From: <sonal.santan@xilinx.com>
To: <dri-devel@lists.freedesktop.org>
Cc: <linux-kernel@vger.kernel.org>, <gregkh@linuxfoundation.org>,
<airlied@redhat.com>, <cyrilc@xilinx.com>, <michals@xilinx.com>,
<lizhih@xilinx.com>, <hyunk@xilinx.com>,
Sonal Santan <sonal.santan@xilinx.com>
Subject: [RFC PATCH Xilinx Alveo 3/6] Add platform drivers for various IPs and frameworks
Date: Tue, 19 Mar 2019 14:53:58 -0700 [thread overview]
Message-ID: <20190319215401.6562-4-sonal.santan@xilinx.com> (raw)
In-Reply-To: <20190319215401.6562-1-sonal.santan@xilinx.com>
From: Sonal Santan <sonal.santan@xilinx.com>
Signed-off-by: Sonal Santan <sonal.santan@xilinx.com>
---
drivers/gpu/drm/xocl/subdev/dna.c | 356 +++
drivers/gpu/drm/xocl/subdev/feature_rom.c | 412 +++
drivers/gpu/drm/xocl/subdev/firewall.c | 389 +++
drivers/gpu/drm/xocl/subdev/fmgr.c | 198 ++
drivers/gpu/drm/xocl/subdev/icap.c | 2859 ++++++++++++++++++
drivers/gpu/drm/xocl/subdev/mailbox.c | 1868 ++++++++++++
drivers/gpu/drm/xocl/subdev/mb_scheduler.c | 3059 ++++++++++++++++++++
drivers/gpu/drm/xocl/subdev/microblaze.c | 722 +++++
drivers/gpu/drm/xocl/subdev/mig.c | 256 ++
drivers/gpu/drm/xocl/subdev/sysmon.c | 385 +++
drivers/gpu/drm/xocl/subdev/xdma.c | 510 ++++
drivers/gpu/drm/xocl/subdev/xmc.c | 1480 ++++++++++
drivers/gpu/drm/xocl/subdev/xvc.c | 461 +++
13 files changed, 12955 insertions(+)
create mode 100644 drivers/gpu/drm/xocl/subdev/dna.c
create mode 100644 drivers/gpu/drm/xocl/subdev/feature_rom.c
create mode 100644 drivers/gpu/drm/xocl/subdev/firewall.c
create mode 100644 drivers/gpu/drm/xocl/subdev/fmgr.c
create mode 100644 drivers/gpu/drm/xocl/subdev/icap.c
create mode 100644 drivers/gpu/drm/xocl/subdev/mailbox.c
create mode 100644 drivers/gpu/drm/xocl/subdev/mb_scheduler.c
create mode 100644 drivers/gpu/drm/xocl/subdev/microblaze.c
create mode 100644 drivers/gpu/drm/xocl/subdev/mig.c
create mode 100644 drivers/gpu/drm/xocl/subdev/sysmon.c
create mode 100644 drivers/gpu/drm/xocl/subdev/xdma.c
create mode 100644 drivers/gpu/drm/xocl/subdev/xmc.c
create mode 100644 drivers/gpu/drm/xocl/subdev/xvc.c
diff --git a/drivers/gpu/drm/xocl/subdev/dna.c b/drivers/gpu/drm/xocl/subdev/dna.c
new file mode 100644
index 000000000000..991d98e5b9aa
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/dna.c
@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Chien-Wei Lan <chienwei@xilinx.com>
+ *
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/vmalloc.h>
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+/* Registers are defined in pg150-ultrascale-memory-ip.pdf:
+ * AXI4-Lite Slave Control/Status Register Map
+ */
+#define XLNX_DNA_MEMORY_MAP_MAGIC_IS_DEFINED (0x3E4D7732)
+#define XLNX_DNA_MAJOR_MINOR_VERSION_REGISTER_OFFSET 0x00 // RO
+#define XLNX_DNA_REVISION_REGISTER_OFFSET 0x04 // RO
+#define XLNX_DNA_CAPABILITY_REGISTER_OFFSET 0x08 // RO
+//#define XLNX_DNA_SCRATCHPAD_REGISTER_OFFSET (0x0C) // RO (31-1) + RW (0)
+#define XLNX_DNA_STATUS_REGISTER_OFFSET 0x10 // RO
+#define XLNX_DNA_FSM_DNA_WORD_WRITE_COUNT_REGISTER_OFFSET (0x14) // RO
+#define XLNX_DNA_FSM_CERTIFICATE_WORD_WRITE_COUNT_REGISTER_OFFSET (0x18) // RO
+#define XLNX_DNA_MESSAGE_START_AXI_ONLY_REGISTER_OFFSET (0x20) // RO (31-1) + RW (0)
+#define XLNX_DNA_READBACK_REGISTER_2_OFFSET 0x40 // RO XLNX_DNA_BOARD_DNA_95_64
+#define XLNX_DNA_READBACK_REGISTER_1_OFFSET 0x44 // RO XLNX_DNA_BOARD_DNA_63_32
+#define XLNX_DNA_READBACK_REGISTER_0_OFFSET 0x48 // RO XLNX_DNA_BOARD_DNA_31_0
+#define XLNX_DNA_DATA_AXI_ONLY_REGISTER_OFFSET (0x80) // WO
+#define XLNX_DNA_CERTIFICATE_DATA_AXI_ONLY_REGISTER_OFFSET (0xC0) // WO - 512 bit aligned.
+#define XLNX_DNA_MAX_ADDRESS_WORDS (0xC4)
+
+struct xocl_xlnx_dna {
+ void __iomem *base;
+ struct device *xlnx_dna_dev;
+ struct mutex xlnx_dna_lock;
+};
+
+static ssize_t status_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ u32 status;
+
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+
+ return sprintf(buf, "0x%x\n", status);
+}
+static DEVICE_ATTR_RO(status);
+
+static ssize_t dna_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ uint32_t dna96_64, dna63_32, dna31_0;
+
+ dna96_64 = ioread32(xlnx_dna->base+XLNX_DNA_READBACK_REGISTER_2_OFFSET);
+ dna63_32 = ioread32(xlnx_dna->base+XLNX_DNA_READBACK_REGISTER_1_OFFSET);
+ dna31_0 = ioread32(xlnx_dna->base+XLNX_DNA_READBACK_REGISTER_0_OFFSET);
+
+ return sprintf(buf, "%08x%08x%08x\n", dna96_64, dna63_32, dna31_0);
+}
+static DEVICE_ATTR_RO(dna);
+
+static ssize_t capability_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ u32 capability;
+
+ capability = ioread32(xlnx_dna->base+XLNX_DNA_CAPABILITY_REGISTER_OFFSET);
+
+ return sprintf(buf, "0x%x\n", capability);
+}
+static DEVICE_ATTR_RO(capability);
+
+
+static ssize_t dna_version_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ u32 version;
+
+ version = ioread32(xlnx_dna->base+XLNX_DNA_MAJOR_MINOR_VERSION_REGISTER_OFFSET);
+
+ return sprintf(buf, "%d.%d\n", version>>16, version & 0xffff);
+}
+static DEVICE_ATTR_RO(dna_version);
+
+static ssize_t revision_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ u32 revision;
+
+ revision = ioread32(xlnx_dna->base+XLNX_DNA_REVISION_REGISTER_OFFSET);
+
+ return sprintf(buf, "%d\n", revision);
+}
+static DEVICE_ATTR_RO(revision);
+
+static struct attribute *xlnx_dna_attributes[] = {
+ &dev_attr_status.attr,
+ &dev_attr_dna.attr,
+ &dev_attr_capability.attr,
+ &dev_attr_dna_version.attr,
+ &dev_attr_revision.attr,
+ NULL
+};
+
+static const struct attribute_group xlnx_dna_attrgroup = {
+ .attrs = xlnx_dna_attributes,
+};
+
+static uint32_t dna_status(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna = platform_get_drvdata(pdev);
+ uint32_t status = 0;
+ uint8_t retries = 10;
+ bool rsa4096done = false;
+
+ if (!xlnx_dna)
+ return status;
+
+ while (!rsa4096done && retries) {
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ if (status>>8 & 0x1) {
+ rsa4096done = true;
+ break;
+ }
+ msleep(1);
+ retries--;
+ }
+
+ if (retries == 0)
+ return -EBUSY;
+
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+
+ return status;
+}
+
+static uint32_t dna_capability(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna = platform_get_drvdata(pdev);
+ u32 capability = 0;
+
+ if (!xlnx_dna)
+ return capability;
+
+ capability = ioread32(xlnx_dna->base+XLNX_DNA_CAPABILITY_REGISTER_OFFSET);
+
+ return capability;
+}
+
+static void dna_write_cert(struct platform_device *pdev, const uint32_t *cert, uint32_t len)
+{
+ struct xocl_xlnx_dna *xlnx_dna = platform_get_drvdata(pdev);
+ int i, j, k;
+ u32 status = 0, words;
+ uint8_t retries = 100;
+ bool sha256done = false;
+ uint32_t convert;
+ uint32_t sign_start, message_words = (len-512)>>2;
+
+ sign_start = message_words;
+
+ if (!xlnx_dna)
+ return;
+
+ iowrite32(0x1, xlnx_dna->base+XLNX_DNA_MESSAGE_START_AXI_ONLY_REGISTER_OFFSET);
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ xocl_info(&pdev->dev, "Start: status %08x", status);
+
+ for (i = 0; i < message_words; i += 16) {
+
+ retries = 100;
+ sha256done = false;
+
+ while (!sha256done && retries) {
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ if (!(status >> 4 & 0x1)) {
+ sha256done = true;
+ break;
+ }
+ msleep(10);
+ retries--;
+ }
+ for (j = 0; j < 16; ++j) {
+ convert = (*(cert+i+j) >> 24 & 0xff) | (*(cert+i+j) >> 8 & 0xff00) |
+ (*(cert+i+j) << 8 & 0xff0000) | ((*(cert+i+j) & 0xff) << 24);
+ iowrite32(convert, xlnx_dna->base+XLNX_DNA_DATA_AXI_ONLY_REGISTER_OFFSET+j*4);
+ }
+ }
+ retries = 100;
+ sha256done = false;
+ while (!sha256done && retries) {
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ if (!(status >> 4 & 0x1)) {
+ sha256done = true;
+ break;
+ }
+ msleep(10);
+ retries--;
+ }
+
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ words = ioread32(xlnx_dna->base+XLNX_DNA_FSM_DNA_WORD_WRITE_COUNT_REGISTER_OFFSET);
+ xocl_info(&pdev->dev, "Message: status %08x dna words %d", status, words);
+
+ for (k = 0; k < 128; k += 16) {
+ for (i = 0; i < 16; i++) {
+ j = k+i+sign_start;
+ convert = (*(cert + j) >> 24 & 0xff) | (*(cert + j) >> 8 & 0xff00) |
+ (*(cert + j) << 8 & 0xff0000) | ((*(cert + j) & 0xff) << 24);
+ iowrite32(convert, xlnx_dna->base+XLNX_DNA_CERTIFICATE_DATA_AXI_ONLY_REGISTER_OFFSET + i * 4);
+ }
+ }
+
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ words = ioread32(xlnx_dna->base+XLNX_DNA_FSM_CERTIFICATE_WORD_WRITE_COUNT_REGISTER_OFFSET);
+ xocl_info(&pdev->dev, "Signature: status %08x certificate words %d", status, words);
+}
+
+static struct xocl_dna_funcs dna_ops = {
+ .status = dna_status,
+ .capability = dna_capability,
+ .write_cert = dna_write_cert,
+};
+
+
+static void mgmt_sysfs_destroy_xlnx_dna(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna;
+
+ xlnx_dna = platform_get_drvdata(pdev);
+
+ sysfs_remove_group(&pdev->dev.kobj, &xlnx_dna_attrgroup);
+
+}
+
+static int mgmt_sysfs_create_xlnx_dna(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna;
+ struct xocl_dev_core *core;
+ int err;
+
+ xlnx_dna = platform_get_drvdata(pdev);
+ core = XDEV(xocl_get_xdev(pdev));
+
+ err = sysfs_create_group(&pdev->dev.kobj, &xlnx_dna_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create pw group failed: 0x%x", err);
+ goto create_grp_failed;
+ }
+
+ return 0;
+
+create_grp_failed:
+ return err;
+}
+
+static int xlnx_dna_probe(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna;
+ struct resource *res;
+ int err;
+
+ xlnx_dna = devm_kzalloc(&pdev->dev, sizeof(*xlnx_dna), GFP_KERNEL);
+ if (!xlnx_dna)
+ return -ENOMEM;
+
+ xlnx_dna->base = devm_kzalloc(&pdev->dev, sizeof(void __iomem *), GFP_KERNEL);
+ if (!xlnx_dna->base)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ xocl_err(&pdev->dev, "resource is NULL");
+ return -EINVAL;
+ }
+ xocl_info(&pdev->dev, "IO start: 0x%llx, end: 0x%llx",
+ res->start, res->end);
+
+ xlnx_dna->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!xlnx_dna->base) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+
+ platform_set_drvdata(pdev, xlnx_dna);
+
+ err = mgmt_sysfs_create_xlnx_dna(pdev);
+ if (err)
+ goto create_xlnx_dna_failed;
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_DNA, &dna_ops);
+
+ return 0;
+
+create_xlnx_dna_failed:
+ platform_set_drvdata(pdev, NULL);
+failed:
+ return err;
+}
+
+
+static int xlnx_dna_remove(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna;
+
+ xlnx_dna = platform_get_drvdata(pdev);
+ if (!xlnx_dna) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ mgmt_sysfs_destroy_xlnx_dna(pdev);
+
+ if (xlnx_dna->base)
+ iounmap(xlnx_dna->base);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, xlnx_dna);
+
+ return 0;
+}
+
+struct platform_device_id xlnx_dna_id_table[] = {
+ { XOCL_DNA, 0 },
+ { },
+};
+
+static struct platform_driver xlnx_dna_driver = {
+ .probe = xlnx_dna_probe,
+ .remove = xlnx_dna_remove,
+ .driver = {
+ .name = "xocl_dna",
+ },
+ .id_table = xlnx_dna_id_table,
+};
+
+int __init xocl_init_dna(void)
+{
+ return platform_driver_register(&xlnx_dna_driver);
+}
+
+void xocl_fini_dna(void)
+{
+ platform_driver_unregister(&xlnx_dna_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/feature_rom.c b/drivers/gpu/drm/xocl/subdev/feature_rom.c
new file mode 100644
index 000000000000..f898af6844aa
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/feature_rom.c
@@ -0,0 +1,412 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include "../xclfeatures.h"
+#include "../xocl_drv.h"
+
+#define MAGIC_NUM 0x786e6c78
+struct feature_rom {
+ void __iomem *base;
+
+ struct FeatureRomHeader header;
+ unsigned int dsa_version;
+ bool unified;
+ bool mb_mgmt_enabled;
+ bool mb_sche_enabled;
+ bool are_dev;
+ bool aws_dev;
+};
+
+static ssize_t VBNV_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%s\n", rom->header.VBNVName);
+}
+static DEVICE_ATTR_RO(VBNV);
+
+static ssize_t dr_base_addr_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ //TODO: Fix: DRBaseAddress no longer required in feature rom
+ if (rom->header.MajorVersion >= 10)
+ return sprintf(buf, "%llu\n", rom->header.DRBaseAddress);
+ else
+ return sprintf(buf, "%u\n", 0);
+}
+static DEVICE_ATTR_RO(dr_base_addr);
+
+static ssize_t ddr_bank_count_max_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%d\n", rom->header.DDRChannelCount);
+}
+static DEVICE_ATTR_RO(ddr_bank_count_max);
+
+static ssize_t ddr_bank_size_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%d\n", rom->header.DDRChannelSize);
+}
+static DEVICE_ATTR_RO(ddr_bank_size);
+
+static ssize_t timestamp_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%llu\n", rom->header.TimeSinceEpoch);
+}
+static DEVICE_ATTR_RO(timestamp);
+
+static ssize_t FPGA_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%s\n", rom->header.FPGAPartName);
+}
+static DEVICE_ATTR_RO(FPGA);
+
+static struct attribute *rom_attrs[] = {
+ &dev_attr_VBNV.attr,
+ &dev_attr_dr_base_addr.attr,
+ &dev_attr_ddr_bank_count_max.attr,
+ &dev_attr_ddr_bank_size.attr,
+ &dev_attr_timestamp.attr,
+ &dev_attr_FPGA.attr,
+ NULL,
+};
+
+static struct attribute_group rom_attr_group = {
+ .attrs = rom_attrs,
+};
+
+static unsigned int dsa_version(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->dsa_version;
+}
+
+static bool is_unified(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->unified;
+}
+
+static bool mb_mgmt_on(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->mb_mgmt_enabled;
+}
+
+static bool mb_sched_on(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->mb_sche_enabled && !XOCL_DSA_MB_SCHE_OFF(xocl_get_xdev(pdev));
+}
+
+static uint32_t *get_cdma_base_addresses(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return (rom->header.FeatureBitMap & CDMA) ? rom->header.CDMABaseAddress : 0;
+}
+
+static u16 get_ddr_channel_count(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->header.DDRChannelCount;
+}
+
+static u64 get_ddr_channel_size(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->header.DDRChannelSize;
+}
+
+static u64 get_timestamp(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->header.TimeSinceEpoch;
+}
+
+static bool is_are(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->are_dev;
+}
+
+static bool is_aws(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->aws_dev;
+}
+
+static bool verify_timestamp(struct platform_device *pdev, u64 timestamp)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ xocl_info(&pdev->dev, "DSA timestamp: 0x%llx",
+ rom->header.TimeSinceEpoch);
+ xocl_info(&pdev->dev, "Verify timestamp: 0x%llx", timestamp);
+ return (rom->header.TimeSinceEpoch == timestamp);
+}
+
+static void get_raw_header(struct platform_device *pdev, void *header)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ memcpy(header, &rom->header, sizeof(rom->header));
+}
+
+static struct xocl_rom_funcs rom_ops = {
+ .dsa_version = dsa_version,
+ .is_unified = is_unified,
+ .mb_mgmt_on = mb_mgmt_on,
+ .mb_sched_on = mb_sched_on,
+ .cdma_addr = get_cdma_base_addresses,
+ .get_ddr_channel_count = get_ddr_channel_count,
+ .get_ddr_channel_size = get_ddr_channel_size,
+ .is_are = is_are,
+ .is_aws = is_aws,
+ .verify_timestamp = verify_timestamp,
+ .get_timestamp = get_timestamp,
+ .get_raw_header = get_raw_header,
+};
+
+static int feature_rom_probe(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+ struct resource *res;
+ u32 val;
+ u16 vendor, did;
+ char *tmp;
+ int ret;
+
+ rom = devm_kzalloc(&pdev->dev, sizeof(*rom), GFP_KERNEL);
+ if (!rom)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ rom->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!rom->base) {
+ ret = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+
+ val = ioread32(rom->base);
+ if (val != MAGIC_NUM) {
+ vendor = XOCL_PL_TO_PCI_DEV(pdev)->vendor;
+ did = XOCL_PL_TO_PCI_DEV(pdev)->device;
+ if (vendor == 0x1d0f && (did == 0x1042 || did == 0xf010)) { // MAGIC, we should define elsewhere
+ xocl_info(&pdev->dev,
+ "Found AWS VU9P Device without featureROM");
+ /*
+ * This is AWS device. Fill the FeatureROM struct.
+ * Right now it doesn't have FeatureROM
+ */
+ memset(rom->header.EntryPointString, 0,
+ sizeof(rom->header.EntryPointString));
+ strncpy(rom->header.EntryPointString, "xlnx", 4);
+ memset(rom->header.FPGAPartName, 0,
+ sizeof(rom->header.FPGAPartName));
+ strncpy(rom->header.FPGAPartName, "AWS VU9P", 8);
+ memset(rom->header.VBNVName, 0,
+ sizeof(rom->header.VBNVName));
+ strncpy(rom->header.VBNVName,
+ "xilinx_aws-vu9p-f1_dynamic_5_0", 35);
+ rom->header.MajorVersion = 4;
+ rom->header.MinorVersion = 0;
+ rom->header.VivadoBuildID = 0xabcd;
+ rom->header.IPBuildID = 0xabcd;
+ rom->header.TimeSinceEpoch = 0xabcd;
+ rom->header.DDRChannelCount = 4;
+ rom->header.DDRChannelSize = 16;
+ rom->header.FeatureBitMap = 0x0;
+ rom->header.FeatureBitMap = UNIFIED_PLATFORM;
+ rom->unified = true;
+ rom->aws_dev = true;
+
+ xocl_info(&pdev->dev, "Enabling AWS dynamic 5.0 DSA");
+ } else {
+ xocl_err(&pdev->dev, "Magic number does not match, actual 0x%x, expected 0x%x",
+ val, MAGIC_NUM);
+ ret = -ENODEV;
+ goto failed;
+ }
+ }
+
+ xocl_memcpy_fromio(&rom->header, rom->base, sizeof(rom->header));
+
+ if (strstr(rom->header.VBNVName, "-xare")) {
+ /*
+ * ARE device, ARE is mapped like another DDR inside FPGA;
+ * map_connects as M04_AXI
+ */
+ rom->header.DDRChannelCount = rom->header.DDRChannelCount - 1;
+ rom->are_dev = true;
+ }
+
+ rom->dsa_version = 0;
+ if (strstr(rom->header.VBNVName, "5_0"))
+ rom->dsa_version = 50;
+ else if (strstr(rom->header.VBNVName, "5_1")
+ || strstr(rom->header.VBNVName, "u200_xdma_201820_1"))
+ rom->dsa_version = 51;
+ else if (strstr(rom->header.VBNVName, "5_2")
+ || strstr(rom->header.VBNVName, "u200_xdma_201820_2")
+ || strstr(rom->header.VBNVName, "u250_xdma_201820_1")
+ || strstr(rom->header.VBNVName, "201830"))
+ rom->dsa_version = 52;
+ else if (strstr(rom->header.VBNVName, "5_3"))
+ rom->dsa_version = 53;
+
+ if (rom->header.FeatureBitMap & UNIFIED_PLATFORM)
+ rom->unified = true;
+
+ if (rom->header.FeatureBitMap & BOARD_MGMT_ENBLD)
+ rom->mb_mgmt_enabled = true;
+
+ if (rom->header.FeatureBitMap & MB_SCHEDULER)
+ rom->mb_sche_enabled = true;
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &rom_attr_group);
+ if (ret) {
+ xocl_err(&pdev->dev, "create sysfs failed");
+ goto failed;
+ }
+
+ tmp = rom->header.EntryPointString;
+ xocl_info(&pdev->dev, "ROM magic : %c%c%c%c",
+ tmp[0], tmp[1], tmp[2], tmp[3]);
+ xocl_info(&pdev->dev, "VBNV: %s", rom->header.VBNVName);
+ xocl_info(&pdev->dev, "DDR channel count : %d",
+ rom->header.DDRChannelCount);
+ xocl_info(&pdev->dev, "DDR channel size: %d GB",
+ rom->header.DDRChannelSize);
+ xocl_info(&pdev->dev, "Major Version: %d", rom->header.MajorVersion);
+ xocl_info(&pdev->dev, "Minor Version: %d", rom->header.MinorVersion);
+ xocl_info(&pdev->dev, "IPBuildID: %u", rom->header.IPBuildID);
+ xocl_info(&pdev->dev, "TimeSinceEpoch: %llx",
+ rom->header.TimeSinceEpoch);
+ xocl_info(&pdev->dev, "FeatureBitMap: %llx", rom->header.FeatureBitMap);
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_FEATURE_ROM, &rom_ops);
+ platform_set_drvdata(pdev, rom);
+
+ return 0;
+
+failed:
+ if (rom->base)
+ iounmap(rom->base);
+ devm_kfree(&pdev->dev, rom);
+ return ret;
+}
+
+static int feature_rom_remove(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ xocl_info(&pdev->dev, "Remove feature rom");
+ rom = platform_get_drvdata(pdev);
+ if (!rom) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+ if (rom->base)
+ iounmap(rom->base);
+
+ sysfs_remove_group(&pdev->dev.kobj, &rom_attr_group);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, rom);
+ return 0;
+}
+
+struct platform_device_id rom_id_table[] = {
+ { XOCL_FEATURE_ROM, 0 },
+ { },
+};
+
+static struct platform_driver feature_rom_driver = {
+ .probe = feature_rom_probe,
+ .remove = feature_rom_remove,
+ .driver = {
+ .name = XOCL_FEATURE_ROM,
+ },
+ .id_table = rom_id_table,
+};
+
+int __init xocl_init_feature_rom(void)
+{
+ return platform_driver_register(&feature_rom_driver);
+}
+
+void xocl_fini_feature_rom(void)
+{
+ return platform_driver_unregister(&feature_rom_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/firewall.c b/drivers/gpu/drm/xocl/subdev/firewall.c
new file mode 100644
index 000000000000..a32766507ae0
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/firewall.c
@@ -0,0 +1,389 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2017-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Utility Functions for AXI firewall IP.
+ * Author: Lizhi.Hou@Xilinx.com
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/ktime.h>
+#include <linux/rtc.h>
+#include "../xocl_drv.h"
+
+/* Firewall registers */
+#define FAULT_STATUS 0x0
+#define SOFT_CTRL 0x4
+#define UNBLOCK_CTRL 0x8
+// Firewall error bits
+#define READ_RESPONSE_BUSY BIT(0)
+#define RECS_ARREADY_MAX_WAIT BIT(1)
+#define RECS_CONTINUOUS_RTRANSFERS_MAX_WAIT BIT(2)
+#define ERRS_RDATA_NUM BIT(3)
+#define ERRS_RID BIT(4)
+#define WRITE_RESPONSE_BUSY BIT(16)
+#define RECS_AWREADY_MAX_WAIT BIT(17)
+#define RECS_WREADY_MAX_WAIT BIT(18)
+#define RECS_WRITE_TO_BVALID_MAX_WAIT BIT(19)
+#define ERRS_BRESP BIT(20)
+
+#define FIREWALL_STATUS_BUSY (READ_RESPONSE_BUSY | WRITE_RESPONSE_BUSY)
+#define CLEAR_RESET_GPIO 0
+
+#define READ_STATUS(fw, id) \
+ XOCL_READ_REG32(fw->base_addrs[id] + FAULT_STATUS)
+#define WRITE_UNBLOCK_CTRL(fw, id, val) \
+ XOCL_WRITE_REG32(val, fw->base_addrs[id] + UNBLOCK_CTRL)
+
+#define IS_FIRED(fw, id) (READ_STATUS(fw, id) & ~FIREWALL_STATUS_BUSY)
+
+#define BUSY_RETRY_COUNT 20
+#define BUSY_RETRY_INTERVAL 100 /* ms */
+#define CLEAR_RETRY_COUNT 4
+#define CLEAR_RETRY_INTERVAL 2 /* ms */
+
+#define MAX_LEVEL 16
+
+struct firewall {
+ void __iomem *base_addrs[MAX_LEVEL];
+ u32 max_level;
+ void __iomem *gpio_addr;
+
+ u32 curr_status;
+ int curr_level;
+
+ u32 err_detected_status;
+ u32 err_detected_level;
+ u64 err_detected_time;
+
+ bool inject_firewall;
+};
+
+static int clear_firewall(struct platform_device *pdev);
+static u32 check_firewall(struct platform_device *pdev, int *level);
+
+static int get_prop(struct platform_device *pdev, u32 prop, void *val)
+{
+ struct firewall *fw;
+
+ fw = platform_get_drvdata(pdev);
+ BUG_ON(!fw);
+
+ check_firewall(pdev, NULL);
+ switch (prop) {
+ case XOCL_AF_PROP_TOTAL_LEVEL:
+ *(u32 *)val = fw->max_level;
+ break;
+ case XOCL_AF_PROP_STATUS:
+ *(u32 *)val = fw->curr_status;
+ break;
+ case XOCL_AF_PROP_LEVEL:
+ *(int *)val = fw->curr_level;
+ break;
+ case XOCL_AF_PROP_DETECTED_STATUS:
+ *(u32 *)val = fw->err_detected_status;
+ break;
+ case XOCL_AF_PROP_DETECTED_LEVEL:
+ *(u32 *)val = fw->err_detected_level;
+ break;
+ case XOCL_AF_PROP_DETECTED_TIME:
+ *(u64 *)val = fw->err_detected_time;
+ break;
+ default:
+ xocl_err(&pdev->dev, "Invalid prop %d", prop);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* sysfs support */
+static ssize_t show_firewall(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+ struct platform_device *pdev = to_platform_device(dev);
+ struct firewall *fw;
+ u64 t;
+ u32 val;
+ int ret;
+
+ fw = platform_get_drvdata(pdev);
+ BUG_ON(!fw);
+
+ if (attr->index == XOCL_AF_PROP_DETECTED_TIME) {
+ get_prop(pdev, attr->index, &t);
+ return sprintf(buf, "%llu\n", t);
+ }
+
+ ret = get_prop(pdev, attr->index, &val);
+ if (ret)
+ return 0;
+
+ return sprintf(buf, "%u\n", val);
+}
+
+static SENSOR_DEVICE_ATTR(status, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_STATUS);
+static SENSOR_DEVICE_ATTR(level, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_LEVEL);
+static SENSOR_DEVICE_ATTR(detected_status, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_DETECTED_STATUS);
+static SENSOR_DEVICE_ATTR(detected_level, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_DETECTED_LEVEL);
+static SENSOR_DEVICE_ATTR(detected_time, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_DETECTED_TIME);
+
+static ssize_t clear_store(struct device *dev, struct device_attribute *da,
+ const char *buf, size_t count)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ u32 val = 0;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val != 1)
+ return -EINVAL;
+
+ clear_firewall(pdev);
+
+ return count;
+}
+static DEVICE_ATTR_WO(clear);
+
+static ssize_t inject_store(struct device *dev, struct device_attribute *da,
+ const char *buf, size_t count)
+{
+ struct firewall *fw = platform_get_drvdata(to_platform_device(dev));
+
+ fw->inject_firewall = true;
+ return count;
+}
+static DEVICE_ATTR_WO(inject);
+
+static struct attribute *firewall_attributes[] = {
+ &sensor_dev_attr_status.dev_attr.attr,
+ &sensor_dev_attr_level.dev_attr.attr,
+ &sensor_dev_attr_detected_status.dev_attr.attr,
+ &sensor_dev_attr_detected_level.dev_attr.attr,
+ &sensor_dev_attr_detected_time.dev_attr.attr,
+ &dev_attr_clear.attr,
+ &dev_attr_inject.attr,
+ NULL
+};
+
+static const struct attribute_group firewall_attrgroup = {
+ .attrs = firewall_attributes,
+};
+
+static u32 check_firewall(struct platform_device *pdev, int *level)
+{
+ struct firewall *fw;
+// struct timeval time;
+ struct timespec64 now;
+ int i;
+ u32 val = 0;
+
+ fw = platform_get_drvdata(pdev);
+ BUG_ON(!fw);
+
+ for (i = 0; i < fw->max_level; i++) {
+ val = IS_FIRED(fw, i);
+ if (val) {
+ xocl_info(&pdev->dev, "AXI Firewall %d tripped, "
+ "status: 0x%x", i, val);
+ if (!fw->curr_status) {
+ fw->err_detected_status = val;
+ fw->err_detected_level = i;
+ ktime_get_ts64(&now);
+ fw->err_detected_time = (u64)(now.tv_sec -
+ (sys_tz.tz_minuteswest * 60));
+ }
+ fw->curr_level = i;
+
+ if (level)
+ *level = i;
+ break;
+ }
+ }
+
+ fw->curr_status = val;
+ fw->curr_level = i >= fw->max_level ? -1 : i;
+
+ /* Inject firewall for testing. */
+ if (fw->curr_level == -1 && fw->inject_firewall) {
+ fw->inject_firewall = false;
+ fw->curr_level = 0;
+ fw->curr_status = 0x1;
+ }
+
+ return fw->curr_status;
+}
+
+static int clear_firewall(struct platform_device *pdev)
+{
+ struct firewall *fw;
+ int i, retry = 0, clear_retry = 0;
+ u32 val;
+ int ret = 0;
+
+ fw = platform_get_drvdata(pdev);
+ BUG_ON(!fw);
+
+ if (!check_firewall(pdev, NULL)) {
+ /* firewall is not tripped */
+ return 0;
+ }
+
+retry_level1:
+ for (i = 0; i < fw->max_level; i++) {
+ for (val = READ_STATUS(fw, i);
+ (val & FIREWALL_STATUS_BUSY) &&
+ retry++ < BUSY_RETRY_COUNT;
+ val = READ_STATUS(fw, i)) {
+ msleep(BUSY_RETRY_INTERVAL);
+ }
+ if (val & FIREWALL_STATUS_BUSY) {
+ xocl_err(&pdev->dev, "firewall %d busy", i);
+ ret = -EBUSY;
+ goto failed;
+ }
+ WRITE_UNBLOCK_CTRL(fw, i, 1);
+ }
+
+ if (check_firewall(pdev, NULL) && clear_retry++ < CLEAR_RETRY_COUNT) {
+ msleep(CLEAR_RETRY_INTERVAL);
+ goto retry_level1;
+ }
+
+ if (!check_firewall(pdev, NULL)) {
+ xocl_info(&pdev->dev, "firewall cleared level 1");
+ return 0;
+ }
+
+ clear_retry = 0;
+
+retry_level2:
+ XOCL_WRITE_REG32(CLEAR_RESET_GPIO, fw->gpio_addr);
+
+ if (check_firewall(pdev, NULL) && clear_retry++ < CLEAR_RETRY_COUNT) {
+ msleep(CLEAR_RETRY_INTERVAL);
+ goto retry_level2;
+ }
+
+ if (!check_firewall(pdev, NULL)) {
+ xocl_info(&pdev->dev, "firewall cleared level 2");
+ return 0;
+ }
+
+ xocl_info(&pdev->dev, "failed clear firewall, level %d, status 0x%x",
+ fw->curr_level, fw->curr_status);
+
+ ret = -EIO;
+
+failed:
+ return ret;
+}
+
+static struct xocl_firewall_funcs fw_ops = {
+ .clear_firewall = clear_firewall,
+ .check_firewall = check_firewall,
+ .get_prop = get_prop,
+};
+
+static int firewall_remove(struct platform_device *pdev)
+{
+ struct firewall *fw;
+ int i;
+
+ fw = platform_get_drvdata(pdev);
+ if (!fw) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ sysfs_remove_group(&pdev->dev.kobj, &firewall_attrgroup);
+
+ for (i = 0; i <= fw->max_level; i++) {
+ if (fw->base_addrs[i])
+ iounmap(fw->base_addrs[i]);
+ }
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, fw);
+
+ return 0;
+}
+
+static int firewall_probe(struct platform_device *pdev)
+{
+ struct firewall *fw;
+ struct resource *res;
+ int i, ret = 0;
+
+ xocl_info(&pdev->dev, "probe");
+
+ fw = devm_kzalloc(&pdev->dev, sizeof(*fw), GFP_KERNEL);
+ if (!fw)
+ return -ENOMEM;
+
+ platform_set_drvdata(pdev, fw);
+
+ fw->curr_level = -1;
+
+ for (i = 0; i < MAX_LEVEL; i++) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+ if (!res) {
+ fw->max_level = i - 1;
+ fw->gpio_addr = fw->base_addrs[i - 1];
+ break;
+ }
+ fw->base_addrs[i] =
+ ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!fw->base_addrs[i]) {
+ ret = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+ }
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &firewall_attrgroup);
+ if (ret) {
+ xocl_err(&pdev->dev, "create attr group failed: %d", ret);
+ goto failed;
+ }
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_AF, &fw_ops);
+
+ return 0;
+
+failed:
+ firewall_remove(pdev);
+ return ret;
+}
+
+struct platform_device_id firewall_id_table[] = {
+ { XOCL_FIREWALL, 0 },
+ { },
+};
+
+static struct platform_driver firewall_driver = {
+ .probe = firewall_probe,
+ .remove = firewall_remove,
+ .driver = {
+ .name = "xocl_firewall",
+ },
+ .id_table = firewall_id_table,
+};
+
+int __init xocl_init_firewall(void)
+{
+ return platform_driver_register(&firewall_driver);
+}
+
+void xocl_fini_firewall(void)
+{
+ return platform_driver_unregister(&firewall_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/fmgr.c b/drivers/gpu/drm/xocl/subdev/fmgr.c
new file mode 100644
index 000000000000..99efd86ccd1b
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/fmgr.c
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * FPGA Manager bindings for XRT driver
+ *
+ * Copyright (C) 2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Sonal Santan
+ *
+ */
+
+#include <linux/fpga/fpga-mgr.h>
+
+#include "../xocl_drv.h"
+#include "../xclbin.h"
+
+/*
+ * Container to capture and cache full xclbin as it is passed in blocks by FPGA
+ * Manager. xocl needs access to full xclbin to walk through xclbin sections. FPGA
+ * Manager's .write() backend sends incremental blocks without any knowledge of
+ * xclbin format forcing us to collect the blocks and stitch them together here.
+ * TODO:
+ * 1. Add a variant of API, icap_download_bitstream_axlf() which works off kernel buffer
+ * 2. Call this new API from FPGA Manager's write complete hook, xocl_pr_write_complete()
+ */
+
+struct xfpga_klass {
+ struct xocl_dev *xdev;
+ struct axlf *blob;
+ char name[64];
+ size_t count;
+ enum fpga_mgr_states state;
+};
+
+static int xocl_pr_write_init(struct fpga_manager *mgr,
+ struct fpga_image_info *info, const char *buf, size_t count)
+{
+ struct xfpga_klass *obj = mgr->priv;
+ const struct axlf *bin = (const struct axlf *)buf;
+
+ if (count < sizeof(struct axlf)) {
+ obj->state = FPGA_MGR_STATE_WRITE_INIT_ERR;
+ return -EINVAL;
+ }
+
+ if (count > bin->m_header.m_length) {
+ obj->state = FPGA_MGR_STATE_WRITE_INIT_ERR;
+ return -EINVAL;
+ }
+
+ /* Free up the previous blob */
+ vfree(obj->blob);
+ obj->blob = vmalloc(bin->m_header.m_length);
+ if (!obj->blob) {
+ obj->state = FPGA_MGR_STATE_WRITE_INIT_ERR;
+ return -ENOMEM;
+ }
+
+ memcpy(obj->blob, buf, count);
+ xocl_info(&mgr->dev, "Begin download of xclbin %pUb of length %lld B", &obj->blob->m_header.uuid,
+ obj->blob->m_header.m_length);
+ obj->count = count;
+ obj->state = FPGA_MGR_STATE_WRITE_INIT;
+ return 0;
+}
+
+static int xocl_pr_write(struct fpga_manager *mgr,
+ const char *buf, size_t count)
+{
+ struct xfpga_klass *obj = mgr->priv;
+ char *curr = (char *)obj->blob;
+
+ if ((obj->state != FPGA_MGR_STATE_WRITE_INIT) && (obj->state != FPGA_MGR_STATE_WRITE)) {
+ obj->state = FPGA_MGR_STATE_WRITE_ERR;
+ return -EINVAL;
+ }
+
+ curr += obj->count;
+ obj->count += count;
+ /* Check if the xclbin buffer is not longer than advertised in the header */
+ if (obj->blob->m_header.m_length < obj->count) {
+ obj->state = FPGA_MGR_STATE_WRITE_ERR;
+ return -EINVAL;
+ }
+ memcpy(curr, buf, count);
+ xocl_info(&mgr->dev, "Next block of %zu B of xclbin %pUb", count, &obj->blob->m_header.uuid);
+ obj->state = FPGA_MGR_STATE_WRITE;
+ return 0;
+}
+
+
+static int xocl_pr_write_complete(struct fpga_manager *mgr,
+ struct fpga_image_info *info)
+{
+ int result;
+ struct xfpga_klass *obj = mgr->priv;
+
+ if (obj->state != FPGA_MGR_STATE_WRITE) {
+ obj->state = FPGA_MGR_STATE_WRITE_COMPLETE_ERR;
+ return -EINVAL;
+ }
+
+ /* Check if we got the complete xclbin */
+ if (obj->blob->m_header.m_length != obj->count) {
+ obj->state = FPGA_MGR_STATE_WRITE_COMPLETE_ERR;
+ return -EINVAL;
+ }
+ /* Send the xclbin blob to actual download framework in icap */
+ result = xocl_icap_download_axlf(obj->xdev, obj->blob);
+ obj->state = result ? FPGA_MGR_STATE_WRITE_COMPLETE_ERR : FPGA_MGR_STATE_WRITE_COMPLETE;
+ xocl_info(&mgr->dev, "Finish download of xclbin %pUb of size %zu B", &obj->blob->m_header.uuid, obj->count);
+ vfree(obj->blob);
+ obj->blob = NULL;
+ obj->count = 0;
+ return result;
+}
+
+static enum fpga_mgr_states xocl_pr_state(struct fpga_manager *mgr)
+{
+ struct xfpga_klass *obj = mgr->priv;
+
+ return obj->state;
+}
+
+static const struct fpga_manager_ops xocl_pr_ops = {
+ .initial_header_size = sizeof(struct axlf),
+ .write_init = xocl_pr_write_init,
+ .write = xocl_pr_write,
+ .write_complete = xocl_pr_write_complete,
+ .state = xocl_pr_state,
+};
+
+
+struct platform_device_id fmgr_id_table[] = {
+ { XOCL_FMGR, 0 },
+ { },
+};
+
+static int fmgr_probe(struct platform_device *pdev)
+{
+ struct fpga_manager *mgr;
+ int ret = 0;
+ struct xfpga_klass *obj = kzalloc(sizeof(struct xfpga_klass), GFP_KERNEL);
+
+ if (!obj)
+ return -ENOMEM;
+
+ obj->xdev = xocl_get_xdev(pdev);
+ snprintf(obj->name, sizeof(obj->name), "Xilinx PCIe FPGA Manager");
+
+ obj->state = FPGA_MGR_STATE_UNKNOWN;
+ mgr = fpga_mgr_create(&pdev->dev, obj->name, &xocl_pr_ops, obj);
+ if (!mgr) {
+ ret = -ENODEV;
+ goto out;
+ }
+ ret = fpga_mgr_register(mgr);
+ if (ret)
+ goto out;
+
+ return ret;
+out:
+ kfree(obj);
+ return ret;
+}
+
+static int fmgr_remove(struct platform_device *pdev)
+{
+ struct fpga_manager *mgr = platform_get_drvdata(pdev);
+ struct xfpga_klass *obj = mgr->priv;
+
+ obj->state = FPGA_MGR_STATE_UNKNOWN;
+ fpga_mgr_unregister(mgr);
+
+ platform_set_drvdata(pdev, NULL);
+ vfree(obj->blob);
+ kfree(obj);
+ return 0;
+}
+
+static struct platform_driver fmgr_driver = {
+ .probe = fmgr_probe,
+ .remove = fmgr_remove,
+ .driver = {
+ .name = "xocl_fmgr",
+ },
+ .id_table = fmgr_id_table,
+};
+
+int __init xocl_init_fmgr(void)
+{
+ return platform_driver_register(&fmgr_driver);
+}
+
+void xocl_fini_fmgr(void)
+{
+ platform_driver_unregister(&fmgr_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/icap.c b/drivers/gpu/drm/xocl/subdev/icap.c
new file mode 100644
index 000000000000..93eb6265a9c4
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/icap.c
@@ -0,0 +1,2859 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2017 Xilinx, Inc. All rights reserved.
+ * Author: Sonal Santan
+ * Code copied verbatim from SDAccel xcldma kernel mode driver
+ *
+ */
+
+/*
+ * TODO: Currently, locking / unlocking bitstream is implemented w/ pid as
+ * identification of bitstream users. We assume that, on bare metal, an app
+ * has only one process and will open both user and mgmt pfs. In this model,
+ * xclmgmt has enough information to handle locking/unlocking alone, but we
+ * still involve user pf and mailbox here so that it'll be easier to support
+ * cloud env later. We'll replace pid with a token that is more appropriate
+ * to identify a user later as well.
+ */
+
+#include <linux/firmware.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/version.h>
+#include <linux/uuid.h>
+#include <linux/pid.h>
+#include "../xclbin.h"
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+#if defined(XOCL_UUID)
+static uuid_t uuid_null = NULL_UUID_LE;
+#endif
+
+#define ICAP_ERR(icap, fmt, arg...) \
+ xocl_err(&(icap)->icap_pdev->dev, fmt "\n", ##arg)
+#define ICAP_INFO(icap, fmt, arg...) \
+ xocl_info(&(icap)->icap_pdev->dev, fmt "\n", ##arg)
+#define ICAP_DBG(icap, fmt, arg...) \
+ xocl_dbg(&(icap)->icap_pdev->dev, fmt "\n", ##arg)
+
+#define ICAP_PRIVILEGED(icap) ((icap)->icap_regs != NULL)
+#define DMA_HWICAP_BITFILE_BUFFER_SIZE 1024
+#define ICAP_MAX_REG_GROUPS ARRAY_SIZE(XOCL_RES_ICAP_MGMT)
+
+#define ICAP_MAX_NUM_CLOCKS 2
+#define OCL_CLKWIZ_STATUS_OFFSET 0x4
+#define OCL_CLKWIZ_CONFIG_OFFSET(n) (0x200 + 4 * (n))
+#define OCL_CLK_FREQ_COUNTER_OFFSET 0x8
+
+/*
+ * Bitstream header information.
+ */
+struct XHwIcap_Bit_Header {
+ unsigned int HeaderLength; /* Length of header in 32 bit words */
+ unsigned int BitstreamLength; /* Length of bitstream to read in bytes*/
+ unsigned char *DesignName; /* Design name read from bitstream header */
+ unsigned char *PartName; /* Part name read from bitstream header */
+ unsigned char *Date; /* Date read from bitstream header */
+ unsigned char *Time; /* Bitstream creation time read from header */
+ unsigned int MagicLength; /* Length of the magic numbers in header */
+};
+
+#define XHI_BIT_HEADER_FAILURE -1
+/* Used for parsing bitstream header */
+#define XHI_EVEN_MAGIC_BYTE 0x0f
+#define XHI_ODD_MAGIC_BYTE 0xf0
+/* Extra mode for IDLE */
+#define XHI_OP_IDLE -1
+/* The imaginary module length register */
+#define XHI_MLR 15
+
+#define GATE_FREEZE_USER 0x0c
+#define GATE_FREEZE_SHELL 0x00
+
+static u32 gate_free_user[] = {0xe, 0xc, 0xe, 0xf};
+static u32 gate_free_shell[] = {0x8, 0xc, 0xe, 0xf};
+
+/*
+ * AXI-HWICAP IP register layout
+ */
+struct icap_reg {
+ u32 ir_rsvd1[7];
+ u32 ir_gier;
+ u32 ir_isr;
+ u32 ir_rsvd2;
+ u32 ir_ier;
+ u32 ir_rsvd3[53];
+ u32 ir_wf;
+ u32 ir_rf;
+ u32 ir_sz;
+ u32 ir_cr;
+ u32 ir_sr;
+ u32 ir_wfv;
+ u32 ir_rfo;
+ u32 ir_asr;
+} __attribute__((packed));
+
+struct icap_generic_state {
+ u32 igs_state;
+} __attribute__((packed));
+
+struct icap_axi_gate {
+ u32 iag_wr;
+ u32 iag_rvsd;
+ u32 iag_rd;
+} __attribute__((packed));
+
+struct icap_bitstream_user {
+ struct list_head ibu_list;
+ pid_t ibu_pid;
+};
+
+struct icap {
+ struct platform_device *icap_pdev;
+ struct mutex icap_lock;
+ struct icap_reg *icap_regs;
+ struct icap_generic_state *icap_state;
+ unsigned int idcode;
+ bool icap_axi_gate_frozen;
+ bool icap_axi_gate_shell_frozen;
+ struct icap_axi_gate *icap_axi_gate;
+
+ u64 icap_bitstream_id;
+ uuid_t icap_bitstream_uuid;
+ int icap_bitstream_ref;
+ struct list_head icap_bitstream_users;
+
+ char *icap_clear_bitstream;
+ unsigned long icap_clear_bitstream_length;
+
+ char *icap_clock_bases[ICAP_MAX_NUM_CLOCKS];
+ unsigned short icap_ocl_frequency[ICAP_MAX_NUM_CLOCKS];
+
+ char *icap_clock_freq_topology;
+ unsigned long icap_clock_freq_topology_length;
+ char *icap_clock_freq_counter;
+ struct mem_topology *mem_topo;
+ struct ip_layout *ip_layout;
+ struct debug_ip_layout *debug_layout;
+ struct connectivity *connectivity;
+
+ char *bit_buffer;
+ unsigned long bit_length;
+};
+
+static inline u32 reg_rd(void __iomem *reg)
+{
+ return XOCL_READ_REG32(reg);
+}
+
+static inline void reg_wr(void __iomem *reg, u32 val)
+{
+ iowrite32(val, reg);
+}
+
+/*
+ * Precomputed table with config0 and config2 register values together with
+ * target frequency. The steps are approximately 5 MHz apart. Table is
+ * generated by wiz.pl.
+ */
+const static struct xclmgmt_ocl_clockwiz {
+ /* target frequency */
+ unsigned short ocl;
+ /* config0 register */
+ unsigned long config0;
+ /* config2 register */
+ unsigned short config2;
+} frequency_table[] = {
+ {/* 600*/ 60, 0x0601, 0x000a},
+ {/* 600*/ 66, 0x0601, 0x0009},
+ {/* 600*/ 75, 0x0601, 0x0008},
+ {/* 800*/ 80, 0x0801, 0x000a},
+ {/* 600*/ 85, 0x0601, 0x0007},
+ {/* 900*/ 90, 0x0901, 0x000a},
+ {/*1000*/ 100, 0x0a01, 0x000a},
+ {/*1100*/ 110, 0x0b01, 0x000a},
+ {/* 700*/ 116, 0x0701, 0x0006},
+ {/*1100*/ 122, 0x0b01, 0x0009},
+ {/* 900*/ 128, 0x0901, 0x0007},
+ {/*1200*/ 133, 0x0c01, 0x0009},
+ {/*1400*/ 140, 0x0e01, 0x000a},
+ {/*1200*/ 150, 0x0c01, 0x0008},
+ {/*1400*/ 155, 0x0e01, 0x0009},
+ {/* 800*/ 160, 0x0801, 0x0005},
+ {/*1000*/ 166, 0x0a01, 0x0006},
+ {/*1200*/ 171, 0x0c01, 0x0007},
+ {/* 900*/ 180, 0x0901, 0x0005},
+ {/*1300*/ 185, 0x0d01, 0x0007},
+ {/*1400*/ 200, 0x0e01, 0x0007},
+ {/*1300*/ 216, 0x0d01, 0x0006},
+ {/* 900*/ 225, 0x0901, 0x0004},
+ {/*1400*/ 233, 0x0e01, 0x0006},
+ {/*1200*/ 240, 0x0c01, 0x0005},
+ {/*1000*/ 250, 0x0a01, 0x0004},
+ {/*1300*/ 260, 0x0d01, 0x0005},
+ {/* 800*/ 266, 0x0801, 0x0003},
+ {/*1100*/ 275, 0x0b01, 0x0004},
+ {/*1400*/ 280, 0x0e01, 0x0005},
+ {/*1200*/ 300, 0x0c01, 0x0004},
+ {/*1300*/ 325, 0x0d01, 0x0004},
+ {/*1000*/ 333, 0x0a01, 0x0003},
+ {/*1400*/ 350, 0x0e01, 0x0004},
+ {/*1100*/ 366, 0x0b01, 0x0003},
+ {/*1200*/ 400, 0x0c01, 0x0003},
+ {/*1300*/ 433, 0x0d01, 0x0003},
+ {/* 900*/ 450, 0x0901, 0x0002},
+ {/*1400*/ 466, 0x0e01, 0x0003},
+ {/*1000*/ 500, 0x0a01, 0x0002}
+};
+
+static int icap_verify_bitstream_axlf(struct platform_device *pdev,
+ struct axlf *xclbin);
+static int icap_parse_bitstream_axlf_section(struct platform_device *pdev,
+ const struct axlf *xclbin, enum axlf_section_kind kind);
+
+static struct icap_bitstream_user *alloc_user(pid_t pid)
+{
+ struct icap_bitstream_user *u =
+ kzalloc(sizeof(struct icap_bitstream_user), GFP_KERNEL);
+
+ if (u) {
+ INIT_LIST_HEAD(&u->ibu_list);
+ u->ibu_pid = pid;
+ }
+ return u;
+}
+
+static void free_user(struct icap_bitstream_user *u)
+{
+ kfree(u);
+}
+
+static struct icap_bitstream_user *obtain_user(struct icap *icap, pid_t pid)
+{
+ struct list_head *pos, *n;
+
+ list_for_each_safe(pos, n, &icap->icap_bitstream_users) {
+ struct icap_bitstream_user *u = list_entry(pos, struct icap_bitstream_user, ibu_list);
+
+ if (u->ibu_pid == pid)
+ return u;
+ }
+
+ return NULL;
+}
+
+static void icap_read_from_peer(struct platform_device *pdev, enum data_kind kind, void *resp, size_t resplen)
+{
+ struct mailbox_subdev_peer subdev_peer = {0};
+ size_t data_len = sizeof(struct mailbox_subdev_peer);
+ struct mailbox_req *mb_req = NULL;
+ size_t reqlen = sizeof(struct mailbox_req) + data_len;
+
+ mb_req = vmalloc(reqlen);
+ if (!mb_req)
+ return;
+
+ mb_req->req = MAILBOX_REQ_PEER_DATA;
+
+ subdev_peer.kind = kind;
+ memcpy(mb_req->data, &subdev_peer, data_len);
+
+ (void) xocl_peer_request(XOCL_PL_DEV_TO_XDEV(pdev),
+ mb_req, reqlen, resp, &resplen, NULL, NULL);
+
+ vfree(mb_req);
+}
+
+
+static int add_user(struct icap *icap, pid_t pid)
+{
+ struct icap_bitstream_user *u;
+
+ u = obtain_user(icap, pid);
+ if (u)
+ return 0;
+
+ u = alloc_user(pid);
+ if (!u)
+ return -ENOMEM;
+
+ list_add_tail(&u->ibu_list, &icap->icap_bitstream_users);
+ icap->icap_bitstream_ref++;
+ return 0;
+}
+
+static int del_user(struct icap *icap, pid_t pid)
+{
+ struct icap_bitstream_user *u = NULL;
+
+ u = obtain_user(icap, pid);
+ if (!u)
+ return -EINVAL;
+
+ list_del(&u->ibu_list);
+ free_user(u);
+ icap->icap_bitstream_ref--;
+ return 0;
+}
+
+static void del_all_users(struct icap *icap)
+{
+ struct icap_bitstream_user *u = NULL;
+ struct list_head *pos, *n;
+
+ if (icap->icap_bitstream_ref == 0)
+ return;
+
+ list_for_each_safe(pos, n, &icap->icap_bitstream_users) {
+ u = list_entry(pos, struct icap_bitstream_user, ibu_list);
+ list_del(&u->ibu_list);
+ free_user(u);
+ }
+
+ ICAP_INFO(icap, "removed %d users", icap->icap_bitstream_ref);
+ icap->icap_bitstream_ref = 0;
+}
+
+static unsigned int find_matching_freq_config(unsigned int freq)
+{
+ unsigned int start = 0;
+ unsigned int end = ARRAY_SIZE(frequency_table) - 1;
+ unsigned int idx = ARRAY_SIZE(frequency_table) - 1;
+
+ if (freq < frequency_table[0].ocl)
+ return 0;
+
+ if (freq > frequency_table[ARRAY_SIZE(frequency_table) - 1].ocl)
+ return ARRAY_SIZE(frequency_table) - 1;
+
+ while (start < end) {
+ if (freq == frequency_table[idx].ocl)
+ break;
+ if (freq < frequency_table[idx].ocl)
+ end = idx;
+ else
+ start = idx + 1;
+ idx = start + (end - start) / 2;
+ }
+ if (freq < frequency_table[idx].ocl)
+ idx--;
+
+ return idx;
+}
+
+static unsigned short icap_get_ocl_frequency(const struct icap *icap, int idx)
+{
+#define XCL_INPUT_FREQ 100
+ const u64 input = XCL_INPUT_FREQ;
+ u32 val;
+ u32 mul0, div0;
+ u32 mul_frac0 = 0;
+ u32 div1;
+ u32 div_frac1 = 0;
+ u64 freq;
+ char *base = NULL;
+
+ if (ICAP_PRIVILEGED(icap)) {
+ base = icap->icap_clock_bases[idx];
+ val = reg_rd(base + OCL_CLKWIZ_STATUS_OFFSET);
+ if ((val & 1) == 0)
+ return 0;
+
+ val = reg_rd(base + OCL_CLKWIZ_CONFIG_OFFSET(0));
+
+ div0 = val & 0xff;
+ mul0 = (val & 0xff00) >> 8;
+ if (val & BIT(26)) {
+ mul_frac0 = val >> 16;
+ mul_frac0 &= 0x3ff;
+ }
+
+ /*
+ * Multiply both numerator (mul0) and the denominator (div0) with 1000
+ * to account for fractional portion of multiplier
+ */
+ mul0 *= 1000;
+ mul0 += mul_frac0;
+ div0 *= 1000;
+
+ val = reg_rd(base + OCL_CLKWIZ_CONFIG_OFFSET(2));
+
+ div1 = val & 0xff;
+ if (val & BIT(18)) {
+ div_frac1 = val >> 8;
+ div_frac1 &= 0x3ff;
+ }
+
+ /*
+ * Multiply both numerator (mul0) and the denominator (div1) with 1000 to
+ * account for fractional portion of divider
+ */
+
+ div1 *= 1000;
+ div1 += div_frac1;
+ div0 *= div1;
+ mul0 *= 1000;
+ if (div0 == 0) {
+ ICAP_ERR(icap, "clockwiz 0 divider");
+ return 0;
+ }
+ freq = (input * mul0) / div0;
+ } else {
+ icap_read_from_peer(icap->icap_pdev, CLOCK_FREQ_0, (u32 *)&freq, sizeof(u32));
+ }
+ return freq;
+}
+
+static unsigned int icap_get_clock_frequency_counter_khz(const struct icap *icap, int idx)
+{
+ u32 freq, status;
+ char *base = icap->icap_clock_freq_counter;
+ int times;
+
+ times = 10;
+ freq = 0;
+ /*
+ * reset and wait until done
+ */
+ if (ICAP_PRIVILEGED(icap)) {
+ if (uuid_is_null(&icap->icap_bitstream_uuid)) {
+ ICAP_ERR(icap, "ERROR: There isn't a xclbin loaded in the dynamic "
+ "region, frequencies counter cannot be determined");
+ return freq;
+ }
+ reg_wr(base, 0x1);
+
+ while (times != 0) {
+ status = reg_rd(base);
+ if (status == 0x2)
+ break;
+ mdelay(1);
+ times--;
+ };
+
+ freq = reg_rd(base + OCL_CLK_FREQ_COUNTER_OFFSET + idx * sizeof(u32));
+ } else {
+ icap_read_from_peer(icap->icap_pdev, FREQ_COUNTER_0, (u32 *)&freq, sizeof(u32));
+ }
+ return freq;
+}
+/*
+ * Based on Clocking Wizard v5.1, section Dynamic Reconfiguration
+ * through AXI4-Lite
+ */
+static int icap_ocl_freqscaling(struct icap *icap, bool force)
+{
+ unsigned int curr_freq;
+ u32 config;
+ int i;
+ int j = 0;
+ u32 val = 0;
+ unsigned int idx = 0;
+ long err = 0;
+
+ for (i = 0; i < ICAP_MAX_NUM_CLOCKS; ++i) {
+ // A value of zero means skip scaling for this clock index
+ if (!icap->icap_ocl_frequency[i])
+ continue;
+
+ idx = find_matching_freq_config(icap->icap_ocl_frequency[i]);
+ curr_freq = icap_get_ocl_frequency(icap, i);
+ ICAP_INFO(icap, "Clock %d, Current %d Mhz, New %d Mhz ",
+ i, curr_freq, icap->icap_ocl_frequency[i]);
+
+ /*
+ * If current frequency is in the same step as the
+ * requested frequency then nothing to do.
+ */
+ if (!force && (find_matching_freq_config(curr_freq) == idx))
+ continue;
+
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_STATUS_OFFSET);
+ if (val != 1) {
+ ICAP_ERR(icap, "clockwiz %d is busy", i);
+ err = -EBUSY;
+ break;
+ }
+
+ config = frequency_table[idx].config0;
+ reg_wr(icap->icap_clock_bases[i] + OCL_CLKWIZ_CONFIG_OFFSET(0),
+ config);
+ config = frequency_table[idx].config2;
+ reg_wr(icap->icap_clock_bases[i] + OCL_CLKWIZ_CONFIG_OFFSET(2),
+ config);
+ msleep(10);
+ reg_wr(icap->icap_clock_bases[i] + OCL_CLKWIZ_CONFIG_OFFSET(23),
+ 0x00000007);
+ msleep(1);
+ reg_wr(icap->icap_clock_bases[i] + OCL_CLKWIZ_CONFIG_OFFSET(23),
+ 0x00000002);
+
+ ICAP_INFO(icap, "clockwiz waiting for locked signal");
+ msleep(100);
+ for (j = 0; j < 100; j++) {
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_STATUS_OFFSET);
+ if (val != 1) {
+ msleep(100);
+ continue;
+ }
+ }
+ if (val != 1) {
+ ICAP_ERR(icap, "clockwiz MMCM/PLL did not lock after %d ms, "
+ "restoring the original configuration", 100 * 100);
+ /* restore the original clock configuration */
+ reg_wr(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_CONFIG_OFFSET(23), 0x00000004);
+ msleep(10);
+ reg_wr(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_CONFIG_OFFSET(23), 0x00000000);
+ err = -ETIMEDOUT;
+ break;
+ }
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_CONFIG_OFFSET(0));
+ ICAP_INFO(icap, "clockwiz CONFIG(0) 0x%x", val);
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_CONFIG_OFFSET(2));
+ ICAP_INFO(icap, "clockwiz CONFIG(2) 0x%x", val);
+ }
+
+ return err;
+}
+
+static bool icap_bitstream_in_use(struct icap *icap, pid_t pid)
+{
+ BUG_ON(icap->icap_bitstream_ref < 0);
+
+ /* Any user counts if pid isn't specified. */
+ if (pid == 0)
+ return icap->icap_bitstream_ref != 0;
+
+ if (icap->icap_bitstream_ref == 0)
+ return false;
+ if ((icap->icap_bitstream_ref == 1) && obtain_user(icap, pid))
+ return false;
+ return true;
+}
+
+static int icap_freeze_axi_gate_shell(struct icap *icap)
+{
+ xdev_handle_t xdev = xocl_get_xdev(icap->icap_pdev);
+
+ ICAP_INFO(icap, "freezing Shell AXI gate");
+ BUG_ON(icap->icap_axi_gate_shell_frozen);
+
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+ reg_wr(&icap->icap_axi_gate->iag_wr, GATE_FREEZE_SHELL);
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+
+ if (!xocl_is_unified(xdev)) {
+ reg_wr(&icap->icap_regs->ir_cr, 0xc);
+ ndelay(20);
+ } else {
+ /* New ICAP reset sequence applicable only to unified dsa. */
+ reg_wr(&icap->icap_regs->ir_cr, 0x8);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x0);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x4);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x0);
+ ndelay(2000);
+ }
+
+ icap->icap_axi_gate_shell_frozen = true;
+
+ return 0;
+}
+
+static int icap_free_axi_gate_shell(struct icap *icap)
+{
+ int i;
+
+ ICAP_INFO(icap, "freeing Shell AXI gate");
+ /*
+ * First pulse the OCL RESET. This is important for PR with multiple
+ * clocks as it resets the edge triggered clock converter FIFO
+ */
+
+ if (!icap->icap_axi_gate_shell_frozen)
+ return 0;
+
+ for (i = 0; i < ARRAY_SIZE(gate_free_shell); i++) {
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+ reg_wr(&icap->icap_axi_gate->iag_wr, gate_free_shell[i]);
+ mdelay(50);
+ }
+
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+
+ icap->icap_axi_gate_shell_frozen = false;
+
+ return 0;
+}
+
+static int icap_freeze_axi_gate(struct icap *icap)
+{
+ xdev_handle_t xdev = xocl_get_xdev(icap->icap_pdev);
+
+ ICAP_INFO(icap, "freezing CL AXI gate");
+ BUG_ON(icap->icap_axi_gate_frozen);
+
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+ reg_wr(&icap->icap_axi_gate->iag_wr, GATE_FREEZE_USER);
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+
+ if (!xocl_is_unified(xdev)) {
+ reg_wr(&icap->icap_regs->ir_cr, 0xc);
+ ndelay(20);
+ } else {
+ /* New ICAP reset sequence applicable only to unified dsa. */
+ reg_wr(&icap->icap_regs->ir_cr, 0x8);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x0);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x4);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x0);
+ ndelay(2000);
+ }
+
+ icap->icap_axi_gate_frozen = true;
+
+ return 0;
+}
+
+static int icap_free_axi_gate(struct icap *icap)
+{
+ int i;
+
+ ICAP_INFO(icap, "freeing CL AXI gate");
+ /*
+ * First pulse the OCL RESET. This is important for PR with multiple
+ * clocks as it resets the edge triggered clock converter FIFO
+ */
+
+ if (!icap->icap_axi_gate_frozen)
+ return 0;
+
+ for (i = 0; i < ARRAY_SIZE(gate_free_user); i++) {
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+ reg_wr(&icap->icap_axi_gate->iag_wr, gate_free_user[i]);
+ ndelay(500);
+ }
+
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+
+ icap->icap_axi_gate_frozen = false;
+
+ return 0;
+}
+
+static void platform_reset_axi_gate(struct platform_device *pdev)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+
+ /* Can only be done from mgmt pf. */
+ if (!ICAP_PRIVILEGED(icap))
+ return;
+
+ mutex_lock(&icap->icap_lock);
+ if (!icap_bitstream_in_use(icap, 0)) {
+ (void) icap_freeze_axi_gate(platform_get_drvdata(pdev));
+ msleep(500);
+ (void) icap_free_axi_gate(platform_get_drvdata(pdev));
+ msleep(500);
+ }
+ mutex_unlock(&icap->icap_lock);
+}
+
+static int set_freqs(struct icap *icap, unsigned short *freqs, int num_freqs)
+{
+ int i;
+ int err;
+ u32 val;
+
+ for (i = 0; i < min(ICAP_MAX_NUM_CLOCKS, num_freqs); ++i) {
+ if (freqs[i] == 0)
+ continue;
+
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_STATUS_OFFSET);
+ if ((val & 0x1) == 0) {
+ ICAP_ERR(icap, "clockwiz %d is busy", i);
+ err = -EBUSY;
+ goto done;
+ }
+ }
+
+ memcpy(icap->icap_ocl_frequency, freqs,
+ sizeof(*freqs) * min(ICAP_MAX_NUM_CLOCKS, num_freqs));
+
+ icap_freeze_axi_gate(icap);
+ err = icap_ocl_freqscaling(icap, false);
+ icap_free_axi_gate(icap);
+
+done:
+ return err;
+
+}
+
+static int set_and_verify_freqs(struct icap *icap, unsigned short *freqs, int num_freqs)
+{
+ int i;
+ int err;
+ u32 clock_freq_counter, request_in_khz, tolerance;
+
+ err = set_freqs(icap, freqs, num_freqs);
+ if (err)
+ return err;
+
+ for (i = 0; i < min(ICAP_MAX_NUM_CLOCKS, num_freqs); ++i) {
+ if (!freqs[i])
+ continue;
+ clock_freq_counter = icap_get_clock_frequency_counter_khz(icap, i);
+ if (clock_freq_counter == 0) {
+ err = -EDOM;
+ break;
+ }
+ request_in_khz = freqs[i]*1000;
+ tolerance = freqs[i]*50;
+ if (tolerance < abs(clock_freq_counter-request_in_khz)) {
+ ICAP_ERR(icap, "Frequency is higher than tolerance value, request %u khz, "
+ "actual %u khz", request_in_khz, clock_freq_counter);
+ err = -EDOM;
+ break;
+ }
+ }
+
+ return err;
+}
+
+static int icap_ocl_set_freqscaling(struct platform_device *pdev,
+ unsigned int region, unsigned short *freqs, int num_freqs)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int err = 0;
+
+ /* Can only be done from mgmt pf. */
+ if (!ICAP_PRIVILEGED(icap))
+ return -EPERM;
+
+ /* For now, only PR region 0 is supported. */
+ if (region != 0)
+ return -EINVAL;
+
+ mutex_lock(&icap->icap_lock);
+
+ err = set_freqs(icap, freqs, num_freqs);
+
+ mutex_unlock(&icap->icap_lock);
+
+ return err;
+}
+
+static int icap_ocl_update_clock_freq_topology(struct platform_device *pdev, struct xclmgmt_ioc_freqscaling *freq_obj)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ struct clock_freq_topology *topology = 0;
+ int num_clocks = 0;
+ int i = 0;
+ int err = 0;
+
+ mutex_lock(&icap->icap_lock);
+ if (icap->icap_clock_freq_topology) {
+ topology = (struct clock_freq_topology *)icap->icap_clock_freq_topology;
+ num_clocks = topology->m_count;
+ ICAP_INFO(icap, "Num clocks is %d", num_clocks);
+ for (i = 0; i < ARRAY_SIZE(freq_obj->ocl_target_freq); i++) {
+ ICAP_INFO(icap, "requested frequency is : "
+ "%d xclbin freq is: %d",
+ freq_obj->ocl_target_freq[i],
+ topology->m_clock_freq[i].m_freq_Mhz);
+ if (freq_obj->ocl_target_freq[i] >
+ topology->m_clock_freq[i].m_freq_Mhz) {
+ ICAP_ERR(icap, "Unable to set frequency as "
+ "requested frequency %d is greater "
+ "than set by xclbin %d",
+ freq_obj->ocl_target_freq[i],
+ topology->m_clock_freq[i].m_freq_Mhz);
+ err = -EDOM;
+ goto done;
+ }
+ }
+ } else {
+ ICAP_ERR(icap, "ERROR: There isn't a hardware accelerator loaded in the dynamic region."
+ " Validation of accelerator frequencies cannot be determine");
+ err = -EDOM;
+ goto done;
+ }
+
+ err = set_and_verify_freqs(icap, freq_obj->ocl_target_freq, ARRAY_SIZE(freq_obj->ocl_target_freq));
+
+done:
+ mutex_unlock(&icap->icap_lock);
+ return err;
+}
+
+static int icap_ocl_get_freqscaling(struct platform_device *pdev,
+ unsigned int region, unsigned short *freqs, int num_freqs)
+{
+ int i;
+ struct icap *icap = platform_get_drvdata(pdev);
+
+ /* For now, only PR region 0 is supported. */
+ if (region != 0)
+ return -EINVAL;
+
+ mutex_lock(&icap->icap_lock);
+ for (i = 0; i < min(ICAP_MAX_NUM_CLOCKS, num_freqs); i++)
+ freqs[i] = icap_get_ocl_frequency(icap, i);
+ mutex_unlock(&icap->icap_lock);
+
+ return 0;
+}
+
+static inline bool mig_calibration_done(struct icap *icap)
+{
+ return (reg_rd(&icap->icap_state->igs_state) & BIT(0)) != 0;
+}
+
+/* Check for MIG calibration. */
+static int calibrate_mig(struct icap *icap)
+{
+ int i;
+
+ for (i = 0; i < 10 && !mig_calibration_done(icap); ++i)
+ msleep(500);
+
+ if (!mig_calibration_done(icap)) {
+ ICAP_ERR(icap,
+ "MIG calibration timeout after bitstream download");
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+static inline void free_clock_freq_topology(struct icap *icap)
+{
+ vfree(icap->icap_clock_freq_topology);
+ icap->icap_clock_freq_topology = NULL;
+ icap->icap_clock_freq_topology_length = 0;
+}
+
+static int icap_setup_clock_freq_topology(struct icap *icap,
+ const char *buffer, unsigned long length)
+{
+ if (length == 0)
+ return 0;
+
+ free_clock_freq_topology(icap);
+
+ icap->icap_clock_freq_topology = vmalloc(length);
+ if (!icap->icap_clock_freq_topology)
+ return -ENOMEM;
+
+ memcpy(icap->icap_clock_freq_topology, buffer, length);
+ icap->icap_clock_freq_topology_length = length;
+
+ return 0;
+}
+
+static inline void free_clear_bitstream(struct icap *icap)
+{
+ vfree(icap->icap_clear_bitstream);
+ icap->icap_clear_bitstream = NULL;
+ icap->icap_clear_bitstream_length = 0;
+}
+
+static int icap_setup_clear_bitstream(struct icap *icap,
+ const char *buffer, unsigned long length)
+{
+ if (length == 0)
+ return 0;
+
+ free_clear_bitstream(icap);
+
+ icap->icap_clear_bitstream = vmalloc(length);
+ if (!icap->icap_clear_bitstream)
+ return -ENOMEM;
+
+ memcpy(icap->icap_clear_bitstream, buffer, length);
+ icap->icap_clear_bitstream_length = length;
+
+ return 0;
+}
+
+static int wait_for_done(struct icap *icap)
+{
+ u32 w;
+ int i = 0;
+
+ for (i = 0; i < 10; i++) {
+ udelay(5);
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ ICAP_INFO(icap, "XHWICAP_SR: %x", w);
+ if (w & 0x5)
+ return 0;
+ }
+
+ ICAP_ERR(icap, "bitstream download timeout");
+ return -ETIMEDOUT;
+}
+
+static int icap_write(struct icap *icap, const u32 *word_buf, int size)
+{
+ int i;
+ u32 value = 0;
+
+ for (i = 0; i < size; i++) {
+ value = be32_to_cpu(word_buf[i]);
+ reg_wr(&icap->icap_regs->ir_wf, value);
+ }
+
+ reg_wr(&icap->icap_regs->ir_cr, 0x1);
+
+ for (i = 0; i < 20; i++) {
+ value = reg_rd(&icap->icap_regs->ir_cr);
+ if ((value & 0x1) == 0)
+ return 0;
+ ndelay(50);
+ }
+
+ ICAP_ERR(icap, "writing %d dwords timeout", size);
+ return -EIO;
+}
+
+static uint64_t icap_get_section_size(struct icap *icap, enum axlf_section_kind kind)
+{
+ uint64_t size = 0;
+
+ switch (kind) {
+ case IP_LAYOUT:
+ size = sizeof_sect(icap->ip_layout, m_ip_data);
+ break;
+ case MEM_TOPOLOGY:
+ size = sizeof_sect(icap->mem_topo, m_mem_data);
+ break;
+ case DEBUG_IP_LAYOUT:
+ size = sizeof_sect(icap->debug_layout, m_debug_ip_data);
+ break;
+ case CONNECTIVITY:
+ size = sizeof_sect(icap->connectivity, m_connection);
+ break;
+ default:
+ break;
+ }
+
+ return size;
+}
+
+static int bitstream_parse_header(struct icap *icap, const unsigned char *Data,
+ unsigned int Size, struct XHwIcap_Bit_Header *Header)
+{
+ unsigned int I;
+ unsigned int Len;
+ unsigned int Tmp;
+ unsigned int Index;
+
+ /* Start Index at start of bitstream */
+ Index = 0;
+
+ /* Initialize HeaderLength. If header returned early inidicates
+ * failure.
+ */
+ Header->HeaderLength = XHI_BIT_HEADER_FAILURE;
+
+ /* Get "Magic" length */
+ Header->MagicLength = Data[Index++];
+ Header->MagicLength = (Header->MagicLength << 8) | Data[Index++];
+
+ /* Read in "magic" */
+ for (I = 0; I < Header->MagicLength - 1; I++) {
+ Tmp = Data[Index++];
+ if (I%2 == 0 && Tmp != XHI_EVEN_MAGIC_BYTE)
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ if (I%2 == 1 && Tmp != XHI_ODD_MAGIC_BYTE)
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ }
+
+ /* Read null end of magic data. */
+ Tmp = Data[Index++];
+
+ /* Read 0x01 (short) */
+ Tmp = Data[Index++];
+ Tmp = (Tmp << 8) | Data[Index++];
+
+ /* Check the "0x01" half word */
+ if (Tmp != 0x01)
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Read 'a' */
+ Tmp = Data[Index++];
+ if (Tmp != 'a')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get Design Name length */
+ Len = Data[Index++];
+ Len = (Len << 8) | Data[Index++];
+
+ /* allocate space for design name and final null character. */
+ Header->DesignName = kmalloc(Len, GFP_KERNEL);
+
+ /* Read in Design Name */
+ for (I = 0; I < Len; I++)
+ Header->DesignName[I] = Data[Index++];
+
+
+ if (Header->DesignName[Len-1] != '\0')
+ return -1;
+
+ /* Read 'b' */
+ Tmp = Data[Index++];
+ if (Tmp != 'b')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get Part Name length */
+ Len = Data[Index++];
+ Len = (Len << 8) | Data[Index++];
+
+ /* allocate space for part name and final null character. */
+ Header->PartName = kmalloc(Len, GFP_KERNEL);
+
+ /* Read in part name */
+ for (I = 0; I < Len; I++)
+ Header->PartName[I] = Data[Index++];
+
+ if (Header->PartName[Len-1] != '\0')
+ return -1;
+
+ /* Read 'c' */
+ Tmp = Data[Index++];
+ if (Tmp != 'c')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get date length */
+ Len = Data[Index++];
+ Len = (Len << 8) | Data[Index++];
+
+ /* allocate space for date and final null character. */
+ Header->Date = kmalloc(Len, GFP_KERNEL);
+
+ /* Read in date name */
+ for (I = 0; I < Len; I++)
+ Header->Date[I] = Data[Index++];
+
+ if (Header->Date[Len - 1] != '\0')
+ return -1;
+
+ /* Read 'd' */
+ Tmp = Data[Index++];
+ if (Tmp != 'd')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get time length */
+ Len = Data[Index++];
+ Len = (Len << 8) | Data[Index++];
+
+ /* allocate space for time and final null character. */
+ Header->Time = kmalloc(Len, GFP_KERNEL);
+
+ /* Read in time name */
+ for (I = 0; I < Len; I++)
+ Header->Time[I] = Data[Index++];
+
+ if (Header->Time[Len - 1] != '\0')
+ return -1;
+
+ /* Read 'e' */
+ Tmp = Data[Index++];
+ if (Tmp != 'e')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get byte length of bitstream */
+ Header->BitstreamLength = Data[Index++];
+ Header->BitstreamLength = (Header->BitstreamLength << 8) | Data[Index++];
+ Header->BitstreamLength = (Header->BitstreamLength << 8) | Data[Index++];
+ Header->BitstreamLength = (Header->BitstreamLength << 8) | Data[Index++];
+ Header->HeaderLength = Index;
+
+ ICAP_INFO(icap, "Design \"%s\"", Header->DesignName);
+ ICAP_INFO(icap, "Part \"%s\"", Header->PartName);
+ ICAP_INFO(icap, "Timestamp \"%s %s\"", Header->Time, Header->Date);
+ ICAP_INFO(icap, "Raw data size 0x%x", Header->BitstreamLength);
+ return 0;
+}
+
+static int bitstream_helper(struct icap *icap, const u32 *word_buffer,
+ unsigned int word_count)
+{
+ unsigned int remain_word;
+ unsigned int word_written = 0;
+ int wr_fifo_vacancy = 0;
+ int err = 0;
+
+ for (remain_word = word_count; remain_word > 0;
+ remain_word -= word_written, word_buffer += word_written) {
+ wr_fifo_vacancy = reg_rd(&icap->icap_regs->ir_wfv);
+ if (wr_fifo_vacancy <= 0) {
+ ICAP_ERR(icap, "no vacancy: %d", wr_fifo_vacancy);
+ err = -EIO;
+ break;
+ }
+ word_written = (wr_fifo_vacancy < remain_word) ?
+ wr_fifo_vacancy : remain_word;
+ if (icap_write(icap, word_buffer, word_written) != 0) {
+ err = -EIO;
+ break;
+ }
+ }
+
+ return err;
+}
+
+static long icap_download(struct icap *icap, const char *buffer,
+ unsigned long length)
+{
+ long err = 0;
+ struct XHwIcap_Bit_Header bit_header = { 0 };
+ unsigned int numCharsRead = DMA_HWICAP_BITFILE_BUFFER_SIZE;
+ unsigned int byte_read;
+
+ BUG_ON(!buffer);
+ BUG_ON(!length);
+
+ if (bitstream_parse_header(icap, buffer,
+ DMA_HWICAP_BITFILE_BUFFER_SIZE, &bit_header)) {
+ err = -EINVAL;
+ goto free_buffers;
+ }
+
+ if ((bit_header.HeaderLength + bit_header.BitstreamLength) > length) {
+ err = -EINVAL;
+ goto free_buffers;
+ }
+
+ buffer += bit_header.HeaderLength;
+
+ for (byte_read = 0; byte_read < bit_header.BitstreamLength;
+ byte_read += numCharsRead) {
+ numCharsRead = bit_header.BitstreamLength - byte_read;
+ if (numCharsRead > DMA_HWICAP_BITFILE_BUFFER_SIZE)
+ numCharsRead = DMA_HWICAP_BITFILE_BUFFER_SIZE;
+
+ err = bitstream_helper(icap, (u32 *)buffer,
+ numCharsRead / sizeof(u32));
+ if (err)
+ goto free_buffers;
+ buffer += numCharsRead;
+ }
+
+ err = wait_for_done(icap);
+
+free_buffers:
+ kfree(bit_header.DesignName);
+ kfree(bit_header.PartName);
+ kfree(bit_header.Date);
+ kfree(bit_header.Time);
+ return err;
+}
+
+static const struct axlf_section_header *get_axlf_section_hdr(
+ struct icap *icap, const struct axlf *top, enum axlf_section_kind kind)
+{
+ int i;
+ const struct axlf_section_header *hdr = NULL;
+
+ ICAP_INFO(icap,
+ "trying to find section header for axlf section %d", kind);
+
+ for (i = 0; i < top->m_header.m_numSections; i++) {
+ ICAP_INFO(icap, "saw section header: %d",
+ top->m_sections[i].m_sectionKind);
+ if (top->m_sections[i].m_sectionKind == kind) {
+ hdr = &top->m_sections[i];
+ break;
+ }
+ }
+
+ if (hdr) {
+ if ((hdr->m_sectionOffset + hdr->m_sectionSize) >
+ top->m_header.m_length) {
+ ICAP_INFO(icap, "found section is invalid");
+ hdr = NULL;
+ } else {
+ ICAP_INFO(icap, "header offset: %llu, size: %llu",
+ hdr->m_sectionOffset, hdr->m_sectionSize);
+ }
+ } else {
+ ICAP_INFO(icap, "could not find section header %d", kind);
+ }
+
+ return hdr;
+}
+
+static int alloc_and_get_axlf_section(struct icap *icap,
+ const struct axlf *top, enum axlf_section_kind kind,
+ void **addr, uint64_t *size)
+{
+ void *section = NULL;
+ const struct axlf_section_header *hdr =
+ get_axlf_section_hdr(icap, top, kind);
+
+ if (hdr == NULL)
+ return -EINVAL;
+
+ section = vmalloc(hdr->m_sectionSize);
+ if (section == NULL)
+ return -ENOMEM;
+
+ memcpy(section, ((const char *)top) + hdr->m_sectionOffset,
+ hdr->m_sectionSize);
+
+ *addr = section;
+ *size = hdr->m_sectionSize;
+ return 0;
+}
+
+static int icap_download_boot_firmware(struct platform_device *pdev)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ struct pci_dev *pcidev = XOCL_PL_TO_PCI_DEV(pdev);
+ struct pci_dev *pcidev_user = NULL;
+ xdev_handle_t xdev = xocl_get_xdev(pdev);
+ int funcid = PCI_FUNC(pcidev->devfn);
+ int slotid = PCI_SLOT(pcidev->devfn);
+ unsigned short deviceid = pcidev->device;
+ struct axlf *bin_obj_axlf;
+ const struct firmware *fw;
+ char fw_name[128];
+ struct XHwIcap_Bit_Header bit_header = { 0 };
+ long err = 0;
+ uint64_t length = 0;
+ uint64_t primaryFirmwareOffset = 0;
+ uint64_t primaryFirmwareLength = 0;
+ uint64_t secondaryFirmwareOffset = 0;
+ uint64_t secondaryFirmwareLength = 0;
+ uint64_t mbBinaryOffset = 0;
+ uint64_t mbBinaryLength = 0;
+ const struct axlf_section_header *primaryHeader = 0;
+ const struct axlf_section_header *secondaryHeader = 0;
+ const struct axlf_section_header *mbHeader = 0;
+ bool load_mbs = false;
+
+ /* Can only be done from mgmt pf. */
+ if (!ICAP_PRIVILEGED(icap))
+ return -EPERM;
+
+ /* Read dsabin from file system. */
+
+ if (funcid != 0) {
+ pcidev_user = pci_get_slot(pcidev->bus,
+ PCI_DEVFN(slotid, funcid - 1));
+ if (!pcidev_user) {
+ pcidev_user = pci_get_device(pcidev->vendor,
+ pcidev->device + 1, NULL);
+ }
+ if (pcidev_user)
+ deviceid = pcidev_user->device;
+ }
+
+ snprintf(fw_name, sizeof(fw_name),
+ "xilinx/%04x-%04x-%04x-%016llx.dsabin",
+ le16_to_cpu(pcidev->vendor),
+ le16_to_cpu(deviceid),
+ le16_to_cpu(pcidev->subsystem_device),
+ le64_to_cpu(xocl_get_timestamp(xdev)));
+ ICAP_INFO(icap, "try load dsabin %s", fw_name);
+ err = request_firmware(&fw, fw_name, &pcidev->dev);
+ if (err) {
+ snprintf(fw_name, sizeof(fw_name),
+ "xilinx/%04x-%04x-%04x-%016llx.dsabin",
+ le16_to_cpu(pcidev->vendor),
+ le16_to_cpu(deviceid + 1),
+ le16_to_cpu(pcidev->subsystem_device),
+ le64_to_cpu(xocl_get_timestamp(xdev)));
+ ICAP_INFO(icap, "try load dsabin %s", fw_name);
+ err = request_firmware(&fw, fw_name, &pcidev->dev);
+ }
+ /* Retry with the legacy dsabin. */
+ if (err) {
+ snprintf(fw_name, sizeof(fw_name),
+ "xilinx/%04x-%04x-%04x-%016llx.dsabin",
+ le16_to_cpu(pcidev->vendor),
+ le16_to_cpu(pcidev->device + 1),
+ le16_to_cpu(pcidev->subsystem_device),
+ le64_to_cpu(0x0000000000000000));
+ ICAP_INFO(icap, "try load dsabin %s", fw_name);
+ err = request_firmware(&fw, fw_name, &pcidev->dev);
+ }
+ if (err) {
+ /* Give up on finding .dsabin. */
+ ICAP_ERR(icap, "unable to find firmware, giving up");
+ return err;
+ }
+
+ /* Grab lock and touch hardware. */
+ mutex_lock(&icap->icap_lock);
+
+ if (xocl_mb_sched_on(xdev)) {
+ /* Try locating the microblaze binary. */
+ bin_obj_axlf = (struct axlf *)fw->data;
+ mbHeader = get_axlf_section_hdr(icap, bin_obj_axlf, SCHED_FIRMWARE);
+ if (mbHeader) {
+ mbBinaryOffset = mbHeader->m_sectionOffset;
+ mbBinaryLength = mbHeader->m_sectionSize;
+ length = bin_obj_axlf->m_header.m_length;
+ xocl_mb_load_sche_image(xdev, fw->data + mbBinaryOffset,
+ mbBinaryLength);
+ ICAP_INFO(icap, "stashed mb sche binary");
+ load_mbs = true;
+ }
+ }
+
+ if (xocl_mb_mgmt_on(xdev)) {
+ /* Try locating the board mgmt binary. */
+ bin_obj_axlf = (struct axlf *)fw->data;
+ mbHeader = get_axlf_section_hdr(icap, bin_obj_axlf, FIRMWARE);
+ if (mbHeader) {
+ mbBinaryOffset = mbHeader->m_sectionOffset;
+ mbBinaryLength = mbHeader->m_sectionSize;
+ length = bin_obj_axlf->m_header.m_length;
+ xocl_mb_load_mgmt_image(xdev, fw->data + mbBinaryOffset,
+ mbBinaryLength);
+ ICAP_INFO(icap, "stashed mb mgmt binary");
+ load_mbs = true;
+ }
+ }
+
+ if (load_mbs)
+ xocl_mb_reset(xdev);
+
+
+ if (memcmp(fw->data, ICAP_XCLBIN_V2, sizeof(ICAP_XCLBIN_V2)) != 0) {
+ ICAP_ERR(icap, "invalid firmware %s", fw_name);
+ err = -EINVAL;
+ goto done;
+ }
+
+ ICAP_INFO(icap, "boot_firmware in axlf format");
+ bin_obj_axlf = (struct axlf *)fw->data;
+ length = bin_obj_axlf->m_header.m_length;
+ /* Match the xclbin with the hardware. */
+ if (!xocl_verify_timestamp(xdev,
+ bin_obj_axlf->m_header.m_featureRomTimeStamp)) {
+ ICAP_ERR(icap, "timestamp of ROM did not match xclbin");
+ err = -EINVAL;
+ goto done;
+ }
+ ICAP_INFO(icap, "VBNV and timestamps matched");
+
+ if (xocl_xrt_version_check(xdev, bin_obj_axlf, true)) {
+ ICAP_ERR(icap, "Major version does not match xrt");
+ err = -EINVAL;
+ goto done;
+ }
+ ICAP_INFO(icap, "runtime version matched");
+
+ primaryHeader = get_axlf_section_hdr(icap, bin_obj_axlf, BITSTREAM);
+ secondaryHeader = get_axlf_section_hdr(icap, bin_obj_axlf,
+ CLEARING_BITSTREAM);
+ if (primaryHeader) {
+ primaryFirmwareOffset = primaryHeader->m_sectionOffset;
+ primaryFirmwareLength = primaryHeader->m_sectionSize;
+ }
+ if (secondaryHeader) {
+ secondaryFirmwareOffset = secondaryHeader->m_sectionOffset;
+ secondaryFirmwareLength = secondaryHeader->m_sectionSize;
+ }
+
+ if (length > fw->size) {
+ err = -EINVAL;
+ goto done;
+ }
+
+ if ((primaryFirmwareOffset + primaryFirmwareLength) > length) {
+ err = -EINVAL;
+ goto done;
+ }
+
+ if ((secondaryFirmwareOffset + secondaryFirmwareLength) > length) {
+ err = -EINVAL;
+ goto done;
+ }
+
+ if (primaryFirmwareLength) {
+ ICAP_INFO(icap,
+ "found second stage bitstream of size 0x%llx in %s",
+ primaryFirmwareLength, fw_name);
+ err = icap_download(icap, fw->data + primaryFirmwareOffset,
+ primaryFirmwareLength);
+ /*
+ * If we loaded a new second stage, we do not need the
+ * previously stashed clearing bitstream if any.
+ */
+ free_clear_bitstream(icap);
+ if (err) {
+ ICAP_ERR(icap,
+ "failed to download second stage bitstream");
+ goto done;
+ }
+ ICAP_INFO(icap, "downloaded second stage bitstream");
+ }
+
+ /*
+ * If both primary and secondary bitstreams have been provided then
+ * ignore the previously stashed bitstream if any. If only secondary
+ * bitstream was provided, but we found a previously stashed bitstream
+ * we should use the latter since it is more appropriate for the
+ * current state of the device
+ */
+ if (secondaryFirmwareLength && (primaryFirmwareLength ||
+ !icap->icap_clear_bitstream)) {
+ free_clear_bitstream(icap);
+ icap->icap_clear_bitstream = vmalloc(secondaryFirmwareLength);
+ if (!icap->icap_clear_bitstream) {
+ err = -ENOMEM;
+ goto done;
+ }
+ icap->icap_clear_bitstream_length = secondaryFirmwareLength;
+ memcpy(icap->icap_clear_bitstream,
+ fw->data + secondaryFirmwareOffset,
+ icap->icap_clear_bitstream_length);
+ ICAP_INFO(icap, "found clearing bitstream of size 0x%lx in %s",
+ icap->icap_clear_bitstream_length, fw_name);
+ } else if (icap->icap_clear_bitstream) {
+ ICAP_INFO(icap,
+ "using existing clearing bitstream of size 0x%lx",
+ icap->icap_clear_bitstream_length);
+ }
+
+ if (icap->icap_clear_bitstream &&
+ bitstream_parse_header(icap, icap->icap_clear_bitstream,
+ DMA_HWICAP_BITFILE_BUFFER_SIZE, &bit_header)) {
+ err = -EINVAL;
+ free_clear_bitstream(icap);
+ }
+
+done:
+ mutex_unlock(&icap->icap_lock);
+ release_firmware(fw);
+ kfree(bit_header.DesignName);
+ kfree(bit_header.PartName);
+ kfree(bit_header.Date);
+ kfree(bit_header.Time);
+ ICAP_INFO(icap, "%s err: %ld", __func__, err);
+ return err;
+}
+
+
+static long icap_download_clear_bitstream(struct icap *icap)
+{
+ long err = 0;
+ const char *buffer = icap->icap_clear_bitstream;
+ unsigned long length = icap->icap_clear_bitstream_length;
+
+ ICAP_INFO(icap, "downloading clear bitstream of length 0x%lx", length);
+
+ if (!buffer)
+ return 0;
+
+ err = icap_download(icap, buffer, length);
+
+ free_clear_bitstream(icap);
+ return err;
+}
+
+/*
+ * This function should be called with icap_mutex lock held
+ */
+static long axlf_set_freqscaling(struct icap *icap, struct platform_device *pdev,
+ const char *clk_buf, unsigned long length)
+{
+ struct clock_freq_topology *freqs = NULL;
+ int clock_type_count = 0;
+ int i = 0;
+ struct clock_freq *freq = NULL;
+ int data_clk_count = 0;
+ int kernel_clk_count = 0;
+ int system_clk_count = 0;
+ unsigned short target_freqs[4] = {0};
+
+ freqs = (struct clock_freq_topology *)clk_buf;
+ if (freqs->m_count > 4) {
+ ICAP_ERR(icap, "More than 4 clocks found in clock topology");
+ return -EDOM;
+ }
+
+ //Error checks - we support 1 data clk (reqd), one kernel clock(reqd) and
+ //at most 2 system clocks (optional/reqd for aws).
+ //Data clk needs to be the first entry, followed by kernel clock
+ //and then system clocks
+ //
+
+ for (i = 0; i < freqs->m_count; i++) {
+ freq = &(freqs->m_clock_freq[i]);
+ if (freq->m_type == CT_DATA)
+ data_clk_count++;
+
+ if (freq->m_type == CT_KERNEL)
+ kernel_clk_count++;
+
+ if (freq->m_type == CT_SYSTEM)
+ system_clk_count++;
+
+ }
+
+ if (data_clk_count != 1) {
+ ICAP_ERR(icap, "Data clock not found in clock topology");
+ return -EDOM;
+ }
+ if (kernel_clk_count != 1) {
+ ICAP_ERR(icap, "Kernel clock not found in clock topology");
+ return -EDOM;
+ }
+ if (system_clk_count > 2) {
+ ICAP_ERR(icap,
+ "More than 2 system clocks found in clock topology");
+ return -EDOM;
+ }
+
+ for (i = 0; i < freqs->m_count; i++) {
+ freq = &(freqs->m_clock_freq[i]);
+ if (freq->m_type == CT_DATA)
+ target_freqs[0] = freq->m_freq_Mhz;
+ }
+
+ for (i = 0; i < freqs->m_count; i++) {
+ freq = &(freqs->m_clock_freq[i]);
+ if (freq->m_type == CT_KERNEL)
+ target_freqs[1] = freq->m_freq_Mhz;
+ }
+
+ clock_type_count = 2;
+ for (i = 0; i < freqs->m_count; i++) {
+ freq = &(freqs->m_clock_freq[i]);
+ if (freq->m_type == CT_SYSTEM)
+ target_freqs[clock_type_count++] = freq->m_freq_Mhz;
+ }
+
+
+ ICAP_INFO(icap, "setting clock freq, "
+ "num: %lu, data_freq: %d , clk_freq: %d, "
+ "sys_freq[0]: %d, sys_freq[1]: %d",
+ ARRAY_SIZE(target_freqs), target_freqs[0], target_freqs[1],
+ target_freqs[2], target_freqs[3]);
+ return set_freqs(icap, target_freqs, 4);
+}
+
+
+static int icap_download_user(struct icap *icap, const char *bit_buf,
+ unsigned long length)
+{
+ long err = 0;
+ struct XHwIcap_Bit_Header bit_header = { 0 };
+ unsigned int numCharsRead = DMA_HWICAP_BITFILE_BUFFER_SIZE;
+ unsigned int byte_read;
+
+ ICAP_INFO(icap, "downloading bitstream, length: %lu", length);
+
+ icap_freeze_axi_gate(icap);
+
+ err = icap_download_clear_bitstream(icap);
+ if (err)
+ goto free_buffers;
+
+ if (bitstream_parse_header(icap, bit_buf,
+ DMA_HWICAP_BITFILE_BUFFER_SIZE, &bit_header)) {
+ err = -EINVAL;
+ goto free_buffers;
+ }
+ if ((bit_header.HeaderLength + bit_header.BitstreamLength) > length) {
+ err = -EINVAL;
+ goto free_buffers;
+ }
+
+ bit_buf += bit_header.HeaderLength;
+ for (byte_read = 0; byte_read < bit_header.BitstreamLength;
+ byte_read += numCharsRead) {
+ numCharsRead = bit_header.BitstreamLength - byte_read;
+ if (numCharsRead > DMA_HWICAP_BITFILE_BUFFER_SIZE)
+ numCharsRead = DMA_HWICAP_BITFILE_BUFFER_SIZE;
+
+ err = bitstream_helper(icap, (u32 *)bit_buf,
+ numCharsRead / sizeof(u32));
+ if (err)
+ goto free_buffers;
+
+ bit_buf += numCharsRead;
+ }
+
+ err = wait_for_done(icap);
+ if (err)
+ goto free_buffers;
+
+ /*
+ * Perform frequency scaling since PR download can silenty overwrite
+ * MMCM settings in static region changing the clock frequencies
+ * although ClockWiz CONFIG registers will misleading report the older
+ * configuration from before bitstream download as if nothing has
+ * changed.
+ */
+ if (!err)
+ err = icap_ocl_freqscaling(icap, true);
+
+free_buffers:
+ icap_free_axi_gate(icap);
+ kfree(bit_header.DesignName);
+ kfree(bit_header.PartName);
+ kfree(bit_header.Date);
+ kfree(bit_header.Time);
+ return err;
+}
+
+
+static int __icap_lock_peer(struct platform_device *pdev, const uuid_t *id)
+{
+ int err = 0;
+ struct icap *icap = platform_get_drvdata(pdev);
+ int resp = 0;
+ size_t resplen = sizeof(resp);
+ struct mailbox_req_bitstream_lock bitstream_lock = {0};
+ size_t data_len = sizeof(struct mailbox_req_bitstream_lock);
+ struct mailbox_req *mb_req = NULL;
+ size_t reqlen = sizeof(struct mailbox_req) + data_len;
+ /* if there is no user there
+ * ask mgmt to lock the bitstream
+ */
+ if (icap->icap_bitstream_ref == 0) {
+ mb_req = vmalloc(reqlen);
+ if (!mb_req) {
+ err = -ENOMEM;
+ goto done;
+ }
+
+ mb_req->req = MAILBOX_REQ_LOCK_BITSTREAM;
+ uuid_copy(&bitstream_lock.uuid, id);
+
+ memcpy(mb_req->data, &bitstream_lock, data_len);
+
+ err = xocl_peer_request(XOCL_PL_DEV_TO_XDEV(pdev),
+ mb_req, reqlen, &resp, &resplen, NULL, NULL);
+
+ if (err) {
+ err = -ENODEV;
+ goto done;
+ }
+
+ if (resp < 0) {
+ err = resp;
+ goto done;
+ }
+ }
+
+done:
+ vfree(mb_req);
+ return err;
+}
+
+static int __icap_unlock_peer(struct platform_device *pdev, const uuid_t *id)
+{
+ int err = 0;
+ struct icap *icap = platform_get_drvdata(pdev);
+ struct mailbox_req_bitstream_lock bitstream_lock = {0};
+ size_t data_len = sizeof(struct mailbox_req_bitstream_lock);
+ struct mailbox_req *mb_req = NULL;
+ size_t reqlen = sizeof(struct mailbox_req) + data_len;
+ /* if there is no user there
+ * ask mgmt to unlock the bitstream
+ */
+ if (icap->icap_bitstream_ref == 0) {
+ mb_req = vmalloc(reqlen);
+ if (!mb_req) {
+ err = -ENOMEM;
+ goto done;
+ }
+
+ mb_req->req = MAILBOX_REQ_UNLOCK_BITSTREAM;
+ memcpy(mb_req->data, &bitstream_lock, data_len);
+
+ err = xocl_peer_notify(XOCL_PL_DEV_TO_XDEV(pdev), mb_req, reqlen);
+ if (err) {
+ err = -ENODEV;
+ goto done;
+ }
+ }
+done:
+ vfree(mb_req);
+ return err;
+}
+
+
+static int icap_download_bitstream_axlf(struct platform_device *pdev,
+ const void *u_xclbin)
+{
+ /*
+ * decouple as 1. download xclbin, 2. parse xclbin 3. verify xclbin
+ */
+ struct icap *icap = platform_get_drvdata(pdev);
+ long err = 0;
+ uint64_t primaryFirmwareOffset = 0;
+ uint64_t primaryFirmwareLength = 0;
+ uint64_t secondaryFirmwareOffset = 0;
+ uint64_t secondaryFirmwareLength = 0;
+ const struct axlf_section_header *primaryHeader = NULL;
+ const struct axlf_section_header *clockHeader = NULL;
+ const struct axlf_section_header *secondaryHeader = NULL;
+ struct axlf *xclbin = (struct axlf *)u_xclbin;
+ char *buffer;
+ xdev_handle_t xdev = xocl_get_xdev(pdev);
+ bool need_download;
+ int msg = -ETIMEDOUT;
+ size_t resplen = sizeof(msg);
+ int pid = pid_nr(task_tgid(current));
+ uint32_t data_len = 0;
+ int peer_connected;
+ struct mailbox_req *mb_req = NULL;
+ struct mailbox_bitstream_kaddr mb_addr = {0};
+ uuid_t peer_uuid;
+
+ if (memcmp(xclbin->m_magic, ICAP_XCLBIN_V2, sizeof(ICAP_XCLBIN_V2)))
+ return -EINVAL;
+
+ if (ICAP_PRIVILEGED(icap)) {
+ if (xocl_xrt_version_check(xdev, xclbin, true)) {
+ ICAP_ERR(icap, "XRT version does not match");
+ return -EINVAL;
+ }
+
+ /* Match the xclbin with the hardware. */
+ if (!xocl_verify_timestamp(xdev,
+ xclbin->m_header.m_featureRomTimeStamp)) {
+ ICAP_ERR(icap, "timestamp of ROM not match Xclbin");
+ xocl_sysfs_error(xdev, "timestamp of ROM not match Xclbin");
+ return -EINVAL;
+ }
+
+ mutex_lock(&icap->icap_lock);
+
+ ICAP_INFO(icap,
+ "incoming xclbin: %016llx, on device xclbin: %016llx",
+ xclbin->m_uniqueId, icap->icap_bitstream_id);
+
+ need_download = (icap->icap_bitstream_id != xclbin->m_uniqueId);
+
+ if (!need_download) {
+ /*
+ * No need to download, if xclbin exists already.
+ * But, still need to reset CUs.
+ */
+ if (!icap_bitstream_in_use(icap, 0)) {
+ icap_freeze_axi_gate(icap);
+ msleep(50);
+ icap_free_axi_gate(icap);
+ msleep(50);
+ }
+ ICAP_INFO(icap, "bitstream already exists, skip downloading");
+ }
+
+ mutex_unlock(&icap->icap_lock);
+
+ if (!need_download)
+ return 0;
+
+ /*
+ * Find sections in xclbin.
+ */
+ ICAP_INFO(icap, "finding CLOCK_FREQ_TOPOLOGY section");
+ /* Read the CLOCK section but defer changing clocks to later */
+ clockHeader = get_axlf_section_hdr(icap, xclbin,
+ CLOCK_FREQ_TOPOLOGY);
+
+ ICAP_INFO(icap, "finding bitstream sections");
+ primaryHeader = get_axlf_section_hdr(icap, xclbin, BITSTREAM);
+ if (primaryHeader == NULL) {
+ err = -EINVAL;
+ goto done;
+ }
+ primaryFirmwareOffset = primaryHeader->m_sectionOffset;
+ primaryFirmwareLength = primaryHeader->m_sectionSize;
+
+ secondaryHeader = get_axlf_section_hdr(icap, xclbin,
+ CLEARING_BITSTREAM);
+ if (secondaryHeader) {
+ if (XOCL_PL_TO_PCI_DEV(pdev)->device == 0x7138) {
+ err = -EINVAL;
+ goto done;
+ } else {
+ secondaryFirmwareOffset =
+ secondaryHeader->m_sectionOffset;
+ secondaryFirmwareLength =
+ secondaryHeader->m_sectionSize;
+ }
+ }
+
+ mutex_lock(&icap->icap_lock);
+
+ if (icap_bitstream_in_use(icap, 0)) {
+ ICAP_ERR(icap, "bitstream is locked, can't download new one");
+ err = -EBUSY;
+ goto done;
+ }
+
+ /* All clear, go ahead and start fiddling with hardware */
+
+ if (clockHeader != NULL) {
+ uint64_t clockFirmwareOffset = clockHeader->m_sectionOffset;
+ uint64_t clockFirmwareLength = clockHeader->m_sectionSize;
+
+ buffer = (char *)xclbin;
+ buffer += clockFirmwareOffset;
+ err = axlf_set_freqscaling(icap, pdev, buffer, clockFirmwareLength);
+ if (err)
+ goto done;
+ err = icap_setup_clock_freq_topology(icap, buffer, clockFirmwareLength);
+ if (err)
+ goto done;
+ }
+
+ icap->icap_bitstream_id = 0;
+ uuid_copy(&icap->icap_bitstream_uuid, &uuid_null);
+
+ buffer = (char *)xclbin;
+ buffer += primaryFirmwareOffset;
+ err = icap_download_user(icap, buffer, primaryFirmwareLength);
+ if (err)
+ goto done;
+
+ buffer = (char *)u_xclbin;
+ buffer += secondaryFirmwareOffset;
+ err = icap_setup_clear_bitstream(icap, buffer, secondaryFirmwareLength);
+ if (err)
+ goto done;
+
+ if ((xocl_is_unified(xdev) || XOCL_DSA_XPR_ON(xdev)))
+ err = calibrate_mig(icap);
+ if (err)
+ goto done;
+
+ /* Remember "this" bitstream, so avoid redownload the next time. */
+ icap->icap_bitstream_id = xclbin->m_uniqueId;
+ if (!uuid_is_null(&xclbin->m_header.uuid)) {
+ uuid_copy(&icap->icap_bitstream_uuid, &xclbin->m_header.uuid);
+ } else {
+ // Legacy xclbin, convert legacy id to new id
+ memcpy(&icap->icap_bitstream_uuid,
+ &xclbin->m_header.m_timeStamp, 8);
+ }
+ } else {
+
+ mutex_lock(&icap->icap_lock);
+
+ if (icap_bitstream_in_use(icap, pid)) {
+ if (!uuid_equal(&xclbin->m_header.uuid, &icap->icap_bitstream_uuid)) {
+ err = -EBUSY;
+ goto done;
+ }
+ }
+
+ icap_read_from_peer(pdev, XCLBIN_UUID, &peer_uuid, sizeof(uuid_t));
+
+ if (!uuid_equal(&peer_uuid, &xclbin->m_header.uuid)) {
+ /*
+ * should replace with userpf download flow
+ */
+ peer_connected = xocl_mailbox_get_data(xdev, PEER_CONN);
+ ICAP_INFO(icap, "%s peer_connected 0x%x", __func__,
+ peer_connected);
+ if (peer_connected < 0) {
+ err = -ENODEV;
+ goto done;
+ }
+
+ if (!(peer_connected & MB_PEER_CONNECTED)) {
+ ICAP_ERR(icap, "%s fail to find peer, abort!",
+ __func__);
+ err = -EFAULT;
+ goto done;
+ }
+
+ if ((peer_connected & 0xF) == MB_PEER_SAMEDOM_CONNECTED) {
+ data_len = sizeof(struct mailbox_req) + sizeof(struct mailbox_bitstream_kaddr);
+ mb_req = vmalloc(data_len);
+ if (!mb_req) {
+ ICAP_ERR(icap, "Unable to create mb_req\n");
+ err = -ENOMEM;
+ goto done;
+ }
+ mb_req->req = MAILBOX_REQ_LOAD_XCLBIN_KADDR;
+ mb_addr.addr = (uint64_t)xclbin;
+ memcpy(mb_req->data, &mb_addr, sizeof(struct mailbox_bitstream_kaddr));
+
+ } else if ((peer_connected & 0xF) == MB_PEER_CONNECTED) {
+ data_len = sizeof(struct mailbox_req) +
+ xclbin->m_header.m_length;
+ mb_req = vmalloc(data_len);
+ if (!mb_req) {
+ ICAP_ERR(icap, "Unable to create mb_req\n");
+ err = -ENOMEM;
+ goto done;
+ }
+ memcpy(mb_req->data, u_xclbin, xclbin->m_header.m_length);
+ mb_req->req = MAILBOX_REQ_LOAD_XCLBIN;
+ }
+
+ mb_req->data_total_len = data_len;
+ (void) xocl_peer_request(xdev,
+ mb_req, data_len, &msg, &resplen, NULL, NULL);
+
+ if (msg != 0) {
+ ICAP_ERR(icap,
+ "%s peer failed to download xclbin",
+ __func__);
+ err = -EFAULT;
+ goto done;
+ }
+ } else
+ ICAP_INFO(icap, "Already downloaded xclbin ID: %016llx",
+ xclbin->m_uniqueId);
+
+ icap->icap_bitstream_id = xclbin->m_uniqueId;
+ if (!uuid_is_null(&xclbin->m_header.uuid)) {
+ uuid_copy(&icap->icap_bitstream_uuid, &xclbin->m_header.uuid);
+ } else {
+ // Legacy xclbin, convert legacy id to new id
+ memcpy(&icap->icap_bitstream_uuid,
+ &xclbin->m_header.m_timeStamp, 8);
+ }
+
+ }
+
+ if (ICAP_PRIVILEGED(icap)) {
+ icap_parse_bitstream_axlf_section(pdev, xclbin, MEM_TOPOLOGY);
+ icap_parse_bitstream_axlf_section(pdev, xclbin, IP_LAYOUT);
+ } else {
+ icap_parse_bitstream_axlf_section(pdev, xclbin, IP_LAYOUT);
+ icap_parse_bitstream_axlf_section(pdev, xclbin, MEM_TOPOLOGY);
+ icap_parse_bitstream_axlf_section(pdev, xclbin, CONNECTIVITY);
+ icap_parse_bitstream_axlf_section(pdev, xclbin, DEBUG_IP_LAYOUT);
+ }
+
+ if (ICAP_PRIVILEGED(icap))
+ err = icap_verify_bitstream_axlf(pdev, xclbin);
+
+done:
+ mutex_unlock(&icap->icap_lock);
+ vfree(mb_req);
+ ICAP_INFO(icap, "%s err: %ld", __func__, err);
+ return err;
+}
+
+static int icap_verify_bitstream_axlf(struct platform_device *pdev,
+ struct axlf *xclbin)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int err = 0, i;
+ xdev_handle_t xdev = xocl_get_xdev(pdev);
+ bool dna_check = false;
+ uint64_t section_size = 0;
+
+ /* Destroy all dynamically add sub-devices*/
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_DNA);
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_MIG);
+
+ /*
+ * Add sub device dynamically.
+ * restrict any dynamically added sub-device and 1 base address,
+ * Has pre-defined length
+ * Ex: "ip_data": {
+ * "m_type": "IP_DNASC",
+ * "properties": "0x0",
+ * "m_base_address": "0x1100000", <-- base address
+ * "m_name": "slr0\/dna_self_check_0"
+ */
+
+ if (!icap->ip_layout) {
+ err = -EFAULT;
+ goto done;
+ }
+ for (i = 0; i < icap->ip_layout->m_count; ++i) {
+ struct xocl_subdev_info subdev_info = { 0 };
+ struct resource res = { 0 };
+ struct ip_data *ip = &icap->ip_layout->m_ip_data[i];
+
+ if (ip->m_type == IP_KERNEL)
+ continue;
+
+ if (ip->m_type == IP_DDR4_CONTROLLER) {
+ uint32_t memidx = ip->properties;
+
+ if (!icap->mem_topo || ip->properties >= icap->mem_topo->m_count ||
+ icap->mem_topo->m_mem_data[memidx].m_type !=
+ MEM_DDR4) {
+ ICAP_ERR(icap, "bad ECC controller index: %u",
+ ip->properties);
+ continue;
+ }
+ if (!icap->mem_topo->m_mem_data[memidx].m_used) {
+ ICAP_INFO(icap,
+ "ignore ECC controller for: %s",
+ icap->mem_topo->m_mem_data[memidx].m_tag);
+ continue;
+ }
+ err = xocl_subdev_get_devinfo(XOCL_SUBDEV_MIG,
+ &subdev_info, &res);
+ if (err) {
+ ICAP_ERR(icap, "can't get MIG subdev info");
+ goto done;
+ }
+ res.start += ip->m_base_address;
+ res.end += ip->m_base_address;
+ subdev_info.priv_data =
+ icap->mem_topo->m_mem_data[memidx].m_tag;
+ subdev_info.data_len =
+ sizeof(icap->mem_topo->m_mem_data[memidx].m_tag);
+ err = xocl_subdev_create_multi_inst(xdev, &subdev_info);
+ if (err) {
+ ICAP_ERR(icap, "can't create MIG subdev");
+ goto done;
+ }
+ }
+ if (ip->m_type == IP_DNASC) {
+ dna_check = true;
+ err = xocl_subdev_get_devinfo(XOCL_SUBDEV_DNA,
+ &subdev_info, &res);
+ if (err) {
+ ICAP_ERR(icap, "can't get DNA subdev info");
+ goto done;
+ }
+ res.start += ip->m_base_address;
+ res.end += ip->m_base_address;
+ err = xocl_subdev_create_one(xdev, &subdev_info);
+ if (err) {
+ ICAP_ERR(icap, "can't create DNA subdev");
+ goto done;
+ }
+ }
+ }
+
+ if (dna_check) {
+ bool is_axi = ((xocl_dna_capability(xdev) & 0x1) != 0);
+
+ /*
+ * Any error occurs here should return -EACCES for app to
+ * know that DNA has failed.
+ */
+ err = -EACCES;
+
+ ICAP_INFO(icap, "DNA version: %s", is_axi ? "AXI" : "BRAM");
+
+ if (is_axi) {
+ uint32_t *cert = NULL;
+
+ if (alloc_and_get_axlf_section(icap, xclbin,
+ DNA_CERTIFICATE,
+ (void **)&cert, §ion_size) != 0) {
+
+ // We keep dna sub device if IP_DNASC presents
+ ICAP_ERR(icap, "Can't get certificate section");
+ goto dna_cert_fail;
+ }
+
+ ICAP_INFO(icap, "DNA Certificate Size 0x%llx", section_size);
+ if (section_size % 64 || section_size < 576)
+ ICAP_ERR(icap, "Invalid certificate size");
+ else
+ xocl_dna_write_cert(xdev, cert, section_size);
+ vfree(cert);
+ }
+
+ /* Check DNA validation result. */
+ if (0x1 & xocl_dna_status(xdev)) {
+ err = 0; /* xclbin is valid */
+ } else {
+ ICAP_ERR(icap, "DNA inside xclbin is invalid");
+ goto dna_cert_fail;
+ }
+ }
+
+done:
+ if (err) {
+ vfree(icap->connectivity);
+ icap->connectivity = NULL;
+ vfree(icap->ip_layout);
+ icap->ip_layout = NULL;
+ vfree(icap->mem_topo);
+ icap->mem_topo = NULL;
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_DNA);
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_MIG);
+ }
+dna_cert_fail:
+ return err;
+}
+
+/*
+ * On x86_64, reset hwicap by loading special bitstream sequence which
+ * forces the FPGA to reload from PROM.
+ */
+static int icap_reset_bitstream(struct platform_device *pdev)
+{
+/*
+ * Booting FPGA from PROM
+ * http://www.xilinx.com/support/documentation/user_guides/ug470_7Series_Config.pdf
+ * Table 7.1
+ */
+#define DUMMY_WORD 0xFFFFFFFF
+#define SYNC_WORD 0xAA995566
+#define TYPE1_NOOP 0x20000000
+#define TYPE1_WRITE_WBSTAR 0x30020001
+#define WBSTAR_ADD10 0x00000000
+#define WBSTAR_ADD11 0x01000000
+#define TYPE1_WRITE_CMD 0x30008001
+#define IPROG_CMD 0x0000000F
+#define SWAP_ENDIAN_32(x) \
+ (unsigned int)((((x) & 0xFF000000) >> 24) | (((x) & 0x00FF0000) >> 8) | \
+ (((x) & 0x0000FF00) << 8) | (((x) & 0x000000FF) << 24))
+ /*
+ * The bitstream is expected in big endian format
+ */
+ const unsigned int fpga_boot_seq[] = {SWAP_ENDIAN_32(DUMMY_WORD),
+ SWAP_ENDIAN_32(SYNC_WORD),
+ SWAP_ENDIAN_32(TYPE1_NOOP),
+ SWAP_ENDIAN_32(TYPE1_WRITE_CMD),
+ SWAP_ENDIAN_32(IPROG_CMD),
+ SWAP_ENDIAN_32(TYPE1_NOOP),
+ SWAP_ENDIAN_32(TYPE1_NOOP)};
+
+ struct icap *icap = platform_get_drvdata(pdev);
+ int i;
+
+ /* Can only be done from mgmt pf. */
+ if (!ICAP_PRIVILEGED(icap))
+ return -EPERM;
+
+ mutex_lock(&icap->icap_lock);
+
+ if (icap_bitstream_in_use(icap, 0)) {
+ mutex_unlock(&icap->icap_lock);
+ ICAP_ERR(icap, "bitstream is locked, can't reset");
+ return -EBUSY;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(fpga_boot_seq); i++) {
+ unsigned int value = be32_to_cpu(fpga_boot_seq[i]);
+
+ reg_wr(&icap->icap_regs->ir_wfv, value);
+ }
+ reg_wr(&icap->icap_regs->ir_cr, 0x1);
+
+ msleep(4000);
+
+ mutex_unlock(&icap->icap_lock);
+
+ ICAP_INFO(icap, "reset bitstream is done");
+ return 0;
+}
+
+static int icap_lock_bitstream(struct platform_device *pdev, const uuid_t *id,
+ pid_t pid)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int err = 0;
+
+ if (uuid_is_null(id)) {
+ ICAP_ERR(icap, "proc %d invalid UUID", pid);
+ return -EINVAL;
+ }
+
+ mutex_lock(&icap->icap_lock);
+
+ if (!ICAP_PRIVILEGED(icap)) {
+ err = __icap_lock_peer(pdev, id);
+ if (err < 0)
+ goto done;
+ }
+
+ if (uuid_equal(id, &icap->icap_bitstream_uuid))
+ err = add_user(icap, pid);
+ else
+ err = -EBUSY;
+
+ if (err >= 0)
+ err = icap->icap_bitstream_ref;
+
+ ICAP_INFO(icap, "proc %d try to lock bitstream %pUb, ref=%d, err=%d",
+ pid, id, icap->icap_bitstream_ref, err);
+done:
+ mutex_unlock(&icap->icap_lock);
+
+ if (!ICAP_PRIVILEGED(icap) && err == 1) /* reset on first reference */
+ xocl_exec_reset(xocl_get_xdev(pdev));
+
+ if (err >= 0)
+ err = 0;
+
+ return err;
+}
+
+static int icap_unlock_bitstream(struct platform_device *pdev, const uuid_t *id,
+ pid_t pid)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int err = 0;
+
+ if (id == NULL)
+ id = &uuid_null;
+
+ mutex_lock(&icap->icap_lock);
+
+ /* Force unlock. */
+ if (uuid_is_null(id))
+ del_all_users(icap);
+ else if (uuid_equal(id, &icap->icap_bitstream_uuid))
+ err = del_user(icap, pid);
+ else
+ err = -EINVAL;
+
+ if (!ICAP_PRIVILEGED(icap))
+ __icap_unlock_peer(pdev, id);
+
+ if (err >= 0)
+ err = icap->icap_bitstream_ref;
+
+ if (!ICAP_PRIVILEGED(icap)) {
+ if (err == 0)
+ xocl_exec_stop(xocl_get_xdev(pdev));
+ }
+
+ ICAP_INFO(icap, "proc %d try to unlock bitstream %pUb, ref=%d, err=%d",
+ pid, id, icap->icap_bitstream_ref, err);
+
+ mutex_unlock(&icap->icap_lock);
+ if (err >= 0)
+ err = 0;
+ return err;
+}
+
+static int icap_parse_bitstream_axlf_section(struct platform_device *pdev,
+ const struct axlf *xclbin, enum axlf_section_kind kind)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ long err = 0;
+ uint64_t section_size = 0, sect_sz = 0;
+ void **target = NULL;
+
+ if (memcmp(xclbin->m_magic, ICAP_XCLBIN_V2, sizeof(ICAP_XCLBIN_V2)))
+ return -EINVAL;
+
+ switch (kind) {
+ case IP_LAYOUT:
+ target = (void **)&icap->ip_layout;
+ break;
+ case MEM_TOPOLOGY:
+ target = (void **)&icap->mem_topo;
+ break;
+ case DEBUG_IP_LAYOUT:
+ target = (void **)&icap->debug_layout;
+ break;
+ case CONNECTIVITY:
+ target = (void **)&icap->connectivity;
+ break;
+ default:
+ break;
+ }
+ if (target) {
+ vfree(*target);
+ *target = NULL;
+ }
+ err = alloc_and_get_axlf_section(icap, xclbin, kind,
+ target, §ion_size);
+ if (err != 0)
+ goto done;
+ sect_sz = icap_get_section_size(icap, kind);
+ if (sect_sz > section_size) {
+ err = -EINVAL;
+ goto done;
+ }
+done:
+ if (err) {
+ vfree(*target);
+ *target = NULL;
+ }
+ ICAP_INFO(icap, "%s kind %d, err: %ld", __func__, kind, err);
+ return err;
+}
+
+static uint64_t icap_get_data(struct platform_device *pdev,
+ enum data_kind kind)
+{
+
+ struct icap *icap = platform_get_drvdata(pdev);
+ uint64_t target = 0;
+
+ mutex_lock(&icap->icap_lock);
+ switch (kind) {
+ case IPLAYOUT_AXLF:
+ target = (uint64_t)icap->ip_layout;
+ break;
+ case MEMTOPO_AXLF:
+ target = (uint64_t)icap->mem_topo;
+ break;
+ case DEBUG_IPLAYOUT_AXLF:
+ target = (uint64_t)icap->debug_layout;
+ break;
+ case CONNECTIVITY_AXLF:
+ target = (uint64_t)icap->connectivity;
+ break;
+ case IDCODE:
+ target = icap->idcode;
+ break;
+ case XCLBIN_UUID:
+ target = (uint64_t)&icap->icap_bitstream_uuid;
+ break;
+ default:
+ break;
+ }
+ mutex_unlock(&icap->icap_lock);
+ return target;
+}
+
+/* Kernel APIs exported from this sub-device driver. */
+static struct xocl_icap_funcs icap_ops = {
+ .reset_axi_gate = platform_reset_axi_gate,
+ .reset_bitstream = icap_reset_bitstream,
+ .download_boot_firmware = icap_download_boot_firmware,
+ .download_bitstream_axlf = icap_download_bitstream_axlf,
+ .ocl_set_freq = icap_ocl_set_freqscaling,
+ .ocl_get_freq = icap_ocl_get_freqscaling,
+ .ocl_update_clock_freq_topology = icap_ocl_update_clock_freq_topology,
+ .ocl_lock_bitstream = icap_lock_bitstream,
+ .ocl_unlock_bitstream = icap_unlock_bitstream,
+ .get_data = icap_get_data,
+};
+
+static ssize_t clock_freq_topology_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct icap *icap = platform_get_drvdata(to_platform_device(dev));
+ ssize_t cnt = 0;
+
+ mutex_lock(&icap->icap_lock);
+ if (ICAP_PRIVILEGED(icap)) {
+ memcpy(buf, icap->icap_clock_freq_topology, icap->icap_clock_freq_topology_length);
+ cnt = icap->icap_clock_freq_topology_length;
+ }
+ mutex_unlock(&icap->icap_lock);
+
+ return cnt;
+
+}
+
+static DEVICE_ATTR_RO(clock_freq_topology);
+
+static ssize_t clock_freqs_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct icap *icap = platform_get_drvdata(to_platform_device(dev));
+ ssize_t cnt = 0;
+ int i;
+ u32 freq_counter, freq, request_in_khz, tolerance;
+
+ mutex_lock(&icap->icap_lock);
+
+ for (i = 0; i < ICAP_MAX_NUM_CLOCKS; i++) {
+ freq = icap_get_ocl_frequency(icap, i);
+ if (!uuid_is_null(&icap->icap_bitstream_uuid)) {
+ freq_counter = icap_get_clock_frequency_counter_khz(icap, i);
+
+ request_in_khz = freq * 1000;
+ tolerance = freq * 50;
+
+ if (abs(freq_counter-request_in_khz) > tolerance)
+ ICAP_INFO(icap, "Frequency mismatch, Should be %u khz, Now is %ukhz",
+ request_in_khz, freq_counter);
+ cnt += sprintf(buf + cnt, "%d\n", DIV_ROUND_CLOSEST(freq_counter, 1000));
+ } else {
+ cnt += sprintf(buf + cnt, "%d\n", freq);
+ }
+ }
+
+ mutex_unlock(&icap->icap_lock);
+
+ return cnt;
+}
+static DEVICE_ATTR_RO(clock_freqs);
+
+static ssize_t icap_rl_program(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t off, size_t count)
+{
+ struct XHwIcap_Bit_Header bit_header = { 0 };
+ struct device *dev = container_of(kobj, struct device, kobj);
+ struct icap *icap = platform_get_drvdata(to_platform_device(dev));
+ ssize_t ret = count;
+
+ if (off == 0) {
+ if (count < DMA_HWICAP_BITFILE_BUFFER_SIZE) {
+ ICAP_ERR(icap, "count is too small %ld", count);
+ return -EINVAL;
+ }
+
+ if (bitstream_parse_header(icap, buffer,
+ DMA_HWICAP_BITFILE_BUFFER_SIZE, &bit_header)) {
+ ICAP_ERR(icap, "parse header failed");
+ return -EINVAL;
+ }
+
+ icap->bit_length = bit_header.HeaderLength +
+ bit_header.BitstreamLength;
+ icap->bit_buffer = vmalloc(icap->bit_length);
+ }
+
+ if (off + count >= icap->bit_length) {
+ /*
+ * assumes all subdevices are removed at this time
+ */
+ memcpy(icap->bit_buffer + off, buffer, icap->bit_length - off);
+ icap_freeze_axi_gate_shell(icap);
+ ret = icap_download(icap, icap->bit_buffer, icap->bit_length);
+ if (ret) {
+ ICAP_ERR(icap, "bitstream download failed");
+ ret = -EIO;
+ } else {
+ ret = count;
+ }
+ icap_free_axi_gate_shell(icap);
+ /* has to reset pci, otherwise firewall trips */
+ xocl_reset(xocl_get_xdev(icap->icap_pdev));
+ icap->icap_bitstream_id = 0;
+ memset(&icap->icap_bitstream_uuid, 0, sizeof(uuid_t));
+ vfree(icap->bit_buffer);
+ icap->bit_buffer = NULL;
+ } else {
+ memcpy(icap->bit_buffer + off, buffer, count);
+ }
+
+ return ret;
+}
+
+static struct bin_attribute shell_program_attr = {
+ .attr = {
+ .name = "shell_program",
+ .mode = 0200
+ },
+ .read = NULL,
+ .write = icap_rl_program,
+ .size = 0
+};
+
+static struct bin_attribute *icap_mgmt_bin_attrs[] = {
+ &shell_program_attr,
+ NULL,
+};
+
+static struct attribute_group icap_mgmt_bin_attr_group = {
+ .bin_attrs = icap_mgmt_bin_attrs,
+};
+
+static ssize_t idcode_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct icap *icap = platform_get_drvdata(to_platform_device(dev));
+ ssize_t cnt = 0;
+ uint32_t val;
+
+ mutex_lock(&icap->icap_lock);
+ if (ICAP_PRIVILEGED(icap)) {
+ cnt = sprintf(buf, "0x%x\n", icap->idcode);
+ } else {
+ icap_read_from_peer(to_platform_device(dev), IDCODE, &val, sizeof(unsigned int));
+ cnt = sprintf(buf, "0x%x\n", val);
+ }
+ mutex_unlock(&icap->icap_lock);
+
+ return cnt;
+}
+
+static DEVICE_ATTR_RO(idcode);
+
+static struct attribute *icap_attrs[] = {
+ &dev_attr_clock_freq_topology.attr,
+ &dev_attr_clock_freqs.attr,
+ &dev_attr_idcode.attr,
+ NULL,
+};
+
+//- Debug IP_layout--
+static ssize_t icap_read_debug_ip_layout(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ struct icap *icap;
+ u32 nread = 0;
+ size_t size = 0;
+
+ icap = (struct icap *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ if (!icap || !icap->debug_layout)
+ return 0;
+
+ mutex_lock(&icap->icap_lock);
+
+ size = sizeof_sect(icap->debug_layout, m_debug_ip_data);
+ if (offset >= size)
+ goto unlock;
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, ((char *)icap->debug_layout) + offset, nread);
+
+unlock:
+ mutex_unlock(&icap->icap_lock);
+ return nread;
+}
+static struct bin_attribute debug_ip_layout_attr = {
+ .attr = {
+ .name = "debug_ip_layout",
+ .mode = 0444
+ },
+ .read = icap_read_debug_ip_layout,
+ .write = NULL,
+ .size = 0
+};
+
+//IP layout
+static ssize_t icap_read_ip_layout(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ struct icap *icap;
+ u32 nread = 0;
+ size_t size = 0;
+
+ icap = (struct icap *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ if (!icap || !icap->ip_layout)
+ return 0;
+
+ mutex_lock(&icap->icap_lock);
+
+ size = sizeof_sect(icap->ip_layout, m_ip_data);
+ if (offset >= size)
+ goto unlock;
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, ((char *)icap->ip_layout) + offset, nread);
+
+unlock:
+ mutex_unlock(&icap->icap_lock);
+ return nread;
+}
+
+static struct bin_attribute ip_layout_attr = {
+ .attr = {
+ .name = "ip_layout",
+ .mode = 0444
+ },
+ .read = icap_read_ip_layout,
+ .write = NULL,
+ .size = 0
+};
+
+//-Connectivity--
+static ssize_t icap_read_connectivity(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ struct icap *icap;
+ u32 nread = 0;
+ size_t size = 0;
+
+ icap = (struct icap *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ if (!icap || !icap->connectivity)
+ return 0;
+
+ mutex_lock(&icap->icap_lock);
+
+ size = sizeof_sect(icap->connectivity, m_connection);
+ if (offset >= size)
+ goto unlock;
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, ((char *)icap->connectivity) + offset, nread);
+
+unlock:
+ mutex_unlock(&icap->icap_lock);
+ return nread;
+}
+
+static struct bin_attribute connectivity_attr = {
+ .attr = {
+ .name = "connectivity",
+ .mode = 0444
+ },
+ .read = icap_read_connectivity,
+ .write = NULL,
+ .size = 0
+};
+
+
+//-Mem_topology--
+static ssize_t icap_read_mem_topology(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ struct icap *icap;
+ u32 nread = 0;
+ size_t size = 0;
+
+ icap = (struct icap *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ if (!icap || !icap->mem_topo)
+ return 0;
+
+ mutex_lock(&icap->icap_lock);
+
+ size = sizeof_sect(icap->mem_topo, m_mem_data);
+ if (offset >= size)
+ goto unlock;
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, ((char *)icap->mem_topo) + offset, nread);
+unlock:
+ mutex_unlock(&icap->icap_lock);
+ return nread;
+}
+
+
+static struct bin_attribute mem_topology_attr = {
+ .attr = {
+ .name = "mem_topology",
+ .mode = 0444
+ },
+ .read = icap_read_mem_topology,
+ .write = NULL,
+ .size = 0
+};
+
+static struct bin_attribute *icap_bin_attrs[] = {
+ &debug_ip_layout_attr,
+ &ip_layout_attr,
+ &connectivity_attr,
+ &mem_topology_attr,
+ NULL,
+};
+
+static struct attribute_group icap_attr_group = {
+ .attrs = icap_attrs,
+ .bin_attrs = icap_bin_attrs,
+};
+
+static int icap_remove(struct platform_device *pdev)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int i;
+
+ BUG_ON(icap == NULL);
+
+ del_all_users(icap);
+ xocl_subdev_register(pdev, XOCL_SUBDEV_ICAP, NULL);
+
+ if (ICAP_PRIVILEGED(icap))
+ sysfs_remove_group(&pdev->dev.kobj, &icap_mgmt_bin_attr_group);
+
+ if (icap->bit_buffer)
+ vfree(icap->bit_buffer);
+
+ iounmap(icap->icap_regs);
+ iounmap(icap->icap_state);
+ iounmap(icap->icap_axi_gate);
+ for (i = 0; i < ICAP_MAX_NUM_CLOCKS; i++)
+ iounmap(icap->icap_clock_bases[i]);
+ free_clear_bitstream(icap);
+ free_clock_freq_topology(icap);
+
+ sysfs_remove_group(&pdev->dev.kobj, &icap_attr_group);
+
+ ICAP_INFO(icap, "cleaned up successfully");
+ platform_set_drvdata(pdev, NULL);
+ vfree(icap->mem_topo);
+ vfree(icap->ip_layout);
+ vfree(icap->debug_layout);
+ vfree(icap->connectivity);
+ kfree(icap);
+ return 0;
+}
+
+/*
+ * Run the following sequence of canned commands to obtain IDCODE of the FPGA
+ */
+static void icap_probe_chip(struct icap *icap)
+{
+ u32 w;
+
+ if (!ICAP_PRIVILEGED(icap))
+ return;
+
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ reg_wr(&icap->icap_regs->ir_gier, 0x0);
+ w = reg_rd(&icap->icap_regs->ir_wfv);
+ reg_wr(&icap->icap_regs->ir_wf, 0xffffffff);
+ reg_wr(&icap->icap_regs->ir_wf, 0xaa995566);
+ reg_wr(&icap->icap_regs->ir_wf, 0x20000000);
+ reg_wr(&icap->icap_regs->ir_wf, 0x20000000);
+ reg_wr(&icap->icap_regs->ir_wf, 0x28018001);
+ reg_wr(&icap->icap_regs->ir_wf, 0x20000000);
+ reg_wr(&icap->icap_regs->ir_wf, 0x20000000);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ reg_wr(&icap->icap_regs->ir_cr, 0x1);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ reg_wr(&icap->icap_regs->ir_sz, 0x1);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ reg_wr(&icap->icap_regs->ir_cr, 0x2);
+ w = reg_rd(&icap->icap_regs->ir_rfo);
+ icap->idcode = reg_rd(&icap->icap_regs->ir_rf);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+}
+
+static int icap_probe(struct platform_device *pdev)
+{
+ struct icap *icap = NULL;
+ struct resource *res;
+ int ret;
+ int reg_grp;
+ void **regs;
+
+ icap = kzalloc(sizeof(struct icap), GFP_KERNEL);
+ if (!icap)
+ return -ENOMEM;
+ platform_set_drvdata(pdev, icap);
+ icap->icap_pdev = pdev;
+ mutex_init(&icap->icap_lock);
+ INIT_LIST_HEAD(&icap->icap_bitstream_users);
+
+ for (reg_grp = 0; reg_grp < ICAP_MAX_REG_GROUPS; reg_grp++) {
+ switch (reg_grp) {
+ case 0:
+ regs = (void **)&icap->icap_regs;
+ break;
+ case 1:
+ regs = (void **)&icap->icap_state;
+ break;
+ case 2:
+ regs = (void **)&icap->icap_axi_gate;
+ break;
+ case 3:
+ regs = (void **)&icap->icap_clock_bases[0];
+ break;
+ case 4:
+ regs = (void **)&icap->icap_clock_bases[1];
+ break;
+ case 5:
+ regs = (void **)&icap->icap_clock_freq_counter;
+ break;
+ default:
+ BUG();
+ break;
+ }
+ res = platform_get_resource(pdev, IORESOURCE_MEM, reg_grp);
+ if (res != NULL) {
+ *regs = ioremap_nocache(res->start,
+ res->end - res->start + 1);
+ if (*regs == NULL) {
+ ICAP_ERR(icap,
+ "failed to map in register group: %d",
+ reg_grp);
+ ret = -EIO;
+ goto failed;
+ } else {
+ ICAP_INFO(icap,
+ "mapped in register group %d @ 0x%p",
+ reg_grp, *regs);
+ }
+ } else {
+ if (reg_grp != 0) {
+ ICAP_ERR(icap,
+ "failed to find register group: %d",
+ reg_grp);
+ ret = -EIO;
+ goto failed;
+ }
+ break;
+ }
+ }
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &icap_attr_group);
+ if (ret) {
+ ICAP_ERR(icap, "create icap attrs failed: %d", ret);
+ goto failed;
+ }
+
+ if (ICAP_PRIVILEGED(icap)) {
+ ret = sysfs_create_group(&pdev->dev.kobj,
+ &icap_mgmt_bin_attr_group);
+ if (ret) {
+ ICAP_ERR(icap, "create icap attrs failed: %d", ret);
+ goto failed;
+ }
+ }
+
+ icap_probe_chip(icap);
+ if (!ICAP_PRIVILEGED(icap))
+ icap_unlock_bitstream(pdev, NULL, 0);
+ ICAP_INFO(icap, "successfully initialized FPGA IDCODE 0x%x",
+ icap->idcode);
+ xocl_subdev_register(pdev, XOCL_SUBDEV_ICAP, &icap_ops);
+ return 0;
+
+failed:
+ (void) icap_remove(pdev);
+ return ret;
+}
+
+
+struct platform_device_id icap_id_table[] = {
+ { XOCL_ICAP, 0 },
+ { },
+};
+
+static struct platform_driver icap_driver = {
+ .probe = icap_probe,
+ .remove = icap_remove,
+ .driver = {
+ .name = XOCL_ICAP,
+ },
+ .id_table = icap_id_table,
+};
+
+int __init xocl_init_icap(void)
+{
+ return platform_driver_register(&icap_driver);
+}
+
+void xocl_fini_icap(void)
+{
+ platform_driver_unregister(&icap_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/mailbox.c b/drivers/gpu/drm/xocl/subdev/mailbox.c
new file mode 100644
index 000000000000..dc4736c9100a
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/mailbox.c
@@ -0,0 +1,1868 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Max Zhen <maxz@xilinx.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+/*
+ * Statement of Theory
+ *
+ * This is the mailbox sub-device driver added into existing xclmgmt / xocl
+ * driver so that user pf and mgmt pf can send and receive messages of
+ * arbitrary length to / from peer. The driver is written based on the spec of
+ * pg114 document (https://www.xilinx.com/support/documentation/
+ * ip_documentation/mailbox/v2_1/pg114-mailbox.pdf). The HW provides one TX
+ * channel and one RX channel, which operate completely independent of each
+ * other. Data can be pushed into or read from a channel in DWORD unit as a
+ * FIFO.
+ *
+ *
+ * Packet layer
+ *
+ * The driver implemented two transport layers - packet and message layer (see
+ * below). A packet is a fixed size chunk of data that can be send through TX
+ * channel or retrieved from RX channel. The TX and RX interrupt happens at
+ * packet boundary, instead of DWORD boundary. The driver will not attempt to
+ * send next packet until the previous one is read by peer. Similarly, the
+ * driver will not attempt to read the data from HW until a full packet has been
+ * written to HW by peer. No polling is implemented. Data transfer is entirely
+ * interrupt driven. So, the interrupt functionality needs to work and enabled
+ * on both mgmt and user pf for mailbox driver to function properly.
+ *
+ * A TX packet is considered as time'd out after sitting in the TX channel of
+ * mailbox HW for two packet ticks (1 packet tick = 1 second, for now) without
+ * being read by peer. Currently, the driver will not try to re-transmit the
+ * packet after timeout. It just simply propagate the error to the upper layer.
+ * A retry at packet layer can be implement later, if considered as appropriate.
+ *
+ *
+ * Message layer
+ *
+ * A message is a data buffer of arbitrary length. The driver will break a
+ * message into multiple packets and transmit them to the peer, which, in turn,
+ * will assemble them into a full message before it's delivered to upper layer
+ * for further processing. One message requires at least one packet to be
+ * transferred to the peer.
+ *
+ * Each message has a unique temporary u64 ID (see communication model below
+ * for more detail). The ID shows up in each packet's header. So, at packet
+ * layer, there is no assumption that adjacent packets belong to the same
+ * message. However, for the sake of simplicity, at message layer, the driver
+ * will not attempt to send the next message until the sending of current one
+ * is finished. I.E., we implement a FIFO for message TX channel. All messages
+ * are sent by driver in the order of received from upper layer. We can
+ * implement messages of different priority later, if needed. There is no
+ * certain order for receiving messages. It's up to the peer side to decide
+ * which message gets enqueued into its own TX queue first, which will be
+ * received first on the other side.
+ *
+ * A message is considered as time'd out when it's transmit (send or receive)
+ * is not finished within 10 packet ticks. This applies to all messages queued
+ * up on both RX and TX channels. Again, no retry for a time'd out message is
+ * implemented. The error will be simply passed to upper layer. Also, a TX
+ * message may time out earlier if it's being transmitted and one of it's
+ * packets time'd out. During normal operation, timeout should never happen.
+ *
+ * The upper layer can choose to queue a message for TX or RX asynchronously
+ * when it provides a callback or wait synchronously when no callback is
+ * provided.
+ *
+ *
+ * Communication model
+ *
+ * At the highest layer, the driver implements a request-response communication
+ * model. A request may or may not require a response, but a response must match
+ * a request, or it'll be silently dropped. The driver provides a few kernel
+ * APIs for mgmt and user pf to talk to each other in this model (see kernel
+ * APIs section below for details). Each request or response is a message by
+ * itself. A request message will automatically be assigned a message ID when
+ * it's enqueued into TX channel for sending. If this request requires a
+ * response, the buffer provided by caller for receiving response will be
+ * enqueued into RX channel as well. The enqueued response message will have
+ * the same message ID as the corresponding request message. The response
+ * message, if provided, will always be enqueued before the request message is
+ * enqueued to avoid race condition.
+ *
+ * The driver will automatically enqueue a special message into the RX channel
+ * for receiving new request after initialized. This request RX message has a
+ * special message ID (id=0) and never time'd out. When a new request comes
+ * from peer, it'll be copied into request RX message then passed to the
+ * callback provided by upper layer through xocl_peer_listen() API for further
+ * processing. Currently, the driver implements only one kernel thread for RX
+ * channel and one for TX channel. So, all message callback happens in the
+ * context of that channel thread. So, the user of mailbox driver needs to be
+ * careful when it calls xocl_peer_request() synchronously in this context.
+ * You may see deadlock when both ends are trying to call xocl_peer_request()
+ * synchronously at the same time.
+ *
+ *
+ * +------------------+ +------------------+
+ * | Request/Response | <--------> | Request/Response |
+ * +------------------+ +------------------+
+ * | Message | <--------> | Message |
+ * +------------------+ +------------------+
+ * | Packet | <--------> | Packet |
+ * +------------------+ +------------------+
+ * | RX/TX Channel | <<======>> | RX/TX Channel |
+ * +------------------+ +------------------+
+ * mgmt pf user pf
+ */
+
+#include <linux/mutex.h>
+#include <linux/completion.h>
+#include <linux/list.h>
+#include <linux/device.h>
+#include <linux/crc32c.h>
+#include <linux/random.h>
+#include "../xocl_drv.h"
+
+int mailbox_no_intr;
+module_param(mailbox_no_intr, int, (S_IRUGO|S_IWUSR));
+MODULE_PARM_DESC(mailbox_no_intr,
+ "Disable mailbox interrupt and do timer-driven msg passing");
+
+#define PACKET_SIZE 16 /* Number of DWORD. */
+
+#define FLAG_STI (1 << 0)
+#define FLAG_RTI (1 << 1)
+
+#define STATUS_EMPTY (1 << 0)
+#define STATUS_FULL (1 << 1)
+#define STATUS_STA (1 << 2)
+#define STATUS_RTA (1 << 3)
+
+#define MBX_ERR(mbx, fmt, arg...) \
+ xocl_err(&mbx->mbx_pdev->dev, fmt "\n", ##arg)
+#define MBX_INFO(mbx, fmt, arg...) \
+ xocl_info(&mbx->mbx_pdev->dev, fmt "\n", ##arg)
+#define MBX_DBG(mbx, fmt, arg...) \
+ xocl_dbg(&mbx->mbx_pdev->dev, fmt "\n", ##arg)
+
+#define MAILBOX_TIMER HZ /* in jiffies */
+#define MSG_TTL 10 /* in MAILBOX_TIMER */
+#define TEST_MSG_LEN 128
+
+#define INVALID_MSG_ID ((u64)-1)
+#define MSG_FLAG_RESPONSE (1 << 0)
+#define MSG_FLAG_REQUEST (1 << 1)
+
+#define MAX_MSG_QUEUE_SZ (PAGE_SIZE << 16)
+#define MAX_MSG_QUEUE_LEN 5
+
+#define MB_CONN_INIT (0x1<<0)
+#define MB_CONN_SYN (0x1<<1)
+#define MB_CONN_ACK (0x1<<2)
+#define MB_CONN_FIN (0x1<<3)
+
+/*
+ * Mailbox IP register layout
+ */
+struct mailbox_reg {
+ u32 mbr_wrdata;
+ u32 mbr_resv1;
+ u32 mbr_rddata;
+ u32 mbr_resv2;
+ u32 mbr_status;
+ u32 mbr_error;
+ u32 mbr_sit;
+ u32 mbr_rit;
+ u32 mbr_is;
+ u32 mbr_ie;
+ u32 mbr_ip;
+ u32 mbr_ctrl;
+} __attribute__((packed));
+
+/*
+ * A message transport by mailbox.
+ */
+struct mailbox_msg {
+ struct list_head mbm_list;
+ struct mailbox_channel *mbm_ch;
+ u64 mbm_req_id;
+ char *mbm_data;
+ size_t mbm_len;
+ int mbm_error;
+ struct completion mbm_complete;
+ mailbox_msg_cb_t mbm_cb;
+ void *mbm_cb_arg;
+ u32 mbm_flags;
+ int mbm_ttl;
+ bool mbm_timer_on;
+};
+
+/*
+ * A packet transport by mailbox.
+ * When extending, only add new data structure to body. Choose to add new flag
+ * if new feature can be safely ignored by peer, other wise, add new type.
+ */
+enum packet_type {
+ PKT_INVALID = 0,
+ PKT_TEST,
+ PKT_MSG_START,
+ PKT_MSG_BODY
+};
+
+
+enum conn_state {
+ CONN_START = 0,
+ CONN_SYN_SENT,
+ CONN_SYN_RECV,
+ CONN_ESTABLISH,
+};
+
+/* Lower 8 bits for type, the rest for flags. */
+#define PKT_TYPE_MASK 0xff
+#define PKT_TYPE_MSG_END (1 << 31)
+struct mailbox_pkt {
+ struct {
+ u32 type;
+ u32 payload_size;
+ } hdr;
+ union {
+ u32 data[PACKET_SIZE - 2];
+ struct {
+ u64 msg_req_id;
+ u32 msg_flags;
+ u32 msg_size;
+ u32 payload[0];
+ } msg_start;
+ struct {
+ u32 payload[0];
+ } msg_body;
+ } body;
+} __attribute__((packed));
+
+/*
+ * Mailbox communication channel.
+ */
+#define MBXCS_BIT_READY 0
+#define MBXCS_BIT_STOP 1
+#define MBXCS_BIT_TICK 2
+#define MBXCS_BIT_CHK_STALL 3
+#define MBXCS_BIT_POLL_MODE 4
+
+struct mailbox_channel;
+typedef void (*chan_func_t)(struct mailbox_channel *ch);
+struct mailbox_channel {
+ struct mailbox *mbc_parent;
+ char *mbc_name;
+
+ struct workqueue_struct *mbc_wq;
+ struct work_struct mbc_work;
+ struct completion mbc_worker;
+ chan_func_t mbc_tran;
+ unsigned long mbc_state;
+
+ struct mutex mbc_mutex;
+ struct list_head mbc_msgs;
+
+ struct mailbox_msg *mbc_cur_msg;
+ int mbc_bytes_done;
+ struct mailbox_pkt mbc_packet;
+
+ struct timer_list mbc_timer;
+ bool mbc_timer_on;
+};
+
+/*
+ * The mailbox softstate.
+ */
+struct mailbox {
+ struct platform_device *mbx_pdev;
+ struct mailbox_reg *mbx_regs;
+ u32 mbx_irq;
+
+ struct mailbox_channel mbx_rx;
+ struct mailbox_channel mbx_tx;
+
+ /* For listening to peer's request. */
+ mailbox_msg_cb_t mbx_listen_cb;
+ void *mbx_listen_cb_arg;
+ struct workqueue_struct *mbx_listen_wq;
+ struct work_struct mbx_listen_worker;
+
+ int mbx_paired;
+ /*
+ * For testing basic intr and mailbox comm functionality via sysfs.
+ * No locking protection, use with care.
+ */
+ struct mailbox_pkt mbx_tst_pkt;
+ char mbx_tst_tx_msg[TEST_MSG_LEN];
+ char mbx_tst_rx_msg[TEST_MSG_LEN];
+ size_t mbx_tst_tx_msg_len;
+
+ /* Req list for all incoming request message */
+ struct completion mbx_comp;
+ struct mutex mbx_lock;
+ struct list_head mbx_req_list;
+ uint8_t mbx_req_cnt;
+ size_t mbx_req_sz;
+
+ struct mutex mbx_conn_lock;
+ uint64_t mbx_conn_id;
+ enum conn_state mbx_state;
+ bool mbx_established;
+ uint32_t mbx_prot_ver;
+
+ void *mbx_kaddr;
+};
+
+static inline const char *reg2name(struct mailbox *mbx, u32 *reg)
+{
+ static const char *reg_names[] = {
+ "wrdata",
+ "reserved1",
+ "rddata",
+ "reserved2",
+ "status",
+ "error",
+ "sit",
+ "rit",
+ "is",
+ "ie",
+ "ip",
+ "ctrl"
+ };
+
+ return reg_names[((uintptr_t)reg -
+ (uintptr_t)mbx->mbx_regs) / sizeof(u32)];
+}
+
+struct mailbox_conn {
+ uint64_t flag;
+ void *kaddr;
+ phys_addr_t paddr;
+ uint32_t crc32;
+ uint32_t ver;
+ uint64_t sec_id;
+};
+
+int mailbox_request(struct platform_device *pdev, void *req, size_t reqlen,
+ void *resp, size_t *resplen, mailbox_msg_cb_t cb, void *cbarg);
+int mailbox_post(struct platform_device *pdev, u64 reqid, void *buf, size_t len);
+static int mailbox_connect_status(struct platform_device *pdev);
+static void connect_state_handler(struct mailbox *mbx, struct mailbox_conn *conn);
+
+static void connect_state_touch(struct mailbox *mbx, uint64_t flag)
+{
+ struct mailbox_conn conn = {0};
+ if (!mbx)
+ return;
+ conn.flag = flag;
+ connect_state_handler(mbx, &conn);
+}
+
+
+static inline u32 mailbox_reg_rd(struct mailbox *mbx, u32 *reg)
+{
+ u32 val = ioread32(reg);
+
+#ifdef MAILBOX_REG_DEBUG
+ MBX_DBG(mbx, "REG_RD(%s)=0x%x", reg2name(mbx, reg), val);
+#endif
+ return val;
+}
+
+static inline void mailbox_reg_wr(struct mailbox *mbx, u32 *reg, u32 val)
+{
+#ifdef MAILBOX_REG_DEBUG
+ MBX_DBG(mbx, "REG_WR(%s, 0x%x)", reg2name(mbx, reg), val);
+#endif
+ iowrite32(val, reg);
+}
+
+static inline void reset_pkt(struct mailbox_pkt *pkt)
+{
+ pkt->hdr.type = PKT_INVALID;
+}
+
+static inline bool valid_pkt(struct mailbox_pkt *pkt)
+{
+ return (pkt->hdr.type != PKT_INVALID);
+}
+
+irqreturn_t mailbox_isr(int irq, void *arg)
+{
+ struct mailbox *mbx = (struct mailbox *)arg;
+ u32 is = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_is);
+
+ while (is) {
+ MBX_DBG(mbx, "intr status: 0x%x", is);
+
+ if ((is & FLAG_STI) != 0) {
+ /* A packet has been sent successfully. */
+ complete(&mbx->mbx_tx.mbc_worker);
+ }
+ if ((is & FLAG_RTI) != 0) {
+ /* A packet is waiting to be received from mailbox. */
+ complete(&mbx->mbx_rx.mbc_worker);
+ }
+ /* Anything else is not expected. */
+ if ((is & (FLAG_STI | FLAG_RTI)) == 0) {
+ MBX_ERR(mbx, "spurious mailbox irq %d, is=0x%x",
+ irq, is);
+ }
+
+ /* Clear intr state for receiving next one. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_is, is);
+
+ is = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_is);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static void chan_timer(struct timer_list *t)
+{
+ struct mailbox_channel *ch = from_timer(ch, t, mbc_timer);
+
+ MBX_DBG(ch->mbc_parent, "%s tick", ch->mbc_name);
+
+ set_bit(MBXCS_BIT_TICK, &ch->mbc_state);
+ complete(&ch->mbc_worker);
+
+ /* We're a periodic timer. */
+ mod_timer(&ch->mbc_timer, jiffies + MAILBOX_TIMER);
+}
+
+static void chan_config_timer(struct mailbox_channel *ch)
+{
+ struct list_head *pos, *n;
+ struct mailbox_msg *msg = NULL;
+ bool on = false;
+
+ mutex_lock(&ch->mbc_mutex);
+
+ if (test_bit(MBXCS_BIT_POLL_MODE, &ch->mbc_state)) {
+ on = true;
+ } else {
+ list_for_each_safe(pos, n, &ch->mbc_msgs) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ if (msg->mbm_req_id == 0)
+ continue;
+ on = true;
+ break;
+ }
+ }
+
+ if (on != ch->mbc_timer_on) {
+ ch->mbc_timer_on = on;
+ if (on)
+ mod_timer(&ch->mbc_timer, jiffies + MAILBOX_TIMER);
+ else
+ del_timer_sync(&ch->mbc_timer);
+ }
+
+ mutex_unlock(&ch->mbc_mutex);
+}
+
+static void free_msg(struct mailbox_msg *msg)
+{
+ vfree(msg);
+}
+
+static void msg_done(struct mailbox_msg *msg, int err)
+{
+ struct mailbox_channel *ch = msg->mbm_ch;
+ struct mailbox *mbx = ch->mbc_parent;
+
+ MBX_DBG(ch->mbc_parent, "%s finishing msg id=0x%llx err=%d",
+ ch->mbc_name, msg->mbm_req_id, err);
+
+ msg->mbm_error = err;
+ if (msg->mbm_cb) {
+ msg->mbm_cb(msg->mbm_cb_arg, msg->mbm_data, msg->mbm_len,
+ msg->mbm_req_id, msg->mbm_error);
+ free_msg(msg);
+ } else {
+ if (msg->mbm_flags & MSG_FLAG_REQUEST) {
+ if ((mbx->mbx_req_sz+msg->mbm_len) >= MAX_MSG_QUEUE_SZ ||
+ mbx->mbx_req_cnt >= MAX_MSG_QUEUE_LEN) {
+ goto done;
+ }
+ mutex_lock(&ch->mbc_parent->mbx_lock);
+ list_add_tail(&msg->mbm_list, &ch->mbc_parent->mbx_req_list);
+ mbx->mbx_req_cnt++;
+ mbx->mbx_req_sz += msg->mbm_len;
+ mutex_unlock(&ch->mbc_parent->mbx_lock);
+
+ complete(&ch->mbc_parent->mbx_comp);
+ } else {
+ complete(&msg->mbm_complete);
+ }
+ }
+done:
+ chan_config_timer(ch);
+}
+
+static void chan_msg_done(struct mailbox_channel *ch, int err)
+{
+ if (!ch->mbc_cur_msg)
+ return;
+
+ msg_done(ch->mbc_cur_msg, err);
+ ch->mbc_cur_msg = NULL;
+ ch->mbc_bytes_done = 0;
+}
+
+void timeout_msg(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_msg *msg = NULL;
+ struct list_head *pos, *n;
+ struct list_head l = LIST_HEAD_INIT(l);
+ bool reschedule = false;
+
+ /* Check active msg first. */
+ msg = ch->mbc_cur_msg;
+ if (msg) {
+
+ if (msg->mbm_ttl == 0) {
+ MBX_ERR(mbx, "found active msg time'd out");
+ chan_msg_done(ch, -ETIME);
+ } else {
+ if (msg->mbm_timer_on) {
+ msg->mbm_ttl--;
+ /* Need to come back again for this one. */
+ reschedule = true;
+ }
+ }
+ }
+
+ mutex_lock(&ch->mbc_mutex);
+
+ list_for_each_safe(pos, n, &ch->mbc_msgs) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ if (!msg->mbm_timer_on)
+ continue;
+ if (msg->mbm_req_id == 0)
+ continue;
+ if (msg->mbm_ttl == 0) {
+ list_del(&msg->mbm_list);
+ list_add_tail(&msg->mbm_list, &l);
+ } else {
+ msg->mbm_ttl--;
+ /* Need to come back again for this one. */
+ reschedule = true;
+ }
+ }
+
+ mutex_unlock(&ch->mbc_mutex);
+
+ if (!list_empty(&l))
+ MBX_ERR(mbx, "found waiting msg time'd out");
+
+ list_for_each_safe(pos, n, &l) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ list_del(&msg->mbm_list);
+ msg_done(msg, -ETIME);
+ }
+}
+
+static void chann_worker(struct work_struct *work)
+{
+ struct mailbox_channel *ch =
+ container_of(work, struct mailbox_channel, mbc_work);
+ struct mailbox *mbx = ch->mbc_parent;
+
+ while (!test_bit(MBXCS_BIT_STOP, &ch->mbc_state)) {
+ MBX_DBG(mbx, "%s worker start", ch->mbc_name);
+ ch->mbc_tran(ch);
+ wait_for_completion_interruptible(&ch->mbc_worker);
+ }
+}
+
+static inline u32 mailbox_chk_err(struct mailbox *mbx)
+{
+ u32 val = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_error);
+
+ /* Ignore bad register value after firewall is tripped. */
+ if (val == 0xffffffff)
+ val = 0;
+
+ /* Error should not be seen, shout when found. */
+ if (val)
+ MBX_ERR(mbx, "mailbox error detected, error=0x%x\n", val);
+ return val;
+}
+
+static int chan_msg_enqueue(struct mailbox_channel *ch, struct mailbox_msg *msg)
+{
+ int rv = 0;
+
+ MBX_DBG(ch->mbc_parent, "%s enqueuing msg, id=0x%llx\n",
+ ch->mbc_name, msg->mbm_req_id);
+
+ BUG_ON(msg->mbm_req_id == INVALID_MSG_ID);
+
+ mutex_lock(&ch->mbc_mutex);
+ if (test_bit(MBXCS_BIT_STOP, &ch->mbc_state)) {
+ rv = -ESHUTDOWN;
+ } else {
+ list_add_tail(&msg->mbm_list, &ch->mbc_msgs);
+ msg->mbm_ch = ch;
+// msg->mbm_ttl = MSG_TTL;
+ }
+ mutex_unlock(&ch->mbc_mutex);
+
+ chan_config_timer(ch);
+
+ return rv;
+}
+
+static struct mailbox_msg *chan_msg_dequeue(struct mailbox_channel *ch,
+ u64 req_id)
+{
+ struct mailbox_msg *msg = NULL;
+ struct list_head *pos;
+
+ mutex_lock(&ch->mbc_mutex);
+
+ /* Take the first msg. */
+ if (req_id == INVALID_MSG_ID) {
+ msg = list_first_entry_or_null(&ch->mbc_msgs,
+ struct mailbox_msg, mbm_list);
+ /* Take the msg w/ specified ID. */
+ } else {
+ list_for_each(pos, &ch->mbc_msgs) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ if (msg->mbm_req_id == req_id)
+ break;
+ }
+ }
+
+ if (msg) {
+ MBX_DBG(ch->mbc_parent, "%s dequeued msg, id=0x%llx\n",
+ ch->mbc_name, msg->mbm_req_id);
+ list_del(&msg->mbm_list);
+ }
+
+ mutex_unlock(&ch->mbc_mutex);
+
+ return msg;
+}
+
+static struct mailbox_msg *alloc_msg(void *buf, size_t len)
+{
+ char *newbuf = NULL;
+ struct mailbox_msg *msg = NULL;
+ /* Give MB*2 secs as time to live */
+ int calculated_ttl = (len >> 19) < MSG_TTL ? MSG_TTL : (len >> 19);
+
+ if (!buf) {
+ msg = vzalloc(sizeof(struct mailbox_msg) + len);
+ if (!msg)
+ return NULL;
+ newbuf = ((char *)msg) + sizeof(struct mailbox_msg);
+ } else {
+ msg = vzalloc(sizeof(struct mailbox_msg));
+ if (!msg)
+ return NULL;
+ newbuf = buf;
+ }
+
+ INIT_LIST_HEAD(&msg->mbm_list);
+ msg->mbm_data = newbuf;
+ msg->mbm_len = len;
+ msg->mbm_ttl = calculated_ttl;
+ msg->mbm_timer_on = false;
+ init_completion(&msg->mbm_complete);
+
+ return msg;
+}
+
+static int chan_init(struct mailbox *mbx, char *nm,
+ struct mailbox_channel *ch, chan_func_t fn)
+{
+ ch->mbc_parent = mbx;
+ ch->mbc_name = nm;
+ ch->mbc_tran = fn;
+ INIT_LIST_HEAD(&ch->mbc_msgs);
+ init_completion(&ch->mbc_worker);
+ mutex_init(&ch->mbc_mutex);
+
+ ch->mbc_cur_msg = NULL;
+ ch->mbc_bytes_done = 0;
+
+ reset_pkt(&ch->mbc_packet);
+ set_bit(MBXCS_BIT_READY, &ch->mbc_state);
+
+ /* One thread for one channel. */
+ ch->mbc_wq =
+ create_singlethread_workqueue(dev_name(&mbx->mbx_pdev->dev));
+ if (!ch->mbc_wq) {
+ ch->mbc_parent = NULL;
+ return -ENOMEM;
+ }
+
+ INIT_WORK(&ch->mbc_work, chann_worker);
+ queue_work(ch->mbc_wq, &ch->mbc_work);
+
+ /* One timer for one channel. */
+ timer_setup(&ch->mbc_timer, chan_timer, 0);
+
+ return 0;
+}
+
+static void chan_fini(struct mailbox_channel *ch)
+{
+ struct mailbox_msg *msg;
+
+ if (!ch->mbc_parent)
+ return;
+
+ /*
+ * Holding mutex to ensure no new msg is enqueued after
+ * flag is set.
+ */
+ mutex_lock(&ch->mbc_mutex);
+ set_bit(MBXCS_BIT_STOP, &ch->mbc_state);
+ mutex_unlock(&ch->mbc_mutex);
+
+ complete(&ch->mbc_worker);
+ cancel_work_sync(&ch->mbc_work);
+ destroy_workqueue(ch->mbc_wq);
+
+ msg = ch->mbc_cur_msg;
+ if (msg)
+ chan_msg_done(ch, -ESHUTDOWN);
+
+ while ((msg = chan_msg_dequeue(ch, INVALID_MSG_ID)) != NULL)
+ msg_done(msg, -ESHUTDOWN);
+
+ del_timer_sync(&ch->mbc_timer);
+}
+
+static void listen_wq_fini(struct mailbox *mbx)
+{
+ BUG_ON(mbx == NULL);
+
+ if (mbx->mbx_listen_wq != NULL) {
+ complete(&mbx->mbx_comp);
+ cancel_work_sync(&mbx->mbx_listen_worker);
+ destroy_workqueue(mbx->mbx_listen_wq);
+ }
+
+}
+
+static void chan_recv_pkt(struct mailbox_channel *ch)
+{
+ int i, retry = 10;
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+
+ BUG_ON(valid_pkt(pkt));
+
+ /* Picking up a packet from HW. */
+ for (i = 0; i < PACKET_SIZE; i++) {
+ while ((mailbox_reg_rd(mbx,
+ &mbx->mbx_regs->mbr_status) & STATUS_EMPTY) &&
+ (retry-- > 0))
+ msleep(100);
+
+ *(((u32 *)pkt) + i) =
+ mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_rddata);
+ }
+
+ if ((mailbox_chk_err(mbx) & STATUS_EMPTY) != 0)
+ reset_pkt(pkt);
+ else
+ MBX_DBG(mbx, "received pkt: type=0x%x", pkt->hdr.type);
+}
+
+static void chan_send_pkt(struct mailbox_channel *ch)
+{
+ int i;
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+
+ BUG_ON(!valid_pkt(pkt));
+
+ MBX_DBG(mbx, "sending pkt: type=0x%x", pkt->hdr.type);
+
+ /* Pushing a packet into HW. */
+ for (i = 0; i < PACKET_SIZE; i++) {
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_wrdata,
+ *(((u32 *)pkt) + i));
+ }
+
+ reset_pkt(pkt);
+ if (ch->mbc_cur_msg)
+ ch->mbc_bytes_done += ch->mbc_packet.hdr.payload_size;
+
+ BUG_ON((mailbox_chk_err(mbx) & STATUS_FULL) != 0);
+}
+
+static int chan_pkt2msg(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ void *msg_data, *pkt_data;
+ struct mailbox_msg *msg = ch->mbc_cur_msg;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+ size_t cnt = pkt->hdr.payload_size;
+ u32 type = (pkt->hdr.type & PKT_TYPE_MASK);
+
+ BUG_ON(((type != PKT_MSG_START) && (type != PKT_MSG_BODY)) || !msg);
+
+ if (type == PKT_MSG_START) {
+ msg->mbm_req_id = pkt->body.msg_start.msg_req_id;
+ BUG_ON(msg->mbm_len < pkt->body.msg_start.msg_size);
+ msg->mbm_len = pkt->body.msg_start.msg_size;
+ pkt_data = pkt->body.msg_start.payload;
+ } else {
+ pkt_data = pkt->body.msg_body.payload;
+ }
+
+ if (cnt > msg->mbm_len - ch->mbc_bytes_done) {
+ MBX_ERR(mbx, "invalid mailbox packet size\n");
+ return -EBADMSG;
+ }
+
+ msg_data = msg->mbm_data + ch->mbc_bytes_done;
+ (void) memcpy(msg_data, pkt_data, cnt);
+ ch->mbc_bytes_done += cnt;
+
+ reset_pkt(pkt);
+ return 0;
+}
+
+/*
+ * Worker for RX channel.
+ */
+static void chan_do_rx(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+ struct mailbox_msg *msg = NULL;
+ bool needs_read = false;
+ u64 id = 0;
+ bool eom;
+ int err;
+ u32 type;
+ u32 st = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_status);
+
+ /* Check if a packet is ready for reading. */
+ if (st == 0xffffffff) {
+ /* Device is still being reset. */
+ needs_read = false;
+ } else if (test_bit(MBXCS_BIT_POLL_MODE, &ch->mbc_state)) {
+ needs_read = ((st & STATUS_EMPTY) == 0);
+ } else {
+ needs_read = ((st & STATUS_RTA) != 0);
+ }
+
+ if (needs_read) {
+ chan_recv_pkt(ch);
+ type = pkt->hdr.type & PKT_TYPE_MASK;
+ eom = ((pkt->hdr.type & PKT_TYPE_MSG_END) != 0);
+
+ switch (type) {
+ case PKT_TEST:
+ (void) memcpy(&mbx->mbx_tst_pkt, &ch->mbc_packet,
+ sizeof(struct mailbox_pkt));
+ reset_pkt(pkt);
+ return;
+ case PKT_MSG_START:
+ if (ch->mbc_cur_msg) {
+ MBX_ERR(mbx, "received partial msg\n");
+ chan_msg_done(ch, -EBADMSG);
+ }
+
+ /* Get a new active msg. */
+ id = 0;
+ if (pkt->body.msg_start.msg_flags & MSG_FLAG_RESPONSE)
+ id = pkt->body.msg_start.msg_req_id;
+ ch->mbc_cur_msg = chan_msg_dequeue(ch, id);
+
+ if (!ch->mbc_cur_msg) {
+ //no msg, alloc dynamically
+ msg = alloc_msg(NULL, pkt->body.msg_start.msg_size);
+
+ msg->mbm_ch = ch;
+ msg->mbm_flags |= MSG_FLAG_REQUEST;
+ ch->mbc_cur_msg = msg;
+
+ } else if (pkt->body.msg_start.msg_size >
+ ch->mbc_cur_msg->mbm_len) {
+ chan_msg_done(ch, -EMSGSIZE);
+ MBX_ERR(mbx, "received msg is too big");
+ reset_pkt(pkt);
+ }
+ break;
+ case PKT_MSG_BODY:
+ if (!ch->mbc_cur_msg) {
+ MBX_ERR(mbx, "got unexpected msg body pkt\n");
+ reset_pkt(pkt);
+ }
+ break;
+ default:
+ MBX_ERR(mbx, "invalid mailbox pkt type\n");
+ reset_pkt(pkt);
+ return;
+ }
+
+ if (valid_pkt(pkt)) {
+ err = chan_pkt2msg(ch);
+ if (err || eom)
+ chan_msg_done(ch, err);
+ }
+ }
+
+ /* Handle timer event. */
+ if (test_bit(MBXCS_BIT_TICK, &ch->mbc_state)) {
+ timeout_msg(ch);
+ clear_bit(MBXCS_BIT_TICK, &ch->mbc_state);
+ }
+}
+
+static void chan_msg2pkt(struct mailbox_channel *ch)
+{
+ size_t cnt = 0;
+ size_t payload_off = 0;
+ void *msg_data, *pkt_data;
+ struct mailbox_msg *msg = ch->mbc_cur_msg;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+ bool is_start = (ch->mbc_bytes_done == 0);
+ bool is_eom = false;
+
+ if (is_start) {
+ payload_off = offsetof(struct mailbox_pkt,
+ body.msg_start.payload);
+ } else {
+ payload_off = offsetof(struct mailbox_pkt,
+ body.msg_body.payload);
+ }
+ cnt = PACKET_SIZE * sizeof(u32) - payload_off;
+ if (cnt >= msg->mbm_len - ch->mbc_bytes_done) {
+ cnt = msg->mbm_len - ch->mbc_bytes_done;
+ is_eom = true;
+ }
+
+ pkt->hdr.type = is_start ? PKT_MSG_START : PKT_MSG_BODY;
+ pkt->hdr.type |= is_eom ? PKT_TYPE_MSG_END : 0;
+ pkt->hdr.payload_size = cnt;
+
+ if (is_start) {
+ pkt->body.msg_start.msg_req_id = msg->mbm_req_id;
+ pkt->body.msg_start.msg_size = msg->mbm_len;
+ pkt->body.msg_start.msg_flags = msg->mbm_flags;
+ pkt_data = pkt->body.msg_start.payload;
+ } else {
+ pkt_data = pkt->body.msg_body.payload;
+ }
+ msg_data = msg->mbm_data + ch->mbc_bytes_done;
+ (void) memcpy(pkt_data, msg_data, cnt);
+}
+
+static void check_tx_stall(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_msg *msg = ch->mbc_cur_msg;
+
+ /*
+ * No stall checking in polling mode. Don't know how often peer will
+ * check the channel.
+ */
+ if ((msg == NULL) || test_bit(MBXCS_BIT_POLL_MODE, &ch->mbc_state))
+ return;
+
+ /*
+ * No tx intr has come since last check.
+ * The TX channel is stalled, reset it.
+ */
+ if (test_bit(MBXCS_BIT_CHK_STALL, &ch->mbc_state)) {
+ MBX_ERR(mbx, "TX channel stall detected, reset...\n");
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_ctrl, 0x1);
+ chan_msg_done(ch, -ETIME);
+ connect_state_touch(mbx, MB_CONN_FIN);
+ /* Mark it for next check. */
+ } else {
+ set_bit(MBXCS_BIT_CHK_STALL, &ch->mbc_state);
+ }
+}
+
+
+
+static void rx_enqueued_msg_timer_on(struct mailbox *mbx, uint64_t req_id)
+{
+ struct list_head *pos, *n;
+ struct mailbox_msg *msg = NULL;
+ struct mailbox_channel *ch = NULL;
+ ch = &mbx->mbx_rx;
+ MBX_DBG(mbx, "try to set ch rx, req_id %llu\n", req_id);
+ mutex_lock(&ch->mbc_mutex);
+
+ list_for_each_safe(pos, n, &ch->mbc_msgs) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ if (msg->mbm_req_id == req_id) {
+ msg->mbm_timer_on = true;
+ MBX_DBG(mbx, "set ch rx, req_id %llu\n", req_id);
+ break;
+ }
+ }
+
+ mutex_unlock(&ch->mbc_mutex);
+
+}
+
+/*
+ * Worker for TX channel.
+ */
+static void chan_do_tx(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ u32 st = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_status);
+
+ /* Check if a packet has been read by peer. */
+ if ((st != 0xffffffff) && ((st & STATUS_STA) != 0)) {
+ clear_bit(MBXCS_BIT_CHK_STALL, &ch->mbc_state);
+
+ /*
+ * The mailbox is free for sending new pkt now. See if we
+ * have something to send.
+ */
+
+ /* Finished sending a whole msg, call it done. */
+ if (ch->mbc_cur_msg &&
+ (ch->mbc_cur_msg->mbm_len == ch->mbc_bytes_done)) {
+ rx_enqueued_msg_timer_on(mbx, ch->mbc_cur_msg->mbm_req_id);
+ chan_msg_done(ch, 0);
+ }
+
+ if (!ch->mbc_cur_msg) {
+ ch->mbc_cur_msg = chan_msg_dequeue(ch, INVALID_MSG_ID);
+ if (ch->mbc_cur_msg)
+ ch->mbc_cur_msg->mbm_timer_on = true;
+ }
+
+ if (ch->mbc_cur_msg) {
+ chan_msg2pkt(ch);
+ } else if (valid_pkt(&mbx->mbx_tst_pkt)) {
+ (void) memcpy(&ch->mbc_packet, &mbx->mbx_tst_pkt,
+ sizeof(struct mailbox_pkt));
+ reset_pkt(&mbx->mbx_tst_pkt);
+ } else {
+ return; /* Nothing to send. */
+ }
+
+ chan_send_pkt(ch);
+ }
+
+ /* Handle timer event. */
+ if (test_bit(MBXCS_BIT_TICK, &ch->mbc_state)) {
+ timeout_msg(ch);
+ check_tx_stall(ch);
+ clear_bit(MBXCS_BIT_TICK, &ch->mbc_state);
+ }
+}
+
+static int mailbox_connect_status(struct platform_device *pdev)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ int ret = 0;
+ mutex_lock(&mbx->mbx_lock);
+ ret = mbx->mbx_paired;
+ mutex_unlock(&mbx->mbx_lock);
+ return ret;
+}
+
+static ssize_t mailbox_ctl_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ u32 *reg = (u32 *)mbx->mbx_regs;
+ int r, n;
+ int nreg = sizeof (struct mailbox_reg) / sizeof (u32);
+
+ for (r = 0, n = 0; r < nreg; r++, reg++) {
+ /* Non-status registers. */
+ if ((reg == &mbx->mbx_regs->mbr_resv1) ||
+ (reg == &mbx->mbx_regs->mbr_wrdata) ||
+ (reg == &mbx->mbx_regs->mbr_rddata) ||
+ (reg == &mbx->mbx_regs->mbr_resv2))
+ continue;
+ /* Write-only status register. */
+ if (reg == &mbx->mbx_regs->mbr_ctrl) {
+ n += sprintf(buf + n, "%02ld %10s = --\n",
+ r * sizeof (u32), reg2name(mbx, reg));
+ /* Read-able status register. */
+ } else {
+ n += sprintf(buf + n, "%02ld %10s = 0x%08x\n",
+ r * sizeof (u32), reg2name(mbx, reg),
+ mailbox_reg_rd(mbx, reg));
+ }
+ }
+
+ return n;
+}
+
+static ssize_t mailbox_ctl_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ u32 off, val;
+ int nreg = sizeof (struct mailbox_reg) / sizeof (u32);
+ u32 *reg = (u32 *)mbx->mbx_regs;
+
+ if (sscanf(buf, "%d:%d", &off, &val) != 2 || (off % sizeof (u32)) ||
+ !(off >= 0 && off < nreg * sizeof (u32))) {
+ MBX_ERR(mbx, "input should be <reg_offset:reg_val>");
+ return -EINVAL;
+ }
+ reg += off / sizeof (u32);
+
+ mailbox_reg_wr(mbx, reg, val);
+ return count;
+}
+/* HW register level debugging i/f. */
+static DEVICE_ATTR_RW(mailbox_ctl);
+
+static ssize_t mailbox_pkt_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ int ret = 0;
+
+ if (valid_pkt(&mbx->mbx_tst_pkt)) {
+ (void) memcpy(buf, mbx->mbx_tst_pkt.body.data,
+ mbx->mbx_tst_pkt.hdr.payload_size);
+ ret = mbx->mbx_tst_pkt.hdr.payload_size;
+ }
+
+ return ret;
+}
+
+static ssize_t mailbox_pkt_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ size_t maxlen = sizeof (mbx->mbx_tst_pkt.body.data);
+
+ if (count > maxlen) {
+ MBX_ERR(mbx, "max input length is %ld", maxlen);
+ return 0;
+ }
+
+ (void) memcpy(mbx->mbx_tst_pkt.body.data, buf, count);
+ mbx->mbx_tst_pkt.hdr.payload_size = count;
+ mbx->mbx_tst_pkt.hdr.type = PKT_TEST;
+ complete(&mbx->mbx_tx.mbc_worker);
+ return count;
+}
+
+/* Packet test i/f. */
+static DEVICE_ATTR_RW(mailbox_pkt);
+
+static ssize_t mailbox_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ struct mailbox_req req;
+ size_t respsz = sizeof (mbx->mbx_tst_rx_msg);
+ int ret = 0;
+
+ req.req = MAILBOX_REQ_TEST_READ;
+ ret = mailbox_request(to_platform_device(dev), &req, sizeof (req),
+ mbx->mbx_tst_rx_msg, &respsz, NULL, NULL);
+ if (ret) {
+ MBX_ERR(mbx, "failed to read test msg from peer: %d", ret);
+ } else if (respsz > 0) {
+ (void) memcpy(buf, mbx->mbx_tst_rx_msg, respsz);
+ ret = respsz;
+ }
+
+ return ret;
+}
+
+static ssize_t mailbox_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ size_t maxlen = sizeof (mbx->mbx_tst_tx_msg);
+ struct mailbox_req req = { 0 };
+
+ if (count > maxlen) {
+ MBX_ERR(mbx, "max input length is %ld", maxlen);
+ return 0;
+ }
+
+ (void) memcpy(mbx->mbx_tst_tx_msg, buf, count);
+ mbx->mbx_tst_tx_msg_len = count;
+ req.req = MAILBOX_REQ_TEST_READY;
+ (void) mailbox_post(mbx->mbx_pdev, 0, &req, sizeof (req));
+
+ return count;
+}
+
+/* Msg test i/f. */
+static DEVICE_ATTR_RW(mailbox);
+
+static ssize_t connection_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ int ret;
+ ret = mailbox_connect_status(pdev);
+ return sprintf(buf, "0x%x\n", ret);
+}
+static DEVICE_ATTR_RO(connection);
+
+
+static struct attribute *mailbox_attrs[] = {
+ &dev_attr_mailbox.attr,
+ &dev_attr_mailbox_ctl.attr,
+ &dev_attr_mailbox_pkt.attr,
+ &dev_attr_connection.attr,
+ NULL,
+};
+
+static const struct attribute_group mailbox_attrgroup = {
+ .attrs = mailbox_attrs,
+};
+
+static void dft_req_msg_cb(void *arg, void *data, size_t len, u64 id, int err)
+{
+ struct mailbox_msg *respmsg;
+ struct mailbox_msg *reqmsg = (struct mailbox_msg *)arg;
+ struct mailbox *mbx = reqmsg->mbm_ch->mbc_parent;
+
+ /*
+ * Can't send out request msg.
+ * Removing corresponding response msg from queue and return error.
+ */
+ if (err) {
+ respmsg = chan_msg_dequeue(&mbx->mbx_rx, reqmsg->mbm_req_id);
+ if (respmsg)
+ msg_done(respmsg, err);
+ }
+}
+
+static void dft_post_msg_cb(void *arg, void *buf, size_t len, u64 id, int err)
+{
+ struct mailbox_msg *msg = (struct mailbox_msg *)arg;
+
+ if (err) {
+ MBX_ERR(msg->mbm_ch->mbc_parent,
+ "failed to post msg, err=%d", err);
+ }
+}
+
+/*
+ * Msg will be sent to peer and reply will be received.
+ */
+int mailbox_request(struct platform_device *pdev, void *req, size_t reqlen,
+ void *resp, size_t *resplen, mailbox_msg_cb_t cb, void *cbarg)
+{
+ int rv = -ENOMEM;
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ struct mailbox_msg *reqmsg = NULL, *respmsg = NULL;
+
+ MBX_INFO(mbx, "sending request: %d", ((struct mailbox_req *)req)->req);
+
+ if (cb) {
+ reqmsg = alloc_msg(NULL, reqlen);
+ if (reqmsg)
+ (void) memcpy(reqmsg->mbm_data, req, reqlen);
+ } else {
+ reqmsg = alloc_msg(req, reqlen);
+ }
+ if (!reqmsg)
+ goto fail;
+ reqmsg->mbm_cb = dft_req_msg_cb;
+ reqmsg->mbm_cb_arg = reqmsg;
+ reqmsg->mbm_req_id = (uintptr_t)reqmsg->mbm_data;
+
+ respmsg = alloc_msg(resp, *resplen);
+ if (!respmsg)
+ goto fail;
+ respmsg->mbm_cb = cb;
+ respmsg->mbm_cb_arg = cbarg;
+ /* Only interested in response w/ same ID. */
+ respmsg->mbm_req_id = reqmsg->mbm_req_id;
+
+ /* Always enqueue RX msg before TX one to avoid race. */
+ rv = chan_msg_enqueue(&mbx->mbx_rx, respmsg);
+ if (rv)
+ goto fail;
+ rv = chan_msg_enqueue(&mbx->mbx_tx, reqmsg);
+ if (rv) {
+ respmsg = chan_msg_dequeue(&mbx->mbx_rx, reqmsg->mbm_req_id);
+ goto fail;
+ }
+
+ /* Kick TX channel to try to send out msg. */
+ complete(&mbx->mbx_tx.mbc_worker);
+
+ if (cb)
+ return 0;
+
+ wait_for_completion(&respmsg->mbm_complete);
+ rv = respmsg->mbm_error;
+ if (rv == 0)
+ *resplen = respmsg->mbm_len;
+
+ free_msg(respmsg);
+ return rv;
+
+fail:
+ if (reqmsg)
+ free_msg(reqmsg);
+ if (respmsg)
+ free_msg(respmsg);
+ return rv;
+}
+
+/*
+ * Msg will be posted, no wait for reply.
+ */
+int mailbox_post(struct platform_device *pdev, u64 reqid, void *buf, size_t len)
+{
+ int rv = 0;
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ struct mailbox_msg *msg = alloc_msg(NULL, len);
+
+ if (reqid == 0) {
+ MBX_DBG(mbx, "posting request: %d",
+ ((struct mailbox_req *)buf)->req);
+ } else {
+ MBX_DBG(mbx, "posting response...");
+ }
+
+ if (!msg)
+ return -ENOMEM;
+
+ (void) memcpy(msg->mbm_data, buf, len);
+ msg->mbm_cb = dft_post_msg_cb;
+ msg->mbm_cb_arg = msg;
+ if (reqid) {
+ msg->mbm_req_id = reqid;
+ msg->mbm_flags |= MSG_FLAG_RESPONSE;
+ } else {
+ msg->mbm_req_id = (uintptr_t)msg->mbm_data;
+ }
+
+ rv = chan_msg_enqueue(&mbx->mbx_tx, msg);
+ if (rv)
+ free_msg(msg);
+
+ /* Kick TX channel to try to send out msg. */
+ complete(&mbx->mbx_tx.mbc_worker);
+
+ return rv;
+}
+/*
+ * should not be called by other than connect_state_handler
+ */
+static int mailbox_connection_notify(struct platform_device *pdev, uint64_t sec_id, uint64_t flag)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ struct mailbox_req *mb_req = NULL;
+ struct mailbox_conn mb_conn = { 0 };
+ int ret = 0;
+ size_t data_len = 0, reqlen = 0;
+ data_len = sizeof(struct mailbox_conn);
+ reqlen = sizeof(struct mailbox_req) + data_len;
+
+ mb_req = (struct mailbox_req *)vmalloc(reqlen);
+ if (!mb_req) {
+ ret = -ENOMEM;
+ goto done;
+ }
+ mb_req->req = MAILBOX_REQ_CONN_EXPL;
+ if (!mbx->mbx_kaddr) {
+ ret = -ENOMEM;
+ goto done;
+ }
+
+ mb_conn.kaddr = mbx->mbx_kaddr;
+ mb_conn.paddr = virt_to_phys(mbx->mbx_kaddr);
+ mb_conn.crc32 = crc32c_le(~0, mbx->mbx_kaddr, PAGE_SIZE);
+ mb_conn.flag = flag;
+ mb_conn.ver = mbx->mbx_prot_ver;
+
+ if (sec_id != 0) {
+ mb_conn.sec_id = sec_id;
+ } else {
+ mb_conn.sec_id = (uint64_t)mbx->mbx_kaddr;
+ mbx->mbx_conn_id = (uint64_t)mbx->mbx_kaddr;
+ }
+
+ memcpy(mb_req->data, &mb_conn, data_len);
+
+ ret = mailbox_post(pdev, 0, mb_req, reqlen);
+
+done:
+ vfree(mb_req);
+ return ret;
+}
+
+static int mailbox_connection_explore(struct platform_device *pdev, struct mailbox_conn *mb_conn)
+{
+ int ret = 0;
+ uint32_t crc_chk;
+ phys_addr_t paddr;
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ if (!mb_conn) {
+ ret = -EFAULT;
+ goto done;
+ }
+
+ paddr = virt_to_phys(mb_conn->kaddr);
+ if (paddr != mb_conn->paddr) {
+ MBX_INFO(mbx, "mb_conn->paddr %llx paddr: %llx\n", mb_conn->paddr, paddr);
+ MBX_INFO(mbx, "Failed to get the same physical addr, running in VMs?\n");
+ ret = -EFAULT;
+ goto done;
+ }
+ crc_chk = crc32c_le(~0, mb_conn->kaddr, PAGE_SIZE);
+
+ if (crc_chk != mb_conn->crc32) {
+ MBX_INFO(mbx, "crc32 : %x, %x\n", mb_conn->crc32, crc_chk);
+ MBX_INFO(mbx, "failed to get the same CRC\n");
+ ret = -EFAULT;
+ goto done;
+ }
+done:
+ return ret;
+}
+
+static int mailbox_get_data(struct platform_device *pdev, enum data_kind kind)
+{
+ int ret = 0;
+ switch (kind) {
+ case PEER_CONN:
+ ret = mailbox_connect_status(pdev);
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+
+static void connect_state_handler(struct mailbox *mbx, struct mailbox_conn *conn)
+{
+ int ret = 0;
+
+ if (!mbx || !conn)
+ return;
+
+ mutex_lock(&mbx->mbx_lock);
+
+ switch (conn->flag) {
+ case MB_CONN_INIT:
+ /* clean up all cached data, */
+ mbx->mbx_paired = 0;
+ mbx->mbx_established = false;
+ kfree(mbx->mbx_kaddr);
+
+ mbx->mbx_kaddr = kzalloc(PAGE_SIZE, GFP_KERNEL);
+ get_random_bytes(mbx->mbx_kaddr, PAGE_SIZE);
+ ret = mailbox_connection_notify(mbx->mbx_pdev, 0, MB_CONN_SYN);
+ if (ret)
+ goto done;
+ mbx->mbx_state = CONN_SYN_SENT;
+ break;
+ case MB_CONN_SYN:
+ if (mbx->mbx_state == CONN_SYN_SENT) {
+ if (!mailbox_connection_explore(mbx->mbx_pdev, conn)) {
+ mbx->mbx_paired |= 0x2;
+ MBX_INFO(mbx, "mailbox mbx_prot_ver %x", mbx->mbx_prot_ver);
+ }
+ ret = mailbox_connection_notify(mbx->mbx_pdev, conn->sec_id, MB_CONN_ACK);
+ if (ret)
+ goto done;
+ mbx->mbx_state = CONN_SYN_RECV;
+ } else
+ mbx->mbx_state = CONN_START;
+ break;
+ case MB_CONN_ACK:
+ if (mbx->mbx_state & (CONN_SYN_SENT | CONN_SYN_RECV)) {
+ if (mbx->mbx_conn_id == (uint64_t)conn->sec_id) {
+ mbx->mbx_paired |= 0x1;
+ mbx->mbx_established = true;
+ mbx->mbx_state = CONN_ESTABLISH;
+ kfree(mbx->mbx_kaddr);
+ mbx->mbx_kaddr = NULL;
+ } else
+ mbx->mbx_state = CONN_START;
+ }
+ break;
+ case MB_CONN_FIN:
+ mbx->mbx_paired = 0;
+ mbx->mbx_established = false;
+ kfree(mbx->mbx_kaddr);
+ mbx->mbx_kaddr = NULL;
+ mbx->mbx_state = CONN_START;
+ break;
+ default:
+ break;
+ }
+done:
+ if (ret) {
+ kfree(mbx->mbx_kaddr);
+ mbx->mbx_kaddr = NULL;
+ mbx->mbx_paired = 0;
+ mbx->mbx_state = CONN_START;
+ }
+ mutex_unlock(&mbx->mbx_lock);
+ MBX_INFO(mbx, "mailbox connection state %d", mbx->mbx_paired);
+}
+
+static void process_request(struct mailbox *mbx, struct mailbox_msg *msg)
+{
+ struct mailbox_req *req = (struct mailbox_req *)msg->mbm_data;
+ struct mailbox_conn *conn = (struct mailbox_conn *)req->data;
+ int rc;
+ const char *recvstr = "received request from peer";
+ const char *sendstr = "sending test msg to peer";
+
+ if (req->req == MAILBOX_REQ_TEST_READ) {
+ MBX_INFO(mbx, "%s: %d", recvstr, req->req);
+ if (mbx->mbx_tst_tx_msg_len) {
+ MBX_INFO(mbx, "%s", sendstr);
+ rc = mailbox_post(mbx->mbx_pdev, msg->mbm_req_id,
+ mbx->mbx_tst_tx_msg, mbx->mbx_tst_tx_msg_len);
+ if (rc)
+ MBX_ERR(mbx, "%s failed: %d", sendstr, rc);
+ else
+ mbx->mbx_tst_tx_msg_len = 0;
+
+ }
+ } else if (req->req == MAILBOX_REQ_TEST_READY) {
+ MBX_INFO(mbx, "%s: %d", recvstr, req->req);
+ } else if (req->req == MAILBOX_REQ_CONN_EXPL) {
+ MBX_INFO(mbx, "%s: %d", recvstr, req->req);
+ if (mbx->mbx_state != CONN_SYN_SENT) {
+ /* if your peer droped without notice,
+ * initial the connection Simultaneously
+ * again.
+ */
+ if (conn->flag == MB_CONN_SYN) {
+ connect_state_touch(mbx, MB_CONN_INIT);
+ }
+ }
+ connect_state_handler(mbx, conn);
+ } else if (mbx->mbx_listen_cb) {
+ /* Call client's registered callback to process request. */
+ MBX_DBG(mbx, "%s: %d, passed on", recvstr, req->req);
+ mbx->mbx_listen_cb(mbx->mbx_listen_cb_arg, msg->mbm_data,
+ msg->mbm_len, msg->mbm_req_id, msg->mbm_error);
+ } else {
+ MBX_INFO(mbx, "%s: %d, dropped", recvstr, req->req);
+ }
+}
+
+/*
+ * Wait for request from peer.
+ */
+static void mailbox_recv_request(struct work_struct *work)
+{
+ int rv = 0;
+ struct mailbox_msg *msg = NULL;
+ struct mailbox *mbx =
+ container_of(work, struct mailbox, mbx_listen_worker);
+
+ for (;;) {
+ /* Only interested in request msg. */
+
+ rv = wait_for_completion_interruptible(&mbx->mbx_comp);
+ if (rv)
+ break;
+ mutex_lock(&mbx->mbx_lock);
+ msg = list_first_entry_or_null(&mbx->mbx_req_list,
+ struct mailbox_msg, mbm_list);
+
+ if (msg) {
+ list_del(&msg->mbm_list);
+ mbx->mbx_req_cnt--;
+ mbx->mbx_req_sz -= msg->mbm_len;
+ mutex_unlock(&mbx->mbx_lock);
+ } else {
+ mutex_unlock(&mbx->mbx_lock);
+ break;
+ }
+
+ process_request(mbx, msg);
+ free_msg(msg);
+ }
+
+ if (rv == -ESHUTDOWN)
+ MBX_INFO(mbx, "channel is closed, no listen to peer");
+ else if (rv != 0)
+ MBX_ERR(mbx, "failed to receive request from peer, err=%d", rv);
+
+ if (msg)
+ free_msg(msg);
+}
+
+int mailbox_listen(struct platform_device *pdev,
+ mailbox_msg_cb_t cb, void *cbarg)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+
+ mbx->mbx_listen_cb_arg = cbarg;
+ /* mbx->mbx_listen_cb is used in another thread as a condition to
+ * call the function. Ensuring that the argument is captured before
+ * the function pointer
+ */
+ wmb();
+ mbx->mbx_listen_cb = cb;
+
+ return 0;
+}
+
+static int mailbox_enable_intr_mode(struct mailbox *mbx)
+{
+ struct resource *res;
+ int ret;
+ struct platform_device *pdev = mbx->mbx_pdev;
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+
+ if (mbx->mbx_irq != -1)
+ return 0;
+
+ res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (res == NULL) {
+ MBX_ERR(mbx, "failed to acquire intr resource");
+ return -EINVAL;
+ }
+
+ ret = xocl_user_interrupt_reg(xdev, res->start, mailbox_isr, mbx);
+ if (ret) {
+ MBX_ERR(mbx, "failed to add intr handler");
+ return ret;
+ }
+ ret = xocl_user_interrupt_config(xdev, res->start, true);
+ BUG_ON(ret != 0);
+
+ /* Only see intr when we have full packet sent or received. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_rit, PACKET_SIZE - 1);
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_sit, 0);
+
+ /* Finally, enable TX / RX intr. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_ie, 0x3);
+
+ clear_bit(MBXCS_BIT_POLL_MODE, &mbx->mbx_rx.mbc_state);
+ chan_config_timer(&mbx->mbx_rx);
+
+ clear_bit(MBXCS_BIT_POLL_MODE, &mbx->mbx_tx.mbc_state);
+ chan_config_timer(&mbx->mbx_tx);
+
+ mbx->mbx_irq = res->start;
+ return 0;
+}
+
+static void mailbox_disable_intr_mode(struct mailbox *mbx)
+{
+ struct platform_device *pdev = mbx->mbx_pdev;
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+
+ /*
+ * No need to turn on polling mode for TX, which has
+ * a channel stall checking timer always on when there is
+ * outstanding TX packet.
+ */
+ set_bit(MBXCS_BIT_POLL_MODE, &mbx->mbx_rx.mbc_state);
+ chan_config_timer(&mbx->mbx_rx);
+
+ /* Disable both TX / RX intrs. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_ie, 0x0);
+
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_rit, 0x0);
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_sit, 0x0);
+
+ if (mbx->mbx_irq == -1)
+ return;
+
+ (void) xocl_user_interrupt_config(xdev, mbx->mbx_irq, false);
+ (void) xocl_user_interrupt_reg(xdev, mbx->mbx_irq, NULL, mbx);
+
+ mbx->mbx_irq = -1;
+}
+
+int mailbox_reset(struct platform_device *pdev, bool end_of_reset)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ int ret = 0;
+
+ if (mailbox_no_intr)
+ return 0;
+
+ if (end_of_reset) {
+ MBX_INFO(mbx, "enable intr mode");
+ if (mailbox_enable_intr_mode(mbx) != 0)
+ MBX_ERR(mbx, "failed to enable intr after reset");
+ } else {
+ MBX_INFO(mbx, "enable polling mode");
+ mailbox_disable_intr_mode(mbx);
+ }
+ return ret;
+}
+
+/* Kernel APIs exported from this sub-device driver. */
+static struct xocl_mailbox_funcs mailbox_ops = {
+ .request = mailbox_request,
+ .post = mailbox_post,
+ .listen = mailbox_listen,
+ .reset = mailbox_reset,
+ .get_data = mailbox_get_data,
+};
+
+static int mailbox_remove(struct platform_device *pdev)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+
+ BUG_ON(mbx == NULL);
+
+ connect_state_touch(mbx, MB_CONN_FIN);
+
+ mailbox_disable_intr_mode(mbx);
+
+ sysfs_remove_group(&pdev->dev.kobj, &mailbox_attrgroup);
+
+ chan_fini(&mbx->mbx_rx);
+ chan_fini(&mbx->mbx_tx);
+ listen_wq_fini(mbx);
+
+ BUG_ON(!(list_empty(&mbx->mbx_req_list)));
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_MAILBOX, NULL);
+
+ if (mbx->mbx_regs)
+ iounmap(mbx->mbx_regs);
+
+ MBX_INFO(mbx, "mailbox cleaned up successfully");
+ platform_set_drvdata(pdev, NULL);
+ kfree(mbx);
+ return 0;
+}
+
+static int mailbox_probe(struct platform_device *pdev)
+{
+ struct mailbox *mbx = NULL;
+ struct resource *res;
+ int ret;
+
+ mbx = kzalloc(sizeof(struct mailbox), GFP_KERNEL);
+ if (!mbx)
+ return -ENOMEM;
+ platform_set_drvdata(pdev, mbx);
+ mbx->mbx_pdev = pdev;
+ mbx->mbx_irq = (u32)-1;
+
+
+ init_completion(&mbx->mbx_comp);
+ mutex_init(&mbx->mbx_lock);
+ INIT_LIST_HEAD(&mbx->mbx_req_list);
+ mbx->mbx_req_cnt = 0;
+ mbx->mbx_req_sz = 0;
+
+ mutex_init(&mbx->mbx_conn_lock);
+ mbx->mbx_established = false;
+ mbx->mbx_conn_id = 0;
+ mbx->mbx_kaddr = NULL;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ mbx->mbx_regs = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!mbx->mbx_regs) {
+ MBX_ERR(mbx, "failed to map in registers");
+ ret = -EIO;
+ goto failed;
+ }
+ /* Reset TX channel, RX channel is managed by peer as his TX. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_ctrl, 0x1);
+
+ /* Set up software communication channels. */
+ ret = chan_init(mbx, "RX", &mbx->mbx_rx, chan_do_rx);
+ if (ret != 0) {
+ MBX_ERR(mbx, "failed to init rx channel");
+ goto failed;
+ }
+ ret = chan_init(mbx, "TX", &mbx->mbx_tx, chan_do_tx);
+ if (ret != 0) {
+ MBX_ERR(mbx, "failed to init tx channel");
+ goto failed;
+ }
+ /* Dedicated thread for listening to peer request. */
+ mbx->mbx_listen_wq =
+ create_singlethread_workqueue(dev_name(&mbx->mbx_pdev->dev));
+ if (!mbx->mbx_listen_wq) {
+ MBX_ERR(mbx, "failed to create request-listen work queue");
+ goto failed;
+ }
+ INIT_WORK(&mbx->mbx_listen_worker, mailbox_recv_request);
+ queue_work(mbx->mbx_listen_wq, &mbx->mbx_listen_worker);
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &mailbox_attrgroup);
+ if (ret != 0) {
+ MBX_ERR(mbx, "failed to init sysfs");
+ goto failed;
+ }
+
+ if (mailbox_no_intr) {
+ MBX_INFO(mbx, "Enabled timer-driven mode");
+ mailbox_disable_intr_mode(mbx);
+ } else {
+ ret = mailbox_enable_intr_mode(mbx);
+ if (ret != 0)
+ goto failed;
+ }
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_MAILBOX, &mailbox_ops);
+
+ connect_state_touch(mbx, MB_CONN_INIT);
+ mbx->mbx_prot_ver = MB_PROTOCOL_VER;
+
+ MBX_INFO(mbx, "successfully initialized");
+ return 0;
+
+failed:
+ mailbox_remove(pdev);
+ return ret;
+}
+
+struct platform_device_id mailbox_id_table[] = {
+ { XOCL_MAILBOX, 0 },
+ { },
+};
+
+static struct platform_driver mailbox_driver = {
+ .probe = mailbox_probe,
+ .remove = mailbox_remove,
+ .driver = {
+ .name = XOCL_MAILBOX,
+ },
+ .id_table = mailbox_id_table,
+};
+
+int __init xocl_init_mailbox(void)
+{
+ BUILD_BUG_ON(sizeof(struct mailbox_pkt) != sizeof(u32) * PACKET_SIZE);
+ return platform_driver_register(&mailbox_driver);
+}
+
+void xocl_fini_mailbox(void)
+{
+ platform_driver_unregister(&mailbox_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/mb_scheduler.c b/drivers/gpu/drm/xocl/subdev/mb_scheduler.c
new file mode 100644
index 000000000000..b3ed3ae0b41a
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/mb_scheduler.c
@@ -0,0 +1,3059 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2018-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ * Soren Soe <soren.soe@xilinx.com>
+ */
+
+/*
+ * Kernel Driver Scheduler (KDS) for XRT
+ *
+ * struct xocl_cmd
+ * - wraps exec BOs create from user space
+ * - transitions through a number of states
+ * - initially added to pending command queue
+ * - consumed by scheduler which manages its execution (state transition)
+ * struct xcol_cu
+ * - compute unit for executing commands
+ * - used only without embedded scheduler (ert)
+ * - talks to HW compute units
+ * struct xocl_ert
+ * - embedded scheduler for executing commands on ert
+ * - talks to HW ERT
+ * struct exec_core
+ * - execution core managing execution on one device
+ * struct xocl_scheduler
+ * - manages execution of cmds on one or more exec cores
+ * - executed in a separate kernel thread
+ * - loops repeatedly when there is work to do
+ * - moves pending commands into a scheduler command queue
+ *
+ * [new -> pending]. The xocl API adds exec BOs to KDS. The exec BOs are
+ * wrapped in a xocl_cmd object and added to a pending command queue.
+ *
+ * [pending -> queued]. Scheduler loops repeatedly and copies pending commands
+ * to its own command queue, then managaes command execution on one or more
+ * execution cores.
+ *
+ * [queued -> submitted]. Commands are submitted for execution on execution
+ * core when the core has room for new commands.
+ *
+ * [submitted -> running]. Once submitted, a command is transition by
+ * scheduler into running state when there is an available compute unit (no
+ * ert) or if ERT is used, then when ERT has room.
+ *
+ * [running -> complete]. Commands running on ERT complete by sending an
+ * interrupt to scheduler. When ERT is not used, commands are running on a
+ * compute unit and are polled for completion.
+ */
+
+#include <linux/bitmap.h>
+#include <linux/list.h>
+#include <linux/eventfd.h>
+#include <linux/kthread.h>
+#include "../ert.h"
+#include "../xocl_drv.h"
+#include "../userpf/common.h"
+
+//#define SCHED_VERBOSE
+
+#if defined(__GNUC__)
+#define SCHED_UNUSED __attribute__((unused))
+#endif
+
+#define sched_debug_packet(packet, size) \
+({ \
+ int i; \
+ u32 *data = (u32 *)packet; \
+ for (i = 0; i < size; ++i) \
+ DRM_INFO("packet(0x%p) data[%d] = 0x%x\n", data, i, data[i]); \
+})
+
+#ifdef SCHED_VERBOSE
+# define SCHED_DEBUG(msg) DRM_INFO(msg)
+# define SCHED_DEBUGF(format, ...) DRM_INFO(format, ##__VA_ARGS__)
+# define SCHED_PRINTF(format, ...) DRM_INFO(format, ##__VA_ARGS__)
+# define SCHED_DEBUG_PACKET(packet, size) sched_debug_packet(packet, size)
+#else
+# define SCHED_DEBUG(msg)
+# define SCHED_DEBUGF(format, ...)
+# define SCHED_PRINTF(format, ...) DRM_INFO(format, ##__VA_ARGS__)
+# define SCHED_DEBUG_PACKET(packet, size)
+#endif
+
+/* constants */
+static const unsigned int no_index = -1;
+
+/* FFA handling */
+static const u32 AP_START = 0x1;
+static const u32 AP_DONE = 0x2;
+static const u32 AP_IDLE = 0x4;
+static const u32 AP_READY = 0x8;
+static const u32 AP_CONTINUE = 0x10;
+
+/* Forward declaration */
+struct exec_core;
+struct exec_ops;
+struct xocl_scheduler;
+
+static int validate(struct platform_device *pdev, struct client_ctx *client,
+ const struct drm_xocl_bo *bo);
+static bool exec_is_flush(struct exec_core *exec);
+static void scheduler_wake_up(struct xocl_scheduler *xs);
+static void scheduler_intr(struct xocl_scheduler *xs);
+static void scheduler_decr_poll(struct xocl_scheduler *xs);
+
+/*
+ */
+static void
+xocl_bitmap_to_arr32(u32 *buf, const unsigned long *bitmap, unsigned int nbits)
+{
+ unsigned int i, halfwords;
+
+ halfwords = DIV_ROUND_UP(nbits, 32);
+ for (i = 0; i < halfwords; i++) {
+ buf[i] = (u32) (bitmap[i/2] & UINT_MAX);
+ if (++i < halfwords)
+ buf[i] = (u32) (bitmap[i/2] >> 32);
+ }
+
+ /* Clear tail bits in last element of array beyond nbits. */
+ if (nbits % BITS_PER_LONG)
+ buf[halfwords - 1] &= (u32) (UINT_MAX >> ((-nbits) & 31));
+}
+
+static void
+xocl_bitmap_from_arr32(unsigned long *bitmap, const u32 *buf, unsigned int nbits)
+{
+ unsigned int i, halfwords;
+
+ halfwords = DIV_ROUND_UP(nbits, 32);
+ for (i = 0; i < halfwords; i++) {
+ bitmap[i/2] = (unsigned long) buf[i];
+ if (++i < halfwords)
+ bitmap[i/2] |= ((unsigned long) buf[i]) << 32;
+ }
+
+ /* Clear tail bits in last word beyond nbits. */
+ if (nbits % BITS_PER_LONG)
+ bitmap[(halfwords - 1) / 2] &= BITMAP_LAST_WORD_MASK(nbits);
+}
+
+
+/**
+ * slot_mask_idx() - Slot mask idx index for a given slot_idx
+ *
+ * @slot_idx: Global [0..127] index of a CQ slot
+ * Return: Index of the slot mask containing the slot_idx
+ */
+static inline unsigned int
+slot_mask_idx(unsigned int slot_idx)
+{
+ return slot_idx >> 5;
+}
+
+/**
+ * slot_idx_in_mask() - Index of command queue slot within the mask that contains it
+ *
+ * @slot_idx: Global [0..127] index of a CQ slot
+ * Return: Index of slot within the mask that contains it
+ */
+static inline unsigned int
+slot_idx_in_mask(unsigned int slot_idx)
+{
+ return slot_idx - (slot_mask_idx(slot_idx) << 5);
+}
+
+/**
+ * Command data used by scheduler
+ *
+ * @list: command object moves from pending to commmand queue list
+ * @cu_list: command object is added to CU list when running (penguin only)
+ *
+ * @bo: underlying drm buffer object
+ * @exec: execution device associated with this command
+ * @client: client (user process) context that created this command
+ * @xs: command scheduler responsible for schedulint this command
+ * @state: state of command object per scheduling
+ * @id: unique id for an active command object
+ * @cu_idx: index of CU executing this cmd object; used in penguin mode only
+ * @slot_idx: command queue index of this command object
+ * @wait_count: number of commands that must trigger this command before it can start
+ * @chain_count: number of commands that this command must trigger when it completes
+ * @chain: list of commands to trigger upon completion; maximum chain depth is 8
+ * @deps: list of commands this object depends on, converted to chain when command is queued
+ * @packet: mapped ert packet object from user space
+ */
+struct xocl_cmd {
+ struct list_head cq_list; // scheduler command queue
+ struct list_head rq_list; // exec core running queue
+
+ /* command packet */
+ struct drm_xocl_bo *bo;
+ union {
+ struct ert_packet *ecmd;
+ struct ert_start_kernel_cmd *kcmd;
+ };
+
+ DECLARE_BITMAP(cu_bitmap, MAX_CUS);
+
+ struct xocl_dev *xdev;
+ struct exec_core *exec;
+ struct client_ctx *client;
+ struct xocl_scheduler *xs;
+ enum ert_cmd_state state;
+
+ /* dependency handling */
+ unsigned int chain_count;
+ unsigned int wait_count;
+ union {
+ struct xocl_cmd *chain[8];
+ struct drm_xocl_bo *deps[8];
+ };
+
+ unsigned long uid; // unique id for this command
+ unsigned int cu_idx; // index of CU running this cmd (penguin mode)
+ unsigned int slot_idx; // index in exec core submit queue
+};
+
+/*
+ * List of free xocl_cmd objects.
+ *
+ * @free_cmds: populated with recycled xocl_cmd objects
+ * @cmd_mutex: mutex lock for cmd_list
+ *
+ * Command objects are recycled for later use and only freed when kernel
+ * module is unloaded.
+ */
+static LIST_HEAD(free_cmds);
+static DEFINE_MUTEX(free_cmds_mutex);
+
+/**
+ * delete_cmd_list() - reclaim memory for all allocated command objects
+ */
+static void
+cmd_list_delete(void)
+{
+ struct xocl_cmd *xcmd;
+ struct list_head *pos, *next;
+
+ mutex_lock(&free_cmds_mutex);
+ list_for_each_safe(pos, next, &free_cmds) {
+ xcmd = list_entry(pos, struct xocl_cmd, cq_list);
+ list_del(pos);
+ kfree(xcmd);
+ }
+ mutex_unlock(&free_cmds_mutex);
+}
+
+/*
+ * opcode() - Command opcode
+ *
+ * @cmd: Command object
+ * Return: Opcode per command packet
+ */
+static inline u32
+cmd_opcode(struct xocl_cmd *xcmd)
+{
+ return xcmd->ecmd->opcode;
+}
+
+/*
+ * type() - Command type
+ *
+ * @cmd: Command object
+ * Return: Type of command
+ */
+static inline u32
+cmd_type(struct xocl_cmd *xcmd)
+{
+ return xcmd->ecmd->type;
+}
+
+/*
+ * exec() - Get execution core
+ */
+static inline struct exec_core *
+cmd_exec(struct xocl_cmd *xcmd)
+{
+ return xcmd->exec;
+}
+
+/*
+ * uid() - Get unique id of command
+ */
+static inline unsigned long
+cmd_uid(struct xocl_cmd *xcmd)
+{
+ return xcmd->uid;
+}
+
+/*
+ */
+static inline unsigned int
+cmd_wait_count(struct xocl_cmd *xcmd)
+{
+ return xcmd->wait_count;
+}
+
+/**
+ * payload_size() - Command payload size
+ *
+ * @xcmd: Command object
+ * Return: Size in number of words of command packet payload
+ */
+static inline unsigned int
+cmd_payload_size(struct xocl_cmd *xcmd)
+{
+ return xcmd->ecmd->count;
+}
+
+/**
+ * cmd_packet_size() - Command packet size
+ *
+ * @xcmd: Command object
+ * Return: Size in number of words of command packet
+ */
+static inline unsigned int
+cmd_packet_size(struct xocl_cmd *xcmd)
+{
+ return cmd_payload_size(xcmd) + 1;
+}
+
+/**
+ * cu_masks() - Number of command packet cu_masks
+ *
+ * @xcmd: Command object
+ * Return: Total number of CU masks in command packet
+ */
+static inline unsigned int
+cmd_cumasks(struct xocl_cmd *xcmd)
+{
+ return 1 + xcmd->kcmd->extra_cu_masks;
+}
+
+/**
+ * regmap_size() - Size of regmap is payload size (n) minus the number of cu_masks
+ *
+ * @xcmd: Command object
+ * Return: Size of register map in number of words
+ */
+static inline unsigned int
+cmd_regmap_size(struct xocl_cmd *xcmd)
+{
+ return cmd_payload_size(xcmd) - cmd_cumasks(xcmd);
+}
+
+/*
+ */
+static inline struct ert_packet*
+cmd_packet(struct xocl_cmd *xcmd)
+{
+ return xcmd->ecmd;
+}
+
+/*
+ */
+static inline u32*
+cmd_regmap(struct xocl_cmd *xcmd)
+{
+ return xcmd->kcmd->data + xcmd->kcmd->extra_cu_masks;
+}
+
+/**
+ * cmd_set_int_state() - Set internal command state used by scheduler only
+ *
+ * @xcmd: command to change internal state on
+ * @state: new command state per ert.h
+ */
+static inline void
+cmd_set_int_state(struct xocl_cmd *xcmd, enum ert_cmd_state state)
+{
+ SCHED_DEBUGF("-> %s(%lu,%d)\n", __func__, xcmd->uid, state);
+ xcmd->state = state;
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * cmd_set_state() - Set both internal and external state of a command
+ *
+ * The state is reflected externally through the command packet
+ * as well as being captured in internal state variable
+ *
+ * @xcmd: command object
+ * @state: new state
+ */
+static inline void
+cmd_set_state(struct xocl_cmd *xcmd, enum ert_cmd_state state)
+{
+ SCHED_DEBUGF("->%s(%lu,%d)\n", __func__, xcmd->uid, state);
+ xcmd->state = state;
+ xcmd->ecmd->state = state;
+ SCHED_DEBUGF("<-%s\n", __func__);
+}
+
+/*
+ * update_state() - Update command state if client has aborted
+ */
+static enum ert_cmd_state
+cmd_update_state(struct xocl_cmd *xcmd)
+{
+ if (xcmd->state != ERT_CMD_STATE_RUNNING && xcmd->client->abort) {
+ userpf_info(xcmd->xdev, "aborting stale client cmd(%lu)", xcmd->uid);
+ cmd_set_state(xcmd, ERT_CMD_STATE_ABORT);
+ }
+ if (exec_is_flush(xcmd->exec)) {
+ userpf_info(xcmd->xdev, "aborting stale exec cmd(%lu)", xcmd->uid);
+ cmd_set_state(xcmd, ERT_CMD_STATE_ABORT);
+ }
+ return xcmd->state;
+}
+
+/*
+ * release_gem_object_reference() -
+ */
+static inline void
+cmd_release_gem_object_reference(struct xocl_cmd *xcmd)
+{
+ if (xcmd->bo)
+ drm_gem_object_put_unlocked(&xcmd->bo->base);
+//PORT4_20
+// drm_gem_object_unreference_unlocked(&xcmd->bo->base);
+}
+
+/*
+ */
+static inline void
+cmd_mark_active(struct xocl_cmd *xcmd)
+{
+ if (xcmd->bo)
+ xcmd->bo->metadata.active = xcmd;
+}
+
+/*
+ */
+static inline void
+cmd_mark_deactive(struct xocl_cmd *xcmd)
+{
+ if (xcmd->bo)
+ xcmd->bo->metadata.active = NULL;
+}
+
+/**
+ * chain_dependencies() - Chain this command to its dependencies
+ *
+ * @xcmd: Command to chain to its dependencies
+ *
+ * This function looks at all incoming explicit BO dependencies, checks if a
+ * corresponding xocl_cmd object exists (is active) in which case that command
+ * object must chain argument xcmd so that it (xcmd) can be triggered when
+ * dependency completes. The chained command has a wait count corresponding to
+ * the number of dependencies that are active.
+ */
+static int
+cmd_chain_dependencies(struct xocl_cmd *xcmd)
+{
+ int didx;
+ int dcount = xcmd->wait_count;
+
+ SCHED_DEBUGF("-> chain_dependencies of xcmd(%lu)\n", xcmd->uid);
+ for (didx = 0; didx < dcount; ++didx) {
+ struct drm_xocl_bo *dbo = xcmd->deps[didx];
+ struct xocl_cmd *chain_to = dbo->metadata.active;
+ // release reference created in ioctl call when dependency was looked up
+ // see comments in xocl_ioctl.c:xocl_execbuf_ioctl()
+//PORT4_20
+// drm_gem_object_unreference_unlocked(&dbo->base);
+ drm_gem_object_put_unlocked(&dbo->base);
+ xcmd->deps[didx] = NULL;
+ if (!chain_to) { /* command may have completed already */
+ --xcmd->wait_count;
+ continue;
+ }
+ if (chain_to->chain_count >= MAX_DEPS) {
+ DRM_INFO("chain count exceeded");
+ return 1;
+ }
+ SCHED_DEBUGF("+ xcmd(%lu)->chain[%d]=xcmd(%lu)", chain_to->uid, chain_to->chain_count, xcmd->uid);
+ chain_to->chain[chain_to->chain_count++] = xcmd;
+ }
+ SCHED_DEBUG("<- chain_dependencies\n");
+ return 0;
+}
+
+/**
+ * trigger_chain() - Trigger the execution of any commands chained to argument command
+ *
+ * @xcmd: Completed command that must trigger its chained (waiting) commands
+ *
+ * The argument command has completed and must trigger the execution of all
+ * chained commands whos wait_count is 0.
+ */
+static void
+cmd_trigger_chain(struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> trigger_chain xcmd(%lu)\n", xcmd->uid);
+ while (xcmd->chain_count) {
+ struct xocl_cmd *trigger = xcmd->chain[--xcmd->chain_count];
+
+ SCHED_DEBUGF("+ cmd(%lu) triggers cmd(%lu) with wait_count(%d)\n",
+ xcmd->uid, trigger->uid, trigger->wait_count);
+ // decrement trigger wait count
+ // scheduler will submit when wait count reaches zero
+ --trigger->wait_count;
+ }
+ SCHED_DEBUG("<- trigger_chain\n");
+}
+
+
+/**
+ * cmd_get() - Get a free command object
+ *
+ * Get from free/recycled list or allocate a new command if necessary.
+ *
+ * Return: Free command object
+ */
+static struct xocl_cmd*
+cmd_get(struct xocl_scheduler *xs, struct exec_core *exec, struct client_ctx *client)
+{
+ struct xocl_cmd *xcmd;
+ static unsigned long count;
+
+ mutex_lock(&free_cmds_mutex);
+ xcmd = list_first_entry_or_null(&free_cmds, struct xocl_cmd, cq_list);
+ if (xcmd)
+ list_del(&xcmd->cq_list);
+ mutex_unlock(&free_cmds_mutex);
+ if (!xcmd)
+ xcmd = kmalloc(sizeof(struct xocl_cmd), GFP_KERNEL);
+ if (!xcmd)
+ return ERR_PTR(-ENOMEM);
+ xcmd->uid = count++;
+ xcmd->exec = exec;
+ xcmd->cu_idx = no_index;
+ xcmd->slot_idx = no_index;
+ xcmd->xs = xs;
+ xcmd->xdev = client->xdev;
+ xcmd->client = client;
+ xcmd->bo = NULL;
+ xcmd->ecmd = NULL;
+ atomic_inc(&client->outstanding_execs);
+ SCHED_DEBUGF("xcmd(%lu) xcmd(%p) [-> new ]\n", xcmd->uid, xcmd);
+ return xcmd;
+}
+
+/**
+ * cmd_free() - free a command object
+ *
+ * @xcmd: command object to free (move to freelist)
+ *
+ * The command *is* in some current list (scheduler command queue)
+ */
+static void
+cmd_free(struct xocl_cmd *xcmd)
+{
+ cmd_release_gem_object_reference(xcmd);
+
+ mutex_lock(&free_cmds_mutex);
+ list_move_tail(&xcmd->cq_list, &free_cmds);
+ mutex_unlock(&free_cmds_mutex);
+
+ atomic_dec(&xcmd->xdev->outstanding_execs);
+ atomic_dec(&xcmd->client->outstanding_execs);
+ SCHED_DEBUGF("xcmd(%lu) [-> free]\n", xcmd->uid);
+}
+
+/**
+ * abort_cmd() - abort command object before it becomes pending
+ *
+ * @xcmd: command object to abort (move to freelist)
+ *
+ * Command object is *not* in any current list
+ *
+ * Return: 0
+ */
+static void
+cmd_abort(struct xocl_cmd *xcmd)
+{
+ mutex_lock(&free_cmds_mutex);
+ list_add_tail(&xcmd->cq_list, &free_cmds);
+ mutex_unlock(&free_cmds_mutex);
+ SCHED_DEBUGF("xcmd(%lu) [-> abort]\n", xcmd->uid);
+}
+
+/*
+ * cmd_bo_init() - Initialize a command object with an exec BO
+ *
+ * In penguin mode, the command object caches the CUs available
+ * to execute the command. When ERT is enabled, the CU info
+ * is not used.
+ */
+static void
+cmd_bo_init(struct xocl_cmd *xcmd, struct drm_xocl_bo *bo,
+ int numdeps, struct drm_xocl_bo **deps, int penguin)
+{
+ SCHED_DEBUGF("%s(%lu,bo,%d,deps,%d)\n", __func__, xcmd->uid, numdeps, penguin);
+ xcmd->bo = bo;
+ xcmd->ecmd = (struct ert_packet *)bo->vmapping;
+
+ if (penguin && cmd_opcode(xcmd) == ERT_START_KERNEL) {
+ unsigned int i = 0;
+ u32 cumasks[4] = {0};
+
+ cumasks[0] = xcmd->kcmd->cu_mask;
+ SCHED_DEBUGF("+ xcmd(%lu) cumask[0]=0x%x\n", xcmd->uid, cumasks[0]);
+ for (i = 0; i < xcmd->kcmd->extra_cu_masks; ++i) {
+ cumasks[i+1] = xcmd->kcmd->data[i];
+ SCHED_DEBUGF("+ xcmd(%lu) cumask[%d]=0x%x\n", xcmd->uid, i+1, cumasks[i+1]);
+ }
+ xocl_bitmap_from_arr32(xcmd->cu_bitmap, cumasks, MAX_CUS);
+ SCHED_DEBUGF("cu_bitmap[0] = %lu\n", xcmd->cu_bitmap[0]);
+ }
+
+ // dependencies are copied here, the anticipated wait_count is number
+ // of specified dependencies. The wait_count is adjusted when the
+ // command is queued in the scheduler based on whether or not a
+ // dependency is active (managed by scheduler)
+ memcpy(xcmd->deps, deps, numdeps*sizeof(struct drm_xocl_bo *));
+ xcmd->wait_count = numdeps;
+ xcmd->chain_count = 0;
+}
+
+/*
+ */
+static void
+cmd_packet_init(struct xocl_cmd *xcmd, struct ert_packet *packet)
+{
+ SCHED_DEBUGF("%s(%lu,packet)\n", __func__, xcmd->uid);
+ xcmd->ecmd = packet;
+}
+
+/*
+ * cmd_has_cu() - Check if this command object can execute on CU
+ *
+ * @cuidx: the index of the CU. Note that CU indicies start from 0.
+ */
+static int
+cmd_has_cu(struct xocl_cmd *xcmd, unsigned int cuidx)
+{
+ SCHED_DEBUGF("%s(%lu,%d) = %d\n", __func__, xcmd->uid, cuidx, test_bit(cuidx, xcmd->cu_bitmap));
+ return test_bit(cuidx, xcmd->cu_bitmap);
+}
+
+/*
+ * struct xocl_cu: Represents a compute unit in penguin mode
+ *
+ * @running_queue: a fifo representing commands running on this CU
+ * @xdev: the xrt device with this CU
+ * @idx: index of this CU
+ * @base: exec base address of this CU
+ * @addr: base address of this CU
+ * @ctrlreg: state of the CU (value of AXI-lite control register)
+ * @done_cnt: number of command that have completed (<=running_queue.size())
+ *
+ */
+struct xocl_cu {
+ struct list_head running_queue;
+ unsigned int idx;
+ void __iomem *base;
+ u32 addr;
+
+ u32 ctrlreg;
+ unsigned int done_cnt;
+ unsigned int run_cnt;
+ unsigned int uid;
+};
+
+/*
+ */
+void
+cu_reset(struct xocl_cu *xcu, unsigned int idx, void __iomem *base, u32 addr)
+{
+ xcu->idx = idx;
+ xcu->base = base;
+ xcu->addr = addr;
+ xcu->ctrlreg = 0;
+ xcu->done_cnt = 0;
+ xcu->run_cnt = 0;
+ SCHED_DEBUGF("%s(uid:%d,idx:%d) @ 0x%x\n", __func__, xcu->uid, xcu->idx, xcu->addr);
+}
+
+/*
+ */
+struct xocl_cu *
+cu_create(void)
+{
+ struct xocl_cu *xcu = kmalloc(sizeof(struct xocl_cu), GFP_KERNEL);
+ static unsigned int uid;
+
+ INIT_LIST_HEAD(&xcu->running_queue);
+ xcu->uid = uid++;
+ SCHED_DEBUGF("%s(uid:%d)\n", __func__, xcu->uid);
+ return xcu;
+}
+
+static inline u32
+cu_base_addr(struct xocl_cu *xcu)
+{
+ return xcu->addr;
+}
+
+/*
+ */
+void
+cu_destroy(struct xocl_cu *xcu)
+{
+ SCHED_DEBUGF("%s(uid:%d)\n", __func__, xcu->uid);
+ kfree(xcu);
+}
+
+/*
+ */
+void
+cu_poll(struct xocl_cu *xcu)
+{
+ // assert !list_empty(&running_queue)
+ xcu->ctrlreg = ioread32(xcu->base + xcu->addr);
+ SCHED_DEBUGF("%s(%d) 0x%x done(%d) run(%d)\n", __func__, xcu->idx, xcu->ctrlreg, xcu->done_cnt, xcu->run_cnt);
+ if (xcu->ctrlreg & AP_DONE) {
+ ++xcu->done_cnt; // assert done_cnt <= |running_queue|
+ --xcu->run_cnt;
+ // acknowledge done
+ iowrite32(AP_CONTINUE, xcu->base + xcu->addr);
+ }
+}
+
+/*
+ * cu_ready() - Check if CU is ready to start another command
+ *
+ * The CU is ready when AP_START is low
+ */
+static int
+cu_ready(struct xocl_cu *xcu)
+{
+ if (xcu->ctrlreg & AP_START)
+ cu_poll(xcu);
+
+ SCHED_DEBUGF("%s(%d) returns %d\n", __func__, xcu->idx, !(xcu->ctrlreg & AP_START));
+ return !(xcu->ctrlreg & AP_START);
+}
+
+/*
+ * cu_first_done() - Get the first completed command from the running queue
+ *
+ * Return: The first command that has completed or nullptr if none
+ */
+static struct xocl_cmd*
+cu_first_done(struct xocl_cu *xcu)
+{
+ if (!xcu->done_cnt)
+ cu_poll(xcu);
+
+ SCHED_DEBUGF("%s(%d) has done_cnt %d\n", __func__, xcu->idx, xcu->done_cnt);
+
+ return xcu->done_cnt
+ ? list_first_entry(&xcu->running_queue, struct xocl_cmd, rq_list)
+ : NULL;
+}
+
+/*
+ * cu_pop_done() - Remove first element from running queue
+ */
+static void
+cu_pop_done(struct xocl_cu *xcu)
+{
+ struct xocl_cmd *xcmd;
+
+ if (!xcu->done_cnt)
+ return;
+ xcmd = list_first_entry(&xcu->running_queue, struct xocl_cmd, rq_list);
+ list_del(&xcmd->rq_list);
+ --xcu->done_cnt;
+ SCHED_DEBUGF("%s(%d) xcmd(%lu) done(%d) run(%d)\n", __func__, xcu->idx, xcmd->uid, xcu->done_cnt, xcu->run_cnt);
+}
+
+/*
+ * cu_start() - Start the CU with a new command.
+ *
+ * The command is pushed onto the running queue
+ */
+static int
+cu_start(struct xocl_cu *xcu, struct xocl_cmd *xcmd)
+{
+ // assert(!(ctrlreg & AP_START), "cu not ready");
+
+ // data past header and cu_masks
+ unsigned int size = cmd_regmap_size(xcmd);
+ u32 *regmap = cmd_regmap(xcmd);
+ unsigned int i;
+
+ // past header, past cumasks
+ SCHED_DEBUG_PACKET(regmap, size);
+
+ // write register map, starting at base + 0xC
+ // 0x4, 0x8 used for interrupt, which is initialized in setu
+ for (i = 1; i < size; ++i)
+ iowrite32(*(regmap + i), xcu->base + xcu->addr + (i << 2));
+
+ // start cu. update local state as we may not be polling prior
+ // to next ready check.
+ xcu->ctrlreg |= AP_START;
+ iowrite32(AP_START, xcu->base + xcu->addr);
+
+ // add cmd to end of running queue
+ list_add_tail(&xcmd->rq_list, &xcu->running_queue);
+ ++xcu->run_cnt;
+
+ SCHED_DEBUGF("%s(%d) started xcmd(%lu) done(%d) run(%d)\n",
+ __func__, xcu->idx, xcmd->uid, xcu->done_cnt, xcu->run_cnt);
+
+ return true;
+}
+
+
+/*
+ * sruct xocl_ert: Represents embedded scheduler in ert mode
+ */
+struct xocl_ert {
+ void __iomem *base;
+ u32 cq_addr;
+ unsigned int uid;
+
+ unsigned int slot_size;
+ unsigned int cq_intr;
+};
+
+/*
+ */
+struct xocl_ert *
+ert_create(void __iomem *base, u32 cq_addr)
+{
+ struct xocl_ert *xert = kmalloc(sizeof(struct xocl_ert), GFP_KERNEL);
+ static unsigned int uid;
+
+ xert->base = base;
+ xert->cq_addr = cq_addr;
+ xert->uid = uid++;
+ xert->slot_size = 0;
+ xert->cq_intr = false;
+ SCHED_DEBUGF("%s(%d,0x%x)\n", __func__, xert->uid, xert->cq_addr);
+ return xert;
+}
+
+/*
+ */
+static void
+ert_destroy(struct xocl_ert *xert)
+{
+ SCHED_DEBUGF("%s(%d)\n", __func__, xert->uid);
+ kfree(xert);
+}
+
+/*
+ */
+static void
+ert_cfg(struct xocl_ert *xert, unsigned int slot_size, unsigned int cq_intr)
+{
+ SCHED_DEBUGF("%s(%d) slot_size(%d) cq_intr(%d)\n", __func__, xert->uid, slot_size, cq_intr);
+ xert->slot_size = slot_size;
+ xert->cq_intr = cq_intr;
+}
+
+/*
+ */
+static bool
+ert_start_cmd(struct xocl_ert *xert, struct xocl_cmd *xcmd)
+{
+ u32 slot_addr = xert->cq_addr + xcmd->slot_idx * xert->slot_size;
+ struct ert_packet *ecmd = cmd_packet(xcmd);
+
+ SCHED_DEBUG_PACKET(ecmd, cmd_packet_size(xcmd));
+
+ SCHED_DEBUGF("-> %s(%d,%lu)\n", __func__, xert->uid, xcmd->uid);
+
+ // write packet minus header
+ SCHED_DEBUGF("++ slot_idx=%d, slot_addr=0x%x\n", xcmd->slot_idx, slot_addr);
+ memcpy_toio(xert->base + slot_addr + 4, ecmd->data, (cmd_packet_size(xcmd) - 1) * sizeof(u32));
+
+ // write header
+ iowrite32(ecmd->header, xert->base + slot_addr);
+
+ // trigger interrupt to embedded scheduler if feature is enabled
+ if (xert->cq_intr) {
+ u32 cq_int_addr = ERT_CQ_STATUS_REGISTER_ADDR + (slot_mask_idx(xcmd->slot_idx) << 2);
+ u32 mask = 1 << slot_idx_in_mask(xcmd->slot_idx);
+
+ SCHED_DEBUGF("++ mb_submit writes slot mask 0x%x to CQ_INT register at addr 0x%x\n",
+ mask, cq_int_addr);
+ iowrite32(mask, xert->base + cq_int_addr);
+ }
+ SCHED_DEBUGF("<- %s returns true\n", __func__);
+ return true;
+}
+
+/*
+ */
+static void
+ert_read_custat(struct xocl_ert *xert, unsigned int num_cus, u32 *cu_usage, struct xocl_cmd *xcmd)
+{
+ u32 slot_addr = xert->cq_addr + xcmd->slot_idx*xert->slot_size;
+
+ memcpy_fromio(cu_usage, xert->base + slot_addr + 4, num_cus * sizeof(u32));
+}
+
+/**
+ * struct exec_ops: scheduler specific operations
+ *
+ * Scheduler can operate in MicroBlaze mode (mb/ert) or in penguin mode. This
+ * struct differentiates specific operations. The struct is per device node,
+ * meaning that one device can operate in ert mode while another can operate
+ * in penguin mode.
+ */
+struct exec_ops {
+ bool (*start)(struct exec_core *exec, struct xocl_cmd *xcmd);
+ void (*query)(struct exec_core *exec, struct xocl_cmd *xcmd);
+};
+
+static struct exec_ops ert_ops;
+static struct exec_ops penguin_ops;
+
+/**
+ * struct exec_core: Core data structure for command execution on a device
+ *
+ * @ctx_list: Context list populated with device context
+ * @exec_lock: Lock for synchronizing external access
+ * @poll_wait_queue: Wait queue for device polling
+ * @scheduler: Command queue scheduler
+ * @submitted_cmds: Tracking of command submitted for execution on this device
+ * @num_slots: Number of command queue slots
+ * @num_cus: Number of CUs in loaded program
+ * @num_cdma: Number of CDMAs in hardware
+ * @polling_mode: If set then poll for command completion
+ * @cq_interrupt: If set then trigger interrupt to MB on new commands
+ * @configured: Flag to indicate that the core data structure has been initialized
+ * @stopped: Flag to indicate that the core data structure cannot be used
+ * @flush: Flag to indicate that commands for this device should be flushed
+ * @cu_usage: Usage count since last reset
+ * @slot_status: Bitmap to track status (busy(1)/free(0)) slots in command queue
+ * @ctrl_busy: Flag to indicate that slot 0 (ctrl commands) is busy
+ * @cu_status: Bitmap to track status (busy(1)/free(0)) of CUs. Unused in ERT mode.
+ * @sr0: If set, then status register [0..31] is pending with completed commands (ERT only).
+ * @sr1: If set, then status register [32..63] is pending with completed commands (ERT only).
+ * @sr2: If set, then status register [64..95] is pending with completed commands (ERT only).
+ * @sr3: If set, then status register [96..127] is pending with completed commands (ERT only).
+ * @ops: Scheduler operations vtable
+ */
+struct exec_core {
+ struct platform_device *pdev;
+
+ struct mutex exec_lock;
+
+ void __iomem *base;
+ u32 intr_base;
+ u32 intr_num;
+
+ wait_queue_head_t poll_wait_queue;
+
+ struct xocl_scheduler *scheduler;
+
+ uuid_t xclbin_id;
+
+ unsigned int num_slots;
+ unsigned int num_cus;
+ unsigned int num_cdma;
+ unsigned int polling_mode;
+ unsigned int cq_interrupt;
+ unsigned int configured;
+ unsigned int stopped;
+ unsigned int flush;
+
+ struct xocl_cu *cus[MAX_CUS];
+ struct xocl_ert *ert;
+
+ u32 cu_usage[MAX_CUS];
+
+ // Bitmap tracks busy(1)/free(0) slots in cmd_slots
+ struct xocl_cmd *submitted_cmds[MAX_SLOTS];
+ DECLARE_BITMAP(slot_status, MAX_SLOTS);
+ unsigned int ctrl_busy;
+
+ // Status register pending complete. Written by ISR,
+ // cleared by scheduler
+ atomic_t sr0;
+ atomic_t sr1;
+ atomic_t sr2;
+ atomic_t sr3;
+
+ // Operations for dynamic indirection dependt on MB
+ // or kernel scheduler
+ struct exec_ops *ops;
+
+ unsigned int uid;
+ unsigned int ip_reference[MAX_CUS];
+};
+
+/**
+ * exec_get_pdev() -
+ */
+static inline struct platform_device *
+exec_get_pdev(struct exec_core *exec)
+{
+ return exec->pdev;
+}
+
+/**
+ * exec_get_xdev() -
+ */
+static inline struct xocl_dev *
+exec_get_xdev(struct exec_core *exec)
+{
+ return xocl_get_xdev(exec->pdev);
+}
+
+/*
+ */
+static inline bool
+exec_is_ert(struct exec_core *exec)
+{
+ return exec->ops == &ert_ops;
+}
+
+/*
+ */
+static inline bool
+exec_is_polling(struct exec_core *exec)
+{
+ return exec->polling_mode;
+}
+
+/*
+ */
+static inline bool
+exec_is_flush(struct exec_core *exec)
+{
+ return exec->flush;
+}
+
+/*
+ */
+static inline u32
+exec_cu_base_addr(struct exec_core *exec, unsigned int cuidx)
+{
+ return cu_base_addr(exec->cus[cuidx]);
+}
+
+/*
+ */
+static inline u32
+exec_cu_usage(struct exec_core *exec, unsigned int cuidx)
+{
+ return exec->cu_usage[cuidx];
+}
+
+/*
+ */
+static void
+exec_cfg(struct exec_core *exec)
+{
+}
+
+/*
+ * to be automated
+ */
+static int
+exec_cfg_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+ struct client_ctx *client = xcmd->client;
+ bool ert = xocl_mb_sched_on(xdev);
+ uint32_t *cdma = xocl_cdma_addr(xdev);
+ unsigned int dsa = xocl_dsa_version(xdev);
+ struct ert_configure_cmd *cfg;
+ int cuidx = 0;
+
+ /* Only allow configuration with one live ctx */
+ if (exec->configured) {
+ DRM_INFO("command scheduler is already configured for this device\n");
+ return 1;
+ }
+
+ DRM_INFO("ert per feature rom = %d\n", ert);
+ DRM_INFO("dsa per feature rom = %d\n", dsa);
+
+ cfg = (struct ert_configure_cmd *)(xcmd->ecmd);
+
+ /* Mark command as control command to force slot 0 execution */
+ cfg->type = ERT_CTRL;
+
+ if (cfg->count != 5 + cfg->num_cus) {
+ DRM_INFO("invalid configure command, count=%d expected 5+num_cus(%d)\n", cfg->count, cfg->num_cus);
+ return 1;
+ }
+
+ SCHED_DEBUG("configuring scheduler\n");
+ exec->num_slots = ERT_CQ_SIZE / cfg->slot_size;
+ exec->num_cus = cfg->num_cus;
+ exec->num_cdma = 0;
+
+ // skip this in polling mode
+ for (cuidx = 0; cuidx < exec->num_cus; ++cuidx) {
+ struct xocl_cu *xcu = exec->cus[cuidx];
+
+ if (!xcu)
+ xcu = exec->cus[cuidx] = cu_create();
+ cu_reset(xcu, cuidx, exec->base, cfg->data[cuidx]);
+ userpf_info(xdev, "%s cu(%d) at 0x%x\n", __func__, xcu->idx, xcu->addr);
+ }
+
+ if (cdma) {
+ uint32_t *addr = 0;
+
+ mutex_lock(&client->lock); /* for modification to client cu_bitmap */
+ for (addr = cdma; addr < cdma+4; ++addr) { /* 4 is from xclfeatures.h */
+ if (*addr) {
+ struct xocl_cu *xcu = exec->cus[cuidx];
+
+ if (!xcu)
+ xcu = exec->cus[cuidx] = cu_create();
+ cu_reset(xcu, cuidx, exec->base, *addr);
+ ++exec->num_cus;
+ ++exec->num_cdma;
+ ++cfg->num_cus;
+ ++cfg->count;
+ cfg->data[cuidx] = *addr;
+ set_bit(cuidx, client->cu_bitmap); /* cdma is shared */
+ userpf_info(xdev, "configure cdma as cu(%d) at 0x%x\n", cuidx, *addr);
+ ++cuidx;
+ }
+ }
+ mutex_unlock(&client->lock);
+ }
+
+ if (ert && cfg->ert) {
+ SCHED_DEBUG("++ configuring embedded scheduler mode\n");
+ if (!exec->ert)
+ exec->ert = ert_create(exec->base, ERT_CQ_BASE_ADDR);
+ ert_cfg(exec->ert, cfg->slot_size, cfg->cq_int);
+ exec->ops = &ert_ops;
+ exec->polling_mode = cfg->polling;
+ exec->cq_interrupt = cfg->cq_int;
+ cfg->dsa52 = (dsa >= 52) ? 1 : 0;
+ cfg->cdma = cdma ? 1 : 0;
+ } else {
+ SCHED_DEBUG("++ configuring penguin scheduler mode\n");
+ exec->ops = &penguin_ops;
+ exec->polling_mode = 1;
+ }
+
+ // reserve slot 0 for control commands
+ set_bit(0, exec->slot_status);
+
+ DRM_INFO("scheduler config ert(%d) slots(%d), cudma(%d), cuisr(%d), cdma(%d), cus(%d)\n"
+ , exec_is_ert(exec)
+ , exec->num_slots
+ , cfg->cu_dma ? 1 : 0
+ , cfg->cu_isr ? 1 : 0
+ , exec->num_cdma
+ , exec->num_cus);
+
+ exec->configured = true;
+ return 0;
+}
+
+/**
+ * exec_reset() - Reset the scheduler
+ *
+ * @exec: Execution core (device) to reset
+ *
+ * TODO: Perform scheduler configuration based on current xclbin
+ * rather than relying of cfg command
+ */
+static void
+exec_reset(struct exec_core *exec)
+{
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+ uuid_t *xclbin_id;
+
+ mutex_lock(&exec->exec_lock);
+
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+
+ userpf_info(xdev, "%s(%d) cfg(%d)\n", __func__, exec->uid, exec->configured);
+
+ // only reconfigure the scheduler on new xclbin
+ if (!xclbin_id || (uuid_equal(&exec->xclbin_id, xclbin_id) && exec->configured)) {
+ exec->stopped = false;
+ exec->configured = false; // TODO: remove, but hangs ERT because of in between AXI resets
+ goto out;
+ }
+
+ userpf_info(xdev, "exec->xclbin(%pUb),xclbin(%pUb)\n", &exec->xclbin_id, xclbin_id);
+ userpf_info(xdev, "%s resets for new xclbin", __func__);
+ memset(exec->cu_usage, 0, MAX_CUS * sizeof(u32));
+ uuid_copy(&exec->xclbin_id, xclbin_id);
+ exec->num_cus = 0;
+ exec->num_cdma = 0;
+
+ exec->num_slots = 16;
+ exec->polling_mode = 1;
+ exec->cq_interrupt = 0;
+ exec->configured = false;
+ exec->stopped = false;
+ exec->flush = false;
+ exec->ops = &penguin_ops;
+
+ bitmap_zero(exec->slot_status, MAX_SLOTS);
+ set_bit(0, exec->slot_status); // reserve for control command
+ exec->ctrl_busy = false;
+
+ atomic_set(&exec->sr0, 0);
+ atomic_set(&exec->sr1, 0);
+ atomic_set(&exec->sr2, 0);
+ atomic_set(&exec->sr3, 0);
+
+ exec_cfg(exec);
+
+out:
+ mutex_unlock(&exec->exec_lock);
+}
+
+/**
+ * exec_stop() - Stop the scheduler from scheduling commands on this core
+ *
+ * @exec: Execution core (device) to stop
+ *
+ * Block access to current exec_core (device). This API must be called prior
+ * to performing an AXI reset and downloading of a new xclbin. Calling this
+ * API flushes the commands running on current device and prevents new
+ * commands from being scheduled on the device. This effectively prevents any
+ * further commands from running on the device
+ */
+static void
+exec_stop(struct exec_core *exec)
+{
+ int idx;
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+ unsigned int outstanding = 0;
+ unsigned int wait_ms = 100;
+ unsigned int retry = 20; // 2 sec
+
+ mutex_lock(&exec->exec_lock);
+ userpf_info(xdev, "%s(%p)\n", __func__, exec);
+ exec->stopped = true;
+ mutex_unlock(&exec->exec_lock);
+
+ // Wait for commands to drain if any
+ outstanding = atomic_read(&xdev->outstanding_execs);
+ while (--retry && outstanding) {
+ userpf_info(xdev, "Waiting for %d outstanding commands to finish", outstanding);
+ msleep(wait_ms);
+ outstanding = atomic_read(&xdev->outstanding_execs);
+ }
+
+ // Last gasp, flush any remaining commands for this device exec core
+ // This is an abnormal case. All exec clients have been destroyed
+ // prior to exec_stop being called (per contract), this implies that
+ // all regular client commands have been flushed.
+ if (outstanding) {
+ // Wake up the scheduler to force one iteration flushing stale
+ // commands for this device
+ exec->flush = 1;
+ scheduler_intr(exec->scheduler);
+
+ // Wait a second
+ msleep(1000);
+ }
+
+ outstanding = atomic_read(&xdev->outstanding_execs);
+ if (outstanding)
+ userpf_err(xdev, "unexpected outstanding commands %d after flush", outstanding);
+
+ // Stale commands were flushed, reset submitted command state
+ for (idx = 0; idx < MAX_SLOTS; ++idx)
+ exec->submitted_cmds[idx] = NULL;
+
+ bitmap_zero(exec->slot_status, MAX_SLOTS);
+ set_bit(0, exec->slot_status); // reserve for control command
+ exec->ctrl_busy = false;
+}
+
+/*
+ */
+static irqreturn_t
+exec_isr(int irq, void *arg)
+{
+ struct exec_core *exec = (struct exec_core *)arg;
+
+ SCHED_DEBUGF("-> xocl_user_event %d\n", irq);
+ if (exec_is_ert(exec) && !exec->polling_mode) {
+
+ if (irq == 0)
+ atomic_set(&exec->sr0, 1);
+ else if (irq == 1)
+ atomic_set(&exec->sr1, 1);
+ else if (irq == 2)
+ atomic_set(&exec->sr2, 1);
+ else if (irq == 3)
+ atomic_set(&exec->sr3, 1);
+
+ /* wake up all scheduler ... currently one only */
+ scheduler_intr(exec->scheduler);
+ } else {
+ userpf_err(exec_get_xdev(exec), "Unhandled isr irq %d, is_ert %d, polling %d",
+ irq, exec_is_ert(exec), exec->polling_mode);
+ }
+ SCHED_DEBUGF("<- xocl_user_event\n");
+ return IRQ_HANDLED;
+}
+
+/*
+ */
+struct exec_core *
+exec_create(struct platform_device *pdev, struct xocl_scheduler *xs)
+{
+ struct exec_core *exec = devm_kzalloc(&pdev->dev, sizeof(struct exec_core), GFP_KERNEL);
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ struct resource *res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ static unsigned int count;
+ unsigned int i;
+
+ if (!exec)
+ return NULL;
+
+ mutex_init(&exec->exec_lock);
+ exec->base = xdev->core.bar_addr;
+
+ exec->intr_base = res->start;
+ exec->intr_num = res->end - res->start + 1;
+ exec->pdev = pdev;
+
+ init_waitqueue_head(&exec->poll_wait_queue);
+ exec->scheduler = xs;
+ exec->uid = count++;
+
+ for (i = 0; i < exec->intr_num; i++) {
+ xocl_user_interrupt_reg(xdev, i+exec->intr_base, exec_isr, exec);
+ xocl_user_interrupt_config(xdev, i + exec->intr_base, true);
+ }
+
+ exec_reset(exec);
+ platform_set_drvdata(pdev, exec);
+
+ SCHED_DEBUGF("%s(%d)\n", __func__, exec->uid);
+
+ return exec;
+}
+
+/*
+ */
+static void
+exec_destroy(struct exec_core *exec)
+{
+ int idx;
+
+ SCHED_DEBUGF("%s(%d)\n", __func__, exec->uid);
+ for (idx = 0; idx < exec->num_cus; ++idx)
+ cu_destroy(exec->cus[idx]);
+ if (exec->ert)
+ ert_destroy(exec->ert);
+ devm_kfree(&exec->pdev->dev, exec);
+}
+
+/*
+ */
+static inline struct xocl_scheduler *
+exec_scheduler(struct exec_core *exec)
+{
+ return exec->scheduler;
+}
+
+/*
+ * acquire_slot_idx() - First available slot index
+ */
+static unsigned int
+exec_acquire_slot_idx(struct exec_core *exec)
+{
+ unsigned int idx = find_first_zero_bit(exec->slot_status, MAX_SLOTS);
+
+ SCHED_DEBUGF("%s(%d) returns %d\n", __func__, exec->uid, idx < exec->num_slots ? idx : no_index);
+ if (idx < exec->num_slots) {
+ set_bit(idx, exec->slot_status);
+ return idx;
+ }
+ return no_index;
+}
+
+
+/**
+ * acquire_slot() - Acquire a slot index for a command
+ *
+ * This function makes a special case for control commands which
+ * must always dispatch to slot 0, otherwise normal acquisition
+ */
+static int
+exec_acquire_slot(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ // slot 0 is reserved for ctrl commands
+ if (cmd_type(xcmd) == ERT_CTRL) {
+ SCHED_DEBUGF("%s(%d,%lu) ctrl cmd\n", __func__, exec->uid, xcmd->uid);
+ if (exec->ctrl_busy)
+ return -1;
+ exec->ctrl_busy = true;
+ return (xcmd->slot_idx = 0);
+ }
+
+ return (xcmd->slot_idx = exec_acquire_slot_idx(exec));
+}
+
+/*
+ * release_slot_idx() - Release specified slot idx
+ */
+static void
+exec_release_slot_idx(struct exec_core *exec, unsigned int slot_idx)
+{
+ clear_bit(slot_idx, exec->slot_status);
+}
+
+/**
+ * release_slot() - Release a slot index for a command
+ *
+ * Special case for control commands that execute in slot 0. This
+ * slot cannot be marked free ever.
+ */
+static void
+exec_release_slot(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ if (xcmd->slot_idx == no_index)
+ return; // already released
+
+ SCHED_DEBUGF("%s(%d) xcmd(%lu) slotidx(%d)\n",
+ __func__, exec->uid, xcmd->uid, xcmd->slot_idx);
+ if (cmd_type(xcmd) == ERT_CTRL) {
+ SCHED_DEBUG("+ ctrl cmd\n");
+ exec->ctrl_busy = false;
+ } else {
+ exec_release_slot_idx(exec, xcmd->slot_idx);
+ }
+ xcmd->slot_idx = no_index;
+}
+
+/*
+ * submit_cmd() - Submit command for execution on this core
+ *
+ * Return: true on success, false if command could not be submitted
+ */
+static bool
+exec_submit_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ unsigned int slotidx = exec_acquire_slot(exec, xcmd);
+
+ if (slotidx == no_index)
+ return false;
+ SCHED_DEBUGF("%s(%d,%lu) slotidx(%d)\n", __func__, exec->uid, xcmd->uid, slotidx);
+ exec->submitted_cmds[slotidx] = xcmd;
+ cmd_set_int_state(xcmd, ERT_CMD_STATE_SUBMITTED);
+ return true;
+}
+
+/*
+ * finish_cmd() - Special post processing of commands after execution
+ */
+static int
+exec_finish_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ if (cmd_opcode(xcmd) == ERT_CU_STAT && exec_is_ert(exec))
+ ert_read_custat(exec->ert, exec->num_cus, exec->cu_usage, xcmd);
+ return 0;
+}
+
+/*
+ * execute_write_cmd() - Execute ERT_WRITE commands
+ */
+static int
+exec_execute_write_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ struct ert_packet *ecmd = xcmd->ecmd;
+ unsigned int idx = 0;
+
+ SCHED_DEBUGF("-> %s(%d,%lu)\n", __func__, exec->uid, xcmd->uid);
+ for (idx = 0; idx < ecmd->count - 1; idx += 2) {
+ u32 addr = ecmd->data[idx];
+ u32 val = ecmd->data[idx+1];
+
+ SCHED_DEBUGF("+ exec_write_cmd base[0x%x] = 0x%x\n", addr, val);
+ iowrite32(val, exec->base + addr);
+ }
+ SCHED_DEBUG("<- exec_write\n");
+ return 0;
+}
+
+/*
+ * notify_host() - Notify user space that a command is complete.
+ */
+static void
+exec_notify_host(struct exec_core *exec)
+{
+ struct list_head *ptr;
+ struct client_ctx *entry;
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+
+ SCHED_DEBUGF("-> %s(%d)\n", __func__, exec->uid);
+
+ /* now for each client update the trigger counter in the context */
+ mutex_lock(&xdev->ctx_list_lock);
+ list_for_each(ptr, &xdev->ctx_list) {
+ entry = list_entry(ptr, struct client_ctx, link);
+ atomic_inc(&entry->trigger);
+ }
+ mutex_unlock(&xdev->ctx_list_lock);
+ /* wake up all the clients */
+ wake_up_interruptible(&exec->poll_wait_queue);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/*
+ * exec_cmd_mark_complete() - Move a command to complete state
+ *
+ * Commands are marked complete in two ways
+ * 1. Through polling of CUs or polling of MB status register
+ * 2. Through interrupts from MB
+ *
+ * @xcmd: Command to mark complete
+ *
+ * The external command state is changed to complete and the host
+ * is notified that some command has completed.
+ */
+static void
+exec_mark_cmd_complete(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> %s(%d,%lu)\n", __func__, exec->uid, xcmd->uid);
+ if (cmd_type(xcmd) == ERT_CTRL)
+ exec_finish_cmd(exec, xcmd);
+
+ cmd_set_state(xcmd, ERT_CMD_STATE_COMPLETED);
+
+ if (exec->polling_mode)
+ scheduler_decr_poll(exec->scheduler);
+
+ exec_release_slot(exec, xcmd);
+ exec_notify_host(exec);
+
+ // Deactivate command and trigger chain of waiting commands
+ cmd_mark_deactive(xcmd);
+ cmd_trigger_chain(xcmd);
+
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * mark_mask_complete() - Move all commands in mask to complete state
+ *
+ * @mask: Bitmask with queried statuses of commands
+ * @mask_idx: Index of the command mask. Used to offset the actual cmd slot index
+ *
+ * Used in ERT mode only. Currently ERT submitted commands remain in exec
+ * submitted queue as ERT doesn't support data flow
+ */
+static void
+exec_mark_mask_complete(struct exec_core *exec, u32 mask, unsigned int mask_idx)
+{
+ int bit_idx = 0, cmd_idx = 0;
+
+ SCHED_DEBUGF("-> %s(0x%x,%d)\n", __func__, mask, mask_idx);
+ if (!mask)
+ return;
+
+ for (bit_idx = 0, cmd_idx = mask_idx<<5; bit_idx < 32; mask >>= 1, ++bit_idx, ++cmd_idx) {
+ // mask could be -1 when firewall trips, double check
+ // exec->submitted_cmds[cmd_idx] to make sure it's not NULL
+ if ((mask & 0x1) && exec->submitted_cmds[cmd_idx])
+ exec_mark_cmd_complete(exec, exec->submitted_cmds[cmd_idx]);
+ }
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/*
+ * penguin_start_cmd() - Start a command in penguin mode
+ */
+static bool
+exec_penguin_start_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ unsigned int cuidx;
+ u32 opcode = cmd_opcode(xcmd);
+
+ SCHED_DEBUGF("-> %s (%d,%lu) opcode(%d)\n", __func__, exec->uid, xcmd->uid, opcode);
+
+ if (opcode == ERT_WRITE && exec_execute_write_cmd(exec, xcmd)) {
+ cmd_set_state(xcmd, ERT_CMD_STATE_ERROR);
+ return false;
+ }
+
+ if (opcode != ERT_START_CU) {
+ SCHED_DEBUGF("<- %s -> true\n", __func__);
+ return true;
+ }
+
+ // Find a ready CU
+ for (cuidx = 0; cuidx < exec->num_cus; ++cuidx) {
+ struct xocl_cu *xcu = exec->cus[cuidx];
+
+ if (cmd_has_cu(xcmd, cuidx) && cu_ready(xcu) && cu_start(xcu, xcmd)) {
+ exec->submitted_cmds[xcmd->slot_idx] = NULL;
+ ++exec->cu_usage[cuidx];
+ exec_release_slot(exec, xcmd);
+ xcmd->cu_idx = cuidx;
+ SCHED_DEBUGF("<- %s -> true\n", __func__);
+ return true;
+ }
+ }
+ SCHED_DEBUGF("<- %s -> false\n", __func__);
+ return false;
+}
+
+/**
+ * penguin_query() - Check command status of argument command
+ *
+ * @xcmd: Command to check
+ *
+ * Function is called in penguin mode (no embedded scheduler).
+ */
+static void
+exec_penguin_query_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ u32 cmdopcode = cmd_opcode(xcmd);
+ u32 cmdtype = cmd_type(xcmd);
+
+ SCHED_DEBUGF("-> %s(%lu) opcode(%d) type(%d) slot_idx=%d\n",
+ __func__, xcmd->uid, cmdopcode, cmdtype, xcmd->slot_idx);
+
+ if (cmdtype == ERT_KDS_LOCAL || cmdtype == ERT_CTRL)
+ exec_mark_cmd_complete(exec, xcmd);
+ else if (cmdopcode == ERT_START_CU) {
+ struct xocl_cu *xcu = exec->cus[xcmd->cu_idx];
+
+ if (cu_first_done(xcu) == xcmd) {
+ cu_pop_done(xcu);
+ exec_mark_cmd_complete(exec, xcmd);
+ }
+ }
+
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+
+/*
+ * ert_start_cmd() - Start a command on ERT
+ */
+static bool
+exec_ert_start_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ // if (cmd_type(xcmd) == ERT_DATAFLOW)
+ // exec_penguin_start_cmd(exec,xcmd);
+ return ert_start_cmd(exec->ert, xcmd);
+}
+
+/*
+ * ert_query_cmd() - Check command completion in ERT
+ *
+ * @xcmd: Command to check
+ *
+ * This function is for ERT mode. In polling mode, check the command status
+ * register containing the slot assigned to the command. In interrupt mode
+ * check the interrupting status register. The function checks all commands
+ * in the same command status register as argument command so more than one
+ * command may be marked complete by this function.
+ */
+static void
+exec_ert_query_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ unsigned int cmd_mask_idx = slot_mask_idx(xcmd->slot_idx);
+
+ SCHED_DEBUGF("-> %s(%lu) slot_idx(%d), cmd_mask_idx(%d)\n", __func__, xcmd->uid, xcmd->slot_idx, cmd_mask_idx);
+
+ if (cmd_type(xcmd) == ERT_KDS_LOCAL) {
+ exec_mark_cmd_complete(exec, xcmd);
+ SCHED_DEBUGF("<- %s local command\n", __func__);
+ return;
+ }
+
+ if (exec->polling_mode
+ || (cmd_mask_idx == 0 && atomic_xchg(&exec->sr0, 0))
+ || (cmd_mask_idx == 1 && atomic_xchg(&exec->sr1, 0))
+ || (cmd_mask_idx == 2 && atomic_xchg(&exec->sr2, 0))
+ || (cmd_mask_idx == 3 && atomic_xchg(&exec->sr3, 0))) {
+ u32 csr_addr = ERT_STATUS_REGISTER_ADDR + (cmd_mask_idx<<2);
+ u32 mask = ioread32(xcmd->exec->base + csr_addr);
+
+ SCHED_DEBUGF("++ %s csr_addr=0x%x mask=0x%x\n", __func__, csr_addr, mask);
+ if (mask)
+ exec_mark_mask_complete(xcmd->exec, mask, cmd_mask_idx);
+ }
+
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/*
+ * start_cmd() - Start execution of a command
+ *
+ * Return: true if successfully started, false otherwise
+ *
+ * Function dispatches based on penguin vs ert mode
+ */
+static bool
+exec_start_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ // assert cmd had been submitted
+ SCHED_DEBUGF("%s(%d,%lu) opcode(%d)\n", __func__, exec->uid, xcmd->uid, cmd_opcode(xcmd));
+
+ if (exec->ops->start(exec, xcmd)) {
+ cmd_set_int_state(xcmd, ERT_CMD_STATE_RUNNING);
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * query_cmd() - Check status of command
+ *
+ * Function dispatches based on penguin vs ert mode. In ERT mode
+ * multiple commands can be marked complete by this function.
+ */
+static void
+exec_query_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("%s(%d,%lu)\n", __func__, exec->uid, xcmd->uid);
+ exec->ops->query(exec, xcmd);
+}
+
+
+
+/**
+ * ert_ops: operations for ERT scheduling
+ */
+static struct exec_ops ert_ops = {
+ .start = exec_ert_start_cmd,
+ .query = exec_ert_query_cmd,
+};
+
+/**
+ * penguin_ops: operations for kernel mode scheduling
+ */
+static struct exec_ops penguin_ops = {
+ .start = exec_penguin_start_cmd,
+ .query = exec_penguin_query_cmd,
+};
+
+/*
+ */
+static inline struct exec_core *
+pdev_get_exec(struct platform_device *pdev)
+{
+ return platform_get_drvdata(pdev);
+}
+
+/*
+ */
+static inline struct exec_core *
+dev_get_exec(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+
+ return pdev ? pdev_get_exec(pdev) : NULL;
+}
+
+/*
+ */
+static inline struct xocl_dev *
+dev_get_xdev(struct device *dev)
+{
+ struct exec_core *exec = dev_get_exec(dev);
+
+ return exec ? exec_get_xdev(exec) : NULL;
+}
+
+/**
+ * List of new pending xocl_cmd objects
+ *
+ * @pending_cmds: populated from user space with new commands for buffer objects
+ * @num_pending: number of pending commands
+ *
+ * Scheduler copies pending commands to its private queue when necessary
+ */
+static LIST_HEAD(pending_cmds);
+static DEFINE_MUTEX(pending_cmds_mutex);
+static atomic_t num_pending = ATOMIC_INIT(0);
+
+static void
+pending_cmds_reset(void)
+{
+ /* clear stale command objects if any */
+ while (!list_empty(&pending_cmds)) {
+ struct xocl_cmd *xcmd = list_first_entry(&pending_cmds, struct xocl_cmd, cq_list);
+
+ DRM_INFO("deleting stale pending cmd\n");
+ cmd_free(xcmd);
+ }
+ atomic_set(&num_pending, 0);
+}
+
+/**
+ * struct xocl_sched: scheduler for xocl_cmd objects
+ *
+ * @scheduler_thread: thread associated with this scheduler
+ * @use_count: use count for this scheduler
+ * @wait_queue: conditional wait queue for scheduler thread
+ * @error: set to 1 to indicate scheduler error
+ * @stop: set to 1 to indicate scheduler should stop
+ * @reset: set to 1 to reset the scheduler
+ * @command_queue: list of command objects managed by scheduler
+ * @intc: boolean flag set when there is a pending interrupt for command completion
+ * @poll: number of running commands in polling mode
+ */
+struct xocl_scheduler {
+ struct task_struct *scheduler_thread;
+ unsigned int use_count;
+
+ wait_queue_head_t wait_queue;
+ unsigned int error;
+ unsigned int stop;
+ unsigned int reset;
+
+ struct list_head command_queue;
+
+ unsigned int intc; /* pending intr shared with isr, word aligned atomic */
+ unsigned int poll; /* number of cmds to poll */
+};
+
+static struct xocl_scheduler scheduler0;
+
+static void
+scheduler_reset(struct xocl_scheduler *xs)
+{
+ xs->error = 0;
+ xs->stop = 0;
+ xs->poll = 0;
+ xs->reset = false;
+ xs->intc = 0;
+}
+
+static void
+scheduler_cq_reset(struct xocl_scheduler *xs)
+{
+ while (!list_empty(&xs->command_queue)) {
+ struct xocl_cmd *xcmd = list_first_entry(&xs->command_queue, struct xocl_cmd, cq_list);
+
+ DRM_INFO("deleting stale scheduler cmd\n");
+ cmd_free(xcmd);
+ }
+}
+
+static void
+scheduler_wake_up(struct xocl_scheduler *xs)
+{
+ wake_up_interruptible(&xs->wait_queue);
+}
+
+static void
+scheduler_intr(struct xocl_scheduler *xs)
+{
+ xs->intc = 1;
+ scheduler_wake_up(xs);
+}
+
+static inline void
+scheduler_decr_poll(struct xocl_scheduler *xs)
+{
+ --xs->poll;
+}
+
+
+/**
+ * scheduler_queue_cmds() - Queue any pending commands
+ *
+ * The scheduler copies pending commands to its internal command queue where
+ * is is now in queued state.
+ */
+static void
+scheduler_queue_cmds(struct xocl_scheduler *xs)
+{
+ struct xocl_cmd *xcmd;
+ struct list_head *pos, *next;
+
+ SCHED_DEBUGF("-> %s\n", __func__);
+ mutex_lock(&pending_cmds_mutex);
+ list_for_each_safe(pos, next, &pending_cmds) {
+ xcmd = list_entry(pos, struct xocl_cmd, cq_list);
+ if (xcmd->xs != xs)
+ continue;
+ SCHED_DEBUGF("+ queueing cmd(%lu)\n", xcmd->uid);
+ list_del(&xcmd->cq_list);
+ list_add_tail(&xcmd->cq_list, &xs->command_queue);
+
+ /* chain active dependencies if any to this command object */
+ if (cmd_wait_count(xcmd) && cmd_chain_dependencies(xcmd))
+ cmd_set_state(xcmd, ERT_CMD_STATE_ERROR);
+ else
+ cmd_set_int_state(xcmd, ERT_CMD_STATE_QUEUED);
+
+ /* this command is now active and can chain other commands */
+ cmd_mark_active(xcmd);
+ atomic_dec(&num_pending);
+ }
+ mutex_unlock(&pending_cmds_mutex);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * queued_to_running() - Move a command from queued to running state if possible
+ *
+ * @xcmd: Command to start
+ *
+ * Upon success, the command is not necessarily running. In ert mode the
+ * command will have been submitted to the embedded scheduler, whereas in
+ * penguin mode the command has been started on a CU.
+ *
+ * Return: %true if command was submitted to device, %false otherwise
+ */
+static bool
+scheduler_queued_to_submitted(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ struct exec_core *exec = cmd_exec(xcmd);
+ bool retval = false;
+
+ if (cmd_wait_count(xcmd))
+ return false;
+
+ SCHED_DEBUGF("-> %s(%lu) opcode(%d)\n", __func__, xcmd->uid, cmd_opcode(xcmd));
+
+ // configure prior to using the core
+ if (cmd_opcode(xcmd) == ERT_CONFIGURE && exec_cfg_cmd(exec, xcmd)) {
+ cmd_set_state(xcmd, ERT_CMD_STATE_ERROR);
+ return false;
+ }
+
+ // submit the command
+ if (exec_submit_cmd(exec, xcmd)) {
+ if (exec->polling_mode)
+ ++xs->poll;
+ retval = true;
+ }
+
+ SCHED_DEBUGF("<- queued_to_submitted returns %d\n", retval);
+
+ return retval;
+}
+
+static bool
+scheduler_submitted_to_running(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ return exec_start_cmd(cmd_exec(xcmd), xcmd);
+}
+
+/**
+ * running_to_complete() - Check status of running commands
+ *
+ * @xcmd: Command is in running state
+ *
+ * When ERT is enabled this function may mark more than just argument
+ * command as complete based on content of command completion register.
+ * Without ERT, only argument command is checked for completion.
+ */
+static void
+scheduler_running_to_complete(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ exec_query_cmd(cmd_exec(xcmd), xcmd);
+}
+
+/**
+ * complete_to_free() - Recycle a complete command objects
+ *
+ * @xcmd: Command is in complete state
+ */
+static void
+scheduler_complete_to_free(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+ cmd_free(xcmd);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+static void
+scheduler_error_to_free(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+ exec_notify_host(cmd_exec(xcmd));
+ scheduler_complete_to_free(xs, xcmd);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+static void
+scheduler_abort_to_free(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+ scheduler_error_to_free(xs, xcmd);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * scheduler_iterator_cmds() - Iterate all commands in scheduler command queue
+ */
+static void
+scheduler_iterate_cmds(struct xocl_scheduler *xs)
+{
+ struct list_head *pos, *next;
+
+ SCHED_DEBUGF("-> %s\n", __func__);
+ list_for_each_safe(pos, next, &xs->command_queue) {
+ struct xocl_cmd *xcmd = list_entry(pos, struct xocl_cmd, cq_list);
+
+ cmd_update_state(xcmd);
+ SCHED_DEBUGF("+ processing cmd(%lu)\n", xcmd->uid);
+
+ /* check running first since queued maybe we waiting for cmd slot */
+ if (xcmd->state == ERT_CMD_STATE_QUEUED)
+ scheduler_queued_to_submitted(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_SUBMITTED)
+ scheduler_submitted_to_running(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_RUNNING)
+ scheduler_running_to_complete(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_COMPLETED)
+ scheduler_complete_to_free(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_ERROR)
+ scheduler_error_to_free(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_ABORT)
+ scheduler_abort_to_free(xs, xcmd);
+ }
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * scheduler_wait_condition() - Check status of scheduler wait condition
+ *
+ * Scheduler must wait (sleep) if
+ * 1. there are no pending commands
+ * 2. no pending interrupt from embedded scheduler
+ * 3. no pending complete commands in polling mode
+ *
+ * Return: 1 if scheduler must wait, 0 othewise
+ */
+static int
+scheduler_wait_condition(struct xocl_scheduler *xs)
+{
+ if (kthread_should_stop()) {
+ xs->stop = 1;
+ SCHED_DEBUG("scheduler wakes kthread_should_stop\n");
+ return 0;
+ }
+
+ if (atomic_read(&num_pending)) {
+ SCHED_DEBUG("scheduler wakes to copy new pending commands\n");
+ return 0;
+ }
+
+ if (xs->intc) {
+ SCHED_DEBUG("scheduler wakes on interrupt\n");
+ xs->intc = 0;
+ return 0;
+ }
+
+ if (xs->poll) {
+ SCHED_DEBUG("scheduler wakes to poll\n");
+ return 0;
+ }
+
+ SCHED_DEBUG("scheduler waits ...\n");
+ return 1;
+}
+
+/**
+ * scheduler_wait() - check if scheduler should wait
+ *
+ * See scheduler_wait_condition().
+ */
+static void
+scheduler_wait(struct xocl_scheduler *xs)
+{
+ wait_event_interruptible(xs->wait_queue, scheduler_wait_condition(xs) == 0);
+}
+
+/**
+ * scheduler_loop() - Run one loop of the scheduler
+ */
+static void
+scheduler_loop(struct xocl_scheduler *xs)
+{
+ static unsigned int loop_cnt;
+
+ SCHED_DEBUGF("%s\n", __func__);
+ scheduler_wait(xs);
+
+ if (xs->error)
+ DRM_INFO("scheduler encountered unexpected error\n");
+
+ if (xs->stop)
+ return;
+
+ if (xs->reset) {
+ SCHED_DEBUG("scheduler is resetting after timeout\n");
+ scheduler_reset(xs);
+ }
+
+ /* queue new pending commands */
+ scheduler_queue_cmds(xs);
+
+ /* iterate all commands */
+ scheduler_iterate_cmds(xs);
+
+ // loop 8 times before explicitly yielding
+ if (++loop_cnt == 8) {
+ loop_cnt = 0;
+ schedule();
+ }
+}
+
+/**
+ * scheduler() - Command scheduler thread routine
+ */
+static int
+scheduler(void *data)
+{
+ struct xocl_scheduler *xs = (struct xocl_scheduler *)data;
+
+ while (!xs->stop)
+ scheduler_loop(xs);
+ DRM_INFO("%s:%d %s thread exits with value %d\n", __FILE__, __LINE__, __func__, xs->error);
+ return xs->error;
+}
+
+
+
+/**
+ * add_xcmd() - Add initialized xcmd object to pending command list
+ *
+ * @xcmd: Command to add
+ *
+ * Scheduler copies pending commands to its internal command queue.
+ *
+ * Return: 0 on success
+ */
+static int
+add_xcmd(struct xocl_cmd *xcmd)
+{
+ struct exec_core *exec = xcmd->exec;
+ struct xocl_dev *xdev = xocl_get_xdev(exec->pdev);
+
+ // Prevent stop and reset
+ mutex_lock(&exec->exec_lock);
+
+ SCHED_DEBUGF("-> %s(%lu) pid(%d)\n", __func__, xcmd->uid, pid_nr(task_tgid(current)));
+ SCHED_DEBUGF("+ exec stopped(%d) configured(%d)\n", exec->stopped, exec->configured);
+
+ if (exec->stopped || (!exec->configured && cmd_opcode(xcmd) != ERT_CONFIGURE))
+ goto err;
+
+ cmd_set_state(xcmd, ERT_CMD_STATE_NEW);
+ mutex_lock(&pending_cmds_mutex);
+ list_add_tail(&xcmd->cq_list, &pending_cmds);
+ atomic_inc(&num_pending);
+ mutex_unlock(&pending_cmds_mutex);
+
+ /* wake scheduler */
+ atomic_inc(&xdev->outstanding_execs);
+ atomic64_inc(&xdev->total_execs);
+ scheduler_wake_up(xcmd->xs);
+
+ SCHED_DEBUGF("<- %s ret(0) opcode(%d) type(%d) num_pending(%d)\n",
+ __func__, cmd_opcode(xcmd), cmd_type(xcmd), atomic_read(&num_pending));
+ mutex_unlock(&exec->exec_lock);
+ return 0;
+
+err:
+ SCHED_DEBUGF("<- %s ret(1) opcode(%d) type(%d) num_pending(%d)\n",
+ __func__, cmd_opcode(xcmd), cmd_type(xcmd), atomic_read(&num_pending));
+ mutex_unlock(&exec->exec_lock);
+ return 1;
+}
+
+
+/**
+ * add_bo_cmd() - Add a new buffer object command to pending list
+ *
+ * @exec: Targeted device
+ * @client: Client context
+ * @bo: Buffer objects from user space from which new command is created
+ * @numdeps: Number of dependencies for this command
+ * @deps: List of @numdeps dependencies
+ *
+ * Scheduler copies pending commands to its internal command queue.
+ *
+ * Return: 0 on success, 1 on failure
+ */
+static int
+add_bo_cmd(struct exec_core *exec, struct client_ctx *client, struct drm_xocl_bo *bo,
+ int numdeps, struct drm_xocl_bo **deps)
+{
+ struct xocl_cmd *xcmd = cmd_get(exec_scheduler(exec), exec, client);
+
+ if (!xcmd)
+ return 1;
+
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+
+ cmd_bo_init(xcmd, bo, numdeps, deps, !exec_is_ert(exec));
+
+ if (add_xcmd(xcmd))
+ goto err;
+
+ SCHED_DEBUGF("<- %s ret(0) opcode(%d) type(%d)\n", __func__, cmd_opcode(xcmd), cmd_type(xcmd));
+ return 0;
+err:
+ cmd_abort(xcmd);
+ SCHED_DEBUGF("<- %s ret(1) opcode(%d) type(%d)\n", __func__, cmd_opcode(xcmd), cmd_type(xcmd));
+ return 1;
+}
+
+static int
+add_ctrl_cmd(struct exec_core *exec, struct client_ctx *client, struct ert_packet *packet)
+{
+ struct xocl_cmd *xcmd = cmd_get(exec_scheduler(exec), exec, client);
+
+ if (!xcmd)
+ return 1;
+
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+
+ cmd_packet_init(xcmd, packet);
+
+ if (add_xcmd(xcmd))
+ goto err;
+
+ SCHED_DEBUGF("<- %s ret(0) opcode(%d) type(%d)\n", __func__, cmd_opcode(xcmd), cmd_type(xcmd));
+ return 0;
+err:
+ cmd_abort(xcmd);
+ SCHED_DEBUGF("<- %s ret(1) opcode(%d) type(%d)\n", __func__, cmd_opcode(xcmd), cmd_type(xcmd));
+ return 1;
+}
+
+
+/**
+ * init_scheduler_thread() - Initialize scheduler thread if necessary
+ *
+ * Return: 0 on success, -errno otherwise
+ */
+static int
+init_scheduler_thread(struct xocl_scheduler *xs)
+{
+ SCHED_DEBUGF("%s use_count=%d\n", __func__, xs->use_count);
+ if (xs->use_count++)
+ return 0;
+
+ init_waitqueue_head(&xs->wait_queue);
+ INIT_LIST_HEAD(&xs->command_queue);
+ scheduler_reset(xs);
+
+ xs->scheduler_thread = kthread_run(scheduler, (void *)xs, "xocl-scheduler-thread0");
+ if (IS_ERR(xs->scheduler_thread)) {
+ int ret = PTR_ERR(xs->scheduler_thread);
+
+ DRM_ERROR(__func__);
+ return ret;
+ }
+ return 0;
+}
+
+/**
+ * fini_scheduler_thread() - Finalize scheduler thread if unused
+ *
+ * Return: 0 on success, -errno otherwise
+ */
+static int
+fini_scheduler_thread(struct xocl_scheduler *xs)
+{
+ int retval = 0;
+
+ SCHED_DEBUGF("%s use_count=%d\n", __func__, xs->use_count);
+ if (--xs->use_count)
+ return 0;
+
+ retval = kthread_stop(xs->scheduler_thread);
+
+ /* clear stale command objects if any */
+ pending_cmds_reset();
+ scheduler_cq_reset(xs);
+
+ /* reclaim memory for allocate command objects */
+ cmd_list_delete();
+
+ return retval;
+}
+
+/**
+ * Entry point for exec buffer.
+ *
+ * Function adds exec buffer to the pending list of commands
+ */
+int
+add_exec_buffer(struct platform_device *pdev, struct client_ctx *client, void *buf,
+ int numdeps, struct drm_xocl_bo **deps)
+{
+ struct exec_core *exec = platform_get_drvdata(pdev);
+ // Add the command to pending list
+ return add_bo_cmd(exec, client, buf, numdeps, deps);
+}
+
+static int
+xocl_client_lock_bitstream_nolock(struct xocl_dev *xdev, struct client_ctx *client)
+{
+ int pid = pid_nr(task_tgid(current));
+ uuid_t *xclbin_id;
+
+ if (client->xclbin_locked)
+ return 0;
+
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+ if (!xclbin_id || !uuid_equal(xclbin_id, &client->xclbin_id)) {
+ userpf_err(xdev,
+ "device xclbin does not match context xclbin, cannot obtain lock for process %d",
+ pid);
+ return 1;
+ }
+
+ if (xocl_icap_lock_bitstream(xdev, &client->xclbin_id, pid) < 0) {
+ userpf_err(xdev, "could not lock bitstream for process %d", pid);
+ return 1;
+ }
+
+ client->xclbin_locked = true;
+ userpf_info(xdev, "process %d successfully locked xcblin", pid);
+ return 0;
+}
+
+static int
+xocl_client_lock_bitstream(struct xocl_dev *xdev, struct client_ctx *client)
+{
+ int ret = 0;
+
+ mutex_lock(&client->lock); // protect current client
+ mutex_lock(&xdev->ctx_list_lock); // protect xdev->xclbin_id
+ ret = xocl_client_lock_bitstream_nolock(xdev, client);
+ mutex_unlock(&xdev->ctx_list_lock);
+ mutex_unlock(&client->lock);
+ return ret;
+}
+
+
+static int
+create_client(struct platform_device *pdev, void **priv)
+{
+ struct client_ctx *client;
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ int ret = 0;
+
+ client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL);
+ if (!client)
+ return -ENOMEM;
+
+ mutex_lock(&xdev->ctx_list_lock);
+
+ if (!xdev->offline) {
+ client->pid = task_tgid(current);
+ mutex_init(&client->lock);
+ client->xclbin_locked = false;
+ client->abort = false;
+ atomic_set(&client->trigger, 0);
+ atomic_set(&client->outstanding_execs, 0);
+ client->num_cus = 0;
+ client->xdev = xocl_get_xdev(pdev);
+ list_add_tail(&client->link, &xdev->ctx_list);
+ *priv = client;
+ } else {
+ /* Do not allow new client to come in while being offline. */
+ devm_kfree(&pdev->dev, client);
+ ret = -EBUSY;
+ }
+
+ mutex_unlock(&xdev->ctx_list_lock);
+
+ DRM_INFO("creating scheduler client for pid(%d), ret: %d\n",
+ pid_nr(task_tgid(current)), ret);
+
+ return ret;
+}
+
+static void destroy_client(struct platform_device *pdev, void **priv)
+{
+ struct client_ctx *client = (struct client_ctx *)(*priv);
+ struct exec_core *exec = platform_get_drvdata(pdev);
+ struct xocl_scheduler *xs = exec_scheduler(exec);
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ unsigned int outstanding = atomic_read(&client->outstanding_execs);
+ unsigned int timeout_loops = 20;
+ unsigned int loops = 0;
+ int pid = pid_nr(task_tgid(current));
+ unsigned int bit;
+ struct ip_layout *layout = XOCL_IP_LAYOUT(xdev);
+
+ bit = layout
+ ? find_first_bit(client->cu_bitmap, layout->m_count)
+ : MAX_CUS;
+
+ /*
+ * This happens when application exists without formally releasing the
+ * contexts on CUs. Give up our contexts on CUs and our lock on xclbin.
+ * Note, that implicit CUs (such as CDMA) do not add to ip_reference.
+ */
+ while (layout && (bit < layout->m_count)) {
+ if (exec->ip_reference[bit]) {
+ userpf_info(xdev, "CTX reclaim (%pUb, %d, %u)",
+ &client->xclbin_id, pid, bit);
+ exec->ip_reference[bit]--;
+ }
+ bit = find_next_bit(client->cu_bitmap, layout->m_count, bit + 1);
+ }
+ bitmap_zero(client->cu_bitmap, MAX_CUS);
+
+ // force scheduler to abort execs for this client
+ client->abort = true;
+
+ // wait for outstanding execs to finish
+ while (outstanding) {
+ unsigned int new;
+
+ userpf_info(xdev, "waiting for %d outstanding execs to finish", outstanding);
+ msleep(500);
+ new = atomic_read(&client->outstanding_execs);
+ loops = (new == outstanding ? (loops + 1) : 0);
+ if (loops == timeout_loops) {
+ userpf_err(xdev,
+ "Giving up with %d outstanding execs, please reset device with 'xbutil reset'\n",
+ outstanding);
+ xdev->needs_reset = true;
+ // reset the scheduler loop
+ xs->reset = true;
+ break;
+ }
+ outstanding = new;
+ }
+
+ DRM_INFO("client exits pid(%d)\n", pid);
+
+ mutex_lock(&xdev->ctx_list_lock);
+ list_del(&client->link);
+ mutex_unlock(&xdev->ctx_list_lock);
+
+ if (client->xclbin_locked)
+ xocl_icap_unlock_bitstream(xdev, &client->xclbin_id, pid);
+ mutex_destroy(&client->lock);
+ devm_kfree(&pdev->dev, client);
+ *priv = NULL;
+}
+
+static uint poll_client(struct platform_device *pdev, struct file *filp,
+ poll_table *wait, void *priv)
+{
+ struct client_ctx *client = (struct client_ctx *)priv;
+ struct exec_core *exec;
+ int counter;
+ uint ret = 0;
+
+ exec = platform_get_drvdata(pdev);
+
+ poll_wait(filp, &exec->poll_wait_queue, wait);
+
+ /*
+ * Mutex lock protects from two threads from the same application
+ * calling poll concurrently using the same file handle
+ */
+ mutex_lock(&client->lock);
+ counter = atomic_read(&client->trigger);
+ if (counter > 0) {
+ /*
+ * Use atomic here since the trigger may be incremented by
+ * interrupt handler running concurrently.
+ */
+ atomic_dec(&client->trigger);
+ ret = POLLIN;
+ }
+ mutex_unlock(&client->lock);
+
+ return ret;
+}
+
+static int client_ioctl_ctx(struct platform_device *pdev,
+ struct client_ctx *client, void *data)
+{
+ bool acquire_lock = false;
+ struct drm_xocl_ctx *args = data;
+ int ret = 0;
+ int pid = pid_nr(task_tgid(current));
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ struct exec_core *exec = platform_get_drvdata(pdev);
+ uuid_t *xclbin_id;
+
+ mutex_lock(&client->lock);
+ mutex_lock(&xdev->ctx_list_lock);
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+ if (!xclbin_id || !uuid_equal(xclbin_id, &args->xclbin_id)) {
+ ret = -EBUSY;
+ goto out;
+ }
+
+ if (args->cu_index >= XOCL_IP_LAYOUT(xdev)->m_count) {
+ userpf_err(xdev, "cuidx(%d) >= numcus(%d)\n",
+ args->cu_index, XOCL_IP_LAYOUT(xdev)->m_count);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (args->op == XOCL_CTX_OP_FREE_CTX) {
+ ret = test_and_clear_bit(args->cu_index, client->cu_bitmap) ? 0 : -EINVAL;
+ if (ret) // No context was previously allocated for this CU
+ goto out;
+
+ // CU unlocked explicitly
+ --exec->ip_reference[args->cu_index];
+ if (!--client->num_cus) {
+ // We just gave up the last context, unlock the xclbin
+ ret = xocl_icap_unlock_bitstream(xdev, xclbin_id, pid);
+ client->xclbin_locked = false;
+ }
+ userpf_info(xdev, "CTX del(%pUb, %d, %u)",
+ xclbin_id, pid, args->cu_index);
+ goto out;
+ }
+
+ if (args->op != XOCL_CTX_OP_ALLOC_CTX) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (args->flags != XOCL_CTX_SHARED) {
+ userpf_err(xdev, "Only shared contexts are supported in this release");
+ ret = -EPERM;
+ goto out;
+ }
+
+ if (!client->num_cus && !client->xclbin_locked)
+ // Process has no other context on any CU yet, hence we need to
+ // lock the xclbin A process uses just one lock for all its ctxs
+ acquire_lock = true;
+
+ if (test_and_set_bit(args->cu_index, client->cu_bitmap)) {
+ userpf_info(xdev, "CTX already allocated by this process");
+ // Context was previously allocated for the same CU,
+ // cannot allocate again
+ ret = 0;
+ goto out;
+ }
+
+ if (acquire_lock) {
+ // This is the first context on any CU for this process,
+ // lock the xclbin
+ ret = xocl_client_lock_bitstream_nolock(xdev, client);
+ if (ret) {
+ // Locking of xclbin failed, give up our context
+ clear_bit(args->cu_index, client->cu_bitmap);
+ goto out;
+ } else {
+ uuid_copy(&client->xclbin_id, xclbin_id);
+ }
+ }
+
+ // Everything is good so far, hence increment the CU reference count
+ ++client->num_cus; // explicitly acquired
+ ++exec->ip_reference[args->cu_index];
+ xocl_info(&pdev->dev, "CTX add(%pUb, %d, %u, %d)",
+ xclbin_id, pid, args->cu_index, acquire_lock);
+out:
+ mutex_unlock(&xdev->ctx_list_lock);
+ mutex_unlock(&client->lock);
+ return ret;
+}
+
+static int
+get_bo_paddr(struct xocl_dev *xdev, struct drm_file *filp,
+ uint32_t bo_hdl, size_t off, size_t size, uint64_t *paddrp)
+{
+ struct drm_device *ddev = filp->minor->dev;
+ struct drm_gem_object *obj;
+ struct drm_xocl_bo *xobj;
+
+ obj = xocl_gem_object_lookup(ddev, filp, bo_hdl);
+ if (!obj) {
+ userpf_err(xdev, "Failed to look up GEM BO 0x%x\n", bo_hdl);
+ return -ENOENT;
+ }
+ xobj = to_xocl_bo(obj);
+
+ if (obj->size <= off || obj->size < off + size || !xobj->mm_node) {
+ userpf_err(xdev, "Failed to get paddr for BO 0x%x\n", bo_hdl);
+//PORT4_20
+// drm_gem_object_unreference_unlocked(obj);
+ drm_gem_object_put_unlocked(obj);
+ return -EINVAL;
+ }
+
+ *paddrp = xobj->mm_node->start + off;
+// drm_gem_object_unreference_unlocked(obj);
+ drm_gem_object_put_unlocked(obj);
+ return 0;
+}
+
+static int
+convert_execbuf(struct xocl_dev *xdev, struct drm_file *filp,
+ struct exec_core *exec, struct drm_xocl_bo *xobj)
+{
+ int i;
+ int ret;
+ size_t src_off;
+ size_t dst_off;
+ size_t sz;
+ uint64_t src_addr;
+ uint64_t dst_addr;
+ struct ert_start_copybo_cmd *scmd = (struct ert_start_copybo_cmd *)xobj->vmapping;
+
+ /* Only convert COPYBO cmd for now. */
+ if (scmd->opcode != ERT_START_COPYBO)
+ return 0;
+
+ sz = scmd->size * COPYBO_UNIT;
+
+ src_off = scmd->src_addr_hi;
+ src_off <<= 32;
+ src_off |= scmd->src_addr_lo;
+ ret = get_bo_paddr(xdev, filp, scmd->src_bo_hdl, src_off, sz, &src_addr);
+ if (ret != 0)
+ return ret;
+
+ dst_off = scmd->dst_addr_hi;
+ dst_off <<= 32;
+ dst_off |= scmd->dst_addr_lo;
+ ret = get_bo_paddr(xdev, filp, scmd->dst_bo_hdl, dst_off, sz, &dst_addr);
+ if (ret != 0)
+ return ret;
+
+ ert_fill_copybo_cmd(scmd, 0, 0, src_addr, dst_addr, sz);
+
+ for (i = exec->num_cus - exec->num_cdma; i < exec->num_cus; i++)
+ scmd->cu_mask[i / 32] |= 1 << (i % 32);
+
+ scmd->opcode = ERT_START_CU;
+
+ return 0;
+}
+
+static int
+client_ioctl_execbuf(struct platform_device *pdev,
+ struct client_ctx *client, void *data, struct drm_file *filp)
+{
+ struct drm_xocl_execbuf *args = data;
+ struct drm_xocl_bo *xobj;
+ struct drm_gem_object *obj;
+ struct drm_xocl_bo *deps[8] = {0};
+ int numdeps = -1;
+ int ret = 0;
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ struct drm_device *ddev = filp->minor->dev;
+
+ if (xdev->needs_reset) {
+ userpf_err(xdev, "device needs reset, use 'xbutil reset -h'");
+ return -EBUSY;
+ }
+
+ /* Look up the gem object corresponding to the BO handle.
+ * This adds a reference to the gem object. The refernece is
+ * passed to kds or released here if errors occur.
+ */
+ obj = xocl_gem_object_lookup(ddev, filp, args->exec_bo_handle);
+ if (!obj) {
+ userpf_err(xdev, "Failed to look up GEM BO %d\n",
+ args->exec_bo_handle);
+ return -ENOENT;
+ }
+
+ /* Convert gem object to xocl_bo extension */
+ xobj = to_xocl_bo(obj);
+ if (!xocl_bo_execbuf(xobj) || convert_execbuf(xdev, filp,
+ platform_get_drvdata(pdev), xobj) != 0) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret = validate(pdev, client, xobj);
+ if (ret) {
+ userpf_err(xdev, "Exec buffer validation failed\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Copy dependencies from user. It is an error if a BO handle specified
+ * as a dependency does not exists. Lookup gem object corresponding to bo
+ * handle. Convert gem object to xocl_bo extension. Note that the
+ * gem lookup acquires a reference to the drm object, this reference
+ * is passed on to the the scheduler via xocl_exec_add_buffer.
+ */
+ for (numdeps = 0; numdeps < 8 && args->deps[numdeps]; ++numdeps) {
+ struct drm_gem_object *gobj =
+ xocl_gem_object_lookup(ddev, filp, args->deps[numdeps]);
+ struct drm_xocl_bo *xbo = gobj ? to_xocl_bo(gobj) : NULL;
+
+ if (!gobj)
+ userpf_err(xdev, "Failed to look up GEM BO %d\n",
+ args->deps[numdeps]);
+ if (!xbo) {
+ ret = -EINVAL;
+ goto out;
+ }
+ deps[numdeps] = xbo;
+ }
+
+ /* acquire lock on xclbin if necessary */
+ ret = xocl_client_lock_bitstream(xdev, client);
+ if (ret) {
+ userpf_err(xdev, "Failed to lock xclbin\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Add exec buffer to scheduler (kds). The scheduler manages the
+ * drm object references acquired by xobj and deps. It is vital
+ * that the references are released properly.
+ */
+ ret = add_exec_buffer(pdev, client, xobj, numdeps, deps);
+ if (ret) {
+ userpf_err(xdev, "Failed to add exec buffer to scheduler\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Return here, noting that the gem objects passed to kds have
+ * references that must be released by kds itself. User manages
+ * a regular reference to all BOs returned as file handles. These
+ * references are released with the BOs are freed.
+ */
+ return ret;
+
+out:
+ for (--numdeps; numdeps >= 0; numdeps--)
+ drm_gem_object_put_unlocked(&deps[numdeps]->base);
+//PORT4_20
+// drm_gem_object_unreference_unlocked(&deps[numdeps]->base);
+ drm_gem_object_put_unlocked(&xobj->base);
+// drm_gem_object_unreference_unlocked(&xobj->base);
+ return ret;
+}
+
+int
+client_ioctl(struct platform_device *pdev, int op, void *data, void *drm_filp)
+{
+ struct drm_file *filp = drm_filp;
+ struct client_ctx *client = filp->driver_priv;
+ int ret;
+
+ switch (op) {
+ case DRM_XOCL_CTX:
+ ret = client_ioctl_ctx(pdev, client, data);
+ break;
+ case DRM_XOCL_EXECBUF:
+ ret = client_ioctl_execbuf(pdev, client, data, drm_filp);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ return ret;
+}
+/**
+ * reset() - Reset device exec data structure
+ *
+ * @pdev: platform device to reset
+ *
+ * [Current 2018.3 situation:]
+ * This function is currently called from mgmt icap on every AXI is
+ * freeze/unfreeze. It ensures that the device exec_core state is reset to
+ * same state as was when scheduler was originally probed for the device.
+ * The callback from icap, ensures that scheduler resets the exec core when
+ * multiple processes are already attached to the device but AXI is reset.
+ *
+ * Even though the very first client created for this device also resets the
+ * exec core, it is possible that further resets are necessary. For example
+ * in multi-process case, there can be 'n' processes that attach to the
+ * device. On first client attach the exec core is reset correctly, but now
+ * assume that 'm' of these processes finishes completely before any remaining
+ * (n-m) processes start using the scheduler. In this case, the n-m clients have
+ * already been created, but icap resets AXI because the xclbin has no
+ * references (arguably this AXI reset is wrong)
+ *
+ * [Work-in-progress:]
+ * Proper contract:
+ * Pre-condition: xocl_exec_stop has been called before xocl_exec_reset.
+ * Pre-condition: new bitstream has been downloaded and AXI has been reset
+ */
+static int
+reset(struct platform_device *pdev)
+{
+ struct exec_core *exec = platform_get_drvdata(pdev);
+
+ exec_stop(exec); // remove when upstream explicitly calls stop()
+ exec_reset(exec);
+ return 0;
+}
+
+/**
+ * stop() - Reset device exec data structure
+ *
+ * This API must be called prior to performing an AXI reset and downloading of
+ * a new xclbin. Calling this API flushes the commands running on current
+ * device and prevents new commands from being scheduled on the device. This
+ * effectively prevents 'xbutil top' from issuing CU_STAT commands while
+ * programming is performed.
+ *
+ * Pre-condition: xocl_client_release has been called, e.g there are no
+ * current clients using the bitstream
+ */
+static int
+stop(struct platform_device *pdev)
+{
+ struct exec_core *exec = platform_get_drvdata(pdev);
+
+ exec_stop(exec);
+ return 0;
+}
+
+/**
+ * validate() - Check if requested cmd is valid in the current context
+ */
+static int
+validate(struct platform_device *pdev, struct client_ctx *client, const struct drm_xocl_bo *bo)
+{
+ struct ert_packet *ecmd = (struct ert_packet *)bo->vmapping;
+ struct ert_start_kernel_cmd *scmd = (struct ert_start_kernel_cmd *)bo->vmapping;
+ unsigned int i = 0;
+ u32 ctx_cus[4] = {0};
+ u32 cumasks = 0;
+ int err = 0;
+
+ SCHED_DEBUGF("-> %s(%d)\n", __func__, ecmd->opcode);
+
+ /* cus for start kernel commands only */
+ if (ecmd->opcode != ERT_START_CU)
+ return 0; /* ok */
+
+ /* client context cu bitmap may not change while validating */
+ mutex_lock(&client->lock);
+
+ /* no specific CUs selected, maybe ctx is not used by client */
+ if (bitmap_empty(client->cu_bitmap, MAX_CUS)) {
+ userpf_err(xocl_get_xdev(pdev), "%s found no CUs in ctx\n", __func__);
+ goto out; /* ok */
+ }
+
+ /* Check CUs in cmd BO against CUs in context */
+ cumasks = 1 + scmd->extra_cu_masks;
+ xocl_bitmap_to_arr32(ctx_cus, client->cu_bitmap, cumasks * 32);
+
+ for (i = 0; i < cumasks; ++i) {
+ uint32_t cmd_cus = ecmd->data[i];
+ /* cmd_cus must be subset of ctx_cus */
+ if (cmd_cus & ~ctx_cus[i]) {
+ SCHED_DEBUGF("<- %s(1), CU mismatch in mask(%d) cmd(0x%x) ctx(0x%x)\n",
+ __func__, i, cmd_cus, ctx_cus[i]);
+ err = 1;
+ goto out; /* error */
+ }
+ }
+
+
+out:
+ mutex_unlock(&client->lock);
+ SCHED_DEBUGF("<- %s(%d) cmd and ctx CUs match\n", __func__, err);
+ return err;
+
+}
+
+struct xocl_mb_scheduler_funcs sche_ops = {
+ .create_client = create_client,
+ .destroy_client = destroy_client,
+ .poll_client = poll_client,
+ .client_ioctl = client_ioctl,
+ .stop = stop,
+ .reset = reset,
+};
+
+/* sysfs */
+static ssize_t
+kds_numcus_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct exec_core *exec = dev_get_exec(dev);
+ unsigned int cus = exec ? exec->num_cus - exec->num_cdma : 0;
+
+ return sprintf(buf, "%d\n", cus);
+}
+static DEVICE_ATTR_RO(kds_numcus);
+
+static ssize_t
+kds_numcdmas_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_xdev(dev);
+ uint32_t *cdma = xocl_cdma_addr(xdev);
+ unsigned int cdmas = cdma ? 1 : 0; //TBD
+
+ return sprintf(buf, "%d\n", cdmas);
+}
+static DEVICE_ATTR_RO(kds_numcdmas);
+
+static ssize_t
+kds_custat_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct exec_core *exec = dev_get_exec(dev);
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+ struct client_ctx client;
+ struct ert_packet packet;
+ unsigned int count = 0;
+ ssize_t sz = 0;
+
+ // minimum required initialization of client
+ client.abort = false;
+ client.xdev = xdev;
+ atomic_set(&client.trigger, 0);
+ atomic_set(&client.outstanding_execs, 0);
+
+ packet.opcode = ERT_CU_STAT;
+ packet.type = ERT_CTRL;
+ packet.count = 1; // data[1]
+
+ if (add_ctrl_cmd(exec, &client, &packet) == 0) {
+ int retry = 5;
+
+ SCHED_DEBUGF("-> custat waiting for command to finish\n");
+ // wait for command completion
+ while (--retry && atomic_read(&client.outstanding_execs))
+ msleep(100);
+ if (retry == 0 && atomic_read(&client.outstanding_execs))
+ userpf_info(xdev, "custat unexpected timeout\n");
+ SCHED_DEBUGF("<- custat retry(%d)\n", retry);
+ }
+
+ for (count = 0; count < exec->num_cus; ++count)
+ sz += sprintf(buf+sz, "CU[@0x%x] : %d\n",
+ exec_cu_base_addr(exec, count),
+ exec_cu_usage(exec, count));
+ if (sz)
+ buf[sz++] = 0;
+
+ return sz;
+}
+static DEVICE_ATTR_RO(kds_custat);
+
+static struct attribute *kds_sysfs_attrs[] = {
+ &dev_attr_kds_numcus.attr,
+ &dev_attr_kds_numcdmas.attr,
+ &dev_attr_kds_custat.attr,
+ NULL
+};
+
+static const struct attribute_group kds_sysfs_attr_group = {
+ .attrs = kds_sysfs_attrs,
+};
+
+static void
+user_sysfs_destroy_kds(struct platform_device *pdev)
+{
+ sysfs_remove_group(&pdev->dev.kobj, &kds_sysfs_attr_group);
+}
+
+static int
+user_sysfs_create_kds(struct platform_device *pdev)
+{
+ int err = sysfs_create_group(&pdev->dev.kobj, &kds_sysfs_attr_group);
+
+ if (err)
+ userpf_err(xocl_get_xdev(pdev), "create kds attr failed: 0x%x", err);
+ return err;
+}
+
+/**
+ * Init scheduler
+ */
+static int mb_scheduler_probe(struct platform_device *pdev)
+{
+ struct exec_core *exec = exec_create(pdev, &scheduler0);
+
+ if (!exec)
+ return -ENOMEM;
+
+ if (user_sysfs_create_kds(pdev))
+ goto err;
+
+ init_scheduler_thread(&scheduler0);
+ xocl_subdev_register(pdev, XOCL_SUBDEV_MB_SCHEDULER, &sche_ops);
+ platform_set_drvdata(pdev, exec);
+
+ DRM_INFO("command scheduler started\n");
+
+ return 0;
+
+err:
+ devm_kfree(&pdev->dev, exec);
+ return 1;
+}
+
+/**
+ * Fini scheduler
+ */
+static int mb_scheduler_remove(struct platform_device *pdev)
+{
+ struct xocl_dev *xdev;
+ int i;
+ struct exec_core *exec = platform_get_drvdata(pdev);
+
+ SCHED_DEBUGF("-> %s\n", __func__);
+ fini_scheduler_thread(exec_scheduler(exec));
+
+ xdev = xocl_get_xdev(pdev);
+ for (i = 0; i < exec->intr_num; i++) {
+ xocl_user_interrupt_config(xdev, i + exec->intr_base, false);
+ xocl_user_interrupt_reg(xdev, i + exec->intr_base,
+ NULL, NULL);
+ }
+ mutex_destroy(&exec->exec_lock);
+
+ user_sysfs_destroy_kds(pdev);
+ exec_destroy(exec);
+ platform_set_drvdata(pdev, NULL);
+
+ SCHED_DEBUGF("<- %s\n", __func__);
+ DRM_INFO("command scheduler removed\n");
+ return 0;
+}
+
+static struct platform_device_id mb_sche_id_table[] = {
+ { XOCL_MB_SCHEDULER, 0 },
+ { },
+};
+
+static struct platform_driver mb_scheduler_driver = {
+ .probe = mb_scheduler_probe,
+ .remove = mb_scheduler_remove,
+ .driver = {
+ .name = "xocl_mb_sche",
+ },
+ .id_table = mb_sche_id_table,
+};
+
+int __init xocl_init_mb_scheduler(void)
+{
+ return platform_driver_register(&mb_scheduler_driver);
+}
+
+void xocl_fini_mb_scheduler(void)
+{
+ SCHED_DEBUGF("-> %s\n", __func__);
+ platform_driver_unregister(&mb_scheduler_driver);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/microblaze.c b/drivers/gpu/drm/xocl/subdev/microblaze.c
new file mode 100644
index 000000000000..38cfbdbb39ef
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/microblaze.c
@@ -0,0 +1,722 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Lizhi.HOu@xilinx.com
+ *
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/vmalloc.h>
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+#define MAX_RETRY 50
+#define RETRY_INTERVAL 100 //ms
+
+#define MAX_IMAGE_LEN 0x20000
+
+#define REG_VERSION 0
+#define REG_ID 0x4
+#define REG_STATUS 0x8
+#define REG_ERR 0xC
+#define REG_CAP 0x10
+#define REG_CTL 0x18
+#define REG_STOP_CONFIRM 0x1C
+#define REG_CURR_BASE 0x20
+#define REG_POWER_CHECKSUM 0x1A4
+
+#define VALID_ID 0x74736574
+
+#define GPIO_RESET 0x0
+#define GPIO_ENABLED 0x1
+
+#define SELF_JUMP(ins) (((ins) & 0xfc00ffff) == 0xb8000000)
+
+enum ctl_mask {
+ CTL_MASK_CLEAR_POW = 0x1,
+ CTL_MASK_CLEAR_ERR = 0x2,
+ CTL_MASK_PAUSE = 0x4,
+ CTL_MASK_STOP = 0x8,
+};
+
+enum status_mask {
+ STATUS_MASK_INIT_DONE = 0x1,
+ STATUS_MASK_STOPPED = 0x2,
+ STATUS_MASK_PAUSE = 0x4,
+};
+
+enum cap_mask {
+ CAP_MASK_PM = 0x1,
+};
+
+enum {
+ MB_STATE_INIT = 0,
+ MB_STATE_RUN,
+ MB_STATE_RESET,
+};
+
+enum {
+ IO_REG,
+ IO_GPIO,
+ IO_IMAGE_MGMT,
+ IO_IMAGE_SCHE,
+ NUM_IOADDR
+};
+
+#define READ_REG32(mb, off) \
+ XOCL_READ_REG32(mb->base_addrs[IO_REG] + off)
+#define WRITE_REG32(mb, val, off) \
+ XOCL_WRITE_REG32(val, mb->base_addrs[IO_REG] + off)
+
+#define READ_GPIO(mb, off) \
+ XOCL_READ_REG32(mb->base_addrs[IO_GPIO] + off)
+#define WRITE_GPIO(mb, val, off) \
+ XOCL_WRITE_REG32(val, mb->base_addrs[IO_GPIO] + off)
+
+#define READ_IMAGE_MGMT(mb, off) \
+ XOCL_READ_REG32(mb->base_addrs[IO_IMAGE_MGMT] + off)
+
+#define COPY_MGMT(mb, buf, len) \
+ xocl_memcpy_toio(mb->base_addrs[IO_IMAGE_MGMT], buf, len)
+#define COPY_SCHE(mb, buf, len) \
+ xocl_memcpy_toio(mb->base_addrs[IO_IMAGE_SCHE], buf, len)
+
+struct xocl_mb {
+ struct platform_device *pdev;
+ void __iomem *base_addrs[NUM_IOADDR];
+
+ struct device *hwmon_dev;
+ bool enabled;
+ u32 state;
+ u32 cap;
+ struct mutex mb_lock;
+
+ char *sche_binary;
+ u32 sche_binary_length;
+ char *mgmt_binary;
+ u32 mgmt_binary_length;
+};
+
+static int mb_stop(struct xocl_mb *mb);
+static int mb_start(struct xocl_mb *mb);
+
+/* sysfs support */
+static void safe_read32(struct xocl_mb *mb, u32 reg, u32 *val)
+{
+ mutex_lock(&mb->mb_lock);
+ if (mb->enabled && mb->state == MB_STATE_RUN)
+ *val = READ_REG32(mb, reg);
+ else
+ *val = 0;
+ mutex_unlock(&mb->mb_lock);
+}
+
+static void safe_write32(struct xocl_mb *mb, u32 reg, u32 val)
+{
+ mutex_lock(&mb->mb_lock);
+ if (mb->enabled && mb->state == MB_STATE_RUN)
+ WRITE_REG32(mb, val, reg);
+ mutex_unlock(&mb->mb_lock);
+}
+
+static ssize_t version_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_VERSION, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(version);
+
+static ssize_t id_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_ID, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(id);
+
+static ssize_t status_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_STATUS, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(status);
+
+static ssize_t error_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_ERR, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(error);
+
+static ssize_t capability_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_CAP, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(capability);
+
+static ssize_t power_checksum_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_POWER_CHECKSUM, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(power_checksum);
+
+static ssize_t pause_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_CTL, &val);
+
+ return sprintf(buf, "%d\n", !!(val & CTL_MASK_PAUSE));
+}
+
+static ssize_t pause_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val > 1)
+ return -EINVAL;
+
+ val = val ? CTL_MASK_PAUSE : 0;
+ safe_write32(mb, REG_CTL, val);
+
+ return count;
+}
+static DEVICE_ATTR_RW(pause);
+
+static ssize_t reset_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val > 1)
+ return -EINVAL;
+
+ if (val) {
+ mb_stop(mb);
+ mb_start(mb);
+ }
+
+ return count;
+}
+static DEVICE_ATTR_WO(reset);
+
+static struct attribute *mb_attrs[] = {
+ &dev_attr_version.attr,
+ &dev_attr_id.attr,
+ &dev_attr_status.attr,
+ &dev_attr_error.attr,
+ &dev_attr_capability.attr,
+ &dev_attr_power_checksum.attr,
+ &dev_attr_pause.attr,
+ &dev_attr_reset.attr,
+ NULL,
+};
+static struct attribute_group mb_attr_group = {
+ .attrs = mb_attrs,
+};
+
+static ssize_t show_mb_pw(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+ struct xocl_mb *mb = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(mb, REG_CURR_BASE + attr->index * sizeof(u32), &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+
+static SENSOR_DEVICE_ATTR(curr1_highest, 0444, show_mb_pw, NULL, 0);
+static SENSOR_DEVICE_ATTR(curr1_average, 0444, show_mb_pw, NULL, 1);
+static SENSOR_DEVICE_ATTR(curr1_input, 0444, show_mb_pw, NULL, 2);
+static SENSOR_DEVICE_ATTR(curr2_highest, 0444, show_mb_pw, NULL, 3);
+static SENSOR_DEVICE_ATTR(curr2_average, 0444, show_mb_pw, NULL, 4);
+static SENSOR_DEVICE_ATTR(curr2_input, 0444, show_mb_pw, NULL, 5);
+static SENSOR_DEVICE_ATTR(curr3_highest, 0444, show_mb_pw, NULL, 6);
+static SENSOR_DEVICE_ATTR(curr3_average, 0444, show_mb_pw, NULL, 7);
+static SENSOR_DEVICE_ATTR(curr3_input, 0444, show_mb_pw, NULL, 8);
+static SENSOR_DEVICE_ATTR(curr4_highest, 0444, show_mb_pw, NULL, 9);
+static SENSOR_DEVICE_ATTR(curr4_average, 0444, show_mb_pw, NULL, 10);
+static SENSOR_DEVICE_ATTR(curr4_input, 0444, show_mb_pw, NULL, 11);
+static SENSOR_DEVICE_ATTR(curr5_highest, 0444, show_mb_pw, NULL, 12);
+static SENSOR_DEVICE_ATTR(curr5_average, 0444, show_mb_pw, NULL, 13);
+static SENSOR_DEVICE_ATTR(curr5_input, 0444, show_mb_pw, NULL, 14);
+static SENSOR_DEVICE_ATTR(curr6_highest, 0444, show_mb_pw, NULL, 15);
+static SENSOR_DEVICE_ATTR(curr6_average, 0444, show_mb_pw, NULL, 16);
+static SENSOR_DEVICE_ATTR(curr6_input, 0444, show_mb_pw, NULL, 17);
+
+static struct attribute *hwmon_mb_attributes[] = {
+ &sensor_dev_attr_curr1_highest.dev_attr.attr,
+ &sensor_dev_attr_curr1_average.dev_attr.attr,
+ &sensor_dev_attr_curr1_input.dev_attr.attr,
+ &sensor_dev_attr_curr2_highest.dev_attr.attr,
+ &sensor_dev_attr_curr2_average.dev_attr.attr,
+ &sensor_dev_attr_curr2_input.dev_attr.attr,
+ &sensor_dev_attr_curr3_highest.dev_attr.attr,
+ &sensor_dev_attr_curr3_average.dev_attr.attr,
+ &sensor_dev_attr_curr3_input.dev_attr.attr,
+ &sensor_dev_attr_curr4_highest.dev_attr.attr,
+ &sensor_dev_attr_curr4_average.dev_attr.attr,
+ &sensor_dev_attr_curr4_input.dev_attr.attr,
+ &sensor_dev_attr_curr5_highest.dev_attr.attr,
+ &sensor_dev_attr_curr5_average.dev_attr.attr,
+ &sensor_dev_attr_curr5_input.dev_attr.attr,
+ &sensor_dev_attr_curr6_highest.dev_attr.attr,
+ &sensor_dev_attr_curr6_average.dev_attr.attr,
+ &sensor_dev_attr_curr6_input.dev_attr.attr,
+ NULL
+};
+
+static const struct attribute_group hwmon_mb_attrgroup = {
+ .attrs = hwmon_mb_attributes,
+};
+
+static ssize_t show_name(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", XCLMGMT_MB_HWMON_NAME);
+}
+
+static struct sensor_device_attribute name_attr =
+ SENSOR_ATTR(name, 0444, show_name, NULL, 0);
+
+static void mgmt_sysfs_destroy_mb(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+
+ mb = platform_get_drvdata(pdev);
+
+ if (!mb->enabled)
+ return;
+
+ if (mb->hwmon_dev) {
+ device_remove_file(mb->hwmon_dev, &name_attr.dev_attr);
+ sysfs_remove_group(&mb->hwmon_dev->kobj,
+ &hwmon_mb_attrgroup);
+ hwmon_device_unregister(mb->hwmon_dev);
+ mb->hwmon_dev = NULL;
+ }
+
+ sysfs_remove_group(&pdev->dev.kobj, &mb_attr_group);
+}
+
+static int mgmt_sysfs_create_mb(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+ struct xocl_dev_core *core;
+ int err;
+
+ mb = platform_get_drvdata(pdev);
+ core = XDEV(xocl_get_xdev(pdev));
+
+ if (!mb->enabled)
+ return 0;
+ err = sysfs_create_group(&pdev->dev.kobj, &mb_attr_group);
+ if (err) {
+ xocl_err(&pdev->dev, "create mb attrs failed: 0x%x", err);
+ goto create_attr_failed;
+ }
+ mb->hwmon_dev = hwmon_device_register(&core->pdev->dev);
+ if (IS_ERR(mb->hwmon_dev)) {
+ err = PTR_ERR(mb->hwmon_dev);
+ xocl_err(&pdev->dev, "register mb hwmon failed: 0x%x", err);
+ goto hwmon_reg_failed;
+ }
+
+ dev_set_drvdata(mb->hwmon_dev, mb);
+
+ err = device_create_file(mb->hwmon_dev, &name_attr.dev_attr);
+ if (err) {
+ xocl_err(&pdev->dev, "create attr name failed: 0x%x", err);
+ goto create_name_failed;
+ }
+
+ err = sysfs_create_group(&mb->hwmon_dev->kobj,
+ &hwmon_mb_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create pw group failed: 0x%x", err);
+ goto create_pw_failed;
+ }
+
+ return 0;
+
+create_pw_failed:
+ device_remove_file(mb->hwmon_dev, &name_attr.dev_attr);
+create_name_failed:
+ hwmon_device_unregister(mb->hwmon_dev);
+ mb->hwmon_dev = NULL;
+hwmon_reg_failed:
+ sysfs_remove_group(&pdev->dev.kobj, &mb_attr_group);
+create_attr_failed:
+ return err;
+}
+
+static int mb_stop(struct xocl_mb *mb)
+{
+ int retry = 0;
+ int ret = 0;
+ u32 reg_val = 0;
+
+ if (!mb->enabled)
+ return 0;
+
+ mutex_lock(&mb->mb_lock);
+ reg_val = READ_GPIO(mb, 0);
+ xocl_info(&mb->pdev->dev, "Reset GPIO 0x%x", reg_val);
+ if (reg_val == GPIO_RESET) {
+ /* MB in reset status */
+ mb->state = MB_STATE_RESET;
+ goto out;
+ }
+
+ xocl_info(&mb->pdev->dev,
+ "MGMT Image magic word, 0x%x, status 0x%x, id 0x%x",
+ READ_IMAGE_MGMT(mb, 0),
+ READ_REG32(mb, REG_STATUS),
+ READ_REG32(mb, REG_ID));
+
+ if (!SELF_JUMP(READ_IMAGE_MGMT(mb, 0))) {
+ /* non cold boot */
+ reg_val = READ_REG32(mb, REG_STATUS);
+ if (!(reg_val & STATUS_MASK_STOPPED)) {
+ // need to stop microblaze
+ xocl_info(&mb->pdev->dev, "stopping microblaze...");
+ WRITE_REG32(mb, CTL_MASK_STOP, REG_CTL);
+ WRITE_REG32(mb, 1, REG_STOP_CONFIRM);
+ while (retry++ < MAX_RETRY &&
+ !(READ_REG32(mb, REG_STATUS) &
+ STATUS_MASK_STOPPED)) {
+ msleep(RETRY_INTERVAL);
+ }
+ if (retry >= MAX_RETRY) {
+ xocl_err(&mb->pdev->dev,
+ "Failed to stop microblaze");
+ xocl_err(&mb->pdev->dev,
+ "Error Reg 0x%x",
+ READ_REG32(mb, REG_ERR));
+ ret = -EIO;
+ goto out;
+ }
+ }
+ xocl_info(&mb->pdev->dev, "Microblaze Stopped, retry %d",
+ retry);
+ }
+
+ /* hold reset */
+ WRITE_GPIO(mb, GPIO_RESET, 0);
+ mb->state = MB_STATE_RESET;
+out:
+ mutex_unlock(&mb->mb_lock);
+
+ return ret;
+}
+
+static int mb_start(struct xocl_mb *mb)
+{
+ int retry = 0;
+ u32 reg_val = 0;
+ int ret = 0;
+ void *xdev_hdl;
+
+ if (!mb->enabled)
+ return 0;
+
+ xdev_hdl = xocl_get_xdev(mb->pdev);
+
+ mutex_lock(&mb->mb_lock);
+ reg_val = READ_GPIO(mb, 0);
+ xocl_info(&mb->pdev->dev, "Reset GPIO 0x%x", reg_val);
+ if (reg_val == GPIO_ENABLED)
+ goto out;
+
+ xocl_info(&mb->pdev->dev, "Start Microblaze...");
+ xocl_info(&mb->pdev->dev, "MGMT Image magic word, 0x%x",
+ READ_IMAGE_MGMT(mb, 0));
+
+ if (xocl_mb_mgmt_on(xdev_hdl)) {
+ xocl_info(&mb->pdev->dev, "Copying mgmt image len %d",
+ mb->mgmt_binary_length);
+ COPY_MGMT(mb, mb->mgmt_binary, mb->mgmt_binary_length);
+ }
+
+ if (xocl_mb_sched_on(xdev_hdl)) {
+ xocl_info(&mb->pdev->dev, "Copying scheduler image len %d",
+ mb->sche_binary_length);
+ COPY_SCHE(mb, mb->sche_binary, mb->sche_binary_length);
+ }
+
+ WRITE_GPIO(mb, GPIO_ENABLED, 0);
+ xocl_info(&mb->pdev->dev,
+ "MGMT Image magic word, 0x%x, status 0x%x, id 0x%x",
+ READ_IMAGE_MGMT(mb, 0),
+ READ_REG32(mb, REG_STATUS),
+ READ_REG32(mb, REG_ID));
+ do {
+ msleep(RETRY_INTERVAL);
+ } while (retry++ < MAX_RETRY && (READ_REG32(mb, REG_STATUS) &
+ STATUS_MASK_STOPPED));
+
+ /* Extra pulse needed as workaround for axi interconnect issue in DSA */
+ if (retry >= MAX_RETRY) {
+ retry = 0;
+ WRITE_GPIO(mb, GPIO_RESET, 0);
+ WRITE_GPIO(mb, GPIO_ENABLED, 0);
+ do {
+ msleep(RETRY_INTERVAL);
+ } while (retry++ < MAX_RETRY && (READ_REG32(mb, REG_STATUS) &
+ STATUS_MASK_STOPPED));
+ }
+
+ if (retry >= MAX_RETRY) {
+ xocl_err(&mb->pdev->dev, "Failed to start microblaze");
+ xocl_err(&mb->pdev->dev, "Error Reg 0x%x",
+ READ_REG32(mb, REG_ERR));
+ ret = -EIO;
+ }
+
+ mb->cap = READ_REG32(mb, REG_CAP);
+ mb->state = MB_STATE_RUN;
+out:
+ mutex_unlock(&mb->mb_lock);
+
+ return ret;
+}
+
+static void mb_reset(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+
+ xocl_info(&pdev->dev, "Reset Microblaze...");
+ mb = platform_get_drvdata(pdev);
+ if (!mb)
+ return;
+
+ mb_stop(mb);
+ mb_start(mb);
+}
+
+static int load_mgmt_image(struct platform_device *pdev, const char *image,
+ u32 len)
+{
+ struct xocl_mb *mb;
+ char *binary;
+
+ if (len > MAX_IMAGE_LEN)
+ return -EINVAL;
+
+ mb = platform_get_drvdata(pdev);
+ if (!mb)
+ return -EINVAL;
+
+ binary = mb->mgmt_binary;
+ mb->mgmt_binary = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
+ if (!mb->mgmt_binary)
+ return -ENOMEM;
+
+ if (binary)
+ devm_kfree(&pdev->dev, binary);
+ memcpy(mb->mgmt_binary, image, len);
+ mb->mgmt_binary_length = len;
+
+ return 0;
+}
+
+static int load_sche_image(struct platform_device *pdev, const char *image,
+ u32 len)
+{
+ struct xocl_mb *mb;
+ char *binary = NULL;
+
+ if (len > MAX_IMAGE_LEN)
+ return -EINVAL;
+
+ mb = platform_get_drvdata(pdev);
+ if (!mb)
+ return -EINVAL;
+
+ binary = mb->sche_binary;
+ mb->sche_binary = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
+ if (!mb->sche_binary)
+ return -ENOMEM;
+
+ if (binary)
+ devm_kfree(&pdev->dev, binary);
+ memcpy(mb->sche_binary, image, len);
+ mb->sche_binary_length = len;
+
+ return 0;
+}
+
+//Have a function stub but don't actually do anything when this is called
+static int mb_ignore(struct platform_device *pdev)
+{
+ return 0;
+}
+
+static struct xocl_mb_funcs mb_ops = {
+ .load_mgmt_image = load_mgmt_image,
+ .load_sche_image = load_sche_image,
+ .reset = mb_reset,
+ .stop = mb_ignore,
+};
+
+
+
+static int mb_remove(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+ int i;
+
+ mb = platform_get_drvdata(pdev);
+ if (!mb)
+ return 0;
+
+ if (mb->mgmt_binary)
+ devm_kfree(&pdev->dev, mb->mgmt_binary);
+ if (mb->sche_binary)
+ devm_kfree(&pdev->dev, mb->sche_binary);
+
+ /*
+ * It is more secure that MB keeps running even driver is unloaded.
+ * Even user unload our driver and use their own stuff, MB will still
+ * be able to monitor the board unless user stops it explicitly
+ */
+ mb_stop(mb);
+
+ mgmt_sysfs_destroy_mb(pdev);
+
+ for (i = 0; i < NUM_IOADDR; i++) {
+ if (mb->base_addrs[i])
+ iounmap(mb->base_addrs[i]);
+ }
+
+ mutex_destroy(&mb->mb_lock);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, mb);
+
+ return 0;
+}
+
+static int mb_probe(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+ struct resource *res;
+ void *xdev_hdl;
+ int i, err;
+
+ mb = devm_kzalloc(&pdev->dev, sizeof(*mb), GFP_KERNEL);
+ if (!mb) {
+ xocl_err(&pdev->dev, "out of memory");
+ return -ENOMEM;
+ }
+
+ mb->pdev = pdev;
+ platform_set_drvdata(pdev, mb);
+
+ xdev_hdl = xocl_get_xdev(pdev);
+ if (xocl_mb_mgmt_on(xdev_hdl) || xocl_mb_sched_on(xdev_hdl)) {
+ xocl_info(&pdev->dev, "Microblaze is supported.");
+ mb->enabled = true;
+ } else {
+ xocl_info(&pdev->dev, "Microblaze is not supported.");
+ devm_kfree(&pdev->dev, mb);
+ platform_set_drvdata(pdev, NULL);
+ return 0;
+ }
+
+ for (i = 0; i < NUM_IOADDR; i++) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+ xocl_info(&pdev->dev, "IO start: 0x%llx, end: 0x%llx",
+ res->start, res->end);
+ mb->base_addrs[i] =
+ ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!mb->base_addrs[i]) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+ }
+
+ err = mgmt_sysfs_create_mb(pdev);
+ if (err) {
+ xocl_err(&pdev->dev, "Create sysfs failed, err %d", err);
+ goto failed;
+ }
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_MB, &mb_ops);
+
+ mutex_init(&mb->mb_lock);
+
+ return 0;
+
+failed:
+ mb_remove(pdev);
+ return err;
+}
+
+struct platform_device_id mb_id_table[] = {
+ { XOCL_MB, 0 },
+ { },
+};
+
+static struct platform_driver mb_driver = {
+ .probe = mb_probe,
+ .remove = mb_remove,
+ .driver = {
+ .name = "xocl_mb",
+ },
+ .id_table = mb_id_table,
+};
+
+int __init xocl_init_mb(void)
+{
+ return platform_driver_register(&mb_driver);
+}
+
+void xocl_fini_mb(void)
+{
+ platform_driver_unregister(&mb_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/mig.c b/drivers/gpu/drm/xocl/subdev/mig.c
new file mode 100644
index 000000000000..5a574f7af796
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/mig.c
@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2018-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Chien-Wei Lan <chienwei@xilinx.com>
+ *
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+/* Registers are defined in pg150-ultrascale-memory-ip.pdf:
+ * AXI4-Lite Slave Control/Status Register Map
+ */
+
+#define MIG_DEBUG
+#define MIG_DEV2MIG(dev) \
+ ((struct xocl_mig *)platform_get_drvdata(to_platform_device(dev)))
+#define MIG_DEV2BASE(dev) (MIG_DEV2MIG(dev)->base)
+
+#define ECC_STATUS 0x0
+#define ECC_ON_OFF 0x8
+#define CE_CNT 0xC
+#define CE_ADDR_LO 0x1C0
+#define CE_ADDR_HI 0x1C4
+#define UE_ADDR_LO 0x2C0
+#define UE_ADDR_HI 0x2C4
+#define INJ_FAULT_REG 0x300
+
+struct xocl_mig {
+ void __iomem *base;
+ struct device *mig_dev;
+};
+
+static ssize_t ecc_ue_ffa_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ uint64_t val = ioread32(MIG_DEV2BASE(dev) + UE_ADDR_HI);
+
+ val <<= 32;
+ val |= ioread32(MIG_DEV2BASE(dev) + UE_ADDR_LO);
+ return sprintf(buf, "0x%llx\n", val);
+}
+static DEVICE_ATTR_RO(ecc_ue_ffa);
+
+
+static ssize_t ecc_ce_ffa_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ uint64_t val = ioread32(MIG_DEV2BASE(dev) + CE_ADDR_HI);
+
+ val <<= 32;
+ val |= ioread32(MIG_DEV2BASE(dev) + CE_ADDR_LO);
+ return sprintf(buf, "0x%llx\n", val);
+}
+static DEVICE_ATTR_RO(ecc_ce_ffa);
+
+
+static ssize_t ecc_ce_cnt_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%u\n", ioread32(MIG_DEV2BASE(dev) + CE_CNT));
+}
+static DEVICE_ATTR_RO(ecc_ce_cnt);
+
+
+static ssize_t ecc_status_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%u\n", ioread32(MIG_DEV2BASE(dev) + ECC_STATUS));
+}
+static DEVICE_ATTR_RO(ecc_status);
+
+
+static ssize_t ecc_reset_store(struct device *dev, struct device_attribute *da,
+ const char *buf, size_t count)
+{
+ iowrite32(0x3, MIG_DEV2BASE(dev) + ECC_STATUS);
+ iowrite32(0, MIG_DEV2BASE(dev) + CE_CNT);
+ return count;
+}
+static DEVICE_ATTR_WO(ecc_reset);
+
+
+static ssize_t ecc_enabled_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%u\n", ioread32(MIG_DEV2BASE(dev) + ECC_ON_OFF));
+}
+static ssize_t ecc_enabled_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ uint32_t val;
+
+ if (sscanf(buf, "%d", &val) != 1 || val > 1) {
+ xocl_err(&to_platform_device(dev)->dev,
+ "usage: echo [0|1] > ecc_enabled");
+ return -EINVAL;
+ }
+
+ iowrite32(val, MIG_DEV2BASE(dev) + ECC_ON_OFF);
+ return count;
+}
+static DEVICE_ATTR_RW(ecc_enabled);
+
+
+#ifdef MIG_DEBUG
+static ssize_t ecc_inject_store(struct device *dev, struct device_attribute *da,
+ const char *buf, size_t count)
+{
+ iowrite32(1, MIG_DEV2BASE(dev) + INJ_FAULT_REG);
+ return count;
+}
+static DEVICE_ATTR_WO(ecc_inject);
+#endif
+
+
+/* Standard sysfs entry for all dynamic subdevices. */
+static ssize_t name_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", XOCL_GET_SUBDEV_PRIV(dev));
+}
+static DEVICE_ATTR_RO(name);
+
+
+static struct attribute *mig_attributes[] = {
+ &dev_attr_name.attr,
+ &dev_attr_ecc_enabled.attr,
+ &dev_attr_ecc_status.attr,
+ &dev_attr_ecc_ce_cnt.attr,
+ &dev_attr_ecc_ce_ffa.attr,
+ &dev_attr_ecc_ue_ffa.attr,
+ &dev_attr_ecc_reset.attr,
+#ifdef MIG_DEBUG
+ &dev_attr_ecc_inject.attr,
+#endif
+ NULL
+};
+
+static const struct attribute_group mig_attrgroup = {
+ .attrs = mig_attributes,
+};
+
+static void mgmt_sysfs_destroy_mig(struct platform_device *pdev)
+{
+ struct xocl_mig *mig;
+
+ mig = platform_get_drvdata(pdev);
+ sysfs_remove_group(&pdev->dev.kobj, &mig_attrgroup);
+}
+
+static int mgmt_sysfs_create_mig(struct platform_device *pdev)
+{
+ struct xocl_mig *mig;
+ int err;
+
+ mig = platform_get_drvdata(pdev);
+ err = sysfs_create_group(&pdev->dev.kobj, &mig_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create pw group failed: 0x%x", err);
+ return err;
+ }
+
+ return 0;
+}
+
+static int mig_probe(struct platform_device *pdev)
+{
+ struct xocl_mig *mig;
+ struct resource *res;
+ int err;
+
+ mig = devm_kzalloc(&pdev->dev, sizeof(*mig), GFP_KERNEL);
+ if (!mig)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ xocl_err(&pdev->dev, "resource is NULL");
+ return -EINVAL;
+ }
+
+ xocl_info(&pdev->dev, "MIG name: %s, IO start: 0x%llx, end: 0x%llx",
+ XOCL_GET_SUBDEV_PRIV(&pdev->dev), res->start, res->end);
+
+ mig->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!mig->base) {
+ xocl_err(&pdev->dev, "Map iomem failed");
+ return -EIO;
+ }
+
+ platform_set_drvdata(pdev, mig);
+
+ err = mgmt_sysfs_create_mig(pdev);
+ if (err) {
+ platform_set_drvdata(pdev, NULL);
+ iounmap(mig->base);
+ return err;
+ }
+
+ return 0;
+}
+
+
+static int mig_remove(struct platform_device *pdev)
+{
+ struct xocl_mig *mig;
+
+ mig = platform_get_drvdata(pdev);
+ if (!mig) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ xocl_info(&pdev->dev, "MIG name: %s", XOCL_GET_SUBDEV_PRIV(&pdev->dev));
+
+ mgmt_sysfs_destroy_mig(pdev);
+
+ if (mig->base)
+ iounmap(mig->base);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, mig);
+
+ return 0;
+}
+
+struct platform_device_id mig_id_table[] = {
+ { XOCL_MIG, 0 },
+ { },
+};
+
+static struct platform_driver mig_driver = {
+ .probe = mig_probe,
+ .remove = mig_remove,
+ .driver = {
+ .name = "xocl_mig",
+ },
+ .id_table = mig_id_table,
+};
+
+int __init xocl_init_mig(void)
+{
+ return platform_driver_register(&mig_driver);
+}
+
+void xocl_fini_mig(void)
+{
+ platform_driver_unregister(&mig_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/sysmon.c b/drivers/gpu/drm/xocl/subdev/sysmon.c
new file mode 100644
index 000000000000..bb5c84485344
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/sysmon.c
@@ -0,0 +1,385 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+#define TEMP 0x400 // TEMPOERATURE REGISTER ADDRESS
+#define VCCINT 0x404 // VCCINT REGISTER OFFSET
+#define VCCAUX 0x408 // VCCAUX REGISTER OFFSET
+#define VCCBRAM 0x418 // VCCBRAM REGISTER OFFSET
+#define TEMP_MAX 0x480
+#define VCCINT_MAX 0x484
+#define VCCAUX_MAX 0x488
+#define VCCBRAM_MAX 0x48c
+#define TEMP_MIN 0x490
+#define VCCINT_MIN 0x494
+#define VCCAUX_MIN 0x498
+#define VCCBRAM_MIN 0x49c
+
+#define SYSMON_TO_MILLDEGREE(val) \
+ (((int64_t)(val) * 501374 >> 16) - 273678)
+#define SYSMON_TO_MILLVOLT(val) \
+ ((val) * 1000 * 3 >> 16)
+
+#define READ_REG32(sysmon, off) \
+ XOCL_READ_REG32(sysmon->base + off)
+#define WRITE_REG32(sysmon, val, off) \
+ XOCL_WRITE_REG32(val, sysmon->base + off)
+
+struct xocl_sysmon {
+ void __iomem *base;
+ struct device *hwmon_dev;
+};
+
+static int get_prop(struct platform_device *pdev, u32 prop, void *val)
+{
+ struct xocl_sysmon *sysmon;
+ u32 tmp;
+
+ sysmon = platform_get_drvdata(pdev);
+ BUG_ON(!sysmon);
+
+ switch (prop) {
+ case XOCL_SYSMON_PROP_TEMP:
+ tmp = READ_REG32(sysmon, TEMP);
+ *(u32 *)val = SYSMON_TO_MILLDEGREE(tmp)/1000;
+ break;
+ case XOCL_SYSMON_PROP_TEMP_MAX:
+ tmp = READ_REG32(sysmon, TEMP_MAX);
+ *(u32 *)val = SYSMON_TO_MILLDEGREE(tmp);
+ break;
+ case XOCL_SYSMON_PROP_TEMP_MIN:
+ tmp = READ_REG32(sysmon, TEMP_MIN);
+ *(u32 *)val = SYSMON_TO_MILLDEGREE(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_INT:
+ tmp = READ_REG32(sysmon, VCCINT);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_INT_MAX:
+ tmp = READ_REG32(sysmon, VCCINT_MAX);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_INT_MIN:
+ tmp = READ_REG32(sysmon, VCCINT_MIN);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_AUX:
+ tmp = READ_REG32(sysmon, VCCAUX);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_AUX_MAX:
+ tmp = READ_REG32(sysmon, VCCAUX_MAX);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_AUX_MIN:
+ tmp = READ_REG32(sysmon, VCCAUX_MIN);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_BRAM:
+ tmp = READ_REG32(sysmon, VCCBRAM);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_BRAM_MAX:
+ tmp = READ_REG32(sysmon, VCCBRAM_MAX);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_BRAM_MIN:
+ tmp = READ_REG32(sysmon, VCCBRAM_MIN);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ default:
+ xocl_err(&pdev->dev, "Invalid prop");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static struct xocl_sysmon_funcs sysmon_ops = {
+ .get_prop = get_prop,
+};
+
+static ssize_t show_sysmon(struct platform_device *pdev, u32 prop, char *buf)
+{
+ u32 val;
+
+ (void) get_prop(pdev, prop, &val);
+ return sprintf(buf, "%u\n", val);
+}
+
+/* sysfs support */
+static ssize_t show_hwmon(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+ struct platform_device *pdev = dev_get_drvdata(dev);
+
+ return show_sysmon(pdev, attr->index, buf);
+}
+
+static ssize_t show_name(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", XCLMGMT_SYSMON_HWMON_NAME);
+}
+
+static SENSOR_DEVICE_ATTR(temp1_input, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_TEMP);
+static SENSOR_DEVICE_ATTR(temp1_highest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_TEMP_MAX);
+static SENSOR_DEVICE_ATTR(temp1_lowest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_TEMP_MIN);
+
+static SENSOR_DEVICE_ATTR(in0_input, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_INT);
+static SENSOR_DEVICE_ATTR(in0_highest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_INT_MAX);
+static SENSOR_DEVICE_ATTR(in0_lowest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_INT_MIN);
+
+static SENSOR_DEVICE_ATTR(in1_input, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_AUX);
+static SENSOR_DEVICE_ATTR(in1_highest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_AUX_MAX);
+static SENSOR_DEVICE_ATTR(in1_lowest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_AUX_MIN);
+
+static SENSOR_DEVICE_ATTR(in2_input, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_BRAM);
+static SENSOR_DEVICE_ATTR(in2_highest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_BRAM_MAX);
+static SENSOR_DEVICE_ATTR(in2_lowest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_BRAM_MIN);
+
+static struct attribute *hwmon_sysmon_attributes[] = {
+ &sensor_dev_attr_temp1_input.dev_attr.attr,
+ &sensor_dev_attr_temp1_highest.dev_attr.attr,
+ &sensor_dev_attr_temp1_lowest.dev_attr.attr,
+ &sensor_dev_attr_in0_input.dev_attr.attr,
+ &sensor_dev_attr_in0_highest.dev_attr.attr,
+ &sensor_dev_attr_in0_lowest.dev_attr.attr,
+ &sensor_dev_attr_in1_input.dev_attr.attr,
+ &sensor_dev_attr_in1_highest.dev_attr.attr,
+ &sensor_dev_attr_in1_lowest.dev_attr.attr,
+ &sensor_dev_attr_in2_input.dev_attr.attr,
+ &sensor_dev_attr_in2_highest.dev_attr.attr,
+ &sensor_dev_attr_in2_lowest.dev_attr.attr,
+ NULL
+};
+
+static const struct attribute_group hwmon_sysmon_attrgroup = {
+ .attrs = hwmon_sysmon_attributes,
+};
+
+static struct sensor_device_attribute sysmon_name_attr =
+ SENSOR_ATTR(name, 0444, show_name, NULL, 0);
+
+static ssize_t temp_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return show_sysmon(to_platform_device(dev), XOCL_SYSMON_PROP_TEMP, buf);
+}
+static DEVICE_ATTR_RO(temp);
+
+static ssize_t vcc_int_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return show_sysmon(to_platform_device(dev), XOCL_SYSMON_PROP_VCC_INT, buf);
+}
+static DEVICE_ATTR_RO(vcc_int);
+
+static ssize_t vcc_aux_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return show_sysmon(to_platform_device(dev), XOCL_SYSMON_PROP_VCC_AUX, buf);
+}
+static DEVICE_ATTR_RO(vcc_aux);
+
+static ssize_t vcc_bram_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return show_sysmon(to_platform_device(dev), XOCL_SYSMON_PROP_VCC_BRAM, buf);
+}
+static DEVICE_ATTR_RO(vcc_bram);
+
+static struct attribute *sysmon_attributes[] = {
+ &dev_attr_temp.attr,
+ &dev_attr_vcc_int.attr,
+ &dev_attr_vcc_aux.attr,
+ &dev_attr_vcc_bram.attr,
+ NULL,
+};
+
+static const struct attribute_group sysmon_attrgroup = {
+ .attrs = sysmon_attributes,
+};
+
+static void mgmt_sysfs_destroy_sysmon(struct platform_device *pdev)
+{
+ struct xocl_sysmon *sysmon;
+
+ sysmon = platform_get_drvdata(pdev);
+
+ device_remove_file(sysmon->hwmon_dev, &sysmon_name_attr.dev_attr);
+ sysfs_remove_group(&sysmon->hwmon_dev->kobj, &hwmon_sysmon_attrgroup);
+ hwmon_device_unregister(sysmon->hwmon_dev);
+ sysmon->hwmon_dev = NULL;
+
+ sysfs_remove_group(&pdev->dev.kobj, &sysmon_attrgroup);
+}
+
+static int mgmt_sysfs_create_sysmon(struct platform_device *pdev)
+{
+ struct xocl_sysmon *sysmon;
+ struct xocl_dev_core *core;
+ int err;
+
+ sysmon = platform_get_drvdata(pdev);
+ core = XDEV(xocl_get_xdev(pdev));
+
+ sysmon->hwmon_dev = hwmon_device_register(&core->pdev->dev);
+ if (IS_ERR(sysmon->hwmon_dev)) {
+ err = PTR_ERR(sysmon->hwmon_dev);
+ xocl_err(&pdev->dev, "register sysmon hwmon failed: 0x%x", err);
+ goto hwmon_reg_failed;
+ }
+
+ dev_set_drvdata(sysmon->hwmon_dev, pdev);
+ err = device_create_file(sysmon->hwmon_dev,
+ &sysmon_name_attr.dev_attr);
+ if (err) {
+ xocl_err(&pdev->dev, "create attr name failed: 0x%x", err);
+ goto create_name_failed;
+ }
+
+ err = sysfs_create_group(&sysmon->hwmon_dev->kobj,
+ &hwmon_sysmon_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create hwmon group failed: 0x%x", err);
+ goto create_hwmon_failed;
+ }
+
+ err = sysfs_create_group(&pdev->dev.kobj, &sysmon_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create sysmon group failed: 0x%x", err);
+ goto create_sysmon_failed;
+ }
+
+ return 0;
+
+create_sysmon_failed:
+ sysfs_remove_group(&sysmon->hwmon_dev->kobj, &hwmon_sysmon_attrgroup);
+create_hwmon_failed:
+ device_remove_file(sysmon->hwmon_dev, &sysmon_name_attr.dev_attr);
+create_name_failed:
+ hwmon_device_unregister(sysmon->hwmon_dev);
+ sysmon->hwmon_dev = NULL;
+hwmon_reg_failed:
+ return err;
+}
+
+static int sysmon_probe(struct platform_device *pdev)
+{
+ struct xocl_sysmon *sysmon;
+ struct resource *res;
+ int err;
+
+ sysmon = devm_kzalloc(&pdev->dev, sizeof(*sysmon), GFP_KERNEL);
+ if (!sysmon)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ xocl_err(&pdev->dev, "resource is NULL");
+ return -EINVAL;
+ }
+ xocl_info(&pdev->dev, "IO start: 0x%llx, end: 0x%llx",
+ res->start, res->end);
+ sysmon->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!sysmon->base) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+
+ platform_set_drvdata(pdev, sysmon);
+
+ err = mgmt_sysfs_create_sysmon(pdev);
+ if (err)
+ goto create_sysmon_failed;
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_SYSMON, &sysmon_ops);
+
+ return 0;
+
+create_sysmon_failed:
+ platform_set_drvdata(pdev, NULL);
+failed:
+ return err;
+}
+
+
+static int sysmon_remove(struct platform_device *pdev)
+{
+ struct xocl_sysmon *sysmon;
+
+ sysmon = platform_get_drvdata(pdev);
+ if (!sysmon) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ mgmt_sysfs_destroy_sysmon(pdev);
+
+ if (sysmon->base)
+ iounmap(sysmon->base);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, sysmon);
+
+ return 0;
+}
+
+struct platform_device_id sysmon_id_table[] = {
+ { XOCL_SYSMON, 0 },
+ { },
+};
+
+static struct platform_driver sysmon_driver = {
+ .probe = sysmon_probe,
+ .remove = sysmon_remove,
+ .driver = {
+ .name = "xocl_sysmon",
+ },
+ .id_table = sysmon_id_table,
+};
+
+int __init xocl_init_sysmon(void)
+{
+ return platform_driver_register(&sysmon_driver);
+}
+
+void xocl_fini_sysmon(void)
+{
+ platform_driver_unregister(&sysmon_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/xdma.c b/drivers/gpu/drm/xocl/subdev/xdma.c
new file mode 100644
index 000000000000..647a69f29a84
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/xdma.c
@@ -0,0 +1,510 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ */
+
+/* XDMA version Memory Mapped DMA */
+
+#include <linux/version.h>
+#include <linux/eventfd.h>
+#include <drm/drmP.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_mm.h>
+#include "../xocl_drv.h"
+#include "../xocl_drm.h"
+#include "../lib/libxdma_api.h"
+
+#define XOCL_FILE_PAGE_OFFSET 0x100000
+#ifndef VM_RESERVED
+#define VM_RESERVED (VM_DONTEXPAND | VM_DONTDUMP)
+#endif
+
+struct xdma_irq {
+ struct eventfd_ctx *event_ctx;
+ bool in_use;
+ bool enabled;
+ irq_handler_t handler;
+ void *arg;
+};
+
+struct xocl_xdma {
+ void *dma_handle;
+ u32 max_user_intr;
+ u32 start_user_intr;
+ struct xdma_irq *user_msix_table;
+ struct mutex user_msix_table_lock;
+
+ struct xocl_drm *drm;
+ /* Number of bidirectional channels */
+ u32 channel;
+ /* Semaphore, one for each direction */
+ struct semaphore channel_sem[2];
+ /*
+ * Channel usage bitmasks, one for each direction
+ * bit 1 indicates channel is free, bit 0 indicates channel is free
+ */
+ unsigned long channel_bitmap[2];
+ unsigned long long *channel_usage[2];
+
+ struct mutex stat_lock;
+};
+
+static ssize_t xdma_migrate_bo(struct platform_device *pdev,
+ struct sg_table *sgt, u32 dir, u64 paddr, u32 channel, u64 len)
+{
+ struct xocl_xdma *xdma;
+ struct page *pg;
+ struct scatterlist *sg = sgt->sgl;
+ int nents = sgt->orig_nents;
+ pid_t pid = current->pid;
+ int i = 0;
+ ssize_t ret;
+ unsigned long long pgaddr;
+
+ xdma = platform_get_drvdata(pdev);
+ xocl_dbg(&pdev->dev, "TID %d, Channel:%d, Offset: 0x%llx, Dir: %d",
+ pid, channel, paddr, dir);
+ ret = xdma_xfer_submit(xdma->dma_handle, channel, dir,
+ paddr, sgt, false, 10000);
+ if (ret >= 0) {
+ xdma->channel_usage[dir][channel] += ret;
+ return ret;
+ }
+
+ xocl_err(&pdev->dev, "DMA failed, Dumping SG Page Table");
+ for (i = 0; i < nents; i++, sg = sg_next(sg)) {
+ if (!sg)
+ break;
+ pg = sg_page(sg);
+ if (!pg)
+ continue;
+ pgaddr = page_to_phys(pg);
+ xocl_err(&pdev->dev, "%i, 0x%llx\n", i, pgaddr);
+ }
+ return ret;
+}
+
+static int acquire_channel(struct platform_device *pdev, u32 dir)
+{
+ struct xocl_xdma *xdma;
+ int channel = 0;
+ int result = 0;
+
+ xdma = platform_get_drvdata(pdev);
+ if (down_interruptible(&xdma->channel_sem[dir])) {
+ channel = -ERESTARTSYS;
+ goto out;
+ }
+
+ for (channel = 0; channel < xdma->channel; channel++) {
+ result = test_and_clear_bit(channel,
+ &xdma->channel_bitmap[dir]);
+ if (result)
+ break;
+ }
+ if (!result) {
+ // How is this possible?
+ up(&xdma->channel_sem[dir]);
+ channel = -EIO;
+ }
+
+out:
+ return channel;
+}
+
+static void release_channel(struct platform_device *pdev, u32 dir, u32 channel)
+{
+ struct xocl_xdma *xdma;
+
+
+ xdma = platform_get_drvdata(pdev);
+ set_bit(channel, &xdma->channel_bitmap[dir]);
+ up(&xdma->channel_sem[dir]);
+}
+
+static u32 get_channel_count(struct platform_device *pdev)
+{
+ struct xocl_xdma *xdma;
+
+ xdma = platform_get_drvdata(pdev);
+ BUG_ON(!xdma);
+
+ return xdma->channel;
+}
+
+static void *get_drm_handle(struct platform_device *pdev)
+{
+ struct xocl_xdma *xdma;
+
+ xdma = platform_get_drvdata(pdev);
+
+ return xdma->drm;
+}
+
+static u64 get_channel_stat(struct platform_device *pdev, u32 channel,
+ u32 write)
+{
+ struct xocl_xdma *xdma;
+
+ xdma = platform_get_drvdata(pdev);
+ BUG_ON(!xdma);
+
+ return xdma->channel_usage[write][channel];
+}
+
+static int user_intr_config(struct platform_device *pdev, u32 intr, bool en)
+{
+ struct xocl_xdma *xdma;
+ const unsigned int mask = 1 << intr;
+ int ret;
+
+ xdma = platform_get_drvdata(pdev);
+
+ if (intr >= xdma->max_user_intr) {
+ xocl_err(&pdev->dev, "Invalid intr %d, user start %d, max %d",
+ intr, xdma->start_user_intr, xdma->max_user_intr);
+ return -EINVAL;
+ }
+
+ mutex_lock(&xdma->user_msix_table_lock);
+ if (xdma->user_msix_table[intr].enabled == en) {
+ ret = 0;
+ goto end;
+ }
+
+ ret = en ? xdma_user_isr_enable(xdma->dma_handle, mask) :
+ xdma_user_isr_disable(xdma->dma_handle, mask);
+ if (!ret)
+ xdma->user_msix_table[intr].enabled = en;
+end:
+ mutex_unlock(&xdma->user_msix_table_lock);
+
+ return ret;
+}
+
+static irqreturn_t xdma_isr(int irq, void *arg)
+{
+ struct xdma_irq *irq_entry = arg;
+ int ret = IRQ_HANDLED;
+
+ if (irq_entry->handler)
+ ret = irq_entry->handler(irq, irq_entry->arg);
+
+ if (!IS_ERR_OR_NULL(irq_entry->event_ctx))
+ eventfd_signal(irq_entry->event_ctx, 1);
+
+ return ret;
+}
+
+static int user_intr_unreg(struct platform_device *pdev, u32 intr)
+{
+ struct xocl_xdma *xdma;
+ const unsigned int mask = 1 << intr;
+ int ret;
+
+ xdma = platform_get_drvdata(pdev);
+
+ if (intr >= xdma->max_user_intr)
+ return -EINVAL;
+
+ mutex_lock(&xdma->user_msix_table_lock);
+ if (!xdma->user_msix_table[intr].in_use) {
+ ret = -EINVAL;
+ goto failed;
+ }
+ xdma->user_msix_table[intr].handler = NULL;
+ xdma->user_msix_table[intr].arg = NULL;
+
+ ret = xdma_user_isr_register(xdma->dma_handle, mask, NULL, NULL);
+ if (ret) {
+ xocl_err(&pdev->dev, "xdma unregister isr failed");
+ goto failed;
+ }
+
+ xdma->user_msix_table[intr].in_use = false;
+
+failed:
+ mutex_unlock(&xdma->user_msix_table_lock);
+ return ret;
+}
+
+static int user_intr_register(struct platform_device *pdev, u32 intr,
+ irq_handler_t handler, void *arg, int event_fd)
+{
+ struct xocl_xdma *xdma;
+ struct eventfd_ctx *trigger = ERR_PTR(-EINVAL);
+ const unsigned int mask = 1 << intr;
+ int ret;
+
+ xdma = platform_get_drvdata(pdev);
+
+ if (intr >= xdma->max_user_intr ||
+ (event_fd >= 0 && intr < xdma->start_user_intr)) {
+ xocl_err(&pdev->dev, "Invalid intr %d, user start %d, max %d",
+ intr, xdma->start_user_intr, xdma->max_user_intr);
+ return -EINVAL;
+ }
+
+ if (event_fd >= 0) {
+ trigger = eventfd_ctx_fdget(event_fd);
+ if (IS_ERR(trigger)) {
+ xocl_err(&pdev->dev, "get event ctx failed");
+ return -EFAULT;
+ }
+ }
+
+ mutex_lock(&xdma->user_msix_table_lock);
+ if (xdma->user_msix_table[intr].in_use) {
+ xocl_err(&pdev->dev, "IRQ %d is in use", intr);
+ ret = -EPERM;
+ goto failed;
+ }
+ xdma->user_msix_table[intr].event_ctx = trigger;
+ xdma->user_msix_table[intr].handler = handler;
+ xdma->user_msix_table[intr].arg = arg;
+
+ ret = xdma_user_isr_register(xdma->dma_handle, mask, xdma_isr,
+ &xdma->user_msix_table[intr]);
+ if (ret) {
+ xocl_err(&pdev->dev, "IRQ register failed");
+ xdma->user_msix_table[intr].handler = NULL;
+ xdma->user_msix_table[intr].arg = NULL;
+ xdma->user_msix_table[intr].event_ctx = NULL;
+ goto failed;
+ }
+
+ xdma->user_msix_table[intr].in_use = true;
+
+ mutex_unlock(&xdma->user_msix_table_lock);
+
+
+ return 0;
+
+failed:
+ mutex_unlock(&xdma->user_msix_table_lock);
+ if (!IS_ERR(trigger))
+ eventfd_ctx_put(trigger);
+
+ return ret;
+}
+
+static struct xocl_dma_funcs xdma_ops = {
+ .migrate_bo = xdma_migrate_bo,
+ .ac_chan = acquire_channel,
+ .rel_chan = release_channel,
+ .get_chan_count = get_channel_count,
+ .get_chan_stat = get_channel_stat,
+ .user_intr_register = user_intr_register,
+ .user_intr_config = user_intr_config,
+ .user_intr_unreg = user_intr_unreg,
+ .get_drm_handle = get_drm_handle,
+};
+
+static ssize_t channel_stat_raw_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ u32 i;
+ ssize_t nbytes = 0;
+ struct platform_device *pdev = to_platform_device(dev);
+ u32 chs = get_channel_count(pdev);
+
+ for (i = 0; i < chs; i++) {
+ nbytes += sprintf(buf + nbytes, "%llu %llu\n",
+ get_channel_stat(pdev, i, 0),
+ get_channel_stat(pdev, i, 1));
+ }
+ return nbytes;
+}
+static DEVICE_ATTR_RO(channel_stat_raw);
+
+static struct attribute *xdma_attrs[] = {
+ &dev_attr_channel_stat_raw.attr,
+ NULL,
+};
+
+static struct attribute_group xdma_attr_group = {
+ .attrs = xdma_attrs,
+};
+
+static int set_max_chan(struct platform_device *pdev,
+ struct xocl_xdma *xdma)
+{
+ xdma->channel_usage[0] = devm_kzalloc(&pdev->dev, sizeof(u64) *
+ xdma->channel, GFP_KERNEL);
+ xdma->channel_usage[1] = devm_kzalloc(&pdev->dev, sizeof(u64) *
+ xdma->channel, GFP_KERNEL);
+ if (!xdma->channel_usage[0] || !xdma->channel_usage[1]) {
+ xocl_err(&pdev->dev, "failed to alloc channel usage");
+ return -ENOMEM;
+ }
+
+ sema_init(&xdma->channel_sem[0], xdma->channel);
+ sema_init(&xdma->channel_sem[1], xdma->channel);
+
+ /* Initialize bit mask to represent individual channels */
+ xdma->channel_bitmap[0] = BIT(xdma->channel);
+ xdma->channel_bitmap[0]--;
+ xdma->channel_bitmap[1] = xdma->channel_bitmap[0];
+
+ return 0;
+}
+
+static int xdma_probe(struct platform_device *pdev)
+{
+ struct xocl_xdma *xdma = NULL;
+ int ret = 0;
+ xdev_handle_t xdev;
+
+ xdev = xocl_get_xdev(pdev);
+ BUG_ON(!xdev);
+
+ xdma = devm_kzalloc(&pdev->dev, sizeof(*xdma), GFP_KERNEL);
+ if (!xdma) {
+ ret = -ENOMEM;
+ goto failed;
+ }
+
+ xdma->dma_handle = xdma_device_open(XOCL_MODULE_NAME, XDEV(xdev)->pdev,
+ &xdma->max_user_intr,
+ &xdma->channel, &xdma->channel);
+ if (xdma->dma_handle == NULL) {
+ xocl_err(&pdev->dev, "XDMA Device Open failed");
+ ret = -EIO;
+ goto failed;
+ }
+
+ xdma->user_msix_table = devm_kzalloc(&pdev->dev,
+ xdma->max_user_intr *
+ sizeof(struct xdma_irq), GFP_KERNEL);
+ if (!xdma->user_msix_table) {
+ xocl_err(&pdev->dev, "alloc user_msix_table failed");
+ ret = -ENOMEM;
+ goto failed;
+ }
+
+ ret = set_max_chan(pdev, xdma);
+ if (ret) {
+ xocl_err(&pdev->dev, "Set max channel failed");
+ goto failed;
+ }
+
+ xdma->drm = xocl_drm_init(xdev);
+ if (!xdma->drm) {
+ ret = -EFAULT;
+ xocl_err(&pdev->dev, "failed to init drm mm");
+ goto failed;
+ }
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &xdma_attr_group);
+ if (ret) {
+ xocl_err(&pdev->dev, "create attrs failed: %d", ret);
+ goto failed;
+ }
+
+ mutex_init(&xdma->stat_lock);
+ mutex_init(&xdma->user_msix_table_lock);
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_DMA, &xdma_ops);
+ platform_set_drvdata(pdev, xdma);
+
+ return 0;
+
+failed:
+ if (xdma) {
+ if (xdma->drm)
+ xocl_drm_fini(xdma->drm);
+ if (xdma->dma_handle)
+ xdma_device_close(XDEV(xdev)->pdev, xdma->dma_handle);
+ if (xdma->channel_usage[0])
+ devm_kfree(&pdev->dev, xdma->channel_usage[0]);
+ if (xdma->channel_usage[1])
+ devm_kfree(&pdev->dev, xdma->channel_usage[1]);
+ if (xdma->user_msix_table)
+ devm_kfree(&pdev->dev, xdma->user_msix_table);
+
+ devm_kfree(&pdev->dev, xdma);
+ }
+
+ platform_set_drvdata(pdev, NULL);
+
+ return ret;
+}
+
+static int xdma_remove(struct platform_device *pdev)
+{
+ struct xocl_xdma *xdma = platform_get_drvdata(pdev);
+ xdev_handle_t xdev;
+ struct xdma_irq *irq_entry;
+ int i;
+
+ if (!xdma) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ xdev = xocl_get_xdev(pdev);
+ BUG_ON(!xdev);
+
+ sysfs_remove_group(&pdev->dev.kobj, &xdma_attr_group);
+
+ if (xdma->drm)
+ xocl_drm_fini(xdma->drm);
+ if (xdma->dma_handle)
+ xdma_device_close(XDEV(xdev)->pdev, xdma->dma_handle);
+
+ for (i = 0; i < xdma->max_user_intr; i++) {
+ irq_entry = &xdma->user_msix_table[i];
+ if (irq_entry->in_use) {
+ if (irq_entry->enabled) {
+ xocl_err(&pdev->dev,
+ "ERROR: Interrupt %d is still on", i);
+ }
+ if (!IS_ERR_OR_NULL(irq_entry->event_ctx))
+ eventfd_ctx_put(irq_entry->event_ctx);
+ }
+ }
+
+ if (xdma->channel_usage[0])
+ devm_kfree(&pdev->dev, xdma->channel_usage[0]);
+ if (xdma->channel_usage[1])
+ devm_kfree(&pdev->dev, xdma->channel_usage[1]);
+
+ mutex_destroy(&xdma->stat_lock);
+ mutex_destroy(&xdma->user_msix_table_lock);
+
+ devm_kfree(&pdev->dev, xdma->user_msix_table);
+ platform_set_drvdata(pdev, NULL);
+
+ devm_kfree(&pdev->dev, xdma);
+
+ return 0;
+}
+
+static struct platform_device_id xdma_id_table[] = {
+ { XOCL_XDMA, 0 },
+ { },
+};
+
+static struct platform_driver xdma_driver = {
+ .probe = xdma_probe,
+ .remove = xdma_remove,
+ .driver = {
+ .name = "xocl_xdma",
+ },
+ .id_table = xdma_id_table,
+};
+
+int __init xocl_init_xdma(void)
+{
+ return platform_driver_register(&xdma_driver);
+}
+
+void xocl_fini_xdma(void)
+{
+ return platform_driver_unregister(&xdma_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/xmc.c b/drivers/gpu/drm/xocl/subdev/xmc.c
new file mode 100644
index 000000000000..d9d620ac09b9
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/xmc.c
@@ -0,0 +1,1480 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: chienwei@xilinx.com
+ *
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include "../ert.h"
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+#define MAX_XMC_RETRY 150 //Retry is set to 15s for XMC
+#define MAX_ERT_RETRY 10 //Retry is set to 1s for ERT
+#define RETRY_INTERVAL 100 //100ms
+
+#define MAX_IMAGE_LEN 0x20000
+
+#define XMC_MAGIC_REG 0x0
+#define XMC_VERSION_REG 0x4
+#define XMC_STATUS_REG 0x8
+#define XMC_ERROR_REG 0xC
+#define XMC_FEATURE_REG 0x10
+#define XMC_SENSOR_REG 0x14
+#define XMC_CONTROL_REG 0x18
+#define XMC_STOP_CONFIRM_REG 0x1C
+#define XMC_12V_PEX_REG 0x20
+#define XMC_3V3_PEX_REG 0x2C
+#define XMC_3V3_AUX_REG 0x38
+#define XMC_12V_AUX_REG 0x44
+#define XMC_DDR4_VPP_BTM_REG 0x50
+#define XMC_SYS_5V5_REG 0x5C
+#define XMC_VCC1V2_TOP_REG 0x68
+#define XMC_VCC1V8_REG 0x74
+#define XMC_VCC0V85_REG 0x80
+#define XMC_DDR4_VPP_TOP_REG 0x8C
+#define XMC_MGT0V9AVCC_REG 0x98
+#define XMC_12V_SW_REG 0xA4
+#define XMC_MGTAVTT_REG 0xB0
+#define XMC_VCC1V2_BTM_REG 0xBC
+#define XMC_12V_PEX_I_IN_REG 0xC8
+#define XMC_12V_AUX_I_IN_REG 0xD4
+#define XMC_VCCINT_V_REG 0xE0
+#define XMC_VCCINT_I_REG 0xEC
+#define XMC_FPGA_TEMP 0xF8
+#define XMC_FAN_TEMP_REG 0x104
+#define XMC_DIMM_TEMP0_REG 0x110
+#define XMC_DIMM_TEMP1_REG 0x11C
+#define XMC_DIMM_TEMP2_REG 0x128
+#define XMC_DIMM_TEMP3_REG 0x134
+#define XMC_FAN_SPEED_REG 0x164
+#define XMC_SE98_TEMP0_REG 0x140
+#define XMC_SE98_TEMP1_REG 0x14C
+#define XMC_SE98_TEMP2_REG 0x158
+#define XMC_CAGE_TEMP0_REG 0x170
+#define XMC_CAGE_TEMP1_REG 0x17C
+#define XMC_CAGE_TEMP2_REG 0x188
+#define XMC_CAGE_TEMP3_REG 0x194
+#define XMC_SNSR_CHKSUM_REG 0x1A4
+#define XMC_SNSR_FLAGS_REG 0x1A8
+#define XMC_HOST_MSG_OFFSET_REG 0x300
+#define XMC_HOST_MSG_ERROR_REG 0x304
+#define XMC_HOST_MSG_HEADER_REG 0x308
+
+
+#define VALID_ID 0x74736574
+
+#define GPIO_RESET 0x0
+#define GPIO_ENABLED 0x1
+
+#define SELF_JUMP(ins) (((ins) & 0xfc00ffff) == 0xb8000000)
+#define XMC_PRIVILEGED(xmc) ((xmc)->base_addrs[0] != NULL)
+
+enum ctl_mask {
+ CTL_MASK_CLEAR_POW = 0x1,
+ CTL_MASK_CLEAR_ERR = 0x2,
+ CTL_MASK_PAUSE = 0x4,
+ CTL_MASK_STOP = 0x8,
+};
+
+enum status_mask {
+ STATUS_MASK_INIT_DONE = 0x1,
+ STATUS_MASK_STOPPED = 0x2,
+ STATUS_MASK_PAUSE = 0x4,
+};
+
+enum cap_mask {
+ CAP_MASK_PM = 0x1,
+};
+
+enum {
+ XMC_STATE_UNKNOWN,
+ XMC_STATE_ENABLED,
+ XMC_STATE_RESET,
+ XMC_STATE_STOPPED,
+ XMC_STATE_ERROR
+};
+
+enum {
+ IO_REG,
+ IO_GPIO,
+ IO_IMAGE_MGMT,
+ IO_IMAGE_SCHED,
+ IO_CQ,
+ NUM_IOADDR
+};
+
+enum {
+ VOLTAGE_MAX,
+ VOLTAGE_AVG,
+ VOLTAGE_INS,
+};
+
+#define READ_REG32(xmc, off) \
+ XOCL_READ_REG32(xmc->base_addrs[IO_REG] + off)
+#define WRITE_REG32(xmc, val, off) \
+ XOCL_WRITE_REG32(val, xmc->base_addrs[IO_REG] + off)
+
+#define READ_GPIO(xmc, off) \
+ XOCL_READ_REG32(xmc->base_addrs[IO_GPIO] + off)
+#define WRITE_GPIO(xmc, val, off) \
+ XOCL_WRITE_REG32(val, xmc->base_addrs[IO_GPIO] + off)
+
+#define READ_IMAGE_MGMT(xmc, off) \
+ XOCL_READ_REG32(xmc->base_addrs[IO_IMAGE_MGMT] + off)
+
+#define READ_IMAGE_SCHED(xmc, off) \
+ XOCL_READ_REG32(xmc->base_addrs[IO_IMAGE_SCHED] + off)
+
+#define COPY_MGMT(xmc, buf, len) \
+ xocl_memcpy_toio(xmc->base_addrs[IO_IMAGE_MGMT], buf, len)
+#define COPY_SCHE(xmc, buf, len) \
+ xocl_memcpy_toio(xmc->base_addrs[IO_IMAGE_SCHED], buf, len)
+
+struct xocl_xmc {
+ struct platform_device *pdev;
+ void __iomem *base_addrs[NUM_IOADDR];
+
+ struct device *hwmon_dev;
+ bool enabled;
+ u32 state;
+ u32 cap;
+ struct mutex xmc_lock;
+
+ char *sche_binary;
+ u32 sche_binary_length;
+ char *mgmt_binary;
+ u32 mgmt_binary_length;
+};
+
+
+static int load_xmc(struct xocl_xmc *xmc);
+static int stop_xmc(struct platform_device *pdev);
+
+static void xmc_read_from_peer(struct platform_device *pdev, enum data_kind kind, void *resp, size_t resplen)
+{
+ struct mailbox_subdev_peer subdev_peer = {0};
+ size_t data_len = sizeof(struct mailbox_subdev_peer);
+ struct mailbox_req *mb_req = NULL;
+ size_t reqlen = sizeof(struct mailbox_req) + data_len;
+
+ mb_req = vmalloc(reqlen);
+ if (!mb_req)
+ return;
+
+ mb_req->req = MAILBOX_REQ_PEER_DATA;
+
+ subdev_peer.kind = kind;
+ memcpy(mb_req->data, &subdev_peer, data_len);
+
+ (void) xocl_peer_request(XOCL_PL_DEV_TO_XDEV(pdev),
+ mb_req, reqlen, resp, &resplen, NULL, NULL);
+ vfree(mb_req);
+}
+
+/* sysfs support */
+static void safe_read32(struct xocl_xmc *xmc, u32 reg, u32 *val)
+{
+ mutex_lock(&xmc->xmc_lock);
+ if (xmc->enabled && xmc->state == XMC_STATE_ENABLED)
+ *val = READ_REG32(xmc, reg);
+ else
+ *val = 0;
+
+ mutex_unlock(&xmc->xmc_lock);
+}
+
+static void safe_write32(struct xocl_xmc *xmc, u32 reg, u32 val)
+{
+ mutex_lock(&xmc->xmc_lock);
+ if (xmc->enabled && xmc->state == XMC_STATE_ENABLED)
+ WRITE_REG32(xmc, val, reg);
+
+ mutex_unlock(&xmc->xmc_lock);
+}
+
+static void safe_read_from_peer(struct xocl_xmc *xmc, struct platform_device *pdev, enum data_kind kind, u32 *val)
+{
+ mutex_lock(&xmc->xmc_lock);
+ if (xmc->enabled)
+ xmc_read_from_peer(pdev, kind, val, sizeof(u32));
+ else
+ *val = 0;
+
+ mutex_unlock(&xmc->xmc_lock);
+}
+
+static int xmc_get_data(struct platform_device *pdev, enum data_kind kind)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(pdev);
+ int val;
+
+ if (XMC_PRIVILEGED(xmc)) {
+ switch (kind) {
+ case VOL_12V_PEX:
+ safe_read32(xmc, XMC_12V_PEX_REG + sizeof(u32)*VOLTAGE_INS, &val);
+ break;
+ default:
+ break;
+ }
+ }
+ return val;
+}
+
+static ssize_t xmc_12v_pex_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 pes_val;
+
+ if (XMC_PRIVILEGED(xmc))
+ safe_read32(xmc, XMC_12V_PEX_REG+sizeof(u32)*VOLTAGE_INS, &pes_val);
+ else
+ safe_read_from_peer(xmc, to_platform_device(dev), VOL_12V_PEX, &pes_val);
+
+ return sprintf(buf, "%d\n", pes_val);
+}
+static DEVICE_ATTR_RO(xmc_12v_pex_vol);
+
+static ssize_t xmc_12v_aux_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_12V_AUX_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_12v_aux_vol);
+
+static ssize_t xmc_12v_pex_curr_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 pes_val;
+
+ safe_read32(xmc, XMC_12V_PEX_I_IN_REG+sizeof(u32)*VOLTAGE_INS, &pes_val);
+
+ return sprintf(buf, "%d\n", pes_val);
+}
+static DEVICE_ATTR_RO(xmc_12v_pex_curr);
+
+static ssize_t xmc_12v_aux_curr_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_12V_AUX_I_IN_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_12v_aux_curr);
+
+static ssize_t xmc_3v3_pex_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_3V3_PEX_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_3v3_pex_vol);
+
+static ssize_t xmc_3v3_aux_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_3V3_AUX_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_3v3_aux_vol);
+
+static ssize_t xmc_ddr_vpp_btm_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DDR4_VPP_BTM_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_ddr_vpp_btm);
+
+static ssize_t xmc_sys_5v5_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SYS_5V5_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_sys_5v5);
+
+static ssize_t xmc_1v2_top_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCC1V2_TOP_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_1v2_top);
+
+static ssize_t xmc_1v8_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCC1V8_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_1v8);
+
+static ssize_t xmc_0v85_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCC0V85_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_0v85);
+
+static ssize_t xmc_ddr_vpp_top_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DDR4_VPP_TOP_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_ddr_vpp_top);
+
+
+static ssize_t xmc_mgt0v9avcc_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_MGT0V9AVCC_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_mgt0v9avcc);
+
+static ssize_t xmc_12v_sw_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_12V_SW_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_12v_sw);
+
+
+static ssize_t xmc_mgtavtt_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_MGTAVTT_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_mgtavtt);
+
+static ssize_t xmc_vcc1v2_btm_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCC1V2_BTM_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_vcc1v2_btm);
+
+static ssize_t xmc_vccint_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCCINT_V_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_vccint_vol);
+
+static ssize_t xmc_vccint_curr_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCCINT_I_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_vccint_curr);
+
+static ssize_t xmc_se98_temp0_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SE98_TEMP0_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_se98_temp0);
+
+static ssize_t xmc_se98_temp1_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SE98_TEMP1_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_se98_temp1);
+
+static ssize_t xmc_se98_temp2_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SE98_TEMP2_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_se98_temp2);
+
+static ssize_t xmc_fpga_temp_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_FPGA_TEMP, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_fpga_temp);
+
+static ssize_t xmc_fan_temp_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_FAN_TEMP_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_fan_temp);
+
+static ssize_t xmc_fan_rpm_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_FAN_SPEED_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_fan_rpm);
+
+
+static ssize_t xmc_dimm_temp0_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DIMM_TEMP0_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_dimm_temp0);
+
+static ssize_t xmc_dimm_temp1_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DIMM_TEMP1_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_dimm_temp1);
+
+static ssize_t xmc_dimm_temp2_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DIMM_TEMP2_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_dimm_temp2);
+
+static ssize_t xmc_dimm_temp3_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DIMM_TEMP3_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_dimm_temp3);
+
+
+static ssize_t xmc_cage_temp0_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_CAGE_TEMP0_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_cage_temp0);
+
+static ssize_t xmc_cage_temp1_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_CAGE_TEMP1_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_cage_temp1);
+
+static ssize_t xmc_cage_temp2_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_CAGE_TEMP2_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_cage_temp2);
+
+static ssize_t xmc_cage_temp3_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_CAGE_TEMP3_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_cage_temp3);
+
+
+static ssize_t version_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_VERSION_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(version);
+
+static ssize_t sensor_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_SENSOR_REG, &val);
+
+ return sprintf(buf, "0x%04x\n", val);
+}
+static DEVICE_ATTR_RO(sensor);
+
+
+static ssize_t id_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_MAGIC_REG, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(id);
+
+static ssize_t status_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_STATUS_REG, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(status);
+
+static ssize_t error_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_ERROR_REG, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(error);
+
+static ssize_t capability_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_FEATURE_REG, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(capability);
+
+static ssize_t power_checksum_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_SNSR_CHKSUM_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(power_checksum);
+
+static ssize_t pause_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_CONTROL_REG, &val);
+
+ return sprintf(buf, "%d\n", !!(val & CTL_MASK_PAUSE));
+}
+
+static ssize_t pause_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val > 1)
+ return -EINVAL;
+
+ val = val ? CTL_MASK_PAUSE : 0;
+ safe_write32(xmc, XMC_CONTROL_REG, val);
+
+ return count;
+}
+static DEVICE_ATTR_RW(pause);
+
+static ssize_t reset_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val > 1)
+ return -EINVAL;
+
+ if (val)
+ load_xmc(xmc);
+
+ return count;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t power_flag_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SNSR_FLAGS_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(power_flag);
+
+static ssize_t host_msg_offset_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_HOST_MSG_OFFSET_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(host_msg_offset);
+
+static ssize_t host_msg_error_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_HOST_MSG_ERROR_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(host_msg_error);
+
+static ssize_t host_msg_header_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_HOST_MSG_HEADER_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(host_msg_header);
+
+
+
+static int get_temp_by_m_tag(struct xocl_xmc *xmc, char *m_tag)
+{
+
+ /**
+ * m_tag get from xclbin must follow this format
+ * DDR[0] or bank1
+ * we check the index in m_tag to decide which temperature
+ * to get from XMC IP base address
+ */
+ char *start = NULL, *left_parentness = NULL, *right_parentness = NULL;
+ long idx;
+ int ret = 0, digit_len = 0;
+ char temp[4];
+
+ if (!xmc)
+ return -ENODEV;
+
+
+ if (!strncmp(m_tag, "bank", 4)) {
+ start = m_tag;
+ // bank0, no left parentness
+ left_parentness = m_tag+3;
+ right_parentness = m_tag+strlen(m_tag)+1;
+ digit_len = right_parentness-(2+left_parentness);
+ } else if (!strncmp(m_tag, "DDR", 3)) {
+
+ start = m_tag;
+ left_parentness = strstr(m_tag, "[");
+ right_parentness = strstr(m_tag, "]");
+ digit_len = right_parentness-(1+left_parentness);
+ }
+
+ if (!left_parentness || !right_parentness)
+ return ret;
+
+ if (!strncmp(m_tag, "DDR", left_parentness-start) || !strncmp(m_tag, "bank", left_parentness-start)) {
+
+ strncpy(temp, left_parentness+1, digit_len);
+ //assumption, temperature won't higher than 3 digits, or the temp[digit_len] should be a null character
+ temp[digit_len] = '\0';
+ //convert to signed long, decimal base
+ if (kstrtol(temp, 10, &idx) == 0 && idx < 4 && idx >= 0)
+ safe_read32(xmc, XMC_DIMM_TEMP0_REG + (3*sizeof(int32_t)) * idx +
+ sizeof(u32)*VOLTAGE_INS, &ret);
+ else
+ ret = 0;
+ }
+
+ return ret;
+
+}
+
+static struct attribute *xmc_attrs[] = {
+ &dev_attr_version.attr,
+ &dev_attr_id.attr,
+ &dev_attr_status.attr,
+ &dev_attr_sensor.attr,
+ &dev_attr_error.attr,
+ &dev_attr_capability.attr,
+ &dev_attr_power_checksum.attr,
+ &dev_attr_xmc_12v_pex_vol.attr,
+ &dev_attr_xmc_12v_aux_vol.attr,
+ &dev_attr_xmc_12v_pex_curr.attr,
+ &dev_attr_xmc_12v_aux_curr.attr,
+ &dev_attr_xmc_3v3_pex_vol.attr,
+ &dev_attr_xmc_3v3_aux_vol.attr,
+ &dev_attr_xmc_ddr_vpp_btm.attr,
+ &dev_attr_xmc_sys_5v5.attr,
+ &dev_attr_xmc_1v2_top.attr,
+ &dev_attr_xmc_1v8.attr,
+ &dev_attr_xmc_0v85.attr,
+ &dev_attr_xmc_ddr_vpp_top.attr,
+ &dev_attr_xmc_mgt0v9avcc.attr,
+ &dev_attr_xmc_12v_sw.attr,
+ &dev_attr_xmc_mgtavtt.attr,
+ &dev_attr_xmc_vcc1v2_btm.attr,
+ &dev_attr_xmc_fpga_temp.attr,
+ &dev_attr_xmc_fan_temp.attr,
+ &dev_attr_xmc_fan_rpm.attr,
+ &dev_attr_xmc_dimm_temp0.attr,
+ &dev_attr_xmc_dimm_temp1.attr,
+ &dev_attr_xmc_dimm_temp2.attr,
+ &dev_attr_xmc_dimm_temp3.attr,
+ &dev_attr_xmc_vccint_vol.attr,
+ &dev_attr_xmc_vccint_curr.attr,
+ &dev_attr_xmc_se98_temp0.attr,
+ &dev_attr_xmc_se98_temp1.attr,
+ &dev_attr_xmc_se98_temp2.attr,
+ &dev_attr_xmc_cage_temp0.attr,
+ &dev_attr_xmc_cage_temp1.attr,
+ &dev_attr_xmc_cage_temp2.attr,
+ &dev_attr_xmc_cage_temp3.attr,
+ &dev_attr_pause.attr,
+ &dev_attr_reset.attr,
+ &dev_attr_power_flag.attr,
+ &dev_attr_host_msg_offset.attr,
+ &dev_attr_host_msg_error.attr,
+ &dev_attr_host_msg_header.attr,
+ NULL,
+};
+
+
+static ssize_t read_temp_by_mem_topology(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ u32 nread = 0;
+ size_t size = 0;
+ u32 i;
+ struct mem_topology *memtopo = NULL;
+ struct xocl_xmc *xmc;
+ uint32_t temp[MAX_M_COUNT] = {0};
+ struct xclmgmt_dev *lro;
+
+ //xocl_icap_lock_bitstream
+ lro = (struct xclmgmt_dev *)dev_get_drvdata(container_of(kobj, struct device, kobj)->parent);
+ xmc = (struct xocl_xmc *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ memtopo = (struct mem_topology *)xocl_icap_get_data(lro, MEMTOPO_AXLF);
+
+ if (!memtopo)
+ return 0;
+
+ size = sizeof(u32)*(memtopo->m_count);
+
+ if (offset >= size)
+ return 0;
+
+ for (i = 0; i < memtopo->m_count; ++i)
+ *(temp+i) = get_temp_by_m_tag(xmc, memtopo->m_mem_data[i].m_tag);
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, temp, nread);
+ //xocl_icap_unlock_bitstream
+ return nread;
+}
+
+static struct bin_attribute bin_dimm_temp_by_mem_topology_attr = {
+ .attr = {
+ .name = "temp_by_mem_topology",
+ .mode = 0444
+ },
+ .read = read_temp_by_mem_topology,
+ .write = NULL,
+ .size = 0
+};
+
+static struct bin_attribute *xmc_bin_attrs[] = {
+ &bin_dimm_temp_by_mem_topology_attr,
+ NULL,
+};
+
+static struct attribute_group xmc_attr_group = {
+ .attrs = xmc_attrs,
+ .bin_attrs = xmc_bin_attrs,
+};
+static ssize_t show_mb_pw(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_12V_PEX_REG + attr->index * sizeof(u32), &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+
+static SENSOR_DEVICE_ATTR(curr1_highest, 0444, show_mb_pw, NULL, 0);
+static SENSOR_DEVICE_ATTR(curr1_average, 0444, show_mb_pw, NULL, 1);
+static SENSOR_DEVICE_ATTR(curr1_input, 0444, show_mb_pw, NULL, 2);
+static SENSOR_DEVICE_ATTR(curr2_highest, 0444, show_mb_pw, NULL, 3);
+static SENSOR_DEVICE_ATTR(curr2_average, 0444, show_mb_pw, NULL, 4);
+static SENSOR_DEVICE_ATTR(curr2_input, 0444, show_mb_pw, NULL, 5);
+static SENSOR_DEVICE_ATTR(curr3_highest, 0444, show_mb_pw, NULL, 6);
+static SENSOR_DEVICE_ATTR(curr3_average, 0444, show_mb_pw, NULL, 7);
+static SENSOR_DEVICE_ATTR(curr3_input, 0444, show_mb_pw, NULL, 8);
+static SENSOR_DEVICE_ATTR(curr4_highest, 0444, show_mb_pw, NULL, 9);
+static SENSOR_DEVICE_ATTR(curr4_average, 0444, show_mb_pw, NULL, 10);
+static SENSOR_DEVICE_ATTR(curr4_input, 0444, show_mb_pw, NULL, 11);
+static SENSOR_DEVICE_ATTR(curr5_highest, 0444, show_mb_pw, NULL, 12);
+static SENSOR_DEVICE_ATTR(curr5_average, 0444, show_mb_pw, NULL, 13);
+static SENSOR_DEVICE_ATTR(curr5_input, 0444, show_mb_pw, NULL, 14);
+static SENSOR_DEVICE_ATTR(curr6_highest, 0444, show_mb_pw, NULL, 15);
+static SENSOR_DEVICE_ATTR(curr6_average, 0444, show_mb_pw, NULL, 16);
+static SENSOR_DEVICE_ATTR(curr6_input, 0444, show_mb_pw, NULL, 17);
+
+static struct attribute *hwmon_xmc_attributes[] = {
+ &sensor_dev_attr_curr1_highest.dev_attr.attr,
+ &sensor_dev_attr_curr1_average.dev_attr.attr,
+ &sensor_dev_attr_curr1_input.dev_attr.attr,
+ &sensor_dev_attr_curr2_highest.dev_attr.attr,
+ &sensor_dev_attr_curr2_average.dev_attr.attr,
+ &sensor_dev_attr_curr2_input.dev_attr.attr,
+ &sensor_dev_attr_curr3_highest.dev_attr.attr,
+ &sensor_dev_attr_curr3_average.dev_attr.attr,
+ &sensor_dev_attr_curr3_input.dev_attr.attr,
+ &sensor_dev_attr_curr4_highest.dev_attr.attr,
+ &sensor_dev_attr_curr4_average.dev_attr.attr,
+ &sensor_dev_attr_curr4_input.dev_attr.attr,
+ &sensor_dev_attr_curr5_highest.dev_attr.attr,
+ &sensor_dev_attr_curr5_average.dev_attr.attr,
+ &sensor_dev_attr_curr5_input.dev_attr.attr,
+ &sensor_dev_attr_curr6_highest.dev_attr.attr,
+ &sensor_dev_attr_curr6_average.dev_attr.attr,
+ &sensor_dev_attr_curr6_input.dev_attr.attr,
+ NULL
+};
+
+static const struct attribute_group hwmon_xmc_attrgroup = {
+ .attrs = hwmon_xmc_attributes,
+};
+
+static ssize_t show_name(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", XCLMGMT_MB_HWMON_NAME);
+}
+
+static struct sensor_device_attribute name_attr =
+ SENSOR_ATTR(name, 0444, show_name, NULL, 0);
+
+static void mgmt_sysfs_destroy_xmc(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+
+ xmc = platform_get_drvdata(pdev);
+
+ if (!xmc->enabled)
+ return;
+
+ if (xmc->hwmon_dev) {
+ device_remove_file(xmc->hwmon_dev, &name_attr.dev_attr);
+ sysfs_remove_group(&xmc->hwmon_dev->kobj,
+ &hwmon_xmc_attrgroup);
+ hwmon_device_unregister(xmc->hwmon_dev);
+ xmc->hwmon_dev = NULL;
+ }
+
+ sysfs_remove_group(&pdev->dev.kobj, &xmc_attr_group);
+}
+
+static int mgmt_sysfs_create_xmc(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ struct xocl_dev_core *core;
+ int err;
+
+ xmc = platform_get_drvdata(pdev);
+ core = XDEV(xocl_get_xdev(pdev));
+
+ if (!xmc->enabled)
+ return 0;
+
+ err = sysfs_create_group(&pdev->dev.kobj, &xmc_attr_group);
+ if (err) {
+ xocl_err(&pdev->dev, "create xmc attrs failed: 0x%x", err);
+ goto create_attr_failed;
+ }
+ xmc->hwmon_dev = hwmon_device_register(&core->pdev->dev);
+ if (IS_ERR(xmc->hwmon_dev)) {
+ err = PTR_ERR(xmc->hwmon_dev);
+ xocl_err(&pdev->dev, "register xmc hwmon failed: 0x%x", err);
+ goto hwmon_reg_failed;
+ }
+
+ dev_set_drvdata(xmc->hwmon_dev, xmc);
+
+ err = device_create_file(xmc->hwmon_dev, &name_attr.dev_attr);
+ if (err) {
+ xocl_err(&pdev->dev, "create attr name failed: 0x%x", err);
+ goto create_name_failed;
+ }
+
+ err = sysfs_create_group(&xmc->hwmon_dev->kobj,
+ &hwmon_xmc_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create pw group failed: 0x%x", err);
+ goto create_pw_failed;
+ }
+
+ return 0;
+
+create_pw_failed:
+ device_remove_file(xmc->hwmon_dev, &name_attr.dev_attr);
+create_name_failed:
+ hwmon_device_unregister(xmc->hwmon_dev);
+ xmc->hwmon_dev = NULL;
+hwmon_reg_failed:
+ sysfs_remove_group(&pdev->dev.kobj, &xmc_attr_group);
+create_attr_failed:
+ return err;
+}
+
+static int stop_xmc_nolock(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ int retry = 0;
+ u32 reg_val = 0;
+ void *xdev_hdl;
+
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return -ENODEV;
+ else if (!xmc->enabled)
+ return -ENODEV;
+
+ xdev_hdl = xocl_get_xdev(xmc->pdev);
+
+ reg_val = READ_GPIO(xmc, 0);
+ xocl_info(&xmc->pdev->dev, "MB Reset GPIO 0x%x", reg_val);
+
+ //Stop XMC and ERT if its currently running
+ if (reg_val == GPIO_ENABLED) {
+ xocl_info(&xmc->pdev->dev,
+ "XMC info, version 0x%x, status 0x%x, id 0x%x",
+ READ_REG32(xmc, XMC_VERSION_REG),
+ READ_REG32(xmc, XMC_STATUS_REG),
+ READ_REG32(xmc, XMC_MAGIC_REG));
+
+ reg_val = READ_REG32(xmc, XMC_STATUS_REG);
+ if (!(reg_val & STATUS_MASK_STOPPED)) {
+ xocl_info(&xmc->pdev->dev, "Stopping XMC...");
+ WRITE_REG32(xmc, CTL_MASK_STOP, XMC_CONTROL_REG);
+ WRITE_REG32(xmc, 1, XMC_STOP_CONFIRM_REG);
+ }
+ //Need to check if ERT is loaded before we attempt to stop it
+ if (!SELF_JUMP(READ_IMAGE_SCHED(xmc, 0))) {
+ reg_val = XOCL_READ_REG32(xmc->base_addrs[IO_CQ]);
+ if (!(reg_val & ERT_STOP_ACK)) {
+ xocl_info(&xmc->pdev->dev, "Stopping scheduler...");
+ XOCL_WRITE_REG32(ERT_STOP_CMD, xmc->base_addrs[IO_CQ]);
+ }
+ }
+
+ retry = 0;
+ while (retry++ < MAX_XMC_RETRY &&
+ !(READ_REG32(xmc, XMC_STATUS_REG) & STATUS_MASK_STOPPED))
+ msleep(RETRY_INTERVAL);
+
+ //Wait for XMC to stop and then check that ERT has also finished
+ if (retry >= MAX_XMC_RETRY) {
+ xocl_err(&xmc->pdev->dev,
+ "Failed to stop XMC");
+ xocl_err(&xmc->pdev->dev,
+ "XMC Error Reg 0x%x",
+ READ_REG32(xmc, XMC_ERROR_REG));
+ xmc->state = XMC_STATE_ERROR;
+ return -ETIMEDOUT;
+ } else if (!SELF_JUMP(READ_IMAGE_SCHED(xmc, 0)) &&
+ !(XOCL_READ_REG32(xmc->base_addrs[IO_CQ]) & ERT_STOP_ACK)) {
+ while (retry++ < MAX_ERT_RETRY &&
+ !(XOCL_READ_REG32(xmc->base_addrs[IO_CQ]) & ERT_STOP_ACK))
+ msleep(RETRY_INTERVAL);
+ if (retry >= MAX_ERT_RETRY) {
+ xocl_err(&xmc->pdev->dev,
+ "Failed to stop sched");
+ xocl_err(&xmc->pdev->dev,
+ "Scheduler CQ status 0x%x",
+ XOCL_READ_REG32(xmc->base_addrs[IO_CQ]));
+ //We don't exit if ERT doesn't stop since it can hang due to bad kernel
+ //xmc->state = XMC_STATE_ERROR;
+ //return -ETIMEDOUT;
+ }
+ }
+
+ xocl_info(&xmc->pdev->dev, "XMC/sched Stopped, retry %d",
+ retry);
+ }
+
+ // Hold XMC in reset now that its safely stopped
+ xocl_info(&xmc->pdev->dev,
+ "XMC info, version 0x%x, status 0x%x, id 0x%x",
+ READ_REG32(xmc, XMC_VERSION_REG),
+ READ_REG32(xmc, XMC_STATUS_REG),
+ READ_REG32(xmc, XMC_MAGIC_REG));
+ WRITE_GPIO(xmc, GPIO_RESET, 0);
+ xmc->state = XMC_STATE_RESET;
+ reg_val = READ_GPIO(xmc, 0);
+ xocl_info(&xmc->pdev->dev, "MB Reset GPIO 0x%x", reg_val);
+ if (reg_val != GPIO_RESET) {
+ //Shouldnt make it here but if we do then exit
+ xmc->state = XMC_STATE_ERROR;
+ return -EIO;
+ }
+
+ return 0;
+}
+static int stop_xmc(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ int ret = 0;
+ void *xdev_hdl;
+
+ xocl_info(&pdev->dev, "Stop Microblaze...");
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return -ENODEV;
+ else if (!xmc->enabled)
+ return -ENODEV;
+
+ xdev_hdl = xocl_get_xdev(xmc->pdev);
+
+ mutex_lock(&xmc->xmc_lock);
+ ret = stop_xmc_nolock(pdev);
+ mutex_unlock(&xmc->xmc_lock);
+
+ return ret;
+}
+
+static int load_xmc(struct xocl_xmc *xmc)
+{
+ int retry = 0;
+ u32 reg_val = 0;
+ int ret = 0;
+ void *xdev_hdl;
+
+ if (!xmc->enabled)
+ return -ENODEV;
+
+ mutex_lock(&xmc->xmc_lock);
+
+ /* Stop XMC first */
+ ret = stop_xmc_nolock(xmc->pdev);
+ if (ret != 0)
+ goto out;
+
+ xdev_hdl = xocl_get_xdev(xmc->pdev);
+
+ /* Load XMC and ERT Image */
+ if (xocl_mb_mgmt_on(xdev_hdl)) {
+ xocl_info(&xmc->pdev->dev, "Copying XMC image len %d",
+ xmc->mgmt_binary_length);
+ COPY_MGMT(xmc, xmc->mgmt_binary, xmc->mgmt_binary_length);
+ }
+
+ if (xocl_mb_sched_on(xdev_hdl)) {
+ xocl_info(&xmc->pdev->dev, "Copying scheduler image len %d",
+ xmc->sche_binary_length);
+ COPY_SCHE(xmc, xmc->sche_binary, xmc->sche_binary_length);
+ }
+
+ /* Take XMC and ERT out of reset */
+ WRITE_GPIO(xmc, GPIO_ENABLED, 0);
+ reg_val = READ_GPIO(xmc, 0);
+ xocl_info(&xmc->pdev->dev, "MB Reset GPIO 0x%x", reg_val);
+ if (reg_val != GPIO_ENABLED) {
+ //Shouldnt make it here but if we do then exit
+ xmc->state = XMC_STATE_ERROR;
+ goto out;
+ }
+
+ /* Wait for XMC to start
+ * Note that ERT will start long before XMC so we don't check anything
+ */
+ reg_val = READ_REG32(xmc, XMC_STATUS_REG);
+ if (!(reg_val & STATUS_MASK_INIT_DONE)) {
+ xocl_info(&xmc->pdev->dev, "Waiting for XMC to finish init...");
+ retry = 0;
+ while (retry++ < MAX_XMC_RETRY &&
+ !(READ_REG32(xmc, XMC_STATUS_REG) & STATUS_MASK_INIT_DONE))
+ msleep(RETRY_INTERVAL);
+ if (retry >= MAX_XMC_RETRY) {
+ xocl_err(&xmc->pdev->dev,
+ "XMC did not finish init sequence!");
+ xocl_err(&xmc->pdev->dev,
+ "Error Reg 0x%x",
+ READ_REG32(xmc, XMC_ERROR_REG));
+ xocl_err(&xmc->pdev->dev,
+ "Status Reg 0x%x",
+ READ_REG32(xmc, XMC_STATUS_REG));
+ ret = -ETIMEDOUT;
+ xmc->state = XMC_STATE_ERROR;
+ goto out;
+ }
+ }
+ xocl_info(&xmc->pdev->dev, "XMC and scheduler Enabled, retry %d",
+ retry);
+ xocl_info(&xmc->pdev->dev,
+ "XMC info, version 0x%x, status 0x%x, id 0x%x",
+ READ_REG32(xmc, XMC_VERSION_REG),
+ READ_REG32(xmc, XMC_STATUS_REG),
+ READ_REG32(xmc, XMC_MAGIC_REG));
+ xmc->state = XMC_STATE_ENABLED;
+
+ xmc->cap = READ_REG32(xmc, XMC_FEATURE_REG);
+out:
+ mutex_unlock(&xmc->xmc_lock);
+
+ return ret;
+}
+
+static void xmc_reset(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+
+ xocl_info(&pdev->dev, "Reset Microblaze...");
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return;
+
+ load_xmc(xmc);
+}
+
+static int load_mgmt_image(struct platform_device *pdev, const char *image,
+ u32 len)
+{
+ struct xocl_xmc *xmc;
+ char *binary;
+
+ if (len > MAX_IMAGE_LEN)
+ return -EINVAL;
+
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return -EINVAL;
+
+ binary = xmc->mgmt_binary;
+ xmc->mgmt_binary = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
+ if (!xmc->mgmt_binary)
+ return -ENOMEM;
+
+ if (binary)
+ devm_kfree(&pdev->dev, binary);
+ memcpy(xmc->mgmt_binary, image, len);
+ xmc->mgmt_binary_length = len;
+
+ return 0;
+}
+
+static int load_sche_image(struct platform_device *pdev, const char *image,
+ u32 len)
+{
+ struct xocl_xmc *xmc;
+ char *binary = NULL;
+
+ if (len > MAX_IMAGE_LEN)
+ return -EINVAL;
+
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return -EINVAL;
+
+ binary = xmc->sche_binary;
+ xmc->sche_binary = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
+ if (!xmc->sche_binary)
+ return -ENOMEM;
+
+ if (binary)
+ devm_kfree(&pdev->dev, binary);
+ memcpy(xmc->sche_binary, image, len);
+ xmc->sche_binary_length = len;
+
+ return 0;
+}
+
+static struct xocl_mb_funcs xmc_ops = {
+ .load_mgmt_image = load_mgmt_image,
+ .load_sche_image = load_sche_image,
+ .reset = xmc_reset,
+ .stop = stop_xmc,
+ .get_data = xmc_get_data,
+};
+
+static int xmc_remove(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ int i;
+
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return 0;
+
+ if (xmc->mgmt_binary)
+ devm_kfree(&pdev->dev, xmc->mgmt_binary);
+ if (xmc->sche_binary)
+ devm_kfree(&pdev->dev, xmc->sche_binary);
+
+ mgmt_sysfs_destroy_xmc(pdev);
+
+ for (i = 0; i < NUM_IOADDR; i++) {
+ if (xmc->base_addrs[i])
+ iounmap(xmc->base_addrs[i]);
+ }
+
+ mutex_destroy(&xmc->xmc_lock);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, xmc);
+
+ return 0;
+}
+
+static int xmc_probe(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ struct resource *res;
+ void *xdev_hdl;
+ int i, err;
+
+ xmc = devm_kzalloc(&pdev->dev, sizeof(*xmc), GFP_KERNEL);
+ if (!xmc) {
+ xocl_err(&pdev->dev, "out of memory");
+ return -ENOMEM;
+ }
+
+ xmc->pdev = pdev;
+ platform_set_drvdata(pdev, xmc);
+
+ xdev_hdl = xocl_get_xdev(pdev);
+ if (xocl_mb_mgmt_on(xdev_hdl) || xocl_mb_sched_on(xdev_hdl)) {
+ xocl_info(&pdev->dev, "Microblaze is supported.");
+ xmc->enabled = true;
+ } else {
+ xocl_err(&pdev->dev, "Microblaze is not supported.");
+ devm_kfree(&pdev->dev, xmc);
+ platform_set_drvdata(pdev, NULL);
+ return 0;
+ }
+
+ for (i = 0; i < NUM_IOADDR; i++) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+ if (res) {
+ xocl_info(&pdev->dev, "IO start: 0x%llx, end: 0x%llx",
+ res->start, res->end);
+ xmc->base_addrs[i] =
+ ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!xmc->base_addrs[i]) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+ } else
+ break;
+ }
+
+ err = mgmt_sysfs_create_xmc(pdev);
+ if (err) {
+ xocl_err(&pdev->dev, "Create sysfs failed, err %d", err);
+ goto failed;
+ }
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_XMC, &xmc_ops);
+
+ mutex_init(&xmc->xmc_lock);
+
+ return 0;
+
+failed:
+ xmc_remove(pdev);
+ return err;
+}
+
+struct platform_device_id xmc_id_table[] = {
+ { XOCL_XMC, 0 },
+ { },
+};
+
+static struct platform_driver xmc_driver = {
+ .probe = xmc_probe,
+ .remove = xmc_remove,
+ .driver = {
+ .name = XOCL_XMC,
+ },
+ .id_table = xmc_id_table,
+};
+
+int __init xocl_init_xmc(void)
+{
+ return platform_driver_register(&xmc_driver);
+}
+
+void xocl_fini_xmc(void)
+{
+ platform_driver_unregister(&xmc_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/xvc.c b/drivers/gpu/drm/xocl/subdev/xvc.c
new file mode 100644
index 000000000000..355dbad30b00
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/xvc.c
@@ -0,0 +1,461 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ *
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/ioctl.h>
+
+#include "../xocl_drv.h"
+
+/* IOCTL interfaces */
+#define XIL_XVC_MAGIC 0x58564344 // "XVCD"
+#define MINOR_PUB_HIGH_BIT 0x00000
+#define MINOR_PRI_HIGH_BIT 0x10000
+#define MINOR_NAME_MASK 0xffffffff
+
+enum xvc_algo_type {
+ XVC_ALGO_NULL,
+ XVC_ALGO_CFG,
+ XVC_ALGO_BAR
+};
+
+struct xil_xvc_ioc {
+ unsigned int opcode;
+ unsigned int length;
+ unsigned char *tms_buf;
+ unsigned char *tdi_buf;
+ unsigned char *tdo_buf;
+};
+
+struct xil_xvc_properties {
+ unsigned int xvc_algo_type;
+ unsigned int config_vsec_id;
+ unsigned int config_vsec_rev;
+ unsigned int bar_index;
+ unsigned int bar_offset;
+};
+
+#define XDMA_IOCXVC _IOWR(XIL_XVC_MAGIC, 1, struct xil_xvc_ioc)
+#define XDMA_RDXVC_PROPS _IOR(XIL_XVC_MAGIC, 2, struct xil_xvc_properties)
+
+#define COMPLETION_LOOP_MAX 100
+
+#define XVC_BAR_LENGTH_REG 0x0
+#define XVC_BAR_TMS_REG 0x4
+#define XVC_BAR_TDI_REG 0x8
+#define XVC_BAR_TDO_REG 0xC
+#define XVC_BAR_CTRL_REG 0x10
+
+#define XVC_DEV_NAME "xvc" SUBDEV_SUFFIX
+
+struct xocl_xvc {
+ void *__iomem base;
+ unsigned int instance;
+ struct cdev *sys_cdev;
+ struct device *sys_device;
+};
+
+static dev_t xvc_dev;
+
+static struct xil_xvc_properties xvc_pci_props;
+
+#ifdef __REG_DEBUG__
+/* SECTION: Function definitions */
+static inline void __write_register(const char *fn, u32 value, void *base,
+ unsigned int off)
+{
+ pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value);
+ iowrite32(value, base + off);
+}
+
+static inline u32 __read_register(const char *fn, void *base, unsigned int off)
+{
+ u32 v = ioread32(base + off);
+
+ pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v);
+ return v;
+}
+#define write_register(v, base, off) __write_register(__func__, v, base, off)
+#define read_register(base, off) __read_register(__func__, base, off)
+
+#else
+#define write_register(v, base, off) iowrite32(v, (base) + (off))
+#define read_register(base, off) ioread32((base) + (off))
+#endif /* #ifdef __REG_DEBUG__ */
+
+
+static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits,
+ u32 *tdo_bits)
+{
+ u32 control;
+ u32 write_reg_data;
+ int count;
+
+ /* set tms bit */
+ write_register(tms_bits, base, XVC_BAR_TMS_REG);
+ /* set tdi bits and shift data out */
+ write_register(tdi_bits, base, XVC_BAR_TDI_REG);
+
+ control = read_register(base, XVC_BAR_CTRL_REG);
+ /* enable shift operation */
+ write_reg_data = control | 0x01;
+ write_register(write_reg_data, base, XVC_BAR_CTRL_REG);
+
+ /* poll for completion */
+ count = COMPLETION_LOOP_MAX;
+ while (count) {
+ /* read control reg to check shift operation completion */
+ control = read_register(base, XVC_BAR_CTRL_REG);
+ if ((control & 0x01) == 0)
+ break;
+
+ count--;
+ }
+
+ if (!count) {
+ pr_warn("XVC bar transaction timed out (0x%0X)\n", control);
+ return -ETIMEDOUT;
+ }
+
+ /* read tdo bits back out */
+ *tdo_bits = read_register(base, XVC_BAR_TDO_REG);
+
+ return 0;
+}
+
+static long xvc_ioctl_helper(struct xocl_xvc *xvc, const void __user *arg)
+{
+ struct xil_xvc_ioc xvc_obj;
+ unsigned int opcode;
+ unsigned int total_bits;
+ unsigned int total_bytes;
+ unsigned int bits, bits_left;
+ unsigned char *buffer = NULL;
+ unsigned char *tms_buf = NULL;
+ unsigned char *tdi_buf = NULL;
+ unsigned char *tdo_buf = NULL;
+ void __iomem *iobase = xvc->base;
+ u32 control_reg_data;
+ u32 write_reg_data;
+ int rv;
+
+ rv = copy_from_user((void *)&xvc_obj, arg,
+ sizeof(struct xil_xvc_ioc));
+ /* anything not copied ? */
+ if (rv) {
+ pr_info("copy_from_user xvc_obj failed: %d.\n", rv);
+ goto cleanup;
+ }
+
+ opcode = xvc_obj.opcode;
+
+ /* Invalid operation type, no operation performed */
+ if (opcode != 0x01 && opcode != 0x02) {
+ pr_info("UNKNOWN opcode 0x%x.\n", opcode);
+ return -EINVAL;
+ }
+
+ total_bits = xvc_obj.length;
+ total_bytes = (total_bits + 7) >> 3;
+
+ buffer = kmalloc(total_bytes * 3, GFP_KERNEL);
+ if (!buffer) {
+ pr_info("OOM %u, op 0x%x, len %u bits, %u bytes.\n",
+ 3 * total_bytes, opcode, total_bits, total_bytes);
+ rv = -ENOMEM;
+ goto cleanup;
+ }
+ tms_buf = buffer;
+ tdi_buf = tms_buf + total_bytes;
+ tdo_buf = tdi_buf + total_bytes;
+
+ rv = copy_from_user((void *)tms_buf, xvc_obj.tms_buf, total_bytes);
+ if (rv) {
+ pr_info("copy tmfs_buf failed: %d/%u.\n", rv, total_bytes);
+ goto cleanup;
+ }
+ rv = copy_from_user((void *)tdi_buf, xvc_obj.tdi_buf, total_bytes);
+ if (rv) {
+ pr_info("copy tdi_buf failed: %d/%u.\n", rv, total_bytes);
+ goto cleanup;
+ }
+
+ // If performing loopback test, set loopback bit (0x02) in control reg
+ if (opcode == 0x02) {
+ control_reg_data = read_register(iobase, XVC_BAR_CTRL_REG);
+ write_reg_data = control_reg_data | 0x02;
+ write_register(write_reg_data, iobase, XVC_BAR_CTRL_REG);
+ }
+
+ /* set length register to 32 initially if more than one
+ * word-transaction is to be done
+ */
+ if (total_bits >= 32)
+ write_register(0x20, iobase, XVC_BAR_LENGTH_REG);
+
+ for (bits = 0, bits_left = total_bits; bits < total_bits; bits += 32,
+ bits_left -= 32) {
+ unsigned int bytes = bits >> 3;
+ unsigned int shift_bytes = 4;
+ u32 tms_store = 0;
+ u32 tdi_store = 0;
+ u32 tdo_store = 0;
+
+ if (bits_left < 32) {
+ /* set number of bits to shift out */
+ write_register(bits_left, iobase, XVC_BAR_LENGTH_REG);
+ shift_bytes = (bits_left + 7) >> 3;
+ }
+
+ memcpy(&tms_store, tms_buf + bytes, shift_bytes);
+ memcpy(&tdi_store, tdi_buf + bytes, shift_bytes);
+
+ /* Shift data out and copy to output buffer */
+ rv = xvc_shift_bits(iobase, tms_store, tdi_store, &tdo_store);
+ if (rv < 0)
+ goto cleanup;
+
+ memcpy(tdo_buf + bytes, &tdo_store, shift_bytes);
+ }
+
+ // If performing loopback test, reset loopback bit in control reg
+ if (opcode == 0x02) {
+ control_reg_data = read_register(iobase, XVC_BAR_CTRL_REG);
+ write_reg_data = control_reg_data & ~(0x02);
+ write_register(write_reg_data, iobase, XVC_BAR_CTRL_REG);
+ }
+
+ rv = copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf, total_bytes);
+ if (rv) {
+ pr_info("copy back tdo_buf failed: %d/%u.\n", rv, total_bytes);
+ rv = -EFAULT;
+ goto cleanup;
+ }
+
+cleanup:
+ kfree(buffer);
+
+ mmiowb();
+
+ return rv;
+}
+
+static long xvc_read_properties(struct xocl_xvc *xvc, const void __user *arg)
+{
+ int status = 0;
+ struct xil_xvc_properties xvc_props_obj;
+
+ xvc_props_obj.xvc_algo_type = (unsigned int) xvc_pci_props.xvc_algo_type;
+ xvc_props_obj.config_vsec_id = xvc_pci_props.config_vsec_id;
+ xvc_props_obj.config_vsec_rev = xvc_pci_props.config_vsec_rev;
+ xvc_props_obj.bar_index = xvc_pci_props.bar_index;
+ xvc_props_obj.bar_offset = xvc_pci_props.bar_offset;
+
+ if (copy_to_user((void *)arg, &xvc_props_obj, sizeof(xvc_props_obj)))
+ status = -ENOMEM;
+
+ mmiowb();
+ return status;
+}
+
+static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+ struct xocl_xvc *xvc = filp->private_data;
+ long status = 0;
+
+ switch (cmd) {
+ case XDMA_IOCXVC:
+ status = xvc_ioctl_helper(xvc, (void __user *)arg);
+ break;
+ case XDMA_RDXVC_PROPS:
+ status = xvc_read_properties(xvc, (void __user *)arg);
+ break;
+ default:
+ status = -ENOIOCTLCMD;
+ break;
+ }
+
+ return status;
+}
+
+static int char_open(struct inode *inode, struct file *file)
+{
+ struct xocl_xvc *xvc = NULL;
+
+ xvc = xocl_drvinst_open(inode->i_cdev);
+ if (!xvc)
+ return -ENXIO;
+
+ /* create a reference to our char device in the opened file */
+ file->private_data = xvc;
+ return 0;
+}
+
+/*
+ * Called when the device goes from used to unused.
+ */
+static int char_close(struct inode *inode, struct file *file)
+{
+ struct xocl_xvc *xvc = file->private_data;
+
+ xocl_drvinst_close(xvc);
+ return 0;
+}
+
+
+/*
+ * character device file operations for the XVC
+ */
+static const struct file_operations xvc_fops = {
+ .owner = THIS_MODULE,
+ .open = char_open,
+ .release = char_close,
+ .unlocked_ioctl = xvc_ioctl,
+};
+
+static int xvc_probe(struct platform_device *pdev)
+{
+ struct xocl_xvc *xvc;
+ struct resource *res;
+ struct xocl_dev_core *core;
+ int err;
+
+ xvc = xocl_drvinst_alloc(&pdev->dev, sizeof(*xvc));
+ if (!xvc)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ xvc->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!xvc->base) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+
+ core = xocl_get_xdev(pdev);
+
+ xvc->sys_cdev = cdev_alloc();
+ xvc->sys_cdev->ops = &xvc_fops;
+ xvc->sys_cdev->owner = THIS_MODULE;
+ xvc->instance = XOCL_DEV_ID(core->pdev) |
+ platform_get_device_id(pdev)->driver_data;
+ xvc->sys_cdev->dev = MKDEV(MAJOR(xvc_dev), core->dev_minor);
+ err = cdev_add(xvc->sys_cdev, xvc->sys_cdev->dev, 1);
+ if (err) {
+ xocl_err(&pdev->dev, "cdev_add failed, %d", err);
+ goto failed;
+ }
+
+ xvc->sys_device = device_create(xrt_class, &pdev->dev,
+ xvc->sys_cdev->dev,
+ NULL, "%s%d",
+ platform_get_device_id(pdev)->name,
+ xvc->instance & MINOR_NAME_MASK);
+ if (IS_ERR(xvc->sys_device)) {
+ err = PTR_ERR(xvc->sys_device);
+ goto failed;
+ }
+
+ xocl_drvinst_set_filedev(xvc, xvc->sys_cdev);
+
+ platform_set_drvdata(pdev, xvc);
+ xocl_info(&pdev->dev, "XVC device instance %d initialized\n",
+ xvc->instance);
+
+ // Update PCIe BAR properties in a global structure
+ xvc_pci_props.xvc_algo_type = XVC_ALGO_BAR;
+ xvc_pci_props.config_vsec_id = 0;
+ xvc_pci_props.config_vsec_rev = 0;
+ xvc_pci_props.bar_index = core->bar_idx;
+ xvc_pci_props.bar_offset = (unsigned int) res->start - (unsigned int)
+ pci_resource_start(core->pdev, core->bar_idx);
+
+ return 0;
+failed:
+ if (!IS_ERR(xvc->sys_device))
+ device_destroy(xrt_class, xvc->sys_cdev->dev);
+ if (xvc->sys_cdev)
+ cdev_del(xvc->sys_cdev);
+ if (xvc->base)
+ iounmap(xvc->base);
+ xocl_drvinst_free(xvc);
+
+ return err;
+}
+
+
+static int xvc_remove(struct platform_device *pdev)
+{
+ struct xocl_xvc *xvc;
+
+ xvc = platform_get_drvdata(pdev);
+ if (!xvc) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+ device_destroy(xrt_class, xvc->sys_cdev->dev);
+ cdev_del(xvc->sys_cdev);
+ if (xvc->base)
+ iounmap(xvc->base);
+
+ platform_set_drvdata(pdev, NULL);
+ xocl_drvinst_free(xvc);
+
+ return 0;
+}
+
+struct platform_device_id xvc_id_table[] = {
+ { XOCL_XVC_PUB, MINOR_PUB_HIGH_BIT },
+ { XOCL_XVC_PRI, MINOR_PRI_HIGH_BIT },
+ { },
+};
+
+static struct platform_driver xvc_driver = {
+ .probe = xvc_probe,
+ .remove = xvc_remove,
+ .driver = {
+ .name = XVC_DEV_NAME,
+ },
+ .id_table = xvc_id_table,
+};
+
+int __init xocl_init_xvc(void)
+{
+ int err = 0;
+
+ err = alloc_chrdev_region(&xvc_dev, 0, XOCL_MAX_DEVICES, XVC_DEV_NAME);
+ if (err < 0)
+ goto err_register_chrdev;
+
+ err = platform_driver_register(&xvc_driver);
+ if (err)
+ goto err_driver_reg;
+ return 0;
+
+err_driver_reg:
+ unregister_chrdev_region(xvc_dev, XOCL_MAX_DEVICES);
+err_register_chrdev:
+ return err;
+}
+
+void xocl_fini_xvc(void)
+{
+ unregister_chrdev_region(xvc_dev, XOCL_MAX_DEVICES);
+ platform_driver_unregister(&xvc_driver);
+}
--
2.17.0
WARNING: multiple messages have this Message-ID (diff)
From: <sonal.santan@xilinx.com>
To: dri-devel@lists.freedesktop.org
Cc: linux-kernel@vger.kernel.org, gregkh@linuxfoundation.org,
airlied@redhat.com, cyrilc@xilinx.com, michals@xilinx.com,
lizhih@xilinx.com, hyunk@xilinx.com,
Sonal Santan <sonal.santan@xilinx.com>
Subject: [RFC PATCH Xilinx Alveo 3/6] Add platform drivers for various IPs and frameworks
Date: Tue, 19 Mar 2019 14:53:58 -0700 [thread overview]
Message-ID: <20190319215401.6562-4-sonal.santan@xilinx.com> (raw)
In-Reply-To: <20190319215401.6562-1-sonal.santan@xilinx.com>
From: Sonal Santan <sonal.santan@xilinx.com>
Signed-off-by: Sonal Santan <sonal.santan@xilinx.com>
---
drivers/gpu/drm/xocl/subdev/dna.c | 356 +++
drivers/gpu/drm/xocl/subdev/feature_rom.c | 412 +++
drivers/gpu/drm/xocl/subdev/firewall.c | 389 +++
drivers/gpu/drm/xocl/subdev/fmgr.c | 198 ++
drivers/gpu/drm/xocl/subdev/icap.c | 2859 ++++++++++++++++++
drivers/gpu/drm/xocl/subdev/mailbox.c | 1868 ++++++++++++
drivers/gpu/drm/xocl/subdev/mb_scheduler.c | 3059 ++++++++++++++++++++
drivers/gpu/drm/xocl/subdev/microblaze.c | 722 +++++
drivers/gpu/drm/xocl/subdev/mig.c | 256 ++
drivers/gpu/drm/xocl/subdev/sysmon.c | 385 +++
drivers/gpu/drm/xocl/subdev/xdma.c | 510 ++++
drivers/gpu/drm/xocl/subdev/xmc.c | 1480 ++++++++++
drivers/gpu/drm/xocl/subdev/xvc.c | 461 +++
13 files changed, 12955 insertions(+)
create mode 100644 drivers/gpu/drm/xocl/subdev/dna.c
create mode 100644 drivers/gpu/drm/xocl/subdev/feature_rom.c
create mode 100644 drivers/gpu/drm/xocl/subdev/firewall.c
create mode 100644 drivers/gpu/drm/xocl/subdev/fmgr.c
create mode 100644 drivers/gpu/drm/xocl/subdev/icap.c
create mode 100644 drivers/gpu/drm/xocl/subdev/mailbox.c
create mode 100644 drivers/gpu/drm/xocl/subdev/mb_scheduler.c
create mode 100644 drivers/gpu/drm/xocl/subdev/microblaze.c
create mode 100644 drivers/gpu/drm/xocl/subdev/mig.c
create mode 100644 drivers/gpu/drm/xocl/subdev/sysmon.c
create mode 100644 drivers/gpu/drm/xocl/subdev/xdma.c
create mode 100644 drivers/gpu/drm/xocl/subdev/xmc.c
create mode 100644 drivers/gpu/drm/xocl/subdev/xvc.c
diff --git a/drivers/gpu/drm/xocl/subdev/dna.c b/drivers/gpu/drm/xocl/subdev/dna.c
new file mode 100644
index 000000000000..991d98e5b9aa
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/dna.c
@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Chien-Wei Lan <chienwei@xilinx.com>
+ *
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/vmalloc.h>
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+/* Registers are defined in pg150-ultrascale-memory-ip.pdf:
+ * AXI4-Lite Slave Control/Status Register Map
+ */
+#define XLNX_DNA_MEMORY_MAP_MAGIC_IS_DEFINED (0x3E4D7732)
+#define XLNX_DNA_MAJOR_MINOR_VERSION_REGISTER_OFFSET 0x00 // RO
+#define XLNX_DNA_REVISION_REGISTER_OFFSET 0x04 // RO
+#define XLNX_DNA_CAPABILITY_REGISTER_OFFSET 0x08 // RO
+//#define XLNX_DNA_SCRATCHPAD_REGISTER_OFFSET (0x0C) // RO (31-1) + RW (0)
+#define XLNX_DNA_STATUS_REGISTER_OFFSET 0x10 // RO
+#define XLNX_DNA_FSM_DNA_WORD_WRITE_COUNT_REGISTER_OFFSET (0x14) // RO
+#define XLNX_DNA_FSM_CERTIFICATE_WORD_WRITE_COUNT_REGISTER_OFFSET (0x18) // RO
+#define XLNX_DNA_MESSAGE_START_AXI_ONLY_REGISTER_OFFSET (0x20) // RO (31-1) + RW (0)
+#define XLNX_DNA_READBACK_REGISTER_2_OFFSET 0x40 // RO XLNX_DNA_BOARD_DNA_95_64
+#define XLNX_DNA_READBACK_REGISTER_1_OFFSET 0x44 // RO XLNX_DNA_BOARD_DNA_63_32
+#define XLNX_DNA_READBACK_REGISTER_0_OFFSET 0x48 // RO XLNX_DNA_BOARD_DNA_31_0
+#define XLNX_DNA_DATA_AXI_ONLY_REGISTER_OFFSET (0x80) // WO
+#define XLNX_DNA_CERTIFICATE_DATA_AXI_ONLY_REGISTER_OFFSET (0xC0) // WO - 512 bit aligned.
+#define XLNX_DNA_MAX_ADDRESS_WORDS (0xC4)
+
+struct xocl_xlnx_dna {
+ void __iomem *base;
+ struct device *xlnx_dna_dev;
+ struct mutex xlnx_dna_lock;
+};
+
+static ssize_t status_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ u32 status;
+
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+
+ return sprintf(buf, "0x%x\n", status);
+}
+static DEVICE_ATTR_RO(status);
+
+static ssize_t dna_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ uint32_t dna96_64, dna63_32, dna31_0;
+
+ dna96_64 = ioread32(xlnx_dna->base+XLNX_DNA_READBACK_REGISTER_2_OFFSET);
+ dna63_32 = ioread32(xlnx_dna->base+XLNX_DNA_READBACK_REGISTER_1_OFFSET);
+ dna31_0 = ioread32(xlnx_dna->base+XLNX_DNA_READBACK_REGISTER_0_OFFSET);
+
+ return sprintf(buf, "%08x%08x%08x\n", dna96_64, dna63_32, dna31_0);
+}
+static DEVICE_ATTR_RO(dna);
+
+static ssize_t capability_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ u32 capability;
+
+ capability = ioread32(xlnx_dna->base+XLNX_DNA_CAPABILITY_REGISTER_OFFSET);
+
+ return sprintf(buf, "0x%x\n", capability);
+}
+static DEVICE_ATTR_RO(capability);
+
+
+static ssize_t dna_version_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ u32 version;
+
+ version = ioread32(xlnx_dna->base+XLNX_DNA_MAJOR_MINOR_VERSION_REGISTER_OFFSET);
+
+ return sprintf(buf, "%d.%d\n", version>>16, version & 0xffff);
+}
+static DEVICE_ATTR_RO(dna_version);
+
+static ssize_t revision_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xlnx_dna *xlnx_dna = dev_get_drvdata(dev);
+ u32 revision;
+
+ revision = ioread32(xlnx_dna->base+XLNX_DNA_REVISION_REGISTER_OFFSET);
+
+ return sprintf(buf, "%d\n", revision);
+}
+static DEVICE_ATTR_RO(revision);
+
+static struct attribute *xlnx_dna_attributes[] = {
+ &dev_attr_status.attr,
+ &dev_attr_dna.attr,
+ &dev_attr_capability.attr,
+ &dev_attr_dna_version.attr,
+ &dev_attr_revision.attr,
+ NULL
+};
+
+static const struct attribute_group xlnx_dna_attrgroup = {
+ .attrs = xlnx_dna_attributes,
+};
+
+static uint32_t dna_status(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna = platform_get_drvdata(pdev);
+ uint32_t status = 0;
+ uint8_t retries = 10;
+ bool rsa4096done = false;
+
+ if (!xlnx_dna)
+ return status;
+
+ while (!rsa4096done && retries) {
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ if (status>>8 & 0x1) {
+ rsa4096done = true;
+ break;
+ }
+ msleep(1);
+ retries--;
+ }
+
+ if (retries == 0)
+ return -EBUSY;
+
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+
+ return status;
+}
+
+static uint32_t dna_capability(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna = platform_get_drvdata(pdev);
+ u32 capability = 0;
+
+ if (!xlnx_dna)
+ return capability;
+
+ capability = ioread32(xlnx_dna->base+XLNX_DNA_CAPABILITY_REGISTER_OFFSET);
+
+ return capability;
+}
+
+static void dna_write_cert(struct platform_device *pdev, const uint32_t *cert, uint32_t len)
+{
+ struct xocl_xlnx_dna *xlnx_dna = platform_get_drvdata(pdev);
+ int i, j, k;
+ u32 status = 0, words;
+ uint8_t retries = 100;
+ bool sha256done = false;
+ uint32_t convert;
+ uint32_t sign_start, message_words = (len-512)>>2;
+
+ sign_start = message_words;
+
+ if (!xlnx_dna)
+ return;
+
+ iowrite32(0x1, xlnx_dna->base+XLNX_DNA_MESSAGE_START_AXI_ONLY_REGISTER_OFFSET);
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ xocl_info(&pdev->dev, "Start: status %08x", status);
+
+ for (i = 0; i < message_words; i += 16) {
+
+ retries = 100;
+ sha256done = false;
+
+ while (!sha256done && retries) {
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ if (!(status >> 4 & 0x1)) {
+ sha256done = true;
+ break;
+ }
+ msleep(10);
+ retries--;
+ }
+ for (j = 0; j < 16; ++j) {
+ convert = (*(cert+i+j) >> 24 & 0xff) | (*(cert+i+j) >> 8 & 0xff00) |
+ (*(cert+i+j) << 8 & 0xff0000) | ((*(cert+i+j) & 0xff) << 24);
+ iowrite32(convert, xlnx_dna->base+XLNX_DNA_DATA_AXI_ONLY_REGISTER_OFFSET+j*4);
+ }
+ }
+ retries = 100;
+ sha256done = false;
+ while (!sha256done && retries) {
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ if (!(status >> 4 & 0x1)) {
+ sha256done = true;
+ break;
+ }
+ msleep(10);
+ retries--;
+ }
+
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ words = ioread32(xlnx_dna->base+XLNX_DNA_FSM_DNA_WORD_WRITE_COUNT_REGISTER_OFFSET);
+ xocl_info(&pdev->dev, "Message: status %08x dna words %d", status, words);
+
+ for (k = 0; k < 128; k += 16) {
+ for (i = 0; i < 16; i++) {
+ j = k+i+sign_start;
+ convert = (*(cert + j) >> 24 & 0xff) | (*(cert + j) >> 8 & 0xff00) |
+ (*(cert + j) << 8 & 0xff0000) | ((*(cert + j) & 0xff) << 24);
+ iowrite32(convert, xlnx_dna->base+XLNX_DNA_CERTIFICATE_DATA_AXI_ONLY_REGISTER_OFFSET + i * 4);
+ }
+ }
+
+ status = ioread32(xlnx_dna->base+XLNX_DNA_STATUS_REGISTER_OFFSET);
+ words = ioread32(xlnx_dna->base+XLNX_DNA_FSM_CERTIFICATE_WORD_WRITE_COUNT_REGISTER_OFFSET);
+ xocl_info(&pdev->dev, "Signature: status %08x certificate words %d", status, words);
+}
+
+static struct xocl_dna_funcs dna_ops = {
+ .status = dna_status,
+ .capability = dna_capability,
+ .write_cert = dna_write_cert,
+};
+
+
+static void mgmt_sysfs_destroy_xlnx_dna(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna;
+
+ xlnx_dna = platform_get_drvdata(pdev);
+
+ sysfs_remove_group(&pdev->dev.kobj, &xlnx_dna_attrgroup);
+
+}
+
+static int mgmt_sysfs_create_xlnx_dna(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna;
+ struct xocl_dev_core *core;
+ int err;
+
+ xlnx_dna = platform_get_drvdata(pdev);
+ core = XDEV(xocl_get_xdev(pdev));
+
+ err = sysfs_create_group(&pdev->dev.kobj, &xlnx_dna_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create pw group failed: 0x%x", err);
+ goto create_grp_failed;
+ }
+
+ return 0;
+
+create_grp_failed:
+ return err;
+}
+
+static int xlnx_dna_probe(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna;
+ struct resource *res;
+ int err;
+
+ xlnx_dna = devm_kzalloc(&pdev->dev, sizeof(*xlnx_dna), GFP_KERNEL);
+ if (!xlnx_dna)
+ return -ENOMEM;
+
+ xlnx_dna->base = devm_kzalloc(&pdev->dev, sizeof(void __iomem *), GFP_KERNEL);
+ if (!xlnx_dna->base)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ xocl_err(&pdev->dev, "resource is NULL");
+ return -EINVAL;
+ }
+ xocl_info(&pdev->dev, "IO start: 0x%llx, end: 0x%llx",
+ res->start, res->end);
+
+ xlnx_dna->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!xlnx_dna->base) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+
+ platform_set_drvdata(pdev, xlnx_dna);
+
+ err = mgmt_sysfs_create_xlnx_dna(pdev);
+ if (err)
+ goto create_xlnx_dna_failed;
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_DNA, &dna_ops);
+
+ return 0;
+
+create_xlnx_dna_failed:
+ platform_set_drvdata(pdev, NULL);
+failed:
+ return err;
+}
+
+
+static int xlnx_dna_remove(struct platform_device *pdev)
+{
+ struct xocl_xlnx_dna *xlnx_dna;
+
+ xlnx_dna = platform_get_drvdata(pdev);
+ if (!xlnx_dna) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ mgmt_sysfs_destroy_xlnx_dna(pdev);
+
+ if (xlnx_dna->base)
+ iounmap(xlnx_dna->base);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, xlnx_dna);
+
+ return 0;
+}
+
+struct platform_device_id xlnx_dna_id_table[] = {
+ { XOCL_DNA, 0 },
+ { },
+};
+
+static struct platform_driver xlnx_dna_driver = {
+ .probe = xlnx_dna_probe,
+ .remove = xlnx_dna_remove,
+ .driver = {
+ .name = "xocl_dna",
+ },
+ .id_table = xlnx_dna_id_table,
+};
+
+int __init xocl_init_dna(void)
+{
+ return platform_driver_register(&xlnx_dna_driver);
+}
+
+void xocl_fini_dna(void)
+{
+ platform_driver_unregister(&xlnx_dna_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/feature_rom.c b/drivers/gpu/drm/xocl/subdev/feature_rom.c
new file mode 100644
index 000000000000..f898af6844aa
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/feature_rom.c
@@ -0,0 +1,412 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include "../xclfeatures.h"
+#include "../xocl_drv.h"
+
+#define MAGIC_NUM 0x786e6c78
+struct feature_rom {
+ void __iomem *base;
+
+ struct FeatureRomHeader header;
+ unsigned int dsa_version;
+ bool unified;
+ bool mb_mgmt_enabled;
+ bool mb_sche_enabled;
+ bool are_dev;
+ bool aws_dev;
+};
+
+static ssize_t VBNV_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%s\n", rom->header.VBNVName);
+}
+static DEVICE_ATTR_RO(VBNV);
+
+static ssize_t dr_base_addr_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ //TODO: Fix: DRBaseAddress no longer required in feature rom
+ if (rom->header.MajorVersion >= 10)
+ return sprintf(buf, "%llu\n", rom->header.DRBaseAddress);
+ else
+ return sprintf(buf, "%u\n", 0);
+}
+static DEVICE_ATTR_RO(dr_base_addr);
+
+static ssize_t ddr_bank_count_max_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%d\n", rom->header.DDRChannelCount);
+}
+static DEVICE_ATTR_RO(ddr_bank_count_max);
+
+static ssize_t ddr_bank_size_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%d\n", rom->header.DDRChannelSize);
+}
+static DEVICE_ATTR_RO(ddr_bank_size);
+
+static ssize_t timestamp_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%llu\n", rom->header.TimeSinceEpoch);
+}
+static DEVICE_ATTR_RO(timestamp);
+
+static ssize_t FPGA_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct feature_rom *rom = platform_get_drvdata(to_platform_device(dev));
+
+ return sprintf(buf, "%s\n", rom->header.FPGAPartName);
+}
+static DEVICE_ATTR_RO(FPGA);
+
+static struct attribute *rom_attrs[] = {
+ &dev_attr_VBNV.attr,
+ &dev_attr_dr_base_addr.attr,
+ &dev_attr_ddr_bank_count_max.attr,
+ &dev_attr_ddr_bank_size.attr,
+ &dev_attr_timestamp.attr,
+ &dev_attr_FPGA.attr,
+ NULL,
+};
+
+static struct attribute_group rom_attr_group = {
+ .attrs = rom_attrs,
+};
+
+static unsigned int dsa_version(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->dsa_version;
+}
+
+static bool is_unified(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->unified;
+}
+
+static bool mb_mgmt_on(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->mb_mgmt_enabled;
+}
+
+static bool mb_sched_on(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->mb_sche_enabled && !XOCL_DSA_MB_SCHE_OFF(xocl_get_xdev(pdev));
+}
+
+static uint32_t *get_cdma_base_addresses(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return (rom->header.FeatureBitMap & CDMA) ? rom->header.CDMABaseAddress : 0;
+}
+
+static u16 get_ddr_channel_count(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->header.DDRChannelCount;
+}
+
+static u64 get_ddr_channel_size(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->header.DDRChannelSize;
+}
+
+static u64 get_timestamp(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->header.TimeSinceEpoch;
+}
+
+static bool is_are(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->are_dev;
+}
+
+static bool is_aws(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ return rom->aws_dev;
+}
+
+static bool verify_timestamp(struct platform_device *pdev, u64 timestamp)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ xocl_info(&pdev->dev, "DSA timestamp: 0x%llx",
+ rom->header.TimeSinceEpoch);
+ xocl_info(&pdev->dev, "Verify timestamp: 0x%llx", timestamp);
+ return (rom->header.TimeSinceEpoch == timestamp);
+}
+
+static void get_raw_header(struct platform_device *pdev, void *header)
+{
+ struct feature_rom *rom;
+
+ rom = platform_get_drvdata(pdev);
+ BUG_ON(!rom);
+
+ memcpy(header, &rom->header, sizeof(rom->header));
+}
+
+static struct xocl_rom_funcs rom_ops = {
+ .dsa_version = dsa_version,
+ .is_unified = is_unified,
+ .mb_mgmt_on = mb_mgmt_on,
+ .mb_sched_on = mb_sched_on,
+ .cdma_addr = get_cdma_base_addresses,
+ .get_ddr_channel_count = get_ddr_channel_count,
+ .get_ddr_channel_size = get_ddr_channel_size,
+ .is_are = is_are,
+ .is_aws = is_aws,
+ .verify_timestamp = verify_timestamp,
+ .get_timestamp = get_timestamp,
+ .get_raw_header = get_raw_header,
+};
+
+static int feature_rom_probe(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+ struct resource *res;
+ u32 val;
+ u16 vendor, did;
+ char *tmp;
+ int ret;
+
+ rom = devm_kzalloc(&pdev->dev, sizeof(*rom), GFP_KERNEL);
+ if (!rom)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ rom->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!rom->base) {
+ ret = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+
+ val = ioread32(rom->base);
+ if (val != MAGIC_NUM) {
+ vendor = XOCL_PL_TO_PCI_DEV(pdev)->vendor;
+ did = XOCL_PL_TO_PCI_DEV(pdev)->device;
+ if (vendor == 0x1d0f && (did == 0x1042 || did == 0xf010)) { // MAGIC, we should define elsewhere
+ xocl_info(&pdev->dev,
+ "Found AWS VU9P Device without featureROM");
+ /*
+ * This is AWS device. Fill the FeatureROM struct.
+ * Right now it doesn't have FeatureROM
+ */
+ memset(rom->header.EntryPointString, 0,
+ sizeof(rom->header.EntryPointString));
+ strncpy(rom->header.EntryPointString, "xlnx", 4);
+ memset(rom->header.FPGAPartName, 0,
+ sizeof(rom->header.FPGAPartName));
+ strncpy(rom->header.FPGAPartName, "AWS VU9P", 8);
+ memset(rom->header.VBNVName, 0,
+ sizeof(rom->header.VBNVName));
+ strncpy(rom->header.VBNVName,
+ "xilinx_aws-vu9p-f1_dynamic_5_0", 35);
+ rom->header.MajorVersion = 4;
+ rom->header.MinorVersion = 0;
+ rom->header.VivadoBuildID = 0xabcd;
+ rom->header.IPBuildID = 0xabcd;
+ rom->header.TimeSinceEpoch = 0xabcd;
+ rom->header.DDRChannelCount = 4;
+ rom->header.DDRChannelSize = 16;
+ rom->header.FeatureBitMap = 0x0;
+ rom->header.FeatureBitMap = UNIFIED_PLATFORM;
+ rom->unified = true;
+ rom->aws_dev = true;
+
+ xocl_info(&pdev->dev, "Enabling AWS dynamic 5.0 DSA");
+ } else {
+ xocl_err(&pdev->dev, "Magic number does not match, actual 0x%x, expected 0x%x",
+ val, MAGIC_NUM);
+ ret = -ENODEV;
+ goto failed;
+ }
+ }
+
+ xocl_memcpy_fromio(&rom->header, rom->base, sizeof(rom->header));
+
+ if (strstr(rom->header.VBNVName, "-xare")) {
+ /*
+ * ARE device, ARE is mapped like another DDR inside FPGA;
+ * map_connects as M04_AXI
+ */
+ rom->header.DDRChannelCount = rom->header.DDRChannelCount - 1;
+ rom->are_dev = true;
+ }
+
+ rom->dsa_version = 0;
+ if (strstr(rom->header.VBNVName, "5_0"))
+ rom->dsa_version = 50;
+ else if (strstr(rom->header.VBNVName, "5_1")
+ || strstr(rom->header.VBNVName, "u200_xdma_201820_1"))
+ rom->dsa_version = 51;
+ else if (strstr(rom->header.VBNVName, "5_2")
+ || strstr(rom->header.VBNVName, "u200_xdma_201820_2")
+ || strstr(rom->header.VBNVName, "u250_xdma_201820_1")
+ || strstr(rom->header.VBNVName, "201830"))
+ rom->dsa_version = 52;
+ else if (strstr(rom->header.VBNVName, "5_3"))
+ rom->dsa_version = 53;
+
+ if (rom->header.FeatureBitMap & UNIFIED_PLATFORM)
+ rom->unified = true;
+
+ if (rom->header.FeatureBitMap & BOARD_MGMT_ENBLD)
+ rom->mb_mgmt_enabled = true;
+
+ if (rom->header.FeatureBitMap & MB_SCHEDULER)
+ rom->mb_sche_enabled = true;
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &rom_attr_group);
+ if (ret) {
+ xocl_err(&pdev->dev, "create sysfs failed");
+ goto failed;
+ }
+
+ tmp = rom->header.EntryPointString;
+ xocl_info(&pdev->dev, "ROM magic : %c%c%c%c",
+ tmp[0], tmp[1], tmp[2], tmp[3]);
+ xocl_info(&pdev->dev, "VBNV: %s", rom->header.VBNVName);
+ xocl_info(&pdev->dev, "DDR channel count : %d",
+ rom->header.DDRChannelCount);
+ xocl_info(&pdev->dev, "DDR channel size: %d GB",
+ rom->header.DDRChannelSize);
+ xocl_info(&pdev->dev, "Major Version: %d", rom->header.MajorVersion);
+ xocl_info(&pdev->dev, "Minor Version: %d", rom->header.MinorVersion);
+ xocl_info(&pdev->dev, "IPBuildID: %u", rom->header.IPBuildID);
+ xocl_info(&pdev->dev, "TimeSinceEpoch: %llx",
+ rom->header.TimeSinceEpoch);
+ xocl_info(&pdev->dev, "FeatureBitMap: %llx", rom->header.FeatureBitMap);
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_FEATURE_ROM, &rom_ops);
+ platform_set_drvdata(pdev, rom);
+
+ return 0;
+
+failed:
+ if (rom->base)
+ iounmap(rom->base);
+ devm_kfree(&pdev->dev, rom);
+ return ret;
+}
+
+static int feature_rom_remove(struct platform_device *pdev)
+{
+ struct feature_rom *rom;
+
+ xocl_info(&pdev->dev, "Remove feature rom");
+ rom = platform_get_drvdata(pdev);
+ if (!rom) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+ if (rom->base)
+ iounmap(rom->base);
+
+ sysfs_remove_group(&pdev->dev.kobj, &rom_attr_group);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, rom);
+ return 0;
+}
+
+struct platform_device_id rom_id_table[] = {
+ { XOCL_FEATURE_ROM, 0 },
+ { },
+};
+
+static struct platform_driver feature_rom_driver = {
+ .probe = feature_rom_probe,
+ .remove = feature_rom_remove,
+ .driver = {
+ .name = XOCL_FEATURE_ROM,
+ },
+ .id_table = rom_id_table,
+};
+
+int __init xocl_init_feature_rom(void)
+{
+ return platform_driver_register(&feature_rom_driver);
+}
+
+void xocl_fini_feature_rom(void)
+{
+ return platform_driver_unregister(&feature_rom_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/firewall.c b/drivers/gpu/drm/xocl/subdev/firewall.c
new file mode 100644
index 000000000000..a32766507ae0
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/firewall.c
@@ -0,0 +1,389 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2017-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Utility Functions for AXI firewall IP.
+ * Author: Lizhi.Hou@Xilinx.com
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/ktime.h>
+#include <linux/rtc.h>
+#include "../xocl_drv.h"
+
+/* Firewall registers */
+#define FAULT_STATUS 0x0
+#define SOFT_CTRL 0x4
+#define UNBLOCK_CTRL 0x8
+// Firewall error bits
+#define READ_RESPONSE_BUSY BIT(0)
+#define RECS_ARREADY_MAX_WAIT BIT(1)
+#define RECS_CONTINUOUS_RTRANSFERS_MAX_WAIT BIT(2)
+#define ERRS_RDATA_NUM BIT(3)
+#define ERRS_RID BIT(4)
+#define WRITE_RESPONSE_BUSY BIT(16)
+#define RECS_AWREADY_MAX_WAIT BIT(17)
+#define RECS_WREADY_MAX_WAIT BIT(18)
+#define RECS_WRITE_TO_BVALID_MAX_WAIT BIT(19)
+#define ERRS_BRESP BIT(20)
+
+#define FIREWALL_STATUS_BUSY (READ_RESPONSE_BUSY | WRITE_RESPONSE_BUSY)
+#define CLEAR_RESET_GPIO 0
+
+#define READ_STATUS(fw, id) \
+ XOCL_READ_REG32(fw->base_addrs[id] + FAULT_STATUS)
+#define WRITE_UNBLOCK_CTRL(fw, id, val) \
+ XOCL_WRITE_REG32(val, fw->base_addrs[id] + UNBLOCK_CTRL)
+
+#define IS_FIRED(fw, id) (READ_STATUS(fw, id) & ~FIREWALL_STATUS_BUSY)
+
+#define BUSY_RETRY_COUNT 20
+#define BUSY_RETRY_INTERVAL 100 /* ms */
+#define CLEAR_RETRY_COUNT 4
+#define CLEAR_RETRY_INTERVAL 2 /* ms */
+
+#define MAX_LEVEL 16
+
+struct firewall {
+ void __iomem *base_addrs[MAX_LEVEL];
+ u32 max_level;
+ void __iomem *gpio_addr;
+
+ u32 curr_status;
+ int curr_level;
+
+ u32 err_detected_status;
+ u32 err_detected_level;
+ u64 err_detected_time;
+
+ bool inject_firewall;
+};
+
+static int clear_firewall(struct platform_device *pdev);
+static u32 check_firewall(struct platform_device *pdev, int *level);
+
+static int get_prop(struct platform_device *pdev, u32 prop, void *val)
+{
+ struct firewall *fw;
+
+ fw = platform_get_drvdata(pdev);
+ BUG_ON(!fw);
+
+ check_firewall(pdev, NULL);
+ switch (prop) {
+ case XOCL_AF_PROP_TOTAL_LEVEL:
+ *(u32 *)val = fw->max_level;
+ break;
+ case XOCL_AF_PROP_STATUS:
+ *(u32 *)val = fw->curr_status;
+ break;
+ case XOCL_AF_PROP_LEVEL:
+ *(int *)val = fw->curr_level;
+ break;
+ case XOCL_AF_PROP_DETECTED_STATUS:
+ *(u32 *)val = fw->err_detected_status;
+ break;
+ case XOCL_AF_PROP_DETECTED_LEVEL:
+ *(u32 *)val = fw->err_detected_level;
+ break;
+ case XOCL_AF_PROP_DETECTED_TIME:
+ *(u64 *)val = fw->err_detected_time;
+ break;
+ default:
+ xocl_err(&pdev->dev, "Invalid prop %d", prop);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+/* sysfs support */
+static ssize_t show_firewall(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+ struct platform_device *pdev = to_platform_device(dev);
+ struct firewall *fw;
+ u64 t;
+ u32 val;
+ int ret;
+
+ fw = platform_get_drvdata(pdev);
+ BUG_ON(!fw);
+
+ if (attr->index == XOCL_AF_PROP_DETECTED_TIME) {
+ get_prop(pdev, attr->index, &t);
+ return sprintf(buf, "%llu\n", t);
+ }
+
+ ret = get_prop(pdev, attr->index, &val);
+ if (ret)
+ return 0;
+
+ return sprintf(buf, "%u\n", val);
+}
+
+static SENSOR_DEVICE_ATTR(status, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_STATUS);
+static SENSOR_DEVICE_ATTR(level, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_LEVEL);
+static SENSOR_DEVICE_ATTR(detected_status, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_DETECTED_STATUS);
+static SENSOR_DEVICE_ATTR(detected_level, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_DETECTED_LEVEL);
+static SENSOR_DEVICE_ATTR(detected_time, 0444, show_firewall, NULL,
+ XOCL_AF_PROP_DETECTED_TIME);
+
+static ssize_t clear_store(struct device *dev, struct device_attribute *da,
+ const char *buf, size_t count)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ u32 val = 0;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val != 1)
+ return -EINVAL;
+
+ clear_firewall(pdev);
+
+ return count;
+}
+static DEVICE_ATTR_WO(clear);
+
+static ssize_t inject_store(struct device *dev, struct device_attribute *da,
+ const char *buf, size_t count)
+{
+ struct firewall *fw = platform_get_drvdata(to_platform_device(dev));
+
+ fw->inject_firewall = true;
+ return count;
+}
+static DEVICE_ATTR_WO(inject);
+
+static struct attribute *firewall_attributes[] = {
+ &sensor_dev_attr_status.dev_attr.attr,
+ &sensor_dev_attr_level.dev_attr.attr,
+ &sensor_dev_attr_detected_status.dev_attr.attr,
+ &sensor_dev_attr_detected_level.dev_attr.attr,
+ &sensor_dev_attr_detected_time.dev_attr.attr,
+ &dev_attr_clear.attr,
+ &dev_attr_inject.attr,
+ NULL
+};
+
+static const struct attribute_group firewall_attrgroup = {
+ .attrs = firewall_attributes,
+};
+
+static u32 check_firewall(struct platform_device *pdev, int *level)
+{
+ struct firewall *fw;
+// struct timeval time;
+ struct timespec64 now;
+ int i;
+ u32 val = 0;
+
+ fw = platform_get_drvdata(pdev);
+ BUG_ON(!fw);
+
+ for (i = 0; i < fw->max_level; i++) {
+ val = IS_FIRED(fw, i);
+ if (val) {
+ xocl_info(&pdev->dev, "AXI Firewall %d tripped, "
+ "status: 0x%x", i, val);
+ if (!fw->curr_status) {
+ fw->err_detected_status = val;
+ fw->err_detected_level = i;
+ ktime_get_ts64(&now);
+ fw->err_detected_time = (u64)(now.tv_sec -
+ (sys_tz.tz_minuteswest * 60));
+ }
+ fw->curr_level = i;
+
+ if (level)
+ *level = i;
+ break;
+ }
+ }
+
+ fw->curr_status = val;
+ fw->curr_level = i >= fw->max_level ? -1 : i;
+
+ /* Inject firewall for testing. */
+ if (fw->curr_level == -1 && fw->inject_firewall) {
+ fw->inject_firewall = false;
+ fw->curr_level = 0;
+ fw->curr_status = 0x1;
+ }
+
+ return fw->curr_status;
+}
+
+static int clear_firewall(struct platform_device *pdev)
+{
+ struct firewall *fw;
+ int i, retry = 0, clear_retry = 0;
+ u32 val;
+ int ret = 0;
+
+ fw = platform_get_drvdata(pdev);
+ BUG_ON(!fw);
+
+ if (!check_firewall(pdev, NULL)) {
+ /* firewall is not tripped */
+ return 0;
+ }
+
+retry_level1:
+ for (i = 0; i < fw->max_level; i++) {
+ for (val = READ_STATUS(fw, i);
+ (val & FIREWALL_STATUS_BUSY) &&
+ retry++ < BUSY_RETRY_COUNT;
+ val = READ_STATUS(fw, i)) {
+ msleep(BUSY_RETRY_INTERVAL);
+ }
+ if (val & FIREWALL_STATUS_BUSY) {
+ xocl_err(&pdev->dev, "firewall %d busy", i);
+ ret = -EBUSY;
+ goto failed;
+ }
+ WRITE_UNBLOCK_CTRL(fw, i, 1);
+ }
+
+ if (check_firewall(pdev, NULL) && clear_retry++ < CLEAR_RETRY_COUNT) {
+ msleep(CLEAR_RETRY_INTERVAL);
+ goto retry_level1;
+ }
+
+ if (!check_firewall(pdev, NULL)) {
+ xocl_info(&pdev->dev, "firewall cleared level 1");
+ return 0;
+ }
+
+ clear_retry = 0;
+
+retry_level2:
+ XOCL_WRITE_REG32(CLEAR_RESET_GPIO, fw->gpio_addr);
+
+ if (check_firewall(pdev, NULL) && clear_retry++ < CLEAR_RETRY_COUNT) {
+ msleep(CLEAR_RETRY_INTERVAL);
+ goto retry_level2;
+ }
+
+ if (!check_firewall(pdev, NULL)) {
+ xocl_info(&pdev->dev, "firewall cleared level 2");
+ return 0;
+ }
+
+ xocl_info(&pdev->dev, "failed clear firewall, level %d, status 0x%x",
+ fw->curr_level, fw->curr_status);
+
+ ret = -EIO;
+
+failed:
+ return ret;
+}
+
+static struct xocl_firewall_funcs fw_ops = {
+ .clear_firewall = clear_firewall,
+ .check_firewall = check_firewall,
+ .get_prop = get_prop,
+};
+
+static int firewall_remove(struct platform_device *pdev)
+{
+ struct firewall *fw;
+ int i;
+
+ fw = platform_get_drvdata(pdev);
+ if (!fw) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ sysfs_remove_group(&pdev->dev.kobj, &firewall_attrgroup);
+
+ for (i = 0; i <= fw->max_level; i++) {
+ if (fw->base_addrs[i])
+ iounmap(fw->base_addrs[i]);
+ }
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, fw);
+
+ return 0;
+}
+
+static int firewall_probe(struct platform_device *pdev)
+{
+ struct firewall *fw;
+ struct resource *res;
+ int i, ret = 0;
+
+ xocl_info(&pdev->dev, "probe");
+
+ fw = devm_kzalloc(&pdev->dev, sizeof(*fw), GFP_KERNEL);
+ if (!fw)
+ return -ENOMEM;
+
+ platform_set_drvdata(pdev, fw);
+
+ fw->curr_level = -1;
+
+ for (i = 0; i < MAX_LEVEL; i++) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+ if (!res) {
+ fw->max_level = i - 1;
+ fw->gpio_addr = fw->base_addrs[i - 1];
+ break;
+ }
+ fw->base_addrs[i] =
+ ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!fw->base_addrs[i]) {
+ ret = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+ }
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &firewall_attrgroup);
+ if (ret) {
+ xocl_err(&pdev->dev, "create attr group failed: %d", ret);
+ goto failed;
+ }
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_AF, &fw_ops);
+
+ return 0;
+
+failed:
+ firewall_remove(pdev);
+ return ret;
+}
+
+struct platform_device_id firewall_id_table[] = {
+ { XOCL_FIREWALL, 0 },
+ { },
+};
+
+static struct platform_driver firewall_driver = {
+ .probe = firewall_probe,
+ .remove = firewall_remove,
+ .driver = {
+ .name = "xocl_firewall",
+ },
+ .id_table = firewall_id_table,
+};
+
+int __init xocl_init_firewall(void)
+{
+ return platform_driver_register(&firewall_driver);
+}
+
+void xocl_fini_firewall(void)
+{
+ return platform_driver_unregister(&firewall_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/fmgr.c b/drivers/gpu/drm/xocl/subdev/fmgr.c
new file mode 100644
index 000000000000..99efd86ccd1b
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/fmgr.c
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * FPGA Manager bindings for XRT driver
+ *
+ * Copyright (C) 2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Sonal Santan
+ *
+ */
+
+#include <linux/fpga/fpga-mgr.h>
+
+#include "../xocl_drv.h"
+#include "../xclbin.h"
+
+/*
+ * Container to capture and cache full xclbin as it is passed in blocks by FPGA
+ * Manager. xocl needs access to full xclbin to walk through xclbin sections. FPGA
+ * Manager's .write() backend sends incremental blocks without any knowledge of
+ * xclbin format forcing us to collect the blocks and stitch them together here.
+ * TODO:
+ * 1. Add a variant of API, icap_download_bitstream_axlf() which works off kernel buffer
+ * 2. Call this new API from FPGA Manager's write complete hook, xocl_pr_write_complete()
+ */
+
+struct xfpga_klass {
+ struct xocl_dev *xdev;
+ struct axlf *blob;
+ char name[64];
+ size_t count;
+ enum fpga_mgr_states state;
+};
+
+static int xocl_pr_write_init(struct fpga_manager *mgr,
+ struct fpga_image_info *info, const char *buf, size_t count)
+{
+ struct xfpga_klass *obj = mgr->priv;
+ const struct axlf *bin = (const struct axlf *)buf;
+
+ if (count < sizeof(struct axlf)) {
+ obj->state = FPGA_MGR_STATE_WRITE_INIT_ERR;
+ return -EINVAL;
+ }
+
+ if (count > bin->m_header.m_length) {
+ obj->state = FPGA_MGR_STATE_WRITE_INIT_ERR;
+ return -EINVAL;
+ }
+
+ /* Free up the previous blob */
+ vfree(obj->blob);
+ obj->blob = vmalloc(bin->m_header.m_length);
+ if (!obj->blob) {
+ obj->state = FPGA_MGR_STATE_WRITE_INIT_ERR;
+ return -ENOMEM;
+ }
+
+ memcpy(obj->blob, buf, count);
+ xocl_info(&mgr->dev, "Begin download of xclbin %pUb of length %lld B", &obj->blob->m_header.uuid,
+ obj->blob->m_header.m_length);
+ obj->count = count;
+ obj->state = FPGA_MGR_STATE_WRITE_INIT;
+ return 0;
+}
+
+static int xocl_pr_write(struct fpga_manager *mgr,
+ const char *buf, size_t count)
+{
+ struct xfpga_klass *obj = mgr->priv;
+ char *curr = (char *)obj->blob;
+
+ if ((obj->state != FPGA_MGR_STATE_WRITE_INIT) && (obj->state != FPGA_MGR_STATE_WRITE)) {
+ obj->state = FPGA_MGR_STATE_WRITE_ERR;
+ return -EINVAL;
+ }
+
+ curr += obj->count;
+ obj->count += count;
+ /* Check if the xclbin buffer is not longer than advertised in the header */
+ if (obj->blob->m_header.m_length < obj->count) {
+ obj->state = FPGA_MGR_STATE_WRITE_ERR;
+ return -EINVAL;
+ }
+ memcpy(curr, buf, count);
+ xocl_info(&mgr->dev, "Next block of %zu B of xclbin %pUb", count, &obj->blob->m_header.uuid);
+ obj->state = FPGA_MGR_STATE_WRITE;
+ return 0;
+}
+
+
+static int xocl_pr_write_complete(struct fpga_manager *mgr,
+ struct fpga_image_info *info)
+{
+ int result;
+ struct xfpga_klass *obj = mgr->priv;
+
+ if (obj->state != FPGA_MGR_STATE_WRITE) {
+ obj->state = FPGA_MGR_STATE_WRITE_COMPLETE_ERR;
+ return -EINVAL;
+ }
+
+ /* Check if we got the complete xclbin */
+ if (obj->blob->m_header.m_length != obj->count) {
+ obj->state = FPGA_MGR_STATE_WRITE_COMPLETE_ERR;
+ return -EINVAL;
+ }
+ /* Send the xclbin blob to actual download framework in icap */
+ result = xocl_icap_download_axlf(obj->xdev, obj->blob);
+ obj->state = result ? FPGA_MGR_STATE_WRITE_COMPLETE_ERR : FPGA_MGR_STATE_WRITE_COMPLETE;
+ xocl_info(&mgr->dev, "Finish download of xclbin %pUb of size %zu B", &obj->blob->m_header.uuid, obj->count);
+ vfree(obj->blob);
+ obj->blob = NULL;
+ obj->count = 0;
+ return result;
+}
+
+static enum fpga_mgr_states xocl_pr_state(struct fpga_manager *mgr)
+{
+ struct xfpga_klass *obj = mgr->priv;
+
+ return obj->state;
+}
+
+static const struct fpga_manager_ops xocl_pr_ops = {
+ .initial_header_size = sizeof(struct axlf),
+ .write_init = xocl_pr_write_init,
+ .write = xocl_pr_write,
+ .write_complete = xocl_pr_write_complete,
+ .state = xocl_pr_state,
+};
+
+
+struct platform_device_id fmgr_id_table[] = {
+ { XOCL_FMGR, 0 },
+ { },
+};
+
+static int fmgr_probe(struct platform_device *pdev)
+{
+ struct fpga_manager *mgr;
+ int ret = 0;
+ struct xfpga_klass *obj = kzalloc(sizeof(struct xfpga_klass), GFP_KERNEL);
+
+ if (!obj)
+ return -ENOMEM;
+
+ obj->xdev = xocl_get_xdev(pdev);
+ snprintf(obj->name, sizeof(obj->name), "Xilinx PCIe FPGA Manager");
+
+ obj->state = FPGA_MGR_STATE_UNKNOWN;
+ mgr = fpga_mgr_create(&pdev->dev, obj->name, &xocl_pr_ops, obj);
+ if (!mgr) {
+ ret = -ENODEV;
+ goto out;
+ }
+ ret = fpga_mgr_register(mgr);
+ if (ret)
+ goto out;
+
+ return ret;
+out:
+ kfree(obj);
+ return ret;
+}
+
+static int fmgr_remove(struct platform_device *pdev)
+{
+ struct fpga_manager *mgr = platform_get_drvdata(pdev);
+ struct xfpga_klass *obj = mgr->priv;
+
+ obj->state = FPGA_MGR_STATE_UNKNOWN;
+ fpga_mgr_unregister(mgr);
+
+ platform_set_drvdata(pdev, NULL);
+ vfree(obj->blob);
+ kfree(obj);
+ return 0;
+}
+
+static struct platform_driver fmgr_driver = {
+ .probe = fmgr_probe,
+ .remove = fmgr_remove,
+ .driver = {
+ .name = "xocl_fmgr",
+ },
+ .id_table = fmgr_id_table,
+};
+
+int __init xocl_init_fmgr(void)
+{
+ return platform_driver_register(&fmgr_driver);
+}
+
+void xocl_fini_fmgr(void)
+{
+ platform_driver_unregister(&fmgr_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/icap.c b/drivers/gpu/drm/xocl/subdev/icap.c
new file mode 100644
index 000000000000..93eb6265a9c4
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/icap.c
@@ -0,0 +1,2859 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2017 Xilinx, Inc. All rights reserved.
+ * Author: Sonal Santan
+ * Code copied verbatim from SDAccel xcldma kernel mode driver
+ *
+ */
+
+/*
+ * TODO: Currently, locking / unlocking bitstream is implemented w/ pid as
+ * identification of bitstream users. We assume that, on bare metal, an app
+ * has only one process and will open both user and mgmt pfs. In this model,
+ * xclmgmt has enough information to handle locking/unlocking alone, but we
+ * still involve user pf and mailbox here so that it'll be easier to support
+ * cloud env later. We'll replace pid with a token that is more appropriate
+ * to identify a user later as well.
+ */
+
+#include <linux/firmware.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include <linux/version.h>
+#include <linux/uuid.h>
+#include <linux/pid.h>
+#include "../xclbin.h"
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+#if defined(XOCL_UUID)
+static uuid_t uuid_null = NULL_UUID_LE;
+#endif
+
+#define ICAP_ERR(icap, fmt, arg...) \
+ xocl_err(&(icap)->icap_pdev->dev, fmt "\n", ##arg)
+#define ICAP_INFO(icap, fmt, arg...) \
+ xocl_info(&(icap)->icap_pdev->dev, fmt "\n", ##arg)
+#define ICAP_DBG(icap, fmt, arg...) \
+ xocl_dbg(&(icap)->icap_pdev->dev, fmt "\n", ##arg)
+
+#define ICAP_PRIVILEGED(icap) ((icap)->icap_regs != NULL)
+#define DMA_HWICAP_BITFILE_BUFFER_SIZE 1024
+#define ICAP_MAX_REG_GROUPS ARRAY_SIZE(XOCL_RES_ICAP_MGMT)
+
+#define ICAP_MAX_NUM_CLOCKS 2
+#define OCL_CLKWIZ_STATUS_OFFSET 0x4
+#define OCL_CLKWIZ_CONFIG_OFFSET(n) (0x200 + 4 * (n))
+#define OCL_CLK_FREQ_COUNTER_OFFSET 0x8
+
+/*
+ * Bitstream header information.
+ */
+struct XHwIcap_Bit_Header {
+ unsigned int HeaderLength; /* Length of header in 32 bit words */
+ unsigned int BitstreamLength; /* Length of bitstream to read in bytes*/
+ unsigned char *DesignName; /* Design name read from bitstream header */
+ unsigned char *PartName; /* Part name read from bitstream header */
+ unsigned char *Date; /* Date read from bitstream header */
+ unsigned char *Time; /* Bitstream creation time read from header */
+ unsigned int MagicLength; /* Length of the magic numbers in header */
+};
+
+#define XHI_BIT_HEADER_FAILURE -1
+/* Used for parsing bitstream header */
+#define XHI_EVEN_MAGIC_BYTE 0x0f
+#define XHI_ODD_MAGIC_BYTE 0xf0
+/* Extra mode for IDLE */
+#define XHI_OP_IDLE -1
+/* The imaginary module length register */
+#define XHI_MLR 15
+
+#define GATE_FREEZE_USER 0x0c
+#define GATE_FREEZE_SHELL 0x00
+
+static u32 gate_free_user[] = {0xe, 0xc, 0xe, 0xf};
+static u32 gate_free_shell[] = {0x8, 0xc, 0xe, 0xf};
+
+/*
+ * AXI-HWICAP IP register layout
+ */
+struct icap_reg {
+ u32 ir_rsvd1[7];
+ u32 ir_gier;
+ u32 ir_isr;
+ u32 ir_rsvd2;
+ u32 ir_ier;
+ u32 ir_rsvd3[53];
+ u32 ir_wf;
+ u32 ir_rf;
+ u32 ir_sz;
+ u32 ir_cr;
+ u32 ir_sr;
+ u32 ir_wfv;
+ u32 ir_rfo;
+ u32 ir_asr;
+} __attribute__((packed));
+
+struct icap_generic_state {
+ u32 igs_state;
+} __attribute__((packed));
+
+struct icap_axi_gate {
+ u32 iag_wr;
+ u32 iag_rvsd;
+ u32 iag_rd;
+} __attribute__((packed));
+
+struct icap_bitstream_user {
+ struct list_head ibu_list;
+ pid_t ibu_pid;
+};
+
+struct icap {
+ struct platform_device *icap_pdev;
+ struct mutex icap_lock;
+ struct icap_reg *icap_regs;
+ struct icap_generic_state *icap_state;
+ unsigned int idcode;
+ bool icap_axi_gate_frozen;
+ bool icap_axi_gate_shell_frozen;
+ struct icap_axi_gate *icap_axi_gate;
+
+ u64 icap_bitstream_id;
+ uuid_t icap_bitstream_uuid;
+ int icap_bitstream_ref;
+ struct list_head icap_bitstream_users;
+
+ char *icap_clear_bitstream;
+ unsigned long icap_clear_bitstream_length;
+
+ char *icap_clock_bases[ICAP_MAX_NUM_CLOCKS];
+ unsigned short icap_ocl_frequency[ICAP_MAX_NUM_CLOCKS];
+
+ char *icap_clock_freq_topology;
+ unsigned long icap_clock_freq_topology_length;
+ char *icap_clock_freq_counter;
+ struct mem_topology *mem_topo;
+ struct ip_layout *ip_layout;
+ struct debug_ip_layout *debug_layout;
+ struct connectivity *connectivity;
+
+ char *bit_buffer;
+ unsigned long bit_length;
+};
+
+static inline u32 reg_rd(void __iomem *reg)
+{
+ return XOCL_READ_REG32(reg);
+}
+
+static inline void reg_wr(void __iomem *reg, u32 val)
+{
+ iowrite32(val, reg);
+}
+
+/*
+ * Precomputed table with config0 and config2 register values together with
+ * target frequency. The steps are approximately 5 MHz apart. Table is
+ * generated by wiz.pl.
+ */
+const static struct xclmgmt_ocl_clockwiz {
+ /* target frequency */
+ unsigned short ocl;
+ /* config0 register */
+ unsigned long config0;
+ /* config2 register */
+ unsigned short config2;
+} frequency_table[] = {
+ {/* 600*/ 60, 0x0601, 0x000a},
+ {/* 600*/ 66, 0x0601, 0x0009},
+ {/* 600*/ 75, 0x0601, 0x0008},
+ {/* 800*/ 80, 0x0801, 0x000a},
+ {/* 600*/ 85, 0x0601, 0x0007},
+ {/* 900*/ 90, 0x0901, 0x000a},
+ {/*1000*/ 100, 0x0a01, 0x000a},
+ {/*1100*/ 110, 0x0b01, 0x000a},
+ {/* 700*/ 116, 0x0701, 0x0006},
+ {/*1100*/ 122, 0x0b01, 0x0009},
+ {/* 900*/ 128, 0x0901, 0x0007},
+ {/*1200*/ 133, 0x0c01, 0x0009},
+ {/*1400*/ 140, 0x0e01, 0x000a},
+ {/*1200*/ 150, 0x0c01, 0x0008},
+ {/*1400*/ 155, 0x0e01, 0x0009},
+ {/* 800*/ 160, 0x0801, 0x0005},
+ {/*1000*/ 166, 0x0a01, 0x0006},
+ {/*1200*/ 171, 0x0c01, 0x0007},
+ {/* 900*/ 180, 0x0901, 0x0005},
+ {/*1300*/ 185, 0x0d01, 0x0007},
+ {/*1400*/ 200, 0x0e01, 0x0007},
+ {/*1300*/ 216, 0x0d01, 0x0006},
+ {/* 900*/ 225, 0x0901, 0x0004},
+ {/*1400*/ 233, 0x0e01, 0x0006},
+ {/*1200*/ 240, 0x0c01, 0x0005},
+ {/*1000*/ 250, 0x0a01, 0x0004},
+ {/*1300*/ 260, 0x0d01, 0x0005},
+ {/* 800*/ 266, 0x0801, 0x0003},
+ {/*1100*/ 275, 0x0b01, 0x0004},
+ {/*1400*/ 280, 0x0e01, 0x0005},
+ {/*1200*/ 300, 0x0c01, 0x0004},
+ {/*1300*/ 325, 0x0d01, 0x0004},
+ {/*1000*/ 333, 0x0a01, 0x0003},
+ {/*1400*/ 350, 0x0e01, 0x0004},
+ {/*1100*/ 366, 0x0b01, 0x0003},
+ {/*1200*/ 400, 0x0c01, 0x0003},
+ {/*1300*/ 433, 0x0d01, 0x0003},
+ {/* 900*/ 450, 0x0901, 0x0002},
+ {/*1400*/ 466, 0x0e01, 0x0003},
+ {/*1000*/ 500, 0x0a01, 0x0002}
+};
+
+static int icap_verify_bitstream_axlf(struct platform_device *pdev,
+ struct axlf *xclbin);
+static int icap_parse_bitstream_axlf_section(struct platform_device *pdev,
+ const struct axlf *xclbin, enum axlf_section_kind kind);
+
+static struct icap_bitstream_user *alloc_user(pid_t pid)
+{
+ struct icap_bitstream_user *u =
+ kzalloc(sizeof(struct icap_bitstream_user), GFP_KERNEL);
+
+ if (u) {
+ INIT_LIST_HEAD(&u->ibu_list);
+ u->ibu_pid = pid;
+ }
+ return u;
+}
+
+static void free_user(struct icap_bitstream_user *u)
+{
+ kfree(u);
+}
+
+static struct icap_bitstream_user *obtain_user(struct icap *icap, pid_t pid)
+{
+ struct list_head *pos, *n;
+
+ list_for_each_safe(pos, n, &icap->icap_bitstream_users) {
+ struct icap_bitstream_user *u = list_entry(pos, struct icap_bitstream_user, ibu_list);
+
+ if (u->ibu_pid == pid)
+ return u;
+ }
+
+ return NULL;
+}
+
+static void icap_read_from_peer(struct platform_device *pdev, enum data_kind kind, void *resp, size_t resplen)
+{
+ struct mailbox_subdev_peer subdev_peer = {0};
+ size_t data_len = sizeof(struct mailbox_subdev_peer);
+ struct mailbox_req *mb_req = NULL;
+ size_t reqlen = sizeof(struct mailbox_req) + data_len;
+
+ mb_req = vmalloc(reqlen);
+ if (!mb_req)
+ return;
+
+ mb_req->req = MAILBOX_REQ_PEER_DATA;
+
+ subdev_peer.kind = kind;
+ memcpy(mb_req->data, &subdev_peer, data_len);
+
+ (void) xocl_peer_request(XOCL_PL_DEV_TO_XDEV(pdev),
+ mb_req, reqlen, resp, &resplen, NULL, NULL);
+
+ vfree(mb_req);
+}
+
+
+static int add_user(struct icap *icap, pid_t pid)
+{
+ struct icap_bitstream_user *u;
+
+ u = obtain_user(icap, pid);
+ if (u)
+ return 0;
+
+ u = alloc_user(pid);
+ if (!u)
+ return -ENOMEM;
+
+ list_add_tail(&u->ibu_list, &icap->icap_bitstream_users);
+ icap->icap_bitstream_ref++;
+ return 0;
+}
+
+static int del_user(struct icap *icap, pid_t pid)
+{
+ struct icap_bitstream_user *u = NULL;
+
+ u = obtain_user(icap, pid);
+ if (!u)
+ return -EINVAL;
+
+ list_del(&u->ibu_list);
+ free_user(u);
+ icap->icap_bitstream_ref--;
+ return 0;
+}
+
+static void del_all_users(struct icap *icap)
+{
+ struct icap_bitstream_user *u = NULL;
+ struct list_head *pos, *n;
+
+ if (icap->icap_bitstream_ref == 0)
+ return;
+
+ list_for_each_safe(pos, n, &icap->icap_bitstream_users) {
+ u = list_entry(pos, struct icap_bitstream_user, ibu_list);
+ list_del(&u->ibu_list);
+ free_user(u);
+ }
+
+ ICAP_INFO(icap, "removed %d users", icap->icap_bitstream_ref);
+ icap->icap_bitstream_ref = 0;
+}
+
+static unsigned int find_matching_freq_config(unsigned int freq)
+{
+ unsigned int start = 0;
+ unsigned int end = ARRAY_SIZE(frequency_table) - 1;
+ unsigned int idx = ARRAY_SIZE(frequency_table) - 1;
+
+ if (freq < frequency_table[0].ocl)
+ return 0;
+
+ if (freq > frequency_table[ARRAY_SIZE(frequency_table) - 1].ocl)
+ return ARRAY_SIZE(frequency_table) - 1;
+
+ while (start < end) {
+ if (freq == frequency_table[idx].ocl)
+ break;
+ if (freq < frequency_table[idx].ocl)
+ end = idx;
+ else
+ start = idx + 1;
+ idx = start + (end - start) / 2;
+ }
+ if (freq < frequency_table[idx].ocl)
+ idx--;
+
+ return idx;
+}
+
+static unsigned short icap_get_ocl_frequency(const struct icap *icap, int idx)
+{
+#define XCL_INPUT_FREQ 100
+ const u64 input = XCL_INPUT_FREQ;
+ u32 val;
+ u32 mul0, div0;
+ u32 mul_frac0 = 0;
+ u32 div1;
+ u32 div_frac1 = 0;
+ u64 freq;
+ char *base = NULL;
+
+ if (ICAP_PRIVILEGED(icap)) {
+ base = icap->icap_clock_bases[idx];
+ val = reg_rd(base + OCL_CLKWIZ_STATUS_OFFSET);
+ if ((val & 1) == 0)
+ return 0;
+
+ val = reg_rd(base + OCL_CLKWIZ_CONFIG_OFFSET(0));
+
+ div0 = val & 0xff;
+ mul0 = (val & 0xff00) >> 8;
+ if (val & BIT(26)) {
+ mul_frac0 = val >> 16;
+ mul_frac0 &= 0x3ff;
+ }
+
+ /*
+ * Multiply both numerator (mul0) and the denominator (div0) with 1000
+ * to account for fractional portion of multiplier
+ */
+ mul0 *= 1000;
+ mul0 += mul_frac0;
+ div0 *= 1000;
+
+ val = reg_rd(base + OCL_CLKWIZ_CONFIG_OFFSET(2));
+
+ div1 = val & 0xff;
+ if (val & BIT(18)) {
+ div_frac1 = val >> 8;
+ div_frac1 &= 0x3ff;
+ }
+
+ /*
+ * Multiply both numerator (mul0) and the denominator (div1) with 1000 to
+ * account for fractional portion of divider
+ */
+
+ div1 *= 1000;
+ div1 += div_frac1;
+ div0 *= div1;
+ mul0 *= 1000;
+ if (div0 == 0) {
+ ICAP_ERR(icap, "clockwiz 0 divider");
+ return 0;
+ }
+ freq = (input * mul0) / div0;
+ } else {
+ icap_read_from_peer(icap->icap_pdev, CLOCK_FREQ_0, (u32 *)&freq, sizeof(u32));
+ }
+ return freq;
+}
+
+static unsigned int icap_get_clock_frequency_counter_khz(const struct icap *icap, int idx)
+{
+ u32 freq, status;
+ char *base = icap->icap_clock_freq_counter;
+ int times;
+
+ times = 10;
+ freq = 0;
+ /*
+ * reset and wait until done
+ */
+ if (ICAP_PRIVILEGED(icap)) {
+ if (uuid_is_null(&icap->icap_bitstream_uuid)) {
+ ICAP_ERR(icap, "ERROR: There isn't a xclbin loaded in the dynamic "
+ "region, frequencies counter cannot be determined");
+ return freq;
+ }
+ reg_wr(base, 0x1);
+
+ while (times != 0) {
+ status = reg_rd(base);
+ if (status == 0x2)
+ break;
+ mdelay(1);
+ times--;
+ };
+
+ freq = reg_rd(base + OCL_CLK_FREQ_COUNTER_OFFSET + idx * sizeof(u32));
+ } else {
+ icap_read_from_peer(icap->icap_pdev, FREQ_COUNTER_0, (u32 *)&freq, sizeof(u32));
+ }
+ return freq;
+}
+/*
+ * Based on Clocking Wizard v5.1, section Dynamic Reconfiguration
+ * through AXI4-Lite
+ */
+static int icap_ocl_freqscaling(struct icap *icap, bool force)
+{
+ unsigned int curr_freq;
+ u32 config;
+ int i;
+ int j = 0;
+ u32 val = 0;
+ unsigned int idx = 0;
+ long err = 0;
+
+ for (i = 0; i < ICAP_MAX_NUM_CLOCKS; ++i) {
+ // A value of zero means skip scaling for this clock index
+ if (!icap->icap_ocl_frequency[i])
+ continue;
+
+ idx = find_matching_freq_config(icap->icap_ocl_frequency[i]);
+ curr_freq = icap_get_ocl_frequency(icap, i);
+ ICAP_INFO(icap, "Clock %d, Current %d Mhz, New %d Mhz ",
+ i, curr_freq, icap->icap_ocl_frequency[i]);
+
+ /*
+ * If current frequency is in the same step as the
+ * requested frequency then nothing to do.
+ */
+ if (!force && (find_matching_freq_config(curr_freq) == idx))
+ continue;
+
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_STATUS_OFFSET);
+ if (val != 1) {
+ ICAP_ERR(icap, "clockwiz %d is busy", i);
+ err = -EBUSY;
+ break;
+ }
+
+ config = frequency_table[idx].config0;
+ reg_wr(icap->icap_clock_bases[i] + OCL_CLKWIZ_CONFIG_OFFSET(0),
+ config);
+ config = frequency_table[idx].config2;
+ reg_wr(icap->icap_clock_bases[i] + OCL_CLKWIZ_CONFIG_OFFSET(2),
+ config);
+ msleep(10);
+ reg_wr(icap->icap_clock_bases[i] + OCL_CLKWIZ_CONFIG_OFFSET(23),
+ 0x00000007);
+ msleep(1);
+ reg_wr(icap->icap_clock_bases[i] + OCL_CLKWIZ_CONFIG_OFFSET(23),
+ 0x00000002);
+
+ ICAP_INFO(icap, "clockwiz waiting for locked signal");
+ msleep(100);
+ for (j = 0; j < 100; j++) {
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_STATUS_OFFSET);
+ if (val != 1) {
+ msleep(100);
+ continue;
+ }
+ }
+ if (val != 1) {
+ ICAP_ERR(icap, "clockwiz MMCM/PLL did not lock after %d ms, "
+ "restoring the original configuration", 100 * 100);
+ /* restore the original clock configuration */
+ reg_wr(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_CONFIG_OFFSET(23), 0x00000004);
+ msleep(10);
+ reg_wr(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_CONFIG_OFFSET(23), 0x00000000);
+ err = -ETIMEDOUT;
+ break;
+ }
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_CONFIG_OFFSET(0));
+ ICAP_INFO(icap, "clockwiz CONFIG(0) 0x%x", val);
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_CONFIG_OFFSET(2));
+ ICAP_INFO(icap, "clockwiz CONFIG(2) 0x%x", val);
+ }
+
+ return err;
+}
+
+static bool icap_bitstream_in_use(struct icap *icap, pid_t pid)
+{
+ BUG_ON(icap->icap_bitstream_ref < 0);
+
+ /* Any user counts if pid isn't specified. */
+ if (pid == 0)
+ return icap->icap_bitstream_ref != 0;
+
+ if (icap->icap_bitstream_ref == 0)
+ return false;
+ if ((icap->icap_bitstream_ref == 1) && obtain_user(icap, pid))
+ return false;
+ return true;
+}
+
+static int icap_freeze_axi_gate_shell(struct icap *icap)
+{
+ xdev_handle_t xdev = xocl_get_xdev(icap->icap_pdev);
+
+ ICAP_INFO(icap, "freezing Shell AXI gate");
+ BUG_ON(icap->icap_axi_gate_shell_frozen);
+
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+ reg_wr(&icap->icap_axi_gate->iag_wr, GATE_FREEZE_SHELL);
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+
+ if (!xocl_is_unified(xdev)) {
+ reg_wr(&icap->icap_regs->ir_cr, 0xc);
+ ndelay(20);
+ } else {
+ /* New ICAP reset sequence applicable only to unified dsa. */
+ reg_wr(&icap->icap_regs->ir_cr, 0x8);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x0);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x4);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x0);
+ ndelay(2000);
+ }
+
+ icap->icap_axi_gate_shell_frozen = true;
+
+ return 0;
+}
+
+static int icap_free_axi_gate_shell(struct icap *icap)
+{
+ int i;
+
+ ICAP_INFO(icap, "freeing Shell AXI gate");
+ /*
+ * First pulse the OCL RESET. This is important for PR with multiple
+ * clocks as it resets the edge triggered clock converter FIFO
+ */
+
+ if (!icap->icap_axi_gate_shell_frozen)
+ return 0;
+
+ for (i = 0; i < ARRAY_SIZE(gate_free_shell); i++) {
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+ reg_wr(&icap->icap_axi_gate->iag_wr, gate_free_shell[i]);
+ mdelay(50);
+ }
+
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+
+ icap->icap_axi_gate_shell_frozen = false;
+
+ return 0;
+}
+
+static int icap_freeze_axi_gate(struct icap *icap)
+{
+ xdev_handle_t xdev = xocl_get_xdev(icap->icap_pdev);
+
+ ICAP_INFO(icap, "freezing CL AXI gate");
+ BUG_ON(icap->icap_axi_gate_frozen);
+
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+ reg_wr(&icap->icap_axi_gate->iag_wr, GATE_FREEZE_USER);
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+
+ if (!xocl_is_unified(xdev)) {
+ reg_wr(&icap->icap_regs->ir_cr, 0xc);
+ ndelay(20);
+ } else {
+ /* New ICAP reset sequence applicable only to unified dsa. */
+ reg_wr(&icap->icap_regs->ir_cr, 0x8);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x0);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x4);
+ ndelay(2000);
+ reg_wr(&icap->icap_regs->ir_cr, 0x0);
+ ndelay(2000);
+ }
+
+ icap->icap_axi_gate_frozen = true;
+
+ return 0;
+}
+
+static int icap_free_axi_gate(struct icap *icap)
+{
+ int i;
+
+ ICAP_INFO(icap, "freeing CL AXI gate");
+ /*
+ * First pulse the OCL RESET. This is important for PR with multiple
+ * clocks as it resets the edge triggered clock converter FIFO
+ */
+
+ if (!icap->icap_axi_gate_frozen)
+ return 0;
+
+ for (i = 0; i < ARRAY_SIZE(gate_free_user); i++) {
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+ reg_wr(&icap->icap_axi_gate->iag_wr, gate_free_user[i]);
+ ndelay(500);
+ }
+
+ (void) reg_rd(&icap->icap_axi_gate->iag_rd);
+
+ icap->icap_axi_gate_frozen = false;
+
+ return 0;
+}
+
+static void platform_reset_axi_gate(struct platform_device *pdev)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+
+ /* Can only be done from mgmt pf. */
+ if (!ICAP_PRIVILEGED(icap))
+ return;
+
+ mutex_lock(&icap->icap_lock);
+ if (!icap_bitstream_in_use(icap, 0)) {
+ (void) icap_freeze_axi_gate(platform_get_drvdata(pdev));
+ msleep(500);
+ (void) icap_free_axi_gate(platform_get_drvdata(pdev));
+ msleep(500);
+ }
+ mutex_unlock(&icap->icap_lock);
+}
+
+static int set_freqs(struct icap *icap, unsigned short *freqs, int num_freqs)
+{
+ int i;
+ int err;
+ u32 val;
+
+ for (i = 0; i < min(ICAP_MAX_NUM_CLOCKS, num_freqs); ++i) {
+ if (freqs[i] == 0)
+ continue;
+
+ val = reg_rd(icap->icap_clock_bases[i] +
+ OCL_CLKWIZ_STATUS_OFFSET);
+ if ((val & 0x1) == 0) {
+ ICAP_ERR(icap, "clockwiz %d is busy", i);
+ err = -EBUSY;
+ goto done;
+ }
+ }
+
+ memcpy(icap->icap_ocl_frequency, freqs,
+ sizeof(*freqs) * min(ICAP_MAX_NUM_CLOCKS, num_freqs));
+
+ icap_freeze_axi_gate(icap);
+ err = icap_ocl_freqscaling(icap, false);
+ icap_free_axi_gate(icap);
+
+done:
+ return err;
+
+}
+
+static int set_and_verify_freqs(struct icap *icap, unsigned short *freqs, int num_freqs)
+{
+ int i;
+ int err;
+ u32 clock_freq_counter, request_in_khz, tolerance;
+
+ err = set_freqs(icap, freqs, num_freqs);
+ if (err)
+ return err;
+
+ for (i = 0; i < min(ICAP_MAX_NUM_CLOCKS, num_freqs); ++i) {
+ if (!freqs[i])
+ continue;
+ clock_freq_counter = icap_get_clock_frequency_counter_khz(icap, i);
+ if (clock_freq_counter == 0) {
+ err = -EDOM;
+ break;
+ }
+ request_in_khz = freqs[i]*1000;
+ tolerance = freqs[i]*50;
+ if (tolerance < abs(clock_freq_counter-request_in_khz)) {
+ ICAP_ERR(icap, "Frequency is higher than tolerance value, request %u khz, "
+ "actual %u khz", request_in_khz, clock_freq_counter);
+ err = -EDOM;
+ break;
+ }
+ }
+
+ return err;
+}
+
+static int icap_ocl_set_freqscaling(struct platform_device *pdev,
+ unsigned int region, unsigned short *freqs, int num_freqs)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int err = 0;
+
+ /* Can only be done from mgmt pf. */
+ if (!ICAP_PRIVILEGED(icap))
+ return -EPERM;
+
+ /* For now, only PR region 0 is supported. */
+ if (region != 0)
+ return -EINVAL;
+
+ mutex_lock(&icap->icap_lock);
+
+ err = set_freqs(icap, freqs, num_freqs);
+
+ mutex_unlock(&icap->icap_lock);
+
+ return err;
+}
+
+static int icap_ocl_update_clock_freq_topology(struct platform_device *pdev, struct xclmgmt_ioc_freqscaling *freq_obj)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ struct clock_freq_topology *topology = 0;
+ int num_clocks = 0;
+ int i = 0;
+ int err = 0;
+
+ mutex_lock(&icap->icap_lock);
+ if (icap->icap_clock_freq_topology) {
+ topology = (struct clock_freq_topology *)icap->icap_clock_freq_topology;
+ num_clocks = topology->m_count;
+ ICAP_INFO(icap, "Num clocks is %d", num_clocks);
+ for (i = 0; i < ARRAY_SIZE(freq_obj->ocl_target_freq); i++) {
+ ICAP_INFO(icap, "requested frequency is : "
+ "%d xclbin freq is: %d",
+ freq_obj->ocl_target_freq[i],
+ topology->m_clock_freq[i].m_freq_Mhz);
+ if (freq_obj->ocl_target_freq[i] >
+ topology->m_clock_freq[i].m_freq_Mhz) {
+ ICAP_ERR(icap, "Unable to set frequency as "
+ "requested frequency %d is greater "
+ "than set by xclbin %d",
+ freq_obj->ocl_target_freq[i],
+ topology->m_clock_freq[i].m_freq_Mhz);
+ err = -EDOM;
+ goto done;
+ }
+ }
+ } else {
+ ICAP_ERR(icap, "ERROR: There isn't a hardware accelerator loaded in the dynamic region."
+ " Validation of accelerator frequencies cannot be determine");
+ err = -EDOM;
+ goto done;
+ }
+
+ err = set_and_verify_freqs(icap, freq_obj->ocl_target_freq, ARRAY_SIZE(freq_obj->ocl_target_freq));
+
+done:
+ mutex_unlock(&icap->icap_lock);
+ return err;
+}
+
+static int icap_ocl_get_freqscaling(struct platform_device *pdev,
+ unsigned int region, unsigned short *freqs, int num_freqs)
+{
+ int i;
+ struct icap *icap = platform_get_drvdata(pdev);
+
+ /* For now, only PR region 0 is supported. */
+ if (region != 0)
+ return -EINVAL;
+
+ mutex_lock(&icap->icap_lock);
+ for (i = 0; i < min(ICAP_MAX_NUM_CLOCKS, num_freqs); i++)
+ freqs[i] = icap_get_ocl_frequency(icap, i);
+ mutex_unlock(&icap->icap_lock);
+
+ return 0;
+}
+
+static inline bool mig_calibration_done(struct icap *icap)
+{
+ return (reg_rd(&icap->icap_state->igs_state) & BIT(0)) != 0;
+}
+
+/* Check for MIG calibration. */
+static int calibrate_mig(struct icap *icap)
+{
+ int i;
+
+ for (i = 0; i < 10 && !mig_calibration_done(icap); ++i)
+ msleep(500);
+
+ if (!mig_calibration_done(icap)) {
+ ICAP_ERR(icap,
+ "MIG calibration timeout after bitstream download");
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+static inline void free_clock_freq_topology(struct icap *icap)
+{
+ vfree(icap->icap_clock_freq_topology);
+ icap->icap_clock_freq_topology = NULL;
+ icap->icap_clock_freq_topology_length = 0;
+}
+
+static int icap_setup_clock_freq_topology(struct icap *icap,
+ const char *buffer, unsigned long length)
+{
+ if (length == 0)
+ return 0;
+
+ free_clock_freq_topology(icap);
+
+ icap->icap_clock_freq_topology = vmalloc(length);
+ if (!icap->icap_clock_freq_topology)
+ return -ENOMEM;
+
+ memcpy(icap->icap_clock_freq_topology, buffer, length);
+ icap->icap_clock_freq_topology_length = length;
+
+ return 0;
+}
+
+static inline void free_clear_bitstream(struct icap *icap)
+{
+ vfree(icap->icap_clear_bitstream);
+ icap->icap_clear_bitstream = NULL;
+ icap->icap_clear_bitstream_length = 0;
+}
+
+static int icap_setup_clear_bitstream(struct icap *icap,
+ const char *buffer, unsigned long length)
+{
+ if (length == 0)
+ return 0;
+
+ free_clear_bitstream(icap);
+
+ icap->icap_clear_bitstream = vmalloc(length);
+ if (!icap->icap_clear_bitstream)
+ return -ENOMEM;
+
+ memcpy(icap->icap_clear_bitstream, buffer, length);
+ icap->icap_clear_bitstream_length = length;
+
+ return 0;
+}
+
+static int wait_for_done(struct icap *icap)
+{
+ u32 w;
+ int i = 0;
+
+ for (i = 0; i < 10; i++) {
+ udelay(5);
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ ICAP_INFO(icap, "XHWICAP_SR: %x", w);
+ if (w & 0x5)
+ return 0;
+ }
+
+ ICAP_ERR(icap, "bitstream download timeout");
+ return -ETIMEDOUT;
+}
+
+static int icap_write(struct icap *icap, const u32 *word_buf, int size)
+{
+ int i;
+ u32 value = 0;
+
+ for (i = 0; i < size; i++) {
+ value = be32_to_cpu(word_buf[i]);
+ reg_wr(&icap->icap_regs->ir_wf, value);
+ }
+
+ reg_wr(&icap->icap_regs->ir_cr, 0x1);
+
+ for (i = 0; i < 20; i++) {
+ value = reg_rd(&icap->icap_regs->ir_cr);
+ if ((value & 0x1) == 0)
+ return 0;
+ ndelay(50);
+ }
+
+ ICAP_ERR(icap, "writing %d dwords timeout", size);
+ return -EIO;
+}
+
+static uint64_t icap_get_section_size(struct icap *icap, enum axlf_section_kind kind)
+{
+ uint64_t size = 0;
+
+ switch (kind) {
+ case IP_LAYOUT:
+ size = sizeof_sect(icap->ip_layout, m_ip_data);
+ break;
+ case MEM_TOPOLOGY:
+ size = sizeof_sect(icap->mem_topo, m_mem_data);
+ break;
+ case DEBUG_IP_LAYOUT:
+ size = sizeof_sect(icap->debug_layout, m_debug_ip_data);
+ break;
+ case CONNECTIVITY:
+ size = sizeof_sect(icap->connectivity, m_connection);
+ break;
+ default:
+ break;
+ }
+
+ return size;
+}
+
+static int bitstream_parse_header(struct icap *icap, const unsigned char *Data,
+ unsigned int Size, struct XHwIcap_Bit_Header *Header)
+{
+ unsigned int I;
+ unsigned int Len;
+ unsigned int Tmp;
+ unsigned int Index;
+
+ /* Start Index at start of bitstream */
+ Index = 0;
+
+ /* Initialize HeaderLength. If header returned early inidicates
+ * failure.
+ */
+ Header->HeaderLength = XHI_BIT_HEADER_FAILURE;
+
+ /* Get "Magic" length */
+ Header->MagicLength = Data[Index++];
+ Header->MagicLength = (Header->MagicLength << 8) | Data[Index++];
+
+ /* Read in "magic" */
+ for (I = 0; I < Header->MagicLength - 1; I++) {
+ Tmp = Data[Index++];
+ if (I%2 == 0 && Tmp != XHI_EVEN_MAGIC_BYTE)
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ if (I%2 == 1 && Tmp != XHI_ODD_MAGIC_BYTE)
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ }
+
+ /* Read null end of magic data. */
+ Tmp = Data[Index++];
+
+ /* Read 0x01 (short) */
+ Tmp = Data[Index++];
+ Tmp = (Tmp << 8) | Data[Index++];
+
+ /* Check the "0x01" half word */
+ if (Tmp != 0x01)
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Read 'a' */
+ Tmp = Data[Index++];
+ if (Tmp != 'a')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get Design Name length */
+ Len = Data[Index++];
+ Len = (Len << 8) | Data[Index++];
+
+ /* allocate space for design name and final null character. */
+ Header->DesignName = kmalloc(Len, GFP_KERNEL);
+
+ /* Read in Design Name */
+ for (I = 0; I < Len; I++)
+ Header->DesignName[I] = Data[Index++];
+
+
+ if (Header->DesignName[Len-1] != '\0')
+ return -1;
+
+ /* Read 'b' */
+ Tmp = Data[Index++];
+ if (Tmp != 'b')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get Part Name length */
+ Len = Data[Index++];
+ Len = (Len << 8) | Data[Index++];
+
+ /* allocate space for part name and final null character. */
+ Header->PartName = kmalloc(Len, GFP_KERNEL);
+
+ /* Read in part name */
+ for (I = 0; I < Len; I++)
+ Header->PartName[I] = Data[Index++];
+
+ if (Header->PartName[Len-1] != '\0')
+ return -1;
+
+ /* Read 'c' */
+ Tmp = Data[Index++];
+ if (Tmp != 'c')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get date length */
+ Len = Data[Index++];
+ Len = (Len << 8) | Data[Index++];
+
+ /* allocate space for date and final null character. */
+ Header->Date = kmalloc(Len, GFP_KERNEL);
+
+ /* Read in date name */
+ for (I = 0; I < Len; I++)
+ Header->Date[I] = Data[Index++];
+
+ if (Header->Date[Len - 1] != '\0')
+ return -1;
+
+ /* Read 'd' */
+ Tmp = Data[Index++];
+ if (Tmp != 'd')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get time length */
+ Len = Data[Index++];
+ Len = (Len << 8) | Data[Index++];
+
+ /* allocate space for time and final null character. */
+ Header->Time = kmalloc(Len, GFP_KERNEL);
+
+ /* Read in time name */
+ for (I = 0; I < Len; I++)
+ Header->Time[I] = Data[Index++];
+
+ if (Header->Time[Len - 1] != '\0')
+ return -1;
+
+ /* Read 'e' */
+ Tmp = Data[Index++];
+ if (Tmp != 'e')
+ return -1; /* INVALID_FILE_HEADER_ERROR */
+
+ /* Get byte length of bitstream */
+ Header->BitstreamLength = Data[Index++];
+ Header->BitstreamLength = (Header->BitstreamLength << 8) | Data[Index++];
+ Header->BitstreamLength = (Header->BitstreamLength << 8) | Data[Index++];
+ Header->BitstreamLength = (Header->BitstreamLength << 8) | Data[Index++];
+ Header->HeaderLength = Index;
+
+ ICAP_INFO(icap, "Design \"%s\"", Header->DesignName);
+ ICAP_INFO(icap, "Part \"%s\"", Header->PartName);
+ ICAP_INFO(icap, "Timestamp \"%s %s\"", Header->Time, Header->Date);
+ ICAP_INFO(icap, "Raw data size 0x%x", Header->BitstreamLength);
+ return 0;
+}
+
+static int bitstream_helper(struct icap *icap, const u32 *word_buffer,
+ unsigned int word_count)
+{
+ unsigned int remain_word;
+ unsigned int word_written = 0;
+ int wr_fifo_vacancy = 0;
+ int err = 0;
+
+ for (remain_word = word_count; remain_word > 0;
+ remain_word -= word_written, word_buffer += word_written) {
+ wr_fifo_vacancy = reg_rd(&icap->icap_regs->ir_wfv);
+ if (wr_fifo_vacancy <= 0) {
+ ICAP_ERR(icap, "no vacancy: %d", wr_fifo_vacancy);
+ err = -EIO;
+ break;
+ }
+ word_written = (wr_fifo_vacancy < remain_word) ?
+ wr_fifo_vacancy : remain_word;
+ if (icap_write(icap, word_buffer, word_written) != 0) {
+ err = -EIO;
+ break;
+ }
+ }
+
+ return err;
+}
+
+static long icap_download(struct icap *icap, const char *buffer,
+ unsigned long length)
+{
+ long err = 0;
+ struct XHwIcap_Bit_Header bit_header = { 0 };
+ unsigned int numCharsRead = DMA_HWICAP_BITFILE_BUFFER_SIZE;
+ unsigned int byte_read;
+
+ BUG_ON(!buffer);
+ BUG_ON(!length);
+
+ if (bitstream_parse_header(icap, buffer,
+ DMA_HWICAP_BITFILE_BUFFER_SIZE, &bit_header)) {
+ err = -EINVAL;
+ goto free_buffers;
+ }
+
+ if ((bit_header.HeaderLength + bit_header.BitstreamLength) > length) {
+ err = -EINVAL;
+ goto free_buffers;
+ }
+
+ buffer += bit_header.HeaderLength;
+
+ for (byte_read = 0; byte_read < bit_header.BitstreamLength;
+ byte_read += numCharsRead) {
+ numCharsRead = bit_header.BitstreamLength - byte_read;
+ if (numCharsRead > DMA_HWICAP_BITFILE_BUFFER_SIZE)
+ numCharsRead = DMA_HWICAP_BITFILE_BUFFER_SIZE;
+
+ err = bitstream_helper(icap, (u32 *)buffer,
+ numCharsRead / sizeof(u32));
+ if (err)
+ goto free_buffers;
+ buffer += numCharsRead;
+ }
+
+ err = wait_for_done(icap);
+
+free_buffers:
+ kfree(bit_header.DesignName);
+ kfree(bit_header.PartName);
+ kfree(bit_header.Date);
+ kfree(bit_header.Time);
+ return err;
+}
+
+static const struct axlf_section_header *get_axlf_section_hdr(
+ struct icap *icap, const struct axlf *top, enum axlf_section_kind kind)
+{
+ int i;
+ const struct axlf_section_header *hdr = NULL;
+
+ ICAP_INFO(icap,
+ "trying to find section header for axlf section %d", kind);
+
+ for (i = 0; i < top->m_header.m_numSections; i++) {
+ ICAP_INFO(icap, "saw section header: %d",
+ top->m_sections[i].m_sectionKind);
+ if (top->m_sections[i].m_sectionKind == kind) {
+ hdr = &top->m_sections[i];
+ break;
+ }
+ }
+
+ if (hdr) {
+ if ((hdr->m_sectionOffset + hdr->m_sectionSize) >
+ top->m_header.m_length) {
+ ICAP_INFO(icap, "found section is invalid");
+ hdr = NULL;
+ } else {
+ ICAP_INFO(icap, "header offset: %llu, size: %llu",
+ hdr->m_sectionOffset, hdr->m_sectionSize);
+ }
+ } else {
+ ICAP_INFO(icap, "could not find section header %d", kind);
+ }
+
+ return hdr;
+}
+
+static int alloc_and_get_axlf_section(struct icap *icap,
+ const struct axlf *top, enum axlf_section_kind kind,
+ void **addr, uint64_t *size)
+{
+ void *section = NULL;
+ const struct axlf_section_header *hdr =
+ get_axlf_section_hdr(icap, top, kind);
+
+ if (hdr == NULL)
+ return -EINVAL;
+
+ section = vmalloc(hdr->m_sectionSize);
+ if (section == NULL)
+ return -ENOMEM;
+
+ memcpy(section, ((const char *)top) + hdr->m_sectionOffset,
+ hdr->m_sectionSize);
+
+ *addr = section;
+ *size = hdr->m_sectionSize;
+ return 0;
+}
+
+static int icap_download_boot_firmware(struct platform_device *pdev)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ struct pci_dev *pcidev = XOCL_PL_TO_PCI_DEV(pdev);
+ struct pci_dev *pcidev_user = NULL;
+ xdev_handle_t xdev = xocl_get_xdev(pdev);
+ int funcid = PCI_FUNC(pcidev->devfn);
+ int slotid = PCI_SLOT(pcidev->devfn);
+ unsigned short deviceid = pcidev->device;
+ struct axlf *bin_obj_axlf;
+ const struct firmware *fw;
+ char fw_name[128];
+ struct XHwIcap_Bit_Header bit_header = { 0 };
+ long err = 0;
+ uint64_t length = 0;
+ uint64_t primaryFirmwareOffset = 0;
+ uint64_t primaryFirmwareLength = 0;
+ uint64_t secondaryFirmwareOffset = 0;
+ uint64_t secondaryFirmwareLength = 0;
+ uint64_t mbBinaryOffset = 0;
+ uint64_t mbBinaryLength = 0;
+ const struct axlf_section_header *primaryHeader = 0;
+ const struct axlf_section_header *secondaryHeader = 0;
+ const struct axlf_section_header *mbHeader = 0;
+ bool load_mbs = false;
+
+ /* Can only be done from mgmt pf. */
+ if (!ICAP_PRIVILEGED(icap))
+ return -EPERM;
+
+ /* Read dsabin from file system. */
+
+ if (funcid != 0) {
+ pcidev_user = pci_get_slot(pcidev->bus,
+ PCI_DEVFN(slotid, funcid - 1));
+ if (!pcidev_user) {
+ pcidev_user = pci_get_device(pcidev->vendor,
+ pcidev->device + 1, NULL);
+ }
+ if (pcidev_user)
+ deviceid = pcidev_user->device;
+ }
+
+ snprintf(fw_name, sizeof(fw_name),
+ "xilinx/%04x-%04x-%04x-%016llx.dsabin",
+ le16_to_cpu(pcidev->vendor),
+ le16_to_cpu(deviceid),
+ le16_to_cpu(pcidev->subsystem_device),
+ le64_to_cpu(xocl_get_timestamp(xdev)));
+ ICAP_INFO(icap, "try load dsabin %s", fw_name);
+ err = request_firmware(&fw, fw_name, &pcidev->dev);
+ if (err) {
+ snprintf(fw_name, sizeof(fw_name),
+ "xilinx/%04x-%04x-%04x-%016llx.dsabin",
+ le16_to_cpu(pcidev->vendor),
+ le16_to_cpu(deviceid + 1),
+ le16_to_cpu(pcidev->subsystem_device),
+ le64_to_cpu(xocl_get_timestamp(xdev)));
+ ICAP_INFO(icap, "try load dsabin %s", fw_name);
+ err = request_firmware(&fw, fw_name, &pcidev->dev);
+ }
+ /* Retry with the legacy dsabin. */
+ if (err) {
+ snprintf(fw_name, sizeof(fw_name),
+ "xilinx/%04x-%04x-%04x-%016llx.dsabin",
+ le16_to_cpu(pcidev->vendor),
+ le16_to_cpu(pcidev->device + 1),
+ le16_to_cpu(pcidev->subsystem_device),
+ le64_to_cpu(0x0000000000000000));
+ ICAP_INFO(icap, "try load dsabin %s", fw_name);
+ err = request_firmware(&fw, fw_name, &pcidev->dev);
+ }
+ if (err) {
+ /* Give up on finding .dsabin. */
+ ICAP_ERR(icap, "unable to find firmware, giving up");
+ return err;
+ }
+
+ /* Grab lock and touch hardware. */
+ mutex_lock(&icap->icap_lock);
+
+ if (xocl_mb_sched_on(xdev)) {
+ /* Try locating the microblaze binary. */
+ bin_obj_axlf = (struct axlf *)fw->data;
+ mbHeader = get_axlf_section_hdr(icap, bin_obj_axlf, SCHED_FIRMWARE);
+ if (mbHeader) {
+ mbBinaryOffset = mbHeader->m_sectionOffset;
+ mbBinaryLength = mbHeader->m_sectionSize;
+ length = bin_obj_axlf->m_header.m_length;
+ xocl_mb_load_sche_image(xdev, fw->data + mbBinaryOffset,
+ mbBinaryLength);
+ ICAP_INFO(icap, "stashed mb sche binary");
+ load_mbs = true;
+ }
+ }
+
+ if (xocl_mb_mgmt_on(xdev)) {
+ /* Try locating the board mgmt binary. */
+ bin_obj_axlf = (struct axlf *)fw->data;
+ mbHeader = get_axlf_section_hdr(icap, bin_obj_axlf, FIRMWARE);
+ if (mbHeader) {
+ mbBinaryOffset = mbHeader->m_sectionOffset;
+ mbBinaryLength = mbHeader->m_sectionSize;
+ length = bin_obj_axlf->m_header.m_length;
+ xocl_mb_load_mgmt_image(xdev, fw->data + mbBinaryOffset,
+ mbBinaryLength);
+ ICAP_INFO(icap, "stashed mb mgmt binary");
+ load_mbs = true;
+ }
+ }
+
+ if (load_mbs)
+ xocl_mb_reset(xdev);
+
+
+ if (memcmp(fw->data, ICAP_XCLBIN_V2, sizeof(ICAP_XCLBIN_V2)) != 0) {
+ ICAP_ERR(icap, "invalid firmware %s", fw_name);
+ err = -EINVAL;
+ goto done;
+ }
+
+ ICAP_INFO(icap, "boot_firmware in axlf format");
+ bin_obj_axlf = (struct axlf *)fw->data;
+ length = bin_obj_axlf->m_header.m_length;
+ /* Match the xclbin with the hardware. */
+ if (!xocl_verify_timestamp(xdev,
+ bin_obj_axlf->m_header.m_featureRomTimeStamp)) {
+ ICAP_ERR(icap, "timestamp of ROM did not match xclbin");
+ err = -EINVAL;
+ goto done;
+ }
+ ICAP_INFO(icap, "VBNV and timestamps matched");
+
+ if (xocl_xrt_version_check(xdev, bin_obj_axlf, true)) {
+ ICAP_ERR(icap, "Major version does not match xrt");
+ err = -EINVAL;
+ goto done;
+ }
+ ICAP_INFO(icap, "runtime version matched");
+
+ primaryHeader = get_axlf_section_hdr(icap, bin_obj_axlf, BITSTREAM);
+ secondaryHeader = get_axlf_section_hdr(icap, bin_obj_axlf,
+ CLEARING_BITSTREAM);
+ if (primaryHeader) {
+ primaryFirmwareOffset = primaryHeader->m_sectionOffset;
+ primaryFirmwareLength = primaryHeader->m_sectionSize;
+ }
+ if (secondaryHeader) {
+ secondaryFirmwareOffset = secondaryHeader->m_sectionOffset;
+ secondaryFirmwareLength = secondaryHeader->m_sectionSize;
+ }
+
+ if (length > fw->size) {
+ err = -EINVAL;
+ goto done;
+ }
+
+ if ((primaryFirmwareOffset + primaryFirmwareLength) > length) {
+ err = -EINVAL;
+ goto done;
+ }
+
+ if ((secondaryFirmwareOffset + secondaryFirmwareLength) > length) {
+ err = -EINVAL;
+ goto done;
+ }
+
+ if (primaryFirmwareLength) {
+ ICAP_INFO(icap,
+ "found second stage bitstream of size 0x%llx in %s",
+ primaryFirmwareLength, fw_name);
+ err = icap_download(icap, fw->data + primaryFirmwareOffset,
+ primaryFirmwareLength);
+ /*
+ * If we loaded a new second stage, we do not need the
+ * previously stashed clearing bitstream if any.
+ */
+ free_clear_bitstream(icap);
+ if (err) {
+ ICAP_ERR(icap,
+ "failed to download second stage bitstream");
+ goto done;
+ }
+ ICAP_INFO(icap, "downloaded second stage bitstream");
+ }
+
+ /*
+ * If both primary and secondary bitstreams have been provided then
+ * ignore the previously stashed bitstream if any. If only secondary
+ * bitstream was provided, but we found a previously stashed bitstream
+ * we should use the latter since it is more appropriate for the
+ * current state of the device
+ */
+ if (secondaryFirmwareLength && (primaryFirmwareLength ||
+ !icap->icap_clear_bitstream)) {
+ free_clear_bitstream(icap);
+ icap->icap_clear_bitstream = vmalloc(secondaryFirmwareLength);
+ if (!icap->icap_clear_bitstream) {
+ err = -ENOMEM;
+ goto done;
+ }
+ icap->icap_clear_bitstream_length = secondaryFirmwareLength;
+ memcpy(icap->icap_clear_bitstream,
+ fw->data + secondaryFirmwareOffset,
+ icap->icap_clear_bitstream_length);
+ ICAP_INFO(icap, "found clearing bitstream of size 0x%lx in %s",
+ icap->icap_clear_bitstream_length, fw_name);
+ } else if (icap->icap_clear_bitstream) {
+ ICAP_INFO(icap,
+ "using existing clearing bitstream of size 0x%lx",
+ icap->icap_clear_bitstream_length);
+ }
+
+ if (icap->icap_clear_bitstream &&
+ bitstream_parse_header(icap, icap->icap_clear_bitstream,
+ DMA_HWICAP_BITFILE_BUFFER_SIZE, &bit_header)) {
+ err = -EINVAL;
+ free_clear_bitstream(icap);
+ }
+
+done:
+ mutex_unlock(&icap->icap_lock);
+ release_firmware(fw);
+ kfree(bit_header.DesignName);
+ kfree(bit_header.PartName);
+ kfree(bit_header.Date);
+ kfree(bit_header.Time);
+ ICAP_INFO(icap, "%s err: %ld", __func__, err);
+ return err;
+}
+
+
+static long icap_download_clear_bitstream(struct icap *icap)
+{
+ long err = 0;
+ const char *buffer = icap->icap_clear_bitstream;
+ unsigned long length = icap->icap_clear_bitstream_length;
+
+ ICAP_INFO(icap, "downloading clear bitstream of length 0x%lx", length);
+
+ if (!buffer)
+ return 0;
+
+ err = icap_download(icap, buffer, length);
+
+ free_clear_bitstream(icap);
+ return err;
+}
+
+/*
+ * This function should be called with icap_mutex lock held
+ */
+static long axlf_set_freqscaling(struct icap *icap, struct platform_device *pdev,
+ const char *clk_buf, unsigned long length)
+{
+ struct clock_freq_topology *freqs = NULL;
+ int clock_type_count = 0;
+ int i = 0;
+ struct clock_freq *freq = NULL;
+ int data_clk_count = 0;
+ int kernel_clk_count = 0;
+ int system_clk_count = 0;
+ unsigned short target_freqs[4] = {0};
+
+ freqs = (struct clock_freq_topology *)clk_buf;
+ if (freqs->m_count > 4) {
+ ICAP_ERR(icap, "More than 4 clocks found in clock topology");
+ return -EDOM;
+ }
+
+ //Error checks - we support 1 data clk (reqd), one kernel clock(reqd) and
+ //at most 2 system clocks (optional/reqd for aws).
+ //Data clk needs to be the first entry, followed by kernel clock
+ //and then system clocks
+ //
+
+ for (i = 0; i < freqs->m_count; i++) {
+ freq = &(freqs->m_clock_freq[i]);
+ if (freq->m_type == CT_DATA)
+ data_clk_count++;
+
+ if (freq->m_type == CT_KERNEL)
+ kernel_clk_count++;
+
+ if (freq->m_type == CT_SYSTEM)
+ system_clk_count++;
+
+ }
+
+ if (data_clk_count != 1) {
+ ICAP_ERR(icap, "Data clock not found in clock topology");
+ return -EDOM;
+ }
+ if (kernel_clk_count != 1) {
+ ICAP_ERR(icap, "Kernel clock not found in clock topology");
+ return -EDOM;
+ }
+ if (system_clk_count > 2) {
+ ICAP_ERR(icap,
+ "More than 2 system clocks found in clock topology");
+ return -EDOM;
+ }
+
+ for (i = 0; i < freqs->m_count; i++) {
+ freq = &(freqs->m_clock_freq[i]);
+ if (freq->m_type == CT_DATA)
+ target_freqs[0] = freq->m_freq_Mhz;
+ }
+
+ for (i = 0; i < freqs->m_count; i++) {
+ freq = &(freqs->m_clock_freq[i]);
+ if (freq->m_type == CT_KERNEL)
+ target_freqs[1] = freq->m_freq_Mhz;
+ }
+
+ clock_type_count = 2;
+ for (i = 0; i < freqs->m_count; i++) {
+ freq = &(freqs->m_clock_freq[i]);
+ if (freq->m_type == CT_SYSTEM)
+ target_freqs[clock_type_count++] = freq->m_freq_Mhz;
+ }
+
+
+ ICAP_INFO(icap, "setting clock freq, "
+ "num: %lu, data_freq: %d , clk_freq: %d, "
+ "sys_freq[0]: %d, sys_freq[1]: %d",
+ ARRAY_SIZE(target_freqs), target_freqs[0], target_freqs[1],
+ target_freqs[2], target_freqs[3]);
+ return set_freqs(icap, target_freqs, 4);
+}
+
+
+static int icap_download_user(struct icap *icap, const char *bit_buf,
+ unsigned long length)
+{
+ long err = 0;
+ struct XHwIcap_Bit_Header bit_header = { 0 };
+ unsigned int numCharsRead = DMA_HWICAP_BITFILE_BUFFER_SIZE;
+ unsigned int byte_read;
+
+ ICAP_INFO(icap, "downloading bitstream, length: %lu", length);
+
+ icap_freeze_axi_gate(icap);
+
+ err = icap_download_clear_bitstream(icap);
+ if (err)
+ goto free_buffers;
+
+ if (bitstream_parse_header(icap, bit_buf,
+ DMA_HWICAP_BITFILE_BUFFER_SIZE, &bit_header)) {
+ err = -EINVAL;
+ goto free_buffers;
+ }
+ if ((bit_header.HeaderLength + bit_header.BitstreamLength) > length) {
+ err = -EINVAL;
+ goto free_buffers;
+ }
+
+ bit_buf += bit_header.HeaderLength;
+ for (byte_read = 0; byte_read < bit_header.BitstreamLength;
+ byte_read += numCharsRead) {
+ numCharsRead = bit_header.BitstreamLength - byte_read;
+ if (numCharsRead > DMA_HWICAP_BITFILE_BUFFER_SIZE)
+ numCharsRead = DMA_HWICAP_BITFILE_BUFFER_SIZE;
+
+ err = bitstream_helper(icap, (u32 *)bit_buf,
+ numCharsRead / sizeof(u32));
+ if (err)
+ goto free_buffers;
+
+ bit_buf += numCharsRead;
+ }
+
+ err = wait_for_done(icap);
+ if (err)
+ goto free_buffers;
+
+ /*
+ * Perform frequency scaling since PR download can silenty overwrite
+ * MMCM settings in static region changing the clock frequencies
+ * although ClockWiz CONFIG registers will misleading report the older
+ * configuration from before bitstream download as if nothing has
+ * changed.
+ */
+ if (!err)
+ err = icap_ocl_freqscaling(icap, true);
+
+free_buffers:
+ icap_free_axi_gate(icap);
+ kfree(bit_header.DesignName);
+ kfree(bit_header.PartName);
+ kfree(bit_header.Date);
+ kfree(bit_header.Time);
+ return err;
+}
+
+
+static int __icap_lock_peer(struct platform_device *pdev, const uuid_t *id)
+{
+ int err = 0;
+ struct icap *icap = platform_get_drvdata(pdev);
+ int resp = 0;
+ size_t resplen = sizeof(resp);
+ struct mailbox_req_bitstream_lock bitstream_lock = {0};
+ size_t data_len = sizeof(struct mailbox_req_bitstream_lock);
+ struct mailbox_req *mb_req = NULL;
+ size_t reqlen = sizeof(struct mailbox_req) + data_len;
+ /* if there is no user there
+ * ask mgmt to lock the bitstream
+ */
+ if (icap->icap_bitstream_ref == 0) {
+ mb_req = vmalloc(reqlen);
+ if (!mb_req) {
+ err = -ENOMEM;
+ goto done;
+ }
+
+ mb_req->req = MAILBOX_REQ_LOCK_BITSTREAM;
+ uuid_copy(&bitstream_lock.uuid, id);
+
+ memcpy(mb_req->data, &bitstream_lock, data_len);
+
+ err = xocl_peer_request(XOCL_PL_DEV_TO_XDEV(pdev),
+ mb_req, reqlen, &resp, &resplen, NULL, NULL);
+
+ if (err) {
+ err = -ENODEV;
+ goto done;
+ }
+
+ if (resp < 0) {
+ err = resp;
+ goto done;
+ }
+ }
+
+done:
+ vfree(mb_req);
+ return err;
+}
+
+static int __icap_unlock_peer(struct platform_device *pdev, const uuid_t *id)
+{
+ int err = 0;
+ struct icap *icap = platform_get_drvdata(pdev);
+ struct mailbox_req_bitstream_lock bitstream_lock = {0};
+ size_t data_len = sizeof(struct mailbox_req_bitstream_lock);
+ struct mailbox_req *mb_req = NULL;
+ size_t reqlen = sizeof(struct mailbox_req) + data_len;
+ /* if there is no user there
+ * ask mgmt to unlock the bitstream
+ */
+ if (icap->icap_bitstream_ref == 0) {
+ mb_req = vmalloc(reqlen);
+ if (!mb_req) {
+ err = -ENOMEM;
+ goto done;
+ }
+
+ mb_req->req = MAILBOX_REQ_UNLOCK_BITSTREAM;
+ memcpy(mb_req->data, &bitstream_lock, data_len);
+
+ err = xocl_peer_notify(XOCL_PL_DEV_TO_XDEV(pdev), mb_req, reqlen);
+ if (err) {
+ err = -ENODEV;
+ goto done;
+ }
+ }
+done:
+ vfree(mb_req);
+ return err;
+}
+
+
+static int icap_download_bitstream_axlf(struct platform_device *pdev,
+ const void *u_xclbin)
+{
+ /*
+ * decouple as 1. download xclbin, 2. parse xclbin 3. verify xclbin
+ */
+ struct icap *icap = platform_get_drvdata(pdev);
+ long err = 0;
+ uint64_t primaryFirmwareOffset = 0;
+ uint64_t primaryFirmwareLength = 0;
+ uint64_t secondaryFirmwareOffset = 0;
+ uint64_t secondaryFirmwareLength = 0;
+ const struct axlf_section_header *primaryHeader = NULL;
+ const struct axlf_section_header *clockHeader = NULL;
+ const struct axlf_section_header *secondaryHeader = NULL;
+ struct axlf *xclbin = (struct axlf *)u_xclbin;
+ char *buffer;
+ xdev_handle_t xdev = xocl_get_xdev(pdev);
+ bool need_download;
+ int msg = -ETIMEDOUT;
+ size_t resplen = sizeof(msg);
+ int pid = pid_nr(task_tgid(current));
+ uint32_t data_len = 0;
+ int peer_connected;
+ struct mailbox_req *mb_req = NULL;
+ struct mailbox_bitstream_kaddr mb_addr = {0};
+ uuid_t peer_uuid;
+
+ if (memcmp(xclbin->m_magic, ICAP_XCLBIN_V2, sizeof(ICAP_XCLBIN_V2)))
+ return -EINVAL;
+
+ if (ICAP_PRIVILEGED(icap)) {
+ if (xocl_xrt_version_check(xdev, xclbin, true)) {
+ ICAP_ERR(icap, "XRT version does not match");
+ return -EINVAL;
+ }
+
+ /* Match the xclbin with the hardware. */
+ if (!xocl_verify_timestamp(xdev,
+ xclbin->m_header.m_featureRomTimeStamp)) {
+ ICAP_ERR(icap, "timestamp of ROM not match Xclbin");
+ xocl_sysfs_error(xdev, "timestamp of ROM not match Xclbin");
+ return -EINVAL;
+ }
+
+ mutex_lock(&icap->icap_lock);
+
+ ICAP_INFO(icap,
+ "incoming xclbin: %016llx, on device xclbin: %016llx",
+ xclbin->m_uniqueId, icap->icap_bitstream_id);
+
+ need_download = (icap->icap_bitstream_id != xclbin->m_uniqueId);
+
+ if (!need_download) {
+ /*
+ * No need to download, if xclbin exists already.
+ * But, still need to reset CUs.
+ */
+ if (!icap_bitstream_in_use(icap, 0)) {
+ icap_freeze_axi_gate(icap);
+ msleep(50);
+ icap_free_axi_gate(icap);
+ msleep(50);
+ }
+ ICAP_INFO(icap, "bitstream already exists, skip downloading");
+ }
+
+ mutex_unlock(&icap->icap_lock);
+
+ if (!need_download)
+ return 0;
+
+ /*
+ * Find sections in xclbin.
+ */
+ ICAP_INFO(icap, "finding CLOCK_FREQ_TOPOLOGY section");
+ /* Read the CLOCK section but defer changing clocks to later */
+ clockHeader = get_axlf_section_hdr(icap, xclbin,
+ CLOCK_FREQ_TOPOLOGY);
+
+ ICAP_INFO(icap, "finding bitstream sections");
+ primaryHeader = get_axlf_section_hdr(icap, xclbin, BITSTREAM);
+ if (primaryHeader == NULL) {
+ err = -EINVAL;
+ goto done;
+ }
+ primaryFirmwareOffset = primaryHeader->m_sectionOffset;
+ primaryFirmwareLength = primaryHeader->m_sectionSize;
+
+ secondaryHeader = get_axlf_section_hdr(icap, xclbin,
+ CLEARING_BITSTREAM);
+ if (secondaryHeader) {
+ if (XOCL_PL_TO_PCI_DEV(pdev)->device == 0x7138) {
+ err = -EINVAL;
+ goto done;
+ } else {
+ secondaryFirmwareOffset =
+ secondaryHeader->m_sectionOffset;
+ secondaryFirmwareLength =
+ secondaryHeader->m_sectionSize;
+ }
+ }
+
+ mutex_lock(&icap->icap_lock);
+
+ if (icap_bitstream_in_use(icap, 0)) {
+ ICAP_ERR(icap, "bitstream is locked, can't download new one");
+ err = -EBUSY;
+ goto done;
+ }
+
+ /* All clear, go ahead and start fiddling with hardware */
+
+ if (clockHeader != NULL) {
+ uint64_t clockFirmwareOffset = clockHeader->m_sectionOffset;
+ uint64_t clockFirmwareLength = clockHeader->m_sectionSize;
+
+ buffer = (char *)xclbin;
+ buffer += clockFirmwareOffset;
+ err = axlf_set_freqscaling(icap, pdev, buffer, clockFirmwareLength);
+ if (err)
+ goto done;
+ err = icap_setup_clock_freq_topology(icap, buffer, clockFirmwareLength);
+ if (err)
+ goto done;
+ }
+
+ icap->icap_bitstream_id = 0;
+ uuid_copy(&icap->icap_bitstream_uuid, &uuid_null);
+
+ buffer = (char *)xclbin;
+ buffer += primaryFirmwareOffset;
+ err = icap_download_user(icap, buffer, primaryFirmwareLength);
+ if (err)
+ goto done;
+
+ buffer = (char *)u_xclbin;
+ buffer += secondaryFirmwareOffset;
+ err = icap_setup_clear_bitstream(icap, buffer, secondaryFirmwareLength);
+ if (err)
+ goto done;
+
+ if ((xocl_is_unified(xdev) || XOCL_DSA_XPR_ON(xdev)))
+ err = calibrate_mig(icap);
+ if (err)
+ goto done;
+
+ /* Remember "this" bitstream, so avoid redownload the next time. */
+ icap->icap_bitstream_id = xclbin->m_uniqueId;
+ if (!uuid_is_null(&xclbin->m_header.uuid)) {
+ uuid_copy(&icap->icap_bitstream_uuid, &xclbin->m_header.uuid);
+ } else {
+ // Legacy xclbin, convert legacy id to new id
+ memcpy(&icap->icap_bitstream_uuid,
+ &xclbin->m_header.m_timeStamp, 8);
+ }
+ } else {
+
+ mutex_lock(&icap->icap_lock);
+
+ if (icap_bitstream_in_use(icap, pid)) {
+ if (!uuid_equal(&xclbin->m_header.uuid, &icap->icap_bitstream_uuid)) {
+ err = -EBUSY;
+ goto done;
+ }
+ }
+
+ icap_read_from_peer(pdev, XCLBIN_UUID, &peer_uuid, sizeof(uuid_t));
+
+ if (!uuid_equal(&peer_uuid, &xclbin->m_header.uuid)) {
+ /*
+ * should replace with userpf download flow
+ */
+ peer_connected = xocl_mailbox_get_data(xdev, PEER_CONN);
+ ICAP_INFO(icap, "%s peer_connected 0x%x", __func__,
+ peer_connected);
+ if (peer_connected < 0) {
+ err = -ENODEV;
+ goto done;
+ }
+
+ if (!(peer_connected & MB_PEER_CONNECTED)) {
+ ICAP_ERR(icap, "%s fail to find peer, abort!",
+ __func__);
+ err = -EFAULT;
+ goto done;
+ }
+
+ if ((peer_connected & 0xF) == MB_PEER_SAMEDOM_CONNECTED) {
+ data_len = sizeof(struct mailbox_req) + sizeof(struct mailbox_bitstream_kaddr);
+ mb_req = vmalloc(data_len);
+ if (!mb_req) {
+ ICAP_ERR(icap, "Unable to create mb_req\n");
+ err = -ENOMEM;
+ goto done;
+ }
+ mb_req->req = MAILBOX_REQ_LOAD_XCLBIN_KADDR;
+ mb_addr.addr = (uint64_t)xclbin;
+ memcpy(mb_req->data, &mb_addr, sizeof(struct mailbox_bitstream_kaddr));
+
+ } else if ((peer_connected & 0xF) == MB_PEER_CONNECTED) {
+ data_len = sizeof(struct mailbox_req) +
+ xclbin->m_header.m_length;
+ mb_req = vmalloc(data_len);
+ if (!mb_req) {
+ ICAP_ERR(icap, "Unable to create mb_req\n");
+ err = -ENOMEM;
+ goto done;
+ }
+ memcpy(mb_req->data, u_xclbin, xclbin->m_header.m_length);
+ mb_req->req = MAILBOX_REQ_LOAD_XCLBIN;
+ }
+
+ mb_req->data_total_len = data_len;
+ (void) xocl_peer_request(xdev,
+ mb_req, data_len, &msg, &resplen, NULL, NULL);
+
+ if (msg != 0) {
+ ICAP_ERR(icap,
+ "%s peer failed to download xclbin",
+ __func__);
+ err = -EFAULT;
+ goto done;
+ }
+ } else
+ ICAP_INFO(icap, "Already downloaded xclbin ID: %016llx",
+ xclbin->m_uniqueId);
+
+ icap->icap_bitstream_id = xclbin->m_uniqueId;
+ if (!uuid_is_null(&xclbin->m_header.uuid)) {
+ uuid_copy(&icap->icap_bitstream_uuid, &xclbin->m_header.uuid);
+ } else {
+ // Legacy xclbin, convert legacy id to new id
+ memcpy(&icap->icap_bitstream_uuid,
+ &xclbin->m_header.m_timeStamp, 8);
+ }
+
+ }
+
+ if (ICAP_PRIVILEGED(icap)) {
+ icap_parse_bitstream_axlf_section(pdev, xclbin, MEM_TOPOLOGY);
+ icap_parse_bitstream_axlf_section(pdev, xclbin, IP_LAYOUT);
+ } else {
+ icap_parse_bitstream_axlf_section(pdev, xclbin, IP_LAYOUT);
+ icap_parse_bitstream_axlf_section(pdev, xclbin, MEM_TOPOLOGY);
+ icap_parse_bitstream_axlf_section(pdev, xclbin, CONNECTIVITY);
+ icap_parse_bitstream_axlf_section(pdev, xclbin, DEBUG_IP_LAYOUT);
+ }
+
+ if (ICAP_PRIVILEGED(icap))
+ err = icap_verify_bitstream_axlf(pdev, xclbin);
+
+done:
+ mutex_unlock(&icap->icap_lock);
+ vfree(mb_req);
+ ICAP_INFO(icap, "%s err: %ld", __func__, err);
+ return err;
+}
+
+static int icap_verify_bitstream_axlf(struct platform_device *pdev,
+ struct axlf *xclbin)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int err = 0, i;
+ xdev_handle_t xdev = xocl_get_xdev(pdev);
+ bool dna_check = false;
+ uint64_t section_size = 0;
+
+ /* Destroy all dynamically add sub-devices*/
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_DNA);
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_MIG);
+
+ /*
+ * Add sub device dynamically.
+ * restrict any dynamically added sub-device and 1 base address,
+ * Has pre-defined length
+ * Ex: "ip_data": {
+ * "m_type": "IP_DNASC",
+ * "properties": "0x0",
+ * "m_base_address": "0x1100000", <-- base address
+ * "m_name": "slr0\/dna_self_check_0"
+ */
+
+ if (!icap->ip_layout) {
+ err = -EFAULT;
+ goto done;
+ }
+ for (i = 0; i < icap->ip_layout->m_count; ++i) {
+ struct xocl_subdev_info subdev_info = { 0 };
+ struct resource res = { 0 };
+ struct ip_data *ip = &icap->ip_layout->m_ip_data[i];
+
+ if (ip->m_type == IP_KERNEL)
+ continue;
+
+ if (ip->m_type == IP_DDR4_CONTROLLER) {
+ uint32_t memidx = ip->properties;
+
+ if (!icap->mem_topo || ip->properties >= icap->mem_topo->m_count ||
+ icap->mem_topo->m_mem_data[memidx].m_type !=
+ MEM_DDR4) {
+ ICAP_ERR(icap, "bad ECC controller index: %u",
+ ip->properties);
+ continue;
+ }
+ if (!icap->mem_topo->m_mem_data[memidx].m_used) {
+ ICAP_INFO(icap,
+ "ignore ECC controller for: %s",
+ icap->mem_topo->m_mem_data[memidx].m_tag);
+ continue;
+ }
+ err = xocl_subdev_get_devinfo(XOCL_SUBDEV_MIG,
+ &subdev_info, &res);
+ if (err) {
+ ICAP_ERR(icap, "can't get MIG subdev info");
+ goto done;
+ }
+ res.start += ip->m_base_address;
+ res.end += ip->m_base_address;
+ subdev_info.priv_data =
+ icap->mem_topo->m_mem_data[memidx].m_tag;
+ subdev_info.data_len =
+ sizeof(icap->mem_topo->m_mem_data[memidx].m_tag);
+ err = xocl_subdev_create_multi_inst(xdev, &subdev_info);
+ if (err) {
+ ICAP_ERR(icap, "can't create MIG subdev");
+ goto done;
+ }
+ }
+ if (ip->m_type == IP_DNASC) {
+ dna_check = true;
+ err = xocl_subdev_get_devinfo(XOCL_SUBDEV_DNA,
+ &subdev_info, &res);
+ if (err) {
+ ICAP_ERR(icap, "can't get DNA subdev info");
+ goto done;
+ }
+ res.start += ip->m_base_address;
+ res.end += ip->m_base_address;
+ err = xocl_subdev_create_one(xdev, &subdev_info);
+ if (err) {
+ ICAP_ERR(icap, "can't create DNA subdev");
+ goto done;
+ }
+ }
+ }
+
+ if (dna_check) {
+ bool is_axi = ((xocl_dna_capability(xdev) & 0x1) != 0);
+
+ /*
+ * Any error occurs here should return -EACCES for app to
+ * know that DNA has failed.
+ */
+ err = -EACCES;
+
+ ICAP_INFO(icap, "DNA version: %s", is_axi ? "AXI" : "BRAM");
+
+ if (is_axi) {
+ uint32_t *cert = NULL;
+
+ if (alloc_and_get_axlf_section(icap, xclbin,
+ DNA_CERTIFICATE,
+ (void **)&cert, §ion_size) != 0) {
+
+ // We keep dna sub device if IP_DNASC presents
+ ICAP_ERR(icap, "Can't get certificate section");
+ goto dna_cert_fail;
+ }
+
+ ICAP_INFO(icap, "DNA Certificate Size 0x%llx", section_size);
+ if (section_size % 64 || section_size < 576)
+ ICAP_ERR(icap, "Invalid certificate size");
+ else
+ xocl_dna_write_cert(xdev, cert, section_size);
+ vfree(cert);
+ }
+
+ /* Check DNA validation result. */
+ if (0x1 & xocl_dna_status(xdev)) {
+ err = 0; /* xclbin is valid */
+ } else {
+ ICAP_ERR(icap, "DNA inside xclbin is invalid");
+ goto dna_cert_fail;
+ }
+ }
+
+done:
+ if (err) {
+ vfree(icap->connectivity);
+ icap->connectivity = NULL;
+ vfree(icap->ip_layout);
+ icap->ip_layout = NULL;
+ vfree(icap->mem_topo);
+ icap->mem_topo = NULL;
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_DNA);
+ xocl_subdev_destroy_by_id(xdev, XOCL_SUBDEV_MIG);
+ }
+dna_cert_fail:
+ return err;
+}
+
+/*
+ * On x86_64, reset hwicap by loading special bitstream sequence which
+ * forces the FPGA to reload from PROM.
+ */
+static int icap_reset_bitstream(struct platform_device *pdev)
+{
+/*
+ * Booting FPGA from PROM
+ * http://www.xilinx.com/support/documentation/user_guides/ug470_7Series_Config.pdf
+ * Table 7.1
+ */
+#define DUMMY_WORD 0xFFFFFFFF
+#define SYNC_WORD 0xAA995566
+#define TYPE1_NOOP 0x20000000
+#define TYPE1_WRITE_WBSTAR 0x30020001
+#define WBSTAR_ADD10 0x00000000
+#define WBSTAR_ADD11 0x01000000
+#define TYPE1_WRITE_CMD 0x30008001
+#define IPROG_CMD 0x0000000F
+#define SWAP_ENDIAN_32(x) \
+ (unsigned int)((((x) & 0xFF000000) >> 24) | (((x) & 0x00FF0000) >> 8) | \
+ (((x) & 0x0000FF00) << 8) | (((x) & 0x000000FF) << 24))
+ /*
+ * The bitstream is expected in big endian format
+ */
+ const unsigned int fpga_boot_seq[] = {SWAP_ENDIAN_32(DUMMY_WORD),
+ SWAP_ENDIAN_32(SYNC_WORD),
+ SWAP_ENDIAN_32(TYPE1_NOOP),
+ SWAP_ENDIAN_32(TYPE1_WRITE_CMD),
+ SWAP_ENDIAN_32(IPROG_CMD),
+ SWAP_ENDIAN_32(TYPE1_NOOP),
+ SWAP_ENDIAN_32(TYPE1_NOOP)};
+
+ struct icap *icap = platform_get_drvdata(pdev);
+ int i;
+
+ /* Can only be done from mgmt pf. */
+ if (!ICAP_PRIVILEGED(icap))
+ return -EPERM;
+
+ mutex_lock(&icap->icap_lock);
+
+ if (icap_bitstream_in_use(icap, 0)) {
+ mutex_unlock(&icap->icap_lock);
+ ICAP_ERR(icap, "bitstream is locked, can't reset");
+ return -EBUSY;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(fpga_boot_seq); i++) {
+ unsigned int value = be32_to_cpu(fpga_boot_seq[i]);
+
+ reg_wr(&icap->icap_regs->ir_wfv, value);
+ }
+ reg_wr(&icap->icap_regs->ir_cr, 0x1);
+
+ msleep(4000);
+
+ mutex_unlock(&icap->icap_lock);
+
+ ICAP_INFO(icap, "reset bitstream is done");
+ return 0;
+}
+
+static int icap_lock_bitstream(struct platform_device *pdev, const uuid_t *id,
+ pid_t pid)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int err = 0;
+
+ if (uuid_is_null(id)) {
+ ICAP_ERR(icap, "proc %d invalid UUID", pid);
+ return -EINVAL;
+ }
+
+ mutex_lock(&icap->icap_lock);
+
+ if (!ICAP_PRIVILEGED(icap)) {
+ err = __icap_lock_peer(pdev, id);
+ if (err < 0)
+ goto done;
+ }
+
+ if (uuid_equal(id, &icap->icap_bitstream_uuid))
+ err = add_user(icap, pid);
+ else
+ err = -EBUSY;
+
+ if (err >= 0)
+ err = icap->icap_bitstream_ref;
+
+ ICAP_INFO(icap, "proc %d try to lock bitstream %pUb, ref=%d, err=%d",
+ pid, id, icap->icap_bitstream_ref, err);
+done:
+ mutex_unlock(&icap->icap_lock);
+
+ if (!ICAP_PRIVILEGED(icap) && err == 1) /* reset on first reference */
+ xocl_exec_reset(xocl_get_xdev(pdev));
+
+ if (err >= 0)
+ err = 0;
+
+ return err;
+}
+
+static int icap_unlock_bitstream(struct platform_device *pdev, const uuid_t *id,
+ pid_t pid)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int err = 0;
+
+ if (id == NULL)
+ id = &uuid_null;
+
+ mutex_lock(&icap->icap_lock);
+
+ /* Force unlock. */
+ if (uuid_is_null(id))
+ del_all_users(icap);
+ else if (uuid_equal(id, &icap->icap_bitstream_uuid))
+ err = del_user(icap, pid);
+ else
+ err = -EINVAL;
+
+ if (!ICAP_PRIVILEGED(icap))
+ __icap_unlock_peer(pdev, id);
+
+ if (err >= 0)
+ err = icap->icap_bitstream_ref;
+
+ if (!ICAP_PRIVILEGED(icap)) {
+ if (err == 0)
+ xocl_exec_stop(xocl_get_xdev(pdev));
+ }
+
+ ICAP_INFO(icap, "proc %d try to unlock bitstream %pUb, ref=%d, err=%d",
+ pid, id, icap->icap_bitstream_ref, err);
+
+ mutex_unlock(&icap->icap_lock);
+ if (err >= 0)
+ err = 0;
+ return err;
+}
+
+static int icap_parse_bitstream_axlf_section(struct platform_device *pdev,
+ const struct axlf *xclbin, enum axlf_section_kind kind)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ long err = 0;
+ uint64_t section_size = 0, sect_sz = 0;
+ void **target = NULL;
+
+ if (memcmp(xclbin->m_magic, ICAP_XCLBIN_V2, sizeof(ICAP_XCLBIN_V2)))
+ return -EINVAL;
+
+ switch (kind) {
+ case IP_LAYOUT:
+ target = (void **)&icap->ip_layout;
+ break;
+ case MEM_TOPOLOGY:
+ target = (void **)&icap->mem_topo;
+ break;
+ case DEBUG_IP_LAYOUT:
+ target = (void **)&icap->debug_layout;
+ break;
+ case CONNECTIVITY:
+ target = (void **)&icap->connectivity;
+ break;
+ default:
+ break;
+ }
+ if (target) {
+ vfree(*target);
+ *target = NULL;
+ }
+ err = alloc_and_get_axlf_section(icap, xclbin, kind,
+ target, §ion_size);
+ if (err != 0)
+ goto done;
+ sect_sz = icap_get_section_size(icap, kind);
+ if (sect_sz > section_size) {
+ err = -EINVAL;
+ goto done;
+ }
+done:
+ if (err) {
+ vfree(*target);
+ *target = NULL;
+ }
+ ICAP_INFO(icap, "%s kind %d, err: %ld", __func__, kind, err);
+ return err;
+}
+
+static uint64_t icap_get_data(struct platform_device *pdev,
+ enum data_kind kind)
+{
+
+ struct icap *icap = platform_get_drvdata(pdev);
+ uint64_t target = 0;
+
+ mutex_lock(&icap->icap_lock);
+ switch (kind) {
+ case IPLAYOUT_AXLF:
+ target = (uint64_t)icap->ip_layout;
+ break;
+ case MEMTOPO_AXLF:
+ target = (uint64_t)icap->mem_topo;
+ break;
+ case DEBUG_IPLAYOUT_AXLF:
+ target = (uint64_t)icap->debug_layout;
+ break;
+ case CONNECTIVITY_AXLF:
+ target = (uint64_t)icap->connectivity;
+ break;
+ case IDCODE:
+ target = icap->idcode;
+ break;
+ case XCLBIN_UUID:
+ target = (uint64_t)&icap->icap_bitstream_uuid;
+ break;
+ default:
+ break;
+ }
+ mutex_unlock(&icap->icap_lock);
+ return target;
+}
+
+/* Kernel APIs exported from this sub-device driver. */
+static struct xocl_icap_funcs icap_ops = {
+ .reset_axi_gate = platform_reset_axi_gate,
+ .reset_bitstream = icap_reset_bitstream,
+ .download_boot_firmware = icap_download_boot_firmware,
+ .download_bitstream_axlf = icap_download_bitstream_axlf,
+ .ocl_set_freq = icap_ocl_set_freqscaling,
+ .ocl_get_freq = icap_ocl_get_freqscaling,
+ .ocl_update_clock_freq_topology = icap_ocl_update_clock_freq_topology,
+ .ocl_lock_bitstream = icap_lock_bitstream,
+ .ocl_unlock_bitstream = icap_unlock_bitstream,
+ .get_data = icap_get_data,
+};
+
+static ssize_t clock_freq_topology_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct icap *icap = platform_get_drvdata(to_platform_device(dev));
+ ssize_t cnt = 0;
+
+ mutex_lock(&icap->icap_lock);
+ if (ICAP_PRIVILEGED(icap)) {
+ memcpy(buf, icap->icap_clock_freq_topology, icap->icap_clock_freq_topology_length);
+ cnt = icap->icap_clock_freq_topology_length;
+ }
+ mutex_unlock(&icap->icap_lock);
+
+ return cnt;
+
+}
+
+static DEVICE_ATTR_RO(clock_freq_topology);
+
+static ssize_t clock_freqs_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct icap *icap = platform_get_drvdata(to_platform_device(dev));
+ ssize_t cnt = 0;
+ int i;
+ u32 freq_counter, freq, request_in_khz, tolerance;
+
+ mutex_lock(&icap->icap_lock);
+
+ for (i = 0; i < ICAP_MAX_NUM_CLOCKS; i++) {
+ freq = icap_get_ocl_frequency(icap, i);
+ if (!uuid_is_null(&icap->icap_bitstream_uuid)) {
+ freq_counter = icap_get_clock_frequency_counter_khz(icap, i);
+
+ request_in_khz = freq * 1000;
+ tolerance = freq * 50;
+
+ if (abs(freq_counter-request_in_khz) > tolerance)
+ ICAP_INFO(icap, "Frequency mismatch, Should be %u khz, Now is %ukhz",
+ request_in_khz, freq_counter);
+ cnt += sprintf(buf + cnt, "%d\n", DIV_ROUND_CLOSEST(freq_counter, 1000));
+ } else {
+ cnt += sprintf(buf + cnt, "%d\n", freq);
+ }
+ }
+
+ mutex_unlock(&icap->icap_lock);
+
+ return cnt;
+}
+static DEVICE_ATTR_RO(clock_freqs);
+
+static ssize_t icap_rl_program(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t off, size_t count)
+{
+ struct XHwIcap_Bit_Header bit_header = { 0 };
+ struct device *dev = container_of(kobj, struct device, kobj);
+ struct icap *icap = platform_get_drvdata(to_platform_device(dev));
+ ssize_t ret = count;
+
+ if (off == 0) {
+ if (count < DMA_HWICAP_BITFILE_BUFFER_SIZE) {
+ ICAP_ERR(icap, "count is too small %ld", count);
+ return -EINVAL;
+ }
+
+ if (bitstream_parse_header(icap, buffer,
+ DMA_HWICAP_BITFILE_BUFFER_SIZE, &bit_header)) {
+ ICAP_ERR(icap, "parse header failed");
+ return -EINVAL;
+ }
+
+ icap->bit_length = bit_header.HeaderLength +
+ bit_header.BitstreamLength;
+ icap->bit_buffer = vmalloc(icap->bit_length);
+ }
+
+ if (off + count >= icap->bit_length) {
+ /*
+ * assumes all subdevices are removed at this time
+ */
+ memcpy(icap->bit_buffer + off, buffer, icap->bit_length - off);
+ icap_freeze_axi_gate_shell(icap);
+ ret = icap_download(icap, icap->bit_buffer, icap->bit_length);
+ if (ret) {
+ ICAP_ERR(icap, "bitstream download failed");
+ ret = -EIO;
+ } else {
+ ret = count;
+ }
+ icap_free_axi_gate_shell(icap);
+ /* has to reset pci, otherwise firewall trips */
+ xocl_reset(xocl_get_xdev(icap->icap_pdev));
+ icap->icap_bitstream_id = 0;
+ memset(&icap->icap_bitstream_uuid, 0, sizeof(uuid_t));
+ vfree(icap->bit_buffer);
+ icap->bit_buffer = NULL;
+ } else {
+ memcpy(icap->bit_buffer + off, buffer, count);
+ }
+
+ return ret;
+}
+
+static struct bin_attribute shell_program_attr = {
+ .attr = {
+ .name = "shell_program",
+ .mode = 0200
+ },
+ .read = NULL,
+ .write = icap_rl_program,
+ .size = 0
+};
+
+static struct bin_attribute *icap_mgmt_bin_attrs[] = {
+ &shell_program_attr,
+ NULL,
+};
+
+static struct attribute_group icap_mgmt_bin_attr_group = {
+ .bin_attrs = icap_mgmt_bin_attrs,
+};
+
+static ssize_t idcode_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct icap *icap = platform_get_drvdata(to_platform_device(dev));
+ ssize_t cnt = 0;
+ uint32_t val;
+
+ mutex_lock(&icap->icap_lock);
+ if (ICAP_PRIVILEGED(icap)) {
+ cnt = sprintf(buf, "0x%x\n", icap->idcode);
+ } else {
+ icap_read_from_peer(to_platform_device(dev), IDCODE, &val, sizeof(unsigned int));
+ cnt = sprintf(buf, "0x%x\n", val);
+ }
+ mutex_unlock(&icap->icap_lock);
+
+ return cnt;
+}
+
+static DEVICE_ATTR_RO(idcode);
+
+static struct attribute *icap_attrs[] = {
+ &dev_attr_clock_freq_topology.attr,
+ &dev_attr_clock_freqs.attr,
+ &dev_attr_idcode.attr,
+ NULL,
+};
+
+//- Debug IP_layout--
+static ssize_t icap_read_debug_ip_layout(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ struct icap *icap;
+ u32 nread = 0;
+ size_t size = 0;
+
+ icap = (struct icap *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ if (!icap || !icap->debug_layout)
+ return 0;
+
+ mutex_lock(&icap->icap_lock);
+
+ size = sizeof_sect(icap->debug_layout, m_debug_ip_data);
+ if (offset >= size)
+ goto unlock;
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, ((char *)icap->debug_layout) + offset, nread);
+
+unlock:
+ mutex_unlock(&icap->icap_lock);
+ return nread;
+}
+static struct bin_attribute debug_ip_layout_attr = {
+ .attr = {
+ .name = "debug_ip_layout",
+ .mode = 0444
+ },
+ .read = icap_read_debug_ip_layout,
+ .write = NULL,
+ .size = 0
+};
+
+//IP layout
+static ssize_t icap_read_ip_layout(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ struct icap *icap;
+ u32 nread = 0;
+ size_t size = 0;
+
+ icap = (struct icap *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ if (!icap || !icap->ip_layout)
+ return 0;
+
+ mutex_lock(&icap->icap_lock);
+
+ size = sizeof_sect(icap->ip_layout, m_ip_data);
+ if (offset >= size)
+ goto unlock;
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, ((char *)icap->ip_layout) + offset, nread);
+
+unlock:
+ mutex_unlock(&icap->icap_lock);
+ return nread;
+}
+
+static struct bin_attribute ip_layout_attr = {
+ .attr = {
+ .name = "ip_layout",
+ .mode = 0444
+ },
+ .read = icap_read_ip_layout,
+ .write = NULL,
+ .size = 0
+};
+
+//-Connectivity--
+static ssize_t icap_read_connectivity(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ struct icap *icap;
+ u32 nread = 0;
+ size_t size = 0;
+
+ icap = (struct icap *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ if (!icap || !icap->connectivity)
+ return 0;
+
+ mutex_lock(&icap->icap_lock);
+
+ size = sizeof_sect(icap->connectivity, m_connection);
+ if (offset >= size)
+ goto unlock;
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, ((char *)icap->connectivity) + offset, nread);
+
+unlock:
+ mutex_unlock(&icap->icap_lock);
+ return nread;
+}
+
+static struct bin_attribute connectivity_attr = {
+ .attr = {
+ .name = "connectivity",
+ .mode = 0444
+ },
+ .read = icap_read_connectivity,
+ .write = NULL,
+ .size = 0
+};
+
+
+//-Mem_topology--
+static ssize_t icap_read_mem_topology(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ struct icap *icap;
+ u32 nread = 0;
+ size_t size = 0;
+
+ icap = (struct icap *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ if (!icap || !icap->mem_topo)
+ return 0;
+
+ mutex_lock(&icap->icap_lock);
+
+ size = sizeof_sect(icap->mem_topo, m_mem_data);
+ if (offset >= size)
+ goto unlock;
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, ((char *)icap->mem_topo) + offset, nread);
+unlock:
+ mutex_unlock(&icap->icap_lock);
+ return nread;
+}
+
+
+static struct bin_attribute mem_topology_attr = {
+ .attr = {
+ .name = "mem_topology",
+ .mode = 0444
+ },
+ .read = icap_read_mem_topology,
+ .write = NULL,
+ .size = 0
+};
+
+static struct bin_attribute *icap_bin_attrs[] = {
+ &debug_ip_layout_attr,
+ &ip_layout_attr,
+ &connectivity_attr,
+ &mem_topology_attr,
+ NULL,
+};
+
+static struct attribute_group icap_attr_group = {
+ .attrs = icap_attrs,
+ .bin_attrs = icap_bin_attrs,
+};
+
+static int icap_remove(struct platform_device *pdev)
+{
+ struct icap *icap = platform_get_drvdata(pdev);
+ int i;
+
+ BUG_ON(icap == NULL);
+
+ del_all_users(icap);
+ xocl_subdev_register(pdev, XOCL_SUBDEV_ICAP, NULL);
+
+ if (ICAP_PRIVILEGED(icap))
+ sysfs_remove_group(&pdev->dev.kobj, &icap_mgmt_bin_attr_group);
+
+ if (icap->bit_buffer)
+ vfree(icap->bit_buffer);
+
+ iounmap(icap->icap_regs);
+ iounmap(icap->icap_state);
+ iounmap(icap->icap_axi_gate);
+ for (i = 0; i < ICAP_MAX_NUM_CLOCKS; i++)
+ iounmap(icap->icap_clock_bases[i]);
+ free_clear_bitstream(icap);
+ free_clock_freq_topology(icap);
+
+ sysfs_remove_group(&pdev->dev.kobj, &icap_attr_group);
+
+ ICAP_INFO(icap, "cleaned up successfully");
+ platform_set_drvdata(pdev, NULL);
+ vfree(icap->mem_topo);
+ vfree(icap->ip_layout);
+ vfree(icap->debug_layout);
+ vfree(icap->connectivity);
+ kfree(icap);
+ return 0;
+}
+
+/*
+ * Run the following sequence of canned commands to obtain IDCODE of the FPGA
+ */
+static void icap_probe_chip(struct icap *icap)
+{
+ u32 w;
+
+ if (!ICAP_PRIVILEGED(icap))
+ return;
+
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ reg_wr(&icap->icap_regs->ir_gier, 0x0);
+ w = reg_rd(&icap->icap_regs->ir_wfv);
+ reg_wr(&icap->icap_regs->ir_wf, 0xffffffff);
+ reg_wr(&icap->icap_regs->ir_wf, 0xaa995566);
+ reg_wr(&icap->icap_regs->ir_wf, 0x20000000);
+ reg_wr(&icap->icap_regs->ir_wf, 0x20000000);
+ reg_wr(&icap->icap_regs->ir_wf, 0x28018001);
+ reg_wr(&icap->icap_regs->ir_wf, 0x20000000);
+ reg_wr(&icap->icap_regs->ir_wf, 0x20000000);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ reg_wr(&icap->icap_regs->ir_cr, 0x1);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ w = reg_rd(&icap->icap_regs->ir_sr);
+ reg_wr(&icap->icap_regs->ir_sz, 0x1);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+ reg_wr(&icap->icap_regs->ir_cr, 0x2);
+ w = reg_rd(&icap->icap_regs->ir_rfo);
+ icap->idcode = reg_rd(&icap->icap_regs->ir_rf);
+ w = reg_rd(&icap->icap_regs->ir_cr);
+}
+
+static int icap_probe(struct platform_device *pdev)
+{
+ struct icap *icap = NULL;
+ struct resource *res;
+ int ret;
+ int reg_grp;
+ void **regs;
+
+ icap = kzalloc(sizeof(struct icap), GFP_KERNEL);
+ if (!icap)
+ return -ENOMEM;
+ platform_set_drvdata(pdev, icap);
+ icap->icap_pdev = pdev;
+ mutex_init(&icap->icap_lock);
+ INIT_LIST_HEAD(&icap->icap_bitstream_users);
+
+ for (reg_grp = 0; reg_grp < ICAP_MAX_REG_GROUPS; reg_grp++) {
+ switch (reg_grp) {
+ case 0:
+ regs = (void **)&icap->icap_regs;
+ break;
+ case 1:
+ regs = (void **)&icap->icap_state;
+ break;
+ case 2:
+ regs = (void **)&icap->icap_axi_gate;
+ break;
+ case 3:
+ regs = (void **)&icap->icap_clock_bases[0];
+ break;
+ case 4:
+ regs = (void **)&icap->icap_clock_bases[1];
+ break;
+ case 5:
+ regs = (void **)&icap->icap_clock_freq_counter;
+ break;
+ default:
+ BUG();
+ break;
+ }
+ res = platform_get_resource(pdev, IORESOURCE_MEM, reg_grp);
+ if (res != NULL) {
+ *regs = ioremap_nocache(res->start,
+ res->end - res->start + 1);
+ if (*regs == NULL) {
+ ICAP_ERR(icap,
+ "failed to map in register group: %d",
+ reg_grp);
+ ret = -EIO;
+ goto failed;
+ } else {
+ ICAP_INFO(icap,
+ "mapped in register group %d @ 0x%p",
+ reg_grp, *regs);
+ }
+ } else {
+ if (reg_grp != 0) {
+ ICAP_ERR(icap,
+ "failed to find register group: %d",
+ reg_grp);
+ ret = -EIO;
+ goto failed;
+ }
+ break;
+ }
+ }
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &icap_attr_group);
+ if (ret) {
+ ICAP_ERR(icap, "create icap attrs failed: %d", ret);
+ goto failed;
+ }
+
+ if (ICAP_PRIVILEGED(icap)) {
+ ret = sysfs_create_group(&pdev->dev.kobj,
+ &icap_mgmt_bin_attr_group);
+ if (ret) {
+ ICAP_ERR(icap, "create icap attrs failed: %d", ret);
+ goto failed;
+ }
+ }
+
+ icap_probe_chip(icap);
+ if (!ICAP_PRIVILEGED(icap))
+ icap_unlock_bitstream(pdev, NULL, 0);
+ ICAP_INFO(icap, "successfully initialized FPGA IDCODE 0x%x",
+ icap->idcode);
+ xocl_subdev_register(pdev, XOCL_SUBDEV_ICAP, &icap_ops);
+ return 0;
+
+failed:
+ (void) icap_remove(pdev);
+ return ret;
+}
+
+
+struct platform_device_id icap_id_table[] = {
+ { XOCL_ICAP, 0 },
+ { },
+};
+
+static struct platform_driver icap_driver = {
+ .probe = icap_probe,
+ .remove = icap_remove,
+ .driver = {
+ .name = XOCL_ICAP,
+ },
+ .id_table = icap_id_table,
+};
+
+int __init xocl_init_icap(void)
+{
+ return platform_driver_register(&icap_driver);
+}
+
+void xocl_fini_icap(void)
+{
+ platform_driver_unregister(&icap_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/mailbox.c b/drivers/gpu/drm/xocl/subdev/mailbox.c
new file mode 100644
index 000000000000..dc4736c9100a
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/mailbox.c
@@ -0,0 +1,1868 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Max Zhen <maxz@xilinx.com>
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+/*
+ * Statement of Theory
+ *
+ * This is the mailbox sub-device driver added into existing xclmgmt / xocl
+ * driver so that user pf and mgmt pf can send and receive messages of
+ * arbitrary length to / from peer. The driver is written based on the spec of
+ * pg114 document (https://www.xilinx.com/support/documentation/
+ * ip_documentation/mailbox/v2_1/pg114-mailbox.pdf). The HW provides one TX
+ * channel and one RX channel, which operate completely independent of each
+ * other. Data can be pushed into or read from a channel in DWORD unit as a
+ * FIFO.
+ *
+ *
+ * Packet layer
+ *
+ * The driver implemented two transport layers - packet and message layer (see
+ * below). A packet is a fixed size chunk of data that can be send through TX
+ * channel or retrieved from RX channel. The TX and RX interrupt happens at
+ * packet boundary, instead of DWORD boundary. The driver will not attempt to
+ * send next packet until the previous one is read by peer. Similarly, the
+ * driver will not attempt to read the data from HW until a full packet has been
+ * written to HW by peer. No polling is implemented. Data transfer is entirely
+ * interrupt driven. So, the interrupt functionality needs to work and enabled
+ * on both mgmt and user pf for mailbox driver to function properly.
+ *
+ * A TX packet is considered as time'd out after sitting in the TX channel of
+ * mailbox HW for two packet ticks (1 packet tick = 1 second, for now) without
+ * being read by peer. Currently, the driver will not try to re-transmit the
+ * packet after timeout. It just simply propagate the error to the upper layer.
+ * A retry at packet layer can be implement later, if considered as appropriate.
+ *
+ *
+ * Message layer
+ *
+ * A message is a data buffer of arbitrary length. The driver will break a
+ * message into multiple packets and transmit them to the peer, which, in turn,
+ * will assemble them into a full message before it's delivered to upper layer
+ * for further processing. One message requires at least one packet to be
+ * transferred to the peer.
+ *
+ * Each message has a unique temporary u64 ID (see communication model below
+ * for more detail). The ID shows up in each packet's header. So, at packet
+ * layer, there is no assumption that adjacent packets belong to the same
+ * message. However, for the sake of simplicity, at message layer, the driver
+ * will not attempt to send the next message until the sending of current one
+ * is finished. I.E., we implement a FIFO for message TX channel. All messages
+ * are sent by driver in the order of received from upper layer. We can
+ * implement messages of different priority later, if needed. There is no
+ * certain order for receiving messages. It's up to the peer side to decide
+ * which message gets enqueued into its own TX queue first, which will be
+ * received first on the other side.
+ *
+ * A message is considered as time'd out when it's transmit (send or receive)
+ * is not finished within 10 packet ticks. This applies to all messages queued
+ * up on both RX and TX channels. Again, no retry for a time'd out message is
+ * implemented. The error will be simply passed to upper layer. Also, a TX
+ * message may time out earlier if it's being transmitted and one of it's
+ * packets time'd out. During normal operation, timeout should never happen.
+ *
+ * The upper layer can choose to queue a message for TX or RX asynchronously
+ * when it provides a callback or wait synchronously when no callback is
+ * provided.
+ *
+ *
+ * Communication model
+ *
+ * At the highest layer, the driver implements a request-response communication
+ * model. A request may or may not require a response, but a response must match
+ * a request, or it'll be silently dropped. The driver provides a few kernel
+ * APIs for mgmt and user pf to talk to each other in this model (see kernel
+ * APIs section below for details). Each request or response is a message by
+ * itself. A request message will automatically be assigned a message ID when
+ * it's enqueued into TX channel for sending. If this request requires a
+ * response, the buffer provided by caller for receiving response will be
+ * enqueued into RX channel as well. The enqueued response message will have
+ * the same message ID as the corresponding request message. The response
+ * message, if provided, will always be enqueued before the request message is
+ * enqueued to avoid race condition.
+ *
+ * The driver will automatically enqueue a special message into the RX channel
+ * for receiving new request after initialized. This request RX message has a
+ * special message ID (id=0) and never time'd out. When a new request comes
+ * from peer, it'll be copied into request RX message then passed to the
+ * callback provided by upper layer through xocl_peer_listen() API for further
+ * processing. Currently, the driver implements only one kernel thread for RX
+ * channel and one for TX channel. So, all message callback happens in the
+ * context of that channel thread. So, the user of mailbox driver needs to be
+ * careful when it calls xocl_peer_request() synchronously in this context.
+ * You may see deadlock when both ends are trying to call xocl_peer_request()
+ * synchronously at the same time.
+ *
+ *
+ * +------------------+ +------------------+
+ * | Request/Response | <--------> | Request/Response |
+ * +------------------+ +------------------+
+ * | Message | <--------> | Message |
+ * +------------------+ +------------------+
+ * | Packet | <--------> | Packet |
+ * +------------------+ +------------------+
+ * | RX/TX Channel | <<======>> | RX/TX Channel |
+ * +------------------+ +------------------+
+ * mgmt pf user pf
+ */
+
+#include <linux/mutex.h>
+#include <linux/completion.h>
+#include <linux/list.h>
+#include <linux/device.h>
+#include <linux/crc32c.h>
+#include <linux/random.h>
+#include "../xocl_drv.h"
+
+int mailbox_no_intr;
+module_param(mailbox_no_intr, int, (S_IRUGO|S_IWUSR));
+MODULE_PARM_DESC(mailbox_no_intr,
+ "Disable mailbox interrupt and do timer-driven msg passing");
+
+#define PACKET_SIZE 16 /* Number of DWORD. */
+
+#define FLAG_STI (1 << 0)
+#define FLAG_RTI (1 << 1)
+
+#define STATUS_EMPTY (1 << 0)
+#define STATUS_FULL (1 << 1)
+#define STATUS_STA (1 << 2)
+#define STATUS_RTA (1 << 3)
+
+#define MBX_ERR(mbx, fmt, arg...) \
+ xocl_err(&mbx->mbx_pdev->dev, fmt "\n", ##arg)
+#define MBX_INFO(mbx, fmt, arg...) \
+ xocl_info(&mbx->mbx_pdev->dev, fmt "\n", ##arg)
+#define MBX_DBG(mbx, fmt, arg...) \
+ xocl_dbg(&mbx->mbx_pdev->dev, fmt "\n", ##arg)
+
+#define MAILBOX_TIMER HZ /* in jiffies */
+#define MSG_TTL 10 /* in MAILBOX_TIMER */
+#define TEST_MSG_LEN 128
+
+#define INVALID_MSG_ID ((u64)-1)
+#define MSG_FLAG_RESPONSE (1 << 0)
+#define MSG_FLAG_REQUEST (1 << 1)
+
+#define MAX_MSG_QUEUE_SZ (PAGE_SIZE << 16)
+#define MAX_MSG_QUEUE_LEN 5
+
+#define MB_CONN_INIT (0x1<<0)
+#define MB_CONN_SYN (0x1<<1)
+#define MB_CONN_ACK (0x1<<2)
+#define MB_CONN_FIN (0x1<<3)
+
+/*
+ * Mailbox IP register layout
+ */
+struct mailbox_reg {
+ u32 mbr_wrdata;
+ u32 mbr_resv1;
+ u32 mbr_rddata;
+ u32 mbr_resv2;
+ u32 mbr_status;
+ u32 mbr_error;
+ u32 mbr_sit;
+ u32 mbr_rit;
+ u32 mbr_is;
+ u32 mbr_ie;
+ u32 mbr_ip;
+ u32 mbr_ctrl;
+} __attribute__((packed));
+
+/*
+ * A message transport by mailbox.
+ */
+struct mailbox_msg {
+ struct list_head mbm_list;
+ struct mailbox_channel *mbm_ch;
+ u64 mbm_req_id;
+ char *mbm_data;
+ size_t mbm_len;
+ int mbm_error;
+ struct completion mbm_complete;
+ mailbox_msg_cb_t mbm_cb;
+ void *mbm_cb_arg;
+ u32 mbm_flags;
+ int mbm_ttl;
+ bool mbm_timer_on;
+};
+
+/*
+ * A packet transport by mailbox.
+ * When extending, only add new data structure to body. Choose to add new flag
+ * if new feature can be safely ignored by peer, other wise, add new type.
+ */
+enum packet_type {
+ PKT_INVALID = 0,
+ PKT_TEST,
+ PKT_MSG_START,
+ PKT_MSG_BODY
+};
+
+
+enum conn_state {
+ CONN_START = 0,
+ CONN_SYN_SENT,
+ CONN_SYN_RECV,
+ CONN_ESTABLISH,
+};
+
+/* Lower 8 bits for type, the rest for flags. */
+#define PKT_TYPE_MASK 0xff
+#define PKT_TYPE_MSG_END (1 << 31)
+struct mailbox_pkt {
+ struct {
+ u32 type;
+ u32 payload_size;
+ } hdr;
+ union {
+ u32 data[PACKET_SIZE - 2];
+ struct {
+ u64 msg_req_id;
+ u32 msg_flags;
+ u32 msg_size;
+ u32 payload[0];
+ } msg_start;
+ struct {
+ u32 payload[0];
+ } msg_body;
+ } body;
+} __attribute__((packed));
+
+/*
+ * Mailbox communication channel.
+ */
+#define MBXCS_BIT_READY 0
+#define MBXCS_BIT_STOP 1
+#define MBXCS_BIT_TICK 2
+#define MBXCS_BIT_CHK_STALL 3
+#define MBXCS_BIT_POLL_MODE 4
+
+struct mailbox_channel;
+typedef void (*chan_func_t)(struct mailbox_channel *ch);
+struct mailbox_channel {
+ struct mailbox *mbc_parent;
+ char *mbc_name;
+
+ struct workqueue_struct *mbc_wq;
+ struct work_struct mbc_work;
+ struct completion mbc_worker;
+ chan_func_t mbc_tran;
+ unsigned long mbc_state;
+
+ struct mutex mbc_mutex;
+ struct list_head mbc_msgs;
+
+ struct mailbox_msg *mbc_cur_msg;
+ int mbc_bytes_done;
+ struct mailbox_pkt mbc_packet;
+
+ struct timer_list mbc_timer;
+ bool mbc_timer_on;
+};
+
+/*
+ * The mailbox softstate.
+ */
+struct mailbox {
+ struct platform_device *mbx_pdev;
+ struct mailbox_reg *mbx_regs;
+ u32 mbx_irq;
+
+ struct mailbox_channel mbx_rx;
+ struct mailbox_channel mbx_tx;
+
+ /* For listening to peer's request. */
+ mailbox_msg_cb_t mbx_listen_cb;
+ void *mbx_listen_cb_arg;
+ struct workqueue_struct *mbx_listen_wq;
+ struct work_struct mbx_listen_worker;
+
+ int mbx_paired;
+ /*
+ * For testing basic intr and mailbox comm functionality via sysfs.
+ * No locking protection, use with care.
+ */
+ struct mailbox_pkt mbx_tst_pkt;
+ char mbx_tst_tx_msg[TEST_MSG_LEN];
+ char mbx_tst_rx_msg[TEST_MSG_LEN];
+ size_t mbx_tst_tx_msg_len;
+
+ /* Req list for all incoming request message */
+ struct completion mbx_comp;
+ struct mutex mbx_lock;
+ struct list_head mbx_req_list;
+ uint8_t mbx_req_cnt;
+ size_t mbx_req_sz;
+
+ struct mutex mbx_conn_lock;
+ uint64_t mbx_conn_id;
+ enum conn_state mbx_state;
+ bool mbx_established;
+ uint32_t mbx_prot_ver;
+
+ void *mbx_kaddr;
+};
+
+static inline const char *reg2name(struct mailbox *mbx, u32 *reg)
+{
+ static const char *reg_names[] = {
+ "wrdata",
+ "reserved1",
+ "rddata",
+ "reserved2",
+ "status",
+ "error",
+ "sit",
+ "rit",
+ "is",
+ "ie",
+ "ip",
+ "ctrl"
+ };
+
+ return reg_names[((uintptr_t)reg -
+ (uintptr_t)mbx->mbx_regs) / sizeof(u32)];
+}
+
+struct mailbox_conn {
+ uint64_t flag;
+ void *kaddr;
+ phys_addr_t paddr;
+ uint32_t crc32;
+ uint32_t ver;
+ uint64_t sec_id;
+};
+
+int mailbox_request(struct platform_device *pdev, void *req, size_t reqlen,
+ void *resp, size_t *resplen, mailbox_msg_cb_t cb, void *cbarg);
+int mailbox_post(struct platform_device *pdev, u64 reqid, void *buf, size_t len);
+static int mailbox_connect_status(struct platform_device *pdev);
+static void connect_state_handler(struct mailbox *mbx, struct mailbox_conn *conn);
+
+static void connect_state_touch(struct mailbox *mbx, uint64_t flag)
+{
+ struct mailbox_conn conn = {0};
+ if (!mbx)
+ return;
+ conn.flag = flag;
+ connect_state_handler(mbx, &conn);
+}
+
+
+static inline u32 mailbox_reg_rd(struct mailbox *mbx, u32 *reg)
+{
+ u32 val = ioread32(reg);
+
+#ifdef MAILBOX_REG_DEBUG
+ MBX_DBG(mbx, "REG_RD(%s)=0x%x", reg2name(mbx, reg), val);
+#endif
+ return val;
+}
+
+static inline void mailbox_reg_wr(struct mailbox *mbx, u32 *reg, u32 val)
+{
+#ifdef MAILBOX_REG_DEBUG
+ MBX_DBG(mbx, "REG_WR(%s, 0x%x)", reg2name(mbx, reg), val);
+#endif
+ iowrite32(val, reg);
+}
+
+static inline void reset_pkt(struct mailbox_pkt *pkt)
+{
+ pkt->hdr.type = PKT_INVALID;
+}
+
+static inline bool valid_pkt(struct mailbox_pkt *pkt)
+{
+ return (pkt->hdr.type != PKT_INVALID);
+}
+
+irqreturn_t mailbox_isr(int irq, void *arg)
+{
+ struct mailbox *mbx = (struct mailbox *)arg;
+ u32 is = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_is);
+
+ while (is) {
+ MBX_DBG(mbx, "intr status: 0x%x", is);
+
+ if ((is & FLAG_STI) != 0) {
+ /* A packet has been sent successfully. */
+ complete(&mbx->mbx_tx.mbc_worker);
+ }
+ if ((is & FLAG_RTI) != 0) {
+ /* A packet is waiting to be received from mailbox. */
+ complete(&mbx->mbx_rx.mbc_worker);
+ }
+ /* Anything else is not expected. */
+ if ((is & (FLAG_STI | FLAG_RTI)) == 0) {
+ MBX_ERR(mbx, "spurious mailbox irq %d, is=0x%x",
+ irq, is);
+ }
+
+ /* Clear intr state for receiving next one. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_is, is);
+
+ is = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_is);
+ }
+
+ return IRQ_HANDLED;
+}
+
+static void chan_timer(struct timer_list *t)
+{
+ struct mailbox_channel *ch = from_timer(ch, t, mbc_timer);
+
+ MBX_DBG(ch->mbc_parent, "%s tick", ch->mbc_name);
+
+ set_bit(MBXCS_BIT_TICK, &ch->mbc_state);
+ complete(&ch->mbc_worker);
+
+ /* We're a periodic timer. */
+ mod_timer(&ch->mbc_timer, jiffies + MAILBOX_TIMER);
+}
+
+static void chan_config_timer(struct mailbox_channel *ch)
+{
+ struct list_head *pos, *n;
+ struct mailbox_msg *msg = NULL;
+ bool on = false;
+
+ mutex_lock(&ch->mbc_mutex);
+
+ if (test_bit(MBXCS_BIT_POLL_MODE, &ch->mbc_state)) {
+ on = true;
+ } else {
+ list_for_each_safe(pos, n, &ch->mbc_msgs) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ if (msg->mbm_req_id == 0)
+ continue;
+ on = true;
+ break;
+ }
+ }
+
+ if (on != ch->mbc_timer_on) {
+ ch->mbc_timer_on = on;
+ if (on)
+ mod_timer(&ch->mbc_timer, jiffies + MAILBOX_TIMER);
+ else
+ del_timer_sync(&ch->mbc_timer);
+ }
+
+ mutex_unlock(&ch->mbc_mutex);
+}
+
+static void free_msg(struct mailbox_msg *msg)
+{
+ vfree(msg);
+}
+
+static void msg_done(struct mailbox_msg *msg, int err)
+{
+ struct mailbox_channel *ch = msg->mbm_ch;
+ struct mailbox *mbx = ch->mbc_parent;
+
+ MBX_DBG(ch->mbc_parent, "%s finishing msg id=0x%llx err=%d",
+ ch->mbc_name, msg->mbm_req_id, err);
+
+ msg->mbm_error = err;
+ if (msg->mbm_cb) {
+ msg->mbm_cb(msg->mbm_cb_arg, msg->mbm_data, msg->mbm_len,
+ msg->mbm_req_id, msg->mbm_error);
+ free_msg(msg);
+ } else {
+ if (msg->mbm_flags & MSG_FLAG_REQUEST) {
+ if ((mbx->mbx_req_sz+msg->mbm_len) >= MAX_MSG_QUEUE_SZ ||
+ mbx->mbx_req_cnt >= MAX_MSG_QUEUE_LEN) {
+ goto done;
+ }
+ mutex_lock(&ch->mbc_parent->mbx_lock);
+ list_add_tail(&msg->mbm_list, &ch->mbc_parent->mbx_req_list);
+ mbx->mbx_req_cnt++;
+ mbx->mbx_req_sz += msg->mbm_len;
+ mutex_unlock(&ch->mbc_parent->mbx_lock);
+
+ complete(&ch->mbc_parent->mbx_comp);
+ } else {
+ complete(&msg->mbm_complete);
+ }
+ }
+done:
+ chan_config_timer(ch);
+}
+
+static void chan_msg_done(struct mailbox_channel *ch, int err)
+{
+ if (!ch->mbc_cur_msg)
+ return;
+
+ msg_done(ch->mbc_cur_msg, err);
+ ch->mbc_cur_msg = NULL;
+ ch->mbc_bytes_done = 0;
+}
+
+void timeout_msg(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_msg *msg = NULL;
+ struct list_head *pos, *n;
+ struct list_head l = LIST_HEAD_INIT(l);
+ bool reschedule = false;
+
+ /* Check active msg first. */
+ msg = ch->mbc_cur_msg;
+ if (msg) {
+
+ if (msg->mbm_ttl == 0) {
+ MBX_ERR(mbx, "found active msg time'd out");
+ chan_msg_done(ch, -ETIME);
+ } else {
+ if (msg->mbm_timer_on) {
+ msg->mbm_ttl--;
+ /* Need to come back again for this one. */
+ reschedule = true;
+ }
+ }
+ }
+
+ mutex_lock(&ch->mbc_mutex);
+
+ list_for_each_safe(pos, n, &ch->mbc_msgs) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ if (!msg->mbm_timer_on)
+ continue;
+ if (msg->mbm_req_id == 0)
+ continue;
+ if (msg->mbm_ttl == 0) {
+ list_del(&msg->mbm_list);
+ list_add_tail(&msg->mbm_list, &l);
+ } else {
+ msg->mbm_ttl--;
+ /* Need to come back again for this one. */
+ reschedule = true;
+ }
+ }
+
+ mutex_unlock(&ch->mbc_mutex);
+
+ if (!list_empty(&l))
+ MBX_ERR(mbx, "found waiting msg time'd out");
+
+ list_for_each_safe(pos, n, &l) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ list_del(&msg->mbm_list);
+ msg_done(msg, -ETIME);
+ }
+}
+
+static void chann_worker(struct work_struct *work)
+{
+ struct mailbox_channel *ch =
+ container_of(work, struct mailbox_channel, mbc_work);
+ struct mailbox *mbx = ch->mbc_parent;
+
+ while (!test_bit(MBXCS_BIT_STOP, &ch->mbc_state)) {
+ MBX_DBG(mbx, "%s worker start", ch->mbc_name);
+ ch->mbc_tran(ch);
+ wait_for_completion_interruptible(&ch->mbc_worker);
+ }
+}
+
+static inline u32 mailbox_chk_err(struct mailbox *mbx)
+{
+ u32 val = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_error);
+
+ /* Ignore bad register value after firewall is tripped. */
+ if (val == 0xffffffff)
+ val = 0;
+
+ /* Error should not be seen, shout when found. */
+ if (val)
+ MBX_ERR(mbx, "mailbox error detected, error=0x%x\n", val);
+ return val;
+}
+
+static int chan_msg_enqueue(struct mailbox_channel *ch, struct mailbox_msg *msg)
+{
+ int rv = 0;
+
+ MBX_DBG(ch->mbc_parent, "%s enqueuing msg, id=0x%llx\n",
+ ch->mbc_name, msg->mbm_req_id);
+
+ BUG_ON(msg->mbm_req_id == INVALID_MSG_ID);
+
+ mutex_lock(&ch->mbc_mutex);
+ if (test_bit(MBXCS_BIT_STOP, &ch->mbc_state)) {
+ rv = -ESHUTDOWN;
+ } else {
+ list_add_tail(&msg->mbm_list, &ch->mbc_msgs);
+ msg->mbm_ch = ch;
+// msg->mbm_ttl = MSG_TTL;
+ }
+ mutex_unlock(&ch->mbc_mutex);
+
+ chan_config_timer(ch);
+
+ return rv;
+}
+
+static struct mailbox_msg *chan_msg_dequeue(struct mailbox_channel *ch,
+ u64 req_id)
+{
+ struct mailbox_msg *msg = NULL;
+ struct list_head *pos;
+
+ mutex_lock(&ch->mbc_mutex);
+
+ /* Take the first msg. */
+ if (req_id == INVALID_MSG_ID) {
+ msg = list_first_entry_or_null(&ch->mbc_msgs,
+ struct mailbox_msg, mbm_list);
+ /* Take the msg w/ specified ID. */
+ } else {
+ list_for_each(pos, &ch->mbc_msgs) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ if (msg->mbm_req_id == req_id)
+ break;
+ }
+ }
+
+ if (msg) {
+ MBX_DBG(ch->mbc_parent, "%s dequeued msg, id=0x%llx\n",
+ ch->mbc_name, msg->mbm_req_id);
+ list_del(&msg->mbm_list);
+ }
+
+ mutex_unlock(&ch->mbc_mutex);
+
+ return msg;
+}
+
+static struct mailbox_msg *alloc_msg(void *buf, size_t len)
+{
+ char *newbuf = NULL;
+ struct mailbox_msg *msg = NULL;
+ /* Give MB*2 secs as time to live */
+ int calculated_ttl = (len >> 19) < MSG_TTL ? MSG_TTL : (len >> 19);
+
+ if (!buf) {
+ msg = vzalloc(sizeof(struct mailbox_msg) + len);
+ if (!msg)
+ return NULL;
+ newbuf = ((char *)msg) + sizeof(struct mailbox_msg);
+ } else {
+ msg = vzalloc(sizeof(struct mailbox_msg));
+ if (!msg)
+ return NULL;
+ newbuf = buf;
+ }
+
+ INIT_LIST_HEAD(&msg->mbm_list);
+ msg->mbm_data = newbuf;
+ msg->mbm_len = len;
+ msg->mbm_ttl = calculated_ttl;
+ msg->mbm_timer_on = false;
+ init_completion(&msg->mbm_complete);
+
+ return msg;
+}
+
+static int chan_init(struct mailbox *mbx, char *nm,
+ struct mailbox_channel *ch, chan_func_t fn)
+{
+ ch->mbc_parent = mbx;
+ ch->mbc_name = nm;
+ ch->mbc_tran = fn;
+ INIT_LIST_HEAD(&ch->mbc_msgs);
+ init_completion(&ch->mbc_worker);
+ mutex_init(&ch->mbc_mutex);
+
+ ch->mbc_cur_msg = NULL;
+ ch->mbc_bytes_done = 0;
+
+ reset_pkt(&ch->mbc_packet);
+ set_bit(MBXCS_BIT_READY, &ch->mbc_state);
+
+ /* One thread for one channel. */
+ ch->mbc_wq =
+ create_singlethread_workqueue(dev_name(&mbx->mbx_pdev->dev));
+ if (!ch->mbc_wq) {
+ ch->mbc_parent = NULL;
+ return -ENOMEM;
+ }
+
+ INIT_WORK(&ch->mbc_work, chann_worker);
+ queue_work(ch->mbc_wq, &ch->mbc_work);
+
+ /* One timer for one channel. */
+ timer_setup(&ch->mbc_timer, chan_timer, 0);
+
+ return 0;
+}
+
+static void chan_fini(struct mailbox_channel *ch)
+{
+ struct mailbox_msg *msg;
+
+ if (!ch->mbc_parent)
+ return;
+
+ /*
+ * Holding mutex to ensure no new msg is enqueued after
+ * flag is set.
+ */
+ mutex_lock(&ch->mbc_mutex);
+ set_bit(MBXCS_BIT_STOP, &ch->mbc_state);
+ mutex_unlock(&ch->mbc_mutex);
+
+ complete(&ch->mbc_worker);
+ cancel_work_sync(&ch->mbc_work);
+ destroy_workqueue(ch->mbc_wq);
+
+ msg = ch->mbc_cur_msg;
+ if (msg)
+ chan_msg_done(ch, -ESHUTDOWN);
+
+ while ((msg = chan_msg_dequeue(ch, INVALID_MSG_ID)) != NULL)
+ msg_done(msg, -ESHUTDOWN);
+
+ del_timer_sync(&ch->mbc_timer);
+}
+
+static void listen_wq_fini(struct mailbox *mbx)
+{
+ BUG_ON(mbx == NULL);
+
+ if (mbx->mbx_listen_wq != NULL) {
+ complete(&mbx->mbx_comp);
+ cancel_work_sync(&mbx->mbx_listen_worker);
+ destroy_workqueue(mbx->mbx_listen_wq);
+ }
+
+}
+
+static void chan_recv_pkt(struct mailbox_channel *ch)
+{
+ int i, retry = 10;
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+
+ BUG_ON(valid_pkt(pkt));
+
+ /* Picking up a packet from HW. */
+ for (i = 0; i < PACKET_SIZE; i++) {
+ while ((mailbox_reg_rd(mbx,
+ &mbx->mbx_regs->mbr_status) & STATUS_EMPTY) &&
+ (retry-- > 0))
+ msleep(100);
+
+ *(((u32 *)pkt) + i) =
+ mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_rddata);
+ }
+
+ if ((mailbox_chk_err(mbx) & STATUS_EMPTY) != 0)
+ reset_pkt(pkt);
+ else
+ MBX_DBG(mbx, "received pkt: type=0x%x", pkt->hdr.type);
+}
+
+static void chan_send_pkt(struct mailbox_channel *ch)
+{
+ int i;
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+
+ BUG_ON(!valid_pkt(pkt));
+
+ MBX_DBG(mbx, "sending pkt: type=0x%x", pkt->hdr.type);
+
+ /* Pushing a packet into HW. */
+ for (i = 0; i < PACKET_SIZE; i++) {
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_wrdata,
+ *(((u32 *)pkt) + i));
+ }
+
+ reset_pkt(pkt);
+ if (ch->mbc_cur_msg)
+ ch->mbc_bytes_done += ch->mbc_packet.hdr.payload_size;
+
+ BUG_ON((mailbox_chk_err(mbx) & STATUS_FULL) != 0);
+}
+
+static int chan_pkt2msg(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ void *msg_data, *pkt_data;
+ struct mailbox_msg *msg = ch->mbc_cur_msg;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+ size_t cnt = pkt->hdr.payload_size;
+ u32 type = (pkt->hdr.type & PKT_TYPE_MASK);
+
+ BUG_ON(((type != PKT_MSG_START) && (type != PKT_MSG_BODY)) || !msg);
+
+ if (type == PKT_MSG_START) {
+ msg->mbm_req_id = pkt->body.msg_start.msg_req_id;
+ BUG_ON(msg->mbm_len < pkt->body.msg_start.msg_size);
+ msg->mbm_len = pkt->body.msg_start.msg_size;
+ pkt_data = pkt->body.msg_start.payload;
+ } else {
+ pkt_data = pkt->body.msg_body.payload;
+ }
+
+ if (cnt > msg->mbm_len - ch->mbc_bytes_done) {
+ MBX_ERR(mbx, "invalid mailbox packet size\n");
+ return -EBADMSG;
+ }
+
+ msg_data = msg->mbm_data + ch->mbc_bytes_done;
+ (void) memcpy(msg_data, pkt_data, cnt);
+ ch->mbc_bytes_done += cnt;
+
+ reset_pkt(pkt);
+ return 0;
+}
+
+/*
+ * Worker for RX channel.
+ */
+static void chan_do_rx(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+ struct mailbox_msg *msg = NULL;
+ bool needs_read = false;
+ u64 id = 0;
+ bool eom;
+ int err;
+ u32 type;
+ u32 st = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_status);
+
+ /* Check if a packet is ready for reading. */
+ if (st == 0xffffffff) {
+ /* Device is still being reset. */
+ needs_read = false;
+ } else if (test_bit(MBXCS_BIT_POLL_MODE, &ch->mbc_state)) {
+ needs_read = ((st & STATUS_EMPTY) == 0);
+ } else {
+ needs_read = ((st & STATUS_RTA) != 0);
+ }
+
+ if (needs_read) {
+ chan_recv_pkt(ch);
+ type = pkt->hdr.type & PKT_TYPE_MASK;
+ eom = ((pkt->hdr.type & PKT_TYPE_MSG_END) != 0);
+
+ switch (type) {
+ case PKT_TEST:
+ (void) memcpy(&mbx->mbx_tst_pkt, &ch->mbc_packet,
+ sizeof(struct mailbox_pkt));
+ reset_pkt(pkt);
+ return;
+ case PKT_MSG_START:
+ if (ch->mbc_cur_msg) {
+ MBX_ERR(mbx, "received partial msg\n");
+ chan_msg_done(ch, -EBADMSG);
+ }
+
+ /* Get a new active msg. */
+ id = 0;
+ if (pkt->body.msg_start.msg_flags & MSG_FLAG_RESPONSE)
+ id = pkt->body.msg_start.msg_req_id;
+ ch->mbc_cur_msg = chan_msg_dequeue(ch, id);
+
+ if (!ch->mbc_cur_msg) {
+ //no msg, alloc dynamically
+ msg = alloc_msg(NULL, pkt->body.msg_start.msg_size);
+
+ msg->mbm_ch = ch;
+ msg->mbm_flags |= MSG_FLAG_REQUEST;
+ ch->mbc_cur_msg = msg;
+
+ } else if (pkt->body.msg_start.msg_size >
+ ch->mbc_cur_msg->mbm_len) {
+ chan_msg_done(ch, -EMSGSIZE);
+ MBX_ERR(mbx, "received msg is too big");
+ reset_pkt(pkt);
+ }
+ break;
+ case PKT_MSG_BODY:
+ if (!ch->mbc_cur_msg) {
+ MBX_ERR(mbx, "got unexpected msg body pkt\n");
+ reset_pkt(pkt);
+ }
+ break;
+ default:
+ MBX_ERR(mbx, "invalid mailbox pkt type\n");
+ reset_pkt(pkt);
+ return;
+ }
+
+ if (valid_pkt(pkt)) {
+ err = chan_pkt2msg(ch);
+ if (err || eom)
+ chan_msg_done(ch, err);
+ }
+ }
+
+ /* Handle timer event. */
+ if (test_bit(MBXCS_BIT_TICK, &ch->mbc_state)) {
+ timeout_msg(ch);
+ clear_bit(MBXCS_BIT_TICK, &ch->mbc_state);
+ }
+}
+
+static void chan_msg2pkt(struct mailbox_channel *ch)
+{
+ size_t cnt = 0;
+ size_t payload_off = 0;
+ void *msg_data, *pkt_data;
+ struct mailbox_msg *msg = ch->mbc_cur_msg;
+ struct mailbox_pkt *pkt = &ch->mbc_packet;
+ bool is_start = (ch->mbc_bytes_done == 0);
+ bool is_eom = false;
+
+ if (is_start) {
+ payload_off = offsetof(struct mailbox_pkt,
+ body.msg_start.payload);
+ } else {
+ payload_off = offsetof(struct mailbox_pkt,
+ body.msg_body.payload);
+ }
+ cnt = PACKET_SIZE * sizeof(u32) - payload_off;
+ if (cnt >= msg->mbm_len - ch->mbc_bytes_done) {
+ cnt = msg->mbm_len - ch->mbc_bytes_done;
+ is_eom = true;
+ }
+
+ pkt->hdr.type = is_start ? PKT_MSG_START : PKT_MSG_BODY;
+ pkt->hdr.type |= is_eom ? PKT_TYPE_MSG_END : 0;
+ pkt->hdr.payload_size = cnt;
+
+ if (is_start) {
+ pkt->body.msg_start.msg_req_id = msg->mbm_req_id;
+ pkt->body.msg_start.msg_size = msg->mbm_len;
+ pkt->body.msg_start.msg_flags = msg->mbm_flags;
+ pkt_data = pkt->body.msg_start.payload;
+ } else {
+ pkt_data = pkt->body.msg_body.payload;
+ }
+ msg_data = msg->mbm_data + ch->mbc_bytes_done;
+ (void) memcpy(pkt_data, msg_data, cnt);
+}
+
+static void check_tx_stall(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ struct mailbox_msg *msg = ch->mbc_cur_msg;
+
+ /*
+ * No stall checking in polling mode. Don't know how often peer will
+ * check the channel.
+ */
+ if ((msg == NULL) || test_bit(MBXCS_BIT_POLL_MODE, &ch->mbc_state))
+ return;
+
+ /*
+ * No tx intr has come since last check.
+ * The TX channel is stalled, reset it.
+ */
+ if (test_bit(MBXCS_BIT_CHK_STALL, &ch->mbc_state)) {
+ MBX_ERR(mbx, "TX channel stall detected, reset...\n");
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_ctrl, 0x1);
+ chan_msg_done(ch, -ETIME);
+ connect_state_touch(mbx, MB_CONN_FIN);
+ /* Mark it for next check. */
+ } else {
+ set_bit(MBXCS_BIT_CHK_STALL, &ch->mbc_state);
+ }
+}
+
+
+
+static void rx_enqueued_msg_timer_on(struct mailbox *mbx, uint64_t req_id)
+{
+ struct list_head *pos, *n;
+ struct mailbox_msg *msg = NULL;
+ struct mailbox_channel *ch = NULL;
+ ch = &mbx->mbx_rx;
+ MBX_DBG(mbx, "try to set ch rx, req_id %llu\n", req_id);
+ mutex_lock(&ch->mbc_mutex);
+
+ list_for_each_safe(pos, n, &ch->mbc_msgs) {
+ msg = list_entry(pos, struct mailbox_msg, mbm_list);
+ if (msg->mbm_req_id == req_id) {
+ msg->mbm_timer_on = true;
+ MBX_DBG(mbx, "set ch rx, req_id %llu\n", req_id);
+ break;
+ }
+ }
+
+ mutex_unlock(&ch->mbc_mutex);
+
+}
+
+/*
+ * Worker for TX channel.
+ */
+static void chan_do_tx(struct mailbox_channel *ch)
+{
+ struct mailbox *mbx = ch->mbc_parent;
+ u32 st = mailbox_reg_rd(mbx, &mbx->mbx_regs->mbr_status);
+
+ /* Check if a packet has been read by peer. */
+ if ((st != 0xffffffff) && ((st & STATUS_STA) != 0)) {
+ clear_bit(MBXCS_BIT_CHK_STALL, &ch->mbc_state);
+
+ /*
+ * The mailbox is free for sending new pkt now. See if we
+ * have something to send.
+ */
+
+ /* Finished sending a whole msg, call it done. */
+ if (ch->mbc_cur_msg &&
+ (ch->mbc_cur_msg->mbm_len == ch->mbc_bytes_done)) {
+ rx_enqueued_msg_timer_on(mbx, ch->mbc_cur_msg->mbm_req_id);
+ chan_msg_done(ch, 0);
+ }
+
+ if (!ch->mbc_cur_msg) {
+ ch->mbc_cur_msg = chan_msg_dequeue(ch, INVALID_MSG_ID);
+ if (ch->mbc_cur_msg)
+ ch->mbc_cur_msg->mbm_timer_on = true;
+ }
+
+ if (ch->mbc_cur_msg) {
+ chan_msg2pkt(ch);
+ } else if (valid_pkt(&mbx->mbx_tst_pkt)) {
+ (void) memcpy(&ch->mbc_packet, &mbx->mbx_tst_pkt,
+ sizeof(struct mailbox_pkt));
+ reset_pkt(&mbx->mbx_tst_pkt);
+ } else {
+ return; /* Nothing to send. */
+ }
+
+ chan_send_pkt(ch);
+ }
+
+ /* Handle timer event. */
+ if (test_bit(MBXCS_BIT_TICK, &ch->mbc_state)) {
+ timeout_msg(ch);
+ check_tx_stall(ch);
+ clear_bit(MBXCS_BIT_TICK, &ch->mbc_state);
+ }
+}
+
+static int mailbox_connect_status(struct platform_device *pdev)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ int ret = 0;
+ mutex_lock(&mbx->mbx_lock);
+ ret = mbx->mbx_paired;
+ mutex_unlock(&mbx->mbx_lock);
+ return ret;
+}
+
+static ssize_t mailbox_ctl_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ u32 *reg = (u32 *)mbx->mbx_regs;
+ int r, n;
+ int nreg = sizeof (struct mailbox_reg) / sizeof (u32);
+
+ for (r = 0, n = 0; r < nreg; r++, reg++) {
+ /* Non-status registers. */
+ if ((reg == &mbx->mbx_regs->mbr_resv1) ||
+ (reg == &mbx->mbx_regs->mbr_wrdata) ||
+ (reg == &mbx->mbx_regs->mbr_rddata) ||
+ (reg == &mbx->mbx_regs->mbr_resv2))
+ continue;
+ /* Write-only status register. */
+ if (reg == &mbx->mbx_regs->mbr_ctrl) {
+ n += sprintf(buf + n, "%02ld %10s = --\n",
+ r * sizeof (u32), reg2name(mbx, reg));
+ /* Read-able status register. */
+ } else {
+ n += sprintf(buf + n, "%02ld %10s = 0x%08x\n",
+ r * sizeof (u32), reg2name(mbx, reg),
+ mailbox_reg_rd(mbx, reg));
+ }
+ }
+
+ return n;
+}
+
+static ssize_t mailbox_ctl_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ u32 off, val;
+ int nreg = sizeof (struct mailbox_reg) / sizeof (u32);
+ u32 *reg = (u32 *)mbx->mbx_regs;
+
+ if (sscanf(buf, "%d:%d", &off, &val) != 2 || (off % sizeof (u32)) ||
+ !(off >= 0 && off < nreg * sizeof (u32))) {
+ MBX_ERR(mbx, "input should be <reg_offset:reg_val>");
+ return -EINVAL;
+ }
+ reg += off / sizeof (u32);
+
+ mailbox_reg_wr(mbx, reg, val);
+ return count;
+}
+/* HW register level debugging i/f. */
+static DEVICE_ATTR_RW(mailbox_ctl);
+
+static ssize_t mailbox_pkt_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ int ret = 0;
+
+ if (valid_pkt(&mbx->mbx_tst_pkt)) {
+ (void) memcpy(buf, mbx->mbx_tst_pkt.body.data,
+ mbx->mbx_tst_pkt.hdr.payload_size);
+ ret = mbx->mbx_tst_pkt.hdr.payload_size;
+ }
+
+ return ret;
+}
+
+static ssize_t mailbox_pkt_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ size_t maxlen = sizeof (mbx->mbx_tst_pkt.body.data);
+
+ if (count > maxlen) {
+ MBX_ERR(mbx, "max input length is %ld", maxlen);
+ return 0;
+ }
+
+ (void) memcpy(mbx->mbx_tst_pkt.body.data, buf, count);
+ mbx->mbx_tst_pkt.hdr.payload_size = count;
+ mbx->mbx_tst_pkt.hdr.type = PKT_TEST;
+ complete(&mbx->mbx_tx.mbc_worker);
+ return count;
+}
+
+/* Packet test i/f. */
+static DEVICE_ATTR_RW(mailbox_pkt);
+
+static ssize_t mailbox_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ struct mailbox_req req;
+ size_t respsz = sizeof (mbx->mbx_tst_rx_msg);
+ int ret = 0;
+
+ req.req = MAILBOX_REQ_TEST_READ;
+ ret = mailbox_request(to_platform_device(dev), &req, sizeof (req),
+ mbx->mbx_tst_rx_msg, &respsz, NULL, NULL);
+ if (ret) {
+ MBX_ERR(mbx, "failed to read test msg from peer: %d", ret);
+ } else if (respsz > 0) {
+ (void) memcpy(buf, mbx->mbx_tst_rx_msg, respsz);
+ ret = respsz;
+ }
+
+ return ret;
+}
+
+static ssize_t mailbox_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ size_t maxlen = sizeof (mbx->mbx_tst_tx_msg);
+ struct mailbox_req req = { 0 };
+
+ if (count > maxlen) {
+ MBX_ERR(mbx, "max input length is %ld", maxlen);
+ return 0;
+ }
+
+ (void) memcpy(mbx->mbx_tst_tx_msg, buf, count);
+ mbx->mbx_tst_tx_msg_len = count;
+ req.req = MAILBOX_REQ_TEST_READY;
+ (void) mailbox_post(mbx->mbx_pdev, 0, &req, sizeof (req));
+
+ return count;
+}
+
+/* Msg test i/f. */
+static DEVICE_ATTR_RW(mailbox);
+
+static ssize_t connection_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ int ret;
+ ret = mailbox_connect_status(pdev);
+ return sprintf(buf, "0x%x\n", ret);
+}
+static DEVICE_ATTR_RO(connection);
+
+
+static struct attribute *mailbox_attrs[] = {
+ &dev_attr_mailbox.attr,
+ &dev_attr_mailbox_ctl.attr,
+ &dev_attr_mailbox_pkt.attr,
+ &dev_attr_connection.attr,
+ NULL,
+};
+
+static const struct attribute_group mailbox_attrgroup = {
+ .attrs = mailbox_attrs,
+};
+
+static void dft_req_msg_cb(void *arg, void *data, size_t len, u64 id, int err)
+{
+ struct mailbox_msg *respmsg;
+ struct mailbox_msg *reqmsg = (struct mailbox_msg *)arg;
+ struct mailbox *mbx = reqmsg->mbm_ch->mbc_parent;
+
+ /*
+ * Can't send out request msg.
+ * Removing corresponding response msg from queue and return error.
+ */
+ if (err) {
+ respmsg = chan_msg_dequeue(&mbx->mbx_rx, reqmsg->mbm_req_id);
+ if (respmsg)
+ msg_done(respmsg, err);
+ }
+}
+
+static void dft_post_msg_cb(void *arg, void *buf, size_t len, u64 id, int err)
+{
+ struct mailbox_msg *msg = (struct mailbox_msg *)arg;
+
+ if (err) {
+ MBX_ERR(msg->mbm_ch->mbc_parent,
+ "failed to post msg, err=%d", err);
+ }
+}
+
+/*
+ * Msg will be sent to peer and reply will be received.
+ */
+int mailbox_request(struct platform_device *pdev, void *req, size_t reqlen,
+ void *resp, size_t *resplen, mailbox_msg_cb_t cb, void *cbarg)
+{
+ int rv = -ENOMEM;
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ struct mailbox_msg *reqmsg = NULL, *respmsg = NULL;
+
+ MBX_INFO(mbx, "sending request: %d", ((struct mailbox_req *)req)->req);
+
+ if (cb) {
+ reqmsg = alloc_msg(NULL, reqlen);
+ if (reqmsg)
+ (void) memcpy(reqmsg->mbm_data, req, reqlen);
+ } else {
+ reqmsg = alloc_msg(req, reqlen);
+ }
+ if (!reqmsg)
+ goto fail;
+ reqmsg->mbm_cb = dft_req_msg_cb;
+ reqmsg->mbm_cb_arg = reqmsg;
+ reqmsg->mbm_req_id = (uintptr_t)reqmsg->mbm_data;
+
+ respmsg = alloc_msg(resp, *resplen);
+ if (!respmsg)
+ goto fail;
+ respmsg->mbm_cb = cb;
+ respmsg->mbm_cb_arg = cbarg;
+ /* Only interested in response w/ same ID. */
+ respmsg->mbm_req_id = reqmsg->mbm_req_id;
+
+ /* Always enqueue RX msg before TX one to avoid race. */
+ rv = chan_msg_enqueue(&mbx->mbx_rx, respmsg);
+ if (rv)
+ goto fail;
+ rv = chan_msg_enqueue(&mbx->mbx_tx, reqmsg);
+ if (rv) {
+ respmsg = chan_msg_dequeue(&mbx->mbx_rx, reqmsg->mbm_req_id);
+ goto fail;
+ }
+
+ /* Kick TX channel to try to send out msg. */
+ complete(&mbx->mbx_tx.mbc_worker);
+
+ if (cb)
+ return 0;
+
+ wait_for_completion(&respmsg->mbm_complete);
+ rv = respmsg->mbm_error;
+ if (rv == 0)
+ *resplen = respmsg->mbm_len;
+
+ free_msg(respmsg);
+ return rv;
+
+fail:
+ if (reqmsg)
+ free_msg(reqmsg);
+ if (respmsg)
+ free_msg(respmsg);
+ return rv;
+}
+
+/*
+ * Msg will be posted, no wait for reply.
+ */
+int mailbox_post(struct platform_device *pdev, u64 reqid, void *buf, size_t len)
+{
+ int rv = 0;
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ struct mailbox_msg *msg = alloc_msg(NULL, len);
+
+ if (reqid == 0) {
+ MBX_DBG(mbx, "posting request: %d",
+ ((struct mailbox_req *)buf)->req);
+ } else {
+ MBX_DBG(mbx, "posting response...");
+ }
+
+ if (!msg)
+ return -ENOMEM;
+
+ (void) memcpy(msg->mbm_data, buf, len);
+ msg->mbm_cb = dft_post_msg_cb;
+ msg->mbm_cb_arg = msg;
+ if (reqid) {
+ msg->mbm_req_id = reqid;
+ msg->mbm_flags |= MSG_FLAG_RESPONSE;
+ } else {
+ msg->mbm_req_id = (uintptr_t)msg->mbm_data;
+ }
+
+ rv = chan_msg_enqueue(&mbx->mbx_tx, msg);
+ if (rv)
+ free_msg(msg);
+
+ /* Kick TX channel to try to send out msg. */
+ complete(&mbx->mbx_tx.mbc_worker);
+
+ return rv;
+}
+/*
+ * should not be called by other than connect_state_handler
+ */
+static int mailbox_connection_notify(struct platform_device *pdev, uint64_t sec_id, uint64_t flag)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ struct mailbox_req *mb_req = NULL;
+ struct mailbox_conn mb_conn = { 0 };
+ int ret = 0;
+ size_t data_len = 0, reqlen = 0;
+ data_len = sizeof(struct mailbox_conn);
+ reqlen = sizeof(struct mailbox_req) + data_len;
+
+ mb_req = (struct mailbox_req *)vmalloc(reqlen);
+ if (!mb_req) {
+ ret = -ENOMEM;
+ goto done;
+ }
+ mb_req->req = MAILBOX_REQ_CONN_EXPL;
+ if (!mbx->mbx_kaddr) {
+ ret = -ENOMEM;
+ goto done;
+ }
+
+ mb_conn.kaddr = mbx->mbx_kaddr;
+ mb_conn.paddr = virt_to_phys(mbx->mbx_kaddr);
+ mb_conn.crc32 = crc32c_le(~0, mbx->mbx_kaddr, PAGE_SIZE);
+ mb_conn.flag = flag;
+ mb_conn.ver = mbx->mbx_prot_ver;
+
+ if (sec_id != 0) {
+ mb_conn.sec_id = sec_id;
+ } else {
+ mb_conn.sec_id = (uint64_t)mbx->mbx_kaddr;
+ mbx->mbx_conn_id = (uint64_t)mbx->mbx_kaddr;
+ }
+
+ memcpy(mb_req->data, &mb_conn, data_len);
+
+ ret = mailbox_post(pdev, 0, mb_req, reqlen);
+
+done:
+ vfree(mb_req);
+ return ret;
+}
+
+static int mailbox_connection_explore(struct platform_device *pdev, struct mailbox_conn *mb_conn)
+{
+ int ret = 0;
+ uint32_t crc_chk;
+ phys_addr_t paddr;
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ if (!mb_conn) {
+ ret = -EFAULT;
+ goto done;
+ }
+
+ paddr = virt_to_phys(mb_conn->kaddr);
+ if (paddr != mb_conn->paddr) {
+ MBX_INFO(mbx, "mb_conn->paddr %llx paddr: %llx\n", mb_conn->paddr, paddr);
+ MBX_INFO(mbx, "Failed to get the same physical addr, running in VMs?\n");
+ ret = -EFAULT;
+ goto done;
+ }
+ crc_chk = crc32c_le(~0, mb_conn->kaddr, PAGE_SIZE);
+
+ if (crc_chk != mb_conn->crc32) {
+ MBX_INFO(mbx, "crc32 : %x, %x\n", mb_conn->crc32, crc_chk);
+ MBX_INFO(mbx, "failed to get the same CRC\n");
+ ret = -EFAULT;
+ goto done;
+ }
+done:
+ return ret;
+}
+
+static int mailbox_get_data(struct platform_device *pdev, enum data_kind kind)
+{
+ int ret = 0;
+ switch (kind) {
+ case PEER_CONN:
+ ret = mailbox_connect_status(pdev);
+ break;
+ default:
+ break;
+ }
+
+ return ret;
+}
+
+
+static void connect_state_handler(struct mailbox *mbx, struct mailbox_conn *conn)
+{
+ int ret = 0;
+
+ if (!mbx || !conn)
+ return;
+
+ mutex_lock(&mbx->mbx_lock);
+
+ switch (conn->flag) {
+ case MB_CONN_INIT:
+ /* clean up all cached data, */
+ mbx->mbx_paired = 0;
+ mbx->mbx_established = false;
+ kfree(mbx->mbx_kaddr);
+
+ mbx->mbx_kaddr = kzalloc(PAGE_SIZE, GFP_KERNEL);
+ get_random_bytes(mbx->mbx_kaddr, PAGE_SIZE);
+ ret = mailbox_connection_notify(mbx->mbx_pdev, 0, MB_CONN_SYN);
+ if (ret)
+ goto done;
+ mbx->mbx_state = CONN_SYN_SENT;
+ break;
+ case MB_CONN_SYN:
+ if (mbx->mbx_state == CONN_SYN_SENT) {
+ if (!mailbox_connection_explore(mbx->mbx_pdev, conn)) {
+ mbx->mbx_paired |= 0x2;
+ MBX_INFO(mbx, "mailbox mbx_prot_ver %x", mbx->mbx_prot_ver);
+ }
+ ret = mailbox_connection_notify(mbx->mbx_pdev, conn->sec_id, MB_CONN_ACK);
+ if (ret)
+ goto done;
+ mbx->mbx_state = CONN_SYN_RECV;
+ } else
+ mbx->mbx_state = CONN_START;
+ break;
+ case MB_CONN_ACK:
+ if (mbx->mbx_state & (CONN_SYN_SENT | CONN_SYN_RECV)) {
+ if (mbx->mbx_conn_id == (uint64_t)conn->sec_id) {
+ mbx->mbx_paired |= 0x1;
+ mbx->mbx_established = true;
+ mbx->mbx_state = CONN_ESTABLISH;
+ kfree(mbx->mbx_kaddr);
+ mbx->mbx_kaddr = NULL;
+ } else
+ mbx->mbx_state = CONN_START;
+ }
+ break;
+ case MB_CONN_FIN:
+ mbx->mbx_paired = 0;
+ mbx->mbx_established = false;
+ kfree(mbx->mbx_kaddr);
+ mbx->mbx_kaddr = NULL;
+ mbx->mbx_state = CONN_START;
+ break;
+ default:
+ break;
+ }
+done:
+ if (ret) {
+ kfree(mbx->mbx_kaddr);
+ mbx->mbx_kaddr = NULL;
+ mbx->mbx_paired = 0;
+ mbx->mbx_state = CONN_START;
+ }
+ mutex_unlock(&mbx->mbx_lock);
+ MBX_INFO(mbx, "mailbox connection state %d", mbx->mbx_paired);
+}
+
+static void process_request(struct mailbox *mbx, struct mailbox_msg *msg)
+{
+ struct mailbox_req *req = (struct mailbox_req *)msg->mbm_data;
+ struct mailbox_conn *conn = (struct mailbox_conn *)req->data;
+ int rc;
+ const char *recvstr = "received request from peer";
+ const char *sendstr = "sending test msg to peer";
+
+ if (req->req == MAILBOX_REQ_TEST_READ) {
+ MBX_INFO(mbx, "%s: %d", recvstr, req->req);
+ if (mbx->mbx_tst_tx_msg_len) {
+ MBX_INFO(mbx, "%s", sendstr);
+ rc = mailbox_post(mbx->mbx_pdev, msg->mbm_req_id,
+ mbx->mbx_tst_tx_msg, mbx->mbx_tst_tx_msg_len);
+ if (rc)
+ MBX_ERR(mbx, "%s failed: %d", sendstr, rc);
+ else
+ mbx->mbx_tst_tx_msg_len = 0;
+
+ }
+ } else if (req->req == MAILBOX_REQ_TEST_READY) {
+ MBX_INFO(mbx, "%s: %d", recvstr, req->req);
+ } else if (req->req == MAILBOX_REQ_CONN_EXPL) {
+ MBX_INFO(mbx, "%s: %d", recvstr, req->req);
+ if (mbx->mbx_state != CONN_SYN_SENT) {
+ /* if your peer droped without notice,
+ * initial the connection Simultaneously
+ * again.
+ */
+ if (conn->flag == MB_CONN_SYN) {
+ connect_state_touch(mbx, MB_CONN_INIT);
+ }
+ }
+ connect_state_handler(mbx, conn);
+ } else if (mbx->mbx_listen_cb) {
+ /* Call client's registered callback to process request. */
+ MBX_DBG(mbx, "%s: %d, passed on", recvstr, req->req);
+ mbx->mbx_listen_cb(mbx->mbx_listen_cb_arg, msg->mbm_data,
+ msg->mbm_len, msg->mbm_req_id, msg->mbm_error);
+ } else {
+ MBX_INFO(mbx, "%s: %d, dropped", recvstr, req->req);
+ }
+}
+
+/*
+ * Wait for request from peer.
+ */
+static void mailbox_recv_request(struct work_struct *work)
+{
+ int rv = 0;
+ struct mailbox_msg *msg = NULL;
+ struct mailbox *mbx =
+ container_of(work, struct mailbox, mbx_listen_worker);
+
+ for (;;) {
+ /* Only interested in request msg. */
+
+ rv = wait_for_completion_interruptible(&mbx->mbx_comp);
+ if (rv)
+ break;
+ mutex_lock(&mbx->mbx_lock);
+ msg = list_first_entry_or_null(&mbx->mbx_req_list,
+ struct mailbox_msg, mbm_list);
+
+ if (msg) {
+ list_del(&msg->mbm_list);
+ mbx->mbx_req_cnt--;
+ mbx->mbx_req_sz -= msg->mbm_len;
+ mutex_unlock(&mbx->mbx_lock);
+ } else {
+ mutex_unlock(&mbx->mbx_lock);
+ break;
+ }
+
+ process_request(mbx, msg);
+ free_msg(msg);
+ }
+
+ if (rv == -ESHUTDOWN)
+ MBX_INFO(mbx, "channel is closed, no listen to peer");
+ else if (rv != 0)
+ MBX_ERR(mbx, "failed to receive request from peer, err=%d", rv);
+
+ if (msg)
+ free_msg(msg);
+}
+
+int mailbox_listen(struct platform_device *pdev,
+ mailbox_msg_cb_t cb, void *cbarg)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+
+ mbx->mbx_listen_cb_arg = cbarg;
+ /* mbx->mbx_listen_cb is used in another thread as a condition to
+ * call the function. Ensuring that the argument is captured before
+ * the function pointer
+ */
+ wmb();
+ mbx->mbx_listen_cb = cb;
+
+ return 0;
+}
+
+static int mailbox_enable_intr_mode(struct mailbox *mbx)
+{
+ struct resource *res;
+ int ret;
+ struct platform_device *pdev = mbx->mbx_pdev;
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+
+ if (mbx->mbx_irq != -1)
+ return 0;
+
+ res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (res == NULL) {
+ MBX_ERR(mbx, "failed to acquire intr resource");
+ return -EINVAL;
+ }
+
+ ret = xocl_user_interrupt_reg(xdev, res->start, mailbox_isr, mbx);
+ if (ret) {
+ MBX_ERR(mbx, "failed to add intr handler");
+ return ret;
+ }
+ ret = xocl_user_interrupt_config(xdev, res->start, true);
+ BUG_ON(ret != 0);
+
+ /* Only see intr when we have full packet sent or received. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_rit, PACKET_SIZE - 1);
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_sit, 0);
+
+ /* Finally, enable TX / RX intr. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_ie, 0x3);
+
+ clear_bit(MBXCS_BIT_POLL_MODE, &mbx->mbx_rx.mbc_state);
+ chan_config_timer(&mbx->mbx_rx);
+
+ clear_bit(MBXCS_BIT_POLL_MODE, &mbx->mbx_tx.mbc_state);
+ chan_config_timer(&mbx->mbx_tx);
+
+ mbx->mbx_irq = res->start;
+ return 0;
+}
+
+static void mailbox_disable_intr_mode(struct mailbox *mbx)
+{
+ struct platform_device *pdev = mbx->mbx_pdev;
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+
+ /*
+ * No need to turn on polling mode for TX, which has
+ * a channel stall checking timer always on when there is
+ * outstanding TX packet.
+ */
+ set_bit(MBXCS_BIT_POLL_MODE, &mbx->mbx_rx.mbc_state);
+ chan_config_timer(&mbx->mbx_rx);
+
+ /* Disable both TX / RX intrs. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_ie, 0x0);
+
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_rit, 0x0);
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_sit, 0x0);
+
+ if (mbx->mbx_irq == -1)
+ return;
+
+ (void) xocl_user_interrupt_config(xdev, mbx->mbx_irq, false);
+ (void) xocl_user_interrupt_reg(xdev, mbx->mbx_irq, NULL, mbx);
+
+ mbx->mbx_irq = -1;
+}
+
+int mailbox_reset(struct platform_device *pdev, bool end_of_reset)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+ int ret = 0;
+
+ if (mailbox_no_intr)
+ return 0;
+
+ if (end_of_reset) {
+ MBX_INFO(mbx, "enable intr mode");
+ if (mailbox_enable_intr_mode(mbx) != 0)
+ MBX_ERR(mbx, "failed to enable intr after reset");
+ } else {
+ MBX_INFO(mbx, "enable polling mode");
+ mailbox_disable_intr_mode(mbx);
+ }
+ return ret;
+}
+
+/* Kernel APIs exported from this sub-device driver. */
+static struct xocl_mailbox_funcs mailbox_ops = {
+ .request = mailbox_request,
+ .post = mailbox_post,
+ .listen = mailbox_listen,
+ .reset = mailbox_reset,
+ .get_data = mailbox_get_data,
+};
+
+static int mailbox_remove(struct platform_device *pdev)
+{
+ struct mailbox *mbx = platform_get_drvdata(pdev);
+
+ BUG_ON(mbx == NULL);
+
+ connect_state_touch(mbx, MB_CONN_FIN);
+
+ mailbox_disable_intr_mode(mbx);
+
+ sysfs_remove_group(&pdev->dev.kobj, &mailbox_attrgroup);
+
+ chan_fini(&mbx->mbx_rx);
+ chan_fini(&mbx->mbx_tx);
+ listen_wq_fini(mbx);
+
+ BUG_ON(!(list_empty(&mbx->mbx_req_list)));
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_MAILBOX, NULL);
+
+ if (mbx->mbx_regs)
+ iounmap(mbx->mbx_regs);
+
+ MBX_INFO(mbx, "mailbox cleaned up successfully");
+ platform_set_drvdata(pdev, NULL);
+ kfree(mbx);
+ return 0;
+}
+
+static int mailbox_probe(struct platform_device *pdev)
+{
+ struct mailbox *mbx = NULL;
+ struct resource *res;
+ int ret;
+
+ mbx = kzalloc(sizeof(struct mailbox), GFP_KERNEL);
+ if (!mbx)
+ return -ENOMEM;
+ platform_set_drvdata(pdev, mbx);
+ mbx->mbx_pdev = pdev;
+ mbx->mbx_irq = (u32)-1;
+
+
+ init_completion(&mbx->mbx_comp);
+ mutex_init(&mbx->mbx_lock);
+ INIT_LIST_HEAD(&mbx->mbx_req_list);
+ mbx->mbx_req_cnt = 0;
+ mbx->mbx_req_sz = 0;
+
+ mutex_init(&mbx->mbx_conn_lock);
+ mbx->mbx_established = false;
+ mbx->mbx_conn_id = 0;
+ mbx->mbx_kaddr = NULL;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ mbx->mbx_regs = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!mbx->mbx_regs) {
+ MBX_ERR(mbx, "failed to map in registers");
+ ret = -EIO;
+ goto failed;
+ }
+ /* Reset TX channel, RX channel is managed by peer as his TX. */
+ mailbox_reg_wr(mbx, &mbx->mbx_regs->mbr_ctrl, 0x1);
+
+ /* Set up software communication channels. */
+ ret = chan_init(mbx, "RX", &mbx->mbx_rx, chan_do_rx);
+ if (ret != 0) {
+ MBX_ERR(mbx, "failed to init rx channel");
+ goto failed;
+ }
+ ret = chan_init(mbx, "TX", &mbx->mbx_tx, chan_do_tx);
+ if (ret != 0) {
+ MBX_ERR(mbx, "failed to init tx channel");
+ goto failed;
+ }
+ /* Dedicated thread for listening to peer request. */
+ mbx->mbx_listen_wq =
+ create_singlethread_workqueue(dev_name(&mbx->mbx_pdev->dev));
+ if (!mbx->mbx_listen_wq) {
+ MBX_ERR(mbx, "failed to create request-listen work queue");
+ goto failed;
+ }
+ INIT_WORK(&mbx->mbx_listen_worker, mailbox_recv_request);
+ queue_work(mbx->mbx_listen_wq, &mbx->mbx_listen_worker);
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &mailbox_attrgroup);
+ if (ret != 0) {
+ MBX_ERR(mbx, "failed to init sysfs");
+ goto failed;
+ }
+
+ if (mailbox_no_intr) {
+ MBX_INFO(mbx, "Enabled timer-driven mode");
+ mailbox_disable_intr_mode(mbx);
+ } else {
+ ret = mailbox_enable_intr_mode(mbx);
+ if (ret != 0)
+ goto failed;
+ }
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_MAILBOX, &mailbox_ops);
+
+ connect_state_touch(mbx, MB_CONN_INIT);
+ mbx->mbx_prot_ver = MB_PROTOCOL_VER;
+
+ MBX_INFO(mbx, "successfully initialized");
+ return 0;
+
+failed:
+ mailbox_remove(pdev);
+ return ret;
+}
+
+struct platform_device_id mailbox_id_table[] = {
+ { XOCL_MAILBOX, 0 },
+ { },
+};
+
+static struct platform_driver mailbox_driver = {
+ .probe = mailbox_probe,
+ .remove = mailbox_remove,
+ .driver = {
+ .name = XOCL_MAILBOX,
+ },
+ .id_table = mailbox_id_table,
+};
+
+int __init xocl_init_mailbox(void)
+{
+ BUILD_BUG_ON(sizeof(struct mailbox_pkt) != sizeof(u32) * PACKET_SIZE);
+ return platform_driver_register(&mailbox_driver);
+}
+
+void xocl_fini_mailbox(void)
+{
+ platform_driver_unregister(&mailbox_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/mb_scheduler.c b/drivers/gpu/drm/xocl/subdev/mb_scheduler.c
new file mode 100644
index 000000000000..b3ed3ae0b41a
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/mb_scheduler.c
@@ -0,0 +1,3059 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2018-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ * Soren Soe <soren.soe@xilinx.com>
+ */
+
+/*
+ * Kernel Driver Scheduler (KDS) for XRT
+ *
+ * struct xocl_cmd
+ * - wraps exec BOs create from user space
+ * - transitions through a number of states
+ * - initially added to pending command queue
+ * - consumed by scheduler which manages its execution (state transition)
+ * struct xcol_cu
+ * - compute unit for executing commands
+ * - used only without embedded scheduler (ert)
+ * - talks to HW compute units
+ * struct xocl_ert
+ * - embedded scheduler for executing commands on ert
+ * - talks to HW ERT
+ * struct exec_core
+ * - execution core managing execution on one device
+ * struct xocl_scheduler
+ * - manages execution of cmds on one or more exec cores
+ * - executed in a separate kernel thread
+ * - loops repeatedly when there is work to do
+ * - moves pending commands into a scheduler command queue
+ *
+ * [new -> pending]. The xocl API adds exec BOs to KDS. The exec BOs are
+ * wrapped in a xocl_cmd object and added to a pending command queue.
+ *
+ * [pending -> queued]. Scheduler loops repeatedly and copies pending commands
+ * to its own command queue, then managaes command execution on one or more
+ * execution cores.
+ *
+ * [queued -> submitted]. Commands are submitted for execution on execution
+ * core when the core has room for new commands.
+ *
+ * [submitted -> running]. Once submitted, a command is transition by
+ * scheduler into running state when there is an available compute unit (no
+ * ert) or if ERT is used, then when ERT has room.
+ *
+ * [running -> complete]. Commands running on ERT complete by sending an
+ * interrupt to scheduler. When ERT is not used, commands are running on a
+ * compute unit and are polled for completion.
+ */
+
+#include <linux/bitmap.h>
+#include <linux/list.h>
+#include <linux/eventfd.h>
+#include <linux/kthread.h>
+#include "../ert.h"
+#include "../xocl_drv.h"
+#include "../userpf/common.h"
+
+//#define SCHED_VERBOSE
+
+#if defined(__GNUC__)
+#define SCHED_UNUSED __attribute__((unused))
+#endif
+
+#define sched_debug_packet(packet, size) \
+({ \
+ int i; \
+ u32 *data = (u32 *)packet; \
+ for (i = 0; i < size; ++i) \
+ DRM_INFO("packet(0x%p) data[%d] = 0x%x\n", data, i, data[i]); \
+})
+
+#ifdef SCHED_VERBOSE
+# define SCHED_DEBUG(msg) DRM_INFO(msg)
+# define SCHED_DEBUGF(format, ...) DRM_INFO(format, ##__VA_ARGS__)
+# define SCHED_PRINTF(format, ...) DRM_INFO(format, ##__VA_ARGS__)
+# define SCHED_DEBUG_PACKET(packet, size) sched_debug_packet(packet, size)
+#else
+# define SCHED_DEBUG(msg)
+# define SCHED_DEBUGF(format, ...)
+# define SCHED_PRINTF(format, ...) DRM_INFO(format, ##__VA_ARGS__)
+# define SCHED_DEBUG_PACKET(packet, size)
+#endif
+
+/* constants */
+static const unsigned int no_index = -1;
+
+/* FFA handling */
+static const u32 AP_START = 0x1;
+static const u32 AP_DONE = 0x2;
+static const u32 AP_IDLE = 0x4;
+static const u32 AP_READY = 0x8;
+static const u32 AP_CONTINUE = 0x10;
+
+/* Forward declaration */
+struct exec_core;
+struct exec_ops;
+struct xocl_scheduler;
+
+static int validate(struct platform_device *pdev, struct client_ctx *client,
+ const struct drm_xocl_bo *bo);
+static bool exec_is_flush(struct exec_core *exec);
+static void scheduler_wake_up(struct xocl_scheduler *xs);
+static void scheduler_intr(struct xocl_scheduler *xs);
+static void scheduler_decr_poll(struct xocl_scheduler *xs);
+
+/*
+ */
+static void
+xocl_bitmap_to_arr32(u32 *buf, const unsigned long *bitmap, unsigned int nbits)
+{
+ unsigned int i, halfwords;
+
+ halfwords = DIV_ROUND_UP(nbits, 32);
+ for (i = 0; i < halfwords; i++) {
+ buf[i] = (u32) (bitmap[i/2] & UINT_MAX);
+ if (++i < halfwords)
+ buf[i] = (u32) (bitmap[i/2] >> 32);
+ }
+
+ /* Clear tail bits in last element of array beyond nbits. */
+ if (nbits % BITS_PER_LONG)
+ buf[halfwords - 1] &= (u32) (UINT_MAX >> ((-nbits) & 31));
+}
+
+static void
+xocl_bitmap_from_arr32(unsigned long *bitmap, const u32 *buf, unsigned int nbits)
+{
+ unsigned int i, halfwords;
+
+ halfwords = DIV_ROUND_UP(nbits, 32);
+ for (i = 0; i < halfwords; i++) {
+ bitmap[i/2] = (unsigned long) buf[i];
+ if (++i < halfwords)
+ bitmap[i/2] |= ((unsigned long) buf[i]) << 32;
+ }
+
+ /* Clear tail bits in last word beyond nbits. */
+ if (nbits % BITS_PER_LONG)
+ bitmap[(halfwords - 1) / 2] &= BITMAP_LAST_WORD_MASK(nbits);
+}
+
+
+/**
+ * slot_mask_idx() - Slot mask idx index for a given slot_idx
+ *
+ * @slot_idx: Global [0..127] index of a CQ slot
+ * Return: Index of the slot mask containing the slot_idx
+ */
+static inline unsigned int
+slot_mask_idx(unsigned int slot_idx)
+{
+ return slot_idx >> 5;
+}
+
+/**
+ * slot_idx_in_mask() - Index of command queue slot within the mask that contains it
+ *
+ * @slot_idx: Global [0..127] index of a CQ slot
+ * Return: Index of slot within the mask that contains it
+ */
+static inline unsigned int
+slot_idx_in_mask(unsigned int slot_idx)
+{
+ return slot_idx - (slot_mask_idx(slot_idx) << 5);
+}
+
+/**
+ * Command data used by scheduler
+ *
+ * @list: command object moves from pending to commmand queue list
+ * @cu_list: command object is added to CU list when running (penguin only)
+ *
+ * @bo: underlying drm buffer object
+ * @exec: execution device associated with this command
+ * @client: client (user process) context that created this command
+ * @xs: command scheduler responsible for schedulint this command
+ * @state: state of command object per scheduling
+ * @id: unique id for an active command object
+ * @cu_idx: index of CU executing this cmd object; used in penguin mode only
+ * @slot_idx: command queue index of this command object
+ * @wait_count: number of commands that must trigger this command before it can start
+ * @chain_count: number of commands that this command must trigger when it completes
+ * @chain: list of commands to trigger upon completion; maximum chain depth is 8
+ * @deps: list of commands this object depends on, converted to chain when command is queued
+ * @packet: mapped ert packet object from user space
+ */
+struct xocl_cmd {
+ struct list_head cq_list; // scheduler command queue
+ struct list_head rq_list; // exec core running queue
+
+ /* command packet */
+ struct drm_xocl_bo *bo;
+ union {
+ struct ert_packet *ecmd;
+ struct ert_start_kernel_cmd *kcmd;
+ };
+
+ DECLARE_BITMAP(cu_bitmap, MAX_CUS);
+
+ struct xocl_dev *xdev;
+ struct exec_core *exec;
+ struct client_ctx *client;
+ struct xocl_scheduler *xs;
+ enum ert_cmd_state state;
+
+ /* dependency handling */
+ unsigned int chain_count;
+ unsigned int wait_count;
+ union {
+ struct xocl_cmd *chain[8];
+ struct drm_xocl_bo *deps[8];
+ };
+
+ unsigned long uid; // unique id for this command
+ unsigned int cu_idx; // index of CU running this cmd (penguin mode)
+ unsigned int slot_idx; // index in exec core submit queue
+};
+
+/*
+ * List of free xocl_cmd objects.
+ *
+ * @free_cmds: populated with recycled xocl_cmd objects
+ * @cmd_mutex: mutex lock for cmd_list
+ *
+ * Command objects are recycled for later use and only freed when kernel
+ * module is unloaded.
+ */
+static LIST_HEAD(free_cmds);
+static DEFINE_MUTEX(free_cmds_mutex);
+
+/**
+ * delete_cmd_list() - reclaim memory for all allocated command objects
+ */
+static void
+cmd_list_delete(void)
+{
+ struct xocl_cmd *xcmd;
+ struct list_head *pos, *next;
+
+ mutex_lock(&free_cmds_mutex);
+ list_for_each_safe(pos, next, &free_cmds) {
+ xcmd = list_entry(pos, struct xocl_cmd, cq_list);
+ list_del(pos);
+ kfree(xcmd);
+ }
+ mutex_unlock(&free_cmds_mutex);
+}
+
+/*
+ * opcode() - Command opcode
+ *
+ * @cmd: Command object
+ * Return: Opcode per command packet
+ */
+static inline u32
+cmd_opcode(struct xocl_cmd *xcmd)
+{
+ return xcmd->ecmd->opcode;
+}
+
+/*
+ * type() - Command type
+ *
+ * @cmd: Command object
+ * Return: Type of command
+ */
+static inline u32
+cmd_type(struct xocl_cmd *xcmd)
+{
+ return xcmd->ecmd->type;
+}
+
+/*
+ * exec() - Get execution core
+ */
+static inline struct exec_core *
+cmd_exec(struct xocl_cmd *xcmd)
+{
+ return xcmd->exec;
+}
+
+/*
+ * uid() - Get unique id of command
+ */
+static inline unsigned long
+cmd_uid(struct xocl_cmd *xcmd)
+{
+ return xcmd->uid;
+}
+
+/*
+ */
+static inline unsigned int
+cmd_wait_count(struct xocl_cmd *xcmd)
+{
+ return xcmd->wait_count;
+}
+
+/**
+ * payload_size() - Command payload size
+ *
+ * @xcmd: Command object
+ * Return: Size in number of words of command packet payload
+ */
+static inline unsigned int
+cmd_payload_size(struct xocl_cmd *xcmd)
+{
+ return xcmd->ecmd->count;
+}
+
+/**
+ * cmd_packet_size() - Command packet size
+ *
+ * @xcmd: Command object
+ * Return: Size in number of words of command packet
+ */
+static inline unsigned int
+cmd_packet_size(struct xocl_cmd *xcmd)
+{
+ return cmd_payload_size(xcmd) + 1;
+}
+
+/**
+ * cu_masks() - Number of command packet cu_masks
+ *
+ * @xcmd: Command object
+ * Return: Total number of CU masks in command packet
+ */
+static inline unsigned int
+cmd_cumasks(struct xocl_cmd *xcmd)
+{
+ return 1 + xcmd->kcmd->extra_cu_masks;
+}
+
+/**
+ * regmap_size() - Size of regmap is payload size (n) minus the number of cu_masks
+ *
+ * @xcmd: Command object
+ * Return: Size of register map in number of words
+ */
+static inline unsigned int
+cmd_regmap_size(struct xocl_cmd *xcmd)
+{
+ return cmd_payload_size(xcmd) - cmd_cumasks(xcmd);
+}
+
+/*
+ */
+static inline struct ert_packet*
+cmd_packet(struct xocl_cmd *xcmd)
+{
+ return xcmd->ecmd;
+}
+
+/*
+ */
+static inline u32*
+cmd_regmap(struct xocl_cmd *xcmd)
+{
+ return xcmd->kcmd->data + xcmd->kcmd->extra_cu_masks;
+}
+
+/**
+ * cmd_set_int_state() - Set internal command state used by scheduler only
+ *
+ * @xcmd: command to change internal state on
+ * @state: new command state per ert.h
+ */
+static inline void
+cmd_set_int_state(struct xocl_cmd *xcmd, enum ert_cmd_state state)
+{
+ SCHED_DEBUGF("-> %s(%lu,%d)\n", __func__, xcmd->uid, state);
+ xcmd->state = state;
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * cmd_set_state() - Set both internal and external state of a command
+ *
+ * The state is reflected externally through the command packet
+ * as well as being captured in internal state variable
+ *
+ * @xcmd: command object
+ * @state: new state
+ */
+static inline void
+cmd_set_state(struct xocl_cmd *xcmd, enum ert_cmd_state state)
+{
+ SCHED_DEBUGF("->%s(%lu,%d)\n", __func__, xcmd->uid, state);
+ xcmd->state = state;
+ xcmd->ecmd->state = state;
+ SCHED_DEBUGF("<-%s\n", __func__);
+}
+
+/*
+ * update_state() - Update command state if client has aborted
+ */
+static enum ert_cmd_state
+cmd_update_state(struct xocl_cmd *xcmd)
+{
+ if (xcmd->state != ERT_CMD_STATE_RUNNING && xcmd->client->abort) {
+ userpf_info(xcmd->xdev, "aborting stale client cmd(%lu)", xcmd->uid);
+ cmd_set_state(xcmd, ERT_CMD_STATE_ABORT);
+ }
+ if (exec_is_flush(xcmd->exec)) {
+ userpf_info(xcmd->xdev, "aborting stale exec cmd(%lu)", xcmd->uid);
+ cmd_set_state(xcmd, ERT_CMD_STATE_ABORT);
+ }
+ return xcmd->state;
+}
+
+/*
+ * release_gem_object_reference() -
+ */
+static inline void
+cmd_release_gem_object_reference(struct xocl_cmd *xcmd)
+{
+ if (xcmd->bo)
+ drm_gem_object_put_unlocked(&xcmd->bo->base);
+//PORT4_20
+// drm_gem_object_unreference_unlocked(&xcmd->bo->base);
+}
+
+/*
+ */
+static inline void
+cmd_mark_active(struct xocl_cmd *xcmd)
+{
+ if (xcmd->bo)
+ xcmd->bo->metadata.active = xcmd;
+}
+
+/*
+ */
+static inline void
+cmd_mark_deactive(struct xocl_cmd *xcmd)
+{
+ if (xcmd->bo)
+ xcmd->bo->metadata.active = NULL;
+}
+
+/**
+ * chain_dependencies() - Chain this command to its dependencies
+ *
+ * @xcmd: Command to chain to its dependencies
+ *
+ * This function looks at all incoming explicit BO dependencies, checks if a
+ * corresponding xocl_cmd object exists (is active) in which case that command
+ * object must chain argument xcmd so that it (xcmd) can be triggered when
+ * dependency completes. The chained command has a wait count corresponding to
+ * the number of dependencies that are active.
+ */
+static int
+cmd_chain_dependencies(struct xocl_cmd *xcmd)
+{
+ int didx;
+ int dcount = xcmd->wait_count;
+
+ SCHED_DEBUGF("-> chain_dependencies of xcmd(%lu)\n", xcmd->uid);
+ for (didx = 0; didx < dcount; ++didx) {
+ struct drm_xocl_bo *dbo = xcmd->deps[didx];
+ struct xocl_cmd *chain_to = dbo->metadata.active;
+ // release reference created in ioctl call when dependency was looked up
+ // see comments in xocl_ioctl.c:xocl_execbuf_ioctl()
+//PORT4_20
+// drm_gem_object_unreference_unlocked(&dbo->base);
+ drm_gem_object_put_unlocked(&dbo->base);
+ xcmd->deps[didx] = NULL;
+ if (!chain_to) { /* command may have completed already */
+ --xcmd->wait_count;
+ continue;
+ }
+ if (chain_to->chain_count >= MAX_DEPS) {
+ DRM_INFO("chain count exceeded");
+ return 1;
+ }
+ SCHED_DEBUGF("+ xcmd(%lu)->chain[%d]=xcmd(%lu)", chain_to->uid, chain_to->chain_count, xcmd->uid);
+ chain_to->chain[chain_to->chain_count++] = xcmd;
+ }
+ SCHED_DEBUG("<- chain_dependencies\n");
+ return 0;
+}
+
+/**
+ * trigger_chain() - Trigger the execution of any commands chained to argument command
+ *
+ * @xcmd: Completed command that must trigger its chained (waiting) commands
+ *
+ * The argument command has completed and must trigger the execution of all
+ * chained commands whos wait_count is 0.
+ */
+static void
+cmd_trigger_chain(struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> trigger_chain xcmd(%lu)\n", xcmd->uid);
+ while (xcmd->chain_count) {
+ struct xocl_cmd *trigger = xcmd->chain[--xcmd->chain_count];
+
+ SCHED_DEBUGF("+ cmd(%lu) triggers cmd(%lu) with wait_count(%d)\n",
+ xcmd->uid, trigger->uid, trigger->wait_count);
+ // decrement trigger wait count
+ // scheduler will submit when wait count reaches zero
+ --trigger->wait_count;
+ }
+ SCHED_DEBUG("<- trigger_chain\n");
+}
+
+
+/**
+ * cmd_get() - Get a free command object
+ *
+ * Get from free/recycled list or allocate a new command if necessary.
+ *
+ * Return: Free command object
+ */
+static struct xocl_cmd*
+cmd_get(struct xocl_scheduler *xs, struct exec_core *exec, struct client_ctx *client)
+{
+ struct xocl_cmd *xcmd;
+ static unsigned long count;
+
+ mutex_lock(&free_cmds_mutex);
+ xcmd = list_first_entry_or_null(&free_cmds, struct xocl_cmd, cq_list);
+ if (xcmd)
+ list_del(&xcmd->cq_list);
+ mutex_unlock(&free_cmds_mutex);
+ if (!xcmd)
+ xcmd = kmalloc(sizeof(struct xocl_cmd), GFP_KERNEL);
+ if (!xcmd)
+ return ERR_PTR(-ENOMEM);
+ xcmd->uid = count++;
+ xcmd->exec = exec;
+ xcmd->cu_idx = no_index;
+ xcmd->slot_idx = no_index;
+ xcmd->xs = xs;
+ xcmd->xdev = client->xdev;
+ xcmd->client = client;
+ xcmd->bo = NULL;
+ xcmd->ecmd = NULL;
+ atomic_inc(&client->outstanding_execs);
+ SCHED_DEBUGF("xcmd(%lu) xcmd(%p) [-> new ]\n", xcmd->uid, xcmd);
+ return xcmd;
+}
+
+/**
+ * cmd_free() - free a command object
+ *
+ * @xcmd: command object to free (move to freelist)
+ *
+ * The command *is* in some current list (scheduler command queue)
+ */
+static void
+cmd_free(struct xocl_cmd *xcmd)
+{
+ cmd_release_gem_object_reference(xcmd);
+
+ mutex_lock(&free_cmds_mutex);
+ list_move_tail(&xcmd->cq_list, &free_cmds);
+ mutex_unlock(&free_cmds_mutex);
+
+ atomic_dec(&xcmd->xdev->outstanding_execs);
+ atomic_dec(&xcmd->client->outstanding_execs);
+ SCHED_DEBUGF("xcmd(%lu) [-> free]\n", xcmd->uid);
+}
+
+/**
+ * abort_cmd() - abort command object before it becomes pending
+ *
+ * @xcmd: command object to abort (move to freelist)
+ *
+ * Command object is *not* in any current list
+ *
+ * Return: 0
+ */
+static void
+cmd_abort(struct xocl_cmd *xcmd)
+{
+ mutex_lock(&free_cmds_mutex);
+ list_add_tail(&xcmd->cq_list, &free_cmds);
+ mutex_unlock(&free_cmds_mutex);
+ SCHED_DEBUGF("xcmd(%lu) [-> abort]\n", xcmd->uid);
+}
+
+/*
+ * cmd_bo_init() - Initialize a command object with an exec BO
+ *
+ * In penguin mode, the command object caches the CUs available
+ * to execute the command. When ERT is enabled, the CU info
+ * is not used.
+ */
+static void
+cmd_bo_init(struct xocl_cmd *xcmd, struct drm_xocl_bo *bo,
+ int numdeps, struct drm_xocl_bo **deps, int penguin)
+{
+ SCHED_DEBUGF("%s(%lu,bo,%d,deps,%d)\n", __func__, xcmd->uid, numdeps, penguin);
+ xcmd->bo = bo;
+ xcmd->ecmd = (struct ert_packet *)bo->vmapping;
+
+ if (penguin && cmd_opcode(xcmd) == ERT_START_KERNEL) {
+ unsigned int i = 0;
+ u32 cumasks[4] = {0};
+
+ cumasks[0] = xcmd->kcmd->cu_mask;
+ SCHED_DEBUGF("+ xcmd(%lu) cumask[0]=0x%x\n", xcmd->uid, cumasks[0]);
+ for (i = 0; i < xcmd->kcmd->extra_cu_masks; ++i) {
+ cumasks[i+1] = xcmd->kcmd->data[i];
+ SCHED_DEBUGF("+ xcmd(%lu) cumask[%d]=0x%x\n", xcmd->uid, i+1, cumasks[i+1]);
+ }
+ xocl_bitmap_from_arr32(xcmd->cu_bitmap, cumasks, MAX_CUS);
+ SCHED_DEBUGF("cu_bitmap[0] = %lu\n", xcmd->cu_bitmap[0]);
+ }
+
+ // dependencies are copied here, the anticipated wait_count is number
+ // of specified dependencies. The wait_count is adjusted when the
+ // command is queued in the scheduler based on whether or not a
+ // dependency is active (managed by scheduler)
+ memcpy(xcmd->deps, deps, numdeps*sizeof(struct drm_xocl_bo *));
+ xcmd->wait_count = numdeps;
+ xcmd->chain_count = 0;
+}
+
+/*
+ */
+static void
+cmd_packet_init(struct xocl_cmd *xcmd, struct ert_packet *packet)
+{
+ SCHED_DEBUGF("%s(%lu,packet)\n", __func__, xcmd->uid);
+ xcmd->ecmd = packet;
+}
+
+/*
+ * cmd_has_cu() - Check if this command object can execute on CU
+ *
+ * @cuidx: the index of the CU. Note that CU indicies start from 0.
+ */
+static int
+cmd_has_cu(struct xocl_cmd *xcmd, unsigned int cuidx)
+{
+ SCHED_DEBUGF("%s(%lu,%d) = %d\n", __func__, xcmd->uid, cuidx, test_bit(cuidx, xcmd->cu_bitmap));
+ return test_bit(cuidx, xcmd->cu_bitmap);
+}
+
+/*
+ * struct xocl_cu: Represents a compute unit in penguin mode
+ *
+ * @running_queue: a fifo representing commands running on this CU
+ * @xdev: the xrt device with this CU
+ * @idx: index of this CU
+ * @base: exec base address of this CU
+ * @addr: base address of this CU
+ * @ctrlreg: state of the CU (value of AXI-lite control register)
+ * @done_cnt: number of command that have completed (<=running_queue.size())
+ *
+ */
+struct xocl_cu {
+ struct list_head running_queue;
+ unsigned int idx;
+ void __iomem *base;
+ u32 addr;
+
+ u32 ctrlreg;
+ unsigned int done_cnt;
+ unsigned int run_cnt;
+ unsigned int uid;
+};
+
+/*
+ */
+void
+cu_reset(struct xocl_cu *xcu, unsigned int idx, void __iomem *base, u32 addr)
+{
+ xcu->idx = idx;
+ xcu->base = base;
+ xcu->addr = addr;
+ xcu->ctrlreg = 0;
+ xcu->done_cnt = 0;
+ xcu->run_cnt = 0;
+ SCHED_DEBUGF("%s(uid:%d,idx:%d) @ 0x%x\n", __func__, xcu->uid, xcu->idx, xcu->addr);
+}
+
+/*
+ */
+struct xocl_cu *
+cu_create(void)
+{
+ struct xocl_cu *xcu = kmalloc(sizeof(struct xocl_cu), GFP_KERNEL);
+ static unsigned int uid;
+
+ INIT_LIST_HEAD(&xcu->running_queue);
+ xcu->uid = uid++;
+ SCHED_DEBUGF("%s(uid:%d)\n", __func__, xcu->uid);
+ return xcu;
+}
+
+static inline u32
+cu_base_addr(struct xocl_cu *xcu)
+{
+ return xcu->addr;
+}
+
+/*
+ */
+void
+cu_destroy(struct xocl_cu *xcu)
+{
+ SCHED_DEBUGF("%s(uid:%d)\n", __func__, xcu->uid);
+ kfree(xcu);
+}
+
+/*
+ */
+void
+cu_poll(struct xocl_cu *xcu)
+{
+ // assert !list_empty(&running_queue)
+ xcu->ctrlreg = ioread32(xcu->base + xcu->addr);
+ SCHED_DEBUGF("%s(%d) 0x%x done(%d) run(%d)\n", __func__, xcu->idx, xcu->ctrlreg, xcu->done_cnt, xcu->run_cnt);
+ if (xcu->ctrlreg & AP_DONE) {
+ ++xcu->done_cnt; // assert done_cnt <= |running_queue|
+ --xcu->run_cnt;
+ // acknowledge done
+ iowrite32(AP_CONTINUE, xcu->base + xcu->addr);
+ }
+}
+
+/*
+ * cu_ready() - Check if CU is ready to start another command
+ *
+ * The CU is ready when AP_START is low
+ */
+static int
+cu_ready(struct xocl_cu *xcu)
+{
+ if (xcu->ctrlreg & AP_START)
+ cu_poll(xcu);
+
+ SCHED_DEBUGF("%s(%d) returns %d\n", __func__, xcu->idx, !(xcu->ctrlreg & AP_START));
+ return !(xcu->ctrlreg & AP_START);
+}
+
+/*
+ * cu_first_done() - Get the first completed command from the running queue
+ *
+ * Return: The first command that has completed or nullptr if none
+ */
+static struct xocl_cmd*
+cu_first_done(struct xocl_cu *xcu)
+{
+ if (!xcu->done_cnt)
+ cu_poll(xcu);
+
+ SCHED_DEBUGF("%s(%d) has done_cnt %d\n", __func__, xcu->idx, xcu->done_cnt);
+
+ return xcu->done_cnt
+ ? list_first_entry(&xcu->running_queue, struct xocl_cmd, rq_list)
+ : NULL;
+}
+
+/*
+ * cu_pop_done() - Remove first element from running queue
+ */
+static void
+cu_pop_done(struct xocl_cu *xcu)
+{
+ struct xocl_cmd *xcmd;
+
+ if (!xcu->done_cnt)
+ return;
+ xcmd = list_first_entry(&xcu->running_queue, struct xocl_cmd, rq_list);
+ list_del(&xcmd->rq_list);
+ --xcu->done_cnt;
+ SCHED_DEBUGF("%s(%d) xcmd(%lu) done(%d) run(%d)\n", __func__, xcu->idx, xcmd->uid, xcu->done_cnt, xcu->run_cnt);
+}
+
+/*
+ * cu_start() - Start the CU with a new command.
+ *
+ * The command is pushed onto the running queue
+ */
+static int
+cu_start(struct xocl_cu *xcu, struct xocl_cmd *xcmd)
+{
+ // assert(!(ctrlreg & AP_START), "cu not ready");
+
+ // data past header and cu_masks
+ unsigned int size = cmd_regmap_size(xcmd);
+ u32 *regmap = cmd_regmap(xcmd);
+ unsigned int i;
+
+ // past header, past cumasks
+ SCHED_DEBUG_PACKET(regmap, size);
+
+ // write register map, starting at base + 0xC
+ // 0x4, 0x8 used for interrupt, which is initialized in setu
+ for (i = 1; i < size; ++i)
+ iowrite32(*(regmap + i), xcu->base + xcu->addr + (i << 2));
+
+ // start cu. update local state as we may not be polling prior
+ // to next ready check.
+ xcu->ctrlreg |= AP_START;
+ iowrite32(AP_START, xcu->base + xcu->addr);
+
+ // add cmd to end of running queue
+ list_add_tail(&xcmd->rq_list, &xcu->running_queue);
+ ++xcu->run_cnt;
+
+ SCHED_DEBUGF("%s(%d) started xcmd(%lu) done(%d) run(%d)\n",
+ __func__, xcu->idx, xcmd->uid, xcu->done_cnt, xcu->run_cnt);
+
+ return true;
+}
+
+
+/*
+ * sruct xocl_ert: Represents embedded scheduler in ert mode
+ */
+struct xocl_ert {
+ void __iomem *base;
+ u32 cq_addr;
+ unsigned int uid;
+
+ unsigned int slot_size;
+ unsigned int cq_intr;
+};
+
+/*
+ */
+struct xocl_ert *
+ert_create(void __iomem *base, u32 cq_addr)
+{
+ struct xocl_ert *xert = kmalloc(sizeof(struct xocl_ert), GFP_KERNEL);
+ static unsigned int uid;
+
+ xert->base = base;
+ xert->cq_addr = cq_addr;
+ xert->uid = uid++;
+ xert->slot_size = 0;
+ xert->cq_intr = false;
+ SCHED_DEBUGF("%s(%d,0x%x)\n", __func__, xert->uid, xert->cq_addr);
+ return xert;
+}
+
+/*
+ */
+static void
+ert_destroy(struct xocl_ert *xert)
+{
+ SCHED_DEBUGF("%s(%d)\n", __func__, xert->uid);
+ kfree(xert);
+}
+
+/*
+ */
+static void
+ert_cfg(struct xocl_ert *xert, unsigned int slot_size, unsigned int cq_intr)
+{
+ SCHED_DEBUGF("%s(%d) slot_size(%d) cq_intr(%d)\n", __func__, xert->uid, slot_size, cq_intr);
+ xert->slot_size = slot_size;
+ xert->cq_intr = cq_intr;
+}
+
+/*
+ */
+static bool
+ert_start_cmd(struct xocl_ert *xert, struct xocl_cmd *xcmd)
+{
+ u32 slot_addr = xert->cq_addr + xcmd->slot_idx * xert->slot_size;
+ struct ert_packet *ecmd = cmd_packet(xcmd);
+
+ SCHED_DEBUG_PACKET(ecmd, cmd_packet_size(xcmd));
+
+ SCHED_DEBUGF("-> %s(%d,%lu)\n", __func__, xert->uid, xcmd->uid);
+
+ // write packet minus header
+ SCHED_DEBUGF("++ slot_idx=%d, slot_addr=0x%x\n", xcmd->slot_idx, slot_addr);
+ memcpy_toio(xert->base + slot_addr + 4, ecmd->data, (cmd_packet_size(xcmd) - 1) * sizeof(u32));
+
+ // write header
+ iowrite32(ecmd->header, xert->base + slot_addr);
+
+ // trigger interrupt to embedded scheduler if feature is enabled
+ if (xert->cq_intr) {
+ u32 cq_int_addr = ERT_CQ_STATUS_REGISTER_ADDR + (slot_mask_idx(xcmd->slot_idx) << 2);
+ u32 mask = 1 << slot_idx_in_mask(xcmd->slot_idx);
+
+ SCHED_DEBUGF("++ mb_submit writes slot mask 0x%x to CQ_INT register at addr 0x%x\n",
+ mask, cq_int_addr);
+ iowrite32(mask, xert->base + cq_int_addr);
+ }
+ SCHED_DEBUGF("<- %s returns true\n", __func__);
+ return true;
+}
+
+/*
+ */
+static void
+ert_read_custat(struct xocl_ert *xert, unsigned int num_cus, u32 *cu_usage, struct xocl_cmd *xcmd)
+{
+ u32 slot_addr = xert->cq_addr + xcmd->slot_idx*xert->slot_size;
+
+ memcpy_fromio(cu_usage, xert->base + slot_addr + 4, num_cus * sizeof(u32));
+}
+
+/**
+ * struct exec_ops: scheduler specific operations
+ *
+ * Scheduler can operate in MicroBlaze mode (mb/ert) or in penguin mode. This
+ * struct differentiates specific operations. The struct is per device node,
+ * meaning that one device can operate in ert mode while another can operate
+ * in penguin mode.
+ */
+struct exec_ops {
+ bool (*start)(struct exec_core *exec, struct xocl_cmd *xcmd);
+ void (*query)(struct exec_core *exec, struct xocl_cmd *xcmd);
+};
+
+static struct exec_ops ert_ops;
+static struct exec_ops penguin_ops;
+
+/**
+ * struct exec_core: Core data structure for command execution on a device
+ *
+ * @ctx_list: Context list populated with device context
+ * @exec_lock: Lock for synchronizing external access
+ * @poll_wait_queue: Wait queue for device polling
+ * @scheduler: Command queue scheduler
+ * @submitted_cmds: Tracking of command submitted for execution on this device
+ * @num_slots: Number of command queue slots
+ * @num_cus: Number of CUs in loaded program
+ * @num_cdma: Number of CDMAs in hardware
+ * @polling_mode: If set then poll for command completion
+ * @cq_interrupt: If set then trigger interrupt to MB on new commands
+ * @configured: Flag to indicate that the core data structure has been initialized
+ * @stopped: Flag to indicate that the core data structure cannot be used
+ * @flush: Flag to indicate that commands for this device should be flushed
+ * @cu_usage: Usage count since last reset
+ * @slot_status: Bitmap to track status (busy(1)/free(0)) slots in command queue
+ * @ctrl_busy: Flag to indicate that slot 0 (ctrl commands) is busy
+ * @cu_status: Bitmap to track status (busy(1)/free(0)) of CUs. Unused in ERT mode.
+ * @sr0: If set, then status register [0..31] is pending with completed commands (ERT only).
+ * @sr1: If set, then status register [32..63] is pending with completed commands (ERT only).
+ * @sr2: If set, then status register [64..95] is pending with completed commands (ERT only).
+ * @sr3: If set, then status register [96..127] is pending with completed commands (ERT only).
+ * @ops: Scheduler operations vtable
+ */
+struct exec_core {
+ struct platform_device *pdev;
+
+ struct mutex exec_lock;
+
+ void __iomem *base;
+ u32 intr_base;
+ u32 intr_num;
+
+ wait_queue_head_t poll_wait_queue;
+
+ struct xocl_scheduler *scheduler;
+
+ uuid_t xclbin_id;
+
+ unsigned int num_slots;
+ unsigned int num_cus;
+ unsigned int num_cdma;
+ unsigned int polling_mode;
+ unsigned int cq_interrupt;
+ unsigned int configured;
+ unsigned int stopped;
+ unsigned int flush;
+
+ struct xocl_cu *cus[MAX_CUS];
+ struct xocl_ert *ert;
+
+ u32 cu_usage[MAX_CUS];
+
+ // Bitmap tracks busy(1)/free(0) slots in cmd_slots
+ struct xocl_cmd *submitted_cmds[MAX_SLOTS];
+ DECLARE_BITMAP(slot_status, MAX_SLOTS);
+ unsigned int ctrl_busy;
+
+ // Status register pending complete. Written by ISR,
+ // cleared by scheduler
+ atomic_t sr0;
+ atomic_t sr1;
+ atomic_t sr2;
+ atomic_t sr3;
+
+ // Operations for dynamic indirection dependt on MB
+ // or kernel scheduler
+ struct exec_ops *ops;
+
+ unsigned int uid;
+ unsigned int ip_reference[MAX_CUS];
+};
+
+/**
+ * exec_get_pdev() -
+ */
+static inline struct platform_device *
+exec_get_pdev(struct exec_core *exec)
+{
+ return exec->pdev;
+}
+
+/**
+ * exec_get_xdev() -
+ */
+static inline struct xocl_dev *
+exec_get_xdev(struct exec_core *exec)
+{
+ return xocl_get_xdev(exec->pdev);
+}
+
+/*
+ */
+static inline bool
+exec_is_ert(struct exec_core *exec)
+{
+ return exec->ops == &ert_ops;
+}
+
+/*
+ */
+static inline bool
+exec_is_polling(struct exec_core *exec)
+{
+ return exec->polling_mode;
+}
+
+/*
+ */
+static inline bool
+exec_is_flush(struct exec_core *exec)
+{
+ return exec->flush;
+}
+
+/*
+ */
+static inline u32
+exec_cu_base_addr(struct exec_core *exec, unsigned int cuidx)
+{
+ return cu_base_addr(exec->cus[cuidx]);
+}
+
+/*
+ */
+static inline u32
+exec_cu_usage(struct exec_core *exec, unsigned int cuidx)
+{
+ return exec->cu_usage[cuidx];
+}
+
+/*
+ */
+static void
+exec_cfg(struct exec_core *exec)
+{
+}
+
+/*
+ * to be automated
+ */
+static int
+exec_cfg_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+ struct client_ctx *client = xcmd->client;
+ bool ert = xocl_mb_sched_on(xdev);
+ uint32_t *cdma = xocl_cdma_addr(xdev);
+ unsigned int dsa = xocl_dsa_version(xdev);
+ struct ert_configure_cmd *cfg;
+ int cuidx = 0;
+
+ /* Only allow configuration with one live ctx */
+ if (exec->configured) {
+ DRM_INFO("command scheduler is already configured for this device\n");
+ return 1;
+ }
+
+ DRM_INFO("ert per feature rom = %d\n", ert);
+ DRM_INFO("dsa per feature rom = %d\n", dsa);
+
+ cfg = (struct ert_configure_cmd *)(xcmd->ecmd);
+
+ /* Mark command as control command to force slot 0 execution */
+ cfg->type = ERT_CTRL;
+
+ if (cfg->count != 5 + cfg->num_cus) {
+ DRM_INFO("invalid configure command, count=%d expected 5+num_cus(%d)\n", cfg->count, cfg->num_cus);
+ return 1;
+ }
+
+ SCHED_DEBUG("configuring scheduler\n");
+ exec->num_slots = ERT_CQ_SIZE / cfg->slot_size;
+ exec->num_cus = cfg->num_cus;
+ exec->num_cdma = 0;
+
+ // skip this in polling mode
+ for (cuidx = 0; cuidx < exec->num_cus; ++cuidx) {
+ struct xocl_cu *xcu = exec->cus[cuidx];
+
+ if (!xcu)
+ xcu = exec->cus[cuidx] = cu_create();
+ cu_reset(xcu, cuidx, exec->base, cfg->data[cuidx]);
+ userpf_info(xdev, "%s cu(%d) at 0x%x\n", __func__, xcu->idx, xcu->addr);
+ }
+
+ if (cdma) {
+ uint32_t *addr = 0;
+
+ mutex_lock(&client->lock); /* for modification to client cu_bitmap */
+ for (addr = cdma; addr < cdma+4; ++addr) { /* 4 is from xclfeatures.h */
+ if (*addr) {
+ struct xocl_cu *xcu = exec->cus[cuidx];
+
+ if (!xcu)
+ xcu = exec->cus[cuidx] = cu_create();
+ cu_reset(xcu, cuidx, exec->base, *addr);
+ ++exec->num_cus;
+ ++exec->num_cdma;
+ ++cfg->num_cus;
+ ++cfg->count;
+ cfg->data[cuidx] = *addr;
+ set_bit(cuidx, client->cu_bitmap); /* cdma is shared */
+ userpf_info(xdev, "configure cdma as cu(%d) at 0x%x\n", cuidx, *addr);
+ ++cuidx;
+ }
+ }
+ mutex_unlock(&client->lock);
+ }
+
+ if (ert && cfg->ert) {
+ SCHED_DEBUG("++ configuring embedded scheduler mode\n");
+ if (!exec->ert)
+ exec->ert = ert_create(exec->base, ERT_CQ_BASE_ADDR);
+ ert_cfg(exec->ert, cfg->slot_size, cfg->cq_int);
+ exec->ops = &ert_ops;
+ exec->polling_mode = cfg->polling;
+ exec->cq_interrupt = cfg->cq_int;
+ cfg->dsa52 = (dsa >= 52) ? 1 : 0;
+ cfg->cdma = cdma ? 1 : 0;
+ } else {
+ SCHED_DEBUG("++ configuring penguin scheduler mode\n");
+ exec->ops = &penguin_ops;
+ exec->polling_mode = 1;
+ }
+
+ // reserve slot 0 for control commands
+ set_bit(0, exec->slot_status);
+
+ DRM_INFO("scheduler config ert(%d) slots(%d), cudma(%d), cuisr(%d), cdma(%d), cus(%d)\n"
+ , exec_is_ert(exec)
+ , exec->num_slots
+ , cfg->cu_dma ? 1 : 0
+ , cfg->cu_isr ? 1 : 0
+ , exec->num_cdma
+ , exec->num_cus);
+
+ exec->configured = true;
+ return 0;
+}
+
+/**
+ * exec_reset() - Reset the scheduler
+ *
+ * @exec: Execution core (device) to reset
+ *
+ * TODO: Perform scheduler configuration based on current xclbin
+ * rather than relying of cfg command
+ */
+static void
+exec_reset(struct exec_core *exec)
+{
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+ uuid_t *xclbin_id;
+
+ mutex_lock(&exec->exec_lock);
+
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+
+ userpf_info(xdev, "%s(%d) cfg(%d)\n", __func__, exec->uid, exec->configured);
+
+ // only reconfigure the scheduler on new xclbin
+ if (!xclbin_id || (uuid_equal(&exec->xclbin_id, xclbin_id) && exec->configured)) {
+ exec->stopped = false;
+ exec->configured = false; // TODO: remove, but hangs ERT because of in between AXI resets
+ goto out;
+ }
+
+ userpf_info(xdev, "exec->xclbin(%pUb),xclbin(%pUb)\n", &exec->xclbin_id, xclbin_id);
+ userpf_info(xdev, "%s resets for new xclbin", __func__);
+ memset(exec->cu_usage, 0, MAX_CUS * sizeof(u32));
+ uuid_copy(&exec->xclbin_id, xclbin_id);
+ exec->num_cus = 0;
+ exec->num_cdma = 0;
+
+ exec->num_slots = 16;
+ exec->polling_mode = 1;
+ exec->cq_interrupt = 0;
+ exec->configured = false;
+ exec->stopped = false;
+ exec->flush = false;
+ exec->ops = &penguin_ops;
+
+ bitmap_zero(exec->slot_status, MAX_SLOTS);
+ set_bit(0, exec->slot_status); // reserve for control command
+ exec->ctrl_busy = false;
+
+ atomic_set(&exec->sr0, 0);
+ atomic_set(&exec->sr1, 0);
+ atomic_set(&exec->sr2, 0);
+ atomic_set(&exec->sr3, 0);
+
+ exec_cfg(exec);
+
+out:
+ mutex_unlock(&exec->exec_lock);
+}
+
+/**
+ * exec_stop() - Stop the scheduler from scheduling commands on this core
+ *
+ * @exec: Execution core (device) to stop
+ *
+ * Block access to current exec_core (device). This API must be called prior
+ * to performing an AXI reset and downloading of a new xclbin. Calling this
+ * API flushes the commands running on current device and prevents new
+ * commands from being scheduled on the device. This effectively prevents any
+ * further commands from running on the device
+ */
+static void
+exec_stop(struct exec_core *exec)
+{
+ int idx;
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+ unsigned int outstanding = 0;
+ unsigned int wait_ms = 100;
+ unsigned int retry = 20; // 2 sec
+
+ mutex_lock(&exec->exec_lock);
+ userpf_info(xdev, "%s(%p)\n", __func__, exec);
+ exec->stopped = true;
+ mutex_unlock(&exec->exec_lock);
+
+ // Wait for commands to drain if any
+ outstanding = atomic_read(&xdev->outstanding_execs);
+ while (--retry && outstanding) {
+ userpf_info(xdev, "Waiting for %d outstanding commands to finish", outstanding);
+ msleep(wait_ms);
+ outstanding = atomic_read(&xdev->outstanding_execs);
+ }
+
+ // Last gasp, flush any remaining commands for this device exec core
+ // This is an abnormal case. All exec clients have been destroyed
+ // prior to exec_stop being called (per contract), this implies that
+ // all regular client commands have been flushed.
+ if (outstanding) {
+ // Wake up the scheduler to force one iteration flushing stale
+ // commands for this device
+ exec->flush = 1;
+ scheduler_intr(exec->scheduler);
+
+ // Wait a second
+ msleep(1000);
+ }
+
+ outstanding = atomic_read(&xdev->outstanding_execs);
+ if (outstanding)
+ userpf_err(xdev, "unexpected outstanding commands %d after flush", outstanding);
+
+ // Stale commands were flushed, reset submitted command state
+ for (idx = 0; idx < MAX_SLOTS; ++idx)
+ exec->submitted_cmds[idx] = NULL;
+
+ bitmap_zero(exec->slot_status, MAX_SLOTS);
+ set_bit(0, exec->slot_status); // reserve for control command
+ exec->ctrl_busy = false;
+}
+
+/*
+ */
+static irqreturn_t
+exec_isr(int irq, void *arg)
+{
+ struct exec_core *exec = (struct exec_core *)arg;
+
+ SCHED_DEBUGF("-> xocl_user_event %d\n", irq);
+ if (exec_is_ert(exec) && !exec->polling_mode) {
+
+ if (irq == 0)
+ atomic_set(&exec->sr0, 1);
+ else if (irq == 1)
+ atomic_set(&exec->sr1, 1);
+ else if (irq == 2)
+ atomic_set(&exec->sr2, 1);
+ else if (irq == 3)
+ atomic_set(&exec->sr3, 1);
+
+ /* wake up all scheduler ... currently one only */
+ scheduler_intr(exec->scheduler);
+ } else {
+ userpf_err(exec_get_xdev(exec), "Unhandled isr irq %d, is_ert %d, polling %d",
+ irq, exec_is_ert(exec), exec->polling_mode);
+ }
+ SCHED_DEBUGF("<- xocl_user_event\n");
+ return IRQ_HANDLED;
+}
+
+/*
+ */
+struct exec_core *
+exec_create(struct platform_device *pdev, struct xocl_scheduler *xs)
+{
+ struct exec_core *exec = devm_kzalloc(&pdev->dev, sizeof(struct exec_core), GFP_KERNEL);
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ struct resource *res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ static unsigned int count;
+ unsigned int i;
+
+ if (!exec)
+ return NULL;
+
+ mutex_init(&exec->exec_lock);
+ exec->base = xdev->core.bar_addr;
+
+ exec->intr_base = res->start;
+ exec->intr_num = res->end - res->start + 1;
+ exec->pdev = pdev;
+
+ init_waitqueue_head(&exec->poll_wait_queue);
+ exec->scheduler = xs;
+ exec->uid = count++;
+
+ for (i = 0; i < exec->intr_num; i++) {
+ xocl_user_interrupt_reg(xdev, i+exec->intr_base, exec_isr, exec);
+ xocl_user_interrupt_config(xdev, i + exec->intr_base, true);
+ }
+
+ exec_reset(exec);
+ platform_set_drvdata(pdev, exec);
+
+ SCHED_DEBUGF("%s(%d)\n", __func__, exec->uid);
+
+ return exec;
+}
+
+/*
+ */
+static void
+exec_destroy(struct exec_core *exec)
+{
+ int idx;
+
+ SCHED_DEBUGF("%s(%d)\n", __func__, exec->uid);
+ for (idx = 0; idx < exec->num_cus; ++idx)
+ cu_destroy(exec->cus[idx]);
+ if (exec->ert)
+ ert_destroy(exec->ert);
+ devm_kfree(&exec->pdev->dev, exec);
+}
+
+/*
+ */
+static inline struct xocl_scheduler *
+exec_scheduler(struct exec_core *exec)
+{
+ return exec->scheduler;
+}
+
+/*
+ * acquire_slot_idx() - First available slot index
+ */
+static unsigned int
+exec_acquire_slot_idx(struct exec_core *exec)
+{
+ unsigned int idx = find_first_zero_bit(exec->slot_status, MAX_SLOTS);
+
+ SCHED_DEBUGF("%s(%d) returns %d\n", __func__, exec->uid, idx < exec->num_slots ? idx : no_index);
+ if (idx < exec->num_slots) {
+ set_bit(idx, exec->slot_status);
+ return idx;
+ }
+ return no_index;
+}
+
+
+/**
+ * acquire_slot() - Acquire a slot index for a command
+ *
+ * This function makes a special case for control commands which
+ * must always dispatch to slot 0, otherwise normal acquisition
+ */
+static int
+exec_acquire_slot(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ // slot 0 is reserved for ctrl commands
+ if (cmd_type(xcmd) == ERT_CTRL) {
+ SCHED_DEBUGF("%s(%d,%lu) ctrl cmd\n", __func__, exec->uid, xcmd->uid);
+ if (exec->ctrl_busy)
+ return -1;
+ exec->ctrl_busy = true;
+ return (xcmd->slot_idx = 0);
+ }
+
+ return (xcmd->slot_idx = exec_acquire_slot_idx(exec));
+}
+
+/*
+ * release_slot_idx() - Release specified slot idx
+ */
+static void
+exec_release_slot_idx(struct exec_core *exec, unsigned int slot_idx)
+{
+ clear_bit(slot_idx, exec->slot_status);
+}
+
+/**
+ * release_slot() - Release a slot index for a command
+ *
+ * Special case for control commands that execute in slot 0. This
+ * slot cannot be marked free ever.
+ */
+static void
+exec_release_slot(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ if (xcmd->slot_idx == no_index)
+ return; // already released
+
+ SCHED_DEBUGF("%s(%d) xcmd(%lu) slotidx(%d)\n",
+ __func__, exec->uid, xcmd->uid, xcmd->slot_idx);
+ if (cmd_type(xcmd) == ERT_CTRL) {
+ SCHED_DEBUG("+ ctrl cmd\n");
+ exec->ctrl_busy = false;
+ } else {
+ exec_release_slot_idx(exec, xcmd->slot_idx);
+ }
+ xcmd->slot_idx = no_index;
+}
+
+/*
+ * submit_cmd() - Submit command for execution on this core
+ *
+ * Return: true on success, false if command could not be submitted
+ */
+static bool
+exec_submit_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ unsigned int slotidx = exec_acquire_slot(exec, xcmd);
+
+ if (slotidx == no_index)
+ return false;
+ SCHED_DEBUGF("%s(%d,%lu) slotidx(%d)\n", __func__, exec->uid, xcmd->uid, slotidx);
+ exec->submitted_cmds[slotidx] = xcmd;
+ cmd_set_int_state(xcmd, ERT_CMD_STATE_SUBMITTED);
+ return true;
+}
+
+/*
+ * finish_cmd() - Special post processing of commands after execution
+ */
+static int
+exec_finish_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ if (cmd_opcode(xcmd) == ERT_CU_STAT && exec_is_ert(exec))
+ ert_read_custat(exec->ert, exec->num_cus, exec->cu_usage, xcmd);
+ return 0;
+}
+
+/*
+ * execute_write_cmd() - Execute ERT_WRITE commands
+ */
+static int
+exec_execute_write_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ struct ert_packet *ecmd = xcmd->ecmd;
+ unsigned int idx = 0;
+
+ SCHED_DEBUGF("-> %s(%d,%lu)\n", __func__, exec->uid, xcmd->uid);
+ for (idx = 0; idx < ecmd->count - 1; idx += 2) {
+ u32 addr = ecmd->data[idx];
+ u32 val = ecmd->data[idx+1];
+
+ SCHED_DEBUGF("+ exec_write_cmd base[0x%x] = 0x%x\n", addr, val);
+ iowrite32(val, exec->base + addr);
+ }
+ SCHED_DEBUG("<- exec_write\n");
+ return 0;
+}
+
+/*
+ * notify_host() - Notify user space that a command is complete.
+ */
+static void
+exec_notify_host(struct exec_core *exec)
+{
+ struct list_head *ptr;
+ struct client_ctx *entry;
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+
+ SCHED_DEBUGF("-> %s(%d)\n", __func__, exec->uid);
+
+ /* now for each client update the trigger counter in the context */
+ mutex_lock(&xdev->ctx_list_lock);
+ list_for_each(ptr, &xdev->ctx_list) {
+ entry = list_entry(ptr, struct client_ctx, link);
+ atomic_inc(&entry->trigger);
+ }
+ mutex_unlock(&xdev->ctx_list_lock);
+ /* wake up all the clients */
+ wake_up_interruptible(&exec->poll_wait_queue);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/*
+ * exec_cmd_mark_complete() - Move a command to complete state
+ *
+ * Commands are marked complete in two ways
+ * 1. Through polling of CUs or polling of MB status register
+ * 2. Through interrupts from MB
+ *
+ * @xcmd: Command to mark complete
+ *
+ * The external command state is changed to complete and the host
+ * is notified that some command has completed.
+ */
+static void
+exec_mark_cmd_complete(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> %s(%d,%lu)\n", __func__, exec->uid, xcmd->uid);
+ if (cmd_type(xcmd) == ERT_CTRL)
+ exec_finish_cmd(exec, xcmd);
+
+ cmd_set_state(xcmd, ERT_CMD_STATE_COMPLETED);
+
+ if (exec->polling_mode)
+ scheduler_decr_poll(exec->scheduler);
+
+ exec_release_slot(exec, xcmd);
+ exec_notify_host(exec);
+
+ // Deactivate command and trigger chain of waiting commands
+ cmd_mark_deactive(xcmd);
+ cmd_trigger_chain(xcmd);
+
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * mark_mask_complete() - Move all commands in mask to complete state
+ *
+ * @mask: Bitmask with queried statuses of commands
+ * @mask_idx: Index of the command mask. Used to offset the actual cmd slot index
+ *
+ * Used in ERT mode only. Currently ERT submitted commands remain in exec
+ * submitted queue as ERT doesn't support data flow
+ */
+static void
+exec_mark_mask_complete(struct exec_core *exec, u32 mask, unsigned int mask_idx)
+{
+ int bit_idx = 0, cmd_idx = 0;
+
+ SCHED_DEBUGF("-> %s(0x%x,%d)\n", __func__, mask, mask_idx);
+ if (!mask)
+ return;
+
+ for (bit_idx = 0, cmd_idx = mask_idx<<5; bit_idx < 32; mask >>= 1, ++bit_idx, ++cmd_idx) {
+ // mask could be -1 when firewall trips, double check
+ // exec->submitted_cmds[cmd_idx] to make sure it's not NULL
+ if ((mask & 0x1) && exec->submitted_cmds[cmd_idx])
+ exec_mark_cmd_complete(exec, exec->submitted_cmds[cmd_idx]);
+ }
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/*
+ * penguin_start_cmd() - Start a command in penguin mode
+ */
+static bool
+exec_penguin_start_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ unsigned int cuidx;
+ u32 opcode = cmd_opcode(xcmd);
+
+ SCHED_DEBUGF("-> %s (%d,%lu) opcode(%d)\n", __func__, exec->uid, xcmd->uid, opcode);
+
+ if (opcode == ERT_WRITE && exec_execute_write_cmd(exec, xcmd)) {
+ cmd_set_state(xcmd, ERT_CMD_STATE_ERROR);
+ return false;
+ }
+
+ if (opcode != ERT_START_CU) {
+ SCHED_DEBUGF("<- %s -> true\n", __func__);
+ return true;
+ }
+
+ // Find a ready CU
+ for (cuidx = 0; cuidx < exec->num_cus; ++cuidx) {
+ struct xocl_cu *xcu = exec->cus[cuidx];
+
+ if (cmd_has_cu(xcmd, cuidx) && cu_ready(xcu) && cu_start(xcu, xcmd)) {
+ exec->submitted_cmds[xcmd->slot_idx] = NULL;
+ ++exec->cu_usage[cuidx];
+ exec_release_slot(exec, xcmd);
+ xcmd->cu_idx = cuidx;
+ SCHED_DEBUGF("<- %s -> true\n", __func__);
+ return true;
+ }
+ }
+ SCHED_DEBUGF("<- %s -> false\n", __func__);
+ return false;
+}
+
+/**
+ * penguin_query() - Check command status of argument command
+ *
+ * @xcmd: Command to check
+ *
+ * Function is called in penguin mode (no embedded scheduler).
+ */
+static void
+exec_penguin_query_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ u32 cmdopcode = cmd_opcode(xcmd);
+ u32 cmdtype = cmd_type(xcmd);
+
+ SCHED_DEBUGF("-> %s(%lu) opcode(%d) type(%d) slot_idx=%d\n",
+ __func__, xcmd->uid, cmdopcode, cmdtype, xcmd->slot_idx);
+
+ if (cmdtype == ERT_KDS_LOCAL || cmdtype == ERT_CTRL)
+ exec_mark_cmd_complete(exec, xcmd);
+ else if (cmdopcode == ERT_START_CU) {
+ struct xocl_cu *xcu = exec->cus[xcmd->cu_idx];
+
+ if (cu_first_done(xcu) == xcmd) {
+ cu_pop_done(xcu);
+ exec_mark_cmd_complete(exec, xcmd);
+ }
+ }
+
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+
+/*
+ * ert_start_cmd() - Start a command on ERT
+ */
+static bool
+exec_ert_start_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ // if (cmd_type(xcmd) == ERT_DATAFLOW)
+ // exec_penguin_start_cmd(exec,xcmd);
+ return ert_start_cmd(exec->ert, xcmd);
+}
+
+/*
+ * ert_query_cmd() - Check command completion in ERT
+ *
+ * @xcmd: Command to check
+ *
+ * This function is for ERT mode. In polling mode, check the command status
+ * register containing the slot assigned to the command. In interrupt mode
+ * check the interrupting status register. The function checks all commands
+ * in the same command status register as argument command so more than one
+ * command may be marked complete by this function.
+ */
+static void
+exec_ert_query_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ unsigned int cmd_mask_idx = slot_mask_idx(xcmd->slot_idx);
+
+ SCHED_DEBUGF("-> %s(%lu) slot_idx(%d), cmd_mask_idx(%d)\n", __func__, xcmd->uid, xcmd->slot_idx, cmd_mask_idx);
+
+ if (cmd_type(xcmd) == ERT_KDS_LOCAL) {
+ exec_mark_cmd_complete(exec, xcmd);
+ SCHED_DEBUGF("<- %s local command\n", __func__);
+ return;
+ }
+
+ if (exec->polling_mode
+ || (cmd_mask_idx == 0 && atomic_xchg(&exec->sr0, 0))
+ || (cmd_mask_idx == 1 && atomic_xchg(&exec->sr1, 0))
+ || (cmd_mask_idx == 2 && atomic_xchg(&exec->sr2, 0))
+ || (cmd_mask_idx == 3 && atomic_xchg(&exec->sr3, 0))) {
+ u32 csr_addr = ERT_STATUS_REGISTER_ADDR + (cmd_mask_idx<<2);
+ u32 mask = ioread32(xcmd->exec->base + csr_addr);
+
+ SCHED_DEBUGF("++ %s csr_addr=0x%x mask=0x%x\n", __func__, csr_addr, mask);
+ if (mask)
+ exec_mark_mask_complete(xcmd->exec, mask, cmd_mask_idx);
+ }
+
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/*
+ * start_cmd() - Start execution of a command
+ *
+ * Return: true if successfully started, false otherwise
+ *
+ * Function dispatches based on penguin vs ert mode
+ */
+static bool
+exec_start_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ // assert cmd had been submitted
+ SCHED_DEBUGF("%s(%d,%lu) opcode(%d)\n", __func__, exec->uid, xcmd->uid, cmd_opcode(xcmd));
+
+ if (exec->ops->start(exec, xcmd)) {
+ cmd_set_int_state(xcmd, ERT_CMD_STATE_RUNNING);
+ return true;
+ }
+
+ return false;
+}
+
+/*
+ * query_cmd() - Check status of command
+ *
+ * Function dispatches based on penguin vs ert mode. In ERT mode
+ * multiple commands can be marked complete by this function.
+ */
+static void
+exec_query_cmd(struct exec_core *exec, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("%s(%d,%lu)\n", __func__, exec->uid, xcmd->uid);
+ exec->ops->query(exec, xcmd);
+}
+
+
+
+/**
+ * ert_ops: operations for ERT scheduling
+ */
+static struct exec_ops ert_ops = {
+ .start = exec_ert_start_cmd,
+ .query = exec_ert_query_cmd,
+};
+
+/**
+ * penguin_ops: operations for kernel mode scheduling
+ */
+static struct exec_ops penguin_ops = {
+ .start = exec_penguin_start_cmd,
+ .query = exec_penguin_query_cmd,
+};
+
+/*
+ */
+static inline struct exec_core *
+pdev_get_exec(struct platform_device *pdev)
+{
+ return platform_get_drvdata(pdev);
+}
+
+/*
+ */
+static inline struct exec_core *
+dev_get_exec(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+
+ return pdev ? pdev_get_exec(pdev) : NULL;
+}
+
+/*
+ */
+static inline struct xocl_dev *
+dev_get_xdev(struct device *dev)
+{
+ struct exec_core *exec = dev_get_exec(dev);
+
+ return exec ? exec_get_xdev(exec) : NULL;
+}
+
+/**
+ * List of new pending xocl_cmd objects
+ *
+ * @pending_cmds: populated from user space with new commands for buffer objects
+ * @num_pending: number of pending commands
+ *
+ * Scheduler copies pending commands to its private queue when necessary
+ */
+static LIST_HEAD(pending_cmds);
+static DEFINE_MUTEX(pending_cmds_mutex);
+static atomic_t num_pending = ATOMIC_INIT(0);
+
+static void
+pending_cmds_reset(void)
+{
+ /* clear stale command objects if any */
+ while (!list_empty(&pending_cmds)) {
+ struct xocl_cmd *xcmd = list_first_entry(&pending_cmds, struct xocl_cmd, cq_list);
+
+ DRM_INFO("deleting stale pending cmd\n");
+ cmd_free(xcmd);
+ }
+ atomic_set(&num_pending, 0);
+}
+
+/**
+ * struct xocl_sched: scheduler for xocl_cmd objects
+ *
+ * @scheduler_thread: thread associated with this scheduler
+ * @use_count: use count for this scheduler
+ * @wait_queue: conditional wait queue for scheduler thread
+ * @error: set to 1 to indicate scheduler error
+ * @stop: set to 1 to indicate scheduler should stop
+ * @reset: set to 1 to reset the scheduler
+ * @command_queue: list of command objects managed by scheduler
+ * @intc: boolean flag set when there is a pending interrupt for command completion
+ * @poll: number of running commands in polling mode
+ */
+struct xocl_scheduler {
+ struct task_struct *scheduler_thread;
+ unsigned int use_count;
+
+ wait_queue_head_t wait_queue;
+ unsigned int error;
+ unsigned int stop;
+ unsigned int reset;
+
+ struct list_head command_queue;
+
+ unsigned int intc; /* pending intr shared with isr, word aligned atomic */
+ unsigned int poll; /* number of cmds to poll */
+};
+
+static struct xocl_scheduler scheduler0;
+
+static void
+scheduler_reset(struct xocl_scheduler *xs)
+{
+ xs->error = 0;
+ xs->stop = 0;
+ xs->poll = 0;
+ xs->reset = false;
+ xs->intc = 0;
+}
+
+static void
+scheduler_cq_reset(struct xocl_scheduler *xs)
+{
+ while (!list_empty(&xs->command_queue)) {
+ struct xocl_cmd *xcmd = list_first_entry(&xs->command_queue, struct xocl_cmd, cq_list);
+
+ DRM_INFO("deleting stale scheduler cmd\n");
+ cmd_free(xcmd);
+ }
+}
+
+static void
+scheduler_wake_up(struct xocl_scheduler *xs)
+{
+ wake_up_interruptible(&xs->wait_queue);
+}
+
+static void
+scheduler_intr(struct xocl_scheduler *xs)
+{
+ xs->intc = 1;
+ scheduler_wake_up(xs);
+}
+
+static inline void
+scheduler_decr_poll(struct xocl_scheduler *xs)
+{
+ --xs->poll;
+}
+
+
+/**
+ * scheduler_queue_cmds() - Queue any pending commands
+ *
+ * The scheduler copies pending commands to its internal command queue where
+ * is is now in queued state.
+ */
+static void
+scheduler_queue_cmds(struct xocl_scheduler *xs)
+{
+ struct xocl_cmd *xcmd;
+ struct list_head *pos, *next;
+
+ SCHED_DEBUGF("-> %s\n", __func__);
+ mutex_lock(&pending_cmds_mutex);
+ list_for_each_safe(pos, next, &pending_cmds) {
+ xcmd = list_entry(pos, struct xocl_cmd, cq_list);
+ if (xcmd->xs != xs)
+ continue;
+ SCHED_DEBUGF("+ queueing cmd(%lu)\n", xcmd->uid);
+ list_del(&xcmd->cq_list);
+ list_add_tail(&xcmd->cq_list, &xs->command_queue);
+
+ /* chain active dependencies if any to this command object */
+ if (cmd_wait_count(xcmd) && cmd_chain_dependencies(xcmd))
+ cmd_set_state(xcmd, ERT_CMD_STATE_ERROR);
+ else
+ cmd_set_int_state(xcmd, ERT_CMD_STATE_QUEUED);
+
+ /* this command is now active and can chain other commands */
+ cmd_mark_active(xcmd);
+ atomic_dec(&num_pending);
+ }
+ mutex_unlock(&pending_cmds_mutex);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * queued_to_running() - Move a command from queued to running state if possible
+ *
+ * @xcmd: Command to start
+ *
+ * Upon success, the command is not necessarily running. In ert mode the
+ * command will have been submitted to the embedded scheduler, whereas in
+ * penguin mode the command has been started on a CU.
+ *
+ * Return: %true if command was submitted to device, %false otherwise
+ */
+static bool
+scheduler_queued_to_submitted(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ struct exec_core *exec = cmd_exec(xcmd);
+ bool retval = false;
+
+ if (cmd_wait_count(xcmd))
+ return false;
+
+ SCHED_DEBUGF("-> %s(%lu) opcode(%d)\n", __func__, xcmd->uid, cmd_opcode(xcmd));
+
+ // configure prior to using the core
+ if (cmd_opcode(xcmd) == ERT_CONFIGURE && exec_cfg_cmd(exec, xcmd)) {
+ cmd_set_state(xcmd, ERT_CMD_STATE_ERROR);
+ return false;
+ }
+
+ // submit the command
+ if (exec_submit_cmd(exec, xcmd)) {
+ if (exec->polling_mode)
+ ++xs->poll;
+ retval = true;
+ }
+
+ SCHED_DEBUGF("<- queued_to_submitted returns %d\n", retval);
+
+ return retval;
+}
+
+static bool
+scheduler_submitted_to_running(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ return exec_start_cmd(cmd_exec(xcmd), xcmd);
+}
+
+/**
+ * running_to_complete() - Check status of running commands
+ *
+ * @xcmd: Command is in running state
+ *
+ * When ERT is enabled this function may mark more than just argument
+ * command as complete based on content of command completion register.
+ * Without ERT, only argument command is checked for completion.
+ */
+static void
+scheduler_running_to_complete(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ exec_query_cmd(cmd_exec(xcmd), xcmd);
+}
+
+/**
+ * complete_to_free() - Recycle a complete command objects
+ *
+ * @xcmd: Command is in complete state
+ */
+static void
+scheduler_complete_to_free(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+ cmd_free(xcmd);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+static void
+scheduler_error_to_free(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+ exec_notify_host(cmd_exec(xcmd));
+ scheduler_complete_to_free(xs, xcmd);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+static void
+scheduler_abort_to_free(struct xocl_scheduler *xs, struct xocl_cmd *xcmd)
+{
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+ scheduler_error_to_free(xs, xcmd);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * scheduler_iterator_cmds() - Iterate all commands in scheduler command queue
+ */
+static void
+scheduler_iterate_cmds(struct xocl_scheduler *xs)
+{
+ struct list_head *pos, *next;
+
+ SCHED_DEBUGF("-> %s\n", __func__);
+ list_for_each_safe(pos, next, &xs->command_queue) {
+ struct xocl_cmd *xcmd = list_entry(pos, struct xocl_cmd, cq_list);
+
+ cmd_update_state(xcmd);
+ SCHED_DEBUGF("+ processing cmd(%lu)\n", xcmd->uid);
+
+ /* check running first since queued maybe we waiting for cmd slot */
+ if (xcmd->state == ERT_CMD_STATE_QUEUED)
+ scheduler_queued_to_submitted(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_SUBMITTED)
+ scheduler_submitted_to_running(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_RUNNING)
+ scheduler_running_to_complete(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_COMPLETED)
+ scheduler_complete_to_free(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_ERROR)
+ scheduler_error_to_free(xs, xcmd);
+ if (xcmd->state == ERT_CMD_STATE_ABORT)
+ scheduler_abort_to_free(xs, xcmd);
+ }
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
+
+/**
+ * scheduler_wait_condition() - Check status of scheduler wait condition
+ *
+ * Scheduler must wait (sleep) if
+ * 1. there are no pending commands
+ * 2. no pending interrupt from embedded scheduler
+ * 3. no pending complete commands in polling mode
+ *
+ * Return: 1 if scheduler must wait, 0 othewise
+ */
+static int
+scheduler_wait_condition(struct xocl_scheduler *xs)
+{
+ if (kthread_should_stop()) {
+ xs->stop = 1;
+ SCHED_DEBUG("scheduler wakes kthread_should_stop\n");
+ return 0;
+ }
+
+ if (atomic_read(&num_pending)) {
+ SCHED_DEBUG("scheduler wakes to copy new pending commands\n");
+ return 0;
+ }
+
+ if (xs->intc) {
+ SCHED_DEBUG("scheduler wakes on interrupt\n");
+ xs->intc = 0;
+ return 0;
+ }
+
+ if (xs->poll) {
+ SCHED_DEBUG("scheduler wakes to poll\n");
+ return 0;
+ }
+
+ SCHED_DEBUG("scheduler waits ...\n");
+ return 1;
+}
+
+/**
+ * scheduler_wait() - check if scheduler should wait
+ *
+ * See scheduler_wait_condition().
+ */
+static void
+scheduler_wait(struct xocl_scheduler *xs)
+{
+ wait_event_interruptible(xs->wait_queue, scheduler_wait_condition(xs) == 0);
+}
+
+/**
+ * scheduler_loop() - Run one loop of the scheduler
+ */
+static void
+scheduler_loop(struct xocl_scheduler *xs)
+{
+ static unsigned int loop_cnt;
+
+ SCHED_DEBUGF("%s\n", __func__);
+ scheduler_wait(xs);
+
+ if (xs->error)
+ DRM_INFO("scheduler encountered unexpected error\n");
+
+ if (xs->stop)
+ return;
+
+ if (xs->reset) {
+ SCHED_DEBUG("scheduler is resetting after timeout\n");
+ scheduler_reset(xs);
+ }
+
+ /* queue new pending commands */
+ scheduler_queue_cmds(xs);
+
+ /* iterate all commands */
+ scheduler_iterate_cmds(xs);
+
+ // loop 8 times before explicitly yielding
+ if (++loop_cnt == 8) {
+ loop_cnt = 0;
+ schedule();
+ }
+}
+
+/**
+ * scheduler() - Command scheduler thread routine
+ */
+static int
+scheduler(void *data)
+{
+ struct xocl_scheduler *xs = (struct xocl_scheduler *)data;
+
+ while (!xs->stop)
+ scheduler_loop(xs);
+ DRM_INFO("%s:%d %s thread exits with value %d\n", __FILE__, __LINE__, __func__, xs->error);
+ return xs->error;
+}
+
+
+
+/**
+ * add_xcmd() - Add initialized xcmd object to pending command list
+ *
+ * @xcmd: Command to add
+ *
+ * Scheduler copies pending commands to its internal command queue.
+ *
+ * Return: 0 on success
+ */
+static int
+add_xcmd(struct xocl_cmd *xcmd)
+{
+ struct exec_core *exec = xcmd->exec;
+ struct xocl_dev *xdev = xocl_get_xdev(exec->pdev);
+
+ // Prevent stop and reset
+ mutex_lock(&exec->exec_lock);
+
+ SCHED_DEBUGF("-> %s(%lu) pid(%d)\n", __func__, xcmd->uid, pid_nr(task_tgid(current)));
+ SCHED_DEBUGF("+ exec stopped(%d) configured(%d)\n", exec->stopped, exec->configured);
+
+ if (exec->stopped || (!exec->configured && cmd_opcode(xcmd) != ERT_CONFIGURE))
+ goto err;
+
+ cmd_set_state(xcmd, ERT_CMD_STATE_NEW);
+ mutex_lock(&pending_cmds_mutex);
+ list_add_tail(&xcmd->cq_list, &pending_cmds);
+ atomic_inc(&num_pending);
+ mutex_unlock(&pending_cmds_mutex);
+
+ /* wake scheduler */
+ atomic_inc(&xdev->outstanding_execs);
+ atomic64_inc(&xdev->total_execs);
+ scheduler_wake_up(xcmd->xs);
+
+ SCHED_DEBUGF("<- %s ret(0) opcode(%d) type(%d) num_pending(%d)\n",
+ __func__, cmd_opcode(xcmd), cmd_type(xcmd), atomic_read(&num_pending));
+ mutex_unlock(&exec->exec_lock);
+ return 0;
+
+err:
+ SCHED_DEBUGF("<- %s ret(1) opcode(%d) type(%d) num_pending(%d)\n",
+ __func__, cmd_opcode(xcmd), cmd_type(xcmd), atomic_read(&num_pending));
+ mutex_unlock(&exec->exec_lock);
+ return 1;
+}
+
+
+/**
+ * add_bo_cmd() - Add a new buffer object command to pending list
+ *
+ * @exec: Targeted device
+ * @client: Client context
+ * @bo: Buffer objects from user space from which new command is created
+ * @numdeps: Number of dependencies for this command
+ * @deps: List of @numdeps dependencies
+ *
+ * Scheduler copies pending commands to its internal command queue.
+ *
+ * Return: 0 on success, 1 on failure
+ */
+static int
+add_bo_cmd(struct exec_core *exec, struct client_ctx *client, struct drm_xocl_bo *bo,
+ int numdeps, struct drm_xocl_bo **deps)
+{
+ struct xocl_cmd *xcmd = cmd_get(exec_scheduler(exec), exec, client);
+
+ if (!xcmd)
+ return 1;
+
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+
+ cmd_bo_init(xcmd, bo, numdeps, deps, !exec_is_ert(exec));
+
+ if (add_xcmd(xcmd))
+ goto err;
+
+ SCHED_DEBUGF("<- %s ret(0) opcode(%d) type(%d)\n", __func__, cmd_opcode(xcmd), cmd_type(xcmd));
+ return 0;
+err:
+ cmd_abort(xcmd);
+ SCHED_DEBUGF("<- %s ret(1) opcode(%d) type(%d)\n", __func__, cmd_opcode(xcmd), cmd_type(xcmd));
+ return 1;
+}
+
+static int
+add_ctrl_cmd(struct exec_core *exec, struct client_ctx *client, struct ert_packet *packet)
+{
+ struct xocl_cmd *xcmd = cmd_get(exec_scheduler(exec), exec, client);
+
+ if (!xcmd)
+ return 1;
+
+ SCHED_DEBUGF("-> %s(%lu)\n", __func__, xcmd->uid);
+
+ cmd_packet_init(xcmd, packet);
+
+ if (add_xcmd(xcmd))
+ goto err;
+
+ SCHED_DEBUGF("<- %s ret(0) opcode(%d) type(%d)\n", __func__, cmd_opcode(xcmd), cmd_type(xcmd));
+ return 0;
+err:
+ cmd_abort(xcmd);
+ SCHED_DEBUGF("<- %s ret(1) opcode(%d) type(%d)\n", __func__, cmd_opcode(xcmd), cmd_type(xcmd));
+ return 1;
+}
+
+
+/**
+ * init_scheduler_thread() - Initialize scheduler thread if necessary
+ *
+ * Return: 0 on success, -errno otherwise
+ */
+static int
+init_scheduler_thread(struct xocl_scheduler *xs)
+{
+ SCHED_DEBUGF("%s use_count=%d\n", __func__, xs->use_count);
+ if (xs->use_count++)
+ return 0;
+
+ init_waitqueue_head(&xs->wait_queue);
+ INIT_LIST_HEAD(&xs->command_queue);
+ scheduler_reset(xs);
+
+ xs->scheduler_thread = kthread_run(scheduler, (void *)xs, "xocl-scheduler-thread0");
+ if (IS_ERR(xs->scheduler_thread)) {
+ int ret = PTR_ERR(xs->scheduler_thread);
+
+ DRM_ERROR(__func__);
+ return ret;
+ }
+ return 0;
+}
+
+/**
+ * fini_scheduler_thread() - Finalize scheduler thread if unused
+ *
+ * Return: 0 on success, -errno otherwise
+ */
+static int
+fini_scheduler_thread(struct xocl_scheduler *xs)
+{
+ int retval = 0;
+
+ SCHED_DEBUGF("%s use_count=%d\n", __func__, xs->use_count);
+ if (--xs->use_count)
+ return 0;
+
+ retval = kthread_stop(xs->scheduler_thread);
+
+ /* clear stale command objects if any */
+ pending_cmds_reset();
+ scheduler_cq_reset(xs);
+
+ /* reclaim memory for allocate command objects */
+ cmd_list_delete();
+
+ return retval;
+}
+
+/**
+ * Entry point for exec buffer.
+ *
+ * Function adds exec buffer to the pending list of commands
+ */
+int
+add_exec_buffer(struct platform_device *pdev, struct client_ctx *client, void *buf,
+ int numdeps, struct drm_xocl_bo **deps)
+{
+ struct exec_core *exec = platform_get_drvdata(pdev);
+ // Add the command to pending list
+ return add_bo_cmd(exec, client, buf, numdeps, deps);
+}
+
+static int
+xocl_client_lock_bitstream_nolock(struct xocl_dev *xdev, struct client_ctx *client)
+{
+ int pid = pid_nr(task_tgid(current));
+ uuid_t *xclbin_id;
+
+ if (client->xclbin_locked)
+ return 0;
+
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+ if (!xclbin_id || !uuid_equal(xclbin_id, &client->xclbin_id)) {
+ userpf_err(xdev,
+ "device xclbin does not match context xclbin, cannot obtain lock for process %d",
+ pid);
+ return 1;
+ }
+
+ if (xocl_icap_lock_bitstream(xdev, &client->xclbin_id, pid) < 0) {
+ userpf_err(xdev, "could not lock bitstream for process %d", pid);
+ return 1;
+ }
+
+ client->xclbin_locked = true;
+ userpf_info(xdev, "process %d successfully locked xcblin", pid);
+ return 0;
+}
+
+static int
+xocl_client_lock_bitstream(struct xocl_dev *xdev, struct client_ctx *client)
+{
+ int ret = 0;
+
+ mutex_lock(&client->lock); // protect current client
+ mutex_lock(&xdev->ctx_list_lock); // protect xdev->xclbin_id
+ ret = xocl_client_lock_bitstream_nolock(xdev, client);
+ mutex_unlock(&xdev->ctx_list_lock);
+ mutex_unlock(&client->lock);
+ return ret;
+}
+
+
+static int
+create_client(struct platform_device *pdev, void **priv)
+{
+ struct client_ctx *client;
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ int ret = 0;
+
+ client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL);
+ if (!client)
+ return -ENOMEM;
+
+ mutex_lock(&xdev->ctx_list_lock);
+
+ if (!xdev->offline) {
+ client->pid = task_tgid(current);
+ mutex_init(&client->lock);
+ client->xclbin_locked = false;
+ client->abort = false;
+ atomic_set(&client->trigger, 0);
+ atomic_set(&client->outstanding_execs, 0);
+ client->num_cus = 0;
+ client->xdev = xocl_get_xdev(pdev);
+ list_add_tail(&client->link, &xdev->ctx_list);
+ *priv = client;
+ } else {
+ /* Do not allow new client to come in while being offline. */
+ devm_kfree(&pdev->dev, client);
+ ret = -EBUSY;
+ }
+
+ mutex_unlock(&xdev->ctx_list_lock);
+
+ DRM_INFO("creating scheduler client for pid(%d), ret: %d\n",
+ pid_nr(task_tgid(current)), ret);
+
+ return ret;
+}
+
+static void destroy_client(struct platform_device *pdev, void **priv)
+{
+ struct client_ctx *client = (struct client_ctx *)(*priv);
+ struct exec_core *exec = platform_get_drvdata(pdev);
+ struct xocl_scheduler *xs = exec_scheduler(exec);
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ unsigned int outstanding = atomic_read(&client->outstanding_execs);
+ unsigned int timeout_loops = 20;
+ unsigned int loops = 0;
+ int pid = pid_nr(task_tgid(current));
+ unsigned int bit;
+ struct ip_layout *layout = XOCL_IP_LAYOUT(xdev);
+
+ bit = layout
+ ? find_first_bit(client->cu_bitmap, layout->m_count)
+ : MAX_CUS;
+
+ /*
+ * This happens when application exists without formally releasing the
+ * contexts on CUs. Give up our contexts on CUs and our lock on xclbin.
+ * Note, that implicit CUs (such as CDMA) do not add to ip_reference.
+ */
+ while (layout && (bit < layout->m_count)) {
+ if (exec->ip_reference[bit]) {
+ userpf_info(xdev, "CTX reclaim (%pUb, %d, %u)",
+ &client->xclbin_id, pid, bit);
+ exec->ip_reference[bit]--;
+ }
+ bit = find_next_bit(client->cu_bitmap, layout->m_count, bit + 1);
+ }
+ bitmap_zero(client->cu_bitmap, MAX_CUS);
+
+ // force scheduler to abort execs for this client
+ client->abort = true;
+
+ // wait for outstanding execs to finish
+ while (outstanding) {
+ unsigned int new;
+
+ userpf_info(xdev, "waiting for %d outstanding execs to finish", outstanding);
+ msleep(500);
+ new = atomic_read(&client->outstanding_execs);
+ loops = (new == outstanding ? (loops + 1) : 0);
+ if (loops == timeout_loops) {
+ userpf_err(xdev,
+ "Giving up with %d outstanding execs, please reset device with 'xbutil reset'\n",
+ outstanding);
+ xdev->needs_reset = true;
+ // reset the scheduler loop
+ xs->reset = true;
+ break;
+ }
+ outstanding = new;
+ }
+
+ DRM_INFO("client exits pid(%d)\n", pid);
+
+ mutex_lock(&xdev->ctx_list_lock);
+ list_del(&client->link);
+ mutex_unlock(&xdev->ctx_list_lock);
+
+ if (client->xclbin_locked)
+ xocl_icap_unlock_bitstream(xdev, &client->xclbin_id, pid);
+ mutex_destroy(&client->lock);
+ devm_kfree(&pdev->dev, client);
+ *priv = NULL;
+}
+
+static uint poll_client(struct platform_device *pdev, struct file *filp,
+ poll_table *wait, void *priv)
+{
+ struct client_ctx *client = (struct client_ctx *)priv;
+ struct exec_core *exec;
+ int counter;
+ uint ret = 0;
+
+ exec = platform_get_drvdata(pdev);
+
+ poll_wait(filp, &exec->poll_wait_queue, wait);
+
+ /*
+ * Mutex lock protects from two threads from the same application
+ * calling poll concurrently using the same file handle
+ */
+ mutex_lock(&client->lock);
+ counter = atomic_read(&client->trigger);
+ if (counter > 0) {
+ /*
+ * Use atomic here since the trigger may be incremented by
+ * interrupt handler running concurrently.
+ */
+ atomic_dec(&client->trigger);
+ ret = POLLIN;
+ }
+ mutex_unlock(&client->lock);
+
+ return ret;
+}
+
+static int client_ioctl_ctx(struct platform_device *pdev,
+ struct client_ctx *client, void *data)
+{
+ bool acquire_lock = false;
+ struct drm_xocl_ctx *args = data;
+ int ret = 0;
+ int pid = pid_nr(task_tgid(current));
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ struct exec_core *exec = platform_get_drvdata(pdev);
+ uuid_t *xclbin_id;
+
+ mutex_lock(&client->lock);
+ mutex_lock(&xdev->ctx_list_lock);
+ xclbin_id = (uuid_t *)xocl_icap_get_data(xdev, XCLBIN_UUID);
+ if (!xclbin_id || !uuid_equal(xclbin_id, &args->xclbin_id)) {
+ ret = -EBUSY;
+ goto out;
+ }
+
+ if (args->cu_index >= XOCL_IP_LAYOUT(xdev)->m_count) {
+ userpf_err(xdev, "cuidx(%d) >= numcus(%d)\n",
+ args->cu_index, XOCL_IP_LAYOUT(xdev)->m_count);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (args->op == XOCL_CTX_OP_FREE_CTX) {
+ ret = test_and_clear_bit(args->cu_index, client->cu_bitmap) ? 0 : -EINVAL;
+ if (ret) // No context was previously allocated for this CU
+ goto out;
+
+ // CU unlocked explicitly
+ --exec->ip_reference[args->cu_index];
+ if (!--client->num_cus) {
+ // We just gave up the last context, unlock the xclbin
+ ret = xocl_icap_unlock_bitstream(xdev, xclbin_id, pid);
+ client->xclbin_locked = false;
+ }
+ userpf_info(xdev, "CTX del(%pUb, %d, %u)",
+ xclbin_id, pid, args->cu_index);
+ goto out;
+ }
+
+ if (args->op != XOCL_CTX_OP_ALLOC_CTX) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ if (args->flags != XOCL_CTX_SHARED) {
+ userpf_err(xdev, "Only shared contexts are supported in this release");
+ ret = -EPERM;
+ goto out;
+ }
+
+ if (!client->num_cus && !client->xclbin_locked)
+ // Process has no other context on any CU yet, hence we need to
+ // lock the xclbin A process uses just one lock for all its ctxs
+ acquire_lock = true;
+
+ if (test_and_set_bit(args->cu_index, client->cu_bitmap)) {
+ userpf_info(xdev, "CTX already allocated by this process");
+ // Context was previously allocated for the same CU,
+ // cannot allocate again
+ ret = 0;
+ goto out;
+ }
+
+ if (acquire_lock) {
+ // This is the first context on any CU for this process,
+ // lock the xclbin
+ ret = xocl_client_lock_bitstream_nolock(xdev, client);
+ if (ret) {
+ // Locking of xclbin failed, give up our context
+ clear_bit(args->cu_index, client->cu_bitmap);
+ goto out;
+ } else {
+ uuid_copy(&client->xclbin_id, xclbin_id);
+ }
+ }
+
+ // Everything is good so far, hence increment the CU reference count
+ ++client->num_cus; // explicitly acquired
+ ++exec->ip_reference[args->cu_index];
+ xocl_info(&pdev->dev, "CTX add(%pUb, %d, %u, %d)",
+ xclbin_id, pid, args->cu_index, acquire_lock);
+out:
+ mutex_unlock(&xdev->ctx_list_lock);
+ mutex_unlock(&client->lock);
+ return ret;
+}
+
+static int
+get_bo_paddr(struct xocl_dev *xdev, struct drm_file *filp,
+ uint32_t bo_hdl, size_t off, size_t size, uint64_t *paddrp)
+{
+ struct drm_device *ddev = filp->minor->dev;
+ struct drm_gem_object *obj;
+ struct drm_xocl_bo *xobj;
+
+ obj = xocl_gem_object_lookup(ddev, filp, bo_hdl);
+ if (!obj) {
+ userpf_err(xdev, "Failed to look up GEM BO 0x%x\n", bo_hdl);
+ return -ENOENT;
+ }
+ xobj = to_xocl_bo(obj);
+
+ if (obj->size <= off || obj->size < off + size || !xobj->mm_node) {
+ userpf_err(xdev, "Failed to get paddr for BO 0x%x\n", bo_hdl);
+//PORT4_20
+// drm_gem_object_unreference_unlocked(obj);
+ drm_gem_object_put_unlocked(obj);
+ return -EINVAL;
+ }
+
+ *paddrp = xobj->mm_node->start + off;
+// drm_gem_object_unreference_unlocked(obj);
+ drm_gem_object_put_unlocked(obj);
+ return 0;
+}
+
+static int
+convert_execbuf(struct xocl_dev *xdev, struct drm_file *filp,
+ struct exec_core *exec, struct drm_xocl_bo *xobj)
+{
+ int i;
+ int ret;
+ size_t src_off;
+ size_t dst_off;
+ size_t sz;
+ uint64_t src_addr;
+ uint64_t dst_addr;
+ struct ert_start_copybo_cmd *scmd = (struct ert_start_copybo_cmd *)xobj->vmapping;
+
+ /* Only convert COPYBO cmd for now. */
+ if (scmd->opcode != ERT_START_COPYBO)
+ return 0;
+
+ sz = scmd->size * COPYBO_UNIT;
+
+ src_off = scmd->src_addr_hi;
+ src_off <<= 32;
+ src_off |= scmd->src_addr_lo;
+ ret = get_bo_paddr(xdev, filp, scmd->src_bo_hdl, src_off, sz, &src_addr);
+ if (ret != 0)
+ return ret;
+
+ dst_off = scmd->dst_addr_hi;
+ dst_off <<= 32;
+ dst_off |= scmd->dst_addr_lo;
+ ret = get_bo_paddr(xdev, filp, scmd->dst_bo_hdl, dst_off, sz, &dst_addr);
+ if (ret != 0)
+ return ret;
+
+ ert_fill_copybo_cmd(scmd, 0, 0, src_addr, dst_addr, sz);
+
+ for (i = exec->num_cus - exec->num_cdma; i < exec->num_cus; i++)
+ scmd->cu_mask[i / 32] |= 1 << (i % 32);
+
+ scmd->opcode = ERT_START_CU;
+
+ return 0;
+}
+
+static int
+client_ioctl_execbuf(struct platform_device *pdev,
+ struct client_ctx *client, void *data, struct drm_file *filp)
+{
+ struct drm_xocl_execbuf *args = data;
+ struct drm_xocl_bo *xobj;
+ struct drm_gem_object *obj;
+ struct drm_xocl_bo *deps[8] = {0};
+ int numdeps = -1;
+ int ret = 0;
+ struct xocl_dev *xdev = xocl_get_xdev(pdev);
+ struct drm_device *ddev = filp->minor->dev;
+
+ if (xdev->needs_reset) {
+ userpf_err(xdev, "device needs reset, use 'xbutil reset -h'");
+ return -EBUSY;
+ }
+
+ /* Look up the gem object corresponding to the BO handle.
+ * This adds a reference to the gem object. The refernece is
+ * passed to kds or released here if errors occur.
+ */
+ obj = xocl_gem_object_lookup(ddev, filp, args->exec_bo_handle);
+ if (!obj) {
+ userpf_err(xdev, "Failed to look up GEM BO %d\n",
+ args->exec_bo_handle);
+ return -ENOENT;
+ }
+
+ /* Convert gem object to xocl_bo extension */
+ xobj = to_xocl_bo(obj);
+ if (!xocl_bo_execbuf(xobj) || convert_execbuf(xdev, filp,
+ platform_get_drvdata(pdev), xobj) != 0) {
+ ret = -EINVAL;
+ goto out;
+ }
+
+ ret = validate(pdev, client, xobj);
+ if (ret) {
+ userpf_err(xdev, "Exec buffer validation failed\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Copy dependencies from user. It is an error if a BO handle specified
+ * as a dependency does not exists. Lookup gem object corresponding to bo
+ * handle. Convert gem object to xocl_bo extension. Note that the
+ * gem lookup acquires a reference to the drm object, this reference
+ * is passed on to the the scheduler via xocl_exec_add_buffer.
+ */
+ for (numdeps = 0; numdeps < 8 && args->deps[numdeps]; ++numdeps) {
+ struct drm_gem_object *gobj =
+ xocl_gem_object_lookup(ddev, filp, args->deps[numdeps]);
+ struct drm_xocl_bo *xbo = gobj ? to_xocl_bo(gobj) : NULL;
+
+ if (!gobj)
+ userpf_err(xdev, "Failed to look up GEM BO %d\n",
+ args->deps[numdeps]);
+ if (!xbo) {
+ ret = -EINVAL;
+ goto out;
+ }
+ deps[numdeps] = xbo;
+ }
+
+ /* acquire lock on xclbin if necessary */
+ ret = xocl_client_lock_bitstream(xdev, client);
+ if (ret) {
+ userpf_err(xdev, "Failed to lock xclbin\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Add exec buffer to scheduler (kds). The scheduler manages the
+ * drm object references acquired by xobj and deps. It is vital
+ * that the references are released properly.
+ */
+ ret = add_exec_buffer(pdev, client, xobj, numdeps, deps);
+ if (ret) {
+ userpf_err(xdev, "Failed to add exec buffer to scheduler\n");
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Return here, noting that the gem objects passed to kds have
+ * references that must be released by kds itself. User manages
+ * a regular reference to all BOs returned as file handles. These
+ * references are released with the BOs are freed.
+ */
+ return ret;
+
+out:
+ for (--numdeps; numdeps >= 0; numdeps--)
+ drm_gem_object_put_unlocked(&deps[numdeps]->base);
+//PORT4_20
+// drm_gem_object_unreference_unlocked(&deps[numdeps]->base);
+ drm_gem_object_put_unlocked(&xobj->base);
+// drm_gem_object_unreference_unlocked(&xobj->base);
+ return ret;
+}
+
+int
+client_ioctl(struct platform_device *pdev, int op, void *data, void *drm_filp)
+{
+ struct drm_file *filp = drm_filp;
+ struct client_ctx *client = filp->driver_priv;
+ int ret;
+
+ switch (op) {
+ case DRM_XOCL_CTX:
+ ret = client_ioctl_ctx(pdev, client, data);
+ break;
+ case DRM_XOCL_EXECBUF:
+ ret = client_ioctl_execbuf(pdev, client, data, drm_filp);
+ break;
+ default:
+ ret = -EINVAL;
+ break;
+ }
+
+ return ret;
+}
+/**
+ * reset() - Reset device exec data structure
+ *
+ * @pdev: platform device to reset
+ *
+ * [Current 2018.3 situation:]
+ * This function is currently called from mgmt icap on every AXI is
+ * freeze/unfreeze. It ensures that the device exec_core state is reset to
+ * same state as was when scheduler was originally probed for the device.
+ * The callback from icap, ensures that scheduler resets the exec core when
+ * multiple processes are already attached to the device but AXI is reset.
+ *
+ * Even though the very first client created for this device also resets the
+ * exec core, it is possible that further resets are necessary. For example
+ * in multi-process case, there can be 'n' processes that attach to the
+ * device. On first client attach the exec core is reset correctly, but now
+ * assume that 'm' of these processes finishes completely before any remaining
+ * (n-m) processes start using the scheduler. In this case, the n-m clients have
+ * already been created, but icap resets AXI because the xclbin has no
+ * references (arguably this AXI reset is wrong)
+ *
+ * [Work-in-progress:]
+ * Proper contract:
+ * Pre-condition: xocl_exec_stop has been called before xocl_exec_reset.
+ * Pre-condition: new bitstream has been downloaded and AXI has been reset
+ */
+static int
+reset(struct platform_device *pdev)
+{
+ struct exec_core *exec = platform_get_drvdata(pdev);
+
+ exec_stop(exec); // remove when upstream explicitly calls stop()
+ exec_reset(exec);
+ return 0;
+}
+
+/**
+ * stop() - Reset device exec data structure
+ *
+ * This API must be called prior to performing an AXI reset and downloading of
+ * a new xclbin. Calling this API flushes the commands running on current
+ * device and prevents new commands from being scheduled on the device. This
+ * effectively prevents 'xbutil top' from issuing CU_STAT commands while
+ * programming is performed.
+ *
+ * Pre-condition: xocl_client_release has been called, e.g there are no
+ * current clients using the bitstream
+ */
+static int
+stop(struct platform_device *pdev)
+{
+ struct exec_core *exec = platform_get_drvdata(pdev);
+
+ exec_stop(exec);
+ return 0;
+}
+
+/**
+ * validate() - Check if requested cmd is valid in the current context
+ */
+static int
+validate(struct platform_device *pdev, struct client_ctx *client, const struct drm_xocl_bo *bo)
+{
+ struct ert_packet *ecmd = (struct ert_packet *)bo->vmapping;
+ struct ert_start_kernel_cmd *scmd = (struct ert_start_kernel_cmd *)bo->vmapping;
+ unsigned int i = 0;
+ u32 ctx_cus[4] = {0};
+ u32 cumasks = 0;
+ int err = 0;
+
+ SCHED_DEBUGF("-> %s(%d)\n", __func__, ecmd->opcode);
+
+ /* cus for start kernel commands only */
+ if (ecmd->opcode != ERT_START_CU)
+ return 0; /* ok */
+
+ /* client context cu bitmap may not change while validating */
+ mutex_lock(&client->lock);
+
+ /* no specific CUs selected, maybe ctx is not used by client */
+ if (bitmap_empty(client->cu_bitmap, MAX_CUS)) {
+ userpf_err(xocl_get_xdev(pdev), "%s found no CUs in ctx\n", __func__);
+ goto out; /* ok */
+ }
+
+ /* Check CUs in cmd BO against CUs in context */
+ cumasks = 1 + scmd->extra_cu_masks;
+ xocl_bitmap_to_arr32(ctx_cus, client->cu_bitmap, cumasks * 32);
+
+ for (i = 0; i < cumasks; ++i) {
+ uint32_t cmd_cus = ecmd->data[i];
+ /* cmd_cus must be subset of ctx_cus */
+ if (cmd_cus & ~ctx_cus[i]) {
+ SCHED_DEBUGF("<- %s(1), CU mismatch in mask(%d) cmd(0x%x) ctx(0x%x)\n",
+ __func__, i, cmd_cus, ctx_cus[i]);
+ err = 1;
+ goto out; /* error */
+ }
+ }
+
+
+out:
+ mutex_unlock(&client->lock);
+ SCHED_DEBUGF("<- %s(%d) cmd and ctx CUs match\n", __func__, err);
+ return err;
+
+}
+
+struct xocl_mb_scheduler_funcs sche_ops = {
+ .create_client = create_client,
+ .destroy_client = destroy_client,
+ .poll_client = poll_client,
+ .client_ioctl = client_ioctl,
+ .stop = stop,
+ .reset = reset,
+};
+
+/* sysfs */
+static ssize_t
+kds_numcus_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct exec_core *exec = dev_get_exec(dev);
+ unsigned int cus = exec ? exec->num_cus - exec->num_cdma : 0;
+
+ return sprintf(buf, "%d\n", cus);
+}
+static DEVICE_ATTR_RO(kds_numcus);
+
+static ssize_t
+kds_numcdmas_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct xocl_dev *xdev = dev_get_xdev(dev);
+ uint32_t *cdma = xocl_cdma_addr(xdev);
+ unsigned int cdmas = cdma ? 1 : 0; //TBD
+
+ return sprintf(buf, "%d\n", cdmas);
+}
+static DEVICE_ATTR_RO(kds_numcdmas);
+
+static ssize_t
+kds_custat_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct exec_core *exec = dev_get_exec(dev);
+ struct xocl_dev *xdev = exec_get_xdev(exec);
+ struct client_ctx client;
+ struct ert_packet packet;
+ unsigned int count = 0;
+ ssize_t sz = 0;
+
+ // minimum required initialization of client
+ client.abort = false;
+ client.xdev = xdev;
+ atomic_set(&client.trigger, 0);
+ atomic_set(&client.outstanding_execs, 0);
+
+ packet.opcode = ERT_CU_STAT;
+ packet.type = ERT_CTRL;
+ packet.count = 1; // data[1]
+
+ if (add_ctrl_cmd(exec, &client, &packet) == 0) {
+ int retry = 5;
+
+ SCHED_DEBUGF("-> custat waiting for command to finish\n");
+ // wait for command completion
+ while (--retry && atomic_read(&client.outstanding_execs))
+ msleep(100);
+ if (retry == 0 && atomic_read(&client.outstanding_execs))
+ userpf_info(xdev, "custat unexpected timeout\n");
+ SCHED_DEBUGF("<- custat retry(%d)\n", retry);
+ }
+
+ for (count = 0; count < exec->num_cus; ++count)
+ sz += sprintf(buf+sz, "CU[@0x%x] : %d\n",
+ exec_cu_base_addr(exec, count),
+ exec_cu_usage(exec, count));
+ if (sz)
+ buf[sz++] = 0;
+
+ return sz;
+}
+static DEVICE_ATTR_RO(kds_custat);
+
+static struct attribute *kds_sysfs_attrs[] = {
+ &dev_attr_kds_numcus.attr,
+ &dev_attr_kds_numcdmas.attr,
+ &dev_attr_kds_custat.attr,
+ NULL
+};
+
+static const struct attribute_group kds_sysfs_attr_group = {
+ .attrs = kds_sysfs_attrs,
+};
+
+static void
+user_sysfs_destroy_kds(struct platform_device *pdev)
+{
+ sysfs_remove_group(&pdev->dev.kobj, &kds_sysfs_attr_group);
+}
+
+static int
+user_sysfs_create_kds(struct platform_device *pdev)
+{
+ int err = sysfs_create_group(&pdev->dev.kobj, &kds_sysfs_attr_group);
+
+ if (err)
+ userpf_err(xocl_get_xdev(pdev), "create kds attr failed: 0x%x", err);
+ return err;
+}
+
+/**
+ * Init scheduler
+ */
+static int mb_scheduler_probe(struct platform_device *pdev)
+{
+ struct exec_core *exec = exec_create(pdev, &scheduler0);
+
+ if (!exec)
+ return -ENOMEM;
+
+ if (user_sysfs_create_kds(pdev))
+ goto err;
+
+ init_scheduler_thread(&scheduler0);
+ xocl_subdev_register(pdev, XOCL_SUBDEV_MB_SCHEDULER, &sche_ops);
+ platform_set_drvdata(pdev, exec);
+
+ DRM_INFO("command scheduler started\n");
+
+ return 0;
+
+err:
+ devm_kfree(&pdev->dev, exec);
+ return 1;
+}
+
+/**
+ * Fini scheduler
+ */
+static int mb_scheduler_remove(struct platform_device *pdev)
+{
+ struct xocl_dev *xdev;
+ int i;
+ struct exec_core *exec = platform_get_drvdata(pdev);
+
+ SCHED_DEBUGF("-> %s\n", __func__);
+ fini_scheduler_thread(exec_scheduler(exec));
+
+ xdev = xocl_get_xdev(pdev);
+ for (i = 0; i < exec->intr_num; i++) {
+ xocl_user_interrupt_config(xdev, i + exec->intr_base, false);
+ xocl_user_interrupt_reg(xdev, i + exec->intr_base,
+ NULL, NULL);
+ }
+ mutex_destroy(&exec->exec_lock);
+
+ user_sysfs_destroy_kds(pdev);
+ exec_destroy(exec);
+ platform_set_drvdata(pdev, NULL);
+
+ SCHED_DEBUGF("<- %s\n", __func__);
+ DRM_INFO("command scheduler removed\n");
+ return 0;
+}
+
+static struct platform_device_id mb_sche_id_table[] = {
+ { XOCL_MB_SCHEDULER, 0 },
+ { },
+};
+
+static struct platform_driver mb_scheduler_driver = {
+ .probe = mb_scheduler_probe,
+ .remove = mb_scheduler_remove,
+ .driver = {
+ .name = "xocl_mb_sche",
+ },
+ .id_table = mb_sche_id_table,
+};
+
+int __init xocl_init_mb_scheduler(void)
+{
+ return platform_driver_register(&mb_scheduler_driver);
+}
+
+void xocl_fini_mb_scheduler(void)
+{
+ SCHED_DEBUGF("-> %s\n", __func__);
+ platform_driver_unregister(&mb_scheduler_driver);
+ SCHED_DEBUGF("<- %s\n", __func__);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/microblaze.c b/drivers/gpu/drm/xocl/subdev/microblaze.c
new file mode 100644
index 000000000000..38cfbdbb39ef
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/microblaze.c
@@ -0,0 +1,722 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Lizhi.HOu@xilinx.com
+ *
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/vmalloc.h>
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+#define MAX_RETRY 50
+#define RETRY_INTERVAL 100 //ms
+
+#define MAX_IMAGE_LEN 0x20000
+
+#define REG_VERSION 0
+#define REG_ID 0x4
+#define REG_STATUS 0x8
+#define REG_ERR 0xC
+#define REG_CAP 0x10
+#define REG_CTL 0x18
+#define REG_STOP_CONFIRM 0x1C
+#define REG_CURR_BASE 0x20
+#define REG_POWER_CHECKSUM 0x1A4
+
+#define VALID_ID 0x74736574
+
+#define GPIO_RESET 0x0
+#define GPIO_ENABLED 0x1
+
+#define SELF_JUMP(ins) (((ins) & 0xfc00ffff) == 0xb8000000)
+
+enum ctl_mask {
+ CTL_MASK_CLEAR_POW = 0x1,
+ CTL_MASK_CLEAR_ERR = 0x2,
+ CTL_MASK_PAUSE = 0x4,
+ CTL_MASK_STOP = 0x8,
+};
+
+enum status_mask {
+ STATUS_MASK_INIT_DONE = 0x1,
+ STATUS_MASK_STOPPED = 0x2,
+ STATUS_MASK_PAUSE = 0x4,
+};
+
+enum cap_mask {
+ CAP_MASK_PM = 0x1,
+};
+
+enum {
+ MB_STATE_INIT = 0,
+ MB_STATE_RUN,
+ MB_STATE_RESET,
+};
+
+enum {
+ IO_REG,
+ IO_GPIO,
+ IO_IMAGE_MGMT,
+ IO_IMAGE_SCHE,
+ NUM_IOADDR
+};
+
+#define READ_REG32(mb, off) \
+ XOCL_READ_REG32(mb->base_addrs[IO_REG] + off)
+#define WRITE_REG32(mb, val, off) \
+ XOCL_WRITE_REG32(val, mb->base_addrs[IO_REG] + off)
+
+#define READ_GPIO(mb, off) \
+ XOCL_READ_REG32(mb->base_addrs[IO_GPIO] + off)
+#define WRITE_GPIO(mb, val, off) \
+ XOCL_WRITE_REG32(val, mb->base_addrs[IO_GPIO] + off)
+
+#define READ_IMAGE_MGMT(mb, off) \
+ XOCL_READ_REG32(mb->base_addrs[IO_IMAGE_MGMT] + off)
+
+#define COPY_MGMT(mb, buf, len) \
+ xocl_memcpy_toio(mb->base_addrs[IO_IMAGE_MGMT], buf, len)
+#define COPY_SCHE(mb, buf, len) \
+ xocl_memcpy_toio(mb->base_addrs[IO_IMAGE_SCHE], buf, len)
+
+struct xocl_mb {
+ struct platform_device *pdev;
+ void __iomem *base_addrs[NUM_IOADDR];
+
+ struct device *hwmon_dev;
+ bool enabled;
+ u32 state;
+ u32 cap;
+ struct mutex mb_lock;
+
+ char *sche_binary;
+ u32 sche_binary_length;
+ char *mgmt_binary;
+ u32 mgmt_binary_length;
+};
+
+static int mb_stop(struct xocl_mb *mb);
+static int mb_start(struct xocl_mb *mb);
+
+/* sysfs support */
+static void safe_read32(struct xocl_mb *mb, u32 reg, u32 *val)
+{
+ mutex_lock(&mb->mb_lock);
+ if (mb->enabled && mb->state == MB_STATE_RUN)
+ *val = READ_REG32(mb, reg);
+ else
+ *val = 0;
+ mutex_unlock(&mb->mb_lock);
+}
+
+static void safe_write32(struct xocl_mb *mb, u32 reg, u32 val)
+{
+ mutex_lock(&mb->mb_lock);
+ if (mb->enabled && mb->state == MB_STATE_RUN)
+ WRITE_REG32(mb, val, reg);
+ mutex_unlock(&mb->mb_lock);
+}
+
+static ssize_t version_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_VERSION, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(version);
+
+static ssize_t id_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_ID, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(id);
+
+static ssize_t status_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_STATUS, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(status);
+
+static ssize_t error_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_ERR, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(error);
+
+static ssize_t capability_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_CAP, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(capability);
+
+static ssize_t power_checksum_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_POWER_CHECKSUM, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(power_checksum);
+
+static ssize_t pause_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(mb, REG_CTL, &val);
+
+ return sprintf(buf, "%d\n", !!(val & CTL_MASK_PAUSE));
+}
+
+static ssize_t pause_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val > 1)
+ return -EINVAL;
+
+ val = val ? CTL_MASK_PAUSE : 0;
+ safe_write32(mb, REG_CTL, val);
+
+ return count;
+}
+static DEVICE_ATTR_RW(pause);
+
+static ssize_t reset_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_mb *mb = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val > 1)
+ return -EINVAL;
+
+ if (val) {
+ mb_stop(mb);
+ mb_start(mb);
+ }
+
+ return count;
+}
+static DEVICE_ATTR_WO(reset);
+
+static struct attribute *mb_attrs[] = {
+ &dev_attr_version.attr,
+ &dev_attr_id.attr,
+ &dev_attr_status.attr,
+ &dev_attr_error.attr,
+ &dev_attr_capability.attr,
+ &dev_attr_power_checksum.attr,
+ &dev_attr_pause.attr,
+ &dev_attr_reset.attr,
+ NULL,
+};
+static struct attribute_group mb_attr_group = {
+ .attrs = mb_attrs,
+};
+
+static ssize_t show_mb_pw(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+ struct xocl_mb *mb = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(mb, REG_CURR_BASE + attr->index * sizeof(u32), &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+
+static SENSOR_DEVICE_ATTR(curr1_highest, 0444, show_mb_pw, NULL, 0);
+static SENSOR_DEVICE_ATTR(curr1_average, 0444, show_mb_pw, NULL, 1);
+static SENSOR_DEVICE_ATTR(curr1_input, 0444, show_mb_pw, NULL, 2);
+static SENSOR_DEVICE_ATTR(curr2_highest, 0444, show_mb_pw, NULL, 3);
+static SENSOR_DEVICE_ATTR(curr2_average, 0444, show_mb_pw, NULL, 4);
+static SENSOR_DEVICE_ATTR(curr2_input, 0444, show_mb_pw, NULL, 5);
+static SENSOR_DEVICE_ATTR(curr3_highest, 0444, show_mb_pw, NULL, 6);
+static SENSOR_DEVICE_ATTR(curr3_average, 0444, show_mb_pw, NULL, 7);
+static SENSOR_DEVICE_ATTR(curr3_input, 0444, show_mb_pw, NULL, 8);
+static SENSOR_DEVICE_ATTR(curr4_highest, 0444, show_mb_pw, NULL, 9);
+static SENSOR_DEVICE_ATTR(curr4_average, 0444, show_mb_pw, NULL, 10);
+static SENSOR_DEVICE_ATTR(curr4_input, 0444, show_mb_pw, NULL, 11);
+static SENSOR_DEVICE_ATTR(curr5_highest, 0444, show_mb_pw, NULL, 12);
+static SENSOR_DEVICE_ATTR(curr5_average, 0444, show_mb_pw, NULL, 13);
+static SENSOR_DEVICE_ATTR(curr5_input, 0444, show_mb_pw, NULL, 14);
+static SENSOR_DEVICE_ATTR(curr6_highest, 0444, show_mb_pw, NULL, 15);
+static SENSOR_DEVICE_ATTR(curr6_average, 0444, show_mb_pw, NULL, 16);
+static SENSOR_DEVICE_ATTR(curr6_input, 0444, show_mb_pw, NULL, 17);
+
+static struct attribute *hwmon_mb_attributes[] = {
+ &sensor_dev_attr_curr1_highest.dev_attr.attr,
+ &sensor_dev_attr_curr1_average.dev_attr.attr,
+ &sensor_dev_attr_curr1_input.dev_attr.attr,
+ &sensor_dev_attr_curr2_highest.dev_attr.attr,
+ &sensor_dev_attr_curr2_average.dev_attr.attr,
+ &sensor_dev_attr_curr2_input.dev_attr.attr,
+ &sensor_dev_attr_curr3_highest.dev_attr.attr,
+ &sensor_dev_attr_curr3_average.dev_attr.attr,
+ &sensor_dev_attr_curr3_input.dev_attr.attr,
+ &sensor_dev_attr_curr4_highest.dev_attr.attr,
+ &sensor_dev_attr_curr4_average.dev_attr.attr,
+ &sensor_dev_attr_curr4_input.dev_attr.attr,
+ &sensor_dev_attr_curr5_highest.dev_attr.attr,
+ &sensor_dev_attr_curr5_average.dev_attr.attr,
+ &sensor_dev_attr_curr5_input.dev_attr.attr,
+ &sensor_dev_attr_curr6_highest.dev_attr.attr,
+ &sensor_dev_attr_curr6_average.dev_attr.attr,
+ &sensor_dev_attr_curr6_input.dev_attr.attr,
+ NULL
+};
+
+static const struct attribute_group hwmon_mb_attrgroup = {
+ .attrs = hwmon_mb_attributes,
+};
+
+static ssize_t show_name(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", XCLMGMT_MB_HWMON_NAME);
+}
+
+static struct sensor_device_attribute name_attr =
+ SENSOR_ATTR(name, 0444, show_name, NULL, 0);
+
+static void mgmt_sysfs_destroy_mb(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+
+ mb = platform_get_drvdata(pdev);
+
+ if (!mb->enabled)
+ return;
+
+ if (mb->hwmon_dev) {
+ device_remove_file(mb->hwmon_dev, &name_attr.dev_attr);
+ sysfs_remove_group(&mb->hwmon_dev->kobj,
+ &hwmon_mb_attrgroup);
+ hwmon_device_unregister(mb->hwmon_dev);
+ mb->hwmon_dev = NULL;
+ }
+
+ sysfs_remove_group(&pdev->dev.kobj, &mb_attr_group);
+}
+
+static int mgmt_sysfs_create_mb(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+ struct xocl_dev_core *core;
+ int err;
+
+ mb = platform_get_drvdata(pdev);
+ core = XDEV(xocl_get_xdev(pdev));
+
+ if (!mb->enabled)
+ return 0;
+ err = sysfs_create_group(&pdev->dev.kobj, &mb_attr_group);
+ if (err) {
+ xocl_err(&pdev->dev, "create mb attrs failed: 0x%x", err);
+ goto create_attr_failed;
+ }
+ mb->hwmon_dev = hwmon_device_register(&core->pdev->dev);
+ if (IS_ERR(mb->hwmon_dev)) {
+ err = PTR_ERR(mb->hwmon_dev);
+ xocl_err(&pdev->dev, "register mb hwmon failed: 0x%x", err);
+ goto hwmon_reg_failed;
+ }
+
+ dev_set_drvdata(mb->hwmon_dev, mb);
+
+ err = device_create_file(mb->hwmon_dev, &name_attr.dev_attr);
+ if (err) {
+ xocl_err(&pdev->dev, "create attr name failed: 0x%x", err);
+ goto create_name_failed;
+ }
+
+ err = sysfs_create_group(&mb->hwmon_dev->kobj,
+ &hwmon_mb_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create pw group failed: 0x%x", err);
+ goto create_pw_failed;
+ }
+
+ return 0;
+
+create_pw_failed:
+ device_remove_file(mb->hwmon_dev, &name_attr.dev_attr);
+create_name_failed:
+ hwmon_device_unregister(mb->hwmon_dev);
+ mb->hwmon_dev = NULL;
+hwmon_reg_failed:
+ sysfs_remove_group(&pdev->dev.kobj, &mb_attr_group);
+create_attr_failed:
+ return err;
+}
+
+static int mb_stop(struct xocl_mb *mb)
+{
+ int retry = 0;
+ int ret = 0;
+ u32 reg_val = 0;
+
+ if (!mb->enabled)
+ return 0;
+
+ mutex_lock(&mb->mb_lock);
+ reg_val = READ_GPIO(mb, 0);
+ xocl_info(&mb->pdev->dev, "Reset GPIO 0x%x", reg_val);
+ if (reg_val == GPIO_RESET) {
+ /* MB in reset status */
+ mb->state = MB_STATE_RESET;
+ goto out;
+ }
+
+ xocl_info(&mb->pdev->dev,
+ "MGMT Image magic word, 0x%x, status 0x%x, id 0x%x",
+ READ_IMAGE_MGMT(mb, 0),
+ READ_REG32(mb, REG_STATUS),
+ READ_REG32(mb, REG_ID));
+
+ if (!SELF_JUMP(READ_IMAGE_MGMT(mb, 0))) {
+ /* non cold boot */
+ reg_val = READ_REG32(mb, REG_STATUS);
+ if (!(reg_val & STATUS_MASK_STOPPED)) {
+ // need to stop microblaze
+ xocl_info(&mb->pdev->dev, "stopping microblaze...");
+ WRITE_REG32(mb, CTL_MASK_STOP, REG_CTL);
+ WRITE_REG32(mb, 1, REG_STOP_CONFIRM);
+ while (retry++ < MAX_RETRY &&
+ !(READ_REG32(mb, REG_STATUS) &
+ STATUS_MASK_STOPPED)) {
+ msleep(RETRY_INTERVAL);
+ }
+ if (retry >= MAX_RETRY) {
+ xocl_err(&mb->pdev->dev,
+ "Failed to stop microblaze");
+ xocl_err(&mb->pdev->dev,
+ "Error Reg 0x%x",
+ READ_REG32(mb, REG_ERR));
+ ret = -EIO;
+ goto out;
+ }
+ }
+ xocl_info(&mb->pdev->dev, "Microblaze Stopped, retry %d",
+ retry);
+ }
+
+ /* hold reset */
+ WRITE_GPIO(mb, GPIO_RESET, 0);
+ mb->state = MB_STATE_RESET;
+out:
+ mutex_unlock(&mb->mb_lock);
+
+ return ret;
+}
+
+static int mb_start(struct xocl_mb *mb)
+{
+ int retry = 0;
+ u32 reg_val = 0;
+ int ret = 0;
+ void *xdev_hdl;
+
+ if (!mb->enabled)
+ return 0;
+
+ xdev_hdl = xocl_get_xdev(mb->pdev);
+
+ mutex_lock(&mb->mb_lock);
+ reg_val = READ_GPIO(mb, 0);
+ xocl_info(&mb->pdev->dev, "Reset GPIO 0x%x", reg_val);
+ if (reg_val == GPIO_ENABLED)
+ goto out;
+
+ xocl_info(&mb->pdev->dev, "Start Microblaze...");
+ xocl_info(&mb->pdev->dev, "MGMT Image magic word, 0x%x",
+ READ_IMAGE_MGMT(mb, 0));
+
+ if (xocl_mb_mgmt_on(xdev_hdl)) {
+ xocl_info(&mb->pdev->dev, "Copying mgmt image len %d",
+ mb->mgmt_binary_length);
+ COPY_MGMT(mb, mb->mgmt_binary, mb->mgmt_binary_length);
+ }
+
+ if (xocl_mb_sched_on(xdev_hdl)) {
+ xocl_info(&mb->pdev->dev, "Copying scheduler image len %d",
+ mb->sche_binary_length);
+ COPY_SCHE(mb, mb->sche_binary, mb->sche_binary_length);
+ }
+
+ WRITE_GPIO(mb, GPIO_ENABLED, 0);
+ xocl_info(&mb->pdev->dev,
+ "MGMT Image magic word, 0x%x, status 0x%x, id 0x%x",
+ READ_IMAGE_MGMT(mb, 0),
+ READ_REG32(mb, REG_STATUS),
+ READ_REG32(mb, REG_ID));
+ do {
+ msleep(RETRY_INTERVAL);
+ } while (retry++ < MAX_RETRY && (READ_REG32(mb, REG_STATUS) &
+ STATUS_MASK_STOPPED));
+
+ /* Extra pulse needed as workaround for axi interconnect issue in DSA */
+ if (retry >= MAX_RETRY) {
+ retry = 0;
+ WRITE_GPIO(mb, GPIO_RESET, 0);
+ WRITE_GPIO(mb, GPIO_ENABLED, 0);
+ do {
+ msleep(RETRY_INTERVAL);
+ } while (retry++ < MAX_RETRY && (READ_REG32(mb, REG_STATUS) &
+ STATUS_MASK_STOPPED));
+ }
+
+ if (retry >= MAX_RETRY) {
+ xocl_err(&mb->pdev->dev, "Failed to start microblaze");
+ xocl_err(&mb->pdev->dev, "Error Reg 0x%x",
+ READ_REG32(mb, REG_ERR));
+ ret = -EIO;
+ }
+
+ mb->cap = READ_REG32(mb, REG_CAP);
+ mb->state = MB_STATE_RUN;
+out:
+ mutex_unlock(&mb->mb_lock);
+
+ return ret;
+}
+
+static void mb_reset(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+
+ xocl_info(&pdev->dev, "Reset Microblaze...");
+ mb = platform_get_drvdata(pdev);
+ if (!mb)
+ return;
+
+ mb_stop(mb);
+ mb_start(mb);
+}
+
+static int load_mgmt_image(struct platform_device *pdev, const char *image,
+ u32 len)
+{
+ struct xocl_mb *mb;
+ char *binary;
+
+ if (len > MAX_IMAGE_LEN)
+ return -EINVAL;
+
+ mb = platform_get_drvdata(pdev);
+ if (!mb)
+ return -EINVAL;
+
+ binary = mb->mgmt_binary;
+ mb->mgmt_binary = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
+ if (!mb->mgmt_binary)
+ return -ENOMEM;
+
+ if (binary)
+ devm_kfree(&pdev->dev, binary);
+ memcpy(mb->mgmt_binary, image, len);
+ mb->mgmt_binary_length = len;
+
+ return 0;
+}
+
+static int load_sche_image(struct platform_device *pdev, const char *image,
+ u32 len)
+{
+ struct xocl_mb *mb;
+ char *binary = NULL;
+
+ if (len > MAX_IMAGE_LEN)
+ return -EINVAL;
+
+ mb = platform_get_drvdata(pdev);
+ if (!mb)
+ return -EINVAL;
+
+ binary = mb->sche_binary;
+ mb->sche_binary = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
+ if (!mb->sche_binary)
+ return -ENOMEM;
+
+ if (binary)
+ devm_kfree(&pdev->dev, binary);
+ memcpy(mb->sche_binary, image, len);
+ mb->sche_binary_length = len;
+
+ return 0;
+}
+
+//Have a function stub but don't actually do anything when this is called
+static int mb_ignore(struct platform_device *pdev)
+{
+ return 0;
+}
+
+static struct xocl_mb_funcs mb_ops = {
+ .load_mgmt_image = load_mgmt_image,
+ .load_sche_image = load_sche_image,
+ .reset = mb_reset,
+ .stop = mb_ignore,
+};
+
+
+
+static int mb_remove(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+ int i;
+
+ mb = platform_get_drvdata(pdev);
+ if (!mb)
+ return 0;
+
+ if (mb->mgmt_binary)
+ devm_kfree(&pdev->dev, mb->mgmt_binary);
+ if (mb->sche_binary)
+ devm_kfree(&pdev->dev, mb->sche_binary);
+
+ /*
+ * It is more secure that MB keeps running even driver is unloaded.
+ * Even user unload our driver and use their own stuff, MB will still
+ * be able to monitor the board unless user stops it explicitly
+ */
+ mb_stop(mb);
+
+ mgmt_sysfs_destroy_mb(pdev);
+
+ for (i = 0; i < NUM_IOADDR; i++) {
+ if (mb->base_addrs[i])
+ iounmap(mb->base_addrs[i]);
+ }
+
+ mutex_destroy(&mb->mb_lock);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, mb);
+
+ return 0;
+}
+
+static int mb_probe(struct platform_device *pdev)
+{
+ struct xocl_mb *mb;
+ struct resource *res;
+ void *xdev_hdl;
+ int i, err;
+
+ mb = devm_kzalloc(&pdev->dev, sizeof(*mb), GFP_KERNEL);
+ if (!mb) {
+ xocl_err(&pdev->dev, "out of memory");
+ return -ENOMEM;
+ }
+
+ mb->pdev = pdev;
+ platform_set_drvdata(pdev, mb);
+
+ xdev_hdl = xocl_get_xdev(pdev);
+ if (xocl_mb_mgmt_on(xdev_hdl) || xocl_mb_sched_on(xdev_hdl)) {
+ xocl_info(&pdev->dev, "Microblaze is supported.");
+ mb->enabled = true;
+ } else {
+ xocl_info(&pdev->dev, "Microblaze is not supported.");
+ devm_kfree(&pdev->dev, mb);
+ platform_set_drvdata(pdev, NULL);
+ return 0;
+ }
+
+ for (i = 0; i < NUM_IOADDR; i++) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+ xocl_info(&pdev->dev, "IO start: 0x%llx, end: 0x%llx",
+ res->start, res->end);
+ mb->base_addrs[i] =
+ ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!mb->base_addrs[i]) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+ }
+
+ err = mgmt_sysfs_create_mb(pdev);
+ if (err) {
+ xocl_err(&pdev->dev, "Create sysfs failed, err %d", err);
+ goto failed;
+ }
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_MB, &mb_ops);
+
+ mutex_init(&mb->mb_lock);
+
+ return 0;
+
+failed:
+ mb_remove(pdev);
+ return err;
+}
+
+struct platform_device_id mb_id_table[] = {
+ { XOCL_MB, 0 },
+ { },
+};
+
+static struct platform_driver mb_driver = {
+ .probe = mb_probe,
+ .remove = mb_remove,
+ .driver = {
+ .name = "xocl_mb",
+ },
+ .id_table = mb_id_table,
+};
+
+int __init xocl_init_mb(void)
+{
+ return platform_driver_register(&mb_driver);
+}
+
+void xocl_fini_mb(void)
+{
+ platform_driver_unregister(&mb_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/mig.c b/drivers/gpu/drm/xocl/subdev/mig.c
new file mode 100644
index 000000000000..5a574f7af796
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/mig.c
@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2018-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: Chien-Wei Lan <chienwei@xilinx.com>
+ *
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+/* Registers are defined in pg150-ultrascale-memory-ip.pdf:
+ * AXI4-Lite Slave Control/Status Register Map
+ */
+
+#define MIG_DEBUG
+#define MIG_DEV2MIG(dev) \
+ ((struct xocl_mig *)platform_get_drvdata(to_platform_device(dev)))
+#define MIG_DEV2BASE(dev) (MIG_DEV2MIG(dev)->base)
+
+#define ECC_STATUS 0x0
+#define ECC_ON_OFF 0x8
+#define CE_CNT 0xC
+#define CE_ADDR_LO 0x1C0
+#define CE_ADDR_HI 0x1C4
+#define UE_ADDR_LO 0x2C0
+#define UE_ADDR_HI 0x2C4
+#define INJ_FAULT_REG 0x300
+
+struct xocl_mig {
+ void __iomem *base;
+ struct device *mig_dev;
+};
+
+static ssize_t ecc_ue_ffa_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ uint64_t val = ioread32(MIG_DEV2BASE(dev) + UE_ADDR_HI);
+
+ val <<= 32;
+ val |= ioread32(MIG_DEV2BASE(dev) + UE_ADDR_LO);
+ return sprintf(buf, "0x%llx\n", val);
+}
+static DEVICE_ATTR_RO(ecc_ue_ffa);
+
+
+static ssize_t ecc_ce_ffa_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ uint64_t val = ioread32(MIG_DEV2BASE(dev) + CE_ADDR_HI);
+
+ val <<= 32;
+ val |= ioread32(MIG_DEV2BASE(dev) + CE_ADDR_LO);
+ return sprintf(buf, "0x%llx\n", val);
+}
+static DEVICE_ATTR_RO(ecc_ce_ffa);
+
+
+static ssize_t ecc_ce_cnt_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%u\n", ioread32(MIG_DEV2BASE(dev) + CE_CNT));
+}
+static DEVICE_ATTR_RO(ecc_ce_cnt);
+
+
+static ssize_t ecc_status_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%u\n", ioread32(MIG_DEV2BASE(dev) + ECC_STATUS));
+}
+static DEVICE_ATTR_RO(ecc_status);
+
+
+static ssize_t ecc_reset_store(struct device *dev, struct device_attribute *da,
+ const char *buf, size_t count)
+{
+ iowrite32(0x3, MIG_DEV2BASE(dev) + ECC_STATUS);
+ iowrite32(0, MIG_DEV2BASE(dev) + CE_CNT);
+ return count;
+}
+static DEVICE_ATTR_WO(ecc_reset);
+
+
+static ssize_t ecc_enabled_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%u\n", ioread32(MIG_DEV2BASE(dev) + ECC_ON_OFF));
+}
+static ssize_t ecc_enabled_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ uint32_t val;
+
+ if (sscanf(buf, "%d", &val) != 1 || val > 1) {
+ xocl_err(&to_platform_device(dev)->dev,
+ "usage: echo [0|1] > ecc_enabled");
+ return -EINVAL;
+ }
+
+ iowrite32(val, MIG_DEV2BASE(dev) + ECC_ON_OFF);
+ return count;
+}
+static DEVICE_ATTR_RW(ecc_enabled);
+
+
+#ifdef MIG_DEBUG
+static ssize_t ecc_inject_store(struct device *dev, struct device_attribute *da,
+ const char *buf, size_t count)
+{
+ iowrite32(1, MIG_DEV2BASE(dev) + INJ_FAULT_REG);
+ return count;
+}
+static DEVICE_ATTR_WO(ecc_inject);
+#endif
+
+
+/* Standard sysfs entry for all dynamic subdevices. */
+static ssize_t name_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", XOCL_GET_SUBDEV_PRIV(dev));
+}
+static DEVICE_ATTR_RO(name);
+
+
+static struct attribute *mig_attributes[] = {
+ &dev_attr_name.attr,
+ &dev_attr_ecc_enabled.attr,
+ &dev_attr_ecc_status.attr,
+ &dev_attr_ecc_ce_cnt.attr,
+ &dev_attr_ecc_ce_ffa.attr,
+ &dev_attr_ecc_ue_ffa.attr,
+ &dev_attr_ecc_reset.attr,
+#ifdef MIG_DEBUG
+ &dev_attr_ecc_inject.attr,
+#endif
+ NULL
+};
+
+static const struct attribute_group mig_attrgroup = {
+ .attrs = mig_attributes,
+};
+
+static void mgmt_sysfs_destroy_mig(struct platform_device *pdev)
+{
+ struct xocl_mig *mig;
+
+ mig = platform_get_drvdata(pdev);
+ sysfs_remove_group(&pdev->dev.kobj, &mig_attrgroup);
+}
+
+static int mgmt_sysfs_create_mig(struct platform_device *pdev)
+{
+ struct xocl_mig *mig;
+ int err;
+
+ mig = platform_get_drvdata(pdev);
+ err = sysfs_create_group(&pdev->dev.kobj, &mig_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create pw group failed: 0x%x", err);
+ return err;
+ }
+
+ return 0;
+}
+
+static int mig_probe(struct platform_device *pdev)
+{
+ struct xocl_mig *mig;
+ struct resource *res;
+ int err;
+
+ mig = devm_kzalloc(&pdev->dev, sizeof(*mig), GFP_KERNEL);
+ if (!mig)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ xocl_err(&pdev->dev, "resource is NULL");
+ return -EINVAL;
+ }
+
+ xocl_info(&pdev->dev, "MIG name: %s, IO start: 0x%llx, end: 0x%llx",
+ XOCL_GET_SUBDEV_PRIV(&pdev->dev), res->start, res->end);
+
+ mig->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!mig->base) {
+ xocl_err(&pdev->dev, "Map iomem failed");
+ return -EIO;
+ }
+
+ platform_set_drvdata(pdev, mig);
+
+ err = mgmt_sysfs_create_mig(pdev);
+ if (err) {
+ platform_set_drvdata(pdev, NULL);
+ iounmap(mig->base);
+ return err;
+ }
+
+ return 0;
+}
+
+
+static int mig_remove(struct platform_device *pdev)
+{
+ struct xocl_mig *mig;
+
+ mig = platform_get_drvdata(pdev);
+ if (!mig) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ xocl_info(&pdev->dev, "MIG name: %s", XOCL_GET_SUBDEV_PRIV(&pdev->dev));
+
+ mgmt_sysfs_destroy_mig(pdev);
+
+ if (mig->base)
+ iounmap(mig->base);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, mig);
+
+ return 0;
+}
+
+struct platform_device_id mig_id_table[] = {
+ { XOCL_MIG, 0 },
+ { },
+};
+
+static struct platform_driver mig_driver = {
+ .probe = mig_probe,
+ .remove = mig_remove,
+ .driver = {
+ .name = "xocl_mig",
+ },
+ .id_table = mig_id_table,
+};
+
+int __init xocl_init_mig(void)
+{
+ return platform_driver_register(&mig_driver);
+}
+
+void xocl_fini_mig(void)
+{
+ platform_driver_unregister(&mig_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/sysmon.c b/drivers/gpu/drm/xocl/subdev/sysmon.c
new file mode 100644
index 000000000000..bb5c84485344
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/sysmon.c
@@ -0,0 +1,385 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+#define TEMP 0x400 // TEMPOERATURE REGISTER ADDRESS
+#define VCCINT 0x404 // VCCINT REGISTER OFFSET
+#define VCCAUX 0x408 // VCCAUX REGISTER OFFSET
+#define VCCBRAM 0x418 // VCCBRAM REGISTER OFFSET
+#define TEMP_MAX 0x480
+#define VCCINT_MAX 0x484
+#define VCCAUX_MAX 0x488
+#define VCCBRAM_MAX 0x48c
+#define TEMP_MIN 0x490
+#define VCCINT_MIN 0x494
+#define VCCAUX_MIN 0x498
+#define VCCBRAM_MIN 0x49c
+
+#define SYSMON_TO_MILLDEGREE(val) \
+ (((int64_t)(val) * 501374 >> 16) - 273678)
+#define SYSMON_TO_MILLVOLT(val) \
+ ((val) * 1000 * 3 >> 16)
+
+#define READ_REG32(sysmon, off) \
+ XOCL_READ_REG32(sysmon->base + off)
+#define WRITE_REG32(sysmon, val, off) \
+ XOCL_WRITE_REG32(val, sysmon->base + off)
+
+struct xocl_sysmon {
+ void __iomem *base;
+ struct device *hwmon_dev;
+};
+
+static int get_prop(struct platform_device *pdev, u32 prop, void *val)
+{
+ struct xocl_sysmon *sysmon;
+ u32 tmp;
+
+ sysmon = platform_get_drvdata(pdev);
+ BUG_ON(!sysmon);
+
+ switch (prop) {
+ case XOCL_SYSMON_PROP_TEMP:
+ tmp = READ_REG32(sysmon, TEMP);
+ *(u32 *)val = SYSMON_TO_MILLDEGREE(tmp)/1000;
+ break;
+ case XOCL_SYSMON_PROP_TEMP_MAX:
+ tmp = READ_REG32(sysmon, TEMP_MAX);
+ *(u32 *)val = SYSMON_TO_MILLDEGREE(tmp);
+ break;
+ case XOCL_SYSMON_PROP_TEMP_MIN:
+ tmp = READ_REG32(sysmon, TEMP_MIN);
+ *(u32 *)val = SYSMON_TO_MILLDEGREE(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_INT:
+ tmp = READ_REG32(sysmon, VCCINT);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_INT_MAX:
+ tmp = READ_REG32(sysmon, VCCINT_MAX);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_INT_MIN:
+ tmp = READ_REG32(sysmon, VCCINT_MIN);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_AUX:
+ tmp = READ_REG32(sysmon, VCCAUX);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_AUX_MAX:
+ tmp = READ_REG32(sysmon, VCCAUX_MAX);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_AUX_MIN:
+ tmp = READ_REG32(sysmon, VCCAUX_MIN);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_BRAM:
+ tmp = READ_REG32(sysmon, VCCBRAM);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_BRAM_MAX:
+ tmp = READ_REG32(sysmon, VCCBRAM_MAX);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ case XOCL_SYSMON_PROP_VCC_BRAM_MIN:
+ tmp = READ_REG32(sysmon, VCCBRAM_MIN);
+ *(u32 *)val = SYSMON_TO_MILLVOLT(tmp);
+ break;
+ default:
+ xocl_err(&pdev->dev, "Invalid prop");
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static struct xocl_sysmon_funcs sysmon_ops = {
+ .get_prop = get_prop,
+};
+
+static ssize_t show_sysmon(struct platform_device *pdev, u32 prop, char *buf)
+{
+ u32 val;
+
+ (void) get_prop(pdev, prop, &val);
+ return sprintf(buf, "%u\n", val);
+}
+
+/* sysfs support */
+static ssize_t show_hwmon(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+ struct platform_device *pdev = dev_get_drvdata(dev);
+
+ return show_sysmon(pdev, attr->index, buf);
+}
+
+static ssize_t show_name(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", XCLMGMT_SYSMON_HWMON_NAME);
+}
+
+static SENSOR_DEVICE_ATTR(temp1_input, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_TEMP);
+static SENSOR_DEVICE_ATTR(temp1_highest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_TEMP_MAX);
+static SENSOR_DEVICE_ATTR(temp1_lowest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_TEMP_MIN);
+
+static SENSOR_DEVICE_ATTR(in0_input, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_INT);
+static SENSOR_DEVICE_ATTR(in0_highest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_INT_MAX);
+static SENSOR_DEVICE_ATTR(in0_lowest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_INT_MIN);
+
+static SENSOR_DEVICE_ATTR(in1_input, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_AUX);
+static SENSOR_DEVICE_ATTR(in1_highest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_AUX_MAX);
+static SENSOR_DEVICE_ATTR(in1_lowest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_AUX_MIN);
+
+static SENSOR_DEVICE_ATTR(in2_input, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_BRAM);
+static SENSOR_DEVICE_ATTR(in2_highest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_BRAM_MAX);
+static SENSOR_DEVICE_ATTR(in2_lowest, 0444, show_hwmon, NULL,
+ XOCL_SYSMON_PROP_VCC_BRAM_MIN);
+
+static struct attribute *hwmon_sysmon_attributes[] = {
+ &sensor_dev_attr_temp1_input.dev_attr.attr,
+ &sensor_dev_attr_temp1_highest.dev_attr.attr,
+ &sensor_dev_attr_temp1_lowest.dev_attr.attr,
+ &sensor_dev_attr_in0_input.dev_attr.attr,
+ &sensor_dev_attr_in0_highest.dev_attr.attr,
+ &sensor_dev_attr_in0_lowest.dev_attr.attr,
+ &sensor_dev_attr_in1_input.dev_attr.attr,
+ &sensor_dev_attr_in1_highest.dev_attr.attr,
+ &sensor_dev_attr_in1_lowest.dev_attr.attr,
+ &sensor_dev_attr_in2_input.dev_attr.attr,
+ &sensor_dev_attr_in2_highest.dev_attr.attr,
+ &sensor_dev_attr_in2_lowest.dev_attr.attr,
+ NULL
+};
+
+static const struct attribute_group hwmon_sysmon_attrgroup = {
+ .attrs = hwmon_sysmon_attributes,
+};
+
+static struct sensor_device_attribute sysmon_name_attr =
+ SENSOR_ATTR(name, 0444, show_name, NULL, 0);
+
+static ssize_t temp_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return show_sysmon(to_platform_device(dev), XOCL_SYSMON_PROP_TEMP, buf);
+}
+static DEVICE_ATTR_RO(temp);
+
+static ssize_t vcc_int_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return show_sysmon(to_platform_device(dev), XOCL_SYSMON_PROP_VCC_INT, buf);
+}
+static DEVICE_ATTR_RO(vcc_int);
+
+static ssize_t vcc_aux_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return show_sysmon(to_platform_device(dev), XOCL_SYSMON_PROP_VCC_AUX, buf);
+}
+static DEVICE_ATTR_RO(vcc_aux);
+
+static ssize_t vcc_bram_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return show_sysmon(to_platform_device(dev), XOCL_SYSMON_PROP_VCC_BRAM, buf);
+}
+static DEVICE_ATTR_RO(vcc_bram);
+
+static struct attribute *sysmon_attributes[] = {
+ &dev_attr_temp.attr,
+ &dev_attr_vcc_int.attr,
+ &dev_attr_vcc_aux.attr,
+ &dev_attr_vcc_bram.attr,
+ NULL,
+};
+
+static const struct attribute_group sysmon_attrgroup = {
+ .attrs = sysmon_attributes,
+};
+
+static void mgmt_sysfs_destroy_sysmon(struct platform_device *pdev)
+{
+ struct xocl_sysmon *sysmon;
+
+ sysmon = platform_get_drvdata(pdev);
+
+ device_remove_file(sysmon->hwmon_dev, &sysmon_name_attr.dev_attr);
+ sysfs_remove_group(&sysmon->hwmon_dev->kobj, &hwmon_sysmon_attrgroup);
+ hwmon_device_unregister(sysmon->hwmon_dev);
+ sysmon->hwmon_dev = NULL;
+
+ sysfs_remove_group(&pdev->dev.kobj, &sysmon_attrgroup);
+}
+
+static int mgmt_sysfs_create_sysmon(struct platform_device *pdev)
+{
+ struct xocl_sysmon *sysmon;
+ struct xocl_dev_core *core;
+ int err;
+
+ sysmon = platform_get_drvdata(pdev);
+ core = XDEV(xocl_get_xdev(pdev));
+
+ sysmon->hwmon_dev = hwmon_device_register(&core->pdev->dev);
+ if (IS_ERR(sysmon->hwmon_dev)) {
+ err = PTR_ERR(sysmon->hwmon_dev);
+ xocl_err(&pdev->dev, "register sysmon hwmon failed: 0x%x", err);
+ goto hwmon_reg_failed;
+ }
+
+ dev_set_drvdata(sysmon->hwmon_dev, pdev);
+ err = device_create_file(sysmon->hwmon_dev,
+ &sysmon_name_attr.dev_attr);
+ if (err) {
+ xocl_err(&pdev->dev, "create attr name failed: 0x%x", err);
+ goto create_name_failed;
+ }
+
+ err = sysfs_create_group(&sysmon->hwmon_dev->kobj,
+ &hwmon_sysmon_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create hwmon group failed: 0x%x", err);
+ goto create_hwmon_failed;
+ }
+
+ err = sysfs_create_group(&pdev->dev.kobj, &sysmon_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create sysmon group failed: 0x%x", err);
+ goto create_sysmon_failed;
+ }
+
+ return 0;
+
+create_sysmon_failed:
+ sysfs_remove_group(&sysmon->hwmon_dev->kobj, &hwmon_sysmon_attrgroup);
+create_hwmon_failed:
+ device_remove_file(sysmon->hwmon_dev, &sysmon_name_attr.dev_attr);
+create_name_failed:
+ hwmon_device_unregister(sysmon->hwmon_dev);
+ sysmon->hwmon_dev = NULL;
+hwmon_reg_failed:
+ return err;
+}
+
+static int sysmon_probe(struct platform_device *pdev)
+{
+ struct xocl_sysmon *sysmon;
+ struct resource *res;
+ int err;
+
+ sysmon = devm_kzalloc(&pdev->dev, sizeof(*sysmon), GFP_KERNEL);
+ if (!sysmon)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ xocl_err(&pdev->dev, "resource is NULL");
+ return -EINVAL;
+ }
+ xocl_info(&pdev->dev, "IO start: 0x%llx, end: 0x%llx",
+ res->start, res->end);
+ sysmon->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!sysmon->base) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+
+ platform_set_drvdata(pdev, sysmon);
+
+ err = mgmt_sysfs_create_sysmon(pdev);
+ if (err)
+ goto create_sysmon_failed;
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_SYSMON, &sysmon_ops);
+
+ return 0;
+
+create_sysmon_failed:
+ platform_set_drvdata(pdev, NULL);
+failed:
+ return err;
+}
+
+
+static int sysmon_remove(struct platform_device *pdev)
+{
+ struct xocl_sysmon *sysmon;
+
+ sysmon = platform_get_drvdata(pdev);
+ if (!sysmon) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ mgmt_sysfs_destroy_sysmon(pdev);
+
+ if (sysmon->base)
+ iounmap(sysmon->base);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, sysmon);
+
+ return 0;
+}
+
+struct platform_device_id sysmon_id_table[] = {
+ { XOCL_SYSMON, 0 },
+ { },
+};
+
+static struct platform_driver sysmon_driver = {
+ .probe = sysmon_probe,
+ .remove = sysmon_remove,
+ .driver = {
+ .name = "xocl_sysmon",
+ },
+ .id_table = sysmon_id_table,
+};
+
+int __init xocl_init_sysmon(void)
+{
+ return platform_driver_register(&sysmon_driver);
+}
+
+void xocl_fini_sysmon(void)
+{
+ platform_driver_unregister(&sysmon_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/xdma.c b/drivers/gpu/drm/xocl/subdev/xdma.c
new file mode 100644
index 000000000000..647a69f29a84
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/xdma.c
@@ -0,0 +1,510 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ */
+
+/* XDMA version Memory Mapped DMA */
+
+#include <linux/version.h>
+#include <linux/eventfd.h>
+#include <drm/drmP.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_mm.h>
+#include "../xocl_drv.h"
+#include "../xocl_drm.h"
+#include "../lib/libxdma_api.h"
+
+#define XOCL_FILE_PAGE_OFFSET 0x100000
+#ifndef VM_RESERVED
+#define VM_RESERVED (VM_DONTEXPAND | VM_DONTDUMP)
+#endif
+
+struct xdma_irq {
+ struct eventfd_ctx *event_ctx;
+ bool in_use;
+ bool enabled;
+ irq_handler_t handler;
+ void *arg;
+};
+
+struct xocl_xdma {
+ void *dma_handle;
+ u32 max_user_intr;
+ u32 start_user_intr;
+ struct xdma_irq *user_msix_table;
+ struct mutex user_msix_table_lock;
+
+ struct xocl_drm *drm;
+ /* Number of bidirectional channels */
+ u32 channel;
+ /* Semaphore, one for each direction */
+ struct semaphore channel_sem[2];
+ /*
+ * Channel usage bitmasks, one for each direction
+ * bit 1 indicates channel is free, bit 0 indicates channel is free
+ */
+ unsigned long channel_bitmap[2];
+ unsigned long long *channel_usage[2];
+
+ struct mutex stat_lock;
+};
+
+static ssize_t xdma_migrate_bo(struct platform_device *pdev,
+ struct sg_table *sgt, u32 dir, u64 paddr, u32 channel, u64 len)
+{
+ struct xocl_xdma *xdma;
+ struct page *pg;
+ struct scatterlist *sg = sgt->sgl;
+ int nents = sgt->orig_nents;
+ pid_t pid = current->pid;
+ int i = 0;
+ ssize_t ret;
+ unsigned long long pgaddr;
+
+ xdma = platform_get_drvdata(pdev);
+ xocl_dbg(&pdev->dev, "TID %d, Channel:%d, Offset: 0x%llx, Dir: %d",
+ pid, channel, paddr, dir);
+ ret = xdma_xfer_submit(xdma->dma_handle, channel, dir,
+ paddr, sgt, false, 10000);
+ if (ret >= 0) {
+ xdma->channel_usage[dir][channel] += ret;
+ return ret;
+ }
+
+ xocl_err(&pdev->dev, "DMA failed, Dumping SG Page Table");
+ for (i = 0; i < nents; i++, sg = sg_next(sg)) {
+ if (!sg)
+ break;
+ pg = sg_page(sg);
+ if (!pg)
+ continue;
+ pgaddr = page_to_phys(pg);
+ xocl_err(&pdev->dev, "%i, 0x%llx\n", i, pgaddr);
+ }
+ return ret;
+}
+
+static int acquire_channel(struct platform_device *pdev, u32 dir)
+{
+ struct xocl_xdma *xdma;
+ int channel = 0;
+ int result = 0;
+
+ xdma = platform_get_drvdata(pdev);
+ if (down_interruptible(&xdma->channel_sem[dir])) {
+ channel = -ERESTARTSYS;
+ goto out;
+ }
+
+ for (channel = 0; channel < xdma->channel; channel++) {
+ result = test_and_clear_bit(channel,
+ &xdma->channel_bitmap[dir]);
+ if (result)
+ break;
+ }
+ if (!result) {
+ // How is this possible?
+ up(&xdma->channel_sem[dir]);
+ channel = -EIO;
+ }
+
+out:
+ return channel;
+}
+
+static void release_channel(struct platform_device *pdev, u32 dir, u32 channel)
+{
+ struct xocl_xdma *xdma;
+
+
+ xdma = platform_get_drvdata(pdev);
+ set_bit(channel, &xdma->channel_bitmap[dir]);
+ up(&xdma->channel_sem[dir]);
+}
+
+static u32 get_channel_count(struct platform_device *pdev)
+{
+ struct xocl_xdma *xdma;
+
+ xdma = platform_get_drvdata(pdev);
+ BUG_ON(!xdma);
+
+ return xdma->channel;
+}
+
+static void *get_drm_handle(struct platform_device *pdev)
+{
+ struct xocl_xdma *xdma;
+
+ xdma = platform_get_drvdata(pdev);
+
+ return xdma->drm;
+}
+
+static u64 get_channel_stat(struct platform_device *pdev, u32 channel,
+ u32 write)
+{
+ struct xocl_xdma *xdma;
+
+ xdma = platform_get_drvdata(pdev);
+ BUG_ON(!xdma);
+
+ return xdma->channel_usage[write][channel];
+}
+
+static int user_intr_config(struct platform_device *pdev, u32 intr, bool en)
+{
+ struct xocl_xdma *xdma;
+ const unsigned int mask = 1 << intr;
+ int ret;
+
+ xdma = platform_get_drvdata(pdev);
+
+ if (intr >= xdma->max_user_intr) {
+ xocl_err(&pdev->dev, "Invalid intr %d, user start %d, max %d",
+ intr, xdma->start_user_intr, xdma->max_user_intr);
+ return -EINVAL;
+ }
+
+ mutex_lock(&xdma->user_msix_table_lock);
+ if (xdma->user_msix_table[intr].enabled == en) {
+ ret = 0;
+ goto end;
+ }
+
+ ret = en ? xdma_user_isr_enable(xdma->dma_handle, mask) :
+ xdma_user_isr_disable(xdma->dma_handle, mask);
+ if (!ret)
+ xdma->user_msix_table[intr].enabled = en;
+end:
+ mutex_unlock(&xdma->user_msix_table_lock);
+
+ return ret;
+}
+
+static irqreturn_t xdma_isr(int irq, void *arg)
+{
+ struct xdma_irq *irq_entry = arg;
+ int ret = IRQ_HANDLED;
+
+ if (irq_entry->handler)
+ ret = irq_entry->handler(irq, irq_entry->arg);
+
+ if (!IS_ERR_OR_NULL(irq_entry->event_ctx))
+ eventfd_signal(irq_entry->event_ctx, 1);
+
+ return ret;
+}
+
+static int user_intr_unreg(struct platform_device *pdev, u32 intr)
+{
+ struct xocl_xdma *xdma;
+ const unsigned int mask = 1 << intr;
+ int ret;
+
+ xdma = platform_get_drvdata(pdev);
+
+ if (intr >= xdma->max_user_intr)
+ return -EINVAL;
+
+ mutex_lock(&xdma->user_msix_table_lock);
+ if (!xdma->user_msix_table[intr].in_use) {
+ ret = -EINVAL;
+ goto failed;
+ }
+ xdma->user_msix_table[intr].handler = NULL;
+ xdma->user_msix_table[intr].arg = NULL;
+
+ ret = xdma_user_isr_register(xdma->dma_handle, mask, NULL, NULL);
+ if (ret) {
+ xocl_err(&pdev->dev, "xdma unregister isr failed");
+ goto failed;
+ }
+
+ xdma->user_msix_table[intr].in_use = false;
+
+failed:
+ mutex_unlock(&xdma->user_msix_table_lock);
+ return ret;
+}
+
+static int user_intr_register(struct platform_device *pdev, u32 intr,
+ irq_handler_t handler, void *arg, int event_fd)
+{
+ struct xocl_xdma *xdma;
+ struct eventfd_ctx *trigger = ERR_PTR(-EINVAL);
+ const unsigned int mask = 1 << intr;
+ int ret;
+
+ xdma = platform_get_drvdata(pdev);
+
+ if (intr >= xdma->max_user_intr ||
+ (event_fd >= 0 && intr < xdma->start_user_intr)) {
+ xocl_err(&pdev->dev, "Invalid intr %d, user start %d, max %d",
+ intr, xdma->start_user_intr, xdma->max_user_intr);
+ return -EINVAL;
+ }
+
+ if (event_fd >= 0) {
+ trigger = eventfd_ctx_fdget(event_fd);
+ if (IS_ERR(trigger)) {
+ xocl_err(&pdev->dev, "get event ctx failed");
+ return -EFAULT;
+ }
+ }
+
+ mutex_lock(&xdma->user_msix_table_lock);
+ if (xdma->user_msix_table[intr].in_use) {
+ xocl_err(&pdev->dev, "IRQ %d is in use", intr);
+ ret = -EPERM;
+ goto failed;
+ }
+ xdma->user_msix_table[intr].event_ctx = trigger;
+ xdma->user_msix_table[intr].handler = handler;
+ xdma->user_msix_table[intr].arg = arg;
+
+ ret = xdma_user_isr_register(xdma->dma_handle, mask, xdma_isr,
+ &xdma->user_msix_table[intr]);
+ if (ret) {
+ xocl_err(&pdev->dev, "IRQ register failed");
+ xdma->user_msix_table[intr].handler = NULL;
+ xdma->user_msix_table[intr].arg = NULL;
+ xdma->user_msix_table[intr].event_ctx = NULL;
+ goto failed;
+ }
+
+ xdma->user_msix_table[intr].in_use = true;
+
+ mutex_unlock(&xdma->user_msix_table_lock);
+
+
+ return 0;
+
+failed:
+ mutex_unlock(&xdma->user_msix_table_lock);
+ if (!IS_ERR(trigger))
+ eventfd_ctx_put(trigger);
+
+ return ret;
+}
+
+static struct xocl_dma_funcs xdma_ops = {
+ .migrate_bo = xdma_migrate_bo,
+ .ac_chan = acquire_channel,
+ .rel_chan = release_channel,
+ .get_chan_count = get_channel_count,
+ .get_chan_stat = get_channel_stat,
+ .user_intr_register = user_intr_register,
+ .user_intr_config = user_intr_config,
+ .user_intr_unreg = user_intr_unreg,
+ .get_drm_handle = get_drm_handle,
+};
+
+static ssize_t channel_stat_raw_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ u32 i;
+ ssize_t nbytes = 0;
+ struct platform_device *pdev = to_platform_device(dev);
+ u32 chs = get_channel_count(pdev);
+
+ for (i = 0; i < chs; i++) {
+ nbytes += sprintf(buf + nbytes, "%llu %llu\n",
+ get_channel_stat(pdev, i, 0),
+ get_channel_stat(pdev, i, 1));
+ }
+ return nbytes;
+}
+static DEVICE_ATTR_RO(channel_stat_raw);
+
+static struct attribute *xdma_attrs[] = {
+ &dev_attr_channel_stat_raw.attr,
+ NULL,
+};
+
+static struct attribute_group xdma_attr_group = {
+ .attrs = xdma_attrs,
+};
+
+static int set_max_chan(struct platform_device *pdev,
+ struct xocl_xdma *xdma)
+{
+ xdma->channel_usage[0] = devm_kzalloc(&pdev->dev, sizeof(u64) *
+ xdma->channel, GFP_KERNEL);
+ xdma->channel_usage[1] = devm_kzalloc(&pdev->dev, sizeof(u64) *
+ xdma->channel, GFP_KERNEL);
+ if (!xdma->channel_usage[0] || !xdma->channel_usage[1]) {
+ xocl_err(&pdev->dev, "failed to alloc channel usage");
+ return -ENOMEM;
+ }
+
+ sema_init(&xdma->channel_sem[0], xdma->channel);
+ sema_init(&xdma->channel_sem[1], xdma->channel);
+
+ /* Initialize bit mask to represent individual channels */
+ xdma->channel_bitmap[0] = BIT(xdma->channel);
+ xdma->channel_bitmap[0]--;
+ xdma->channel_bitmap[1] = xdma->channel_bitmap[0];
+
+ return 0;
+}
+
+static int xdma_probe(struct platform_device *pdev)
+{
+ struct xocl_xdma *xdma = NULL;
+ int ret = 0;
+ xdev_handle_t xdev;
+
+ xdev = xocl_get_xdev(pdev);
+ BUG_ON(!xdev);
+
+ xdma = devm_kzalloc(&pdev->dev, sizeof(*xdma), GFP_KERNEL);
+ if (!xdma) {
+ ret = -ENOMEM;
+ goto failed;
+ }
+
+ xdma->dma_handle = xdma_device_open(XOCL_MODULE_NAME, XDEV(xdev)->pdev,
+ &xdma->max_user_intr,
+ &xdma->channel, &xdma->channel);
+ if (xdma->dma_handle == NULL) {
+ xocl_err(&pdev->dev, "XDMA Device Open failed");
+ ret = -EIO;
+ goto failed;
+ }
+
+ xdma->user_msix_table = devm_kzalloc(&pdev->dev,
+ xdma->max_user_intr *
+ sizeof(struct xdma_irq), GFP_KERNEL);
+ if (!xdma->user_msix_table) {
+ xocl_err(&pdev->dev, "alloc user_msix_table failed");
+ ret = -ENOMEM;
+ goto failed;
+ }
+
+ ret = set_max_chan(pdev, xdma);
+ if (ret) {
+ xocl_err(&pdev->dev, "Set max channel failed");
+ goto failed;
+ }
+
+ xdma->drm = xocl_drm_init(xdev);
+ if (!xdma->drm) {
+ ret = -EFAULT;
+ xocl_err(&pdev->dev, "failed to init drm mm");
+ goto failed;
+ }
+
+ ret = sysfs_create_group(&pdev->dev.kobj, &xdma_attr_group);
+ if (ret) {
+ xocl_err(&pdev->dev, "create attrs failed: %d", ret);
+ goto failed;
+ }
+
+ mutex_init(&xdma->stat_lock);
+ mutex_init(&xdma->user_msix_table_lock);
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_DMA, &xdma_ops);
+ platform_set_drvdata(pdev, xdma);
+
+ return 0;
+
+failed:
+ if (xdma) {
+ if (xdma->drm)
+ xocl_drm_fini(xdma->drm);
+ if (xdma->dma_handle)
+ xdma_device_close(XDEV(xdev)->pdev, xdma->dma_handle);
+ if (xdma->channel_usage[0])
+ devm_kfree(&pdev->dev, xdma->channel_usage[0]);
+ if (xdma->channel_usage[1])
+ devm_kfree(&pdev->dev, xdma->channel_usage[1]);
+ if (xdma->user_msix_table)
+ devm_kfree(&pdev->dev, xdma->user_msix_table);
+
+ devm_kfree(&pdev->dev, xdma);
+ }
+
+ platform_set_drvdata(pdev, NULL);
+
+ return ret;
+}
+
+static int xdma_remove(struct platform_device *pdev)
+{
+ struct xocl_xdma *xdma = platform_get_drvdata(pdev);
+ xdev_handle_t xdev;
+ struct xdma_irq *irq_entry;
+ int i;
+
+ if (!xdma) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+
+ xdev = xocl_get_xdev(pdev);
+ BUG_ON(!xdev);
+
+ sysfs_remove_group(&pdev->dev.kobj, &xdma_attr_group);
+
+ if (xdma->drm)
+ xocl_drm_fini(xdma->drm);
+ if (xdma->dma_handle)
+ xdma_device_close(XDEV(xdev)->pdev, xdma->dma_handle);
+
+ for (i = 0; i < xdma->max_user_intr; i++) {
+ irq_entry = &xdma->user_msix_table[i];
+ if (irq_entry->in_use) {
+ if (irq_entry->enabled) {
+ xocl_err(&pdev->dev,
+ "ERROR: Interrupt %d is still on", i);
+ }
+ if (!IS_ERR_OR_NULL(irq_entry->event_ctx))
+ eventfd_ctx_put(irq_entry->event_ctx);
+ }
+ }
+
+ if (xdma->channel_usage[0])
+ devm_kfree(&pdev->dev, xdma->channel_usage[0]);
+ if (xdma->channel_usage[1])
+ devm_kfree(&pdev->dev, xdma->channel_usage[1]);
+
+ mutex_destroy(&xdma->stat_lock);
+ mutex_destroy(&xdma->user_msix_table_lock);
+
+ devm_kfree(&pdev->dev, xdma->user_msix_table);
+ platform_set_drvdata(pdev, NULL);
+
+ devm_kfree(&pdev->dev, xdma);
+
+ return 0;
+}
+
+static struct platform_device_id xdma_id_table[] = {
+ { XOCL_XDMA, 0 },
+ { },
+};
+
+static struct platform_driver xdma_driver = {
+ .probe = xdma_probe,
+ .remove = xdma_remove,
+ .driver = {
+ .name = "xocl_xdma",
+ },
+ .id_table = xdma_id_table,
+};
+
+int __init xocl_init_xdma(void)
+{
+ return platform_driver_register(&xdma_driver);
+}
+
+void xocl_fini_xdma(void)
+{
+ return platform_driver_unregister(&xdma_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/xmc.c b/drivers/gpu/drm/xocl/subdev/xmc.c
new file mode 100644
index 000000000000..d9d620ac09b9
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/xmc.c
@@ -0,0 +1,1480 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2018 Xilinx, Inc. All rights reserved.
+ *
+ * Authors: chienwei@xilinx.com
+ *
+ */
+
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/vmalloc.h>
+#include <linux/string.h>
+#include "../ert.h"
+#include "../xocl_drv.h"
+#include <drm/xmgmt_drm.h>
+
+#define MAX_XMC_RETRY 150 //Retry is set to 15s for XMC
+#define MAX_ERT_RETRY 10 //Retry is set to 1s for ERT
+#define RETRY_INTERVAL 100 //100ms
+
+#define MAX_IMAGE_LEN 0x20000
+
+#define XMC_MAGIC_REG 0x0
+#define XMC_VERSION_REG 0x4
+#define XMC_STATUS_REG 0x8
+#define XMC_ERROR_REG 0xC
+#define XMC_FEATURE_REG 0x10
+#define XMC_SENSOR_REG 0x14
+#define XMC_CONTROL_REG 0x18
+#define XMC_STOP_CONFIRM_REG 0x1C
+#define XMC_12V_PEX_REG 0x20
+#define XMC_3V3_PEX_REG 0x2C
+#define XMC_3V3_AUX_REG 0x38
+#define XMC_12V_AUX_REG 0x44
+#define XMC_DDR4_VPP_BTM_REG 0x50
+#define XMC_SYS_5V5_REG 0x5C
+#define XMC_VCC1V2_TOP_REG 0x68
+#define XMC_VCC1V8_REG 0x74
+#define XMC_VCC0V85_REG 0x80
+#define XMC_DDR4_VPP_TOP_REG 0x8C
+#define XMC_MGT0V9AVCC_REG 0x98
+#define XMC_12V_SW_REG 0xA4
+#define XMC_MGTAVTT_REG 0xB0
+#define XMC_VCC1V2_BTM_REG 0xBC
+#define XMC_12V_PEX_I_IN_REG 0xC8
+#define XMC_12V_AUX_I_IN_REG 0xD4
+#define XMC_VCCINT_V_REG 0xE0
+#define XMC_VCCINT_I_REG 0xEC
+#define XMC_FPGA_TEMP 0xF8
+#define XMC_FAN_TEMP_REG 0x104
+#define XMC_DIMM_TEMP0_REG 0x110
+#define XMC_DIMM_TEMP1_REG 0x11C
+#define XMC_DIMM_TEMP2_REG 0x128
+#define XMC_DIMM_TEMP3_REG 0x134
+#define XMC_FAN_SPEED_REG 0x164
+#define XMC_SE98_TEMP0_REG 0x140
+#define XMC_SE98_TEMP1_REG 0x14C
+#define XMC_SE98_TEMP2_REG 0x158
+#define XMC_CAGE_TEMP0_REG 0x170
+#define XMC_CAGE_TEMP1_REG 0x17C
+#define XMC_CAGE_TEMP2_REG 0x188
+#define XMC_CAGE_TEMP3_REG 0x194
+#define XMC_SNSR_CHKSUM_REG 0x1A4
+#define XMC_SNSR_FLAGS_REG 0x1A8
+#define XMC_HOST_MSG_OFFSET_REG 0x300
+#define XMC_HOST_MSG_ERROR_REG 0x304
+#define XMC_HOST_MSG_HEADER_REG 0x308
+
+
+#define VALID_ID 0x74736574
+
+#define GPIO_RESET 0x0
+#define GPIO_ENABLED 0x1
+
+#define SELF_JUMP(ins) (((ins) & 0xfc00ffff) == 0xb8000000)
+#define XMC_PRIVILEGED(xmc) ((xmc)->base_addrs[0] != NULL)
+
+enum ctl_mask {
+ CTL_MASK_CLEAR_POW = 0x1,
+ CTL_MASK_CLEAR_ERR = 0x2,
+ CTL_MASK_PAUSE = 0x4,
+ CTL_MASK_STOP = 0x8,
+};
+
+enum status_mask {
+ STATUS_MASK_INIT_DONE = 0x1,
+ STATUS_MASK_STOPPED = 0x2,
+ STATUS_MASK_PAUSE = 0x4,
+};
+
+enum cap_mask {
+ CAP_MASK_PM = 0x1,
+};
+
+enum {
+ XMC_STATE_UNKNOWN,
+ XMC_STATE_ENABLED,
+ XMC_STATE_RESET,
+ XMC_STATE_STOPPED,
+ XMC_STATE_ERROR
+};
+
+enum {
+ IO_REG,
+ IO_GPIO,
+ IO_IMAGE_MGMT,
+ IO_IMAGE_SCHED,
+ IO_CQ,
+ NUM_IOADDR
+};
+
+enum {
+ VOLTAGE_MAX,
+ VOLTAGE_AVG,
+ VOLTAGE_INS,
+};
+
+#define READ_REG32(xmc, off) \
+ XOCL_READ_REG32(xmc->base_addrs[IO_REG] + off)
+#define WRITE_REG32(xmc, val, off) \
+ XOCL_WRITE_REG32(val, xmc->base_addrs[IO_REG] + off)
+
+#define READ_GPIO(xmc, off) \
+ XOCL_READ_REG32(xmc->base_addrs[IO_GPIO] + off)
+#define WRITE_GPIO(xmc, val, off) \
+ XOCL_WRITE_REG32(val, xmc->base_addrs[IO_GPIO] + off)
+
+#define READ_IMAGE_MGMT(xmc, off) \
+ XOCL_READ_REG32(xmc->base_addrs[IO_IMAGE_MGMT] + off)
+
+#define READ_IMAGE_SCHED(xmc, off) \
+ XOCL_READ_REG32(xmc->base_addrs[IO_IMAGE_SCHED] + off)
+
+#define COPY_MGMT(xmc, buf, len) \
+ xocl_memcpy_toio(xmc->base_addrs[IO_IMAGE_MGMT], buf, len)
+#define COPY_SCHE(xmc, buf, len) \
+ xocl_memcpy_toio(xmc->base_addrs[IO_IMAGE_SCHED], buf, len)
+
+struct xocl_xmc {
+ struct platform_device *pdev;
+ void __iomem *base_addrs[NUM_IOADDR];
+
+ struct device *hwmon_dev;
+ bool enabled;
+ u32 state;
+ u32 cap;
+ struct mutex xmc_lock;
+
+ char *sche_binary;
+ u32 sche_binary_length;
+ char *mgmt_binary;
+ u32 mgmt_binary_length;
+};
+
+
+static int load_xmc(struct xocl_xmc *xmc);
+static int stop_xmc(struct platform_device *pdev);
+
+static void xmc_read_from_peer(struct platform_device *pdev, enum data_kind kind, void *resp, size_t resplen)
+{
+ struct mailbox_subdev_peer subdev_peer = {0};
+ size_t data_len = sizeof(struct mailbox_subdev_peer);
+ struct mailbox_req *mb_req = NULL;
+ size_t reqlen = sizeof(struct mailbox_req) + data_len;
+
+ mb_req = vmalloc(reqlen);
+ if (!mb_req)
+ return;
+
+ mb_req->req = MAILBOX_REQ_PEER_DATA;
+
+ subdev_peer.kind = kind;
+ memcpy(mb_req->data, &subdev_peer, data_len);
+
+ (void) xocl_peer_request(XOCL_PL_DEV_TO_XDEV(pdev),
+ mb_req, reqlen, resp, &resplen, NULL, NULL);
+ vfree(mb_req);
+}
+
+/* sysfs support */
+static void safe_read32(struct xocl_xmc *xmc, u32 reg, u32 *val)
+{
+ mutex_lock(&xmc->xmc_lock);
+ if (xmc->enabled && xmc->state == XMC_STATE_ENABLED)
+ *val = READ_REG32(xmc, reg);
+ else
+ *val = 0;
+
+ mutex_unlock(&xmc->xmc_lock);
+}
+
+static void safe_write32(struct xocl_xmc *xmc, u32 reg, u32 val)
+{
+ mutex_lock(&xmc->xmc_lock);
+ if (xmc->enabled && xmc->state == XMC_STATE_ENABLED)
+ WRITE_REG32(xmc, val, reg);
+
+ mutex_unlock(&xmc->xmc_lock);
+}
+
+static void safe_read_from_peer(struct xocl_xmc *xmc, struct platform_device *pdev, enum data_kind kind, u32 *val)
+{
+ mutex_lock(&xmc->xmc_lock);
+ if (xmc->enabled)
+ xmc_read_from_peer(pdev, kind, val, sizeof(u32));
+ else
+ *val = 0;
+
+ mutex_unlock(&xmc->xmc_lock);
+}
+
+static int xmc_get_data(struct platform_device *pdev, enum data_kind kind)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(pdev);
+ int val;
+
+ if (XMC_PRIVILEGED(xmc)) {
+ switch (kind) {
+ case VOL_12V_PEX:
+ safe_read32(xmc, XMC_12V_PEX_REG + sizeof(u32)*VOLTAGE_INS, &val);
+ break;
+ default:
+ break;
+ }
+ }
+ return val;
+}
+
+static ssize_t xmc_12v_pex_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 pes_val;
+
+ if (XMC_PRIVILEGED(xmc))
+ safe_read32(xmc, XMC_12V_PEX_REG+sizeof(u32)*VOLTAGE_INS, &pes_val);
+ else
+ safe_read_from_peer(xmc, to_platform_device(dev), VOL_12V_PEX, &pes_val);
+
+ return sprintf(buf, "%d\n", pes_val);
+}
+static DEVICE_ATTR_RO(xmc_12v_pex_vol);
+
+static ssize_t xmc_12v_aux_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_12V_AUX_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_12v_aux_vol);
+
+static ssize_t xmc_12v_pex_curr_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 pes_val;
+
+ safe_read32(xmc, XMC_12V_PEX_I_IN_REG+sizeof(u32)*VOLTAGE_INS, &pes_val);
+
+ return sprintf(buf, "%d\n", pes_val);
+}
+static DEVICE_ATTR_RO(xmc_12v_pex_curr);
+
+static ssize_t xmc_12v_aux_curr_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_12V_AUX_I_IN_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_12v_aux_curr);
+
+static ssize_t xmc_3v3_pex_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_3V3_PEX_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_3v3_pex_vol);
+
+static ssize_t xmc_3v3_aux_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_3V3_AUX_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_3v3_aux_vol);
+
+static ssize_t xmc_ddr_vpp_btm_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DDR4_VPP_BTM_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_ddr_vpp_btm);
+
+static ssize_t xmc_sys_5v5_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SYS_5V5_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_sys_5v5);
+
+static ssize_t xmc_1v2_top_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCC1V2_TOP_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_1v2_top);
+
+static ssize_t xmc_1v8_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCC1V8_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_1v8);
+
+static ssize_t xmc_0v85_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCC0V85_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_0v85);
+
+static ssize_t xmc_ddr_vpp_top_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DDR4_VPP_TOP_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_ddr_vpp_top);
+
+
+static ssize_t xmc_mgt0v9avcc_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_MGT0V9AVCC_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_mgt0v9avcc);
+
+static ssize_t xmc_12v_sw_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_12V_SW_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_12v_sw);
+
+
+static ssize_t xmc_mgtavtt_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_MGTAVTT_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_mgtavtt);
+
+static ssize_t xmc_vcc1v2_btm_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCC1V2_BTM_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_vcc1v2_btm);
+
+static ssize_t xmc_vccint_vol_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCCINT_V_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_vccint_vol);
+
+static ssize_t xmc_vccint_curr_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_VCCINT_I_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_vccint_curr);
+
+static ssize_t xmc_se98_temp0_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SE98_TEMP0_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_se98_temp0);
+
+static ssize_t xmc_se98_temp1_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SE98_TEMP1_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_se98_temp1);
+
+static ssize_t xmc_se98_temp2_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SE98_TEMP2_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_se98_temp2);
+
+static ssize_t xmc_fpga_temp_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_FPGA_TEMP, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_fpga_temp);
+
+static ssize_t xmc_fan_temp_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_FAN_TEMP_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_fan_temp);
+
+static ssize_t xmc_fan_rpm_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_FAN_SPEED_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_fan_rpm);
+
+
+static ssize_t xmc_dimm_temp0_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DIMM_TEMP0_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_dimm_temp0);
+
+static ssize_t xmc_dimm_temp1_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DIMM_TEMP1_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_dimm_temp1);
+
+static ssize_t xmc_dimm_temp2_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DIMM_TEMP2_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_dimm_temp2);
+
+static ssize_t xmc_dimm_temp3_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_DIMM_TEMP3_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_dimm_temp3);
+
+
+static ssize_t xmc_cage_temp0_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_CAGE_TEMP0_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_cage_temp0);
+
+static ssize_t xmc_cage_temp1_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_CAGE_TEMP1_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_cage_temp1);
+
+static ssize_t xmc_cage_temp2_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_CAGE_TEMP2_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_cage_temp2);
+
+static ssize_t xmc_cage_temp3_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_CAGE_TEMP3_REG+sizeof(u32)*VOLTAGE_INS, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(xmc_cage_temp3);
+
+
+static ssize_t version_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_VERSION_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(version);
+
+static ssize_t sensor_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_SENSOR_REG, &val);
+
+ return sprintf(buf, "0x%04x\n", val);
+}
+static DEVICE_ATTR_RO(sensor);
+
+
+static ssize_t id_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_MAGIC_REG, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(id);
+
+static ssize_t status_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_STATUS_REG, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(status);
+
+static ssize_t error_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_ERROR_REG, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(error);
+
+static ssize_t capability_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_FEATURE_REG, &val);
+
+ return sprintf(buf, "%x\n", val);
+}
+static DEVICE_ATTR_RO(capability);
+
+static ssize_t power_checksum_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_SNSR_CHKSUM_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(power_checksum);
+
+static ssize_t pause_show(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ safe_read32(xmc, XMC_CONTROL_REG, &val);
+
+ return sprintf(buf, "%d\n", !!(val & CTL_MASK_PAUSE));
+}
+
+static ssize_t pause_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val > 1)
+ return -EINVAL;
+
+ val = val ? CTL_MASK_PAUSE : 0;
+ safe_write32(xmc, XMC_CONTROL_REG, val);
+
+ return count;
+}
+static DEVICE_ATTR_RW(pause);
+
+static ssize_t reset_store(struct device *dev,
+ struct device_attribute *da, const char *buf, size_t count)
+{
+ struct xocl_xmc *xmc = platform_get_drvdata(to_platform_device(dev));
+ u32 val;
+
+ if (kstrtou32(buf, 10, &val) == -EINVAL || val > 1)
+ return -EINVAL;
+
+ if (val)
+ load_xmc(xmc);
+
+ return count;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t power_flag_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_SNSR_FLAGS_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(power_flag);
+
+static ssize_t host_msg_offset_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_HOST_MSG_OFFSET_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(host_msg_offset);
+
+static ssize_t host_msg_error_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_HOST_MSG_ERROR_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(host_msg_error);
+
+static ssize_t host_msg_header_show(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_HOST_MSG_HEADER_REG, &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+static DEVICE_ATTR_RO(host_msg_header);
+
+
+
+static int get_temp_by_m_tag(struct xocl_xmc *xmc, char *m_tag)
+{
+
+ /**
+ * m_tag get from xclbin must follow this format
+ * DDR[0] or bank1
+ * we check the index in m_tag to decide which temperature
+ * to get from XMC IP base address
+ */
+ char *start = NULL, *left_parentness = NULL, *right_parentness = NULL;
+ long idx;
+ int ret = 0, digit_len = 0;
+ char temp[4];
+
+ if (!xmc)
+ return -ENODEV;
+
+
+ if (!strncmp(m_tag, "bank", 4)) {
+ start = m_tag;
+ // bank0, no left parentness
+ left_parentness = m_tag+3;
+ right_parentness = m_tag+strlen(m_tag)+1;
+ digit_len = right_parentness-(2+left_parentness);
+ } else if (!strncmp(m_tag, "DDR", 3)) {
+
+ start = m_tag;
+ left_parentness = strstr(m_tag, "[");
+ right_parentness = strstr(m_tag, "]");
+ digit_len = right_parentness-(1+left_parentness);
+ }
+
+ if (!left_parentness || !right_parentness)
+ return ret;
+
+ if (!strncmp(m_tag, "DDR", left_parentness-start) || !strncmp(m_tag, "bank", left_parentness-start)) {
+
+ strncpy(temp, left_parentness+1, digit_len);
+ //assumption, temperature won't higher than 3 digits, or the temp[digit_len] should be a null character
+ temp[digit_len] = '\0';
+ //convert to signed long, decimal base
+ if (kstrtol(temp, 10, &idx) == 0 && idx < 4 && idx >= 0)
+ safe_read32(xmc, XMC_DIMM_TEMP0_REG + (3*sizeof(int32_t)) * idx +
+ sizeof(u32)*VOLTAGE_INS, &ret);
+ else
+ ret = 0;
+ }
+
+ return ret;
+
+}
+
+static struct attribute *xmc_attrs[] = {
+ &dev_attr_version.attr,
+ &dev_attr_id.attr,
+ &dev_attr_status.attr,
+ &dev_attr_sensor.attr,
+ &dev_attr_error.attr,
+ &dev_attr_capability.attr,
+ &dev_attr_power_checksum.attr,
+ &dev_attr_xmc_12v_pex_vol.attr,
+ &dev_attr_xmc_12v_aux_vol.attr,
+ &dev_attr_xmc_12v_pex_curr.attr,
+ &dev_attr_xmc_12v_aux_curr.attr,
+ &dev_attr_xmc_3v3_pex_vol.attr,
+ &dev_attr_xmc_3v3_aux_vol.attr,
+ &dev_attr_xmc_ddr_vpp_btm.attr,
+ &dev_attr_xmc_sys_5v5.attr,
+ &dev_attr_xmc_1v2_top.attr,
+ &dev_attr_xmc_1v8.attr,
+ &dev_attr_xmc_0v85.attr,
+ &dev_attr_xmc_ddr_vpp_top.attr,
+ &dev_attr_xmc_mgt0v9avcc.attr,
+ &dev_attr_xmc_12v_sw.attr,
+ &dev_attr_xmc_mgtavtt.attr,
+ &dev_attr_xmc_vcc1v2_btm.attr,
+ &dev_attr_xmc_fpga_temp.attr,
+ &dev_attr_xmc_fan_temp.attr,
+ &dev_attr_xmc_fan_rpm.attr,
+ &dev_attr_xmc_dimm_temp0.attr,
+ &dev_attr_xmc_dimm_temp1.attr,
+ &dev_attr_xmc_dimm_temp2.attr,
+ &dev_attr_xmc_dimm_temp3.attr,
+ &dev_attr_xmc_vccint_vol.attr,
+ &dev_attr_xmc_vccint_curr.attr,
+ &dev_attr_xmc_se98_temp0.attr,
+ &dev_attr_xmc_se98_temp1.attr,
+ &dev_attr_xmc_se98_temp2.attr,
+ &dev_attr_xmc_cage_temp0.attr,
+ &dev_attr_xmc_cage_temp1.attr,
+ &dev_attr_xmc_cage_temp2.attr,
+ &dev_attr_xmc_cage_temp3.attr,
+ &dev_attr_pause.attr,
+ &dev_attr_reset.attr,
+ &dev_attr_power_flag.attr,
+ &dev_attr_host_msg_offset.attr,
+ &dev_attr_host_msg_error.attr,
+ &dev_attr_host_msg_header.attr,
+ NULL,
+};
+
+
+static ssize_t read_temp_by_mem_topology(struct file *filp, struct kobject *kobj,
+ struct bin_attribute *attr, char *buffer, loff_t offset, size_t count)
+{
+ u32 nread = 0;
+ size_t size = 0;
+ u32 i;
+ struct mem_topology *memtopo = NULL;
+ struct xocl_xmc *xmc;
+ uint32_t temp[MAX_M_COUNT] = {0};
+ struct xclmgmt_dev *lro;
+
+ //xocl_icap_lock_bitstream
+ lro = (struct xclmgmt_dev *)dev_get_drvdata(container_of(kobj, struct device, kobj)->parent);
+ xmc = (struct xocl_xmc *)dev_get_drvdata(container_of(kobj, struct device, kobj));
+
+ memtopo = (struct mem_topology *)xocl_icap_get_data(lro, MEMTOPO_AXLF);
+
+ if (!memtopo)
+ return 0;
+
+ size = sizeof(u32)*(memtopo->m_count);
+
+ if (offset >= size)
+ return 0;
+
+ for (i = 0; i < memtopo->m_count; ++i)
+ *(temp+i) = get_temp_by_m_tag(xmc, memtopo->m_mem_data[i].m_tag);
+
+ if (count < size - offset)
+ nread = count;
+ else
+ nread = size - offset;
+
+ memcpy(buffer, temp, nread);
+ //xocl_icap_unlock_bitstream
+ return nread;
+}
+
+static struct bin_attribute bin_dimm_temp_by_mem_topology_attr = {
+ .attr = {
+ .name = "temp_by_mem_topology",
+ .mode = 0444
+ },
+ .read = read_temp_by_mem_topology,
+ .write = NULL,
+ .size = 0
+};
+
+static struct bin_attribute *xmc_bin_attrs[] = {
+ &bin_dimm_temp_by_mem_topology_attr,
+ NULL,
+};
+
+static struct attribute_group xmc_attr_group = {
+ .attrs = xmc_attrs,
+ .bin_attrs = xmc_bin_attrs,
+};
+static ssize_t show_mb_pw(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
+ struct xocl_xmc *xmc = dev_get_drvdata(dev);
+ u32 val;
+
+ safe_read32(xmc, XMC_12V_PEX_REG + attr->index * sizeof(u32), &val);
+
+ return sprintf(buf, "%d\n", val);
+}
+
+static SENSOR_DEVICE_ATTR(curr1_highest, 0444, show_mb_pw, NULL, 0);
+static SENSOR_DEVICE_ATTR(curr1_average, 0444, show_mb_pw, NULL, 1);
+static SENSOR_DEVICE_ATTR(curr1_input, 0444, show_mb_pw, NULL, 2);
+static SENSOR_DEVICE_ATTR(curr2_highest, 0444, show_mb_pw, NULL, 3);
+static SENSOR_DEVICE_ATTR(curr2_average, 0444, show_mb_pw, NULL, 4);
+static SENSOR_DEVICE_ATTR(curr2_input, 0444, show_mb_pw, NULL, 5);
+static SENSOR_DEVICE_ATTR(curr3_highest, 0444, show_mb_pw, NULL, 6);
+static SENSOR_DEVICE_ATTR(curr3_average, 0444, show_mb_pw, NULL, 7);
+static SENSOR_DEVICE_ATTR(curr3_input, 0444, show_mb_pw, NULL, 8);
+static SENSOR_DEVICE_ATTR(curr4_highest, 0444, show_mb_pw, NULL, 9);
+static SENSOR_DEVICE_ATTR(curr4_average, 0444, show_mb_pw, NULL, 10);
+static SENSOR_DEVICE_ATTR(curr4_input, 0444, show_mb_pw, NULL, 11);
+static SENSOR_DEVICE_ATTR(curr5_highest, 0444, show_mb_pw, NULL, 12);
+static SENSOR_DEVICE_ATTR(curr5_average, 0444, show_mb_pw, NULL, 13);
+static SENSOR_DEVICE_ATTR(curr5_input, 0444, show_mb_pw, NULL, 14);
+static SENSOR_DEVICE_ATTR(curr6_highest, 0444, show_mb_pw, NULL, 15);
+static SENSOR_DEVICE_ATTR(curr6_average, 0444, show_mb_pw, NULL, 16);
+static SENSOR_DEVICE_ATTR(curr6_input, 0444, show_mb_pw, NULL, 17);
+
+static struct attribute *hwmon_xmc_attributes[] = {
+ &sensor_dev_attr_curr1_highest.dev_attr.attr,
+ &sensor_dev_attr_curr1_average.dev_attr.attr,
+ &sensor_dev_attr_curr1_input.dev_attr.attr,
+ &sensor_dev_attr_curr2_highest.dev_attr.attr,
+ &sensor_dev_attr_curr2_average.dev_attr.attr,
+ &sensor_dev_attr_curr2_input.dev_attr.attr,
+ &sensor_dev_attr_curr3_highest.dev_attr.attr,
+ &sensor_dev_attr_curr3_average.dev_attr.attr,
+ &sensor_dev_attr_curr3_input.dev_attr.attr,
+ &sensor_dev_attr_curr4_highest.dev_attr.attr,
+ &sensor_dev_attr_curr4_average.dev_attr.attr,
+ &sensor_dev_attr_curr4_input.dev_attr.attr,
+ &sensor_dev_attr_curr5_highest.dev_attr.attr,
+ &sensor_dev_attr_curr5_average.dev_attr.attr,
+ &sensor_dev_attr_curr5_input.dev_attr.attr,
+ &sensor_dev_attr_curr6_highest.dev_attr.attr,
+ &sensor_dev_attr_curr6_average.dev_attr.attr,
+ &sensor_dev_attr_curr6_input.dev_attr.attr,
+ NULL
+};
+
+static const struct attribute_group hwmon_xmc_attrgroup = {
+ .attrs = hwmon_xmc_attributes,
+};
+
+static ssize_t show_name(struct device *dev, struct device_attribute *da,
+ char *buf)
+{
+ return sprintf(buf, "%s\n", XCLMGMT_MB_HWMON_NAME);
+}
+
+static struct sensor_device_attribute name_attr =
+ SENSOR_ATTR(name, 0444, show_name, NULL, 0);
+
+static void mgmt_sysfs_destroy_xmc(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+
+ xmc = platform_get_drvdata(pdev);
+
+ if (!xmc->enabled)
+ return;
+
+ if (xmc->hwmon_dev) {
+ device_remove_file(xmc->hwmon_dev, &name_attr.dev_attr);
+ sysfs_remove_group(&xmc->hwmon_dev->kobj,
+ &hwmon_xmc_attrgroup);
+ hwmon_device_unregister(xmc->hwmon_dev);
+ xmc->hwmon_dev = NULL;
+ }
+
+ sysfs_remove_group(&pdev->dev.kobj, &xmc_attr_group);
+}
+
+static int mgmt_sysfs_create_xmc(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ struct xocl_dev_core *core;
+ int err;
+
+ xmc = platform_get_drvdata(pdev);
+ core = XDEV(xocl_get_xdev(pdev));
+
+ if (!xmc->enabled)
+ return 0;
+
+ err = sysfs_create_group(&pdev->dev.kobj, &xmc_attr_group);
+ if (err) {
+ xocl_err(&pdev->dev, "create xmc attrs failed: 0x%x", err);
+ goto create_attr_failed;
+ }
+ xmc->hwmon_dev = hwmon_device_register(&core->pdev->dev);
+ if (IS_ERR(xmc->hwmon_dev)) {
+ err = PTR_ERR(xmc->hwmon_dev);
+ xocl_err(&pdev->dev, "register xmc hwmon failed: 0x%x", err);
+ goto hwmon_reg_failed;
+ }
+
+ dev_set_drvdata(xmc->hwmon_dev, xmc);
+
+ err = device_create_file(xmc->hwmon_dev, &name_attr.dev_attr);
+ if (err) {
+ xocl_err(&pdev->dev, "create attr name failed: 0x%x", err);
+ goto create_name_failed;
+ }
+
+ err = sysfs_create_group(&xmc->hwmon_dev->kobj,
+ &hwmon_xmc_attrgroup);
+ if (err) {
+ xocl_err(&pdev->dev, "create pw group failed: 0x%x", err);
+ goto create_pw_failed;
+ }
+
+ return 0;
+
+create_pw_failed:
+ device_remove_file(xmc->hwmon_dev, &name_attr.dev_attr);
+create_name_failed:
+ hwmon_device_unregister(xmc->hwmon_dev);
+ xmc->hwmon_dev = NULL;
+hwmon_reg_failed:
+ sysfs_remove_group(&pdev->dev.kobj, &xmc_attr_group);
+create_attr_failed:
+ return err;
+}
+
+static int stop_xmc_nolock(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ int retry = 0;
+ u32 reg_val = 0;
+ void *xdev_hdl;
+
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return -ENODEV;
+ else if (!xmc->enabled)
+ return -ENODEV;
+
+ xdev_hdl = xocl_get_xdev(xmc->pdev);
+
+ reg_val = READ_GPIO(xmc, 0);
+ xocl_info(&xmc->pdev->dev, "MB Reset GPIO 0x%x", reg_val);
+
+ //Stop XMC and ERT if its currently running
+ if (reg_val == GPIO_ENABLED) {
+ xocl_info(&xmc->pdev->dev,
+ "XMC info, version 0x%x, status 0x%x, id 0x%x",
+ READ_REG32(xmc, XMC_VERSION_REG),
+ READ_REG32(xmc, XMC_STATUS_REG),
+ READ_REG32(xmc, XMC_MAGIC_REG));
+
+ reg_val = READ_REG32(xmc, XMC_STATUS_REG);
+ if (!(reg_val & STATUS_MASK_STOPPED)) {
+ xocl_info(&xmc->pdev->dev, "Stopping XMC...");
+ WRITE_REG32(xmc, CTL_MASK_STOP, XMC_CONTROL_REG);
+ WRITE_REG32(xmc, 1, XMC_STOP_CONFIRM_REG);
+ }
+ //Need to check if ERT is loaded before we attempt to stop it
+ if (!SELF_JUMP(READ_IMAGE_SCHED(xmc, 0))) {
+ reg_val = XOCL_READ_REG32(xmc->base_addrs[IO_CQ]);
+ if (!(reg_val & ERT_STOP_ACK)) {
+ xocl_info(&xmc->pdev->dev, "Stopping scheduler...");
+ XOCL_WRITE_REG32(ERT_STOP_CMD, xmc->base_addrs[IO_CQ]);
+ }
+ }
+
+ retry = 0;
+ while (retry++ < MAX_XMC_RETRY &&
+ !(READ_REG32(xmc, XMC_STATUS_REG) & STATUS_MASK_STOPPED))
+ msleep(RETRY_INTERVAL);
+
+ //Wait for XMC to stop and then check that ERT has also finished
+ if (retry >= MAX_XMC_RETRY) {
+ xocl_err(&xmc->pdev->dev,
+ "Failed to stop XMC");
+ xocl_err(&xmc->pdev->dev,
+ "XMC Error Reg 0x%x",
+ READ_REG32(xmc, XMC_ERROR_REG));
+ xmc->state = XMC_STATE_ERROR;
+ return -ETIMEDOUT;
+ } else if (!SELF_JUMP(READ_IMAGE_SCHED(xmc, 0)) &&
+ !(XOCL_READ_REG32(xmc->base_addrs[IO_CQ]) & ERT_STOP_ACK)) {
+ while (retry++ < MAX_ERT_RETRY &&
+ !(XOCL_READ_REG32(xmc->base_addrs[IO_CQ]) & ERT_STOP_ACK))
+ msleep(RETRY_INTERVAL);
+ if (retry >= MAX_ERT_RETRY) {
+ xocl_err(&xmc->pdev->dev,
+ "Failed to stop sched");
+ xocl_err(&xmc->pdev->dev,
+ "Scheduler CQ status 0x%x",
+ XOCL_READ_REG32(xmc->base_addrs[IO_CQ]));
+ //We don't exit if ERT doesn't stop since it can hang due to bad kernel
+ //xmc->state = XMC_STATE_ERROR;
+ //return -ETIMEDOUT;
+ }
+ }
+
+ xocl_info(&xmc->pdev->dev, "XMC/sched Stopped, retry %d",
+ retry);
+ }
+
+ // Hold XMC in reset now that its safely stopped
+ xocl_info(&xmc->pdev->dev,
+ "XMC info, version 0x%x, status 0x%x, id 0x%x",
+ READ_REG32(xmc, XMC_VERSION_REG),
+ READ_REG32(xmc, XMC_STATUS_REG),
+ READ_REG32(xmc, XMC_MAGIC_REG));
+ WRITE_GPIO(xmc, GPIO_RESET, 0);
+ xmc->state = XMC_STATE_RESET;
+ reg_val = READ_GPIO(xmc, 0);
+ xocl_info(&xmc->pdev->dev, "MB Reset GPIO 0x%x", reg_val);
+ if (reg_val != GPIO_RESET) {
+ //Shouldnt make it here but if we do then exit
+ xmc->state = XMC_STATE_ERROR;
+ return -EIO;
+ }
+
+ return 0;
+}
+static int stop_xmc(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ int ret = 0;
+ void *xdev_hdl;
+
+ xocl_info(&pdev->dev, "Stop Microblaze...");
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return -ENODEV;
+ else if (!xmc->enabled)
+ return -ENODEV;
+
+ xdev_hdl = xocl_get_xdev(xmc->pdev);
+
+ mutex_lock(&xmc->xmc_lock);
+ ret = stop_xmc_nolock(pdev);
+ mutex_unlock(&xmc->xmc_lock);
+
+ return ret;
+}
+
+static int load_xmc(struct xocl_xmc *xmc)
+{
+ int retry = 0;
+ u32 reg_val = 0;
+ int ret = 0;
+ void *xdev_hdl;
+
+ if (!xmc->enabled)
+ return -ENODEV;
+
+ mutex_lock(&xmc->xmc_lock);
+
+ /* Stop XMC first */
+ ret = stop_xmc_nolock(xmc->pdev);
+ if (ret != 0)
+ goto out;
+
+ xdev_hdl = xocl_get_xdev(xmc->pdev);
+
+ /* Load XMC and ERT Image */
+ if (xocl_mb_mgmt_on(xdev_hdl)) {
+ xocl_info(&xmc->pdev->dev, "Copying XMC image len %d",
+ xmc->mgmt_binary_length);
+ COPY_MGMT(xmc, xmc->mgmt_binary, xmc->mgmt_binary_length);
+ }
+
+ if (xocl_mb_sched_on(xdev_hdl)) {
+ xocl_info(&xmc->pdev->dev, "Copying scheduler image len %d",
+ xmc->sche_binary_length);
+ COPY_SCHE(xmc, xmc->sche_binary, xmc->sche_binary_length);
+ }
+
+ /* Take XMC and ERT out of reset */
+ WRITE_GPIO(xmc, GPIO_ENABLED, 0);
+ reg_val = READ_GPIO(xmc, 0);
+ xocl_info(&xmc->pdev->dev, "MB Reset GPIO 0x%x", reg_val);
+ if (reg_val != GPIO_ENABLED) {
+ //Shouldnt make it here but if we do then exit
+ xmc->state = XMC_STATE_ERROR;
+ goto out;
+ }
+
+ /* Wait for XMC to start
+ * Note that ERT will start long before XMC so we don't check anything
+ */
+ reg_val = READ_REG32(xmc, XMC_STATUS_REG);
+ if (!(reg_val & STATUS_MASK_INIT_DONE)) {
+ xocl_info(&xmc->pdev->dev, "Waiting for XMC to finish init...");
+ retry = 0;
+ while (retry++ < MAX_XMC_RETRY &&
+ !(READ_REG32(xmc, XMC_STATUS_REG) & STATUS_MASK_INIT_DONE))
+ msleep(RETRY_INTERVAL);
+ if (retry >= MAX_XMC_RETRY) {
+ xocl_err(&xmc->pdev->dev,
+ "XMC did not finish init sequence!");
+ xocl_err(&xmc->pdev->dev,
+ "Error Reg 0x%x",
+ READ_REG32(xmc, XMC_ERROR_REG));
+ xocl_err(&xmc->pdev->dev,
+ "Status Reg 0x%x",
+ READ_REG32(xmc, XMC_STATUS_REG));
+ ret = -ETIMEDOUT;
+ xmc->state = XMC_STATE_ERROR;
+ goto out;
+ }
+ }
+ xocl_info(&xmc->pdev->dev, "XMC and scheduler Enabled, retry %d",
+ retry);
+ xocl_info(&xmc->pdev->dev,
+ "XMC info, version 0x%x, status 0x%x, id 0x%x",
+ READ_REG32(xmc, XMC_VERSION_REG),
+ READ_REG32(xmc, XMC_STATUS_REG),
+ READ_REG32(xmc, XMC_MAGIC_REG));
+ xmc->state = XMC_STATE_ENABLED;
+
+ xmc->cap = READ_REG32(xmc, XMC_FEATURE_REG);
+out:
+ mutex_unlock(&xmc->xmc_lock);
+
+ return ret;
+}
+
+static void xmc_reset(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+
+ xocl_info(&pdev->dev, "Reset Microblaze...");
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return;
+
+ load_xmc(xmc);
+}
+
+static int load_mgmt_image(struct platform_device *pdev, const char *image,
+ u32 len)
+{
+ struct xocl_xmc *xmc;
+ char *binary;
+
+ if (len > MAX_IMAGE_LEN)
+ return -EINVAL;
+
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return -EINVAL;
+
+ binary = xmc->mgmt_binary;
+ xmc->mgmt_binary = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
+ if (!xmc->mgmt_binary)
+ return -ENOMEM;
+
+ if (binary)
+ devm_kfree(&pdev->dev, binary);
+ memcpy(xmc->mgmt_binary, image, len);
+ xmc->mgmt_binary_length = len;
+
+ return 0;
+}
+
+static int load_sche_image(struct platform_device *pdev, const char *image,
+ u32 len)
+{
+ struct xocl_xmc *xmc;
+ char *binary = NULL;
+
+ if (len > MAX_IMAGE_LEN)
+ return -EINVAL;
+
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return -EINVAL;
+
+ binary = xmc->sche_binary;
+ xmc->sche_binary = devm_kzalloc(&pdev->dev, len, GFP_KERNEL);
+ if (!xmc->sche_binary)
+ return -ENOMEM;
+
+ if (binary)
+ devm_kfree(&pdev->dev, binary);
+ memcpy(xmc->sche_binary, image, len);
+ xmc->sche_binary_length = len;
+
+ return 0;
+}
+
+static struct xocl_mb_funcs xmc_ops = {
+ .load_mgmt_image = load_mgmt_image,
+ .load_sche_image = load_sche_image,
+ .reset = xmc_reset,
+ .stop = stop_xmc,
+ .get_data = xmc_get_data,
+};
+
+static int xmc_remove(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ int i;
+
+ xmc = platform_get_drvdata(pdev);
+ if (!xmc)
+ return 0;
+
+ if (xmc->mgmt_binary)
+ devm_kfree(&pdev->dev, xmc->mgmt_binary);
+ if (xmc->sche_binary)
+ devm_kfree(&pdev->dev, xmc->sche_binary);
+
+ mgmt_sysfs_destroy_xmc(pdev);
+
+ for (i = 0; i < NUM_IOADDR; i++) {
+ if (xmc->base_addrs[i])
+ iounmap(xmc->base_addrs[i]);
+ }
+
+ mutex_destroy(&xmc->xmc_lock);
+
+ platform_set_drvdata(pdev, NULL);
+ devm_kfree(&pdev->dev, xmc);
+
+ return 0;
+}
+
+static int xmc_probe(struct platform_device *pdev)
+{
+ struct xocl_xmc *xmc;
+ struct resource *res;
+ void *xdev_hdl;
+ int i, err;
+
+ xmc = devm_kzalloc(&pdev->dev, sizeof(*xmc), GFP_KERNEL);
+ if (!xmc) {
+ xocl_err(&pdev->dev, "out of memory");
+ return -ENOMEM;
+ }
+
+ xmc->pdev = pdev;
+ platform_set_drvdata(pdev, xmc);
+
+ xdev_hdl = xocl_get_xdev(pdev);
+ if (xocl_mb_mgmt_on(xdev_hdl) || xocl_mb_sched_on(xdev_hdl)) {
+ xocl_info(&pdev->dev, "Microblaze is supported.");
+ xmc->enabled = true;
+ } else {
+ xocl_err(&pdev->dev, "Microblaze is not supported.");
+ devm_kfree(&pdev->dev, xmc);
+ platform_set_drvdata(pdev, NULL);
+ return 0;
+ }
+
+ for (i = 0; i < NUM_IOADDR; i++) {
+ res = platform_get_resource(pdev, IORESOURCE_MEM, i);
+ if (res) {
+ xocl_info(&pdev->dev, "IO start: 0x%llx, end: 0x%llx",
+ res->start, res->end);
+ xmc->base_addrs[i] =
+ ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!xmc->base_addrs[i]) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+ } else
+ break;
+ }
+
+ err = mgmt_sysfs_create_xmc(pdev);
+ if (err) {
+ xocl_err(&pdev->dev, "Create sysfs failed, err %d", err);
+ goto failed;
+ }
+
+ xocl_subdev_register(pdev, XOCL_SUBDEV_XMC, &xmc_ops);
+
+ mutex_init(&xmc->xmc_lock);
+
+ return 0;
+
+failed:
+ xmc_remove(pdev);
+ return err;
+}
+
+struct platform_device_id xmc_id_table[] = {
+ { XOCL_XMC, 0 },
+ { },
+};
+
+static struct platform_driver xmc_driver = {
+ .probe = xmc_probe,
+ .remove = xmc_remove,
+ .driver = {
+ .name = XOCL_XMC,
+ },
+ .id_table = xmc_id_table,
+};
+
+int __init xocl_init_xmc(void)
+{
+ return platform_driver_register(&xmc_driver);
+}
+
+void xocl_fini_xmc(void)
+{
+ platform_driver_unregister(&xmc_driver);
+}
diff --git a/drivers/gpu/drm/xocl/subdev/xvc.c b/drivers/gpu/drm/xocl/subdev/xvc.c
new file mode 100644
index 000000000000..355dbad30b00
--- /dev/null
+++ b/drivers/gpu/drm/xocl/subdev/xvc.c
@@ -0,0 +1,461 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * A GEM style device manager for PCIe based OpenCL accelerators.
+ *
+ * Copyright (C) 2016-2019 Xilinx, Inc. All rights reserved.
+ *
+ * Authors:
+ *
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__
+
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/vmalloc.h>
+#include <linux/slab.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/ioctl.h>
+
+#include "../xocl_drv.h"
+
+/* IOCTL interfaces */
+#define XIL_XVC_MAGIC 0x58564344 // "XVCD"
+#define MINOR_PUB_HIGH_BIT 0x00000
+#define MINOR_PRI_HIGH_BIT 0x10000
+#define MINOR_NAME_MASK 0xffffffff
+
+enum xvc_algo_type {
+ XVC_ALGO_NULL,
+ XVC_ALGO_CFG,
+ XVC_ALGO_BAR
+};
+
+struct xil_xvc_ioc {
+ unsigned int opcode;
+ unsigned int length;
+ unsigned char *tms_buf;
+ unsigned char *tdi_buf;
+ unsigned char *tdo_buf;
+};
+
+struct xil_xvc_properties {
+ unsigned int xvc_algo_type;
+ unsigned int config_vsec_id;
+ unsigned int config_vsec_rev;
+ unsigned int bar_index;
+ unsigned int bar_offset;
+};
+
+#define XDMA_IOCXVC _IOWR(XIL_XVC_MAGIC, 1, struct xil_xvc_ioc)
+#define XDMA_RDXVC_PROPS _IOR(XIL_XVC_MAGIC, 2, struct xil_xvc_properties)
+
+#define COMPLETION_LOOP_MAX 100
+
+#define XVC_BAR_LENGTH_REG 0x0
+#define XVC_BAR_TMS_REG 0x4
+#define XVC_BAR_TDI_REG 0x8
+#define XVC_BAR_TDO_REG 0xC
+#define XVC_BAR_CTRL_REG 0x10
+
+#define XVC_DEV_NAME "xvc" SUBDEV_SUFFIX
+
+struct xocl_xvc {
+ void *__iomem base;
+ unsigned int instance;
+ struct cdev *sys_cdev;
+ struct device *sys_device;
+};
+
+static dev_t xvc_dev;
+
+static struct xil_xvc_properties xvc_pci_props;
+
+#ifdef __REG_DEBUG__
+/* SECTION: Function definitions */
+static inline void __write_register(const char *fn, u32 value, void *base,
+ unsigned int off)
+{
+ pr_info("%s: 0x%p, W reg 0x%lx, 0x%x.\n", fn, base, off, value);
+ iowrite32(value, base + off);
+}
+
+static inline u32 __read_register(const char *fn, void *base, unsigned int off)
+{
+ u32 v = ioread32(base + off);
+
+ pr_info("%s: 0x%p, R reg 0x%lx, 0x%x.\n", fn, base, off, v);
+ return v;
+}
+#define write_register(v, base, off) __write_register(__func__, v, base, off)
+#define read_register(base, off) __read_register(__func__, base, off)
+
+#else
+#define write_register(v, base, off) iowrite32(v, (base) + (off))
+#define read_register(base, off) ioread32((base) + (off))
+#endif /* #ifdef __REG_DEBUG__ */
+
+
+static int xvc_shift_bits(void *base, u32 tms_bits, u32 tdi_bits,
+ u32 *tdo_bits)
+{
+ u32 control;
+ u32 write_reg_data;
+ int count;
+
+ /* set tms bit */
+ write_register(tms_bits, base, XVC_BAR_TMS_REG);
+ /* set tdi bits and shift data out */
+ write_register(tdi_bits, base, XVC_BAR_TDI_REG);
+
+ control = read_register(base, XVC_BAR_CTRL_REG);
+ /* enable shift operation */
+ write_reg_data = control | 0x01;
+ write_register(write_reg_data, base, XVC_BAR_CTRL_REG);
+
+ /* poll for completion */
+ count = COMPLETION_LOOP_MAX;
+ while (count) {
+ /* read control reg to check shift operation completion */
+ control = read_register(base, XVC_BAR_CTRL_REG);
+ if ((control & 0x01) == 0)
+ break;
+
+ count--;
+ }
+
+ if (!count) {
+ pr_warn("XVC bar transaction timed out (0x%0X)\n", control);
+ return -ETIMEDOUT;
+ }
+
+ /* read tdo bits back out */
+ *tdo_bits = read_register(base, XVC_BAR_TDO_REG);
+
+ return 0;
+}
+
+static long xvc_ioctl_helper(struct xocl_xvc *xvc, const void __user *arg)
+{
+ struct xil_xvc_ioc xvc_obj;
+ unsigned int opcode;
+ unsigned int total_bits;
+ unsigned int total_bytes;
+ unsigned int bits, bits_left;
+ unsigned char *buffer = NULL;
+ unsigned char *tms_buf = NULL;
+ unsigned char *tdi_buf = NULL;
+ unsigned char *tdo_buf = NULL;
+ void __iomem *iobase = xvc->base;
+ u32 control_reg_data;
+ u32 write_reg_data;
+ int rv;
+
+ rv = copy_from_user((void *)&xvc_obj, arg,
+ sizeof(struct xil_xvc_ioc));
+ /* anything not copied ? */
+ if (rv) {
+ pr_info("copy_from_user xvc_obj failed: %d.\n", rv);
+ goto cleanup;
+ }
+
+ opcode = xvc_obj.opcode;
+
+ /* Invalid operation type, no operation performed */
+ if (opcode != 0x01 && opcode != 0x02) {
+ pr_info("UNKNOWN opcode 0x%x.\n", opcode);
+ return -EINVAL;
+ }
+
+ total_bits = xvc_obj.length;
+ total_bytes = (total_bits + 7) >> 3;
+
+ buffer = kmalloc(total_bytes * 3, GFP_KERNEL);
+ if (!buffer) {
+ pr_info("OOM %u, op 0x%x, len %u bits, %u bytes.\n",
+ 3 * total_bytes, opcode, total_bits, total_bytes);
+ rv = -ENOMEM;
+ goto cleanup;
+ }
+ tms_buf = buffer;
+ tdi_buf = tms_buf + total_bytes;
+ tdo_buf = tdi_buf + total_bytes;
+
+ rv = copy_from_user((void *)tms_buf, xvc_obj.tms_buf, total_bytes);
+ if (rv) {
+ pr_info("copy tmfs_buf failed: %d/%u.\n", rv, total_bytes);
+ goto cleanup;
+ }
+ rv = copy_from_user((void *)tdi_buf, xvc_obj.tdi_buf, total_bytes);
+ if (rv) {
+ pr_info("copy tdi_buf failed: %d/%u.\n", rv, total_bytes);
+ goto cleanup;
+ }
+
+ // If performing loopback test, set loopback bit (0x02) in control reg
+ if (opcode == 0x02) {
+ control_reg_data = read_register(iobase, XVC_BAR_CTRL_REG);
+ write_reg_data = control_reg_data | 0x02;
+ write_register(write_reg_data, iobase, XVC_BAR_CTRL_REG);
+ }
+
+ /* set length register to 32 initially if more than one
+ * word-transaction is to be done
+ */
+ if (total_bits >= 32)
+ write_register(0x20, iobase, XVC_BAR_LENGTH_REG);
+
+ for (bits = 0, bits_left = total_bits; bits < total_bits; bits += 32,
+ bits_left -= 32) {
+ unsigned int bytes = bits >> 3;
+ unsigned int shift_bytes = 4;
+ u32 tms_store = 0;
+ u32 tdi_store = 0;
+ u32 tdo_store = 0;
+
+ if (bits_left < 32) {
+ /* set number of bits to shift out */
+ write_register(bits_left, iobase, XVC_BAR_LENGTH_REG);
+ shift_bytes = (bits_left + 7) >> 3;
+ }
+
+ memcpy(&tms_store, tms_buf + bytes, shift_bytes);
+ memcpy(&tdi_store, tdi_buf + bytes, shift_bytes);
+
+ /* Shift data out and copy to output buffer */
+ rv = xvc_shift_bits(iobase, tms_store, tdi_store, &tdo_store);
+ if (rv < 0)
+ goto cleanup;
+
+ memcpy(tdo_buf + bytes, &tdo_store, shift_bytes);
+ }
+
+ // If performing loopback test, reset loopback bit in control reg
+ if (opcode == 0x02) {
+ control_reg_data = read_register(iobase, XVC_BAR_CTRL_REG);
+ write_reg_data = control_reg_data & ~(0x02);
+ write_register(write_reg_data, iobase, XVC_BAR_CTRL_REG);
+ }
+
+ rv = copy_to_user((void *)xvc_obj.tdo_buf, tdo_buf, total_bytes);
+ if (rv) {
+ pr_info("copy back tdo_buf failed: %d/%u.\n", rv, total_bytes);
+ rv = -EFAULT;
+ goto cleanup;
+ }
+
+cleanup:
+ kfree(buffer);
+
+ mmiowb();
+
+ return rv;
+}
+
+static long xvc_read_properties(struct xocl_xvc *xvc, const void __user *arg)
+{
+ int status = 0;
+ struct xil_xvc_properties xvc_props_obj;
+
+ xvc_props_obj.xvc_algo_type = (unsigned int) xvc_pci_props.xvc_algo_type;
+ xvc_props_obj.config_vsec_id = xvc_pci_props.config_vsec_id;
+ xvc_props_obj.config_vsec_rev = xvc_pci_props.config_vsec_rev;
+ xvc_props_obj.bar_index = xvc_pci_props.bar_index;
+ xvc_props_obj.bar_offset = xvc_pci_props.bar_offset;
+
+ if (copy_to_user((void *)arg, &xvc_props_obj, sizeof(xvc_props_obj)))
+ status = -ENOMEM;
+
+ mmiowb();
+ return status;
+}
+
+static long xvc_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
+{
+ struct xocl_xvc *xvc = filp->private_data;
+ long status = 0;
+
+ switch (cmd) {
+ case XDMA_IOCXVC:
+ status = xvc_ioctl_helper(xvc, (void __user *)arg);
+ break;
+ case XDMA_RDXVC_PROPS:
+ status = xvc_read_properties(xvc, (void __user *)arg);
+ break;
+ default:
+ status = -ENOIOCTLCMD;
+ break;
+ }
+
+ return status;
+}
+
+static int char_open(struct inode *inode, struct file *file)
+{
+ struct xocl_xvc *xvc = NULL;
+
+ xvc = xocl_drvinst_open(inode->i_cdev);
+ if (!xvc)
+ return -ENXIO;
+
+ /* create a reference to our char device in the opened file */
+ file->private_data = xvc;
+ return 0;
+}
+
+/*
+ * Called when the device goes from used to unused.
+ */
+static int char_close(struct inode *inode, struct file *file)
+{
+ struct xocl_xvc *xvc = file->private_data;
+
+ xocl_drvinst_close(xvc);
+ return 0;
+}
+
+
+/*
+ * character device file operations for the XVC
+ */
+static const struct file_operations xvc_fops = {
+ .owner = THIS_MODULE,
+ .open = char_open,
+ .release = char_close,
+ .unlocked_ioctl = xvc_ioctl,
+};
+
+static int xvc_probe(struct platform_device *pdev)
+{
+ struct xocl_xvc *xvc;
+ struct resource *res;
+ struct xocl_dev_core *core;
+ int err;
+
+ xvc = xocl_drvinst_alloc(&pdev->dev, sizeof(*xvc));
+ if (!xvc)
+ return -ENOMEM;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ xvc->base = ioremap_nocache(res->start, res->end - res->start + 1);
+ if (!xvc->base) {
+ err = -EIO;
+ xocl_err(&pdev->dev, "Map iomem failed");
+ goto failed;
+ }
+
+ core = xocl_get_xdev(pdev);
+
+ xvc->sys_cdev = cdev_alloc();
+ xvc->sys_cdev->ops = &xvc_fops;
+ xvc->sys_cdev->owner = THIS_MODULE;
+ xvc->instance = XOCL_DEV_ID(core->pdev) |
+ platform_get_device_id(pdev)->driver_data;
+ xvc->sys_cdev->dev = MKDEV(MAJOR(xvc_dev), core->dev_minor);
+ err = cdev_add(xvc->sys_cdev, xvc->sys_cdev->dev, 1);
+ if (err) {
+ xocl_err(&pdev->dev, "cdev_add failed, %d", err);
+ goto failed;
+ }
+
+ xvc->sys_device = device_create(xrt_class, &pdev->dev,
+ xvc->sys_cdev->dev,
+ NULL, "%s%d",
+ platform_get_device_id(pdev)->name,
+ xvc->instance & MINOR_NAME_MASK);
+ if (IS_ERR(xvc->sys_device)) {
+ err = PTR_ERR(xvc->sys_device);
+ goto failed;
+ }
+
+ xocl_drvinst_set_filedev(xvc, xvc->sys_cdev);
+
+ platform_set_drvdata(pdev, xvc);
+ xocl_info(&pdev->dev, "XVC device instance %d initialized\n",
+ xvc->instance);
+
+ // Update PCIe BAR properties in a global structure
+ xvc_pci_props.xvc_algo_type = XVC_ALGO_BAR;
+ xvc_pci_props.config_vsec_id = 0;
+ xvc_pci_props.config_vsec_rev = 0;
+ xvc_pci_props.bar_index = core->bar_idx;
+ xvc_pci_props.bar_offset = (unsigned int) res->start - (unsigned int)
+ pci_resource_start(core->pdev, core->bar_idx);
+
+ return 0;
+failed:
+ if (!IS_ERR(xvc->sys_device))
+ device_destroy(xrt_class, xvc->sys_cdev->dev);
+ if (xvc->sys_cdev)
+ cdev_del(xvc->sys_cdev);
+ if (xvc->base)
+ iounmap(xvc->base);
+ xocl_drvinst_free(xvc);
+
+ return err;
+}
+
+
+static int xvc_remove(struct platform_device *pdev)
+{
+ struct xocl_xvc *xvc;
+
+ xvc = platform_get_drvdata(pdev);
+ if (!xvc) {
+ xocl_err(&pdev->dev, "driver data is NULL");
+ return -EINVAL;
+ }
+ device_destroy(xrt_class, xvc->sys_cdev->dev);
+ cdev_del(xvc->sys_cdev);
+ if (xvc->base)
+ iounmap(xvc->base);
+
+ platform_set_drvdata(pdev, NULL);
+ xocl_drvinst_free(xvc);
+
+ return 0;
+}
+
+struct platform_device_id xvc_id_table[] = {
+ { XOCL_XVC_PUB, MINOR_PUB_HIGH_BIT },
+ { XOCL_XVC_PRI, MINOR_PRI_HIGH_BIT },
+ { },
+};
+
+static struct platform_driver xvc_driver = {
+ .probe = xvc_probe,
+ .remove = xvc_remove,
+ .driver = {
+ .name = XVC_DEV_NAME,
+ },
+ .id_table = xvc_id_table,
+};
+
+int __init xocl_init_xvc(void)
+{
+ int err = 0;
+
+ err = alloc_chrdev_region(&xvc_dev, 0, XOCL_MAX_DEVICES, XVC_DEV_NAME);
+ if (err < 0)
+ goto err_register_chrdev;
+
+ err = platform_driver_register(&xvc_driver);
+ if (err)
+ goto err_driver_reg;
+ return 0;
+
+err_driver_reg:
+ unregister_chrdev_region(xvc_dev, XOCL_MAX_DEVICES);
+err_register_chrdev:
+ return err;
+}
+
+void xocl_fini_xvc(void)
+{
+ unregister_chrdev_region(xvc_dev, XOCL_MAX_DEVICES);
+ platform_driver_unregister(&xvc_driver);
+}
--
2.17.0
next prev parent reply other threads:[~2019-03-19 21:54 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-03-19 21:53 [RFC PATCH Xilinx Alveo 0/6] Xilinx PCIe accelerator driver sonal.santan
2019-03-19 21:53 ` sonal.santan
2019-03-19 21:53 ` [RFC PATCH Xilinx Alveo 1/6] Add skeleton code: ioctl definitions and build hooks sonal.santan
2019-03-19 21:53 ` sonal.santan
2019-03-19 21:53 ` [RFC PATCH Xilinx Alveo 2/6] Global data structures shared between xocl and xmgmt drivers sonal.santan
2019-03-19 21:53 ` sonal.santan
2019-03-19 21:53 ` sonal.santan [this message]
2019-03-19 21:53 ` [RFC PATCH Xilinx Alveo 3/6] Add platform drivers for various IPs and frameworks sonal.santan
2019-03-19 21:53 ` [RFC PATCH Xilinx Alveo 4/6] Add core of XDMA driver sonal.santan
2019-03-19 21:53 ` sonal.santan
2019-03-19 21:54 ` [RFC PATCH Xilinx Alveo 5/6] Add management driver sonal.santan
2019-03-19 21:54 ` sonal.santan
2019-03-19 21:54 ` [RFC PATCH Xilinx Alveo 6/6] Add user physical function driver sonal.santan
2019-03-19 21:54 ` sonal.santan
2019-03-25 20:28 ` [RFC PATCH Xilinx Alveo 0/6] Xilinx PCIe accelerator driver Daniel Vetter
2019-03-25 20:28 ` Daniel Vetter
2019-03-26 23:30 ` Sonal Santan
2019-03-27 8:22 ` Daniel Vetter
2019-03-27 12:50 ` Sonal Santan
2019-03-27 14:11 ` Daniel Vetter
2019-03-27 14:11 ` Daniel Vetter
2019-03-28 0:13 ` Sonal Santan
2019-03-29 4:56 ` Dave Airlie
2019-03-30 1:09 ` Ronan KERYELL
2019-03-30 1:09 ` Ronan KERYELL
2019-04-03 13:14 ` Daniel Vetter
2019-04-03 13:14 ` Daniel Vetter
2019-04-03 14:17 ` Moritz Fischer
2019-04-03 14:53 ` Daniel Vetter
2019-04-03 15:47 ` Jerome Glisse
2019-04-03 15:47 ` Jerome Glisse
2019-04-05 22:15 ` Sonal Santan
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190319215401.6562-4-sonal.santan@xilinx.com \
--to=sonal.santan@xilinx.com \
--cc=airlied@redhat.com \
--cc=cyrilc@xilinx.com \
--cc=dri-devel@lists.freedesktop.org \
--cc=gregkh@linuxfoundation.org \
--cc=hyunk@xilinx.com \
--cc=linux-kernel@vger.kernel.org \
--cc=lizhih@xilinx.com \
--cc=michals@xilinx.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.