[PATCH v2 0/5] misc: Add Add Synopsys DesignWare xData IP driver

[PATCH v2 0/5] misc: Add Add Synopsys DesignWare xData IP driver

[Gustavo Pimentel]

This patch series adds a new driver called xData-pcie for the Synopsys
DesignWare PCIe prototype.

The driver configures and enables the Synopsys DesignWare PCIe traffic
generator IP inside of prototype Endpoint which will generate upstream
and downstream PCIe traffic. This allows to quickly test the PCIe link
throughput speed and check is the prototype solution has some limitation
or not.

Cc: Derek Kiernan <derek.kiernan@xilinx.com>
Cc: Dragan Cvetic <dragan.cvetic@xilinx.com>
Cc: Arnd Bergmann <arnd@arndb.de>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: Jonathan Corbet <corbet@lwn.net>
Cc: linux-pci@vger.kernel.org
Cc: linux-doc@vger.kernel.org
Cc: linux-kernel@vger.kernel.org

Changes:
 V2: Rework driver according to Greg Kroah-Hartman feedback 

Gustavo Pimentel (5):
  misc: Add Synopsys DesignWare xData IP driver
  misc: Add Synopsys DesignWare xData IP driver to Makefile
  misc: Add Synopsys DesignWare xData IP driver to Kconfig
  Documentation: misc-devices: Add Documentation for dw-xdata-pcie
    driver
  MAINTAINERS: Add Synopsys xData IP driver maintainer

 Documentation/misc-devices/dw-xdata-pcie.rst |  40 +++
 MAINTAINERS                                  |   7 +
 drivers/misc/Kconfig                         |  11 +
 drivers/misc/Makefile                        |   1 +
 drivers/misc/dw-xdata-pcie.c                 | 390 +++++++++++++++++++++++++++
 5 files changed, 449 insertions(+)
 create mode 100644 Documentation/misc-devices/dw-xdata-pcie.rst
 create mode 100644 drivers/misc/dw-xdata-pcie.c

-- 
2.7.4

[PATCH v2 1/5] misc: Add Synopsys DesignWare xData IP driver

[Gustavo Pimentel]

Add Synopsys DesignWare xData IP driver. This driver enables/disables
the PCI traffic generator module pertain to the Synopsys DesignWare
prototype.

Signed-off-by: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
---
 drivers/misc/dw-xdata-pcie.c | 390 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 390 insertions(+)
 create mode 100644 drivers/misc/dw-xdata-pcie.c

diff --git a/drivers/misc/dw-xdata-pcie.c b/drivers/misc/dw-xdata-pcie.c
new file mode 100644
index 00000000..d74ff22
--- /dev/null
+++ b/drivers/misc/dw-xdata-pcie.c
@@ -0,0 +1,390 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2020 Synopsys, Inc. and/or its affiliates.
+ * Synopsys DesignWare xData driver
+ *
+ * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/pci-epf.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/pci.h>
+
+#define DW_XDATA_DRIVER_NAME		"dw-xdata-pcie"
+
+#define DW_XDATA_EP_MEM_OFFSET		0x8000000
+
+struct dw_xdata_pcie_data {
+	/* xData registers location */
+	enum pci_barno			rg_bar;
+	off_t				rg_off;
+	size_t				rg_sz;
+};
+
+static const struct dw_xdata_pcie_data snps_edda_data = {
+	/* xData registers location */
+	.rg_bar				= BAR_0,
+	.rg_off				= 0x00000000,   /*   0 Kbytes */
+	.rg_sz				= 0x0000012c,   /* 300  bytes */
+};
+
+#define STATUS_DONE			BIT(0)
+
+#define CONTROL_DOORBELL		BIT(0)
+#define CONTROL_IS_WRITE		BIT(1)
+#define CONTROL_LENGTH(a)		FIELD_PREP(GENMASK(13, 2), a)
+#define CONTROL_PATTERN_INC		BIT(16)
+#define CONTROL_NO_ADDR_INC		BIT(18)
+
+#define XPERF_CONTROL_ENABLE		BIT(5)
+
+union _addr {
+	u64 reg;
+	struct {
+		u32 lsb;
+		u32 msb;
+	};
+};
+
+struct dw_xdata_regs {
+	union _addr addr;				/* 0x000..0x004 */
+	u32 burst_cnt;					/* 0x008 */
+	u32 control;					/* 0x00c */
+	u32 pattern;					/* 0x010 */
+	u32 status;					/* 0x014 */
+	u32 RAM_addr;					/* 0x018 */
+	u32 RAM_port;					/* 0x01c */
+	u32 _reserved0[14];				/* 0x020..0x054 */
+	u32 perf_control;				/* 0x058 */
+	u32 _reserved1[41];				/* 0x05c..0x0fc */
+	union _addr wr_cnt;				/* 0x100..0x104 */
+	union _addr rd_cnt;				/* 0x108..0x10c */
+} __packed;
+
+struct dw_xdata_region {
+	phys_addr_t paddr;				/* physical address */
+	void __iomem *vaddr;				/* virtual address */
+	size_t sz;					/* size */
+};
+
+struct dw_xdata {
+	struct dw_xdata_region rg_region;		/* registers */
+	size_t max_wr_len;				/* max wr xfer len */
+	size_t max_rd_len;				/* max rd xfer len */
+	struct pci_dev *pdev;
+};
+
+static inline struct dw_xdata_regs __iomem *__dw_regs(struct dw_xdata *dw)
+{
+	return dw->rg_region.vaddr;
+}
+
+static void dw_xdata_stop(struct dw_xdata *dw)
+{
+	u32 burst = readl(&(__dw_regs(dw)->burst_cnt));
+
+	if (burst & BIT(31)) {
+		burst &= ~(u32)BIT(31);
+		writel(burst, &(__dw_regs(dw)->burst_cnt));
+	}
+}
+
+static void dw_xdata_start(struct dw_xdata *dw, bool write)
+{
+	u32 control, status;
+
+	/* Stop first if xfer in progress */
+	dw_xdata_stop(dw);
+
+	/* Clear status register */
+	writel(0x0, &(__dw_regs(dw)->status));
+
+	/* Burst count register set for continuous until stopped */
+	writel(0x80001001, &(__dw_regs(dw)->burst_cnt));
+
+	/* Pattern register */
+	writel(0x0, &(__dw_regs(dw)->pattern));
+
+	/* Control register */
+	control = CONTROL_DOORBELL | CONTROL_PATTERN_INC | CONTROL_NO_ADDR_INC;
+	if (write) {
+		control |= CONTROL_IS_WRITE;
+		control |= CONTROL_LENGTH(dw->max_wr_len);
+	} else {
+		control |= CONTROL_LENGTH(dw->max_rd_len);
+	}
+	writel(control, &(__dw_regs(dw)->control));
+
+	usleep_range(100, 150);
+
+	status = readl(&(__dw_regs(dw)->status));
+	if (!(status & STATUS_DONE))
+		pci_dbg(dw->pdev, "xData: started %s direction\n",
+			write ? "write" : "read");
+}
+
+static void dw_xdata_perf_meas(struct dw_xdata *dw, u64 *data, bool write)
+{
+	union _addr *cnt;
+
+	if (write)
+		cnt = &(__dw_regs(dw)->wr_cnt);
+	else
+		cnt = &(__dw_regs(dw)->rd_cnt);
+
+#ifdef CONFIG_64BIT
+	*data = readq(&cnt->reg);
+#else /* CONFIG_64BIT */
+	*data = readl(&cnt->msb);
+	*data <<= 32;
+	*data |= readl(&cnt->lsb);
+#endif /* CONFIG_64BIT */
+}
+
+static u64 dw_xdata_perf_diff(u64 *m1, u64 *m2, u64 time)
+{
+	u64 rate = (*m1 - *m2);
+
+	rate *= (1000 * 1000 * 1000);
+	rate >>= 20;
+	rate = DIV_ROUND_CLOSEST_ULL(rate, time);
+
+	return rate;
+}
+
+static void dw_xdata_perf(struct dw_xdata *dw, u64 *rate, bool write)
+{
+	u64 data[2], time[2], diff;
+
+	/* First measurement */
+	writel(0x0, &(__dw_regs(dw)->perf_control));
+	dw_xdata_perf_meas(dw, &data[0], write);
+	time[0] = jiffies;
+	writel((u32)XPERF_CONTROL_ENABLE, &(__dw_regs(dw)->perf_control));
+
+	/* Delay 100ms */
+	mdelay(100);
+
+	/* Second measurement */
+	writel(0x0, &(__dw_regs(dw)->perf_control));
+	dw_xdata_perf_meas(dw, &data[1], write);
+	time[1] = jiffies;
+	writel((u32)XPERF_CONTROL_ENABLE, &(__dw_regs(dw)->perf_control));
+
+	/* Calculations */
+	diff = jiffies_to_nsecs(time[1] - time[0]);
+	*rate = dw_xdata_perf_diff(&data[1], &data[0], diff);
+
+	pci_dbg(dw->pdev, "xData: time=%llu us, %s=%llu MB/s\n",
+		diff, write ? "write" : "read", *rate);
+}
+
+static inline struct device *kobj2device(struct kobject *kobj)
+{
+	return container_of(kobj, struct device, kobj);
+}
+
+static inline struct pci_dev *device2pci_dev(struct device *dev)
+{
+	return container_of(dev, struct pci_dev, dev);
+}
+
+static ssize_t sysfs_write_show(struct kobject *kobj,
+				struct kobj_attribute *attr, char *buf)
+{
+	struct device *dev = kobj2device(kobj);
+	struct pci_dev *pdev = device2pci_dev(dev);
+	struct dw_xdata *dw = pci_get_drvdata(pdev);
+	u64 rate;
+
+	dw_xdata_perf(dw, &rate, true);
+	return sprintf(buf, "%llu MB/s\n", rate);
+}
+
+static ssize_t sysfs_write_store(struct kobject *kobj,
+				 struct kobj_attribute *attr, const char *buf,
+				 size_t count)
+{
+	struct device *dev = kobj2device(kobj);
+	struct pci_dev *pdev = device2pci_dev(dev);
+	struct dw_xdata *dw = pci_get_drvdata(pdev);
+
+	pci_dbg(pdev, "xData: requested write transfer\n");
+	dw_xdata_start(dw, true);
+
+	return count;
+}
+
+struct kobj_attribute sysfs_write_attr = __ATTR(write, 0644,
+						sysfs_write_show,
+						sysfs_write_store);
+
+static ssize_t sysfs_read_show(struct kobject *kobj,
+			       struct kobj_attribute *attr, char *buf)
+{
+	struct device *dev = kobj2device(kobj);
+	struct pci_dev *pdev = device2pci_dev(dev);
+	struct dw_xdata *dw = pci_get_drvdata(pdev);
+	u64 rate;
+
+	dw_xdata_perf(dw, &rate, false);
+	return sprintf(buf, "%llu MB/s\n", rate);
+}
+
+static ssize_t sysfs_read_store(struct kobject *kobj,
+				struct kobj_attribute *attr, const char *buf,
+				size_t count)
+{
+	struct device *dev = kobj2device(kobj);
+	struct pci_dev *pdev = device2pci_dev(dev);
+	struct dw_xdata *dw = pci_get_drvdata(pdev);
+
+	pci_dbg(pdev, "xData: requested read transfer\n");
+	dw_xdata_start(dw, false);
+
+	return count;
+}
+
+struct kobj_attribute sysfs_read_attr = __ATTR(read, 0644,
+					       sysfs_read_show,
+					       sysfs_read_store);
+
+static ssize_t sysfs_stop_store(struct kobject *kobj,
+				struct kobj_attribute *attr, const char *buf,
+				size_t count)
+{
+	struct device *dev = kobj2device(kobj);
+	struct pci_dev *pdev = device2pci_dev(dev);
+	struct dw_xdata *dw = pci_get_drvdata(pdev);
+
+	pci_dbg(pdev, "xData: requested stop any transfer\n");
+	dw_xdata_stop(dw);
+
+	return count;
+}
+
+struct kobj_attribute sysfs_stop_attr = __ATTR(stop, 0644,
+					       NULL,
+					       sysfs_stop_store);
+
+static int dw_xdata_pcie_probe(struct pci_dev *pdev,
+			       const struct pci_device_id *pid)
+{
+	const struct dw_xdata_pcie_data *pdata = (void *)pid->driver_data;
+	struct device *dev = &pdev->dev;
+	struct dw_xdata *dw;
+	u64 addr;
+	int err;
+
+	/* Enable PCI device */
+	err = pcim_enable_device(pdev);
+	if (err) {
+		pci_err(pdev, "enabling device failed\n");
+		return err;
+	}
+
+	/* Mapping PCI BAR regions */
+	err = pcim_iomap_regions(pdev, BIT(pdata->rg_bar), pci_name(pdev));
+	if (err) {
+		pci_err(pdev, "xData BAR I/O remapping failed\n");
+		return err;
+	}
+
+	pci_set_master(pdev);
+
+	/* Allocate memory */
+	dw = devm_kzalloc(&pdev->dev, sizeof(*dw), GFP_KERNEL);
+	if (!dw)
+		return -ENOMEM;
+
+	/* Data structure initialization */
+	dw->rg_region.vaddr = pcim_iomap_table(pdev)[pdata->rg_bar];
+	if (!dw->rg_region.vaddr)
+		return -ENOMEM;
+
+	dw->rg_region.vaddr += pdata->rg_off;
+	dw->rg_region.paddr = pdev->resource[pdata->rg_bar].start;
+	dw->rg_region.paddr += pdata->rg_off;
+	dw->rg_region.sz = pdata->rg_sz;
+
+	dw->max_wr_len = pcie_get_mps(pdev);
+	dw->max_wr_len >>= 2;
+
+	dw->max_rd_len = pcie_get_readrq(pdev);
+	dw->max_rd_len >>= 2;
+
+	dw->pdev = pdev;
+
+	writel(0x0, &(__dw_regs(dw)->RAM_addr));
+	writel(0x0, &(__dw_regs(dw)->RAM_port));
+
+	addr = dw->rg_region.paddr + DW_XDATA_EP_MEM_OFFSET;
+#ifdef CONFIG_64BIT
+	writeq(addr, &(__dw_regs(dw)->addr.reg));
+#else /* CONFIG_64BIT */
+	writel(lower_32_bits(addr), &(__dw_regs(dw)->addr.lsb));
+	writel(upper_32_bits(addr), &(__dw_regs(dw)->addr.msb));
+#endif /* CONFIG_64BIT */
+	pci_dbg(pdev, "xData: target address = 0x%.16llx\n", addr);
+
+	pci_dbg(pdev, "xData: wr_len=%zu, rd_len=%zu\n",
+		dw->max_wr_len * 4, dw->max_rd_len * 4);
+
+	err = sysfs_create_file(&dev->kobj, &sysfs_write_attr.attr);
+	if (err)
+		return err;
+
+	err = sysfs_create_file(&dev->kobj, &sysfs_read_attr.attr);
+	if (err)
+		return err;
+
+	err = sysfs_create_file(&dev->kobj, &sysfs_stop_attr.attr);
+	if (err)
+		return err;
+
+	err = sysfs_create_link(kernel_kobj, &dev->kobj, DW_XDATA_DRIVER_NAME);
+	if (err)
+		return err;
+
+	/* Saving data structure reference */
+	pci_set_drvdata(pdev, dw);
+
+	return 0;
+}
+
+static void dw_xdata_pcie_remove(struct pci_dev *pdev)
+{
+	struct dw_xdata *dw = pci_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+
+	if (dw)
+		dw_xdata_stop(dw);
+
+	sysfs_remove_link(kernel_kobj, DW_XDATA_DRIVER_NAME);
+	kobject_put(&dev->kobj);
+}
+
+static const struct pci_device_id dw_xdata_pcie_id_table[] = {
+	{ PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) },
+	{ }
+};
+MODULE_DEVICE_TABLE(pci, dw_xdata_pcie_id_table);
+
+static struct pci_driver dw_xdata_pcie_driver = {
+	.name		= DW_XDATA_DRIVER_NAME,
+	.id_table	= dw_xdata_pcie_id_table,
+	.probe		= dw_xdata_pcie_probe,
+	.remove		= dw_xdata_pcie_remove,
+};
+
+module_pci_driver(dw_xdata_pcie_driver);
+
+MODULE_LICENSE("GPL v2");
+MODULE_DESCRIPTION("Synopsys DesignWare xData PCIe driver");
+MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>");
+
-- 
2.7.4

[PATCH v2 4/5] Documentation: misc-devices: Add Documentation for dw-xdata-pcie driver

[Gustavo Pimentel]

Add Documentation for dw-xdata-pcie driver.

Signed-off-by: Gustavo Pimentel <gustavo.pimentel@synopsys.com>
---
 Documentation/misc-devices/dw-xdata-pcie.rst | 40 ++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 Documentation/misc-devices/dw-xdata-pcie.rst

diff --git a/Documentation/misc-devices/dw-xdata-pcie.rst b/Documentation/misc-devices/dw-xdata-pcie.rst
new file mode 100644
index 00000000..3af9fad
--- /dev/null
+++ b/Documentation/misc-devices/dw-xdata-pcie.rst
@@ -0,0 +1,40 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+===========================================================================
+Driver for Synopsys DesignWare PCIe traffic generator (also known as xData)
+===========================================================================
+
+This driver should be used as a host-side (Root Complex) driver and Synopsys
+DesignWare prototype that includes this IP.
+
+The "dw-xdata-pcie" driver can be used to enable/disable PCIe traffic
+generator in either direction (mutual exclusion) besides allowing the
+PCIe link performance analysis.
+
+The interaction with this driver is done through the module parameter and
+can be changed in runtime. The driver outputs the requested command state
+information to /var/log/kern.log or dmesg.
+
+Request write TLPs traffic generation - Root Complex to Endpoint direction
+- Command:
+	echo 1 > /sys/kernel/dw-xdata-pcie/write
+
+Get write TLPs traffic link throughput
+- Command:
+        cat /sys/kernel/dw-xdata-pcie/write
+- Output example:
+	204 MB/s
+
+Request read TLPs traffic generation - Endpoint to Root Complex direction:
+- Command:
+	echo 1 > /sys/kernel/dw-xdata-pcie/read
+
+Get read TLPs traffic link throughput
+- Command:
+        cat /sys/kernel/dw-xdata-pcie/read
+- Output example:
+	199 MB/s
+
+Request to stop any current TLP transfer:
+- Command:
+	echo 1 > /sys/kernel/dw-xdata-pcie/stop
-- 
2.7.4

Re: [PATCH v2 0/5] misc: Add Add Synopsys DesignWare xData IP driver

[Arnd Bergmann]

On Fri, Nov 13, 2020 at 11:37 PM Gustavo Pimentel
<Gustavo.Pimentel@synopsys.com> wrote:
>
> This patch series adds a new driver called xData-pcie for the Synopsys
> DesignWare PCIe prototype.
>
> The driver configures and enables the Synopsys DesignWare PCIe traffic
> generator IP inside of prototype Endpoint which will generate upstream
> and downstream PCIe traffic. This allows to quickly test the PCIe link
> throughput speed and check is the prototype solution has some limitation
> or not.

I don't quite understand what this hardware is, based on your description.
Is this a specific piece of hardware that only serves as a traffic generator,
or a particular hardware feature of the DesignWare endpoint, or is it
software running on a SoC in endpoint mode while plugged into a Linux
system running this driver on the host?

Most importantly; Is there any relation between this driver and the driver
we have for the DesignWare PCIe endpoint itself?

My feeling is that this should be located more closely to drivers/pci/,
but that depends on what it actually does.

     Arnd

RE: [PATCH v2 0/5] misc: Add Add Synopsys DesignWare xData IP driver

[Gustavo Pimentel]

On Tue, Nov 17, 2020 at 14:4:49, Arnd Bergmann <arnd@kernel.org> wrote:

> On Fri, Nov 13, 2020 at 11:37 PM Gustavo Pimentel
> <Gustavo.Pimentel@synopsys.com> wrote:
> >
> > This patch series adds a new driver called xData-pcie for the Synopsys
> > DesignWare PCIe prototype.
> >
> > The driver configures and enables the Synopsys DesignWare PCIe traffic
> > generator IP inside of prototype Endpoint which will generate upstream
> > and downstream PCIe traffic. This allows to quickly test the PCIe link
> > throughput speed and check is the prototype solution has some limitation
> > or not.
> 
> I don't quite understand what this hardware is, based on your description.
> Is this a specific piece of hardware that only serves as a traffic generator,
> or a particular hardware feature of the DesignWare endpoint, or is it
> software running on a SoC in endpoint mode while plugged into a Linux
> system running this driver on the host?

Hi Arnd,

Firstly you have to have in mind that we are talking about an HW 
prototype based on FPGA. This PCIe Endpoint HW prototype from Synopsys 
might have multiple HW blocks inside (depends on the HW design), in this 
particular prototype case, it has an HW block is called xData (available 
internally to Synopsys only) which is a PCIe traffic generator, this 
block has no practical usage, unless for HW validation and testing new 
designs that push forward new PCIe speeds.

> 
> Most importantly; Is there any relation between this driver and the driver
> we have for the DesignWare PCIe endpoint itself?

The scopes are different. The DesignWare PCIe endpoint is a framework 
that allows to test some PCIe generic functionalities (not related to 
xData) using pcitest.

> 
> My feeling is that this should be located more closely to drivers/pci/,
> but that depends on what it actually does.

I thought to put on /misc because the purpose is very limited and doesn't 
fit in a normal case.

-Gustavo

> 
>      Arnd

Re: [PATCH v2 0/5] misc: Add Add Synopsys DesignWare xData IP driver

[Arnd Bergmann]

On Tue, Nov 17, 2020 at 3:53 PM Gustavo Pimentel
<Gustavo.Pimentel@synopsys.com> wrote:
> On Tue, Nov 17, 2020 at 14:4:49, Arnd Bergmann <arnd@kernel.org> wrote:
> > On Fri, Nov 13, 2020 at 11:37 PM Gustavo Pimentel <Gustavo.Pimentel@synopsys.com> wrote:
> > >
> > > This patch series adds a new driver called xData-pcie for the Synopsys
> > > DesignWare PCIe prototype.
> > >
> > > The driver configures and enables the Synopsys DesignWare PCIe traffic
> > > generator IP inside of prototype Endpoint which will generate upstream
> > > and downstream PCIe traffic. This allows to quickly test the PCIe link
> > > throughput speed and check is the prototype solution has some limitation
> > > or not.
> >
> > I don't quite understand what this hardware is, based on your description.
> > Is this a specific piece of hardware that only serves as a traffic generator,
> > or a particular hardware feature of the DesignWare endpoint, or is it
> > software running on a SoC in endpoint mode while plugged into a Linux
> > system running this driver on the host?
>
> Firstly you have to have in mind that we are talking about an HW
> prototype based on FPGA. This PCIe Endpoint HW prototype from Synopsys
> might have multiple HW blocks inside (depends on the HW design), in this
> particular prototype case, it has an HW block is called xData (available
> internally to Synopsys only) which is a PCIe traffic generator, this
> block has no practical usage, unless for HW validation and testing new
> designs that push forward new PCIe speeds.

Ok, got it. Thanks for the explanation.

> > My feeling is that this should be located more closely to drivers/pci/,
> > but that depends on what it actually does.
>
> I thought to put on /misc because the purpose is very limited and doesn't
> fit in a normal case.

Makes sense. I usually try to ensure we don't add anything to drivers/misc
that could reasonably be grouped with related code elsewhere, but
I agree there isn't much that fits into this category today, so let's leave
it there unless someone comes up with a better idea.

The only alternative I could see would be drivers/pci/testing/

      Arnd