[RFC PATCH 0/2] add support for CDX bus MSI domain

[RFC PATCH 0/2] add support for CDX bus MSI domain

[Nipun Gupta]

Devices in FPGA can be added/modified dynamically on run-time.
These devices are exposed on system bus to embedded CPUs.

CDX is an upcoming bus, that caters to the requirement for 
dynamically discovered FPGA devices. These devices are added
as platform devices where fwnode is created using 'software
nodes' in Linux framework.

This RFC targets to solves 2 issues when adding devices
dynamically using platform_device_register API.

1. It creates a MSI domain for CDX bus devices, which can 
   discover device ID used by GIC ITS without depending
   on of_node.
2. Since these devices are not present in device tree, it
   creates a sysfs entry to expose the compatible string.

Nipun Gupta (2):
  irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent
  driver core: add compatible string in sysfs for platform devices

 drivers/base/platform.c                  |  15 +++
 drivers/irqchip/Makefile                 |   1 +
 drivers/irqchip/irq-gic-v3-its-cdx-msi.c | 114 +++++++++++++++++++++++
 include/linux/cdx/cdx.h                  |  15 +++
 4 files changed, 145 insertions(+)
 create mode 100644 drivers/irqchip/irq-gic-v3-its-cdx-msi.c
 create mode 100644 include/linux/cdx/cdx.h

-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent

[Nipun Gupta]

Devices on cdx bus are dynamically detected and registered using
platform_device_register API. As these devices are not linked to
of node they need a separate MSI domain for handling device ID
to be provided to the GIC ITS domain.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
Signed-off-by: Nikhil Agarwal <nikhil.agarwal@amd.com>
---
CONFIG_CDX_BUS and device tree bindings for xlnx,cdx-controller-1.0
would be added as part of CDX bus patches

 drivers/irqchip/Makefile                 |   1 +
 drivers/irqchip/irq-gic-v3-its-cdx-msi.c | 113 +++++++++++++++++++++++
 include/linux/cdx/cdx.h                  |  15 +++
 3 files changed, 129 insertions(+)
 create mode 100644 drivers/irqchip/irq-gic-v3-its-cdx-msi.c
 create mode 100644 include/linux/cdx/cdx.h

diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 5b67450a9538..623adb8a1f20 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -115,3 +115,4 @@ obj-$(CONFIG_WPCM450_AIC)		+= irq-wpcm450-aic.o
 obj-$(CONFIG_IRQ_IDT3243X)		+= irq-idt3243x.o
 obj-$(CONFIG_APPLE_AIC)			+= irq-apple-aic.o
 obj-$(CONFIG_MCHP_EIC)			+= irq-mchp-eic.o
+obj-$(CONFIG_CDX_BUS)			+= irq-gic-v3-its-cdx-msi.o
diff --git a/drivers/irqchip/irq-gic-v3-its-cdx-msi.c b/drivers/irqchip/irq-gic-v3-its-cdx-msi.c
new file mode 100644
index 000000000000..eb17b74efdc5
--- /dev/null
+++ b/drivers/irqchip/irq-gic-v3-its-cdx-msi.c
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD CDX bus driver MSI support
+ *
+ * Copyright(C) 2022 Xilinx Inc.
+ */
+
+#include <linux/irq.h>
+#include <linux/msi.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_irq.h>
+#include <linux/cdx/cdx.h>
+
+static struct irq_chip its_msi_irq_chip = {
+	.name = "ITS-fMSI",
+	.irq_mask = irq_chip_mask_parent,
+	.irq_unmask = irq_chip_unmask_parent,
+	.irq_eoi = irq_chip_eoi_parent,
+	.irq_set_affinity = msi_domain_set_affinity
+};
+
+static int its_cdx_msi_prepare(struct irq_domain *msi_domain,
+				  struct device *dev,
+				  int nvec, msi_alloc_info_t *info)
+{
+	struct msi_domain_info *msi_info;
+	struct cdx_device_data *dev_data;
+	u32 dev_id;
+
+	dev_data = dev->platform_data;
+	dev_id = dev_data->dev_id;
+
+	/* Set the device Id to be passed to the GIC-ITS */
+	info->scratchpad[0].ul = dev_id;
+
+	msi_info = msi_get_domain_info(msi_domain->parent);
+
+	/* Allocate at least 32 MSIs, and always as a power of 2 */
+	nvec = max_t(int, 32, roundup_pow_of_two(nvec));
+	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
+}
+
+static struct msi_domain_ops its_cdx_msi_ops __ro_after_init = {
+	.msi_prepare = its_cdx_msi_prepare,
+};
+
+static struct msi_domain_info its_cdx_msi_domain_info = {
+	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
+	.ops	= &its_cdx_msi_ops,
+	.chip	= &its_msi_irq_chip,
+};
+
+static const struct of_device_id cdx_device_id[] = {
+	{.compatible = "xlnx,cdx-controller-1.0", },
+	{},
+};
+
+struct irq_domain *get_parent(struct fwnode_handle *handle)
+{
+	return irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
+}
+
+static void __init its_cdx_msi_init_one(struct device_node *np,
+				    const char *name)
+{
+	struct irq_domain *parent;
+	struct irq_domain *cdx_msi_domain;
+	struct fwnode_handle *fwnode_handle;
+	struct device_node *parent_node;
+
+	parent_node = of_parse_phandle(np, "msi-parent", 0);
+
+	parent = get_parent(of_node_to_fwnode(parent_node));
+	if (!parent || !msi_get_domain_info(parent)) {
+		pr_err("%s: unable to locate ITS domain\n", name);
+		return;
+	}
+
+	fwnode_handle = of_node_to_fwnode(np);
+	cdx_msi_domain = platform_msi_create_irq_domain(fwnode_handle,
+						&its_cdx_msi_domain_info,
+						parent);
+	if (!cdx_msi_domain) {
+		pr_err("%s: unable to create cdx bus domain\n", name);
+		return;
+	}
+
+	pr_info("cdx bus MSI: %s domain created\n", name);
+}
+
+static void __init its_cdx_of_msi_init(void)
+{
+	struct device_node *np;
+
+	for (np = of_find_matching_node(NULL, cdx_device_id); np;
+	     np = of_find_matching_node(np, cdx_device_id)) {
+		if (!of_device_is_available(np))
+			continue;
+
+		its_cdx_msi_init_one(np, np->full_name);
+	}
+}
+
+static int __init its_cdx_msi_init(void)
+{
+	its_cdx_of_msi_init();
+
+	return 0;
+}
+
+early_initcall(its_cdx_msi_init);
diff --git a/include/linux/cdx/cdx.h b/include/linux/cdx/cdx.h
new file mode 100644
index 000000000000..244ad721771d
--- /dev/null
+++ b/include/linux/cdx/cdx.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * AMD CDX bus public interface
+ *
+ * Copyright(C) 2022 Xilinx Inc.
+ */
+
+#ifndef _CDX_H_
+#define _CDX_H_
+
+struct cdx_device_data {
+	int dev_id;
+};
+
+#endif /* _CDX_H_ */
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent

[Greg KH]

On Wed, Aug 03, 2022 at 05:56:54PM +0530, Nipun Gupta wrote:
> Devices on cdx bus are dynamically detected and registered using
> platform_device_register API. As these devices are not linked to
> of node they need a separate MSI domain for handling device ID
> to be provided to the GIC ITS domain.
> 
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> Signed-off-by: Nikhil Agarwal <nikhil.agarwal@amd.com>
> ---
> CONFIG_CDX_BUS and device tree bindings for xlnx,cdx-controller-1.0
> would be added as part of CDX bus patches
> 
>  drivers/irqchip/Makefile                 |   1 +
>  drivers/irqchip/irq-gic-v3-its-cdx-msi.c | 113 +++++++++++++++++++++++
>  include/linux/cdx/cdx.h                  |  15 +++
>  3 files changed, 129 insertions(+)
>  create mode 100644 drivers/irqchip/irq-gic-v3-its-cdx-msi.c
>  create mode 100644 include/linux/cdx/cdx.h
> 
> diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
> index 5b67450a9538..623adb8a1f20 100644
> --- a/drivers/irqchip/Makefile
> +++ b/drivers/irqchip/Makefile
> @@ -115,3 +115,4 @@ obj-$(CONFIG_WPCM450_AIC)		+= irq-wpcm450-aic.o
>  obj-$(CONFIG_IRQ_IDT3243X)		+= irq-idt3243x.o
>  obj-$(CONFIG_APPLE_AIC)			+= irq-apple-aic.o
>  obj-$(CONFIG_MCHP_EIC)			+= irq-mchp-eic.o
> +obj-$(CONFIG_CDX_BUS)			+= irq-gic-v3-its-cdx-msi.o
> diff --git a/drivers/irqchip/irq-gic-v3-its-cdx-msi.c b/drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> new file mode 100644
> index 000000000000..eb17b74efdc5
> --- /dev/null
> +++ b/drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> @@ -0,0 +1,113 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * AMD CDX bus driver MSI support
> + *
> + * Copyright(C) 2022 Xilinx Inc.
> + */
> +
> +#include <linux/irq.h>
> +#include <linux/msi.h>
> +#include <linux/of.h>
> +#include <linux/of_address.h>
> +#include <linux/of_device.h>
> +#include <linux/of_irq.h>
> +#include <linux/cdx/cdx.h>
> +
> +static struct irq_chip its_msi_irq_chip = {
> +	.name = "ITS-fMSI",
> +	.irq_mask = irq_chip_mask_parent,
> +	.irq_unmask = irq_chip_unmask_parent,
> +	.irq_eoi = irq_chip_eoi_parent,
> +	.irq_set_affinity = msi_domain_set_affinity
> +};
> +
> +static int its_cdx_msi_prepare(struct irq_domain *msi_domain,
> +				  struct device *dev,
> +				  int nvec, msi_alloc_info_t *info)
> +{
> +	struct msi_domain_info *msi_info;
> +	struct cdx_device_data *dev_data;
> +	u32 dev_id;
> +
> +	dev_data = dev->platform_data;
> +	dev_id = dev_data->dev_id;
> +
> +	/* Set the device Id to be passed to the GIC-ITS */
> +	info->scratchpad[0].ul = dev_id;
> +
> +	msi_info = msi_get_domain_info(msi_domain->parent);
> +
> +	/* Allocate at least 32 MSIs, and always as a power of 2 */
> +	nvec = max_t(int, 32, roundup_pow_of_two(nvec));
> +	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
> +}
> +
> +static struct msi_domain_ops its_cdx_msi_ops __ro_after_init = {
> +	.msi_prepare = its_cdx_msi_prepare,
> +};
> +
> +static struct msi_domain_info its_cdx_msi_domain_info = {
> +	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
> +	.ops	= &its_cdx_msi_ops,
> +	.chip	= &its_msi_irq_chip,
> +};
> +
> +static const struct of_device_id cdx_device_id[] = {
> +	{.compatible = "xlnx,cdx-controller-1.0", },
> +	{},
> +};
> +
> +struct irq_domain *get_parent(struct fwnode_handle *handle)
> +{
> +	return irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
> +}
> +
> +static void __init its_cdx_msi_init_one(struct device_node *np,
> +				    const char *name)
> +{
> +	struct irq_domain *parent;
> +	struct irq_domain *cdx_msi_domain;
> +	struct fwnode_handle *fwnode_handle;
> +	struct device_node *parent_node;
> +
> +	parent_node = of_parse_phandle(np, "msi-parent", 0);
> +
> +	parent = get_parent(of_node_to_fwnode(parent_node));
> +	if (!parent || !msi_get_domain_info(parent)) {
> +		pr_err("%s: unable to locate ITS domain\n", name);
> +		return;
> +	}
> +
> +	fwnode_handle = of_node_to_fwnode(np);
> +	cdx_msi_domain = platform_msi_create_irq_domain(fwnode_handle,
> +						&its_cdx_msi_domain_info,
> +						parent);
> +	if (!cdx_msi_domain) {
> +		pr_err("%s: unable to create cdx bus domain\n", name);
> +		return;
> +	}
> +
> +	pr_info("cdx bus MSI: %s domain created\n", name);

When drivers work properly, they should be quiet.

Also, when drivers print lines to the kernel log, they need to use the
dev_*() calls, not "raw" pr_*() calls please.

thanks,

greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Greg KH <gregkh@linuxfoundation.org>
> Sent: Wednesday, August 3, 2022 6:03 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
> rafael@kernel.org; maz@kernel.org; tglx@linutronix.de; okaya@kernel.org;
> Anand, Harpreet <harpreet.anand@amd.com>; Simek, Michal
> <michal.simek@amd.com>; Agarwal, Nikhil <nikhil.agarwal@amd.com>
> Subject: Re: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-its
> domain as parent
> 
> On Wed, Aug 03, 2022 at 05:56:54PM +0530, Nipun Gupta wrote:
> > Devices on cdx bus are dynamically detected and registered using
> > platform_device_register API. As these devices are not linked to
> > of node they need a separate MSI domain for handling device ID
> > to be provided to the GIC ITS domain.
> >
> > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> > Signed-off-by: Nikhil Agarwal <nikhil.agarwal@amd.com>
> > ---
> > CONFIG_CDX_BUS and device tree bindings for xlnx,cdx-controller-1.0
> > would be added as part of CDX bus patches
> >
> >  drivers/irqchip/Makefile                 |   1 +
> >  drivers/irqchip/irq-gic-v3-its-cdx-msi.c | 113 +++++++++++++++++++++++
> >  include/linux/cdx/cdx.h                  |  15 +++
> >  3 files changed, 129 insertions(+)
> >  create mode 100644 drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> >  create mode 100644 include/linux/cdx/cdx.h
> >
> > diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
> > index 5b67450a9538..623adb8a1f20 100644
> > --- a/drivers/irqchip/Makefile
> > +++ b/drivers/irqchip/Makefile
> > @@ -115,3 +115,4 @@ obj-$(CONFIG_WPCM450_AIC)         += irq-wpcm450-
> aic.o
> >  obj-$(CONFIG_IRQ_IDT3243X)           += irq-idt3243x.o
> >  obj-$(CONFIG_APPLE_AIC)                      += irq-apple-aic.o
> >  obj-$(CONFIG_MCHP_EIC)                       += irq-mchp-eic.o
> > +obj-$(CONFIG_CDX_BUS)                        += irq-gic-v3-its-cdx-msi.o
> > diff --git a/drivers/irqchip/irq-gic-v3-its-cdx-msi.c b/drivers/irqchip/irq-gic-v3-
> its-cdx-msi.c
> > new file mode 100644
> > index 000000000000..eb17b74efdc5
> > --- /dev/null
> > +++ b/drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> > @@ -0,0 +1,113 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * AMD CDX bus driver MSI support
> > + *
> > + * Copyright(C) 2022 Xilinx Inc.
> > + */
> > +
> > +#include <linux/irq.h>
> > +#include <linux/msi.h>
> > +#include <linux/of.h>
> > +#include <linux/of_address.h>
> > +#include <linux/of_device.h>
> > +#include <linux/of_irq.h>
> > +#include <linux/cdx/cdx.h>
> > +
> > +static struct irq_chip its_msi_irq_chip = {
> > +     .name = "ITS-fMSI",
> > +     .irq_mask = irq_chip_mask_parent,
> > +     .irq_unmask = irq_chip_unmask_parent,
> > +     .irq_eoi = irq_chip_eoi_parent,
> > +     .irq_set_affinity = msi_domain_set_affinity
> > +};
> > +
> > +static int its_cdx_msi_prepare(struct irq_domain *msi_domain,
> > +                               struct device *dev,
> > +                               int nvec, msi_alloc_info_t *info)
> > +{
> > +     struct msi_domain_info *msi_info;
> > +     struct cdx_device_data *dev_data;
> > +     u32 dev_id;
> > +
> > +     dev_data = dev->platform_data;
> > +     dev_id = dev_data->dev_id;
> > +
> > +     /* Set the device Id to be passed to the GIC-ITS */
> > +     info->scratchpad[0].ul = dev_id;
> > +
> > +     msi_info = msi_get_domain_info(msi_domain->parent);
> > +
> > +     /* Allocate at least 32 MSIs, and always as a power of 2 */
> > +     nvec = max_t(int, 32, roundup_pow_of_two(nvec));
> > +     return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
> > +}
> > +
> > +static struct msi_domain_ops its_cdx_msi_ops __ro_after_init = {
> > +     .msi_prepare = its_cdx_msi_prepare,
> > +};
> > +
> > +static struct msi_domain_info its_cdx_msi_domain_info = {
> > +     .flags  = (MSI_FLAG_USE_DEF_DOM_OPS |
> MSI_FLAG_USE_DEF_CHIP_OPS),
> > +     .ops    = &its_cdx_msi_ops,
> > +     .chip   = &its_msi_irq_chip,
> > +};
> > +
> > +static const struct of_device_id cdx_device_id[] = {
> > +     {.compatible = "xlnx,cdx-controller-1.0", },
> > +     {},
> > +};
> > +
> > +struct irq_domain *get_parent(struct fwnode_handle *handle)
> > +{
> > +     return irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
> > +}
> > +
> > +static void __init its_cdx_msi_init_one(struct device_node *np,
> > +                                 const char *name)
> > +{
> > +     struct irq_domain *parent;
> > +     struct irq_domain *cdx_msi_domain;
> > +     struct fwnode_handle *fwnode_handle;
> > +     struct device_node *parent_node;
> > +
> > +     parent_node = of_parse_phandle(np, "msi-parent", 0);
> > +
> > +     parent = get_parent(of_node_to_fwnode(parent_node));
> > +     if (!parent || !msi_get_domain_info(parent)) {
> > +             pr_err("%s: unable to locate ITS domain\n", name);
> > +             return;
> > +     }
> > +
> > +     fwnode_handle = of_node_to_fwnode(np);
> > +     cdx_msi_domain = platform_msi_create_irq_domain(fwnode_handle,
> > +                                             &its_cdx_msi_domain_info,
> > +                                             parent);
> > +     if (!cdx_msi_domain) {
> > +             pr_err("%s: unable to create cdx bus domain\n", name);
> > +             return;
> > +     }
> > +
> > +     pr_info("cdx bus MSI: %s domain created\n", name);
> 
> When drivers work properly, they should be quiet.
> 
> Also, when drivers print lines to the kernel log, they need to use the
> dev_*() calls, not "raw" pr_*() calls please.

Totally agree on this.
Will be updated in the formal series.

Thanks,
Nipun

> 
> thanks,
> 
> greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Wed, 03 Aug 2022 13:26:54 +0100,
Nipun Gupta <nipun.gupta@amd.com> wrote:
> 
> Devices on cdx bus are dynamically detected and registered using
> platform_device_register API. As these devices are not linked to
> of node they need a separate MSI domain for handling device ID
> to be provided to the GIC ITS domain.
> 
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> Signed-off-by: Nikhil Agarwal <nikhil.agarwal@amd.com>
> ---
> CONFIG_CDX_BUS and device tree bindings for xlnx,cdx-controller-1.0
> would be added as part of CDX bus patches
> 
>  drivers/irqchip/Makefile                 |   1 +
>  drivers/irqchip/irq-gic-v3-its-cdx-msi.c | 113 +++++++++++++++++++++++
>  include/linux/cdx/cdx.h                  |  15 +++
>  3 files changed, 129 insertions(+)
>  create mode 100644 drivers/irqchip/irq-gic-v3-its-cdx-msi.c
>  create mode 100644 include/linux/cdx/cdx.h
> 
> diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
> index 5b67450a9538..623adb8a1f20 100644
> --- a/drivers/irqchip/Makefile
> +++ b/drivers/irqchip/Makefile
> @@ -115,3 +115,4 @@ obj-$(CONFIG_WPCM450_AIC)		+= irq-wpcm450-aic.o
>  obj-$(CONFIG_IRQ_IDT3243X)		+= irq-idt3243x.o
>  obj-$(CONFIG_APPLE_AIC)			+= irq-apple-aic.o
>  obj-$(CONFIG_MCHP_EIC)			+= irq-mchp-eic.o
> +obj-$(CONFIG_CDX_BUS)			+= irq-gic-v3-its-cdx-msi.o
> diff --git a/drivers/irqchip/irq-gic-v3-its-cdx-msi.c b/drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> new file mode 100644
> index 000000000000..eb17b74efdc5
> --- /dev/null
> +++ b/drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> @@ -0,0 +1,113 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * AMD CDX bus driver MSI support
> + *
> + * Copyright(C) 2022 Xilinx Inc.
> + */
> +
> +#include <linux/irq.h>
> +#include <linux/msi.h>
> +#include <linux/of.h>
> +#include <linux/of_address.h>
> +#include <linux/of_device.h>
> +#include <linux/of_irq.h>
> +#include <linux/cdx/cdx.h>
> +
> +static struct irq_chip its_msi_irq_chip = {
> +	.name = "ITS-fMSI",
> +	.irq_mask = irq_chip_mask_parent,
> +	.irq_unmask = irq_chip_unmask_parent,
> +	.irq_eoi = irq_chip_eoi_parent,
> +	.irq_set_affinity = msi_domain_set_affinity
> +};
> +
> +static int its_cdx_msi_prepare(struct irq_domain *msi_domain,
> +				  struct device *dev,
> +				  int nvec, msi_alloc_info_t *info)
> +{
> +	struct msi_domain_info *msi_info;
> +	struct cdx_device_data *dev_data;
> +	u32 dev_id;
> +
> +	dev_data = dev->platform_data;
> +	dev_id = dev_data->dev_id;
> +
> +	/* Set the device Id to be passed to the GIC-ITS */
> +	info->scratchpad[0].ul = dev_id;
> +
> +	msi_info = msi_get_domain_info(msi_domain->parent);
> +
> +	/* Allocate at least 32 MSIs, and always as a power of 2 */
> +	nvec = max_t(int, 32, roundup_pow_of_two(nvec));
> +	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
> +}
> +
> +static struct msi_domain_ops its_cdx_msi_ops __ro_after_init = {
> +	.msi_prepare = its_cdx_msi_prepare,
> +};
> +
> +static struct msi_domain_info its_cdx_msi_domain_info = {
> +	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
> +	.ops	= &its_cdx_msi_ops,
> +	.chip	= &its_msi_irq_chip,
> +};
> +
> +static const struct of_device_id cdx_device_id[] = {
> +	{.compatible = "xlnx,cdx-controller-1.0", },

What is this? If this is supposed to represent am ITS, it really
should say so.

> +	{},
> +};
> +
> +struct irq_domain *get_parent(struct fwnode_handle *handle)
> +{
> +	return irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
> +}
> +
> +static void __init its_cdx_msi_init_one(struct device_node *np,
> +				    const char *name)
> +{
> +	struct irq_domain *parent;
> +	struct irq_domain *cdx_msi_domain;
> +	struct fwnode_handle *fwnode_handle;
> +	struct device_node *parent_node;
> +
> +	parent_node = of_parse_phandle(np, "msi-parent", 0);
> +
> +	parent = get_parent(of_node_to_fwnode(parent_node));
> +	if (!parent || !msi_get_domain_info(parent)) {
> +		pr_err("%s: unable to locate ITS domain\n", name);
> +		return;
> +	}
> +
> +	fwnode_handle = of_node_to_fwnode(np);
> +	cdx_msi_domain = platform_msi_create_irq_domain(fwnode_handle,
> +						&its_cdx_msi_domain_info,
> +						parent);
> +	if (!cdx_msi_domain) {
> +		pr_err("%s: unable to create cdx bus domain\n", name);
> +		return;
> +	}
> +
> +	pr_info("cdx bus MSI: %s domain created\n", name);
> +}
> +
> +static void __init its_cdx_of_msi_init(void)
> +{
> +	struct device_node *np;
> +
> +	for (np = of_find_matching_node(NULL, cdx_device_id); np;
> +	     np = of_find_matching_node(np, cdx_device_id)) {
> +		if (!of_device_is_available(np))
> +			continue;
> +
> +		its_cdx_msi_init_one(np, np->full_name);
> +	}
> +}
> +
> +static int __init its_cdx_msi_init(void)
> +{
> +	its_cdx_of_msi_init();
> +
> +	return 0;
> +}
> +
> +early_initcall(its_cdx_msi_init);

I really don't think we should have any more of this muck. Yes, the
other busses are also doing that, but here's our chance to do
something better.

Why can't the *bus* driver (I assume there is one) perform these
tasks? It would really help if this patch was shown in context,
because I have no idea how this fits in the grand scheme of things.

Thanks,

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Marc Zyngier <maz@kernel.org>
> Sent: Thursday, August 4, 2022 2:20 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; tglx@linutronix.de;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Simek,
> Michal <michal.simek@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>
> Subject: Re: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-
> its domain as parent
> 
> [CAUTION: External Email]
> 
> On Wed, 03 Aug 2022 13:26:54 +0100,
> Nipun Gupta <mailto:nipun.gupta@amd.com> wrote:
> >
> > Devices on cdx bus are dynamically detected and registered using
> > platform_device_register API. As these devices are not linked to
> > of node they need a separate MSI domain for handling device ID
> > to be provided to the GIC ITS domain.
> >
> > Signed-off-by: Nipun Gupta <mailto:nipun.gupta@amd.com>
> > Signed-off-by: Nikhil Agarwal <mailto:nikhil.agarwal@amd.com>
> > ---
> > CONFIG_CDX_BUS and device tree bindings for xlnx,cdx-controller-1.0
> > would be added as part of CDX bus patches
> >
> >  drivers/irqchip/Makefile                 |   1 +
> >  drivers/irqchip/irq-gic-v3-its-cdx-msi.c | 113 +++++++++++++++++++++++
> >  include/linux/cdx/cdx.h                  |  15 +++
> >  3 files changed, 129 insertions(+)
> >  create mode 100644 drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> >  create mode 100644 include/linux/cdx/cdx.h
> >
> > diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
> > index 5b67450a9538..623adb8a1f20 100644
> > --- a/drivers/irqchip/Makefile
> > +++ b/drivers/irqchip/Makefile
> > @@ -115,3 +115,4 @@ obj-$(CONFIG_WPCM450_AIC)         += irq-
> wpcm450-aic.o
> >  obj-$(CONFIG_IRQ_IDT3243X)           += irq-idt3243x.o
> >  obj-$(CONFIG_APPLE_AIC)                      += irq-apple-aic.o
> >  obj-$(CONFIG_MCHP_EIC)                       += irq-mchp-eic.o
> > +obj-$(CONFIG_CDX_BUS)                        += irq-gic-v3-its-cdx-msi.o
> > diff --git a/drivers/irqchip/irq-gic-v3-its-cdx-msi.c b/drivers/irqchip/irq-gic-
> v3-its-cdx-msi.c
> > new file mode 100644
> > index 000000000000..eb17b74efdc5
> > --- /dev/null
> > +++ b/drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> > @@ -0,0 +1,113 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * AMD CDX bus driver MSI support
> > + *
> > + * Copyright(C) 2022 Xilinx Inc.
> > + */
> > +
> > +#include <linux/irq.h>
> > +#include <linux/msi.h>
> > +#include <linux/of.h>
> > +#include <linux/of_address.h>
> > +#include <linux/of_device.h>
> > +#include <linux/of_irq.h>
> > +#include <linux/cdx/cdx.h>
> > +
> > +static struct irq_chip its_msi_irq_chip = {
> > +     .name = "ITS-fMSI",
> > +     .irq_mask = irq_chip_mask_parent,
> > +     .irq_unmask = irq_chip_unmask_parent,
> > +     .irq_eoi = irq_chip_eoi_parent,
> > +     .irq_set_affinity = msi_domain_set_affinity
> > +};
> > +
> > +static int its_cdx_msi_prepare(struct irq_domain *msi_domain,
> > +                               struct device *dev,
> > +                               int nvec, msi_alloc_info_t *info)
> > +{
> > +     struct msi_domain_info *msi_info;
> > +     struct cdx_device_data *dev_data;
> > +     u32 dev_id;
> > +
> > +     dev_data = dev->platform_data;
> > +     dev_id = dev_data->dev_id;
> > +
> > +     /* Set the device Id to be passed to the GIC-ITS */
> > +     info->scratchpad[0].ul = dev_id;
> > +
> > +     msi_info = msi_get_domain_info(msi_domain->parent);
> > +
> > +     /* Allocate at least 32 MSIs, and always as a power of 2 */
> > +     nvec = max_t(int, 32, roundup_pow_of_two(nvec));
> > +     return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec,
> info);
> > +}
> > +
> > +static struct msi_domain_ops its_cdx_msi_ops __ro_after_init = {
> > +     .msi_prepare = its_cdx_msi_prepare,
> > +};
> > +
> > +static struct msi_domain_info its_cdx_msi_domain_info = {
> > +     .flags  = (MSI_FLAG_USE_DEF_DOM_OPS |
> MSI_FLAG_USE_DEF_CHIP_OPS),
> > +     .ops    = &its_cdx_msi_ops,
> > +     .chip   = &its_msi_irq_chip,
> > +};
> > +
> > +static const struct of_device_id cdx_device_id[] = {
> > +     {.compatible = "xlnx,cdx-controller-1.0", },
> 
> What is this? If this is supposed to represent am ITS, it really
> should say so.

This is a CDX bus controller, not an ITS. This will be added as a part of
device-tree documentation when we add formal CDX bus patches.
CDX is an upcoming AMD bus, supporting dynamically discovered
FPGA devices.

> 
> > +     {},
> > +};
> > +
> > +struct irq_domain *get_parent(struct fwnode_handle *handle)
> > +{
> > +     return irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
> > +}
> > +
> > +static void __init its_cdx_msi_init_one(struct device_node *np,
> > +                                 const char *name)
> > +{
> > +     struct irq_domain *parent;
> > +     struct irq_domain *cdx_msi_domain;
> > +     struct fwnode_handle *fwnode_handle;
> > +     struct device_node *parent_node;
> > +
> > +     parent_node = of_parse_phandle(np, "msi-parent", 0);
> > +
> > +     parent = get_parent(of_node_to_fwnode(parent_node));
> > +     if (!parent || !msi_get_domain_info(parent)) {
> > +             pr_err("%s: unable to locate ITS domain\n", name);
> > +             return;
> > +     }
> > +
> > +     fwnode_handle = of_node_to_fwnode(np);
> > +     cdx_msi_domain = platform_msi_create_irq_domain(fwnode_handle,
> > +                                             &its_cdx_msi_domain_info,
> > +                                             parent);
> > +     if (!cdx_msi_domain) {
> > +             pr_err("%s: unable to create cdx bus domain\n", name);
> > +             return;
> > +     }
> > +
> > +     pr_info("cdx bus MSI: %s domain created\n", name);
> > +}
> > +
> > +static void __init its_cdx_of_msi_init(void)
> > +{
> > +     struct device_node *np;
> > +
> > +     for (np = of_find_matching_node(NULL, cdx_device_id); np;
> > +          np = of_find_matching_node(np, cdx_device_id)) {
> > +             if (!of_device_is_available(np))
> > +                     continue;
> > +
> > +             its_cdx_msi_init_one(np, np->full_name);
> > +     }
> > +}
> > +
> > +static int __init its_cdx_msi_init(void)
> > +{
> > +     its_cdx_of_msi_init();
> > +
> > +     return 0;
> > +}
> > +
> > +early_initcall(its_cdx_msi_init);
> 
> I really don't think we should have any more of this muck. Yes, the
> other busses are also doing that, but here's our chance to do
> something better.
> 
> Why can't the *bus* driver (I assume there is one) perform these
> tasks? It would really help if this patch was shown in context,
> because I have no idea how this fits in the grand scheme of things.

Agree, bus driver could do this stuff, and I was considering it too, but as this
was the norm so added these changes in this IRQ chip :).

Bus driver is not out yet, and we have pushed these as RFC changes for
early feedback (mentioned in the cover letter), but yes, these patches would
be clubbed as a part of overall changes.

And we can move it as a part of bus driver itself. I hope that suits?

Thanks,
Nipun

> 
> Thanks,
> 
>         M.
> 
> --
> Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Thu, 04 Aug 2022 10:18:19 +0100,
"Gupta, Nipun" <Nipun.Gupta@amd.com> wrote:

> > > +static const struct of_device_id cdx_device_id[] = {
> > > +     {.compatible = "xlnx,cdx-controller-1.0", },
> > 
> > What is this? If this is supposed to represent am ITS, it really
> > should say so.
> 
> This is a CDX bus controller, not an ITS. This will be added as a part of
> device-tree documentation when we add formal CDX bus patches.
> CDX is an upcoming AMD bus, supporting dynamically discovered
> FPGA devices.

Right. So that's yet another reason to not have this glue code at all.

> 
> > 
> > > +     {},
> > > +};
> > > +
> > > +struct irq_domain *get_parent(struct fwnode_handle *handle)
> > > +{
> > > +     return irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
> > > +}
> > > +
> > > +static void __init its_cdx_msi_init_one(struct device_node *np,
> > > +                                 const char *name)
> > > +{
> > > +     struct irq_domain *parent;
> > > +     struct irq_domain *cdx_msi_domain;
> > > +     struct fwnode_handle *fwnode_handle;
> > > +     struct device_node *parent_node;
> > > +
> > > +     parent_node = of_parse_phandle(np, "msi-parent", 0);
> > > +
> > > +     parent = get_parent(of_node_to_fwnode(parent_node));
> > > +     if (!parent || !msi_get_domain_info(parent)) {
> > > +             pr_err("%s: unable to locate ITS domain\n", name);
> > > +             return;
> > > +     }
> > > +
> > > +     fwnode_handle = of_node_to_fwnode(np);
> > > +     cdx_msi_domain = platform_msi_create_irq_domain(fwnode_handle,
> > > +                                             &its_cdx_msi_domain_info,
> > > +                                             parent);
> > > +     if (!cdx_msi_domain) {
> > > +             pr_err("%s: unable to create cdx bus domain\n", name);
> > > +             return;
> > > +     }
> > > +
> > > +     pr_info("cdx bus MSI: %s domain created\n", name);
> > > +}
> > > +
> > > +static void __init its_cdx_of_msi_init(void)
> > > +{
> > > +     struct device_node *np;
> > > +
> > > +     for (np = of_find_matching_node(NULL, cdx_device_id); np;
> > > +          np = of_find_matching_node(np, cdx_device_id)) {
> > > +             if (!of_device_is_available(np))
> > > +                     continue;
> > > +
> > > +             its_cdx_msi_init_one(np, np->full_name);
> > > +     }
> > > +}
> > > +
> > > +static int __init its_cdx_msi_init(void)
> > > +{
> > > +     its_cdx_of_msi_init();
> > > +
> > > +     return 0;
> > > +}
> > > +
> > > +early_initcall(its_cdx_msi_init);
> > 
> > I really don't think we should have any more of this muck. Yes, the
> > other busses are also doing that, but here's our chance to do
> > something better.
> > 
> > Why can't the *bus* driver (I assume there is one) perform these
> > tasks? It would really help if this patch was shown in context,
> > because I have no idea how this fits in the grand scheme of things.
> 
> Agree, bus driver could do this stuff, and I was considering it too,
> but as this was the norm so added these changes in this IRQ chip :).

Well, let's not miss this opportunity to improve things! :-)

> Bus driver is not out yet, and we have pushed these as RFC changes
> for early feedback (mentioned in the cover letter), but yes, these
> patches would be clubbed as a part of overall changes.
> 
> And we can move it as a part of bus driver itself. I hope that suits?

It certainly sounds better than the above looks! Please Cc me on the
submission of this infrastructure.

Thanks,

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Marc Zyngier <maz@kernel.org>
> Sent: Thursday, August 4, 2022 4:09 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; tglx@linutronix.de;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Simek,
> Michal <michal.simek@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>
> Subject: Re: [RFC PATCH 1/2] irqchip: cdx-bus: add cdx-MSI domain with gic-
> its domain as parent

<snip>

> > > > +static int __init its_cdx_msi_init(void)
> > > > +{
> > > > +     its_cdx_of_msi_init();
> > > > +
> > > > +     return 0;
> > > > +}
> > > > +
> > > > +early_initcall(its_cdx_msi_init);
> > >
> > > I really don't think we should have any more of this muck. Yes, the
> > > other busses are also doing that, but here's our chance to do
> > > something better.
> > >
> > > Why can't the *bus* driver (I assume there is one) perform these
> > > tasks? It would really help if this patch was shown in context,
> > > because I have no idea how this fits in the grand scheme of things.
> >
> > Agree, bus driver could do this stuff, and I was considering it too,
> > but as this was the norm so added these changes in this IRQ chip :).
> 
> Well, let's not miss this opportunity to improve things! :-)
> 
> > Bus driver is not out yet, and we have pushed these as RFC changes
> > for early feedback (mentioned in the cover letter), but yes, these
> > patches would be clubbed as a part of overall changes.
> >
> > And we can move it as a part of bus driver itself. I hope that suits?
> 
> It certainly sounds better than the above looks! Please Cc me on the
> submission of this infrastructure.

Sounds good!! Then we will move this inside the bus infra itself and would
add you as a reviewer for the bus series.

> 
> Thanks,
> 
>         M.
> 
> --
> Without deviation from the norm, progress is not possible.

On a lighter note, on the earlier mail I thought this was a comment for the
patch, as this change matches where norm is bus drivers adding domain in IRQ
chip, but now it seems this is the tagline you have :)

Thanks,
Nipun
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH 2/2] driver core: add compatible string in sysfs for platform devices

[Nipun Gupta]

For devices registered dynamically using platform_device_register
API, this patch exposes the sysfs entry for the compatible string.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 drivers/base/platform.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 51bb2289865c..89949f88a0a1 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -1289,10 +1289,25 @@ static ssize_t driver_override_store(struct device *dev,
 }
 static DEVICE_ATTR_RW(driver_override);
 
+static ssize_t compatible_show(struct device *dev, struct device_attribute *attr,
+			      char *buf)
+{
+	const char *compat;
+	int ret;
+
+	ret = device_property_read_string(dev, "compatible", &compat);
+	if (ret != 0)
+		return 0;
+
+	return sysfs_emit(buf, "%s", compat);
+}
+static DEVICE_ATTR_RO(compatible);
+
 static struct attribute *platform_dev_attrs[] = {
 	&dev_attr_modalias.attr,
 	&dev_attr_numa_node.attr,
 	&dev_attr_driver_override.attr,
+	&dev_attr_compatible.attr,
 	NULL,
 };
 
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH 2/2] driver core: add compatible string in sysfs for platform devices

[Greg KH]

On Wed, Aug 03, 2022 at 05:56:55PM +0530, Nipun Gupta wrote:
> For devices registered dynamically using platform_device_register
> API, this patch exposes the sysfs entry for the compatible string.
> 
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> ---
>  drivers/base/platform.c | 15 +++++++++++++++
>  1 file changed, 15 insertions(+)
> 
> diff --git a/drivers/base/platform.c b/drivers/base/platform.c
> index 51bb2289865c..89949f88a0a1 100644
> --- a/drivers/base/platform.c
> +++ b/drivers/base/platform.c
> @@ -1289,10 +1289,25 @@ static ssize_t driver_override_store(struct device *dev,
>  }
>  static DEVICE_ATTR_RW(driver_override);
>  
> +static ssize_t compatible_show(struct device *dev, struct device_attribute *attr,
> +			      char *buf)
> +{
> +	const char *compat;
> +	int ret;
> +
> +	ret = device_property_read_string(dev, "compatible", &compat);
> +	if (ret != 0)
> +		return 0;
> +
> +	return sysfs_emit(buf, "%s", compat);
> +}
> +static DEVICE_ATTR_RO(compatible);

You forgot the Documentation/ABI/ update :(

Also, what happens if there is no such string to read?  Why are you
returning 0?  You should not create the attribute at all then, right?

thanks,

greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH 2/2] driver core: add compatible string in sysfs for platform devices

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Greg KH <gregkh@linuxfoundation.org>
> Sent: Wednesday, August 3, 2022 6:02 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
> rafael@kernel.org; maz@kernel.org; tglx@linutronix.de; okaya@kernel.org;
> Anand, Harpreet <harpreet.anand@amd.com>; Simek, Michal
> <michal.simek@amd.com>; Agarwal, Nikhil <nikhil.agarwal@amd.com>
> Subject: Re: [RFC PATCH 2/2] driver core: add compatible string in sysfs for
> platform devices
> 
> [CAUTION: External Email]
> 
> On Wed, Aug 03, 2022 at 05:56:55PM +0530, Nipun Gupta wrote:
> > For devices registered dynamically using platform_device_register
> > API, this patch exposes the sysfs entry for the compatible string.
> >
> > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> > ---
> >  drivers/base/platform.c | 15 +++++++++++++++
> >  1 file changed, 15 insertions(+)
> >
> > diff --git a/drivers/base/platform.c b/drivers/base/platform.c
> > index 51bb2289865c..89949f88a0a1 100644
> > --- a/drivers/base/platform.c
> > +++ b/drivers/base/platform.c
> > @@ -1289,10 +1289,25 @@ static ssize_t driver_override_store(struct device
> *dev,
> >  }
> >  static DEVICE_ATTR_RW(driver_override);
> >
> > +static ssize_t compatible_show(struct device *dev, struct device_attribute
> *attr,
> > +                           char *buf)
> > +{
> > +     const char *compat;
> > +     int ret;
> > +
> > +     ret = device_property_read_string(dev, "compatible", &compat);
> > +     if (ret != 0)
> > +             return 0;
> > +
> > +     return sysfs_emit(buf, "%s", compat);
> > +}
> > +static DEVICE_ATTR_RO(compatible);
> 
> You forgot the Documentation/ABI/ update :(

Thanks for pointing. Will be adding the new sysfs entry as part
of the documentation.

> 
> Also, what happens if there is no such string to read?  Why are you
> returning 0?  You should not create the attribute at all then, right?

Will be updating platform_dev_attrs_visible() API to have it visible
depending on if the attribute is available. Yes, then we may not need
this check here and would be part of the platform_dev_attrs_visible() API.

Thanks,
Nipun

> 
> thanks,
> 
> greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH 0/2] add support for CDX bus MSI domain

[Robin Murphy]

On 2022-08-03 13:26, Nipun Gupta wrote:
> Devices in FPGA can be added/modified dynamically on run-time.
> These devices are exposed on system bus to embedded CPUs.
> 
> CDX is an upcoming bus, that caters to the requirement for
> dynamically discovered FPGA devices. These devices are added
> as platform devices where fwnode is created using 'software
> nodes' in Linux framework.
> 
> This RFC targets to solves 2 issues when adding devices
> dynamically using platform_device_register API.
> 
> 1. It creates a MSI domain for CDX bus devices, which can
>     discover device ID used by GIC ITS without depending
>     on of_node.
> 2. Since these devices are not present in device tree, it
>     creates a sysfs entry to expose the compatible string.

Isn't this pretty much what CONFIG_OF_DYNAMIC is for? From the look of 
these patches this thing is still completely tied to devicetree, so why 
reinvent that wheel?

Robin.

> Nipun Gupta (2):
>    irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent
>    driver core: add compatible string in sysfs for platform devices
> 
>   drivers/base/platform.c                  |  15 +++
>   drivers/irqchip/Makefile                 |   1 +
>   drivers/irqchip/irq-gic-v3-its-cdx-msi.c | 114 +++++++++++++++++++++++
>   include/linux/cdx/cdx.h                  |  15 +++
>   4 files changed, 145 insertions(+)
>   create mode 100644 drivers/irqchip/irq-gic-v3-its-cdx-msi.c
>   create mode 100644 include/linux/cdx/cdx.h
> 

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH 0/2] add support for CDX bus MSI domain

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Robin Murphy <robin.murphy@arm.com>
> Sent: Wednesday, August 3, 2022 7:46 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>; linux-kernel@vger.kernel.org;
> linux-arm-kernel@lists.infradead.org; gregkh@linuxfoundation.org;
> rafael@kernel.org; maz@kernel.org; tglx@linutronix.de
> Cc: okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>;
> Simek, Michal <michal.simek@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>
> Subject: Re: [RFC PATCH 0/2] add support for CDX bus MSI domain
> 
> [CAUTION: External Email]
> 
> On 2022-08-03 13:26, Nipun Gupta wrote:
> > Devices in FPGA can be added/modified dynamically on run-time.
> > These devices are exposed on system bus to embedded CPUs.
> >
> > CDX is an upcoming bus, that caters to the requirement for
> > dynamically discovered FPGA devices. These devices are added
> > as platform devices where fwnode is created using 'software
> > nodes' in Linux framework.
> >
> > This RFC targets to solves 2 issues when adding devices
> > dynamically using platform_device_register API.
> >
> > 1. It creates a MSI domain for CDX bus devices, which can
> >     discover device ID used by GIC ITS without depending
> >     on of_node.
> > 2. Since these devices are not present in device tree, it
> >     creates a sysfs entry to expose the compatible string.
> 
> Isn't this pretty much what CONFIG_OF_DYNAMIC is for? From the look of
> these patches this thing is still completely tied to devicetree, so why
> reinvent that wheel?

According to our analysis, device tree modification via OF_DYNAMIC is
mostly designed as a boot time patching mechanism but it was never
designed to be runtime used as a "bus" mechanism.

Most firmware patches ACPI/DT during boot but we want to go beyond
boot time configuration and would like to be able to dynamically
remove/insert objects during FPGA servicing. As an example, a new FPGA
can be programmed to the system where either the attributes are
changed or new devices are introduced.

Another concern about OF_DYNAMIC is about security.
We'd like to keep device-tree contents intact at boot and never
touch again as a good security principle.

> 
> Robin.
> 
> > Nipun Gupta (2):
> >    irqchip: cdx-bus: add cdx-MSI domain with gic-its domain as parent
> >    driver core: add compatible string in sysfs for platform devices
> >
> >   drivers/base/platform.c                  |  15 +++
> >   drivers/irqchip/Makefile                 |   1 +
> >   drivers/irqchip/irq-gic-v3-its-cdx-msi.c | 114 +++++++++++++++++++++++
> >   include/linux/cdx/cdx.h                  |  15 +++
> >   4 files changed, 145 insertions(+)
> >   create mode 100644 drivers/irqchip/irq-gic-v3-its-cdx-msi.c
> >   create mode 100644 include/linux/cdx/cdx.h
> >
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH 0/2] add support for CDX bus MSI domain

[Jason Gunthorpe]

On Wed, Aug 03, 2022 at 03:16:11PM +0100, Robin Murphy wrote:
> On 2022-08-03 13:26, Nipun Gupta wrote:
> > Devices in FPGA can be added/modified dynamically on run-time.
> > These devices are exposed on system bus to embedded CPUs.
> > 
> > CDX is an upcoming bus, that caters to the requirement for
> > dynamically discovered FPGA devices. These devices are added
> > as platform devices where fwnode is created using 'software
> > nodes' in Linux framework.
> > 
> > This RFC targets to solves 2 issues when adding devices
> > dynamically using platform_device_register API.
> > 
> > 1. It creates a MSI domain for CDX bus devices, which can
> >     discover device ID used by GIC ITS without depending
> >     on of_node.
> > 2. Since these devices are not present in device tree, it
> >     creates a sysfs entry to expose the compatible string.
> 
> Isn't this pretty much what CONFIG_OF_DYNAMIC is for? From the look of these
> patches this thing is still completely tied to devicetree, so why reinvent
> that wheel?

+1

Since the v2 was posted, I strongly agree with this.

The idea that "FW" should somehow provide the FPGA DT components seems
completely backwards. The DT components of a FPGA should come along
with the bitfile that is loaded into the FPGA, and not be part of FW
at all - unless FW is loading the FPGA, then FW can install it in the
main DT and we don't need this.

OF_DYNAMIC or some other version of it like this CDX, should be used
to patch in an entire DT for the FPGA loaded from the filesystem.

This is important because of a lot of useful use cases on FPGAs are
things like I2C, SPI, PCI and flash controllers that rely on DT sub
nodes to be functional.

And this explains why this is creating a platform device - because if
you have a DT node and want to instantiate an OF driver for it, you
pretty create a platform device since platform device has been turned
into the "universal device for OF nodes"

This CDX thing isn't even a bus!

Look at what cdx_bus_device_discovery() does, look at its YAML. There
is no halfway sane bus here like PCI, it is all driven by a DT. The
code to actually get the DT is all "FIXME'D" out, but that seems to be
the design.

The only thing the CDX does is provide some fairly hacky support for
mapping the stream IDs and devices IDs in the FPGA fabric so IOMMU and
MSI can work because the DT fragment it is loading can't be properly
connected back to the main DT due to how it was loaded.

Jason

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v3 0/7] add support for CDX bus

[Nipun Gupta]

Devices in FPGA can be added/modified dynamically on run-time.
This patch series introduces AMD CDX bus, which provides a
mechanism to discover/rescan FPGA devices on run-time. These
devices are memory mapped on system bus for embedded CPUs, and
added as CDX devices in Linux framework.

This RFC:
- Intrduces the CDX bus controller and CDX devices.
- Adds rescan and reset support for the CDX bus.
- Add support for reset for CDX devices.
- Support for CDX bus in arm-smmu-v3 driver
- Support for CDX-MSI domain.
- Vfio-cdx driver support for CDX devices.

Please NOTE: This RFC change does not support the CDX bus firmware
interface as it is under development, and this series aims to get
an early feedback from the community. Firmware interaction are
stubbed as MCDI APIs which is a protocol used by AMD to interact
with Firmware.

Changes in v2:
- introduce CDX bus infrastructure
- fixed code for making compatible visible for devices
  having the 'compatible' property only (Greg K-H)
- moved CDX-MSI domain as part of CDX bus infrastructure.
  previously it was part of irqchip (Marc Zynger).
- fixed few prints (Greg K-H)
- support rescan and reset of CDX bus
- add VFIO reset module for CDX bus based devices

Changes in v3:
- Move CDX bus as a new bus type in kernel rather than
  using the platform devices (Greg K-H, Marc Zynger)
- Correspondingly update ARM SMMU v3
- Add support for vfio-cdx driver
- Updated device tree yaml with correct binding information
  (Krzysztof Kozlowski)
- remove 'compatible' sysfs platform patch which was requried
  for CDX devices exposed as platform devices


Following text provides a basic overview of CDX bus Architecture.

Contents summary
   - CDX overview
   - CDX Linux driver architecture overview
        - Bus driver
        - VFIO driver

CDX Overview
------------

CDX is a Hardware Architecture designed for AMD FPGA devices. It
consists of sophisticated mechanism for interaction between FPGA,
Firmware and the APUs (Application CPUs).

Firmware resides on RPU (Realtime CPUs) which interacts with
the FPGA program manager and the APUs. The RPU provides memory-mapped
interface (RPU if) which is used to communicate with APUs.

The diagram below shows an overview of the CDX architecture:


         +--------------------------------------+
         |    Application CPUs (APU)            |
         |                                      |
         |                    CDX device drivers|
         |     Linux OS                |        |
         |                        CDX bus       |
         |                             |        |
         +-----------------------------|--------+
                                       | (discover, config,
                                       |  reset, rescan)
                                       |
         +------------------------| RPU if |----+
         |                             |        |
         |                             V        |
         |          Realtime CPUs (RPU)         |
         |                                      |
         +--------------------------------------+
                               |
         +---------------------|----------------+
         |  FPGA               |                |
         |      +-----------------------+       |
         |      |           |           |       |
         | +-------+    +-------+   +-------+   |
         | | dev 1 |    | dev 2 |   | dev 3 |   |
         | +-------+    +-------+   +-------+   |
         +--------------------------------------+


The RPU firmware extracts the device information from the loaded FPGA
image and implements a mechanism that allows the APU drivers to
enumerate such devices (device personality and resource details) via
a dedicated communication channel. RPU mediates operations such as
discover, reset and rescan of the FPGA devices for the APU. This is
done using memory mapped interface provided by the RPU to APU.

Nipun Gupta (7):
  dt-bindings: bus: add CDX bus device tree bindings
  bus/cdx: add the cdx bus driver
  iommu/arm-smmu-v3: support ops registration for CDX bus
  bus/cdx: add cdx-MSI domain with gic-its domain as parent
  bus/cdx: add bus and device attributes
  vfio/cdx: add support for CDX bus
  vfio/cdx: add interrupt support

 Documentation/ABI/testing/sysfs-bus-cdx       |  54 ++
 .../devicetree/bindings/bus/xlnx,cdx.yaml     |  75 +++
 MAINTAINERS                                   |   9 +
 drivers/bus/Kconfig                           |   1 +
 drivers/bus/Makefile                          |   3 +
 drivers/bus/cdx/Kconfig                       |   7 +
 drivers/bus/cdx/Makefile                      |   3 +
 drivers/bus/cdx/cdx.c                         | 603 ++++++++++++++++++
 drivers/bus/cdx/cdx.h                         |  53 ++
 drivers/bus/cdx/cdx_msi.c                     | 236 +++++++
 drivers/bus/cdx/mcdi_stubs.c                  |  61 ++
 drivers/bus/cdx/mcdi_stubs.h                  |  87 +++
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   |  16 +-
 drivers/vfio/Makefile                         |   1 +
 drivers/vfio/cdx/Kconfig                      |  10 +
 drivers/vfio/cdx/Makefile                     |   4 +
 drivers/vfio/cdx/vfio_cdx.c                   | 337 ++++++++++
 drivers/vfio/cdx/vfio_cdx_intr.c              | 212 ++++++
 drivers/vfio/cdx/vfio_cdx_private.h           |  50 ++
 include/linux/cdx/cdx_bus.h                   | 120 ++++
 include/linux/mod_devicetable.h               |  13 +
 scripts/mod/devicetable-offsets.c             |   4 +
 scripts/mod/file2alias.c                      |  12 +
 23 files changed, 1969 insertions(+), 2 deletions(-)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-cdx
 create mode 100644 Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
 create mode 100644 drivers/bus/cdx/Kconfig
 create mode 100644 drivers/bus/cdx/Makefile
 create mode 100644 drivers/bus/cdx/cdx.c
 create mode 100644 drivers/bus/cdx/cdx.h
 create mode 100644 drivers/bus/cdx/cdx_msi.c
 create mode 100644 drivers/bus/cdx/mcdi_stubs.c
 create mode 100644 drivers/bus/cdx/mcdi_stubs.h
 create mode 100644 drivers/vfio/cdx/Kconfig
 create mode 100644 drivers/vfio/cdx/Makefile
 create mode 100644 drivers/vfio/cdx/vfio_cdx.c
 create mode 100644 drivers/vfio/cdx/vfio_cdx_intr.c
 create mode 100644 drivers/vfio/cdx/vfio_cdx_private.h
 create mode 100644 include/linux/cdx/cdx_bus.h

-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v3 1/7] dt-bindings: bus: add CDX bus device tree bindings

[Nipun Gupta]

This patch adds a devicetree binding documentation for CDX
bus.

CDX bus controller dynamically detects CDX bus and the
devices on these bus using CDX firmware.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 .../devicetree/bindings/bus/xlnx,cdx.yaml     | 75 +++++++++++++++++++
 MAINTAINERS                                   |  6 ++
 2 files changed, 81 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/bus/xlnx,cdx.yaml

diff --git a/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml b/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
new file mode 100644
index 000000000000..0aa5599ada8e
--- /dev/null
+++ b/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/bus/xlnx,cdx.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: AMD CDX bus controller
+
+description: |
+  CDX bus controller for AMD devices is implemented to dynamically
+  detect CDX bus and devices on these bus using the firmware.
+  The CDX bus manages multiple FPGA based hardware devices, which
+  can support network, crypto or any other specialized type of
+  devices. These FPGA based devices can be added/modified dynamically
+  on run-time.
+
+  All devices on the CDX bus will have a unique streamid (for IOMMU)
+  and a unique device ID (for MSI) corresponding to a requestor ID
+  (one to one associated with the device). The streamid and deviceid
+  are used to configure SMMU and GIC-ITS respectively.
+
+  iommu-map property is used to define the set of stream ids
+  corresponding to each device and the associated IOMMU.
+
+  The MSI writes are accompanied by sideband data (Device ID).
+  The msi-map property is used to associate the devices with the
+  device ID as well as the associated ITS controller.
+
+maintainers:
+  - Nipun Gupta <nipun.gupta@amd.com>
+  - Nikhil Agarwal <nikhil.agarwal@amd.com>
+
+properties:
+  compatible:
+    const: xlnx,cdxbus-controller-1.0
+
+  reg:
+    maxItems: 1
+
+  iommu-map: true
+
+  msi-map: true
+
+required:
+  - compatible
+  - reg
+  - iommu-map
+  - msi-map
+
+additionalProperties: false
+
+examples:
+  - |
+    smmu@ec000000 {
+        compatible = "arm,smmu-v3";
+        #iommu-cells = <1>;
+    };
+
+    gic@e2000000 {
+        compatible = "arm,gic-v3";
+        interrupt-controller;
+        its: gic-its@e2040000 {
+            compatible = "arm,gic-v3-its";
+            msi-controller;
+        };
+    };
+
+    cdx: cdx@4000000 {
+        compatible = "xlnx,cdxbus-controller-1.0";
+        reg = <0x00000000 0x04000000 0 0x1000>;
+        /* define map for RIDs 250-259 */
+        iommu-map = <250 &smmu 250 10>;
+        /* define msi map for RIDs 250-259 */
+        msi-map = <250 &its 250 10>;
+    };
diff --git a/MAINTAINERS b/MAINTAINERS
index 9d7f64dc0efe..f0598b3d731c 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -934,6 +934,12 @@ S:	Supported
 F:	drivers/crypto/ccp/
 F:	include/linux/ccp.h
 
+AMD CDX BUS DRIVER
+M:	Nipun Gupta <nipun.gupta@amd.com>
+M:	Nikhil Agarwal <nikhil.agarwal@amd.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
+
 AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER - SEV SUPPORT
 M:	Brijesh Singh <brijesh.singh@amd.com>
 M:	Tom Lendacky <thomas.lendacky@amd.com>
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 1/7] dt-bindings: bus: add CDX bus device tree bindings

[Rob Herring]

On Tue, 06 Sep 2022 19:17:55 +0530, Nipun Gupta wrote:
> This patch adds a devicetree binding documentation for CDX
> bus.
> 
> CDX bus controller dynamically detects CDX bus and the
> devices on these bus using CDX firmware.
> 
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> ---
>  .../devicetree/bindings/bus/xlnx,cdx.yaml     | 75 +++++++++++++++++++
>  MAINTAINERS                                   |  6 ++
>  2 files changed, 81 insertions(+)
>  create mode 100644 Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> 

My bot found errors running 'make DT_CHECKER_FLAGS=-m dt_binding_check'
on your patch (DT_CHECKER_FLAGS is new in v5.13):

yamllint warnings/errors:

dtschema/dtc warnings/errors:
Documentation/devicetree/bindings/bus/xlnx,cdx.example.dts:18.23-21.11: Warning (unit_address_vs_reg): /example-0/smmu@ec000000: node has a unit name, but no reg or ranges property
Documentation/devicetree/bindings/bus/xlnx,cdx.example.dts:23.22-30.11: Warning (unit_address_vs_reg): /example-0/gic@e2000000: node has a unit name, but no reg or ranges property
Documentation/devicetree/bindings/bus/xlnx,cdx.example.dts:26.35-29.15: Warning (unit_address_vs_reg): /example-0/gic@e2000000/gic-its@e2040000: node has a unit name, but no reg or ranges property
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: smmu@ec000000: $nodename:0: 'smmu@ec000000' does not match '^iommu@[0-9a-f]*'
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: smmu@ec000000: 'reg' is a required property
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: gic@e2000000: $nodename:0: 'gic@e2000000' does not match '^interrupt-controller(@[0-9a-f,]+)*$'
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: gic@e2000000: '#interrupt-cells' is a dependency of 'interrupt-controller'
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: gic@e2000000: 'reg' is a required property
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: gic@e2000000: gic-its@e2040000: False schema does not allow {'compatible': ['arm,gic-v3-its'], 'msi-controller': True, 'phandle': [[1]]}
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: gic@e2000000: gic-its@e2040000: '#msi-cells' is a required property
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: gic@e2000000: gic-its@e2040000: 'reg' is a required property
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: gic@e2000000: 'oneOf' conditional failed, one must be fixed:
	'interrupts' is a required property
	'interrupts-extended' is a required property
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.yaml
/builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb: cdx@4000000: reg: [[0, 67108864], [0, 4096]] is too long
	From schema: /builds/robherring/linux-dt-review/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml

doc reference errors (make refcheckdocs):

See https://patchwork.ozlabs.org/patch/

This check can fail if there are any dependencies. The base for a patch
series is generally the most recent rc1.

If you already ran 'make dt_binding_check' and didn't see the above
error(s), then make sure 'yamllint' is installed and dt-schema is up to
date:

pip3 install dtschema --upgrade

Please check and re-submit.


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 1/7] dt-bindings: bus: add CDX bus device tree bindings

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Rob Herring <robh@kernel.org>
> Sent: Tuesday, September 6, 2022 11:17 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: gregkh@linuxfoundation.org; eric.auger@redhat.com;
> devicetree@vger.kernel.org; linux-kbuild@vger.kernel.org;
> jeffrey.l.hugo@gmail.com; maz@kernel.org; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; Michael.Srba@seznam.cz; cohuck@redhat.com;
> will@kernel.org; masahiroy@kernel.org; mchehab+huawei@kernel.org;
> joro@8bytes.org; okaya@kernel.org; alex.williamson@redhat.com;
> song.bao.hua@hisilicon.com; jgg@nvidia.com; mani@kernel.org; linux-
> kernel@vger.kernel.org; robin.murphy@arm.com; robh+dt@kernel.org;
> krzysztof.kozlowski+dt@linaro.org; rafael@kernel.org; f.fainelli@gmail.com;
> jgg@ziepe.ca; kvm@vger.kernel.org; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Anand, Harpreet <harpreet.anand@amd.com>;
> yishaih@nvidia.com; linux-arm-kernel@lists.infradead.org; git (AMD-Xilinx)
> <git@amd.com>; saravanak@google.com; Radovanovic, Aleksandar
> <aleksandar.radovanovic@amd.com>; Simek, Michal
> <michal.simek@amd.com>; ndesaulniers@google.com
> Subject: Re: [RFC PATCH v3 1/7] dt-bindings: bus: add CDX bus device tree
> bindings
> 
> [CAUTION: External Email]
> 
> On Tue, 06 Sep 2022 19:17:55 +0530, Nipun Gupta wrote:
> > This patch adds a devicetree binding documentation for CDX
> > bus.
> >
> > CDX bus controller dynamically detects CDX bus and the
> > devices on these bus using CDX firmware.
> >
> > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> > ---
> >  .../devicetree/bindings/bus/xlnx,cdx.yaml     | 75 +++++++++++++++++++
> >  MAINTAINERS                                   |  6 ++
> >  2 files changed, 81 insertions(+)
> >  create mode 100644 Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> >
> 
> My bot found errors running 'make DT_CHECKER_FLAGS=-m dt_binding_check'
> on your patch (DT_CHECKER_FLAGS is new in v5.13):
> 
> yamllint warnings/errors:
> 
> dtschema/dtc warnings/errors:
> Documentation/devicetree/bindings/bus/xlnx,cdx.example.dts:18.23-21.11:
> Warning (unit_address_vs_reg): /example-0/smmu@ec000000: node has a unit
> name, but no reg or ranges property
> Documentation/devicetree/bindings/bus/xlnx,cdx.example.dts:23.22-30.11:
> Warning (unit_address_vs_reg): /example-0/gic@e2000000: node has a unit
> name, but no reg or ranges property
> Documentation/devicetree/bindings/bus/xlnx,cdx.example.dts:26.35-29.15:
> Warning (unit_address_vs_reg): /example-0/gic@e2000000/gic-its@e2040000:
> node has a unit name, but no reg or ranges property
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> smmu@ec000000: $nodename:0: 'smmu@ec000000' does not match
> '^iommu@[0-9a-f]*'
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> smmu@ec000000: 'reg' is a required property
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/iommu/arm,smmu-v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> gic@e2000000: $nodename:0: 'gic@e2000000' does not match '^interrupt-
> controller(@[0-9a-f,]+)*$'
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-
> v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> gic@e2000000: '#interrupt-cells' is a dependency of 'interrupt-controller'
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-
> v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> gic@e2000000: 'reg' is a required property
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-
> v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> gic@e2000000: gic-its@e2040000: False schema does not allow {'compatible':
> ['arm,gic-v3-its'], 'msi-controller': True, 'phandle': [[1]]}
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-
> v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> gic@e2000000: gic-its@e2040000: '#msi-cells' is a required property
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-
> v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> gic@e2000000: gic-its@e2040000: 'reg' is a required property
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-
> v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> gic@e2000000: 'oneOf' conditional failed, one must be fixed:
>         'interrupts' is a required property
>         'interrupts-extended' is a required property
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/interrupt-controller/arm,gic-
> v3.yaml
> /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.example.dtb:
> cdx@4000000: reg: [[0, 67108864], [0, 4096]] is too long
>         From schema: /builds/robherring/linux-dt-
> review/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> 
> doc reference errors (make refcheckdocs):
> 
> See
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpatchw
> ork.ozlabs.org%2Fpatch%2F&amp;data=05%7C01%7Cnipun.gupta%40amd.com
> %7C47f53d11f4024ba765f408da902fc525%7C3dd8961fe4884e608e11a82d994e
> 183d%7C0%7C0%7C637980832144301226%7CUnknown%7CTWFpbGZsb3d8eyJ
> WIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C
> 3000%7C%7C%7C&amp;sdata=BXO1d2OWdCQ1lb%2BEDNiUvmfSPPfxfzJET%2B
> uuAL9EdEY%3D&amp;reserved=0
> 
> This check can fail if there are any dependencies. The base for a patch
> series is generally the most recent rc1.
> 
> If you already ran 'make dt_binding_check' and didn't see the above
> error(s), then make sure 'yamllint' is installed and dt-schema is up to
> date:
> 
> pip3 install dtschema --upgrade
> 
> Please check and re-submit.

I did run make dt_binding_check, but did not see the issue.
Will update the dtschema and fix this.

Thanks,
Nipun

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 1/7] dt-bindings: bus: add CDX bus device tree bindings

[Krzysztof Kozlowski]

On 07/09/2022 05:13, Gupta, Nipun wrote:
>>
>> If you already ran 'make dt_binding_check' and didn't see the above
>> error(s), then make sure 'yamllint' is installed and dt-schema is up to
>> date:
>>
>> pip3 install dtschema --upgrade
>>
>> Please check and re-submit.
> 
> I did run make dt_binding_check, but did not see the issue.
> Will update the dtschema and fix this.

Regardless whether you saw it or not, the example DTS is incorrect...
Even if there are no errors, please do not add incorrect DTS.

Best regards,
Krzysztof

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver

[Nipun Gupta]

CDX bus supports the scanning and probing of FPGA based
devices. These devices are registers as CDX devices.

The bus driver sets up the basic infrastructure and fetches
the device related information from the firmware.

CDX bus is capable of scanning devices dynamically,
supporting rescanning of dynamically added, removed or
updated devices.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---

Basic overview of CDX bus architecture is provided in [patch 0/7].

Please NOTE: This RFC change does not support the CDX bus firmware
interface as it is under development, and this series aims to get
an early feedback from the community. Firmware interaction are
stubbed as MCDI APIs which is a protocol used by AMD to interact
with Firmware.

 MAINTAINERS                       |   2 +
 drivers/bus/Kconfig               |   1 +
 drivers/bus/Makefile              |   3 +
 drivers/bus/cdx/Kconfig           |   7 +
 drivers/bus/cdx/Makefile          |   3 +
 drivers/bus/cdx/cdx.c             | 437 ++++++++++++++++++++++++++++++
 drivers/bus/cdx/cdx.h             |  34 +++
 drivers/bus/cdx/mcdi_stubs.c      |  54 ++++
 drivers/bus/cdx/mcdi_stubs.h      |  76 ++++++
 include/linux/cdx/cdx_bus.h       |  93 +++++++
 include/linux/mod_devicetable.h   |  13 +
 scripts/mod/devicetable-offsets.c |   4 +
 scripts/mod/file2alias.c          |  12 +
 13 files changed, 739 insertions(+)
 create mode 100644 drivers/bus/cdx/Kconfig
 create mode 100644 drivers/bus/cdx/Makefile
 create mode 100644 drivers/bus/cdx/cdx.c
 create mode 100644 drivers/bus/cdx/cdx.h
 create mode 100644 drivers/bus/cdx/mcdi_stubs.c
 create mode 100644 drivers/bus/cdx/mcdi_stubs.h
 create mode 100644 include/linux/cdx/cdx_bus.h

diff --git a/MAINTAINERS b/MAINTAINERS
index f0598b3d731c..61af11c9fe06 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -939,6 +939,8 @@ M:	Nipun Gupta <nipun.gupta@amd.com>
 M:	Nikhil Agarwal <nikhil.agarwal@amd.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
+F:	drivers/bus/cdx/*
+F:	include/linux/cdx/*
 
 AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER - SEV SUPPORT
 M:	Brijesh Singh <brijesh.singh@amd.com>
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 7bfe998f3514..b0324efb9a6a 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -251,5 +251,6 @@ config DA8XX_MSTPRI
 
 source "drivers/bus/fsl-mc/Kconfig"
 source "drivers/bus/mhi/Kconfig"
+source "drivers/bus/cdx/Kconfig"
 
 endmenu
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index d90eed189a65..88649111c395 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -20,6 +20,9 @@ obj-$(CONFIG_INTEL_IXP4XX_EB)	+= intel-ixp4xx-eb.o
 obj-$(CONFIG_MIPS_CDMM)		+= mips_cdmm.o
 obj-$(CONFIG_MVEBU_MBUS) 	+= mvebu-mbus.o
 
+#CDX bus
+obj-$(CONFIG_CDX_BUS)		+= cdx/
+
 # Interconnect bus driver for OMAP SoCs.
 obj-$(CONFIG_OMAP_INTERCONNECT)	+= omap_l3_smx.o omap_l3_noc.o
 
diff --git a/drivers/bus/cdx/Kconfig b/drivers/bus/cdx/Kconfig
new file mode 100644
index 000000000000..c1eed5225328
--- /dev/null
+++ b/drivers/bus/cdx/Kconfig
@@ -0,0 +1,7 @@
+config CDX_BUS
+	bool "CDX Bus driver"
+	help
+		Driver to enable CDX Bus infrastructure. CDX bus is
+		capable of scanning devices dynamically, supporting
+		rescanning of dynamically added, removed or updated
+		devices.
diff --git a/drivers/bus/cdx/Makefile b/drivers/bus/cdx/Makefile
new file mode 100644
index 000000000000..e91bfe706294
--- /dev/null
+++ b/drivers/bus/cdx/Makefile
@@ -0,0 +1,3 @@
+obj-$(CONFIG_CDX_BUS) += cdx-bus-driver.o
+
+cdx-bus-driver-objs := cdx.o cdx_msi.o mcdi_stubs.o
diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
new file mode 100644
index 000000000000..fc417c32c59b
--- /dev/null
+++ b/drivers/bus/cdx/cdx.c
@@ -0,0 +1,437 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Platform driver for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/property.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-map-ops.h>
+#include <linux/property.h>
+#include <linux/iommu.h>
+#include <linux/cdx/cdx_bus.h>
+
+#include "cdx.h"
+#include "mcdi_stubs.h"
+
+/*
+ * Default DMA mask for devices on a CDX bus
+ */
+#define CDX_DEFAULT_DMA_MASK	(~0ULL)
+
+struct platform_device *cdx_controller_pdev;
+
+static int cdx_bus_device_discovery(struct platform_device *pdev);
+
+static int cdx_unregister_device(struct device *dev,
+				 void * __always_unused data)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	kfree(cdx_dev->driver_override);
+	cdx_dev->driver_override = NULL;
+
+	/*
+	 * Do not free cdx_dev here as it would be freed in
+	 * cdx_device_release() called from within put_device().
+	 */
+	device_del(&cdx_dev->dev);
+	put_device(&cdx_dev->dev);
+
+	return 0;
+}
+
+void cdx_unregister_devices(struct bus_type *bus)
+{
+	int ret;
+
+	/* Reset all the devices attached to cdx bus */
+	ret = bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device);
+	if (ret)
+		pr_err("error in CDX unregister for all devices\n");
+}
+
+/**
+ * cdx_match_one_device - Tell if a CDX device structure has a matching
+ *			  CDX device id structure
+ * @id: single CDX device id structure to match
+ * @dev: the CDX device structure to match against
+ *
+ * Returns the matching cdx_device_id structure or %NULL if there is no match.
+ */
+static inline const struct cdx_device_id *
+cdx_match_one_device(const struct cdx_device_id *id,
+		     const struct cdx_device *dev)
+{
+	if ((id->vendor == dev->vendor) && (id->device == dev->device))
+		return id;
+	return NULL;
+}
+
+/**
+ * cdx_match_id - See if a CDX device matches a given cdx_id table
+ * @ids: array of CDX device ID structures to search in
+ * @dev: the CDX device structure to match against.
+ *
+ * Used by a driver to check whether a CDX device is in its list of
+ * supported devices.  Returns the matching cdx_device_id structure or
+ * %NULL if there is no match.
+ */
+static inline const struct cdx_device_id *
+cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev)
+{
+	if (ids) {
+		while (ids->vendor || ids->device) {
+			if (cdx_match_one_device(ids, dev))
+				return ids;
+			ids++;
+		}
+	}
+	return NULL;
+}
+
+/**
+ * cdx_bus_match - device to driver matching callback
+ * @dev: the cdx device to match against
+ * @drv: the device driver to search for matching cdx device
+ * structures
+ *
+ * Returns 1 on success, 0 otherwise.
+ */
+static int cdx_bus_match(struct device *dev, struct device_driver *drv)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	struct cdx_driver *cdx_drv = to_cdx_driver(drv);
+	const struct cdx_device_id *found_id;
+
+	/* When driver_override is set, only bind to the matching driver */
+	if (cdx_dev->driver_override)
+		return !strcmp(cdx_dev->driver_override, cdx_drv->driver.name);
+
+	found_id = cdx_match_id(cdx_drv->match_id_table, cdx_dev);
+	if (found_id)
+		return true;
+
+	return false;
+}
+
+static int cdx_dma_configure(struct device *dev)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	u32 input_id = cdx_dev->req_id;
+	int ret;
+
+	ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id);
+	if (ret) {
+		dev_err(dev, "of_dma_configure_id() failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+struct bus_type cdx_bus_type = {
+	.name = "cdx",
+	.match = cdx_bus_match,
+	.dma_configure  = cdx_dma_configure,
+};
+EXPORT_SYMBOL_GPL(cdx_bus_type);
+
+static int cdx_driver_probe(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	int error;
+
+	error = cdx_drv->probe(cdx_dev);
+	if (error < 0) {
+		if (error != -EPROBE_DEFER)
+			dev_err(dev, "%s failed: %d\n", __func__, error);
+		return error;
+	}
+
+	return 0;
+}
+
+static int cdx_driver_remove(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	int error;
+
+	error = cdx_drv->remove(cdx_dev);
+	if (error < 0) {
+		dev_err(dev, "%s failed: %d\n", __func__, error);
+		return error;
+	}
+
+	return 0;
+}
+
+static void cdx_driver_shutdown(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	cdx_drv->shutdown(cdx_dev);
+}
+
+/*
+ * __cdx_driver_register - registers a CDX device driver
+ */
+int __cdx_driver_register(struct cdx_driver *cdx_driver,
+			  struct module *owner)
+{
+	int error;
+
+	cdx_driver->driver.owner = owner;
+	cdx_driver->driver.bus = &cdx_bus_type;
+
+	if (cdx_driver->probe)
+		cdx_driver->driver.probe = cdx_driver_probe;
+
+	if (cdx_driver->remove)
+		cdx_driver->driver.remove = cdx_driver_remove;
+
+	if (cdx_driver->shutdown)
+		cdx_driver->driver.shutdown = cdx_driver_shutdown;
+
+	error = driver_register(&cdx_driver->driver);
+	if (error < 0) {
+		pr_err("driver_register() failed for %s: %d\n",
+		       cdx_driver->driver.name, error);
+		return error;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(__cdx_driver_register);
+
+/*
+ * cdx_driver_unregister - unregisters a device driver from the
+ * CDX bus
+ */
+void cdx_driver_unregister(struct cdx_driver *cdx_driver)
+{
+	driver_unregister(&cdx_driver->driver);
+}
+EXPORT_SYMBOL_GPL(cdx_driver_unregister);
+
+static void cdx_device_release(struct device *dev)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	kfree(cdx_dev);
+}
+
+static int cdx_device_add(struct device *parent,
+			  struct cdx_dev_params_t *dev_params)
+{
+	struct cdx_device *cdx_dev;
+	int ret;
+
+	cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
+	if (!cdx_dev) {
+		dev_err(parent,
+			"memory allocation for cdx dev failed\n");
+		return -ENOMEM;
+	}
+
+	/* Populate resource */
+	memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
+		dev_params->res_count);
+	cdx_dev->res_count = dev_params->res_count;
+
+	/* Populate CDX dev params */
+	cdx_dev->req_id = dev_params->req_id;
+	cdx_dev->vendor = dev_params->vendor;
+	cdx_dev->device = dev_params->device;
+	cdx_dev->bus_id = dev_params->bus_id;
+	cdx_dev->func_id = dev_params->func_id;
+	cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
+
+	/* Initiaize generic device */
+	device_initialize(&cdx_dev->dev);
+	cdx_dev->dev.parent = parent;
+	cdx_dev->dev.bus = &cdx_bus_type;
+	cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
+	cdx_dev->dev.release = cdx_device_release;
+
+	/* Set Name */
+	dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", cdx_dev->bus_id,
+			cdx_dev->func_id);
+
+	ret = device_add(&cdx_dev->dev);
+	if (ret != 0) {
+		dev_err(&cdx_dev->dev,
+			"cdx device add failed: %d", ret);
+		goto fail;
+	}
+
+	return 0;
+
+fail:
+	/*
+	 * Do not free cdx_dev here as it would be freed in
+	 * cdx_device_release() called from within put_device().
+	 */
+	put_device(&cdx_dev->dev);
+
+	return ret;
+}
+
+static int cdx_bus_device_discovery(struct platform_device *pdev)
+{
+	struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(pdev);
+	int num_cdx_bus, num_cdx_func;
+	uint8_t bus_id, func_id;
+	int ret;
+
+	cdx_controller_pdev = pdev;
+
+	/* MCDI FW Read: Fetch the number of CDX buses present*/
+	num_cdx_bus = cdx_mcdi_get_num_buses(cdx_mcdi);
+
+	for (bus_id = 0; bus_id < num_cdx_bus; bus_id++) {
+		/* MCDI FW Read: Fetch the number of devices present */
+		num_cdx_func = cdx_mcdi_get_num_funcs(cdx_mcdi, bus_id);
+
+		for (func_id = 0; func_id < num_cdx_func; func_id++) {
+			struct cdx_dev_params_t dev_params;
+
+			/* MCDI FW: Get the device config */
+			ret = cdx_mcdi_get_func_config(cdx_mcdi, bus_id,
+					func_id, &dev_params);
+			if (ret) {
+				dev_err(&pdev->dev,
+					"CDX device config get failed for bus: %d\n",
+					ret);
+				goto fail;
+			}
+
+			/* Add the device to the cdx bus */
+			ret = cdx_device_add(&pdev->dev, &dev_params);
+			if (ret == -EPROBE_DEFER) {
+				goto fail;
+			} else if (ret) {
+				dev_err(&pdev->dev,
+					"registering cdx dev: %d failed: %d\n",
+					func_id, ret);
+				goto fail;
+			} else {
+				dev_dbg(&pdev->dev,
+					"CDX dev: %d on cdx bus: %d created\n",
+					func_id, bus_id);
+			}
+		}
+	}
+
+	return 0;
+fail:
+	cdx_unregister_devices(&cdx_bus_type);
+	return ret;
+}
+
+static int cdx_bus_probe(struct platform_device *pdev)
+{
+	struct cdx_mcdi_t *cdx_mcdi;
+	int ret;
+
+	cdx_mcdi = kzalloc(sizeof(*cdx_mcdi), GFP_KERNEL);
+	if (!cdx_mcdi) {
+		dev_err(&pdev->dev, "Failed to allocate memory for cdx_mcdi\n");
+		return -ENOMEM;
+	}
+
+	/* MCDI FW: Initialize the FW path */
+	ret = cdx_mcdi_init(cdx_mcdi);
+	if (ret) {
+		dev_err(&pdev->dev, "MCDI Initialization failed: %d\n", ret);
+		goto mcdi_init_fail;
+	}
+	platform_set_drvdata(pdev, cdx_mcdi);
+
+	/* Discover all the devices on the bus */
+	ret = cdx_bus_device_discovery(pdev);
+	if (ret) {
+		dev_err(&pdev->dev,
+			"CDX bus device discovery failed: %d\n", ret);
+		goto device_discovery_fail;
+	}
+
+	return 0;
+
+mcdi_init_fail:
+	kfree(cdx_mcdi);
+device_discovery_fail:
+	cdx_mcdi_finish(cdx_mcdi);
+
+	return ret;
+}
+
+static int cdx_bus_remove(struct platform_device *pdev)
+{
+	struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(pdev);
+
+	cdx_unregister_devices(&cdx_bus_type);
+
+	cdx_mcdi_finish(cdx_mcdi);
+	kfree(cdx_mcdi);
+
+	return 0;
+}
+
+static void cdx_bus_shutdown(struct platform_device *pdev)
+{
+	cdx_bus_remove(pdev);
+}
+
+static const struct of_device_id cdx_match_table[] = {
+	{.compatible = "xlnx,cdxbus-controller-1.0",},
+	{ },
+};
+
+MODULE_DEVICE_TABLE(of, cdx_match_table);
+
+static struct platform_driver cdx_bus_driver = {
+	.driver = {
+		   .name = "cdx-bus",
+		   .pm = NULL,
+		   .of_match_table = cdx_match_table,
+		   },
+	.probe = cdx_bus_probe,
+	.remove = cdx_bus_remove,
+	.shutdown = cdx_bus_shutdown,
+};
+
+static int __init cdx_bus_driver_init(void)
+{
+	int ret;
+
+	ret = bus_register(&cdx_bus_type);
+	if (ret < 0) {
+		pr_err("bus type registration failed for CDX: %d\n", ret);
+		return ret;
+	}
+
+	ret = platform_driver_register(&cdx_bus_driver);
+	if (ret < 0) {
+		pr_err("platform_driver_register() failed: %d\n", ret);
+		goto fail;
+	}
+
+	return 0;
+
+fail:
+	bus_unregister(&cdx_bus_type);
+	return ret;
+}
+postcore_initcall(cdx_bus_driver_init);
diff --git a/drivers/bus/cdx/cdx.h b/drivers/bus/cdx/cdx.h
new file mode 100644
index 000000000000..db0569431c10
--- /dev/null
+++ b/drivers/bus/cdx/cdx.h
@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Header file for the CDX Bus
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _CDX_H_
+#define _CDX_H_
+
+#include <linux/cdx/cdx_bus.h>
+#include <linux/irqdomain.h>
+
+/**
+ * struct cdx_dev_params_t - CDX device parameters
+ * @vendor: Vendor ID for CDX device
+ * @device: Device ID for CDX device
+ * @bus_id: Bus ID for this CDX device
+ * @func_id: Function ID for this device
+ * @res: array of MMIO region entries
+ * @res_count: number of valid MMIO regions
+ * @req_id: Requestor ID associated with CDX device
+ */
+struct cdx_dev_params_t {
+	u16 vendor;
+	u16 device;
+	u8 bus_id;
+	u8 func_id;
+	struct resource res[MAX_CDX_DEV_RESOURCES];
+	u8 res_count;
+	u32 req_id;
+};
+
+#endif /* _CDX_H_ */
diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
new file mode 100644
index 000000000000..cc9d30fa02f8
--- /dev/null
+++ b/drivers/bus/cdx/mcdi_stubs.c
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * MCDI Firmware interaction for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/ioport.h>
+
+#include "cdx.h"
+#include "mcdi_stubs.h"
+
+int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi)
+{
+	cdx_mcdi->id = 0;
+	cdx_mcdi->flags = 0;
+
+	return 0;
+}
+
+void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi)
+{
+}
+
+int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi)
+{
+	return 1;
+}
+
+int cdx_mcdi_get_num_funcs(struct cdx_mcdi_t *cdx_mcdi, int bus_num)
+{
+	return 1;
+}
+
+int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
+			     uint8_t bus_id, uint8_t func_id,
+			     struct cdx_dev_params_t *dev_params)
+{
+	dev_params->res[0].start = 0xe4020000;
+	dev_params->res[0].end = 0xe4020FFF;
+	dev_params->res[0].flags = IORESOURCE_MEM;
+	dev_params->res[1].start = 0xe4100000;
+	dev_params->res[1].end = 0xE411FFFF;
+	dev_params->res[1].flags = IORESOURCE_MEM;
+	dev_params->res_count = 2;
+
+	dev_params->req_id = 0x250;
+	dev_params->vendor = 0x10ee;
+	dev_params->device = 0x8084;
+	dev_params->bus_id = bus_id;
+	dev_params->func_id = func_id;
+
+	return 0;
+}
diff --git a/drivers/bus/cdx/mcdi_stubs.h b/drivers/bus/cdx/mcdi_stubs.h
new file mode 100644
index 000000000000..7b6f79d48ce9
--- /dev/null
+++ b/drivers/bus/cdx/mcdi_stubs.h
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Header file for MCDI FW interaction for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _MCDI_STUBS_H_
+#define _MCDI_STUBS_H_
+
+#include "cdx.h"
+
+/**
+ * struct cdx_mcdi_t - CDX MCDI Firmware interface, to interact
+ *		       with CDX controller.
+ * @id: ID for MCDI Firmware interface
+ * @flags: Associated flags
+ */
+struct cdx_mcdi_t {
+	u32 id;
+	u32 flags;
+	/* Have more MCDI interface related data */
+};
+
+/**
+ * cdx_mcdi_init - Initialize the MCDI Firmware interface
+ *	for the CDX controller.
+ * @cdx_mcdi: pointer to MCDI interface
+ *
+ * Return 0 on success, <0 on failure
+ */
+int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_finish - Close the MCDI Firmware interface.
+ * @cdx_mcdi: pointer to MCDI interface
+ */
+void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_get_num_buses - Get the total number of busses on
+ *	the controller.
+ * @cdx_mcdi: pointer to MCDI interface.
+ *
+ * Return total number of busses available on the controller,
+ *	<0 on failure
+ */
+int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_get_num_funcs - Get the total number of functions on
+ *	a particular bus of the controller.
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ *
+ * Return total number of functions available on the bus, <0 on failure
+ */
+int cdx_mcdi_get_num_funcs(struct cdx_mcdi_t *cdx_mcdi, int bus_num);
+
+/**
+ * cdx_mcdi_get_func_config - Get configuration for a particular
+ *	bus_id:func_id
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @func_id: Function number.
+ * @dev_params: Pointer to cdx_dev_params_t, this is populated by this
+ *	function with the configuration corresponding to the provided
+ *	bus_id:func_id.
+ *
+ * Return 0 total number of functions available on the bus, <0 on failure
+ */
+int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
+			     uint8_t bus_id, uint8_t func_id,
+			     struct cdx_dev_params_t *dev_params);
+
+#endif /* _MCDI_STUBS_H_ */
diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
new file mode 100644
index 000000000000..6e870b2c87d9
--- /dev/null
+++ b/include/linux/cdx/cdx_bus.h
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CDX bus public interface
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ *
+ */
+#ifndef _CDX_BUS_H_
+#define _CDX_BUS_H_
+
+#include <linux/mod_devicetable.h>
+#include <linux/device.h>
+
+#define MAX_CDX_DEV_RESOURCES	6
+
+/**
+ * struct cdx_device - CDX device object
+ * @dev: Linux driver model device object
+ * @vendor: Vendor ID for CDX device
+ * @device: Device ID for CDX device
+ * @bus_id: Bus ID for this CDX device
+ * @func_id: Function ID for this device
+ * @res: array of MMIO region entries
+ * @res_count: number of valid MMIO regions
+ * @dma_mask: Default DMA mask
+ * @flags: CDX device flags
+ * @req_id: Requestor ID associated with CDX device
+ * @driver_override: driver name to force a match; do not set directly,
+ *                   because core frees it; use driver_set_override() to
+ *                   set or clear it.
+ */
+struct cdx_device {
+	struct device dev;
+	u16 vendor;
+	u16 device;
+	u8 bus_id;
+	u8 func_id;
+	struct resource res[MAX_CDX_DEV_RESOURCES];
+	u8 res_count;
+	u64 dma_mask;
+	u16 flags;
+	u32 req_id;
+	const char *driver_override;
+};
+
+#define to_cdx_device(_dev) \
+	container_of(_dev, struct cdx_device, dev)
+
+/**
+ * struct cdx_driver - CDX device driver
+ * @driver: Generic device driver
+ * @match_id_table: table of supported device matching Ids
+ * @probe: Function called when a device is added
+ * @remove: Function called when a device is removed
+ * @shutdown: Function called at shutdown time to quiesce the device
+ * @suspend: Function called when a device is stopped
+ * @resume: Function called when a device is resumed
+ * @driver_managed_dma: Device driver doesn't use kernel DMA API for DMA.
+ *		For most device drivers, no need to care about this flag
+ *		as long as all DMAs are handled through the kernel DMA API.
+ *		For some special ones, for example VFIO drivers, they know
+ *		how to manage the DMA themselves and set this flag so that
+ *		the IOMMU layer will allow them to setup and manage their
+ *		own I/O address space.
+ */
+struct cdx_driver {
+	struct device_driver driver;
+	const struct cdx_device_id *match_id_table;
+	int (*probe)(struct cdx_device *dev);
+	int (*remove)(struct cdx_device *dev);
+	void (*shutdown)(struct cdx_device *dev);
+	int (*suspend)(struct cdx_device *dev, pm_message_t state);
+	int (*resume)(struct cdx_device *dev);
+	bool driver_managed_dma;
+};
+
+#define to_cdx_driver(_drv) \
+	container_of(_drv, struct cdx_driver, driver)
+
+/*
+ * Macro to avoid include chaining to get THIS_MODULE
+ */
+#define cdx_driver_register(drv) \
+	__cdx_driver_register(drv, THIS_MODULE)
+
+int __must_check __cdx_driver_register(struct cdx_driver *cdx_driver,
+				       struct module *owner);
+
+void cdx_driver_unregister(struct cdx_driver *driver);
+
+extern struct bus_type cdx_bus_type;
+
+#endif /* _CDX_BUS_H_ */
diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h
index 549590e9c644..9b94be83d53e 100644
--- a/include/linux/mod_devicetable.h
+++ b/include/linux/mod_devicetable.h
@@ -911,4 +911,17 @@ struct ishtp_device_id {
 	kernel_ulong_t driver_data;
 };
 
+/**
+ * struct cdx_device_id - CDX device identifier
+ * @vendor: vendor ID
+ * @device: Device ID
+ *
+ * Type of entries in the "device Id" table for CDX devices supported by
+ * a CDX device driver.
+ */
+struct cdx_device_id {
+	__u16 vendor;
+	__u16 device;
+};
+
 #endif /* LINUX_MOD_DEVICETABLE_H */
diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c
index c0d3bcb99138..62dc988df84d 100644
--- a/scripts/mod/devicetable-offsets.c
+++ b/scripts/mod/devicetable-offsets.c
@@ -262,5 +262,9 @@ int main(void)
 	DEVID(ishtp_device_id);
 	DEVID_FIELD(ishtp_device_id, guid);
 
+	DEVID(cdx_device_id);
+	DEVID_FIELD(cdx_device_id, vendor);
+	DEVID_FIELD(cdx_device_id, device);
+
 	return 0;
 }
diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c
index 80d973144fde..c36e1f624e39 100644
--- a/scripts/mod/file2alias.c
+++ b/scripts/mod/file2alias.c
@@ -1452,6 +1452,17 @@ static int do_dfl_entry(const char *filename, void *symval, char *alias)
 	return 1;
 }
 
+/* Looks like: cdx:vNdN */
+static int do_cdx_entry(const char *filename, void *symval,
+			   char *alias)
+{
+	DEF_FIELD(symval, cdx_device_id, vendor);
+	DEF_FIELD(symval, cdx_device_id, device);
+
+	sprintf(alias, "cdx:v%08Xd%08Xd", vendor, device);
+	return 1;
+}
+
 /* Does namelen bytes of name exactly match the symbol? */
 static bool sym_is(const char *name, unsigned namelen, const char *symbol)
 {
@@ -1531,6 +1542,7 @@ static const struct devtable devtable[] = {
 	{"ssam", SIZE_ssam_device_id, do_ssam_entry},
 	{"dfl", SIZE_dfl_device_id, do_dfl_entry},
 	{"ishtp", SIZE_ishtp_device_id, do_ishtp_entry},
+	{"cdx", SIZE_cdx_device_id, do_cdx_entry},
 };
 
 /* Create MODULE_ALIAS() statements.
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver

[Saravana Kannan]

On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
>
> CDX bus supports the scanning and probing of FPGA based
> devices. These devices are registers as CDX devices.
>
> The bus driver sets up the basic infrastructure and fetches
> the device related information from the firmware.
>
> CDX bus is capable of scanning devices dynamically,
> supporting rescanning of dynamically added, removed or
> updated devices.
>
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> ---
>
> Basic overview of CDX bus architecture is provided in [patch 0/7].
>
> Please NOTE: This RFC change does not support the CDX bus firmware
> interface as it is under development, and this series aims to get
> an early feedback from the community. Firmware interaction are
> stubbed as MCDI APIs which is a protocol used by AMD to interact
> with Firmware.
>
>  MAINTAINERS                       |   2 +
>  drivers/bus/Kconfig               |   1 +
>  drivers/bus/Makefile              |   3 +
>  drivers/bus/cdx/Kconfig           |   7 +
>  drivers/bus/cdx/Makefile          |   3 +
>  drivers/bus/cdx/cdx.c             | 437 ++++++++++++++++++++++++++++++
>  drivers/bus/cdx/cdx.h             |  34 +++
>  drivers/bus/cdx/mcdi_stubs.c      |  54 ++++
>  drivers/bus/cdx/mcdi_stubs.h      |  76 ++++++
>  include/linux/cdx/cdx_bus.h       |  93 +++++++
>  include/linux/mod_devicetable.h   |  13 +
>  scripts/mod/devicetable-offsets.c |   4 +
>  scripts/mod/file2alias.c          |  12 +
>  13 files changed, 739 insertions(+)
>  create mode 100644 drivers/bus/cdx/Kconfig
>  create mode 100644 drivers/bus/cdx/Makefile
>  create mode 100644 drivers/bus/cdx/cdx.c
>  create mode 100644 drivers/bus/cdx/cdx.h
>  create mode 100644 drivers/bus/cdx/mcdi_stubs.c
>  create mode 100644 drivers/bus/cdx/mcdi_stubs.h
>  create mode 100644 include/linux/cdx/cdx_bus.h
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index f0598b3d731c..61af11c9fe06 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -939,6 +939,8 @@ M:  Nipun Gupta <nipun.gupta@amd.com>
>  M:     Nikhil Agarwal <nikhil.agarwal@amd.com>
>  S:     Maintained
>  F:     Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> +F:     drivers/bus/cdx/*
> +F:     include/linux/cdx/*
>
>  AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER - SEV SUPPORT
>  M:     Brijesh Singh <brijesh.singh@amd.com>
> diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
> index 7bfe998f3514..b0324efb9a6a 100644
> --- a/drivers/bus/Kconfig
> +++ b/drivers/bus/Kconfig
> @@ -251,5 +251,6 @@ config DA8XX_MSTPRI
>
>  source "drivers/bus/fsl-mc/Kconfig"
>  source "drivers/bus/mhi/Kconfig"
> +source "drivers/bus/cdx/Kconfig"
>
>  endmenu
> diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
> index d90eed189a65..88649111c395 100644
> --- a/drivers/bus/Makefile
> +++ b/drivers/bus/Makefile
> @@ -20,6 +20,9 @@ obj-$(CONFIG_INTEL_IXP4XX_EB) += intel-ixp4xx-eb.o
>  obj-$(CONFIG_MIPS_CDMM)                += mips_cdmm.o
>  obj-$(CONFIG_MVEBU_MBUS)       += mvebu-mbus.o
>
> +#CDX bus
> +obj-$(CONFIG_CDX_BUS)          += cdx/
> +
>  # Interconnect bus driver for OMAP SoCs.
>  obj-$(CONFIG_OMAP_INTERCONNECT)        += omap_l3_smx.o omap_l3_noc.o
>
> diff --git a/drivers/bus/cdx/Kconfig b/drivers/bus/cdx/Kconfig
> new file mode 100644
> index 000000000000..c1eed5225328
> --- /dev/null
> +++ b/drivers/bus/cdx/Kconfig
> @@ -0,0 +1,7 @@
> +config CDX_BUS
> +       bool "CDX Bus driver"
> +       help
> +               Driver to enable CDX Bus infrastructure. CDX bus is
> +               capable of scanning devices dynamically, supporting
> +               rescanning of dynamically added, removed or updated
> +               devices.
> diff --git a/drivers/bus/cdx/Makefile b/drivers/bus/cdx/Makefile
> new file mode 100644
> index 000000000000..e91bfe706294
> --- /dev/null
> +++ b/drivers/bus/cdx/Makefile
> @@ -0,0 +1,3 @@
> +obj-$(CONFIG_CDX_BUS) += cdx-bus-driver.o
> +
> +cdx-bus-driver-objs := cdx.o cdx_msi.o mcdi_stubs.o
> diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
> new file mode 100644
> index 000000000000..fc417c32c59b
> --- /dev/null
> +++ b/drivers/bus/cdx/cdx.c
> @@ -0,0 +1,437 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Platform driver for CDX bus.
> + *
> + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> + */
> +
> +#include <linux/init.h>
> +#include <linux/module.h>
> +#include <linux/kernel.h>
> +#include <linux/of.h>
> +#include <linux/property.h>
> +#include <linux/of_address.h>
> +#include <linux/of_irq.h>
> +#include <linux/of_platform.h>
> +#include <linux/dma-mapping.h>
> +#include <linux/dma-map-ops.h>
> +#include <linux/property.h>
> +#include <linux/iommu.h>
> +#include <linux/cdx/cdx_bus.h>
> +
> +#include "cdx.h"
> +#include "mcdi_stubs.h"
> +
> +/*
> + * Default DMA mask for devices on a CDX bus
> + */
> +#define CDX_DEFAULT_DMA_MASK   (~0ULL)
> +
> +struct platform_device *cdx_controller_pdev;
> +
> +static int cdx_bus_device_discovery(struct platform_device *pdev);
> +
> +static int cdx_unregister_device(struct device *dev,
> +                                void * __always_unused data)
> +{
> +       struct cdx_device *cdx_dev = to_cdx_device(dev);
> +
> +       kfree(cdx_dev->driver_override);
> +       cdx_dev->driver_override = NULL;
> +
> +       /*
> +        * Do not free cdx_dev here as it would be freed in
> +        * cdx_device_release() called from within put_device().
> +        */
> +       device_del(&cdx_dev->dev);
> +       put_device(&cdx_dev->dev);
> +
> +       return 0;
> +}
> +
> +void cdx_unregister_devices(struct bus_type *bus)
> +{
> +       int ret;
> +
> +       /* Reset all the devices attached to cdx bus */
> +       ret = bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device);
> +       if (ret)
> +               pr_err("error in CDX unregister for all devices\n");
> +}
> +
> +/**
> + * cdx_match_one_device - Tell if a CDX device structure has a matching
> + *                       CDX device id structure
> + * @id: single CDX device id structure to match
> + * @dev: the CDX device structure to match against
> + *
> + * Returns the matching cdx_device_id structure or %NULL if there is no match.
> + */
> +static inline const struct cdx_device_id *
> +cdx_match_one_device(const struct cdx_device_id *id,
> +                    const struct cdx_device *dev)
> +{
> +       if ((id->vendor == dev->vendor) && (id->device == dev->device))
> +               return id;
> +       return NULL;
> +}
> +
> +/**
> + * cdx_match_id - See if a CDX device matches a given cdx_id table
> + * @ids: array of CDX device ID structures to search in
> + * @dev: the CDX device structure to match against.
> + *
> + * Used by a driver to check whether a CDX device is in its list of
> + * supported devices.  Returns the matching cdx_device_id structure or
> + * %NULL if there is no match.
> + */
> +static inline const struct cdx_device_id *
> +cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev)
> +{
> +       if (ids) {
> +               while (ids->vendor || ids->device) {
> +                       if (cdx_match_one_device(ids, dev))
> +                               return ids;
> +                       ids++;
> +               }
> +       }
> +       return NULL;
> +}
> +
> +/**
> + * cdx_bus_match - device to driver matching callback
> + * @dev: the cdx device to match against
> + * @drv: the device driver to search for matching cdx device
> + * structures
> + *
> + * Returns 1 on success, 0 otherwise.
> + */
> +static int cdx_bus_match(struct device *dev, struct device_driver *drv)
> +{
> +       struct cdx_device *cdx_dev = to_cdx_device(dev);
> +       struct cdx_driver *cdx_drv = to_cdx_driver(drv);
> +       const struct cdx_device_id *found_id;
> +
> +       /* When driver_override is set, only bind to the matching driver */
> +       if (cdx_dev->driver_override)
> +               return !strcmp(cdx_dev->driver_override, cdx_drv->driver.name);
> +
> +       found_id = cdx_match_id(cdx_drv->match_id_table, cdx_dev);
> +       if (found_id)
> +               return true;
> +
> +       return false;
> +}
> +
> +static int cdx_dma_configure(struct device *dev)
> +{
> +       struct cdx_device *cdx_dev = to_cdx_device(dev);
> +       u32 input_id = cdx_dev->req_id;
> +       int ret;
> +
> +       ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id);
> +       if (ret) {
> +               dev_err(dev, "of_dma_configure_id() failed\n");
> +               return ret;
> +       }
> +
> +       return 0;
> +}
> +
> +struct bus_type cdx_bus_type = {
> +       .name = "cdx",
> +       .match = cdx_bus_match,
> +       .dma_configure  = cdx_dma_configure,
> +};
> +EXPORT_SYMBOL_GPL(cdx_bus_type);
> +
> +static int cdx_driver_probe(struct device *dev)
> +{
> +       struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
> +       struct cdx_device *cdx_dev = to_cdx_device(dev);
> +       int error;
> +
> +       error = cdx_drv->probe(cdx_dev);
> +       if (error < 0) {
> +               if (error != -EPROBE_DEFER)
> +                       dev_err(dev, "%s failed: %d\n", __func__, error);
> +               return error;
> +       }
> +
> +       return 0;
> +}
> +
> +static int cdx_driver_remove(struct device *dev)
> +{
> +       struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
> +       struct cdx_device *cdx_dev = to_cdx_device(dev);
> +       int error;
> +
> +       error = cdx_drv->remove(cdx_dev);
> +       if (error < 0) {
> +               dev_err(dev, "%s failed: %d\n", __func__, error);
> +               return error;
> +       }
> +
> +       return 0;
> +}
> +
> +static void cdx_driver_shutdown(struct device *dev)
> +{
> +       struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
> +       struct cdx_device *cdx_dev = to_cdx_device(dev);
> +
> +       cdx_drv->shutdown(cdx_dev);
> +}
> +
> +/*
> + * __cdx_driver_register - registers a CDX device driver
> + */
> +int __cdx_driver_register(struct cdx_driver *cdx_driver,
> +                         struct module *owner)
> +{
> +       int error;
> +
> +       cdx_driver->driver.owner = owner;
> +       cdx_driver->driver.bus = &cdx_bus_type;
> +
> +       if (cdx_driver->probe)
> +               cdx_driver->driver.probe = cdx_driver_probe;
> +
> +       if (cdx_driver->remove)
> +               cdx_driver->driver.remove = cdx_driver_remove;
> +
> +       if (cdx_driver->shutdown)
> +               cdx_driver->driver.shutdown = cdx_driver_shutdown;
> +
> +       error = driver_register(&cdx_driver->driver);
> +       if (error < 0) {
> +               pr_err("driver_register() failed for %s: %d\n",
> +                      cdx_driver->driver.name, error);
> +               return error;
> +       }
> +
> +       return 0;
> +}
> +EXPORT_SYMBOL_GPL(__cdx_driver_register);
> +
> +/*
> + * cdx_driver_unregister - unregisters a device driver from the
> + * CDX bus
> + */
> +void cdx_driver_unregister(struct cdx_driver *cdx_driver)
> +{
> +       driver_unregister(&cdx_driver->driver);
> +}
> +EXPORT_SYMBOL_GPL(cdx_driver_unregister);
> +
> +static void cdx_device_release(struct device *dev)
> +{
> +       struct cdx_device *cdx_dev = to_cdx_device(dev);
> +
> +       kfree(cdx_dev);
> +}
> +
> +static int cdx_device_add(struct device *parent,
> +                         struct cdx_dev_params_t *dev_params)
> +{
> +       struct cdx_device *cdx_dev;
> +       int ret;
> +
> +       cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
> +       if (!cdx_dev) {
> +               dev_err(parent,
> +                       "memory allocation for cdx dev failed\n");
> +               return -ENOMEM;
> +       }
> +
> +       /* Populate resource */
> +       memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
> +               dev_params->res_count);
> +       cdx_dev->res_count = dev_params->res_count;
> +
> +       /* Populate CDX dev params */
> +       cdx_dev->req_id = dev_params->req_id;
> +       cdx_dev->vendor = dev_params->vendor;
> +       cdx_dev->device = dev_params->device;
> +       cdx_dev->bus_id = dev_params->bus_id;
> +       cdx_dev->func_id = dev_params->func_id;
> +       cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
> +
> +       /* Initiaize generic device */
> +       device_initialize(&cdx_dev->dev);
> +       cdx_dev->dev.parent = parent;
> +       cdx_dev->dev.bus = &cdx_bus_type;
> +       cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
> +       cdx_dev->dev.release = cdx_device_release;

How will these devices get resources like clocks, regulators, etc that
might be provided by other DT based devices? Is that not possible?

I also see that v2 of this series has a "swnode" implementation that
was used to set the fwnode of these CDX devices. Why are these devices
no longer getting the fwnode set? Also, swnode doesn't have support
for the fwnode.add_links() ops. It also doesn't seem to support the
parent of a swnode being another fwnode of a different type (DT). I'm
not sure about the history behind that, but maybe swnode is not the
right fit or you might need to add support for these to swnode. All of
this is to say that if you set these things up correctly, fw_devlink
can work for CDX devices and that might be helpful.

-Saravana
> +
> +       /* Set Name */
> +       dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", cdx_dev->bus_id,
> +                       cdx_dev->func_id);
> +
> +       ret = device_add(&cdx_dev->dev);
> +       if (ret != 0) {
> +               dev_err(&cdx_dev->dev,
> +                       "cdx device add failed: %d", ret);
> +               goto fail;
> +       }
> +
> +       return 0;
> +
> +fail:
> +       /*
> +        * Do not free cdx_dev here as it would be freed in
> +        * cdx_device_release() called from within put_device().
> +        */
> +       put_device(&cdx_dev->dev);
> +
> +       return ret;
> +}
> +
> +static int cdx_bus_device_discovery(struct platform_device *pdev)
> +{
> +       struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(pdev);
> +       int num_cdx_bus, num_cdx_func;
> +       uint8_t bus_id, func_id;
> +       int ret;
> +
> +       cdx_controller_pdev = pdev;
> +
> +       /* MCDI FW Read: Fetch the number of CDX buses present*/
> +       num_cdx_bus = cdx_mcdi_get_num_buses(cdx_mcdi);
> +
> +       for (bus_id = 0; bus_id < num_cdx_bus; bus_id++) {
> +               /* MCDI FW Read: Fetch the number of devices present */
> +               num_cdx_func = cdx_mcdi_get_num_funcs(cdx_mcdi, bus_id);
> +
> +               for (func_id = 0; func_id < num_cdx_func; func_id++) {
> +                       struct cdx_dev_params_t dev_params;
> +
> +                       /* MCDI FW: Get the device config */
> +                       ret = cdx_mcdi_get_func_config(cdx_mcdi, bus_id,
> +                                       func_id, &dev_params);
> +                       if (ret) {
> +                               dev_err(&pdev->dev,
> +                                       "CDX device config get failed for bus: %d\n",
> +                                       ret);
> +                               goto fail;
> +                       }
> +
> +                       /* Add the device to the cdx bus */
> +                       ret = cdx_device_add(&pdev->dev, &dev_params);
> +                       if (ret == -EPROBE_DEFER) {
> +                               goto fail;
> +                       } else if (ret) {
> +                               dev_err(&pdev->dev,
> +                                       "registering cdx dev: %d failed: %d\n",
> +                                       func_id, ret);
> +                               goto fail;
> +                       } else {
> +                               dev_dbg(&pdev->dev,
> +                                       "CDX dev: %d on cdx bus: %d created\n",
> +                                       func_id, bus_id);
> +                       }
> +               }
> +       }
> +
> +       return 0;
> +fail:
> +       cdx_unregister_devices(&cdx_bus_type);
> +       return ret;
> +}
> +
> +static int cdx_bus_probe(struct platform_device *pdev)
> +{
> +       struct cdx_mcdi_t *cdx_mcdi;
> +       int ret;
> +
> +       cdx_mcdi = kzalloc(sizeof(*cdx_mcdi), GFP_KERNEL);
> +       if (!cdx_mcdi) {
> +               dev_err(&pdev->dev, "Failed to allocate memory for cdx_mcdi\n");
> +               return -ENOMEM;
> +       }
> +
> +       /* MCDI FW: Initialize the FW path */
> +       ret = cdx_mcdi_init(cdx_mcdi);
> +       if (ret) {
> +               dev_err(&pdev->dev, "MCDI Initialization failed: %d\n", ret);
> +               goto mcdi_init_fail;
> +       }
> +       platform_set_drvdata(pdev, cdx_mcdi);
> +
> +       /* Discover all the devices on the bus */
> +       ret = cdx_bus_device_discovery(pdev);
> +       if (ret) {
> +               dev_err(&pdev->dev,
> +                       "CDX bus device discovery failed: %d\n", ret);
> +               goto device_discovery_fail;
> +       }
> +
> +       return 0;
> +
> +mcdi_init_fail:
> +       kfree(cdx_mcdi);
> +device_discovery_fail:
> +       cdx_mcdi_finish(cdx_mcdi);
> +
> +       return ret;
> +}
> +
> +static int cdx_bus_remove(struct platform_device *pdev)
> +{
> +       struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(pdev);
> +
> +       cdx_unregister_devices(&cdx_bus_type);
> +
> +       cdx_mcdi_finish(cdx_mcdi);
> +       kfree(cdx_mcdi);
> +
> +       return 0;
> +}
> +
> +static void cdx_bus_shutdown(struct platform_device *pdev)
> +{
> +       cdx_bus_remove(pdev);
> +}
> +
> +static const struct of_device_id cdx_match_table[] = {
> +       {.compatible = "xlnx,cdxbus-controller-1.0",},
> +       { },
> +};
> +
> +MODULE_DEVICE_TABLE(of, cdx_match_table);
> +
> +static struct platform_driver cdx_bus_driver = {
> +       .driver = {
> +                  .name = "cdx-bus",
> +                  .pm = NULL,
> +                  .of_match_table = cdx_match_table,
> +                  },
> +       .probe = cdx_bus_probe,
> +       .remove = cdx_bus_remove,
> +       .shutdown = cdx_bus_shutdown,
> +};
> +
> +static int __init cdx_bus_driver_init(void)
> +{
> +       int ret;
> +
> +       ret = bus_register(&cdx_bus_type);
> +       if (ret < 0) {
> +               pr_err("bus type registration failed for CDX: %d\n", ret);
> +               return ret;
> +       }
> +
> +       ret = platform_driver_register(&cdx_bus_driver);
> +       if (ret < 0) {
> +               pr_err("platform_driver_register() failed: %d\n", ret);
> +               goto fail;
> +       }
> +
> +       return 0;
> +
> +fail:
> +       bus_unregister(&cdx_bus_type);
> +       return ret;
> +}
> +postcore_initcall(cdx_bus_driver_init);
> diff --git a/drivers/bus/cdx/cdx.h b/drivers/bus/cdx/cdx.h
> new file mode 100644
> index 000000000000..db0569431c10
> --- /dev/null
> +++ b/drivers/bus/cdx/cdx.h
> @@ -0,0 +1,34 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Header file for the CDX Bus
> + *
> + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> + */
> +
> +#ifndef _CDX_H_
> +#define _CDX_H_
> +
> +#include <linux/cdx/cdx_bus.h>
> +#include <linux/irqdomain.h>
> +
> +/**
> + * struct cdx_dev_params_t - CDX device parameters
> + * @vendor: Vendor ID for CDX device
> + * @device: Device ID for CDX device
> + * @bus_id: Bus ID for this CDX device
> + * @func_id: Function ID for this device
> + * @res: array of MMIO region entries
> + * @res_count: number of valid MMIO regions
> + * @req_id: Requestor ID associated with CDX device
> + */
> +struct cdx_dev_params_t {
> +       u16 vendor;
> +       u16 device;
> +       u8 bus_id;
> +       u8 func_id;
> +       struct resource res[MAX_CDX_DEV_RESOURCES];
> +       u8 res_count;
> +       u32 req_id;
> +};
> +
> +#endif /* _CDX_H_ */
> diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
> new file mode 100644
> index 000000000000..cc9d30fa02f8
> --- /dev/null
> +++ b/drivers/bus/cdx/mcdi_stubs.c
> @@ -0,0 +1,54 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * MCDI Firmware interaction for CDX bus.
> + *
> + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> + */
> +
> +#include <linux/ioport.h>
> +
> +#include "cdx.h"
> +#include "mcdi_stubs.h"
> +
> +int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi)
> +{
> +       cdx_mcdi->id = 0;
> +       cdx_mcdi->flags = 0;
> +
> +       return 0;
> +}
> +
> +void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi)
> +{
> +}
> +
> +int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi)
> +{
> +       return 1;
> +}
> +
> +int cdx_mcdi_get_num_funcs(struct cdx_mcdi_t *cdx_mcdi, int bus_num)
> +{
> +       return 1;
> +}
> +
> +int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
> +                            uint8_t bus_id, uint8_t func_id,
> +                            struct cdx_dev_params_t *dev_params)
> +{
> +       dev_params->res[0].start = 0xe4020000;
> +       dev_params->res[0].end = 0xe4020FFF;
> +       dev_params->res[0].flags = IORESOURCE_MEM;
> +       dev_params->res[1].start = 0xe4100000;
> +       dev_params->res[1].end = 0xE411FFFF;
> +       dev_params->res[1].flags = IORESOURCE_MEM;
> +       dev_params->res_count = 2;
> +
> +       dev_params->req_id = 0x250;
> +       dev_params->vendor = 0x10ee;
> +       dev_params->device = 0x8084;
> +       dev_params->bus_id = bus_id;
> +       dev_params->func_id = func_id;
> +
> +       return 0;
> +}
> diff --git a/drivers/bus/cdx/mcdi_stubs.h b/drivers/bus/cdx/mcdi_stubs.h
> new file mode 100644
> index 000000000000..7b6f79d48ce9
> --- /dev/null
> +++ b/drivers/bus/cdx/mcdi_stubs.h
> @@ -0,0 +1,76 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Header file for MCDI FW interaction for CDX bus.
> + *
> + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> + */
> +
> +#ifndef _MCDI_STUBS_H_
> +#define _MCDI_STUBS_H_
> +
> +#include "cdx.h"
> +
> +/**
> + * struct cdx_mcdi_t - CDX MCDI Firmware interface, to interact
> + *                    with CDX controller.
> + * @id: ID for MCDI Firmware interface
> + * @flags: Associated flags
> + */
> +struct cdx_mcdi_t {
> +       u32 id;
> +       u32 flags;
> +       /* Have more MCDI interface related data */
> +};
> +
> +/**
> + * cdx_mcdi_init - Initialize the MCDI Firmware interface
> + *     for the CDX controller.
> + * @cdx_mcdi: pointer to MCDI interface
> + *
> + * Return 0 on success, <0 on failure
> + */
> +int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi);
> +
> +/**
> + * cdx_mcdi_finish - Close the MCDI Firmware interface.
> + * @cdx_mcdi: pointer to MCDI interface
> + */
> +void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi);
> +
> +/**
> + * cdx_mcdi_get_num_buses - Get the total number of busses on
> + *     the controller.
> + * @cdx_mcdi: pointer to MCDI interface.
> + *
> + * Return total number of busses available on the controller,
> + *     <0 on failure
> + */
> +int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi);
> +
> +/**
> + * cdx_mcdi_get_num_funcs - Get the total number of functions on
> + *     a particular bus of the controller.
> + * @cdx_mcdi: pointer to MCDI interface.
> + * @bus_num: Bus number.
> + *
> + * Return total number of functions available on the bus, <0 on failure
> + */
> +int cdx_mcdi_get_num_funcs(struct cdx_mcdi_t *cdx_mcdi, int bus_num);
> +
> +/**
> + * cdx_mcdi_get_func_config - Get configuration for a particular
> + *     bus_id:func_id
> + * @cdx_mcdi: pointer to MCDI interface.
> + * @bus_num: Bus number.
> + * @func_id: Function number.
> + * @dev_params: Pointer to cdx_dev_params_t, this is populated by this
> + *     function with the configuration corresponding to the provided
> + *     bus_id:func_id.
> + *
> + * Return 0 total number of functions available on the bus, <0 on failure
> + */
> +int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
> +                            uint8_t bus_id, uint8_t func_id,
> +                            struct cdx_dev_params_t *dev_params);
> +
> +#endif /* _MCDI_STUBS_H_ */
> diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
> new file mode 100644
> index 000000000000..6e870b2c87d9
> --- /dev/null
> +++ b/include/linux/cdx/cdx_bus.h
> @@ -0,0 +1,93 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * CDX bus public interface
> + *
> + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> + *
> + */
> +#ifndef _CDX_BUS_H_
> +#define _CDX_BUS_H_
> +
> +#include <linux/mod_devicetable.h>
> +#include <linux/device.h>
> +
> +#define MAX_CDX_DEV_RESOURCES  6
> +
> +/**
> + * struct cdx_device - CDX device object
> + * @dev: Linux driver model device object
> + * @vendor: Vendor ID for CDX device
> + * @device: Device ID for CDX device
> + * @bus_id: Bus ID for this CDX device
> + * @func_id: Function ID for this device
> + * @res: array of MMIO region entries
> + * @res_count: number of valid MMIO regions
> + * @dma_mask: Default DMA mask
> + * @flags: CDX device flags
> + * @req_id: Requestor ID associated with CDX device
> + * @driver_override: driver name to force a match; do not set directly,
> + *                   because core frees it; use driver_set_override() to
> + *                   set or clear it.
> + */
> +struct cdx_device {
> +       struct device dev;
> +       u16 vendor;
> +       u16 device;
> +       u8 bus_id;
> +       u8 func_id;
> +       struct resource res[MAX_CDX_DEV_RESOURCES];
> +       u8 res_count;
> +       u64 dma_mask;
> +       u16 flags;
> +       u32 req_id;
> +       const char *driver_override;
> +};
> +
> +#define to_cdx_device(_dev) \
> +       container_of(_dev, struct cdx_device, dev)
> +
> +/**
> + * struct cdx_driver - CDX device driver
> + * @driver: Generic device driver
> + * @match_id_table: table of supported device matching Ids
> + * @probe: Function called when a device is added
> + * @remove: Function called when a device is removed
> + * @shutdown: Function called at shutdown time to quiesce the device
> + * @suspend: Function called when a device is stopped
> + * @resume: Function called when a device is resumed
> + * @driver_managed_dma: Device driver doesn't use kernel DMA API for DMA.
> + *             For most device drivers, no need to care about this flag
> + *             as long as all DMAs are handled through the kernel DMA API.
> + *             For some special ones, for example VFIO drivers, they know
> + *             how to manage the DMA themselves and set this flag so that
> + *             the IOMMU layer will allow them to setup and manage their
> + *             own I/O address space.
> + */
> +struct cdx_driver {
> +       struct device_driver driver;
> +       const struct cdx_device_id *match_id_table;
> +       int (*probe)(struct cdx_device *dev);
> +       int (*remove)(struct cdx_device *dev);
> +       void (*shutdown)(struct cdx_device *dev);
> +       int (*suspend)(struct cdx_device *dev, pm_message_t state);
> +       int (*resume)(struct cdx_device *dev);
> +       bool driver_managed_dma;
> +};
> +
> +#define to_cdx_driver(_drv) \
> +       container_of(_drv, struct cdx_driver, driver)
> +
> +/*
> + * Macro to avoid include chaining to get THIS_MODULE
> + */
> +#define cdx_driver_register(drv) \
> +       __cdx_driver_register(drv, THIS_MODULE)
> +
> +int __must_check __cdx_driver_register(struct cdx_driver *cdx_driver,
> +                                      struct module *owner);
> +
> +void cdx_driver_unregister(struct cdx_driver *driver);
> +
> +extern struct bus_type cdx_bus_type;
> +
> +#endif /* _CDX_BUS_H_ */
> diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h
> index 549590e9c644..9b94be83d53e 100644
> --- a/include/linux/mod_devicetable.h
> +++ b/include/linux/mod_devicetable.h
> @@ -911,4 +911,17 @@ struct ishtp_device_id {
>         kernel_ulong_t driver_data;
>  };
>
> +/**
> + * struct cdx_device_id - CDX device identifier
> + * @vendor: vendor ID
> + * @device: Device ID
> + *
> + * Type of entries in the "device Id" table for CDX devices supported by
> + * a CDX device driver.
> + */
> +struct cdx_device_id {
> +       __u16 vendor;
> +       __u16 device;
> +};
> +
>  #endif /* LINUX_MOD_DEVICETABLE_H */
> diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c
> index c0d3bcb99138..62dc988df84d 100644
> --- a/scripts/mod/devicetable-offsets.c
> +++ b/scripts/mod/devicetable-offsets.c
> @@ -262,5 +262,9 @@ int main(void)
>         DEVID(ishtp_device_id);
>         DEVID_FIELD(ishtp_device_id, guid);
>
> +       DEVID(cdx_device_id);
> +       DEVID_FIELD(cdx_device_id, vendor);
> +       DEVID_FIELD(cdx_device_id, device);
> +
>         return 0;
>  }
> diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c
> index 80d973144fde..c36e1f624e39 100644
> --- a/scripts/mod/file2alias.c
> +++ b/scripts/mod/file2alias.c
> @@ -1452,6 +1452,17 @@ static int do_dfl_entry(const char *filename, void *symval, char *alias)
>         return 1;
>  }
>
> +/* Looks like: cdx:vNdN */
> +static int do_cdx_entry(const char *filename, void *symval,
> +                          char *alias)
> +{
> +       DEF_FIELD(symval, cdx_device_id, vendor);
> +       DEF_FIELD(symval, cdx_device_id, device);
> +
> +       sprintf(alias, "cdx:v%08Xd%08Xd", vendor, device);
> +       return 1;
> +}
> +
>  /* Does namelen bytes of name exactly match the symbol? */
>  static bool sym_is(const char *name, unsigned namelen, const char *symbol)
>  {
> @@ -1531,6 +1542,7 @@ static const struct devtable devtable[] = {
>         {"ssam", SIZE_ssam_device_id, do_ssam_entry},
>         {"dfl", SIZE_dfl_device_id, do_dfl_entry},
>         {"ishtp", SIZE_ishtp_device_id, do_ishtp_entry},
> +       {"cdx", SIZE_cdx_device_id, do_cdx_entry},
>  };
>
>  /* Create MODULE_ALIAS() statements.
> --
> 2.25.1
>

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Saravana Kannan <saravanak@google.com>
> Sent: Wednesday, September 7, 2022 6:03 AM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; Michael.Srba@seznam.cz; mani@kernel.org;
> yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com;
> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
> Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver
> 
> [CAUTION: External Email]
> 
> On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
> >
> > CDX bus supports the scanning and probing of FPGA based
> > devices. These devices are registers as CDX devices.
> >
> > The bus driver sets up the basic infrastructure and fetches
> > the device related information from the firmware.
> >
> > CDX bus is capable of scanning devices dynamically,
> > supporting rescanning of dynamically added, removed or
> > updated devices.
> >
> > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> > ---
> >
> > Basic overview of CDX bus architecture is provided in [patch 0/7].
> >
> > Please NOTE: This RFC change does not support the CDX bus firmware
> > interface as it is under development, and this series aims to get
> > an early feedback from the community. Firmware interaction are
> > stubbed as MCDI APIs which is a protocol used by AMD to interact
> > with Firmware.
> >
> >  MAINTAINERS                       |   2 +
> >  drivers/bus/Kconfig               |   1 +
> >  drivers/bus/Makefile              |   3 +
> >  drivers/bus/cdx/Kconfig           |   7 +
> >  drivers/bus/cdx/Makefile          |   3 +
> >  drivers/bus/cdx/cdx.c             | 437 ++++++++++++++++++++++++++++++
> >  drivers/bus/cdx/cdx.h             |  34 +++
> >  drivers/bus/cdx/mcdi_stubs.c      |  54 ++++
> >  drivers/bus/cdx/mcdi_stubs.h      |  76 ++++++
> >  include/linux/cdx/cdx_bus.h       |  93 +++++++
> >  include/linux/mod_devicetable.h   |  13 +
> >  scripts/mod/devicetable-offsets.c |   4 +
> >  scripts/mod/file2alias.c          |  12 +
> >  13 files changed, 739 insertions(+)
> >  create mode 100644 drivers/bus/cdx/Kconfig
> >  create mode 100644 drivers/bus/cdx/Makefile
> >  create mode 100644 drivers/bus/cdx/cdx.c
> >  create mode 100644 drivers/bus/cdx/cdx.h
> >  create mode 100644 drivers/bus/cdx/mcdi_stubs.c
> >  create mode 100644 drivers/bus/cdx/mcdi_stubs.h
> >  create mode 100644 include/linux/cdx/cdx_bus.h
> >

<snip>

> > +
> > +static int cdx_device_add(struct device *parent,
> > +                         struct cdx_dev_params_t *dev_params)
> > +{
> > +       struct cdx_device *cdx_dev;
> > +       int ret;
> > +
> > +       cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
> > +       if (!cdx_dev) {
> > +               dev_err(parent,
> > +                       "memory allocation for cdx dev failed\n");
> > +               return -ENOMEM;
> > +       }
> > +
> > +       /* Populate resource */
> > +       memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
> > +               dev_params->res_count);
> > +       cdx_dev->res_count = dev_params->res_count;
> > +
> > +       /* Populate CDX dev params */
> > +       cdx_dev->req_id = dev_params->req_id;
> > +       cdx_dev->vendor = dev_params->vendor;
> > +       cdx_dev->device = dev_params->device;
> > +       cdx_dev->bus_id = dev_params->bus_id;
> > +       cdx_dev->func_id = dev_params->func_id;
> > +       cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
> > +
> > +       /* Initiaize generic device */
> > +       device_initialize(&cdx_dev->dev);
> > +       cdx_dev->dev.parent = parent;
> > +       cdx_dev->dev.bus = &cdx_bus_type;
> > +       cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
> > +       cdx_dev->dev.release = cdx_device_release;
> 
> How will these devices get resources like clocks, regulators, etc that
> might be provided by other DT based devices? Is that not possible?
> 
> I also see that v2 of this series has a "swnode" implementation that
> was used to set the fwnode of these CDX devices. Why are these devices
> no longer getting the fwnode set? Also, swnode doesn't have support
> for the fwnode.add_links() ops. It also doesn't seem to support the
> parent of a swnode being another fwnode of a different type (DT). I'm
> not sure about the history behind that, but maybe swnode is not the
> right fit or you might need to add support for these to swnode. All of
> this is to say that if you set these things up correctly, fw_devlink
> can work for CDX devices and that might be helpful.

In the RFC v2 we were trying to use the platform bus for CDX devices,
and in that implementation sw_node was used, but now we understand
that using platform bus for dynamic devices is not a good approach.

So, changes in RFC v3 are to have a CDX bus for CDX devices.

Regards,
Nipun

> 
> -Saravana
> > +
> > +       /* Set Name */
> > +       dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", cdx_dev->bus_id,
> > +                       cdx_dev->func_id);
> > +
> > +       ret = device_add(&cdx_dev->dev);
> > +       if (ret != 0) {
> > +               dev_err(&cdx_dev->dev,
> > +                       "cdx device add failed: %d", ret);
> > +               goto fail;
> > +       }
> > +
> > +       return 0;
> > +
> > +fail:
> > +       /*
> > +        * Do not free cdx_dev here as it would be freed in
> > +        * cdx_device_release() called from within put_device().
> > +        */
> > +       put_device(&cdx_dev->dev);
> > +
> > +       return ret;
> > +}
> > +
> > +static int cdx_bus_device_discovery(struct platform_device *pdev)
> > +{
> > +       struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(pdev);
> > +       int num_cdx_bus, num_cdx_func;
> > +       uint8_t bus_id, func_id;
> > +       int ret;
> > +
> > +       cdx_controller_pdev = pdev;
> > +
> > +       /* MCDI FW Read: Fetch the number of CDX buses present*/
> > +       num_cdx_bus = cdx_mcdi_get_num_buses(cdx_mcdi);
> > +
> > +       for (bus_id = 0; bus_id < num_cdx_bus; bus_id++) {
> > +               /* MCDI FW Read: Fetch the number of devices present */
> > +               num_cdx_func = cdx_mcdi_get_num_funcs(cdx_mcdi, bus_id);
> > +
> > +               for (func_id = 0; func_id < num_cdx_func; func_id++) {
> > +                       struct cdx_dev_params_t dev_params;
> > +
> > +                       /* MCDI FW: Get the device config */
> > +                       ret = cdx_mcdi_get_func_config(cdx_mcdi, bus_id,
> > +                                       func_id, &dev_params);
> > +                       if (ret) {
> > +                               dev_err(&pdev->dev,
> > +                                       "CDX device config get failed for bus: %d\n",
> > +                                       ret);
> > +                               goto fail;
> > +                       }
> > +
> > +                       /* Add the device to the cdx bus */
> > +                       ret = cdx_device_add(&pdev->dev, &dev_params);
> > +                       if (ret == -EPROBE_DEFER) {
> > +                               goto fail;
> > +                       } else if (ret) {
> > +                               dev_err(&pdev->dev,
> > +                                       "registering cdx dev: %d failed: %d\n",
> > +                                       func_id, ret);
> > +                               goto fail;
> > +                       } else {
> > +                               dev_dbg(&pdev->dev,
> > +                                       "CDX dev: %d on cdx bus: %d created\n",
> > +                                       func_id, bus_id);
> > +                       }
> > +               }
> > +       }
> > +
> > +       return 0;
> > +fail:
> > +       cdx_unregister_devices(&cdx_bus_type);
> > +       return ret;
> > +}
> > +
> > +static int cdx_bus_probe(struct platform_device *pdev)
> > +{
> > +       struct cdx_mcdi_t *cdx_mcdi;
> > +       int ret;
> > +
> > +       cdx_mcdi = kzalloc(sizeof(*cdx_mcdi), GFP_KERNEL);
> > +       if (!cdx_mcdi) {
> > +               dev_err(&pdev->dev, "Failed to allocate memory for cdx_mcdi\n");
> > +               return -ENOMEM;
> > +       }
> > +
> > +       /* MCDI FW: Initialize the FW path */
> > +       ret = cdx_mcdi_init(cdx_mcdi);
> > +       if (ret) {
> > +               dev_err(&pdev->dev, "MCDI Initialization failed: %d\n", ret);
> > +               goto mcdi_init_fail;
> > +       }
> > +       platform_set_drvdata(pdev, cdx_mcdi);
> > +
> > +       /* Discover all the devices on the bus */
> > +       ret = cdx_bus_device_discovery(pdev);
> > +       if (ret) {
> > +               dev_err(&pdev->dev,
> > +                       "CDX bus device discovery failed: %d\n", ret);
> > +               goto device_discovery_fail;
> > +       }
> > +
> > +       return 0;
> > +
> > +mcdi_init_fail:
> > +       kfree(cdx_mcdi);
> > +device_discovery_fail:
> > +       cdx_mcdi_finish(cdx_mcdi);
> > +
> > +       return ret;
> > +}
> > +
> > +static int cdx_bus_remove(struct platform_device *pdev)
> > +{
> > +       struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(pdev);
> > +
> > +       cdx_unregister_devices(&cdx_bus_type);
> > +
> > +       cdx_mcdi_finish(cdx_mcdi);
> > +       kfree(cdx_mcdi);
> > +
> > +       return 0;
> > +}
> > +
> > +static void cdx_bus_shutdown(struct platform_device *pdev)
> > +{
> > +       cdx_bus_remove(pdev);
> > +}
> > +
> > +static const struct of_device_id cdx_match_table[] = {
> > +       {.compatible = "xlnx,cdxbus-controller-1.0",},
> > +       { },
> > +};
> > +
> > +MODULE_DEVICE_TABLE(of, cdx_match_table);
> > +
> > +static struct platform_driver cdx_bus_driver = {
> > +       .driver = {
> > +                  .name = "cdx-bus",
> > +                  .pm = NULL,
> > +                  .of_match_table = cdx_match_table,
> > +                  },
> > +       .probe = cdx_bus_probe,
> > +       .remove = cdx_bus_remove,
> > +       .shutdown = cdx_bus_shutdown,
> > +};
> > +
> > +static int __init cdx_bus_driver_init(void)
> > +{
> > +       int ret;
> > +
> > +       ret = bus_register(&cdx_bus_type);
> > +       if (ret < 0) {
> > +               pr_err("bus type registration failed for CDX: %d\n", ret);
> > +               return ret;
> > +       }
> > +
> > +       ret = platform_driver_register(&cdx_bus_driver);
> > +       if (ret < 0) {
> > +               pr_err("platform_driver_register() failed: %d\n", ret);
> > +               goto fail;
> > +       }
> > +
> > +       return 0;
> > +
> > +fail:
> > +       bus_unregister(&cdx_bus_type);
> > +       return ret;
> > +}
> > +postcore_initcall(cdx_bus_driver_init);
> > diff --git a/drivers/bus/cdx/cdx.h b/drivers/bus/cdx/cdx.h
> > new file mode 100644
> > index 000000000000..db0569431c10
> > --- /dev/null
> > +++ b/drivers/bus/cdx/cdx.h
> > @@ -0,0 +1,34 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Header file for the CDX Bus
> > + *
> > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > + */
> > +
> > +#ifndef _CDX_H_
> > +#define _CDX_H_
> > +
> > +#include <linux/cdx/cdx_bus.h>
> > +#include <linux/irqdomain.h>
> > +
> > +/**
> > + * struct cdx_dev_params_t - CDX device parameters
> > + * @vendor: Vendor ID for CDX device
> > + * @device: Device ID for CDX device
> > + * @bus_id: Bus ID for this CDX device
> > + * @func_id: Function ID for this device
> > + * @res: array of MMIO region entries
> > + * @res_count: number of valid MMIO regions
> > + * @req_id: Requestor ID associated with CDX device
> > + */
> > +struct cdx_dev_params_t {
> > +       u16 vendor;
> > +       u16 device;
> > +       u8 bus_id;
> > +       u8 func_id;
> > +       struct resource res[MAX_CDX_DEV_RESOURCES];
> > +       u8 res_count;
> > +       u32 req_id;
> > +};
> > +
> > +#endif /* _CDX_H_ */
> > diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
> > new file mode 100644
> > index 000000000000..cc9d30fa02f8
> > --- /dev/null
> > +++ b/drivers/bus/cdx/mcdi_stubs.c
> > @@ -0,0 +1,54 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * MCDI Firmware interaction for CDX bus.
> > + *
> > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > + */
> > +
> > +#include <linux/ioport.h>
> > +
> > +#include "cdx.h"
> > +#include "mcdi_stubs.h"
> > +
> > +int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi)
> > +{
> > +       cdx_mcdi->id = 0;
> > +       cdx_mcdi->flags = 0;
> > +
> > +       return 0;
> > +}
> > +
> > +void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi)
> > +{
> > +}
> > +
> > +int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi)
> > +{
> > +       return 1;
> > +}
> > +
> > +int cdx_mcdi_get_num_funcs(struct cdx_mcdi_t *cdx_mcdi, int bus_num)
> > +{
> > +       return 1;
> > +}
> > +
> > +int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
> > +                            uint8_t bus_id, uint8_t func_id,
> > +                            struct cdx_dev_params_t *dev_params)
> > +{
> > +       dev_params->res[0].start = 0xe4020000;
> > +       dev_params->res[0].end = 0xe4020FFF;
> > +       dev_params->res[0].flags = IORESOURCE_MEM;
> > +       dev_params->res[1].start = 0xe4100000;
> > +       dev_params->res[1].end = 0xE411FFFF;
> > +       dev_params->res[1].flags = IORESOURCE_MEM;
> > +       dev_params->res_count = 2;
> > +
> > +       dev_params->req_id = 0x250;
> > +       dev_params->vendor = 0x10ee;
> > +       dev_params->device = 0x8084;
> > +       dev_params->bus_id = bus_id;
> > +       dev_params->func_id = func_id;
> > +
> > +       return 0;
> > +}
> > diff --git a/drivers/bus/cdx/mcdi_stubs.h b/drivers/bus/cdx/mcdi_stubs.h
> > new file mode 100644
> > index 000000000000..7b6f79d48ce9
> > --- /dev/null
> > +++ b/drivers/bus/cdx/mcdi_stubs.h
> > @@ -0,0 +1,76 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Header file for MCDI FW interaction for CDX bus.
> > + *
> > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > + */
> > +
> > +#ifndef _MCDI_STUBS_H_
> > +#define _MCDI_STUBS_H_
> > +
> > +#include "cdx.h"
> > +
> > +/**
> > + * struct cdx_mcdi_t - CDX MCDI Firmware interface, to interact
> > + *                    with CDX controller.
> > + * @id: ID for MCDI Firmware interface
> > + * @flags: Associated flags
> > + */
> > +struct cdx_mcdi_t {
> > +       u32 id;
> > +       u32 flags;
> > +       /* Have more MCDI interface related data */
> > +};
> > +
> > +/**
> > + * cdx_mcdi_init - Initialize the MCDI Firmware interface
> > + *     for the CDX controller.
> > + * @cdx_mcdi: pointer to MCDI interface
> > + *
> > + * Return 0 on success, <0 on failure
> > + */
> > +int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi);
> > +
> > +/**
> > + * cdx_mcdi_finish - Close the MCDI Firmware interface.
> > + * @cdx_mcdi: pointer to MCDI interface
> > + */
> > +void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi);
> > +
> > +/**
> > + * cdx_mcdi_get_num_buses - Get the total number of busses on
> > + *     the controller.
> > + * @cdx_mcdi: pointer to MCDI interface.
> > + *
> > + * Return total number of busses available on the controller,
> > + *     <0 on failure
> > + */
> > +int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi);
> > +
> > +/**
> > + * cdx_mcdi_get_num_funcs - Get the total number of functions on
> > + *     a particular bus of the controller.
> > + * @cdx_mcdi: pointer to MCDI interface.
> > + * @bus_num: Bus number.
> > + *
> > + * Return total number of functions available on the bus, <0 on failure
> > + */
> > +int cdx_mcdi_get_num_funcs(struct cdx_mcdi_t *cdx_mcdi, int bus_num);
> > +
> > +/**
> > + * cdx_mcdi_get_func_config - Get configuration for a particular
> > + *     bus_id:func_id
> > + * @cdx_mcdi: pointer to MCDI interface.
> > + * @bus_num: Bus number.
> > + * @func_id: Function number.
> > + * @dev_params: Pointer to cdx_dev_params_t, this is populated by this
> > + *     function with the configuration corresponding to the provided
> > + *     bus_id:func_id.
> > + *
> > + * Return 0 total number of functions available on the bus, <0 on failure
> > + */
> > +int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
> > +                            uint8_t bus_id, uint8_t func_id,
> > +                            struct cdx_dev_params_t *dev_params);
> > +
> > +#endif /* _MCDI_STUBS_H_ */
> > diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
> > new file mode 100644
> > index 000000000000..6e870b2c87d9
> > --- /dev/null
> > +++ b/include/linux/cdx/cdx_bus.h
> > @@ -0,0 +1,93 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * CDX bus public interface
> > + *
> > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > + *
> > + */
> > +#ifndef _CDX_BUS_H_
> > +#define _CDX_BUS_H_
> > +
> > +#include <linux/mod_devicetable.h>
> > +#include <linux/device.h>
> > +
> > +#define MAX_CDX_DEV_RESOURCES  6
> > +
> > +/**
> > + * struct cdx_device - CDX device object
> > + * @dev: Linux driver model device object
> > + * @vendor: Vendor ID for CDX device
> > + * @device: Device ID for CDX device
> > + * @bus_id: Bus ID for this CDX device
> > + * @func_id: Function ID for this device
> > + * @res: array of MMIO region entries
> > + * @res_count: number of valid MMIO regions
> > + * @dma_mask: Default DMA mask
> > + * @flags: CDX device flags
> > + * @req_id: Requestor ID associated with CDX device
> > + * @driver_override: driver name to force a match; do not set directly,
> > + *                   because core frees it; use driver_set_override() to
> > + *                   set or clear it.
> > + */
> > +struct cdx_device {
> > +       struct device dev;
> > +       u16 vendor;
> > +       u16 device;
> > +       u8 bus_id;
> > +       u8 func_id;
> > +       struct resource res[MAX_CDX_DEV_RESOURCES];
> > +       u8 res_count;
> > +       u64 dma_mask;
> > +       u16 flags;
> > +       u32 req_id;
> > +       const char *driver_override;
> > +};
> > +
> > +#define to_cdx_device(_dev) \
> > +       container_of(_dev, struct cdx_device, dev)
> > +
> > +/**
> > + * struct cdx_driver - CDX device driver
> > + * @driver: Generic device driver
> > + * @match_id_table: table of supported device matching Ids
> > + * @probe: Function called when a device is added
> > + * @remove: Function called when a device is removed
> > + * @shutdown: Function called at shutdown time to quiesce the device
> > + * @suspend: Function called when a device is stopped
> > + * @resume: Function called when a device is resumed
> > + * @driver_managed_dma: Device driver doesn't use kernel DMA API for
> DMA.
> > + *             For most device drivers, no need to care about this flag
> > + *             as long as all DMAs are handled through the kernel DMA API.
> > + *             For some special ones, for example VFIO drivers, they know
> > + *             how to manage the DMA themselves and set this flag so that
> > + *             the IOMMU layer will allow them to setup and manage their
> > + *             own I/O address space.
> > + */
> > +struct cdx_driver {
> > +       struct device_driver driver;
> > +       const struct cdx_device_id *match_id_table;
> > +       int (*probe)(struct cdx_device *dev);
> > +       int (*remove)(struct cdx_device *dev);
> > +       void (*shutdown)(struct cdx_device *dev);
> > +       int (*suspend)(struct cdx_device *dev, pm_message_t state);
> > +       int (*resume)(struct cdx_device *dev);
> > +       bool driver_managed_dma;
> > +};
> > +
> > +#define to_cdx_driver(_drv) \
> > +       container_of(_drv, struct cdx_driver, driver)
> > +
> > +/*
> > + * Macro to avoid include chaining to get THIS_MODULE
> > + */
> > +#define cdx_driver_register(drv) \
> > +       __cdx_driver_register(drv, THIS_MODULE)
> > +
> > +int __must_check __cdx_driver_register(struct cdx_driver *cdx_driver,
> > +                                      struct module *owner);
> > +
> > +void cdx_driver_unregister(struct cdx_driver *driver);
> > +
> > +extern struct bus_type cdx_bus_type;
> > +
> > +#endif /* _CDX_BUS_H_ */
> > diff --git a/include/linux/mod_devicetable.h
> b/include/linux/mod_devicetable.h
> > index 549590e9c644..9b94be83d53e 100644
> > --- a/include/linux/mod_devicetable.h
> > +++ b/include/linux/mod_devicetable.h
> > @@ -911,4 +911,17 @@ struct ishtp_device_id {
> >         kernel_ulong_t driver_data;
> >  };
> >
> > +/**
> > + * struct cdx_device_id - CDX device identifier
> > + * @vendor: vendor ID
> > + * @device: Device ID
> > + *
> > + * Type of entries in the "device Id" table for CDX devices supported by
> > + * a CDX device driver.
> > + */
> > +struct cdx_device_id {
> > +       __u16 vendor;
> > +       __u16 device;
> > +};
> > +
> >  #endif /* LINUX_MOD_DEVICETABLE_H */
> > diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-
> offsets.c
> > index c0d3bcb99138..62dc988df84d 100644
> > --- a/scripts/mod/devicetable-offsets.c
> > +++ b/scripts/mod/devicetable-offsets.c
> > @@ -262,5 +262,9 @@ int main(void)
> >         DEVID(ishtp_device_id);
> >         DEVID_FIELD(ishtp_device_id, guid);
> >
> > +       DEVID(cdx_device_id);
> > +       DEVID_FIELD(cdx_device_id, vendor);
> > +       DEVID_FIELD(cdx_device_id, device);
> > +
> >         return 0;
> >  }
> > diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c
> > index 80d973144fde..c36e1f624e39 100644
> > --- a/scripts/mod/file2alias.c
> > +++ b/scripts/mod/file2alias.c
> > @@ -1452,6 +1452,17 @@ static int do_dfl_entry(const char *filename, void
> *symval, char *alias)
> >         return 1;
> >  }
> >
> > +/* Looks like: cdx:vNdN */
> > +static int do_cdx_entry(const char *filename, void *symval,
> > +                          char *alias)
> > +{
> > +       DEF_FIELD(symval, cdx_device_id, vendor);
> > +       DEF_FIELD(symval, cdx_device_id, device);
> > +
> > +       sprintf(alias, "cdx:v%08Xd%08Xd", vendor, device);
> > +       return 1;
> > +}
> > +
> >  /* Does namelen bytes of name exactly match the symbol? */
> >  static bool sym_is(const char *name, unsigned namelen, const char *symbol)
> >  {
> > @@ -1531,6 +1542,7 @@ static const struct devtable devtable[] = {
> >         {"ssam", SIZE_ssam_device_id, do_ssam_entry},
> >         {"dfl", SIZE_dfl_device_id, do_dfl_entry},
> >         {"ishtp", SIZE_ishtp_device_id, do_ishtp_entry},
> > +       {"cdx", SIZE_cdx_device_id, do_cdx_entry},
> >  };
> >
> >  /* Create MODULE_ALIAS() statements.
> > --
> > 2.25.1
> >
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver

[Saravana Kannan]

On Tue, Sep 6, 2022 at 8:21 PM Gupta, Nipun <Nipun.Gupta@amd.com> wrote:
>
> [AMD Official Use Only - General]
>
>
>
> > -----Original Message-----
> > From: Saravana Kannan <saravanak@google.com>
> > Sent: Wednesday, September 7, 2022 6:03 AM
> > To: Gupta, Nipun <Nipun.Gupta@amd.com>
> > Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> > gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> > alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> > <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> > mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
> > jeffrey.l.hugo@gmail.com; Michael.Srba@seznam.cz; mani@kernel.org;
> > yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com;
> > will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> > ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> > kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> > devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
> > Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> > <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> > Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> > <git@amd.com>
> > Subject: Re: [RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver
> >
> > [CAUTION: External Email]
> >
> > On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
> > >
> > > CDX bus supports the scanning and probing of FPGA based
> > > devices. These devices are registers as CDX devices.
> > >
> > > The bus driver sets up the basic infrastructure and fetches
> > > the device related information from the firmware.
> > >
> > > CDX bus is capable of scanning devices dynamically,
> > > supporting rescanning of dynamically added, removed or
> > > updated devices.
> > >
> > > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> > > ---
> > >
> > > Basic overview of CDX bus architecture is provided in [patch 0/7].
> > >
> > > Please NOTE: This RFC change does not support the CDX bus firmware
> > > interface as it is under development, and this series aims to get
> > > an early feedback from the community. Firmware interaction are
> > > stubbed as MCDI APIs which is a protocol used by AMD to interact
> > > with Firmware.
> > >
> > >  MAINTAINERS                       |   2 +
> > >  drivers/bus/Kconfig               |   1 +
> > >  drivers/bus/Makefile              |   3 +
> > >  drivers/bus/cdx/Kconfig           |   7 +
> > >  drivers/bus/cdx/Makefile          |   3 +
> > >  drivers/bus/cdx/cdx.c             | 437 ++++++++++++++++++++++++++++++
> > >  drivers/bus/cdx/cdx.h             |  34 +++
> > >  drivers/bus/cdx/mcdi_stubs.c      |  54 ++++
> > >  drivers/bus/cdx/mcdi_stubs.h      |  76 ++++++
> > >  include/linux/cdx/cdx_bus.h       |  93 +++++++
> > >  include/linux/mod_devicetable.h   |  13 +
> > >  scripts/mod/devicetable-offsets.c |   4 +
> > >  scripts/mod/file2alias.c          |  12 +
> > >  13 files changed, 739 insertions(+)
> > >  create mode 100644 drivers/bus/cdx/Kconfig
> > >  create mode 100644 drivers/bus/cdx/Makefile
> > >  create mode 100644 drivers/bus/cdx/cdx.c
> > >  create mode 100644 drivers/bus/cdx/cdx.h
> > >  create mode 100644 drivers/bus/cdx/mcdi_stubs.c
> > >  create mode 100644 drivers/bus/cdx/mcdi_stubs.h
> > >  create mode 100644 include/linux/cdx/cdx_bus.h
> > >
>
> <snip>
>
> > > +
> > > +static int cdx_device_add(struct device *parent,
> > > +                         struct cdx_dev_params_t *dev_params)
> > > +{
> > > +       struct cdx_device *cdx_dev;
> > > +       int ret;
> > > +
> > > +       cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
> > > +       if (!cdx_dev) {
> > > +               dev_err(parent,
> > > +                       "memory allocation for cdx dev failed\n");
> > > +               return -ENOMEM;
> > > +       }
> > > +
> > > +       /* Populate resource */
> > > +       memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
> > > +               dev_params->res_count);
> > > +       cdx_dev->res_count = dev_params->res_count;
> > > +
> > > +       /* Populate CDX dev params */
> > > +       cdx_dev->req_id = dev_params->req_id;
> > > +       cdx_dev->vendor = dev_params->vendor;
> > > +       cdx_dev->device = dev_params->device;
> > > +       cdx_dev->bus_id = dev_params->bus_id;
> > > +       cdx_dev->func_id = dev_params->func_id;
> > > +       cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
> > > +
> > > +       /* Initiaize generic device */
> > > +       device_initialize(&cdx_dev->dev);
> > > +       cdx_dev->dev.parent = parent;
> > > +       cdx_dev->dev.bus = &cdx_bus_type;
> > > +       cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
> > > +       cdx_dev->dev.release = cdx_device_release;
> >
> > How will these devices get resources like clocks, regulators, etc that
> > might be provided by other DT based devices? Is that not possible?
> >
> > I also see that v2 of this series has a "swnode" implementation that
> > was used to set the fwnode of these CDX devices. Why are these devices
> > no longer getting the fwnode set? Also, swnode doesn't have support
> > for the fwnode.add_links() ops. It also doesn't seem to support the
> > parent of a swnode being another fwnode of a different type (DT). I'm
> > not sure about the history behind that, but maybe swnode is not the
> > right fit or you might need to add support for these to swnode. All of
> > this is to say that if you set these things up correctly, fw_devlink
> > can work for CDX devices and that might be helpful.
>
> In the RFC v2 we were trying to use the platform bus for CDX devices,
> and in that implementation sw_node was used, but now we understand
> that using platform bus for dynamic devices is not a good approach.

Whether you use platform bus or not shouldn't have any impact on where
the device has a fwnode or not.

> So, changes in RFC v3 are to have a CDX bus for CDX devices.

This doesn't answer most of my questions though. Can you answer them please?

-Saravana

>
> Regards,
> Nipun
>
> >
> > -Saravana
> > > +
> > > +       /* Set Name */
> > > +       dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", cdx_dev->bus_id,
> > > +                       cdx_dev->func_id);
> > > +
> > > +       ret = device_add(&cdx_dev->dev);
> > > +       if (ret != 0) {
> > > +               dev_err(&cdx_dev->dev,
> > > +                       "cdx device add failed: %d", ret);
> > > +               goto fail;
> > > +       }
> > > +
> > > +       return 0;
> > > +
> > > +fail:
> > > +       /*
> > > +        * Do not free cdx_dev here as it would be freed in
> > > +        * cdx_device_release() called from within put_device().
> > > +        */
> > > +       put_device(&cdx_dev->dev);
> > > +
> > > +       return ret;
> > > +}
> > > +
> > > +static int cdx_bus_device_discovery(struct platform_device *pdev)
> > > +{
> > > +       struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(pdev);
> > > +       int num_cdx_bus, num_cdx_func;
> > > +       uint8_t bus_id, func_id;
> > > +       int ret;
> > > +
> > > +       cdx_controller_pdev = pdev;
> > > +
> > > +       /* MCDI FW Read: Fetch the number of CDX buses present*/
> > > +       num_cdx_bus = cdx_mcdi_get_num_buses(cdx_mcdi);
> > > +
> > > +       for (bus_id = 0; bus_id < num_cdx_bus; bus_id++) {
> > > +               /* MCDI FW Read: Fetch the number of devices present */
> > > +               num_cdx_func = cdx_mcdi_get_num_funcs(cdx_mcdi, bus_id);
> > > +
> > > +               for (func_id = 0; func_id < num_cdx_func; func_id++) {
> > > +                       struct cdx_dev_params_t dev_params;
> > > +
> > > +                       /* MCDI FW: Get the device config */
> > > +                       ret = cdx_mcdi_get_func_config(cdx_mcdi, bus_id,
> > > +                                       func_id, &dev_params);
> > > +                       if (ret) {
> > > +                               dev_err(&pdev->dev,
> > > +                                       "CDX device config get failed for bus: %d\n",
> > > +                                       ret);
> > > +                               goto fail;
> > > +                       }
> > > +
> > > +                       /* Add the device to the cdx bus */
> > > +                       ret = cdx_device_add(&pdev->dev, &dev_params);
> > > +                       if (ret == -EPROBE_DEFER) {
> > > +                               goto fail;
> > > +                       } else if (ret) {
> > > +                               dev_err(&pdev->dev,
> > > +                                       "registering cdx dev: %d failed: %d\n",
> > > +                                       func_id, ret);
> > > +                               goto fail;
> > > +                       } else {
> > > +                               dev_dbg(&pdev->dev,
> > > +                                       "CDX dev: %d on cdx bus: %d created\n",
> > > +                                       func_id, bus_id);
> > > +                       }
> > > +               }
> > > +       }
> > > +
> > > +       return 0;
> > > +fail:
> > > +       cdx_unregister_devices(&cdx_bus_type);
> > > +       return ret;
> > > +}
> > > +
> > > +static int cdx_bus_probe(struct platform_device *pdev)
> > > +{
> > > +       struct cdx_mcdi_t *cdx_mcdi;
> > > +       int ret;
> > > +
> > > +       cdx_mcdi = kzalloc(sizeof(*cdx_mcdi), GFP_KERNEL);
> > > +       if (!cdx_mcdi) {
> > > +               dev_err(&pdev->dev, "Failed to allocate memory for cdx_mcdi\n");
> > > +               return -ENOMEM;
> > > +       }
> > > +
> > > +       /* MCDI FW: Initialize the FW path */
> > > +       ret = cdx_mcdi_init(cdx_mcdi);
> > > +       if (ret) {
> > > +               dev_err(&pdev->dev, "MCDI Initialization failed: %d\n", ret);
> > > +               goto mcdi_init_fail;
> > > +       }
> > > +       platform_set_drvdata(pdev, cdx_mcdi);
> > > +
> > > +       /* Discover all the devices on the bus */
> > > +       ret = cdx_bus_device_discovery(pdev);
> > > +       if (ret) {
> > > +               dev_err(&pdev->dev,
> > > +                       "CDX bus device discovery failed: %d\n", ret);
> > > +               goto device_discovery_fail;
> > > +       }
> > > +
> > > +       return 0;
> > > +
> > > +mcdi_init_fail:
> > > +       kfree(cdx_mcdi);
> > > +device_discovery_fail:
> > > +       cdx_mcdi_finish(cdx_mcdi);
> > > +
> > > +       return ret;
> > > +}
> > > +
> > > +static int cdx_bus_remove(struct platform_device *pdev)
> > > +{
> > > +       struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(pdev);
> > > +
> > > +       cdx_unregister_devices(&cdx_bus_type);
> > > +
> > > +       cdx_mcdi_finish(cdx_mcdi);
> > > +       kfree(cdx_mcdi);
> > > +
> > > +       return 0;
> > > +}
> > > +
> > > +static void cdx_bus_shutdown(struct platform_device *pdev)
> > > +{
> > > +       cdx_bus_remove(pdev);
> > > +}
> > > +
> > > +static const struct of_device_id cdx_match_table[] = {
> > > +       {.compatible = "xlnx,cdxbus-controller-1.0",},
> > > +       { },
> > > +};
> > > +
> > > +MODULE_DEVICE_TABLE(of, cdx_match_table);
> > > +
> > > +static struct platform_driver cdx_bus_driver = {
> > > +       .driver = {
> > > +                  .name = "cdx-bus",
> > > +                  .pm = NULL,
> > > +                  .of_match_table = cdx_match_table,
> > > +                  },
> > > +       .probe = cdx_bus_probe,
> > > +       .remove = cdx_bus_remove,
> > > +       .shutdown = cdx_bus_shutdown,
> > > +};
> > > +
> > > +static int __init cdx_bus_driver_init(void)
> > > +{
> > > +       int ret;
> > > +
> > > +       ret = bus_register(&cdx_bus_type);
> > > +       if (ret < 0) {
> > > +               pr_err("bus type registration failed for CDX: %d\n", ret);
> > > +               return ret;
> > > +       }
> > > +
> > > +       ret = platform_driver_register(&cdx_bus_driver);
> > > +       if (ret < 0) {
> > > +               pr_err("platform_driver_register() failed: %d\n", ret);
> > > +               goto fail;
> > > +       }
> > > +
> > > +       return 0;
> > > +
> > > +fail:
> > > +       bus_unregister(&cdx_bus_type);
> > > +       return ret;
> > > +}
> > > +postcore_initcall(cdx_bus_driver_init);
> > > diff --git a/drivers/bus/cdx/cdx.h b/drivers/bus/cdx/cdx.h
> > > new file mode 100644
> > > index 000000000000..db0569431c10
> > > --- /dev/null
> > > +++ b/drivers/bus/cdx/cdx.h
> > > @@ -0,0 +1,34 @@
> > > +// SPDX-License-Identifier: GPL-2.0
> > > +/*
> > > + * Header file for the CDX Bus
> > > + *
> > > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > > + */
> > > +
> > > +#ifndef _CDX_H_
> > > +#define _CDX_H_
> > > +
> > > +#include <linux/cdx/cdx_bus.h>
> > > +#include <linux/irqdomain.h>
> > > +
> > > +/**
> > > + * struct cdx_dev_params_t - CDX device parameters
> > > + * @vendor: Vendor ID for CDX device
> > > + * @device: Device ID for CDX device
> > > + * @bus_id: Bus ID for this CDX device
> > > + * @func_id: Function ID for this device
> > > + * @res: array of MMIO region entries
> > > + * @res_count: number of valid MMIO regions
> > > + * @req_id: Requestor ID associated with CDX device
> > > + */
> > > +struct cdx_dev_params_t {
> > > +       u16 vendor;
> > > +       u16 device;
> > > +       u8 bus_id;
> > > +       u8 func_id;
> > > +       struct resource res[MAX_CDX_DEV_RESOURCES];
> > > +       u8 res_count;
> > > +       u32 req_id;
> > > +};
> > > +
> > > +#endif /* _CDX_H_ */
> > > diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
> > > new file mode 100644
> > > index 000000000000..cc9d30fa02f8
> > > --- /dev/null
> > > +++ b/drivers/bus/cdx/mcdi_stubs.c
> > > @@ -0,0 +1,54 @@
> > > +// SPDX-License-Identifier: GPL-2.0
> > > +/*
> > > + * MCDI Firmware interaction for CDX bus.
> > > + *
> > > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > > + */
> > > +
> > > +#include <linux/ioport.h>
> > > +
> > > +#include "cdx.h"
> > > +#include "mcdi_stubs.h"
> > > +
> > > +int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi)
> > > +{
> > > +       cdx_mcdi->id = 0;
> > > +       cdx_mcdi->flags = 0;
> > > +
> > > +       return 0;
> > > +}
> > > +
> > > +void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi)
> > > +{
> > > +}
> > > +
> > > +int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi)
> > > +{
> > > +       return 1;
> > > +}
> > > +
> > > +int cdx_mcdi_get_num_funcs(struct cdx_mcdi_t *cdx_mcdi, int bus_num)
> > > +{
> > > +       return 1;
> > > +}
> > > +
> > > +int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
> > > +                            uint8_t bus_id, uint8_t func_id,
> > > +                            struct cdx_dev_params_t *dev_params)
> > > +{
> > > +       dev_params->res[0].start = 0xe4020000;
> > > +       dev_params->res[0].end = 0xe4020FFF;
> > > +       dev_params->res[0].flags = IORESOURCE_MEM;
> > > +       dev_params->res[1].start = 0xe4100000;
> > > +       dev_params->res[1].end = 0xE411FFFF;
> > > +       dev_params->res[1].flags = IORESOURCE_MEM;
> > > +       dev_params->res_count = 2;
> > > +
> > > +       dev_params->req_id = 0x250;
> > > +       dev_params->vendor = 0x10ee;
> > > +       dev_params->device = 0x8084;
> > > +       dev_params->bus_id = bus_id;
> > > +       dev_params->func_id = func_id;
> > > +
> > > +       return 0;
> > > +}
> > > diff --git a/drivers/bus/cdx/mcdi_stubs.h b/drivers/bus/cdx/mcdi_stubs.h
> > > new file mode 100644
> > > index 000000000000..7b6f79d48ce9
> > > --- /dev/null
> > > +++ b/drivers/bus/cdx/mcdi_stubs.h
> > > @@ -0,0 +1,76 @@
> > > +// SPDX-License-Identifier: GPL-2.0
> > > +/*
> > > + * Header file for MCDI FW interaction for CDX bus.
> > > + *
> > > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > > + */
> > > +
> > > +#ifndef _MCDI_STUBS_H_
> > > +#define _MCDI_STUBS_H_
> > > +
> > > +#include "cdx.h"
> > > +
> > > +/**
> > > + * struct cdx_mcdi_t - CDX MCDI Firmware interface, to interact
> > > + *                    with CDX controller.
> > > + * @id: ID for MCDI Firmware interface
> > > + * @flags: Associated flags
> > > + */
> > > +struct cdx_mcdi_t {
> > > +       u32 id;
> > > +       u32 flags;
> > > +       /* Have more MCDI interface related data */
> > > +};
> > > +
> > > +/**
> > > + * cdx_mcdi_init - Initialize the MCDI Firmware interface
> > > + *     for the CDX controller.
> > > + * @cdx_mcdi: pointer to MCDI interface
> > > + *
> > > + * Return 0 on success, <0 on failure
> > > + */
> > > +int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi);
> > > +
> > > +/**
> > > + * cdx_mcdi_finish - Close the MCDI Firmware interface.
> > > + * @cdx_mcdi: pointer to MCDI interface
> > > + */
> > > +void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi);
> > > +
> > > +/**
> > > + * cdx_mcdi_get_num_buses - Get the total number of busses on
> > > + *     the controller.
> > > + * @cdx_mcdi: pointer to MCDI interface.
> > > + *
> > > + * Return total number of busses available on the controller,
> > > + *     <0 on failure
> > > + */
> > > +int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi);
> > > +
> > > +/**
> > > + * cdx_mcdi_get_num_funcs - Get the total number of functions on
> > > + *     a particular bus of the controller.
> > > + * @cdx_mcdi: pointer to MCDI interface.
> > > + * @bus_num: Bus number.
> > > + *
> > > + * Return total number of functions available on the bus, <0 on failure
> > > + */
> > > +int cdx_mcdi_get_num_funcs(struct cdx_mcdi_t *cdx_mcdi, int bus_num);
> > > +
> > > +/**
> > > + * cdx_mcdi_get_func_config - Get configuration for a particular
> > > + *     bus_id:func_id
> > > + * @cdx_mcdi: pointer to MCDI interface.
> > > + * @bus_num: Bus number.
> > > + * @func_id: Function number.
> > > + * @dev_params: Pointer to cdx_dev_params_t, this is populated by this
> > > + *     function with the configuration corresponding to the provided
> > > + *     bus_id:func_id.
> > > + *
> > > + * Return 0 total number of functions available on the bus, <0 on failure
> > > + */
> > > +int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
> > > +                            uint8_t bus_id, uint8_t func_id,
> > > +                            struct cdx_dev_params_t *dev_params);
> > > +
> > > +#endif /* _MCDI_STUBS_H_ */
> > > diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
> > > new file mode 100644
> > > index 000000000000..6e870b2c87d9
> > > --- /dev/null
> > > +++ b/include/linux/cdx/cdx_bus.h
> > > @@ -0,0 +1,93 @@
> > > +// SPDX-License-Identifier: GPL-2.0
> > > +/*
> > > + * CDX bus public interface
> > > + *
> > > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > > + *
> > > + */
> > > +#ifndef _CDX_BUS_H_
> > > +#define _CDX_BUS_H_
> > > +
> > > +#include <linux/mod_devicetable.h>
> > > +#include <linux/device.h>
> > > +
> > > +#define MAX_CDX_DEV_RESOURCES  6
> > > +
> > > +/**
> > > + * struct cdx_device - CDX device object
> > > + * @dev: Linux driver model device object
> > > + * @vendor: Vendor ID for CDX device
> > > + * @device: Device ID for CDX device
> > > + * @bus_id: Bus ID for this CDX device
> > > + * @func_id: Function ID for this device
> > > + * @res: array of MMIO region entries
> > > + * @res_count: number of valid MMIO regions
> > > + * @dma_mask: Default DMA mask
> > > + * @flags: CDX device flags
> > > + * @req_id: Requestor ID associated with CDX device
> > > + * @driver_override: driver name to force a match; do not set directly,
> > > + *                   because core frees it; use driver_set_override() to
> > > + *                   set or clear it.
> > > + */
> > > +struct cdx_device {
> > > +       struct device dev;
> > > +       u16 vendor;
> > > +       u16 device;
> > > +       u8 bus_id;
> > > +       u8 func_id;
> > > +       struct resource res[MAX_CDX_DEV_RESOURCES];
> > > +       u8 res_count;
> > > +       u64 dma_mask;
> > > +       u16 flags;
> > > +       u32 req_id;
> > > +       const char *driver_override;
> > > +};
> > > +
> > > +#define to_cdx_device(_dev) \
> > > +       container_of(_dev, struct cdx_device, dev)
> > > +
> > > +/**
> > > + * struct cdx_driver - CDX device driver
> > > + * @driver: Generic device driver
> > > + * @match_id_table: table of supported device matching Ids
> > > + * @probe: Function called when a device is added
> > > + * @remove: Function called when a device is removed
> > > + * @shutdown: Function called at shutdown time to quiesce the device
> > > + * @suspend: Function called when a device is stopped
> > > + * @resume: Function called when a device is resumed
> > > + * @driver_managed_dma: Device driver doesn't use kernel DMA API for
> > DMA.
> > > + *             For most device drivers, no need to care about this flag
> > > + *             as long as all DMAs are handled through the kernel DMA API.
> > > + *             For some special ones, for example VFIO drivers, they know
> > > + *             how to manage the DMA themselves and set this flag so that
> > > + *             the IOMMU layer will allow them to setup and manage their
> > > + *             own I/O address space.
> > > + */
> > > +struct cdx_driver {
> > > +       struct device_driver driver;
> > > +       const struct cdx_device_id *match_id_table;
> > > +       int (*probe)(struct cdx_device *dev);
> > > +       int (*remove)(struct cdx_device *dev);
> > > +       void (*shutdown)(struct cdx_device *dev);
> > > +       int (*suspend)(struct cdx_device *dev, pm_message_t state);
> > > +       int (*resume)(struct cdx_device *dev);
> > > +       bool driver_managed_dma;
> > > +};
> > > +
> > > +#define to_cdx_driver(_drv) \
> > > +       container_of(_drv, struct cdx_driver, driver)
> > > +
> > > +/*
> > > + * Macro to avoid include chaining to get THIS_MODULE
> > > + */
> > > +#define cdx_driver_register(drv) \
> > > +       __cdx_driver_register(drv, THIS_MODULE)
> > > +
> > > +int __must_check __cdx_driver_register(struct cdx_driver *cdx_driver,
> > > +                                      struct module *owner);
> > > +
> > > +void cdx_driver_unregister(struct cdx_driver *driver);
> > > +
> > > +extern struct bus_type cdx_bus_type;
> > > +
> > > +#endif /* _CDX_BUS_H_ */
> > > diff --git a/include/linux/mod_devicetable.h
> > b/include/linux/mod_devicetable.h
> > > index 549590e9c644..9b94be83d53e 100644
> > > --- a/include/linux/mod_devicetable.h
> > > +++ b/include/linux/mod_devicetable.h
> > > @@ -911,4 +911,17 @@ struct ishtp_device_id {
> > >         kernel_ulong_t driver_data;
> > >  };
> > >
> > > +/**
> > > + * struct cdx_device_id - CDX device identifier
> > > + * @vendor: vendor ID
> > > + * @device: Device ID
> > > + *
> > > + * Type of entries in the "device Id" table for CDX devices supported by
> > > + * a CDX device driver.
> > > + */
> > > +struct cdx_device_id {
> > > +       __u16 vendor;
> > > +       __u16 device;
> > > +};
> > > +
> > >  #endif /* LINUX_MOD_DEVICETABLE_H */
> > > diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-
> > offsets.c
> > > index c0d3bcb99138..62dc988df84d 100644
> > > --- a/scripts/mod/devicetable-offsets.c
> > > +++ b/scripts/mod/devicetable-offsets.c
> > > @@ -262,5 +262,9 @@ int main(void)
> > >         DEVID(ishtp_device_id);
> > >         DEVID_FIELD(ishtp_device_id, guid);
> > >
> > > +       DEVID(cdx_device_id);
> > > +       DEVID_FIELD(cdx_device_id, vendor);
> > > +       DEVID_FIELD(cdx_device_id, device);
> > > +
> > >         return 0;
> > >  }
> > > diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c
> > > index 80d973144fde..c36e1f624e39 100644
> > > --- a/scripts/mod/file2alias.c
> > > +++ b/scripts/mod/file2alias.c
> > > @@ -1452,6 +1452,17 @@ static int do_dfl_entry(const char *filename, void
> > *symval, char *alias)
> > >         return 1;
> > >  }
> > >
> > > +/* Looks like: cdx:vNdN */
> > > +static int do_cdx_entry(const char *filename, void *symval,
> > > +                          char *alias)
> > > +{
> > > +       DEF_FIELD(symval, cdx_device_id, vendor);
> > > +       DEF_FIELD(symval, cdx_device_id, device);
> > > +
> > > +       sprintf(alias, "cdx:v%08Xd%08Xd", vendor, device);
> > > +       return 1;
> > > +}
> > > +
> > >  /* Does namelen bytes of name exactly match the symbol? */
> > >  static bool sym_is(const char *name, unsigned namelen, const char *symbol)
> > >  {
> > > @@ -1531,6 +1542,7 @@ static const struct devtable devtable[] = {
> > >         {"ssam", SIZE_ssam_device_id, do_ssam_entry},
> > >         {"dfl", SIZE_dfl_device_id, do_dfl_entry},
> > >         {"ishtp", SIZE_ishtp_device_id, do_ishtp_entry},
> > > +       {"cdx", SIZE_cdx_device_id, do_cdx_entry},
> > >  };
> > >
> > >  /* Create MODULE_ALIAS() statements.
> > > --
> > > 2.25.1
> > >

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver

[Greg KH]

On Tue, Sep 06, 2022 at 07:17:56PM +0530, Nipun Gupta wrote:
> CDX bus supports the scanning and probing of FPGA based
> devices. These devices are registers as CDX devices.
> 
> The bus driver sets up the basic infrastructure and fetches
> the device related information from the firmware.
> 
> CDX bus is capable of scanning devices dynamically,
> supporting rescanning of dynamically added, removed or
> updated devices.

Really?  Then why is the platform driver mess still in here?


> --- /dev/null
> +++ b/drivers/bus/cdx/cdx.c
> @@ -0,0 +1,437 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Platform driver for CDX bus.

Again, this should not be a platform driver.

Now you can have a CDX "bus" driver, that is a platform driver, but that
needs to be in a separate file and as a separate module and totally
independant of the CDX bus code entirely.

Otherwise this is a mess to try to sift through and determine what is,
and is not, going on.  Please split that up and get rid of all of the
platform driver stuff here and put it in a separate patch that happens
after the CDX bus logic is added.

thanks,

greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Greg KH <gregkh@linuxfoundation.org>
> Sent: Wednesday, September 7, 2022 6:03 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org; rafael@kernel.org;
> eric.auger@redhat.com; alex.williamson@redhat.com; cohuck@redhat.com;
> Gupta, Puneet (DCG-ENG) <puneet.gupta@amd.com>;
> song.bao.hua@hisilicon.com; mchehab+huawei@kernel.org; maz@kernel.org;
> f.fainelli@gmail.com; jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com; will@kernel.org;
> joro@8bytes.org; masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v3 2/7] bus/cdx: add the cdx bus driver
> 
> [CAUTION: External Email]
> 
> On Tue, Sep 06, 2022 at 07:17:56PM +0530, Nipun Gupta wrote:
> > CDX bus supports the scanning and probing of FPGA based
> > devices. These devices are registers as CDX devices.
> >
> > The bus driver sets up the basic infrastructure and fetches
> > the device related information from the firmware.
> >
> > CDX bus is capable of scanning devices dynamically,
> > supporting rescanning of dynamically added, removed or
> > updated devices.
> 
> Really?  Then why is the platform driver mess still in here?
> 
> 
> > --- /dev/null
> > +++ b/drivers/bus/cdx/cdx.c
> > @@ -0,0 +1,437 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Platform driver for CDX bus.
> 
> Again, this should not be a platform driver.
> 
> Now you can have a CDX "bus" driver, that is a platform driver, but that
> needs to be in a separate file and as a separate module and totally
> independant of the CDX bus code entirely.
> 
> Otherwise this is a mess to try to sift through and determine what is,
> and is not, going on.  Please split that up and get rid of all of the
> platform driver stuff here and put it in a separate patch that happens
> after the CDX bus logic is added.

Platform device/driver is essentially the CDX bus controller here and there are
existing busses like fsl-mc and rsb bus which are having similar bus and platform
driver in same file.

As you mention to make things cleaner (and which seems more appropriate) we
will segregate the bus code and platform bus controller into different patches
and files. We will include the change in the next RFC.

Thanks,
Nipun

> 
> thanks,
> 
> greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration for CDX bus

[Nipun Gupta]

With new CDX bus supported for AMD FPGA devices on ARM
platform, the bus requires registration for the SMMU v3
driver.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index d32b02336411..8ec9f2baf12d 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -29,6 +29,7 @@
 #include <linux/platform_device.h>
 
 #include <linux/amba/bus.h>
+#include <linux/cdx/cdx_bus.h>
 
 #include "arm-smmu-v3.h"
 #include "../../iommu-sva-lib.h"
@@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
 		if (err)
 			goto err_reset_pci_ops;
 	}
+#endif
+#ifdef CONFIG_CDX_BUS
+	if (cdx_bus_type.iommu_ops != ops) {
+		err = bus_set_iommu(&cdx_bus_type, ops);
+		if (err)
+			goto err_reset_amba_ops;
+	}
 #endif
 	if (platform_bus_type.iommu_ops != ops) {
 		err = bus_set_iommu(&platform_bus_type, ops);
 		if (err)
-			goto err_reset_amba_ops;
+			goto err_reset_cdx_ops;
 	}
 
 	return 0;
 
-err_reset_amba_ops:
+err_reset_cdx_ops:
+#ifdef CONFIG_CDX_BUS
+	bus_set_iommu(&cdx_bus_type, NULL);
+#endif
+err_reset_amba_ops: __maybe_unused;
 #ifdef CONFIG_ARM_AMBA
 	bus_set_iommu(&amba_bustype, NULL);
 #endif
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration for CDX bus

[Saravana Kannan]

On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
>
> With new CDX bus supported for AMD FPGA devices on ARM
> platform, the bus requires registration for the SMMU v3
> driver.
>
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> ---
>  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
>  1 file changed, 14 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> index d32b02336411..8ec9f2baf12d 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> @@ -29,6 +29,7 @@
>  #include <linux/platform_device.h>
>
>  #include <linux/amba/bus.h>
> +#include <linux/cdx/cdx_bus.h>
>
>  #include "arm-smmu-v3.h"
>  #include "../../iommu-sva-lib.h"
> @@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
>                 if (err)
>                         goto err_reset_pci_ops;
>         }
> +#endif
> +#ifdef CONFIG_CDX_BUS
> +       if (cdx_bus_type.iommu_ops != ops) {
> +               err = bus_set_iommu(&cdx_bus_type, ops);
> +               if (err)
> +                       goto err_reset_amba_ops;
> +       }

I'm not an expert on IOMMUs, so apologies if the question is stupid.

Why does the CDX bus need special treatment here (like PCI) when there
are so many other busses (eg: I2C, SPI, etc) that don't need any
changes here?

-Saravana

>  #endif
>         if (platform_bus_type.iommu_ops != ops) {
>                 err = bus_set_iommu(&platform_bus_type, ops);
>                 if (err)
> -                       goto err_reset_amba_ops;
> +                       goto err_reset_cdx_ops;
>         }
>
>         return 0;
>
> -err_reset_amba_ops:
> +err_reset_cdx_ops:
> +#ifdef CONFIG_CDX_BUS
> +       bus_set_iommu(&cdx_bus_type, NULL);
> +#endif
> +err_reset_amba_ops: __maybe_unused;
>  #ifdef CONFIG_ARM_AMBA
>         bus_set_iommu(&amba_bustype, NULL);
>  #endif
> --
> 2.25.1
>

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration for CDX bus

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Saravana Kannan <saravanak@google.com>
> Sent: Wednesday, September 7, 2022 5:41 AM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; Michael.Srba@seznam.cz; mani@kernel.org;
> yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com;
> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
> Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration
> for CDX bus
> 
> [CAUTION: External Email]
> 
> On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
> >
> > With new CDX bus supported for AMD FPGA devices on ARM
> > platform, the bus requires registration for the SMMU v3
> > driver.
> >
> > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> > ---
> >  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
> >  1 file changed, 14 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> > index d32b02336411..8ec9f2baf12d 100644
> > --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> > +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> > @@ -29,6 +29,7 @@
> >  #include <linux/platform_device.h>
> >
> >  #include <linux/amba/bus.h>
> > +#include <linux/cdx/cdx_bus.h>
> >
> >  #include "arm-smmu-v3.h"
> >  #include "../../iommu-sva-lib.h"
> > @@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct
> iommu_ops *ops)
> >                 if (err)
> >                         goto err_reset_pci_ops;
> >         }
> > +#endif
> > +#ifdef CONFIG_CDX_BUS
> > +       if (cdx_bus_type.iommu_ops != ops) {
> > +               err = bus_set_iommu(&cdx_bus_type, ops);
> > +               if (err)
> > +                       goto err_reset_amba_ops;
> > +       }
> 
> I'm not an expert on IOMMUs, so apologies if the question is stupid.
> 
> Why does the CDX bus need special treatment here (like PCI) when there
> are so many other busses (eg: I2C, SPI, etc) that don't need any
> changes here?

AFAIU, the devices on I2C/SPI does not use SMMU. Apart from PCI/AMBA,
FSL-MC is another similar bus (on SMMUv2) which uses SMMU ops.

The devices here are behind SMMU. Robin can kindly correct or add
more here from SMMU perspective.

Thanks,
Nipun

> 
> -Saravana
> 
> >  #endif
> >         if (platform_bus_type.iommu_ops != ops) {
> >                 err = bus_set_iommu(&platform_bus_type, ops);
> >                 if (err)
> > -                       goto err_reset_amba_ops;
> > +                       goto err_reset_cdx_ops;
> >         }
> >
> >         return 0;
> >
> > -err_reset_amba_ops:
> > +err_reset_cdx_ops:
> > +#ifdef CONFIG_CDX_BUS
> > +       bus_set_iommu(&cdx_bus_type, NULL);
> > +#endif
> > +err_reset_amba_ops: __maybe_unused;
> >  #ifdef CONFIG_ARM_AMBA
> >         bus_set_iommu(&amba_bustype, NULL);
> >  #endif
> > --
> > 2.25.1
> >
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration for CDX bus

[Robin Murphy]

On 2022-09-07 04:17, Gupta, Nipun wrote:
> [AMD Official Use Only - General]
> 
> 
> 
>> -----Original Message-----
>> From: Saravana Kannan <saravanak@google.com>
>> Sent: Wednesday, September 7, 2022 5:41 AM
>> To: Gupta, Nipun <Nipun.Gupta@amd.com>
>> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
>> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
>> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
>> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
>> mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
>> jeffrey.l.hugo@gmail.com; Michael.Srba@seznam.cz; mani@kernel.org;
>> yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com;
>> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
>> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
>> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
>> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
>> Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
>> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
>> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
>> <git@amd.com>
>> Subject: Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration
>> for CDX bus
>>
>> [CAUTION: External Email]
>>
>> On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
>>>
>>> With new CDX bus supported for AMD FPGA devices on ARM
>>> platform, the bus requires registration for the SMMU v3
>>> driver.
>>>
>>> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
>>> ---
>>>   drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
>>>   1 file changed, 14 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
>> b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
>>> index d32b02336411..8ec9f2baf12d 100644
>>> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
>>> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
>>> @@ -29,6 +29,7 @@
>>>   #include <linux/platform_device.h>
>>>
>>>   #include <linux/amba/bus.h>
>>> +#include <linux/cdx/cdx_bus.h>
>>>
>>>   #include "arm-smmu-v3.h"
>>>   #include "../../iommu-sva-lib.h"
>>> @@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct
>> iommu_ops *ops)
>>>                  if (err)
>>>                          goto err_reset_pci_ops;
>>>          }
>>> +#endif
>>> +#ifdef CONFIG_CDX_BUS
>>> +       if (cdx_bus_type.iommu_ops != ops) {
>>> +               err = bus_set_iommu(&cdx_bus_type, ops);
>>> +               if (err)
>>> +                       goto err_reset_amba_ops;
>>> +       }
>>
>> I'm not an expert on IOMMUs, so apologies if the question is stupid.
>>
>> Why does the CDX bus need special treatment here (like PCI) when there
>> are so many other busses (eg: I2C, SPI, etc) that don't need any
>> changes here?
> 
> AFAIU, the devices on I2C/SPI does not use SMMU. Apart from PCI/AMBA,
> FSL-MC is another similar bus (on SMMUv2) which uses SMMU ops.
> 
> The devices here are behind SMMU. Robin can kindly correct or add
> more here from SMMU perspective.

Indeed, there is no need to describe and handle how DMA may or may not 
be translated for I2C/SPI/USB/etc. because they are not DMA-capable 
buses (in those cases the relevant bus *controller* often does DMA, but 
it does that for itself as the platform/PCI/etc. device it is).

Note that I have a series pending[1] that will make this patch a whole 
lot simpler.

Thanks,
Robin.

[1] 
https://lore.kernel.org/linux-iommu/cover.1660572783.git.robin.murphy@arm.com/T/#t

> 
> Thanks,
> Nipun
> 
>>
>> -Saravana
>>
>>>   #endif
>>>          if (platform_bus_type.iommu_ops != ops) {
>>>                  err = bus_set_iommu(&platform_bus_type, ops);
>>>                  if (err)
>>> -                       goto err_reset_amba_ops;
>>> +                       goto err_reset_cdx_ops;
>>>          }
>>>
>>>          return 0;
>>>
>>> -err_reset_amba_ops:
>>> +err_reset_cdx_ops:
>>> +#ifdef CONFIG_CDX_BUS
>>> +       bus_set_iommu(&cdx_bus_type, NULL);
>>> +#endif
>>> +err_reset_amba_ops: __maybe_unused;
>>>   #ifdef CONFIG_ARM_AMBA
>>>          bus_set_iommu(&amba_bustype, NULL);
>>>   #endif
>>> --
>>> 2.25.1
>>>

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration for CDX bus

[Saravana Kannan]

On Wed, Sep 7, 2022 at 1:27 AM Robin Murphy <robin.murphy@arm.com> wrote:
>
> On 2022-09-07 04:17, Gupta, Nipun wrote:
> > [AMD Official Use Only - General]
> >
> >
> >
> >> -----Original Message-----
> >> From: Saravana Kannan <saravanak@google.com>
> >> Sent: Wednesday, September 7, 2022 5:41 AM
> >> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> >> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> >> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> >> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> >> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> >> mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
> >> jeffrey.l.hugo@gmail.com; Michael.Srba@seznam.cz; mani@kernel.org;
> >> yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com;
> >> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> >> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> >> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> >> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
> >> Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> >> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> >> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> >> <git@amd.com>
> >> Subject: Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration
> >> for CDX bus
> >>
> >> [CAUTION: External Email]
> >>
> >> On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
> >>>
> >>> With new CDX bus supported for AMD FPGA devices on ARM
> >>> platform, the bus requires registration for the SMMU v3
> >>> driver.
> >>>
> >>> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> >>> ---
> >>>   drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
> >>>   1 file changed, 14 insertions(+), 2 deletions(-)
> >>>
> >>> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> >> b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> >>> index d32b02336411..8ec9f2baf12d 100644
> >>> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> >>> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> >>> @@ -29,6 +29,7 @@
> >>>   #include <linux/platform_device.h>
> >>>
> >>>   #include <linux/amba/bus.h>
> >>> +#include <linux/cdx/cdx_bus.h>
> >>>
> >>>   #include "arm-smmu-v3.h"
> >>>   #include "../../iommu-sva-lib.h"
> >>> @@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct
> >> iommu_ops *ops)
> >>>                  if (err)
> >>>                          goto err_reset_pci_ops;
> >>>          }
> >>> +#endif
> >>> +#ifdef CONFIG_CDX_BUS
> >>> +       if (cdx_bus_type.iommu_ops != ops) {
> >>> +               err = bus_set_iommu(&cdx_bus_type, ops);
> >>> +               if (err)
> >>> +                       goto err_reset_amba_ops;
> >>> +       }
> >>
> >> I'm not an expert on IOMMUs, so apologies if the question is stupid.
> >>
> >> Why does the CDX bus need special treatment here (like PCI) when there
> >> are so many other busses (eg: I2C, SPI, etc) that don't need any
> >> changes here?
> >
> > AFAIU, the devices on I2C/SPI does not use SMMU. Apart from PCI/AMBA,
> > FSL-MC is another similar bus (on SMMUv2) which uses SMMU ops.
> >
> > The devices here are behind SMMU. Robin can kindly correct or add
> > more here from SMMU perspective.
>
> Indeed, there is no need to describe and handle how DMA may or may not
> be translated for I2C/SPI/USB/etc. because they are not DMA-capable
> buses (in those cases the relevant bus *controller* often does DMA, but
> it does that for itself as the platform/PCI/etc. device it is).

Ok this is what I was guessing was the reason, but didn't want to make
that assumption.

So if there are other cases like AMBA, FSL-MC where the devices can do
direct DMA, why do those buses not need a #ifdef section in this
function like CDX? Or put another way, why does CDX need special treatment?

> Note that I have a series pending[1] that will make this patch a whole
> lot simpler.

Thanks for the pointer. I'll make some comments in that series about
bus notifiers.


-Saravana

>
> Thanks,
> Robin.
>
> [1]
> https://lore.kernel.org/linux-iommu/cover.1660572783.git.robin.murphy@arm.com/T/#t
>
> >
> > Thanks,
> > Nipun
> >
> >>
> >> -Saravana
> >>
> >>>   #endif
> >>>          if (platform_bus_type.iommu_ops != ops) {
> >>>                  err = bus_set_iommu(&platform_bus_type, ops);
> >>>                  if (err)
> >>> -                       goto err_reset_amba_ops;
> >>> +                       goto err_reset_cdx_ops;
> >>>          }
> >>>
> >>>          return 0;
> >>>
> >>> -err_reset_amba_ops:
> >>> +err_reset_cdx_ops:
> >>> +#ifdef CONFIG_CDX_BUS
> >>> +       bus_set_iommu(&cdx_bus_type, NULL);
> >>> +#endif
> >>> +err_reset_amba_ops: __maybe_unused;
> >>>   #ifdef CONFIG_ARM_AMBA
> >>>          bus_set_iommu(&amba_bustype, NULL);
> >>>   #endif
> >>> --
> >>> 2.25.1
> >>>

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration for CDX bus

[Robin Murphy]

On 2022-09-07 19:24, Saravana Kannan wrote:
> On Wed, Sep 7, 2022 at 1:27 AM Robin Murphy <robin.murphy@arm.com> wrote:
>>
>> On 2022-09-07 04:17, Gupta, Nipun wrote:
>>> [AMD Official Use Only - General]
>>>
>>>
>>>
>>>> -----Original Message-----
>>>> From: Saravana Kannan <saravanak@google.com>
>>>> Sent: Wednesday, September 7, 2022 5:41 AM
>>>> To: Gupta, Nipun <Nipun.Gupta@amd.com>
>>>> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
>>>> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
>>>> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
>>>> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
>>>> mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
>>>> jeffrey.l.hugo@gmail.com; Michael.Srba@seznam.cz; mani@kernel.org;
>>>> yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com;
>>>> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
>>>> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
>>>> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
>>>> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
>>>> Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
>>>> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
>>>> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
>>>> <git@amd.com>
>>>> Subject: Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration
>>>> for CDX bus
>>>>
>>>> [CAUTION: External Email]
>>>>
>>>> On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
>>>>>
>>>>> With new CDX bus supported for AMD FPGA devices on ARM
>>>>> platform, the bus requires registration for the SMMU v3
>>>>> driver.
>>>>>
>>>>> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
>>>>> ---
>>>>>    drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
>>>>>    1 file changed, 14 insertions(+), 2 deletions(-)
>>>>>
>>>>> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
>>>> b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
>>>>> index d32b02336411..8ec9f2baf12d 100644
>>>>> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
>>>>> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
>>>>> @@ -29,6 +29,7 @@
>>>>>    #include <linux/platform_device.h>
>>>>>
>>>>>    #include <linux/amba/bus.h>
>>>>> +#include <linux/cdx/cdx_bus.h>
>>>>>
>>>>>    #include "arm-smmu-v3.h"
>>>>>    #include "../../iommu-sva-lib.h"
>>>>> @@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct
>>>> iommu_ops *ops)
>>>>>                   if (err)
>>>>>                           goto err_reset_pci_ops;
>>>>>           }
>>>>> +#endif
>>>>> +#ifdef CONFIG_CDX_BUS
>>>>> +       if (cdx_bus_type.iommu_ops != ops) {
>>>>> +               err = bus_set_iommu(&cdx_bus_type, ops);
>>>>> +               if (err)
>>>>> +                       goto err_reset_amba_ops;
>>>>> +       }
>>>>
>>>> I'm not an expert on IOMMUs, so apologies if the question is stupid.
>>>>
>>>> Why does the CDX bus need special treatment here (like PCI) when there
>>>> are so many other busses (eg: I2C, SPI, etc) that don't need any
>>>> changes here?
>>>
>>> AFAIU, the devices on I2C/SPI does not use SMMU. Apart from PCI/AMBA,
>>> FSL-MC is another similar bus (on SMMUv2) which uses SMMU ops.
>>>
>>> The devices here are behind SMMU. Robin can kindly correct or add
>>> more here from SMMU perspective.
>>
>> Indeed, there is no need to describe and handle how DMA may or may not
>> be translated for I2C/SPI/USB/etc. because they are not DMA-capable
>> buses (in those cases the relevant bus *controller* often does DMA, but
>> it does that for itself as the platform/PCI/etc. device it is).
> 
> Ok this is what I was guessing was the reason, but didn't want to make
> that assumption.
> 
> So if there are other cases like AMBA, FSL-MC where the devices can do
> direct DMA, why do those buses not need a #ifdef section in this
> function like CDX? Or put another way, why does CDX need special treatment?

Er, it doesn't? The only non-optional bus here is platform, since the 
others *can* be configured out and *are* #ifdefed accordingly. This 
patch is fine for the kernel it was based on, it'll just want rewriting 
now that I've cleaned all this horrible driver boilerplate up. And 
according to the thread on patch #4 there might need to be additional 
changes for CDX to express a reserved MSI region for SMMU support to 
actually work properly.

Robin.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration for CDX bus

[Saravana Kannan]

On Wed, Sep 7, 2022 at 1:40 PM Robin Murphy <robin.murphy@arm.com> wrote:
>
> On 2022-09-07 19:24, Saravana Kannan wrote:
> > On Wed, Sep 7, 2022 at 1:27 AM Robin Murphy <robin.murphy@arm.com> wrote:
> >>
> >> On 2022-09-07 04:17, Gupta, Nipun wrote:
> >>> [AMD Official Use Only - General]
> >>>
> >>>
> >>>
> >>>> -----Original Message-----
> >>>> From: Saravana Kannan <saravanak@google.com>
> >>>> Sent: Wednesday, September 7, 2022 5:41 AM
> >>>> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> >>>> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> >>>> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> >>>> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> >>>> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> >>>> mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
> >>>> jeffrey.l.hugo@gmail.com; Michael.Srba@seznam.cz; mani@kernel.org;
> >>>> yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com;
> >>>> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> >>>> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> >>>> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> >>>> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
> >>>> Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> >>>> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> >>>> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> >>>> <git@amd.com>
> >>>> Subject: Re: [RFC PATCH v3 3/7] iommu/arm-smmu-v3: support ops registration
> >>>> for CDX bus
> >>>>
> >>>> [CAUTION: External Email]
> >>>>
> >>>> On Tue, Sep 6, 2022 at 6:48 AM Nipun Gupta <nipun.gupta@amd.com> wrote:
> >>>>>
> >>>>> With new CDX bus supported for AMD FPGA devices on ARM
> >>>>> platform, the bus requires registration for the SMMU v3
> >>>>> driver.
> >>>>>
> >>>>> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> >>>>> ---
> >>>>>    drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
> >>>>>    1 file changed, 14 insertions(+), 2 deletions(-)
> >>>>>
> >>>>> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> >>>> b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> >>>>> index d32b02336411..8ec9f2baf12d 100644
> >>>>> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> >>>>> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> >>>>> @@ -29,6 +29,7 @@
> >>>>>    #include <linux/platform_device.h>
> >>>>>
> >>>>>    #include <linux/amba/bus.h>
> >>>>> +#include <linux/cdx/cdx_bus.h>
> >>>>>
> >>>>>    #include "arm-smmu-v3.h"
> >>>>>    #include "../../iommu-sva-lib.h"
> >>>>> @@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct
> >>>> iommu_ops *ops)
> >>>>>                   if (err)
> >>>>>                           goto err_reset_pci_ops;
> >>>>>           }
> >>>>> +#endif
> >>>>> +#ifdef CONFIG_CDX_BUS
> >>>>> +       if (cdx_bus_type.iommu_ops != ops) {
> >>>>> +               err = bus_set_iommu(&cdx_bus_type, ops);
> >>>>> +               if (err)
> >>>>> +                       goto err_reset_amba_ops;
> >>>>> +       }
> >>>>
> >>>> I'm not an expert on IOMMUs, so apologies if the question is stupid.
> >>>>
> >>>> Why does the CDX bus need special treatment here (like PCI) when there
> >>>> are so many other busses (eg: I2C, SPI, etc) that don't need any
> >>>> changes here?
> >>>
> >>> AFAIU, the devices on I2C/SPI does not use SMMU. Apart from PCI/AMBA,
> >>> FSL-MC is another similar bus (on SMMUv2) which uses SMMU ops.
> >>>
> >>> The devices here are behind SMMU. Robin can kindly correct or add
> >>> more here from SMMU perspective.
> >>
> >> Indeed, there is no need to describe and handle how DMA may or may not
> >> be translated for I2C/SPI/USB/etc. because they are not DMA-capable
> >> buses (in those cases the relevant bus *controller* often does DMA, but
> >> it does that for itself as the platform/PCI/etc. device it is).
> >
> > Ok this is what I was guessing was the reason, but didn't want to make
> > that assumption.
> >
> > So if there are other cases like AMBA, FSL-MC where the devices can do
> > direct DMA, why do those buses not need a #ifdef section in this
> > function like CDX? Or put another way, why does CDX need special treatment?
>
> Er, it doesn't? The only non-optional bus here is platform, since the
> others *can* be configured out and *are* #ifdefed accordingly.

Ah ok. Also I somehow missed the #ifdef AMBA there and thought there
was only #ifdef PCI and the rest of the buses somehow got it working
without having to muck around arm-smmu-v3.c.

Thanks for the explanation. I'm done here :)

-Saravana

> This
> patch is fine for the kernel it was based on, it'll just want rewriting
> now that I've cleaned all this horrible driver boilerplate up. And
> according to the thread on patch #4 there might need to be additional
> changes for CDX to express a reserved MSI region for SMMU support to
> actually work properly.
>
> Robin.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Nipun Gupta]

Devices on cdx bus are dynamically detected and registered using
platform_device_register API. As these devices are not linked to
of node they need a separate MSI domain for handling device ID
to be provided to the GIC ITS domain.

This also introduces APIs to alloc and free IRQs for CDX domain.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
Signed-off-by: Nikhil Agarwal <nikhil.agarwal@amd.com>
---
 drivers/bus/cdx/cdx.c        |  18 +++
 drivers/bus/cdx/cdx.h        |  19 +++
 drivers/bus/cdx/cdx_msi.c    | 236 +++++++++++++++++++++++++++++++++++
 drivers/bus/cdx/mcdi_stubs.c |   1 +
 include/linux/cdx/cdx_bus.h  |  19 +++
 5 files changed, 293 insertions(+)
 create mode 100644 drivers/bus/cdx/cdx_msi.c

diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
index fc417c32c59b..02ececce1c84 100644
--- a/drivers/bus/cdx/cdx.c
+++ b/drivers/bus/cdx/cdx.c
@@ -17,6 +17,7 @@
 #include <linux/dma-map-ops.h>
 #include <linux/property.h>
 #include <linux/iommu.h>
+#include <linux/msi.h>
 #include <linux/cdx/cdx_bus.h>
 
 #include "cdx.h"
@@ -236,6 +237,7 @@ static int cdx_device_add(struct device *parent,
 			  struct cdx_dev_params_t *dev_params)
 {
 	struct cdx_device *cdx_dev;
+	struct irq_domain *cdx_msi_domain;
 	int ret;
 
 	cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
@@ -252,6 +254,7 @@ static int cdx_device_add(struct device *parent,
 
 	/* Populate CDX dev params */
 	cdx_dev->req_id = dev_params->req_id;
+	cdx_dev->num_msi = dev_params->num_msi;
 	cdx_dev->vendor = dev_params->vendor;
 	cdx_dev->device = dev_params->device;
 	cdx_dev->bus_id = dev_params->bus_id;
@@ -269,6 +272,21 @@ static int cdx_device_add(struct device *parent,
 	dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", cdx_dev->bus_id,
 			cdx_dev->func_id);
 
+	/* If CDX MSI domain is not created, create one. */
+	cdx_msi_domain = cdx_find_msi_domain(parent);
+	if (!cdx_msi_domain) {
+		cdx_msi_domain = cdx_msi_domain_init(parent);
+		if (!cdx_msi_domain) {
+			dev_err(&cdx_dev->dev,
+				"cdx_msi_domain_init() failed: %d", ret);
+			kfree(cdx_dev);
+			return -1;
+		}
+	}
+
+	/* Set the MSI domain */
+	dev_set_msi_domain(&cdx_dev->dev, cdx_msi_domain);
+
 	ret = device_add(&cdx_dev->dev);
 	if (ret != 0) {
 		dev_err(&cdx_dev->dev,
diff --git a/drivers/bus/cdx/cdx.h b/drivers/bus/cdx/cdx.h
index db0569431c10..95df440ebd73 100644
--- a/drivers/bus/cdx/cdx.h
+++ b/drivers/bus/cdx/cdx.h
@@ -20,6 +20,7 @@
  * @res: array of MMIO region entries
  * @res_count: number of valid MMIO regions
  * @req_id: Requestor ID associated with CDX device
+ * @num_msi: Number of MSI's supported by the device
  */
 struct cdx_dev_params_t {
 	u16 vendor;
@@ -29,6 +30,24 @@ struct cdx_dev_params_t {
 	struct resource res[MAX_CDX_DEV_RESOURCES];
 	u8 res_count;
 	u32 req_id;
+	u32 num_msi;
 };
 
+/**
+ * cdx_msi_domain_init - Init the CDX bus MSI domain.
+ * @dev: Device of the CDX bus controller
+ *
+ * Return CDX MSI domain, NULL on failure
+ */
+struct irq_domain *cdx_msi_domain_init(struct device *dev);
+
+/**
+ * cdx_find_msi_domain - Get the CDX-MSI domain.
+ * @dev: CDX controller generic device
+ *
+ * Return CDX MSI domain, NULL on error or if CDX-MSI domain is
+ *   not yet created.
+ */
+struct irq_domain *cdx_find_msi_domain(struct device *parent);
+
 #endif /* _CDX_H_ */
diff --git a/drivers/bus/cdx/cdx_msi.c b/drivers/bus/cdx/cdx_msi.c
new file mode 100644
index 000000000000..2fb7bac18393
--- /dev/null
+++ b/drivers/bus/cdx/cdx_msi.c
@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD CDX bus driver MSI support
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ *
+ */
+
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/msi.h>
+#include <linux/cdx/cdx_bus.h>
+
+#include "cdx.h"
+
+#ifdef GENERIC_MSI_DOMAIN_OPS
+/*
+ * Generate a unique ID identifying the interrupt (only used within the MSI
+ * irqdomain.  Combine the req_id with the interrupt index.
+ */
+static irq_hw_number_t cdx_domain_calc_hwirq(struct cdx_device *dev,
+					     struct msi_desc *desc)
+{
+	/*
+	 * Make the base hwirq value for req_id*10000 so it is readable
+	 * as a decimal value in /proc/interrupts.
+	 */
+	return (irq_hw_number_t)(desc->msi_index + (dev->req_id * 10000));
+}
+
+static void cdx_msi_set_desc(msi_alloc_info_t *arg,
+			     struct msi_desc *desc)
+{
+	arg->desc = desc;
+	arg->hwirq = cdx_domain_calc_hwirq(to_cdx_device(desc->dev), desc);
+}
+#else
+#define cdx_msi_set_desc NULL
+#endif
+
+static void cdx_msi_update_dom_ops(struct msi_domain_info *info)
+{
+	struct msi_domain_ops *ops = info->ops;
+
+	if (!ops)
+		return;
+
+	/* set_desc should not be set by the caller */
+	if (!ops->set_desc)
+		ops->set_desc = cdx_msi_set_desc;
+}
+
+static void cdx_msi_write_msg(struct irq_data *irq_data,
+			      struct msi_msg *msg)
+{
+	/*
+	 * Do nothing as CDX devices have these pre-populated
+	 * in the hardware itself.
+	 */
+}
+
+static void cdx_msi_update_chip_ops(struct msi_domain_info *info)
+{
+	struct irq_chip *chip = info->chip;
+
+	if (!chip)
+		return;
+
+	/*
+	 * irq_write_msi_msg should not be set by the caller
+	 */
+	if (!chip->irq_write_msi_msg)
+		chip->irq_write_msi_msg = cdx_msi_write_msg;
+}
+/**
+ * cdx_msi_create_irq_domain - Create a CDX MSI interrupt domain
+ * @fwnode:	Optional firmware node of the interrupt controller
+ * @info:	MSI domain info
+ * @parent:	Parent irq domain
+ *
+ * Updates the domain and chip ops and creates a CDX MSI
+ * interrupt domain.
+ *
+ * Returns:
+ * A domain pointer or NULL in case of failure.
+ */
+static struct irq_domain *cdx_msi_create_irq_domain(struct fwnode_handle *fwnode,
+						    struct msi_domain_info *info,
+						    struct irq_domain *parent)
+{
+	if (WARN_ON((info->flags & MSI_FLAG_LEVEL_CAPABLE)))
+		info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
+	if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
+		cdx_msi_update_dom_ops(info);
+	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
+		cdx_msi_update_chip_ops(info);
+	info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS;
+
+	return msi_create_irq_domain(fwnode, info, parent);
+}
+
+int cdx_msi_domain_alloc_irqs(struct device *dev, unsigned int irq_count)
+{
+	struct irq_domain *msi_domain;
+	int ret;
+
+	msi_domain = dev_get_msi_domain(dev);
+	if (!msi_domain) {
+		dev_err(dev, "msi domain get failed\n");
+		return -EINVAL;
+	}
+
+	ret = msi_setup_device_data(dev);
+	if (ret) {
+		dev_err(dev, "msi setup device failed: %d\n", ret);
+		return ret;
+	}
+
+	msi_lock_descs(dev);
+	if (msi_first_desc(dev, MSI_DESC_ALL))
+		ret = -EINVAL;
+	msi_unlock_descs(dev);
+	if (ret) {
+		dev_err(dev, "msi setup device failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = msi_domain_alloc_irqs(msi_domain, dev, irq_count);
+	if (ret)
+		dev_err(dev, "Failed to allocate IRQs\n");
+
+	return ret;
+}
+EXPORT_SYMBOL(cdx_msi_domain_alloc_irqs);
+
+void cdx_msi_domain_free_irqs(struct device *dev)
+{
+	struct irq_domain *msi_domain;
+
+	msi_domain = dev_get_msi_domain(dev);
+	if (!msi_domain)
+		return;
+
+	msi_domain_free_irqs(msi_domain, dev);
+}
+EXPORT_SYMBOL(cdx_msi_domain_free_irqs);
+
+static struct irq_chip cdx_msi_irq_chip = {
+	.name = "CDX-MSI",
+	.irq_mask = irq_chip_mask_parent,
+	.irq_unmask = irq_chip_unmask_parent,
+	.irq_eoi = irq_chip_eoi_parent,
+	.irq_set_affinity = msi_domain_set_affinity
+};
+
+static int cdx_msi_prepare(struct irq_domain *msi_domain,
+			   struct device *dev,
+			   int nvec, msi_alloc_info_t *info)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	struct msi_domain_info *msi_info;
+	struct device *parent = dev->parent;
+	u32 dev_id;
+	int ret;
+
+	/* Retrieve device ID from requestor ID using parent device */
+	ret = of_map_id(parent->of_node, cdx_dev->req_id, "msi-map",
+			"msi-map-mask",	NULL, &dev_id);
+	if (ret) {
+		dev_err(dev, "of_map_id failed for MSI: %d\n", ret);
+		return ret;
+	}
+
+	/* Set the device Id to be passed to the GIC-ITS */
+	info->scratchpad[0].ul = dev_id;
+
+	msi_info = msi_get_domain_info(msi_domain->parent);
+
+	/* Allocate at least 32 MSIs, and always as a power of 2 */
+	nvec = max_t(int, 32, roundup_pow_of_two(nvec));
+	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
+}
+
+static struct msi_domain_ops cdx_msi_ops __ro_after_init = {
+	.msi_prepare = cdx_msi_prepare,
+};
+
+static struct msi_domain_info cdx_msi_domain_info = {
+	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
+	.ops	= &cdx_msi_ops,
+	.chip	= &cdx_msi_irq_chip,
+};
+
+struct irq_domain *cdx_msi_domain_init(struct device *dev)
+{
+	struct irq_domain *parent;
+	struct irq_domain *cdx_msi_domain;
+	struct fwnode_handle *fwnode_handle;
+	struct device_node *parent_node;
+	struct device_node *np = dev->of_node;
+
+	fwnode_handle = of_node_to_fwnode(np);
+
+	parent_node = of_parse_phandle(np, "msi-map", 1);
+	if (!parent_node) {
+		dev_err(dev, "msi-map not present on cdx controller\n");
+		return NULL;
+	}
+
+	parent = irq_find_matching_fwnode(of_node_to_fwnode(parent_node),
+			DOMAIN_BUS_NEXUS);
+	if (!parent || !msi_get_domain_info(parent)) {
+		dev_err(dev, "unable to locate ITS domain\n");
+		return NULL;
+	}
+
+	cdx_msi_domain = cdx_msi_create_irq_domain(fwnode_handle,
+				&cdx_msi_domain_info, parent);
+	if (!cdx_msi_domain) {
+		dev_err(dev, "unable to create CDX-MSI domain\n");
+		return NULL;
+	}
+
+	dev_dbg(dev, "CDX-MSI domain created\n");
+
+	return cdx_msi_domain;
+}
+
+struct irq_domain *cdx_find_msi_domain(struct device *parent)
+{
+	return irq_find_host(parent->of_node);
+}
diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
index cc9d30fa02f8..2c8db1f5a057 100644
--- a/drivers/bus/cdx/mcdi_stubs.c
+++ b/drivers/bus/cdx/mcdi_stubs.c
@@ -45,6 +45,7 @@ int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
 	dev_params->res_count = 2;
 
 	dev_params->req_id = 0x250;
+	dev_params->num_msi = 4;
 	dev_params->vendor = 0x10ee;
 	dev_params->device = 0x8084;
 	dev_params->bus_id = bus_id;
diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
index 6e870b2c87d9..bf86024783d2 100644
--- a/include/linux/cdx/cdx_bus.h
+++ b/include/linux/cdx/cdx_bus.h
@@ -25,6 +25,7 @@
  * @dma_mask: Default DMA mask
  * @flags: CDX device flags
  * @req_id: Requestor ID associated with CDX device
+ * @num_msi: Number of MSI's supported by the device
  * @driver_override: driver name to force a match; do not set directly,
  *                   because core frees it; use driver_set_override() to
  *                   set or clear it.
@@ -40,6 +41,7 @@ struct cdx_device {
 	u64 dma_mask;
 	u16 flags;
 	u32 req_id;
+	u32 num_msi;
 	const char *driver_override;
 };
 
@@ -90,4 +92,21 @@ void cdx_driver_unregister(struct cdx_driver *driver);
 
 extern struct bus_type cdx_bus_type;
 
+/**
+ * cdx_msi_domain_alloc_irqs - Allocate MSI's for the CDX device
+ * @dev: device pointer
+ * @irq_count: Number of MSI's to be allocated
+ *
+ * Return 0 for success, -errno on failure
+ */
+int cdx_msi_domain_alloc_irqs(struct device *dev, unsigned int irq_count);
+
+/**
+ * cdx_msi_domain_free_irqs - Free MSI's for CDX device
+ * @dev: device pointer
+ *
+ * Return 0 for success, -errno on failure
+ */
+void cdx_msi_domain_free_irqs(struct device *dev);
+
 #endif /* _CDX_BUS_H_ */
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Jason Gunthorpe]

On Tue, Sep 06, 2022 at 07:17:58PM +0530, Nipun Gupta wrote:

> +static void cdx_msi_write_msg(struct irq_data *irq_data,
> +			      struct msi_msg *msg)
> +{
> +	/*
> +	 * Do nothing as CDX devices have these pre-populated
> +	 * in the hardware itself.
> +	 */
> +}

Huh?

There is no way it can be pre-populated, the addr/data pair,
especially on ARM, is completely under SW control. There is some
commonly used IOVA base in Linux for the ITS page, but no HW should
hardwire that.

Jason

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Tue, 06 Sep 2022 18:19:06 +0100,
Jason Gunthorpe <jgg@nvidia.com> wrote:
> 
> On Tue, Sep 06, 2022 at 07:17:58PM +0530, Nipun Gupta wrote:
> 
> > +static void cdx_msi_write_msg(struct irq_data *irq_data,
> > +			      struct msi_msg *msg)
> > +{
> > +	/*
> > +	 * Do nothing as CDX devices have these pre-populated
> > +	 * in the hardware itself.
> > +	 */
> > +}
> 
> Huh?
> 
> There is no way it can be pre-populated, the addr/data pair,
> especially on ARM, is completely under SW control.

There is nothing in the GIC spec that says that.

> There is some commonly used IOVA base in Linux for the ITS page, but
> no HW should hardwire that.

That's not strictly true. It really depends on how this block is
integrated, and there is a number of existing blocks that know *in HW*
how to signal an LPI.

See, as the canonical example, how the mbigen driver doesn't need to
know about the address of GITS_TRANSLATER.

Yes, this messes with translation (the access is downstream of the
SMMU) if you relied on it to have some isolation, and it has a "black
hole" effect as nobody can have an IOVA that overlaps with the
physical address of the GITS_TRANSLATER register.

But is it illegal as per the architecture? No. It's just stupid.

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Robin Murphy]

On 2022-09-07 12:17, Marc Zyngier wrote:
> On Tue, 06 Sep 2022 18:19:06 +0100,
> Jason Gunthorpe <jgg@nvidia.com> wrote:
>>
>> On Tue, Sep 06, 2022 at 07:17:58PM +0530, Nipun Gupta wrote:
>>
>>> +static void cdx_msi_write_msg(struct irq_data *irq_data,
>>> +			      struct msi_msg *msg)
>>> +{
>>> +	/*
>>> +	 * Do nothing as CDX devices have these pre-populated
>>> +	 * in the hardware itself.
>>> +	 */
>>> +}
>>
>> Huh?
>>
>> There is no way it can be pre-populated, the addr/data pair,
>> especially on ARM, is completely under SW control.
> 
> There is nothing in the GIC spec that says that.
> 
>> There is some commonly used IOVA base in Linux for the ITS page, but
>> no HW should hardwire that.
> 
> That's not strictly true. It really depends on how this block is
> integrated, and there is a number of existing blocks that know *in HW*
> how to signal an LPI.
> 
> See, as the canonical example, how the mbigen driver doesn't need to
> know about the address of GITS_TRANSLATER.
> 
> Yes, this messes with translation (the access is downstream of the
> SMMU) if you relied on it to have some isolation, and it has a "black
> hole" effect as nobody can have an IOVA that overlaps with the
> physical address of the GITS_TRANSLATER register.
> 
> But is it illegal as per the architecture? No. It's just stupid.

If that were the case, then we'd also need a platform quirk so the SMMU 
driver knows about it. Yuck.

But even then, are you suggesting there is some way to convince the ITS 
driver to allocate a specific predetermined EventID when a driver 
requests an MSI? Asking for a friend...

Cheers,
Robin.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Wed, 07 Sep 2022 12:33:12 +0100,
Robin Murphy <robin.murphy@arm.com> wrote:
> 
> On 2022-09-07 12:17, Marc Zyngier wrote:
> > On Tue, 06 Sep 2022 18:19:06 +0100,
> > Jason Gunthorpe <jgg@nvidia.com> wrote:
> >> 
> >> On Tue, Sep 06, 2022 at 07:17:58PM +0530, Nipun Gupta wrote:
> >> 
> >>> +static void cdx_msi_write_msg(struct irq_data *irq_data,
> >>> +			      struct msi_msg *msg)
> >>> +{
> >>> +	/*
> >>> +	 * Do nothing as CDX devices have these pre-populated
> >>> +	 * in the hardware itself.
> >>> +	 */
> >>> +}
> >> 
> >> Huh?
> >> 
> >> There is no way it can be pre-populated, the addr/data pair,
> >> especially on ARM, is completely under SW control.
> > 
> > There is nothing in the GIC spec that says that.
> > 
> >> There is some commonly used IOVA base in Linux for the ITS page, but
> >> no HW should hardwire that.
> > 
> > That's not strictly true. It really depends on how this block is
> > integrated, and there is a number of existing blocks that know *in HW*
> > how to signal an LPI.
> > 
> > See, as the canonical example, how the mbigen driver doesn't need to
> > know about the address of GITS_TRANSLATER.
> > 
> > Yes, this messes with translation (the access is downstream of the
> > SMMU) if you relied on it to have some isolation, and it has a "black
> > hole" effect as nobody can have an IOVA that overlaps with the
> > physical address of the GITS_TRANSLATER register.
> > 
> > But is it illegal as per the architecture? No. It's just stupid.
> 
> If that were the case, then we'd also need a platform quirk so the
> SMMU driver knows about it. Yuck.

Yup. As I said, this is stupid.

> But even then, are you suggesting there is some way to convince the
> ITS driver to allocate a specific predetermined EventID when a driver
> requests an MSI? Asking for a friend...

Of course not. Whoever did that has decided to hardcode the Linux
behaviour into the HW, because it is well known that SW behaviour
never changes. Nononono.

I am >this< tempted to sneak a change into the allocation scheme to
start at 5 or 13 (alternatively), and to map LPIs top-down. That
should get people thinking.

Cheers,

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Radovanovic, Aleksandar]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Marc Zyngier <maz@kernel.org>
> Sent: 07 September 2022 12:17
> To: Jason Gunthorpe <jgg@nvidia.com>
> Cc: Gupta, Nipun <Nipun.Gupta@amd.com>; robh+dt@kernel.org;
> krzysztof.kozlowski+dt@linaro.org; gregkh@linuxfoundation.org;
> rafael@kernel.org; eric.auger@redhat.com; alex.williamson@redhat.com;
> cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> robin.murphy@arm.com; will@kernel.org; joro@8bytes.org;
> masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-
> Xilinx) <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> [CAUTION: External Email]
> 
> On Tue, 06 Sep 2022 18:19:06 +0100,
> Jason Gunthorpe <jgg@nvidia.com> wrote:
> >
> > On Tue, Sep 06, 2022 at 07:17:58PM +0530, Nipun Gupta wrote:
> >
> > > +static void cdx_msi_write_msg(struct irq_data *irq_data,
> > > +                         struct msi_msg *msg) {
> > > +   /*
> > > +    * Do nothing as CDX devices have these pre-populated
> > > +    * in the hardware itself.
> > > +    */
> > > +}
> >
> > Huh?
> >
> > There is no way it can be pre-populated, the addr/data pair,
> > especially on ARM, is completely under SW control.
> 
> There is nothing in the GIC spec that says that.
> 
> > There is some commonly used IOVA base in Linux for the ITS page, but
> > no HW should hardwire that.
> 
> That's not strictly true. It really depends on how this block is integrated, and
> there is a number of existing blocks that know *in HW* how to signal an LPI.
> 
> See, as the canonical example, how the mbigen driver doesn't need to know
> about the address of GITS_TRANSLATER.
> 
> Yes, this messes with translation (the access is downstream of the
> SMMU) if you relied on it to have some isolation, and it has a "black hole"
> effect as nobody can have an IOVA that overlaps with the physical address of
> the GITS_TRANSLATER register.
> 
> But is it illegal as per the architecture? No. It's just stupid.
> 
>         M.
> 
> --
> Without deviation from the norm, progress is not possible.

To give some context, CDX devices are specific to embedded ARM CPUs on the FPGA and a lot of the CDX hardware core is under the control of the system firmware, not the application CPUs. 

That being said, the MSI address is always going to be the GIC GITS_TRANSLATER, which is known to the system firmware, as it is fixed per FPGA platform. At present, we do not allow the application CPU OS to change this - I believe this is for security reasons, but this may or may not be a good idea in general. As Marc mentions, CDX MSI writes are downstream of the SMMU and, if SMMU does not provide identity mapping for GITS_TRANSLATER, then we have a problem and may need to allow the OS to write the address part. However, even if we did, the CDX hardware is limited in that it can only take one GITS_TRANSLATER register target address per system, not per CDX device, nor per MSI vector.

As for the data part (EventID in GIC parlance), this is always going to be the CDX device-relative vector number - I believe this can't be changed, it is a hardware limitation (but I need to double-check). That should be OK, though, as I believe this is exactly what Linux would write anyway, as each CDX device should be in its own IRQ domain (i.e. have its own ITS device table).

The best I can propose is to pass the addr/data info to firmware here, which will then decide what to do with it. At least, it can assert that the values are what the hardware expects and fail loudly if not, rather than having a silently misconfigured system.

Aleksandar

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Wed, 07 Sep 2022 12:35:54 +0100,
"Radovanovic, Aleksandar" <aleksandar.radovanovic@amd.com> wrote:
> 
> [AMD Official Use Only - General]
> 
> 
> 
> > -----Original Message-----
> > From: Marc Zyngier <maz@kernel.org>
> > Sent: 07 September 2022 12:17
> > To: Jason Gunthorpe <jgg@nvidia.com>
> > Cc: Gupta, Nipun <Nipun.Gupta@amd.com>; robh+dt@kernel.org;
> > krzysztof.kozlowski+dt@linaro.org; gregkh@linuxfoundation.org;
> > rafael@kernel.org; eric.auger@redhat.com; alex.williamson@redhat.com;
> > cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> > <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> > mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> > jeffrey.l.hugo@gmail.com; saravanak@google.com;
> > Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> > robin.murphy@arm.com; will@kernel.org; joro@8bytes.org;
> > masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> > kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> > kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> > okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> > Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> > Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-
> > Xilinx) <git@amd.com>
> > Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> > domain as parent
> > 
> > [CAUTION: External Email]
> > 
> > On Tue, 06 Sep 2022 18:19:06 +0100,
> > Jason Gunthorpe <jgg@nvidia.com> wrote:
> > >
> > > On Tue, Sep 06, 2022 at 07:17:58PM +0530, Nipun Gupta wrote:
> > >
> > > > +static void cdx_msi_write_msg(struct irq_data *irq_data,
> > > > +                         struct msi_msg *msg) {
> > > > +   /*
> > > > +    * Do nothing as CDX devices have these pre-populated
> > > > +    * in the hardware itself.
> > > > +    */
> > > > +}
> > >
> > > Huh?
> > >
> > > There is no way it can be pre-populated, the addr/data pair,
> > > especially on ARM, is completely under SW control.
> > 
> > There is nothing in the GIC spec that says that.
> > 
> > > There is some commonly used IOVA base in Linux for the ITS page, but
> > > no HW should hardwire that.
> > 
> > That's not strictly true. It really depends on how this block is integrated, and
> > there is a number of existing blocks that know *in HW* how to signal an LPI.
> > 
> > See, as the canonical example, how the mbigen driver doesn't need to know
> > about the address of GITS_TRANSLATER.
> > 
> > Yes, this messes with translation (the access is downstream of the
> > SMMU) if you relied on it to have some isolation, and it has a "black hole"
> > effect as nobody can have an IOVA that overlaps with the physical address of
> > the GITS_TRANSLATER register.
> > 
> > But is it illegal as per the architecture? No. It's just stupid.
> > 
> >         M.
> > 
> > --
> > Without deviation from the norm, progress is not possible.
> 
> To give some context, CDX devices are specific to embedded ARM CPUs
> on the FPGA and a lot of the CDX hardware core is under the control
> of the system firmware, not the application CPUs.
> 
> That being said, the MSI address is always going to be the GIC
> GITS_TRANSLATER, which is known to the system firmware, as it is
> fixed per FPGA platform. At present, we do not allow the application
> CPU OS to change this - I believe this is for security reasons, but
> this may or may not be a good idea in general.

I'm sure that being downstream of the SMMU is a security feature...

> As Marc mentions, CDX
> MSI writes are downstream of the SMMU and, if SMMU does not provide
> identity mapping for GITS_TRANSLATER, then we have a problem and may
> need to allow the OS to write the address part. However, even if we
> did, the CDX hardware is limited in that it can only take one
> GITS_TRANSLATER register target address per system, not per CDX
> device, nor per MSI vector.

If the MSI generation is downstream of the SMMU, why should the SMMU
provide a 1:1 mapping for GITS_TRANSLATER? I don't think it should
provide a mapping at all in this case. But it looks like I don't
really understand how these things are placed relative to each
other... :-/

> 
> As for the data part (EventID in GIC parlance), this is always going
> to be the CDX device-relative vector number - I believe this can't
> be changed, it is a hardware limitation (but I need to
> double-check). That should be OK, though, as I believe this is
> exactly what Linux would write anyway, as each CDX device should be
> in its own IRQ domain (i.e. have its own ITS device table).

But that's really the worse part. You have hardcoded what is the
*current* Linux behaviour. Things change. And baking SW behaviour into
a piece of HW looks incredibly shortsighted...

> The best I can propose is to pass the addr/data info to firmware
> here, which will then decide what to do with it. At least, it can
> assert that the values are what the hardware expects and fail loudly
> if not, rather than having a silently misconfigured system.

And then what? It means that by agreeing to support this bus, we are
agreeing to *never* change the EventID allocation scheme.

I'm not signing up to this.

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Radovanovic, Aleksandar]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Marc Zyngier <maz@kernel.org>
> Sent: 07 September 2022 13:33
> To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
> Cc: Jason Gunthorpe <jgg@nvidia.com>; Gupta, Nipun
> <Nipun.Gupta@amd.com>; robh+dt@kernel.org;
> krzysztof.kozlowski+dt@linaro.org; gregkh@linuxfoundation.org;
> rafael@kernel.org; eric.auger@redhat.com; alex.williamson@redhat.com;
> cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> robin.murphy@arm.com; will@kernel.org; joro@8bytes.org;
> masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> git (AMD-Xilinx) <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> [CAUTION: External Email]
> 
> > As Marc mentions, CDX
> > MSI writes are downstream of the SMMU and, if SMMU does not provide
> > identity mapping for GITS_TRANSLATER, then we have a problem and may
> > need to allow the OS to write the address part. However, even if we
> > did, the CDX hardware is limited in that it can only take one
> > GITS_TRANSLATER register target address per system, not per CDX
> > device, nor per MSI vector.
> 
> If the MSI generation is downstream of the SMMU, why should the SMMU
> provide a 1:1 mapping for GITS_TRANSLATER? I don't think it should provide a
> mapping at all in this case. But it looks like I don't really understand how
> these things are placed relative to each other... :-/
> 

Apologies, I got my streams confused. It is _upstream_ of the SMMU, it does go through SMMU mapping.

> >
> > As for the data part (EventID in GIC parlance), this is always going
> > to be the CDX device-relative vector number - I believe this can't be
> > changed, it is a hardware limitation (but I need to double-check).
> > That should be OK, though, as I believe this is exactly what Linux
> > would write anyway, as each CDX device should be in its own IRQ domain
> > (i.e. have its own ITS device table).
> 
> But that's really the worse part. You have hardcoded what is the
> *current* Linux behaviour. Things change. And baking SW behaviour into a
> piece of HW looks incredibly shortsighted...

For posterity, I'm not an RTL designer/architect, so share your sentiment to a certain extent. That said, I expect the decision was not based on Linux or any other SW behaviour, but because it is the most straightforward and least expensive way to do it. Having an EventID register for each and every MSI source just so you can program it in any random order costs flops and all the associated complexity of extra RTL logic (think timing closure, etc.), so trade-offs are made. The fact that it matches current Linux behaviour means we just got lucky... 

Anyway, I'm straying off topic here, I'll check with the system architects if there's anything that can be done here. But I'm not feeling hopeful.

Aleksandar

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Wed, 07 Sep 2022 14:18:52 +0100,
"Radovanovic, Aleksandar" <aleksandar.radovanovic@amd.com> wrote:
> >
> > > As Marc mentions, CDX
> > > MSI writes are downstream of the SMMU and, if SMMU does not provide
> > > identity mapping for GITS_TRANSLATER, then we have a problem and may
> > > need to allow the OS to write the address part. However, even if we
> > > did, the CDX hardware is limited in that it can only take one
> > > GITS_TRANSLATER register target address per system, not per CDX
> > > device, nor per MSI vector.
> > 
> > If the MSI generation is downstream of the SMMU, why should the SMMU
> > provide a 1:1 mapping for GITS_TRANSLATER? I don't think it should provide a
> > mapping at all in this case. But it looks like I don't really understand how
> > these things are placed relative to each other... :-/
> > 
> 
> Apologies, I got my streams confused. It is _upstream_ of the SMMU,
> it does go through SMMU mapping.

OK, so you definitely need a mapping, but it cannot be a translation,
and it needs to be in all the possible address spaces. OMG.

> > > As for the data part (EventID in GIC parlance), this is always going
> > > to be the CDX device-relative vector number - I believe this can't be
> > > changed, it is a hardware limitation (but I need to double-check).
> > > That should be OK, though, as I believe this is exactly what Linux
> > > would write anyway, as each CDX device should be in its own IRQ domain
> > > (i.e. have its own ITS device table).
> > 
> > But that's really the worse part. You have hardcoded what is the
> > *current* Linux behaviour. Things change. And baking SW behaviour into a
> > piece of HW looks incredibly shortsighted...
> 
> For posterity, I'm not an RTL designer/architect, so share your
> sentiment to a certain extent. That said, I expect the decision was
> not based on Linux or any other SW behaviour, but because it is the
> most straightforward and least expensive way to do it. Having an
> EventID register for each and every MSI source just so you can
> program it in any random order costs flops and all the associated
> complexity of extra RTL logic (think timing closure, etc.), so
> trade-offs are made. The fact that it matches current Linux
> behaviour means we just got lucky...

Yeah, but that's not the only problem: there is no guarantee that we
have enough LPIs to allocate for the device, so we'll perform a
partial allocation (8 instead of 32 LPIs, for example).

How do you tell the device which events are valid? As far as I can
see, you can't, and I guess it will fire on all of them, resulting in
interrupts being dropped on the floor.

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Radovanovic, Aleksandar]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Marc Zyngier <maz@kernel.org>
> Sent: 08 September 2022 09:08
> To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
> Cc: Jason Gunthorpe <jgg@nvidia.com>; Gupta, Nipun
> <Nipun.Gupta@amd.com>; robh+dt@kernel.org;
> krzysztof.kozlowski+dt@linaro.org; gregkh@linuxfoundation.org;
> rafael@kernel.org; eric.auger@redhat.com; alex.williamson@redhat.com;
> cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> robin.murphy@arm.com; will@kernel.org; joro@8bytes.org;
> masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> git (AMD-Xilinx) <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> [CAUTION: External Email]
>  
> OK, so you definitely need a mapping, but it cannot be a translation, and it
> needs to be in all the possible address spaces. OMG.

Could you elaborate why it needs to be in all the possible address spaces? I'm in no way familiar with kernel IOVA allocation, so not sure I understand this requirement. Note that each CDX device will have its own unique StreamID (in general case, equal to DeviceID sent to the GIC), so, from a SMMU perspective, the mapping can be specific to that device. As long as that IOVA is not allocated to any DMA region for _that_ device, things should be OK? But, I appreciate it might not be that simple from a kernel perspective.

> > > > As for the data part (EventID in GIC parlance), this is always
> > > > going to be the CDX device-relative vector number - I believe this
> > > > can't be changed, it is a hardware limitation (but I need to double-
> check).
> > > > That should be OK, though, as I believe this is exactly what Linux
> > > > would write anyway, as each CDX device should be in its own IRQ
> > > > domain (i.e. have its own ITS device table).
> > >
> > > But that's really the worse part. You have hardcoded what is the
> > > *current* Linux behaviour. Things change. And baking SW behaviour
> > > into a piece of HW looks incredibly shortsighted...
> >
> > For posterity, I'm not an RTL designer/architect, so share your
> > sentiment to a certain extent. That said, I expect the decision was
> > not based on Linux or any other SW behaviour, but because it is the
> > most straightforward and least expensive way to do it. Having an
> > EventID register for each and every MSI source just so you can program
> > it in any random order costs flops and all the associated complexity
> > of extra RTL logic (think timing closure, etc.), so trade-offs are
> > made. The fact that it matches current Linux behaviour means we just
> > got lucky...
> 
> Yeah, but that's not the only problem: there is no guarantee that we have
> enough LPIs to allocate for the device, so we'll perform a partial allocation (8
> instead of 32 LPIs, for example).

Why should that be a problem? The driver will know in advance the number of vectors required by the device. I expect it will need to call some equivalent of  platform_msi_domain_alloc_irqs(), shouldn't that fail if not enough IRQs are allocated (and ultimately fail the probe)? Even if not, we can still inform the firmware in write_msg, which will serve as an indication that a particular vector is enabled. The firmware can decide what to do with the device if not all of the vectors are enabled.

Aleksandar
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Robin Murphy]

On 2022-09-08 10:51, Radovanovic, Aleksandar wrote:
> [AMD Official Use Only - General]
> 
> 
> 
>> -----Original Message-----
>> From: Marc Zyngier <maz@kernel.org>
>> Sent: 08 September 2022 09:08
>> To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
>> Cc: Jason Gunthorpe <jgg@nvidia.com>; Gupta, Nipun
>> <Nipun.Gupta@amd.com>; robh+dt@kernel.org;
>> krzysztof.kozlowski+dt@linaro.org; gregkh@linuxfoundation.org;
>> rafael@kernel.org; eric.auger@redhat.com; alex.williamson@redhat.com;
>> cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
>> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
>> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
>> jeffrey.l.hugo@gmail.com; saravanak@google.com;
>> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
>> robin.murphy@arm.com; will@kernel.org; joro@8bytes.org;
>> masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
>> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
>> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
>> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
>> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
>> git (AMD-Xilinx) <git@amd.com>
>> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
>> domain as parent
>>
>> [CAUTION: External Email]
>>   
>> OK, so you definitely need a mapping, but it cannot be a translation, and it
>> needs to be in all the possible address spaces. OMG.
> 
> Could you elaborate why it needs to be in all the possible address spaces? I'm in no way familiar with kernel IOVA allocation, so not sure I understand this requirement. Note that each CDX device will have its own unique StreamID (in general case, equal to DeviceID sent to the GIC), so, from a SMMU perspective, the mapping can be specific to that device. As long as that IOVA is not allocated to any DMA region for _that_ device, things should be OK? But, I appreciate it might not be that simple from a kernel perspective.

That's the point - any device could could have its own mapping, 
therefore that hole has to be punched in *every* mapping that any of 
those devices could use, so that MSI writes don't unexpectedly fault, or 
corrupt memory if that address is free to be used to map a DMA buffer. 
At least the HiSilicon PCI quirk is functionally similar (for slightly 
different underlying reasons) so there's already precedent and an 
example that you can follow to a reasonable degree.

Robin.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Thu, 08 Sep 2022 10:51:01 +0100,
"Radovanovic, Aleksandar" <aleksandar.radovanovic@amd.com> wrote:
> 
> [AMD Official Use Only - General]
> 
> 
> 
> > -----Original Message-----
> > From: Marc Zyngier <maz@kernel.org>
> > Sent: 08 September 2022 09:08
> > To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
> > Cc: Jason Gunthorpe <jgg@nvidia.com>; Gupta, Nipun
> > <Nipun.Gupta@amd.com>; robh+dt@kernel.org;
> > krzysztof.kozlowski+dt@linaro.org; gregkh@linuxfoundation.org;
> > rafael@kernel.org; eric.auger@redhat.com; alex.williamson@redhat.com;
> > cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> > <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> > mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> > jeffrey.l.hugo@gmail.com; saravanak@google.com;
> > Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> > robin.murphy@arm.com; will@kernel.org; joro@8bytes.org;
> > masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> > kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> > kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> > okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> > Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> > git (AMD-Xilinx) <git@amd.com>
> > Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> > domain as parent
> > 
> > [CAUTION: External Email]
> >  
> > OK, so you definitely need a mapping, but it cannot be a translation, and it
> > needs to be in all the possible address spaces. OMG.
> 
> Could you elaborate why it needs to be in all the possible address
> spaces? I'm in no way familiar with kernel IOVA allocation, so not
> sure I understand this requirement. Note that each CDX device will
> have its own unique StreamID (in general case, equal to DeviceID
> sent to the GIC), so, from a SMMU perspective, the mapping can be
> specific to that device. As long as that IOVA is not allocated to
> any DMA region for _that_ device, things should be OK? But, I
> appreciate it might not be that simple from a kernel perspective.

Robin answered that one. This also has direct impacts on vfio and
virtualisation, as your userspace/VM cannot have any memory at the
address of the ITS (because you cannot DMA to it).

>
> > > > > As for the data part (EventID in GIC parlance), this is always
> > > > > going to be the CDX device-relative vector number - I believe this
> > > > > can't be changed, it is a hardware limitation (but I need to double-
> > check).
> > > > > That should be OK, though, as I believe this is exactly what Linux
> > > > > would write anyway, as each CDX device should be in its own IRQ
> > > > > domain (i.e. have its own ITS device table).
> > > >
> > > > But that's really the worse part. You have hardcoded what is the
> > > > *current* Linux behaviour. Things change. And baking SW behaviour
> > > > into a piece of HW looks incredibly shortsighted...
> > >
> > > For posterity, I'm not an RTL designer/architect, so share your
> > > sentiment to a certain extent. That said, I expect the decision was
> > > not based on Linux or any other SW behaviour, but because it is the
> > > most straightforward and least expensive way to do it. Having an
> > > EventID register for each and every MSI source just so you can program
> > > it in any random order costs flops and all the associated complexity
> > > of extra RTL logic (think timing closure, etc.), so trade-offs are
> > > made. The fact that it matches current Linux behaviour means we just
> > > got lucky...
> > 
> > Yeah, but that's not the only problem: there is no guarantee that we have
> > enough LPIs to allocate for the device, so we'll perform a partial allocation (8
> > instead of 32 LPIs, for example).
> 
> Why should that be a problem? The driver will know in advance the
> number of vectors required by the device. I expect it will need to
> call some equivalent of platform_msi_domain_alloc_irqs(), shouldn't
> that fail if not enough IRQs are allocated (and ultimately fail the
> probe)?

No, that's not how MSIs work in Linux: this is a best effort
allocation, where the allocator is free to dish out the number of MSIs
it can allocate at this point in time (the ITS will divide the
allocation by two until it succeeds). If the end-point driver doesn't
like it, it can of course decide to fail its own probe.

> Even if not, we can still inform the firmware in write_msg,
> which will serve as an indication that a particular vector is
> enabled. The firmware can decide what to do with the device if not
> all of the vectors are enabled.

Do you anticipate that the FW will be involved at the point where the
driver is finalising the device configuration? It seems that you need
to come up with some sort of spec for this if there isn't one yet.

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Tue, 06 Sep 2022 14:47:58 +0100,
Nipun Gupta <nipun.gupta@amd.com> wrote:
> 
> Devices on cdx bus are dynamically detected and registered using
> platform_device_register API. As these devices are not linked to
> of node they need a separate MSI domain for handling device ID
> to be provided to the GIC ITS domain.
> 
> This also introduces APIs to alloc and free IRQs for CDX domain.
> 
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> Signed-off-by: Nikhil Agarwal <nikhil.agarwal@amd.com>
> ---
>  drivers/bus/cdx/cdx.c        |  18 +++
>  drivers/bus/cdx/cdx.h        |  19 +++
>  drivers/bus/cdx/cdx_msi.c    | 236 +++++++++++++++++++++++++++++++++++
>  drivers/bus/cdx/mcdi_stubs.c |   1 +
>  include/linux/cdx/cdx_bus.h  |  19 +++
>  5 files changed, 293 insertions(+)
>  create mode 100644 drivers/bus/cdx/cdx_msi.c
> 
> diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
> index fc417c32c59b..02ececce1c84 100644
> --- a/drivers/bus/cdx/cdx.c
> +++ b/drivers/bus/cdx/cdx.c
> @@ -17,6 +17,7 @@
>  #include <linux/dma-map-ops.h>
>  #include <linux/property.h>
>  #include <linux/iommu.h>
> +#include <linux/msi.h>
>  #include <linux/cdx/cdx_bus.h>
>  
>  #include "cdx.h"
> @@ -236,6 +237,7 @@ static int cdx_device_add(struct device *parent,
>  			  struct cdx_dev_params_t *dev_params)
>  {
>  	struct cdx_device *cdx_dev;
> +	struct irq_domain *cdx_msi_domain;
>  	int ret;
>  
>  	cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
> @@ -252,6 +254,7 @@ static int cdx_device_add(struct device *parent,
>  
>  	/* Populate CDX dev params */
>  	cdx_dev->req_id = dev_params->req_id;
> +	cdx_dev->num_msi = dev_params->num_msi;
>  	cdx_dev->vendor = dev_params->vendor;
>  	cdx_dev->device = dev_params->device;
>  	cdx_dev->bus_id = dev_params->bus_id;
> @@ -269,6 +272,21 @@ static int cdx_device_add(struct device *parent,
>  	dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", cdx_dev->bus_id,
>  			cdx_dev->func_id);
>  
> +	/* If CDX MSI domain is not created, create one. */
> +	cdx_msi_domain = cdx_find_msi_domain(parent);

Why do we need such a wrapper around find_host_domain()?

> +	if (!cdx_msi_domain) {
> +		cdx_msi_domain = cdx_msi_domain_init(parent);

This is racy. If device are populated in parallel, bad things will
happen.

> +		if (!cdx_msi_domain) {
> +			dev_err(&cdx_dev->dev,
> +				"cdx_msi_domain_init() failed: %d", ret);
> +			kfree(cdx_dev);
> +			return -1;

Use standard error codes.

> +		}
> +	}
> +
> +	/* Set the MSI domain */
> +	dev_set_msi_domain(&cdx_dev->dev, cdx_msi_domain);
> +
>  	ret = device_add(&cdx_dev->dev);
>  	if (ret != 0) {
>  		dev_err(&cdx_dev->dev,
> diff --git a/drivers/bus/cdx/cdx.h b/drivers/bus/cdx/cdx.h
> index db0569431c10..95df440ebd73 100644
> --- a/drivers/bus/cdx/cdx.h
> +++ b/drivers/bus/cdx/cdx.h
> @@ -20,6 +20,7 @@
>   * @res: array of MMIO region entries
>   * @res_count: number of valid MMIO regions
>   * @req_id: Requestor ID associated with CDX device
> + * @num_msi: Number of MSI's supported by the device
>   */
>  struct cdx_dev_params_t {
>  	u16 vendor;
> @@ -29,6 +30,24 @@ struct cdx_dev_params_t {
>  	struct resource res[MAX_CDX_DEV_RESOURCES];
>  	u8 res_count;
>  	u32 req_id;
> +	u32 num_msi;
>  };
>  
> +/**
> + * cdx_msi_domain_init - Init the CDX bus MSI domain.
> + * @dev: Device of the CDX bus controller
> + *
> + * Return CDX MSI domain, NULL on failure
> + */
> +struct irq_domain *cdx_msi_domain_init(struct device *dev);
> +
> +/**
> + * cdx_find_msi_domain - Get the CDX-MSI domain.
> + * @dev: CDX controller generic device
> + *
> + * Return CDX MSI domain, NULL on error or if CDX-MSI domain is
> + *   not yet created.
> + */
> +struct irq_domain *cdx_find_msi_domain(struct device *parent);
> +
>  #endif /* _CDX_H_ */
> diff --git a/drivers/bus/cdx/cdx_msi.c b/drivers/bus/cdx/cdx_msi.c
> new file mode 100644
> index 000000000000..2fb7bac18393
> --- /dev/null
> +++ b/drivers/bus/cdx/cdx_msi.c
> @@ -0,0 +1,236 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * AMD CDX bus driver MSI support
> + *
> + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> + *
> + */
> +
> +#include <linux/of.h>
> +#include <linux/of_device.h>
> +#include <linux/of_address.h>
> +#include <linux/of_irq.h>
> +#include <linux/irq.h>
> +#include <linux/irqdomain.h>
> +#include <linux/msi.h>
> +#include <linux/cdx/cdx_bus.h>
> +
> +#include "cdx.h"
> +
> +#ifdef GENERIC_MSI_DOMAIN_OPS
> +/*
> + * Generate a unique ID identifying the interrupt (only used within the MSI
> + * irqdomain.  Combine the req_id with the interrupt index.
> + */
> +static irq_hw_number_t cdx_domain_calc_hwirq(struct cdx_device *dev,
> +					     struct msi_desc *desc)
> +{
> +	/*
> +	 * Make the base hwirq value for req_id*10000 so it is readable
> +	 * as a decimal value in /proc/interrupts.
> +	 */
> +	return (irq_hw_number_t)(desc->msi_index + (dev->req_id * 10000));

No, please. Use shifts, and use a script if decimal conversion fails
you. We're not playing these games. And the cast is pointless.

Yes, you have lifted it from the FSL code, bad move. /me makes a note
to go and clean-up this crap.

> +}
> +
> +static void cdx_msi_set_desc(msi_alloc_info_t *arg,
> +			     struct msi_desc *desc)
> +{
> +	arg->desc = desc;
> +	arg->hwirq = cdx_domain_calc_hwirq(to_cdx_device(desc->dev), desc);
> +}
> +#else
> +#define cdx_msi_set_desc NULL

Why the ifdefery? This should *only* be supported with
GENERIC_MSI_DOMAIN_OPS.

> +#endif
> +
> +static void cdx_msi_update_dom_ops(struct msi_domain_info *info)
> +{
> +	struct msi_domain_ops *ops = info->ops;
> +
> +	if (!ops)
> +		return;
> +
> +	/* set_desc should not be set by the caller */
> +	if (!ops->set_desc)
> +		ops->set_desc = cdx_msi_set_desc;

Then why are you allowing this to be overridden?

> +}
> +
> +static void cdx_msi_write_msg(struct irq_data *irq_data,
> +			      struct msi_msg *msg)
> +{
> +	/*
> +	 * Do nothing as CDX devices have these pre-populated
> +	 * in the hardware itself.
> +	 */

We talked about this in a separate thread. This is a major problem.

> +}
> +
> +static void cdx_msi_update_chip_ops(struct msi_domain_info *info)
> +{
> +	struct irq_chip *chip = info->chip;
> +
> +	if (!chip)
> +		return;
> +
> +	/*
> +	 * irq_write_msi_msg should not be set by the caller
> +	 */
> +	if (!chip->irq_write_msi_msg)
> +		chip->irq_write_msi_msg = cdx_msi_write_msg;

Then why the check?

> +}
> +/**
> + * cdx_msi_create_irq_domain - Create a CDX MSI interrupt domain
> + * @fwnode:	Optional firmware node of the interrupt controller
> + * @info:	MSI domain info
> + * @parent:	Parent irq domain
> + *
> + * Updates the domain and chip ops and creates a CDX MSI
> + * interrupt domain.
> + *
> + * Returns:
> + * A domain pointer or NULL in case of failure.
> + */
> +static struct irq_domain *cdx_msi_create_irq_domain(struct fwnode_handle *fwnode,
> +						    struct msi_domain_info *info,
> +						    struct irq_domain *parent)
> +{
> +	if (WARN_ON((info->flags & MSI_FLAG_LEVEL_CAPABLE)))
> +		info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;

No. Just fail the domain creation. We shouldn't paper over these things.

> +	if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
> +		cdx_msi_update_dom_ops(info);
> +	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
> +		cdx_msi_update_chip_ops(info);

Under what circumstances would the default ops not be used? The only
caller is in this file and has pre-computed values.

This looks like a copy/paste from platform-msi.c.

> +	info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS;
> +
> +	return msi_create_irq_domain(fwnode, info, parent);

This whole function makes no sense. You should move everything to the
relevant structures, and simply call msi_create_irq_domain() from the
sole caller of this function.

> +}
> +
> +int cdx_msi_domain_alloc_irqs(struct device *dev, unsigned int irq_count)
> +{
> +	struct irq_domain *msi_domain;
> +	int ret;
> +
> +	msi_domain = dev_get_msi_domain(dev);
> +	if (!msi_domain) {

How can that happen?

> +		dev_err(dev, "msi domain get failed\n");
> +		return -EINVAL;
> +	}
> +
> +	ret = msi_setup_device_data(dev);
> +	if (ret) {
> +		dev_err(dev, "msi setup device failed: %d\n", ret);
> +		return ret;
> +	}
> +
> +	msi_lock_descs(dev);
> +	if (msi_first_desc(dev, MSI_DESC_ALL))
> +		ret = -EINVAL;
> +	msi_unlock_descs(dev);
> +	if (ret) {
> +		dev_err(dev, "msi setup device failed: %d\n", ret);

Same message twice, not very useful. Consider grouping these things at
the end of the function and make use of a (oh Gawd) goto...

> +		return ret;
> +	}
> +
> +	ret = msi_domain_alloc_irqs(msi_domain, dev, irq_count);
> +	if (ret)
> +		dev_err(dev, "Failed to allocate IRQs\n");
> +
> +	return ret;
> +}
> +EXPORT_SYMBOL(cdx_msi_domain_alloc_irqs);

EXPORT_SYMBOL_GPL(), please, for all the exports.

> +
> +void cdx_msi_domain_free_irqs(struct device *dev)
> +{
> +	struct irq_domain *msi_domain;
> +
> +	msi_domain = dev_get_msi_domain(dev);
> +	if (!msi_domain)

Again, how can that happen?

> +		return;
> +
> +	msi_domain_free_irqs(msi_domain, dev);
> +}
> +EXPORT_SYMBOL(cdx_msi_domain_free_irqs);

This feels like a very pointless helper, and again a copy/paste from
the FSL code. I'd rather you change msi_domain_free_irqs() to only
take a device and use the implicit MSI domain.

> +
> +static struct irq_chip cdx_msi_irq_chip = {
> +	.name = "CDX-MSI",
> +	.irq_mask = irq_chip_mask_parent,
> +	.irq_unmask = irq_chip_unmask_parent,
> +	.irq_eoi = irq_chip_eoi_parent,
> +	.irq_set_affinity = msi_domain_set_affinity

nit: please align things vertically.

> +};
> +
> +static int cdx_msi_prepare(struct irq_domain *msi_domain,
> +			   struct device *dev,
> +			   int nvec, msi_alloc_info_t *info)
> +{
> +	struct cdx_device *cdx_dev = to_cdx_device(dev);
> +	struct msi_domain_info *msi_info;
> +	struct device *parent = dev->parent;
> +	u32 dev_id;
> +	int ret;
> +
> +	/* Retrieve device ID from requestor ID using parent device */
> +	ret = of_map_id(parent->of_node, cdx_dev->req_id, "msi-map",
> +			"msi-map-mask",	NULL, &dev_id);
> +	if (ret) {
> +		dev_err(dev, "of_map_id failed for MSI: %d\n", ret);
> +		return ret;
> +	}
> +
> +	/* Set the device Id to be passed to the GIC-ITS */
> +	info->scratchpad[0].ul = dev_id;
> +
> +	msi_info = msi_get_domain_info(msi_domain->parent);
> +
> +	/* Allocate at least 32 MSIs, and always as a power of 2 */

Where is this requirement coming from?

> +	nvec = max_t(int, 32, roundup_pow_of_two(nvec));
> +	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
> +}
> +
> +static struct msi_domain_ops cdx_msi_ops __ro_after_init = {
> +	.msi_prepare = cdx_msi_prepare,
> +};
> +
> +static struct msi_domain_info cdx_msi_domain_info = {
> +	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
> +	.ops	= &cdx_msi_ops,
> +	.chip	= &cdx_msi_irq_chip,
> +};
> +
> +struct irq_domain *cdx_msi_domain_init(struct device *dev)
> +{
> +	struct irq_domain *parent;
> +	struct irq_domain *cdx_msi_domain;
> +	struct fwnode_handle *fwnode_handle;
> +	struct device_node *parent_node;
> +	struct device_node *np = dev->of_node;
> +
> +	fwnode_handle = of_node_to_fwnode(np);
> +
> +	parent_node = of_parse_phandle(np, "msi-map", 1);

Huh. This only works because you are stuck with a single ITS per system.

> +	if (!parent_node) {
> +		dev_err(dev, "msi-map not present on cdx controller\n");
> +		return NULL;
> +	}
> +
> +	parent = irq_find_matching_fwnode(of_node_to_fwnode(parent_node),
> +			DOMAIN_BUS_NEXUS);
> +	if (!parent || !msi_get_domain_info(parent)) {
> +		dev_err(dev, "unable to locate ITS domain\n");
> +		return NULL;
> +	}
> +
> +	cdx_msi_domain = cdx_msi_create_irq_domain(fwnode_handle,
> +				&cdx_msi_domain_info, parent);
> +	if (!cdx_msi_domain) {
> +		dev_err(dev, "unable to create CDX-MSI domain\n");
> +		return NULL;
> +	}
> +
> +	dev_dbg(dev, "CDX-MSI domain created\n");
> +
> +	return cdx_msi_domain;
> +}
> +
> +struct irq_domain *cdx_find_msi_domain(struct device *parent)
> +{
> +	return irq_find_host(parent->of_node);
> +}
> diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
> index cc9d30fa02f8..2c8db1f5a057 100644
> --- a/drivers/bus/cdx/mcdi_stubs.c
> +++ b/drivers/bus/cdx/mcdi_stubs.c
> @@ -45,6 +45,7 @@ int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
>  	dev_params->res_count = 2;
>  
>  	dev_params->req_id = 0x250;
> +	dev_params->num_msi = 4;

Why the hardcoded 4? Is that part of the firmware emulation stuff?

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Marc Zyngier <maz@kernel.org>
> Sent: Wednesday, September 7, 2022 6:48 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-
> ENG) <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com; will@kernel.org;
> joro@8bytes.org; masahiroy@kernel.org; ndesaulniers@google.com; linux-
> arm-kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-
> Xilinx) <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> [CAUTION: External Email]
> 
> On Tue, 06 Sep 2022 14:47:58 +0100,
> Nipun Gupta <nipun.gupta@amd.com> wrote:
> >
> > Devices on cdx bus are dynamically detected and registered using
> > platform_device_register API. As these devices are not linked to
> > of node they need a separate MSI domain for handling device ID
> > to be provided to the GIC ITS domain.
> >
> > This also introduces APIs to alloc and free IRQs for CDX domain.
> >
> > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> > Signed-off-by: Nikhil Agarwal <nikhil.agarwal@amd.com>
> > ---
> >  drivers/bus/cdx/cdx.c        |  18 +++
> >  drivers/bus/cdx/cdx.h        |  19 +++
> >  drivers/bus/cdx/cdx_msi.c    | 236
> +++++++++++++++++++++++++++++++++++
> >  drivers/bus/cdx/mcdi_stubs.c |   1 +
> >  include/linux/cdx/cdx_bus.h  |  19 +++
> >  5 files changed, 293 insertions(+)
> >  create mode 100644 drivers/bus/cdx/cdx_msi.c
> >

<..>

> > +             return;
> > +
> > +     msi_domain_free_irqs(msi_domain, dev);
> > +}
> > +EXPORT_SYMBOL(cdx_msi_domain_free_irqs);
> 
> This feels like a very pointless helper, and again a copy/paste from
> the FSL code. I'd rather you change msi_domain_free_irqs() to only
> take a device and use the implicit MSI domain.

I agree with other comments except this one.

In current implementation we have an API "cdx_msi_domain_alloc_irqs()",
so having "cdx_msi_domain_free_irqs()" seems legitimate, as the caller
would allocate and free MSI's using a similar APIs (cdx_msi_domain*).

Changing msi_domain_free_irqs() to use implicit msi domain in case
msi_domain is not provided by the caller seems appropriate, Ill change the
same for "msi_domain_alloc_irqs()" too.

<..>

> > diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
> > index cc9d30fa02f8..2c8db1f5a057 100644
> > --- a/drivers/bus/cdx/mcdi_stubs.c
> > +++ b/drivers/bus/cdx/mcdi_stubs.c
> > @@ -45,6 +45,7 @@ int cdx_mcdi_get_func_config(struct cdx_mcdi_t
> *cdx_mcdi,
> >       dev_params->res_count = 2;
> >
> >       dev_params->req_id = 0x250;
> > +     dev_params->num_msi = 4;
> 
> Why the hardcoded 4? Is that part of the firmware emulation stuff?

Yes, this is currently part of emulation, and would change with proper
emulation support.

> 
>         M.
> 
> --
> Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Marc Zyngier]

On Thu, 08 Sep 2022 15:13:31 +0100,
"Gupta, Nipun" <Nipun.Gupta@amd.com> wrote:
> 
>
> > > +             return;
> > > +
> > > +     msi_domain_free_irqs(msi_domain, dev);
> > > +}
> > > +EXPORT_SYMBOL(cdx_msi_domain_free_irqs);
> > 
> > This feels like a very pointless helper, and again a copy/paste from
> > the FSL code. I'd rather you change msi_domain_free_irqs() to only
> > take a device and use the implicit MSI domain.
> 
> I agree with other comments except this one.
> 
> In current implementation we have an API "cdx_msi_domain_alloc_irqs()",
> so having "cdx_msi_domain_free_irqs()" seems legitimate, as the caller
> would allocate and free MSI's using a similar APIs (cdx_msi_domain*).

Why would that be a problem? Using generic functions when they apply
should be the default, and "specialised" helpers are only here as a
reminder that our MSI API still needs serious improvement.

> Changing msi_domain_free_irqs() to use implicit msi domain in case
> msi_domain is not provided by the caller seems appropriate, Ill change the
> same for "msi_domain_alloc_irqs()" too.

What I'm asking is that there is no explicit msi_domain anymore. We
always use the one referenced by the device. And if that can be done
on the allocation path too, great.

> <..>
> 
> > > diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
> > > index cc9d30fa02f8..2c8db1f5a057 100644
> > > --- a/drivers/bus/cdx/mcdi_stubs.c
> > > +++ b/drivers/bus/cdx/mcdi_stubs.c
> > > @@ -45,6 +45,7 @@ int cdx_mcdi_get_func_config(struct cdx_mcdi_t
> > *cdx_mcdi,
> > >       dev_params->res_count = 2;
> > >
> > >       dev_params->req_id = 0x250;
> > > +     dev_params->num_msi = 4;
> > 
> > Why the hardcoded 4? Is that part of the firmware emulation stuff?
> 
> Yes, this is currently part of emulation, and would change with proper
> emulation support.

What "proper emulation support"? I expect no emulation at all, but
instead a well defined probing method.

	M.

-- 
Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Marc Zyngier <maz@kernel.org>
> Sent: Thursday, September 8, 2022 8:00 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com; jeffrey.l.hugo@gmail.com;
> saravanak@google.com; Michael.Srba@seznam.cz; mani@kernel.org;
> yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com;
> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
> Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> [CAUTION: External Email]
> 
> On Thu, 08 Sep 2022 15:13:31 +0100,
> "Gupta, Nipun" <Nipun.Gupta@amd.com> wrote:
> >
> >
> > > > +             return;
> > > > +
> > > > +     msi_domain_free_irqs(msi_domain, dev);
> > > > +}
> > > > +EXPORT_SYMBOL(cdx_msi_domain_free_irqs);
> > >
> > > This feels like a very pointless helper, and again a copy/paste from
> > > the FSL code. I'd rather you change msi_domain_free_irqs() to only
> > > take a device and use the implicit MSI domain.
> >
> > I agree with other comments except this one.
> >
> > In current implementation we have an API "cdx_msi_domain_alloc_irqs()",
> > so having "cdx_msi_domain_free_irqs()" seems legitimate, as the caller
> > would allocate and free MSI's using a similar APIs (cdx_msi_domain*).
> 
> Why would that be a problem? Using generic functions when they apply
> should be the default, and "specialised" helpers are only here as a
> reminder that our MSI API still needs serious improvement.

We can remove the wrapper API, rather have a #define to provide same name
convention for alloc and free IRQ APIs for CDX drivers. But both ways if we use
#define or direct use of msi_domain_free_irqs() API, we need
msi_domain_free_irqs() symbol exported I hope having export symbol to this
API would not be a problem.

> 
> > Changing msi_domain_free_irqs() to use implicit msi domain in case
> > msi_domain is not provided by the caller seems appropriate, Ill change the
> > same for "msi_domain_alloc_irqs()" too.
> 
> What I'm asking is that there is no explicit msi_domain anymore. We
> always use the one referenced by the device. And if that can be done
> on the allocation path too, great.

I think it can be removed from both the APIs. Also, API's
msi_domain_alloc_irqs_descs_locked() and msi_domain_free_irqs_descs_locked()
can have similar change.

> 
> > <..>
> >
> > > > diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
> > > > index cc9d30fa02f8..2c8db1f5a057 100644
> > > > --- a/drivers/bus/cdx/mcdi_stubs.c
> > > > +++ b/drivers/bus/cdx/mcdi_stubs.c
> > > > @@ -45,6 +45,7 @@ int cdx_mcdi_get_func_config(struct cdx_mcdi_t
> > > *cdx_mcdi,
> > > >       dev_params->res_count = 2;
> > > >
> > > >       dev_params->req_id = 0x250;
> > > > +     dev_params->num_msi = 4;
> > >
> > > Why the hardcoded 4? Is that part of the firmware emulation stuff?
> >
> > Yes, this is currently part of emulation, and would change with proper
> > emulation support.
> 
> What "proper emulation support"? I expect no emulation at all, but
> instead a well defined probing method.

I meant proper firmware interfacing support for probing.

> 
>         M.
> 
> --
> Without deviation from the norm, progress is not possible.

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Gupta, Nipun]

[AMD Official Use Only - General]


<snip>

> 
> > +}
> > +
> > +static void cdx_msi_write_msg(struct irq_data *irq_data,
> > +                           struct msi_msg *msg)
> > +{
> > +     /*
> > +      * Do nothing as CDX devices have these pre-populated
> > +      * in the hardware itself.
> > +      */
> 
> We talked about this in a separate thread. This is a major problem.

We discussed this further with the hardware design team and now have the
correct and complete understanding here. As the CDX devices are FPGA based,
they don't exist yet, so it would be possible to construct them in such a way
that the eventid is programable.

To make it generic for CDX devices, we have added a firmware API which provide
the mappings (MSI vector ID to eventID) to the fabric, that can be referred by the
device while generating the MSI interrupt.

Also, there is an existing table to have GITS_TRANSLATOR iova address (address in
msi_msg) for CDX devices, which can be programmed by the firmware. So, providing
IOVA address to device would also not be a problem here.

We would be rolling out RFC v4 with these changes soon.

Regards,
Nipun

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Radovanovic, Aleksandar]

[AMD Official Use Only - General]

> -----Original Message-----
> From: Gupta, Nipun <Nipun.Gupta@amd.com>
> Sent: 12 October 2022 11:04
> To: Marc Zyngier <maz@kernel.org>; Robin Murphy
> <robin.murphy@arm.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-
> ENG) <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> jgg@ziepe.ca; jgg@nvidia.com; will@kernel.org; joro@8bytes.org;
> masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-
> Xilinx) <git@amd.com>
> Subject: RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> [AMD Official Use Only - General]
> 
> 
> <snip>
> 
> >
> > > +}
> > > +
> > > +static void cdx_msi_write_msg(struct irq_data *irq_data,
> > > +                           struct msi_msg *msg) {
> > > +     /*
> > > +      * Do nothing as CDX devices have these pre-populated
> > > +      * in the hardware itself.
> > > +      */
> >
> > We talked about this in a separate thread. This is a major problem.
> 
> We discussed this further with the hardware design team and now have the
> correct and complete understanding here. As the CDX devices are FPGA
> based, they don't exist yet, so it would be possible to construct them in such
> a way that the eventid is programable.
> 
> To make it generic for CDX devices, we have added a firmware API which
> provide the mappings (MSI vector ID to eventID) to the fabric, that can be
> referred by the device while generating the MSI interrupt.
> 
> Also, there is an existing table to have GITS_TRANSLATOR iova address
> (address in
> msi_msg) for CDX devices, which can be programmed by the firmware. So,
> providing IOVA address to device would also not be a problem here.
> 
> We would be rolling out RFC v4 with these changes soon.
> 
> Regards,
> Nipun

Just to be clear, there will be some HW limitations with the proposed solution,
so let's just make sure that we're all OK with it.

For the MSI EventID, the HW interrupt logic assumes the MSI write value is 
equal to the MSI vector number. However, the vector number is programmable
for most (all) of the interrupt sources, which isn't exactly the same as saying
EventID is programmable for a vector number, but can be used to emulate the
desired behaviour, with a translation table in firmware. 

The limitation here is that we support at most 16 bits of EventID (and this still
needs to be confirmed for all interrupt sources)

As for GITS_TRANSLATER, we can take up to 4 different IOVAs, which limits us
to 4 CDX devices (should be sufficient for current HW use-cases). Also, it means
that the address part must be the same for all vectors within a single CDX 
device. I'm assuming this is OK as it is going to be a single interrupt and IOMMU
domain anyway.

Thanks,
Aleksandar
_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Jason Gunthorpe]

On Wed, Oct 12, 2022 at 10:34:23AM +0000, Radovanovic, Aleksandar wrote:

> For the MSI EventID, the HW interrupt logic assumes the MSI write value is 
> equal to the MSI vector number. However, the vector number is programmable
> for most (all) of the interrupt sources, which isn't exactly the same as saying
> EventID is programmable for a vector number, but can be used to emulate the
> desired behaviour, with a translation table in firmware. 

If you do this stuff wrong you will eventually run into situations
that don't work. Like VFIO/VMs and things.

> As for GITS_TRANSLATER, we can take up to 4 different IOVAs, which limits us
> to 4 CDX devices (should be sufficient for current HW use-cases). Also, it means
> that the address part must be the same for all vectors within a single CDX 
> device. I'm assuming this is OK as it is going to be a single interrupt and IOMMU
> domain anyway.

This is not at all how MSI is supposed to work.

Jason

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Radovanovic, Aleksandar]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Jason Gunthorpe <jgg@ziepe.ca>
> Sent: 12 October 2022 14:02
> To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
> Cc: Gupta, Nipun <Nipun.Gupta@amd.com>; Marc Zyngier
> <maz@kernel.org>; Robin Murphy <robin.murphy@arm.com>;
> robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-
> ENG) <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org;
> Anand, Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>; git
> (AMD-Xilinx) <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> Caution: This message originated from an External Source. Use proper
> caution when opening attachments, clicking links, or responding.
> 
> 
> On Wed, Oct 12, 2022 at 10:34:23AM +0000, Radovanovic, Aleksandar wrote:
> 
> 
> > As for GITS_TRANSLATER, we can take up to 4 different IOVAs, which
> > limits us to 4 CDX devices (should be sufficient for current HW
> > use-cases). Also, it means that the address part must be the same for
> > all vectors within a single CDX device. I'm assuming this is OK as it
> > is going to be a single interrupt and IOMMU domain anyway.
> 
> This is not at all how MSI is supposed to work.

In the general case, no, they're not. However, this is an embedded device
with a GICv3, so the general case does not really apply. On GIC, the MSI 
target address is always a fixed register in the ITS (GIC_TRANSLATER), 
possibly SMMU translated. As long as the translation is consistent across
a single device (and I see no reason why or how the kernel would do it 
otherwise, given that a single CDX device generates the same StreamID
for all MSI writes), the GIC IOVA should be the same for all vectors of 
the device.

It is worth noting that this limitation is not going to be baked in the 
proposed MSI configuration interface, it will still take both the address
and data parts for each vector. It is just that this particular 
implementation will throw an error if you supply a different target
address across device MSI vectors. It does not preclude some future
device accepting different addresses per vector over the same 
interface.

Thanks,
Aleksandar


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Jason Gunthorpe]

On Wed, Oct 12, 2022 at 01:37:54PM +0000, Radovanovic, Aleksandar wrote:
> > On Wed, Oct 12, 2022 at 10:34:23AM +0000, Radovanovic, Aleksandar wrote:
> > 
> > 
> > > As for GITS_TRANSLATER, we can take up to 4 different IOVAs, which
> > > limits us to 4 CDX devices (should be sufficient for current HW
> > > use-cases). Also, it means that the address part must be the same for
> > > all vectors within a single CDX device. I'm assuming this is OK as it
> > > is going to be a single interrupt and IOMMU domain anyway.
> > 
> > This is not at all how MSI is supposed to work.
> 
> In the general case, no, they're not.

I don't mean that you can hack this to work - I mean that in MSI the
addr/data is supposed to come from the end point itself, not from some
kind of shared structure. This is important because the actual act of
generating the write has to be coherent with the DMA the device is
doing, as the MSI write must push any DMA data to visibility to meet
the "producer / consumer" model.

So it is really weird/wrong to have a HW design where the MSI
infrastructure is shared across many devices.

Jason

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Radovanovic, Aleksandar]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Jason Gunthorpe <jgg@ziepe.ca>
> Sent: 12 October 2022 15:38
> To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
> Cc: Gupta, Nipun <Nipun.Gupta@amd.com>; Marc Zyngier
> <maz@kernel.org>; Robin Murphy <robin.murphy@arm.com>;
> robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-
> ENG) <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org;
> Anand, Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>; git
> (AMD-Xilinx) <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> Caution: This message originated from an External Source. Use proper
> caution when opening attachments, clicking links, or responding.
> 
> 
> On Wed, Oct 12, 2022 at 01:37:54PM +0000, Radovanovic, Aleksandar wrote:
> > > On Wed, Oct 12, 2022 at 10:34:23AM +0000, Radovanovic, Aleksandar
> wrote:
> > >
> > >
> > > > As for GITS_TRANSLATER, we can take up to 4 different IOVAs, which
> > > > limits us to 4 CDX devices (should be sufficient for current HW
> > > > use-cases). Also, it means that the address part must be the same
> > > > for all vectors within a single CDX device. I'm assuming this is
> > > > OK as it is going to be a single interrupt and IOMMU domain anyway.
> > >
> > > This is not at all how MSI is supposed to work.
> >
> > In the general case, no, they're not.
> 
> I don't mean that you can hack this to work - I mean that in MSI the
> addr/data is supposed to come from the end point itself, not from some kind
> of shared structure. This is important because the actual act of generating
> the write has to be coherent with the DMA the device is doing, as the MSI
> write must push any DMA data to visibility to meet the "producer /
> consumer" model.
> 

I'm not sure I follow your argument, the limitation here is that the MSI
address value is shared between vectors of the same device (requester id
or endpoint, whichever way you prefer to call it), not between devices. 
This in no way implies that it is unordered with respect to device DMA - 
it is ordered and takes the same AXI path into the CPU cluster, so the
producer/consumer semantics are preserved.

Thanks,
Aleksandar

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Jason Gunthorpe]

On Wed, Oct 12, 2022 at 03:09:26PM +0000, Radovanovic, Aleksandar wrote:

> > On Wed, Oct 12, 2022 at 01:37:54PM +0000, Radovanovic, Aleksandar wrote:
> > > > On Wed, Oct 12, 2022 at 10:34:23AM +0000, Radovanovic, Aleksandar
> > wrote:
> > > >
> > > >
> > > > > As for GITS_TRANSLATER, we can take up to 4 different IOVAs, which
> > > > > limits us to 4 CDX devices (should be sufficient for current HW
> > > > > use-cases). Also, it means that the address part must be the same
> > > > > for all vectors within a single CDX device. I'm assuming this is
> > > > > OK as it is going to be a single interrupt and IOMMU domain anyway.
> > > >
> > > > This is not at all how MSI is supposed to work.
> > >
> > > In the general case, no, they're not.
> > 
> > I don't mean that you can hack this to work - I mean that in MSI the
> > addr/data is supposed to come from the end point itself, not from some kind
> > of shared structure. This is important because the actual act of generating
> > the write has to be coherent with the DMA the device is doing, as the MSI
> > write must push any DMA data to visibility to meet the "producer /
> > consumer" model.
> > 
> 
> I'm not sure I follow your argument, the limitation here is that the MSI
> address value is shared between vectors of the same device (requester id
> or endpoint, whichever way you prefer to call it), not between
> devices.

That isn't what you said, you said "we can take up to 4 different
IOVAs, which limits us to 4 CDX devices" - which sounds like HW being
shared across devices??

Jason

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Radovanovic, Aleksandar]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Jason Gunthorpe <jgg@ziepe.ca>
> Sent: 13 October 2022 13:43
> To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
> Cc: Gupta, Nipun <Nipun.Gupta@amd.com>; Marc Zyngier
> <maz@kernel.org>; Robin Murphy <robin.murphy@arm.com>;
> robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-
> ENG) <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org;
> Anand, Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>; git
> (AMD-Xilinx) <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> Caution: This message originated from an External Source. Use proper
> caution when opening attachments, clicking links, or responding.
> 
> 
> On Wed, Oct 12, 2022 at 03:09:26PM +0000, Radovanovic, Aleksandar wrote:
> 
> > > On Wed, Oct 12, 2022 at 01:37:54PM +0000, Radovanovic, Aleksandar
> wrote:
> > > > > On Wed, Oct 12, 2022 at 10:34:23AM +0000, Radovanovic,
> > > > > Aleksandar
> > > wrote:
> > > > >
> > > > >
> > > > > > As for GITS_TRANSLATER, we can take up to 4 different IOVAs,
> > > > > > which limits us to 4 CDX devices (should be sufficient for
> > > > > > current HW use-cases). Also, it means that the address part
> > > > > > must be the same for all vectors within a single CDX device.
> > > > > > I'm assuming this is OK as it is going to be a single interrupt and
> IOMMU domain anyway.
> > > > >
> > > > > This is not at all how MSI is supposed to work.
> > > >
> > > > In the general case, no, they're not.
> > >
> > > I don't mean that you can hack this to work - I mean that in MSI the
> > > addr/data is supposed to come from the end point itself, not from
> > > some kind of shared structure. This is important because the actual
> > > act of generating the write has to be coherent with the DMA the
> > > device is doing, as the MSI write must push any DMA data to
> > > visibility to meet the "producer / consumer" model.
> > >
> >
> > I'm not sure I follow your argument, the limitation here is that the
> > MSI address value is shared between vectors of the same device
> > (requester id or endpoint, whichever way you prefer to call it), not
> > between devices.
> 
> That isn't what you said, you said "we can take up to 4 different IOVAs, which
> limits us to 4 CDX devices" - which sounds like HW being shared across
> devices??

And that still does not imply lack of ordering or sharing of MSI target addresses between devices.

This is a highly programmable IP block, at the core of which is an interconnect interfacing to programmable logic (PL), a number of PCIe controllers (either endpoint or root-port), DMA engines, offload engines, the embedded processor subsystem (PSX), etc. DMA and interrupts can be routed across it in almost any (meaningful) direction. The datapath 'endpoints' request DMA and interrupts, but don't concern themselves with the mechanics of delivering that in the target domain. It is the responsibility of the egress bridges to the target domains to convert the interconnect interrupt transactions to whatever the interrupt delivery mechanism for that domain is. E.g. for PCIe controllers in endpoint mode, that would be through PCIe MSI-X tables internal to the controller (and managed by the PCIe host), for PSX that would be the PSX bridge (partially managed by the PSX OS, mediated through firmware, i.e. through CDX bus driver) and so on. It is the responsibility of the interconnect to maintain transaction ordering (including DMA vs. interrupts). It is the responsibility of the firmware to manage the bridges according to the implemented use-case, so everything works as expected. 

The CDX bus driver manages a single aspect of this and that is endpoints implemented in PL/engines, targeting the PSX.

So, yes, the hardware that translates interrupt transactions to GIC AXI writes is shared between endpoints, but what I said above still applies. And that doesn't necessarily make it weird/wrong, it's just more complex than you might think.

Anyway, I think we're straying off topic here, none of this is visible to the kernel anyway. The question that we still need to answer is, are you OK with the limitations I listed originally?

Thanks,
Aleksandar

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[gregkh]

On Fri, Oct 14, 2022 at 11:18:36AM +0000, Radovanovic, Aleksandar wrote:
> Anyway, I think we're straying off topic here, none of this is visible to the kernel anyway. The question that we still need to answer is, are you OK with the limitations I listed originally?

What original limitations?  This thread is crazy and complex and you
need to use your \n key more :)

thanks,

greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Radovanovic, Aleksandar]

[AMD Official Use Only - General]



> -----Original Message-----
> From: gregkh@linuxfoundation.org <gregkh@linuxfoundation.org>
> Sent: 14 October 2022 12:55
> To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
> Cc: Jason Gunthorpe <jgg@ziepe.ca>; Gupta, Nipun
> <Nipun.Gupta@amd.com>; Marc Zyngier <maz@kernel.org>; Robin Murphy
> <robin.murphy@arm.com>; robh+dt@kernel.org;
> krzysztof.kozlowski+dt@linaro.org; rafael@kernel.org;
> eric.auger@redhat.com; alex.williamson@redhat.com; cohuck@redhat.com;
> Gupta, Puneet (DCG-ENG) <puneet.gupta@amd.com>;
> song.bao.hua@hisilicon.com; mchehab+huawei@kernel.org;
> f.fainelli@gmail.com; jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org;
> Anand, Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>; git
> (AMD-Xilinx) <git@amd.com>
> Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> domain as parent
> 
> Caution: This message originated from an External Source. Use proper
> caution when opening attachments, clicking links, or responding.
> 
> 
> On Fri, Oct 14, 2022 at 11:18:36AM +0000, Radovanovic, Aleksandar wrote:
> > Anyway, I think we're straying off topic here, none of this is visible to the
> kernel anyway. The question that we still need to answer is, are you OK with
> the limitations I listed originally?
> 
> What original limitations?  

Limitations with regards to MSI message configuration of a CDX device:

1) MSI write value is at most 16 useable bits

2) MSI address value must be the same across all vectors of a single CDX device
This would be the (potentially IOMMU translated) I/O address of
GITS_TRANSLATER. As long as that IOMMU translation is consistent across a 
single device, I think we should be OK.

> This thread is crazy and complex and you need to
> use your \n key more :)

I do try, but the corporate mail client that we're stuck with seems to just merge
everything back together. Sigh.

> thanks,
> 
> greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[gregkh]

On Fri, Oct 14, 2022 at 12:13:45PM +0000, Radovanovic, Aleksandar wrote:
> [AMD Official Use Only - General]
> 
> 
> 
> > -----Original Message-----
> > From: gregkh@linuxfoundation.org <gregkh@linuxfoundation.org>
> > Sent: 14 October 2022 12:55
> > To: Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>
> > Cc: Jason Gunthorpe <jgg@ziepe.ca>; Gupta, Nipun
> > <Nipun.Gupta@amd.com>; Marc Zyngier <maz@kernel.org>; Robin Murphy
> > <robin.murphy@arm.com>; robh+dt@kernel.org;
> > krzysztof.kozlowski+dt@linaro.org; rafael@kernel.org;
> > eric.auger@redhat.com; alex.williamson@redhat.com; cohuck@redhat.com;
> > Gupta, Puneet (DCG-ENG) <puneet.gupta@amd.com>;
> > song.bao.hua@hisilicon.com; mchehab+huawei@kernel.org;
> > f.fainelli@gmail.com; jeffrey.l.hugo@gmail.com; saravanak@google.com;
> > Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> > will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> > ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> > kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> > devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org;
> > Anand, Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> > <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>; git
> > (AMD-Xilinx) <git@amd.com>
> > Subject: Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its
> > domain as parent
> > 
> > Caution: This message originated from an External Source. Use proper
> > caution when opening attachments, clicking links, or responding.
> > 
> > 
> > On Fri, Oct 14, 2022 at 11:18:36AM +0000, Radovanovic, Aleksandar wrote:
> > > Anyway, I think we're straying off topic here, none of this is visible to the
> > kernel anyway. The question that we still need to answer is, are you OK with
> > the limitations I listed originally?
> > 
> > What original limitations?  
> 
> Limitations with regards to MSI message configuration of a CDX device:
> 
> 1) MSI write value is at most 16 useable bits
> 
> 2) MSI address value must be the same across all vectors of a single CDX device
> This would be the (potentially IOMMU translated) I/O address of
> GITS_TRANSLATER. As long as that IOMMU translation is consistent across a 
> single device, I think we should be OK.

It's been a while since I read the PCI spec, but this feels like it does
not follow what MSI is supposed to allow.  Is the "CDX" spec anywhere
that mentions any of this as to what is supposed to be allowed and
supported?

And what is a "single device" here in how the kernel knows about it?  Is
it a PCI device, or some other new structure that is handed to the
kernel from the BIOS?

thanks,

greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 4/7] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Jason Gunthorpe]

On Fri, Oct 14, 2022 at 11:18:36AM +0000, Radovanovic, Aleksandar wrote:

> And that still does not imply lack of ordering or sharing of MSI
> target addresses between devices.

Either the end point generates the MSI, and maybe the bridge mangles
it, or it raises a lot of suspicion that this is not right. If the end
point generates the MSI then it raises the question why do we need to
tolerate these limits?

> This is a highly programmable IP block, at the core of which is an
> interconnect interfacing to programmable logic (PL), a number of
> PCIe controllers (either endpoint or root-port), DMA engines,
> offload engines, the embedded processor subsystem (PSX), etc. DMA
> and interrupts can be routed across it in almost any (meaningful)
> direction. The datapath 'endpoints' request DMA and interrupts, but
> don't concern themselves with the mechanics of delivering that in
> the target domain. It is the responsibility of the egress bridges to
> the target domains to convert the interconnect interrupt
> transactions to whatever the interrupt delivery mechanism for that
> domain is. E.g. for PCIe controllers in endpoint mode, that would be
> through PCIe MSI-X tables internal to the controller (and managed by
> the PCIe host), for PSX that would be the PSX bridge (partially
> managed by the PSX OS, mediated through firmware, i.e. through CDX
> bus driver) and so on. It is the responsibility of the interconnect
> to maintain transaction ordering (including DMA vs. interrupts). It
> is the responsibility of the firmware to manage the bridges
> according to the implemented use-case, so everything works as
> expected.

Again, this all just seems wrongly designed. MSI should not be part
of an interconnect bridge. We did that already 20 years ago, it was
called IOAPICs on x86 and I think everyone is happy to see it gone.

If you want to build IOAPICs again, I guess you can, but that is a
slightly different SW setup than the MSI you are trying to use here,
and even that didn't have the same limitations you are proposing.

> So, yes, the hardware that translates interrupt transactions to GIC
> AXI writes is shared between endpoints, but what I said above still
> applies. And that doesn't necessarily make it weird/wrong, it's just
> more complex than you might think.

If it doesn't fit the architecture, then I think it must be considered
wrong. Mis-using platform architected components like MSI in HW is
problematic.

You should design the HW properly so you don't have these
problems. Involving FW in the MSI setup is also a bad idea, POWER did
this and it made a big mess of their arch code :(

Jason

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v3 5/7] bus/cdx: add bus and device attributes

[Nipun Gupta]

This change adds te support for rescanning and reset
of the CDX buses, as well as option to optionally reset
any devices on the bus. It also enables sysfs entry for
vendor and device id.

Sysfs entries are provided in CDX controller:
- rescan of the CDX controller.
- reset all the devices present on CDX buses.

Sysfs entry is provided in each of the platform device
detected by the CDX controller
- vendor id
- device id
- modalias
- reset of the device.

Signed-off-by: Puneet Gupta <puneet.gupta@amd.com>
Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 Documentation/ABI/testing/sysfs-bus-cdx |  54 +++++++++
 drivers/bus/cdx/cdx.c                   | 148 ++++++++++++++++++++++++
 drivers/bus/cdx/mcdi_stubs.c            |   6 +
 drivers/bus/cdx/mcdi_stubs.h            |  11 ++
 include/linux/cdx/cdx_bus.h             |   8 ++
 5 files changed, 227 insertions(+)
 create mode 100644 Documentation/ABI/testing/sysfs-bus-cdx

diff --git a/Documentation/ABI/testing/sysfs-bus-cdx b/Documentation/ABI/testing/sysfs-bus-cdx
new file mode 100644
index 000000000000..5f84f0a93a66
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-bus-cdx
@@ -0,0 +1,54 @@
+What:		/sys/bus/cdx/rescan
+Date:		August 2022
+Contact:	puneet.gupta@amd.com
+Description:
+		Writing 1 to this would cause rescan of the bus
+		and devices on the CDX bus. Any new devices would
+		be scanned and added to the list of linux devices
+		and any devices removed are also deleted from linux.
+
+                For example::
+
+		  # echo 1 > /sys/bus/cdx/rescan
+
+What:		/sys/bus/cdx/reset_all
+Date:		August 2022
+Contact:	puneet.gupta@amd.com
+Description:
+		Writing 1 to this would reset all the devices present
+		on the CDX bus
+
+                For example::
+
+		  # echo 1 > /sys/bus/cdx/reset_all
+
+What:		/sys/bus/cdx/devices/.../vendor
+Date:		August 2022
+Contact:	nipun.gupta@amd.com
+Description:
+		Vendor ID for this CDX device
+
+What:		/sys/bus/cdx/devices/.../device
+Date:		August 2022
+Contact:	nipun.gupta@amd.com
+Description:
+		Device ID for this CDX device
+
+What:		/sys/bus/cdx/devices/.../reset
+What:		/sys/bus/cdx/devices/.../reset
+Date:		August 2022
+Contact:	nipun.gupta@amd.com
+Description:
+		Writing 1 to this would reset the CDX device
+
+                For example::
+
+		  # echo 1 > /sys/bus/cdx/.../reset
+
+What:		/sys/bus/cdx/devices/.../modalias
+Date:		August 2022
+Contact:	nipun.gupta@amd.com
+Description:
+		A CDX device has modalias:
+
+			- v`vendor`d`device`
diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
index 02ececce1c84..9b7a69de6a8f 100644
--- a/drivers/bus/cdx/cdx.c
+++ b/drivers/bus/cdx/cdx.c
@@ -32,6 +32,23 @@ struct platform_device *cdx_controller_pdev;
 
 static int cdx_bus_device_discovery(struct platform_device *pdev);
 
+static int reset_cdx_device(struct device *dev, void * __always_unused data)
+{
+	struct platform_device *cdx_bus_pdev = to_platform_device(dev->parent);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	struct cdx_mcdi_t *cdx_mcdi = platform_get_drvdata(cdx_bus_pdev);
+
+	/* MCDI FW: reset particular device represented by bus:func */
+	cdx_mcdi_func_reset(cdx_mcdi, cdx_dev->bus_id, cdx_dev->func_id);
+
+	return 0;
+}
+
+int cdx_dev_reset(struct device *dev)
+{
+	return reset_cdx_device(dev, NULL);
+}
+
 static int cdx_unregister_device(struct device *dev,
 				 void * __always_unused data)
 {
@@ -139,10 +156,141 @@ static int cdx_dma_configure(struct device *dev)
 	return 0;
 }
 
+static ssize_t vendor_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return sprintf(buf, "0x%x\n", cdx_dev->vendor);
+}
+static DEVICE_ATTR_RO(vendor);
+
+static ssize_t device_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return sprintf(buf, "0x%x\n", cdx_dev->device);
+}
+static DEVICE_ATTR_RO(device);
+
+static ssize_t reset_store(struct device *dev, struct device_attribute *attr,
+			   const char *buf, size_t count)
+{
+	int ret = 0;
+	bool reset = count > 0 && *buf != '0';
+
+	if (!reset)
+		return count;
+
+	ret = reset_cdx_device(dev, NULL);
+	if (ret)
+		return ret;
+
+	return count;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return sprintf(buf, "cdx:v%08Xd%d\n", cdx_dev->vendor,
+			cdx_dev->device);
+}
+static DEVICE_ATTR_RO(modalias);
+
+static ssize_t driver_override_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	int ret;
+
+	if (WARN_ON(dev->bus != &cdx_bus_type))
+		return -EINVAL;
+
+	ret = driver_set_override(dev, &cdx_dev->driver_override, buf, count);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static ssize_t driver_override_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%s\n", cdx_dev->driver_override);
+}
+static DEVICE_ATTR_RW(driver_override);
+
+static struct attribute *cdx_dev_attrs[] = {
+	&dev_attr_reset.attr,
+	&dev_attr_vendor.attr,
+	&dev_attr_device.attr,
+	&dev_attr_modalias.attr,
+	&dev_attr_driver_override.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(cdx_dev);
+
+static ssize_t rescan_store(struct bus_type *bus,
+			    const char *buf, size_t count)
+{
+	bool rescan = count > 0 && *buf != '0';
+	int ret = 0;
+
+	if (!rescan)
+		return count;
+
+	/* Unregister all the devices on the bus */
+	cdx_unregister_devices(&cdx_bus_type);
+
+	/* Rescan all the devices */
+	ret = cdx_bus_device_discovery(cdx_controller_pdev);
+	if (ret)
+		return ret;
+
+	return count;
+}
+static BUS_ATTR_WO(rescan);
+
+static ssize_t reset_all_store(struct bus_type *bus,
+			       const char *buf, size_t count)
+{
+	bool reset = count > 0 && *buf != '0';
+	int ret = 0;
+
+	if (!reset)
+		return count;
+
+	/* Reset all the devices attached to cdx bus */
+	ret = bus_for_each_dev(bus, NULL, NULL, reset_cdx_device);
+	if (ret) {
+		pr_err("error in CDX bus reset\n");
+		return 0;
+	}
+
+	return count;
+}
+static BUS_ATTR_WO(reset_all);
+
+static struct attribute *cdx_bus_attrs[] = {
+	&bus_attr_rescan.attr,
+	&bus_attr_reset_all.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(cdx_bus);
+
 struct bus_type cdx_bus_type = {
 	.name = "cdx",
 	.match = cdx_bus_match,
 	.dma_configure  = cdx_dma_configure,
+	.dev_groups = cdx_dev_groups,
+	.bus_groups = cdx_bus_groups,
 };
 EXPORT_SYMBOL_GPL(cdx_bus_type);
 
diff --git a/drivers/bus/cdx/mcdi_stubs.c b/drivers/bus/cdx/mcdi_stubs.c
index 2c8db1f5a057..2bc042d2b061 100644
--- a/drivers/bus/cdx/mcdi_stubs.c
+++ b/drivers/bus/cdx/mcdi_stubs.c
@@ -53,3 +53,9 @@ int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
 
 	return 0;
 }
+
+int cdx_mcdi_func_reset(struct cdx_mcdi_t *cdx_mcdi,
+			uint8_t bus_id, uint8_t func_id)
+{
+	return 0;
+}
diff --git a/drivers/bus/cdx/mcdi_stubs.h b/drivers/bus/cdx/mcdi_stubs.h
index 7b6f79d48ce9..535218bcdce0 100644
--- a/drivers/bus/cdx/mcdi_stubs.h
+++ b/drivers/bus/cdx/mcdi_stubs.h
@@ -73,4 +73,15 @@ int cdx_mcdi_get_func_config(struct cdx_mcdi_t *cdx_mcdi,
 			     uint8_t bus_id, uint8_t func_id,
 			     struct cdx_dev_params_t *dev_params);
 
+/**
+ * cdx_mcdi_func_reset - Reset cdx device represented by bus_id:func_id
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @func_id: Function number.
+ *
+ * Return 0 on success, <0 on failure
+ */
+int cdx_mcdi_func_reset(struct cdx_mcdi_t *cdx_mcdi,
+			uint8_t bus_id, uint8_t func_id);
+
 #endif /* _MCDI_STUBS_H_ */
diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
index bf86024783d2..8306b4d3fd82 100644
--- a/include/linux/cdx/cdx_bus.h
+++ b/include/linux/cdx/cdx_bus.h
@@ -109,4 +109,12 @@ int cdx_msi_domain_alloc_irqs(struct device *dev, unsigned int irq_count);
  */
 void cdx_msi_domain_free_irqs(struct device *dev);
 
+/**
+ * cdx_dev_reset - Reset CDX device
+ * @dev: device pointer
+ *
+ * Return 0 for success, -errno on failure
+ */
+int cdx_dev_reset(struct device *dev);
+
 #endif /* _CDX_BUS_H_ */
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v3 6/7] vfio/cdx: add support for CDX bus

[Nipun Gupta]

vfio-cdx driver enables IOCTLs for user space to query
MMIO regions for CDX devices and mmap them. This change
also adds support for reset of CDX devices.

This change adds the VFIO CDX driver and enables the following
ioctls for CDX devices:
 - VFIO_DEVICE_GET_INFO:
 - VFIO_DEVICE_GET_REGION_INFO
 - VFIO_DEVICE_RESET

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 MAINTAINERS                         |   1 +
 drivers/vfio/Makefile               |   1 +
 drivers/vfio/cdx/Kconfig            |  10 +
 drivers/vfio/cdx/Makefile           |   4 +
 drivers/vfio/cdx/vfio_cdx.c         | 284 ++++++++++++++++++++++++++++
 drivers/vfio/cdx/vfio_cdx_private.h |  32 ++++
 6 files changed, 332 insertions(+)
 create mode 100644 drivers/vfio/cdx/Kconfig
 create mode 100644 drivers/vfio/cdx/Makefile
 create mode 100644 drivers/vfio/cdx/vfio_cdx.c
 create mode 100644 drivers/vfio/cdx/vfio_cdx_private.h

diff --git a/MAINTAINERS b/MAINTAINERS
index 61af11c9fe06..e66a93ca2218 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -940,6 +940,7 @@ M:	Nikhil Agarwal <nikhil.agarwal@amd.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
 F:	drivers/bus/cdx/*
+F:	drivers/vfio/cdx/*
 F:	include/linux/cdx/*
 
 AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER - SEV SUPPORT
diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
index 1a32357592e3..7f4f5c891c28 100644
--- a/drivers/vfio/Makefile
+++ b/drivers/vfio/Makefile
@@ -12,3 +12,4 @@ obj-$(CONFIG_VFIO_PCI) += pci/
 obj-$(CONFIG_VFIO_PLATFORM) += platform/
 obj-$(CONFIG_VFIO_MDEV) += mdev/
 obj-$(CONFIG_VFIO_FSL_MC) += fsl-mc/
+obj-$(CONFIG_CDX_BUS) += cdx/
diff --git a/drivers/vfio/cdx/Kconfig b/drivers/vfio/cdx/Kconfig
new file mode 100644
index 000000000000..728d7294a2c7
--- /dev/null
+++ b/drivers/vfio/cdx/Kconfig
@@ -0,0 +1,10 @@
+config VFIO_CDX
+	tristate "VFIO support for CDX bus devices"
+	depends on CDX_BUS
+	select EVENTFD
+	help
+	  Driver to enable VFIO support for the devices on CDX bus.
+	  This is required to make use of CDX devices present in
+	  the system using the VFIO framework.
+
+	  If you don't know what to do here, say N.
diff --git a/drivers/vfio/cdx/Makefile b/drivers/vfio/cdx/Makefile
new file mode 100644
index 000000000000..7a4183d76802
--- /dev/null
+++ b/drivers/vfio/cdx/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+
+#vfio-fsl-mc-y := vfio_cdx.o vfio_cdx_intr.o
+obj-y += vfio_cdx.o vfio_cdx_intr.o
diff --git a/drivers/vfio/cdx/vfio_cdx.c b/drivers/vfio/cdx/vfio_cdx.c
new file mode 100644
index 000000000000..2e5bd494057a
--- /dev/null
+++ b/drivers/vfio/cdx/vfio_cdx.c
@@ -0,0 +1,284 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/device.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/vfio.h>
+#include <linux/cdx/cdx_bus.h>
+#include <linux/delay.h>
+#include <linux/io-64-nonatomic-hi-lo.h>
+
+#include "vfio_cdx_private.h"
+
+static struct cdx_driver vfio_cdx_driver;
+
+static int vfio_cdx_open_device(struct vfio_device *core_vdev)
+{
+	struct vfio_cdx_device *vdev =
+		container_of(core_vdev, struct vfio_cdx_device, vdev);
+	struct cdx_device *cdx_dev = vdev->cdx_dev;
+	int count = cdx_dev->res_count;
+	int i;
+
+	vdev->regions = kcalloc(count, sizeof(struct vfio_cdx_region),
+				GFP_KERNEL);
+	if (!vdev->regions)
+		return -ENOMEM;
+
+	for (i = 0; i < count; i++) {
+		struct resource *res = &cdx_dev->res[i];
+
+		vdev->regions[i].addr = res->start;
+		vdev->regions[i].size = resource_size(res);
+		vdev->regions[i].type = res->flags;
+		/*
+		 * Only regions addressed with PAGE granularity may be
+		 * MMAPed securely.
+		 */
+		if (!(vdev->regions[i].addr & ~PAGE_MASK) &&
+				!(vdev->regions[i].size & ~PAGE_MASK))
+			vdev->regions[i].flags |=
+					VFIO_REGION_INFO_FLAG_MMAP;
+		vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_READ;
+		if (!(cdx_dev->res[i].flags & IORESOURCE_READONLY))
+			vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_WRITE;
+	}
+
+	return 0;
+}
+
+static void vfio_cdx_regions_cleanup(struct vfio_cdx_device *vdev)
+{
+	kfree(vdev->regions);
+}
+
+static int vfio_cdx_reset_device(struct vfio_cdx_device *vdev)
+{
+	struct vfio_device *core_vdev = &vdev->vdev;
+
+	return cdx_dev_reset(core_vdev->dev);
+}
+
+static void vfio_cdx_close_device(struct vfio_device *core_vdev)
+{
+	struct vfio_cdx_device *vdev =
+		container_of(core_vdev, struct vfio_cdx_device, vdev);
+	int ret;
+
+	vfio_cdx_regions_cleanup(vdev);
+
+	/* reset the device before cleaning up the interrupts */
+	ret = vfio_cdx_reset_device(vdev);
+	if (WARN_ON(ret))
+		dev_warn(core_vdev->dev,
+			 "VFIO_CDX: reset device has failed (%d)\n", ret);
+}
+
+static long vfio_cdx_ioctl(struct vfio_device *core_vdev,
+			   unsigned int cmd, unsigned long arg)
+{
+	unsigned long minsz;
+	struct vfio_cdx_device *vdev =
+		container_of(core_vdev, struct vfio_cdx_device, vdev);
+	struct cdx_device *cdx_dev = vdev->cdx_dev;
+
+	switch (cmd) {
+	case VFIO_DEVICE_GET_INFO:
+	{
+		struct vfio_device_info info;
+
+		minsz = offsetofend(struct vfio_device_info, num_irqs);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		info.flags = VFIO_DEVICE_FLAGS_RESET;
+
+		info.num_regions = cdx_dev->res_count;
+		info.num_irqs = 1;
+
+		return copy_to_user((void __user *)arg, &info, minsz) ?
+			-EFAULT : 0;
+	}
+	case VFIO_DEVICE_GET_REGION_INFO:
+	{
+		struct vfio_region_info info;
+
+		minsz = offsetofend(struct vfio_region_info, offset);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		if (info.index >= cdx_dev->res_count)
+			return -EINVAL;
+
+		/* map offset to the physical address  */
+		info.offset = VFIO_CDX_INDEX_TO_OFFSET(info.index);
+		info.size = vdev->regions[info.index].size;
+		info.flags = vdev->regions[info.index].flags;
+
+		if (copy_to_user((void __user *)arg, &info, minsz))
+			return -EFAULT;
+		return 0;
+	}
+	case VFIO_DEVICE_RESET:
+	{
+		return vfio_cdx_reset_device(vdev);
+	}
+	default:
+		return -ENOTTY;
+	}
+}
+
+static int vfio_cdx_mmap_mmio(struct vfio_cdx_region region,
+			      struct vm_area_struct *vma)
+{
+	u64 size = vma->vm_end - vma->vm_start;
+	u64 pgoff, base;
+	u8 region_cacheable;
+
+	pgoff = vma->vm_pgoff &
+		((1U << (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
+	base = pgoff << PAGE_SHIFT;
+
+	if (region.size < PAGE_SIZE || base + size > region.size)
+		return -EINVAL;
+
+	region_cacheable = (region.type & IORESOURCE_MEM);
+	if (!region_cacheable)
+		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+
+	vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
+
+	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+			       size, vma->vm_page_prot);
+}
+
+static int vfio_cdx_mmap(struct vfio_device *core_vdev,
+			    struct vm_area_struct *vma)
+{
+	struct vfio_cdx_device *vdev =
+		container_of(core_vdev, struct vfio_cdx_device, vdev);
+	struct cdx_device *cdx_dev = vdev->cdx_dev;
+	unsigned int index;
+
+	index = vma->vm_pgoff >> (VFIO_CDX_OFFSET_SHIFT - PAGE_SHIFT);
+
+	if (vma->vm_end < vma->vm_start)
+		return -EINVAL;
+	if (vma->vm_start & ~PAGE_MASK)
+		return -EINVAL;
+	if (vma->vm_end & ~PAGE_MASK)
+		return -EINVAL;
+	if (!(vma->vm_flags & VM_SHARED))
+		return -EINVAL;
+	if (index >= cdx_dev->res_count)
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP))
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ)
+			&& (vma->vm_flags & VM_READ))
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)
+			&& (vma->vm_flags & VM_WRITE))
+		return -EINVAL;
+
+	vma->vm_private_data = cdx_dev;
+
+	return vfio_cdx_mmap_mmio(vdev->regions[index], vma);
+}
+
+static const struct vfio_device_ops vfio_cdx_ops = {
+	.name		= "vfio-cdx",
+	.open_device	= vfio_cdx_open_device,
+	.close_device	= vfio_cdx_close_device,
+	.ioctl		= vfio_cdx_ioctl,
+	.mmap		= vfio_cdx_mmap,
+};
+
+static int vfio_cdx_probe(struct cdx_device *cdx_dev)
+{
+	struct vfio_cdx_device *vdev;
+	struct device *dev = &cdx_dev->dev;
+	int ret;
+
+	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
+	if (!vdev)
+		return -ENOMEM;
+
+	vfio_init_group_dev(&vdev->vdev, dev, &vfio_cdx_ops);
+	vdev->cdx_dev = cdx_dev;
+	vdev->dev = dev;
+
+	ret = vfio_assign_device_set(&vdev->vdev, dev);
+	if (ret) {
+		dev_err(dev, "VFIO_CDX: vfio_assign_device_set failed\n");
+		goto out_uninit;
+	}
+
+	ret = vfio_register_group_dev(&vdev->vdev);
+	if (ret) {
+		dev_err(dev, "VFIO_CDX: Failed to add to vfio group\n");
+		goto out_uninit;
+	}
+
+	dev_set_drvdata(dev, vdev);
+	return 0;
+
+out_uninit:
+	vfio_uninit_group_dev(&vdev->vdev);
+	kfree(vdev);
+	return ret;
+}
+
+static int vfio_cdx_remove(struct cdx_device *cdx_dev)
+{
+	struct device *dev = &cdx_dev->dev;
+	struct vfio_cdx_device *vdev = dev_get_drvdata(dev);
+
+	vfio_unregister_group_dev(&vdev->vdev);
+
+	vfio_uninit_group_dev(&vdev->vdev);
+	kfree(vdev);
+	return 0;
+}
+
+static struct cdx_driver vfio_cdx_driver = {
+	.probe		= vfio_cdx_probe,
+	.remove		= vfio_cdx_remove,
+	.driver	= {
+		.name	= "vfio-cdx",
+		.owner	= THIS_MODULE,
+	},
+	.driver_managed_dma = true,
+};
+
+static int __init vfio_cdx_driver_init(void)
+{
+	return cdx_driver_register(&vfio_cdx_driver);
+}
+
+static void __exit vfio_cdx_driver_exit(void)
+{
+	cdx_driver_unregister(&vfio_cdx_driver);
+}
+
+module_init(vfio_cdx_driver_init);
+module_exit(vfio_cdx_driver_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("VFIO for CDX devices - User Level meta-driver");
diff --git a/drivers/vfio/cdx/vfio_cdx_private.h b/drivers/vfio/cdx/vfio_cdx_private.h
new file mode 100644
index 000000000000..d87b55663462
--- /dev/null
+++ b/drivers/vfio/cdx/vfio_cdx_private.h
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#ifndef VFIO_CDX_PRIVATE_H
+#define VFIO_CDX_PRIVATE_H
+
+#define VFIO_CDX_OFFSET_SHIFT    40
+#define VFIO_CDX_OFFSET_MASK (((u64)(1) << VFIO_CDX_OFFSET_SHIFT) - 1)
+
+#define VFIO_CDX_OFFSET_TO_INDEX(off) ((off) >> VFIO_CDX_OFFSET_SHIFT)
+
+#define VFIO_CDX_INDEX_TO_OFFSET(index)	\
+	((u64)(index) << VFIO_CDX_OFFSET_SHIFT)
+
+struct vfio_cdx_region {
+	u32			flags;
+	u32			type;
+	u64			addr;
+	resource_size_t		size;
+	void __iomem		*ioaddr;
+};
+
+struct vfio_cdx_device {
+	struct vfio_device		vdev;
+	struct cdx_device		*cdx_dev;
+	struct device			*dev;
+	struct vfio_cdx_region		*regions;
+};
+
+#endif /* VFIO_CDX_PRIVATE_H */
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v3 6/7] vfio/cdx: add support for CDX bus

[Jason Gunthorpe]

On Tue, Sep 06, 2022 at 07:18:00PM +0530, Nipun Gupta wrote:
> vfio-cdx driver enables IOCTLs for user space to query
> MMIO regions for CDX devices and mmap them. This change
> also adds support for reset of CDX devices.
> 
> This change adds the VFIO CDX driver and enables the following
> ioctls for CDX devices:
>  - VFIO_DEVICE_GET_INFO:
>  - VFIO_DEVICE_GET_REGION_INFO
>  - VFIO_DEVICE_RESET

For non-RFC posting please post VFIO separately after getting the base
bus merged.

I think you will need to get some work done to consolidate some of
this duplicated code that was cut & pasted here..

Jason

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v3 6/7] vfio/cdx: add support for CDX bus

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Jason Gunthorpe <jgg@nvidia.com>
> Sent: Tuesday, September 6, 2022 10:51 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com; Michael.Srba@seznam.cz;
> mani@kernel.org; yishaih@nvidia.com; robin.murphy@arm.com;
> will@kernel.org; joro@8bytes.org; masahiroy@kernel.org;
> ndesaulniers@google.com; linux-arm-kernel@lists.infradead.org; linux-
> kbuild@vger.kernel.org; linux-kernel@vger.kernel.org;
> devicetree@vger.kernel.org; kvm@vger.kernel.org; okaya@kernel.org; Anand,
> Harpreet <harpreet.anand@amd.com>; Agarwal, Nikhil
> <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v3 6/7] vfio/cdx: add support for CDX bus
> 
> [CAUTION: External Email]
> 
> On Tue, Sep 06, 2022 at 07:18:00PM +0530, Nipun Gupta wrote:
> > vfio-cdx driver enables IOCTLs for user space to query
> > MMIO regions for CDX devices and mmap them. This change
> > also adds support for reset of CDX devices.
> >
> > This change adds the VFIO CDX driver and enables the following
> > ioctls for CDX devices:
> >  - VFIO_DEVICE_GET_INFO:
> >  - VFIO_DEVICE_GET_REGION_INFO
> >  - VFIO_DEVICE_RESET
> 
> For non-RFC posting please post VFIO separately after getting the base
> bus merged.

Agree, with non-RFC changes we will get bus merged first and then
have the vfio patches as a separate set.

Thanks,
Nipun

> 
> I think you will need to get some work done to consolidate some of
> this duplicated code that was cut & pasted here..
> 
> Jason

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v3 7/7] vfio/cdx: add interrupt support

[Nipun Gupta]

This patch allows to set an eventfd for cdx device interrupts
and also to trigger the interrupt eventfd from userspace.
All CDX device interrupts are MSIs. The MSIs are allocated from
the CDX-MSI domain.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 drivers/vfio/cdx/vfio_cdx.c         |  53 +++++++
 drivers/vfio/cdx/vfio_cdx_intr.c    | 212 ++++++++++++++++++++++++++++
 drivers/vfio/cdx/vfio_cdx_private.h |  18 +++
 3 files changed, 283 insertions(+)
 create mode 100644 drivers/vfio/cdx/vfio_cdx_intr.c

diff --git a/drivers/vfio/cdx/vfio_cdx.c b/drivers/vfio/cdx/vfio_cdx.c
index 2e5bd494057a..4591b8057b2f 100644
--- a/drivers/vfio/cdx/vfio_cdx.c
+++ b/drivers/vfio/cdx/vfio_cdx.c
@@ -77,6 +77,8 @@ static void vfio_cdx_close_device(struct vfio_device *core_vdev)
 	if (WARN_ON(ret))
 		dev_warn(core_vdev->dev,
 			 "VFIO_CDX: reset device has failed (%d)\n", ret);
+
+	vfio_cdx_irqs_cleanup(vdev);
 }
 
 static long vfio_cdx_ioctl(struct vfio_device *core_vdev,
@@ -132,6 +134,57 @@ static long vfio_cdx_ioctl(struct vfio_device *core_vdev,
 			return -EFAULT;
 		return 0;
 	}
+	case VFIO_DEVICE_GET_IRQ_INFO:
+	{
+		struct vfio_irq_info info;
+
+		minsz = offsetofend(struct vfio_irq_info, count);
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		if (info.index >= 1)
+			return -EINVAL;
+
+		info.flags = VFIO_IRQ_INFO_EVENTFD;
+		info.count = cdx_dev->num_msi;
+
+		if (copy_to_user((void __user *)arg, &info, minsz))
+			return -EFAULT;
+		return 0;
+	}
+	case VFIO_DEVICE_SET_IRQS:
+	{
+		struct vfio_irq_set hdr;
+		u8 *data = NULL;
+		int ret = 0;
+		size_t data_size = 0;
+
+		minsz = offsetofend(struct vfio_irq_set, count);
+
+		if (copy_from_user(&hdr, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr,
+				cdx_dev->num_msi, 1, &data_size);
+		if (ret)
+			return ret;
+
+		if (data_size) {
+			data = memdup_user((void __user *)(arg + minsz),
+				   data_size);
+			if (IS_ERR(data))
+				return PTR_ERR(data);
+		}
+
+		ret = vfio_cdx_set_irqs_ioctl(vdev, hdr.flags,
+				hdr.index, hdr.start, hdr.count, data);
+		kfree(data);
+
+		return ret;
+	}
 	case VFIO_DEVICE_RESET:
 	{
 		return vfio_cdx_reset_device(vdev);
diff --git a/drivers/vfio/cdx/vfio_cdx_intr.c b/drivers/vfio/cdx/vfio_cdx_intr.c
new file mode 100644
index 000000000000..20fe87bce464
--- /dev/null
+++ b/drivers/vfio/cdx/vfio_cdx_intr.c
@@ -0,0 +1,212 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/vfio.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/eventfd.h>
+#include <linux/msi.h>
+#include <linux/interrupt.h>
+
+#include "linux/cdx/cdx_bus.h"
+#include "vfio_cdx_private.h"
+
+static irqreturn_t vfio_cdx_msihandler(int irq_no, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static int vfio_cdx_msi_enable(struct vfio_cdx_device *vdev, int nvec)
+{
+	struct device *dev = vdev->dev;
+	int msi_idx, ret;
+
+	vdev->cdx_irqs = kcalloc(nvec, sizeof(struct vfio_cdx_irq), GFP_KERNEL);
+	if (!vdev->cdx_irqs)
+		return -ENOMEM;
+
+	/* Allocate cdx MSIs */
+	ret = cdx_msi_domain_alloc_irqs(dev, nvec);
+	if (ret < 0) {
+		kfree(vdev->cdx_irqs);
+		return ret;
+	}
+
+	for (msi_idx = 0; msi_idx < nvec; msi_idx++)
+		vdev->cdx_irqs[msi_idx].irq_no = msi_get_virq(dev, msi_idx);
+
+	vdev->irq_count = nvec;
+	vdev->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_cdx_msi_set_vector_signal(struct vfio_cdx_device *vdev,
+					  int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq_no, ret;
+
+	if (vector < 0 || vector >= vdev->irq_count)
+		return -EINVAL;
+
+	irq_no = vdev->cdx_irqs[vector].irq_no;
+
+	if (vdev->cdx_irqs[vector].trigger) {
+		free_irq(irq_no, vdev->cdx_irqs[vector].trigger);
+		kfree(vdev->cdx_irqs[vector].name);
+		eventfd_ctx_put(vdev->cdx_irqs[vector].trigger);
+		vdev->cdx_irqs[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	vdev->cdx_irqs[vector].name = kasprintf(GFP_KERNEL, "vfio-msi[%d](%s)",
+						vector, dev_name(vdev->dev));
+	if (!vdev->cdx_irqs[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(vdev->cdx_irqs[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq_no, vfio_cdx_msihandler, 0,
+			  vdev->cdx_irqs[vector].name, trigger);
+	if (ret) {
+		kfree(vdev->cdx_irqs[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	vdev->cdx_irqs[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_cdx_msi_set_block(struct vfio_cdx_device *vdev,
+				  unsigned int start, unsigned int count,
+				  int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= vdev->irq_count || start + count > vdev->irq_count)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_cdx_msi_set_vector_signal(vdev, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_cdx_msi_set_vector_signal(vdev, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_cdx_msi_disable(struct vfio_cdx_device *vdev)
+{
+	struct device *dev = vdev->dev;
+
+	vfio_cdx_msi_set_block(vdev, 0, vdev->irq_count, NULL);
+
+	if (vdev->config_msi == 1)
+		cdx_msi_domain_free_irqs(dev);
+
+	vdev->config_msi = 0;
+	vdev->irq_count = 0;
+
+	kfree(vdev->cdx_irqs);
+}
+
+static int vfio_cdx_set_msi_trigger(struct vfio_cdx_device *vdev,
+				    unsigned int index, unsigned int start,
+				    unsigned int count, uint32_t flags,
+				    void *data)
+{
+	struct cdx_device *cdx_dev = vdev->cdx_dev;
+	int i;
+
+	if (start + count > cdx_dev->num_msi)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_cdx_msi_disable(vdev);
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (vdev->config_msi)
+			return vfio_cdx_msi_set_block(vdev, start, count,
+						  fds);
+		ret = vfio_cdx_msi_enable(vdev, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_cdx_msi_set_block(vdev, start, count, fds);
+		if (ret)
+			vfio_cdx_msi_disable(vdev);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!vdev->cdx_irqs[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(vdev->cdx_irqs[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(vdev->cdx_irqs[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
+int vfio_cdx_set_irqs_ioctl(struct vfio_cdx_device *vdev,
+			    u32 flags, unsigned int index,
+			    unsigned int start, unsigned int count,
+			    void *data)
+{
+	if (flags & VFIO_IRQ_SET_ACTION_TRIGGER)
+		return vfio_cdx_set_msi_trigger(vdev, index, start,
+			  count, flags, data);
+	else
+		return -EINVAL;
+}
+
+/* Free All IRQs for the given device */
+void vfio_cdx_irqs_cleanup(struct vfio_cdx_device *vdev)
+{
+	/*
+	 * Device does not support any interrupt or the interrupts
+	 * were not configured
+	 */
+	if (!vdev->cdx_irqs)
+		return;
+
+	vfio_cdx_set_msi_trigger(vdev, 1, 0, 0,
+			VFIO_IRQ_SET_DATA_NONE, NULL);
+
+	if (vdev->config_msi) {
+		cdx_msi_domain_free_irqs(vdev->dev);
+		kfree(vdev->cdx_irqs);
+	}
+	vdev->cdx_irqs = NULL;
+}
diff --git a/drivers/vfio/cdx/vfio_cdx_private.h b/drivers/vfio/cdx/vfio_cdx_private.h
index d87b55663462..9f93fc8cabfd 100644
--- a/drivers/vfio/cdx/vfio_cdx_private.h
+++ b/drivers/vfio/cdx/vfio_cdx_private.h
@@ -14,6 +14,14 @@
 #define VFIO_CDX_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_CDX_OFFSET_SHIFT)
 
+struct vfio_cdx_irq {
+	u32			flags;
+	u32			count;
+	int			irq_no;
+	struct eventfd_ctx	*trigger;
+	char			*name;
+};
+
 struct vfio_cdx_region {
 	u32			flags;
 	u32			type;
@@ -27,6 +35,16 @@ struct vfio_cdx_device {
 	struct cdx_device		*cdx_dev;
 	struct device			*dev;
 	struct vfio_cdx_region		*regions;
+	struct vfio_cdx_irq		*cdx_irqs;
+	uint32_t			irq_count;
+	uint32_t			config_msi;
 };
 
+int vfio_cdx_set_irqs_ioctl(struct vfio_cdx_device *vdev,
+		u32 flags, unsigned int index,
+		unsigned int start, unsigned int count,
+		void *data);
+
+void vfio_cdx_irqs_cleanup(struct vfio_cdx_device *vdev);
+
 #endif /* VFIO_CDX_PRIVATE_H */
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 0/8] add support for CDX bus

[Nipun Gupta]

This patch series introduces AMD CDX bus, which provides a
mechanism to discover/rescan FPGA devices on run-time. These
devices are memory mapped on system bus for embedded CPUs, and 
added as CDX devices in Linux framework.

This RFC:
- Intrduces the CDX bus controller and CDX devices.
- Adds rescan and reset support for the CDX bus.
- Add support for reset for CDX devices.
- Support for CDX bus in arm-smmu-v3 driver
- Support for CDX-MSI domain.

Please NOTE: This RFC change does not support the CDX bus firmware
interface as it is under development, and this series aims to get 
an early feedback from the community. Firmware interaction are 
stubbed as MCDI APIs which is a protocol used by AMD to interact
with Firmware.

Changes in v2: 
- introduce CDX bus infrastructure
- fixed code for making compatible visible for devices
  having the 'compatible' property only (Greg K-H)
- moved CDX-MSI domain as part of CDX bus infrastructure.
  previously it was part of irqchip (Marc Zynger).
- fixed few prints (Greg K-H)
- support rescan and reset of CDX bus 
- add VFIO reset module for CDX bus based devices

Changes in v3: 
- Move CDX bus as a new bus type in kernel rather than
  using the platform devices (Greg K-H, Marc Zynger)
- Correspondingly update ARM SMMU v3
- Add support for vfio-cdx driver
- Updated device tree yaml with correct binding information
  (Krzysztof Kozlowski)
- remove 'compatible' sysfs platform patch which was required
  for CDX devices exposed as platform devices

Changes in v4: 
 - Separate CDX bus and CDX controller driver (Greg K-H)
 - Added MSI interfacing to Firmware for writing MSI message
   to firmware so it can be provided to the device.
 - Fix MSI review comments - multiple cleanups (Mark Zynger)
 - Fix the device tree yaml compilation (Rob Herring, Krzysztof)
 - removed vfio-cdx from this series. It will be added after bus
   support is complete (Jason)

Nipun Gupta (8):
  dt-bindings: bus: add CDX bus device tree bindings
  bus/cdx: add the cdx bus driver
  iommu/arm-smmu-v3: support ops registration for CDX bus
  bux/cdx: support dma configuration for CDX devices
  bus/cdx: add bus and device attributes
  irq/msi: use implicit msi domain for alloc and free
  bus/cdx: add cdx-MSI domain with gic-its domain as parent
  bus/cdx: add cdx controller

 .../devicetree/bindings/bus/xlnx,cdx.yaml     |  66 +++
 MAINTAINERS                                   |   8 +
 drivers/base/platform-msi.c                   |   4 +-
 drivers/bus/Kconfig                           |   1 +
 drivers/bus/Makefile                          |   3 +
 drivers/bus/cdx/Kconfig                       |  17 +
 drivers/bus/cdx/Makefile                      |   8 +
 drivers/bus/cdx/cdx.c                         | 557 ++++++++++++++++++
 drivers/bus/cdx/cdx.h                         |  76 +++
 drivers/bus/cdx/cdx_msi.c                     | 161 +++++
 drivers/bus/cdx/controller/Kconfig            |  16 +
 drivers/bus/cdx/controller/Makefile           |   8 +
 drivers/bus/cdx/controller/cdx_controller.c   | 210 +++++++
 drivers/bus/cdx/controller/mcdi_stubs.c       |  68 +++
 drivers/bus/cdx/controller/mcdi_stubs.h       | 101 ++++
 drivers/bus/fsl-mc/fsl-mc-msi.c               |   4 +-
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c   |  16 +-
 drivers/pci/msi/irqdomain.c                   |   4 +-
 drivers/soc/ti/ti_sci_inta_msi.c              |   2 +-
 include/linux/cdx/cdx_bus.h                   | 165 ++++++
 include/linux/mod_devicetable.h               |  13 +
 include/linux/msi.h                           |  10 +-
 kernel/irq/msi.c                              |  22 +-
 scripts/mod/devicetable-offsets.c             |   4 +
 scripts/mod/file2alias.c                      |  12 +
 25 files changed, 1529 insertions(+), 27 deletions(-)
 create mode 100644 Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
 create mode 100644 drivers/bus/cdx/Kconfig
 create mode 100644 drivers/bus/cdx/Makefile
 create mode 100644 drivers/bus/cdx/cdx.c
 create mode 100644 drivers/bus/cdx/cdx.h
 create mode 100644 drivers/bus/cdx/cdx_msi.c
 create mode 100644 drivers/bus/cdx/controller/Kconfig
 create mode 100644 drivers/bus/cdx/controller/Makefile
 create mode 100644 drivers/bus/cdx/controller/cdx_controller.c
 create mode 100644 drivers/bus/cdx/controller/mcdi_stubs.c
 create mode 100644 drivers/bus/cdx/controller/mcdi_stubs.h
 create mode 100644 include/linux/cdx/cdx_bus.h

-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 1/8] dt-bindings: bus: add CDX bus device tree bindings

[Nipun Gupta]

This patch adds a devicetree binding documentation for CDX
bus.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 .../devicetree/bindings/bus/xlnx,cdx.yaml     | 65 +++++++++++++++++++
 MAINTAINERS                                   |  6 ++
 2 files changed, 71 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/bus/xlnx,cdx.yaml

diff --git a/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml b/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
new file mode 100644
index 000000000000..984ff65b668a
--- /dev/null
+++ b/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
@@ -0,0 +1,65 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/bus/xlnx,cdx.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: AMD CDX bus controller
+
+description: |
+  CDX bus controller detects CDX devices using CDX firmware and
+  add those to cdx bus. The CDX bus manages multiple FPGA based
+  hardware devices, which can support network, crypto or any other
+  specialized type of devices. These FPGA based devices can be
+  added/modified dynamically on run-time.
+
+  All devices on the CDX bus will have a unique streamid (for IOMMU)
+  and a unique device ID (for MSI) corresponding to a requestor ID
+  (one to one associated with the device). The streamid and deviceid
+  are used to configure SMMU and GIC-ITS respectively.
+
+  iommu-map property is used to define the set of stream ids
+  corresponding to each device and the associated IOMMU.
+
+  The MSI writes are accompanied by sideband data (Device ID).
+  The msi-map property is used to associate the devices with the
+  device ID as well as the associated ITS controller.
+
+maintainers:
+  - Nipun Gupta <nipun.gupta@amd.com>
+  - Nikhil Agarwal <nikhil.agarwal@amd.com>
+
+properties:
+  compatible:
+    const: xlnx,cdxbus-controller-1.0
+
+  reg:
+    maxItems: 1
+
+  iommu-map: true
+
+  msi-map: true
+
+required:
+  - compatible
+  - reg
+  - iommu-map
+  - msi-map
+
+additionalProperties: false
+
+examples:
+  - |
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        cdx: cdx@4000000 {
+            compatible = "xlnx,cdxbus-controller-1.0";
+            reg = <0x00000000 0x04000000 0 0x1000>;
+            /* define map for RIDs 250-259 */
+            iommu-map = <250 &smmu 250 10>;
+            /* define msi map for RIDs 250-259 */
+            msi-map = <250 &its 250 10>;
+        };
+    };
diff --git a/MAINTAINERS b/MAINTAINERS
index f5ca4aefd184..5f48f11fe0c3 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -935,6 +935,12 @@ S:	Supported
 F:	drivers/crypto/ccp/
 F:	include/linux/ccp.h
 
+AMD CDX BUS DRIVER
+M:	Nipun Gupta <nipun.gupta@amd.com>
+M:	Nikhil Agarwal <nikhil.agarwal@amd.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
+
 AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER - SEV SUPPORT
 M:	Brijesh Singh <brijesh.singh@amd.com>
 M:	Tom Lendacky <thomas.lendacky@amd.com>
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v4 1/8] dt-bindings: bus: add CDX bus device tree bindings

[Rob Herring]

On Fri, Oct 14, 2022 at 10:10:42AM +0530, Nipun Gupta wrote:
> This patch adds a devicetree binding documentation for CDX
> bus.

Please read submitting-patches.rst and what it says about commit 
messages.

> 
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> ---
>  .../devicetree/bindings/bus/xlnx,cdx.yaml     | 65 +++++++++++++++++++
>  MAINTAINERS                                   |  6 ++
>  2 files changed, 71 insertions(+)
>  create mode 100644 Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> 
> diff --git a/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml b/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> new file mode 100644
> index 000000000000..984ff65b668a
> --- /dev/null
> +++ b/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> @@ -0,0 +1,65 @@
> +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
> +%YAML 1.2
> +---
> +$id: http://devicetree.org/schemas/bus/xlnx,cdx.yaml#
> +$schema: http://devicetree.org/meta-schemas/core.yaml#
> +
> +title: AMD CDX bus controller
> +
> +description: |
> +  CDX bus controller detects CDX devices using CDX firmware and
> +  add those to cdx bus. The CDX bus manages multiple FPGA based
> +  hardware devices, which can support network, crypto or any other
> +  specialized type of devices. These FPGA based devices can be
> +  added/modified dynamically on run-time.
> +
> +  All devices on the CDX bus will have a unique streamid (for IOMMU)
> +  and a unique device ID (for MSI) corresponding to a requestor ID
> +  (one to one associated with the device). The streamid and deviceid
> +  are used to configure SMMU and GIC-ITS respectively.
> +
> +  iommu-map property is used to define the set of stream ids
> +  corresponding to each device and the associated IOMMU.
> +
> +  The MSI writes are accompanied by sideband data (Device ID).
> +  The msi-map property is used to associate the devices with the
> +  device ID as well as the associated ITS controller.
> +
> +maintainers:
> +  - Nipun Gupta <nipun.gupta@amd.com>
> +  - Nikhil Agarwal <nikhil.agarwal@amd.com>
> +
> +properties:
> +  compatible:
> +    const: xlnx,cdxbus-controller-1.0

Where does 1.0 come from?

> +
> +  reg:
> +    maxItems: 1
> +
> +  iommu-map: true
> +
> +  msi-map: true
> +
> +required:
> +  - compatible
> +  - reg
> +  - iommu-map
> +  - msi-map
> +
> +additionalProperties: false
> +
> +examples:
> +  - |
> +    soc {
> +        #address-cells = <2>;
> +        #size-cells = <2>;
> +
> +        cdx: cdx@4000000 {

bus@...

> +            compatible = "xlnx,cdxbus-controller-1.0";
> +            reg = <0x00000000 0x04000000 0 0x1000>;
> +            /* define map for RIDs 250-259 */
> +            iommu-map = <250 &smmu 250 10>;
> +            /* define msi map for RIDs 250-259 */
> +            msi-map = <250 &its 250 10>;
> +        };
> +    };
> diff --git a/MAINTAINERS b/MAINTAINERS
> index f5ca4aefd184..5f48f11fe0c3 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -935,6 +935,12 @@ S:	Supported
>  F:	drivers/crypto/ccp/
>  F:	include/linux/ccp.h
>  
> +AMD CDX BUS DRIVER
> +M:	Nipun Gupta <nipun.gupta@amd.com>
> +M:	Nikhil Agarwal <nikhil.agarwal@amd.com>
> +S:	Maintained
> +F:	Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> +
>  AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER - SEV SUPPORT
>  M:	Brijesh Singh <brijesh.singh@amd.com>
>  M:	Tom Lendacky <thomas.lendacky@amd.com>
> -- 
> 2.25.1
> 
> 

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v4 1/8] dt-bindings: bus: add CDX bus device tree bindings

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Rob Herring <robh@kernel.org>
> Sent: Friday, October 14, 2022 7:47 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: krzysztof.kozlowski+dt@linaro.org; gregkh@linuxfoundation.org;
> rafael@kernel.org; eric.auger@redhat.com; alex.williamson@redhat.com;
> cohuck@redhat.com; Gupta, Puneet (DCG-ENG) <puneet.gupta@amd.com>;
> song.bao.hua@hisilicon.com; mchehab+huawei@kernel.org; maz@kernel.org;
> f.fainelli@gmail.com; jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com; will@kernel.org;
> joro@8bytes.org; masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v4 1/8] dt-bindings: bus: add CDX bus device tree
> bindings
> 
> Caution: This message originated from an External Source. Use proper caution
> when opening attachments, clicking links, or responding.
> 
> 
> On Fri, Oct 14, 2022 at 10:10:42AM +0530, Nipun Gupta wrote:
> > This patch adds a devicetree binding documentation for CDX
> > bus.
> 
> Please read submitting-patches.rst and what it says about commit
> messages.

Thanks for pointing. I had a look at the file and will update accordingly

> 
> >
> > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> > ---
> >  .../devicetree/bindings/bus/xlnx,cdx.yaml     | 65 +++++++++++++++++++
> >  MAINTAINERS                                   |  6 ++
> >  2 files changed, 71 insertions(+)
> >  create mode 100644 Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> >
> > diff --git a/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> b/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> > new file mode 100644
> > index 000000000000..984ff65b668a
> > --- /dev/null
> > +++ b/Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> > @@ -0,0 +1,65 @@
> > +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
> > +%YAML 1.2
> > +---
> > +$id:
> https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdevicetre
> e.org%2Fschemas%2Fbus%2Fxlnx%2Ccdx.yaml%23&amp;data=05%7C01%7Cnip
> un.gupta%40amd.com%7C16a836a178fa4e373c9308daadeecd32%7C3dd8961f
> e4884e608e11a82d994e183d%7C0%7C0%7C638013538406495866%7CUnknow
> n%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haW
> wiLCJXVCI6Mn0%3D%7C3000%7C%7C%7C&amp;sdata=44F%2Bw51YMeCyE78g
> 3EmevxqYx%2FybbeKdlkDv1X6XDcM%3D&amp;reserved=0
> > +$schema:
> https://nam11.safelinks.protection.outlook.com/?url=http%3A%2F%2Fdevicetre
> e.org%2Fmeta-
> schemas%2Fcore.yaml%23&amp;data=05%7C01%7Cnipun.gupta%40amd.com%
> 7C16a836a178fa4e373c9308daadeecd32%7C3dd8961fe4884e608e11a82d994e
> 183d%7C0%7C0%7C638013538406495866%7CUnknown%7CTWFpbGZsb3d8eyJ
> WIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C
> 3000%7C%7C%7C&amp;sdata=FPADx3bdmJg%2BfnEs%2BUtNTMPo3U1aiX3wv
> w57EQvleK4%3D&amp;reserved=0
> > +
> > +title: AMD CDX bus controller
> > +
> > +description: |
> > +  CDX bus controller detects CDX devices using CDX firmware and
> > +  add those to cdx bus. The CDX bus manages multiple FPGA based
> > +  hardware devices, which can support network, crypto or any other
> > +  specialized type of devices. These FPGA based devices can be
> > +  added/modified dynamically on run-time.
> > +
> > +  All devices on the CDX bus will have a unique streamid (for IOMMU)
> > +  and a unique device ID (for MSI) corresponding to a requestor ID
> > +  (one to one associated with the device). The streamid and deviceid
> > +  are used to configure SMMU and GIC-ITS respectively.
> > +
> > +  iommu-map property is used to define the set of stream ids
> > +  corresponding to each device and the associated IOMMU.
> > +
> > +  The MSI writes are accompanied by sideband data (Device ID).
> > +  The msi-map property is used to associate the devices with the
> > +  device ID as well as the associated ITS controller.
> > +
> > +maintainers:
> > +  - Nipun Gupta <nipun.gupta@amd.com>
> > +  - Nikhil Agarwal <nikhil.agarwal@amd.com>
> > +
> > +properties:
> > +  compatible:
> > +    const: xlnx,cdxbus-controller-1.0
> 
> Where does 1.0 come from?

This came up in internal review comment, but seems that there is no need for
having the controller versioning. We will remove in the next set of patches.

> 
> > +
> > +  reg:
> > +    maxItems: 1
> > +
> > +  iommu-map: true
> > +
> > +  msi-map: true
> > +
> > +required:
> > +  - compatible
> > +  - reg
> > +  - iommu-map
> > +  - msi-map
> > +
> > +additionalProperties: false
> > +
> > +examples:
> > +  - |
> > +    soc {
> > +        #address-cells = <2>;
> > +        #size-cells = <2>;
> > +
> > +        cdx: cdx@4000000 {
> 
> bus@...

Sure. Will update.

Thanks,
Nipun

> 
> > +            compatible = "xlnx,cdxbus-controller-1.0";
> > +            reg = <0x00000000 0x04000000 0 0x1000>;
> > +            /* define map for RIDs 250-259 */
> > +            iommu-map = <250 &smmu 250 10>;
> > +            /* define msi map for RIDs 250-259 */
> > +            msi-map = <250 &its 250 10>;
> > +        };
> > +    };
> > diff --git a/MAINTAINERS b/MAINTAINERS
> > index f5ca4aefd184..5f48f11fe0c3 100644
> > --- a/MAINTAINERS
> > +++ b/MAINTAINERS
> > @@ -935,6 +935,12 @@ S:       Supported
> >  F:   drivers/crypto/ccp/
> >  F:   include/linux/ccp.h
> >
> > +AMD CDX BUS DRIVER
> > +M:   Nipun Gupta <nipun.gupta@amd.com>
> > +M:   Nikhil Agarwal <nikhil.agarwal@amd.com>
> > +S:   Maintained
> > +F:   Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
> > +
> >  AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER - SEV SUPPORT
> >  M:   Brijesh Singh <brijesh.singh@amd.com>
> >  M:   Tom Lendacky <thomas.lendacky@amd.com>
> > --
> > 2.25.1
> >
> >

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 2/8] bus/cdx: add the cdx bus driver

[Nipun Gupta]

CDX bus supports the scanning and probing of FPGA based
devices. These devices are registered as CDX devices.

The bus driver sets up the basic infrastructure and triggers
the cdx controller to scan the cdx devices once registered.

CDX bus uses ops registered by the CDX controller to scan,
reset and write MSI message on the CDX devices.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 MAINTAINERS                       |   2 +
 drivers/bus/Kconfig               |   1 +
 drivers/bus/Makefile              |   3 +
 drivers/bus/cdx/Kconfig           |  13 ++
 drivers/bus/cdx/Makefile          |   8 +
 drivers/bus/cdx/cdx.c             | 366 ++++++++++++++++++++++++++++++
 drivers/bus/cdx/cdx.h             |  66 ++++++
 include/linux/cdx/cdx_bus.h       | 139 ++++++++++++
 include/linux/mod_devicetable.h   |  13 ++
 scripts/mod/devicetable-offsets.c |   4 +
 scripts/mod/file2alias.c          |  12 +
 11 files changed, 627 insertions(+)
 create mode 100644 drivers/bus/cdx/Kconfig
 create mode 100644 drivers/bus/cdx/Makefile
 create mode 100644 drivers/bus/cdx/cdx.c
 create mode 100644 drivers/bus/cdx/cdx.h
 create mode 100644 include/linux/cdx/cdx_bus.h

diff --git a/MAINTAINERS b/MAINTAINERS
index 5f48f11fe0c3..6b7b3c098839 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -940,6 +940,8 @@ M:	Nipun Gupta <nipun.gupta@amd.com>
 M:	Nikhil Agarwal <nikhil.agarwal@amd.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/bus/xlnx,cdx.yaml
+F:	drivers/bus/cdx/*
+F:	include/linux/cdx/*
 
 AMD CRYPTOGRAPHIC COPROCESSOR (CCP) DRIVER - SEV SUPPORT
 M:	Brijesh Singh <brijesh.singh@amd.com>
diff --git a/drivers/bus/Kconfig b/drivers/bus/Kconfig
index 7bfe998f3514..b0324efb9a6a 100644
--- a/drivers/bus/Kconfig
+++ b/drivers/bus/Kconfig
@@ -251,5 +251,6 @@ config DA8XX_MSTPRI
 
 source "drivers/bus/fsl-mc/Kconfig"
 source "drivers/bus/mhi/Kconfig"
+source "drivers/bus/cdx/Kconfig"
 
 endmenu
diff --git a/drivers/bus/Makefile b/drivers/bus/Makefile
index d90eed189a65..88649111c395 100644
--- a/drivers/bus/Makefile
+++ b/drivers/bus/Makefile
@@ -20,6 +20,9 @@ obj-$(CONFIG_INTEL_IXP4XX_EB)	+= intel-ixp4xx-eb.o
 obj-$(CONFIG_MIPS_CDMM)		+= mips_cdmm.o
 obj-$(CONFIG_MVEBU_MBUS) 	+= mvebu-mbus.o
 
+#CDX bus
+obj-$(CONFIG_CDX_BUS)		+= cdx/
+
 # Interconnect bus driver for OMAP SoCs.
 obj-$(CONFIG_OMAP_INTERCONNECT)	+= omap_l3_smx.o omap_l3_noc.o
 
diff --git a/drivers/bus/cdx/Kconfig b/drivers/bus/cdx/Kconfig
new file mode 100644
index 000000000000..98ec05ad708d
--- /dev/null
+++ b/drivers/bus/cdx/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# CDX bus configuration
+#
+# Copyright (C) 2022, Advanced Micro Devices, Inc.
+#
+
+config CDX_BUS
+	bool "CDX Bus driver"
+	help
+		Driver to enable CDX Bus infrastructure. CDX bus uses
+		CDX controller and firmware to scan the FPGA based
+		devices.
diff --git a/drivers/bus/cdx/Makefile b/drivers/bus/cdx/Makefile
new file mode 100644
index 000000000000..2e8f42611dfc
--- /dev/null
+++ b/drivers/bus/cdx/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for CDX
+#
+# Copyright (C) 2022, Advanced Micro Devices, Inc.
+#
+
+obj-$(CONFIG_CDX_BUS) += cdx.o
diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
new file mode 100644
index 000000000000..5a366f4ae69c
--- /dev/null
+++ b/drivers/bus/cdx/cdx.c
@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CDX bus driver.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+/*
+ * Architecture Overview
+ * =====================
+ * CDX is a Hardware Architecture designed for AMD FPGA devices. It
+ * consists of sophisticated mechanism for interaction between FPGA,
+ * Firmware and the APUs (Application CPUs).
+ *
+ * Firmware resides on RPU (Realtime CPUs) which interacts with
+ * the FPGA program manager and the APUs. The RPU provides memory-mapped
+ * interface (RPU if) which is used to communicate with APUs.
+ *
+ * The diagram below shows an overview of the CDX architecture:
+ *
+ *          +--------------------------------------+
+ *          |    Application CPUs (APU)            |
+ *          |                                      |
+ *          |                    CDX device drivers|
+ *          |     Linux OS                |        |
+ *          |                        CDX bus       |
+ *          |                             |        |
+ *          |                     CDX controller   |
+ *          |                             |        |
+ *          +-----------------------------|--------+
+ *                                        | (discover, config,
+ *                                        |  reset, rescan)
+ *                                        |
+ *          +------------------------| RPU if |----+
+ *          |                             |        |
+ *          |                             V        |
+ *          |          Realtime CPUs (RPU)         |
+ *          |                                      |
+ *          +--------------------------------------+
+ *                                |
+ *          +---------------------|----------------+
+ *          |  FPGA               |                |
+ *          |      +-----------------------+       |
+ *          |      |           |           |       |
+ *          | +-------+    +-------+   +-------+   |
+ *          | | dev 1 |    | dev 2 |   | dev 3 |   |
+ *          | +-------+    +-------+   +-------+   |
+ *          +--------------------------------------+
+ *
+ * The RPU firmware extracts the device information from the loaded FPGA
+ * image and implements a mechanism that allows the APU drivers to
+ * enumerate such devices (device personality and resource details) via
+ * a dedicated communication channel. RPU mediates operations such as
+ * discover, reset and rescan of the FPGA devices for the APU. This is
+ * done using memory mapped interface provided by the RPU to APU.
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/cdx/cdx_bus.h>
+
+#include "cdx.h"
+
+/*
+ * Default DMA mask for devices on a CDX bus
+ */
+#define CDX_DEFAULT_DMA_MASK	(~0ULL)
+
+static struct cdx_controller_t *cdx_controller;
+
+static int reset_cdx_device(struct device *dev, void * __always_unused data)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	struct cdx_controller_t *cdx = cdx_dev->cdx;
+	int ret;
+
+	ret = cdx->ops.reset_dev(cdx, cdx_dev->bus_num, cdx_dev->dev_num);
+	if (ret)
+		dev_err(dev, "cdx device reset failed\n");
+
+	return ret;
+}
+
+int cdx_dev_reset(struct device *dev)
+{
+	return reset_cdx_device(dev, NULL);
+}
+EXPORT_SYMBOL_GPL(cdx_dev_reset);
+
+static int cdx_unregister_device(struct device *dev,
+				 void * __always_unused data)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	kfree(cdx_dev->driver_override);
+	cdx_dev->driver_override = NULL;
+
+	/*
+	 * Do not free cdx_dev here as it would be freed in
+	 * cdx_device_release() called from within put_device().
+	 */
+	device_del(&cdx_dev->dev);
+	put_device(&cdx_dev->dev);
+
+	return 0;
+}
+
+/**
+ * cdx_match_one_device - Tell if a CDX device structure has a matching
+ *			  CDX device id structure
+ * @id: single CDX device id structure to match
+ * @dev: the CDX device structure to match against
+ *
+ * Returns the matching cdx_device_id structure or NULL if there is no match.
+ */
+static inline const struct cdx_device_id *
+cdx_match_one_device(const struct cdx_device_id *id,
+		     const struct cdx_device *dev)
+{
+	/* Use vendor ID and device ID for matching */
+	if ((id->vendor == dev->vendor) && (id->device == dev->device))
+		return id;
+	return NULL;
+}
+
+/**
+ * cdx_match_id - See if a CDX device matches a given cdx_id table
+ * @ids: array of CDX device ID structures to search in
+ * @dev: the CDX device structure to match against.
+ *
+ * Used by a driver to check whether a CDX device is in its list of
+ * supported devices.  Returns the matching cdx_device_id structure or
+ * NULL if there is no match.
+ */
+static inline const struct cdx_device_id *
+cdx_match_id(const struct cdx_device_id *ids, struct cdx_device *dev)
+{
+	if (ids) {
+		while (ids->vendor || ids->device) {
+			if (cdx_match_one_device(ids, dev))
+				return ids;
+			ids++;
+		}
+	}
+	return NULL;
+}
+
+/**
+ * cdx_bus_match - device to driver matching callback
+ * @dev: the cdx device to match against
+ * @drv: the device driver to search for matching cdx device
+ * structures
+ *
+ * Returns 1 on success, 0 otherwise.
+ */
+static int cdx_bus_match(struct device *dev, struct device_driver *drv)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	struct cdx_driver *cdx_drv = to_cdx_driver(drv);
+	const struct cdx_device_id *found_id;
+
+	/* When driver_override is set, only bind to the matching driver */
+	if (cdx_dev->driver_override)
+		return !strcmp(cdx_dev->driver_override, cdx_drv->driver.name);
+
+	found_id = cdx_match_id(cdx_drv->match_id_table, cdx_dev);
+	if (found_id)
+		return true;
+
+	return false;
+}
+
+static void cdx_remove(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	if (cdx_drv->remove)
+		cdx_drv->remove(cdx_dev);
+}
+
+struct bus_type cdx_bus_type = {
+	.name		= "cdx",
+	.match		= cdx_bus_match,
+	.remove		= cdx_remove,
+};
+EXPORT_SYMBOL_GPL(cdx_bus_type);
+
+static int cdx_driver_probe(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	int error;
+
+	error = cdx_drv->probe(cdx_dev);
+	if (error < 0) {
+		if (error != -EPROBE_DEFER)
+			dev_err(dev, "%s failed: %d\n", __func__, error);
+		return error;
+	}
+
+	return 0;
+}
+
+static int cdx_driver_remove(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	int ret;
+
+	if (cdx_drv->remove) {
+		ret = cdx_drv->remove(cdx_dev);
+		if (ret < 0) {
+			dev_err(dev, "%s failed: %d\n", __func__, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void cdx_driver_shutdown(struct device *dev)
+{
+	struct cdx_driver *cdx_drv = to_cdx_driver(dev->driver);
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	cdx_drv->shutdown(cdx_dev);
+}
+
+int __cdx_driver_register(struct cdx_driver *cdx_driver,
+			  struct module *owner)
+{
+	int error;
+
+	cdx_driver->driver.owner = owner;
+	cdx_driver->driver.bus = &cdx_bus_type;
+
+	if (cdx_driver->probe)
+		cdx_driver->driver.probe = cdx_driver_probe;
+
+	if (cdx_driver->remove)
+		cdx_driver->driver.remove = cdx_driver_remove;
+
+	if (cdx_driver->shutdown)
+		cdx_driver->driver.shutdown = cdx_driver_shutdown;
+
+	error = driver_register(&cdx_driver->driver);
+	if (error < 0) {
+		pr_err("driver_register() failed for %s: %d\n",
+		       cdx_driver->driver.name, error);
+		return error;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(__cdx_driver_register);
+
+void cdx_driver_unregister(struct cdx_driver *cdx_driver)
+{
+	driver_unregister(&cdx_driver->driver);
+}
+EXPORT_SYMBOL_GPL(cdx_driver_unregister);
+
+static void cdx_device_release(struct device *dev)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	kfree(cdx_dev);
+}
+
+int cdx_device_add(struct cdx_dev_params_t *dev_params)
+{
+	struct cdx_controller_t *cdx = dev_params->cdx;
+	struct device *parent = cdx->dev;
+	struct cdx_device *cdx_dev;
+	int ret;
+
+	cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
+	if (!cdx_dev) {
+		dev_err(parent,
+			"memory allocation for cdx dev failed\n");
+		return -ENOMEM;
+	}
+
+	/* Populate resource */
+	memcpy(cdx_dev->res, dev_params->res, sizeof(struct resource) *
+		dev_params->res_count);
+	cdx_dev->res_count = dev_params->res_count;
+
+	/* Populate CDX dev params */
+	cdx_dev->req_id = dev_params->req_id;
+	cdx_dev->vendor = dev_params->vendor;
+	cdx_dev->device = dev_params->device;
+	cdx_dev->bus_num = dev_params->bus_num;
+	cdx_dev->dev_num = dev_params->dev_num;
+	cdx_dev->cdx = dev_params->cdx;
+	cdx_dev->dma_mask = CDX_DEFAULT_DMA_MASK;
+
+	/* Initiaize generic device */
+	device_initialize(&cdx_dev->dev);
+	cdx_dev->dev.parent = parent;
+	cdx_dev->dev.bus = &cdx_bus_type;
+	cdx_dev->dev.dma_mask = &cdx_dev->dma_mask;
+	cdx_dev->dev.release = cdx_device_release;
+
+	/* Set Name */
+	dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", cdx_dev->bus_num,
+			cdx_dev->dev_num);
+
+	ret = device_add(&cdx_dev->dev);
+	if (ret != 0) {
+		dev_err(&cdx_dev->dev,
+			"cdx device add failed: %d", ret);
+		goto fail;
+	}
+
+	/* Reset the device before use */
+	ret = cdx->ops.reset_dev(cdx, cdx_dev->bus_num, cdx_dev->dev_num);
+	if (ret) {
+		dev_err(&cdx_dev->dev, "cdx device reset failed\n");
+		goto fail;
+	}
+
+	return 0;
+
+fail:
+	/*
+	 * Do not free cdx_dev here as it would be freed in
+	 * cdx_device_release() called from put_device().
+	 */
+	put_device(&cdx_dev->dev);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cdx_device_add);
+
+int cdx_register_controller(struct cdx_controller_t *cdx)
+{
+	int ret;
+
+	/* Scan all the devices */
+	ret = cdx->ops.scan(cdx);
+	if (ret) {
+		dev_err(cdx->dev, "CDX scanning failed: %d\n", ret);
+		return ret;
+	}
+
+	cdx_controller = cdx;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(cdx_register_controller);
+
+void cdx_unregister_controller(struct cdx_controller_t *cdx)
+{
+	device_for_each_child(cdx->dev, NULL, cdx_unregister_device);
+	cdx_controller = NULL;
+}
+EXPORT_SYMBOL_GPL(cdx_unregister_controller);
+
+static int __init cdx_bus_init(void)
+{
+	return bus_register(&cdx_bus_type);
+}
+postcore_initcall(cdx_bus_init);
diff --git a/drivers/bus/cdx/cdx.h b/drivers/bus/cdx/cdx.h
new file mode 100644
index 000000000000..80496865ae9c
--- /dev/null
+++ b/drivers/bus/cdx/cdx.h
@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Header file for the CDX Bus
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _CDX_H_
+#define _CDX_H_
+
+#include <linux/cdx/cdx_bus.h>
+
+/**
+ * struct cdx_dev_params_t - CDX device parameters
+ * @cdx: CDX controller assocated with the device
+ * @parent: Associated CDX controller
+ * @vendor: Vendor ID for CDX device
+ * @device: Device ID for CDX device
+ * @subsys_vendor: Sub vendor ID for CDX device
+ * @subsys_device: Sub device ID for CDX device
+ * @bus_num: Bus number for this CDX device
+ * @dev_num: Device number for this device
+ * @res: array of MMIO region entries
+ * @res_count: number of valid MMIO regions
+ * @req_id: Requestor ID associated with CDX device
+ */
+struct cdx_dev_params_t {
+	struct cdx_controller_t *cdx;
+	u16 vendor;
+	u16 device;
+	u16 subsys_vendor;
+	u16 subsys_device;
+	u8 bus_num;
+	u8 dev_num;
+	struct resource res[MAX_CDX_DEV_RESOURCES];
+	u8 res_count;
+	u32 req_id;
+};
+
+/**
+ * cdx_register_controller - Register a CDX controller and its ports
+ *		on the CDX bus.
+ * @cdx: The CDX controller to register
+ *
+ * Returns -errno on failure, 0 on success.
+ */
+int cdx_register_controller(struct cdx_controller_t *cdx);
+
+/**
+ * cdx_unregister_controller - Unregister a CDX controller
+ * @cdx: The CDX controller to unregister
+ */
+void cdx_unregister_controller(struct cdx_controller_t *cdx);
+
+/**
+ * cdx_device_add - Add a CDX device. This function adds a CDX device
+ *		on the CDX bus as per the device paramteres provided
+ *		by caller. It also creates and registers an associated
+ *		Linux generic device.
+ * @dev_params: device parameters associated with the device to be created.
+ *
+ * Returns -errno on failure, 0 on success.
+ */
+int cdx_device_add(struct cdx_dev_params_t *dev_params);
+
+#endif /* _CDX_H_ */
diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
new file mode 100644
index 000000000000..9e6872a03215
--- /dev/null
+++ b/include/linux/cdx/cdx_bus.h
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * CDX bus public interface
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ *
+ */
+#ifndef _CDX_BUS_H_
+#define _CDX_BUS_H_
+
+#include <linux/mod_devicetable.h>
+#include <linux/device.h>
+
+#define MAX_CDX_DEV_RESOURCES	4
+
+/* Forward declaration for CDX controller */
+struct cdx_controller_t;
+
+typedef int (*cdx_scan_t)(struct cdx_controller_t *cdx);
+
+typedef int (*cdx_dev_reset_t)(struct cdx_controller_t *cdx,
+			       uint8_t bus_num, uint8_t dev_num);
+
+/**
+ * Callbacks supported by CDX controller.
+ * @scan: scan the devices on the controller
+ * @reset_dev: reset a CDX device
+ */
+struct cdx_ops_t {
+	cdx_scan_t scan;
+	cdx_dev_reset_t reset_dev;
+};
+
+/**
+ * struct cdx_controller: CDX controller object
+ * @dev: Linux device associated with the CDX controller.
+ * @priv: private data
+ * @ops: CDX controller ops
+ */
+struct cdx_controller_t {
+	struct device *dev;
+	void *priv;
+
+	struct cdx_ops_t ops;
+};
+
+/**
+ * struct cdx_device - CDX device object
+ * @dev: Linux driver model device object
+ * @cdx: CDX controller assocated with the device
+ * @vendor: Vendor ID for CDX device
+ * @device: Device ID for CDX device
+ * @bus_num: Bus number for this CDX device
+ * @dev_num: Device number for this device
+ * @res: array of MMIO region entries
+ * @res_count: number of valid MMIO regions
+ * @dma_mask: Default DMA mask
+ * @flags: CDX device flags
+ * @req_id: Requestor ID associated with CDX device
+ * @driver_override: driver name to force a match; do not set directly,
+ *                   because core frees it; use driver_set_override() to
+ *                   set or clear it.
+ */
+struct cdx_device {
+	struct device dev;
+	struct cdx_controller_t *cdx;
+	u16 vendor;
+	u16 device;
+	u8 bus_num;
+	u8 dev_num;
+	struct resource res[MAX_CDX_DEV_RESOURCES];
+	u8 res_count;
+	u64 dma_mask;
+	u16 flags;
+	u32 req_id;
+	const char *driver_override;
+};
+
+#define to_cdx_device(_dev) \
+	container_of(_dev, struct cdx_device, dev)
+
+/**
+ * struct cdx_driver - CDX device driver
+ * @driver: Generic device driver
+ * @match_id_table: table of supported device matching Ids
+ * @probe: Function called when a device is added
+ * @remove: Function called when a device is removed
+ * @shutdown: Function called at shutdown time to quiesce the device
+ * @suspend: Function called when a device is stopped
+ * @resume: Function called when a device is resumed
+ * @driver_managed_dma: Device driver doesn't use kernel DMA API for DMA.
+ *		For most device drivers, no need to care about this flag
+ *		as long as all DMAs are handled through the kernel DMA API.
+ *		For some special ones, for example VFIO drivers, they know
+ *		how to manage the DMA themselves and set this flag so that
+ *		the IOMMU layer will allow them to setup and manage their
+ *		own I/O address space.
+ */
+struct cdx_driver {
+	struct device_driver driver;
+	const struct cdx_device_id *match_id_table;
+	int (*probe)(struct cdx_device *dev);
+	int (*remove)(struct cdx_device *dev);
+	void (*shutdown)(struct cdx_device *dev);
+	int (*suspend)(struct cdx_device *dev, pm_message_t state);
+	int (*resume)(struct cdx_device *dev);
+	bool driver_managed_dma;
+};
+
+#define to_cdx_driver(_drv) \
+	container_of(_drv, struct cdx_driver, driver)
+
+/* Macro to avoid include chaining to get THIS_MODULE */
+#define cdx_driver_register(drv) \
+	__cdx_driver_register(drv, THIS_MODULE)
+
+/**
+ * __cdx_driver_register - registers a CDX device driver
+ */
+int __must_check __cdx_driver_register(struct cdx_driver *cdx_driver,
+				       struct module *owner);
+
+/**
+ * cdx_driver_unregister - unregisters a device driver from the
+ * CDX bus.
+ */
+void cdx_driver_unregister(struct cdx_driver *driver);
+
+extern struct bus_type cdx_bus_type;
+
+/**
+ * cdx_dev_reset - Reset CDX device
+ * @dev: device pointer
+ *
+ * Return 0 for success, -errno on failure
+ */
+int cdx_dev_reset(struct device *dev);
+
+#endif /* _CDX_BUS_H_ */
diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h
index 549590e9c644..9b94be83d53e 100644
--- a/include/linux/mod_devicetable.h
+++ b/include/linux/mod_devicetable.h
@@ -911,4 +911,17 @@ struct ishtp_device_id {
 	kernel_ulong_t driver_data;
 };
 
+/**
+ * struct cdx_device_id - CDX device identifier
+ * @vendor: vendor ID
+ * @device: Device ID
+ *
+ * Type of entries in the "device Id" table for CDX devices supported by
+ * a CDX device driver.
+ */
+struct cdx_device_id {
+	__u16 vendor;
+	__u16 device;
+};
+
 #endif /* LINUX_MOD_DEVICETABLE_H */
diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c
index c0d3bcb99138..62dc988df84d 100644
--- a/scripts/mod/devicetable-offsets.c
+++ b/scripts/mod/devicetable-offsets.c
@@ -262,5 +262,9 @@ int main(void)
 	DEVID(ishtp_device_id);
 	DEVID_FIELD(ishtp_device_id, guid);
 
+	DEVID(cdx_device_id);
+	DEVID_FIELD(cdx_device_id, vendor);
+	DEVID_FIELD(cdx_device_id, device);
+
 	return 0;
 }
diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c
index 80d973144fde..c36e1f624e39 100644
--- a/scripts/mod/file2alias.c
+++ b/scripts/mod/file2alias.c
@@ -1452,6 +1452,17 @@ static int do_dfl_entry(const char *filename, void *symval, char *alias)
 	return 1;
 }
 
+/* Looks like: cdx:vNdN */
+static int do_cdx_entry(const char *filename, void *symval,
+			   char *alias)
+{
+	DEF_FIELD(symval, cdx_device_id, vendor);
+	DEF_FIELD(symval, cdx_device_id, device);
+
+	sprintf(alias, "cdx:v%08Xd%08Xd", vendor, device);
+	return 1;
+}
+
 /* Does namelen bytes of name exactly match the symbol? */
 static bool sym_is(const char *name, unsigned namelen, const char *symbol)
 {
@@ -1531,6 +1542,7 @@ static const struct devtable devtable[] = {
 	{"ssam", SIZE_ssam_device_id, do_ssam_entry},
 	{"dfl", SIZE_dfl_device_id, do_dfl_entry},
 	{"ishtp", SIZE_ishtp_device_id, do_ishtp_entry},
+	{"cdx", SIZE_cdx_device_id, do_cdx_entry},
 };
 
 /* Create MODULE_ALIAS() statements.
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v4 2/8] bus/cdx: add the cdx bus driver

[Greg KH]

On Fri, Oct 14, 2022 at 10:10:43AM +0530, Nipun Gupta wrote:
> CDX bus supports the scanning and probing of FPGA based
> devices. These devices are registered as CDX devices.
> 
> The bus driver sets up the basic infrastructure and triggers
> the cdx controller to scan the cdx devices once registered.
> 
> CDX bus uses ops registered by the CDX controller to scan,
> reset and write MSI message on the CDX devices.
> 
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>

Why is this all still "RFC"?  RFC means to me, "I don't know what to do
here, so here's a half-baked proposal".  When you are on v4 of a RFC,
that is a huge sign that this all isn't working at all.  Also, where is
the questions that you still have that you need help with here?

Also, I don't review RFC changes normally as that means the submitter
doesn't think they are good enough to be submitted, so why would I
review them when I have hundreds of other changes that submitters do
think are good enough?

Can you just submit these "for real" if they work properly for you and
you have tested them and you would be happy if they are accepted.

thanks,

greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v4 2/8] bus/cdx: add the cdx bus driver

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Greg KH <gregkh@linuxfoundation.org>
> Sent: Friday, October 14, 2022 12:45 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org; rafael@kernel.org;
> eric.auger@redhat.com; alex.williamson@redhat.com; cohuck@redhat.com;
> Gupta, Puneet (DCG-ENG) <puneet.gupta@amd.com>;
> song.bao.hua@hisilicon.com; mchehab+huawei@kernel.org; maz@kernel.org;
> f.fainelli@gmail.com; jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com; will@kernel.org;
> joro@8bytes.org; masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v4 2/8] bus/cdx: add the cdx bus driver
> 
> Caution: This message originated from an External Source. Use proper caution
> when opening attachments, clicking links, or responding.
> 
> 
> On Fri, Oct 14, 2022 at 10:10:43AM +0530, Nipun Gupta wrote:
> > CDX bus supports the scanning and probing of FPGA based
> > devices. These devices are registered as CDX devices.
> >
> > The bus driver sets up the basic infrastructure and triggers
> > the cdx controller to scan the cdx devices once registered.
> >
> > CDX bus uses ops registered by the CDX controller to scan,
> > reset and write MSI message on the CDX devices.
> >
> > Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> 
> Why is this all still "RFC"?  RFC means to me, "I don't know what to do
> here, so here's a half-baked proposal".  When you are on v4 of a RFC,
> that is a huge sign that this all isn't working at all.  Also, where is
> the questions that you still have that you need help with here?
> 
> Also, I don't review RFC changes normally as that means the submitter
> doesn't think they are good enough to be submitted, so why would I
> review them when I have hundreds of other changes that submitters do
> think are good enough?
> 
> Can you just submit these "for real" if they work properly for you and
> you have tested them and you would be happy if they are accepted.

We are in the process of merging the firmware support which is stubbed
in the current series. We will send out a non-RFC change in the next spin
with the integrated firmware support.

Thanks,
Nipun

> 
> thanks,
> 
> greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v4 2/8] bus/cdx: add the cdx bus driver

[Greg KH]

On Fri, Oct 14, 2022 at 10:10:43AM +0530, Nipun Gupta wrote:
> --- a/drivers/bus/Makefile
> +++ b/drivers/bus/Makefile
> @@ -20,6 +20,9 @@ obj-$(CONFIG_INTEL_IXP4XX_EB)	+= intel-ixp4xx-eb.o
>  obj-$(CONFIG_MIPS_CDMM)		+= mips_cdmm.o
>  obj-$(CONFIG_MVEBU_MBUS) 	+= mvebu-mbus.o
>  
> +#CDX bus

No need for a comment like this :)

> +obj-$(CONFIG_CDX_BUS)		+= cdx/
> +
>  # Interconnect bus driver for OMAP SoCs.
>  obj-$(CONFIG_OMAP_INTERCONNECT)	+= omap_l3_smx.o omap_l3_noc.o
>  
> diff --git a/drivers/bus/cdx/Kconfig b/drivers/bus/cdx/Kconfig
> new file mode 100644
> index 000000000000..98ec05ad708d
> --- /dev/null
> +++ b/drivers/bus/cdx/Kconfig
> @@ -0,0 +1,13 @@
> +# SPDX-License-Identifier: GPL-2.0
> +#
> +# CDX bus configuration
> +#
> +# Copyright (C) 2022, Advanced Micro Devices, Inc.
> +#
> +
> +config CDX_BUS
> +	bool "CDX Bus driver"
> +	help
> +		Driver to enable CDX Bus infrastructure. CDX bus uses
> +		CDX controller and firmware to scan the FPGA based
> +		devices.

Why bool?  Not as a module?  That's broken.

And "FPGA based devices" means nothing to me, please expand on what this
all is as it is not descriptive at all.

Also list the module name please.



> diff --git a/drivers/bus/cdx/Makefile b/drivers/bus/cdx/Makefile
> new file mode 100644
> index 000000000000..2e8f42611dfc
> --- /dev/null
> +++ b/drivers/bus/cdx/Makefile
> @@ -0,0 +1,8 @@
> +# SPDX-License-Identifier: GPL-2.0
> +#
> +# Makefile for CDX
> +#
> +# Copyright (C) 2022, Advanced Micro Devices, Inc.
> +#
> +
> +obj-$(CONFIG_CDX_BUS) += cdx.o
> diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
> new file mode 100644
> index 000000000000..5a366f4ae69c
> --- /dev/null
> +++ b/drivers/bus/cdx/cdx.c
> @@ -0,0 +1,366 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * CDX bus driver.
> + *
> + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> + */
> +
> +/*
> + * Architecture Overview
> + * =====================
> + * CDX is a Hardware Architecture designed for AMD FPGA devices. It
> + * consists of sophisticated mechanism for interaction between FPGA,
> + * Firmware and the APUs (Application CPUs).
> + *
> + * Firmware resides on RPU (Realtime CPUs) which interacts with
> + * the FPGA program manager and the APUs. The RPU provides memory-mapped
> + * interface (RPU if) which is used to communicate with APUs.
> + *
> + * The diagram below shows an overview of the CDX architecture:
> + *
> + *          +--------------------------------------+
> + *          |    Application CPUs (APU)            |
> + *          |                                      |
> + *          |                    CDX device drivers|
> + *          |     Linux OS                |        |
> + *          |                        CDX bus       |
> + *          |                             |        |
> + *          |                     CDX controller   |
> + *          |                             |        |
> + *          +-----------------------------|--------+
> + *                                        | (discover, config,
> + *                                        |  reset, rescan)
> + *                                        |
> + *          +------------------------| RPU if |----+
> + *          |                             |        |
> + *          |                             V        |
> + *          |          Realtime CPUs (RPU)         |
> + *          |                                      |
> + *          +--------------------------------------+
> + *                                |
> + *          +---------------------|----------------+
> + *          |  FPGA               |                |
> + *          |      +-----------------------+       |
> + *          |      |           |           |       |
> + *          | +-------+    +-------+   +-------+   |
> + *          | | dev 1 |    | dev 2 |   | dev 3 |   |
> + *          | +-------+    +-------+   +-------+   |
> + *          +--------------------------------------+
> + *
> + * The RPU firmware extracts the device information from the loaded FPGA
> + * image and implements a mechanism that allows the APU drivers to
> + * enumerate such devices (device personality and resource details) via
> + * a dedicated communication channel. RPU mediates operations such as
> + * discover, reset and rescan of the FPGA devices for the APU. This is
> + * done using memory mapped interface provided by the RPU to APU.
> + */
> +
> +#include <linux/init.h>
> +#include <linux/kernel.h>
> +#include <linux/slab.h>
> +#include <linux/cdx/cdx_bus.h>
> +
> +#include "cdx.h"
> +
> +/*
> + * Default DMA mask for devices on a CDX bus
> + */
> +#define CDX_DEFAULT_DMA_MASK	(~0ULL)
> +
> +static struct cdx_controller_t *cdx_controller;

You don't get a static device, sorry, everything must be dynamic or your
design is broken.

And remove the crazy "_t" from your structure name, checkpatch should
have complained about that.



> +
> +static int reset_cdx_device(struct device *dev, void * __always_unused data)

__always_unused is never needed.

But the funny thing is, you added that variable, why are you not using
it?  If it's not used, then don't have it.

Try to get others at AMD to review your code first, before sending your
next batch out, this is really really odd...

greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v4 2/8] bus/cdx: add the cdx bus driver

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Greg KH <gregkh@linuxfoundation.org>
> Sent: Friday, October 14, 2022 12:48 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org; rafael@kernel.org;
> eric.auger@redhat.com; alex.williamson@redhat.com; cohuck@redhat.com;
> Gupta, Puneet (DCG-ENG) <puneet.gupta@amd.com>;
> song.bao.hua@hisilicon.com; mchehab+huawei@kernel.org; maz@kernel.org;
> f.fainelli@gmail.com; jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com; will@kernel.org;
> joro@8bytes.org; masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v4 2/8] bus/cdx: add the cdx bus driver
> 
> Caution: This message originated from an External Source. Use proper caution
> when opening attachments, clicking links, or responding.
> 
> 
> On Fri, Oct 14, 2022 at 10:10:43AM +0530, Nipun Gupta wrote:
> > --- a/drivers/bus/Makefile
> > +++ b/drivers/bus/Makefile
> > @@ -20,6 +20,9 @@ obj-$(CONFIG_INTEL_IXP4XX_EB)       += intel-ixp4xx-
> eb.o
> >  obj-$(CONFIG_MIPS_CDMM)              += mips_cdmm.o
> >  obj-$(CONFIG_MVEBU_MBUS)     += mvebu-mbus.o
> >
> > +#CDX bus
> 
> No need for a comment like this :)

Sure, will remove.

> 
> > +obj-$(CONFIG_CDX_BUS)                += cdx/
> > +
> >  # Interconnect bus driver for OMAP SoCs.
> >  obj-$(CONFIG_OMAP_INTERCONNECT)      += omap_l3_smx.o omap_l3_noc.o
> >
> > diff --git a/drivers/bus/cdx/Kconfig b/drivers/bus/cdx/Kconfig
> > new file mode 100644
> > index 000000000000..98ec05ad708d
> > --- /dev/null
> > +++ b/drivers/bus/cdx/Kconfig
> > @@ -0,0 +1,13 @@
> > +# SPDX-License-Identifier: GPL-2.0
> > +#
> > +# CDX bus configuration
> > +#
> > +# Copyright (C) 2022, Advanced Micro Devices, Inc.
> > +#
> > +
> > +config CDX_BUS
> > +     bool "CDX Bus driver"
> > +     help
> > +             Driver to enable CDX Bus infrastructure. CDX bus uses
> > +             CDX controller and firmware to scan the FPGA based
> > +             devices.
> 
> Why bool?  Not as a module?  That's broken.

Okay. We see that most of other busses like amba, fslmc, pci are bool only.
As we have MSI domain as part of this bus, we need to export multiple
symbols to make it work as a module.

> 
> And "FPGA based devices" means nothing to me, please expand on what this
> all is as it is not descriptive at all.
> 
> Also list the module name please.
> 
> 
> 
> > diff --git a/drivers/bus/cdx/Makefile b/drivers/bus/cdx/Makefile
> > new file mode 100644
> > index 000000000000..2e8f42611dfc
> > --- /dev/null
> > +++ b/drivers/bus/cdx/Makefile
> > @@ -0,0 +1,8 @@
> > +# SPDX-License-Identifier: GPL-2.0
> > +#
> > +# Makefile for CDX
> > +#
> > +# Copyright (C) 2022, Advanced Micro Devices, Inc.
> > +#
> > +
> > +obj-$(CONFIG_CDX_BUS) += cdx.o
> > diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
> > new file mode 100644
> > index 000000000000..5a366f4ae69c
> > --- /dev/null
> > +++ b/drivers/bus/cdx/cdx.c
> > @@ -0,0 +1,366 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * CDX bus driver.
> > + *
> > + * Copyright (C) 2022, Advanced Micro Devices, Inc.
> > + */
> > +
> > +/*
> > + * Architecture Overview
> > + * =====================
> > + * CDX is a Hardware Architecture designed for AMD FPGA devices. It
> > + * consists of sophisticated mechanism for interaction between FPGA,
> > + * Firmware and the APUs (Application CPUs).
> > + *
> > + * Firmware resides on RPU (Realtime CPUs) which interacts with
> > + * the FPGA program manager and the APUs. The RPU provides memory-
> mapped
> > + * interface (RPU if) which is used to communicate with APUs.
> > + *
> > + * The diagram below shows an overview of the CDX architecture:
> > + *
> > + *          +--------------------------------------+
> > + *          |    Application CPUs (APU)            |
> > + *          |                                      |
> > + *          |                    CDX device drivers|
> > + *          |     Linux OS                |        |
> > + *          |                        CDX bus       |
> > + *          |                             |        |
> > + *          |                     CDX controller   |
> > + *          |                             |        |
> > + *          +-----------------------------|--------+
> > + *                                        | (discover, config,
> > + *                                        |  reset, rescan)
> > + *                                        |
> > + *          +------------------------| RPU if |----+
> > + *          |                             |        |
> > + *          |                             V        |
> > + *          |          Realtime CPUs (RPU)         |
> > + *          |                                      |
> > + *          +--------------------------------------+
> > + *                                |
> > + *          +---------------------|----------------+
> > + *          |  FPGA               |                |
> > + *          |      +-----------------------+       |
> > + *          |      |           |           |       |
> > + *          | +-------+    +-------+   +-------+   |
> > + *          | | dev 1 |    | dev 2 |   | dev 3 |   |
> > + *          | +-------+    +-------+   +-------+   |
> > + *          +--------------------------------------+
> > + *
> > + * The RPU firmware extracts the device information from the loaded FPGA
> > + * image and implements a mechanism that allows the APU drivers to
> > + * enumerate such devices (device personality and resource details) via
> > + * a dedicated communication channel. RPU mediates operations such as
> > + * discover, reset and rescan of the FPGA devices for the APU. This is
> > + * done using memory mapped interface provided by the RPU to APU.
> > + */
> > +
> > +#include <linux/init.h>
> > +#include <linux/kernel.h>
> > +#include <linux/slab.h>
> > +#include <linux/cdx/cdx_bus.h>
> > +
> > +#include "cdx.h"
> > +
> > +/*
> > + * Default DMA mask for devices on a CDX bus
> > + */
> > +#define CDX_DEFAULT_DMA_MASK (~0ULL)
> > +
> > +static struct cdx_controller_t *cdx_controller;
> 
> You don't get a static device, sorry, everything must be dynamic or your
> design is broken.

We will remove this static device here.

> 
> And remove the crazy "_t" from your structure name, checkpatch should
> have complained about that.

Sure. Will remove from other places too.

> 
> 
> 
> > +
> > +static int reset_cdx_device(struct device *dev, void * __always_unused data)
> 
> __always_unused is never needed.
> 
> But the funny thing is, you added that variable, why are you not using
> it?  If it's not used, then don't have it.

The 'data' parameter is here because the same reset_cdx_device() API is later used in
device attributes patch where it is passed as an argument to bus_for_each_dev API,
which needs the function to have these two arguments. Agree, that this parameter
should be added as part of the device attribute patch, will be updated for next spin.

Thanks,
Nipun

> 
> Try to get others at AMD to review your code first, before sending your
> next batch out, this is really really odd...
> 
> greg k-h

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 3/8] iommu/arm-smmu-v3: support ops registration for CDX bus

[Nipun Gupta]

With new CDX bus supported for AMD FPGA devices on ARM
platform, the bus requires registration for the SMMU v3
driver.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index d32b02336411..8ec9f2baf12d 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -29,6 +29,7 @@
 #include <linux/platform_device.h>
 
 #include <linux/amba/bus.h>
+#include <linux/cdx/cdx_bus.h>
 
 #include "arm-smmu-v3.h"
 #include "../../iommu-sva-lib.h"
@@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct iommu_ops *ops)
 		if (err)
 			goto err_reset_pci_ops;
 	}
+#endif
+#ifdef CONFIG_CDX_BUS
+	if (cdx_bus_type.iommu_ops != ops) {
+		err = bus_set_iommu(&cdx_bus_type, ops);
+		if (err)
+			goto err_reset_amba_ops;
+	}
 #endif
 	if (platform_bus_type.iommu_ops != ops) {
 		err = bus_set_iommu(&platform_bus_type, ops);
 		if (err)
-			goto err_reset_amba_ops;
+			goto err_reset_cdx_ops;
 	}
 
 	return 0;
 
-err_reset_amba_ops:
+err_reset_cdx_ops:
+#ifdef CONFIG_CDX_BUS
+	bus_set_iommu(&cdx_bus_type, NULL);
+#endif
+err_reset_amba_ops: __maybe_unused;
 #ifdef CONFIG_ARM_AMBA
 	bus_set_iommu(&amba_bustype, NULL);
 #endif
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v4 3/8] iommu/arm-smmu-v3: support ops registration for CDX bus

[Gupta, Nipun]

[AMD Official Use Only - General]

This patch will be rebased on top of https://lore.kernel.org/linux-iommu/cover.1660572783.git.robin.murphy@arm.com/T/#t
in next respin.

Regards,
Nipun

> -----Original Message-----
> From: Nipun Gupta <nipun.gupta@amd.com>
> Sent: Friday, October 14, 2022 10:11 AM
> To: robh+dt@kernel.org; krzysztof.kozlowski+dt@linaro.org;
> gregkh@linuxfoundation.org; rafael@kernel.org; eric.auger@redhat.com;
> alex.williamson@redhat.com; cohuck@redhat.com; Gupta, Puneet (DCG-ENG)
> <puneet.gupta@amd.com>; song.bao.hua@hisilicon.com;
> mchehab+huawei@kernel.org; maz@kernel.org; f.fainelli@gmail.com;
> jeffrey.l.hugo@gmail.com; saravanak@google.com; Michael.Srba@seznam.cz;
> mani@kernel.org; yishaih@nvidia.com; jgg@ziepe.ca; jgg@nvidia.com;
> robin.murphy@arm.com; will@kernel.org; joro@8bytes.org;
> masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org
> Cc: okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>; Gupta, Nipun <Nipun.Gupta@amd.com>
> Subject: [RFC PATCH v4 3/8] iommu/arm-smmu-v3: support ops registration for
> CDX bus
> 
> With new CDX bus supported for AMD FPGA devices on ARM
> platform, the bus requires registration for the SMMU v3
> driver.
> 
> Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
> ---
>  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 16 ++++++++++++++--
>  1 file changed, 14 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> index d32b02336411..8ec9f2baf12d 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> @@ -29,6 +29,7 @@
>  #include <linux/platform_device.h>
> 
>  #include <linux/amba/bus.h>
> +#include <linux/cdx/cdx_bus.h>
> 
>  #include "arm-smmu-v3.h"
>  #include "../../iommu-sva-lib.h"
> @@ -3690,16 +3691,27 @@ static int arm_smmu_set_bus_ops(struct
> iommu_ops *ops)
>  		if (err)
>  			goto err_reset_pci_ops;
>  	}
> +#endif
> +#ifdef CONFIG_CDX_BUS
> +	if (cdx_bus_type.iommu_ops != ops) {
> +		err = bus_set_iommu(&cdx_bus_type, ops);
> +		if (err)
> +			goto err_reset_amba_ops;
> +	}
>  #endif
>  	if (platform_bus_type.iommu_ops != ops) {
>  		err = bus_set_iommu(&platform_bus_type, ops);
>  		if (err)
> -			goto err_reset_amba_ops;
> +			goto err_reset_cdx_ops;
>  	}
> 
>  	return 0;
> 
> -err_reset_amba_ops:
> +err_reset_cdx_ops:
> +#ifdef CONFIG_CDX_BUS
> +	bus_set_iommu(&cdx_bus_type, NULL);
> +#endif
> +err_reset_amba_ops: __maybe_unused;
>  #ifdef CONFIG_ARM_AMBA
>  	bus_set_iommu(&amba_bustype, NULL);
>  #endif
> --
> 2.25.1

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 4/8] bux/cdx: support dma configuration for CDX devices

[Nipun Gupta]

With CDX bus supported in SMMU, devices on CDX bus can be
configured to support DMA configuration. This change adds
the dma configure callback for CDX bus.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 drivers/bus/cdx/cdx.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
index 5a366f4ae69c..b2a7e0b34df8 100644
--- a/drivers/bus/cdx/cdx.c
+++ b/drivers/bus/cdx/cdx.c
@@ -57,6 +57,7 @@
 
 #include <linux/init.h>
 #include <linux/kernel.h>
+#include <linux/of_device.h>
 #include <linux/slab.h>
 #include <linux/cdx/cdx_bus.h>
 
@@ -180,10 +181,26 @@ static void cdx_remove(struct device *dev)
 		cdx_drv->remove(cdx_dev);
 }
 
+static int cdx_dma_configure(struct device *dev)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	u32 input_id = cdx_dev->req_id;
+	int ret;
+
+	ret = of_dma_configure_id(dev, dev->parent->of_node, 0, &input_id);
+	if (ret) {
+		dev_err(dev, "of_dma_configure_id() failed\n");
+		return ret;
+	}
+
+	return 0;
+}
+
 struct bus_type cdx_bus_type = {
 	.name		= "cdx",
 	.match		= cdx_bus_match,
 	.remove		= cdx_remove,
+	.dma_configure  = cdx_dma_configure,
 };
 EXPORT_SYMBOL_GPL(cdx_bus_type);
 
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 5/8] bus/cdx: add bus and device attributes

[Nipun Gupta]

This change adds te support for rescanning and reset
of the CDX buses, as well as option to reset any device
on the bus. It also enables sysfs entry for vendor id
and device id.

Sysfs entries are provided in CDX controller:
- rescan of the CDX controller.
- reset all the devices present on CDX buses.

Sysfs entry is provided in each of the platform device
detected by the CDX controller
- vendor id
- device id
- modalias
- reset of the device.

Signed-off-by: Puneet Gupta <puneet.gupta@amd.com>
Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 drivers/bus/cdx/cdx.c | 158 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)

diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
index b2a7e0b34df8..41c61bf6d5f0 100644
--- a/drivers/bus/cdx/cdx.c
+++ b/drivers/bus/cdx/cdx.c
@@ -107,6 +107,12 @@ static int cdx_unregister_device(struct device *dev,
 	return 0;
 }
 
+static void cdx_unregister_devices(struct bus_type *bus)
+{
+	/* Reset all the devices attached to cdx bus */
+	bus_for_each_dev(bus, NULL, NULL, cdx_unregister_device);
+}
+
 /**
  * cdx_match_one_device - Tell if a CDX device structure has a matching
  *			  CDX device id structure
@@ -196,11 +202,163 @@ static int cdx_dma_configure(struct device *dev)
 	return 0;
 }
 
+static ssize_t vendor_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return sprintf(buf, "0x%x\n", cdx_dev->vendor);
+}
+static DEVICE_ATTR_RO(vendor);
+
+static ssize_t device_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return sprintf(buf, "0x%x\n", cdx_dev->device);
+}
+static DEVICE_ATTR_RO(device);
+
+static ssize_t reset_store(struct device *dev, struct device_attribute *attr,
+			   const char *buf, size_t count)
+{
+	int ret = 0;
+	bool reset = count > 0 && *buf != '0';
+
+	if (!reset)
+		return count;
+
+	ret = reset_cdx_device(dev, NULL);
+	if (ret)
+		return ret;
+
+	return count;
+}
+static DEVICE_ATTR_WO(reset);
+
+static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return sprintf(buf, "cdx:v%08Xd%d\n", cdx_dev->vendor,
+			cdx_dev->device);
+}
+static DEVICE_ATTR_RO(modalias);
+
+static ssize_t driver_override_store(struct device *dev,
+				     struct device_attribute *attr,
+				     const char *buf, size_t count)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	const char *old = cdx_dev->driver_override;
+	char *driver_override;
+	char *cp;
+
+	if (WARN_ON(dev->bus != &cdx_bus_type))
+		return -EINVAL;
+
+	if (count >= (PAGE_SIZE - 1))
+		return -EINVAL;
+
+	driver_override = kstrndup(buf, count, GFP_KERNEL);
+	if (!driver_override)
+		return -ENOMEM;
+
+	cp = strchr(driver_override, '\n');
+	if (cp)
+		*cp = '\0';
+
+	if (strlen(driver_override)) {
+		cdx_dev->driver_override = driver_override;
+	} else {
+		kfree(driver_override);
+		cdx_dev->driver_override = NULL;
+	}
+
+	kfree(old);
+
+	return count;
+}
+
+static ssize_t driver_override_show(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+
+	return snprintf(buf, PAGE_SIZE, "%s\n", cdx_dev->driver_override);
+}
+static DEVICE_ATTR_RW(driver_override);
+
+static struct attribute *cdx_dev_attrs[] = {
+	&dev_attr_reset.attr,
+	&dev_attr_vendor.attr,
+	&dev_attr_device.attr,
+	&dev_attr_modalias.attr,
+	&dev_attr_driver_override.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(cdx_dev);
+
+static ssize_t rescan_store(struct bus_type *bus,
+			    const char *buf, size_t count)
+{
+	bool rescan = count > 0 && *buf != '0';
+	int ret = 0;
+
+	if (!rescan)
+		return count;
+
+	if (!cdx_controller)
+		return -EINVAL;
+
+	/* Unregister all the devices on the bus */
+	cdx_unregister_devices(&cdx_bus_type);
+
+	/* Rescan all the devices */
+	ret = cdx_controller->ops.scan(cdx_controller);
+	if (ret)
+		return ret;
+
+	return count;
+}
+static BUS_ATTR_WO(rescan);
+
+static ssize_t reset_all_store(struct bus_type *bus,
+			       const char *buf, size_t count)
+{
+	bool reset = count > 0 && *buf != '0';
+	int ret = 0;
+
+	if (!reset)
+		return count;
+
+	/* Reset all the devices attached to cdx bus */
+	ret = bus_for_each_dev(bus, NULL, NULL, reset_cdx_device);
+	if (ret) {
+		pr_err("error in CDX bus reset\n");
+		return 0;
+	}
+
+	return count;
+}
+static BUS_ATTR_WO(reset_all);
+
+static struct attribute *cdx_bus_attrs[] = {
+	&bus_attr_rescan.attr,
+	&bus_attr_reset_all.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(cdx_bus);
+
 struct bus_type cdx_bus_type = {
 	.name		= "cdx",
 	.match		= cdx_bus_match,
 	.remove		= cdx_remove,
 	.dma_configure  = cdx_dma_configure,
+	.dev_groups	= cdx_dev_groups,
+	.bus_groups	= cdx_bus_groups,
 };
 EXPORT_SYMBOL_GPL(cdx_bus_type);
 
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 6/8] irq/msi: use implicit msi domain for alloc and free

[Nipun Gupta]

The domain can be extracted out of the device rather than
passing it explicitly in the msi domain alloc and free APIs.
So this change removes taking irq domain input parameter for
these APIs

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 drivers/base/platform-msi.c      |  4 ++--
 drivers/bus/fsl-mc/fsl-mc-msi.c  |  4 ++--
 drivers/pci/msi/irqdomain.c      |  4 ++--
 drivers/soc/ti/ti_sci_inta_msi.c |  2 +-
 include/linux/msi.h              | 10 ++++------
 kernel/irq/msi.c                 | 21 +++++++++------------
 6 files changed, 20 insertions(+), 25 deletions(-)

diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index 296ea673d661..62664a207230 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -212,7 +212,7 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec,
 	if (err)
 		return err;
 
-	err = msi_domain_alloc_irqs(dev->msi.domain, dev, nvec);
+	err = msi_domain_alloc_irqs(dev, nvec);
 	if (err)
 		platform_msi_free_priv_data(dev);
 
@@ -226,7 +226,7 @@ EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs);
  */
 void platform_msi_domain_free_irqs(struct device *dev)
 {
-	msi_domain_free_irqs(dev->msi.domain, dev);
+	msi_domain_free_irqs(dev);
 	platform_msi_free_priv_data(dev);
 }
 EXPORT_SYMBOL_GPL(platform_msi_domain_free_irqs);
diff --git a/drivers/bus/fsl-mc/fsl-mc-msi.c b/drivers/bus/fsl-mc/fsl-mc-msi.c
index 0cfe859a4ac4..0522e80a34ac 100644
--- a/drivers/bus/fsl-mc/fsl-mc-msi.c
+++ b/drivers/bus/fsl-mc/fsl-mc-msi.c
@@ -235,7 +235,7 @@ int fsl_mc_msi_domain_alloc_irqs(struct device *dev,  unsigned int irq_count)
 	 * NOTE: Calling this function will trigger the invocation of the
 	 * its_fsl_mc_msi_prepare() callback
 	 */
-	error = msi_domain_alloc_irqs(msi_domain, dev, irq_count);
+	error = msi_domain_alloc_irqs(dev, irq_count);
 
 	if (error)
 		dev_err(dev, "Failed to allocate IRQs\n");
@@ -250,5 +250,5 @@ void fsl_mc_msi_domain_free_irqs(struct device *dev)
 	if (!msi_domain)
 		return;
 
-	msi_domain_free_irqs(msi_domain, dev);
+	msi_domain_free_irqs(dev);
 }
diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c
index e9cf318e6670..00ac782afd1d 100644
--- a/drivers/pci/msi/irqdomain.c
+++ b/drivers/pci/msi/irqdomain.c
@@ -14,7 +14,7 @@ int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
 
 	domain = dev_get_msi_domain(&dev->dev);
 	if (domain && irq_domain_is_hierarchy(domain))
-		return msi_domain_alloc_irqs_descs_locked(domain, &dev->dev, nvec);
+		return msi_domain_alloc_irqs_descs_locked(&dev->dev, nvec);
 
 	return pci_msi_legacy_setup_msi_irqs(dev, nvec, type);
 }
@@ -25,7 +25,7 @@ void pci_msi_teardown_msi_irqs(struct pci_dev *dev)
 
 	domain = dev_get_msi_domain(&dev->dev);
 	if (domain && irq_domain_is_hierarchy(domain))
-		msi_domain_free_irqs_descs_locked(domain, &dev->dev);
+		msi_domain_free_irqs_descs_locked(&dev->dev);
 	else
 		pci_msi_legacy_teardown_msi_irqs(dev);
 	msi_free_msi_descs(&dev->dev);
diff --git a/drivers/soc/ti/ti_sci_inta_msi.c b/drivers/soc/ti/ti_sci_inta_msi.c
index 991c78b34745..ae10b88d1ff0 100644
--- a/drivers/soc/ti/ti_sci_inta_msi.c
+++ b/drivers/soc/ti/ti_sci_inta_msi.c
@@ -114,7 +114,7 @@ int ti_sci_inta_msi_domain_alloc_irqs(struct device *dev,
 		goto unlock;
 	}
 
-	ret = msi_domain_alloc_irqs_descs_locked(msi_domain, dev, nvec);
+	ret = msi_domain_alloc_irqs_descs_locked(dev, nvec);
 	if (ret)
 		dev_err(dev, "Failed to allocate IRQs %d\n", ret);
 unlock:
diff --git a/include/linux/msi.h b/include/linux/msi.h
index fc918a658d48..76f997bdcb52 100644
--- a/include/linux/msi.h
+++ b/include/linux/msi.h
@@ -428,13 +428,11 @@ struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode,
 					 struct irq_domain *parent);
 int __msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
 			    int nvec);
-int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev,
-				       int nvec);
-int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev,
-			  int nvec);
+int msi_domain_alloc_irqs_descs_locked(struct device *dev, int nvec);
+int msi_domain_alloc_irqs(struct device *dev, int nvec);
 void __msi_domain_free_irqs(struct irq_domain *domain, struct device *dev);
-void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev);
-void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev);
+void msi_domain_free_irqs_descs_locked(struct device *dev);
+void msi_domain_free_irqs(struct device *dev);
 struct msi_domain_info *msi_get_domain_info(struct irq_domain *domain);
 
 struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode,
diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c
index a9ee535293eb..14983c82a9e3 100644
--- a/kernel/irq/msi.c
+++ b/kernel/irq/msi.c
@@ -925,7 +925,6 @@ static int msi_domain_add_simple_msi_descs(struct msi_domain_info *info,
 
 /**
  * msi_domain_alloc_irqs_descs_locked - Allocate interrupts from a MSI interrupt domain
- * @domain:	The domain to allocate from
  * @dev:	Pointer to device struct of the device for which the interrupts
  *		are allocated
  * @nvec:	The number of interrupts to allocate
@@ -936,9 +935,9 @@ static int msi_domain_add_simple_msi_descs(struct msi_domain_info *info,
  *
  * Return: %0 on success or an error code.
  */
-int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device *dev,
-				       int nvec)
+int msi_domain_alloc_irqs_descs_locked(struct device *dev, int nvec)
 {
+	struct irq_domain *domain = dev_get_msi_domain(dev);
 	struct msi_domain_info *info = domain->host_data;
 	struct msi_domain_ops *ops = info->ops;
 	int ret;
@@ -951,25 +950,24 @@ int msi_domain_alloc_irqs_descs_locked(struct irq_domain *domain, struct device
 
 	ret = ops->domain_alloc_irqs(domain, dev, nvec);
 	if (ret)
-		msi_domain_free_irqs_descs_locked(domain, dev);
+		msi_domain_free_irqs_descs_locked(dev);
 	return ret;
 }
 
 /**
  * msi_domain_alloc_irqs - Allocate interrupts from a MSI interrupt domain
- * @domain:	The domain to allocate from
  * @dev:	Pointer to device struct of the device for which the interrupts
  *		are allocated
  * @nvec:	The number of interrupts to allocate
  *
  * Return: %0 on success or an error code.
  */
-int msi_domain_alloc_irqs(struct irq_domain *domain, struct device *dev, int nvec)
+int msi_domain_alloc_irqs(struct device *dev, int nvec)
 {
 	int ret;
 
 	msi_lock_descs(dev);
-	ret = msi_domain_alloc_irqs_descs_locked(domain, dev, nvec);
+	ret = msi_domain_alloc_irqs_descs_locked(dev, nvec);
 	msi_unlock_descs(dev);
 	return ret;
 }
@@ -1006,7 +1004,6 @@ static void msi_domain_free_msi_descs(struct msi_domain_info *info,
 
 /**
  * msi_domain_free_irqs_descs_locked - Free interrupts from a MSI interrupt @domain associated to @dev
- * @domain:	The domain to managing the interrupts
  * @dev:	Pointer to device struct of the device for which the interrupts
  *		are free
  *
@@ -1014,8 +1011,9 @@ static void msi_domain_free_msi_descs(struct msi_domain_info *info,
  * pair. Use this for MSI irqdomains which implement their own vector
  * allocation.
  */
-void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device *dev)
+void msi_domain_free_irqs_descs_locked(struct device *dev)
 {
+	struct irq_domain *domain = dev_get_msi_domain(dev);
 	struct msi_domain_info *info = domain->host_data;
 	struct msi_domain_ops *ops = info->ops;
 
@@ -1027,14 +1025,13 @@ void msi_domain_free_irqs_descs_locked(struct irq_domain *domain, struct device
 
 /**
  * msi_domain_free_irqs - Free interrupts from a MSI interrupt @domain associated to @dev
- * @domain:	The domain to managing the interrupts
  * @dev:	Pointer to device struct of the device for which the interrupts
  *		are free
  */
-void msi_domain_free_irqs(struct irq_domain *domain, struct device *dev)
+void msi_domain_free_irqs(struct device *dev)
 {
 	msi_lock_descs(dev);
-	msi_domain_free_irqs_descs_locked(domain, dev);
+	msi_domain_free_irqs_descs_locked(dev);
 	msi_unlock_descs(dev);
 }
 
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 7/8] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Nipun Gupta]

Since CDX devices are not linked to of node they need a
separate MSI domain for handling device ID to be provided to
the GIC ITS domain.

This also introduces APIs to alloc and free IRQs for CDX domain.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
Signed-off-by: Nikhil Agarwal <nikhil.agarwal@amd.com>
---
 drivers/bus/cdx/Kconfig     |   1 +
 drivers/bus/cdx/Makefile    |   2 +-
 drivers/bus/cdx/cdx.c       |  18 ++++
 drivers/bus/cdx/cdx.h       |  10 +++
 drivers/bus/cdx/cdx_msi.c   | 161 ++++++++++++++++++++++++++++++++++++
 include/linux/cdx/cdx_bus.h |  26 ++++++
 kernel/irq/msi.c            |   1 +
 7 files changed, 218 insertions(+), 1 deletion(-)
 create mode 100644 drivers/bus/cdx/cdx_msi.c

diff --git a/drivers/bus/cdx/Kconfig b/drivers/bus/cdx/Kconfig
index 98ec05ad708d..062443080d6f 100644
--- a/drivers/bus/cdx/Kconfig
+++ b/drivers/bus/cdx/Kconfig
@@ -7,6 +7,7 @@
 
 config CDX_BUS
 	bool "CDX Bus driver"
+	select GENERIC_MSI_IRQ_DOMAIN
 	help
 		Driver to enable CDX Bus infrastructure. CDX bus uses
 		CDX controller and firmware to scan the FPGA based
diff --git a/drivers/bus/cdx/Makefile b/drivers/bus/cdx/Makefile
index 2e8f42611dfc..95f8fa3b4db8 100644
--- a/drivers/bus/cdx/Makefile
+++ b/drivers/bus/cdx/Makefile
@@ -5,4 +5,4 @@
 # Copyright (C) 2022, Advanced Micro Devices, Inc.
 #
 
-obj-$(CONFIG_CDX_BUS) += cdx.o
+obj-$(CONFIG_CDX_BUS) += cdx.o cdx_msi.o
diff --git a/drivers/bus/cdx/cdx.c b/drivers/bus/cdx/cdx.c
index 41c61bf6d5f0..a3accde0421f 100644
--- a/drivers/bus/cdx/cdx.c
+++ b/drivers/bus/cdx/cdx.c
@@ -56,6 +56,7 @@
  */
 
 #include <linux/init.h>
+#include <linux/irqdomain.h>
 #include <linux/kernel.h>
 #include <linux/of_device.h>
 #include <linux/slab.h>
@@ -449,6 +450,7 @@ int cdx_device_add(struct cdx_dev_params_t *dev_params)
 	struct cdx_controller_t *cdx = dev_params->cdx;
 	struct device *parent = cdx->dev;
 	struct cdx_device *cdx_dev;
+	struct irq_domain *cdx_msi_domain;
 	int ret;
 
 	cdx_dev = kzalloc(sizeof(*cdx_dev), GFP_KERNEL);
@@ -465,6 +467,7 @@ int cdx_device_add(struct cdx_dev_params_t *dev_params)
 
 	/* Populate CDX dev params */
 	cdx_dev->req_id = dev_params->req_id;
+	cdx_dev->num_msi = dev_params->num_msi;
 	cdx_dev->vendor = dev_params->vendor;
 	cdx_dev->device = dev_params->device;
 	cdx_dev->bus_num = dev_params->bus_num;
@@ -483,6 +486,21 @@ int cdx_device_add(struct cdx_dev_params_t *dev_params)
 	dev_set_name(&cdx_dev->dev, "cdx-%02x:%02x", cdx_dev->bus_num,
 			cdx_dev->dev_num);
 
+	/* If CDX MSI domain is not created, create one. */
+	cdx_msi_domain = irq_find_host(parent->of_node);
+	if (!cdx_msi_domain) {
+		cdx_msi_domain = cdx_msi_domain_init(parent);
+		if (!cdx_msi_domain) {
+			dev_err(&cdx_dev->dev,
+				"cdx_msi_domain_init() failed: %d", ret);
+			kfree(cdx_dev);
+			return -ENODEV;
+		}
+	}
+
+	/* Set the MSI domain */
+	dev_set_msi_domain(&cdx_dev->dev, cdx_msi_domain);
+
 	ret = device_add(&cdx_dev->dev);
 	if (ret != 0) {
 		dev_err(&cdx_dev->dev,
diff --git a/drivers/bus/cdx/cdx.h b/drivers/bus/cdx/cdx.h
index 80496865ae9c..5fd40c7e633e 100644
--- a/drivers/bus/cdx/cdx.h
+++ b/drivers/bus/cdx/cdx.h
@@ -23,6 +23,7 @@
  * @res: array of MMIO region entries
  * @res_count: number of valid MMIO regions
  * @req_id: Requestor ID associated with CDX device
+ * @num_msi: Number of MSI's supported by the device
  */
 struct cdx_dev_params_t {
 	struct cdx_controller_t *cdx;
@@ -35,6 +36,7 @@ struct cdx_dev_params_t {
 	struct resource res[MAX_CDX_DEV_RESOURCES];
 	u8 res_count;
 	u32 req_id;
+	u32 num_msi;
 };
 
 /**
@@ -63,4 +65,12 @@ void cdx_unregister_controller(struct cdx_controller_t *cdx);
  */
 int cdx_device_add(struct cdx_dev_params_t *dev_params);
 
+/**
+ * cdx_msi_domain_init - Init the CDX bus MSI domain.
+ * @dev: Device of the CDX bus controller
+ *
+ * Return CDX MSI domain, NULL on failure
+ */
+struct irq_domain *cdx_msi_domain_init(struct device *dev);
+
 #endif /* _CDX_H_ */
diff --git a/drivers/bus/cdx/cdx_msi.c b/drivers/bus/cdx/cdx_msi.c
new file mode 100644
index 000000000000..b9e7e9d6fb51
--- /dev/null
+++ b/drivers/bus/cdx/cdx_msi.c
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD CDX bus driver MSI support
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ *
+ */
+
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/msi.h>
+#include <linux/cdx/cdx_bus.h>
+
+#include "cdx.h"
+
+#define REQ_ID_SHIFT	10
+
+/*
+ * Convert an msi_desc to a globaly unique identifier (per-device
+ * reqid + msi_desc position in the msi_list).
+ */
+static irq_hw_number_t cdx_domain_calc_hwirq(struct cdx_device *dev,
+					     struct msi_desc *desc)
+{
+	return (dev->req_id << REQ_ID_SHIFT) | desc->msi_index;
+}
+
+static void cdx_msi_set_desc(msi_alloc_info_t *arg,
+			     struct msi_desc *desc)
+{
+	arg->desc = desc;
+	arg->hwirq = cdx_domain_calc_hwirq(to_cdx_device(desc->dev), desc);
+}
+
+static void cdx_msi_write_msg(struct irq_data *irq_data,
+			      struct msi_msg *msg)
+{
+	struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data);
+	struct cdx_device *cdx_dev = to_cdx_device(msi_desc->dev);
+	struct cdx_controller_t *cdx = cdx_dev->cdx;
+	uint64_t addr;
+	int ret;
+
+	addr = ((uint64_t)(msi_desc->msg.address_hi) << 32) |
+	       msi_desc->msg.address_lo;
+
+	ret = cdx->ops.write_msi(cdx, cdx_dev->bus_num, cdx_dev->dev_num,
+				 msi_desc->msi_index, msi_desc->msg.data,
+				 addr);
+	if (ret)
+		dev_err(&cdx_dev->dev, "Write MSI failed to CDX controller\n");
+}
+
+int cdx_msi_domain_alloc_irqs(struct device *dev, unsigned int irq_count)
+{
+	int ret;
+
+	ret = msi_setup_device_data(dev);
+	if (ret)
+		return ret;
+
+	msi_lock_descs(dev);
+	if (msi_first_desc(dev, MSI_DESC_ALL))
+		ret = -EINVAL;
+	msi_unlock_descs(dev);
+	if (ret)
+		return ret;
+
+	ret = msi_domain_alloc_irqs(dev, irq_count);
+	if (ret)
+		dev_err(dev, "Failed to allocate IRQs\n");
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(cdx_msi_domain_alloc_irqs);
+
+static struct irq_chip cdx_msi_irq_chip = {
+	.name			= "CDX-MSI",
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_set_affinity	= msi_domain_set_affinity,
+	.irq_write_msi_msg	= cdx_msi_write_msg
+};
+
+static int cdx_msi_prepare(struct irq_domain *msi_domain,
+			   struct device *dev,
+			   int nvec, msi_alloc_info_t *info)
+{
+	struct cdx_device *cdx_dev = to_cdx_device(dev);
+	struct msi_domain_info *msi_info;
+	struct device *parent = dev->parent;
+	u32 dev_id;
+	int ret;
+
+	/* Retrieve device ID from requestor ID using parent device */
+	ret = of_map_id(parent->of_node, cdx_dev->req_id, "msi-map",
+			"msi-map-mask",	NULL, &dev_id);
+	if (ret) {
+		dev_err(dev, "of_map_id failed for MSI: %d\n", ret);
+		return ret;
+	}
+
+	/* Set the device Id to be passed to the GIC-ITS */
+	info->scratchpad[0].ul = dev_id;
+
+	msi_info = msi_get_domain_info(msi_domain->parent);
+
+	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
+}
+
+static struct msi_domain_ops cdx_msi_ops __ro_after_init = {
+	.msi_prepare	= cdx_msi_prepare,
+	.set_desc	= cdx_msi_set_desc
+};
+
+static struct msi_domain_info cdx_msi_domain_info = {
+	.ops	= &cdx_msi_ops,
+	.chip	= &cdx_msi_irq_chip,
+	.flags	= MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
+		  MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS
+};
+
+struct irq_domain *cdx_msi_domain_init(struct device *dev)
+{
+	struct irq_domain *parent;
+	struct irq_domain *cdx_msi_domain;
+	struct fwnode_handle *fwnode_handle;
+	struct device_node *parent_node;
+	struct device_node *np = dev->of_node;
+
+	fwnode_handle = of_node_to_fwnode(np);
+
+	parent_node = of_parse_phandle(np, "msi-map", 1);
+	if (!parent_node) {
+		dev_err(dev, "msi-map not present on cdx controller\n");
+		return NULL;
+	}
+
+	parent = irq_find_matching_fwnode(of_node_to_fwnode(parent_node),
+			DOMAIN_BUS_NEXUS);
+	if (!parent || !msi_get_domain_info(parent)) {
+		dev_err(dev, "unable to locate ITS domain\n");
+		return NULL;
+	}
+
+	cdx_msi_domain = msi_create_irq_domain(fwnode_handle,
+				&cdx_msi_domain_info, parent);
+	if (!cdx_msi_domain) {
+		dev_err(dev, "unable to create CDX-MSI domain\n");
+		return NULL;
+	}
+
+	dev_dbg(dev, "CDX-MSI domain created\n");
+
+	return cdx_msi_domain;
+}
diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h
index 9e6872a03215..dfc0bd289834 100644
--- a/include/linux/cdx/cdx_bus.h
+++ b/include/linux/cdx/cdx_bus.h
@@ -21,14 +21,21 @@ typedef int (*cdx_scan_t)(struct cdx_controller_t *cdx);
 typedef int (*cdx_dev_reset_t)(struct cdx_controller_t *cdx,
 			       uint8_t bus_num, uint8_t dev_num);
 
+typedef int (*cdx_write_msi_msg_t)(struct cdx_controller_t *cdx,
+				   uint8_t bus_num, uint8_t dev_num,
+				   uint16_t msi_index, uint32_t data,
+				   uint64_t addr);
+
 /**
  * Callbacks supported by CDX controller.
  * @scan: scan the devices on the controller
  * @reset_dev: reset a CDX device
+ * @write_msi: callback to write the MSI message
  */
 struct cdx_ops_t {
 	cdx_scan_t scan;
 	cdx_dev_reset_t reset_dev;
+	cdx_write_msi_msg_t write_msi;
 };
 
 /**
@@ -57,6 +64,7 @@ struct cdx_controller_t {
  * @dma_mask: Default DMA mask
  * @flags: CDX device flags
  * @req_id: Requestor ID associated with CDX device
+ * @num_msi: Number of MSI's supported by the device
  * @driver_override: driver name to force a match; do not set directly,
  *                   because core frees it; use driver_set_override() to
  *                   set or clear it.
@@ -73,6 +81,7 @@ struct cdx_device {
 	u64 dma_mask;
 	u16 flags;
 	u32 req_id;
+	u32 num_msi;
 	const char *driver_override;
 };
 
@@ -136,4 +145,21 @@ extern struct bus_type cdx_bus_type;
  */
 int cdx_dev_reset(struct device *dev);
 
+/**
+ * cdx_msi_domain_alloc_irqs - Allocate MSI's for the CDX device
+ * @dev: device pointer
+ * @irq_count: Number of MSI's to be allocated
+ *
+ * Return 0 for success, -errno on failure
+ */
+int cdx_msi_domain_alloc_irqs(struct device *dev, unsigned int irq_count);
+
+/**
+ * cdx_msi_domain_free_irqs - Free MSI's for CDX device
+ * @dev: device pointer
+ *
+ * Return 0 for success, -errno on failure
+ */
+#define cdx_msi_domain_free_irqs msi_domain_free_irqs
+
 #endif /* _CDX_BUS_H_ */
diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c
index 14983c82a9e3..f55a5e395686 100644
--- a/kernel/irq/msi.c
+++ b/kernel/irq/msi.c
@@ -1034,6 +1034,7 @@ void msi_domain_free_irqs(struct device *dev)
 	msi_domain_free_irqs_descs_locked(dev);
 	msi_unlock_descs(dev);
 }
+EXPORT_SYMBOL_GPL(msi_domain_free_irqs);
 
 /**
  * msi_get_domain_info - Get the MSI interrupt domain info for @domain
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v4 7/8] bus/cdx: add cdx-MSI domain with gic-its domain as parent

[Thomas Gleixner]

On Fri, Oct 14 2022 at 10:10, Nipun Gupta wrote:
> Since CDX devices are not linked to of node they need a
> separate MSI domain for handling device ID to be provided to
> the GIC ITS domain.
>
> This also introduces APIs to alloc and free IRQs for CDX domain.

This is not really much of an explanation.

> +#include "cdx.h"
> +
> +#define REQ_ID_SHIFT	10
> +
> +/*
> + * Convert an msi_desc to a globaly unique identifier (per-device
> + * reqid + msi_desc position in the msi_list).
> + */
> +static irq_hw_number_t cdx_domain_calc_hwirq(struct cdx_device *dev,
> +					     struct msi_desc *desc)
> +{
> +	return (dev->req_id << REQ_ID_SHIFT) | desc->msi_index;
> +}
> +
> +static void cdx_msi_set_desc(msi_alloc_info_t *arg,
> +			     struct msi_desc *desc)
> +{
> +	arg->desc = desc;
> +	arg->hwirq = cdx_domain_calc_hwirq(to_cdx_device(desc->dev), desc);
> +}
> +
> +static void cdx_msi_write_msg(struct irq_data *irq_data,
> +			      struct msi_msg *msg)
> +{
> +	struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data);
> +	struct cdx_device *cdx_dev = to_cdx_device(msi_desc->dev);
> +	struct cdx_controller_t *cdx = cdx_dev->cdx;
> +	uint64_t addr;
> +	int ret;
> +
> +	addr = ((uint64_t)(msi_desc->msg.address_hi) << 32) |
> +	       msi_desc->msg.address_lo;
> +
> +	ret = cdx->ops.write_msi(cdx, cdx_dev->bus_num, cdx_dev->dev_num,
> +				 msi_desc->msi_index, msi_desc->msg.data,
> +				 addr);
> +	if (ret)
> +		dev_err(&cdx_dev->dev, "Write MSI failed to CDX controller\n");
> +}

So this is yet another variant of platform-MSI which is about to be
kicked from the planet.

Please check:

     https://lore.kernel.org/r/20221111131813.914374272@linutronix.de
     https://lore.kernel.org/r/20221111133158.196269823@linutronix.de

and the ARM/GIC conversion in:

     https://git.kernel.org/pub/scm/linux/kernel/git/tglx/devel.git/log/?h=devmsi-arm

The latter is WIP, but functional.

I'm sure you replace your yet another magic MSI domain which only
provides a device specific MSI write function with the new
infrastructure trivially.

Thanks,

        tglx



_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[RFC PATCH v4 8/8] bus/cdx: add cdx controller

[Nipun Gupta]

Add driver for CDX controller. The controller discovers the
CDX devices with the help of firmware and add those devices
to the CDX bus.

CDX controller also provide ops to be registered to CDX
bus for scanning, resetting and writing MSI data to CDX
devices.

Signed-off-by: Nipun Gupta <nipun.gupta@amd.com>
---
 drivers/bus/cdx/Kconfig                     |   2 +
 drivers/bus/cdx/Makefile                    |   2 +-
 drivers/bus/cdx/controller/Kconfig          |  16 ++
 drivers/bus/cdx/controller/Makefile         |   8 +
 drivers/bus/cdx/controller/cdx_controller.c | 210 ++++++++++++++++++++
 drivers/bus/cdx/controller/mcdi_stubs.c     |  68 +++++++
 drivers/bus/cdx/controller/mcdi_stubs.h     | 101 ++++++++++
 7 files changed, 406 insertions(+), 1 deletion(-)
 create mode 100644 drivers/bus/cdx/controller/Kconfig
 create mode 100644 drivers/bus/cdx/controller/Makefile
 create mode 100644 drivers/bus/cdx/controller/cdx_controller.c
 create mode 100644 drivers/bus/cdx/controller/mcdi_stubs.c
 create mode 100644 drivers/bus/cdx/controller/mcdi_stubs.h

diff --git a/drivers/bus/cdx/Kconfig b/drivers/bus/cdx/Kconfig
index 062443080d6f..8036d754a0a6 100644
--- a/drivers/bus/cdx/Kconfig
+++ b/drivers/bus/cdx/Kconfig
@@ -12,3 +12,5 @@ config CDX_BUS
 		Driver to enable CDX Bus infrastructure. CDX bus uses
 		CDX controller and firmware to scan the FPGA based
 		devices.
+
+source "drivers/bus/cdx/controller/Kconfig"
diff --git a/drivers/bus/cdx/Makefile b/drivers/bus/cdx/Makefile
index 95f8fa3b4db8..285fd7d51174 100644
--- a/drivers/bus/cdx/Makefile
+++ b/drivers/bus/cdx/Makefile
@@ -5,4 +5,4 @@
 # Copyright (C) 2022, Advanced Micro Devices, Inc.
 #
 
-obj-$(CONFIG_CDX_BUS) += cdx.o cdx_msi.o
+obj-$(CONFIG_CDX_BUS) += cdx.o cdx_msi.o controller/
diff --git a/drivers/bus/cdx/controller/Kconfig b/drivers/bus/cdx/controller/Kconfig
new file mode 100644
index 000000000000..29bd45626d38
--- /dev/null
+++ b/drivers/bus/cdx/controller/Kconfig
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# CDX controller configuration
+#
+# Copyright (C) 2022, Advanced Micro Devices, Inc.
+#
+
+config CDX_CONTROLLER
+	tristate "CDX hardware driver"
+	depends on CDX_BUS
+	help
+	  CDX controller drives the CDX bus. It interacts with
+	  firmware to get the hardware devices and registers with
+	  the CDX bus. Say Y to enable the CDX hardware driver.
+
+	  If unsure, say N.
diff --git a/drivers/bus/cdx/controller/Makefile b/drivers/bus/cdx/controller/Makefile
new file mode 100644
index 000000000000..83bcd4ae1874
--- /dev/null
+++ b/drivers/bus/cdx/controller/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Makefile for CDX controller drivers
+#
+# Copyright (C) 2022, Advanced Micro Devices, Inc.
+#
+
+obj-$(CONFIG_CDX_CONTROLLER) += cdx_controller.o mcdi_stubs.o
diff --git a/drivers/bus/cdx/controller/cdx_controller.c b/drivers/bus/cdx/controller/cdx_controller.c
new file mode 100644
index 000000000000..792fbb8314d5
--- /dev/null
+++ b/drivers/bus/cdx/controller/cdx_controller.c
@@ -0,0 +1,210 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Platform driver for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+#include <linux/of_platform.h>
+#include <linux/cdx/cdx_bus.h>
+
+#include "../cdx.h"
+#include "mcdi_stubs.h"
+
+static int cdx_reset_device(struct cdx_controller_t *cdx,
+			    uint8_t bus_num, uint8_t dev_num)
+{
+	return cdx_mcdi_reset_dev(cdx->priv, bus_num, dev_num);
+}
+
+static int cdx_scan_devices(struct cdx_controller_t *cdx)
+{
+	struct cdx_mcdi_t *cdx_mcdi = cdx->priv;
+	int num_cdx_bus, num_cdx_dev;
+	uint8_t bus_num, dev_num;
+	int ret;
+
+	/* MCDI FW Read: Fetch the number of CDX buses present*/
+	num_cdx_bus = cdx_mcdi_get_num_buses(cdx_mcdi);
+
+	for (bus_num = 0; bus_num < num_cdx_bus; bus_num++) {
+		/* MCDI FW Read: Fetch the number of devices present */
+		num_cdx_dev = cdx_mcdi_get_num_devs(cdx_mcdi, bus_num);
+
+		for (dev_num = 0; dev_num < num_cdx_dev; dev_num++) {
+			struct cdx_dev_params_t dev_params;
+
+			/* MCDI FW: Get the device config */
+			ret = cdx_mcdi_get_dev_config(cdx_mcdi, bus_num,
+						      dev_num, &dev_params);
+			if (ret) {
+				dev_err(cdx->dev,
+					"CDX device config get failed for bus: %d\n", ret);
+				return ret;
+			}
+			dev_params.cdx = cdx;
+
+			/* Add the device to the cdx bus */
+			ret = cdx_device_add(&dev_params);
+			if (ret) {
+				dev_err(cdx->dev, "registering cdx dev: %d failed: %d\n",
+					dev_num, ret);
+				return ret;
+			}
+
+			dev_dbg(cdx->dev, "CDX dev: %d on cdx bus: %d created\n",
+				dev_num, bus_num);
+		}
+	}
+
+	return 0;
+}
+
+static int cdx_write_msi(struct cdx_controller_t *cdx, uint8_t bus_num,
+			 uint8_t dev_num, uint16_t msi_index,
+			 uint32_t data, uint64_t addr)
+{
+	return cdx_mcdi_write_msi(cdx->priv, bus_num, dev_num,
+				  msi_index, data, addr);
+}
+
+static int cdx_probe(struct platform_device *pdev)
+{
+	struct cdx_mcdi_t *cdx_mcdi;
+	struct cdx_controller_t *cdx;
+	struct device_node *iommu_node;
+	struct platform_device *plat_iommu_dev;
+	int ret;
+
+	/*
+	 * Defer probe CDX controller in case IOMMU is not yet initialized.
+	 * We do not add CDX devices before the IOMMU device is probed, as
+	 * in cleanup or shutdown time (device_shutdonw()), which traverse
+	 * device list in reverse order of device addition, we need to remove
+	 * CDX devices, before IOMMU device is removed.
+	 */
+	iommu_node = of_parse_phandle(pdev->dev.of_node, "iommu-map", 1);
+	if (!of_device_is_available(iommu_node)) {
+		dev_err(&pdev->dev, "No associated IOMMU node found\n");
+		return -EINVAL;
+	}
+
+	plat_iommu_dev = of_find_device_by_node(iommu_node);
+	if (!plat_iommu_dev) {
+		ret = -EPROBE_DEFER;
+		goto iommu_of_cleanup;
+	}
+
+	if (!plat_iommu_dev->dev.driver) {
+		ret = -EPROBE_DEFER;
+		goto iommu_of_cleanup;
+	}
+
+	cdx_mcdi = kzalloc(sizeof(*cdx_mcdi), GFP_KERNEL);
+	if (!cdx_mcdi) {
+		dev_err(&pdev->dev, "Failed to allocate memory for cdx_mcdi\n");
+		ret = -ENOMEM;
+		goto iommu_of_cleanup;
+	}
+
+	/* MCDI FW: Initialize the FW path */
+	ret = cdx_mcdi_init(cdx_mcdi);
+	if (ret) {
+		dev_err(&pdev->dev, "MCDI Initialization failed: %d\n", ret);
+		goto mcdi_init_fail;
+	}
+
+	cdx = kzalloc(sizeof(*cdx), GFP_KERNEL);
+	if (!cdx) {
+		dev_err(&pdev->dev, "Failed to allocate memory for cdx\n");
+		ret = -ENOMEM;
+		goto cdx_alloc_fail;
+	}
+	platform_set_drvdata(pdev, cdx);
+
+	cdx->dev = &pdev->dev;
+	cdx->priv = cdx_mcdi;
+	cdx->ops.scan = cdx_scan_devices;
+	cdx->ops.reset_dev = cdx_reset_device;
+	cdx->ops.write_msi = cdx_write_msi;
+
+	/* Register CDX controller with CDX bus driver */
+	ret = cdx_register_controller(cdx);
+	if (ret) {
+		dev_err(&pdev->dev, "Failed to register CDX controller\n");
+		goto cdx_register_fail;
+	}
+
+	return 0;
+
+cdx_register_fail:
+	kfree(cdx);
+cdx_alloc_fail:
+	cdx_mcdi_finish(cdx_mcdi);
+mcdi_init_fail:
+	kfree(cdx_mcdi);
+iommu_of_cleanup:
+	of_node_put(iommu_node);
+
+	return ret;
+}
+
+static int cdx_remove(struct platform_device *pdev)
+{
+	struct cdx_controller_t *cdx = platform_get_drvdata(pdev);
+	struct cdx_mcdi_t *cdx_mcdi = cdx->priv;
+
+	cdx_unregister_controller(cdx);
+	kfree(cdx);
+
+	cdx_mcdi_finish(cdx_mcdi);
+	kfree(cdx_mcdi);
+
+	return 0;
+}
+
+static void cdx_shutdown(struct platform_device *pdev)
+{
+	cdx_remove(pdev);
+}
+
+static const struct of_device_id cdx_match_table[] = {
+	{.compatible = "xlnx,cdxbus-controller-1.0",},
+	{ },
+};
+
+MODULE_DEVICE_TABLE(of, cdx_match_table);
+
+static struct platform_driver cdx_pdriver = {
+	.driver = {
+		   .name = "cdx-controller",
+		   .pm = NULL,
+		   .of_match_table = cdx_match_table,
+		   },
+	.probe = cdx_probe,
+	.remove = cdx_remove,
+	.shutdown = cdx_shutdown,
+};
+
+static int __init cdx_controller_init(void)
+{
+	int ret;
+
+	ret = platform_driver_register(&cdx_pdriver);
+	if (ret < 0)
+		pr_err("platform_driver_register() failed: %d\n", ret);
+
+	return ret;
+}
+module_init(cdx_controller_init);
+
+static void __exit cdx_controller_exit(void)
+{
+	platform_driver_unregister(&cdx_pdriver);
+}
+module_exit(cdx_controller_exit);
diff --git a/drivers/bus/cdx/controller/mcdi_stubs.c b/drivers/bus/cdx/controller/mcdi_stubs.c
new file mode 100644
index 000000000000..17737ffeae1c
--- /dev/null
+++ b/drivers/bus/cdx/controller/mcdi_stubs.c
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * MCDI Firmware interaction for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/ioport.h>
+
+#include "mcdi_stubs.h"
+
+int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi)
+{
+	cdx_mcdi->id = 0;
+	cdx_mcdi->flags = 0;
+
+	return 0;
+}
+
+void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi)
+{
+}
+
+int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi)
+{
+	return 1;
+}
+
+int cdx_mcdi_get_num_devs(struct cdx_mcdi_t *cdx_mcdi, int bus_num)
+{
+	return 1;
+}
+
+int cdx_mcdi_get_dev_config(struct cdx_mcdi_t *cdx_mcdi,
+			    uint8_t bus_num, uint8_t dev_num,
+			    struct cdx_dev_params_t *dev_params)
+{
+	dev_params->res[0].start = 0xe4020000;
+	dev_params->res[0].end = 0xe4020FFF;
+	dev_params->res[0].flags = IORESOURCE_MEM;
+	dev_params->res[1].start = 0xe4100000;
+	dev_params->res[1].end = 0xE411FFFF;
+	dev_params->res[1].flags = IORESOURCE_MEM;
+	dev_params->res_count = 2;
+
+	dev_params->req_id = 0x250;
+	dev_params->num_msi = 4;
+	dev_params->vendor = 0x10ee;
+	dev_params->device = 0x8084;
+	dev_params->bus_num = bus_num;
+	dev_params->dev_num = dev_num;
+
+	return 0;
+}
+
+int cdx_mcdi_reset_dev(struct cdx_mcdi_t *cdx_mcdi,
+		       uint8_t bus_num, uint8_t dev_num)
+{
+	return 0;
+}
+
+int cdx_mcdi_write_msi(struct cdx_mcdi_t *cdx_mcdi,
+		       uint8_t bus_num, uint8_t dev_num,
+		       uint16_t msi_index, uint32_t data,
+		       uint64_t addr)
+{
+	return 0;
+}
diff --git a/drivers/bus/cdx/controller/mcdi_stubs.h b/drivers/bus/cdx/controller/mcdi_stubs.h
new file mode 100644
index 000000000000..3256f0e306fa
--- /dev/null
+++ b/drivers/bus/cdx/controller/mcdi_stubs.h
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Header file for MCDI FW interaction for CDX bus.
+ *
+ * Copyright (C) 2022, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _MCDI_STUBS_H_
+#define _MCDI_STUBS_H_
+
+#include "../cdx.h"
+
+/**
+ * struct cdx_mcdi_t - CDX MCDI Firmware interface, to interact
+ *		       with CDX controller.
+ * @id: ID for MCDI Firmware interface
+ * @flags: Associated flags
+ */
+struct cdx_mcdi_t {
+	u32 id;
+	u32 flags;
+	/* Have more MCDI interface related data */
+};
+
+/**
+ * cdx_mcdi_init - Initialize the MCDI Firmware interface
+ *	for the CDX controller.
+ * @cdx_mcdi: pointer to MCDI interface
+ *
+ * Return 0 on success, <0 on failure
+ */
+int cdx_mcdi_init(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_finish - Close the MCDI Firmware interface.
+ * @cdx_mcdi: pointer to MCDI interface
+ */
+void cdx_mcdi_finish(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_get_num_buses - Get the total number of busses on
+ *	the controller.
+ * @cdx_mcdi: pointer to MCDI interface.
+ *
+ * Return total number of busses available on the controller,
+ *	<0 on failure
+ */
+int cdx_mcdi_get_num_buses(struct cdx_mcdi_t *cdx_mcdi);
+
+/**
+ * cdx_mcdi_get_num_devs - Get the total number of devices on
+ *	a particular bus.
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ *
+ * Return total number of devices available on the bus, <0 on failure
+ */
+int cdx_mcdi_get_num_devs(struct cdx_mcdi_t *cdx_mcdi, int bus_num);
+
+/**
+ * cdx_mcdi_get_dev_config - Get configuration for a particular
+ *	bus_num:dev_num
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ * @dev_params: Pointer to cdx_dev_params_t, this is populated by this
+ *	function with the configuration corresponding to the provided
+ *	bus_num:dev_num.
+ *
+ * Return 0 total number of devices available on the bus, <0 on failure
+ */
+int cdx_mcdi_get_dev_config(struct cdx_mcdi_t *cdx_mcdi,
+			    uint8_t bus_num, uint8_t dev_num,
+			    struct cdx_dev_params_t *dev_params);
+
+/**
+ * cdx_mcdi_reset_dev - Reset cdx device represented by bus_num:dev_num
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ *
+ * Return 0 on success, <0 on failure
+ */
+int cdx_mcdi_reset_dev(struct cdx_mcdi_t *cdx_mcdi,
+		       uint8_t bus_num, uint8_t dev_num);
+
+/**
+ * cdx_mcdi_write_msi_data - Write MSI related info to the Firmware
+ * @cdx_mcdi: pointer to MCDI interface.
+ * @bus_num: Bus number.
+ * @dev_num: Device number.
+ * @msi_index: MSI Index for which the data and address is provided
+ * @data: MSI data (i.e. eventID) for the interrupt
+ * @addr: the address on which the MSI is to be raised
+ */
+int cdx_mcdi_write_msi(struct cdx_mcdi_t *cdx_mcdi,
+		       uint8_t bus_num, uint8_t dev_num,
+		       uint16_t msi_index, uint32_t data,
+		       uint64_t addr);
+
+#endif /* _MCDI_STUBS_H_ */
-- 
2.25.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

Re: [RFC PATCH v4 0/8] add support for CDX bus

[Rob Herring]

On Fri, Oct 14, 2022 at 10:10:41AM +0530, Nipun Gupta wrote:
> This patch series introduces AMD CDX bus, which provides a
> mechanism to discover/rescan FPGA devices on run-time. These
> devices are memory mapped on system bus for embedded CPUs, and 
> added as CDX devices in Linux framework.

Please don't send a new version of a series as a reply to older 
versions. Just provide a lore link to older versions.

Rob

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [RFC PATCH v4 0/8] add support for CDX bus

[Gupta, Nipun]

[AMD Official Use Only - General]



> -----Original Message-----
> From: Rob Herring <robh@kernel.org>
> Sent: Friday, October 14, 2022 7:40 PM
> To: Gupta, Nipun <Nipun.Gupta@amd.com>
> Cc: krzysztof.kozlowski+dt@linaro.org; gregkh@linuxfoundation.org;
> rafael@kernel.org; eric.auger@redhat.com; alex.williamson@redhat.com;
> cohuck@redhat.com; Gupta, Puneet (DCG-ENG) <puneet.gupta@amd.com>;
> song.bao.hua@hisilicon.com; mchehab+huawei@kernel.org; maz@kernel.org;
> f.fainelli@gmail.com; jeffrey.l.hugo@gmail.com; saravanak@google.com;
> Michael.Srba@seznam.cz; mani@kernel.org; yishaih@nvidia.com;
> jgg@ziepe.ca; jgg@nvidia.com; robin.murphy@arm.com; will@kernel.org;
> joro@8bytes.org; masahiroy@kernel.org; ndesaulniers@google.com; linux-arm-
> kernel@lists.infradead.org; linux-kbuild@vger.kernel.org; linux-
> kernel@vger.kernel.org; devicetree@vger.kernel.org; kvm@vger.kernel.org;
> okaya@kernel.org; Anand, Harpreet <harpreet.anand@amd.com>; Agarwal,
> Nikhil <nikhil.agarwal@amd.com>; Simek, Michal <michal.simek@amd.com>;
> Radovanovic, Aleksandar <aleksandar.radovanovic@amd.com>; git (AMD-Xilinx)
> <git@amd.com>
> Subject: Re: [RFC PATCH v4 0/8] add support for CDX bus
> 
> Caution: This message originated from an External Source. Use proper caution
> when opening attachments, clicking links, or responding.
> 
> 
> On Fri, Oct 14, 2022 at 10:10:41AM +0530, Nipun Gupta wrote:
> > This patch series introduces AMD CDX bus, which provides a
> > mechanism to discover/rescan FPGA devices on run-time. These
> > devices are memory mapped on system bus for embedded CPUs, and
> > added as CDX devices in Linux framework.
> 
> Please don't send a new version of a series as a reply to older
> versions. Just provide a lore link to older versions.

Okay. Will provide lore link instead of in-reply from next version.

Thanks,
Nipun

> 
> Rob

_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel