[RFC, v0 0/3] msi support for platform devices

[RFC, v0 0/3] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 1189 bytes --]

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using
   callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Vikas Gupta (3):
  vfio/platform: add support for msi
  vfio/platform: change cleanup order
  vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 235 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 include/uapi/linux/vfio.h                     |   1 +
 9 files changed, 417 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c

-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC, v0 1/3] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 15647 bytes --]

MSI support for platform devices.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/vfio_platform_common.c  |  84 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 235 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 include/uapi/linux/vfio.h                     |   1 +
 4 files changed, 329 insertions(+), 14 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index c0771a9567fb..c713f4e4c552 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -26,6 +26,10 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+
+/* devices having MSI support */
+static LIST_HEAD(msi_list);
+
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +51,26 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
 	return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi = false;
+	struct vfio_platform_msi_node *iter;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry(iter, &msi_list, link) {
+		if (!strcmp(iter->compat, vdev->compat) &&
+		    try_module_get(iter->owner)) {
+			vdev->msi_module = iter->owner;
+			vdev->of_get_msi = iter->of_get_msi;
+			vdev->of_msi_write = iter->of_msi_write;
+			has_msi = true;
+			break;
+		}
+	}
+	mutex_unlock(&driver_lock);
+	return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
 				    struct device *dev)
 {
@@ -110,6 +134,11 @@ static bool vfio_platform_has_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? true : false;
 }
 
+static bool vfio_platform_has_msi(struct vfio_platform_device *vdev)
+{
+	return vdev->of_get_msi ? true : false;
+}
+
 static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -126,6 +155,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi;
+
+	has_msi = vfio_platform_lookup_msi(vdev);
+	if (!has_msi) {
+		request_module("vfio-msi:%s", vdev->compat);
+		has_msi = vfio_platform_lookup_msi(vdev);
+	}
+
+	return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +177,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 		module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+	if (vdev->of_get_msi)
+		module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
@@ -313,6 +361,10 @@ static long vfio_platform_ioctl(void *device_data,
 
 		if (vfio_platform_has_reset(vdev))
 			vdev->flags |= VFIO_DEVICE_FLAGS_RESET;
+
+		if (vfio_platform_has_msi(vdev))
+			vdev->flags |= VFIO_DEVICE_FLAGS_MSI;
+
 		info.flags = vdev->flags;
 		info.num_regions = vdev->num_regions;
 		info.num_irqs = vdev->num_irqs;
@@ -679,11 +731,15 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 		return ret;
 	}
 
+	ret = vfio_platform_get_msi(vdev);
+	if (ret)
+		dev_info(vdev->device, "msi not supported\n");
+
 	group = vfio_iommu_group_get(dev);
 	if (!group) {
 		dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
 		ret = -EINVAL;
-		goto put_reset;
+		goto put_msi;
 	}
 
 	ret = vfio_add_group_dev(dev, &vfio_platform_ops, vdev);
@@ -697,6 +753,8 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 
 put_iommu:
 	vfio_iommu_group_put(group, dev);
+put_msi:
+	vfio_platform_put_msi(vdev);
 put_reset:
 	vfio_platform_put_reset(vdev);
 	return ret;
@@ -745,6 +803,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+	mutex_lock(&driver_lock);
+	list_add(&node->link, &msi_list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+	struct vfio_platform_msi_node *iter, *temp;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry_safe(iter, temp, &msi_list, link) {
+		if (!strcmp(iter->compat, compat)) {
+			list_del(&iter->link);
+			break;
+		}
+	}
+
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index c5b09ec0a3c9..14b544c3aa42 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,176 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	struct device *dev = msi_desc_to_dev(desc);
+	struct vfio_device *device = dev_get_drvdata(dev);
+	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+						vfio_device_data(device);
+
+	vdev->of_msi_write(vdev, desc, msg);
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev, int nvec)
+{
+	int ret;
+	int msi_idx = 0;
+	struct msi_desc *desc;
+	struct device *dev = vdev->device;
+	u32 msi_off = vdev->num_irqs - vdev->num_msis;
+
+	/* Allocate platform MSIs */
+	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+	if (ret < 0)
+		return ret;
+
+	for_each_msi_entry(desc, dev) {
+		vdev->irqs[msi_off + msi_idx].hwirq = desc->irq;
+		msi_idx++;
+	}
+
+	vdev->num_msis = nvec;
+	vdev->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_device *vdev,
+				      int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq, ret;
+	u32 msi_off = vdev->num_irqs - vdev->num_msis;
+
+	if (vector < 0 || vector >= vdev->num_msis)
+		return -EINVAL;
+
+	irq = vdev->irqs[msi_off + vector].hwirq;
+
+	if (vdev->irqs[vector].trigger) {
+		free_irq(irq, vdev->irqs[vector].trigger);
+		kfree(vdev->irqs[vector].name);
+		eventfd_ctx_put(vdev->irqs[vector].trigger);
+		vdev->irqs[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	vdev->irqs[vector].name = kasprintf(GFP_KERNEL,
+					    "vfio-msi[%d]", vector);
+	if (!vdev->irqs[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(vdev->irqs[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq, vfio_msihandler, 0,
+			  vdev->irqs[vector].name, trigger);
+	if (ret) {
+		kfree(vdev->irqs[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	vdev->irqs[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_device *vdev, unsigned int start,
+			      unsigned int count, int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= vdev->num_msis || start + count > vdev->num_msis)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_msi_set_vector_signal(vdev, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_msi_set_vector_signal(vdev, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev)
+{
+	struct device *dev = vdev->device;
+
+	vfio_msi_set_block(vdev, 0, vdev->num_msis, NULL);
+	platform_msi_domain_free_irqs(dev);
+
+	vdev->config_msi = 0;
+	vdev->num_msis = 0;
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+				unsigned int index, unsigned int start,
+				unsigned int count, uint32_t flags, void *data)
+{
+	int i;
+
+	if (start + count > vdev->num_msis)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_msi_disable(vdev);
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (vdev->config_msi)
+			return vfio_msi_set_block(vdev, start, count,
+						  fds);
+		ret = vfio_msi_enable(vdev, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_msi_set_block(vdev, start, count, fds);
+		if (ret)
+			vfio_msi_disable(vdev);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!vdev->irqs[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(vdev->irqs[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(vdev->irqs[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 				 uint32_t flags, unsigned index, unsigned start,
 				 unsigned count, void *data)
@@ -261,16 +433,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 		    unsigned start, unsigned count, uint32_t flags,
 		    void *data) = NULL;
 
-	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-	case VFIO_IRQ_SET_ACTION_MASK:
-		func = vfio_platform_set_irq_mask;
-		break;
-	case VFIO_IRQ_SET_ACTION_UNMASK:
-		func = vfio_platform_set_irq_unmask;
-		break;
-	case VFIO_IRQ_SET_ACTION_TRIGGER:
-		func = vfio_platform_set_irq_trigger;
-		break;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (!irq->is_msi) {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+			func = vfio_platform_set_irq_mask;
+			break;
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			func = vfio_platform_set_irq_unmask;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_platform_set_irq_trigger;
+			break;
+		}
+	} else {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_set_msi_trigger;
+			break;
+		}
 	}
 
 	if (!func)
@@ -282,11 +467,17 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
+	int msi_cnt = 0;
 
 	while (vdev->get_irq(vdev, cnt) >= 0)
 		cnt++;
 
-	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	if (vdev->of_get_msi)
+		msi_cnt = vdev->of_get_msi(vdev);
+	vdev->num_msis = msi_cnt;
+
+	vdev->irqs = kcalloc(cnt + msi_cnt, sizeof(struct vfio_platform_irq),
+			     GFP_KERNEL);
 	if (!vdev->irqs)
 		return -ENOMEM;
 
@@ -309,7 +500,18 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		vdev->irqs[i].masked = false;
 	}
 
-	vdev->num_irqs = cnt;
+	for (i = cnt; i < msi_cnt + cnt; i++) {
+		spin_lock_init(&vdev->irqs[i].lock);
+
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+
+		vdev->irqs[i].count = 1;
+		vdev->irqs[i].hwirq = 0;
+		vdev->irqs[i].masked = false;
+		vdev->irqs[i].is_msi = true;
+	}
+
+	vdev->num_irqs = cnt + msi_cnt;
 
 	return 0;
 err:
@@ -320,10 +522,17 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
 	int i;
+	int non_msi_cnt = vdev->num_irqs - vdev->num_msis;
 
-	for (i = 0; i < vdev->num_irqs; i++)
+	for (i = 0; i < non_msi_cnt; i++)
 		vfio_set_trigger(vdev, i, -1, NULL);
 
+	if (vdev->num_msis) {
+		vfio_set_msi_trigger(vdev, 0, 0, 0,
+				     VFIO_IRQ_SET_DATA_NONE, NULL);
+		vdev->num_msis = 0;
+	}
+
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
 }
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index 289089910643..2aea445e1071 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -26,6 +27,7 @@ struct vfio_platform_irq {
 	char			*name;
 	struct eventfd_ctx	*trigger;
 	bool			masked;
+	bool			is_msi;
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
@@ -46,13 +48,16 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	u32				num_msis;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
 	const char			*compat;
 	const char			*acpihid;
 	struct module			*reset_module;
+	struct module			*msi_module;
 	struct device			*device;
+	int				config_msi;
 
 	/*
 	 * These fields should be filled by the bus specific binder
@@ -65,11 +70,19 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
+	int	(*of_msi_write)(struct vfio_platform_device *vdev,
+				struct msi_desc *desc,
+				struct msi_msg *msg);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
+typedef int (*vfio_platform_msi_write_fn_t)(struct vfio_platform_device *vdev,
+					    struct msi_desc *desc,
+					    struct msi_msg *msg);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
@@ -78,6 +91,14 @@ struct vfio_platform_reset_node {
 	vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+	struct list_head link;
+	char *compat;
+	struct module *owner;
+	vfio_platform_get_msi_fn_t of_get_msi;
+	vfio_platform_msi_write_fn_t of_msi_write;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 				      struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +115,8 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
 					   vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
 #define vfio_platform_register_reset(__compat, __reset)		\
 static struct vfio_platform_reset_node __reset ## _node = {	\
 	.owner = THIS_MODULE,					\
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 2f313a238a8f..aab051e8338d 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -203,6 +203,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_AP	(1 << 5)	/* vfio-ap device */
 #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)	/* vfio-fsl-mc device */
 #define VFIO_DEVICE_FLAGS_CAPS	(1 << 7)	/* Info supports caps */
+#define VFIO_DEVICE_FLAGS_MSI	(1 << 8)	/* Device supports msi */
 	__u32	num_regions;	/* Max region index + 1 */
 	__u32	num_irqs;	/* Max IRQ index + 1 */
 	__u32   cap_offset;	/* Offset within info struct of first cap */
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Alex Williamson]

On Thu,  5 Nov 2020 11:32:55 +0530
Vikas Gupta <vikas.gupta@broadcom.com> wrote:

> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index 2f313a238a8f..aab051e8338d 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -203,6 +203,7 @@ struct vfio_device_info {
>  #define VFIO_DEVICE_FLAGS_AP	(1 << 5)	/* vfio-ap device */
>  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)	/* vfio-fsl-mc device */
>  #define VFIO_DEVICE_FLAGS_CAPS	(1 << 7)	/* Info supports caps */
> +#define VFIO_DEVICE_FLAGS_MSI	(1 << 8)	/* Device supports msi */
>  	__u32	num_regions;	/* Max region index + 1 */
>  	__u32	num_irqs;	/* Max IRQ index + 1 */
>  	__u32   cap_offset;	/* Offset within info struct of first cap */

This doesn't make any sense to me, MSIs are just edge triggered
interrupts to userspace, so why isn't this fully described via
VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
this seems incomplete, which indexes are MSI (IRQ_INFO can describe
that)?  We also already support MSI with vfio-pci, so a global flag for
the device advertising this still seems wrong.  Thanks,

Alex

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 1946 bytes --]

Hi Alex,

On Thu, Nov 5, 2020 at 12:38 PM Alex Williamson
<alex.williamson@redhat.com> wrote:
>
> On Thu,  5 Nov 2020 11:32:55 +0530
> Vikas Gupta <vikas.gupta@broadcom.com> wrote:
>
> > diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> > index 2f313a238a8f..aab051e8338d 100644
> > --- a/include/uapi/linux/vfio.h
> > +++ b/include/uapi/linux/vfio.h
> > @@ -203,6 +203,7 @@ struct vfio_device_info {
> >  #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
> >  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
> >  #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
> > +#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
> >       __u32   num_regions;    /* Max region index + 1 */
> >       __u32   num_irqs;       /* Max IRQ index + 1 */
> >       __u32   cap_offset;     /* Offset within info struct of first cap */
>
> This doesn't make any sense to me, MSIs are just edge triggered
> interrupts to userspace, so why isn't this fully described via
> VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
> this seems incomplete, which indexes are MSI (IRQ_INFO can describe
> that)?  We also already support MSI with vfio-pci, so a global flag for
> the device advertising this still seems wrong.  Thanks,
>
> Alex
>
Since VFIO platform uses indexes for IRQ numbers so I think MSI(s)
cannot be described using indexes.
In the patch set there is no difference between MSI and normal
interrupt for VFIO_DEVICE_GET_IRQ_INFO.
The patch set adds MSI(s), say as an extension, to the normal
interrupts and handled accordingly. Do you see this is a violation? If
yes, then we`ll think of other possible ways to support MSI for the
platform devices.
Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it
collides with an already supported vfio-pci or if not necessary, we
can remove this flag.

Thanks,
Vikas

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Alex Williamson]

On Fri, 6 Nov 2020 08:24:26 +0530
Vikas Gupta <vikas.gupta@broadcom.com> wrote:

> Hi Alex,
> 
> On Thu, Nov 5, 2020 at 12:38 PM Alex Williamson
> <alex.williamson@redhat.com> wrote:
> >
> > On Thu,  5 Nov 2020 11:32:55 +0530
> > Vikas Gupta <vikas.gupta@broadcom.com> wrote:
> >  
> > > diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> > > index 2f313a238a8f..aab051e8338d 100644
> > > --- a/include/uapi/linux/vfio.h
> > > +++ b/include/uapi/linux/vfio.h
> > > @@ -203,6 +203,7 @@ struct vfio_device_info {
> > >  #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
> > >  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
> > >  #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
> > > +#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
> > >       __u32   num_regions;    /* Max region index + 1 */
> > >       __u32   num_irqs;       /* Max IRQ index + 1 */
> > >       __u32   cap_offset;     /* Offset within info struct of first cap */  
> >
> > This doesn't make any sense to me, MSIs are just edge triggered
> > interrupts to userspace, so why isn't this fully described via
> > VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
> > this seems incomplete, which indexes are MSI (IRQ_INFO can describe
> > that)?  We also already support MSI with vfio-pci, so a global flag for
> > the device advertising this still seems wrong.  Thanks,
> >
> > Alex
> >  
> Since VFIO platform uses indexes for IRQ numbers so I think MSI(s)
> cannot be described using indexes.

That would be news for vfio-pci which has been describing MSIs with
sub-indexes within indexes since vfio started.

> In the patch set there is no difference between MSI and normal
> interrupt for VFIO_DEVICE_GET_IRQ_INFO.

Then what exactly is a global device flag indicating?  Does it indicate
all IRQs are MSI?

> The patch set adds MSI(s), say as an extension, to the normal
> interrupts and handled accordingly.

So we have both "normal" IRQs and MSIs?  How does the user know which
indexes are which?

> Do you see this is a violation? If

Seems pretty unclear and dubious use of a global device flag.

> yes, then we`ll think of other possible ways to support MSI for the
> platform devices.
> Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it
> collides with an already supported vfio-pci or if not necessary, we
> can remove this flag.

If nothing else you're using a global flag to describe a platform
device specific augmentation.  We've recently added capabilities on the
device info return that would be more appropriate for this, but
fundamentally I don't understand why the irq info isn't sufficient.
Thanks,

Alex

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 3751 bytes --]

Hi Alex,

On Fri, Nov 6, 2020 at 8:42 AM Alex Williamson
<alex.williamson@redhat.com> wrote:
>
> On Fri, 6 Nov 2020 08:24:26 +0530
> Vikas Gupta <vikas.gupta@broadcom.com> wrote:
>
> > Hi Alex,
> >
> > On Thu, Nov 5, 2020 at 12:38 PM Alex Williamson
> > <alex.williamson@redhat.com> wrote:
> > >
> > > On Thu,  5 Nov 2020 11:32:55 +0530
> > > Vikas Gupta <vikas.gupta@broadcom.com> wrote:
> > >
> > > > diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> > > > index 2f313a238a8f..aab051e8338d 100644
> > > > --- a/include/uapi/linux/vfio.h
> > > > +++ b/include/uapi/linux/vfio.h
> > > > @@ -203,6 +203,7 @@ struct vfio_device_info {
> > > >  #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
> > > >  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
> > > >  #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
> > > > +#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
> > > >       __u32   num_regions;    /* Max region index + 1 */
> > > >       __u32   num_irqs;       /* Max IRQ index + 1 */
> > > >       __u32   cap_offset;     /* Offset within info struct of first cap */
> > >
> > > This doesn't make any sense to me, MSIs are just edge triggered
> > > interrupts to userspace, so why isn't this fully described via
> > > VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
> > > this seems incomplete, which indexes are MSI (IRQ_INFO can describe
> > > that)?  We also already support MSI with vfio-pci, so a global flag for
> > > the device advertising this still seems wrong.  Thanks,
> > >
> > > Alex
> > >
> > Since VFIO platform uses indexes for IRQ numbers so I think MSI(s)
> > cannot be described using indexes.
>
> That would be news for vfio-pci which has been describing MSIs with
> sub-indexes within indexes since vfio started.
>
> > In the patch set there is no difference between MSI and normal
> > interrupt for VFIO_DEVICE_GET_IRQ_INFO.
>
> Then what exactly is a global device flag indicating?  Does it indicate
> all IRQs are MSI?

No, it's not indicating that all are MSI.
The rationale behind adding the flag to tell user-space that platform
device supports MSI as well. As you mentioned recently added
capabilities can help on this, I`ll go through that.

>
> > The patch set adds MSI(s), say as an extension, to the normal
> > interrupts and handled accordingly.
>
> So we have both "normal" IRQs and MSIs?  How does the user know which
> indexes are which?

With this patch set, I think this is missing and user space cannot
know that particular index is MSI interrupt.
For platform devices there is no such mechanism, like index and
sub-indexes to differentiate between legacy, MSI or MSIX as it’s there
in PCI.
I believe for a particular IRQ index if the flag
VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ
index has MSI(s). Does it make sense?
Suggestions on this would be helpful.

Thanks,
Vikas
>
> > Do you see this is a violation? If
>
> Seems pretty unclear and dubious use of a global device flag.
>
> > yes, then we`ll think of other possible ways to support MSI for the
> > platform devices.
> > Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it
> > collides with an already supported vfio-pci or if not necessary, we
> > can remove this flag.
>
> If nothing else you're using a global flag to describe a platform
> device specific augmentation.  We've recently added capabilities on the
> device info return that would be more appropriate for this, but
> fundamentally I don't understand why the irq info isn't sufficient.
> Thanks,
>
> Alex
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 11/9/20 7:41 AM, Vikas Gupta wrote:
> Hi Alex,
> 
> On Fri, Nov 6, 2020 at 8:42 AM Alex Williamson
> <alex.williamson@redhat.com> wrote:
>>
>> On Fri, 6 Nov 2020 08:24:26 +0530
>> Vikas Gupta <vikas.gupta@broadcom.com> wrote:
>>
>>> Hi Alex,
>>>
>>> On Thu, Nov 5, 2020 at 12:38 PM Alex Williamson
>>> <alex.williamson@redhat.com> wrote:
>>>>
>>>> On Thu,  5 Nov 2020 11:32:55 +0530
>>>> Vikas Gupta <vikas.gupta@broadcom.com> wrote:
>>>>
>>>>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>>>>> index 2f313a238a8f..aab051e8338d 100644
>>>>> --- a/include/uapi/linux/vfio.h
>>>>> +++ b/include/uapi/linux/vfio.h
>>>>> @@ -203,6 +203,7 @@ struct vfio_device_info {
>>>>>  #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
>>>>>  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
>>>>>  #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
>>>>> +#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
>>>>>       __u32   num_regions;    /* Max region index + 1 */
>>>>>       __u32   num_irqs;       /* Max IRQ index + 1 */
>>>>>       __u32   cap_offset;     /* Offset within info struct of first cap */
>>>>
>>>> This doesn't make any sense to me, MSIs are just edge triggered
>>>> interrupts to userspace, so why isn't this fully described via
>>>> VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
>>>> this seems incomplete, which indexes are MSI (IRQ_INFO can describe
>>>> that)?  We also already support MSI with vfio-pci, so a global flag for
>>>> the device advertising this still seems wrong.  Thanks,
>>>>
>>>> Alex
>>>>
>>> Since VFIO platform uses indexes for IRQ numbers so I think MSI(s)
>>> cannot be described using indexes.
>>
>> That would be news for vfio-pci which has been describing MSIs with
>> sub-indexes within indexes since vfio started.
>>
>>> In the patch set there is no difference between MSI and normal
>>> interrupt for VFIO_DEVICE_GET_IRQ_INFO.
>>
>> Then what exactly is a global device flag indicating?  Does it indicate
>> all IRQs are MSI?
> 
> No, it's not indicating that all are MSI.
> The rationale behind adding the flag to tell user-space that platform
> device supports MSI as well. As you mentioned recently added
> capabilities can help on this, I`ll go through that.
> 
>>
>>> The patch set adds MSI(s), say as an extension, to the normal
>>> interrupts and handled accordingly.
>>
>> So we have both "normal" IRQs and MSIs?  How does the user know which
>> indexes are which?
> 
> With this patch set, I think this is missing and user space cannot
> know that particular index is MSI interrupt.
> For platform devices there is no such mechanism, like index and
> sub-indexes to differentiate between legacy, MSI or MSIX as it’s there
> in PCI.
Wht can't you use the count field (as per vfio_pci_get_irq_count())?
> I believe for a particular IRQ index if the flag
> VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ
> index has MSI(s). Does it make sense?
I don't think it is the same semantics.

Thanks

Eric
> Suggestions on this would be helpful.
> 
> Thanks,
> Vikas
>>
>>> Do you see this is a violation? If
>>
>> Seems pretty unclear and dubious use of a global device flag.
>>
>>> yes, then we`ll think of other possible ways to support MSI for the
>>> platform devices.
>>> Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it
>>> collides with an already supported vfio-pci or if not necessary, we
>>> can remove this flag.
>>
>> If nothing else you're using a global flag to describe a platform
>> device specific augmentation.  We've recently added capabilities on the
>> device info return that would be more appropriate for this, but
>> fundamentally I don't understand why the irq info isn't sufficient.
>> Thanks,
>>
>> Alex
>>

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Alex Williamson]

On Mon, 9 Nov 2020 12:11:15 +0530
Vikas Gupta <vikas.gupta@broadcom.com> wrote:

> Hi Alex,
> 
> On Fri, Nov 6, 2020 at 8:42 AM Alex Williamson
> <alex.williamson@redhat.com> wrote:
> >
> > On Fri, 6 Nov 2020 08:24:26 +0530
> > Vikas Gupta <vikas.gupta@broadcom.com> wrote:
> >  
> > > Hi Alex,
> > >
> > > On Thu, Nov 5, 2020 at 12:38 PM Alex Williamson
> > > <alex.williamson@redhat.com> wrote:  
> > > >
> > > > On Thu,  5 Nov 2020 11:32:55 +0530
> > > > Vikas Gupta <vikas.gupta@broadcom.com> wrote:
> > > >  
> > > > > diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> > > > > index 2f313a238a8f..aab051e8338d 100644
> > > > > --- a/include/uapi/linux/vfio.h
> > > > > +++ b/include/uapi/linux/vfio.h
> > > > > @@ -203,6 +203,7 @@ struct vfio_device_info {
> > > > >  #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
> > > > >  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
> > > > >  #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
> > > > > +#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
> > > > >       __u32   num_regions;    /* Max region index + 1 */
> > > > >       __u32   num_irqs;       /* Max IRQ index + 1 */
> > > > >       __u32   cap_offset;     /* Offset within info struct of first cap */  
> > > >
> > > > This doesn't make any sense to me, MSIs are just edge triggered
> > > > interrupts to userspace, so why isn't this fully described via
> > > > VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
> > > > this seems incomplete, which indexes are MSI (IRQ_INFO can describe
> > > > that)?  We also already support MSI with vfio-pci, so a global flag for
> > > > the device advertising this still seems wrong.  Thanks,
> > > >
> > > > Alex
> > > >  
> > > Since VFIO platform uses indexes for IRQ numbers so I think MSI(s)
> > > cannot be described using indexes.  
> >
> > That would be news for vfio-pci which has been describing MSIs with
> > sub-indexes within indexes since vfio started.
> >  
> > > In the patch set there is no difference between MSI and normal
> > > interrupt for VFIO_DEVICE_GET_IRQ_INFO.  
> >
> > Then what exactly is a global device flag indicating?  Does it indicate
> > all IRQs are MSI?  
> 
> No, it's not indicating that all are MSI.
> The rationale behind adding the flag to tell user-space that platform
> device supports MSI as well. As you mentioned recently added
> capabilities can help on this, I`ll go through that.


It still seems questionable to me to use a device info capability to
describe an interrupt index specific feature.  The scope seems wrong.
Why does userspace need to know that this IRQ is MSI rather than
indicating it's simply an edge triggered interrupt?  That can be done
using only vfio_irq_info.flags.


> > > The patch set adds MSI(s), say as an extension, to the normal
> > > interrupts and handled accordingly.  
> >
> > So we have both "normal" IRQs and MSIs?  How does the user know which
> > indexes are which?  
> 
> With this patch set, I think this is missing and user space cannot
> know that particular index is MSI interrupt.
> For platform devices there is no such mechanism, like index and
> sub-indexes to differentiate between legacy, MSI or MSIX as it’s there
> in PCI.

Indexes and sub-indexes are a grouping mechanism of vfio to describe
related interrupts.  That terminology doesn't exist on PCI either, it's
meant to be used generically.  It's left to the vfio bus driver how
userspace associates a given index to a device feature.

> I believe for a particular IRQ index if the flag
> VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ
> index has MSI(s). Does it make sense?


No, no-resize is an implementation detail, not an indication of the
interrupt mechanism.  It's still not clear to me why it's important to
expose to userspace that a given interrupt is MSI versus simply
exposing it as an edge interrupt (ie. automasked = false).  If it is
necessary, the most direct approach might be to expose a capability
extension in the vfio_irq_info structure to describe it.  Even then
though, I don't think simply exposing a index as MSI is very
meaningful.  What is userspace intended to do differently based on this
information?  Thanks,

Alex

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 5175 bytes --]

Hi Alex,

On Mon, Nov 9, 2020 at 8:58 PM Alex Williamson
<alex.williamson@redhat.com> wrote:
>
> On Mon, 9 Nov 2020 12:11:15 +0530
> Vikas Gupta <vikas.gupta@broadcom.com> wrote:
>
> > Hi Alex,
> >
> > On Fri, Nov 6, 2020 at 8:42 AM Alex Williamson
> > <alex.williamson@redhat.com> wrote:
> > >
> > > On Fri, 6 Nov 2020 08:24:26 +0530
> > > Vikas Gupta <vikas.gupta@broadcom.com> wrote:
> > >
> > > > Hi Alex,
> > > >
> > > > On Thu, Nov 5, 2020 at 12:38 PM Alex Williamson
> > > > <alex.williamson@redhat.com> wrote:
> > > > >
> > > > > On Thu,  5 Nov 2020 11:32:55 +0530
> > > > > Vikas Gupta <vikas.gupta@broadcom.com> wrote:
> > > > >
> > > > > > diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> > > > > > index 2f313a238a8f..aab051e8338d 100644
> > > > > > --- a/include/uapi/linux/vfio.h
> > > > > > +++ b/include/uapi/linux/vfio.h
> > > > > > @@ -203,6 +203,7 @@ struct vfio_device_info {
> > > > > >  #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
> > > > > >  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
> > > > > >  #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
> > > > > > +#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
> > > > > >       __u32   num_regions;    /* Max region index + 1 */
> > > > > >       __u32   num_irqs;       /* Max IRQ index + 1 */
> > > > > >       __u32   cap_offset;     /* Offset within info struct of first cap */
> > > > >
> > > > > This doesn't make any sense to me, MSIs are just edge triggered
> > > > > interrupts to userspace, so why isn't this fully described via
> > > > > VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
> > > > > this seems incomplete, which indexes are MSI (IRQ_INFO can describe
> > > > > that)?  We also already support MSI with vfio-pci, so a global flag for
> > > > > the device advertising this still seems wrong.  Thanks,
> > > > >
> > > > > Alex
> > > > >
> > > > Since VFIO platform uses indexes for IRQ numbers so I think MSI(s)
> > > > cannot be described using indexes.
> > >
> > > That would be news for vfio-pci which has been describing MSIs with
> > > sub-indexes within indexes since vfio started.
> > >
> > > > In the patch set there is no difference between MSI and normal
> > > > interrupt for VFIO_DEVICE_GET_IRQ_INFO.
> > >
> > > Then what exactly is a global device flag indicating?  Does it indicate
> > > all IRQs are MSI?
> >
> > No, it's not indicating that all are MSI.
> > The rationale behind adding the flag to tell user-space that platform
> > device supports MSI as well. As you mentioned recently added
> > capabilities can help on this, I`ll go through that.
>
>
> It still seems questionable to me to use a device info capability to
> describe an interrupt index specific feature.  The scope seems wrong.
> Why does userspace need to know that this IRQ is MSI rather than
> indicating it's simply an edge triggered interrupt?  That can be done
> using only vfio_irq_info.flags.

Ok. In the next patch set I`ll remove the device flag (VFIO_DEVICE_FLAGS_MSI) as
vfio_irq_info.flags should have enough information for edge triggered interrupt.

>
>
> > > > The patch set adds MSI(s), say as an extension, to the normal
> > > > interrupts and handled accordingly.
> > >
> > > So we have both "normal" IRQs and MSIs?  How does the user know which
> > > indexes are which?
> >
> > With this patch set, I think this is missing and user space cannot
> > know that particular index is MSI interrupt.
> > For platform devices there is no such mechanism, like index and
> > sub-indexes to differentiate between legacy, MSI or MSIX as it’s there
> > in PCI.
>
> Indexes and sub-indexes are a grouping mechanism of vfio to describe
> related interrupts.  That terminology doesn't exist on PCI either, it's
> meant to be used generically.  It's left to the vfio bus driver how
> userspace associates a given index to a device feature.
>
> > I believe for a particular IRQ index if the flag
> > VFIO_IRQ_INFO_NORESIZE is used then user space can know which IRQ
> > index has MSI(s). Does it make sense?
>
>
> No, no-resize is an implementation detail, not an indication of the
> interrupt mechanism.  It's still not clear to me why it's important to
> expose to userspace that a given interrupt is MSI versus simply
> exposing it as an edge interrupt (ie. automasked = false).  If it is
> necessary, the most direct approach might be to expose a capability
> extension in the vfio_irq_info structure to describe it.  Even then
> though, I don't think simply exposing a index as MSI is very
> meaningful.  What is userspace intended to do differently based on this
> information?  Thanks,
The current patch set is not setting VFIO_IRQ_INFO_AUTOMASKED
(automasked=false) for MSIs so I believe this much is information
enough for user space to know that this is an edge triggered
interrupt.
 I agree that exposing an index as MSI is not meaningful as user space
has nothing special to do with this information.
>
> Alex
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 11/6/20 3:54 AM, Vikas Gupta wrote:
> Hi Alex,
> 
> On Thu, Nov 5, 2020 at 12:38 PM Alex Williamson
> <alex.williamson@redhat.com> wrote:
>>
>> On Thu,  5 Nov 2020 11:32:55 +0530
>> Vikas Gupta <vikas.gupta@broadcom.com> wrote:
>>
>>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>>> index 2f313a238a8f..aab051e8338d 100644
>>> --- a/include/uapi/linux/vfio.h
>>> +++ b/include/uapi/linux/vfio.h
>>> @@ -203,6 +203,7 @@ struct vfio_device_info {
>>>  #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
>>>  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
>>>  #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
>>> +#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
>>>       __u32   num_regions;    /* Max region index + 1 */
>>>       __u32   num_irqs;       /* Max IRQ index + 1 */
>>>       __u32   cap_offset;     /* Offset within info struct of first cap */
>>
>> This doesn't make any sense to me, MSIs are just edge triggered
>> interrupts to userspace, so why isn't this fully described via
>> VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
>> this seems incomplete, which indexes are MSI (IRQ_INFO can describe
>> that)?  We also already support MSI with vfio-pci, so a global flag for
>> the device advertising this still seems wrong.  Thanks,
>>
>> Alex
>>
> Since VFIO platform uses indexes for IRQ numbers so I think MSI(s)
> cannot be described using indexes.
> In the patch set there is no difference between MSI and normal
> interrupt for VFIO_DEVICE_GET_IRQ_INFO.
in vfio_platform_irq_init() we first iterate on normal interrupts using
get_irq(). Can't we add an MSI index at the end of this list with
vdev->irqs[i].count > 1 and set vdev->num_irqs accordingly?

Thanks

Eric
> The patch set adds MSI(s), say as an extension, to the normal
> interrupts and handled accordingly. Do you see this is a violation? If
> yes, then we`ll think of other possible ways to support MSI for the
> platform devices.
> Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it
> collides with an already supported vfio-pci or if not necessary, we
> can remove this flag.
> 
> Thanks,
> Vikas
> 

Re: [RFC, v0 1/3] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 2599 bytes --]

Hi Eric,

On Mon, Nov 9, 2020 at 8:35 PM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 11/6/20 3:54 AM, Vikas Gupta wrote:
> > Hi Alex,
> >
> > On Thu, Nov 5, 2020 at 12:38 PM Alex Williamson
> > <alex.williamson@redhat.com> wrote:
> >>
> >> On Thu,  5 Nov 2020 11:32:55 +0530
> >> Vikas Gupta <vikas.gupta@broadcom.com> wrote:
> >>
> >>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> >>> index 2f313a238a8f..aab051e8338d 100644
> >>> --- a/include/uapi/linux/vfio.h
> >>> +++ b/include/uapi/linux/vfio.h
> >>> @@ -203,6 +203,7 @@ struct vfio_device_info {
> >>>  #define VFIO_DEVICE_FLAGS_AP (1 << 5)        /* vfio-ap device */
> >>>  #define VFIO_DEVICE_FLAGS_FSL_MC (1 << 6)    /* vfio-fsl-mc device */
> >>>  #define VFIO_DEVICE_FLAGS_CAPS       (1 << 7)        /* Info supports caps */
> >>> +#define VFIO_DEVICE_FLAGS_MSI        (1 << 8)        /* Device supports msi */
> >>>       __u32   num_regions;    /* Max region index + 1 */
> >>>       __u32   num_irqs;       /* Max IRQ index + 1 */
> >>>       __u32   cap_offset;     /* Offset within info struct of first cap */
> >>
> >> This doesn't make any sense to me, MSIs are just edge triggered
> >> interrupts to userspace, so why isn't this fully described via
> >> VFIO_DEVICE_GET_IRQ_INFO?  If we do need something new to describe it,
> >> this seems incomplete, which indexes are MSI (IRQ_INFO can describe
> >> that)?  We also already support MSI with vfio-pci, so a global flag for
> >> the device advertising this still seems wrong.  Thanks,
> >>
> >> Alex
> >>
> > Since VFIO platform uses indexes for IRQ numbers so I think MSI(s)
> > cannot be described using indexes.
> > In the patch set there is no difference between MSI and normal
> > interrupt for VFIO_DEVICE_GET_IRQ_INFO.
> in vfio_platform_irq_init() we first iterate on normal interrupts using
> get_irq(). Can't we add an MSI index at the end of this list with
> vdev->irqs[i].count > 1 and set vdev->num_irqs accordingly?
Yes, I think MSI can be added to the end of list with setting
vdev->irqs[i].count > 1.
I`ll consider changing in the next patch set.
Thanks,
Vikas
>
> Thanks
>
> Eric
> > The patch set adds MSI(s), say as an extension, to the normal
> > interrupts and handled accordingly. Do you see this is a violation? If
> > yes, then we`ll think of other possible ways to support MSI for the
> > platform devices.
> > Macro VFIO_DEVICE_FLAGS_MSI can be changed to any other name if it
> > collides with an already supported vfio-pci or if not necessary, we
> > can remove this flag.
> >
> > Thanks,
> > Vikas
> >
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC, v0 2/3] vfio/platform: change cleanup order

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 846 bytes --]

In the case of msi, vendor specific msi module may require
region access to handle msi cleanup so we need to cleanup region
after irq cleanup only.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/vfio_platform_common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index c713f4e4c552..b9d68ccdf92f 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -283,8 +283,8 @@ static void vfio_platform_release(void *device_data)
 			WARN_ON(1);
 		}
 		pm_runtime_put(vdev->device);
-		vfio_platform_regions_cleanup(vdev);
 		vfio_platform_irq_cleanup(vdev);
+		vfio_platform_regions_cleanup(vdev);
 	}
 
 	mutex_unlock(&driver_lock);
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC, v0 3/3] vfio/platform: add Broadcom msi module

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 4413 bytes --]

Add Broadcom msi module for platform devices.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 +++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 74 +++++++++++++++++++
 5 files changed, 87 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c

diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
index dc1a3c44f2c6..7b8696febe61 100644
--- a/drivers/vfio/platform/Kconfig
+++ b/drivers/vfio/platform/Kconfig
@@ -21,3 +21,4 @@ config VFIO_AMBA
 	  If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
--- a/drivers/vfio/platform/Makefile
+++ b/drivers/vfio/platform/Makefile
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
diff --git a/drivers/vfio/platform/msi/Kconfig b/drivers/vfio/platform/msi/Kconfig
new file mode 100644
index 000000000000..54d6b70e1e32
--- /dev/null
+++ b/drivers/vfio/platform/msi/Kconfig
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+	tristate "MSI support for Broadcom platform devices"
+	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+	default ARCH_BCM_IPROC
+	help
+	  Enables the VFIO platform driver to handle msi for Broadcom devices
+
+	  If you don't know what to do here, say N.
diff --git a/drivers/vfio/platform/msi/Makefile b/drivers/vfio/platform/msi/Makefile
new file mode 100644
index 000000000000..27422d45cecb
--- /dev/null
+++ b/drivers/vfio/platform/msi/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
diff --git a/drivers/vfio/platform/msi/vfio_platform_bcmplt.c b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
new file mode 100644
index 000000000000..b97a571e90a9
--- /dev/null
+++ b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Broadcom.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/msi.h>
+#include <linux/vfio.h>
+
+#include "../vfio_platform_private.h"
+
+#define RING_SIZE	(64 << 10)
+
+#define RING_MSI_ADDR_LS	0x03c
+#define RING_MSI_ADDR_MS	0x040
+#define RING_MSI_DATA_VALUE	0x064
+
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	return (reg->size / RING_SIZE);
+}
+
+static int bcm_write_msi(struct vfio_platform_device *vdev,
+			 struct msi_desc *desc,
+			 struct msi_msg *msg)
+{
+	void __iomem *ring;
+	u32 msi_off = vdev->num_irqs - vdev->num_msis;
+	int ring_num = desc->irq - vdev->irqs[msi_off].hwirq;
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	if (!reg->ioaddr) {
+		reg->ioaddr = ioremap(reg->addr, reg->size);
+		if (!reg->ioaddr)
+			return -ENOMEM;
+	}
+
+	ring = reg->ioaddr + ring_num * RING_SIZE;
+
+	writel_relaxed(msg->address_lo, ring + RING_MSI_ADDR_LS);
+	writel_relaxed(msg->address_hi, ring + RING_MSI_ADDR_MS);
+	writel_relaxed(msg->data, ring + RING_MSI_DATA_VALUE);
+
+	return 0;
+}
+
+static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
+	.owner = THIS_MODULE,
+	.compat = "brcm,iproc-flexrm-mbox",
+	.of_get_msi = bcm_num_msi,
+	.of_msi_write = bcm_write_msi
+};
+
+static int __init vfio_platform_bcmflexrm_msi_module_init(void)
+{
+	__vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
+
+	return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
+{
+	vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
+}
+
+module_init(vfio_platform_bcmflexrm_msi_module_init);
+module_exit(vfio_platform_bcmflexrm_msi_module_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Broadcom");
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC, v1 0/3] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 1743 bytes --]

This RFC adds support for MSI for platform devices.
a) MSI(s) is/are added in addition to the normal interrupts.
b) The vendor specific MSI configuration can be done using
   callbacks which is implemented as msi module.
c) Adds a msi handling module for the Broadcom platform devices.

Changes from:
-------------
 v0 to v1:
   i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
   ii) Add MSI(s) at the end of the irq list of platform IRQs.
       MSI(s) with first entry of MSI block has count and flag
       information.
       IRQ list: Allocation for IRQs + MSIs are allocated as below
       Example: if there are 'n' IRQs and 'k' MSIs
       -------------------------------------------------------
       |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
       -------------------------------------------------------
       MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (3):
  vfio/platform: add support for msi
  vfio/platform: change cleanup order
  vfio/platform: add Broadcom msi module

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
 drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 8 files changed, 419 insertions(+), 15 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c

-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC v1 1/3] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 15255 bytes --]

MSI support for platform devices.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/vfio_platform_common.c  |  84 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  23 ++
 3 files changed, 331 insertions(+), 14 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index c0771a9567fb..226aa8083751 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -26,6 +26,10 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+
+/* devices having MSI support */
+static LIST_HEAD(msi_list);
+
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +51,26 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
 	return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi = false;
+	struct vfio_platform_msi_node *iter;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry(iter, &msi_list, link) {
+		if (!strcmp(iter->compat, vdev->compat) &&
+		    try_module_get(iter->owner)) {
+			vdev->msi_module = iter->owner;
+			vdev->of_get_msi = iter->of_get_msi;
+			vdev->of_msi_write = iter->of_msi_write;
+			has_msi = true;
+			break;
+		}
+	}
+	mutex_unlock(&driver_lock);
+	return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
 				    struct device *dev)
 {
@@ -126,6 +150,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi;
+
+	has_msi = vfio_platform_lookup_msi(vdev);
+	if (!has_msi) {
+		request_module("vfio-msi:%s", vdev->compat);
+		has_msi = vfio_platform_lookup_msi(vdev);
+	}
+
+	return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +172,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 		module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+	if (vdev->of_get_msi)
+		module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
@@ -313,6 +356,7 @@ static long vfio_platform_ioctl(void *device_data,
 
 		if (vfio_platform_has_reset(vdev))
 			vdev->flags |= VFIO_DEVICE_FLAGS_RESET;
+
 		info.flags = vdev->flags;
 		info.num_regions = vdev->num_regions;
 		info.num_irqs = vdev->num_irqs;
@@ -366,6 +410,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,7 +418,12 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
 						 vdev->num_irqs, &data_size);
 		if (ret)
 			return ret;
@@ -679,11 +729,15 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 		return ret;
 	}
 
+	ret = vfio_platform_get_msi(vdev);
+	if (ret)
+		dev_info(vdev->device, "msi not supported\n");
+
 	group = vfio_iommu_group_get(dev);
 	if (!group) {
 		dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
 		ret = -EINVAL;
-		goto put_reset;
+		goto put_msi;
 	}
 
 	ret = vfio_add_group_dev(dev, &vfio_platform_ops, vdev);
@@ -697,6 +751,8 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 
 put_iommu:
 	vfio_iommu_group_put(group, dev);
+put_msi:
+	vfio_platform_put_msi(vdev);
 put_reset:
 	vfio_platform_put_reset(vdev);
 	return ret;
@@ -745,6 +801,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+	mutex_lock(&driver_lock);
+	list_add(&node->link, &msi_list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+	struct vfio_platform_msi_node *iter, *temp;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry_safe(iter, temp, &msi_list, link) {
+		if (!strcmp(iter->compat, compat)) {
+			list_del(&iter->link);
+			break;
+		}
+	}
+
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index c5b09ec0a3c9..3ee736dcb3c5 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,176 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	struct device *dev = msi_desc_to_dev(desc);
+	struct vfio_device *device = dev_get_drvdata(dev);
+	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+						vfio_device_data(device);
+
+	vdev->of_msi_write(vdev, desc, msg);
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev, int nvec)
+{
+	int ret;
+	int msi_idx = 0;
+	struct msi_desc *desc;
+	struct device *dev = vdev->device;
+	int msi_off = vdev->num_irqs - 1;
+
+	/* Allocate platform MSIs */
+	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+	if (ret < 0)
+		return ret;
+
+	for_each_msi_entry(desc, dev) {
+		vdev->irqs[msi_off + msi_idx].hwirq = desc->irq;
+		msi_idx++;
+	}
+
+	vdev->num_msis = nvec;
+	vdev->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_device *vdev,
+				      int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq, ret;
+	int msi_off = vdev->num_irqs - 1;
+
+	if (vector < 0 || vector >= vdev->num_msis)
+		return -EINVAL;
+
+	irq = vdev->irqs[msi_off + vector].hwirq;
+
+	if (vdev->irqs[vector].trigger) {
+		free_irq(irq, vdev->irqs[vector].trigger);
+		kfree(vdev->irqs[vector].name);
+		eventfd_ctx_put(vdev->irqs[vector].trigger);
+		vdev->irqs[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	vdev->irqs[vector].name = kasprintf(GFP_KERNEL,
+					    "vfio-msi[%d]", vector);
+	if (!vdev->irqs[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(vdev->irqs[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq, vfio_msihandler, 0,
+			  vdev->irqs[vector].name, trigger);
+	if (ret) {
+		kfree(vdev->irqs[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	vdev->irqs[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_device *vdev, unsigned int start,
+			      unsigned int count, int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= vdev->num_msis || start + count > vdev->num_msis)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_msi_set_vector_signal(vdev, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_msi_set_vector_signal(vdev, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev)
+{
+	struct device *dev = vdev->device;
+
+	vfio_msi_set_block(vdev, 0, vdev->num_msis, NULL);
+	platform_msi_domain_free_irqs(dev);
+
+	vdev->config_msi = 0;
+	vdev->num_msis = 0;
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+				unsigned int index, unsigned int start,
+				unsigned int count, uint32_t flags, void *data)
+{
+	int i;
+
+	if (start + count > vdev->num_msis)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_msi_disable(vdev);
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (vdev->config_msi)
+			return vfio_msi_set_block(vdev, start, count,
+						  fds);
+		ret = vfio_msi_enable(vdev, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_msi_set_block(vdev, start, count, fds);
+		if (ret)
+			vfio_msi_disable(vdev);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!vdev->irqs[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(vdev->irqs[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(vdev->irqs[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 				 uint32_t flags, unsigned index, unsigned start,
 				 unsigned count, void *data)
@@ -261,16 +433,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 		    unsigned start, unsigned count, uint32_t flags,
 		    void *data) = NULL;
 
-	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-	case VFIO_IRQ_SET_ACTION_MASK:
-		func = vfio_platform_set_irq_mask;
-		break;
-	case VFIO_IRQ_SET_ACTION_UNMASK:
-		func = vfio_platform_set_irq_unmask;
-		break;
-	case VFIO_IRQ_SET_ACTION_TRIGGER:
-		func = vfio_platform_set_irq_trigger;
-		break;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (!irq->is_msi) {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+			func = vfio_platform_set_irq_mask;
+			break;
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			func = vfio_platform_set_irq_unmask;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_platform_set_irq_trigger;
+			break;
+		}
+	} else {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_set_msi_trigger;
+			break;
+		}
 	}
 
 	if (!func)
@@ -282,11 +467,17 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
+	int msi_cnt = 0;
 
 	while (vdev->get_irq(vdev, cnt) >= 0)
 		cnt++;
 
-	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	if (vdev->of_get_msi)
+		msi_cnt = vdev->of_get_msi(vdev);
+	vdev->num_msis = msi_cnt;
+
+	vdev->irqs = kcalloc(cnt + msi_cnt, sizeof(struct vfio_platform_irq),
+			     GFP_KERNEL);
 	if (!vdev->irqs)
 		return -ENOMEM;
 
@@ -311,6 +502,21 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 
 	vdev->num_irqs = cnt;
 
+	/*
+	 * MSI(s) are counted as '1' as far as number of IRQs
+	 * is concerned and it is at last index. We fill the
+	 * count in the first index of MSI(s).
+	 */
+	if (msi_cnt > 0) {
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+		vdev->irqs[i].count = msi_cnt;
+		vdev->irqs[i].hwirq = 0;
+		vdev->irqs[i].masked = false;
+		vdev->irqs[i].is_msi = true;
+
+		vdev->num_irqs++;
+	}
+
 	return 0;
 err:
 	kfree(vdev->irqs);
@@ -320,10 +526,18 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
 	int i;
+	int non_msi_irqs = vdev->num_msis > 0 ?
+				 vdev->num_irqs - 1 : vdev->num_irqs;
 
-	for (i = 0; i < vdev->num_irqs; i++)
+	for (i = 0; i < non_msi_irqs; i++)
 		vfio_set_trigger(vdev, i, -1, NULL);
 
+	if (vdev->num_msis) {
+		vfio_set_msi_trigger(vdev, 0, 0, 0,
+				     VFIO_IRQ_SET_DATA_NONE, NULL);
+		vdev->num_msis = 0;
+	}
+
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
 }
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index 289089910643..2aea445e1071 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -26,6 +27,7 @@ struct vfio_platform_irq {
 	char			*name;
 	struct eventfd_ctx	*trigger;
 	bool			masked;
+	bool			is_msi;
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
@@ -46,13 +48,16 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	u32				num_msis;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
 	const char			*compat;
 	const char			*acpihid;
 	struct module			*reset_module;
+	struct module			*msi_module;
 	struct device			*device;
+	int				config_msi;
 
 	/*
 	 * These fields should be filled by the bus specific binder
@@ -65,11 +70,19 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
+	int	(*of_msi_write)(struct vfio_platform_device *vdev,
+				struct msi_desc *desc,
+				struct msi_msg *msg);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
+typedef int (*vfio_platform_msi_write_fn_t)(struct vfio_platform_device *vdev,
+					    struct msi_desc *desc,
+					    struct msi_msg *msg);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
@@ -78,6 +91,14 @@ struct vfio_platform_reset_node {
 	vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+	struct list_head link;
+	char *compat;
+	struct module *owner;
+	vfio_platform_get_msi_fn_t of_get_msi;
+	vfio_platform_msi_write_fn_t of_msi_write;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 				      struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +115,8 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
 					   vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
 #define vfio_platform_register_reset(__compat, __reset)		\
 static struct vfio_platform_reset_node __reset ## _node = {	\
 	.owner = THIS_MODULE,					\
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC v1 2/3] vfio/platform: change cleanup order

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 846 bytes --]

In the case of msi, vendor specific msi module may require
region access to handle msi cleanup so we need to cleanup region
after irq cleanup only.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/vfio_platform_common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index 226aa8083751..ca4a01c488b8 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -278,8 +278,8 @@ static void vfio_platform_release(void *device_data)
 			WARN_ON(1);
 		}
 		pm_runtime_put(vdev->device);
-		vfio_platform_regions_cleanup(vdev);
 		vfio_platform_irq_cleanup(vdev);
+		vfio_platform_regions_cleanup(vdev);
 	}
 
 	mutex_unlock(&driver_lock);
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC v1 3/3] vfio/platform: add Broadcom msi module

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 4401 bytes --]

Add Broadcom msi module for platform devices.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 +++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 74 +++++++++++++++++++
 5 files changed, 87 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c

diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
index dc1a3c44f2c6..7b8696febe61 100644
--- a/drivers/vfio/platform/Kconfig
+++ b/drivers/vfio/platform/Kconfig
@@ -21,3 +21,4 @@ config VFIO_AMBA
 	  If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
--- a/drivers/vfio/platform/Makefile
+++ b/drivers/vfio/platform/Makefile
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
diff --git a/drivers/vfio/platform/msi/Kconfig b/drivers/vfio/platform/msi/Kconfig
new file mode 100644
index 000000000000..54d6b70e1e32
--- /dev/null
+++ b/drivers/vfio/platform/msi/Kconfig
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+	tristate "MSI support for Broadcom platform devices"
+	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+	default ARCH_BCM_IPROC
+	help
+	  Enables the VFIO platform driver to handle msi for Broadcom devices
+
+	  If you don't know what to do here, say N.
diff --git a/drivers/vfio/platform/msi/Makefile b/drivers/vfio/platform/msi/Makefile
new file mode 100644
index 000000000000..27422d45cecb
--- /dev/null
+++ b/drivers/vfio/platform/msi/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
diff --git a/drivers/vfio/platform/msi/vfio_platform_bcmplt.c b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
new file mode 100644
index 000000000000..7f44d7d0c95d
--- /dev/null
+++ b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Broadcom.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/msi.h>
+#include <linux/vfio.h>
+
+#include "../vfio_platform_private.h"
+
+#define RING_SIZE		(64 << 10)
+
+#define RING_MSI_ADDR_LS	0x03c
+#define RING_MSI_ADDR_MS	0x040
+#define RING_MSI_DATA_VALUE	0x064
+
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	return (reg->size / RING_SIZE);
+}
+
+static int bcm_write_msi(struct vfio_platform_device *vdev,
+			 struct msi_desc *desc,
+			 struct msi_msg *msg)
+{
+	void __iomem *ring;
+	int msi_off = vdev->num_irqs - 1;
+	int ring_num = desc->irq - vdev->irqs[msi_off].hwirq;
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	if (!reg->ioaddr) {
+		reg->ioaddr = ioremap(reg->addr, reg->size);
+		if (!reg->ioaddr)
+			return -ENOMEM;
+	}
+
+	ring = reg->ioaddr + ring_num * RING_SIZE;
+
+	writel_relaxed(msg->address_lo, ring + RING_MSI_ADDR_LS);
+	writel_relaxed(msg->address_hi, ring + RING_MSI_ADDR_MS);
+	writel_relaxed(msg->data, ring + RING_MSI_DATA_VALUE);
+
+	return 0;
+}
+
+static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
+	.owner = THIS_MODULE,
+	.compat = "brcm,iproc-flexrm-mbox",
+	.of_get_msi = bcm_num_msi,
+	.of_msi_write = bcm_write_msi
+};
+
+static int __init vfio_platform_bcmflexrm_msi_module_init(void)
+{
+	__vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
+
+	return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
+{
+	vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
+}
+
+module_init(vfio_platform_bcmflexrm_msi_module_init);
+module_exit(vfio_platform_bcmflexrm_msi_module_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Broadcom");
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC, v1 0/3] msi support for platform devices

[Auger Eric]

Hi Vikas,

On 11/12/20 6:58 PM, Vikas Gupta wrote:
> This RFC adds support for MSI for platform devices.
> a) MSI(s) is/are added in addition to the normal interrupts.
> b) The vendor specific MSI configuration can be done using
>    callbacks which is implemented as msi module.
> c) Adds a msi handling module for the Broadcom platform devices.
> 
> Changes from:
> -------------
>  v0 to v1:
>    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
>    ii) Add MSI(s) at the end of the irq list of platform IRQs.
>        MSI(s) with first entry of MSI block has count and flag
>        information.
>        IRQ list: Allocation for IRQs + MSIs are allocated as below
>        Example: if there are 'n' IRQs and 'k' MSIs
>        -------------------------------------------------------
>        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
>        -------------------------------------------------------
I have not taken time yet to look at your series, but to me you should have
|IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
then for setting a given MSIX (i) you would select the MSIx index and
then set start=i count=1.
to me individual MSIs are encoded in the subindex and not in the index.
The index just selects the "type" of interrupt.

For PCI you just have:
        VFIO_PCI_INTX_IRQ_INDEX,
        VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with
start/count
        VFIO_PCI_MSIX_IRQ_INDEX,
        VFIO_PCI_ERR_IRQ_INDEX,
        VFIO_PCI_REQ_IRQ_INDEX,

(include/uapi/linux/vfio.h)

Thanks

Eric
>        MSI-0 will have count=k set and flags set accordingly.
> 
> Vikas Gupta (3):
>   vfio/platform: add support for msi
>   vfio/platform: change cleanup order
>   vfio/platform: add Broadcom msi module
> 
>  drivers/vfio/platform/Kconfig                 |   1 +
>  drivers/vfio/platform/Makefile                |   1 +
>  drivers/vfio/platform/msi/Kconfig             |   9 +
>  drivers/vfio/platform/msi/Makefile            |   2 +
>  .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
>  drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
>  drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
>  drivers/vfio/platform/vfio_platform_private.h |  23 ++
>  8 files changed, 419 insertions(+), 15 deletions(-)
>  create mode 100644 drivers/vfio/platform/msi/Kconfig
>  create mode 100644 drivers/vfio/platform/msi/Makefile
>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> 

Re: [RFC, v1 0/3] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 4041 bytes --]

Hi Eric,

On Fri, Nov 13, 2020 at 12:10 AM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 11/12/20 6:58 PM, Vikas Gupta wrote:
> > This RFC adds support for MSI for platform devices.
> > a) MSI(s) is/are added in addition to the normal interrupts.
> > b) The vendor specific MSI configuration can be done using
> >    callbacks which is implemented as msi module.
> > c) Adds a msi handling module for the Broadcom platform devices.
> >
> > Changes from:
> > -------------
> >  v0 to v1:
> >    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
> >    ii) Add MSI(s) at the end of the irq list of platform IRQs.
> >        MSI(s) with first entry of MSI block has count and flag
> >        information.
> >        IRQ list: Allocation for IRQs + MSIs are allocated as below
> >        Example: if there are 'n' IRQs and 'k' MSIs
> >        -------------------------------------------------------
> >        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
> >        -------------------------------------------------------
> I have not taken time yet to look at your series, but to me you should have
> |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
> then for setting a given MSIX (i) you would select the MSIx index and
> then set start=i count=1.

As per your suggestion, we should have, if there are n-IRQs, k-MSIXs
and m-MSIs, allocation of IRQs should be done as below

|IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
                                             |        |
                                             |
|MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
                                             |MSI0||MSI1||MSI2|....|MSI-(m-1)|
With this implementation user space can know that, at indexes n and
n+1, edge triggered interrupts are present.
   We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
   struct vfio_platform_irq{
   .....
   .....
   struct vfio_platform_irq *block; => this points to the block
allocation for MSIs/MSIXs and all msi/msix are type of IRQs.
   };
                         OR
Another structure can be defined in 'vfio_pci_private.h'
struct vfio_msi_ctx {
        struct eventfd_ctx      *trigger;
        char                    *name;
};
and
struct vfio_platform_irq {
  .....
  .....
  struct vfio_msi_ctx *block; => this points to the block allocation
for MSIs/MSIXs
};
Which of the above two options sounds OK to you? Please suggest.

> to me individual MSIs are encoded in the subindex and not in the index.
> The index just selects the "type" of interrupt.
>
> For PCI you just have:
>         VFIO_PCI_INTX_IRQ_INDEX,
>         VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with
> start/count
>         VFIO_PCI_MSIX_IRQ_INDEX,
>         VFIO_PCI_ERR_IRQ_INDEX,
>         VFIO_PCI_REQ_IRQ_INDEX,
>
> (include/uapi/linux/vfio.h)

In pci case, type of interrupts is fixed so they can be 'indexed' by
these enums but for VFIO platform user space will need to iterate all
(num_irqs) indexes to know at which indexes edge triggered interrupts
are present.

Thanks,
Vikas
>
> Thanks
>
> Eric
> >        MSI-0 will have count=k set and flags set accordingly.
> >
> > Vikas Gupta (3):
> >   vfio/platform: add support for msi
> >   vfio/platform: change cleanup order
> >   vfio/platform: add Broadcom msi module
> >
> >  drivers/vfio/platform/Kconfig                 |   1 +
> >  drivers/vfio/platform/Makefile                |   1 +
> >  drivers/vfio/platform/msi/Kconfig             |   9 +
> >  drivers/vfio/platform/msi/Makefile            |   2 +
> >  .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
> >  drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
> >  drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
> >  drivers/vfio/platform/vfio_platform_private.h |  23 ++
> >  8 files changed, 419 insertions(+), 15 deletions(-)
> >  create mode 100644 drivers/vfio/platform/msi/Kconfig
> >  create mode 100644 drivers/vfio/platform/msi/Makefile
> >  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> >
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC, v1 0/3] msi support for platform devices

[Auger Eric]

Hi Vikas,

On 11/13/20 6:24 PM, Vikas Gupta wrote:
> Hi Eric,
> 
> On Fri, Nov 13, 2020 at 12:10 AM Auger Eric <eric.auger@redhat.com> wrote:
>>
>> Hi Vikas,
>>
>> On 11/12/20 6:58 PM, Vikas Gupta wrote:
>>> This RFC adds support for MSI for platform devices.
>>> a) MSI(s) is/are added in addition to the normal interrupts.
>>> b) The vendor specific MSI configuration can be done using
>>>    callbacks which is implemented as msi module.
>>> c) Adds a msi handling module for the Broadcom platform devices.
>>>
>>> Changes from:
>>> -------------
>>>  v0 to v1:
>>>    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
>>>    ii) Add MSI(s) at the end of the irq list of platform IRQs.
>>>        MSI(s) with first entry of MSI block has count and flag
>>>        information.
>>>        IRQ list: Allocation for IRQs + MSIs are allocated as below
>>>        Example: if there are 'n' IRQs and 'k' MSIs
>>>        -------------------------------------------------------
>>>        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
>>>        -------------------------------------------------------
>> I have not taken time yet to look at your series, but to me you should have
>> |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
>> then for setting a given MSIX (i) you would select the MSIx index and
>> then set start=i count=1.
> 
> As per your suggestion, we should have, if there are n-IRQs, k-MSIXs
> and m-MSIs, allocation of IRQs should be done as below
> 
> |IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
>                                              |        |
>                                              |
> |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
>                                              |MSI0||MSI1||MSI2|....|MSI-(m-1)|
No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ
if ERR/REQ were to be added.

I think the userspace could query the total number of indices using
VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire
interrupts + MSI index + MSIX index)

Then userspace can loop on all the indices using
VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the
first indices related to wire interrupts (count = 1). Then comes the MSI
index, and after the MSI index. If any of those is supported, count >1,
otherwise count=0. The only thing I am dubious about is can the device
use a single MSI/MSIX? Because my hypothesis here is we use count to
discriminate between wire first indices and other indices.



> With this implementation user space can know that, at indexes n and
> n+1, edge triggered interrupts are present.
note wired interrupts can also be edge ones.
>    We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
>    struct vfio_platform_irq{
>    .....
>    .....
>    struct vfio_platform_irq *block; => this points to the block
> allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
array of wired interrupts (the existing vdev->irqs) and a new array for
MSI(x) as done in the PCI code.

vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);

Does it make sense?

>    };
>                          OR
> Another structure can be defined in 'vfio_pci_private.h'
> struct vfio_msi_ctx {
>         struct eventfd_ctx      *trigger;
>         char                    *name;
> };
> and
> struct vfio_platform_irq {
>   .....
>   .....
>   struct vfio_msi_ctx *block; => this points to the block allocation
> for MSIs/MSIXs
> };
> Which of the above two options sounds OK to you? Please suggest.
> 
>> to me individual MSIs are encoded in the subindex and not in the index.
>> The index just selects the "type" of interrupt.
>>
>> For PCI you just have:
>>         VFIO_PCI_INTX_IRQ_INDEX,
>>         VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with
>> start/count
>>         VFIO_PCI_MSIX_IRQ_INDEX,
>>         VFIO_PCI_ERR_IRQ_INDEX,
>>         VFIO_PCI_REQ_IRQ_INDEX,
>>
>> (include/uapi/linux/vfio.h)
> 
> In pci case, type of interrupts is fixed so they can be 'indexed' by
> these enums but for VFIO platform user space will need to iterate all
> (num_irqs) indexes to know at which indexes edge triggered interrupts
> are present.
indeed, but can't you loop over all indices looking until count !=1? At
this point you know if have finished emurating the wires. Holds if
MSI(x) count !=1 of course.

Thanks

Eric

> 
> Thanks,
> Vikas
>>
>> Thanks
>>
>> Eric
>>>        MSI-0 will have count=k set and flags set accordingly.
>>>
>>> Vikas Gupta (3):
>>>   vfio/platform: add support for msi
>>>   vfio/platform: change cleanup order
>>>   vfio/platform: add Broadcom msi module
>>>
>>>  drivers/vfio/platform/Kconfig                 |   1 +
>>>  drivers/vfio/platform/Makefile                |   1 +
>>>  drivers/vfio/platform/msi/Kconfig             |   9 +
>>>  drivers/vfio/platform/msi/Makefile            |   2 +
>>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
>>>  drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
>>>  drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
>>>  drivers/vfio/platform/vfio_platform_private.h |  23 ++
>>>  8 files changed, 419 insertions(+), 15 deletions(-)
>>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
>>>  create mode 100644 drivers/vfio/platform/msi/Makefile
>>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>>>
>>

Re: [RFC, v1 0/3] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 6483 bytes --]

Hi Eric,

On Mon, Nov 16, 2020 at 6:44 PM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 11/13/20 6:24 PM, Vikas Gupta wrote:
> > Hi Eric,
> >
> > On Fri, Nov 13, 2020 at 12:10 AM Auger Eric <eric.auger@redhat.com> wrote:
> >>
> >> Hi Vikas,
> >>
> >> On 11/12/20 6:58 PM, Vikas Gupta wrote:
> >>> This RFC adds support for MSI for platform devices.
> >>> a) MSI(s) is/are added in addition to the normal interrupts.
> >>> b) The vendor specific MSI configuration can be done using
> >>>    callbacks which is implemented as msi module.
> >>> c) Adds a msi handling module for the Broadcom platform devices.
> >>>
> >>> Changes from:
> >>> -------------
> >>>  v0 to v1:
> >>>    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
> >>>    ii) Add MSI(s) at the end of the irq list of platform IRQs.
> >>>        MSI(s) with first entry of MSI block has count and flag
> >>>        information.
> >>>        IRQ list: Allocation for IRQs + MSIs are allocated as below
> >>>        Example: if there are 'n' IRQs and 'k' MSIs
> >>>        -------------------------------------------------------
> >>>        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
> >>>        -------------------------------------------------------
> >> I have not taken time yet to look at your series, but to me you should have
> >> |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
> >> then for setting a given MSIX (i) you would select the MSIx index and
> >> then set start=i count=1.
> >
> > As per your suggestion, we should have, if there are n-IRQs, k-MSIXs
> > and m-MSIs, allocation of IRQs should be done as below
> >
> > |IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
> >                                              |        |
> >                                              |
> > |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
> >                                              |MSI0||MSI1||MSI2|....|MSI-(m-1)|
> No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
> and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ
> if ERR/REQ were to be added.
I agree on this. Actually the map I drew incorrectly above but wanted
to demonstrate the same. It was a child-parent relationship for MSI
and its members and similarly for MSIX as well.
>
> I think the userspace could query the total number of indices using
> VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire
> interrupts + MSI index + MSIX index)
>
> Then userspace can loop on all the indices using
> VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the
> first indices related to wire interrupts (count = 1). Then comes the MSI
> index, and after the MSI index. If any of those is supported, count >1,
> otherwise count=0. The only thing I am dubious about is can the device
> use a single MSI/MSIX? Because my hypothesis here is we use count to
> discriminate between wire first indices and other indices.
I believe count can be one as well, especially for ERR/REQ as you
mentioned above.I think we can not rely on the count > 1. Now, this is
blocking and we are not left with options unless we consider adding
more enums in flags in vfio_irq_info to tell userspace that particular
index is wired, MSI, MSIX etc. for the platform device.
What do you think?
>
>
>
> > With this implementation user space can know that, at indexes n and
> > n+1, edge triggered interrupts are present.
> note wired interrupts can also be edge ones.
> >    We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
> >    struct vfio_platform_irq{
> >    .....
> >    .....
> >    struct vfio_platform_irq *block; => this points to the block
> > allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
> array of wired interrupts (the existing vdev->irqs) and a new array for
> MSI(x) as done in the PCI code.
>
> vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
>
> Does it make sense?
Yes, we can use similar kinds of allocations.

Thanks,
Vikas
>
> >    };
> >                          OR
> > Another structure can be defined in 'vfio_pci_private.h'
> > struct vfio_msi_ctx {
> >         struct eventfd_ctx      *trigger;
> >         char                    *name;
> > };
> > and
> > struct vfio_platform_irq {
> >   .....
> >   .....
> >   struct vfio_msi_ctx *block; => this points to the block allocation
> > for MSIs/MSIXs
> > };
> > Which of the above two options sounds OK to you? Please suggest.
> >
> >> to me individual MSIs are encoded in the subindex and not in the index.
> >> The index just selects the "type" of interrupt.
> >>
> >> For PCI you just have:
> >>         VFIO_PCI_INTX_IRQ_INDEX,
> >>         VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with
> >> start/count
> >>         VFIO_PCI_MSIX_IRQ_INDEX,
> >>         VFIO_PCI_ERR_IRQ_INDEX,
> >>         VFIO_PCI_REQ_IRQ_INDEX,
> >>
> >> (include/uapi/linux/vfio.h)
> >
> > In pci case, type of interrupts is fixed so they can be 'indexed' by
> > these enums but for VFIO platform user space will need to iterate all
> > (num_irqs) indexes to know at which indexes edge triggered interrupts
> > are present.
> indeed, but can't you loop over all indices looking until count !=1? At
> this point you know if have finished emurating the wires. Holds if
> MSI(x) count !=1 of course.
>
> Thanks
>
> Eric
>
> >
> > Thanks,
> > Vikas
> >>
> >> Thanks
> >>
> >> Eric
> >>>        MSI-0 will have count=k set and flags set accordingly.
> >>>
> >>> Vikas Gupta (3):
> >>>   vfio/platform: add support for msi
> >>>   vfio/platform: change cleanup order
> >>>   vfio/platform: add Broadcom msi module
> >>>
> >>>  drivers/vfio/platform/Kconfig                 |   1 +
> >>>  drivers/vfio/platform/Makefile                |   1 +
> >>>  drivers/vfio/platform/msi/Kconfig             |   9 +
> >>>  drivers/vfio/platform/msi/Makefile            |   2 +
> >>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
> >>>  drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
> >>>  drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
> >>>  drivers/vfio/platform/vfio_platform_private.h |  23 ++
> >>>  8 files changed, 419 insertions(+), 15 deletions(-)
> >>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
> >>>  create mode 100644 drivers/vfio/platform/msi/Makefile
> >>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> >>>
> >>
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC, v1 0/3] msi support for platform devices

[Auger Eric]

Hi Vikas,

On 11/17/20 7:25 AM, Vikas Gupta wrote:
> Hi Eric,
> 
> On Mon, Nov 16, 2020 at 6:44 PM Auger Eric <eric.auger@redhat.com> wrote:
>>
>> Hi Vikas,
>>
>> On 11/13/20 6:24 PM, Vikas Gupta wrote:
>>> Hi Eric,
>>>
>>> On Fri, Nov 13, 2020 at 12:10 AM Auger Eric <eric.auger@redhat.com> wrote:
>>>>
>>>> Hi Vikas,
>>>>
>>>> On 11/12/20 6:58 PM, Vikas Gupta wrote:
>>>>> This RFC adds support for MSI for platform devices.
>>>>> a) MSI(s) is/are added in addition to the normal interrupts.
>>>>> b) The vendor specific MSI configuration can be done using
>>>>>    callbacks which is implemented as msi module.
>>>>> c) Adds a msi handling module for the Broadcom platform devices.
>>>>>
>>>>> Changes from:
>>>>> -------------
>>>>>  v0 to v1:
>>>>>    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
>>>>>    ii) Add MSI(s) at the end of the irq list of platform IRQs.
>>>>>        MSI(s) with first entry of MSI block has count and flag
>>>>>        information.
>>>>>        IRQ list: Allocation for IRQs + MSIs are allocated as below
>>>>>        Example: if there are 'n' IRQs and 'k' MSIs
>>>>>        -------------------------------------------------------
>>>>>        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
>>>>>        -------------------------------------------------------
>>>> I have not taken time yet to look at your series, but to me you should have
>>>> |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
>>>> then for setting a given MSIX (i) you would select the MSIx index and
>>>> then set start=i count=1.
>>>
>>> As per your suggestion, we should have, if there are n-IRQs, k-MSIXs
>>> and m-MSIs, allocation of IRQs should be done as below
>>>
>>> |IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
>>>                                              |        |
>>>                                              |
>>> |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
>>>                                              |MSI0||MSI1||MSI2|....|MSI-(m-1)|
>> No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
>> and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ
>> if ERR/REQ were to be added.
> I agree on this. Actually the map I drew incorrectly above but wanted
> to demonstrate the same. It was a child-parent relationship for MSI
> and its members and similarly for MSIX as well.
>>
>> I think the userspace could query the total number of indices using
>> VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire
>> interrupts + MSI index + MSIX index)
>>
>> Then userspace can loop on all the indices using
>> VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the
>> first indices related to wire interrupts (count = 1). Then comes the MSI
>> index, and after the MSI index. If any of those is supported, count >1,
>> otherwise count=0. The only thing I am dubious about is can the device
>> use a single MSI/MSIX? Because my hypothesis here is we use count to
>> discriminate between wire first indices and other indices.
> I believe count can be one as well, especially for ERR/REQ as you
> mentioned above.
Given ERR and REQ indices would follow MSI and MSIX ones, MSI index
could be recognized by the first index whose count != 1. But indeed I am
not sure the number of supported vectors cannot be 1. In your case it is
induced by the size of the ring so it is OK but for other devices this
may be different.

I think we can not rely on the count > 1. Now, this is
> blocking and we are not left with options unless we consider adding
> more enums in flags in vfio_irq_info to tell userspace that particular
> index is wired, MSI, MSIX etc. for the platform device.
> What do you think?
If count is not reliable to discriminate the first n wired interrupts
from the subsequen MSI and MSIx index, Alex suggested to add a
capability extension in the vfio_irq_info structure. Something similar
to what was done for vfio_region_info.

Such kind of thing was attempted in
https://lore.kernel.org/kvmarm/20201116110030.32335-8-eric.auger@redhat.com/T/#u

` [PATCH v11 07/13] vfio: Use capability chains to handle device
specific irq
` [PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices
` [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting

Note this has not been reviewed yet.

Thanks

Eric

>>
>>
>>
>>> With this implementation user space can know that, at indexes n and
>>> n+1, edge triggered interrupts are present.
>> note wired interrupts can also be edge ones.
>>>    We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
>>>    struct vfio_platform_irq{
>>>    .....
>>>    .....
>>>    struct vfio_platform_irq *block; => this points to the block
>>> allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
>> array of wired interrupts (the existing vdev->irqs) and a new array for
>> MSI(x) as done in the PCI code.
>>
>> vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
>>
>> Does it make sense?
> Yes, we can use similar kinds of allocations.
> 
> Thanks,
> Vikas
>>
>>>    };
>>>                          OR
>>> Another structure can be defined in 'vfio_pci_private.h'
>>> struct vfio_msi_ctx {
>>>         struct eventfd_ctx      *trigger;
>>>         char                    *name;
>>> };
>>> and
>>> struct vfio_platform_irq {
>>>   .....
>>>   .....
>>>   struct vfio_msi_ctx *block; => this points to the block allocation
>>> for MSIs/MSIXs
>>> };
>>> Which of the above two options sounds OK to you? Please suggest.
>>>
>>>> to me individual MSIs are encoded in the subindex and not in the index.
>>>> The index just selects the "type" of interrupt.
>>>>
>>>> For PCI you just have:
>>>>         VFIO_PCI_INTX_IRQ_INDEX,
>>>>         VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with
>>>> start/count
>>>>         VFIO_PCI_MSIX_IRQ_INDEX,
>>>>         VFIO_PCI_ERR_IRQ_INDEX,
>>>>         VFIO_PCI_REQ_IRQ_INDEX,
>>>>
>>>> (include/uapi/linux/vfio.h)
>>>
>>> In pci case, type of interrupts is fixed so they can be 'indexed' by
>>> these enums but for VFIO platform user space will need to iterate all
>>> (num_irqs) indexes to know at which indexes edge triggered interrupts
>>> are present.
>> indeed, but can't you loop over all indices looking until count !=1? At
>> this point you know if have finished emurating the wires. Holds if
>> MSI(x) count !=1 of course.
>>
>> Thanks
>>
>> Eric
>>
>>>
>>> Thanks,
>>> Vikas
>>>>
>>>> Thanks
>>>>
>>>> Eric
>>>>>        MSI-0 will have count=k set and flags set accordingly.
>>>>>
>>>>> Vikas Gupta (3):
>>>>>   vfio/platform: add support for msi
>>>>>   vfio/platform: change cleanup order
>>>>>   vfio/platform: add Broadcom msi module
>>>>>
>>>>>  drivers/vfio/platform/Kconfig                 |   1 +
>>>>>  drivers/vfio/platform/Makefile                |   1 +
>>>>>  drivers/vfio/platform/msi/Kconfig             |   9 +
>>>>>  drivers/vfio/platform/msi/Makefile            |   2 +
>>>>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
>>>>>  drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
>>>>>  drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
>>>>>  drivers/vfio/platform/vfio_platform_private.h |  23 ++
>>>>>  8 files changed, 419 insertions(+), 15 deletions(-)
>>>>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
>>>>>  create mode 100644 drivers/vfio/platform/msi/Makefile
>>>>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>>>>>
>>>>
>>

Re: [RFC, v1 0/3] msi support for platform devices

[Auger Eric]

Hi Vikas,

On 11/17/20 9:05 AM, Auger Eric wrote:
> Hi Vikas,
> 
> On 11/17/20 7:25 AM, Vikas Gupta wrote:
>> Hi Eric,
>>
>> On Mon, Nov 16, 2020 at 6:44 PM Auger Eric <eric.auger@redhat.com> wrote:
>>>
>>> Hi Vikas,
>>>
>>> On 11/13/20 6:24 PM, Vikas Gupta wrote:
>>>> Hi Eric,
>>>>
>>>> On Fri, Nov 13, 2020 at 12:10 AM Auger Eric <eric.auger@redhat.com> wrote:
>>>>>
>>>>> Hi Vikas,
>>>>>
>>>>> On 11/12/20 6:58 PM, Vikas Gupta wrote:
>>>>>> This RFC adds support for MSI for platform devices.
>>>>>> a) MSI(s) is/are added in addition to the normal interrupts.
>>>>>> b) The vendor specific MSI configuration can be done using
>>>>>>    callbacks which is implemented as msi module.
>>>>>> c) Adds a msi handling module for the Broadcom platform devices.
>>>>>>
>>>>>> Changes from:
>>>>>> -------------
>>>>>>  v0 to v1:
>>>>>>    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
>>>>>>    ii) Add MSI(s) at the end of the irq list of platform IRQs.
>>>>>>        MSI(s) with first entry of MSI block has count and flag
>>>>>>        information.
>>>>>>        IRQ list: Allocation for IRQs + MSIs are allocated as below
>>>>>>        Example: if there are 'n' IRQs and 'k' MSIs
>>>>>>        -------------------------------------------------------
>>>>>>        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
>>>>>>        -------------------------------------------------------
>>>>> I have not taken time yet to look at your series, but to me you should have
>>>>> |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
>>>>> then for setting a given MSIX (i) you would select the MSIx index and
>>>>> then set start=i count=1.
>>>>
>>>> As per your suggestion, we should have, if there are n-IRQs, k-MSIXs
>>>> and m-MSIs, allocation of IRQs should be done as below
>>>>
>>>> |IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
>>>>                                              |        |
>>>>                                              |
>>>> |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
>>>>                                              |MSI0||MSI1||MSI2|....|MSI-(m-1)|
>>> No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
>>> and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ
>>> if ERR/REQ were to be added.
>> I agree on this. Actually the map I drew incorrectly above but wanted
>> to demonstrate the same. It was a child-parent relationship for MSI
>> and its members and similarly for MSIX as well.
>>>
>>> I think the userspace could query the total number of indices using
>>> VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire
>>> interrupts + MSI index + MSIX index)
>>>
>>> Then userspace can loop on all the indices using
>>> VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the
>>> first indices related to wire interrupts (count = 1). Then comes the MSI
>>> index, and after the MSI index. If any of those is supported, count >1,
>>> otherwise count=0. The only thing I am dubious about is can the device
>>> use a single MSI/MSIX? Because my hypothesis here is we use count to
>>> discriminate between wire first indices and other indices.
>> I believe count can be one as well, especially for ERR/REQ as you
>> mentioned above.
> Given ERR and REQ indices would follow MSI and MSIX ones, MSI index
> could be recognized by the first index whose count != 1. But indeed I am
> not sure the number of supported vectors cannot be 1. In your case it is
> induced by the size of the ring so it is OK but for other devices this
> may be different.
> 
> I think we can not rely on the count > 1. Now, this is
>> blocking and we are not left with options unless we consider adding
>> more enums in flags in vfio_irq_info to tell userspace that particular
>> index is wired, MSI, MSIX etc. for the platform device.
>> What do you think?
> If count is not reliable to discriminate the first n wired interrupts
> from the subsequen MSI and MSIx index, Alex suggested to add a
> capability extension in the vfio_irq_info structure. Something similar
> to what was done for vfio_region_info.
> 
> Such kind of thing was attempted in
> https://lore.kernel.org/kvmarm/20201116110030.32335-8-eric.auger@redhat.com/T/#u
> 
> ` [PATCH v11 07/13] vfio: Use capability chains to handle device
> specific irq
> ` [PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices
> ` [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting

By the way I was mentionning MSI/MSIx in my previous reply but, as Alex
pointed out, with platform device only a single MSI index does make
sense, no?

Thanks

Eric
> 
> Note this has not been reviewed yet.
> 
> Thanks
> 
> Eric
> 
>>>
>>>
>>>
>>>> With this implementation user space can know that, at indexes n and
>>>> n+1, edge triggered interrupts are present.
>>> note wired interrupts can also be edge ones.
>>>>    We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
>>>>    struct vfio_platform_irq{
>>>>    .....
>>>>    .....
>>>>    struct vfio_platform_irq *block; => this points to the block
>>>> allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
>>> array of wired interrupts (the existing vdev->irqs) and a new array for
>>> MSI(x) as done in the PCI code.
>>>
>>> vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
>>>
>>> Does it make sense?
>> Yes, we can use similar kinds of allocations.
>>
>> Thanks,
>> Vikas
>>>
>>>>    };
>>>>                          OR
>>>> Another structure can be defined in 'vfio_pci_private.h'
>>>> struct vfio_msi_ctx {
>>>>         struct eventfd_ctx      *trigger;
>>>>         char                    *name;
>>>> };
>>>> and
>>>> struct vfio_platform_irq {
>>>>   .....
>>>>   .....
>>>>   struct vfio_msi_ctx *block; => this points to the block allocation
>>>> for MSIs/MSIXs
>>>> };
>>>> Which of the above two options sounds OK to you? Please suggest.
>>>>
>>>>> to me individual MSIs are encoded in the subindex and not in the index.
>>>>> The index just selects the "type" of interrupt.
>>>>>
>>>>> For PCI you just have:
>>>>>         VFIO_PCI_INTX_IRQ_INDEX,
>>>>>         VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with
>>>>> start/count
>>>>>         VFIO_PCI_MSIX_IRQ_INDEX,
>>>>>         VFIO_PCI_ERR_IRQ_INDEX,
>>>>>         VFIO_PCI_REQ_IRQ_INDEX,
>>>>>
>>>>> (include/uapi/linux/vfio.h)
>>>>
>>>> In pci case, type of interrupts is fixed so they can be 'indexed' by
>>>> these enums but for VFIO platform user space will need to iterate all
>>>> (num_irqs) indexes to know at which indexes edge triggered interrupts
>>>> are present.
>>> indeed, but can't you loop over all indices looking until count !=1? At
>>> this point you know if have finished emurating the wires. Holds if
>>> MSI(x) count !=1 of course.
>>>
>>> Thanks
>>>
>>> Eric
>>>
>>>>
>>>> Thanks,
>>>> Vikas
>>>>>
>>>>> Thanks
>>>>>
>>>>> Eric
>>>>>>        MSI-0 will have count=k set and flags set accordingly.
>>>>>>
>>>>>> Vikas Gupta (3):
>>>>>>   vfio/platform: add support for msi
>>>>>>   vfio/platform: change cleanup order
>>>>>>   vfio/platform: add Broadcom msi module
>>>>>>
>>>>>>  drivers/vfio/platform/Kconfig                 |   1 +
>>>>>>  drivers/vfio/platform/Makefile                |   1 +
>>>>>>  drivers/vfio/platform/msi/Kconfig             |   9 +
>>>>>>  drivers/vfio/platform/msi/Makefile            |   2 +
>>>>>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
>>>>>>  drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
>>>>>>  drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
>>>>>>  drivers/vfio/platform/vfio_platform_private.h |  23 ++
>>>>>>  8 files changed, 419 insertions(+), 15 deletions(-)
>>>>>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
>>>>>>  create mode 100644 drivers/vfio/platform/msi/Makefile
>>>>>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>>>>>>
>>>>>
>>>

Re: [RFC, v1 0/3] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 8645 bytes --]

Hi Eric,

On Tue, Nov 17, 2020 at 1:55 PM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 11/17/20 9:05 AM, Auger Eric wrote:
> > Hi Vikas,
> >
> > On 11/17/20 7:25 AM, Vikas Gupta wrote:
> >> Hi Eric,
> >>
> >> On Mon, Nov 16, 2020 at 6:44 PM Auger Eric <eric.auger@redhat.com> wrote:
> >>>
> >>> Hi Vikas,
> >>>
> >>> On 11/13/20 6:24 PM, Vikas Gupta wrote:
> >>>> Hi Eric,
> >>>>
> >>>> On Fri, Nov 13, 2020 at 12:10 AM Auger Eric <eric.auger@redhat.com> wrote:
> >>>>>
> >>>>> Hi Vikas,
> >>>>>
> >>>>> On 11/12/20 6:58 PM, Vikas Gupta wrote:
> >>>>>> This RFC adds support for MSI for platform devices.
> >>>>>> a) MSI(s) is/are added in addition to the normal interrupts.
> >>>>>> b) The vendor specific MSI configuration can be done using
> >>>>>>    callbacks which is implemented as msi module.
> >>>>>> c) Adds a msi handling module for the Broadcom platform devices.
> >>>>>>
> >>>>>> Changes from:
> >>>>>> -------------
> >>>>>>  v0 to v1:
> >>>>>>    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
> >>>>>>    ii) Add MSI(s) at the end of the irq list of platform IRQs.
> >>>>>>        MSI(s) with first entry of MSI block has count and flag
> >>>>>>        information.
> >>>>>>        IRQ list: Allocation for IRQs + MSIs are allocated as below
> >>>>>>        Example: if there are 'n' IRQs and 'k' MSIs
> >>>>>>        -------------------------------------------------------
> >>>>>>        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
> >>>>>>        -------------------------------------------------------
> >>>>> I have not taken time yet to look at your series, but to me you should have
> >>>>> |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
> >>>>> then for setting a given MSIX (i) you would select the MSIx index and
> >>>>> then set start=i count=1.
> >>>>
> >>>> As per your suggestion, we should have, if there are n-IRQs, k-MSIXs
> >>>> and m-MSIs, allocation of IRQs should be done as below
> >>>>
> >>>> |IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
> >>>>                                              |        |
> >>>>                                              |
> >>>> |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
> >>>>                                              |MSI0||MSI1||MSI2|....|MSI-(m-1)|
> >>> No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
> >>> and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ
> >>> if ERR/REQ were to be added.
> >> I agree on this. Actually the map I drew incorrectly above but wanted
> >> to demonstrate the same. It was a child-parent relationship for MSI
> >> and its members and similarly for MSIX as well.
> >>>
> >>> I think the userspace could query the total number of indices using
> >>> VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire
> >>> interrupts + MSI index + MSIX index)
> >>>
> >>> Then userspace can loop on all the indices using
> >>> VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the
> >>> first indices related to wire interrupts (count = 1). Then comes the MSI
> >>> index, and after the MSI index. If any of those is supported, count >1,
> >>> otherwise count=0. The only thing I am dubious about is can the device
> >>> use a single MSI/MSIX? Because my hypothesis here is we use count to
> >>> discriminate between wire first indices and other indices.
> >> I believe count can be one as well, especially for ERR/REQ as you
> >> mentioned above.
> > Given ERR and REQ indices would follow MSI and MSIX ones, MSI index
> > could be recognized by the first index whose count != 1. But indeed I am
> > not sure the number of supported vectors cannot be 1. In your case it is
> > induced by the size of the ring so it is OK but for other devices this
> > may be different.
> >
> > I think we can not rely on the count > 1. Now, this is
> >> blocking and we are not left with options unless we consider adding
> >> more enums in flags in vfio_irq_info to tell userspace that particular
> >> index is wired, MSI, MSIX etc. for the platform device.
> >> What do you think?
> > If count is not reliable to discriminate the first n wired interrupts
> > from the subsequen MSI and MSIx index, Alex suggested to add a
> > capability extension in the vfio_irq_info structure. Something similar
> > to what was done for vfio_region_info.
> >
> > Such kind of thing was attempted in
> > https://lore.kernel.org/kvmarm/20201116110030.32335-8-eric.auger@redhat.com/T/#u
> >
> > ` [PATCH v11 07/13] vfio: Use capability chains to handle device
> > specific irq
> > ` [PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices
> > ` [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting
>
> By the way I was mentionning MSI/MSIx in my previous reply but, as Alex
> pointed out, with platform device only a single MSI index does make
> sense, no?
Yes, I think single MSI should be OK.
This single MSI index should be implemented as ext_irqs, similar to,
as you implemented in the mentioned patch. Is my understanding
correct?
Thanks,
Vikas
>
> Thanks
>
> Eric
> >
> > Note this has not been reviewed yet.
> >
> > Thanks
> >
> > Eric
> >
> >>>
> >>>
> >>>
> >>>> With this implementation user space can know that, at indexes n and
> >>>> n+1, edge triggered interrupts are present.
> >>> note wired interrupts can also be edge ones.
> >>>>    We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
> >>>>    struct vfio_platform_irq{
> >>>>    .....
> >>>>    .....
> >>>>    struct vfio_platform_irq *block; => this points to the block
> >>>> allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
> >>> array of wired interrupts (the existing vdev->irqs) and a new array for
> >>> MSI(x) as done in the PCI code.
> >>>
> >>> vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
> >>>
> >>> Does it make sense?
> >> Yes, we can use similar kinds of allocations.
> >>
> >> Thanks,
> >> Vikas
> >>>
> >>>>    };
> >>>>                          OR
> >>>> Another structure can be defined in 'vfio_pci_private.h'
> >>>> struct vfio_msi_ctx {
> >>>>         struct eventfd_ctx      *trigger;
> >>>>         char                    *name;
> >>>> };
> >>>> and
> >>>> struct vfio_platform_irq {
> >>>>   .....
> >>>>   .....
> >>>>   struct vfio_msi_ctx *block; => this points to the block allocation
> >>>> for MSIs/MSIXs
> >>>> };
> >>>> Which of the above two options sounds OK to you? Please suggest.
> >>>>
> >>>>> to me individual MSIs are encoded in the subindex and not in the index.
> >>>>> The index just selects the "type" of interrupt.
> >>>>>
> >>>>> For PCI you just have:
> >>>>>         VFIO_PCI_INTX_IRQ_INDEX,
> >>>>>         VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with
> >>>>> start/count
> >>>>>         VFIO_PCI_MSIX_IRQ_INDEX,
> >>>>>         VFIO_PCI_ERR_IRQ_INDEX,
> >>>>>         VFIO_PCI_REQ_IRQ_INDEX,
> >>>>>
> >>>>> (include/uapi/linux/vfio.h)
> >>>>
> >>>> In pci case, type of interrupts is fixed so they can be 'indexed' by
> >>>> these enums but for VFIO platform user space will need to iterate all
> >>>> (num_irqs) indexes to know at which indexes edge triggered interrupts
> >>>> are present.
> >>> indeed, but can't you loop over all indices looking until count !=1? At
> >>> this point you know if have finished emurating the wires. Holds if
> >>> MSI(x) count !=1 of course.
> >>>
> >>> Thanks
> >>>
> >>> Eric
> >>>
> >>>>
> >>>> Thanks,
> >>>> Vikas
> >>>>>
> >>>>> Thanks
> >>>>>
> >>>>> Eric
> >>>>>>        MSI-0 will have count=k set and flags set accordingly.
> >>>>>>
> >>>>>> Vikas Gupta (3):
> >>>>>>   vfio/platform: add support for msi
> >>>>>>   vfio/platform: change cleanup order
> >>>>>>   vfio/platform: add Broadcom msi module
> >>>>>>
> >>>>>>  drivers/vfio/platform/Kconfig                 |   1 +
> >>>>>>  drivers/vfio/platform/Makefile                |   1 +
> >>>>>>  drivers/vfio/platform/msi/Kconfig             |   9 +
> >>>>>>  drivers/vfio/platform/msi/Makefile            |   2 +
> >>>>>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
> >>>>>>  drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
> >>>>>>  drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
> >>>>>>  drivers/vfio/platform/vfio_platform_private.h |  23 ++
> >>>>>>  8 files changed, 419 insertions(+), 15 deletions(-)
> >>>>>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
> >>>>>>  create mode 100644 drivers/vfio/platform/msi/Makefile
> >>>>>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> >>>>>>
> >>>>>
> >>>
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC, v1 0/3] msi support for platform devices

[Auger Eric]

Hi Vikas,

On 11/17/20 5:36 PM, Vikas Gupta wrote:
> Hi Eric,
> 
> On Tue, Nov 17, 2020 at 1:55 PM Auger Eric <eric.auger@redhat.com> wrote:
>>
>> Hi Vikas,
>>
>> On 11/17/20 9:05 AM, Auger Eric wrote:
>>> Hi Vikas,
>>>
>>> On 11/17/20 7:25 AM, Vikas Gupta wrote:
>>>> Hi Eric,
>>>>
>>>> On Mon, Nov 16, 2020 at 6:44 PM Auger Eric <eric.auger@redhat.com> wrote:
>>>>>
>>>>> Hi Vikas,
>>>>>
>>>>> On 11/13/20 6:24 PM, Vikas Gupta wrote:
>>>>>> Hi Eric,
>>>>>>
>>>>>> On Fri, Nov 13, 2020 at 12:10 AM Auger Eric <eric.auger@redhat.com> wrote:
>>>>>>>
>>>>>>> Hi Vikas,
>>>>>>>
>>>>>>> On 11/12/20 6:58 PM, Vikas Gupta wrote:
>>>>>>>> This RFC adds support for MSI for platform devices.
>>>>>>>> a) MSI(s) is/are added in addition to the normal interrupts.
>>>>>>>> b) The vendor specific MSI configuration can be done using
>>>>>>>>    callbacks which is implemented as msi module.
>>>>>>>> c) Adds a msi handling module for the Broadcom platform devices.
>>>>>>>>
>>>>>>>> Changes from:
>>>>>>>> -------------
>>>>>>>>  v0 to v1:
>>>>>>>>    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
>>>>>>>>    ii) Add MSI(s) at the end of the irq list of platform IRQs.
>>>>>>>>        MSI(s) with first entry of MSI block has count and flag
>>>>>>>>        information.
>>>>>>>>        IRQ list: Allocation for IRQs + MSIs are allocated as below
>>>>>>>>        Example: if there are 'n' IRQs and 'k' MSIs
>>>>>>>>        -------------------------------------------------------
>>>>>>>>        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
>>>>>>>>        -------------------------------------------------------
>>>>>>> I have not taken time yet to look at your series, but to me you should have
>>>>>>> |IRQ-0|IRQ-1|....|IRQ-n|MSI|MSIX
>>>>>>> then for setting a given MSIX (i) you would select the MSIx index and
>>>>>>> then set start=i count=1.
>>>>>>
>>>>>> As per your suggestion, we should have, if there are n-IRQs, k-MSIXs
>>>>>> and m-MSIs, allocation of IRQs should be done as below
>>>>>>
>>>>>> |IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
>>>>>>                                              |        |
>>>>>>                                              |
>>>>>> |MSIX0||MSIX1||MSXI2|....|MSIX-(k-1)|
>>>>>>                                              |MSI0||MSI1||MSI2|....|MSI-(m-1)|
>>>>> No I really meant this list of indices: IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX|
>>>>> and potentially later on IRQ0|IRQ1|......|IRQ-(n-1)|MSI|MSIX| ERR| REQ
>>>>> if ERR/REQ were to be added.
>>>> I agree on this. Actually the map I drew incorrectly above but wanted
>>>> to demonstrate the same. It was a child-parent relationship for MSI
>>>> and its members and similarly for MSIX as well.
>>>>>
>>>>> I think the userspace could query the total number of indices using
>>>>> VFIO_DEVICE_GET_INFO and retrieve num_irqs (corresponding to the n wire
>>>>> interrupts + MSI index + MSIX index)
>>>>>
>>>>> Then userspace can loop on all the indices using
>>>>> VFIO_DEVICE_GET_IRQ_INFO. For each index it uses count to determine the
>>>>> first indices related to wire interrupts (count = 1). Then comes the MSI
>>>>> index, and after the MSI index. If any of those is supported, count >1,
>>>>> otherwise count=0. The only thing I am dubious about is can the device
>>>>> use a single MSI/MSIX? Because my hypothesis here is we use count to
>>>>> discriminate between wire first indices and other indices.
>>>> I believe count can be one as well, especially for ERR/REQ as you
>>>> mentioned above.
>>> Given ERR and REQ indices would follow MSI and MSIX ones, MSI index
>>> could be recognized by the first index whose count != 1. But indeed I am
>>> not sure the number of supported vectors cannot be 1. In your case it is
>>> induced by the size of the ring so it is OK but for other devices this
>>> may be different.
>>>
>>> I think we can not rely on the count > 1. Now, this is
>>>> blocking and we are not left with options unless we consider adding
>>>> more enums in flags in vfio_irq_info to tell userspace that particular
>>>> index is wired, MSI, MSIX etc. for the platform device.
>>>> What do you think?
>>> If count is not reliable to discriminate the first n wired interrupts
>>> from the subsequen MSI and MSIx index, Alex suggested to add a
>>> capability extension in the vfio_irq_info structure. Something similar
>>> to what was done for vfio_region_info.
>>>
>>> Such kind of thing was attempted in
>>> https://lore.kernel.org/kvmarm/20201116110030.32335-8-eric.auger@redhat.com/T/#u
>>>
>>> ` [PATCH v11 07/13] vfio: Use capability chains to handle device
>>> specific irq
>>> ` [PATCH v11 08/13] vfio/pci: Add framework for custom interrupt indices
>>> ` [PATCH v11 09/13] vfio: Add new IRQ for DMA fault reporting
>>
>> By the way I was mentionning MSI/MSIx in my previous reply but, as Alex
>> pointed out, with platform device only a single MSI index does make
>> sense, no?
> Yes, I think single MSI should be OK.
> This single MSI index should be implemented as ext_irqs, similar to,
> as you implemented in the mentioned patch. Is my understanding
> correct?
Yes, if count !=1 cannot be used to detect the MSI index, I think using
a capability would do the job and this is aligned with last Alex'
suggestion.

Thanks

Eric
> Thanks,
> Vikas
>>
>> Thanks
>>
>> Eric
>>>
>>> Note this has not been reviewed yet.
>>>
>>> Thanks
>>>
>>> Eric
>>>
>>>>>
>>>>>
>>>>>
>>>>>> With this implementation user space can know that, at indexes n and
>>>>>> n+1, edge triggered interrupts are present.
>>>>> note wired interrupts can also be edge ones.
>>>>>>    We may add an element in vfio_platform_irq itself to allocate MSIs/MSIXs
>>>>>>    struct vfio_platform_irq{
>>>>>>    .....
>>>>>>    .....
>>>>>>    struct vfio_platform_irq *block; => this points to the block
>>>>>> allocation for MSIs/MSIXs and all msi/msix are type of IRQs.As wired interrupts and MSI interrupts coexist, I would store in vdev an
>>>>> array of wired interrupts (the existing vdev->irqs) and a new array for
>>>>> MSI(x) as done in the PCI code.
>>>>>
>>>>> vdev->ctx = kcalloc(nvec, sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL);
>>>>>
>>>>> Does it make sense?
>>>> Yes, we can use similar kinds of allocations.
>>>>
>>>> Thanks,
>>>> Vikas
>>>>>
>>>>>>    };
>>>>>>                          OR
>>>>>> Another structure can be defined in 'vfio_pci_private.h'
>>>>>> struct vfio_msi_ctx {
>>>>>>         struct eventfd_ctx      *trigger;
>>>>>>         char                    *name;
>>>>>> };
>>>>>> and
>>>>>> struct vfio_platform_irq {
>>>>>>   .....
>>>>>>   .....
>>>>>>   struct vfio_msi_ctx *block; => this points to the block allocation
>>>>>> for MSIs/MSIXs
>>>>>> };
>>>>>> Which of the above two options sounds OK to you? Please suggest.
>>>>>>
>>>>>>> to me individual MSIs are encoded in the subindex and not in the index.
>>>>>>> The index just selects the "type" of interrupt.
>>>>>>>
>>>>>>> For PCI you just have:
>>>>>>>         VFIO_PCI_INTX_IRQ_INDEX,
>>>>>>>         VFIO_PCI_MSI_IRQ_INDEX, -> MSI index and then you play with
>>>>>>> start/count
>>>>>>>         VFIO_PCI_MSIX_IRQ_INDEX,
>>>>>>>         VFIO_PCI_ERR_IRQ_INDEX,
>>>>>>>         VFIO_PCI_REQ_IRQ_INDEX,
>>>>>>>
>>>>>>> (include/uapi/linux/vfio.h)
>>>>>>
>>>>>> In pci case, type of interrupts is fixed so they can be 'indexed' by
>>>>>> these enums but for VFIO platform user space will need to iterate all
>>>>>> (num_irqs) indexes to know at which indexes edge triggered interrupts
>>>>>> are present.
>>>>> indeed, but can't you loop over all indices looking until count !=1? At
>>>>> this point you know if have finished emurating the wires. Holds if
>>>>> MSI(x) count !=1 of course.
>>>>>
>>>>> Thanks
>>>>>
>>>>> Eric
>>>>>
>>>>>>
>>>>>> Thanks,
>>>>>> Vikas
>>>>>>>
>>>>>>> Thanks
>>>>>>>
>>>>>>> Eric
>>>>>>>>        MSI-0 will have count=k set and flags set accordingly.
>>>>>>>>
>>>>>>>> Vikas Gupta (3):
>>>>>>>>   vfio/platform: add support for msi
>>>>>>>>   vfio/platform: change cleanup order
>>>>>>>>   vfio/platform: add Broadcom msi module
>>>>>>>>
>>>>>>>>  drivers/vfio/platform/Kconfig                 |   1 +
>>>>>>>>  drivers/vfio/platform/Makefile                |   1 +
>>>>>>>>  drivers/vfio/platform/msi/Kconfig             |   9 +
>>>>>>>>  drivers/vfio/platform/msi/Makefile            |   2 +
>>>>>>>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  |  74 ++++++
>>>>>>>>  drivers/vfio/platform/vfio_platform_common.c  |  86 ++++++-
>>>>>>>>  drivers/vfio/platform/vfio_platform_irq.c     | 238 +++++++++++++++++-
>>>>>>>>  drivers/vfio/platform/vfio_platform_private.h |  23 ++
>>>>>>>>  8 files changed, 419 insertions(+), 15 deletions(-)
>>>>>>>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
>>>>>>>>  create mode 100644 drivers/vfio/platform/msi/Makefile
>>>>>>>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>>>>>>>>
>>>>>>>
>>>>>
>>

[RFC, v2 0/1] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 1744 bytes --]

This RFC adds support for MSI for platform devices.
MSI block is added as an ext irq along with the existing
wired interrupt implementation.

Changes from:
-------------
 v1 to v2:
	1) IRQ allocation has been implemented as below:
	       ----------------------------
	       |IRQ-0|IRQ-1|....|IRQ-n|MSI|
       	       ----------------------------
		MSI block has msi contexts and its implemneted
		as ext irq.

	2) Removed vendor specific module for msi handling so
	   previously patch2 and patch3 are not required.

	3) MSI related data is exported to userspace using 'caps'.
	 Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
	is taken from the Eric`s patch
        https://patchwork.kernel.org/project/kvm/patch/20201116110030.32335-8-eric.auger@redhat.com/


 v0 to v1:
   i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
   ii) Add MSI(s) at the end of the irq list of platform IRQs.
       MSI(s) with first entry of MSI block has count and flag
       information.
       IRQ list: Allocation for IRQs + MSIs are allocated as below
       Example: if there are 'n' IRQs and 'k' MSIs
       -------------------------------------------------------
       |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
       -------------------------------------------------------
       MSI-0 will have count=k set and flags set accordingly.

Vikas Gupta (1):
  vfio/platform: add support for msi

 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)

-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC v2 1/1] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 15165 bytes --]

MSI support for platform devices.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  16 ++
 include/uapi/linux/vfio.h                     |  43 +++
 4 files changed, 401 insertions(+), 17 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index c0771a9567fb..b0bfc0f4ee1f 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
+		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+		struct vfio_irq_info_cap_msi *msi_info = NULL;
+		unsigned long capsz;
+		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
 
 		minsz = offsetofend(struct vfio_irq_info, count);
 
+		/* For backward compatibility, cannot require this */
+		capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 
@@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
 		if (info.index >= vdev->num_irqs)
 			return -EINVAL;
 
+		if (info.argsz >= capsz)
+			minsz = capsz;
+
+		if (info.index == ext_irq_index) {
+			struct vfio_irq_info_cap_type cap_type = {
+				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
+				.header.version = 1 };
+			int i;
+			int ret;
+			int num_msgs;
+			size_t msi_info_size;
+			struct vfio_platform_irq *irq;
+
+			info.index = array_index_nospec(info.index,
+							vdev->num_irqs);
+
+			irq = &vdev->irqs[info.index];
+
+			info.flags = irq->flags;
+			cap_type.type = irq->type;
+			cap_type.subtype = irq->subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						       &cap_type.header,
+						       sizeof(cap_type));
+			if (ret)
+				return ret;
+
+			num_msgs = irq->num_ctx;
+
+			msi_info_size = struct_size(msi_info, msgs, num_msgs);
+
+			msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+			if (!msi_info) {
+				kfree(caps.buf);
+				return -ENOMEM;
+			}
+
+			msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
+			msi_info->header.version = 1;
+			msi_info->nr_msgs = num_msgs;
+
+			for (i = 0; i < num_msgs; i++) {
+				struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+				msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
+				msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
+				msi_info->msgs[i].data = ctx->msg.data;
+			}
+
+			ret = vfio_info_add_capability(&caps, &msi_info->header,
+						       msi_info_size);
+			if (ret) {
+				kfree(msi_info);
+				kfree(caps.buf);
+				return ret;
+			}
+		}
+
 		info.flags = vdev->irqs[info.index].flags;
 		info.count = vdev->irqs[info.index].count;
 
+		if (caps.size) {
+			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+			if (info.argsz < sizeof(info) + caps.size) {
+				info.argsz = sizeof(info) + caps.size;
+				info.cap_offset = 0;
+			} else {
+				vfio_info_cap_shift(&caps, sizeof(info));
+				if (copy_to_user((void __user *)arg +
+						  sizeof(info), caps.buf,
+						  caps.size)) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return -EFAULT;
+				}
+				info.cap_offset = sizeof(info);
+			}
+
+			kfree(msi_info);
+			kfree(caps.buf);
+		}
+
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
 
@@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-						 vdev->num_irqs, &data_size);
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+							 vdev->num_irqs,
+							 &data_size);
 		if (ret)
 			return ret;
 
diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index c5b09ec0a3c9..4066223e5b2e 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	int i;
+	struct vfio_platform_irq *irq;
+	u16 index = desc->platform.msi_index;
+	struct device *dev = msi_desc_to_dev(desc);
+	struct vfio_device *device = dev_get_drvdata(dev);
+	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+						vfio_device_data(device);
+
+	for (i = 0; i < vdev->num_irqs; i++)
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			irq = &vdev->irqs[i];
+
+	irq->ctx[index].msg.address_lo = msg->address_lo;
+	irq->ctx[index].msg.address_hi = msg->address_hi;
+	irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+			   struct vfio_platform_irq *irq, int nvec)
+{
+	int ret;
+	int msi_idx = 0;
+	struct msi_desc *desc;
+	struct device *dev = vdev->device;
+
+	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+	if (!irq->ctx)
+		return -ENOMEM;
+
+	/* Allocate platform MSIs */
+	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+	if (ret < 0) {
+		kfree(irq->ctx);
+		return ret;
+	}
+
+	for_each_msi_entry(desc, dev) {
+		irq->ctx[msi_idx].hwirq = desc->irq;
+		msi_idx++;
+	}
+
+	irq->num_ctx = nvec;
+	irq->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+				      int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq_num, ret;
+
+	if (vector < 0 || vector >= irq->num_ctx)
+		return -EINVAL;
+
+	irq_num = irq->ctx[vector].hwirq;
+
+	if (irq->ctx[vector].trigger) {
+		free_irq(irq_num, irq->ctx[vector].trigger);
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(irq->ctx[vector].trigger);
+		irq->ctx[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+					  "vfio-msi[%d]", vector);
+	if (!irq->ctx[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(irq->ctx[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq_num, vfio_msihandler, 0,
+			  irq->ctx[vector].name, trigger);
+	if (ret) {
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	irq->ctx[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+			      unsigned int count, int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= irq->num_ctx || start + count > irq->num_ctx)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_msi_set_vector_signal(irq, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_msi_set_vector_signal(irq, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+			     struct vfio_platform_irq *irq)
+{
+	struct device *dev = vdev->device;
+
+	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+	platform_msi_domain_free_irqs(dev);
+
+	irq->config_msi = 0;
+	irq->num_ctx = 0;
+
+	kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+				unsigned int index, unsigned int start,
+				unsigned int count, uint32_t flags, void *data)
+{
+	int i;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (start + count > irq->count)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_msi_disable(vdev, irq);
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (irq->config_msi)
+			return vfio_msi_set_block(irq, start, count,
+						  fds);
+		ret = vfio_msi_enable(vdev, irq, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_msi_set_block(irq, start, count, fds);
+		if (ret)
+			vfio_msi_disable(vdev, irq);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!irq->ctx[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(irq->ctx[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(irq->ctx[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 				 uint32_t flags, unsigned index, unsigned start,
 				 unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 		    unsigned start, unsigned count, uint32_t flags,
 		    void *data) = NULL;
 
-	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-	case VFIO_IRQ_SET_ACTION_MASK:
-		func = vfio_platform_set_irq_mask;
-		break;
-	case VFIO_IRQ_SET_ACTION_UNMASK:
-		func = vfio_platform_set_irq_unmask;
-		break;
-	case VFIO_IRQ_SET_ACTION_TRIGGER:
-		func = vfio_platform_set_irq_trigger;
-		break;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (irq->type == VFIO_IRQ_TYPE_MSI) {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_set_msi_trigger;
+			break;
+		}
+	} else {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+			func = vfio_platform_set_irq_mask;
+			break;
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			func = vfio_platform_set_irq_unmask;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_platform_set_irq_trigger;
+			break;
+		}
 	}
 
 	if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-	int cnt = 0, i;
+	int i;
+	int cnt = 0;
+	int num_irqs;
+	struct device *dev = vdev->device;
 
 	while (vdev->get_irq(vdev, cnt) >= 0)
 		cnt++;
 
-	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	num_irqs = cnt;
+
+	if (dev->msi_domain)
+		num_irqs++;
+
+	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+			     GFP_KERNEL);
 	if (!vdev->irqs)
 		return -ENOMEM;
 
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		vdev->irqs[i].masked = false;
 	}
 
-	vdev->num_irqs = cnt;
+	/*
+	 * MSI block is added at last index and its an ext irq
+	 */
+	if (dev->msi_domain) {
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+		vdev->irqs[i].count = NR_IRQS;
+		vdev->irqs[i].hwirq = 0;
+		vdev->irqs[i].masked = false;
+		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+		vdev->num_ext_irqs = 1;
+	}
+
+	vdev->num_irqs = num_irqs;
 
 	return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
 	int i;
 
-	for (i = 0; i < vdev->num_irqs; i++)
-		vfio_set_trigger(vdev, i, -1, NULL);
+	for (i = 0; i < vdev->num_irqs; i++) {
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			vfio_set_msi_trigger(vdev, i, 0, 0,
+					     VFIO_IRQ_SET_DATA_NONE, NULL);
+		else
+			vfio_set_trigger(vdev, i, -1, NULL);
+	}
 
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index 289089910643..7bbb05988705 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,18 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
@@ -29,6 +39,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +61,7 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 2f313a238a8f..598d1c944283 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -697,11 +697,54 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
 	__u32	index;		/* IRQ index */
 	__u32	count;		/* Number of IRQs within this index */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE		3
+
+struct vfio_irq_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;     /* global per bus driver */
+	__u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI		(1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS	4
+
+struct vfio_irq_msi_msg {
+	__u32	addr_lo;
+	__u32	addr_hi;
+	__u32	data;
+};
+
+struct vfio_irq_info_cap_msi {
+	struct vfio_info_cap_header header;
+	__u32 nr_msgs;
+	struct vfio_irq_msi_msg msgs[];
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v2 1/1] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 11/24/20 5:16 PM, Vikas Gupta wrote:
> MSI support for platform devices.
> 
> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> ---
>  drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
>  drivers/vfio/platform/vfio_platform_private.h |  16 ++
>  include/uapi/linux/vfio.h                     |  43 +++
>  4 files changed, 401 insertions(+), 17 deletions(-)
> 
> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> index c0771a9567fb..b0bfc0f4ee1f 100644
> --- a/drivers/vfio/platform/vfio_platform_common.c
> +++ b/drivers/vfio/platform/vfio_platform_common.c
> @@ -16,6 +16,7 @@
>  #include <linux/types.h>
>  #include <linux/uaccess.h>
>  #include <linux/vfio.h>
> +#include <linux/nospec.h>
>  
>  #include "vfio_platform_private.h"
>  
> @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
>  
>  	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
>  		struct vfio_irq_info info;
> +		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
> +		struct vfio_irq_info_cap_msi *msi_info = NULL;
> +		unsigned long capsz;
> +		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
>  
>  		minsz = offsetofend(struct vfio_irq_info, count);
>  
> +		/* For backward compatibility, cannot require this */
> +		capsz = offsetofend(struct vfio_irq_info, cap_offset);
> +
>  		if (copy_from_user(&info, (void __user *)arg, minsz))
>  			return -EFAULT;
>  
> @@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
>  		if (info.index >= vdev->num_irqs)
>  			return -EINVAL;
>  
> +		if (info.argsz >= capsz)
> +			minsz = capsz;
> +
> +		if (info.index == ext_irq_index) {
nit: n case we add new ext indices afterwards, I would check info.index
-  ext_irq_index against an VFIO_EXT_IRQ_MSI define.
> +			struct vfio_irq_info_cap_type cap_type = {
> +				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
> +				.header.version = 1 };
> +			int i;
> +			int ret;
> +			int num_msgs;
> +			size_t msi_info_size;
> +			struct vfio_platform_irq *irq;
nit: I think generally the opposite order (length) is chosen. This also
would better match the existing style in this file
> +
> +			info.index = array_index_nospec(info.index,
> +							vdev->num_irqs);
> +
> +			irq = &vdev->irqs[info.index];
> +
> +			info.flags = irq->flags;
I think this can be removed given [*]
> +			cap_type.type = irq->type;
> +			cap_type.subtype = irq->subtype;
> +
> +			ret = vfio_info_add_capability(&caps,
> +						       &cap_type.header,
> +						       sizeof(cap_type));
> +			if (ret)
> +				return ret;
> +
> +			num_msgs = irq->num_ctx;
do would want to return the cap even if !num_ctx?
> +
> +			msi_info_size = struct_size(msi_info, msgs, num_msgs);
> +
> +			msi_info = kzalloc(msi_info_size, GFP_KERNEL);
> +			if (!msi_info) {
> +				kfree(caps.buf);
> +				return -ENOMEM;
> +			}
> +
> +			msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
> +			msi_info->header.version = 1;
> +			msi_info->nr_msgs = num_msgs;
> +
> +			for (i = 0; i < num_msgs; i++) {
> +				struct vfio_irq_ctx *ctx = &irq->ctx[i];
> +
> +				msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
> +				msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
> +				msi_info->msgs[i].data = ctx->msg.data;
> +			}
> +
> +			ret = vfio_info_add_capability(&caps, &msi_info->header,
> +						       msi_info_size);
> +			if (ret) {
> +				kfree(msi_info);
> +				kfree(caps.buf);
> +				return ret;
> +			}
> +		}
> +
>  		info.flags = vdev->irqs[info.index].flags;
[*]
>  		info.count = vdev->irqs[info.index].count;
>  
> +		if (caps.size) {
> +			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
> +			if (info.argsz < sizeof(info) + caps.size) {
> +				info.argsz = sizeof(info) + caps.size;
> +				info.cap_offset = 0;
> +			} else {
> +				vfio_info_cap_shift(&caps, sizeof(info));
> +				if (copy_to_user((void __user *)arg +
> +						  sizeof(info), caps.buf,
> +						  caps.size)) {
> +					kfree(msi_info);
> +					kfree(caps.buf);
> +					return -EFAULT;
> +				}
> +				info.cap_offset = sizeof(info);
> +			}
> +
> +			kfree(msi_info);
> +			kfree(caps.buf);
> +		}
> +
>  		return copy_to_user((void __user *)arg, &info, minsz) ?
>  			-EFAULT : 0;
>  
> @@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
>  		struct vfio_irq_set hdr;
>  		u8 *data = NULL;
>  		int ret = 0;
> +		int max;
>  		size_t data_size = 0;
>  
>  		minsz = offsetofend(struct vfio_irq_set, count);
> @@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
>  		if (copy_from_user(&hdr, (void __user *)arg, minsz))
>  			return -EFAULT;
>  
> -		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
> -						 vdev->num_irqs, &data_size);
> +		if (hdr.index >= vdev->num_irqs)
> +			return -EINVAL;
> +
> +		max = vdev->irqs[hdr.index].count;
> +
> +		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
> +							 vdev->num_irqs,
> +							 &data_size);
>  		if (ret)
>  			return ret;
>  
> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> index c5b09ec0a3c9..4066223e5b2e 100644
> --- a/drivers/vfio/platform/vfio_platform_irq.c
> +++ b/drivers/vfio/platform/vfio_platform_irq.c
> @@ -8,10 +8,12 @@
>  
>  #include <linux/eventfd.h>
>  #include <linux/interrupt.h>
> +#include <linux/eventfd.h>
>  #include <linux/slab.h>
>  #include <linux/types.h>
>  #include <linux/vfio.h>
>  #include <linux/irq.h>
> +#include <linux/msi.h>
>  
>  #include "vfio_platform_private.h"
>  
> @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>  	return 0;
>  }
>  
> +/* MSI/MSIX */
> +static irqreturn_t vfio_msihandler(int irq, void *arg)
> +{
> +	struct eventfd_ctx *trigger = arg;
> +
> +	eventfd_signal(trigger, 1);
> +	return IRQ_HANDLED;
> +}
> +
> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
> +{
> +	int i;
> +	struct vfio_platform_irq *irq;
> +	u16 index = desc->platform.msi_index;
> +	struct device *dev = msi_desc_to_dev(desc);
> +	struct vfio_device *device = dev_get_drvdata(dev);
> +	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
> +						vfio_device_data(device);
> +
> +	for (i = 0; i < vdev->num_irqs; i++)
> +		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> +			irq = &vdev->irqs[i];
> +
> +	irq->ctx[index].msg.address_lo = msg->address_lo;
> +	irq->ctx[index].msg.address_hi = msg->address_hi;
> +	irq->ctx[index].msg.data = msg->data;
> +}
> +
> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
> +			   struct vfio_platform_irq *irq, int nvec)
> +{
> +	int ret;
> +	int msi_idx = 0;
> +	struct msi_desc *desc;
> +	struct device *dev = vdev->device;
> +
> +	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
> +	if (!irq->ctx)
> +		return -ENOMEM;
> +
> +	/* Allocate platform MSIs */
> +	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
> +	if (ret < 0) {
> +		kfree(irq->ctx);
> +		return ret;
> +	}
> +
> +	for_each_msi_entry(desc, dev) {
> +		irq->ctx[msi_idx].hwirq = desc->irq;
> +		msi_idx++;
> +	}
> +
> +	irq->num_ctx = nvec;
> +	irq->config_msi = 1;
> +
> +	return 0;
> +}
> +
> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
> +				      int vector, int fd)
> +{
> +	struct eventfd_ctx *trigger;
> +	int irq_num, ret;
> +
> +	if (vector < 0 || vector >= irq->num_ctx)
> +		return -EINVAL;
> +
> +	irq_num = irq->ctx[vector].hwirq;
> +
> +	if (irq->ctx[vector].trigger) {
> +		free_irq(irq_num, irq->ctx[vector].trigger);
> +		kfree(irq->ctx[vector].name);
> +		eventfd_ctx_put(irq->ctx[vector].trigger);
> +		irq->ctx[vector].trigger = NULL;
> +	}
> +
> +	if (fd < 0)
> +		return 0;
> +
> +	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
> +					  "vfio-msi[%d]", vector);
> +	if (!irq->ctx[vector].name)
> +		return -ENOMEM;
> +
> +	trigger = eventfd_ctx_fdget(fd);
> +	if (IS_ERR(trigger)) {
> +		kfree(irq->ctx[vector].name);
> +		return PTR_ERR(trigger);
> +	}
> +
> +	ret = request_irq(irq_num, vfio_msihandler, 0,
> +			  irq->ctx[vector].name, trigger);
> +	if (ret) {
> +		kfree(irq->ctx[vector].name);
> +		eventfd_ctx_put(trigger);
> +		return ret;
> +	}
> +
> +	irq->ctx[vector].trigger = trigger;
> +
> +	return 0;
> +}
> +
> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
> +			      unsigned int count, int32_t *fds)
> +{
> +	int i, j, ret = 0;
> +
> +	if (start >= irq->num_ctx || start + count > irq->num_ctx)
> +		return -EINVAL;
> +
> +	for (i = 0, j = start; i < count && !ret; i++, j++) {
> +		int fd = fds ? fds[i] : -1;
> +
> +		ret = vfio_msi_set_vector_signal(irq, j, fd);
> +	}
> +
> +	if (ret) {
> +		for (--j; j >= (int)start; j--)
> +			vfio_msi_set_vector_signal(irq, j, -1);
> +	}
> +
> +	return ret;
> +}
> +
> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
> +			     struct vfio_platform_irq *irq)
> +{
> +	struct device *dev = vdev->device;
> +
> +	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
> +
> +	platform_msi_domain_free_irqs(dev);
> +
> +	irq->config_msi = 0;
> +	irq->num_ctx = 0;
> +
> +	kfree(irq->ctx);
> +}
> +
> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
> +				unsigned int index, unsigned int start,
> +				unsigned int count, uint32_t flags, void *data)
> +{
> +	int i;
> +	struct vfio_platform_irq *irq = &vdev->irqs[index];
> +
> +	if (start + count > irq->count)
> +		return -EINVAL;
> +
> +	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
> +		vfio_msi_disable(vdev, irq);
> +		return 0;
> +	}
> +
> +	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
> +		s32 *fds = data;
> +		int ret;
> +
> +		if (irq->config_msi)
> +			return vfio_msi_set_block(irq, start, count,
> +						  fds);
> +		ret = vfio_msi_enable(vdev, irq, start + count);
> +		if (ret)
> +			return ret;
> +
> +		ret = vfio_msi_set_block(irq, start, count, fds);
> +		if (ret)
> +			vfio_msi_disable(vdev, irq);
> +
> +		return ret;
> +	}
> +
> +	for (i = start; i < start + count; i++) {
> +		if (!irq->ctx[i].trigger)
> +			continue;
> +		if (flags & VFIO_IRQ_SET_DATA_NONE) {
> +			eventfd_signal(irq->ctx[i].trigger, 1);
> +		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> +			u8 *bools = data;
> +
> +			if (bools[i - start])
> +				eventfd_signal(irq->ctx[i].trigger, 1);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  				 uint32_t flags, unsigned index, unsigned start,
>  				 unsigned count, void *data)
> @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  		    unsigned start, unsigned count, uint32_t flags,
>  		    void *data) = NULL;
>  
> -	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> -	case VFIO_IRQ_SET_ACTION_MASK:
> -		func = vfio_platform_set_irq_mask;
> -		break;
> -	case VFIO_IRQ_SET_ACTION_UNMASK:
> -		func = vfio_platform_set_irq_unmask;
> -		break;
> -	case VFIO_IRQ_SET_ACTION_TRIGGER:
> -		func = vfio_platform_set_irq_trigger;
> -		break;
> +	struct vfio_platform_irq *irq = &vdev->irqs[index];
> +
> +	if (irq->type == VFIO_IRQ_TYPE_MSI) {
> +		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> +		case VFIO_IRQ_SET_ACTION_MASK:
> +		case VFIO_IRQ_SET_ACTION_UNMASK:
> +			break;
> +		case VFIO_IRQ_SET_ACTION_TRIGGER:
> +			func = vfio_set_msi_trigger;
> +			break;
> +		}
> +	} else {
> +		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> +		case VFIO_IRQ_SET_ACTION_MASK:
> +			func = vfio_platform_set_irq_mask;
> +			break;
> +		case VFIO_IRQ_SET_ACTION_UNMASK:
> +			func = vfio_platform_set_irq_unmask;
> +			break;
> +		case VFIO_IRQ_SET_ACTION_TRIGGER:
> +			func = vfio_platform_set_irq_trigger;
> +			break;
> +		}
>  	}
>  
>  	if (!func)
> @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  
>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>  {
> -	int cnt = 0, i;
> +	int i;
> +	int cnt = 0;
> +	int num_irqs;
> +	struct device *dev = vdev->device;
>  
>  	while (vdev->get_irq(vdev, cnt) >= 0)
>  		cnt++;
>  
> -	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
> +	num_irqs = cnt;
> +
> +	if (dev->msi_domain)
> +		num_irqs++;
> +
> +	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
> +			     GFP_KERNEL);
>  	if (!vdev->irqs)
>  		return -ENOMEM;
>  
> @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>  		vdev->irqs[i].masked = false;
>  	}
>  
> -	vdev->num_irqs = cnt;
> +	/*
> +	 * MSI block is added at last index and its an ext irq
it is
> +	 */
> +	if (dev->msi_domain) {
> +		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
> +		vdev->irqs[i].count = NR_IRQS;
why NR_IRQS?
> +		vdev->irqs[i].hwirq = 0;
> +		vdev->irqs[i].masked = false;
> +		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
> +		vdev->num_ext_irqs = 1;
> +	}
> +
> +	vdev->num_irqs = num_irqs;
>  
>  	return 0;
>  err:
> @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>  {
>  	int i;
>  
> -	for (i = 0; i < vdev->num_irqs; i++)
> -		vfio_set_trigger(vdev, i, -1, NULL);
> +	for (i = 0; i < vdev->num_irqs; i++) {
> +		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> +			vfio_set_msi_trigger(vdev, i, 0, 0,
> +					     VFIO_IRQ_SET_DATA_NONE, NULL);
> +		else
> +			vfio_set_trigger(vdev, i, -1, NULL);
> +	}
>  
>  	vdev->num_irqs = 0;
>  	kfree(vdev->irqs);
> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> index 289089910643..7bbb05988705 100644
> --- a/drivers/vfio/platform/vfio_platform_private.h
> +++ b/drivers/vfio/platform/vfio_platform_private.h
> @@ -9,6 +9,7 @@
>  
>  #include <linux/types.h>
>  #include <linux/interrupt.h>
> +#include <linux/msi.h>
>  
>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
> @@ -19,9 +20,18 @@
>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
>  	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>  
> +struct vfio_irq_ctx {
> +	int			hwirq;
> +	char			*name;
> +	struct msi_msg		msg;
> +	struct eventfd_ctx	*trigger;
> +};
> +
>  struct vfio_platform_irq {
>  	u32			flags;
>  	u32			count;
> +	int			num_ctx;
> +	struct vfio_irq_ctx	*ctx;
>  	int			hwirq;
>  	char			*name;
>  	struct eventfd_ctx	*trigger;
> @@ -29,6 +39,11 @@ struct vfio_platform_irq {
>  	spinlock_t		lock;
>  	struct virqfd		*unmask;
>  	struct virqfd		*mask;
> +
> +	/* for extended irqs */
> +	u32			type;
> +	u32			subtype;
> +	int			config_msi;
>  };
>  
>  struct vfio_platform_region {
> @@ -46,6 +61,7 @@ struct vfio_platform_device {
>  	u32				num_regions;
>  	struct vfio_platform_irq	*irqs;
>  	u32				num_irqs;
> +	int				num_ext_irqs;
>  	int				refcnt;
>  	struct mutex			igate;
>  	struct module			*parent_module;
> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index 2f313a238a8f..598d1c944283 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -697,11 +697,54 @@ struct vfio_irq_info {
>  #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
>  #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
>  #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
> +#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
>  	__u32	index;		/* IRQ index */
>  	__u32	count;		/* Number of IRQs within this index */
> +	__u32	cap_offset;	/* Offset within info struct of first cap */
>  };
>  #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
>  
> +/*
> + * The irq type capability allows IRQs unique to a specific device or
> + * class of devices to be exposed.
> + *
> + * The structures below define version 1 of this capability.
> + */
> +#define VFIO_IRQ_INFO_CAP_TYPE		3
> +
> +struct vfio_irq_info_cap_type {
> +	struct vfio_info_cap_header header;
> +	__u32 type;     /* global per bus driver */
> +	__u32 subtype;  /* type specific */
> +};
> +
> +/*
> + * List of IRQ types, global per bus driver.
> + * If you introduce a new type, please add it here.
> + */
> +
> +/* Non PCI devices having MSI(s) support */
> +#define VFIO_IRQ_TYPE_MSI		(1)
> +
> +/*
> + * The msi capability allows the user to use the msi msg to
> + * configure the iova for the msi configuration.
> + * The structures below define version 1 of this capability.
> + */
> +#define VFIO_IRQ_INFO_CAP_MSI_DESCS	4
> +
> +struct vfio_irq_msi_msg {
> +	__u32	addr_lo;
> +	__u32	addr_hi;
> +	__u32	data;
> +};
> +
> +struct vfio_irq_info_cap_msi {
> +	struct vfio_info_cap_header header;
> +	__u32 nr_msgs;
I think you should align a __u32   reserved field to have a 64b alignment
> +	struct vfio_irq_msi_msg msgs[];
Please can you clarify why this cap is needed versus your prior approach.
> +};
> +
>  /**
>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
>   *
> 
Thanks

Eric

Re: [RFC v2 1/1] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 23277 bytes --]

Hi Eric,

On Wed, Dec 2, 2020 at 8:14 PM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 11/24/20 5:16 PM, Vikas Gupta wrote:
> > MSI support for platform devices.
> >
> > Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> > ---
> >  drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
> >  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
> >  drivers/vfio/platform/vfio_platform_private.h |  16 ++
> >  include/uapi/linux/vfio.h                     |  43 +++
> >  4 files changed, 401 insertions(+), 17 deletions(-)
> >
> > diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> > index c0771a9567fb..b0bfc0f4ee1f 100644
> > --- a/drivers/vfio/platform/vfio_platform_common.c
> > +++ b/drivers/vfio/platform/vfio_platform_common.c
> > @@ -16,6 +16,7 @@
> >  #include <linux/types.h>
> >  #include <linux/uaccess.h>
> >  #include <linux/vfio.h>
> > +#include <linux/nospec.h>
> >
> >  #include "vfio_platform_private.h"
> >
> > @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
> >
> >       } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
> >               struct vfio_irq_info info;
> > +             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
> > +             struct vfio_irq_info_cap_msi *msi_info = NULL;
> > +             unsigned long capsz;
> > +             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
> >
> >               minsz = offsetofend(struct vfio_irq_info, count);
> >
> > +             /* For backward compatibility, cannot require this */
> > +             capsz = offsetofend(struct vfio_irq_info, cap_offset);
> > +
> >               if (copy_from_user(&info, (void __user *)arg, minsz))
> >                       return -EFAULT;
> >
> > @@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
> >               if (info.index >= vdev->num_irqs)
> >                       return -EINVAL;
> >
> > +             if (info.argsz >= capsz)
> > +                     minsz = capsz;
> > +
> > +             if (info.index == ext_irq_index) {
> nit: n case we add new ext indices afterwards, I would check info.index
> -  ext_irq_index against an VFIO_EXT_IRQ_MSI define.
> > +                     struct vfio_irq_info_cap_type cap_type = {
> > +                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
> > +                             .header.version = 1 };
> > +                     int i;
> > +                     int ret;
> > +                     int num_msgs;
> > +                     size_t msi_info_size;
> > +                     struct vfio_platform_irq *irq;
> nit: I think generally the opposite order (length) is chosen. This also
> would better match the existing style in this file
Ok will fix it
> > +
> > +                     info.index = array_index_nospec(info.index,
> > +                                                     vdev->num_irqs);
> > +
> > +                     irq = &vdev->irqs[info.index];
> > +
> > +                     info.flags = irq->flags;
> I think this can be removed given [*]
Sure.
> > +                     cap_type.type = irq->type;
> > +                     cap_type.subtype = irq->subtype;
> > +
> > +                     ret = vfio_info_add_capability(&caps,
> > +                                                    &cap_type.header,
> > +                                                    sizeof(cap_type));
> > +                     if (ret)
> > +                             return ret;
> > +
> > +                     num_msgs = irq->num_ctx;
> do would want to return the cap even if !num_ctx?
If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written.
I`ll take care of same.
> > +
> > +                     msi_info_size = struct_size(msi_info, msgs, num_msgs);
> > +
> > +                     msi_info = kzalloc(msi_info_size, GFP_KERNEL);
> > +                     if (!msi_info) {
> > +                             kfree(caps.buf);
> > +                             return -ENOMEM;
> > +                     }
> > +
> > +                     msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
> > +                     msi_info->header.version = 1;
> > +                     msi_info->nr_msgs = num_msgs;
> > +
> > +                     for (i = 0; i < num_msgs; i++) {
> > +                             struct vfio_irq_ctx *ctx = &irq->ctx[i];
> > +
> > +                             msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
> > +                             msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
> > +                             msi_info->msgs[i].data = ctx->msg.data;
> > +                     }
> > +
> > +                     ret = vfio_info_add_capability(&caps, &msi_info->header,
> > +                                                    msi_info_size);
> > +                     if (ret) {
> > +                             kfree(msi_info);
> > +                             kfree(caps.buf);
> > +                             return ret;
> > +                     }
> > +             }
> > +
> >               info.flags = vdev->irqs[info.index].flags;
> [*]
Will fix it.
> >               info.count = vdev->irqs[info.index].count;
> >
> > +             if (caps.size) {
> > +                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
> > +                     if (info.argsz < sizeof(info) + caps.size) {
> > +                             info.argsz = sizeof(info) + caps.size;
> > +                             info.cap_offset = 0;
> > +                     } else {
> > +                             vfio_info_cap_shift(&caps, sizeof(info));
> > +                             if (copy_to_user((void __user *)arg +
> > +                                               sizeof(info), caps.buf,
> > +                                               caps.size)) {
> > +                                     kfree(msi_info);
> > +                                     kfree(caps.buf);
> > +                                     return -EFAULT;
> > +                             }
> > +                             info.cap_offset = sizeof(info);
> > +                     }
> > +
> > +                     kfree(msi_info);
> > +                     kfree(caps.buf);
> > +             }
> > +
> >               return copy_to_user((void __user *)arg, &info, minsz) ?
> >                       -EFAULT : 0;
> >
> > @@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
> >               struct vfio_irq_set hdr;
> >               u8 *data = NULL;
> >               int ret = 0;
> > +             int max;
> >               size_t data_size = 0;
> >
> >               minsz = offsetofend(struct vfio_irq_set, count);
> > @@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
> >               if (copy_from_user(&hdr, (void __user *)arg, minsz))
> >                       return -EFAULT;
> >
> > -             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
> > -                                              vdev->num_irqs, &data_size);
> > +             if (hdr.index >= vdev->num_irqs)
> > +                     return -EINVAL;
> > +
> > +             max = vdev->irqs[hdr.index].count;
> > +
> > +             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
> > +                                                      vdev->num_irqs,
> > +                                                      &data_size);
> >               if (ret)
> >                       return ret;
> >
> > diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> > index c5b09ec0a3c9..4066223e5b2e 100644
> > --- a/drivers/vfio/platform/vfio_platform_irq.c
> > +++ b/drivers/vfio/platform/vfio_platform_irq.c
> > @@ -8,10 +8,12 @@
> >
> >  #include <linux/eventfd.h>
> >  #include <linux/interrupt.h>
> > +#include <linux/eventfd.h>
> >  #include <linux/slab.h>
> >  #include <linux/types.h>
> >  #include <linux/vfio.h>
> >  #include <linux/irq.h>
> > +#include <linux/msi.h>
> >
> >  #include "vfio_platform_private.h"
> >
> > @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
> >       return 0;
> >  }
> >
> > +/* MSI/MSIX */
> > +static irqreturn_t vfio_msihandler(int irq, void *arg)
> > +{
> > +     struct eventfd_ctx *trigger = arg;
> > +
> > +     eventfd_signal(trigger, 1);
> > +     return IRQ_HANDLED;
> > +}
> > +
> > +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
> > +{
> > +     int i;
> > +     struct vfio_platform_irq *irq;
> > +     u16 index = desc->platform.msi_index;
> > +     struct device *dev = msi_desc_to_dev(desc);
> > +     struct vfio_device *device = dev_get_drvdata(dev);
> > +     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
> > +                                             vfio_device_data(device);
> > +
> > +     for (i = 0; i < vdev->num_irqs; i++)
> > +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> > +                     irq = &vdev->irqs[i];
> > +
> > +     irq->ctx[index].msg.address_lo = msg->address_lo;
> > +     irq->ctx[index].msg.address_hi = msg->address_hi;
> > +     irq->ctx[index].msg.data = msg->data;
> > +}
> > +
> > +static int vfio_msi_enable(struct vfio_platform_device *vdev,
> > +                        struct vfio_platform_irq *irq, int nvec)
> > +{
> > +     int ret;
> > +     int msi_idx = 0;
> > +     struct msi_desc *desc;
> > +     struct device *dev = vdev->device;
> > +
> > +     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
> > +     if (!irq->ctx)
> > +             return -ENOMEM;
> > +
> > +     /* Allocate platform MSIs */
> > +     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
> > +     if (ret < 0) {
> > +             kfree(irq->ctx);
> > +             return ret;
> > +     }
> > +
> > +     for_each_msi_entry(desc, dev) {
> > +             irq->ctx[msi_idx].hwirq = desc->irq;
> > +             msi_idx++;
> > +     }
> > +
> > +     irq->num_ctx = nvec;
> > +     irq->config_msi = 1;
> > +
> > +     return 0;
> > +}
> > +
> > +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
> > +                                   int vector, int fd)
> > +{
> > +     struct eventfd_ctx *trigger;
> > +     int irq_num, ret;
> > +
> > +     if (vector < 0 || vector >= irq->num_ctx)
> > +             return -EINVAL;
> > +
> > +     irq_num = irq->ctx[vector].hwirq;
> > +
> > +     if (irq->ctx[vector].trigger) {
> > +             free_irq(irq_num, irq->ctx[vector].trigger);
> > +             kfree(irq->ctx[vector].name);
> > +             eventfd_ctx_put(irq->ctx[vector].trigger);
> > +             irq->ctx[vector].trigger = NULL;
> > +     }
> > +
> > +     if (fd < 0)
> > +             return 0;
> > +
> > +     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
> > +                                       "vfio-msi[%d]", vector);
> > +     if (!irq->ctx[vector].name)
> > +             return -ENOMEM;
> > +
> > +     trigger = eventfd_ctx_fdget(fd);
> > +     if (IS_ERR(trigger)) {
> > +             kfree(irq->ctx[vector].name);
> > +             return PTR_ERR(trigger);
> > +     }
> > +
> > +     ret = request_irq(irq_num, vfio_msihandler, 0,
> > +                       irq->ctx[vector].name, trigger);
> > +     if (ret) {
> > +             kfree(irq->ctx[vector].name);
> > +             eventfd_ctx_put(trigger);
> > +             return ret;
> > +     }
> > +
> > +     irq->ctx[vector].trigger = trigger;
> > +
> > +     return 0;
> > +}
> > +
> > +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
> > +                           unsigned int count, int32_t *fds)
> > +{
> > +     int i, j, ret = 0;
> > +
> > +     if (start >= irq->num_ctx || start + count > irq->num_ctx)
> > +             return -EINVAL;
> > +
> > +     for (i = 0, j = start; i < count && !ret; i++, j++) {
> > +             int fd = fds ? fds[i] : -1;
> > +
> > +             ret = vfio_msi_set_vector_signal(irq, j, fd);
> > +     }
> > +
> > +     if (ret) {
> > +             for (--j; j >= (int)start; j--)
> > +                     vfio_msi_set_vector_signal(irq, j, -1);
> > +     }
> > +
> > +     return ret;
> > +}
> > +
> > +static void vfio_msi_disable(struct vfio_platform_device *vdev,
> > +                          struct vfio_platform_irq *irq)
> > +{
> > +     struct device *dev = vdev->device;
> > +
> > +     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
> > +
> > +     platform_msi_domain_free_irqs(dev);
> > +
> > +     irq->config_msi = 0;
> > +     irq->num_ctx = 0;
> > +
> > +     kfree(irq->ctx);
> > +}
> > +
> > +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
> > +                             unsigned int index, unsigned int start,
> > +                             unsigned int count, uint32_t flags, void *data)
> > +{
> > +     int i;
> > +     struct vfio_platform_irq *irq = &vdev->irqs[index];
> > +
> > +     if (start + count > irq->count)
> > +             return -EINVAL;
> > +
> > +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
> > +             vfio_msi_disable(vdev, irq);
> > +             return 0;
> > +     }
> > +
> > +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
> > +             s32 *fds = data;
> > +             int ret;
> > +
> > +             if (irq->config_msi)
> > +                     return vfio_msi_set_block(irq, start, count,
> > +                                               fds);
> > +             ret = vfio_msi_enable(vdev, irq, start + count);
> > +             if (ret)
> > +                     return ret;
> > +
> > +             ret = vfio_msi_set_block(irq, start, count, fds);
> > +             if (ret)
> > +                     vfio_msi_disable(vdev, irq);
> > +
> > +             return ret;
> > +     }
> > +
> > +     for (i = start; i < start + count; i++) {
> > +             if (!irq->ctx[i].trigger)
> > +                     continue;
> > +             if (flags & VFIO_IRQ_SET_DATA_NONE) {
> > +                     eventfd_signal(irq->ctx[i].trigger, 1);
> > +             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> > +                     u8 *bools = data;
> > +
> > +                     if (bools[i - start])
> > +                             eventfd_signal(irq->ctx[i].trigger, 1);
> > +             }
> > +     }
> > +
> > +     return 0;
> > +}
> > +
> >  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >                                uint32_t flags, unsigned index, unsigned start,
> >                                unsigned count, void *data)
> > @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >                   unsigned start, unsigned count, uint32_t flags,
> >                   void *data) = NULL;
> >
> > -     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> > -     case VFIO_IRQ_SET_ACTION_MASK:
> > -             func = vfio_platform_set_irq_mask;
> > -             break;
> > -     case VFIO_IRQ_SET_ACTION_UNMASK:
> > -             func = vfio_platform_set_irq_unmask;
> > -             break;
> > -     case VFIO_IRQ_SET_ACTION_TRIGGER:
> > -             func = vfio_platform_set_irq_trigger;
> > -             break;
> > +     struct vfio_platform_irq *irq = &vdev->irqs[index];
> > +
> > +     if (irq->type == VFIO_IRQ_TYPE_MSI) {
> > +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> > +             case VFIO_IRQ_SET_ACTION_MASK:
> > +             case VFIO_IRQ_SET_ACTION_UNMASK:
> > +                     break;
> > +             case VFIO_IRQ_SET_ACTION_TRIGGER:
> > +                     func = vfio_set_msi_trigger;
> > +                     break;
> > +             }
> > +     } else {
> > +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> > +             case VFIO_IRQ_SET_ACTION_MASK:
> > +                     func = vfio_platform_set_irq_mask;
> > +                     break;
> > +             case VFIO_IRQ_SET_ACTION_UNMASK:
> > +                     func = vfio_platform_set_irq_unmask;
> > +                     break;
> > +             case VFIO_IRQ_SET_ACTION_TRIGGER:
> > +                     func = vfio_platform_set_irq_trigger;
> > +                     break;
> > +             }
> >       }
> >
> >       if (!func)
> > @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >
> >  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> >  {
> > -     int cnt = 0, i;
> > +     int i;
> > +     int cnt = 0;
> > +     int num_irqs;
> > +     struct device *dev = vdev->device;
> >
> >       while (vdev->get_irq(vdev, cnt) >= 0)
> >               cnt++;
> >
> > -     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
> > +     num_irqs = cnt;
> > +
> > +     if (dev->msi_domain)
> > +             num_irqs++;
> > +
> > +     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
> > +                          GFP_KERNEL);
> >       if (!vdev->irqs)
> >               return -ENOMEM;
> >
> > @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> >               vdev->irqs[i].masked = false;
> >       }
> >
> > -     vdev->num_irqs = cnt;
> > +     /*
> > +      * MSI block is added at last index and its an ext irq
> it is
> > +      */
> > +     if (dev->msi_domain) {
> > +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
> > +             vdev->irqs[i].count = NR_IRQS;
> why NR_IRQS?
 Since different devices can have different numbers of MSI(s) so we
need to initialize with max possible values. Can you please suggest if
this value does not seem appropriate?
> > +             vdev->irqs[i].hwirq = 0;
> > +             vdev->irqs[i].masked = false;
> > +             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
> > +             vdev->num_ext_irqs = 1;
> > +     }
> > +
> > +     vdev->num_irqs = num_irqs;
> >
> >       return 0;
> >  err:
> > @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
> >  {
> >       int i;
> >
> > -     for (i = 0; i < vdev->num_irqs; i++)
> > -             vfio_set_trigger(vdev, i, -1, NULL);
> > +     for (i = 0; i < vdev->num_irqs; i++) {
> > +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> > +                     vfio_set_msi_trigger(vdev, i, 0, 0,
> > +                                          VFIO_IRQ_SET_DATA_NONE, NULL);
> > +             else
> > +                     vfio_set_trigger(vdev, i, -1, NULL);
> > +     }
> >
> >       vdev->num_irqs = 0;
> >       kfree(vdev->irqs);
> > diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> > index 289089910643..7bbb05988705 100644
> > --- a/drivers/vfio/platform/vfio_platform_private.h
> > +++ b/drivers/vfio/platform/vfio_platform_private.h
> > @@ -9,6 +9,7 @@
> >
> >  #include <linux/types.h>
> >  #include <linux/interrupt.h>
> > +#include <linux/msi.h>
> >
> >  #define VFIO_PLATFORM_OFFSET_SHIFT   40
> >  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
> > @@ -19,9 +20,18 @@
> >  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
> >       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
> >
> > +struct vfio_irq_ctx {
> > +     int                     hwirq;
> > +     char                    *name;
> > +     struct msi_msg          msg;
> > +     struct eventfd_ctx      *trigger;
> > +};
> > +
> >  struct vfio_platform_irq {
> >       u32                     flags;
> >       u32                     count;
> > +     int                     num_ctx;
> > +     struct vfio_irq_ctx     *ctx;
> >       int                     hwirq;
> >       char                    *name;
> >       struct eventfd_ctx      *trigger;
> > @@ -29,6 +39,11 @@ struct vfio_platform_irq {
> >       spinlock_t              lock;
> >       struct virqfd           *unmask;
> >       struct virqfd           *mask;
> > +
> > +     /* for extended irqs */
> > +     u32                     type;
> > +     u32                     subtype;
> > +     int                     config_msi;
> >  };
> >
> >  struct vfio_platform_region {
> > @@ -46,6 +61,7 @@ struct vfio_platform_device {
> >       u32                             num_regions;
> >       struct vfio_platform_irq        *irqs;
> >       u32                             num_irqs;
> > +     int                             num_ext_irqs;
> >       int                             refcnt;
> >       struct mutex                    igate;
> >       struct module                   *parent_module;
> > diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> > index 2f313a238a8f..598d1c944283 100644
> > --- a/include/uapi/linux/vfio.h
> > +++ b/include/uapi/linux/vfio.h
> > @@ -697,11 +697,54 @@ struct vfio_irq_info {
> >  #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
> >  #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
> >  #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
> > +#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
> >       __u32   index;          /* IRQ index */
> >       __u32   count;          /* Number of IRQs within this index */
> > +     __u32   cap_offset;     /* Offset within info struct of first cap */
> >  };
> >  #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
> >
> > +/*
> > + * The irq type capability allows IRQs unique to a specific device or
> > + * class of devices to be exposed.
> > + *
> > + * The structures below define version 1 of this capability.
> > + */
> > +#define VFIO_IRQ_INFO_CAP_TYPE               3
> > +
> > +struct vfio_irq_info_cap_type {
> > +     struct vfio_info_cap_header header;
> > +     __u32 type;     /* global per bus driver */
> > +     __u32 subtype;  /* type specific */
> > +};
> > +
> > +/*
> > + * List of IRQ types, global per bus driver.
> > + * If you introduce a new type, please add it here.
> > + */
> > +
> > +/* Non PCI devices having MSI(s) support */
> > +#define VFIO_IRQ_TYPE_MSI            (1)
> > +
> > +/*
> > + * The msi capability allows the user to use the msi msg to
> > + * configure the iova for the msi configuration.
> > + * The structures below define version 1 of this capability.
> > + */
> > +#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
> > +
> > +struct vfio_irq_msi_msg {
> > +     __u32   addr_lo;
> > +     __u32   addr_hi;
> > +     __u32   data;
> > +};
> > +
> > +struct vfio_irq_info_cap_msi {
> > +     struct vfio_info_cap_header header;
> > +     __u32 nr_msgs;
> I think you should align a __u32   reserved field to have a 64b alignment
Sure.
> > +     struct vfio_irq_msi_msg msgs[];
> Please can you clarify why this cap is needed versus your prior approach.
In the previous patch, the reset module was configuring the device
with MSI msg/data now since the module is not available, user space
needs to have this data.
With this approach userspace just needs the pairs <msg and ctx >
associated with the MSI(s) and it can choose to configure the MSI(s)
sources accordingly.
Let me know if this approach does not look appropriate.

Thanks,
Vikas
> > +};
> > +
> >  /**
> >   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
> >   *
> >
> Thanks
>
> Eric
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v2 1/1] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 12/3/20 3:50 PM, Vikas Gupta wrote:
> Hi Eric,
> 
> On Wed, Dec 2, 2020 at 8:14 PM Auger Eric <eric.auger@redhat.com> wrote:
>>
>> Hi Vikas,
>>
>> On 11/24/20 5:16 PM, Vikas Gupta wrote:
>>> MSI support for platform devices.
>>>
>>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
>>> ---
>>>  drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
>>>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
>>>  drivers/vfio/platform/vfio_platform_private.h |  16 ++
>>>  include/uapi/linux/vfio.h                     |  43 +++
>>>  4 files changed, 401 insertions(+), 17 deletions(-)
>>>
>>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>>> index c0771a9567fb..b0bfc0f4ee1f 100644
>>> --- a/drivers/vfio/platform/vfio_platform_common.c
>>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>>> @@ -16,6 +16,7 @@
>>>  #include <linux/types.h>
>>>  #include <linux/uaccess.h>
>>>  #include <linux/vfio.h>
>>> +#include <linux/nospec.h>
>>>
>>>  #include "vfio_platform_private.h"
>>>
>>> @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
>>>
>>>       } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
>>>               struct vfio_irq_info info;
>>> +             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
>>> +             struct vfio_irq_info_cap_msi *msi_info = NULL;
>>> +             unsigned long capsz;
>>> +             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
>>>
>>>               minsz = offsetofend(struct vfio_irq_info, count);
>>>
>>> +             /* For backward compatibility, cannot require this */
>>> +             capsz = offsetofend(struct vfio_irq_info, cap_offset);
>>> +
>>>               if (copy_from_user(&info, (void __user *)arg, minsz))
>>>                       return -EFAULT;
>>>
>>> @@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
>>>               if (info.index >= vdev->num_irqs)
>>>                       return -EINVAL;
>>>
>>> +             if (info.argsz >= capsz)
>>> +                     minsz = capsz;
>>> +
>>> +             if (info.index == ext_irq_index) {
>> nit: n case we add new ext indices afterwards, I would check info.index
>> -  ext_irq_index against an VFIO_EXT_IRQ_MSI define.
>>> +                     struct vfio_irq_info_cap_type cap_type = {
>>> +                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
>>> +                             .header.version = 1 };
>>> +                     int i;
>>> +                     int ret;
>>> +                     int num_msgs;
>>> +                     size_t msi_info_size;
>>> +                     struct vfio_platform_irq *irq;
>> nit: I think generally the opposite order (length) is chosen. This also
>> would better match the existing style in this file
> Ok will fix it
>>> +
>>> +                     info.index = array_index_nospec(info.index,
>>> +                                                     vdev->num_irqs);
>>> +
>>> +                     irq = &vdev->irqs[info.index];
>>> +
>>> +                     info.flags = irq->flags;
>> I think this can be removed given [*]
> Sure.
>>> +                     cap_type.type = irq->type;
>>> +                     cap_type.subtype = irq->subtype;
>>> +
>>> +                     ret = vfio_info_add_capability(&caps,
>>> +                                                    &cap_type.header,
>>> +                                                    sizeof(cap_type));
>>> +                     if (ret)
>>> +                             return ret;
>>> +
>>> +                     num_msgs = irq->num_ctx;
>> do would want to return the cap even if !num_ctx?
> If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written.
> I`ll take care of same.
>>> +
>>> +                     msi_info_size = struct_size(msi_info, msgs, num_msgs);
>>> +
>>> +                     msi_info = kzalloc(msi_info_size, GFP_KERNEL);
>>> +                     if (!msi_info) {
>>> +                             kfree(caps.buf);
>>> +                             return -ENOMEM;
>>> +                     }
>>> +
>>> +                     msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
>>> +                     msi_info->header.version = 1;
>>> +                     msi_info->nr_msgs = num_msgs;
>>> +
>>> +                     for (i = 0; i < num_msgs; i++) {
>>> +                             struct vfio_irq_ctx *ctx = &irq->ctx[i];
>>> +
>>> +                             msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
>>> +                             msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
>>> +                             msi_info->msgs[i].data = ctx->msg.data;
>>> +                     }
>>> +
>>> +                     ret = vfio_info_add_capability(&caps, &msi_info->header,
>>> +                                                    msi_info_size);
>>> +                     if (ret) {
>>> +                             kfree(msi_info);
>>> +                             kfree(caps.buf);
>>> +                             return ret;
>>> +                     }
>>> +             }
>>> +
>>>               info.flags = vdev->irqs[info.index].flags;
>> [*]
> Will fix it.
>>>               info.count = vdev->irqs[info.index].count;
>>>
>>> +             if (caps.size) {
>>> +                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
>>> +                     if (info.argsz < sizeof(info) + caps.size) {
>>> +                             info.argsz = sizeof(info) + caps.size;
>>> +                             info.cap_offset = 0;
>>> +                     } else {
>>> +                             vfio_info_cap_shift(&caps, sizeof(info));
>>> +                             if (copy_to_user((void __user *)arg +
>>> +                                               sizeof(info), caps.buf,
>>> +                                               caps.size)) {
>>> +                                     kfree(msi_info);
>>> +                                     kfree(caps.buf);
>>> +                                     return -EFAULT;
>>> +                             }
>>> +                             info.cap_offset = sizeof(info);
>>> +                     }
>>> +
>>> +                     kfree(msi_info);
>>> +                     kfree(caps.buf);
>>> +             }
>>> +
>>>               return copy_to_user((void __user *)arg, &info, minsz) ?
>>>                       -EFAULT : 0;
>>>
>>> @@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
>>>               struct vfio_irq_set hdr;
>>>               u8 *data = NULL;
>>>               int ret = 0;
>>> +             int max;
>>>               size_t data_size = 0;
>>>
>>>               minsz = offsetofend(struct vfio_irq_set, count);
>>> @@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
>>>               if (copy_from_user(&hdr, (void __user *)arg, minsz))
>>>                       return -EFAULT;
>>>
>>> -             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
>>> -                                              vdev->num_irqs, &data_size);
>>> +             if (hdr.index >= vdev->num_irqs)
>>> +                     return -EINVAL;
>>> +
>>> +             max = vdev->irqs[hdr.index].count;
>>> +
>>> +             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
>>> +                                                      vdev->num_irqs,
>>> +                                                      &data_size);
>>>               if (ret)
>>>                       return ret;
>>>
>>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
>>> index c5b09ec0a3c9..4066223e5b2e 100644
>>> --- a/drivers/vfio/platform/vfio_platform_irq.c
>>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
>>> @@ -8,10 +8,12 @@
>>>
>>>  #include <linux/eventfd.h>
>>>  #include <linux/interrupt.h>
>>> +#include <linux/eventfd.h>
>>>  #include <linux/slab.h>
>>>  #include <linux/types.h>
>>>  #include <linux/vfio.h>
>>>  #include <linux/irq.h>
>>> +#include <linux/msi.h>
>>>
>>>  #include "vfio_platform_private.h"
>>>
>>> @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>>>       return 0;
>>>  }
>>>
>>> +/* MSI/MSIX */
>>> +static irqreturn_t vfio_msihandler(int irq, void *arg)
>>> +{
>>> +     struct eventfd_ctx *trigger = arg;
>>> +
>>> +     eventfd_signal(trigger, 1);
>>> +     return IRQ_HANDLED;
>>> +}
>>> +
>>> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
>>> +{
>>> +     int i;
>>> +     struct vfio_platform_irq *irq;
>>> +     u16 index = desc->platform.msi_index;
>>> +     struct device *dev = msi_desc_to_dev(desc);
>>> +     struct vfio_device *device = dev_get_drvdata(dev);
>>> +     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
>>> +                                             vfio_device_data(device);
>>> +
>>> +     for (i = 0; i < vdev->num_irqs; i++)
>>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
>>> +                     irq = &vdev->irqs[i];
>>> +
>>> +     irq->ctx[index].msg.address_lo = msg->address_lo;
>>> +     irq->ctx[index].msg.address_hi = msg->address_hi;
>>> +     irq->ctx[index].msg.data = msg->data;
>>> +}
>>> +
>>> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
>>> +                        struct vfio_platform_irq *irq, int nvec)
>>> +{
>>> +     int ret;
>>> +     int msi_idx = 0;
>>> +     struct msi_desc *desc;
>>> +     struct device *dev = vdev->device;
>>> +
>>> +     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
>>> +     if (!irq->ctx)
>>> +             return -ENOMEM;
>>> +
>>> +     /* Allocate platform MSIs */
>>> +     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
>>> +     if (ret < 0) {
>>> +             kfree(irq->ctx);
>>> +             return ret;
>>> +     }
>>> +
>>> +     for_each_msi_entry(desc, dev) {
>>> +             irq->ctx[msi_idx].hwirq = desc->irq;
>>> +             msi_idx++;
>>> +     }
>>> +
>>> +     irq->num_ctx = nvec;
>>> +     irq->config_msi = 1;
>>> +
>>> +     return 0;
>>> +}
>>> +
>>> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
>>> +                                   int vector, int fd)
>>> +{
>>> +     struct eventfd_ctx *trigger;
>>> +     int irq_num, ret;
>>> +
>>> +     if (vector < 0 || vector >= irq->num_ctx)
>>> +             return -EINVAL;
>>> +
>>> +     irq_num = irq->ctx[vector].hwirq;
>>> +
>>> +     if (irq->ctx[vector].trigger) {
>>> +             free_irq(irq_num, irq->ctx[vector].trigger);
>>> +             kfree(irq->ctx[vector].name);
>>> +             eventfd_ctx_put(irq->ctx[vector].trigger);
>>> +             irq->ctx[vector].trigger = NULL;
>>> +     }
>>> +
>>> +     if (fd < 0)
>>> +             return 0;
>>> +
>>> +     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
>>> +                                       "vfio-msi[%d]", vector);
>>> +     if (!irq->ctx[vector].name)
>>> +             return -ENOMEM;
>>> +
>>> +     trigger = eventfd_ctx_fdget(fd);
>>> +     if (IS_ERR(trigger)) {
>>> +             kfree(irq->ctx[vector].name);
>>> +             return PTR_ERR(trigger);
>>> +     }
>>> +
>>> +     ret = request_irq(irq_num, vfio_msihandler, 0,
>>> +                       irq->ctx[vector].name, trigger);
>>> +     if (ret) {
>>> +             kfree(irq->ctx[vector].name);
>>> +             eventfd_ctx_put(trigger);
>>> +             return ret;
>>> +     }
>>> +
>>> +     irq->ctx[vector].trigger = trigger;
>>> +
>>> +     return 0;
>>> +}
>>> +
>>> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
>>> +                           unsigned int count, int32_t *fds)
>>> +{
>>> +     int i, j, ret = 0;
>>> +
>>> +     if (start >= irq->num_ctx || start + count > irq->num_ctx)
>>> +             return -EINVAL;
>>> +
>>> +     for (i = 0, j = start; i < count && !ret; i++, j++) {
>>> +             int fd = fds ? fds[i] : -1;
>>> +
>>> +             ret = vfio_msi_set_vector_signal(irq, j, fd);
>>> +     }
>>> +
>>> +     if (ret) {
>>> +             for (--j; j >= (int)start; j--)
>>> +                     vfio_msi_set_vector_signal(irq, j, -1);
>>> +     }
>>> +
>>> +     return ret;
>>> +}
>>> +
>>> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
>>> +                          struct vfio_platform_irq *irq)
>>> +{
>>> +     struct device *dev = vdev->device;
>>> +
>>> +     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
>>> +
>>> +     platform_msi_domain_free_irqs(dev);
>>> +
>>> +     irq->config_msi = 0;
>>> +     irq->num_ctx = 0;
>>> +
>>> +     kfree(irq->ctx);
>>> +}
>>> +
>>> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
>>> +                             unsigned int index, unsigned int start,
>>> +                             unsigned int count, uint32_t flags, void *data)
>>> +{
>>> +     int i;
>>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>>> +
>>> +     if (start + count > irq->count)
>>> +             return -EINVAL;
>>> +
>>> +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
>>> +             vfio_msi_disable(vdev, irq);
>>> +             return 0;
>>> +     }
>>> +
>>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
>>> +             s32 *fds = data;
>>> +             int ret;
>>> +
>>> +             if (irq->config_msi)
>>> +                     return vfio_msi_set_block(irq, start, count,
>>> +                                               fds);
>>> +             ret = vfio_msi_enable(vdev, irq, start + count);
>>> +             if (ret)
>>> +                     return ret;
>>> +
>>> +             ret = vfio_msi_set_block(irq, start, count, fds);
>>> +             if (ret)
>>> +                     vfio_msi_disable(vdev, irq);
>>> +
>>> +             return ret;
>>> +     }
>>> +
>>> +     for (i = start; i < start + count; i++) {
>>> +             if (!irq->ctx[i].trigger)
>>> +                     continue;
>>> +             if (flags & VFIO_IRQ_SET_DATA_NONE) {
>>> +                     eventfd_signal(irq->ctx[i].trigger, 1);
>>> +             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
>>> +                     u8 *bools = data;
>>> +
>>> +                     if (bools[i - start])
>>> +                             eventfd_signal(irq->ctx[i].trigger, 1);
>>> +             }
>>> +     }
>>> +
>>> +     return 0;
>>> +}
>>> +
>>>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>                                uint32_t flags, unsigned index, unsigned start,
>>>                                unsigned count, void *data)
>>> @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>                   unsigned start, unsigned count, uint32_t flags,
>>>                   void *data) = NULL;
>>>
>>> -     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>> -     case VFIO_IRQ_SET_ACTION_MASK:
>>> -             func = vfio_platform_set_irq_mask;
>>> -             break;
>>> -     case VFIO_IRQ_SET_ACTION_UNMASK:
>>> -             func = vfio_platform_set_irq_unmask;
>>> -             break;
>>> -     case VFIO_IRQ_SET_ACTION_TRIGGER:
>>> -             func = vfio_platform_set_irq_trigger;
>>> -             break;
>>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>>> +
>>> +     if (irq->type == VFIO_IRQ_TYPE_MSI) {
>>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>> +             case VFIO_IRQ_SET_ACTION_MASK:
>>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
>>> +                     break;
>>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
>>> +                     func = vfio_set_msi_trigger;
>>> +                     break;
>>> +             }
>>> +     } else {
>>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>> +             case VFIO_IRQ_SET_ACTION_MASK:
>>> +                     func = vfio_platform_set_irq_mask;
>>> +                     break;
>>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
>>> +                     func = vfio_platform_set_irq_unmask;
>>> +                     break;
>>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
>>> +                     func = vfio_platform_set_irq_trigger;
>>> +                     break;
>>> +             }
>>>       }
>>>
>>>       if (!func)
>>> @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>
>>>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>  {
>>> -     int cnt = 0, i;
>>> +     int i;
>>> +     int cnt = 0;
>>> +     int num_irqs;
>>> +     struct device *dev = vdev->device;
>>>
>>>       while (vdev->get_irq(vdev, cnt) >= 0)
>>>               cnt++;
>>>
>>> -     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
>>> +     num_irqs = cnt;
>>> +
>>> +     if (dev->msi_domain)
>>> +             num_irqs++;
>>> +
>>> +     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
>>> +                          GFP_KERNEL);
>>>       if (!vdev->irqs)
>>>               return -ENOMEM;
>>>
>>> @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>               vdev->irqs[i].masked = false;
>>>       }
>>>
>>> -     vdev->num_irqs = cnt;
>>> +     /*
>>> +      * MSI block is added at last index and its an ext irq
>> it is
>>> +      */
>>> +     if (dev->msi_domain) {
>>> +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
>>> +             vdev->irqs[i].count = NR_IRQS;
>> why NR_IRQS?
>  Since different devices can have different numbers of MSI(s) so we
> need to initialize with max possible values. Can you please suggest if
> this value does not seem appropriate?
As opposed to PCIe, the userspace has no real way to guess how many
vectors can be set (what vfio_pci_get_irq_count does). This also means
we do not fully implement the original API as we are not able to report
an accurate value for .count. How will the user determine how many
vectors he can use?
>>> +             vdev->irqs[i].hwirq = 0;
>>> +             vdev->irqs[i].masked = false;
>>> +             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
>>> +             vdev->num_ext_irqs = 1;
>>> +     }
>>> +
>>> +     vdev->num_irqs = num_irqs;
>>>
>>>       return 0;
>>>  err:
>>> @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>>>  {
>>>       int i;
>>>
>>> -     for (i = 0; i < vdev->num_irqs; i++)
>>> -             vfio_set_trigger(vdev, i, -1, NULL);
>>> +     for (i = 0; i < vdev->num_irqs; i++) {
>>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
>>> +                     vfio_set_msi_trigger(vdev, i, 0, 0,
>>> +                                          VFIO_IRQ_SET_DATA_NONE, NULL);
>>> +             else
>>> +                     vfio_set_trigger(vdev, i, -1, NULL);
>>> +     }
>>>
>>>       vdev->num_irqs = 0;
>>>       kfree(vdev->irqs);
>>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>>> index 289089910643..7bbb05988705 100644
>>> --- a/drivers/vfio/platform/vfio_platform_private.h
>>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>>> @@ -9,6 +9,7 @@
>>>
>>>  #include <linux/types.h>
>>>  #include <linux/interrupt.h>
>>> +#include <linux/msi.h>
>>>
>>>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
>>>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
>>> @@ -19,9 +20,18 @@
>>>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
>>>       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>>>
>>> +struct vfio_irq_ctx {
>>> +     int                     hwirq;
>>> +     char                    *name;
>>> +     struct msi_msg          msg;
>>> +     struct eventfd_ctx      *trigger;
>>> +};
>>> +
>>>  struct vfio_platform_irq {
>>>       u32                     flags;
>>>       u32                     count;
>>> +     int                     num_ctx;
>>> +     struct vfio_irq_ctx     *ctx;
>>>       int                     hwirq;
>>>       char                    *name;
>>>       struct eventfd_ctx      *trigger;
>>> @@ -29,6 +39,11 @@ struct vfio_platform_irq {
>>>       spinlock_t              lock;
>>>       struct virqfd           *unmask;
>>>       struct virqfd           *mask;
>>> +
>>> +     /* for extended irqs */
>>> +     u32                     type;
>>> +     u32                     subtype;
>>> +     int                     config_msi;
>>>  };
>>>
>>>  struct vfio_platform_region {
>>> @@ -46,6 +61,7 @@ struct vfio_platform_device {
>>>       u32                             num_regions;
>>>       struct vfio_platform_irq        *irqs;
>>>       u32                             num_irqs;
>>> +     int                             num_ext_irqs;
>>>       int                             refcnt;
>>>       struct mutex                    igate;
>>>       struct module                   *parent_module;
>>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>>> index 2f313a238a8f..598d1c944283 100644
>>> --- a/include/uapi/linux/vfio.h
>>> +++ b/include/uapi/linux/vfio.h
>>> @@ -697,11 +697,54 @@ struct vfio_irq_info {
>>>  #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
>>>  #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
>>>  #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
>>> +#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
>>>       __u32   index;          /* IRQ index */
>>>       __u32   count;          /* Number of IRQs within this index */
>>> +     __u32   cap_offset;     /* Offset within info struct of first cap */
>>>  };
>>>  #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
>>>
>>> +/*
>>> + * The irq type capability allows IRQs unique to a specific device or
>>> + * class of devices to be exposed.
>>> + *
>>> + * The structures below define version 1 of this capability.
>>> + */
>>> +#define VFIO_IRQ_INFO_CAP_TYPE               3
>>> +
>>> +struct vfio_irq_info_cap_type {
>>> +     struct vfio_info_cap_header header;
>>> +     __u32 type;     /* global per bus driver */
>>> +     __u32 subtype;  /* type specific */
>>> +};
>>> +
>>> +/*
>>> + * List of IRQ types, global per bus driver.
>>> + * If you introduce a new type, please add it here.
>>> + */
>>> +
>>> +/* Non PCI devices having MSI(s) support */
>>> +#define VFIO_IRQ_TYPE_MSI            (1)
>>> +
>>> +/*
>>> + * The msi capability allows the user to use the msi msg to
>>> + * configure the iova for the msi configuration.
>>> + * The structures below define version 1 of this capability.
>>> + */
>>> +#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
>>> +
>>> +struct vfio_irq_msi_msg {
>>> +     __u32   addr_lo;
>>> +     __u32   addr_hi;
>>> +     __u32   data;
>>> +};
>>> +
>>> +struct vfio_irq_info_cap_msi {
>>> +     struct vfio_info_cap_header header;
>>> +     __u32 nr_msgs;
>> I think you should align a __u32   reserved field to have a 64b alignment
> Sure.
>>> +     struct vfio_irq_msi_msg msgs[];
>> Please can you clarify why this cap is needed versus your prior approach.
> In the previous patch, the reset module was configuring the device
> with MSI msg/data now since the module is not available, user space
> needs to have this data.
> With this approach userspace just needs the pairs <msg and ctx >
> associated with the MSI(s) and it can choose to configure the MSI(s)
> sources accordingly.
> Let me know if this approach does not look appropriate.
This comes to the question I asked in my previous email, ie. could you
give some more info about the expected MSI setup sequence? May be the
opportunity to enhance the commit message ;-)

Thanks

Eric
> 
> Thanks,
> Vikas
>>> +};
>>> +
>>>  /**
>>>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
>>>   *
>>>
>> Thanks
>>
>> Eric
>>

Re: [RFC v2 1/1] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 28281 bytes --]

HI Eric,

On Tue, Dec 8, 2020 at 2:13 AM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 12/3/20 3:50 PM, Vikas Gupta wrote:
> > Hi Eric,
> >
> > On Wed, Dec 2, 2020 at 8:14 PM Auger Eric <eric.auger@redhat.com> wrote:
> >>
> >> Hi Vikas,
> >>
> >> On 11/24/20 5:16 PM, Vikas Gupta wrote:
> >>> MSI support for platform devices.
> >>>
> >>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> >>> ---
> >>>  drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
> >>>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
> >>>  drivers/vfio/platform/vfio_platform_private.h |  16 ++
> >>>  include/uapi/linux/vfio.h                     |  43 +++
> >>>  4 files changed, 401 insertions(+), 17 deletions(-)
> >>>
> >>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> >>> index c0771a9567fb..b0bfc0f4ee1f 100644
> >>> --- a/drivers/vfio/platform/vfio_platform_common.c
> >>> +++ b/drivers/vfio/platform/vfio_platform_common.c
> >>> @@ -16,6 +16,7 @@
> >>>  #include <linux/types.h>
> >>>  #include <linux/uaccess.h>
> >>>  #include <linux/vfio.h>
> >>> +#include <linux/nospec.h>
> >>>
> >>>  #include "vfio_platform_private.h"
> >>>
> >>> @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
> >>>
> >>>       } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
> >>>               struct vfio_irq_info info;
> >>> +             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
> >>> +             struct vfio_irq_info_cap_msi *msi_info = NULL;
> >>> +             unsigned long capsz;
> >>> +             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
> >>>
> >>>               minsz = offsetofend(struct vfio_irq_info, count);
> >>>
> >>> +             /* For backward compatibility, cannot require this */
> >>> +             capsz = offsetofend(struct vfio_irq_info, cap_offset);
> >>> +
> >>>               if (copy_from_user(&info, (void __user *)arg, minsz))
> >>>                       return -EFAULT;
> >>>
> >>> @@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
> >>>               if (info.index >= vdev->num_irqs)
> >>>                       return -EINVAL;
> >>>
> >>> +             if (info.argsz >= capsz)
> >>> +                     minsz = capsz;
> >>> +
> >>> +             if (info.index == ext_irq_index) {
> >> nit: n case we add new ext indices afterwards, I would check info.index
> >> -  ext_irq_index against an VFIO_EXT_IRQ_MSI define.
> >>> +                     struct vfio_irq_info_cap_type cap_type = {
> >>> +                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
> >>> +                             .header.version = 1 };
> >>> +                     int i;
> >>> +                     int ret;
> >>> +                     int num_msgs;
> >>> +                     size_t msi_info_size;
> >>> +                     struct vfio_platform_irq *irq;
> >> nit: I think generally the opposite order (length) is chosen. This also
> >> would better match the existing style in this file
> > Ok will fix it
> >>> +
> >>> +                     info.index = array_index_nospec(info.index,
> >>> +                                                     vdev->num_irqs);
> >>> +
> >>> +                     irq = &vdev->irqs[info.index];
> >>> +
> >>> +                     info.flags = irq->flags;
> >> I think this can be removed given [*]
> > Sure.
> >>> +                     cap_type.type = irq->type;
> >>> +                     cap_type.subtype = irq->subtype;
> >>> +
> >>> +                     ret = vfio_info_add_capability(&caps,
> >>> +                                                    &cap_type.header,
> >>> +                                                    sizeof(cap_type));
> >>> +                     if (ret)
> >>> +                             return ret;
> >>> +
> >>> +                     num_msgs = irq->num_ctx;
> >> do would want to return the cap even if !num_ctx?
> > If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written.
> > I`ll take care of same.
> >>> +
> >>> +                     msi_info_size = struct_size(msi_info, msgs, num_msgs);
> >>> +
> >>> +                     msi_info = kzalloc(msi_info_size, GFP_KERNEL);
> >>> +                     if (!msi_info) {
> >>> +                             kfree(caps.buf);
> >>> +                             return -ENOMEM;
> >>> +                     }
> >>> +
> >>> +                     msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
> >>> +                     msi_info->header.version = 1;
> >>> +                     msi_info->nr_msgs = num_msgs;
> >>> +
> >>> +                     for (i = 0; i < num_msgs; i++) {
> >>> +                             struct vfio_irq_ctx *ctx = &irq->ctx[i];
> >>> +
> >>> +                             msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
> >>> +                             msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
> >>> +                             msi_info->msgs[i].data = ctx->msg.data;
> >>> +                     }
> >>> +
> >>> +                     ret = vfio_info_add_capability(&caps, &msi_info->header,
> >>> +                                                    msi_info_size);
> >>> +                     if (ret) {
> >>> +                             kfree(msi_info);
> >>> +                             kfree(caps.buf);
> >>> +                             return ret;
> >>> +                     }
> >>> +             }
> >>> +
> >>>               info.flags = vdev->irqs[info.index].flags;
> >> [*]
> > Will fix it.
> >>>               info.count = vdev->irqs[info.index].count;
> >>>
> >>> +             if (caps.size) {
> >>> +                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
> >>> +                     if (info.argsz < sizeof(info) + caps.size) {
> >>> +                             info.argsz = sizeof(info) + caps.size;
> >>> +                             info.cap_offset = 0;
> >>> +                     } else {
> >>> +                             vfio_info_cap_shift(&caps, sizeof(info));
> >>> +                             if (copy_to_user((void __user *)arg +
> >>> +                                               sizeof(info), caps.buf,
> >>> +                                               caps.size)) {
> >>> +                                     kfree(msi_info);
> >>> +                                     kfree(caps.buf);
> >>> +                                     return -EFAULT;
> >>> +                             }
> >>> +                             info.cap_offset = sizeof(info);
> >>> +                     }
> >>> +
> >>> +                     kfree(msi_info);
> >>> +                     kfree(caps.buf);
> >>> +             }
> >>> +
> >>>               return copy_to_user((void __user *)arg, &info, minsz) ?
> >>>                       -EFAULT : 0;
> >>>
> >>> @@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
> >>>               struct vfio_irq_set hdr;
> >>>               u8 *data = NULL;
> >>>               int ret = 0;
> >>> +             int max;
> >>>               size_t data_size = 0;
> >>>
> >>>               minsz = offsetofend(struct vfio_irq_set, count);
> >>> @@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
> >>>               if (copy_from_user(&hdr, (void __user *)arg, minsz))
> >>>                       return -EFAULT;
> >>>
> >>> -             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
> >>> -                                              vdev->num_irqs, &data_size);
> >>> +             if (hdr.index >= vdev->num_irqs)
> >>> +                     return -EINVAL;
> >>> +
> >>> +             max = vdev->irqs[hdr.index].count;
> >>> +
> >>> +             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
> >>> +                                                      vdev->num_irqs,
> >>> +                                                      &data_size);
> >>>               if (ret)
> >>>                       return ret;
> >>>
> >>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> >>> index c5b09ec0a3c9..4066223e5b2e 100644
> >>> --- a/drivers/vfio/platform/vfio_platform_irq.c
> >>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
> >>> @@ -8,10 +8,12 @@
> >>>
> >>>  #include <linux/eventfd.h>
> >>>  #include <linux/interrupt.h>
> >>> +#include <linux/eventfd.h>
> >>>  #include <linux/slab.h>
> >>>  #include <linux/types.h>
> >>>  #include <linux/vfio.h>
> >>>  #include <linux/irq.h>
> >>> +#include <linux/msi.h>
> >>>
> >>>  #include "vfio_platform_private.h"
> >>>
> >>> @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
> >>>       return 0;
> >>>  }
> >>>
> >>> +/* MSI/MSIX */
> >>> +static irqreturn_t vfio_msihandler(int irq, void *arg)
> >>> +{
> >>> +     struct eventfd_ctx *trigger = arg;
> >>> +
> >>> +     eventfd_signal(trigger, 1);
> >>> +     return IRQ_HANDLED;
> >>> +}
> >>> +
> >>> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
> >>> +{
> >>> +     int i;
> >>> +     struct vfio_platform_irq *irq;
> >>> +     u16 index = desc->platform.msi_index;
> >>> +     struct device *dev = msi_desc_to_dev(desc);
> >>> +     struct vfio_device *device = dev_get_drvdata(dev);
> >>> +     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
> >>> +                                             vfio_device_data(device);
> >>> +
> >>> +     for (i = 0; i < vdev->num_irqs; i++)
> >>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> >>> +                     irq = &vdev->irqs[i];
> >>> +
> >>> +     irq->ctx[index].msg.address_lo = msg->address_lo;
> >>> +     irq->ctx[index].msg.address_hi = msg->address_hi;
> >>> +     irq->ctx[index].msg.data = msg->data;
> >>> +}
> >>> +
> >>> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
> >>> +                        struct vfio_platform_irq *irq, int nvec)
> >>> +{
> >>> +     int ret;
> >>> +     int msi_idx = 0;
> >>> +     struct msi_desc *desc;
> >>> +     struct device *dev = vdev->device;
> >>> +
> >>> +     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
> >>> +     if (!irq->ctx)
> >>> +             return -ENOMEM;
> >>> +
> >>> +     /* Allocate platform MSIs */
> >>> +     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
> >>> +     if (ret < 0) {
> >>> +             kfree(irq->ctx);
> >>> +             return ret;
> >>> +     }
> >>> +
> >>> +     for_each_msi_entry(desc, dev) {
> >>> +             irq->ctx[msi_idx].hwirq = desc->irq;
> >>> +             msi_idx++;
> >>> +     }
> >>> +
> >>> +     irq->num_ctx = nvec;
> >>> +     irq->config_msi = 1;
> >>> +
> >>> +     return 0;
> >>> +}
> >>> +
> >>> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
> >>> +                                   int vector, int fd)
> >>> +{
> >>> +     struct eventfd_ctx *trigger;
> >>> +     int irq_num, ret;
> >>> +
> >>> +     if (vector < 0 || vector >= irq->num_ctx)
> >>> +             return -EINVAL;
> >>> +
> >>> +     irq_num = irq->ctx[vector].hwirq;
> >>> +
> >>> +     if (irq->ctx[vector].trigger) {
> >>> +             free_irq(irq_num, irq->ctx[vector].trigger);
> >>> +             kfree(irq->ctx[vector].name);
> >>> +             eventfd_ctx_put(irq->ctx[vector].trigger);
> >>> +             irq->ctx[vector].trigger = NULL;
> >>> +     }
> >>> +
> >>> +     if (fd < 0)
> >>> +             return 0;
> >>> +
> >>> +     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
> >>> +                                       "vfio-msi[%d]", vector);
> >>> +     if (!irq->ctx[vector].name)
> >>> +             return -ENOMEM;
> >>> +
> >>> +     trigger = eventfd_ctx_fdget(fd);
> >>> +     if (IS_ERR(trigger)) {
> >>> +             kfree(irq->ctx[vector].name);
> >>> +             return PTR_ERR(trigger);
> >>> +     }
> >>> +
> >>> +     ret = request_irq(irq_num, vfio_msihandler, 0,
> >>> +                       irq->ctx[vector].name, trigger);
> >>> +     if (ret) {
> >>> +             kfree(irq->ctx[vector].name);
> >>> +             eventfd_ctx_put(trigger);
> >>> +             return ret;
> >>> +     }
> >>> +
> >>> +     irq->ctx[vector].trigger = trigger;
> >>> +
> >>> +     return 0;
> >>> +}
> >>> +
> >>> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
> >>> +                           unsigned int count, int32_t *fds)
> >>> +{
> >>> +     int i, j, ret = 0;
> >>> +
> >>> +     if (start >= irq->num_ctx || start + count > irq->num_ctx)
> >>> +             return -EINVAL;
> >>> +
> >>> +     for (i = 0, j = start; i < count && !ret; i++, j++) {
> >>> +             int fd = fds ? fds[i] : -1;
> >>> +
> >>> +             ret = vfio_msi_set_vector_signal(irq, j, fd);
> >>> +     }
> >>> +
> >>> +     if (ret) {
> >>> +             for (--j; j >= (int)start; j--)
> >>> +                     vfio_msi_set_vector_signal(irq, j, -1);
> >>> +     }
> >>> +
> >>> +     return ret;
> >>> +}
> >>> +
> >>> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
> >>> +                          struct vfio_platform_irq *irq)
> >>> +{
> >>> +     struct device *dev = vdev->device;
> >>> +
> >>> +     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
> >>> +
> >>> +     platform_msi_domain_free_irqs(dev);
> >>> +
> >>> +     irq->config_msi = 0;
> >>> +     irq->num_ctx = 0;
> >>> +
> >>> +     kfree(irq->ctx);
> >>> +}
> >>> +
> >>> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
> >>> +                             unsigned int index, unsigned int start,
> >>> +                             unsigned int count, uint32_t flags, void *data)
> >>> +{
> >>> +     int i;
> >>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
> >>> +
> >>> +     if (start + count > irq->count)
> >>> +             return -EINVAL;
> >>> +
> >>> +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
> >>> +             vfio_msi_disable(vdev, irq);
> >>> +             return 0;
> >>> +     }
> >>> +
> >>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
> >>> +             s32 *fds = data;
> >>> +             int ret;
> >>> +
> >>> +             if (irq->config_msi)
> >>> +                     return vfio_msi_set_block(irq, start, count,
> >>> +                                               fds);
> >>> +             ret = vfio_msi_enable(vdev, irq, start + count);
> >>> +             if (ret)
> >>> +                     return ret;
> >>> +
> >>> +             ret = vfio_msi_set_block(irq, start, count, fds);
> >>> +             if (ret)
> >>> +                     vfio_msi_disable(vdev, irq);
> >>> +
> >>> +             return ret;
> >>> +     }
> >>> +
> >>> +     for (i = start; i < start + count; i++) {
> >>> +             if (!irq->ctx[i].trigger)
> >>> +                     continue;
> >>> +             if (flags & VFIO_IRQ_SET_DATA_NONE) {
> >>> +                     eventfd_signal(irq->ctx[i].trigger, 1);
> >>> +             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> >>> +                     u8 *bools = data;
> >>> +
> >>> +                     if (bools[i - start])
> >>> +                             eventfd_signal(irq->ctx[i].trigger, 1);
> >>> +             }
> >>> +     }
> >>> +
> >>> +     return 0;
> >>> +}
> >>> +
> >>>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >>>                                uint32_t flags, unsigned index, unsigned start,
> >>>                                unsigned count, void *data)
> >>> @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >>>                   unsigned start, unsigned count, uint32_t flags,
> >>>                   void *data) = NULL;
> >>>
> >>> -     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> >>> -     case VFIO_IRQ_SET_ACTION_MASK:
> >>> -             func = vfio_platform_set_irq_mask;
> >>> -             break;
> >>> -     case VFIO_IRQ_SET_ACTION_UNMASK:
> >>> -             func = vfio_platform_set_irq_unmask;
> >>> -             break;
> >>> -     case VFIO_IRQ_SET_ACTION_TRIGGER:
> >>> -             func = vfio_platform_set_irq_trigger;
> >>> -             break;
> >>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
> >>> +
> >>> +     if (irq->type == VFIO_IRQ_TYPE_MSI) {
> >>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> >>> +             case VFIO_IRQ_SET_ACTION_MASK:
> >>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
> >>> +                     break;
> >>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
> >>> +                     func = vfio_set_msi_trigger;
> >>> +                     break;
> >>> +             }
> >>> +     } else {
> >>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> >>> +             case VFIO_IRQ_SET_ACTION_MASK:
> >>> +                     func = vfio_platform_set_irq_mask;
> >>> +                     break;
> >>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
> >>> +                     func = vfio_platform_set_irq_unmask;
> >>> +                     break;
> >>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
> >>> +                     func = vfio_platform_set_irq_trigger;
> >>> +                     break;
> >>> +             }
> >>>       }
> >>>
> >>>       if (!func)
> >>> @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >>>
> >>>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> >>>  {
> >>> -     int cnt = 0, i;
> >>> +     int i;
> >>> +     int cnt = 0;
> >>> +     int num_irqs;
> >>> +     struct device *dev = vdev->device;
> >>>
> >>>       while (vdev->get_irq(vdev, cnt) >= 0)
> >>>               cnt++;
> >>>
> >>> -     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
> >>> +     num_irqs = cnt;
> >>> +
> >>> +     if (dev->msi_domain)
> >>> +             num_irqs++;
> >>> +
> >>> +     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
> >>> +                          GFP_KERNEL);
> >>>       if (!vdev->irqs)
> >>>               return -ENOMEM;
> >>>
> >>> @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> >>>               vdev->irqs[i].masked = false;
> >>>       }
> >>>
> >>> -     vdev->num_irqs = cnt;
> >>> +     /*
> >>> +      * MSI block is added at last index and its an ext irq
> >> it is
> >>> +      */
> >>> +     if (dev->msi_domain) {
> >>> +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
> >>> +             vdev->irqs[i].count = NR_IRQS;
> >> why NR_IRQS?
> >  Since different devices can have different numbers of MSI(s) so we
> > need to initialize with max possible values. Can you please suggest if
> > this value does not seem appropriate?
> As opposed to PCIe, the userspace has no real way to guess how many
> vectors can be set (what vfio_pci_get_irq_count does). This also means
> we do not fully implement the original API as we are not able to report
> an accurate value for .count. How will the user determine how many
> vectors he can use?

I believe user space will know how many MSIs platform device can have.
We have assigned NR_IRQS because that’s what maximum number of
interrupt can be supported on a specific platform. In case user space
errantly tries to allocate any number of MSIs in kernel
platform_msi_domain_alloc_irqs should fail.
Do you think this approach can create an issue as .count is not
exactly the same as MSIs are existing with device? Is it necessary to
have a PCIe kind of approach as PCIe is standard but platform devices
do not have standards?

> >>> +             vdev->irqs[i].hwirq = 0;
> >>> +             vdev->irqs[i].masked = false;
> >>> +             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
> >>> +             vdev->num_ext_irqs = 1;
> >>> +     }
> >>> +
> >>> +     vdev->num_irqs = num_irqs;
> >>>
> >>>       return 0;
> >>>  err:
> >>> @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
> >>>  {
> >>>       int i;
> >>>
> >>> -     for (i = 0; i < vdev->num_irqs; i++)
> >>> -             vfio_set_trigger(vdev, i, -1, NULL);
> >>> +     for (i = 0; i < vdev->num_irqs; i++) {
> >>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> >>> +                     vfio_set_msi_trigger(vdev, i, 0, 0,
> >>> +                                          VFIO_IRQ_SET_DATA_NONE, NULL);
> >>> +             else
> >>> +                     vfio_set_trigger(vdev, i, -1, NULL);
> >>> +     }
> >>>
> >>>       vdev->num_irqs = 0;
> >>>       kfree(vdev->irqs);
> >>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> >>> index 289089910643..7bbb05988705 100644
> >>> --- a/drivers/vfio/platform/vfio_platform_private.h
> >>> +++ b/drivers/vfio/platform/vfio_platform_private.h
> >>> @@ -9,6 +9,7 @@
> >>>
> >>>  #include <linux/types.h>
> >>>  #include <linux/interrupt.h>
> >>> +#include <linux/msi.h>
> >>>
> >>>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
> >>>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
> >>> @@ -19,9 +20,18 @@
> >>>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
> >>>       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
> >>>
> >>> +struct vfio_irq_ctx {
> >>> +     int                     hwirq;
> >>> +     char                    *name;
> >>> +     struct msi_msg          msg;
> >>> +     struct eventfd_ctx      *trigger;
> >>> +};
> >>> +
> >>>  struct vfio_platform_irq {
> >>>       u32                     flags;
> >>>       u32                     count;
> >>> +     int                     num_ctx;
> >>> +     struct vfio_irq_ctx     *ctx;
> >>>       int                     hwirq;
> >>>       char                    *name;
> >>>       struct eventfd_ctx      *trigger;
> >>> @@ -29,6 +39,11 @@ struct vfio_platform_irq {
> >>>       spinlock_t              lock;
> >>>       struct virqfd           *unmask;
> >>>       struct virqfd           *mask;
> >>> +
> >>> +     /* for extended irqs */
> >>> +     u32                     type;
> >>> +     u32                     subtype;
> >>> +     int                     config_msi;
> >>>  };
> >>>
> >>>  struct vfio_platform_region {
> >>> @@ -46,6 +61,7 @@ struct vfio_platform_device {
> >>>       u32                             num_regions;
> >>>       struct vfio_platform_irq        *irqs;
> >>>       u32                             num_irqs;
> >>> +     int                             num_ext_irqs;
> >>>       int                             refcnt;
> >>>       struct mutex                    igate;
> >>>       struct module                   *parent_module;
> >>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> >>> index 2f313a238a8f..598d1c944283 100644
> >>> --- a/include/uapi/linux/vfio.h
> >>> +++ b/include/uapi/linux/vfio.h
> >>> @@ -697,11 +697,54 @@ struct vfio_irq_info {
> >>>  #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
> >>>  #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
> >>>  #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
> >>> +#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
> >>>       __u32   index;          /* IRQ index */
> >>>       __u32   count;          /* Number of IRQs within this index */
> >>> +     __u32   cap_offset;     /* Offset within info struct of first cap */
> >>>  };
> >>>  #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
> >>>
> >>> +/*
> >>> + * The irq type capability allows IRQs unique to a specific device or
> >>> + * class of devices to be exposed.
> >>> + *
> >>> + * The structures below define version 1 of this capability.
> >>> + */
> >>> +#define VFIO_IRQ_INFO_CAP_TYPE               3
> >>> +
> >>> +struct vfio_irq_info_cap_type {
> >>> +     struct vfio_info_cap_header header;
> >>> +     __u32 type;     /* global per bus driver */
> >>> +     __u32 subtype;  /* type specific */
> >>> +};
> >>> +
> >>> +/*
> >>> + * List of IRQ types, global per bus driver.
> >>> + * If you introduce a new type, please add it here.
> >>> + */
> >>> +
> >>> +/* Non PCI devices having MSI(s) support */
> >>> +#define VFIO_IRQ_TYPE_MSI            (1)
> >>> +
> >>> +/*
> >>> + * The msi capability allows the user to use the msi msg to
> >>> + * configure the iova for the msi configuration.
> >>> + * The structures below define version 1 of this capability.
> >>> + */
> >>> +#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
> >>> +
> >>> +struct vfio_irq_msi_msg {
> >>> +     __u32   addr_lo;
> >>> +     __u32   addr_hi;
> >>> +     __u32   data;
> >>> +};
> >>> +
> >>> +struct vfio_irq_info_cap_msi {
> >>> +     struct vfio_info_cap_header header;
> >>> +     __u32 nr_msgs;
> >> I think you should align a __u32   reserved field to have a 64b alignment
> > Sure.
> >>> +     struct vfio_irq_msi_msg msgs[];
> >> Please can you clarify why this cap is needed versus your prior approach.
> > In the previous patch, the reset module was configuring the device
> > with MSI msg/data now since the module is not available, user space
> > needs to have this data.
> > With this approach userspace just needs the pairs <msg and ctx >
> > associated with the MSI(s) and it can choose to configure the MSI(s)
> > sources accordingly.
> > Let me know if this approach does not look appropriate.
> This comes to the question I asked in my previous email, ie. could you
> give some more info about the expected MSI setup sequence? May be the
> opportunity to enhance the commit message ;-)

With our proposal, user space needs to know how many MSI sources there
are as we want user space to map msi-msg(s) to specific MSI sources.
Let us assume there are max ‘n’ MSI sources a platform device have,
user-space requests ‘n’ msi(s) allocation in the kernel which in turn
gets ‘n’ msi_msg with help of cap (VFIO_IRQ_INFO_CAP_MSI_DESCS). User
space is free to map/configure msi_msg->data value to any particular
source. Since msi_msg->data is one to one mapped to eventfd,
user-space knows that particular eventfd corresponds to the same MSI
source which it has configured with the msi_msg->data value.

Steps:
a)   User space allocates nvec in kernel. Ioctl VFIO_DEVICE_SET_IRQS
b)   Kernel allocates nvec and does associated IRQ allocation, eventfd etc.
c)   User space gets the msi_msgs VFIO_DEVICE_GET_IRQ_INFO.
d)   User space configures the particular MSI source with msi_msg info.
Do these steps look OK?
We can add more information where this cap is introduced in the file
uapi/linux/vfio.h.

Thanks,
Vikas
>
> Thanks
>
> Eric
> >
> > Thanks,
> > Vikas
> >>> +};
> >>> +
> >>>  /**
> >>>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
> >>>   *
> >>>
> >> Thanks
> >>
> >> Eric
> >>
>

-- 
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v2 1/1] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 12/10/20 8:34 AM, Vikas Gupta wrote:
> HI Eric,
> 
> On Tue, Dec 8, 2020 at 2:13 AM Auger Eric <eric.auger@redhat.com> wrote:
>>
>> Hi Vikas,
>>
>> On 12/3/20 3:50 PM, Vikas Gupta wrote:
>>> Hi Eric,
>>>
>>> On Wed, Dec 2, 2020 at 8:14 PM Auger Eric <eric.auger@redhat.com> wrote:
>>>>
>>>> Hi Vikas,
>>>>
>>>> On 11/24/20 5:16 PM, Vikas Gupta wrote:
>>>>> MSI support for platform devices.
>>>>>
>>>>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
>>>>> ---
>>>>>  drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
>>>>>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
>>>>>  drivers/vfio/platform/vfio_platform_private.h |  16 ++
>>>>>  include/uapi/linux/vfio.h                     |  43 +++
>>>>>  4 files changed, 401 insertions(+), 17 deletions(-)
>>>>>
>>>>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>>>>> index c0771a9567fb..b0bfc0f4ee1f 100644
>>>>> --- a/drivers/vfio/platform/vfio_platform_common.c
>>>>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>>>>> @@ -16,6 +16,7 @@
>>>>>  #include <linux/types.h>
>>>>>  #include <linux/uaccess.h>
>>>>>  #include <linux/vfio.h>
>>>>> +#include <linux/nospec.h>
>>>>>
>>>>>  #include "vfio_platform_private.h"
>>>>>
>>>>> @@ -344,9 +345,16 @@ static long vfio_platform_ioctl(void *device_data,
>>>>>
>>>>>       } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
>>>>>               struct vfio_irq_info info;
>>>>> +             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
>>>>> +             struct vfio_irq_info_cap_msi *msi_info = NULL;
>>>>> +             unsigned long capsz;
>>>>> +             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
>>>>>
>>>>>               minsz = offsetofend(struct vfio_irq_info, count);
>>>>>
>>>>> +             /* For backward compatibility, cannot require this */
>>>>> +             capsz = offsetofend(struct vfio_irq_info, cap_offset);
>>>>> +
>>>>>               if (copy_from_user(&info, (void __user *)arg, minsz))
>>>>>                       return -EFAULT;
>>>>>
>>>>> @@ -356,9 +364,89 @@ static long vfio_platform_ioctl(void *device_data,
>>>>>               if (info.index >= vdev->num_irqs)
>>>>>                       return -EINVAL;
>>>>>
>>>>> +             if (info.argsz >= capsz)
>>>>> +                     minsz = capsz;
>>>>> +
>>>>> +             if (info.index == ext_irq_index) {
>>>> nit: n case we add new ext indices afterwards, I would check info.index
>>>> -  ext_irq_index against an VFIO_EXT_IRQ_MSI define.
>>>>> +                     struct vfio_irq_info_cap_type cap_type = {
>>>>> +                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
>>>>> +                             .header.version = 1 };
>>>>> +                     int i;
>>>>> +                     int ret;
>>>>> +                     int num_msgs;
>>>>> +                     size_t msi_info_size;
>>>>> +                     struct vfio_platform_irq *irq;
>>>> nit: I think generally the opposite order (length) is chosen. This also
>>>> would better match the existing style in this file
>>> Ok will fix it
>>>>> +
>>>>> +                     info.index = array_index_nospec(info.index,
>>>>> +                                                     vdev->num_irqs);
>>>>> +
>>>>> +                     irq = &vdev->irqs[info.index];
>>>>> +
>>>>> +                     info.flags = irq->flags;
>>>> I think this can be removed given [*]
>>> Sure.
>>>>> +                     cap_type.type = irq->type;
>>>>> +                     cap_type.subtype = irq->subtype;
>>>>> +
>>>>> +                     ret = vfio_info_add_capability(&caps,
>>>>> +                                                    &cap_type.header,
>>>>> +                                                    sizeof(cap_type));
>>>>> +                     if (ret)
>>>>> +                             return ret;
>>>>> +
>>>>> +                     num_msgs = irq->num_ctx;
>>>> do would want to return the cap even if !num_ctx?
>>> If num_ctx = 0 then VFIO_IRQ_INFO_CAP_MSI_DESCS should not be written.
>>> I`ll take care of same.
>>>>> +
>>>>> +                     msi_info_size = struct_size(msi_info, msgs, num_msgs);
>>>>> +
>>>>> +                     msi_info = kzalloc(msi_info_size, GFP_KERNEL);
>>>>> +                     if (!msi_info) {
>>>>> +                             kfree(caps.buf);
>>>>> +                             return -ENOMEM;
>>>>> +                     }
>>>>> +
>>>>> +                     msi_info->header.id = VFIO_IRQ_INFO_CAP_MSI_DESCS;
>>>>> +                     msi_info->header.version = 1;
>>>>> +                     msi_info->nr_msgs = num_msgs;
>>>>> +
>>>>> +                     for (i = 0; i < num_msgs; i++) {
>>>>> +                             struct vfio_irq_ctx *ctx = &irq->ctx[i];
>>>>> +
>>>>> +                             msi_info->msgs[i].addr_lo = ctx->msg.address_lo;
>>>>> +                             msi_info->msgs[i].addr_hi = ctx->msg.address_hi;
>>>>> +                             msi_info->msgs[i].data = ctx->msg.data;
>>>>> +                     }
>>>>> +
>>>>> +                     ret = vfio_info_add_capability(&caps, &msi_info->header,
>>>>> +                                                    msi_info_size);
>>>>> +                     if (ret) {
>>>>> +                             kfree(msi_info);
>>>>> +                             kfree(caps.buf);
>>>>> +                             return ret;
>>>>> +                     }
>>>>> +             }
>>>>> +
>>>>>               info.flags = vdev->irqs[info.index].flags;
>>>> [*]
>>> Will fix it.
>>>>>               info.count = vdev->irqs[info.index].count;
>>>>>
>>>>> +             if (caps.size) {
>>>>> +                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
>>>>> +                     if (info.argsz < sizeof(info) + caps.size) {
>>>>> +                             info.argsz = sizeof(info) + caps.size;
>>>>> +                             info.cap_offset = 0;
>>>>> +                     } else {
>>>>> +                             vfio_info_cap_shift(&caps, sizeof(info));
>>>>> +                             if (copy_to_user((void __user *)arg +
>>>>> +                                               sizeof(info), caps.buf,
>>>>> +                                               caps.size)) {
>>>>> +                                     kfree(msi_info);
>>>>> +                                     kfree(caps.buf);
>>>>> +                                     return -EFAULT;
>>>>> +                             }
>>>>> +                             info.cap_offset = sizeof(info);
>>>>> +                     }
>>>>> +
>>>>> +                     kfree(msi_info);
>>>>> +                     kfree(caps.buf);
>>>>> +             }
>>>>> +
>>>>>               return copy_to_user((void __user *)arg, &info, minsz) ?
>>>>>                       -EFAULT : 0;
>>>>>
>>>>> @@ -366,6 +454,7 @@ static long vfio_platform_ioctl(void *device_data,
>>>>>               struct vfio_irq_set hdr;
>>>>>               u8 *data = NULL;
>>>>>               int ret = 0;
>>>>> +             int max;
>>>>>               size_t data_size = 0;
>>>>>
>>>>>               minsz = offsetofend(struct vfio_irq_set, count);
>>>>> @@ -373,8 +462,14 @@ static long vfio_platform_ioctl(void *device_data,
>>>>>               if (copy_from_user(&hdr, (void __user *)arg, minsz))
>>>>>                       return -EFAULT;
>>>>>
>>>>> -             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
>>>>> -                                              vdev->num_irqs, &data_size);
>>>>> +             if (hdr.index >= vdev->num_irqs)
>>>>> +                     return -EINVAL;
>>>>> +
>>>>> +             max = vdev->irqs[hdr.index].count;
>>>>> +
>>>>> +             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
>>>>> +                                                      vdev->num_irqs,
>>>>> +                                                      &data_size);
>>>>>               if (ret)
>>>>>                       return ret;
>>>>>
>>>>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
>>>>> index c5b09ec0a3c9..4066223e5b2e 100644
>>>>> --- a/drivers/vfio/platform/vfio_platform_irq.c
>>>>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
>>>>> @@ -8,10 +8,12 @@
>>>>>
>>>>>  #include <linux/eventfd.h>
>>>>>  #include <linux/interrupt.h>
>>>>> +#include <linux/eventfd.h>
>>>>>  #include <linux/slab.h>
>>>>>  #include <linux/types.h>
>>>>>  #include <linux/vfio.h>
>>>>>  #include <linux/irq.h>
>>>>> +#include <linux/msi.h>
>>>>>
>>>>>  #include "vfio_platform_private.h"
>>>>>
>>>>> @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>>>>>       return 0;
>>>>>  }
>>>>>
>>>>> +/* MSI/MSIX */
>>>>> +static irqreturn_t vfio_msihandler(int irq, void *arg)
>>>>> +{
>>>>> +     struct eventfd_ctx *trigger = arg;
>>>>> +
>>>>> +     eventfd_signal(trigger, 1);
>>>>> +     return IRQ_HANDLED;
>>>>> +}
>>>>> +
>>>>> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
>>>>> +{
>>>>> +     int i;
>>>>> +     struct vfio_platform_irq *irq;
>>>>> +     u16 index = desc->platform.msi_index;
>>>>> +     struct device *dev = msi_desc_to_dev(desc);
>>>>> +     struct vfio_device *device = dev_get_drvdata(dev);
>>>>> +     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
>>>>> +                                             vfio_device_data(device);
>>>>> +
>>>>> +     for (i = 0; i < vdev->num_irqs; i++)
>>>>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
>>>>> +                     irq = &vdev->irqs[i];
>>>>> +
>>>>> +     irq->ctx[index].msg.address_lo = msg->address_lo;
>>>>> +     irq->ctx[index].msg.address_hi = msg->address_hi;
>>>>> +     irq->ctx[index].msg.data = msg->data;
>>>>> +}
>>>>> +
>>>>> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
>>>>> +                        struct vfio_platform_irq *irq, int nvec)
>>>>> +{
>>>>> +     int ret;
>>>>> +     int msi_idx = 0;
>>>>> +     struct msi_desc *desc;
>>>>> +     struct device *dev = vdev->device;
>>>>> +
>>>>> +     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
>>>>> +     if (!irq->ctx)
>>>>> +             return -ENOMEM;
>>>>> +
>>>>> +     /* Allocate platform MSIs */
>>>>> +     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
>>>>> +     if (ret < 0) {
>>>>> +             kfree(irq->ctx);
>>>>> +             return ret;
>>>>> +     }
>>>>> +
>>>>> +     for_each_msi_entry(desc, dev) {
>>>>> +             irq->ctx[msi_idx].hwirq = desc->irq;
>>>>> +             msi_idx++;
>>>>> +     }
>>>>> +
>>>>> +     irq->num_ctx = nvec;
>>>>> +     irq->config_msi = 1;
>>>>> +
>>>>> +     return 0;
>>>>> +}
>>>>> +
>>>>> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
>>>>> +                                   int vector, int fd)
>>>>> +{
>>>>> +     struct eventfd_ctx *trigger;
>>>>> +     int irq_num, ret;
>>>>> +
>>>>> +     if (vector < 0 || vector >= irq->num_ctx)
>>>>> +             return -EINVAL;
>>>>> +
>>>>> +     irq_num = irq->ctx[vector].hwirq;
>>>>> +
>>>>> +     if (irq->ctx[vector].trigger) {
>>>>> +             free_irq(irq_num, irq->ctx[vector].trigger);
>>>>> +             kfree(irq->ctx[vector].name);
>>>>> +             eventfd_ctx_put(irq->ctx[vector].trigger);
>>>>> +             irq->ctx[vector].trigger = NULL;
>>>>> +     }
>>>>> +
>>>>> +     if (fd < 0)
>>>>> +             return 0;
>>>>> +
>>>>> +     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
>>>>> +                                       "vfio-msi[%d]", vector);
>>>>> +     if (!irq->ctx[vector].name)
>>>>> +             return -ENOMEM;
>>>>> +
>>>>> +     trigger = eventfd_ctx_fdget(fd);
>>>>> +     if (IS_ERR(trigger)) {
>>>>> +             kfree(irq->ctx[vector].name);
>>>>> +             return PTR_ERR(trigger);
>>>>> +     }
>>>>> +
>>>>> +     ret = request_irq(irq_num, vfio_msihandler, 0,
>>>>> +                       irq->ctx[vector].name, trigger);
>>>>> +     if (ret) {
>>>>> +             kfree(irq->ctx[vector].name);
>>>>> +             eventfd_ctx_put(trigger);
>>>>> +             return ret;
>>>>> +     }
>>>>> +
>>>>> +     irq->ctx[vector].trigger = trigger;
>>>>> +
>>>>> +     return 0;
>>>>> +}
>>>>> +
>>>>> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
>>>>> +                           unsigned int count, int32_t *fds)
>>>>> +{
>>>>> +     int i, j, ret = 0;
>>>>> +
>>>>> +     if (start >= irq->num_ctx || start + count > irq->num_ctx)
>>>>> +             return -EINVAL;
>>>>> +
>>>>> +     for (i = 0, j = start; i < count && !ret; i++, j++) {
>>>>> +             int fd = fds ? fds[i] : -1;
>>>>> +
>>>>> +             ret = vfio_msi_set_vector_signal(irq, j, fd);
>>>>> +     }
>>>>> +
>>>>> +     if (ret) {
>>>>> +             for (--j; j >= (int)start; j--)
>>>>> +                     vfio_msi_set_vector_signal(irq, j, -1);
>>>>> +     }
>>>>> +
>>>>> +     return ret;
>>>>> +}
>>>>> +
>>>>> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
>>>>> +                          struct vfio_platform_irq *irq)
>>>>> +{
>>>>> +     struct device *dev = vdev->device;
>>>>> +
>>>>> +     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
>>>>> +
>>>>> +     platform_msi_domain_free_irqs(dev);
>>>>> +
>>>>> +     irq->config_msi = 0;
>>>>> +     irq->num_ctx = 0;
>>>>> +
>>>>> +     kfree(irq->ctx);
>>>>> +}
>>>>> +
>>>>> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
>>>>> +                             unsigned int index, unsigned int start,
>>>>> +                             unsigned int count, uint32_t flags, void *data)
>>>>> +{
>>>>> +     int i;
>>>>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>>>>> +
>>>>> +     if (start + count > irq->count)
>>>>> +             return -EINVAL;
>>>>> +
>>>>> +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
>>>>> +             vfio_msi_disable(vdev, irq);
>>>>> +             return 0;
>>>>> +     }
>>>>> +
>>>>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
>>>>> +             s32 *fds = data;
>>>>> +             int ret;
>>>>> +
>>>>> +             if (irq->config_msi)
>>>>> +                     return vfio_msi_set_block(irq, start, count,
>>>>> +                                               fds);
>>>>> +             ret = vfio_msi_enable(vdev, irq, start + count);
>>>>> +             if (ret)
>>>>> +                     return ret;
>>>>> +
>>>>> +             ret = vfio_msi_set_block(irq, start, count, fds);
>>>>> +             if (ret)
>>>>> +                     vfio_msi_disable(vdev, irq);
>>>>> +
>>>>> +             return ret;
>>>>> +     }
>>>>> +
>>>>> +     for (i = start; i < start + count; i++) {
>>>>> +             if (!irq->ctx[i].trigger)
>>>>> +                     continue;
>>>>> +             if (flags & VFIO_IRQ_SET_DATA_NONE) {
>>>>> +                     eventfd_signal(irq->ctx[i].trigger, 1);
>>>>> +             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
>>>>> +                     u8 *bools = data;
>>>>> +
>>>>> +                     if (bools[i - start])
>>>>> +                             eventfd_signal(irq->ctx[i].trigger, 1);
>>>>> +             }
>>>>> +     }
>>>>> +
>>>>> +     return 0;
>>>>> +}
>>>>> +
>>>>>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>>>                                uint32_t flags, unsigned index, unsigned start,
>>>>>                                unsigned count, void *data)
>>>>> @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>>>                   unsigned start, unsigned count, uint32_t flags,
>>>>>                   void *data) = NULL;
>>>>>
>>>>> -     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>>>> -     case VFIO_IRQ_SET_ACTION_MASK:
>>>>> -             func = vfio_platform_set_irq_mask;
>>>>> -             break;
>>>>> -     case VFIO_IRQ_SET_ACTION_UNMASK:
>>>>> -             func = vfio_platform_set_irq_unmask;
>>>>> -             break;
>>>>> -     case VFIO_IRQ_SET_ACTION_TRIGGER:
>>>>> -             func = vfio_platform_set_irq_trigger;
>>>>> -             break;
>>>>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>>>>> +
>>>>> +     if (irq->type == VFIO_IRQ_TYPE_MSI) {
>>>>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>>>> +             case VFIO_IRQ_SET_ACTION_MASK:
>>>>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
>>>>> +                     break;
>>>>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
>>>>> +                     func = vfio_set_msi_trigger;
>>>>> +                     break;
>>>>> +             }
>>>>> +     } else {
>>>>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>>>> +             case VFIO_IRQ_SET_ACTION_MASK:
>>>>> +                     func = vfio_platform_set_irq_mask;
>>>>> +                     break;
>>>>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
>>>>> +                     func = vfio_platform_set_irq_unmask;
>>>>> +                     break;
>>>>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
>>>>> +                     func = vfio_platform_set_irq_trigger;
>>>>> +                     break;
>>>>> +             }
>>>>>       }
>>>>>
>>>>>       if (!func)
>>>>> @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>>>
>>>>>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>>>  {
>>>>> -     int cnt = 0, i;
>>>>> +     int i;
>>>>> +     int cnt = 0;
>>>>> +     int num_irqs;
>>>>> +     struct device *dev = vdev->device;
>>>>>
>>>>>       while (vdev->get_irq(vdev, cnt) >= 0)
>>>>>               cnt++;
>>>>>
>>>>> -     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
>>>>> +     num_irqs = cnt;
>>>>> +
>>>>> +     if (dev->msi_domain)
>>>>> +             num_irqs++;
>>>>> +
>>>>> +     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
>>>>> +                          GFP_KERNEL);
>>>>>       if (!vdev->irqs)
>>>>>               return -ENOMEM;
>>>>>
>>>>> @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>>>               vdev->irqs[i].masked = false;
>>>>>       }
>>>>>
>>>>> -     vdev->num_irqs = cnt;
>>>>> +     /*
>>>>> +      * MSI block is added at last index and its an ext irq
>>>> it is
>>>>> +      */
>>>>> +     if (dev->msi_domain) {
>>>>> +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
>>>>> +             vdev->irqs[i].count = NR_IRQS;
>>>> why NR_IRQS?
>>>  Since different devices can have different numbers of MSI(s) so we
>>> need to initialize with max possible values. Can you please suggest if
>>> this value does not seem appropriate?
>> As opposed to PCIe, the userspace has no real way to guess how many
>> vectors can be set (what vfio_pci_get_irq_count does). This also means
>> we do not fully implement the original API as we are not able to report
>> an accurate value for .count. How will the user determine how many
>> vectors he can use?
> 
> I believe user space will know how many MSIs platform device can have.
> We have assigned NR_IRQS because that’s what maximum number of
> interrupt can be supported on a specific platform. In case user space
> errantly tries to allocate any number of MSIs in kernel
> platform_msi_domain_alloc_irqs should fail.
> Do you think this approach can create an issue as .count is not
> exactly the same as MSIs are existing with device? Is it necessary to
> have a PCIe kind of approach as PCIe is standard but platform devices
> do not have standards?
I had a short discussion about this with Alex and I think we came to the
conclusion that is an issue not to be able to report accurately the
number of supported vectors. Assuming that the userspace knows the
number of vectors does not really look as a valid assumption and does
not really fit the vfio spirit.

The previous approach beased on a kernel module allowed to report their
actual number. Even if it is heavier it allows to fill the info supposed
to be returned by the GET_IRQ ioctl and we think it is better.

Thanks

Eric
> 
>>>>> +             vdev->irqs[i].hwirq = 0;
>>>>> +             vdev->irqs[i].masked = false;
>>>>> +             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
>>>>> +             vdev->num_ext_irqs = 1;
>>>>> +     }
>>>>> +
>>>>> +     vdev->num_irqs = num_irqs;
>>>>>
>>>>>       return 0;
>>>>>  err:
>>>>> @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>>>>>  {
>>>>>       int i;
>>>>>
>>>>> -     for (i = 0; i < vdev->num_irqs; i++)
>>>>> -             vfio_set_trigger(vdev, i, -1, NULL);
>>>>> +     for (i = 0; i < vdev->num_irqs; i++) {
>>>>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
>>>>> +                     vfio_set_msi_trigger(vdev, i, 0, 0,
>>>>> +                                          VFIO_IRQ_SET_DATA_NONE, NULL);
>>>>> +             else
>>>>> +                     vfio_set_trigger(vdev, i, -1, NULL);
>>>>> +     }
>>>>>
>>>>>       vdev->num_irqs = 0;
>>>>>       kfree(vdev->irqs);
>>>>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>>>>> index 289089910643..7bbb05988705 100644
>>>>> --- a/drivers/vfio/platform/vfio_platform_private.h
>>>>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>>>>> @@ -9,6 +9,7 @@
>>>>>
>>>>>  #include <linux/types.h>
>>>>>  #include <linux/interrupt.h>
>>>>> +#include <linux/msi.h>
>>>>>
>>>>>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
>>>>>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
>>>>> @@ -19,9 +20,18 @@
>>>>>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
>>>>>       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>>>>>
>>>>> +struct vfio_irq_ctx {
>>>>> +     int                     hwirq;
>>>>> +     char                    *name;
>>>>> +     struct msi_msg          msg;
>>>>> +     struct eventfd_ctx      *trigger;
>>>>> +};
>>>>> +
>>>>>  struct vfio_platform_irq {
>>>>>       u32                     flags;
>>>>>       u32                     count;
>>>>> +     int                     num_ctx;
>>>>> +     struct vfio_irq_ctx     *ctx;
>>>>>       int                     hwirq;
>>>>>       char                    *name;
>>>>>       struct eventfd_ctx      *trigger;
>>>>> @@ -29,6 +39,11 @@ struct vfio_platform_irq {
>>>>>       spinlock_t              lock;
>>>>>       struct virqfd           *unmask;
>>>>>       struct virqfd           *mask;
>>>>> +
>>>>> +     /* for extended irqs */
>>>>> +     u32                     type;
>>>>> +     u32                     subtype;
>>>>> +     int                     config_msi;
>>>>>  };
>>>>>
>>>>>  struct vfio_platform_region {
>>>>> @@ -46,6 +61,7 @@ struct vfio_platform_device {
>>>>>       u32                             num_regions;
>>>>>       struct vfio_platform_irq        *irqs;
>>>>>       u32                             num_irqs;
>>>>> +     int                             num_ext_irqs;
>>>>>       int                             refcnt;
>>>>>       struct mutex                    igate;
>>>>>       struct module                   *parent_module;
>>>>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>>>>> index 2f313a238a8f..598d1c944283 100644
>>>>> --- a/include/uapi/linux/vfio.h
>>>>> +++ b/include/uapi/linux/vfio.h
>>>>> @@ -697,11 +697,54 @@ struct vfio_irq_info {
>>>>>  #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
>>>>>  #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
>>>>>  #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
>>>>> +#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
>>>>>       __u32   index;          /* IRQ index */
>>>>>       __u32   count;          /* Number of IRQs within this index */
>>>>> +     __u32   cap_offset;     /* Offset within info struct of first cap */
>>>>>  };
>>>>>  #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
>>>>>
>>>>> +/*
>>>>> + * The irq type capability allows IRQs unique to a specific device or
>>>>> + * class of devices to be exposed.
>>>>> + *
>>>>> + * The structures below define version 1 of this capability.
>>>>> + */
>>>>> +#define VFIO_IRQ_INFO_CAP_TYPE               3
>>>>> +
>>>>> +struct vfio_irq_info_cap_type {
>>>>> +     struct vfio_info_cap_header header;
>>>>> +     __u32 type;     /* global per bus driver */
>>>>> +     __u32 subtype;  /* type specific */
>>>>> +};
>>>>> +
>>>>> +/*
>>>>> + * List of IRQ types, global per bus driver.
>>>>> + * If you introduce a new type, please add it here.
>>>>> + */
>>>>> +
>>>>> +/* Non PCI devices having MSI(s) support */
>>>>> +#define VFIO_IRQ_TYPE_MSI            (1)
>>>>> +
>>>>> +/*
>>>>> + * The msi capability allows the user to use the msi msg to
>>>>> + * configure the iova for the msi configuration.
>>>>> + * The structures below define version 1 of this capability.
>>>>> + */
>>>>> +#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
>>>>> +
>>>>> +struct vfio_irq_msi_msg {
>>>>> +     __u32   addr_lo;
>>>>> +     __u32   addr_hi;
>>>>> +     __u32   data;
>>>>> +};
>>>>> +
>>>>> +struct vfio_irq_info_cap_msi {
>>>>> +     struct vfio_info_cap_header header;
>>>>> +     __u32 nr_msgs;
>>>> I think you should align a __u32   reserved field to have a 64b alignment
>>> Sure.
>>>>> +     struct vfio_irq_msi_msg msgs[];
>>>> Please can you clarify why this cap is needed versus your prior approach.
>>> In the previous patch, the reset module was configuring the device
>>> with MSI msg/data now since the module is not available, user space
>>> needs to have this data.
>>> With this approach userspace just needs the pairs <msg and ctx >
>>> associated with the MSI(s) and it can choose to configure the MSI(s)
>>> sources accordingly.
>>> Let me know if this approach does not look appropriate.
>> This comes to the question I asked in my previous email, ie. could you
>> give some more info about the expected MSI setup sequence? May be the
>> opportunity to enhance the commit message ;-)
> 
> With our proposal, user space needs to know how many MSI sources there
> are as we want user space to map msi-msg(s) to specific MSI sources.
> Let us assume there are max ‘n’ MSI sources a platform device have,
> user-space requests ‘n’ msi(s) allocation in the kernel which in turn
> gets ‘n’ msi_msg with help of cap (VFIO_IRQ_INFO_CAP_MSI_DESCS). User
> space is free to map/configure msi_msg->data value to any particular
> source. Since msi_msg->data is one to one mapped to eventfd,
> user-space knows that particular eventfd corresponds to the same MSI
> source which it has configured with the msi_msg->data value.
> 
> Steps:
> a)   User space allocates nvec in kernel. Ioctl VFIO_DEVICE_SET_IRQS
> b)   Kernel allocates nvec and does associated IRQ allocation, eventfd etc.
> c)   User space gets the msi_msgs VFIO_DEVICE_GET_IRQ_INFO.
> d)   User space configures the particular MSI source with msi_msg info.
> Do these steps look OK?
> We can add more information where this cap is introduced in the file
> uapi/linux/vfio.h.
> 
> Thanks,
> Vikas
>>
>> Thanks
>>
>> Eric
>>>
>>> Thanks,
>>> Vikas
>>>>> +};
>>>>> +
>>>>>  /**
>>>>>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
>>>>>   *
>>>>>
>>>> Thanks
>>>>
>>>> Eric
>>>>
>>
> 

Re: [RFC, v2 0/1] msi support for platform devices

[Auger Eric]

Hi Vikas,
On 11/24/20 5:16 PM, Vikas Gupta wrote:
> This RFC adds support for MSI for platform devices.
> MSI block is added as an ext irq along with the existing
> wired interrupt implementation.
> 
> Changes from:
> -------------
>  v1 to v2:
> 	1) IRQ allocation has been implemented as below:
> 	       ----------------------------
> 	       |IRQ-0|IRQ-1|....|IRQ-n|MSI|
>        	       ----------------------------
> 		MSI block has msi contexts and its implemneted
it is implemented
> 		as ext irq.
> 
> 	2) Removed vendor specific module for msi handling so
> 	   previously patch2 and patch3 are not required.
> 
> 	3) MSI related data is exported to userspace using 'caps'.
> 	 Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
> 	is taken from the Eric`s patch
>         https://patchwork.kernel.org/project/kvm/patch/20201116110030.32335-8-eric.auger@redhat.com/
So do you mean that by exposing the vectors, now you do not need the msi
module anymore?


Thanks

Eric
> 
> 
>  v0 to v1:
>    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
>    ii) Add MSI(s) at the end of the irq list of platform IRQs.
>        MSI(s) with first entry of MSI block has count and flag
>        information.
>        IRQ list: Allocation for IRQs + MSIs are allocated as below
>        Example: if there are 'n' IRQs and 'k' MSIs
>        -------------------------------------------------------
>        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
>        -------------------------------------------------------
>        MSI-0 will have count=k set and flags set accordingly.
> 
> Vikas Gupta (1):
>   vfio/platform: add support for msi
> 
>  drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
>  drivers/vfio/platform/vfio_platform_private.h |  16 ++
>  include/uapi/linux/vfio.h                     |  43 +++
>  4 files changed, 401 insertions(+), 17 deletions(-)
> 

Re: [RFC, v2 0/1] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 2435 bytes --]

HI Eric,

On Wed, Dec 2, 2020 at 8:13 PM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
> On 11/24/20 5:16 PM, Vikas Gupta wrote:
> > This RFC adds support for MSI for platform devices.
> > MSI block is added as an ext irq along with the existing
> > wired interrupt implementation.
> >
> > Changes from:
> > -------------
> >  v1 to v2:
> >       1) IRQ allocation has been implemented as below:
> >              ----------------------------
> >              |IRQ-0|IRQ-1|....|IRQ-n|MSI|
> >                      ----------------------------
> >               MSI block has msi contexts and its implemneted
> it is implemented
> >               as ext irq.
> >
> >       2) Removed vendor specific module for msi handling so
> >          previously patch2 and patch3 are not required.
> >
> >       3) MSI related data is exported to userspace using 'caps'.
> >        Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
> >       is taken from the Eric`s patch
> >         https://patchwork.kernel.org/project/kvm/patch/20201116110030.32335-8-eric.auger@redhat.com/
> So do you mean that by exposing the vectors, now you do not need the msi
> module anymore?
Yes, with the support of caps we can expose the MSI related data to
userspace and userspace can configure the device, which previous
patches were doing in the kernel module.

Thanks,
Vikas
>
>
> Thanks
>
> Eric
> >
> >
> >  v0 to v1:
> >    i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
> >    ii) Add MSI(s) at the end of the irq list of platform IRQs.
> >        MSI(s) with first entry of MSI block has count and flag
> >        information.
> >        IRQ list: Allocation for IRQs + MSIs are allocated as below
> >        Example: if there are 'n' IRQs and 'k' MSIs
> >        -------------------------------------------------------
> >        |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
> >        -------------------------------------------------------
> >        MSI-0 will have count=k set and flags set accordingly.
> >
> > Vikas Gupta (1):
> >   vfio/platform: add support for msi
> >
> >  drivers/vfio/platform/vfio_platform_common.c  |  99 ++++++-
> >  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
> >  drivers/vfio/platform/vfio_platform_private.h |  16 ++
> >  include/uapi/linux/vfio.h                     |  43 +++
> >  4 files changed, 401 insertions(+), 17 deletions(-)
> >
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC, v3 0/2] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 3288 bytes --]

This RFC adds support for MSI for platform devices.
MSI block is added as an ext irq along with the existing
wired interrupt implementation. The patchset exports two
caps for MSI and related data to configure MSI source device.

Changes from:
-------------
 v2 to v3:
	1) Restored the vendor specific module to get max number
	   of MSIs supported and .count value initialized.
	2) Comments from Eric addressed.

 v1 to v2:
	1) IRQ allocation has been implemented as below:
	       ----------------------------
	       |IRQ-0|IRQ-1|....|IRQ-n|MSI|
       	       ----------------------------
		MSI block has msi contexts and its implemneted
		as ext irq.

	2) Removed vendor specific module for msi handling so
	   previously patch2 and patch3 are not required.

	3) MSI related data is exported to userspace using 'caps'.
	 Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
	is taken from the Eric`s patch
        https://patchwork.kernel.org/project/kvm/patch/20201116110030.32335-8-eric.auger@redhat.com/


 v0 to v1:
   i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
   ii) Add MSI(s) at the end of the irq list of platform IRQs.
       MSI(s) with first entry of MSI block has count and flag
       information.
       IRQ list: Allocation for IRQs + MSIs are allocated as below
       Example: if there are 'n' IRQs and 'k' MSIs
       -------------------------------------------------------
       |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
       -------------------------------------------------------
       MSI-0 will have count=k set and flags set accordingly.


Vikas Gupta (2):
  vfio/platform: add support for msi
  vfio/platform: msi: add Broadcom platform devices

 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/Makefile                |   1 +
 drivers/vfio/platform/msi/Kconfig             |   9 +
 drivers/vfio/platform/msi/Makefile            |   2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  |  49 ++++
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 9 files changed, 558 insertions(+), 19 deletions(-)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c

-- 
2.17.1


-- 
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC v3 1/2] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 21239 bytes --]

MSI support for platform devices.The MSI block
is added as an extended IRQ which exports caps
VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  32 +++
 include/uapi/linux/vfio.h                     |  44 +++
 4 files changed, 496 insertions(+), 19 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index fb4b385191f2..c936852f35d7 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -26,6 +27,8 @@
 #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
 
 static LIST_HEAD(reset_list);
+/* devices having MSI support */
+static LIST_HEAD(msi_list);
 static DEFINE_MUTEX(driver_lock);
 
 static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
@@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
 	return reset_fn;
 }
 
+static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi = false;
+	struct vfio_platform_msi_node *iter;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry(iter, &msi_list, link) {
+		if (!strcmp(iter->compat, vdev->compat) &&
+		    try_module_get(iter->owner)) {
+			vdev->msi_module = iter->owner;
+			vdev->of_get_msi = iter->of_get_msi;
+			has_msi = true;
+			break;
+		}
+	}
+	mutex_unlock(&driver_lock);
+	return has_msi;
+}
+
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
 				    struct device *dev)
 {
@@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 	return vdev->of_reset ? 0 : -ENOENT;
 }
 
+static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
+{
+	bool has_msi;
+
+	has_msi = vfio_platform_lookup_msi(vdev);
+	if (!has_msi) {
+		request_module("vfio-msi:%s", vdev->compat);
+		has_msi = vfio_platform_lookup_msi(vdev);
+	}
+
+	return has_msi ? 0 : -ENOENT;
+}
+
 static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 {
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
@@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
 		module_put(vdev->reset_module);
 }
 
+static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
+{
+	if (vdev->of_get_msi)
+		module_put(vdev->msi_module);
+}
+
 static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
@@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
+		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+		struct vfio_irq_info_cap_msi *msi_info = NULL;
+		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+		unsigned long capsz;
+		u32 index;
 
 		minsz = offsetofend(struct vfio_irq_info, count);
 
+		/* For backward compatibility, cannot require this */
+		capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 
@@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
 		if (info.index >= vdev->num_irqs)
 			return -EINVAL;
 
-		info.flags = vdev->irqs[info.index].flags;
-		info.count = vdev->irqs[info.index].count;
+		if (info.argsz >= capsz)
+			minsz = capsz;
+
+		index = info.index;
+
+		info.flags = vdev->irqs[index].flags;
+		info.count = vdev->irqs[index].count;
+
+		if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+			struct vfio_irq_info_cap_type cap_type = {
+				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
+				.header.version = 1 };
+			struct vfio_platform_irq *irq;
+			size_t msi_info_size;
+			int num_msgs;
+			int ret;
+			int i;
+
+			index = array_index_nospec(index,
+						   vdev->num_irqs);
+			irq = &vdev->irqs[index];
+
+			cap_type.type = irq->type;
+			cap_type.subtype = irq->subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						       &cap_type.header,
+						       sizeof(cap_type));
+			if (ret)
+				return ret;
+
+			num_msgs = irq->num_ctx;
+			if (num_msgs) {
+				msi_info_size = struct_size(msi_info,
+							    msgs, num_msgs);
+
+				msi_info = kzalloc(msi_info_size, GFP_KERNEL);
+				if (!msi_info) {
+					kfree(caps.buf);
+					return -ENOMEM;
+				}
+
+				msi_info->header.id =
+						VFIO_IRQ_INFO_CAP_MSI_DESCS;
+				msi_info->header.version = 1;
+				msi_info->nr_msgs = num_msgs;
+
+				for (i = 0; i < num_msgs; i++) {
+					struct vfio_irq_ctx *ctx = &irq->ctx[i];
+
+					msi_info->msgs[i].addr_lo =
+							ctx->msg.address_lo;
+					msi_info->msgs[i].addr_hi =
+							ctx->msg.address_hi;
+					msi_info->msgs[i].data = ctx->msg.data;
+				}
+
+				ret = vfio_info_add_capability(&caps,
+							&msi_info->header,
+							msi_info_size);
+				if (ret) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return ret;
+				}
+			}
+		}
+
+		if (caps.size) {
+			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+			if (info.argsz < sizeof(info) + caps.size) {
+				info.argsz = sizeof(info) + caps.size;
+				info.cap_offset = 0;
+			} else {
+				vfio_info_cap_shift(&caps, sizeof(info));
+				if (copy_to_user((void __user *)arg +
+						  sizeof(info), caps.buf,
+						  caps.size)) {
+					kfree(msi_info);
+					kfree(caps.buf);
+					return -EFAULT;
+				}
+				info.cap_offset = sizeof(info);
+			}
+
+			kfree(msi_info);
+			kfree(caps.buf);
+		}
 
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
@@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-						 vdev->num_irqs, &data_size);
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+							 vdev->num_irqs,
+							 &data_size);
 		if (ret)
 			return ret;
 
@@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 		return ret;
 	}
 
+	ret = vfio_platform_get_msi(vdev);
+	if (ret)
+		dev_info(vdev->device, "msi not supported\n");
+
 	group = vfio_iommu_group_get(dev);
 	if (!group) {
 		dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
@@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 put_iommu:
 	vfio_iommu_group_put(group, dev);
 put_reset:
+	vfio_platform_put_msi(vdev);
 	vfio_platform_put_reset(vdev);
 	return ret;
 }
@@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
 }
 EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
 
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
+{
+	mutex_lock(&driver_lock);
+	list_add(&node->link, &msi_list);
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
+
+void vfio_platform_unregister_msi(const char *compat)
+{
+	struct vfio_platform_msi_node *iter, *temp;
+
+	mutex_lock(&driver_lock);
+	list_for_each_entry_safe(iter, temp, &msi_list, link) {
+		if (!strcmp(iter->compat, compat)) {
+			list_del(&iter->link);
+			break;
+		}
+	}
+
+	mutex_unlock(&driver_lock);
+}
+EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
+
 MODULE_VERSION(DRIVER_VERSION);
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR(DRIVER_AUTHOR);
diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index c5b09ec0a3c9..e0f4696afedd 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	int i;
+	struct vfio_platform_irq *irq;
+	u16 index = desc->platform.msi_index;
+	struct device *dev = msi_desc_to_dev(desc);
+	struct vfio_device *device = dev_get_drvdata(dev);
+	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+						vfio_device_data(device);
+
+	for (i = 0; i < vdev->num_irqs; i++)
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			irq = &vdev->irqs[i];
+
+	irq->ctx[index].msg.address_lo = msg->address_lo;
+	irq->ctx[index].msg.address_hi = msg->address_hi;
+	irq->ctx[index].msg.data = msg->data;
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+			   struct vfio_platform_irq *irq, int nvec)
+{
+	int ret;
+	int msi_idx = 0;
+	struct msi_desc *desc;
+	struct device *dev = vdev->device;
+
+	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+	if (!irq->ctx)
+		return -ENOMEM;
+
+	/* Allocate platform MSIs */
+	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+	if (ret < 0) {
+		kfree(irq->ctx);
+		return ret;
+	}
+
+	for_each_msi_entry(desc, dev) {
+		irq->ctx[msi_idx].hwirq = desc->irq;
+		msi_idx++;
+	}
+
+	irq->num_ctx = nvec;
+	irq->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+				      int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq_num, ret;
+
+	if (vector < 0 || vector >= irq->num_ctx)
+		return -EINVAL;
+
+	irq_num = irq->ctx[vector].hwirq;
+
+	if (irq->ctx[vector].trigger) {
+		free_irq(irq_num, irq->ctx[vector].trigger);
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(irq->ctx[vector].trigger);
+		irq->ctx[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+					  "vfio-msi[%d]", vector);
+	if (!irq->ctx[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(irq->ctx[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq_num, vfio_msihandler, 0,
+			  irq->ctx[vector].name, trigger);
+	if (ret) {
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	irq->ctx[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+			      unsigned int count, int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= irq->num_ctx || start + count > irq->num_ctx)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_msi_set_vector_signal(irq, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_msi_set_vector_signal(irq, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+			     struct vfio_platform_irq *irq)
+{
+	struct device *dev = vdev->device;
+
+	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+	platform_msi_domain_free_irqs(dev);
+
+	irq->config_msi = 0;
+	irq->num_ctx = 0;
+
+	kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+				unsigned int index, unsigned int start,
+				unsigned int count, uint32_t flags, void *data)
+{
+	int i;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (start + count > irq->count)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_msi_disable(vdev, irq);
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (irq->config_msi)
+			return vfio_msi_set_block(irq, start, count,
+						  fds);
+		ret = vfio_msi_enable(vdev, irq, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_msi_set_block(irq, start, count, fds);
+		if (ret)
+			vfio_msi_disable(vdev, irq);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!irq->ctx[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(irq->ctx[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(irq->ctx[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 				 uint32_t flags, unsigned index, unsigned start,
 				 unsigned count, void *data)
@@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 		    unsigned start, unsigned count, uint32_t flags,
 		    void *data) = NULL;
 
-	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-	case VFIO_IRQ_SET_ACTION_MASK:
-		func = vfio_platform_set_irq_mask;
-		break;
-	case VFIO_IRQ_SET_ACTION_UNMASK:
-		func = vfio_platform_set_irq_unmask;
-		break;
-	case VFIO_IRQ_SET_ACTION_TRIGGER:
-		func = vfio_platform_set_irq_trigger;
-		break;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (irq->type == VFIO_IRQ_TYPE_MSI) {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_set_msi_trigger;
+			break;
+		}
+	} else {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+			func = vfio_platform_set_irq_mask;
+			break;
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			func = vfio_platform_set_irq_unmask;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_platform_set_irq_trigger;
+			break;
+		}
 	}
 
 	if (!func)
@@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-	int cnt = 0, i;
+	int i;
+	int cnt = 0;
+	int num_irqs;
+	int msi_cnt = 0;
 
 	while (vdev->get_irq(vdev, cnt) >= 0)
 		cnt++;
 
-	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	if (vdev->of_get_msi) {
+		msi_cnt = vdev->of_get_msi(vdev);
+		num_irqs++;
+	}
+
+	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+			     GFP_KERNEL);
 	if (!vdev->irqs)
 		return -ENOMEM;
 
@@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		vdev->irqs[i].masked = false;
 	}
 
-	vdev->num_irqs = cnt;
+	/*
+	 * MSI block is added at last index and it is an ext irq
+	 */
+	if (msi_cnt > 0) {
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+		vdev->irqs[i].count = msi_cnt;
+		vdev->irqs[i].hwirq = 0;
+		vdev->irqs[i].masked = false;
+		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+		vdev->num_ext_irqs = 1;
+	}
+
+	vdev->num_irqs = num_irqs;
 
 	return 0;
 err:
@@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
 	int i;
 
-	for (i = 0; i < vdev->num_irqs; i++)
-		vfio_set_trigger(vdev, i, -1, NULL);
+	for (i = 0; i < vdev->num_irqs; i++) {
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			vfio_set_msi_trigger(vdev, i, 0, 0,
+					     VFIO_IRQ_SET_DATA_NONE, NULL);
+		else
+			vfio_set_trigger(vdev, i, -1, NULL);
+	}
 
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index 289089910643..1307feddda21 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI	0
+
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,12 +64,14 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
 	const char			*compat;
 	const char			*acpihid;
 	struct module			*reset_module;
+	struct module			*msi_module;
 	struct device			*device;
 
 	/*
@@ -65,11 +85,13 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
@@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
 	vfio_platform_reset_fn_t of_reset;
 };
 
+struct vfio_platform_msi_node {
+	struct list_head link;
+	char *compat;
+	struct module *owner;
+	vfio_platform_get_msi_fn_t of_get_msi;
+};
+
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 				      struct device *dev);
 extern struct vfio_platform_device *vfio_platform_remove_common
@@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
 extern void vfio_platform_unregister_reset(const char *compat,
 					   vfio_platform_reset_fn_t fn);
+void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
+void vfio_platform_unregister_msi(const char *compat);
+
 #define vfio_platform_register_reset(__compat, __reset)		\
 static struct vfio_platform_reset_node __reset ## _node = {	\
 	.owner = THIS_MODULE,					\
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index d1812777139f..53a7ff2b524e 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -697,11 +697,55 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
 	__u32	index;		/* IRQ index */
 	__u32	count;		/* Number of IRQs within this index */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE		3
+
+struct vfio_irq_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;     /* global per bus driver */
+	__u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI		(1)
+
+/*
+ * The msi capability allows the user to use the msi msg to
+ * configure the iova for the msi configuration.
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_MSI_DESCS	4
+
+struct vfio_irq_msi_msg {
+	__u32	addr_lo;
+	__u32	addr_hi;
+	__u32	data;
+};
+
+struct vfio_irq_info_cap_msi {
+	struct vfio_info_cap_header header;
+	__u32	nr_msgs;
+	__u32	reserved;
+	struct vfio_irq_msi_msg msgs[];
+};
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
-- 
2.17.1


-- 
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v3 1/2] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 12/14/20 6:45 PM, Vikas Gupta wrote:
> MSI support for platform devices.The MSI block
> is added as an extended IRQ which exports caps
> VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.
> 
> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> ---
>  drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
>  drivers/vfio/platform/vfio_platform_private.h |  32 +++
>  include/uapi/linux/vfio.h                     |  44 +++
>  4 files changed, 496 insertions(+), 19 deletions(-)
> 
> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> index fb4b385191f2..c936852f35d7 100644
> --- a/drivers/vfio/platform/vfio_platform_common.c
> +++ b/drivers/vfio/platform/vfio_platform_common.c
> @@ -16,6 +16,7 @@
>  #include <linux/types.h>
>  #include <linux/uaccess.h>
>  #include <linux/vfio.h>
> +#include <linux/nospec.h>
>  
>  #include "vfio_platform_private.h"
>  
> @@ -26,6 +27,8 @@
>  #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
>  
>  static LIST_HEAD(reset_list);
> +/* devices having MSI support */
nit: for devices using MSIs?
> +static LIST_HEAD(msi_list);
>  static DEFINE_MUTEX(driver_lock);
>  
>  static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
> @@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
>  	return reset_fn;
>  }
>  
> +static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
> +{
> +	bool has_msi = false;
> +	struct vfio_platform_msi_node *iter;
> +
> +	mutex_lock(&driver_lock);
> +	list_for_each_entry(iter, &msi_list, link) {
> +		if (!strcmp(iter->compat, vdev->compat) &&
> +		    try_module_get(iter->owner)) {
> +			vdev->msi_module = iter->owner;
> +			vdev->of_get_msi = iter->of_get_msi;
> +			has_msi = true;
> +			break;
> +		}
> +	}
> +	mutex_unlock(&driver_lock);
> +	return has_msi;
> +}
> +
>  static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
>  				    struct device *dev)
>  {
> @@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
>  	return vdev->of_reset ? 0 : -ENOENT;
>  }
>  
> +static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
> +{
> +	bool has_msi;
> +
> +	has_msi = vfio_platform_lookup_msi(vdev);
> +	if (!has_msi) {
> +		request_module("vfio-msi:%s", vdev->compat);
> +		has_msi = vfio_platform_lookup_msi(vdev);
> +	}
> +
> +	return has_msi ? 0 : -ENOENT;
> +}
> +
>  static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
>  {
>  	if (VFIO_PLATFORM_IS_ACPI(vdev))
> @@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
>  		module_put(vdev->reset_module);
>  }
>  
> +static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
> +{
> +	if (vdev->of_get_msi)
> +		module_put(vdev->msi_module);
> +}
> +
>  static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
>  {
>  	int cnt = 0, i;
> @@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
>  
>  	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
>  		struct vfio_irq_info info;
> +		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
> +		struct vfio_irq_info_cap_msi *msi_info = NULL;
> +		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
> +		unsigned long capsz;
> +		u32 index;
>  
>  		minsz = offsetofend(struct vfio_irq_info, count);
>  
> +		/* For backward compatibility, cannot require this */
> +		capsz = offsetofend(struct vfio_irq_info, cap_offset);
> +
>  		if (copy_from_user(&info, (void __user *)arg, minsz))
>  			return -EFAULT;
>  
> @@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
>  		if (info.index >= vdev->num_irqs)
>  			return -EINVAL;
>  
> -		info.flags = vdev->irqs[info.index].flags;
> -		info.count = vdev->irqs[info.index].count;
> +		if (info.argsz >= capsz)
> +			minsz = capsz;
> +
> +		index = info.index;
> +
> +		info.flags = vdev->irqs[index].flags;
> +		info.count = vdev->irqs[index].count;
> +
> +		if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
> +			struct vfio_irq_info_cap_type cap_type = {
> +				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
> +				.header.version = 1 };
> +			struct vfio_platform_irq *irq;
> +			size_t msi_info_size;
> +			int num_msgs;
> +			int ret;
> +			int i;
> +
> +			index = array_index_nospec(index,
> +						   vdev->num_irqs);
> +			irq = &vdev->irqs[index];
> +
> +			cap_type.type = irq->type;
> +			cap_type.subtype = irq->subtype;
> +
> +			ret = vfio_info_add_capability(&caps,
> +						       &cap_type.header,
> +						       sizeof(cap_type));
> +			if (ret)
> +				return ret;
> +
> +			num_msgs = irq->num_ctx;
> +			if (num_msgs) {
> +				msi_info_size = struct_size(msi_info,
> +							    msgs, num_msgs);
> +
> +				msi_info = kzalloc(msi_info_size, GFP_KERNEL);
> +				if (!msi_info) {
> +					kfree(caps.buf);
> +					return -ENOMEM;
> +				}
> +
> +				msi_info->header.id =
> +						VFIO_IRQ_INFO_CAP_MSI_DESCS;
I thought you would remove this cap now the module is back. What is the
motivation to keep it?

Thanks

Eric
> +				msi_info->header.version = 1;
> +				msi_info->nr_msgs = num_msgs;
> +
> +				for (i = 0; i < num_msgs; i++) {
> +					struct vfio_irq_ctx *ctx = &irq->ctx[i];
> +
> +					msi_info->msgs[i].addr_lo =
> +							ctx->msg.address_lo;
> +					msi_info->msgs[i].addr_hi =
> +							ctx->msg.address_hi;
> +					msi_info->msgs[i].data = ctx->msg.data;
> +				}
> +
> +				ret = vfio_info_add_capability(&caps,
> +							&msi_info->header,
> +							msi_info_size);
> +				if (ret) {
> +					kfree(msi_info);
> +					kfree(caps.buf);
> +					return ret;
> +				}
> +			}
> +		}
> +
> +		if (caps.size) {
> +			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
> +			if (info.argsz < sizeof(info) + caps.size) {
> +				info.argsz = sizeof(info) + caps.size;
> +				info.cap_offset = 0;
> +			} else {
> +				vfio_info_cap_shift(&caps, sizeof(info));
> +				if (copy_to_user((void __user *)arg +
> +						  sizeof(info), caps.buf,
> +						  caps.size)) {
> +					kfree(msi_info);
> +					kfree(caps.buf);
> +					return -EFAULT;
> +				}
> +				info.cap_offset = sizeof(info);
> +			}
> +
> +			kfree(msi_info);
> +			kfree(caps.buf);
> +		}
>  
>  		return copy_to_user((void __user *)arg, &info, minsz) ?
>  			-EFAULT : 0;
> @@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
>  		struct vfio_irq_set hdr;
>  		u8 *data = NULL;
>  		int ret = 0;
> +		int max;
>  		size_t data_size = 0;
>  
>  		minsz = offsetofend(struct vfio_irq_set, count);
> @@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
>  		if (copy_from_user(&hdr, (void __user *)arg, minsz))
>  			return -EFAULT;
>  
> -		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
> -						 vdev->num_irqs, &data_size);
> +		if (hdr.index >= vdev->num_irqs)
> +			return -EINVAL;
> +
> +		max = vdev->irqs[hdr.index].count;
> +
> +		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
> +							 vdev->num_irqs,
> +							 &data_size);
>  		if (ret)
>  			return ret;
>  
> @@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>  		return ret;
>  	}
>  
> +	ret = vfio_platform_get_msi(vdev);
> +	if (ret)
> +		dev_info(vdev->device, "msi not supported\n");
I don't think we should output this message. This is printed for
amd-xgbe which does not have msi so this is misleading. I would say
either the vfio_platform_get_msi() can return an actual error or we
return a void?

Thanks

Eric
> +
>  	group = vfio_iommu_group_get(dev);
>  	if (!group) {
>  		dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
> @@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>  put_iommu:
>  	vfio_iommu_group_put(group, dev);
>  put_reset:
> +	vfio_platform_put_msi(vdev);
>  	vfio_platform_put_reset(vdev);
>  	return ret;
>  }
> @@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
>  }
>  EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
>  
> +void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
> +{
> +	mutex_lock(&driver_lock);
> +	list_add(&node->link, &msi_list);
> +	mutex_unlock(&driver_lock);
> +}
> +EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
> +
> +void vfio_platform_unregister_msi(const char *compat)
> +{
> +	struct vfio_platform_msi_node *iter, *temp;
> +
> +	mutex_lock(&driver_lock);
> +	list_for_each_entry_safe(iter, temp, &msi_list, link) {
> +		if (!strcmp(iter->compat, compat)) {
> +			list_del(&iter->link);
> +			break;
> +		}
> +	}
> +
> +	mutex_unlock(&driver_lock);
> +}
> +EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
> +
>  MODULE_VERSION(DRIVER_VERSION);
>  MODULE_LICENSE("GPL v2");
>  MODULE_AUTHOR(DRIVER_AUTHOR);
> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> index c5b09ec0a3c9..e0f4696afedd 100644
> --- a/drivers/vfio/platform/vfio_platform_irq.c
> +++ b/drivers/vfio/platform/vfio_platform_irq.c
> @@ -8,10 +8,12 @@
>  
>  #include <linux/eventfd.h>
>  #include <linux/interrupt.h>
> +#include <linux/eventfd.h>
>  #include <linux/slab.h>
>  #include <linux/types.h>
>  #include <linux/vfio.h>
>  #include <linux/irq.h>
> +#include <linux/msi.h>
>  
>  #include "vfio_platform_private.h"
>  
> @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>  	return 0;
>  }
>  
> +/* MSI/MSIX */
> +static irqreturn_t vfio_msihandler(int irq, void *arg)
> +{
> +	struct eventfd_ctx *trigger = arg;
> +
> +	eventfd_signal(trigger, 1);
> +	return IRQ_HANDLED;
> +}
> +
> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
> +{
> +	int i;
> +	struct vfio_platform_irq *irq;
> +	u16 index = desc->platform.msi_index;
> +	struct device *dev = msi_desc_to_dev(desc);
> +	struct vfio_device *device = dev_get_drvdata(dev);
> +	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
> +						vfio_device_data(device);
> +
> +	for (i = 0; i < vdev->num_irqs; i++)
> +		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> +			irq = &vdev->irqs[i];
> +
> +	irq->ctx[index].msg.address_lo = msg->address_lo;
> +	irq->ctx[index].msg.address_hi = msg->address_hi;
> +	irq->ctx[index].msg.data = msg->data;
> +}
> +
> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
> +			   struct vfio_platform_irq *irq, int nvec)
> +{
> +	int ret;
> +	int msi_idx = 0;
> +	struct msi_desc *desc;
> +	struct device *dev = vdev->device;
> +
> +	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
> +	if (!irq->ctx)
> +		return -ENOMEM;
> +
> +	/* Allocate platform MSIs */
> +	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
> +	if (ret < 0) {
> +		kfree(irq->ctx);
> +		return ret;
> +	}
> +
> +	for_each_msi_entry(desc, dev) {
> +		irq->ctx[msi_idx].hwirq = desc->irq;
> +		msi_idx++;
> +	}
> +
> +	irq->num_ctx = nvec;
> +	irq->config_msi = 1;
> +
> +	return 0;
> +}
> +
> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
> +				      int vector, int fd)
> +{
> +	struct eventfd_ctx *trigger;
> +	int irq_num, ret;
> +
> +	if (vector < 0 || vector >= irq->num_ctx)
> +		return -EINVAL;
> +
> +	irq_num = irq->ctx[vector].hwirq;
> +
> +	if (irq->ctx[vector].trigger) {
> +		free_irq(irq_num, irq->ctx[vector].trigger);
> +		kfree(irq->ctx[vector].name);
> +		eventfd_ctx_put(irq->ctx[vector].trigger);
> +		irq->ctx[vector].trigger = NULL;
> +	}
> +
> +	if (fd < 0)
> +		return 0;
> +
> +	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
> +					  "vfio-msi[%d]", vector);
> +	if (!irq->ctx[vector].name)
> +		return -ENOMEM;
> +
> +	trigger = eventfd_ctx_fdget(fd);
> +	if (IS_ERR(trigger)) {
> +		kfree(irq->ctx[vector].name);
> +		return PTR_ERR(trigger);
> +	}
> +
> +	ret = request_irq(irq_num, vfio_msihandler, 0,
> +			  irq->ctx[vector].name, trigger);
> +	if (ret) {
> +		kfree(irq->ctx[vector].name);
> +		eventfd_ctx_put(trigger);
> +		return ret;
> +	}
> +
> +	irq->ctx[vector].trigger = trigger;
> +
> +	return 0;
> +}
> +
> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
> +			      unsigned int count, int32_t *fds)
> +{
> +	int i, j, ret = 0;
> +
> +	if (start >= irq->num_ctx || start + count > irq->num_ctx)
> +		return -EINVAL;
> +
> +	for (i = 0, j = start; i < count && !ret; i++, j++) {
> +		int fd = fds ? fds[i] : -1;
> +
> +		ret = vfio_msi_set_vector_signal(irq, j, fd);
> +	}
> +
> +	if (ret) {
> +		for (--j; j >= (int)start; j--)
> +			vfio_msi_set_vector_signal(irq, j, -1);
> +	}
> +
> +	return ret;
> +}
> +
> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
> +			     struct vfio_platform_irq *irq)
> +{
> +	struct device *dev = vdev->device;
> +
> +	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
> +
> +	platform_msi_domain_free_irqs(dev);
> +
> +	irq->config_msi = 0;
> +	irq->num_ctx = 0;
> +
> +	kfree(irq->ctx);
> +}
> +
> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
> +				unsigned int index, unsigned int start,
> +				unsigned int count, uint32_t flags, void *data)
> +{
> +	int i;
> +	struct vfio_platform_irq *irq = &vdev->irqs[index];
> +
> +	if (start + count > irq->count)
> +		return -EINVAL;
> +
> +	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
> +		vfio_msi_disable(vdev, irq);
> +		return 0;
> +	}
> +
> +	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
> +		s32 *fds = data;
> +		int ret;
> +
> +		if (irq->config_msi)
> +			return vfio_msi_set_block(irq, start, count,
> +						  fds);
> +		ret = vfio_msi_enable(vdev, irq, start + count);
> +		if (ret)
> +			return ret;
> +
> +		ret = vfio_msi_set_block(irq, start, count, fds);
> +		if (ret)
> +			vfio_msi_disable(vdev, irq);
> +
> +		return ret;
> +	}
> +
> +	for (i = start; i < start + count; i++) {
> +		if (!irq->ctx[i].trigger)
> +			continue;
> +		if (flags & VFIO_IRQ_SET_DATA_NONE) {
> +			eventfd_signal(irq->ctx[i].trigger, 1);
> +		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> +			u8 *bools = data;
> +
> +			if (bools[i - start])
> +				eventfd_signal(irq->ctx[i].trigger, 1);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  				 uint32_t flags, unsigned index, unsigned start,
>  				 unsigned count, void *data)
> @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  		    unsigned start, unsigned count, uint32_t flags,
>  		    void *data) = NULL;
>  
> -	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> -	case VFIO_IRQ_SET_ACTION_MASK:
> -		func = vfio_platform_set_irq_mask;
> -		break;
> -	case VFIO_IRQ_SET_ACTION_UNMASK:
> -		func = vfio_platform_set_irq_unmask;
> -		break;
> -	case VFIO_IRQ_SET_ACTION_TRIGGER:
> -		func = vfio_platform_set_irq_trigger;
> -		break;
> +	struct vfio_platform_irq *irq = &vdev->irqs[index];
> +
> +	if (irq->type == VFIO_IRQ_TYPE_MSI) {
> +		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> +		case VFIO_IRQ_SET_ACTION_MASK:
> +		case VFIO_IRQ_SET_ACTION_UNMASK:
> +			break;
> +		case VFIO_IRQ_SET_ACTION_TRIGGER:
> +			func = vfio_set_msi_trigger;
> +			break;
> +		}
> +	} else {
> +		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> +		case VFIO_IRQ_SET_ACTION_MASK:
> +			func = vfio_platform_set_irq_mask;
> +			break;
> +		case VFIO_IRQ_SET_ACTION_UNMASK:
> +			func = vfio_platform_set_irq_unmask;
> +			break;
> +		case VFIO_IRQ_SET_ACTION_TRIGGER:
> +			func = vfio_platform_set_irq_trigger;
> +			break;
> +		}
>  	}
>  
>  	if (!func)
> @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  
>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>  {
> -	int cnt = 0, i;
> +	int i;
> +	int cnt = 0;
> +	int num_irqs;
> +	int msi_cnt = 0;
>  
>  	while (vdev->get_irq(vdev, cnt) >= 0)
>  		cnt++;
>  
> -	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
> +	if (vdev->of_get_msi) {
> +		msi_cnt = vdev->of_get_msi(vdev);
> +		num_irqs++;
> +	}
> +
> +	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
> +			     GFP_KERNEL);
>  	if (!vdev->irqs)
>  		return -ENOMEM;
>  
> @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>  		vdev->irqs[i].masked = false;
>  	}
>  
> -	vdev->num_irqs = cnt;
> +	/*
> +	 * MSI block is added at last index and it is an ext irq
> +	 */
> +	if (msi_cnt > 0) {
> +		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
> +		vdev->irqs[i].count = msi_cnt;
> +		vdev->irqs[i].hwirq = 0;
> +		vdev->irqs[i].masked = false;
> +		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
> +		vdev->num_ext_irqs = 1;
> +	}
> +
> +	vdev->num_irqs = num_irqs;
>  
>  	return 0;
>  err:
> @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>  {
>  	int i;
>  
> -	for (i = 0; i < vdev->num_irqs; i++)
> -		vfio_set_trigger(vdev, i, -1, NULL);
> +	for (i = 0; i < vdev->num_irqs; i++) {
> +		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> +			vfio_set_msi_trigger(vdev, i, 0, 0,
> +					     VFIO_IRQ_SET_DATA_NONE, NULL);
> +		else
> +			vfio_set_trigger(vdev, i, -1, NULL);
> +	}
>  
>  	vdev->num_irqs = 0;
>  	kfree(vdev->irqs);
> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> index 289089910643..1307feddda21 100644
> --- a/drivers/vfio/platform/vfio_platform_private.h
> +++ b/drivers/vfio/platform/vfio_platform_private.h
> @@ -9,6 +9,7 @@
>  
>  #include <linux/types.h>
>  #include <linux/interrupt.h>
> +#include <linux/msi.h>
>  
>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
> @@ -19,9 +20,21 @@
>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
>  	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>  
> +/* IRQ index for MSI in ext IRQs */
> +#define VFIO_EXT_IRQ_MSI	0
> +
> +struct vfio_irq_ctx {
> +	int			hwirq;
> +	char			*name;
> +	struct msi_msg		msg;
> +	struct eventfd_ctx	*trigger;
> +};
> +
>  struct vfio_platform_irq {
>  	u32			flags;
>  	u32			count;
> +	int			num_ctx;
> +	struct vfio_irq_ctx	*ctx;
>  	int			hwirq;
>  	char			*name;
>  	struct eventfd_ctx	*trigger;
> @@ -29,6 +42,11 @@ struct vfio_platform_irq {
>  	spinlock_t		lock;
>  	struct virqfd		*unmask;
>  	struct virqfd		*mask;
> +
> +	/* for extended irqs */
> +	u32			type;
> +	u32			subtype;
> +	int			config_msi;
>  };
>  
>  struct vfio_platform_region {
> @@ -46,12 +64,14 @@ struct vfio_platform_device {
>  	u32				num_regions;
>  	struct vfio_platform_irq	*irqs;
>  	u32				num_irqs;
> +	int				num_ext_irqs;
>  	int				refcnt;
>  	struct mutex			igate;
>  	struct module			*parent_module;
>  	const char			*compat;
>  	const char			*acpihid;
>  	struct module			*reset_module;
> +	struct module			*msi_module;
>  	struct device			*device;
>  
>  	/*
> @@ -65,11 +85,13 @@ struct vfio_platform_device {
>  		(*get_resource)(struct vfio_platform_device *vdev, int i);
>  	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
>  	int	(*of_reset)(struct vfio_platform_device *vdev);
> +	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
>  
>  	bool				reset_required;
>  };
>  
>  typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
> +typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
>  
>  struct vfio_platform_reset_node {
>  	struct list_head link;
> @@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
>  	vfio_platform_reset_fn_t of_reset;
>  };
>  
> +struct vfio_platform_msi_node {
> +	struct list_head link;
> +	char *compat;
> +	struct module *owner;
> +	vfio_platform_get_msi_fn_t of_get_msi;
> +};
> +
>  extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>  				      struct device *dev);
>  extern struct vfio_platform_device *vfio_platform_remove_common
> @@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
>  extern void vfio_platform_unregister_reset(const char *compat,
>  					   vfio_platform_reset_fn_t fn);
> +void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
> +void vfio_platform_unregister_msi(const char *compat);
> +
>  #define vfio_platform_register_reset(__compat, __reset)		\
>  static struct vfio_platform_reset_node __reset ## _node = {	\
>  	.owner = THIS_MODULE,					\
> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index d1812777139f..53a7ff2b524e 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -697,11 +697,55 @@ struct vfio_irq_info {
>  #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
>  #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
>  #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
> +#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
>  	__u32	index;		/* IRQ index */
>  	__u32	count;		/* Number of IRQs within this index */
> +	__u32	cap_offset;	/* Offset within info struct of first cap */
>  };
>  #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
>  
> +/*
> + * The irq type capability allows IRQs unique to a specific device or
> + * class of devices to be exposed.
> + *
> + * The structures below define version 1 of this capability.
> + */
> +#define VFIO_IRQ_INFO_CAP_TYPE		3
> +
> +struct vfio_irq_info_cap_type {
> +	struct vfio_info_cap_header header;
> +	__u32 type;     /* global per bus driver */
> +	__u32 subtype;  /* type specific */
> +};
> +
> +/*
> + * List of IRQ types, global per bus driver.
> + * If you introduce a new type, please add it here.
> + */
> +
> +/* Non PCI devices having MSI(s) support */
> +#define VFIO_IRQ_TYPE_MSI		(1)
> +
> +/*
> + * The msi capability allows the user to use the msi msg to
> + * configure the iova for the msi configuration.
> + * The structures below define version 1 of this capability.
> + */
> +#define VFIO_IRQ_INFO_CAP_MSI_DESCS	4
> +
> +struct vfio_irq_msi_msg {
> +	__u32	addr_lo;
> +	__u32	addr_hi;
> +	__u32	data;
> +};
> +
> +struct vfio_irq_info_cap_msi {
> +	struct vfio_info_cap_header header;
> +	__u32	nr_msgs;
> +	__u32	reserved;
> +	struct vfio_irq_msi_msg msgs[];
> +};
> +
>  /**
>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
>   *
> 

Re: [RFC v3 1/2] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 32096 bytes --]

On Tue, Dec 22, 2020 at 10:57 PM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 12/14/20 6:45 PM, Vikas Gupta wrote:
> > MSI support for platform devices.The MSI block
> > is added as an extended IRQ which exports caps
> > VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.
> >
> > Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> > ---
> >  drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
> >  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
> >  drivers/vfio/platform/vfio_platform_private.h |  32 +++
> >  include/uapi/linux/vfio.h                     |  44 +++
> >  4 files changed, 496 insertions(+), 19 deletions(-)
> >
> > diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> > index fb4b385191f2..c936852f35d7 100644
> > --- a/drivers/vfio/platform/vfio_platform_common.c
> > +++ b/drivers/vfio/platform/vfio_platform_common.c
> > @@ -16,6 +16,7 @@
> >  #include <linux/types.h>
> >  #include <linux/uaccess.h>
> >  #include <linux/vfio.h>
> > +#include <linux/nospec.h>
> >
> >  #include "vfio_platform_private.h"
> >
> > @@ -26,6 +27,8 @@
> >  #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
> >
> >  static LIST_HEAD(reset_list);
> > +/* devices having MSI support */
> nit: for devices using MSIs?
> > +static LIST_HEAD(msi_list);
> >  static DEFINE_MUTEX(driver_lock);
> >
> >  static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
> > @@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
> >       return reset_fn;
> >  }
> >
> > +static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
> > +{
> > +     bool has_msi = false;
> > +     struct vfio_platform_msi_node *iter;
> > +
> > +     mutex_lock(&driver_lock);
> > +     list_for_each_entry(iter, &msi_list, link) {
> > +             if (!strcmp(iter->compat, vdev->compat) &&
> > +                 try_module_get(iter->owner)) {
> > +                     vdev->msi_module = iter->owner;
> > +                     vdev->of_get_msi = iter->of_get_msi;
> > +                     has_msi = true;
> > +                     break;
> > +             }
> > +     }
> > +     mutex_unlock(&driver_lock);
> > +     return has_msi;
> > +}
> > +
> >  static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
> >                                   struct device *dev)
> >  {
> > @@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
> >       return vdev->of_reset ? 0 : -ENOENT;
> >  }
> >
> > +static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
> > +{
> > +     bool has_msi;
> > +
> > +     has_msi = vfio_platform_lookup_msi(vdev);
> > +     if (!has_msi) {
> > +             request_module("vfio-msi:%s", vdev->compat);
> > +             has_msi = vfio_platform_lookup_msi(vdev);
> > +     }
> > +
> > +     return has_msi ? 0 : -ENOENT;
> > +}
> > +
> >  static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
> >  {
> >       if (VFIO_PLATFORM_IS_ACPI(vdev))
> > @@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
> >               module_put(vdev->reset_module);
> >  }
> >
> > +static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
> > +{
> > +     if (vdev->of_get_msi)
> > +             module_put(vdev->msi_module);
> > +}
> > +
> >  static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
> >  {
> >       int cnt = 0, i;
> > @@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
> >
> >       } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
> >               struct vfio_irq_info info;
> > +             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
> > +             struct vfio_irq_info_cap_msi *msi_info = NULL;
> > +             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
> > +             unsigned long capsz;
> > +             u32 index;
> >
> >               minsz = offsetofend(struct vfio_irq_info, count);
> >
> > +             /* For backward compatibility, cannot require this */
> > +             capsz = offsetofend(struct vfio_irq_info, cap_offset);
> > +
> >               if (copy_from_user(&info, (void __user *)arg, minsz))
> >                       return -EFAULT;
> >
> > @@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
> >               if (info.index >= vdev->num_irqs)
> >                       return -EINVAL;
> >
> > -             info.flags = vdev->irqs[info.index].flags;
> > -             info.count = vdev->irqs[info.index].count;
> > +             if (info.argsz >= capsz)
> > +                     minsz = capsz;
> > +
> > +             index = info.index;
> > +
> > +             info.flags = vdev->irqs[index].flags;
> > +             info.count = vdev->irqs[index].count;
> > +
> > +             if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
> > +                     struct vfio_irq_info_cap_type cap_type = {
> > +                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
> > +                             .header.version = 1 };
> > +                     struct vfio_platform_irq *irq;
> > +                     size_t msi_info_size;
> > +                     int num_msgs;
> > +                     int ret;
> > +                     int i;
> > +
> > +                     index = array_index_nospec(index,
> > +                                                vdev->num_irqs);
> > +                     irq = &vdev->irqs[index];
> > +
> > +                     cap_type.type = irq->type;
> > +                     cap_type.subtype = irq->subtype;
> > +
> > +                     ret = vfio_info_add_capability(&caps,
> > +                                                    &cap_type.header,
> > +                                                    sizeof(cap_type));
> > +                     if (ret)
> > +                             return ret;
> > +
> > +                     num_msgs = irq->num_ctx;
> > +                     if (num_msgs) {
> > +                             msi_info_size = struct_size(msi_info,
> > +                                                         msgs, num_msgs);
> > +
> > +                             msi_info = kzalloc(msi_info_size, GFP_KERNEL);
> > +                             if (!msi_info) {
> > +                                     kfree(caps.buf);
> > +                                     return -ENOMEM;
> > +                             }
> > +
> > +                             msi_info->header.id =
> > +                                             VFIO_IRQ_INFO_CAP_MSI_DESCS;
> I thought you would remove this cap now the module is back. What is the
> motivation to keep it?

Hi Eric,
The earlier module was serving two purposes
a)   Max number of MSI(s) supported by the device. Since the module is
back this information can be filled in .count.
b)   Writing msi_msg(s) to devices for MSI sources.
 We want b) to be handled in the user space so that they need not be
dependent on the kernel. VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user
space to get MSI configuration data and thus they can independently
configure MSI sources. For example, user space can just ask for ‘N’
MSI vectors and related msi_msg for ‘N’ vectors is exported to user
space using this CAP. User space can now use ‘N’ vectors to configure
any ‘N’ MSI sources and these sources might be in any order.
However, this design is different from vfio-pci as MSI configuration
is being handled in kernel only.
Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit
with the CAP framework.

Thanks,
Vikas

 >
> Thanks
>
> Eric
> > +                             msi_info->header.version = 1;
> > +                             msi_info->nr_msgs = num_msgs;
> > +
> > +                             for (i = 0; i < num_msgs; i++) {
> > +                                     struct vfio_irq_ctx *ctx = &irq->ctx[i];
> > +
> > +                                     msi_info->msgs[i].addr_lo =
> > +                                                     ctx->msg.address_lo;
> > +                                     msi_info->msgs[i].addr_hi =
> > +                                                     ctx->msg.address_hi;
> > +                                     msi_info->msgs[i].data = ctx->msg.data;
> > +                             }
> > +
> > +                             ret = vfio_info_add_capability(&caps,
> > +                                                     &msi_info->header,
> > +                                                     msi_info_size);
> > +                             if (ret) {
> > +                                     kfree(msi_info);
> > +                                     kfree(caps.buf);
> > +                                     return ret;
> > +                             }
> > +                     }
> > +             }
> > +
> > +             if (caps.size) {
> > +                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
> > +                     if (info.argsz < sizeof(info) + caps.size) {
> > +                             info.argsz = sizeof(info) + caps.size;
> > +                             info.cap_offset = 0;
> > +                     } else {
> > +                             vfio_info_cap_shift(&caps, sizeof(info));
> > +                             if (copy_to_user((void __user *)arg +
> > +                                               sizeof(info), caps.buf,
> > +                                               caps.size)) {
> > +                                     kfree(msi_info);
> > +                                     kfree(caps.buf);
> > +                                     return -EFAULT;
> > +                             }
> > +                             info.cap_offset = sizeof(info);
> > +                     }
> > +
> > +                     kfree(msi_info);
> > +                     kfree(caps.buf);
> > +             }
> >
> >               return copy_to_user((void __user *)arg, &info, minsz) ?
> >                       -EFAULT : 0;
> > @@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
> >               struct vfio_irq_set hdr;
> >               u8 *data = NULL;
> >               int ret = 0;
> > +             int max;
> >               size_t data_size = 0;
> >
> >               minsz = offsetofend(struct vfio_irq_set, count);
> > @@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
> >               if (copy_from_user(&hdr, (void __user *)arg, minsz))
> >                       return -EFAULT;
> >
> > -             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
> > -                                              vdev->num_irqs, &data_size);
> > +             if (hdr.index >= vdev->num_irqs)
> > +                     return -EINVAL;
> > +
> > +             max = vdev->irqs[hdr.index].count;
> > +
> > +             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
> > +                                                      vdev->num_irqs,
> > +                                                      &data_size);
> >               if (ret)
> >                       return ret;
> >
> > @@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> >               return ret;
> >       }
> >
> > +     ret = vfio_platform_get_msi(vdev);
> > +     if (ret)
> > +             dev_info(vdev->device, "msi not supported\n");
> I don't think we should output this message. This is printed for
> amd-xgbe which does not have msi so this is misleading. I would say
> either the vfio_platform_get_msi() can return an actual error or we
> return a void?
Will check on this what
>
> Thanks
>
> Eric
> > +
> >       group = vfio_iommu_group_get(dev);
> >       if (!group) {
> >               dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
> > @@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> >  put_iommu:
> >       vfio_iommu_group_put(group, dev);
> >  put_reset:
> > +     vfio_platform_put_msi(vdev);
> >       vfio_platform_put_reset(vdev);
> >       return ret;
> >  }
> > @@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
> >  }
> >  EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
> >
> > +void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
> > +{
> > +     mutex_lock(&driver_lock);
> > +     list_add(&node->link, &msi_list);
> > +     mutex_unlock(&driver_lock);
> > +}
> > +EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
> > +
> > +void vfio_platform_unregister_msi(const char *compat)
> > +{
> > +     struct vfio_platform_msi_node *iter, *temp;
> > +
> > +     mutex_lock(&driver_lock);
> > +     list_for_each_entry_safe(iter, temp, &msi_list, link) {
> > +             if (!strcmp(iter->compat, compat)) {
> > +                     list_del(&iter->link);
> > +                     break;
> > +             }
> > +     }
> > +
> > +     mutex_unlock(&driver_lock);
> > +}
> > +EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
> > +
> >  MODULE_VERSION(DRIVER_VERSION);
> >  MODULE_LICENSE("GPL v2");
> >  MODULE_AUTHOR(DRIVER_AUTHOR);
> > diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> > index c5b09ec0a3c9..e0f4696afedd 100644
> > --- a/drivers/vfio/platform/vfio_platform_irq.c
> > +++ b/drivers/vfio/platform/vfio_platform_irq.c
> > @@ -8,10 +8,12 @@
> >
> >  #include <linux/eventfd.h>
> >  #include <linux/interrupt.h>
> > +#include <linux/eventfd.h>
> >  #include <linux/slab.h>
> >  #include <linux/types.h>
> >  #include <linux/vfio.h>
> >  #include <linux/irq.h>
> > +#include <linux/msi.h>
> >
> >  #include "vfio_platform_private.h"
> >
> > @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
> >       return 0;
> >  }
> >
> > +/* MSI/MSIX */
> > +static irqreturn_t vfio_msihandler(int irq, void *arg)
> > +{
> > +     struct eventfd_ctx *trigger = arg;
> > +
> > +     eventfd_signal(trigger, 1);
> > +     return IRQ_HANDLED;
> > +}
> > +
> > +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
> > +{
> > +     int i;
> > +     struct vfio_platform_irq *irq;
> > +     u16 index = desc->platform.msi_index;
> > +     struct device *dev = msi_desc_to_dev(desc);
> > +     struct vfio_device *device = dev_get_drvdata(dev);
> > +     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
> > +                                             vfio_device_data(device);
> > +
> > +     for (i = 0; i < vdev->num_irqs; i++)
> > +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> > +                     irq = &vdev->irqs[i];
> > +
> > +     irq->ctx[index].msg.address_lo = msg->address_lo;
> > +     irq->ctx[index].msg.address_hi = msg->address_hi;
> > +     irq->ctx[index].msg.data = msg->data;
> > +}
> > +
> > +static int vfio_msi_enable(struct vfio_platform_device *vdev,
> > +                        struct vfio_platform_irq *irq, int nvec)
> > +{
> > +     int ret;
> > +     int msi_idx = 0;
> > +     struct msi_desc *desc;
> > +     struct device *dev = vdev->device;
> > +
> > +     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
> > +     if (!irq->ctx)
> > +             return -ENOMEM;
> > +
> > +     /* Allocate platform MSIs */
> > +     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
> > +     if (ret < 0) {
> > +             kfree(irq->ctx);
> > +             return ret;
> > +     }
> > +
> > +     for_each_msi_entry(desc, dev) {
> > +             irq->ctx[msi_idx].hwirq = desc->irq;
> > +             msi_idx++;
> > +     }
> > +
> > +     irq->num_ctx = nvec;
> > +     irq->config_msi = 1;
> > +
> > +     return 0;
> > +}
> > +
> > +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
> > +                                   int vector, int fd)
> > +{
> > +     struct eventfd_ctx *trigger;
> > +     int irq_num, ret;
> > +
> > +     if (vector < 0 || vector >= irq->num_ctx)
> > +             return -EINVAL;
> > +
> > +     irq_num = irq->ctx[vector].hwirq;
> > +
> > +     if (irq->ctx[vector].trigger) {
> > +             free_irq(irq_num, irq->ctx[vector].trigger);
> > +             kfree(irq->ctx[vector].name);
> > +             eventfd_ctx_put(irq->ctx[vector].trigger);
> > +             irq->ctx[vector].trigger = NULL;
> > +     }
> > +
> > +     if (fd < 0)
> > +             return 0;
> > +
> > +     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
> > +                                       "vfio-msi[%d]", vector);
> > +     if (!irq->ctx[vector].name)
> > +             return -ENOMEM;
> > +
> > +     trigger = eventfd_ctx_fdget(fd);
> > +     if (IS_ERR(trigger)) {
> > +             kfree(irq->ctx[vector].name);
> > +             return PTR_ERR(trigger);
> > +     }
> > +
> > +     ret = request_irq(irq_num, vfio_msihandler, 0,
> > +                       irq->ctx[vector].name, trigger);
> > +     if (ret) {
> > +             kfree(irq->ctx[vector].name);
> > +             eventfd_ctx_put(trigger);
> > +             return ret;
> > +     }
> > +
> > +     irq->ctx[vector].trigger = trigger;
> > +
> > +     return 0;
> > +}
> > +
> > +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
> > +                           unsigned int count, int32_t *fds)
> > +{
> > +     int i, j, ret = 0;
> > +
> > +     if (start >= irq->num_ctx || start + count > irq->num_ctx)
> > +             return -EINVAL;
> > +
> > +     for (i = 0, j = start; i < count && !ret; i++, j++) {
> > +             int fd = fds ? fds[i] : -1;
> > +
> > +             ret = vfio_msi_set_vector_signal(irq, j, fd);
> > +     }
> > +
> > +     if (ret) {
> > +             for (--j; j >= (int)start; j--)
> > +                     vfio_msi_set_vector_signal(irq, j, -1);
> > +     }
> > +
> > +     return ret;
> > +}
> > +
> > +static void vfio_msi_disable(struct vfio_platform_device *vdev,
> > +                          struct vfio_platform_irq *irq)
> > +{
> > +     struct device *dev = vdev->device;
> > +
> > +     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
> > +
> > +     platform_msi_domain_free_irqs(dev);
> > +
> > +     irq->config_msi = 0;
> > +     irq->num_ctx = 0;
> > +
> > +     kfree(irq->ctx);
> > +}
> > +
> > +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
> > +                             unsigned int index, unsigned int start,
> > +                             unsigned int count, uint32_t flags, void *data)
> > +{
> > +     int i;
> > +     struct vfio_platform_irq *irq = &vdev->irqs[index];
> > +
> > +     if (start + count > irq->count)
> > +             return -EINVAL;
> > +
> > +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
> > +             vfio_msi_disable(vdev, irq);
> > +             return 0;
> > +     }
> > +
> > +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
> > +             s32 *fds = data;
> > +             int ret;
> > +
> > +             if (irq->config_msi)
> > +                     return vfio_msi_set_block(irq, start, count,
> > +                                               fds);
> > +             ret = vfio_msi_enable(vdev, irq, start + count);
> > +             if (ret)
> > +                     return ret;
> > +
> > +             ret = vfio_msi_set_block(irq, start, count, fds);
> > +             if (ret)
> > +                     vfio_msi_disable(vdev, irq);
> > +
> > +             return ret;
> > +     }
> > +
> > +     for (i = start; i < start + count; i++) {
> > +             if (!irq->ctx[i].trigger)
> > +                     continue;
> > +             if (flags & VFIO_IRQ_SET_DATA_NONE) {
> > +                     eventfd_signal(irq->ctx[i].trigger, 1);
> > +             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> > +                     u8 *bools = data;
> > +
> > +                     if (bools[i - start])
> > +                             eventfd_signal(irq->ctx[i].trigger, 1);
> > +             }
> > +     }
> > +
> > +     return 0;
> > +}
> > +
> >  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >                                uint32_t flags, unsigned index, unsigned start,
> >                                unsigned count, void *data)
> > @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >                   unsigned start, unsigned count, uint32_t flags,
> >                   void *data) = NULL;
> >
> > -     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> > -     case VFIO_IRQ_SET_ACTION_MASK:
> > -             func = vfio_platform_set_irq_mask;
> > -             break;
> > -     case VFIO_IRQ_SET_ACTION_UNMASK:
> > -             func = vfio_platform_set_irq_unmask;
> > -             break;
> > -     case VFIO_IRQ_SET_ACTION_TRIGGER:
> > -             func = vfio_platform_set_irq_trigger;
> > -             break;
> > +     struct vfio_platform_irq *irq = &vdev->irqs[index];
> > +
> > +     if (irq->type == VFIO_IRQ_TYPE_MSI) {
> > +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> > +             case VFIO_IRQ_SET_ACTION_MASK:
> > +             case VFIO_IRQ_SET_ACTION_UNMASK:
> > +                     break;
> > +             case VFIO_IRQ_SET_ACTION_TRIGGER:
> > +                     func = vfio_set_msi_trigger;
> > +                     break;
> > +             }
> > +     } else {
> > +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> > +             case VFIO_IRQ_SET_ACTION_MASK:
> > +                     func = vfio_platform_set_irq_mask;
> > +                     break;
> > +             case VFIO_IRQ_SET_ACTION_UNMASK:
> > +                     func = vfio_platform_set_irq_unmask;
> > +                     break;
> > +             case VFIO_IRQ_SET_ACTION_TRIGGER:
> > +                     func = vfio_platform_set_irq_trigger;
> > +                     break;
> > +             }
> >       }
> >
> >       if (!func)
> > @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >
> >  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> >  {
> > -     int cnt = 0, i;
> > +     int i;
> > +     int cnt = 0;
> > +     int num_irqs;
> > +     int msi_cnt = 0;
> >
> >       while (vdev->get_irq(vdev, cnt) >= 0)
> >               cnt++;
> >
> > -     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
> > +     if (vdev->of_get_msi) {
> > +             msi_cnt = vdev->of_get_msi(vdev);
> > +             num_irqs++;
> > +     }
> > +
> > +     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
> > +                          GFP_KERNEL);
> >       if (!vdev->irqs)
> >               return -ENOMEM;
> >
> > @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> >               vdev->irqs[i].masked = false;
> >       }
> >
> > -     vdev->num_irqs = cnt;
> > +     /*
> > +      * MSI block is added at last index and it is an ext irq
> > +      */
> > +     if (msi_cnt > 0) {
> > +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
> > +             vdev->irqs[i].count = msi_cnt;
> > +             vdev->irqs[i].hwirq = 0;
> > +             vdev->irqs[i].masked = false;
> > +             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
> > +             vdev->num_ext_irqs = 1;
> > +     }
> > +
> > +     vdev->num_irqs = num_irqs;
> >
> >       return 0;
> >  err:
> > @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
> >  {
> >       int i;
> >
> > -     for (i = 0; i < vdev->num_irqs; i++)
> > -             vfio_set_trigger(vdev, i, -1, NULL);
> > +     for (i = 0; i < vdev->num_irqs; i++) {
> > +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> > +                     vfio_set_msi_trigger(vdev, i, 0, 0,
> > +                                          VFIO_IRQ_SET_DATA_NONE, NULL);
> > +             else
> > +                     vfio_set_trigger(vdev, i, -1, NULL);
> > +     }
> >
> >       vdev->num_irqs = 0;
> >       kfree(vdev->irqs);
> > diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> > index 289089910643..1307feddda21 100644
> > --- a/drivers/vfio/platform/vfio_platform_private.h
> > +++ b/drivers/vfio/platform/vfio_platform_private.h
> > @@ -9,6 +9,7 @@
> >
> >  #include <linux/types.h>
> >  #include <linux/interrupt.h>
> > +#include <linux/msi.h>
> >
> >  #define VFIO_PLATFORM_OFFSET_SHIFT   40
> >  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
> > @@ -19,9 +20,21 @@
> >  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
> >       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
> >
> > +/* IRQ index for MSI in ext IRQs */
> > +#define VFIO_EXT_IRQ_MSI     0
> > +
> > +struct vfio_irq_ctx {
> > +     int                     hwirq;
> > +     char                    *name;
> > +     struct msi_msg          msg;
> > +     struct eventfd_ctx      *trigger;
> > +};
> > +
> >  struct vfio_platform_irq {
> >       u32                     flags;
> >       u32                     count;
> > +     int                     num_ctx;
> > +     struct vfio_irq_ctx     *ctx;
> >       int                     hwirq;
> >       char                    *name;
> >       struct eventfd_ctx      *trigger;
> > @@ -29,6 +42,11 @@ struct vfio_platform_irq {
> >       spinlock_t              lock;
> >       struct virqfd           *unmask;
> >       struct virqfd           *mask;
> > +
> > +     /* for extended irqs */
> > +     u32                     type;
> > +     u32                     subtype;
> > +     int                     config_msi;
> >  };
> >
> >  struct vfio_platform_region {
> > @@ -46,12 +64,14 @@ struct vfio_platform_device {
> >       u32                             num_regions;
> >       struct vfio_platform_irq        *irqs;
> >       u32                             num_irqs;
> > +     int                             num_ext_irqs;
> >       int                             refcnt;
> >       struct mutex                    igate;
> >       struct module                   *parent_module;
> >       const char                      *compat;
> >       const char                      *acpihid;
> >       struct module                   *reset_module;
> > +     struct module                   *msi_module;
> >       struct device                   *device;
> >
> >       /*
> > @@ -65,11 +85,13 @@ struct vfio_platform_device {
> >               (*get_resource)(struct vfio_platform_device *vdev, int i);
> >       int     (*get_irq)(struct vfio_platform_device *vdev, int i);
> >       int     (*of_reset)(struct vfio_platform_device *vdev);
> > +     u32     (*of_get_msi)(struct vfio_platform_device *vdev);
> >
> >       bool                            reset_required;
> >  };
> >
> >  typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
> > +typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
> >
> >  struct vfio_platform_reset_node {
> >       struct list_head link;
> > @@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
> >       vfio_platform_reset_fn_t of_reset;
> >  };
> >
> > +struct vfio_platform_msi_node {
> > +     struct list_head link;
> > +     char *compat;
> > +     struct module *owner;
> > +     vfio_platform_get_msi_fn_t of_get_msi;
> > +};
> > +
> >  extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> >                                     struct device *dev);
> >  extern struct vfio_platform_device *vfio_platform_remove_common
> > @@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >  extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
> >  extern void vfio_platform_unregister_reset(const char *compat,
> >                                          vfio_platform_reset_fn_t fn);
> > +void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
> > +void vfio_platform_unregister_msi(const char *compat);
> > +
> >  #define vfio_platform_register_reset(__compat, __reset)              \
> >  static struct vfio_platform_reset_node __reset ## _node = {  \
> >       .owner = THIS_MODULE,                                   \
> > diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> > index d1812777139f..53a7ff2b524e 100644
> > --- a/include/uapi/linux/vfio.h
> > +++ b/include/uapi/linux/vfio.h
> > @@ -697,11 +697,55 @@ struct vfio_irq_info {
> >  #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
> >  #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
> >  #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
> > +#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
> >       __u32   index;          /* IRQ index */
> >       __u32   count;          /* Number of IRQs within this index */
> > +     __u32   cap_offset;     /* Offset within info struct of first cap */
> >  };
> >  #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
> >
> > +/*
> > + * The irq type capability allows IRQs unique to a specific device or
> > + * class of devices to be exposed.
> > + *
> > + * The structures below define version 1 of this capability.
> > + */
> > +#define VFIO_IRQ_INFO_CAP_TYPE               3
> > +
> > +struct vfio_irq_info_cap_type {
> > +     struct vfio_info_cap_header header;
> > +     __u32 type;     /* global per bus driver */
> > +     __u32 subtype;  /* type specific */
> > +};
> > +
> > +/*
> > + * List of IRQ types, global per bus driver.
> > + * If you introduce a new type, please add it here.
> > + */
> > +
> > +/* Non PCI devices having MSI(s) support */
> > +#define VFIO_IRQ_TYPE_MSI            (1)
> > +
> > +/*
> > + * The msi capability allows the user to use the msi msg to
> > + * configure the iova for the msi configuration.
> > + * The structures below define version 1 of this capability.
> > + */
> > +#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
> > +
> > +struct vfio_irq_msi_msg {
> > +     __u32   addr_lo;
> > +     __u32   addr_hi;
> > +     __u32   data;
> > +};
> > +
> > +struct vfio_irq_info_cap_msi {
> > +     struct vfio_info_cap_header header;
> > +     __u32   nr_msgs;
> > +     __u32   reserved;
> > +     struct vfio_irq_msi_msg msgs[];
> > +};
> > +
> >  /**
> >   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
> >   *
> >
>

-- 
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v3 1/2] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 1/5/21 6:53 AM, Vikas Gupta wrote:
> On Tue, Dec 22, 2020 at 10:57 PM Auger Eric <eric.auger@redhat.com> wrote:
>>
>> Hi Vikas,
>>
>> On 12/14/20 6:45 PM, Vikas Gupta wrote:
>>> MSI support for platform devices.The MSI block
>>> is added as an extended IRQ which exports caps
>>> VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.
>>>
>>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
>>> ---
>>>  drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
>>>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
>>>  drivers/vfio/platform/vfio_platform_private.h |  32 +++
>>>  include/uapi/linux/vfio.h                     |  44 +++
>>>  4 files changed, 496 insertions(+), 19 deletions(-)
>>>
>>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>>> index fb4b385191f2..c936852f35d7 100644
>>> --- a/drivers/vfio/platform/vfio_platform_common.c
>>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>>> @@ -16,6 +16,7 @@
>>>  #include <linux/types.h>
>>>  #include <linux/uaccess.h>
>>>  #include <linux/vfio.h>
>>> +#include <linux/nospec.h>
>>>
>>>  #include "vfio_platform_private.h"
>>>
>>> @@ -26,6 +27,8 @@
>>>  #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
>>>
>>>  static LIST_HEAD(reset_list);
>>> +/* devices having MSI support */
>> nit: for devices using MSIs?
>>> +static LIST_HEAD(msi_list);
>>>  static DEFINE_MUTEX(driver_lock);
>>>
>>>  static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
>>> @@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
>>>       return reset_fn;
>>>  }
>>>
>>> +static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
>>> +{
>>> +     bool has_msi = false;
>>> +     struct vfio_platform_msi_node *iter;
>>> +
>>> +     mutex_lock(&driver_lock);
>>> +     list_for_each_entry(iter, &msi_list, link) {
>>> +             if (!strcmp(iter->compat, vdev->compat) &&
>>> +                 try_module_get(iter->owner)) {
>>> +                     vdev->msi_module = iter->owner;
>>> +                     vdev->of_get_msi = iter->of_get_msi;
>>> +                     has_msi = true;
>>> +                     break;
>>> +             }
>>> +     }
>>> +     mutex_unlock(&driver_lock);
>>> +     return has_msi;
>>> +}
>>> +
>>>  static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
>>>                                   struct device *dev)
>>>  {
>>> @@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
>>>       return vdev->of_reset ? 0 : -ENOENT;
>>>  }
>>>
>>> +static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
>>> +{
>>> +     bool has_msi;
>>> +
>>> +     has_msi = vfio_platform_lookup_msi(vdev);
>>> +     if (!has_msi) {
>>> +             request_module("vfio-msi:%s", vdev->compat);
>>> +             has_msi = vfio_platform_lookup_msi(vdev);
>>> +     }
>>> +
>>> +     return has_msi ? 0 : -ENOENT;
>>> +}
>>> +
>>>  static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
>>>  {
>>>       if (VFIO_PLATFORM_IS_ACPI(vdev))
>>> @@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
>>>               module_put(vdev->reset_module);
>>>  }
>>>
>>> +static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
>>> +{
>>> +     if (vdev->of_get_msi)
>>> +             module_put(vdev->msi_module);
>>> +}
>>> +
>>>  static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
>>>  {
>>>       int cnt = 0, i;
>>> @@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
>>>
>>>       } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
>>>               struct vfio_irq_info info;
>>> +             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
>>> +             struct vfio_irq_info_cap_msi *msi_info = NULL;
>>> +             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
>>> +             unsigned long capsz;
>>> +             u32 index;
>>>
>>>               minsz = offsetofend(struct vfio_irq_info, count);
>>>
>>> +             /* For backward compatibility, cannot require this */
>>> +             capsz = offsetofend(struct vfio_irq_info, cap_offset);
>>> +
>>>               if (copy_from_user(&info, (void __user *)arg, minsz))
>>>                       return -EFAULT;
>>>
>>> @@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
>>>               if (info.index >= vdev->num_irqs)
>>>                       return -EINVAL;
>>>
>>> -             info.flags = vdev->irqs[info.index].flags;
>>> -             info.count = vdev->irqs[info.index].count;
>>> +             if (info.argsz >= capsz)
>>> +                     minsz = capsz;
>>> +
>>> +             index = info.index;
>>> +
>>> +             info.flags = vdev->irqs[index].flags;
>>> +             info.count = vdev->irqs[index].count;
>>> +
>>> +             if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
>>> +                     struct vfio_irq_info_cap_type cap_type = {
>>> +                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
>>> +                             .header.version = 1 };
>>> +                     struct vfio_platform_irq *irq;
>>> +                     size_t msi_info_size;
>>> +                     int num_msgs;
>>> +                     int ret;
>>> +                     int i;
>>> +
>>> +                     index = array_index_nospec(index,
>>> +                                                vdev->num_irqs);
>>> +                     irq = &vdev->irqs[index];
>>> +
>>> +                     cap_type.type = irq->type;
>>> +                     cap_type.subtype = irq->subtype;
>>> +
>>> +                     ret = vfio_info_add_capability(&caps,
>>> +                                                    &cap_type.header,
>>> +                                                    sizeof(cap_type));
>>> +                     if (ret)
>>> +                             return ret;
>>> +
>>> +                     num_msgs = irq->num_ctx;
>>> +                     if (num_msgs) {
>>> +                             msi_info_size = struct_size(msi_info,
>>> +                                                         msgs, num_msgs);
>>> +
>>> +                             msi_info = kzalloc(msi_info_size, GFP_KERNEL);
>>> +                             if (!msi_info) {
>>> +                                     kfree(caps.buf);
>>> +                                     return -ENOMEM;
>>> +                             }
>>> +
>>> +                             msi_info->header.id =
>>> +                                             VFIO_IRQ_INFO_CAP_MSI_DESCS;
>> I thought you would remove this cap now the module is back. What is the
>> motivation to keep it?
> 
> Hi Eric,
> The earlier module was serving two purposes
> a)   Max number of MSI(s) supported by the device. Since the module is
> back this information can be filled in .count.
understood
> b)   Writing msi_msg(s) to devices for MSI sources.
>  We want b) to be handled in the user space so that they need not be
> dependent on the kernel.
I fail to understand the problem with the "dependency on the kernel".
What is the functional difference when exposing this to the userspace,
please could you highlight the benefits?

 VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user
> space to get MSI configuration data and thus they can independently
> configure MSI sources. For example, user space can just ask for ‘N’
> MSI vectors and related msi_msg for ‘N’ vectors
this is induced by vfio_msi_enable(), right?


 is exported to user
> space using this CAP. User space can now use ‘N’ vectors to configure
> any ‘N’ MSI sources and these sources might be in any order.
> However, this design is different from vfio-pci as MSI configuration
> is being handled in kernel only.
> Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit
> with the CAP framework.
I think it is OK reporting them through the cap but I fail to understand
the benefits.

Thanks

Eric
> 
> Thanks,
> Vikas
> 
>  >
>> Thanks
>>
>> Eric
>>> +                             msi_info->header.version = 1;
>>> +                             msi_info->nr_msgs = num_msgs;
>>> +
>>> +                             for (i = 0; i < num_msgs; i++) {
>>> +                                     struct vfio_irq_ctx *ctx = &irq->ctx[i];
>>> +
>>> +                                     msi_info->msgs[i].addr_lo =
>>> +                                                     ctx->msg.address_lo;
>>> +                                     msi_info->msgs[i].addr_hi =
>>> +                                                     ctx->msg.address_hi;
>>> +                                     msi_info->msgs[i].data = ctx->msg.data;
>>> +                             }
>>> +
>>> +                             ret = vfio_info_add_capability(&caps,
>>> +                                                     &msi_info->header,
>>> +                                                     msi_info_size);
>>> +                             if (ret) {
>>> +                                     kfree(msi_info);
>>> +                                     kfree(caps.buf);
>>> +                                     return ret;
>>> +                             }
>>> +                     }
>>> +             }
>>> +
>>> +             if (caps.size) {
>>> +                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
>>> +                     if (info.argsz < sizeof(info) + caps.size) {
>>> +                             info.argsz = sizeof(info) + caps.size;
>>> +                             info.cap_offset = 0;
>>> +                     } else {
>>> +                             vfio_info_cap_shift(&caps, sizeof(info));
>>> +                             if (copy_to_user((void __user *)arg +
>>> +                                               sizeof(info), caps.buf,
>>> +                                               caps.size)) {
>>> +                                     kfree(msi_info);
>>> +                                     kfree(caps.buf);
>>> +                                     return -EFAULT;
>>> +                             }
>>> +                             info.cap_offset = sizeof(info);
>>> +                     }
>>> +
>>> +                     kfree(msi_info);
>>> +                     kfree(caps.buf);
>>> +             }
>>>
>>>               return copy_to_user((void __user *)arg, &info, minsz) ?
>>>                       -EFAULT : 0;
>>> @@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
>>>               struct vfio_irq_set hdr;
>>>               u8 *data = NULL;
>>>               int ret = 0;
>>> +             int max;
>>>               size_t data_size = 0;
>>>
>>>               minsz = offsetofend(struct vfio_irq_set, count);
>>> @@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
>>>               if (copy_from_user(&hdr, (void __user *)arg, minsz))
>>>                       return -EFAULT;
>>>
>>> -             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
>>> -                                              vdev->num_irqs, &data_size);
>>> +             if (hdr.index >= vdev->num_irqs)
>>> +                     return -EINVAL;
>>> +
>>> +             max = vdev->irqs[hdr.index].count;
>>> +
>>> +             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
>>> +                                                      vdev->num_irqs,
>>> +                                                      &data_size);
>>>               if (ret)
>>>                       return ret;
>>>
>>> @@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>>>               return ret;
>>>       }
>>>
>>> +     ret = vfio_platform_get_msi(vdev);
>>> +     if (ret)
>>> +             dev_info(vdev->device, "msi not supported\n");
>> I don't think we should output this message. This is printed for
>> amd-xgbe which does not have msi so this is misleading. I would say
>> either the vfio_platform_get_msi() can return an actual error or we
>> return a void?
> Will check on this what
>>
>> Thanks
>>
>> Eric
>>> +
>>>       group = vfio_iommu_group_get(dev);
>>>       if (!group) {
>>>               dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
>>> @@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>>>  put_iommu:
>>>       vfio_iommu_group_put(group, dev);
>>>  put_reset:
>>> +     vfio_platform_put_msi(vdev);
>>>       vfio_platform_put_reset(vdev);
>>>       return ret;
>>>  }
>>> @@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
>>>  }
>>>  EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
>>>
>>> +void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
>>> +{
>>> +     mutex_lock(&driver_lock);
>>> +     list_add(&node->link, &msi_list);
>>> +     mutex_unlock(&driver_lock);
>>> +}
>>> +EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
>>> +
>>> +void vfio_platform_unregister_msi(const char *compat)
>>> +{
>>> +     struct vfio_platform_msi_node *iter, *temp;
>>> +
>>> +     mutex_lock(&driver_lock);
>>> +     list_for_each_entry_safe(iter, temp, &msi_list, link) {
>>> +             if (!strcmp(iter->compat, compat)) {
>>> +                     list_del(&iter->link);
>>> +                     break;
>>> +             }
>>> +     }
>>> +
>>> +     mutex_unlock(&driver_lock);
>>> +}
>>> +EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
>>> +
>>>  MODULE_VERSION(DRIVER_VERSION);
>>>  MODULE_LICENSE("GPL v2");
>>>  MODULE_AUTHOR(DRIVER_AUTHOR);
>>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
>>> index c5b09ec0a3c9..e0f4696afedd 100644
>>> --- a/drivers/vfio/platform/vfio_platform_irq.c
>>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
>>> @@ -8,10 +8,12 @@
>>>
>>>  #include <linux/eventfd.h>
>>>  #include <linux/interrupt.h>
>>> +#include <linux/eventfd.h>
>>>  #include <linux/slab.h>
>>>  #include <linux/types.h>
>>>  #include <linux/vfio.h>
>>>  #include <linux/irq.h>
>>> +#include <linux/msi.h>
>>>
>>>  #include "vfio_platform_private.h"
>>>
>>> @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>>>       return 0;
>>>  }
>>>
>>> +/* MSI/MSIX */
>>> +static irqreturn_t vfio_msihandler(int irq, void *arg)
>>> +{
>>> +     struct eventfd_ctx *trigger = arg;
>>> +
>>> +     eventfd_signal(trigger, 1);
>>> +     return IRQ_HANDLED;
>>> +}
>>> +
>>> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
>>> +{
>>> +     int i;
>>> +     struct vfio_platform_irq *irq;
>>> +     u16 index = desc->platform.msi_index;
>>> +     struct device *dev = msi_desc_to_dev(desc);
>>> +     struct vfio_device *device = dev_get_drvdata(dev);
>>> +     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
>>> +                                             vfio_device_data(device);
>>> +
>>> +     for (i = 0; i < vdev->num_irqs; i++)
>>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
>>> +                     irq = &vdev->irqs[i];
>>> +
>>> +     irq->ctx[index].msg.address_lo = msg->address_lo;
>>> +     irq->ctx[index].msg.address_hi = msg->address_hi;
>>> +     irq->ctx[index].msg.data = msg->data;
>>> +}
>>> +
>>> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
>>> +                        struct vfio_platform_irq *irq, int nvec)
>>> +{
>>> +     int ret;
>>> +     int msi_idx = 0;
>>> +     struct msi_desc *desc;
>>> +     struct device *dev = vdev->device;
>>> +
>>> +     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
>>> +     if (!irq->ctx)
>>> +             return -ENOMEM;
>>> +
>>> +     /* Allocate platform MSIs */
>>> +     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
>>> +     if (ret < 0) {
>>> +             kfree(irq->ctx);
>>> +             return ret;
>>> +     }
>>> +
>>> +     for_each_msi_entry(desc, dev) {
>>> +             irq->ctx[msi_idx].hwirq = desc->irq;
>>> +             msi_idx++;
>>> +     }
>>> +
>>> +     irq->num_ctx = nvec;
>>> +     irq->config_msi = 1;
>>> +
>>> +     return 0;
>>> +}
>>> +
>>> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
>>> +                                   int vector, int fd)
>>> +{
>>> +     struct eventfd_ctx *trigger;
>>> +     int irq_num, ret;
>>> +
>>> +     if (vector < 0 || vector >= irq->num_ctx)
>>> +             return -EINVAL;
>>> +
>>> +     irq_num = irq->ctx[vector].hwirq;
>>> +
>>> +     if (irq->ctx[vector].trigger) {
>>> +             free_irq(irq_num, irq->ctx[vector].trigger);
>>> +             kfree(irq->ctx[vector].name);
>>> +             eventfd_ctx_put(irq->ctx[vector].trigger);
>>> +             irq->ctx[vector].trigger = NULL;
>>> +     }
>>> +
>>> +     if (fd < 0)
>>> +             return 0;
>>> +
>>> +     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
>>> +                                       "vfio-msi[%d]", vector);
>>> +     if (!irq->ctx[vector].name)
>>> +             return -ENOMEM;
>>> +
>>> +     trigger = eventfd_ctx_fdget(fd);
>>> +     if (IS_ERR(trigger)) {
>>> +             kfree(irq->ctx[vector].name);
>>> +             return PTR_ERR(trigger);
>>> +     }
>>> +
>>> +     ret = request_irq(irq_num, vfio_msihandler, 0,
>>> +                       irq->ctx[vector].name, trigger);
>>> +     if (ret) {
>>> +             kfree(irq->ctx[vector].name);
>>> +             eventfd_ctx_put(trigger);
>>> +             return ret;
>>> +     }
>>> +
>>> +     irq->ctx[vector].trigger = trigger;
>>> +
>>> +     return 0;
>>> +}
>>> +
>>> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
>>> +                           unsigned int count, int32_t *fds)
>>> +{
>>> +     int i, j, ret = 0;
>>> +
>>> +     if (start >= irq->num_ctx || start + count > irq->num_ctx)
>>> +             return -EINVAL;
>>> +
>>> +     for (i = 0, j = start; i < count && !ret; i++, j++) {
>>> +             int fd = fds ? fds[i] : -1;
>>> +
>>> +             ret = vfio_msi_set_vector_signal(irq, j, fd);
>>> +     }
>>> +
>>> +     if (ret) {
>>> +             for (--j; j >= (int)start; j--)
>>> +                     vfio_msi_set_vector_signal(irq, j, -1);
>>> +     }
>>> +
>>> +     return ret;
>>> +}
>>> +
>>> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
>>> +                          struct vfio_platform_irq *irq)
>>> +{
>>> +     struct device *dev = vdev->device;
>>> +
>>> +     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
>>> +
>>> +     platform_msi_domain_free_irqs(dev);
>>> +
>>> +     irq->config_msi = 0;
>>> +     irq->num_ctx = 0;
>>> +
>>> +     kfree(irq->ctx);
>>> +}
>>> +
>>> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
>>> +                             unsigned int index, unsigned int start,
>>> +                             unsigned int count, uint32_t flags, void *data)
>>> +{
>>> +     int i;
>>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>>> +
>>> +     if (start + count > irq->count)
>>> +             return -EINVAL;
>>> +
>>> +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
>>> +             vfio_msi_disable(vdev, irq);
>>> +             return 0;
>>> +     }
>>> +
>>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
>>> +             s32 *fds = data;
>>> +             int ret;
>>> +
>>> +             if (irq->config_msi)
>>> +                     return vfio_msi_set_block(irq, start, count,
>>> +                                               fds);
>>> +             ret = vfio_msi_enable(vdev, irq, start + count);
>>> +             if (ret)
>>> +                     return ret;
>>> +
>>> +             ret = vfio_msi_set_block(irq, start, count, fds);
>>> +             if (ret)
>>> +                     vfio_msi_disable(vdev, irq);
>>> +
>>> +             return ret;
>>> +     }
>>> +
>>> +     for (i = start; i < start + count; i++) {
>>> +             if (!irq->ctx[i].trigger)
>>> +                     continue;
>>> +             if (flags & VFIO_IRQ_SET_DATA_NONE) {
>>> +                     eventfd_signal(irq->ctx[i].trigger, 1);
>>> +             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
>>> +                     u8 *bools = data;
>>> +
>>> +                     if (bools[i - start])
>>> +                             eventfd_signal(irq->ctx[i].trigger, 1);
>>> +             }
>>> +     }
>>> +
>>> +     return 0;
>>> +}
>>> +
>>>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>                                uint32_t flags, unsigned index, unsigned start,
>>>                                unsigned count, void *data)
>>> @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>                   unsigned start, unsigned count, uint32_t flags,
>>>                   void *data) = NULL;
>>>
>>> -     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>> -     case VFIO_IRQ_SET_ACTION_MASK:
>>> -             func = vfio_platform_set_irq_mask;
>>> -             break;
>>> -     case VFIO_IRQ_SET_ACTION_UNMASK:
>>> -             func = vfio_platform_set_irq_unmask;
>>> -             break;
>>> -     case VFIO_IRQ_SET_ACTION_TRIGGER:
>>> -             func = vfio_platform_set_irq_trigger;
>>> -             break;
>>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>>> +
>>> +     if (irq->type == VFIO_IRQ_TYPE_MSI) {
>>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>> +             case VFIO_IRQ_SET_ACTION_MASK:
>>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
>>> +                     break;
>>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
>>> +                     func = vfio_set_msi_trigger;
>>> +                     break;
>>> +             }
>>> +     } else {
>>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>> +             case VFIO_IRQ_SET_ACTION_MASK:
>>> +                     func = vfio_platform_set_irq_mask;
>>> +                     break;
>>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
>>> +                     func = vfio_platform_set_irq_unmask;
>>> +                     break;
>>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
>>> +                     func = vfio_platform_set_irq_trigger;
>>> +                     break;
>>> +             }
>>>       }
>>>
>>>       if (!func)
>>> @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>
>>>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>  {
>>> -     int cnt = 0, i;
>>> +     int i;
>>> +     int cnt = 0;
>>> +     int num_irqs;
>>> +     int msi_cnt = 0;
>>>
>>>       while (vdev->get_irq(vdev, cnt) >= 0)
>>>               cnt++;
>>>
>>> -     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
>>> +     if (vdev->of_get_msi) {
>>> +             msi_cnt = vdev->of_get_msi(vdev);
>>> +             num_irqs++;
>>> +     }
>>> +
>>> +     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
>>> +                          GFP_KERNEL);
>>>       if (!vdev->irqs)
>>>               return -ENOMEM;
>>>
>>> @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>               vdev->irqs[i].masked = false;
>>>       }
>>>
>>> -     vdev->num_irqs = cnt;
>>> +     /*
>>> +      * MSI block is added at last index and it is an ext irq
>>> +      */
>>> +     if (msi_cnt > 0) {
>>> +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
>>> +             vdev->irqs[i].count = msi_cnt;
>>> +             vdev->irqs[i].hwirq = 0;
>>> +             vdev->irqs[i].masked = false;
>>> +             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
>>> +             vdev->num_ext_irqs = 1;
>>> +     }
>>> +
>>> +     vdev->num_irqs = num_irqs;
>>>
>>>       return 0;
>>>  err:
>>> @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>>>  {
>>>       int i;
>>>
>>> -     for (i = 0; i < vdev->num_irqs; i++)
>>> -             vfio_set_trigger(vdev, i, -1, NULL);
>>> +     for (i = 0; i < vdev->num_irqs; i++) {
>>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
>>> +                     vfio_set_msi_trigger(vdev, i, 0, 0,
>>> +                                          VFIO_IRQ_SET_DATA_NONE, NULL);
>>> +             else
>>> +                     vfio_set_trigger(vdev, i, -1, NULL);
>>> +     }
>>>
>>>       vdev->num_irqs = 0;
>>>       kfree(vdev->irqs);
>>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>>> index 289089910643..1307feddda21 100644
>>> --- a/drivers/vfio/platform/vfio_platform_private.h
>>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>>> @@ -9,6 +9,7 @@
>>>
>>>  #include <linux/types.h>
>>>  #include <linux/interrupt.h>
>>> +#include <linux/msi.h>
>>>
>>>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
>>>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
>>> @@ -19,9 +20,21 @@
>>>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
>>>       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>>>
>>> +/* IRQ index for MSI in ext IRQs */
>>> +#define VFIO_EXT_IRQ_MSI     0
>>> +
>>> +struct vfio_irq_ctx {
>>> +     int                     hwirq;
>>> +     char                    *name;
>>> +     struct msi_msg          msg;
>>> +     struct eventfd_ctx      *trigger;
>>> +};
>>> +
>>>  struct vfio_platform_irq {
>>>       u32                     flags;
>>>       u32                     count;
>>> +     int                     num_ctx;
>>> +     struct vfio_irq_ctx     *ctx;
>>>       int                     hwirq;
>>>       char                    *name;
>>>       struct eventfd_ctx      *trigger;
>>> @@ -29,6 +42,11 @@ struct vfio_platform_irq {
>>>       spinlock_t              lock;
>>>       struct virqfd           *unmask;
>>>       struct virqfd           *mask;
>>> +
>>> +     /* for extended irqs */
>>> +     u32                     type;
>>> +     u32                     subtype;
>>> +     int                     config_msi;
>>>  };
>>>
>>>  struct vfio_platform_region {
>>> @@ -46,12 +64,14 @@ struct vfio_platform_device {
>>>       u32                             num_regions;
>>>       struct vfio_platform_irq        *irqs;
>>>       u32                             num_irqs;
>>> +     int                             num_ext_irqs;
>>>       int                             refcnt;
>>>       struct mutex                    igate;
>>>       struct module                   *parent_module;
>>>       const char                      *compat;
>>>       const char                      *acpihid;
>>>       struct module                   *reset_module;
>>> +     struct module                   *msi_module;
>>>       struct device                   *device;
>>>
>>>       /*
>>> @@ -65,11 +85,13 @@ struct vfio_platform_device {
>>>               (*get_resource)(struct vfio_platform_device *vdev, int i);
>>>       int     (*get_irq)(struct vfio_platform_device *vdev, int i);
>>>       int     (*of_reset)(struct vfio_platform_device *vdev);
>>> +     u32     (*of_get_msi)(struct vfio_platform_device *vdev);
>>>
>>>       bool                            reset_required;
>>>  };
>>>
>>>  typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
>>> +typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
>>>
>>>  struct vfio_platform_reset_node {
>>>       struct list_head link;
>>> @@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
>>>       vfio_platform_reset_fn_t of_reset;
>>>  };
>>>
>>> +struct vfio_platform_msi_node {
>>> +     struct list_head link;
>>> +     char *compat;
>>> +     struct module *owner;
>>> +     vfio_platform_get_msi_fn_t of_get_msi;
>>> +};
>>> +
>>>  extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>>>                                     struct device *dev);
>>>  extern struct vfio_platform_device *vfio_platform_remove_common
>>> @@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>  extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
>>>  extern void vfio_platform_unregister_reset(const char *compat,
>>>                                          vfio_platform_reset_fn_t fn);
>>> +void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
>>> +void vfio_platform_unregister_msi(const char *compat);
>>> +
>>>  #define vfio_platform_register_reset(__compat, __reset)              \
>>>  static struct vfio_platform_reset_node __reset ## _node = {  \
>>>       .owner = THIS_MODULE,                                   \
>>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>>> index d1812777139f..53a7ff2b524e 100644
>>> --- a/include/uapi/linux/vfio.h
>>> +++ b/include/uapi/linux/vfio.h
>>> @@ -697,11 +697,55 @@ struct vfio_irq_info {
>>>  #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
>>>  #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
>>>  #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
>>> +#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
>>>       __u32   index;          /* IRQ index */
>>>       __u32   count;          /* Number of IRQs within this index */
>>> +     __u32   cap_offset;     /* Offset within info struct of first cap */
>>>  };
>>>  #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
>>>
>>> +/*
>>> + * The irq type capability allows IRQs unique to a specific device or
>>> + * class of devices to be exposed.
>>> + *
>>> + * The structures below define version 1 of this capability.
>>> + */
>>> +#define VFIO_IRQ_INFO_CAP_TYPE               3
>>> +
>>> +struct vfio_irq_info_cap_type {
>>> +     struct vfio_info_cap_header header;
>>> +     __u32 type;     /* global per bus driver */
>>> +     __u32 subtype;  /* type specific */
>>> +};
>>> +
>>> +/*
>>> + * List of IRQ types, global per bus driver.
>>> + * If you introduce a new type, please add it here.
>>> + */
>>> +
>>> +/* Non PCI devices having MSI(s) support */
>>> +#define VFIO_IRQ_TYPE_MSI            (1)
>>> +
>>> +/*
>>> + * The msi capability allows the user to use the msi msg to
>>> + * configure the iova for the msi configuration.
>>> + * The structures below define version 1 of this capability.
>>> + */
>>> +#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
>>> +
>>> +struct vfio_irq_msi_msg {
>>> +     __u32   addr_lo;
>>> +     __u32   addr_hi;
>>> +     __u32   data;
>>> +};
>>> +
>>> +struct vfio_irq_info_cap_msi {
>>> +     struct vfio_info_cap_header header;
>>> +     __u32   nr_msgs;
>>> +     __u32   reserved;
>>> +     struct vfio_irq_msi_msg msgs[];
>>> +};
>>> +
>>>  /**
>>>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
>>>   *
>>>
>>
> 

Re: [RFC v3 1/2] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 33818 bytes --]

Hi Eric,

On Tue, Jan 12, 2021 at 2:30 PM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 1/5/21 6:53 AM, Vikas Gupta wrote:
> > On Tue, Dec 22, 2020 at 10:57 PM Auger Eric <eric.auger@redhat.com> wrote:
> >>
> >> Hi Vikas,
> >>
> >> On 12/14/20 6:45 PM, Vikas Gupta wrote:
> >>> MSI support for platform devices.The MSI block
> >>> is added as an extended IRQ which exports caps
> >>> VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.
> >>>
> >>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> >>> ---
> >>>  drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
> >>>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
> >>>  drivers/vfio/platform/vfio_platform_private.h |  32 +++
> >>>  include/uapi/linux/vfio.h                     |  44 +++
> >>>  4 files changed, 496 insertions(+), 19 deletions(-)
> >>>
> >>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> >>> index fb4b385191f2..c936852f35d7 100644
> >>> --- a/drivers/vfio/platform/vfio_platform_common.c
> >>> +++ b/drivers/vfio/platform/vfio_platform_common.c
> >>> @@ -16,6 +16,7 @@
> >>>  #include <linux/types.h>
> >>>  #include <linux/uaccess.h>
> >>>  #include <linux/vfio.h>
> >>> +#include <linux/nospec.h>
> >>>
> >>>  #include "vfio_platform_private.h"
> >>>
> >>> @@ -26,6 +27,8 @@
> >>>  #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
> >>>
> >>>  static LIST_HEAD(reset_list);
> >>> +/* devices having MSI support */
> >> nit: for devices using MSIs?
> >>> +static LIST_HEAD(msi_list);
> >>>  static DEFINE_MUTEX(driver_lock);
> >>>
> >>>  static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
> >>> @@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
> >>>       return reset_fn;
> >>>  }
> >>>
> >>> +static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
> >>> +{
> >>> +     bool has_msi = false;
> >>> +     struct vfio_platform_msi_node *iter;
> >>> +
> >>> +     mutex_lock(&driver_lock);
> >>> +     list_for_each_entry(iter, &msi_list, link) {
> >>> +             if (!strcmp(iter->compat, vdev->compat) &&
> >>> +                 try_module_get(iter->owner)) {
> >>> +                     vdev->msi_module = iter->owner;
> >>> +                     vdev->of_get_msi = iter->of_get_msi;
> >>> +                     has_msi = true;
> >>> +                     break;
> >>> +             }
> >>> +     }
> >>> +     mutex_unlock(&driver_lock);
> >>> +     return has_msi;
> >>> +}
> >>> +
> >>>  static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
> >>>                                   struct device *dev)
> >>>  {
> >>> @@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
> >>>       return vdev->of_reset ? 0 : -ENOENT;
> >>>  }
> >>>
> >>> +static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
> >>> +{
> >>> +     bool has_msi;
> >>> +
> >>> +     has_msi = vfio_platform_lookup_msi(vdev);
> >>> +     if (!has_msi) {
> >>> +             request_module("vfio-msi:%s", vdev->compat);
> >>> +             has_msi = vfio_platform_lookup_msi(vdev);
> >>> +     }
> >>> +
> >>> +     return has_msi ? 0 : -ENOENT;
> >>> +}
> >>> +
> >>>  static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
> >>>  {
> >>>       if (VFIO_PLATFORM_IS_ACPI(vdev))
> >>> @@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
> >>>               module_put(vdev->reset_module);
> >>>  }
> >>>
> >>> +static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
> >>> +{
> >>> +     if (vdev->of_get_msi)
> >>> +             module_put(vdev->msi_module);
> >>> +}
> >>> +
> >>>  static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
> >>>  {
> >>>       int cnt = 0, i;
> >>> @@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
> >>>
> >>>       } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
> >>>               struct vfio_irq_info info;
> >>> +             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
> >>> +             struct vfio_irq_info_cap_msi *msi_info = NULL;
> >>> +             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
> >>> +             unsigned long capsz;
> >>> +             u32 index;
> >>>
> >>>               minsz = offsetofend(struct vfio_irq_info, count);
> >>>
> >>> +             /* For backward compatibility, cannot require this */
> >>> +             capsz = offsetofend(struct vfio_irq_info, cap_offset);
> >>> +
> >>>               if (copy_from_user(&info, (void __user *)arg, minsz))
> >>>                       return -EFAULT;
> >>>
> >>> @@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
> >>>               if (info.index >= vdev->num_irqs)
> >>>                       return -EINVAL;
> >>>
> >>> -             info.flags = vdev->irqs[info.index].flags;
> >>> -             info.count = vdev->irqs[info.index].count;
> >>> +             if (info.argsz >= capsz)
> >>> +                     minsz = capsz;
> >>> +
> >>> +             index = info.index;
> >>> +
> >>> +             info.flags = vdev->irqs[index].flags;
> >>> +             info.count = vdev->irqs[index].count;
> >>> +
> >>> +             if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
> >>> +                     struct vfio_irq_info_cap_type cap_type = {
> >>> +                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
> >>> +                             .header.version = 1 };
> >>> +                     struct vfio_platform_irq *irq;
> >>> +                     size_t msi_info_size;
> >>> +                     int num_msgs;
> >>> +                     int ret;
> >>> +                     int i;
> >>> +
> >>> +                     index = array_index_nospec(index,
> >>> +                                                vdev->num_irqs);
> >>> +                     irq = &vdev->irqs[index];
> >>> +
> >>> +                     cap_type.type = irq->type;
> >>> +                     cap_type.subtype = irq->subtype;
> >>> +
> >>> +                     ret = vfio_info_add_capability(&caps,
> >>> +                                                    &cap_type.header,
> >>> +                                                    sizeof(cap_type));
> >>> +                     if (ret)
> >>> +                             return ret;
> >>> +
> >>> +                     num_msgs = irq->num_ctx;
> >>> +                     if (num_msgs) {
> >>> +                             msi_info_size = struct_size(msi_info,
> >>> +                                                         msgs, num_msgs);
> >>> +
> >>> +                             msi_info = kzalloc(msi_info_size, GFP_KERNEL);
> >>> +                             if (!msi_info) {
> >>> +                                     kfree(caps.buf);
> >>> +                                     return -ENOMEM;
> >>> +                             }
> >>> +
> >>> +                             msi_info->header.id =
> >>> +                                             VFIO_IRQ_INFO_CAP_MSI_DESCS;
> >> I thought you would remove this cap now the module is back. What is the
> >> motivation to keep it?
> >
> > Hi Eric,
> > The earlier module was serving two purposes
> > a)   Max number of MSI(s) supported by the device. Since the module is
> > back this information can be filled in .count.
> understood
> > b)   Writing msi_msg(s) to devices for MSI sources.
> >  We want b) to be handled in the user space so that they need not be
> > dependent on the kernel.
> I fail to understand the problem with the "dependency on the kernel".
> What is the functional difference when exposing this to the userspace,
> please could you highlight the benefits?
We do not see any functional difference here, the benefit we see is
that user space can configure MSI sources with msi_msg rather than
doing it in the kernel.

>
>  VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user
> > space to get MSI configuration data and thus they can independently
> > configure MSI sources. For example, user space can just ask for ‘N’
> > MSI vectors and related msi_msg for ‘N’ vectors
> this is induced by vfio_msi_enable(), right?
>
Yes
>
>  is exported to user
> > space using this CAP. User space can now use ‘N’ vectors to configure
> > any ‘N’ MSI sources and these sources might be in any order.
> > However, this design is different from vfio-pci as MSI configuration
> > is being handled in kernel only.
> > Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit
> > with the CAP framework.
> I think it is OK reporting them through the cap but I fail to understand
> the benefits.

I believe, since there is no significant benefit this CAP should be
avoided. I plan to remove this CAP in the next patch set.

Thanks,
Vikas
>
> Thanks
>
> Eric
> >
> > Thanks,
> > Vikas
> >
> >  >
> >> Thanks
> >>
> >> Eric
> >>> +                             msi_info->header.version = 1;
> >>> +                             msi_info->nr_msgs = num_msgs;
> >>> +
> >>> +                             for (i = 0; i < num_msgs; i++) {
> >>> +                                     struct vfio_irq_ctx *ctx = &irq->ctx[i];
> >>> +
> >>> +                                     msi_info->msgs[i].addr_lo =
> >>> +                                                     ctx->msg.address_lo;
> >>> +                                     msi_info->msgs[i].addr_hi =
> >>> +                                                     ctx->msg.address_hi;
> >>> +                                     msi_info->msgs[i].data = ctx->msg.data;
> >>> +                             }
> >>> +
> >>> +                             ret = vfio_info_add_capability(&caps,
> >>> +                                                     &msi_info->header,
> >>> +                                                     msi_info_size);
> >>> +                             if (ret) {
> >>> +                                     kfree(msi_info);
> >>> +                                     kfree(caps.buf);
> >>> +                                     return ret;
> >>> +                             }
> >>> +                     }
> >>> +             }
> >>> +
> >>> +             if (caps.size) {
> >>> +                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
> >>> +                     if (info.argsz < sizeof(info) + caps.size) {
> >>> +                             info.argsz = sizeof(info) + caps.size;
> >>> +                             info.cap_offset = 0;
> >>> +                     } else {
> >>> +                             vfio_info_cap_shift(&caps, sizeof(info));
> >>> +                             if (copy_to_user((void __user *)arg +
> >>> +                                               sizeof(info), caps.buf,
> >>> +                                               caps.size)) {
> >>> +                                     kfree(msi_info);
> >>> +                                     kfree(caps.buf);
> >>> +                                     return -EFAULT;
> >>> +                             }
> >>> +                             info.cap_offset = sizeof(info);
> >>> +                     }
> >>> +
> >>> +                     kfree(msi_info);
> >>> +                     kfree(caps.buf);
> >>> +             }
> >>>
> >>>               return copy_to_user((void __user *)arg, &info, minsz) ?
> >>>                       -EFAULT : 0;
> >>> @@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
> >>>               struct vfio_irq_set hdr;
> >>>               u8 *data = NULL;
> >>>               int ret = 0;
> >>> +             int max;
> >>>               size_t data_size = 0;
> >>>
> >>>               minsz = offsetofend(struct vfio_irq_set, count);
> >>> @@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
> >>>               if (copy_from_user(&hdr, (void __user *)arg, minsz))
> >>>                       return -EFAULT;
> >>>
> >>> -             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
> >>> -                                              vdev->num_irqs, &data_size);
> >>> +             if (hdr.index >= vdev->num_irqs)
> >>> +                     return -EINVAL;
> >>> +
> >>> +             max = vdev->irqs[hdr.index].count;
> >>> +
> >>> +             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
> >>> +                                                      vdev->num_irqs,
> >>> +                                                      &data_size);
> >>>               if (ret)
> >>>                       return ret;
> >>>
> >>> @@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> >>>               return ret;
> >>>       }
> >>>
> >>> +     ret = vfio_platform_get_msi(vdev);
> >>> +     if (ret)
> >>> +             dev_info(vdev->device, "msi not supported\n");
> >> I don't think we should output this message. This is printed for
> >> amd-xgbe which does not have msi so this is misleading. I would say
> >> either the vfio_platform_get_msi() can return an actual error or we
> >> return a void?
> > Will check on this what
> >>
> >> Thanks
> >>
> >> Eric
> >>> +
> >>>       group = vfio_iommu_group_get(dev);
> >>>       if (!group) {
> >>>               dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
> >>> @@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> >>>  put_iommu:
> >>>       vfio_iommu_group_put(group, dev);
> >>>  put_reset:
> >>> +     vfio_platform_put_msi(vdev);
> >>>       vfio_platform_put_reset(vdev);
> >>>       return ret;
> >>>  }
> >>> @@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
> >>>  }
> >>>  EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
> >>>
> >>> +void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
> >>> +{
> >>> +     mutex_lock(&driver_lock);
> >>> +     list_add(&node->link, &msi_list);
> >>> +     mutex_unlock(&driver_lock);
> >>> +}
> >>> +EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
> >>> +
> >>> +void vfio_platform_unregister_msi(const char *compat)
> >>> +{
> >>> +     struct vfio_platform_msi_node *iter, *temp;
> >>> +
> >>> +     mutex_lock(&driver_lock);
> >>> +     list_for_each_entry_safe(iter, temp, &msi_list, link) {
> >>> +             if (!strcmp(iter->compat, compat)) {
> >>> +                     list_del(&iter->link);
> >>> +                     break;
> >>> +             }
> >>> +     }
> >>> +
> >>> +     mutex_unlock(&driver_lock);
> >>> +}
> >>> +EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
> >>> +
> >>>  MODULE_VERSION(DRIVER_VERSION);
> >>>  MODULE_LICENSE("GPL v2");
> >>>  MODULE_AUTHOR(DRIVER_AUTHOR);
> >>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> >>> index c5b09ec0a3c9..e0f4696afedd 100644
> >>> --- a/drivers/vfio/platform/vfio_platform_irq.c
> >>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
> >>> @@ -8,10 +8,12 @@
> >>>
> >>>  #include <linux/eventfd.h>
> >>>  #include <linux/interrupt.h>
> >>> +#include <linux/eventfd.h>
> >>>  #include <linux/slab.h>
> >>>  #include <linux/types.h>
> >>>  #include <linux/vfio.h>
> >>>  #include <linux/irq.h>
> >>> +#include <linux/msi.h>
> >>>
> >>>  #include "vfio_platform_private.h"
> >>>
> >>> @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
> >>>       return 0;
> >>>  }
> >>>
> >>> +/* MSI/MSIX */
> >>> +static irqreturn_t vfio_msihandler(int irq, void *arg)
> >>> +{
> >>> +     struct eventfd_ctx *trigger = arg;
> >>> +
> >>> +     eventfd_signal(trigger, 1);
> >>> +     return IRQ_HANDLED;
> >>> +}
> >>> +
> >>> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
> >>> +{
> >>> +     int i;
> >>> +     struct vfio_platform_irq *irq;
> >>> +     u16 index = desc->platform.msi_index;
> >>> +     struct device *dev = msi_desc_to_dev(desc);
> >>> +     struct vfio_device *device = dev_get_drvdata(dev);
> >>> +     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
> >>> +                                             vfio_device_data(device);
> >>> +
> >>> +     for (i = 0; i < vdev->num_irqs; i++)
> >>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> >>> +                     irq = &vdev->irqs[i];
> >>> +
> >>> +     irq->ctx[index].msg.address_lo = msg->address_lo;
> >>> +     irq->ctx[index].msg.address_hi = msg->address_hi;
> >>> +     irq->ctx[index].msg.data = msg->data;
> >>> +}
> >>> +
> >>> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
> >>> +                        struct vfio_platform_irq *irq, int nvec)
> >>> +{
> >>> +     int ret;
> >>> +     int msi_idx = 0;
> >>> +     struct msi_desc *desc;
> >>> +     struct device *dev = vdev->device;
> >>> +
> >>> +     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
> >>> +     if (!irq->ctx)
> >>> +             return -ENOMEM;
> >>> +
> >>> +     /* Allocate platform MSIs */
> >>> +     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
> >>> +     if (ret < 0) {
> >>> +             kfree(irq->ctx);
> >>> +             return ret;
> >>> +     }
> >>> +
> >>> +     for_each_msi_entry(desc, dev) {
> >>> +             irq->ctx[msi_idx].hwirq = desc->irq;
> >>> +             msi_idx++;
> >>> +     }
> >>> +
> >>> +     irq->num_ctx = nvec;
> >>> +     irq->config_msi = 1;
> >>> +
> >>> +     return 0;
> >>> +}
> >>> +
> >>> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
> >>> +                                   int vector, int fd)
> >>> +{
> >>> +     struct eventfd_ctx *trigger;
> >>> +     int irq_num, ret;
> >>> +
> >>> +     if (vector < 0 || vector >= irq->num_ctx)
> >>> +             return -EINVAL;
> >>> +
> >>> +     irq_num = irq->ctx[vector].hwirq;
> >>> +
> >>> +     if (irq->ctx[vector].trigger) {
> >>> +             free_irq(irq_num, irq->ctx[vector].trigger);
> >>> +             kfree(irq->ctx[vector].name);
> >>> +             eventfd_ctx_put(irq->ctx[vector].trigger);
> >>> +             irq->ctx[vector].trigger = NULL;
> >>> +     }
> >>> +
> >>> +     if (fd < 0)
> >>> +             return 0;
> >>> +
> >>> +     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
> >>> +                                       "vfio-msi[%d]", vector);
> >>> +     if (!irq->ctx[vector].name)
> >>> +             return -ENOMEM;
> >>> +
> >>> +     trigger = eventfd_ctx_fdget(fd);
> >>> +     if (IS_ERR(trigger)) {
> >>> +             kfree(irq->ctx[vector].name);
> >>> +             return PTR_ERR(trigger);
> >>> +     }
> >>> +
> >>> +     ret = request_irq(irq_num, vfio_msihandler, 0,
> >>> +                       irq->ctx[vector].name, trigger);
> >>> +     if (ret) {
> >>> +             kfree(irq->ctx[vector].name);
> >>> +             eventfd_ctx_put(trigger);
> >>> +             return ret;
> >>> +     }
> >>> +
> >>> +     irq->ctx[vector].trigger = trigger;
> >>> +
> >>> +     return 0;
> >>> +}
> >>> +
> >>> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
> >>> +                           unsigned int count, int32_t *fds)
> >>> +{
> >>> +     int i, j, ret = 0;
> >>> +
> >>> +     if (start >= irq->num_ctx || start + count > irq->num_ctx)
> >>> +             return -EINVAL;
> >>> +
> >>> +     for (i = 0, j = start; i < count && !ret; i++, j++) {
> >>> +             int fd = fds ? fds[i] : -1;
> >>> +
> >>> +             ret = vfio_msi_set_vector_signal(irq, j, fd);
> >>> +     }
> >>> +
> >>> +     if (ret) {
> >>> +             for (--j; j >= (int)start; j--)
> >>> +                     vfio_msi_set_vector_signal(irq, j, -1);
> >>> +     }
> >>> +
> >>> +     return ret;
> >>> +}
> >>> +
> >>> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
> >>> +                          struct vfio_platform_irq *irq)
> >>> +{
> >>> +     struct device *dev = vdev->device;
> >>> +
> >>> +     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
> >>> +
> >>> +     platform_msi_domain_free_irqs(dev);
> >>> +
> >>> +     irq->config_msi = 0;
> >>> +     irq->num_ctx = 0;
> >>> +
> >>> +     kfree(irq->ctx);
> >>> +}
> >>> +
> >>> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
> >>> +                             unsigned int index, unsigned int start,
> >>> +                             unsigned int count, uint32_t flags, void *data)
> >>> +{
> >>> +     int i;
> >>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
> >>> +
> >>> +     if (start + count > irq->count)
> >>> +             return -EINVAL;
> >>> +
> >>> +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
> >>> +             vfio_msi_disable(vdev, irq);
> >>> +             return 0;
> >>> +     }
> >>> +
> >>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
> >>> +             s32 *fds = data;
> >>> +             int ret;
> >>> +
> >>> +             if (irq->config_msi)
> >>> +                     return vfio_msi_set_block(irq, start, count,
> >>> +                                               fds);
> >>> +             ret = vfio_msi_enable(vdev, irq, start + count);
> >>> +             if (ret)
> >>> +                     return ret;
> >>> +
> >>> +             ret = vfio_msi_set_block(irq, start, count, fds);
> >>> +             if (ret)
> >>> +                     vfio_msi_disable(vdev, irq);
> >>> +
> >>> +             return ret;
> >>> +     }
> >>> +
> >>> +     for (i = start; i < start + count; i++) {
> >>> +             if (!irq->ctx[i].trigger)
> >>> +                     continue;
> >>> +             if (flags & VFIO_IRQ_SET_DATA_NONE) {
> >>> +                     eventfd_signal(irq->ctx[i].trigger, 1);
> >>> +             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> >>> +                     u8 *bools = data;
> >>> +
> >>> +                     if (bools[i - start])
> >>> +                             eventfd_signal(irq->ctx[i].trigger, 1);
> >>> +             }
> >>> +     }
> >>> +
> >>> +     return 0;
> >>> +}
> >>> +
> >>>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >>>                                uint32_t flags, unsigned index, unsigned start,
> >>>                                unsigned count, void *data)
> >>> @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >>>                   unsigned start, unsigned count, uint32_t flags,
> >>>                   void *data) = NULL;
> >>>
> >>> -     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> >>> -     case VFIO_IRQ_SET_ACTION_MASK:
> >>> -             func = vfio_platform_set_irq_mask;
> >>> -             break;
> >>> -     case VFIO_IRQ_SET_ACTION_UNMASK:
> >>> -             func = vfio_platform_set_irq_unmask;
> >>> -             break;
> >>> -     case VFIO_IRQ_SET_ACTION_TRIGGER:
> >>> -             func = vfio_platform_set_irq_trigger;
> >>> -             break;
> >>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
> >>> +
> >>> +     if (irq->type == VFIO_IRQ_TYPE_MSI) {
> >>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> >>> +             case VFIO_IRQ_SET_ACTION_MASK:
> >>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
> >>> +                     break;
> >>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
> >>> +                     func = vfio_set_msi_trigger;
> >>> +                     break;
> >>> +             }
> >>> +     } else {
> >>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> >>> +             case VFIO_IRQ_SET_ACTION_MASK:
> >>> +                     func = vfio_platform_set_irq_mask;
> >>> +                     break;
> >>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
> >>> +                     func = vfio_platform_set_irq_unmask;
> >>> +                     break;
> >>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
> >>> +                     func = vfio_platform_set_irq_trigger;
> >>> +                     break;
> >>> +             }
> >>>       }
> >>>
> >>>       if (!func)
> >>> @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >>>
> >>>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> >>>  {
> >>> -     int cnt = 0, i;
> >>> +     int i;
> >>> +     int cnt = 0;
> >>> +     int num_irqs;
> >>> +     int msi_cnt = 0;
> >>>
> >>>       while (vdev->get_irq(vdev, cnt) >= 0)
> >>>               cnt++;
> >>>
> >>> -     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
> >>> +     if (vdev->of_get_msi) {
> >>> +             msi_cnt = vdev->of_get_msi(vdev);
> >>> +             num_irqs++;
> >>> +     }
> >>> +
> >>> +     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
> >>> +                          GFP_KERNEL);
> >>>       if (!vdev->irqs)
> >>>               return -ENOMEM;
> >>>
> >>> @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> >>>               vdev->irqs[i].masked = false;
> >>>       }
> >>>
> >>> -     vdev->num_irqs = cnt;
> >>> +     /*
> >>> +      * MSI block is added at last index and it is an ext irq
> >>> +      */
> >>> +     if (msi_cnt > 0) {
> >>> +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
> >>> +             vdev->irqs[i].count = msi_cnt;
> >>> +             vdev->irqs[i].hwirq = 0;
> >>> +             vdev->irqs[i].masked = false;
> >>> +             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
> >>> +             vdev->num_ext_irqs = 1;
> >>> +     }
> >>> +
> >>> +     vdev->num_irqs = num_irqs;
> >>>
> >>>       return 0;
> >>>  err:
> >>> @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
> >>>  {
> >>>       int i;
> >>>
> >>> -     for (i = 0; i < vdev->num_irqs; i++)
> >>> -             vfio_set_trigger(vdev, i, -1, NULL);
> >>> +     for (i = 0; i < vdev->num_irqs; i++) {
> >>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> >>> +                     vfio_set_msi_trigger(vdev, i, 0, 0,
> >>> +                                          VFIO_IRQ_SET_DATA_NONE, NULL);
> >>> +             else
> >>> +                     vfio_set_trigger(vdev, i, -1, NULL);
> >>> +     }
> >>>
> >>>       vdev->num_irqs = 0;
> >>>       kfree(vdev->irqs);
> >>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> >>> index 289089910643..1307feddda21 100644
> >>> --- a/drivers/vfio/platform/vfio_platform_private.h
> >>> +++ b/drivers/vfio/platform/vfio_platform_private.h
> >>> @@ -9,6 +9,7 @@
> >>>
> >>>  #include <linux/types.h>
> >>>  #include <linux/interrupt.h>
> >>> +#include <linux/msi.h>
> >>>
> >>>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
> >>>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
> >>> @@ -19,9 +20,21 @@
> >>>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
> >>>       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
> >>>
> >>> +/* IRQ index for MSI in ext IRQs */
> >>> +#define VFIO_EXT_IRQ_MSI     0
> >>> +
> >>> +struct vfio_irq_ctx {
> >>> +     int                     hwirq;
> >>> +     char                    *name;
> >>> +     struct msi_msg          msg;
> >>> +     struct eventfd_ctx      *trigger;
> >>> +};
> >>> +
> >>>  struct vfio_platform_irq {
> >>>       u32                     flags;
> >>>       u32                     count;
> >>> +     int                     num_ctx;
> >>> +     struct vfio_irq_ctx     *ctx;
> >>>       int                     hwirq;
> >>>       char                    *name;
> >>>       struct eventfd_ctx      *trigger;
> >>> @@ -29,6 +42,11 @@ struct vfio_platform_irq {
> >>>       spinlock_t              lock;
> >>>       struct virqfd           *unmask;
> >>>       struct virqfd           *mask;
> >>> +
> >>> +     /* for extended irqs */
> >>> +     u32                     type;
> >>> +     u32                     subtype;
> >>> +     int                     config_msi;
> >>>  };
> >>>
> >>>  struct vfio_platform_region {
> >>> @@ -46,12 +64,14 @@ struct vfio_platform_device {
> >>>       u32                             num_regions;
> >>>       struct vfio_platform_irq        *irqs;
> >>>       u32                             num_irqs;
> >>> +     int                             num_ext_irqs;
> >>>       int                             refcnt;
> >>>       struct mutex                    igate;
> >>>       struct module                   *parent_module;
> >>>       const char                      *compat;
> >>>       const char                      *acpihid;
> >>>       struct module                   *reset_module;
> >>> +     struct module                   *msi_module;
> >>>       struct device                   *device;
> >>>
> >>>       /*
> >>> @@ -65,11 +85,13 @@ struct vfio_platform_device {
> >>>               (*get_resource)(struct vfio_platform_device *vdev, int i);
> >>>       int     (*get_irq)(struct vfio_platform_device *vdev, int i);
> >>>       int     (*of_reset)(struct vfio_platform_device *vdev);
> >>> +     u32     (*of_get_msi)(struct vfio_platform_device *vdev);
> >>>
> >>>       bool                            reset_required;
> >>>  };
> >>>
> >>>  typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
> >>> +typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
> >>>
> >>>  struct vfio_platform_reset_node {
> >>>       struct list_head link;
> >>> @@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
> >>>       vfio_platform_reset_fn_t of_reset;
> >>>  };
> >>>
> >>> +struct vfio_platform_msi_node {
> >>> +     struct list_head link;
> >>> +     char *compat;
> >>> +     struct module *owner;
> >>> +     vfio_platform_get_msi_fn_t of_get_msi;
> >>> +};
> >>> +
> >>>  extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> >>>                                     struct device *dev);
> >>>  extern struct vfio_platform_device *vfio_platform_remove_common
> >>> @@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> >>>  extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
> >>>  extern void vfio_platform_unregister_reset(const char *compat,
> >>>                                          vfio_platform_reset_fn_t fn);
> >>> +void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
> >>> +void vfio_platform_unregister_msi(const char *compat);
> >>> +
> >>>  #define vfio_platform_register_reset(__compat, __reset)              \
> >>>  static struct vfio_platform_reset_node __reset ## _node = {  \
> >>>       .owner = THIS_MODULE,                                   \
> >>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> >>> index d1812777139f..53a7ff2b524e 100644
> >>> --- a/include/uapi/linux/vfio.h
> >>> +++ b/include/uapi/linux/vfio.h
> >>> @@ -697,11 +697,55 @@ struct vfio_irq_info {
> >>>  #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
> >>>  #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
> >>>  #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
> >>> +#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
> >>>       __u32   index;          /* IRQ index */
> >>>       __u32   count;          /* Number of IRQs within this index */
> >>> +     __u32   cap_offset;     /* Offset within info struct of first cap */
> >>>  };
> >>>  #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
> >>>
> >>> +/*
> >>> + * The irq type capability allows IRQs unique to a specific device or
> >>> + * class of devices to be exposed.
> >>> + *
> >>> + * The structures below define version 1 of this capability.
> >>> + */
> >>> +#define VFIO_IRQ_INFO_CAP_TYPE               3
> >>> +
> >>> +struct vfio_irq_info_cap_type {
> >>> +     struct vfio_info_cap_header header;
> >>> +     __u32 type;     /* global per bus driver */
> >>> +     __u32 subtype;  /* type specific */
> >>> +};
> >>> +
> >>> +/*
> >>> + * List of IRQ types, global per bus driver.
> >>> + * If you introduce a new type, please add it here.
> >>> + */
> >>> +
> >>> +/* Non PCI devices having MSI(s) support */
> >>> +#define VFIO_IRQ_TYPE_MSI            (1)
> >>> +
> >>> +/*
> >>> + * The msi capability allows the user to use the msi msg to
> >>> + * configure the iova for the msi configuration.
> >>> + * The structures below define version 1 of this capability.
> >>> + */
> >>> +#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
> >>> +
> >>> +struct vfio_irq_msi_msg {
> >>> +     __u32   addr_lo;
> >>> +     __u32   addr_hi;
> >>> +     __u32   data;
> >>> +};
> >>> +
> >>> +struct vfio_irq_info_cap_msi {
> >>> +     struct vfio_info_cap_header header;
> >>> +     __u32   nr_msgs;
> >>> +     __u32   reserved;
> >>> +     struct vfio_irq_msi_msg msgs[];
> >>> +};
> >>> +
> >>>  /**
> >>>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
> >>>   *
> >>>
> >>
> >
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v3 1/2] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 1/15/21 7:26 AM, Vikas Gupta wrote:
> Hi Eric,
> 
> On Tue, Jan 12, 2021 at 2:30 PM Auger Eric <eric.auger@redhat.com> wrote:
>>
>> Hi Vikas,
>>
>> On 1/5/21 6:53 AM, Vikas Gupta wrote:
>>> On Tue, Dec 22, 2020 at 10:57 PM Auger Eric <eric.auger@redhat.com> wrote:
>>>>
>>>> Hi Vikas,
>>>>
>>>> On 12/14/20 6:45 PM, Vikas Gupta wrote:
>>>>> MSI support for platform devices.The MSI block
>>>>> is added as an extended IRQ which exports caps
>>>>> VFIO_IRQ_INFO_CAP_TYPE and VFIO_IRQ_INFO_CAP_MSI_DESCS.
>>>>>
>>>>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
>>>>> ---
>>>>>  drivers/vfio/platform/vfio_platform_common.c  | 179 +++++++++++-
>>>>>  drivers/vfio/platform/vfio_platform_irq.c     | 260 +++++++++++++++++-
>>>>>  drivers/vfio/platform/vfio_platform_private.h |  32 +++
>>>>>  include/uapi/linux/vfio.h                     |  44 +++
>>>>>  4 files changed, 496 insertions(+), 19 deletions(-)
>>>>>
>>>>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>>>>> index fb4b385191f2..c936852f35d7 100644
>>>>> --- a/drivers/vfio/platform/vfio_platform_common.c
>>>>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>>>>> @@ -16,6 +16,7 @@
>>>>>  #include <linux/types.h>
>>>>>  #include <linux/uaccess.h>
>>>>>  #include <linux/vfio.h>
>>>>> +#include <linux/nospec.h>
>>>>>
>>>>>  #include "vfio_platform_private.h"
>>>>>
>>>>> @@ -26,6 +27,8 @@
>>>>>  #define VFIO_PLATFORM_IS_ACPI(vdev) ((vdev)->acpihid != NULL)
>>>>>
>>>>>  static LIST_HEAD(reset_list);
>>>>> +/* devices having MSI support */
>>>> nit: for devices using MSIs?
>>>>> +static LIST_HEAD(msi_list);
>>>>>  static DEFINE_MUTEX(driver_lock);
>>>>>
>>>>>  static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
>>>>> @@ -47,6 +50,25 @@ static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
>>>>>       return reset_fn;
>>>>>  }
>>>>>
>>>>> +static bool vfio_platform_lookup_msi(struct vfio_platform_device *vdev)
>>>>> +{
>>>>> +     bool has_msi = false;
>>>>> +     struct vfio_platform_msi_node *iter;
>>>>> +
>>>>> +     mutex_lock(&driver_lock);
>>>>> +     list_for_each_entry(iter, &msi_list, link) {
>>>>> +             if (!strcmp(iter->compat, vdev->compat) &&
>>>>> +                 try_module_get(iter->owner)) {
>>>>> +                     vdev->msi_module = iter->owner;
>>>>> +                     vdev->of_get_msi = iter->of_get_msi;
>>>>> +                     has_msi = true;
>>>>> +                     break;
>>>>> +             }
>>>>> +     }
>>>>> +     mutex_unlock(&driver_lock);
>>>>> +     return has_msi;
>>>>> +}
>>>>> +
>>>>>  static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
>>>>>                                   struct device *dev)
>>>>>  {
>>>>> @@ -126,6 +148,19 @@ static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
>>>>>       return vdev->of_reset ? 0 : -ENOENT;
>>>>>  }
>>>>>
>>>>> +static int vfio_platform_get_msi(struct vfio_platform_device *vdev)
>>>>> +{
>>>>> +     bool has_msi;
>>>>> +
>>>>> +     has_msi = vfio_platform_lookup_msi(vdev);
>>>>> +     if (!has_msi) {
>>>>> +             request_module("vfio-msi:%s", vdev->compat);
>>>>> +             has_msi = vfio_platform_lookup_msi(vdev);
>>>>> +     }
>>>>> +
>>>>> +     return has_msi ? 0 : -ENOENT;
>>>>> +}
>>>>> +
>>>>>  static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
>>>>>  {
>>>>>       if (VFIO_PLATFORM_IS_ACPI(vdev))
>>>>> @@ -135,6 +170,12 @@ static void vfio_platform_put_reset(struct vfio_platform_device *vdev)
>>>>>               module_put(vdev->reset_module);
>>>>>  }
>>>>>
>>>>> +static void vfio_platform_put_msi(struct vfio_platform_device *vdev)
>>>>> +{
>>>>> +     if (vdev->of_get_msi)
>>>>> +             module_put(vdev->msi_module);
>>>>> +}
>>>>> +
>>>>>  static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
>>>>>  {
>>>>>       int cnt = 0, i;
>>>>> @@ -343,9 +384,17 @@ static long vfio_platform_ioctl(void *device_data,
>>>>>
>>>>>       } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
>>>>>               struct vfio_irq_info info;
>>>>> +             struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
>>>>> +             struct vfio_irq_info_cap_msi *msi_info = NULL;
>>>>> +             int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
>>>>> +             unsigned long capsz;
>>>>> +             u32 index;
>>>>>
>>>>>               minsz = offsetofend(struct vfio_irq_info, count);
>>>>>
>>>>> +             /* For backward compatibility, cannot require this */
>>>>> +             capsz = offsetofend(struct vfio_irq_info, cap_offset);
>>>>> +
>>>>>               if (copy_from_user(&info, (void __user *)arg, minsz))
>>>>>                       return -EFAULT;
>>>>>
>>>>> @@ -355,8 +404,94 @@ static long vfio_platform_ioctl(void *device_data,
>>>>>               if (info.index >= vdev->num_irqs)
>>>>>                       return -EINVAL;
>>>>>
>>>>> -             info.flags = vdev->irqs[info.index].flags;
>>>>> -             info.count = vdev->irqs[info.index].count;
>>>>> +             if (info.argsz >= capsz)
>>>>> +                     minsz = capsz;
>>>>> +
>>>>> +             index = info.index;
>>>>> +
>>>>> +             info.flags = vdev->irqs[index].flags;
>>>>> +             info.count = vdev->irqs[index].count;
>>>>> +
>>>>> +             if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
>>>>> +                     struct vfio_irq_info_cap_type cap_type = {
>>>>> +                             .header.id = VFIO_IRQ_INFO_CAP_TYPE,
>>>>> +                             .header.version = 1 };
>>>>> +                     struct vfio_platform_irq *irq;
>>>>> +                     size_t msi_info_size;
>>>>> +                     int num_msgs;
>>>>> +                     int ret;
>>>>> +                     int i;
>>>>> +
>>>>> +                     index = array_index_nospec(index,
>>>>> +                                                vdev->num_irqs);
>>>>> +                     irq = &vdev->irqs[index];
>>>>> +
>>>>> +                     cap_type.type = irq->type;
>>>>> +                     cap_type.subtype = irq->subtype;
>>>>> +
>>>>> +                     ret = vfio_info_add_capability(&caps,
>>>>> +                                                    &cap_type.header,
>>>>> +                                                    sizeof(cap_type));
>>>>> +                     if (ret)
>>>>> +                             return ret;
>>>>> +
>>>>> +                     num_msgs = irq->num_ctx;
>>>>> +                     if (num_msgs) {
>>>>> +                             msi_info_size = struct_size(msi_info,
>>>>> +                                                         msgs, num_msgs);
>>>>> +
>>>>> +                             msi_info = kzalloc(msi_info_size, GFP_KERNEL);
>>>>> +                             if (!msi_info) {
>>>>> +                                     kfree(caps.buf);
>>>>> +                                     return -ENOMEM;
>>>>> +                             }
>>>>> +
>>>>> +                             msi_info->header.id =
>>>>> +                                             VFIO_IRQ_INFO_CAP_MSI_DESCS;
>>>> I thought you would remove this cap now the module is back. What is the
>>>> motivation to keep it?
>>>
>>> Hi Eric,
>>> The earlier module was serving two purposes
>>> a)   Max number of MSI(s) supported by the device. Since the module is
>>> back this information can be filled in .count.
>> understood
>>> b)   Writing msi_msg(s) to devices for MSI sources.
>>>  We want b) to be handled in the user space so that they need not be
>>> dependent on the kernel.
>> I fail to understand the problem with the "dependency on the kernel".
>> What is the functional difference when exposing this to the userspace,
>> please could you highlight the benefits?
> We do not see any functional difference here, the benefit we see is
> that user space can configure MSI sources with msi_msg rather than
> doing it in the kernel.
> 
>>
>>  VFIO_IRQ_INFO_CAP_MSI_DESCS is helping user
>>> space to get MSI configuration data and thus they can independently
>>> configure MSI sources. For example, user space can just ask for ‘N’
>>> MSI vectors and related msi_msg for ‘N’ vectors
>> this is induced by vfio_msi_enable(), right?
>>
> Yes
>>
>>  is exported to user
>>> space using this CAP. User space can now use ‘N’ vectors to configure
>>> any ‘N’ MSI sources and these sources might be in any order.
>>> However, this design is different from vfio-pci as MSI configuration
>>> is being handled in kernel only.
>>> Let us know if implementing VFIO_IRQ_INFO_CAP_MSI_DESCS does not fit
>>> with the CAP framework.
>> I think it is OK reporting them through the cap but I fail to understand
>> the benefits.
> 
> I believe, since there is no significant benefit this CAP should be
> avoided. I plan to remove this CAP in the next patch set.

OK thank you for the confirmation.

Thanks

Eric
> 
> Thanks,
> Vikas
>>
>> Thanks
>>
>> Eric
>>>
>>> Thanks,
>>> Vikas
>>>
>>>  >
>>>> Thanks
>>>>
>>>> Eric
>>>>> +                             msi_info->header.version = 1;
>>>>> +                             msi_info->nr_msgs = num_msgs;
>>>>> +
>>>>> +                             for (i = 0; i < num_msgs; i++) {
>>>>> +                                     struct vfio_irq_ctx *ctx = &irq->ctx[i];
>>>>> +
>>>>> +                                     msi_info->msgs[i].addr_lo =
>>>>> +                                                     ctx->msg.address_lo;
>>>>> +                                     msi_info->msgs[i].addr_hi =
>>>>> +                                                     ctx->msg.address_hi;
>>>>> +                                     msi_info->msgs[i].data = ctx->msg.data;
>>>>> +                             }
>>>>> +
>>>>> +                             ret = vfio_info_add_capability(&caps,
>>>>> +                                                     &msi_info->header,
>>>>> +                                                     msi_info_size);
>>>>> +                             if (ret) {
>>>>> +                                     kfree(msi_info);
>>>>> +                                     kfree(caps.buf);
>>>>> +                                     return ret;
>>>>> +                             }
>>>>> +                     }
>>>>> +             }
>>>>> +
>>>>> +             if (caps.size) {
>>>>> +                     info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
>>>>> +                     if (info.argsz < sizeof(info) + caps.size) {
>>>>> +                             info.argsz = sizeof(info) + caps.size;
>>>>> +                             info.cap_offset = 0;
>>>>> +                     } else {
>>>>> +                             vfio_info_cap_shift(&caps, sizeof(info));
>>>>> +                             if (copy_to_user((void __user *)arg +
>>>>> +                                               sizeof(info), caps.buf,
>>>>> +                                               caps.size)) {
>>>>> +                                     kfree(msi_info);
>>>>> +                                     kfree(caps.buf);
>>>>> +                                     return -EFAULT;
>>>>> +                             }
>>>>> +                             info.cap_offset = sizeof(info);
>>>>> +                     }
>>>>> +
>>>>> +                     kfree(msi_info);
>>>>> +                     kfree(caps.buf);
>>>>> +             }
>>>>>
>>>>>               return copy_to_user((void __user *)arg, &info, minsz) ?
>>>>>                       -EFAULT : 0;
>>>>> @@ -365,6 +500,7 @@ static long vfio_platform_ioctl(void *device_data,
>>>>>               struct vfio_irq_set hdr;
>>>>>               u8 *data = NULL;
>>>>>               int ret = 0;
>>>>> +             int max;
>>>>>               size_t data_size = 0;
>>>>>
>>>>>               minsz = offsetofend(struct vfio_irq_set, count);
>>>>> @@ -372,8 +508,14 @@ static long vfio_platform_ioctl(void *device_data,
>>>>>               if (copy_from_user(&hdr, (void __user *)arg, minsz))
>>>>>                       return -EFAULT;
>>>>>
>>>>> -             ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
>>>>> -                                              vdev->num_irqs, &data_size);
>>>>> +             if (hdr.index >= vdev->num_irqs)
>>>>> +                     return -EINVAL;
>>>>> +
>>>>> +             max = vdev->irqs[hdr.index].count;
>>>>> +
>>>>> +             ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
>>>>> +                                                      vdev->num_irqs,
>>>>> +                                                      &data_size);
>>>>>               if (ret)
>>>>>                       return ret;
>>>>>
>>>>> @@ -678,6 +820,10 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>>>>>               return ret;
>>>>>       }
>>>>>
>>>>> +     ret = vfio_platform_get_msi(vdev);
>>>>> +     if (ret)
>>>>> +             dev_info(vdev->device, "msi not supported\n");
>>>> I don't think we should output this message. This is printed for
>>>> amd-xgbe which does not have msi so this is misleading. I would say
>>>> either the vfio_platform_get_msi() can return an actual error or we
>>>> return a void?
>>> Will check on this what
>>>>
>>>> Thanks
>>>>
>>>> Eric
>>>>> +
>>>>>       group = vfio_iommu_group_get(dev);
>>>>>       if (!group) {
>>>>>               dev_err(dev, "No IOMMU group for device %s\n", vdev->name);
>>>>> @@ -697,6 +843,7 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>>>>>  put_iommu:
>>>>>       vfio_iommu_group_put(group, dev);
>>>>>  put_reset:
>>>>> +     vfio_platform_put_msi(vdev);
>>>>>       vfio_platform_put_reset(vdev);
>>>>>       return ret;
>>>>>  }
>>>>> @@ -744,6 +891,30 @@ void vfio_platform_unregister_reset(const char *compat,
>>>>>  }
>>>>>  EXPORT_SYMBOL_GPL(vfio_platform_unregister_reset);
>>>>>
>>>>> +void __vfio_platform_register_msi(struct vfio_platform_msi_node *node)
>>>>> +{
>>>>> +     mutex_lock(&driver_lock);
>>>>> +     list_add(&node->link, &msi_list);
>>>>> +     mutex_unlock(&driver_lock);
>>>>> +}
>>>>> +EXPORT_SYMBOL_GPL(__vfio_platform_register_msi);
>>>>> +
>>>>> +void vfio_platform_unregister_msi(const char *compat)
>>>>> +{
>>>>> +     struct vfio_platform_msi_node *iter, *temp;
>>>>> +
>>>>> +     mutex_lock(&driver_lock);
>>>>> +     list_for_each_entry_safe(iter, temp, &msi_list, link) {
>>>>> +             if (!strcmp(iter->compat, compat)) {
>>>>> +                     list_del(&iter->link);
>>>>> +                     break;
>>>>> +             }
>>>>> +     }
>>>>> +
>>>>> +     mutex_unlock(&driver_lock);
>>>>> +}
>>>>> +EXPORT_SYMBOL_GPL(vfio_platform_unregister_msi);
>>>>> +
>>>>>  MODULE_VERSION(DRIVER_VERSION);
>>>>>  MODULE_LICENSE("GPL v2");
>>>>>  MODULE_AUTHOR(DRIVER_AUTHOR);
>>>>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
>>>>> index c5b09ec0a3c9..e0f4696afedd 100644
>>>>> --- a/drivers/vfio/platform/vfio_platform_irq.c
>>>>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
>>>>> @@ -8,10 +8,12 @@
>>>>>
>>>>>  #include <linux/eventfd.h>
>>>>>  #include <linux/interrupt.h>
>>>>> +#include <linux/eventfd.h>
>>>>>  #include <linux/slab.h>
>>>>>  #include <linux/types.h>
>>>>>  #include <linux/vfio.h>
>>>>>  #include <linux/irq.h>
>>>>> +#include <linux/msi.h>
>>>>>
>>>>>  #include "vfio_platform_private.h"
>>>>>
>>>>> @@ -253,6 +255,195 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>>>>>       return 0;
>>>>>  }
>>>>>
>>>>> +/* MSI/MSIX */
>>>>> +static irqreturn_t vfio_msihandler(int irq, void *arg)
>>>>> +{
>>>>> +     struct eventfd_ctx *trigger = arg;
>>>>> +
>>>>> +     eventfd_signal(trigger, 1);
>>>>> +     return IRQ_HANDLED;
>>>>> +}
>>>>> +
>>>>> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
>>>>> +{
>>>>> +     int i;
>>>>> +     struct vfio_platform_irq *irq;
>>>>> +     u16 index = desc->platform.msi_index;
>>>>> +     struct device *dev = msi_desc_to_dev(desc);
>>>>> +     struct vfio_device *device = dev_get_drvdata(dev);
>>>>> +     struct vfio_platform_device *vdev = (struct vfio_platform_device *)
>>>>> +                                             vfio_device_data(device);
>>>>> +
>>>>> +     for (i = 0; i < vdev->num_irqs; i++)
>>>>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
>>>>> +                     irq = &vdev->irqs[i];
>>>>> +
>>>>> +     irq->ctx[index].msg.address_lo = msg->address_lo;
>>>>> +     irq->ctx[index].msg.address_hi = msg->address_hi;
>>>>> +     irq->ctx[index].msg.data = msg->data;
>>>>> +}
>>>>> +
>>>>> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
>>>>> +                        struct vfio_platform_irq *irq, int nvec)
>>>>> +{
>>>>> +     int ret;
>>>>> +     int msi_idx = 0;
>>>>> +     struct msi_desc *desc;
>>>>> +     struct device *dev = vdev->device;
>>>>> +
>>>>> +     irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
>>>>> +     if (!irq->ctx)
>>>>> +             return -ENOMEM;
>>>>> +
>>>>> +     /* Allocate platform MSIs */
>>>>> +     ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
>>>>> +     if (ret < 0) {
>>>>> +             kfree(irq->ctx);
>>>>> +             return ret;
>>>>> +     }
>>>>> +
>>>>> +     for_each_msi_entry(desc, dev) {
>>>>> +             irq->ctx[msi_idx].hwirq = desc->irq;
>>>>> +             msi_idx++;
>>>>> +     }
>>>>> +
>>>>> +     irq->num_ctx = nvec;
>>>>> +     irq->config_msi = 1;
>>>>> +
>>>>> +     return 0;
>>>>> +}
>>>>> +
>>>>> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
>>>>> +                                   int vector, int fd)
>>>>> +{
>>>>> +     struct eventfd_ctx *trigger;
>>>>> +     int irq_num, ret;
>>>>> +
>>>>> +     if (vector < 0 || vector >= irq->num_ctx)
>>>>> +             return -EINVAL;
>>>>> +
>>>>> +     irq_num = irq->ctx[vector].hwirq;
>>>>> +
>>>>> +     if (irq->ctx[vector].trigger) {
>>>>> +             free_irq(irq_num, irq->ctx[vector].trigger);
>>>>> +             kfree(irq->ctx[vector].name);
>>>>> +             eventfd_ctx_put(irq->ctx[vector].trigger);
>>>>> +             irq->ctx[vector].trigger = NULL;
>>>>> +     }
>>>>> +
>>>>> +     if (fd < 0)
>>>>> +             return 0;
>>>>> +
>>>>> +     irq->ctx[vector].name = kasprintf(GFP_KERNEL,
>>>>> +                                       "vfio-msi[%d]", vector);
>>>>> +     if (!irq->ctx[vector].name)
>>>>> +             return -ENOMEM;
>>>>> +
>>>>> +     trigger = eventfd_ctx_fdget(fd);
>>>>> +     if (IS_ERR(trigger)) {
>>>>> +             kfree(irq->ctx[vector].name);
>>>>> +             return PTR_ERR(trigger);
>>>>> +     }
>>>>> +
>>>>> +     ret = request_irq(irq_num, vfio_msihandler, 0,
>>>>> +                       irq->ctx[vector].name, trigger);
>>>>> +     if (ret) {
>>>>> +             kfree(irq->ctx[vector].name);
>>>>> +             eventfd_ctx_put(trigger);
>>>>> +             return ret;
>>>>> +     }
>>>>> +
>>>>> +     irq->ctx[vector].trigger = trigger;
>>>>> +
>>>>> +     return 0;
>>>>> +}
>>>>> +
>>>>> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
>>>>> +                           unsigned int count, int32_t *fds)
>>>>> +{
>>>>> +     int i, j, ret = 0;
>>>>> +
>>>>> +     if (start >= irq->num_ctx || start + count > irq->num_ctx)
>>>>> +             return -EINVAL;
>>>>> +
>>>>> +     for (i = 0, j = start; i < count && !ret; i++, j++) {
>>>>> +             int fd = fds ? fds[i] : -1;
>>>>> +
>>>>> +             ret = vfio_msi_set_vector_signal(irq, j, fd);
>>>>> +     }
>>>>> +
>>>>> +     if (ret) {
>>>>> +             for (--j; j >= (int)start; j--)
>>>>> +                     vfio_msi_set_vector_signal(irq, j, -1);
>>>>> +     }
>>>>> +
>>>>> +     return ret;
>>>>> +}
>>>>> +
>>>>> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
>>>>> +                          struct vfio_platform_irq *irq)
>>>>> +{
>>>>> +     struct device *dev = vdev->device;
>>>>> +
>>>>> +     vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
>>>>> +
>>>>> +     platform_msi_domain_free_irqs(dev);
>>>>> +
>>>>> +     irq->config_msi = 0;
>>>>> +     irq->num_ctx = 0;
>>>>> +
>>>>> +     kfree(irq->ctx);
>>>>> +}
>>>>> +
>>>>> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
>>>>> +                             unsigned int index, unsigned int start,
>>>>> +                             unsigned int count, uint32_t flags, void *data)
>>>>> +{
>>>>> +     int i;
>>>>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>>>>> +
>>>>> +     if (start + count > irq->count)
>>>>> +             return -EINVAL;
>>>>> +
>>>>> +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
>>>>> +             vfio_msi_disable(vdev, irq);
>>>>> +             return 0;
>>>>> +     }
>>>>> +
>>>>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
>>>>> +             s32 *fds = data;
>>>>> +             int ret;
>>>>> +
>>>>> +             if (irq->config_msi)
>>>>> +                     return vfio_msi_set_block(irq, start, count,
>>>>> +                                               fds);
>>>>> +             ret = vfio_msi_enable(vdev, irq, start + count);
>>>>> +             if (ret)
>>>>> +                     return ret;
>>>>> +
>>>>> +             ret = vfio_msi_set_block(irq, start, count, fds);
>>>>> +             if (ret)
>>>>> +                     vfio_msi_disable(vdev, irq);
>>>>> +
>>>>> +             return ret;
>>>>> +     }
>>>>> +
>>>>> +     for (i = start; i < start + count; i++) {
>>>>> +             if (!irq->ctx[i].trigger)
>>>>> +                     continue;
>>>>> +             if (flags & VFIO_IRQ_SET_DATA_NONE) {
>>>>> +                     eventfd_signal(irq->ctx[i].trigger, 1);
>>>>> +             } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
>>>>> +                     u8 *bools = data;
>>>>> +
>>>>> +                     if (bools[i - start])
>>>>> +                             eventfd_signal(irq->ctx[i].trigger, 1);
>>>>> +             }
>>>>> +     }
>>>>> +
>>>>> +     return 0;
>>>>> +}
>>>>> +
>>>>>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>>>                                uint32_t flags, unsigned index, unsigned start,
>>>>>                                unsigned count, void *data)
>>>>> @@ -261,16 +452,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>>>                   unsigned start, unsigned count, uint32_t flags,
>>>>>                   void *data) = NULL;
>>>>>
>>>>> -     switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>>>> -     case VFIO_IRQ_SET_ACTION_MASK:
>>>>> -             func = vfio_platform_set_irq_mask;
>>>>> -             break;
>>>>> -     case VFIO_IRQ_SET_ACTION_UNMASK:
>>>>> -             func = vfio_platform_set_irq_unmask;
>>>>> -             break;
>>>>> -     case VFIO_IRQ_SET_ACTION_TRIGGER:
>>>>> -             func = vfio_platform_set_irq_trigger;
>>>>> -             break;
>>>>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>>>>> +
>>>>> +     if (irq->type == VFIO_IRQ_TYPE_MSI) {
>>>>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>>>> +             case VFIO_IRQ_SET_ACTION_MASK:
>>>>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
>>>>> +                     break;
>>>>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
>>>>> +                     func = vfio_set_msi_trigger;
>>>>> +                     break;
>>>>> +             }
>>>>> +     } else {
>>>>> +             switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
>>>>> +             case VFIO_IRQ_SET_ACTION_MASK:
>>>>> +                     func = vfio_platform_set_irq_mask;
>>>>> +                     break;
>>>>> +             case VFIO_IRQ_SET_ACTION_UNMASK:
>>>>> +                     func = vfio_platform_set_irq_unmask;
>>>>> +                     break;
>>>>> +             case VFIO_IRQ_SET_ACTION_TRIGGER:
>>>>> +                     func = vfio_platform_set_irq_trigger;
>>>>> +                     break;
>>>>> +             }
>>>>>       }
>>>>>
>>>>>       if (!func)
>>>>> @@ -281,12 +485,21 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>>>
>>>>>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>>>  {
>>>>> -     int cnt = 0, i;
>>>>> +     int i;
>>>>> +     int cnt = 0;
>>>>> +     int num_irqs;
>>>>> +     int msi_cnt = 0;
>>>>>
>>>>>       while (vdev->get_irq(vdev, cnt) >= 0)
>>>>>               cnt++;
>>>>>
>>>>> -     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
>>>>> +     if (vdev->of_get_msi) {
>>>>> +             msi_cnt = vdev->of_get_msi(vdev);
>>>>> +             num_irqs++;
>>>>> +     }
>>>>> +
>>>>> +     vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
>>>>> +                          GFP_KERNEL);
>>>>>       if (!vdev->irqs)
>>>>>               return -ENOMEM;
>>>>>
>>>>> @@ -309,7 +522,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>>>               vdev->irqs[i].masked = false;
>>>>>       }
>>>>>
>>>>> -     vdev->num_irqs = cnt;
>>>>> +     /*
>>>>> +      * MSI block is added at last index and it is an ext irq
>>>>> +      */
>>>>> +     if (msi_cnt > 0) {
>>>>> +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
>>>>> +             vdev->irqs[i].count = msi_cnt;
>>>>> +             vdev->irqs[i].hwirq = 0;
>>>>> +             vdev->irqs[i].masked = false;
>>>>> +             vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
>>>>> +             vdev->num_ext_irqs = 1;
>>>>> +     }
>>>>> +
>>>>> +     vdev->num_irqs = num_irqs;
>>>>>
>>>>>       return 0;
>>>>>  err:
>>>>> @@ -321,8 +546,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>>>>>  {
>>>>>       int i;
>>>>>
>>>>> -     for (i = 0; i < vdev->num_irqs; i++)
>>>>> -             vfio_set_trigger(vdev, i, -1, NULL);
>>>>> +     for (i = 0; i < vdev->num_irqs; i++) {
>>>>> +             if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
>>>>> +                     vfio_set_msi_trigger(vdev, i, 0, 0,
>>>>> +                                          VFIO_IRQ_SET_DATA_NONE, NULL);
>>>>> +             else
>>>>> +                     vfio_set_trigger(vdev, i, -1, NULL);
>>>>> +     }
>>>>>
>>>>>       vdev->num_irqs = 0;
>>>>>       kfree(vdev->irqs);
>>>>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>>>>> index 289089910643..1307feddda21 100644
>>>>> --- a/drivers/vfio/platform/vfio_platform_private.h
>>>>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>>>>> @@ -9,6 +9,7 @@
>>>>>
>>>>>  #include <linux/types.h>
>>>>>  #include <linux/interrupt.h>
>>>>> +#include <linux/msi.h>
>>>>>
>>>>>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
>>>>>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
>>>>> @@ -19,9 +20,21 @@
>>>>>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
>>>>>       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>>>>>
>>>>> +/* IRQ index for MSI in ext IRQs */
>>>>> +#define VFIO_EXT_IRQ_MSI     0
>>>>> +
>>>>> +struct vfio_irq_ctx {
>>>>> +     int                     hwirq;
>>>>> +     char                    *name;
>>>>> +     struct msi_msg          msg;
>>>>> +     struct eventfd_ctx      *trigger;
>>>>> +};
>>>>> +
>>>>>  struct vfio_platform_irq {
>>>>>       u32                     flags;
>>>>>       u32                     count;
>>>>> +     int                     num_ctx;
>>>>> +     struct vfio_irq_ctx     *ctx;
>>>>>       int                     hwirq;
>>>>>       char                    *name;
>>>>>       struct eventfd_ctx      *trigger;
>>>>> @@ -29,6 +42,11 @@ struct vfio_platform_irq {
>>>>>       spinlock_t              lock;
>>>>>       struct virqfd           *unmask;
>>>>>       struct virqfd           *mask;
>>>>> +
>>>>> +     /* for extended irqs */
>>>>> +     u32                     type;
>>>>> +     u32                     subtype;
>>>>> +     int                     config_msi;
>>>>>  };
>>>>>
>>>>>  struct vfio_platform_region {
>>>>> @@ -46,12 +64,14 @@ struct vfio_platform_device {
>>>>>       u32                             num_regions;
>>>>>       struct vfio_platform_irq        *irqs;
>>>>>       u32                             num_irqs;
>>>>> +     int                             num_ext_irqs;
>>>>>       int                             refcnt;
>>>>>       struct mutex                    igate;
>>>>>       struct module                   *parent_module;
>>>>>       const char                      *compat;
>>>>>       const char                      *acpihid;
>>>>>       struct module                   *reset_module;
>>>>> +     struct module                   *msi_module;
>>>>>       struct device                   *device;
>>>>>
>>>>>       /*
>>>>> @@ -65,11 +85,13 @@ struct vfio_platform_device {
>>>>>               (*get_resource)(struct vfio_platform_device *vdev, int i);
>>>>>       int     (*get_irq)(struct vfio_platform_device *vdev, int i);
>>>>>       int     (*of_reset)(struct vfio_platform_device *vdev);
>>>>> +     u32     (*of_get_msi)(struct vfio_platform_device *vdev);
>>>>>
>>>>>       bool                            reset_required;
>>>>>  };
>>>>>
>>>>>  typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
>>>>> +typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
>>>>>
>>>>>  struct vfio_platform_reset_node {
>>>>>       struct list_head link;
>>>>> @@ -78,6 +100,13 @@ struct vfio_platform_reset_node {
>>>>>       vfio_platform_reset_fn_t of_reset;
>>>>>  };
>>>>>
>>>>> +struct vfio_platform_msi_node {
>>>>> +     struct list_head link;
>>>>> +     char *compat;
>>>>> +     struct module *owner;
>>>>> +     vfio_platform_get_msi_fn_t of_get_msi;
>>>>> +};
>>>>> +
>>>>>  extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>>>>>                                     struct device *dev);
>>>>>  extern struct vfio_platform_device *vfio_platform_remove_common
>>>>> @@ -94,6 +123,9 @@ extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>>>>  extern void __vfio_platform_register_reset(struct vfio_platform_reset_node *n);
>>>>>  extern void vfio_platform_unregister_reset(const char *compat,
>>>>>                                          vfio_platform_reset_fn_t fn);
>>>>> +void __vfio_platform_register_msi(struct vfio_platform_msi_node *n);
>>>>> +void vfio_platform_unregister_msi(const char *compat);
>>>>> +
>>>>>  #define vfio_platform_register_reset(__compat, __reset)              \
>>>>>  static struct vfio_platform_reset_node __reset ## _node = {  \
>>>>>       .owner = THIS_MODULE,                                   \
>>>>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>>>>> index d1812777139f..53a7ff2b524e 100644
>>>>> --- a/include/uapi/linux/vfio.h
>>>>> +++ b/include/uapi/linux/vfio.h
>>>>> @@ -697,11 +697,55 @@ struct vfio_irq_info {
>>>>>  #define VFIO_IRQ_INFO_MASKABLE               (1 << 1)
>>>>>  #define VFIO_IRQ_INFO_AUTOMASKED     (1 << 2)
>>>>>  #define VFIO_IRQ_INFO_NORESIZE               (1 << 3)
>>>>> +#define VFIO_IRQ_INFO_FLAG_CAPS              (1 << 4) /* Info supports caps */
>>>>>       __u32   index;          /* IRQ index */
>>>>>       __u32   count;          /* Number of IRQs within this index */
>>>>> +     __u32   cap_offset;     /* Offset within info struct of first cap */
>>>>>  };
>>>>>  #define VFIO_DEVICE_GET_IRQ_INFO     _IO(VFIO_TYPE, VFIO_BASE + 9)
>>>>>
>>>>> +/*
>>>>> + * The irq type capability allows IRQs unique to a specific device or
>>>>> + * class of devices to be exposed.
>>>>> + *
>>>>> + * The structures below define version 1 of this capability.
>>>>> + */
>>>>> +#define VFIO_IRQ_INFO_CAP_TYPE               3
>>>>> +
>>>>> +struct vfio_irq_info_cap_type {
>>>>> +     struct vfio_info_cap_header header;
>>>>> +     __u32 type;     /* global per bus driver */
>>>>> +     __u32 subtype;  /* type specific */
>>>>> +};
>>>>> +
>>>>> +/*
>>>>> + * List of IRQ types, global per bus driver.
>>>>> + * If you introduce a new type, please add it here.
>>>>> + */
>>>>> +
>>>>> +/* Non PCI devices having MSI(s) support */
>>>>> +#define VFIO_IRQ_TYPE_MSI            (1)
>>>>> +
>>>>> +/*
>>>>> + * The msi capability allows the user to use the msi msg to
>>>>> + * configure the iova for the msi configuration.
>>>>> + * The structures below define version 1 of this capability.
>>>>> + */
>>>>> +#define VFIO_IRQ_INFO_CAP_MSI_DESCS  4
>>>>> +
>>>>> +struct vfio_irq_msi_msg {
>>>>> +     __u32   addr_lo;
>>>>> +     __u32   addr_hi;
>>>>> +     __u32   data;
>>>>> +};
>>>>> +
>>>>> +struct vfio_irq_info_cap_msi {
>>>>> +     struct vfio_info_cap_header header;
>>>>> +     __u32   nr_msgs;
>>>>> +     __u32   reserved;
>>>>> +     struct vfio_irq_msi_msg msgs[];
>>>>> +};
>>>>> +
>>>>>  /**
>>>>>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
>>>>>   *
>>>>>
>>>>
>>>
>>

[RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 4506 bytes --]

Add msi support for Broadcom platform devices

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/Kconfig                 |  1 +
 drivers/vfio/platform/Makefile                |  1 +
 drivers/vfio/platform/msi/Kconfig             |  9 ++++
 drivers/vfio/platform/msi/Makefile            |  2 +
 .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
 5 files changed, 62 insertions(+)
 create mode 100644 drivers/vfio/platform/msi/Kconfig
 create mode 100644 drivers/vfio/platform/msi/Makefile
 create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c

diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
index dc1a3c44f2c6..7b8696febe61 100644
--- a/drivers/vfio/platform/Kconfig
+++ b/drivers/vfio/platform/Kconfig
@@ -21,3 +21,4 @@ config VFIO_AMBA
 	  If you don't know what to do here, say N.
 
 source "drivers/vfio/platform/reset/Kconfig"
+source "drivers/vfio/platform/msi/Kconfig"
diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
--- a/drivers/vfio/platform/Makefile
+++ b/drivers/vfio/platform/Makefile
@@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
 obj-$(CONFIG_VFIO_PLATFORM) += reset/
+obj-$(CONFIG_VFIO_PLATFORM) += msi/
 
 vfio-amba-y := vfio_amba.o
 
diff --git a/drivers/vfio/platform/msi/Kconfig b/drivers/vfio/platform/msi/Kconfig
new file mode 100644
index 000000000000..54d6b70e1e32
--- /dev/null
+++ b/drivers/vfio/platform/msi/Kconfig
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VFIO_PLATFORM_BCMPLT_MSI
+	tristate "MSI support for Broadcom platform devices"
+	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+	default ARCH_BCM_IPROC
+	help
+	  Enables the VFIO platform driver to handle msi for Broadcom devices
+
+	  If you don't know what to do here, say N.
diff --git a/drivers/vfio/platform/msi/Makefile b/drivers/vfio/platform/msi/Makefile
new file mode 100644
index 000000000000..27422d45cecb
--- /dev/null
+++ b/drivers/vfio/platform/msi/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
diff --git a/drivers/vfio/platform/msi/vfio_platform_bcmplt.c b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
new file mode 100644
index 000000000000..a074b5e92d77
--- /dev/null
+++ b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2020 Broadcom.
+ */
+
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/msi.h>
+#include <linux/vfio.h>
+
+#include "../vfio_platform_private.h"
+
+#define RING_SIZE		(64 << 10)
+
+#define RING_MSI_ADDR_LS	0x03c
+#define RING_MSI_ADDR_MS	0x040
+#define RING_MSI_DATA_VALUE	0x064
+
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	return (reg->size / RING_SIZE);
+}
+
+static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
+	.owner = THIS_MODULE,
+	.compat = "brcm,iproc-flexrm-mbox",
+	.of_get_msi = bcm_num_msi,
+};
+
+static int __init vfio_platform_bcmflexrm_msi_module_init(void)
+{
+	__vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
+
+	return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
+{
+	vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
+}
+
+module_init(vfio_platform_bcmflexrm_msi_module_init);
+module_exit(vfio_platform_bcmflexrm_msi_module_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Broadcom");
-- 
2.17.1


-- 
This electronic communication and the information and any files transmitted 
with it, or attached to it, are confidential and are intended solely for 
the use of the individual or entity to whom it is addressed and may contain 
information that is confidential, legally privileged, protected by privacy 
laws, or otherwise restricted from disclosure to anyone else. If you are 
not the intended recipient or the person responsible for delivering the 
e-mail to the intended recipient, you are hereby notified that any use, 
copying, distributing, dissemination, forwarding, printing, or copying of 
this e-mail is strictly prohibited. If you received this e-mail in error, 
please return the e-mail to the sender, delete it from your computer, and 
destroy any printed copy of it.

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

[Auger Eric]

Hi Vikas,

On 12/14/20 6:45 PM, Vikas Gupta wrote:
> Add msi support for Broadcom platform devices
> 
> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> ---
>  drivers/vfio/platform/Kconfig                 |  1 +
>  drivers/vfio/platform/Makefile                |  1 +
>  drivers/vfio/platform/msi/Kconfig             |  9 ++++
>  drivers/vfio/platform/msi/Makefile            |  2 +
>  .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
>  5 files changed, 62 insertions(+)
>  create mode 100644 drivers/vfio/platform/msi/Kconfig
>  create mode 100644 drivers/vfio/platform/msi/Makefile
>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
what does plt mean?
> 
> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
> index dc1a3c44f2c6..7b8696febe61 100644
> --- a/drivers/vfio/platform/Kconfig
> +++ b/drivers/vfio/platform/Kconfig
> @@ -21,3 +21,4 @@ config VFIO_AMBA
>  	  If you don't know what to do here, say N.
>  
>  source "drivers/vfio/platform/reset/Kconfig"
> +source "drivers/vfio/platform/msi/Kconfig"
> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
> index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
> --- a/drivers/vfio/platform/Makefile
> +++ b/drivers/vfio/platform/Makefile
> @@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
>  obj-$(CONFIG_VFIO_PLATFORM) += reset/
> +obj-$(CONFIG_VFIO_PLATFORM) += msi/
>  
>  vfio-amba-y := vfio_amba.o
>  
> diff --git a/drivers/vfio/platform/msi/Kconfig b/drivers/vfio/platform/msi/Kconfig
> new file mode 100644
> index 000000000000..54d6b70e1e32
> --- /dev/null
> +++ b/drivers/vfio/platform/msi/Kconfig
> @@ -0,0 +1,9 @@
> +# SPDX-License-Identifier: GPL-2.0-only
> +config VFIO_PLATFORM_BCMPLT_MSI
> +	tristate "MSI support for Broadcom platform devices"
> +	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
> +	default ARCH_BCM_IPROC
> +	help
> +	  Enables the VFIO platform driver to handle msi for Broadcom devices
> +
> +	  If you don't know what to do here, say N.
> diff --git a/drivers/vfio/platform/msi/Makefile b/drivers/vfio/platform/msi/Makefile
> new file mode 100644
> index 000000000000..27422d45cecb
> --- /dev/null
> +++ b/drivers/vfio/platform/msi/Makefile
> @@ -0,0 +1,2 @@
> +# SPDX-License-Identifier: GPL-2.0
> +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
> diff --git a/drivers/vfio/platform/msi/vfio_platform_bcmplt.c b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> new file mode 100644
> index 000000000000..a074b5e92d77
> --- /dev/null
> +++ b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> @@ -0,0 +1,49 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Copyright 2020 Broadcom.
> + */
> +
> +#include <linux/module.h>
> +#include <linux/device.h>
> +#include <linux/interrupt.h>
> +#include <linux/msi.h>
> +#include <linux/vfio.h>
> +
> +#include "../vfio_platform_private.h"
> +
> +#define RING_SIZE		(64 << 10)
> +
> +#define RING_MSI_ADDR_LS	0x03c
> +#define RING_MSI_ADDR_MS	0x040
> +#define RING_MSI_DATA_VALUE	0x064
Those 3 defines would not be needed anymore with that implementation option.
> +
> +static u32 bcm_num_msi(struct vfio_platform_device *vdev)
> +{
> +	struct vfio_platform_region *reg = &vdev->regions[0];
> +
> +	return (reg->size / RING_SIZE);
> +}
> +
> +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
> +	.owner = THIS_MODULE,
> +	.compat = "brcm,iproc-flexrm-mbox",
> +	.of_get_msi = bcm_num_msi,
> +};
> +
> +static int __init vfio_platform_bcmflexrm_msi_module_init(void)
> +{
> +	__vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
> +
> +	return 0;
> +}
> +
> +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
> +{
> +	vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
> +}
> +
> +module_init(vfio_platform_bcmflexrm_msi_module_init);
> +module_exit(vfio_platform_bcmflexrm_msi_module_exit);
One thing I would like to discuss with Alex.

As the reset module is mandated (except if reset_required is forced to
0), I am wondering if we shouldn't try to turn the reset module into a
"specialization" module and put the msi hooks there. I am afraid we may
end up having modules for each and every vfio platform feature
specialization. At the moment that's fully bearable but I can't predict
what's next.

As the mandated feature is the reset capability maybe we could just keep
the config/module name terminology, tune the kconfig help message to
mention the msi support in case of flex-rm?

What do you think?

Thanks

Eric




> +
> +MODULE_LICENSE("GPL v2");
> +MODULE_AUTHOR("Broadcom");
> 

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 5866 bytes --]

Hi Eric,

On Tue, Jan 12, 2021 at 2:52 PM Auger Eric <eric.auger@redhat.com> wrote:
>
> Hi Vikas,
>
> On 12/14/20 6:45 PM, Vikas Gupta wrote:
> > Add msi support for Broadcom platform devices
> >
> > Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> > ---
> >  drivers/vfio/platform/Kconfig                 |  1 +
> >  drivers/vfio/platform/Makefile                |  1 +
> >  drivers/vfio/platform/msi/Kconfig             |  9 ++++
> >  drivers/vfio/platform/msi/Makefile            |  2 +
> >  .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
> >  5 files changed, 62 insertions(+)
> >  create mode 100644 drivers/vfio/platform/msi/Kconfig
> >  create mode 100644 drivers/vfio/platform/msi/Makefile
> >  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> what does plt mean?
This(plt) is a generic name for Broadcom platform devices, which we`ll
 plan to add in this file. Currently we have only one in this file.
Do you think this name does not sound good here?
> >
> > diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
> > index dc1a3c44f2c6..7b8696febe61 100644
> > --- a/drivers/vfio/platform/Kconfig
> > +++ b/drivers/vfio/platform/Kconfig
> > @@ -21,3 +21,4 @@ config VFIO_AMBA
> >         If you don't know what to do here, say N.
> >
> >  source "drivers/vfio/platform/reset/Kconfig"
> > +source "drivers/vfio/platform/msi/Kconfig"
> > diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
> > index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
> > --- a/drivers/vfio/platform/Makefile
> > +++ b/drivers/vfio/platform/Makefile
> > @@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
> >  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
> >  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
> >  obj-$(CONFIG_VFIO_PLATFORM) += reset/
> > +obj-$(CONFIG_VFIO_PLATFORM) += msi/
> >
> >  vfio-amba-y := vfio_amba.o
> >
> > diff --git a/drivers/vfio/platform/msi/Kconfig b/drivers/vfio/platform/msi/Kconfig
> > new file mode 100644
> > index 000000000000..54d6b70e1e32
> > --- /dev/null
> > +++ b/drivers/vfio/platform/msi/Kconfig
> > @@ -0,0 +1,9 @@
> > +# SPDX-License-Identifier: GPL-2.0-only
> > +config VFIO_PLATFORM_BCMPLT_MSI
> > +     tristate "MSI support for Broadcom platform devices"
> > +     depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
> > +     default ARCH_BCM_IPROC
> > +     help
> > +       Enables the VFIO platform driver to handle msi for Broadcom devices
> > +
> > +       If you don't know what to do here, say N.
> > diff --git a/drivers/vfio/platform/msi/Makefile b/drivers/vfio/platform/msi/Makefile
> > new file mode 100644
> > index 000000000000..27422d45cecb
> > --- /dev/null
> > +++ b/drivers/vfio/platform/msi/Makefile
> > @@ -0,0 +1,2 @@
> > +# SPDX-License-Identifier: GPL-2.0
> > +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
> > diff --git a/drivers/vfio/platform/msi/vfio_platform_bcmplt.c b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> > new file mode 100644
> > index 000000000000..a074b5e92d77
> > --- /dev/null
> > +++ b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> > @@ -0,0 +1,49 @@
> > +// SPDX-License-Identifier: GPL-2.0
> > +/*
> > + * Copyright 2020 Broadcom.
> > + */
> > +
> > +#include <linux/module.h>
> > +#include <linux/device.h>
> > +#include <linux/interrupt.h>
> > +#include <linux/msi.h>
> > +#include <linux/vfio.h>
> > +
> > +#include "../vfio_platform_private.h"
> > +
> > +#define RING_SIZE            (64 << 10)
> > +
> > +#define RING_MSI_ADDR_LS     0x03c
> > +#define RING_MSI_ADDR_MS     0x040
> > +#define RING_MSI_DATA_VALUE  0x064
> Those 3 defines would not be needed anymore with that implementation option.
> > +
> > +static u32 bcm_num_msi(struct vfio_platform_device *vdev)
> > +{
> > +     struct vfio_platform_region *reg = &vdev->regions[0];
> > +
> > +     return (reg->size / RING_SIZE);
> > +}
> > +
> > +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
> > +     .owner = THIS_MODULE,
> > +     .compat = "brcm,iproc-flexrm-mbox",
> > +     .of_get_msi = bcm_num_msi,
> > +};
> > +
> > +static int __init vfio_platform_bcmflexrm_msi_module_init(void)
> > +{
> > +     __vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
> > +
> > +     return 0;
> > +}
> > +
> > +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
> > +{
> > +     vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
> > +}
> > +
> > +module_init(vfio_platform_bcmflexrm_msi_module_init);
> > +module_exit(vfio_platform_bcmflexrm_msi_module_exit);
> One thing I would like to discuss with Alex.
>
> As the reset module is mandated (except if reset_required is forced to
> 0), I am wondering if we shouldn't try to turn the reset module into a
> "specialization" module and put the msi hooks there. I am afraid we may
> end up having modules for each and every vfio platform feature
> specialization. At the moment that's fully bearable but I can't predict
> what's next.
>
> As the mandated feature is the reset capability maybe we could just keep
> the config/module name terminology, tune the kconfig help message to
> mention the msi support in case of flex-rm?
>
As I understand, your proposal is that we should not have a separate
module for MSI, rather we add in the existing reset module for
flex-rm. Thus, this way reset modules do not seem to be specialized
just for reset functionality only but for MSI as well. Apart from this
we need not to load the proposed msi module in this patch series. Is
my understanding correct?
For me it looks OK to consolidate MSI in the existing 'reset' module.
Let me know your views so that I can work for the next patch set accordingly.

Thanks,
Vikas

> What do you think?
>
> Thanks
>
> Eric
>
>
>
>
> > +
> > +MODULE_LICENSE("GPL v2");
> > +MODULE_AUTHOR("Broadcom");
> >
>

[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

[Auger Eric]

Hi Vikas,
On 1/15/21 7:35 AM, Vikas Gupta wrote:
> Hi Eric,
> 
> On Tue, Jan 12, 2021 at 2:52 PM Auger Eric <eric.auger@redhat.com> wrote:
>>
>> Hi Vikas,
>>
>> On 12/14/20 6:45 PM, Vikas Gupta wrote:
>>> Add msi support for Broadcom platform devices
>>>
>>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
>>> ---
>>>  drivers/vfio/platform/Kconfig                 |  1 +
>>>  drivers/vfio/platform/Makefile                |  1 +
>>>  drivers/vfio/platform/msi/Kconfig             |  9 ++++
>>>  drivers/vfio/platform/msi/Makefile            |  2 +
>>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
>>>  5 files changed, 62 insertions(+)
>>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
>>>  create mode 100644 drivers/vfio/platform/msi/Makefile
>>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>> what does plt mean?
> This(plt) is a generic name for Broadcom platform devices, which we`ll
>  plan to add in this file. Currently we have only one in this file.
> Do you think this name does not sound good here?

we have VFIO_PLATFORM_BCMFLEXRM_RESET config which also applied to vfio
flex-rm platform device.

I think it would be more homegenous to have VFIO_PLATFORM_BCMFLEXRM_MSI
in case we keep a separate msi module.

also in reset dir we have vfio_platform_bcmflexrm.c


>>>
>>> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
>>> index dc1a3c44f2c6..7b8696febe61 100644
>>> --- a/drivers/vfio/platform/Kconfig
>>> +++ b/drivers/vfio/platform/Kconfig
>>> @@ -21,3 +21,4 @@ config VFIO_AMBA
>>>         If you don't know what to do here, say N.
>>>
>>>  source "drivers/vfio/platform/reset/Kconfig"
>>> +source "drivers/vfio/platform/msi/Kconfig"
>>> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
>>> index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
>>> --- a/drivers/vfio/platform/Makefile
>>> +++ b/drivers/vfio/platform/Makefile
>>> @@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
>>>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>>>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
>>>  obj-$(CONFIG_VFIO_PLATFORM) += reset/
>>> +obj-$(CONFIG_VFIO_PLATFORM) += msi/
>>>
>>>  vfio-amba-y := vfio_amba.o
>>>
>>> diff --git a/drivers/vfio/platform/msi/Kconfig b/drivers/vfio/platform/msi/Kconfig
>>> new file mode 100644
>>> index 000000000000..54d6b70e1e32
>>> --- /dev/null
>>> +++ b/drivers/vfio/platform/msi/Kconfig
>>> @@ -0,0 +1,9 @@
>>> +# SPDX-License-Identifier: GPL-2.0-only
>>> +config VFIO_PLATFORM_BCMPLT_MSI
>>> +     tristate "MSI support for Broadcom platform devices"
>>> +     depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
>>> +     default ARCH_BCM_IPROC
>>> +     help
>>> +       Enables the VFIO platform driver to handle msi for Broadcom devices
>>> +
>>> +       If you don't know what to do here, say N.
>>> diff --git a/drivers/vfio/platform/msi/Makefile b/drivers/vfio/platform/msi/Makefile
>>> new file mode 100644
>>> index 000000000000..27422d45cecb
>>> --- /dev/null
>>> +++ b/drivers/vfio/platform/msi/Makefile
>>> @@ -0,0 +1,2 @@
>>> +# SPDX-License-Identifier: GPL-2.0
>>> +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
>>> diff --git a/drivers/vfio/platform/msi/vfio_platform_bcmplt.c b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>>> new file mode 100644
>>> index 000000000000..a074b5e92d77
>>> --- /dev/null
>>> +++ b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>>> @@ -0,0 +1,49 @@
>>> +// SPDX-License-Identifier: GPL-2.0
>>> +/*
>>> + * Copyright 2020 Broadcom.
>>> + */
>>> +
>>> +#include <linux/module.h>
>>> +#include <linux/device.h>
>>> +#include <linux/interrupt.h>
>>> +#include <linux/msi.h>
>>> +#include <linux/vfio.h>
>>> +
>>> +#include "../vfio_platform_private.h"
>>> +
>>> +#define RING_SIZE            (64 << 10)
>>> +
>>> +#define RING_MSI_ADDR_LS     0x03c
>>> +#define RING_MSI_ADDR_MS     0x040
>>> +#define RING_MSI_DATA_VALUE  0x064
>> Those 3 defines would not be needed anymore with that implementation option.
>>> +
>>> +static u32 bcm_num_msi(struct vfio_platform_device *vdev)
>>> +{
>>> +     struct vfio_platform_region *reg = &vdev->regions[0];
>>> +
>>> +     return (reg->size / RING_SIZE);
>>> +}
>>> +
>>> +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
>>> +     .owner = THIS_MODULE,
>>> +     .compat = "brcm,iproc-flexrm-mbox",
>>> +     .of_get_msi = bcm_num_msi,
>>> +};
>>> +
>>> +static int __init vfio_platform_bcmflexrm_msi_module_init(void)
>>> +{
>>> +     __vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
>>> +
>>> +     return 0;
>>> +}
>>> +
>>> +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
>>> +{
>>> +     vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
>>> +}
>>> +
>>> +module_init(vfio_platform_bcmflexrm_msi_module_init);
>>> +module_exit(vfio_platform_bcmflexrm_msi_module_exit);
>> One thing I would like to discuss with Alex.
>>
>> As the reset module is mandated (except if reset_required is forced to
>> 0), I am wondering if we shouldn't try to turn the reset module into a
>> "specialization" module and put the msi hooks there. I am afraid we may
>> end up having modules for each and every vfio platform feature
>> specialization. At the moment that's fully bearable but I can't predict
>> what's next.
>>
>> As the mandated feature is the reset capability maybe we could just keep
>> the config/module name terminology, tune the kconfig help message to
>> mention the msi support in case of flex-rm?
>>
> As I understand, your proposal is that we should not have a separate
> module for MSI, rather we add in the existing reset module for
> flex-rm. Thus, this way reset modules do not seem to be specialized
> just for reset functionality only but for MSI as well. Apart from this
> we need not to load the proposed msi module in this patch series. Is
> my understanding correct?

yes it is.
> For me it looks OK to consolidate MSI in the existing 'reset' module.
> Let me know your views so that I can work for the next patch set accordingly.

Before you launch into the rewriting I would like to get the
confirmation Alex is OK or if he prefers to keep separate modules.

Thanks

Eric
> 
> Thanks,
> Vikas
> 
>> What do you think?
>>
>> Thanks
>>
>> Eric
>>
>>
>>
>>
>>> +
>>> +MODULE_LICENSE("GPL v2");
>>> +MODULE_AUTHOR("Broadcom");
>>>
>>

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

[Alex Williamson]

On Fri, 15 Jan 2021 10:24:33 +0100
Auger Eric <eric.auger@redhat.com> wrote:

> Hi Vikas,
> On 1/15/21 7:35 AM, Vikas Gupta wrote:
> > Hi Eric,
> > 
> > On Tue, Jan 12, 2021 at 2:52 PM Auger Eric <eric.auger@redhat.com> wrote:  
> >>
> >> Hi Vikas,
> >>
> >> On 12/14/20 6:45 PM, Vikas Gupta wrote:  
> >>> Add msi support for Broadcom platform devices
> >>>
> >>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> >>> ---
> >>>  drivers/vfio/platform/Kconfig                 |  1 +
> >>>  drivers/vfio/platform/Makefile                |  1 +
> >>>  drivers/vfio/platform/msi/Kconfig             |  9 ++++
> >>>  drivers/vfio/platform/msi/Makefile            |  2 +
> >>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
> >>>  5 files changed, 62 insertions(+)
> >>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
> >>>  create mode 100644 drivers/vfio/platform/msi/Makefile
> >>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c  
> >> what does plt mean?  
> > This(plt) is a generic name for Broadcom platform devices, which we`ll
> >  plan to add in this file. Currently we have only one in this file.
> > Do you think this name does not sound good here?  
> 
> we have VFIO_PLATFORM_BCMFLEXRM_RESET config which also applied to vfio
> flex-rm platform device.
> 
> I think it would be more homegenous to have VFIO_PLATFORM_BCMFLEXRM_MSI
> in case we keep a separate msi module.
> 
> also in reset dir we have vfio_platform_bcmflexrm.c
> 
> 
> >>>
> >>> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
> >>> index dc1a3c44f2c6..7b8696febe61 100644
> >>> --- a/drivers/vfio/platform/Kconfig
> >>> +++ b/drivers/vfio/platform/Kconfig
> >>> @@ -21,3 +21,4 @@ config VFIO_AMBA
> >>>         If you don't know what to do here, say N.
> >>>
> >>>  source "drivers/vfio/platform/reset/Kconfig"
> >>> +source "drivers/vfio/platform/msi/Kconfig"
> >>> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
> >>> index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
> >>> --- a/drivers/vfio/platform/Makefile
> >>> +++ b/drivers/vfio/platform/Makefile
> >>> @@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
> >>>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
> >>>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
> >>>  obj-$(CONFIG_VFIO_PLATFORM) += reset/
> >>> +obj-$(CONFIG_VFIO_PLATFORM) += msi/
> >>>
> >>>  vfio-amba-y := vfio_amba.o
> >>>
> >>> diff --git a/drivers/vfio/platform/msi/Kconfig b/drivers/vfio/platform/msi/Kconfig
> >>> new file mode 100644
> >>> index 000000000000..54d6b70e1e32
> >>> --- /dev/null
> >>> +++ b/drivers/vfio/platform/msi/Kconfig
> >>> @@ -0,0 +1,9 @@
> >>> +# SPDX-License-Identifier: GPL-2.0-only
> >>> +config VFIO_PLATFORM_BCMPLT_MSI
> >>> +     tristate "MSI support for Broadcom platform devices"
> >>> +     depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
> >>> +     default ARCH_BCM_IPROC
> >>> +     help
> >>> +       Enables the VFIO platform driver to handle msi for Broadcom devices
> >>> +
> >>> +       If you don't know what to do here, say N.
> >>> diff --git a/drivers/vfio/platform/msi/Makefile b/drivers/vfio/platform/msi/Makefile
> >>> new file mode 100644
> >>> index 000000000000..27422d45cecb
> >>> --- /dev/null
> >>> +++ b/drivers/vfio/platform/msi/Makefile
> >>> @@ -0,0 +1,2 @@
> >>> +# SPDX-License-Identifier: GPL-2.0
> >>> +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
> >>> diff --git a/drivers/vfio/platform/msi/vfio_platform_bcmplt.c b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> >>> new file mode 100644
> >>> index 000000000000..a074b5e92d77
> >>> --- /dev/null
> >>> +++ b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
> >>> @@ -0,0 +1,49 @@
> >>> +// SPDX-License-Identifier: GPL-2.0
> >>> +/*
> >>> + * Copyright 2020 Broadcom.
> >>> + */
> >>> +
> >>> +#include <linux/module.h>
> >>> +#include <linux/device.h>
> >>> +#include <linux/interrupt.h>
> >>> +#include <linux/msi.h>
> >>> +#include <linux/vfio.h>
> >>> +
> >>> +#include "../vfio_platform_private.h"
> >>> +
> >>> +#define RING_SIZE            (64 << 10)
> >>> +
> >>> +#define RING_MSI_ADDR_LS     0x03c
> >>> +#define RING_MSI_ADDR_MS     0x040
> >>> +#define RING_MSI_DATA_VALUE  0x064  
> >> Those 3 defines would not be needed anymore with that implementation option.  
> >>> +
> >>> +static u32 bcm_num_msi(struct vfio_platform_device *vdev)
> >>> +{
> >>> +     struct vfio_platform_region *reg = &vdev->regions[0];
> >>> +
> >>> +     return (reg->size / RING_SIZE);
> >>> +}
> >>> +
> >>> +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
> >>> +     .owner = THIS_MODULE,
> >>> +     .compat = "brcm,iproc-flexrm-mbox",
> >>> +     .of_get_msi = bcm_num_msi,
> >>> +};
> >>> +
> >>> +static int __init vfio_platform_bcmflexrm_msi_module_init(void)
> >>> +{
> >>> +     __vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
> >>> +
> >>> +     return 0;
> >>> +}
> >>> +
> >>> +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
> >>> +{
> >>> +     vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
> >>> +}
> >>> +
> >>> +module_init(vfio_platform_bcmflexrm_msi_module_init);
> >>> +module_exit(vfio_platform_bcmflexrm_msi_module_exit);  
> >> One thing I would like to discuss with Alex.
> >>
> >> As the reset module is mandated (except if reset_required is forced to
> >> 0), I am wondering if we shouldn't try to turn the reset module into a
> >> "specialization" module and put the msi hooks there. I am afraid we may
> >> end up having modules for each and every vfio platform feature
> >> specialization. At the moment that's fully bearable but I can't predict
> >> what's next.
> >>
> >> As the mandated feature is the reset capability maybe we could just keep
> >> the config/module name terminology, tune the kconfig help message to
> >> mention the msi support in case of flex-rm?
> >>  
> > As I understand, your proposal is that we should not have a separate
> > module for MSI, rather we add in the existing reset module for
> > flex-rm. Thus, this way reset modules do not seem to be specialized
> > just for reset functionality only but for MSI as well. Apart from this
> > we need not to load the proposed msi module in this patch series. Is
> > my understanding correct?  
> 
> yes it is.
> > For me it looks OK to consolidate MSI in the existing 'reset' module.
> > Let me know your views so that I can work for the next patch set accordingly.  
> 
> Before you launch into the rewriting I would like to get the
> confirmation Alex is OK or if he prefers to keep separate modules.

If I understand correctly, the proposal here creates an entirely
parallel vfio-msi request module interface like we have for vfio-reset,
so the question is whether we should simplify vfio-platform-core to do
a single module request per compat string and the device specific
module would register multiple features rather than one per module.  Is
that right?

It seems the submodules are pretty simple, there's not a lot to be
gained from duplicate boilerplate code in the modules themselves.  The
core code would clearly be simplified slightly to avoid multiple module
requests, but for a more grand benefit is seems the registration
interfaces would also need to be consolidated, perhaps providing a
feature "ops" structure.  As you indicate, having only two features at
this point with a fairly small number of modules each, it's not yet too
burdensome, but I could imagine it being a useful project.

More importantly in the short term, I'd expect modules handling the
same compat string to be named similarly and enabled by a common
Kconfig option.  Thanks,

Alex

Re: [RFC v3 2/2] vfio/platform: msi: add Broadcom platform devices

[Auger Eric]

Hi Alex,

On 1/19/21 11:45 PM, Alex Williamson wrote:
> On Fri, 15 Jan 2021 10:24:33 +0100
> Auger Eric <eric.auger@redhat.com> wrote:
> 
>> Hi Vikas,
>> On 1/15/21 7:35 AM, Vikas Gupta wrote:
>>> Hi Eric,
>>>
>>> On Tue, Jan 12, 2021 at 2:52 PM Auger Eric <eric.auger@redhat.com> wrote:  
>>>>
>>>> Hi Vikas,
>>>>
>>>> On 12/14/20 6:45 PM, Vikas Gupta wrote:  
>>>>> Add msi support for Broadcom platform devices
>>>>>
>>>>> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
>>>>> ---
>>>>>  drivers/vfio/platform/Kconfig                 |  1 +
>>>>>  drivers/vfio/platform/Makefile                |  1 +
>>>>>  drivers/vfio/platform/msi/Kconfig             |  9 ++++
>>>>>  drivers/vfio/platform/msi/Makefile            |  2 +
>>>>>  .../vfio/platform/msi/vfio_platform_bcmplt.c  | 49 +++++++++++++++++++
>>>>>  5 files changed, 62 insertions(+)
>>>>>  create mode 100644 drivers/vfio/platform/msi/Kconfig
>>>>>  create mode 100644 drivers/vfio/platform/msi/Makefile
>>>>>  create mode 100644 drivers/vfio/platform/msi/vfio_platform_bcmplt.c  
>>>> what does plt mean?  
>>> This(plt) is a generic name for Broadcom platform devices, which we`ll
>>>  plan to add in this file. Currently we have only one in this file.
>>> Do you think this name does not sound good here?  
>>
>> we have VFIO_PLATFORM_BCMFLEXRM_RESET config which also applied to vfio
>> flex-rm platform device.
>>
>> I think it would be more homegenous to have VFIO_PLATFORM_BCMFLEXRM_MSI
>> in case we keep a separate msi module.
>>
>> also in reset dir we have vfio_platform_bcmflexrm.c
>>
>>
>>>>>
>>>>> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
>>>>> index dc1a3c44f2c6..7b8696febe61 100644
>>>>> --- a/drivers/vfio/platform/Kconfig
>>>>> +++ b/drivers/vfio/platform/Kconfig
>>>>> @@ -21,3 +21,4 @@ config VFIO_AMBA
>>>>>         If you don't know what to do here, say N.
>>>>>
>>>>>  source "drivers/vfio/platform/reset/Kconfig"
>>>>> +source "drivers/vfio/platform/msi/Kconfig"
>>>>> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
>>>>> index 3f3a24e7c4ef..9ccdcdbf0e7e 100644
>>>>> --- a/drivers/vfio/platform/Makefile
>>>>> +++ b/drivers/vfio/platform/Makefile
>>>>> @@ -5,6 +5,7 @@ vfio-platform-y := vfio_platform.o
>>>>>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>>>>>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform-base.o
>>>>>  obj-$(CONFIG_VFIO_PLATFORM) += reset/
>>>>> +obj-$(CONFIG_VFIO_PLATFORM) += msi/
>>>>>
>>>>>  vfio-amba-y := vfio_amba.o
>>>>>
>>>>> diff --git a/drivers/vfio/platform/msi/Kconfig b/drivers/vfio/platform/msi/Kconfig
>>>>> new file mode 100644
>>>>> index 000000000000..54d6b70e1e32
>>>>> --- /dev/null
>>>>> +++ b/drivers/vfio/platform/msi/Kconfig
>>>>> @@ -0,0 +1,9 @@
>>>>> +# SPDX-License-Identifier: GPL-2.0-only
>>>>> +config VFIO_PLATFORM_BCMPLT_MSI
>>>>> +     tristate "MSI support for Broadcom platform devices"
>>>>> +     depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
>>>>> +     default ARCH_BCM_IPROC
>>>>> +     help
>>>>> +       Enables the VFIO platform driver to handle msi for Broadcom devices
>>>>> +
>>>>> +       If you don't know what to do here, say N.
>>>>> diff --git a/drivers/vfio/platform/msi/Makefile b/drivers/vfio/platform/msi/Makefile
>>>>> new file mode 100644
>>>>> index 000000000000..27422d45cecb
>>>>> --- /dev/null
>>>>> +++ b/drivers/vfio/platform/msi/Makefile
>>>>> @@ -0,0 +1,2 @@
>>>>> +# SPDX-License-Identifier: GPL-2.0
>>>>> +obj-$(CONFIG_VFIO_PLATFORM_BCMPLT_MSI) += vfio_platform_bcmplt.o
>>>>> diff --git a/drivers/vfio/platform/msi/vfio_platform_bcmplt.c b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>>>>> new file mode 100644
>>>>> index 000000000000..a074b5e92d77
>>>>> --- /dev/null
>>>>> +++ b/drivers/vfio/platform/msi/vfio_platform_bcmplt.c
>>>>> @@ -0,0 +1,49 @@
>>>>> +// SPDX-License-Identifier: GPL-2.0
>>>>> +/*
>>>>> + * Copyright 2020 Broadcom.
>>>>> + */
>>>>> +
>>>>> +#include <linux/module.h>
>>>>> +#include <linux/device.h>
>>>>> +#include <linux/interrupt.h>
>>>>> +#include <linux/msi.h>
>>>>> +#include <linux/vfio.h>
>>>>> +
>>>>> +#include "../vfio_platform_private.h"
>>>>> +
>>>>> +#define RING_SIZE            (64 << 10)
>>>>> +
>>>>> +#define RING_MSI_ADDR_LS     0x03c
>>>>> +#define RING_MSI_ADDR_MS     0x040
>>>>> +#define RING_MSI_DATA_VALUE  0x064  
>>>> Those 3 defines would not be needed anymore with that implementation option.  
>>>>> +
>>>>> +static u32 bcm_num_msi(struct vfio_platform_device *vdev)
>>>>> +{
>>>>> +     struct vfio_platform_region *reg = &vdev->regions[0];
>>>>> +
>>>>> +     return (reg->size / RING_SIZE);
>>>>> +}
>>>>> +
>>>>> +static struct vfio_platform_msi_node vfio_platform_bcmflexrm_msi_node = {
>>>>> +     .owner = THIS_MODULE,
>>>>> +     .compat = "brcm,iproc-flexrm-mbox",
>>>>> +     .of_get_msi = bcm_num_msi,
>>>>> +};
>>>>> +
>>>>> +static int __init vfio_platform_bcmflexrm_msi_module_init(void)
>>>>> +{
>>>>> +     __vfio_platform_register_msi(&vfio_platform_bcmflexrm_msi_node);
>>>>> +
>>>>> +     return 0;
>>>>> +}
>>>>> +
>>>>> +static void __exit vfio_platform_bcmflexrm_msi_module_exit(void)
>>>>> +{
>>>>> +     vfio_platform_unregister_msi("brcm,iproc-flexrm-mbox");
>>>>> +}
>>>>> +
>>>>> +module_init(vfio_platform_bcmflexrm_msi_module_init);
>>>>> +module_exit(vfio_platform_bcmflexrm_msi_module_exit);  
>>>> One thing I would like to discuss with Alex.
>>>>
>>>> As the reset module is mandated (except if reset_required is forced to
>>>> 0), I am wondering if we shouldn't try to turn the reset module into a
>>>> "specialization" module and put the msi hooks there. I am afraid we may
>>>> end up having modules for each and every vfio platform feature
>>>> specialization. At the moment that's fully bearable but I can't predict
>>>> what's next.
>>>>
>>>> As the mandated feature is the reset capability maybe we could just keep
>>>> the config/module name terminology, tune the kconfig help message to
>>>> mention the msi support in case of flex-rm?
>>>>  
>>> As I understand, your proposal is that we should not have a separate
>>> module for MSI, rather we add in the existing reset module for
>>> flex-rm. Thus, this way reset modules do not seem to be specialized
>>> just for reset functionality only but for MSI as well. Apart from this
>>> we need not to load the proposed msi module in this patch series. Is
>>> my understanding correct?  
>>
>> yes it is.
>>> For me it looks OK to consolidate MSI in the existing 'reset' module.
>>> Let me know your views so that I can work for the next patch set accordingly.  
>>
>> Before you launch into the rewriting I would like to get the
>> confirmation Alex is OK or if he prefers to keep separate modules.
> 
> If I understand correctly, the proposal here creates an entirely
> parallel vfio-msi request module interface like we have for vfio-reset,
> so the question is whether we should simplify vfio-platform-core to do
> a single module request per compat string and the device specific
> module would register multiple features rather than one per module.  Is
> that right?
Yes that's correct, the so-called "reset" module would also implement
msi hooks and if new specialization are needed in the future they also
could be put there.
> 
> It seems the submodules are pretty simple, there's not a lot to be
> gained from duplicate boilerplate code in the modules themselves.  The
> core code would clearly be simplified slightly to avoid multiple module
> requests, but for a more grand benefit is seems the registration
> interfaces would also need to be consolidated, perhaps providing a
> feature "ops" structure.  As you indicate, having only two features at
> this point with a fairly small number of modules each, it's not yet too
> burdensome, but I could imagine it being a useful project.
Yes the registration must be reworked anyway.
> 
> More importantly in the short term, I'd expect modules handling the
> same compat string to be named similarly and enabled by a common
> Kconfig option.  Thanks,
I agree with you.

Thanks

Eric
> 
> Alex
> 

[RFC v4 0/3] msi support for platform devices

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 2484 bytes --]

This RFC adds support for MSI for platform devices.
MSI block is added as an ext irq along with the existing
wired interrupt implementation. The patchset exports two
caps for MSI and related data to configure MSI source device.

Changes from:
-------------
 v3 to v4:
	1) Removed the 'cap' for exporting MSI info to userspace and
	   restored into vedor specific module.
	2) Enable GENERIC_MSI_IRQ_DOMAIN in Kconfig.
	3) Removed the vendor specific, Broadcom, 'msi' module and 
	   integrated the MSI relates ops into the 'reset' module for
	   MSI support.  

 v2 to v3:
	1) Restored the vendor specific module to get max number
	   of MSIs supported and .count value initialized.
	2) Comments from Eric addressed.

 v1 to v2:
	1) IRQ allocation has been implemented as below:
	       ----------------------------
	       |IRQ-0|IRQ-1|....|IRQ-n|MSI|
       	       ----------------------------
		MSI block has msi contexts and its implemneted
		as ext irq.

	2) Removed vendor specific module for msi handling so
	   previously patch2 and patch3 are not required.

	3) MSI related data is exported to userspace using 'caps'.
	 Please note VFIO_IRQ_INFO_CAP_TYPE in include/uapi/linux/vfio.h implementation
	is taken from the Eric`s patch
        https://patchwork.kernel.org/project/kvm/patch/20201116110030.32335-8-eric.auger@redhat.com/


 v0 to v1:
   i)  Removed MSI device flag VFIO_DEVICE_FLAGS_MSI.
   ii) Add MSI(s) at the end of the irq list of platform IRQs.
       MSI(s) with first entry of MSI block has count and flag
       information.
       IRQ list: Allocation for IRQs + MSIs are allocated as below
       Example: if there are 'n' IRQs and 'k' MSIs
       -------------------------------------------------------
       |IRQ-0|IRQ-1|....|IRQ-n|MSI-0|MSI-1|MSI-2|......|MSI-k|
       -------------------------------------------------------
       MSI-0 will have count=k set and flags set accordingly.


Vikas Gupta (3):
  vfio/platform: add support for msi
  vfio/platform: change cleanup order
  vfio: platform: reset: add msi support

 drivers/vfio/platform/Kconfig                 |   1 +
 .../platform/reset/vfio_platform_bcmflexrm.c  |  72 ++++-
 drivers/vfio/platform/vfio_platform_common.c  |  97 +++++--
 drivers/vfio/platform/vfio_platform_irq.c     | 253 ++++++++++++++++--
 drivers/vfio/platform/vfio_platform_private.h |  29 ++
 include/uapi/linux/vfio.h                     |  24 ++
 6 files changed, 444 insertions(+), 32 deletions(-)

-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

[RFC v4 1/3] vfio/platform: add support for msi

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 17153 bytes --]

MSI support for platform devices. MSI is added
as a single 'index' with 'count' as the number of
MSI(s) supported by the devices.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/Kconfig                 |   1 +
 drivers/vfio/platform/vfio_platform_common.c  |  95 ++++++-
 drivers/vfio/platform/vfio_platform_irq.c     | 253 ++++++++++++++++--
 drivers/vfio/platform/vfio_platform_private.h |  29 ++
 include/uapi/linux/vfio.h                     |  24 ++
 5 files changed, 373 insertions(+), 29 deletions(-)

diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
index dc1a3c44f2c6..d4bbc9f27763 100644
--- a/drivers/vfio/platform/Kconfig
+++ b/drivers/vfio/platform/Kconfig
@@ -3,6 +3,7 @@ config VFIO_PLATFORM
 	tristate "VFIO support for platform devices"
 	depends on VFIO && EVENTFD && (ARM || ARM64)
 	select VFIO_VIRQFD
+	select GENERIC_MSI_IRQ_DOMAIN
 	help
 	  Support for platform devices with VFIO. This is required to make
 	  use of platform devices present on the system using the VFIO
diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index fb4b385191f2..f2b1f0c3bfcc 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/uaccess.h>
 #include <linux/vfio.h>
+#include <linux/nospec.h>
 
 #include "vfio_platform_private.h"
 
@@ -28,23 +29,22 @@
 static LIST_HEAD(reset_list);
 static DEFINE_MUTEX(driver_lock);
 
-static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
-					struct module **module)
+static void vfio_platform_lookup_reset(const char *compat,
+				       struct module **module,
+				       struct vfio_platform_reset_node **node)
 {
 	struct vfio_platform_reset_node *iter;
-	vfio_platform_reset_fn_t reset_fn = NULL;
 
 	mutex_lock(&driver_lock);
 	list_for_each_entry(iter, &reset_list, link) {
 		if (!strcmp(iter->compat, compat) &&
 			try_module_get(iter->owner)) {
 			*module = iter->owner;
-			reset_fn = iter->of_reset;
+			*node = iter;
 			break;
 		}
 	}
 	mutex_unlock(&driver_lock);
-	return reset_fn;
 }
 
 static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,
@@ -112,15 +112,23 @@ static bool vfio_platform_has_reset(struct vfio_platform_device *vdev)
 
 static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
 {
+	struct vfio_platform_reset_node *node = NULL;
+
 	if (VFIO_PLATFORM_IS_ACPI(vdev))
 		return vfio_platform_acpi_has_reset(vdev) ? 0 : -ENOENT;
 
-	vdev->of_reset = vfio_platform_lookup_reset(vdev->compat,
-						    &vdev->reset_module);
-	if (!vdev->of_reset) {
+	vfio_platform_lookup_reset(vdev->compat, &vdev->reset_module,
+				   &node);
+	if (!node) {
 		request_module("vfio-reset:%s", vdev->compat);
-		vdev->of_reset = vfio_platform_lookup_reset(vdev->compat,
-							&vdev->reset_module);
+		vfio_platform_lookup_reset(vdev->compat, &vdev->reset_module,
+					   &node);
+	}
+
+	if (node) {
+		vdev->of_reset = node->of_reset;
+		vdev->of_get_msi = node->of_get_msi;
+		vdev->of_msi_write = node->of_msi_write;
 	}
 
 	return vdev->of_reset ? 0 : -ENOENT;
@@ -343,9 +351,16 @@ static long vfio_platform_ioctl(void *device_data,
 
 	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
 		struct vfio_irq_info info;
+		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
+		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
+		unsigned long capsz;
+		u32 index;
 
 		minsz = offsetofend(struct vfio_irq_info, count);
 
+		/* For backward compatibility, cannot require this */
+		capsz = offsetofend(struct vfio_irq_info, cap_offset);
+
 		if (copy_from_user(&info, (void __user *)arg, minsz))
 			return -EFAULT;
 
@@ -355,8 +370,53 @@ static long vfio_platform_ioctl(void *device_data,
 		if (info.index >= vdev->num_irqs)
 			return -EINVAL;
 
-		info.flags = vdev->irqs[info.index].flags;
-		info.count = vdev->irqs[info.index].count;
+		if (info.argsz >= capsz)
+			minsz = capsz;
+
+		index = info.index;
+
+		info.flags = vdev->irqs[index].flags;
+		info.count = vdev->irqs[index].count;
+
+		if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
+			struct vfio_irq_info_cap_type cap_type = {
+				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
+				.header.version = 1 };
+			struct vfio_platform_irq *irq;
+			int ret;
+
+			index = array_index_nospec(index,
+						   vdev->num_irqs);
+			irq = &vdev->irqs[index];
+
+			cap_type.type = irq->type;
+			cap_type.subtype = irq->subtype;
+
+			ret = vfio_info_add_capability(&caps,
+						       &cap_type.header,
+						       sizeof(cap_type));
+			if (ret)
+				return ret;
+		}
+
+		if (caps.size) {
+			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
+			if (info.argsz < sizeof(info) + caps.size) {
+				info.argsz = sizeof(info) + caps.size;
+				info.cap_offset = 0;
+			} else {
+				vfio_info_cap_shift(&caps, sizeof(info));
+				if (copy_to_user((void __user *)arg +
+						  sizeof(info), caps.buf,
+						  caps.size)) {
+					kfree(caps.buf);
+					return -EFAULT;
+				}
+				info.cap_offset = sizeof(info);
+			}
+
+			kfree(caps.buf);
+		}
 
 		return copy_to_user((void __user *)arg, &info, minsz) ?
 			-EFAULT : 0;
@@ -365,6 +425,7 @@ static long vfio_platform_ioctl(void *device_data,
 		struct vfio_irq_set hdr;
 		u8 *data = NULL;
 		int ret = 0;
+		int max;
 		size_t data_size = 0;
 
 		minsz = offsetofend(struct vfio_irq_set, count);
@@ -372,8 +433,14 @@ static long vfio_platform_ioctl(void *device_data,
 		if (copy_from_user(&hdr, (void __user *)arg, minsz))
 			return -EFAULT;
 
-		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
-						 vdev->num_irqs, &data_size);
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		max = vdev->irqs[hdr.index].count;
+
+		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
+							 vdev->num_irqs,
+							 &data_size);
 		if (ret)
 			return ret;
 
diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index c5b09ec0a3c9..db60240d27ca 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -8,10 +8,12 @@
 
 #include <linux/eventfd.h>
 #include <linux/interrupt.h>
+#include <linux/eventfd.h>
 #include <linux/slab.h>
 #include <linux/types.h>
 #include <linux/vfio.h>
 #include <linux/irq.h>
+#include <linux/msi.h>
 
 #include "vfio_platform_private.h"
 
@@ -253,6 +255,186 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	return 0;
 }
 
+/* MSI/MSIX */
+static irqreturn_t vfio_msihandler(int irq, void *arg)
+{
+	struct eventfd_ctx *trigger = arg;
+
+	eventfd_signal(trigger, 1);
+	return IRQ_HANDLED;
+}
+
+static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
+{
+	struct device *dev = msi_desc_to_dev(desc);
+	struct vfio_device *device = dev_get_drvdata(dev);
+	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
+						vfio_device_data(device);
+
+	vdev->of_msi_write(vdev, desc, msg);
+}
+
+static int vfio_msi_enable(struct vfio_platform_device *vdev,
+			   struct vfio_platform_irq *irq, int nvec)
+{
+	int ret;
+	int msi_idx = 0;
+	struct msi_desc *desc;
+	struct device *dev = vdev->device;
+
+	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
+	if (!irq->ctx)
+		return -ENOMEM;
+
+	/* Allocate platform MSIs */
+	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
+	if (ret < 0) {
+		kfree(irq->ctx);
+		return ret;
+	}
+
+	for_each_msi_entry(desc, dev) {
+		irq->ctx[msi_idx].hwirq = desc->irq;
+		msi_idx++;
+	}
+
+	irq->num_ctx = nvec;
+	irq->config_msi = 1;
+
+	return 0;
+}
+
+static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
+				      int vector, int fd)
+{
+	struct eventfd_ctx *trigger;
+	int irq_num, ret;
+
+	if (vector < 0 || vector >= irq->num_ctx)
+		return -EINVAL;
+
+	irq_num = irq->ctx[vector].hwirq;
+
+	if (irq->ctx[vector].trigger) {
+		free_irq(irq_num, irq->ctx[vector].trigger);
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(irq->ctx[vector].trigger);
+		irq->ctx[vector].trigger = NULL;
+	}
+
+	if (fd < 0)
+		return 0;
+
+	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
+					  "vfio-msi[%d]", vector);
+	if (!irq->ctx[vector].name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(irq->ctx[vector].name);
+		return PTR_ERR(trigger);
+	}
+
+	ret = request_irq(irq_num, vfio_msihandler, 0,
+			  irq->ctx[vector].name, trigger);
+	if (ret) {
+		kfree(irq->ctx[vector].name);
+		eventfd_ctx_put(trigger);
+		return ret;
+	}
+
+	irq->ctx[vector].trigger = trigger;
+
+	return 0;
+}
+
+static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
+			      unsigned int count, int32_t *fds)
+{
+	int i, j, ret = 0;
+
+	if (start >= irq->num_ctx || start + count > irq->num_ctx)
+		return -EINVAL;
+
+	for (i = 0, j = start; i < count && !ret; i++, j++) {
+		int fd = fds ? fds[i] : -1;
+
+		ret = vfio_msi_set_vector_signal(irq, j, fd);
+	}
+
+	if (ret) {
+		for (--j; j >= (int)start; j--)
+			vfio_msi_set_vector_signal(irq, j, -1);
+	}
+
+	return ret;
+}
+
+static void vfio_msi_disable(struct vfio_platform_device *vdev,
+			     struct vfio_platform_irq *irq)
+{
+	struct device *dev = vdev->device;
+
+	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
+
+	platform_msi_domain_free_irqs(dev);
+
+	irq->config_msi = 0;
+	irq->num_ctx = 0;
+
+	kfree(irq->ctx);
+}
+
+static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
+				unsigned int index, unsigned int start,
+				unsigned int count, uint32_t flags, void *data)
+{
+	int i;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (start + count > irq->count)
+		return -EINVAL;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+		vfio_msi_disable(vdev, irq);
+		return 0;
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		s32 *fds = data;
+		int ret;
+
+		if (irq->config_msi)
+			return vfio_msi_set_block(irq, start, count,
+						  fds);
+		ret = vfio_msi_enable(vdev, irq, start + count);
+		if (ret)
+			return ret;
+
+		ret = vfio_msi_set_block(irq, start, count, fds);
+		if (ret)
+			vfio_msi_disable(vdev, irq);
+
+		return ret;
+	}
+
+	for (i = start; i < start + count; i++) {
+		if (!irq->ctx[i].trigger)
+			continue;
+		if (flags & VFIO_IRQ_SET_DATA_NONE) {
+			eventfd_signal(irq->ctx[i].trigger, 1);
+		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+			u8 *bools = data;
+
+			if (bools[i - start])
+				eventfd_signal(irq->ctx[i].trigger, 1);
+		}
+	}
+
+	return 0;
+}
+
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 				 uint32_t flags, unsigned index, unsigned start,
 				 unsigned count, void *data)
@@ -261,16 +443,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 		    unsigned start, unsigned count, uint32_t flags,
 		    void *data) = NULL;
 
-	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
-	case VFIO_IRQ_SET_ACTION_MASK:
-		func = vfio_platform_set_irq_mask;
-		break;
-	case VFIO_IRQ_SET_ACTION_UNMASK:
-		func = vfio_platform_set_irq_unmask;
-		break;
-	case VFIO_IRQ_SET_ACTION_TRIGGER:
-		func = vfio_platform_set_irq_trigger;
-		break;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+
+	if (irq->type == VFIO_IRQ_TYPE_MSI) {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_set_msi_trigger;
+			break;
+		}
+	} else {
+		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+		case VFIO_IRQ_SET_ACTION_MASK:
+			func = vfio_platform_set_irq_mask;
+			break;
+		case VFIO_IRQ_SET_ACTION_UNMASK:
+			func = vfio_platform_set_irq_unmask;
+			break;
+		case VFIO_IRQ_SET_ACTION_TRIGGER:
+			func = vfio_platform_set_irq_trigger;
+			break;
+		}
 	}
 
 	if (!func)
@@ -281,12 +476,23 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
 
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
-	int cnt = 0, i;
+	int i;
+	int cnt = 0;
+	int num_irqs = 0;
+	int msi_cnt = 0;
 
 	while (vdev->get_irq(vdev, cnt) >= 0)
 		cnt++;
 
-	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	num_irqs = cnt;
+
+	if (vdev->of_get_msi) {
+		msi_cnt = vdev->of_get_msi(vdev);
+		num_irqs++;
+	}
+
+	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
+			     GFP_KERNEL);
 	if (!vdev->irqs)
 		return -ENOMEM;
 
@@ -309,7 +515,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		vdev->irqs[i].masked = false;
 	}
 
-	vdev->num_irqs = cnt;
+	/*
+	 * MSI block is added at last index and it is an ext irq
+	 */
+	if (msi_cnt > 0) {
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+		vdev->irqs[i].count = msi_cnt;
+		vdev->irqs[i].hwirq = 0;
+		vdev->irqs[i].masked = false;
+		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
+		vdev->num_ext_irqs = 1;
+	}
+
+	vdev->num_irqs = num_irqs;
 
 	return 0;
 err:
@@ -321,8 +539,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
 	int i;
 
-	for (i = 0; i < vdev->num_irqs; i++)
-		vfio_set_trigger(vdev, i, -1, NULL);
+	for (i = 0; i < vdev->num_irqs; i++) {
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			vfio_set_msi_trigger(vdev, i, 0, 0,
+					     VFIO_IRQ_SET_DATA_NONE, NULL);
+		else
+			vfio_set_trigger(vdev, i, -1, NULL);
+	}
 
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index 289089910643..b8dd892aec53 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/interrupt.h>
+#include <linux/msi.h>
 
 #define VFIO_PLATFORM_OFFSET_SHIFT   40
 #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
@@ -19,9 +20,21 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+/* IRQ index for MSI in ext IRQs */
+#define VFIO_EXT_IRQ_MSI	0
+
+struct vfio_irq_ctx {
+	int			hwirq;
+	char			*name;
+	struct msi_msg		msg;
+	struct eventfd_ctx	*trigger;
+};
+
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			num_ctx;
+	struct vfio_irq_ctx	*ctx;
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
@@ -29,6 +42,11 @@ struct vfio_platform_irq {
 	spinlock_t		lock;
 	struct virqfd		*unmask;
 	struct virqfd		*mask;
+
+	/* for extended irqs */
+	u32			type;
+	u32			subtype;
+	int			config_msi;
 };
 
 struct vfio_platform_region {
@@ -46,6 +64,7 @@ struct vfio_platform_device {
 	u32				num_regions;
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
+	int				num_ext_irqs;
 	int				refcnt;
 	struct mutex			igate;
 	struct module			*parent_module;
@@ -65,17 +84,27 @@ struct vfio_platform_device {
 		(*get_resource)(struct vfio_platform_device *vdev, int i);
 	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
 	int	(*of_reset)(struct vfio_platform_device *vdev);
+	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
+	void	(*of_msi_write)(struct vfio_platform_device *vdev,
+				struct msi_desc *desc,
+				struct msi_msg *msg);
 
 	bool				reset_required;
 };
 
 typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
+typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
+typedef void (*vfio_platform_msi_write_fn_t)(struct vfio_platform_device *vdev,
+					     struct msi_desc *desc,
+					     struct msi_msg *msg);
 
 struct vfio_platform_reset_node {
 	struct list_head link;
 	char *compat;
 	struct module *owner;
 	vfio_platform_reset_fn_t of_reset;
+	vfio_platform_get_msi_fn_t of_get_msi;
+	vfio_platform_msi_write_fn_t of_msi_write;
 };
 
 extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index d1812777139f..8e2c0131781d 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -697,11 +697,35 @@ struct vfio_irq_info {
 #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
 #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
 #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
 	__u32	index;		/* IRQ index */
 	__u32	count;		/* Number of IRQs within this index */
+	__u32	cap_offset;	/* Offset within info struct of first cap */
 };
 #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
 
+/*
+ * The irq type capability allows IRQs unique to a specific device or
+ * class of devices to be exposed.
+ *
+ * The structures below define version 1 of this capability.
+ */
+#define VFIO_IRQ_INFO_CAP_TYPE		3
+
+struct vfio_irq_info_cap_type {
+	struct vfio_info_cap_header header;
+	__u32 type;     /* global per bus driver */
+	__u32 subtype;  /* type specific */
+};
+
+/*
+ * List of IRQ types, global per bus driver.
+ * If you introduce a new type, please add it here.
+ */
+
+/* Non PCI devices having MSI(s) support */
+#define VFIO_IRQ_TYPE_MSI		(1)
+
 /**
  * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
  *
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v4 1/3] vfio/platform: add support for msi

[Auger Eric]

Hi Vikas,

On 1/29/21 6:24 PM, Vikas Gupta wrote:
> MSI support for platform devices. MSI is added
s/MSI support/ Add MSI support
> as a single 'index' with 'count' as the number of
> MSI(s) supported by the devices.
as a single 'index' following the last wired irq index index, with count.

It allows to associate eventfds to MSIs.

If MSI is supported, specialization callbacks need to be implemented in
the reset module (of_get_msi and of_msi_write).
> 
> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> ---
>  drivers/vfio/platform/Kconfig                 |   1 +
>  drivers/vfio/platform/vfio_platform_common.c  |  95 ++++++-
>  drivers/vfio/platform/vfio_platform_irq.c     | 253 ++++++++++++++++--
>  drivers/vfio/platform/vfio_platform_private.h |  29 ++
>  include/uapi/linux/vfio.h                     |  24 ++
>  5 files changed, 373 insertions(+), 29 deletions(-)
> 
> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
> index dc1a3c44f2c6..d4bbc9f27763 100644
> --- a/drivers/vfio/platform/Kconfig
> +++ b/drivers/vfio/platform/Kconfig
> @@ -3,6 +3,7 @@ config VFIO_PLATFORM
>  	tristate "VFIO support for platform devices"
>  	depends on VFIO && EVENTFD && (ARM || ARM64)
>  	select VFIO_VIRQFD
> +	select GENERIC_MSI_IRQ_DOMAIN
>  	help
>  	  Support for platform devices with VFIO. This is required to make
>  	  use of platform devices present on the system using the VFIO
> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> index fb4b385191f2..f2b1f0c3bfcc 100644
> --- a/drivers/vfio/platform/vfio_platform_common.c
> +++ b/drivers/vfio/platform/vfio_platform_common.c
> @@ -16,6 +16,7 @@
>  #include <linux/types.h>
>  #include <linux/uaccess.h>
>  #include <linux/vfio.h>
> +#include <linux/nospec.h>
>  
>  #include "vfio_platform_private.h"
>  
> @@ -28,23 +29,22 @@
>  static LIST_HEAD(reset_list);
>  static DEFINE_MUTEX(driver_lock);
>  
> -static vfio_platform_reset_fn_t vfio_platform_lookup_reset(const char *compat,
> -					struct module **module)
> +static void vfio_platform_lookup_reset(const char *compat,
> +				       struct module **module,
> +				       struct vfio_platform_reset_node **node)
nit: I would prefer this function directly returns a
struct vfio_platform_reset_node *
>  {
>  	struct vfio_platform_reset_node *iter;
> -	vfio_platform_reset_fn_t reset_fn = NULL;
>  
>  	mutex_lock(&driver_lock);
>  	list_for_each_entry(iter, &reset_list, link) {
>  		if (!strcmp(iter->compat, compat) &&
>  			try_module_get(iter->owner)) {
>  			*module = iter->owner;
> -			reset_fn = iter->of_reset;
> +			*node = iter;
>  			break;
>  		}
>  	}
>  	mutex_unlock(&driver_lock);
> -	return reset_fn;
>  }
>  
>  static int vfio_platform_acpi_probe(struct vfio_platform_device *vdev,> @@ -112,15 +112,23 @@ static bool vfio_platform_has_reset(struct
vfio_platform_device *vdev)
> 
>  static int vfio_platform_get_reset(struct vfio_platform_device *vdev)
>  {
> +	struct vfio_platform_reset_node *node = NULL;
> +
>  	if (VFIO_PLATFORM_IS_ACPI(vdev))
>  		return vfio_platform_acpi_has_reset(vdev) ? 0 : -ENOENT;
>  
> -	vdev->of_reset = vfio_platform_lookup_reset(vdev->compat,
> -						    &vdev->reset_module);
> -	if (!vdev->of_reset) {
> +	vfio_platform_lookup_reset(vdev->compat, &vdev->reset_module,
> +				   &node);
> +	if (!node) {
>  		request_module("vfio-reset:%s", vdev->compat);
> -		vdev->of_reset = vfio_platform_lookup_reset(vdev->compat,
> -							&vdev->reset_module);
> +		vfio_platform_lookup_reset(vdev->compat, &vdev->reset_module,
> +					   &node);
> +	}
> +
> +	if (node) {
> +		vdev->of_reset = node->of_reset;
> +		vdev->of_get_msi = node->of_get_msi;
> +		vdev->of_msi_write = node->of_msi_write;>  	}
>  
>  	return vdev->of_reset ? 0 : -ENOENT;
> @@ -343,9 +351,16 @@ static long vfio_platform_ioctl(void *device_data,
>  
>  	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
>  		struct vfio_irq_info info;
> +		struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
> +		int ext_irq_index = vdev->num_irqs - vdev->num_ext_irqs;
> +		unsigned long capsz;
> +		u32 index;
>  
>  		minsz = offsetofend(struct vfio_irq_info, count);
>  
> +		/* For backward compatibility, cannot require this */
> +		capsz = offsetofend(struct vfio_irq_info, cap_offset);
> +
>  		if (copy_from_user(&info, (void __user *)arg, minsz))
>  			return -EFAULT;
>  
> @@ -355,8 +370,53 @@ static long vfio_platform_ioctl(void *device_data,
>  		if (info.index >= vdev->num_irqs)
>  			return -EINVAL;
>  
> -		info.flags = vdev->irqs[info.index].flags;
> -		info.count = vdev->irqs[info.index].count;
> +		if (info.argsz >= capsz)
> +			minsz = capsz;
> +
> +		index = info.index;
> +
> +		info.flags = vdev->irqs[index].flags;
> +		info.count = vdev->irqs[index].count;
> +
> +		if (ext_irq_index - index == VFIO_EXT_IRQ_MSI) {
> +			struct vfio_irq_info_cap_type cap_type = {
> +				.header.id = VFIO_IRQ_INFO_CAP_TYPE,
> +				.header.version = 1 };
> +			struct vfio_platform_irq *irq;
> +			int ret;
> +
> +			index = array_index_nospec(index,
> +						   vdev->num_irqs);
> +			irq = &vdev->irqs[index];
> +
> +			cap_type.type = irq->type;
> +			cap_type.subtype = irq->subtype;
> +
> +			ret = vfio_info_add_capability(&caps,
> +						       &cap_type.header,
> +						       sizeof(cap_type));
> +			if (ret)
> +				return ret;
> +		}
> +
> +		if (caps.size) {
> +			info.flags |= VFIO_IRQ_INFO_FLAG_CAPS;
> +			if (info.argsz < sizeof(info) + caps.size) {
> +				info.argsz = sizeof(info) + caps.size;
> +				info.cap_offset = 0;
> +			} else {
> +				vfio_info_cap_shift(&caps, sizeof(info));
> +				if (copy_to_user((void __user *)arg +
> +						  sizeof(info), caps.buf,
> +						  caps.size)) {
> +					kfree(caps.buf);
> +					return -EFAULT;
> +				}
> +				info.cap_offset = sizeof(info);
> +			}
> +
> +			kfree(caps.buf);
> +		}
>  
>  		return copy_to_user((void __user *)arg, &info, minsz) ?
>  			-EFAULT : 0;
> @@ -365,6 +425,7 @@ static long vfio_platform_ioctl(void *device_data,
>  		struct vfio_irq_set hdr;
>  		u8 *data = NULL;
>  		int ret = 0;
> +		int max;
>  		size_t data_size = 0;
>  
>  		minsz = offsetofend(struct vfio_irq_set, count);
> @@ -372,8 +433,14 @@ static long vfio_platform_ioctl(void *device_data,
>  		if (copy_from_user(&hdr, (void __user *)arg, minsz))
>  			return -EFAULT;
>  
> -		ret = vfio_set_irqs_validate_and_prepare(&hdr, vdev->num_irqs,
> -						 vdev->num_irqs, &data_size);
> +		if (hdr.index >= vdev->num_irqs)
> +			return -EINVAL;
> +
> +		max = vdev->irqs[hdr.index].count;
> +
> +		ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
> +							 vdev->num_irqs,
> +							 &data_size);
>  		if (ret)
>  			return ret;
>  
> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> index c5b09ec0a3c9..db60240d27ca 100644
> --- a/drivers/vfio/platform/vfio_platform_irq.c
> +++ b/drivers/vfio/platform/vfio_platform_irq.c
> @@ -8,10 +8,12 @@
>  
>  #include <linux/eventfd.h>
>  #include <linux/interrupt.h>
> +#include <linux/eventfd.h>
>  #include <linux/slab.h>
>  #include <linux/types.h>
>  #include <linux/vfio.h>
>  #include <linux/irq.h>
> +#include <linux/msi.h>
>  
>  #include "vfio_platform_private.h"
>  
> @@ -253,6 +255,186 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>  	return 0;
>  }
>  
> +/* MSI/MSIX */
> +static irqreturn_t vfio_msihandler(int irq, void *arg)
> +{
> +	struct eventfd_ctx *trigger = arg;
> +
> +	eventfd_signal(trigger, 1);
> +	return IRQ_HANDLED;
> +}
> +
> +static void msi_write(struct msi_desc *desc, struct msi_msg *msg)
> +{
> +	struct device *dev = msi_desc_to_dev(desc);
> +	struct vfio_device *device = dev_get_drvdata(dev);
> +	struct vfio_platform_device *vdev = (struct vfio_platform_device *)
> +						vfio_device_data(device);
> +
> +	vdev->of_msi_write(vdev, desc, msg);
> +}
> +
> +static int vfio_msi_enable(struct vfio_platform_device *vdev,
> +			   struct vfio_platform_irq *irq, int nvec)
> +{
> +	int ret;
> +	int msi_idx = 0;
> +	struct msi_desc *desc;
> +	struct device *dev = vdev->device;
> +
> +	irq->ctx = kcalloc(nvec, sizeof(struct vfio_irq_ctx), GFP_KERNEL);
> +	if (!irq->ctx)
> +		return -ENOMEM;
> +
> +	/* Allocate platform MSIs */
> +	ret = platform_msi_domain_alloc_irqs(dev, nvec, msi_write);
> +	if (ret < 0) {
> +		kfree(irq->ctx);
> +		return ret;
> +	}
> +
> +	for_each_msi_entry(desc, dev) {
> +		irq->ctx[msi_idx].hwirq = desc->irq;
> +		msi_idx++;
> +	}
> +
> +	irq->num_ctx = nvec;
> +	irq->config_msi = 1;
> +
> +	return 0;
> +}
> +
> +static int vfio_msi_set_vector_signal(struct vfio_platform_irq *irq,
> +				      int vector, int fd)
> +{
> +	struct eventfd_ctx *trigger;
> +	int irq_num, ret;
> +
> +	if (vector < 0 || vector >= irq->num_ctx)
> +		return -EINVAL;
> +
> +	irq_num = irq->ctx[vector].hwirq;
> +
> +	if (irq->ctx[vector].trigger) {
> +		free_irq(irq_num, irq->ctx[vector].trigger);
> +		kfree(irq->ctx[vector].name);
> +		eventfd_ctx_put(irq->ctx[vector].trigger);
> +		irq->ctx[vector].trigger = NULL;
> +	}
> +
> +	if (fd < 0)
> +		return 0;
> +
> +	irq->ctx[vector].name = kasprintf(GFP_KERNEL,
> +					  "vfio-msi[%d]", vector);
> +	if (!irq->ctx[vector].name)
> +		return -ENOMEM;
> +
> +	trigger = eventfd_ctx_fdget(fd);
> +	if (IS_ERR(trigger)) {
> +		kfree(irq->ctx[vector].name);
> +		return PTR_ERR(trigger);
> +	}
> +
> +	ret = request_irq(irq_num, vfio_msihandler, 0,
> +			  irq->ctx[vector].name, trigger);
> +	if (ret) {
> +		kfree(irq->ctx[vector].name);
> +		eventfd_ctx_put(trigger);
> +		return ret;
> +	}
> +
> +	irq->ctx[vector].trigger = trigger;
> +
> +	return 0;
> +}
> +
> +static int vfio_msi_set_block(struct vfio_platform_irq *irq, unsigned int start,
> +			      unsigned int count, int32_t *fds)
> +{
> +	int i, j, ret = 0;
> +
> +	if (start >= irq->num_ctx || start + count > irq->num_ctx)
> +		return -EINVAL;
> +
> +	for (i = 0, j = start; i < count && !ret; i++, j++) {
> +		int fd = fds ? fds[i] : -1;
> +
> +		ret = vfio_msi_set_vector_signal(irq, j, fd);
> +	}
> +
> +	if (ret) {
> +		for (--j; j >= (int)start; j--)
> +			vfio_msi_set_vector_signal(irq, j, -1);
> +	}
> +
> +	return ret;
> +}
> +
> +static void vfio_msi_disable(struct vfio_platform_device *vdev,
> +			     struct vfio_platform_irq *irq)
> +{
> +	struct device *dev = vdev->device;
> +
> +	vfio_msi_set_block(irq, 0, irq->num_ctx, NULL);
> +
> +	platform_msi_domain_free_irqs(dev);
> +
> +	irq->config_msi = 0;
> +	irq->num_ctx = 0;
> +
> +	kfree(irq->ctx);
> +}
> +
> +static int vfio_set_msi_trigger(struct vfio_platform_device *vdev,
> +				unsigned int index, unsigned int start,
> +				unsigned int count, uint32_t flags, void *data)
> +{
> +	int i;
> +	struct vfio_platform_irq *irq = &vdev->irqs[index];
> +
> +	if (start + count > irq->count)
> +		return -EINVAL;
> +
> +	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
> +		vfio_msi_disable(vdev, irq);
> +		return 0;
> +	}
> +
> +	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
> +		s32 *fds = data;
> +		int ret;
> +
> +		if (irq->config_msi)
> +			return vfio_msi_set_block(irq, start, count,
> +						  fds);
looks a bit weird to me that in case of failure we do not do the same as
below (ie. vfio_msi_disable). The pci code does the same but I fail to
understand the logic.
> +		ret = vfio_msi_enable(vdev, irq, start + count);
> +		if (ret)
> +			return ret;
> +
> +		ret = vfio_msi_set_block(irq, start, count, fds);
> +		if (ret)
> +			vfio_msi_disable(vdev, irq);
> +
> +		return ret;
> +	}
> +
> +	for (i = start; i < start + count; i++) {
> +		if (!irq->ctx[i].trigger)
> +			continue;
> +		if (flags & VFIO_IRQ_SET_DATA_NONE) {
> +			eventfd_signal(irq->ctx[i].trigger, 1);
> +		} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> +			u8 *bools = data;
> +
> +			if (bools[i - start])
> +				eventfd_signal(irq->ctx[i].trigger, 1);
> +		}
> +	}
> +
> +	return 0;
> +}
> +
>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  				 uint32_t flags, unsigned index, unsigned start,
>  				 unsigned count, void *data)
> @@ -261,16 +443,29 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  		    unsigned start, unsigned count, uint32_t flags,
>  		    void *data) = NULL;
>  
> -	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> -	case VFIO_IRQ_SET_ACTION_MASK:
> -		func = vfio_platform_set_irq_mask;
> -		break;
> -	case VFIO_IRQ_SET_ACTION_UNMASK:
> -		func = vfio_platform_set_irq_unmask;
> -		break;
> -	case VFIO_IRQ_SET_ACTION_TRIGGER:
> -		func = vfio_platform_set_irq_trigger;
> -		break;
> +	struct vfio_platform_irq *irq = &vdev->irqs[index];
> +
> +	if (irq->type == VFIO_IRQ_TYPE_MSI) {
> +		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> +		case VFIO_IRQ_SET_ACTION_MASK:
> +		case VFIO_IRQ_SET_ACTION_UNMASK:
> +			break;
> +		case VFIO_IRQ_SET_ACTION_TRIGGER:
> +			func = vfio_set_msi_trigger;
> +			break;
> +		}
> +	} else {
> +		switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
> +		case VFIO_IRQ_SET_ACTION_MASK:
> +			func = vfio_platform_set_irq_mask;
> +			break;
> +		case VFIO_IRQ_SET_ACTION_UNMASK:
> +			func = vfio_platform_set_irq_unmask;
> +			break;
> +		case VFIO_IRQ_SET_ACTION_TRIGGER:
> +			func = vfio_platform_set_irq_trigger;
> +			break;
> +		}
>  	}
>  
>  	if (!func)
> @@ -281,12 +476,23 @@ int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>  
>  int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>  {
> -	int cnt = 0, i;
> +	int i;
> +	int cnt = 0;
> +	int num_irqs = 0;
> +	int msi_cnt = 0;
>  
>  	while (vdev->get_irq(vdev, cnt) >= 0)
>  		cnt++;
>  
> -	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
> +	num_irqs = cnt;
> +
> +	if (vdev->of_get_msi) {
I think you also need to test that of_msi_write also is non null?
> +		msi_cnt = vdev->of_get_msi(vdev);
> +		num_irqs++;
> +	}
> +
> +	vdev->irqs = kcalloc(num_irqs, sizeof(struct vfio_platform_irq),
> +			     GFP_KERNEL);
>  	if (!vdev->irqs)
>  		return -ENOMEM;
>  
> @@ -309,7 +515,19 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>  		vdev->irqs[i].masked = false;
>  	}
>  
> -	vdev->num_irqs = cnt;
> +	/*
> +	 * MSI block is added at last index and it is an ext irq
> +	 */
> +	if (msi_cnt > 0) {
> +		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
> +		vdev->irqs[i].count = msi_cnt;
> +		vdev->irqs[i].hwirq = 0;
> +		vdev->irqs[i].masked = false;
> +		vdev->irqs[i].type = VFIO_IRQ_TYPE_MSI;
> +		vdev->num_ext_irqs = 1;
> +	}
> +
> +	vdev->num_irqs = num_irqs;
>  
>  	return 0;
>  err:
> @@ -321,8 +539,13 @@ void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>  {
>  	int i;
>  
> -	for (i = 0; i < vdev->num_irqs; i++)
> -		vfio_set_trigger(vdev, i, -1, NULL);
> +	for (i = 0; i < vdev->num_irqs; i++) {
> +		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> +			vfio_set_msi_trigger(vdev, i, 0, 0,
> +					     VFIO_IRQ_SET_DATA_NONE, NULL);
> +		else
> +			vfio_set_trigger(vdev, i, -1, NULL);
> +	}
>  
>  	vdev->num_irqs = 0;
>  	kfree(vdev->irqs);
> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> index 289089910643..b8dd892aec53 100644
> --- a/drivers/vfio/platform/vfio_platform_private.h
> +++ b/drivers/vfio/platform/vfio_platform_private.h
> @@ -9,6 +9,7 @@
>  
>  #include <linux/types.h>
>  #include <linux/interrupt.h>
> +#include <linux/msi.h>
>  
>  #define VFIO_PLATFORM_OFFSET_SHIFT   40
>  #define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
> @@ -19,9 +20,21 @@
>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
>  	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>  
> +/* IRQ index for MSI in ext IRQs */
> +#define VFIO_EXT_IRQ_MSI	0
> +
> +struct vfio_irq_ctx {
> +	int			hwirq;
> +	char			*name;
> +	struct msi_msg		msg;
> +	struct eventfd_ctx	*trigger;
> +};
> +
>  struct vfio_platform_irq {
>  	u32			flags;
>  	u32			count;
> +	int			num_ctx;
> +	struct vfio_irq_ctx	*ctx;
>  	int			hwirq;
>  	char			*name;
>  	struct eventfd_ctx	*trigger;
> @@ -29,6 +42,11 @@ struct vfio_platform_irq {
>  	spinlock_t		lock;
>  	struct virqfd		*unmask;
>  	struct virqfd		*mask;
> +
> +	/* for extended irqs */
> +	u32			type;
> +	u32			subtype;
> +	int			config_msi;
bool?
>  };
>  
>  struct vfio_platform_region {
> @@ -46,6 +64,7 @@ struct vfio_platform_device {
>  	u32				num_regions;
>  	struct vfio_platform_irq	*irqs;
>  	u32				num_irqs;
> +	int				num_ext_irqs;
>  	int				refcnt;
>  	struct mutex			igate;
>  	struct module			*parent_module;
> @@ -65,17 +84,27 @@ struct vfio_platform_device {
>  		(*get_resource)(struct vfio_platform_device *vdev, int i);
>  	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
>  	int	(*of_reset)(struct vfio_platform_device *vdev);
> +	u32	(*of_get_msi)(struct vfio_platform_device *vdev);
> +	void	(*of_msi_write)(struct vfio_platform_device *vdev,
> +				struct msi_desc *desc,
> +				struct msi_msg *msg);
>  
>  	bool				reset_required;
>  };
>  
>  typedef int (*vfio_platform_reset_fn_t)(struct vfio_platform_device *vdev);
> +typedef u32 (*vfio_platform_get_msi_fn_t)(struct vfio_platform_device *vdev);
> +typedef void (*vfio_platform_msi_write_fn_t)(struct vfio_platform_device *vdev,
> +					     struct msi_desc *desc,
> +					     struct msi_msg *msg);
>  
>  struct vfio_platform_reset_node {
>  	struct list_head link;
>  	char *compat;
>  	struct module *owner;
>  	vfio_platform_reset_fn_t of_reset;
> +	vfio_platform_get_msi_fn_t of_get_msi;
> +	vfio_platform_msi_write_fn_t of_msi_write;
>  };
>  
>  extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index d1812777139f..8e2c0131781d 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -697,11 +697,35 @@ struct vfio_irq_info {
>  #define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
>  #define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
>  #define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
> +#define VFIO_IRQ_INFO_FLAG_CAPS		(1 << 4) /* Info supports caps */
>  	__u32	index;		/* IRQ index */
>  	__u32	count;		/* Number of IRQs within this index */
> +	__u32	cap_offset;	/* Offset within info struct of first cap */
>  };
>  #define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
>  
> +/*
> + * The irq type capability allows IRQs unique to a specific device or
> + * class of devices to be exposed.
> + *
> + * The structures below define version 1 of this capability.
> + */
> +#define VFIO_IRQ_INFO_CAP_TYPE		3
> +
> +struct vfio_irq_info_cap_type {
> +	struct vfio_info_cap_header header;
> +	__u32 type;     /* global per bus driver */
> +	__u32 subtype;  /* type specific */
> +};
> +
> +/*
> + * List of IRQ types, global per bus driver.
> + * If you introduce a new type, please add it here.
> + */
> +
> +/* Non PCI devices having MSI(s) support */
> +#define VFIO_IRQ_TYPE_MSI		(1)
> +
>  /**
>   * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
>   *
> 
Besides looks good to me.

Thanks

Eric

[RFC v4 2/3] vfio/platform: change cleanup order

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 846 bytes --]

In the case of msi, vendor specific msi module may require
region access to handle msi cleanup so we need to cleanup region
after irq cleanup only.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 drivers/vfio/platform/vfio_platform_common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index f2b1f0c3bfcc..1cc040e3ed1f 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -243,8 +243,8 @@ static void vfio_platform_release(void *device_data)
 			WARN_ON(1);
 		}
 		pm_runtime_put(vdev->device);
-		vfio_platform_regions_cleanup(vdev);
 		vfio_platform_irq_cleanup(vdev);
+		vfio_platform_regions_cleanup(vdev);
 	}
 
 	mutex_unlock(&driver_lock);
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v4 2/3] vfio/platform: change cleanup order

[Auger Eric]

Hi Vikas,

On 1/29/21 6:24 PM, Vikas Gupta wrote:
> In the case of msi, vendor specific msi module may require
> region access to handle msi cleanup so we need to cleanup region
> after irq cleanup only.
> 
> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
Acked-by: Eric Auger <eric.auger@redhat.com>

Thanks

Eric

> ---
>  drivers/vfio/platform/vfio_platform_common.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> index f2b1f0c3bfcc..1cc040e3ed1f 100644
> --- a/drivers/vfio/platform/vfio_platform_common.c
> +++ b/drivers/vfio/platform/vfio_platform_common.c
> @@ -243,8 +243,8 @@ static void vfio_platform_release(void *device_data)
>  			WARN_ON(1);
>  		}
>  		pm_runtime_put(vdev->device);
> -		vfio_platform_regions_cleanup(vdev);
>  		vfio_platform_irq_cleanup(vdev);
> +		vfio_platform_regions_cleanup(vdev);
>  	}
>  
>  	mutex_unlock(&driver_lock);
> 

[RFC v4 3/3] vfio: platform: reset: add msi support

[Vikas Gupta]

[-- Attachment #1: Type: text/plain, Size: 2984 bytes --]

Add msi support for Broadcom FlexRm device.

Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
---
 .../platform/reset/vfio_platform_bcmflexrm.c  | 72 ++++++++++++++++++-
 1 file changed, 70 insertions(+), 2 deletions(-)

diff --git a/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c b/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c
index 96064ef8f629..6ca4ca12575b 100644
--- a/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c
+++ b/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c
@@ -21,7 +21,9 @@
 #include <linux/init.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/msi.h>
 #include <linux/module.h>
+#include <linux/vfio.h>
 
 #include "../vfio_platform_private.h"
 
@@ -33,6 +35,9 @@
 #define RING_VER					0x000
 #define RING_CONTROL					0x034
 #define RING_FLUSH_DONE					0x038
+#define RING_MSI_ADDR_LS				0x03c
+#define RING_MSI_ADDR_MS				0x040
+#define RING_MSI_DATA_VALUE				0x064
 
 /* Register RING_CONTROL fields */
 #define CONTROL_FLUSH_SHIFT				5
@@ -105,8 +110,71 @@ static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev)
 	return ret;
 }
 
-module_vfio_reset_handler("brcm,iproc-flexrm-mbox",
-			  vfio_platform_bcmflexrm_reset);
+static u32 bcm_num_msi(struct vfio_platform_device *vdev)
+{
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	return (reg->size / RING_REGS_SIZE);
+}
+
+static void bcm_write_msi(struct vfio_platform_device *vdev,
+		struct msi_desc *desc,
+		struct msi_msg *msg)
+{
+	int i;
+	int hwirq = -1;
+	int msi_src;
+	void __iomem *ring;
+	struct vfio_platform_region *reg = &vdev->regions[0];
+
+	if (!reg)
+		return;
+
+	for (i = 0; i < vdev->num_irqs; i++)
+		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
+			hwirq = vdev->irqs[i].ctx[0].hwirq;
+
+	if (hwirq < 0)
+		return;
+
+	msi_src = desc->irq - hwirq;
+
+	if (!reg->ioaddr) {
+		reg->ioaddr = ioremap(reg->addr, reg->size);
+		if (!reg->ioaddr)
+			return;
+	}
+
+	ring = reg->ioaddr + msi_src * RING_REGS_SIZE;
+
+	writel_relaxed(msg->address_lo, ring + RING_MSI_ADDR_LS);
+	writel_relaxed(msg->address_hi, ring + RING_MSI_ADDR_MS);
+	writel_relaxed(msg->data, ring + RING_MSI_DATA_VALUE);
+}
+
+static struct vfio_platform_reset_node vfio_platform_bcmflexrm_reset_node = {
+	.owner = THIS_MODULE,
+	.compat = "brcm,iproc-flexrm-mbox",
+	.of_reset = vfio_platform_bcmflexrm_reset,
+	.of_get_msi = bcm_num_msi,
+	.of_msi_write = bcm_write_msi
+};
+
+static int __init vfio_platform_bcmflexrm_reset_module_init(void)
+{
+	__vfio_platform_register_reset(&vfio_platform_bcmflexrm_reset_node);
+
+	return 0;
+}
+
+static void __exit vfio_platform_bcmflexrm_reset_module_exit(void)
+{
+	vfio_platform_unregister_reset("brcm,iproc-flexrm-mbox",
+				       vfio_platform_bcmflexrm_reset);
+}
+
+module_init(vfio_platform_bcmflexrm_reset_module_init);
+module_exit(vfio_platform_bcmflexrm_reset_module_exit);
 
 MODULE_LICENSE("GPL v2");
 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
-- 
2.17.1


[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4163 bytes --]

Re: [RFC v4 3/3] vfio: platform: reset: add msi support

[Auger Eric]

Hi Vikas,

On 1/29/21 6:24 PM, Vikas Gupta wrote:
> Add msi support for Broadcom FlexRm device.
> 
> Signed-off-by: Vikas Gupta <vikas.gupta@broadcom.com>
> ---
>  .../platform/reset/vfio_platform_bcmflexrm.c  | 72 ++++++++++++++++++-
>  1 file changed, 70 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c b/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c
> index 96064ef8f629..6ca4ca12575b 100644
> --- a/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c
> +++ b/drivers/vfio/platform/reset/vfio_platform_bcmflexrm.c
> @@ -21,7 +21,9 @@
>  #include <linux/init.h>
>  #include <linux/io.h>
>  #include <linux/kernel.h>
> +#include <linux/msi.h>
>  #include <linux/module.h>
> +#include <linux/vfio.h>
>  
>  #include "../vfio_platform_private.h"
>  
> @@ -33,6 +35,9 @@
>  #define RING_VER					0x000
>  #define RING_CONTROL					0x034
>  #define RING_FLUSH_DONE					0x038
> +#define RING_MSI_ADDR_LS				0x03c
> +#define RING_MSI_ADDR_MS				0x040
> +#define RING_MSI_DATA_VALUE				0x064
>  
>  /* Register RING_CONTROL fields */
>  #define CONTROL_FLUSH_SHIFT				5
> @@ -105,8 +110,71 @@ static int vfio_platform_bcmflexrm_reset(struct vfio_platform_device *vdev)
>  	return ret;
>  }
>  
> -module_vfio_reset_handler("brcm,iproc-flexrm-mbox",
> -			  vfio_platform_bcmflexrm_reset);
> +static u32 bcm_num_msi(struct vfio_platform_device *vdev)
Please use the same prefix as for reset, ie. vfio_platform_bcmflexrm_get_msi
> +{
> +	struct vfio_platform_region *reg = &vdev->regions[0];> +
> +	return (reg->size / RING_REGS_SIZE);
> +}
> +
> +static void bcm_write_msi(struct vfio_platform_device *vdev,
vfio_platform_bcmflexrm_msi_write?
> +		struct msi_desc *desc,
> +		struct msi_msg *msg)
> +{
> +	int i;
> +	int hwirq = -1;
> +	int msi_src;
> +	void __iomem *ring;
> +	struct vfio_platform_region *reg = &vdev->regions[0];
> +
> +	if (!reg)
> +		return;
why do you need this check? For this to be called, SET_IRQ IOCTL must
have been called. This can only happen after vfio_platform_open which
first calls vfio_platform_regions_init and then vfio_platform_irq_init
> +
> +	for (i = 0; i < vdev->num_irqs; i++)
> +		if (vdev->irqs[i].type == VFIO_IRQ_TYPE_MSI)
> +			hwirq = vdev->irqs[i].ctx[0].hwirq;
nit: It would have been easier to record in vdev the last index of wired
interrupts and just add the index of the MSI
> +
> +	if (hwirq < 0)
> +		return;
> +
> +	msi_src = desc->irq - hwirq;
> +
> +	if (!reg->ioaddr) {
> +		reg->ioaddr = ioremap(reg->addr, reg->size);
> +		if (!reg->ioaddr)
pr_warn_once("")?
> +			return;
> +	}
> +
> +	ring = reg->ioaddr + msi_src * RING_REGS_SIZE;
> +
> +	writel_relaxed(msg->address_lo, ring + RING_MSI_ADDR_LS);
> +	writel_relaxed(msg->address_hi, ring + RING_MSI_ADDR_MS);
> +	writel_relaxed(msg->data, ring + RING_MSI_DATA_VALUE);
> +}
> +
> +static struct vfio_platform_reset_node vfio_platform_bcmflexrm_reset_node = {
> +	.owner = THIS_MODULE,
> +	.compat = "brcm,iproc-flexrm-mbox",
> +	.of_reset = vfio_platform_bcmflexrm_reset,
> +	.of_get_msi = bcm_num_msi,
> +	.of_msi_write = bcm_write_msi
> +};
> +
> +static int __init vfio_platform_bcmflexrm_reset_module_init(void)
> +{
> +	__vfio_platform_register_reset(&vfio_platform_bcmflexrm_reset_node);
> +
> +	return 0;
> +}
> +
> +static void __exit vfio_platform_bcmflexrm_reset_module_exit(void)
> +{
> +	vfio_platform_unregister_reset("brcm,iproc-flexrm-mbox",
> +				       vfio_platform_bcmflexrm_reset);
> +}
> +
> +module_init(vfio_platform_bcmflexrm_reset_module_init);
> +module_exit(vfio_platform_bcmflexrm_reset_module_exit);
>  
>  MODULE_LICENSE("GPL v2");
>  MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>");
> 

I think you should move the whole series in PATCH now.

Thanks

Eric