* [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to platform devices
@ 2021-08-13 3:56 Barry Song
2021-08-13 3:56 ` [PATCH v3 1/2] genirq/msi: Extract common sysfs populate entries to MSI core from PCI Barry Song
` (2 more replies)
0 siblings, 3 replies; 6+ messages in thread
From: Barry Song @ 2021-08-13 3:56 UTC (permalink / raw)
To: linux-kernel, maz, tglx
Cc: bhelgaas, dwmw, gregkh, linux-pci, linuxarm, lorenzo.pieralisi,
rafael, robin.murphy, song.bao.hua, will
From: Barry Song <song.bao.hua@hisilicon.com>
-v3:
added Acked-by of Greg Kroah-Hartman and Bjorn Helgaas, thanks!
refined commit log with respect to Bjorn's comments
Hi Thomas & Marc,
Would you please handle this series? Thanks!
-v2:
extract common code for msi_irqs sysfs populate/destory from PCI to MSI core,
platform_device can directly reuse common code;
https://lore.kernel.org/lkml/20210812105341.51657-1-21cnbao@gmail.com/
-v1:
https://lore.kernel.org/lkml/20210811105020.12980-1-song.bao.hua@hisilicon.com/
Just like pci devices have msi_irqs which can be used by userspace irq affinity
tools or applications to bind irqs, platform devices also widely support msi
irqs.
For platform devices, for example ARM SMMU, userspaces also care about its msi
irqs as applications can know the mapping between devices and irqs and then
make smarter decision on handling irq affinity. For example, for SVA mode,
it is better to pin io page fault to the numa node applications are running
on. Otherwise, io page fault will get a remote page from the node iopf happens
rather than from the node applications are running on.
The first patch extracts the sysfs populate/destory code from PCI to
MSI core. The 2nd patch lets platform-msi export msi_irqs entry so that
userspace can know the mapping between devices and irqs for platform
devices.
Barry Song (2):
genirq/msi: Extract common sysfs populate entries to MSI core from PCI
platform-msi: Add ABI to show msi_irqs of platform devices
Documentation/ABI/testing/sysfs-bus-platform | 14 +++
drivers/base/platform-msi.c | 10 ++
drivers/pci/msi.c | 124 ++-----------------------
include/linux/msi.h | 4 +
kernel/irq/msi.c | 134 +++++++++++++++++++++++++++
5 files changed, 171 insertions(+), 115 deletions(-)
--
1.8.3.1
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH v3 1/2] genirq/msi: Extract common sysfs populate entries to MSI core from PCI
2021-08-13 3:56 [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to platform devices Barry Song
@ 2021-08-13 3:56 ` Barry Song
2021-08-24 7:20 ` [tip: irq/core] genirq/msi: Move MSI sysfs handling from PCI to MSI core tip-bot2 for Barry Song
2021-08-13 3:56 ` [PATCH v3 2/2] platform-msi: Add ABI to show msi_irqs of platform devices Barry Song
2021-08-14 9:45 ` [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to " Marc Zyngier
2 siblings, 1 reply; 6+ messages in thread
From: Barry Song @ 2021-08-13 3:56 UTC (permalink / raw)
To: linux-kernel, maz, tglx
Cc: bhelgaas, dwmw, gregkh, linux-pci, linuxarm, lorenzo.pieralisi,
rafael, robin.murphy, song.bao.hua, will
From: Barry Song <song.bao.hua@hisilicon.com>
This patch makes PCI's MSI sysfs code common in MSI so that other busses
such as platform can reuse it.
Acked-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Acked-by: Bjorn Helgaas <bhelgaas@google.com>
Signed-off-by: Barry Song <song.bao.hua@hisilicon.com>
---
drivers/pci/msi.c | 124 ++++--------------------------------------------
include/linux/msi.h | 4 ++
kernel/irq/msi.c | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 147 insertions(+), 115 deletions(-)
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index 9232255..4b9c0bb 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -338,9 +338,7 @@ static void free_msi_irqs(struct pci_dev *dev)
{
struct list_head *msi_list = dev_to_msi_list(&dev->dev);
struct msi_desc *entry, *tmp;
- struct attribute **msi_attrs;
- struct device_attribute *dev_attr;
- int i, count = 0;
+ int i;
for_each_pci_msi_entry(entry, dev)
if (entry->irq)
@@ -360,18 +358,7 @@ static void free_msi_irqs(struct pci_dev *dev)
}
if (dev->msi_irq_groups) {
- sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
- msi_attrs = dev->msi_irq_groups[0]->attrs;
- while (msi_attrs[count]) {
- dev_attr = container_of(msi_attrs[count],
- struct device_attribute, attr);
- kfree(dev_attr->attr.name);
- kfree(dev_attr);
- ++count;
- }
- kfree(msi_attrs);
- kfree(dev->msi_irq_groups[0]);
- kfree(dev->msi_irq_groups);
+ msi_destroy_sysfs(&dev->dev, dev->msi_irq_groups);
dev->msi_irq_groups = NULL;
}
}
@@ -452,102 +439,6 @@ void pci_restore_msi_state(struct pci_dev *dev)
}
EXPORT_SYMBOL_GPL(pci_restore_msi_state);
-static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
- char *buf)
-{
- struct msi_desc *entry;
- unsigned long irq;
- int retval;
-
- retval = kstrtoul(attr->attr.name, 10, &irq);
- if (retval)
- return retval;
-
- entry = irq_get_msi_desc(irq);
- if (!entry)
- return -ENODEV;
-
- return sysfs_emit(buf, "%s\n",
- entry->msi_attrib.is_msix ? "msix" : "msi");
-}
-
-static int populate_msi_sysfs(struct pci_dev *pdev)
-{
- struct attribute **msi_attrs;
- struct attribute *msi_attr;
- struct device_attribute *msi_dev_attr;
- struct attribute_group *msi_irq_group;
- const struct attribute_group **msi_irq_groups;
- struct msi_desc *entry;
- int ret = -ENOMEM;
- int num_msi = 0;
- int count = 0;
- int i;
-
- /* Determine how many msi entries we have */
- for_each_pci_msi_entry(entry, pdev)
- num_msi += entry->nvec_used;
- if (!num_msi)
- return 0;
-
- /* Dynamically create the MSI attributes for the PCI device */
- msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL);
- if (!msi_attrs)
- return -ENOMEM;
- for_each_pci_msi_entry(entry, pdev) {
- for (i = 0; i < entry->nvec_used; i++) {
- msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
- if (!msi_dev_attr)
- goto error_attrs;
- msi_attrs[count] = &msi_dev_attr->attr;
-
- sysfs_attr_init(&msi_dev_attr->attr);
- msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
- entry->irq + i);
- if (!msi_dev_attr->attr.name)
- goto error_attrs;
- msi_dev_attr->attr.mode = S_IRUGO;
- msi_dev_attr->show = msi_mode_show;
- ++count;
- }
- }
-
- msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
- if (!msi_irq_group)
- goto error_attrs;
- msi_irq_group->name = "msi_irqs";
- msi_irq_group->attrs = msi_attrs;
-
- msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL);
- if (!msi_irq_groups)
- goto error_irq_group;
- msi_irq_groups[0] = msi_irq_group;
-
- ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
- if (ret)
- goto error_irq_groups;
- pdev->msi_irq_groups = msi_irq_groups;
-
- return 0;
-
-error_irq_groups:
- kfree(msi_irq_groups);
-error_irq_group:
- kfree(msi_irq_group);
-error_attrs:
- count = 0;
- msi_attr = msi_attrs[count];
- while (msi_attr) {
- msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
- kfree(msi_attr->name);
- kfree(msi_dev_attr);
- ++count;
- msi_attr = msi_attrs[count];
- }
- kfree(msi_attrs);
- return ret;
-}
-
static struct msi_desc *
msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
{
@@ -648,8 +539,9 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
return ret;
}
- ret = populate_msi_sysfs(dev);
- if (ret) {
+ dev->msi_irq_groups = msi_populate_sysfs(&dev->dev);
+ if (IS_ERR(dev->msi_irq_groups)) {
+ ret = PTR_ERR(dev->msi_irq_groups);
msi_mask_irq(entry, mask, ~mask);
free_msi_irqs(dev);
return ret;
@@ -804,9 +696,11 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
msix_program_entries(dev, entries);
- ret = populate_msi_sysfs(dev);
- if (ret)
+ dev->msi_irq_groups = msi_populate_sysfs(&dev->dev);
+ if (IS_ERR(dev->msi_irq_groups)) {
+ ret = PTR_ERR(dev->msi_irq_groups);
goto out_free;
+ }
/* Set MSI-X enabled bits and unmask the function */
pci_intx_for_msi(dev, 0);
diff --git a/include/linux/msi.h b/include/linux/msi.h
index 6aff469..375f1f9 100644
--- a/include/linux/msi.h
+++ b/include/linux/msi.h
@@ -237,6 +237,10 @@ struct msi_desc *alloc_msi_entry(struct device *dev, int nvec,
void pci_msi_mask_irq(struct irq_data *data);
void pci_msi_unmask_irq(struct irq_data *data);
+const struct attribute_group **msi_populate_sysfs(struct device *dev);
+void msi_destroy_sysfs(struct device *dev,
+ const struct attribute_group **msi_irq_groups);
+
/*
* The arch hooks to setup up msi irqs. Default functions are implemented
* as weak symbols so that they /can/ be overriden by architecture specific
diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c
index c41965e..f153bca 100644
--- a/kernel/irq/msi.c
+++ b/kernel/irq/msi.c
@@ -14,6 +14,7 @@
#include <linux/irqdomain.h>
#include <linux/msi.h>
#include <linux/slab.h>
+#include <linux/pci.h>
#include "internals.h"
@@ -69,6 +70,139 @@ void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)
}
EXPORT_SYMBOL_GPL(get_cached_msi_msg);
+static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct msi_desc *entry;
+ unsigned long irq;
+ int retval;
+ bool is_msix = false;
+
+ retval = kstrtoul(attr->attr.name, 10, &irq);
+ if (retval)
+ return retval;
+
+ entry = irq_get_msi_desc(irq);
+ if (!entry)
+ return -ENODEV;
+
+ if (dev_is_pci(dev))
+ is_msix = entry->msi_attrib.is_msix;
+
+ return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi");
+}
+
+/**
+ * msi_populate_sysfs - Populate msi_irqs sysfs entries for devices
+ * @dev: The device(PCI, platform etc) who will get sysfs entries
+ *
+ * Return attribute_group ** so that specific bus MSI can save it to
+ * somewhere during initilizing msi irqs. If devices has no MSI irq,
+ * return NULL; if it fails to populate sysfs, return ERR_PTR
+ */
+const struct attribute_group **msi_populate_sysfs(struct device *dev)
+{
+ struct attribute **msi_attrs;
+ struct attribute *msi_attr;
+ struct device_attribute *msi_dev_attr;
+ struct attribute_group *msi_irq_group;
+ const struct attribute_group **msi_irq_groups;
+ struct msi_desc *entry;
+ int ret = -ENOMEM;
+ int num_msi = 0;
+ int count = 0;
+ int i;
+
+ /* Determine how many msi entries we have */
+ for_each_msi_entry(entry, dev)
+ num_msi += entry->nvec_used;
+ if (!num_msi)
+ return NULL;
+
+ /* Dynamically create the MSI attributes for the device */
+ msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL);
+ if (!msi_attrs)
+ return ERR_PTR(-ENOMEM);
+ for_each_msi_entry(entry, dev) {
+ for (i = 0; i < entry->nvec_used; i++) {
+ msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
+ if (!msi_dev_attr)
+ goto error_attrs;
+ msi_attrs[count] = &msi_dev_attr->attr;
+
+ sysfs_attr_init(&msi_dev_attr->attr);
+ msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
+ entry->irq + i);
+ if (!msi_dev_attr->attr.name)
+ goto error_attrs;
+ msi_dev_attr->attr.mode = 0444;
+ msi_dev_attr->show = msi_mode_show;
+ ++count;
+ }
+ }
+
+ msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
+ if (!msi_irq_group)
+ goto error_attrs;
+ msi_irq_group->name = "msi_irqs";
+ msi_irq_group->attrs = msi_attrs;
+
+ msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL);
+ if (!msi_irq_groups)
+ goto error_irq_group;
+ msi_irq_groups[0] = msi_irq_group;
+
+ ret = sysfs_create_groups(&dev->kobj, msi_irq_groups);
+ if (ret)
+ goto error_irq_groups;
+
+ return msi_irq_groups;
+
+error_irq_groups:
+ kfree(msi_irq_groups);
+error_irq_group:
+ kfree(msi_irq_group);
+error_attrs:
+ count = 0;
+ msi_attr = msi_attrs[count];
+ while (msi_attr) {
+ msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
+ kfree(msi_attr->name);
+ kfree(msi_dev_attr);
+ ++count;
+ msi_attr = msi_attrs[count];
+ }
+ kfree(msi_attrs);
+ return ERR_PTR(ret);
+}
+
+/**
+ * msi_destroy_sysfs - Destroy msi_irqs sysfs entries for devices
+ * @dev: The device(PCI, platform etc) who will remove sysfs entries
+ * @msi_irq_groups: attribute_group for device msi_irqs entries
+ */
+void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups)
+{
+ struct attribute **msi_attrs;
+ struct device_attribute *dev_attr;
+ int count = 0;
+
+ if (msi_irq_groups) {
+ sysfs_remove_groups(&dev->kobj, msi_irq_groups);
+ msi_attrs = msi_irq_groups[0]->attrs;
+ while (msi_attrs[count]) {
+ dev_attr = container_of(msi_attrs[count],
+ struct device_attribute, attr);
+ kfree(dev_attr->attr.name);
+ kfree(dev_attr);
+ ++count;
+ }
+ kfree(msi_attrs);
+ kfree(msi_irq_groups[0]);
+ kfree(msi_irq_groups);
+ }
+}
+
#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
static inline void irq_chip_write_msi_msg(struct irq_data *data,
struct msi_msg *msg)
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [PATCH v3 2/2] platform-msi: Add ABI to show msi_irqs of platform devices
2021-08-13 3:56 [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to platform devices Barry Song
2021-08-13 3:56 ` [PATCH v3 1/2] genirq/msi: Extract common sysfs populate entries to MSI core from PCI Barry Song
@ 2021-08-13 3:56 ` Barry Song
2021-08-24 7:20 ` [tip: irq/core] " tip-bot2 for Barry Song
2021-08-14 9:45 ` [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to " Marc Zyngier
2 siblings, 1 reply; 6+ messages in thread
From: Barry Song @ 2021-08-13 3:56 UTC (permalink / raw)
To: linux-kernel, maz, tglx
Cc: bhelgaas, dwmw, gregkh, linux-pci, linuxarm, lorenzo.pieralisi,
rafael, robin.murphy, song.bao.hua, will, Zhou Wang
From: Barry Song <song.bao.hua@hisilicon.com>
Just like PCI devices have msi_irqs which can be used by userspace IRQ
affinity tools or applications to bind IRQs, platform devices also widely
support MSI IRQs. For some platform devices such as ARM SMMU, userspaces
also care about its MSI IRQs as applications can know the mapping between
devices and IRQs and then make smarter decision on handling IRQ affinity.
For example, in SVA mode, it is better to pin I/O page fault to the NUMA
node applications are running on. Otherwise, I/O page fault will get a
remote page from the node IOPF happens.
With this patch, a system with multiple ARM SMMUs in multiple different
NUMA nodes can get the mapping between devices and IRQs now:
root@ubuntu:/sys/devices/platform# ls -l arm-smmu-v3.*/msi_irqs/*
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.0.auto/msi_irqs/25
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.0.auto/msi_irqs/26
-r--r--r-- 1 root root 4096 Aug 11 10:28 arm-smmu-v3.1.auto/msi_irqs/27
-r--r--r-- 1 root root 4096 Aug 11 10:28 arm-smmu-v3.1.auto/msi_irqs/28
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.2.auto/msi_irqs/29
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.2.auto/msi_irqs/30
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.3.auto/msi_irqs/31
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.3.auto/msi_irqs/32
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.4.auto/msi_irqs/33
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.4.auto/msi_irqs/34
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.5.auto/msi_irqs/35
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.5.auto/msi_irqs/36
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.6.auto/msi_irqs/37
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.6.auto/msi_irqs/38
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.7.auto/msi_irqs/39
-r--r--r-- 1 root root 4096 Aug 11 10:29 arm-smmu-v3.7.auto/msi_irqs/40
Applications can use the mapping and the NUMA node information to pin
IRQs by further leveraging the numa information which has also been
exported:
root@ubuntu:/sys/devices/platform# cat arm-smmu-v3.0.auto/numa_node
0
root@ubuntu:/sys/devices/platform# cat arm-smmu-v3.4.auto/numa_node
2
Acked-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Acked-by: Bjorn Helgaas <bhelgaas@google.com>
Cc: Zhou Wang <wangzhou1@hisilicon.com>
Signed-off-by: Barry Song <song.bao.hua@hisilicon.com>
---
Documentation/ABI/testing/sysfs-bus-platform | 14 ++++++++++++++
drivers/base/platform-msi.c | 10 ++++++++++
2 files changed, 24 insertions(+)
diff --git a/Documentation/ABI/testing/sysfs-bus-platform b/Documentation/ABI/testing/sysfs-bus-platform
index 194ca70..ff30728 100644
--- a/Documentation/ABI/testing/sysfs-bus-platform
+++ b/Documentation/ABI/testing/sysfs-bus-platform
@@ -28,3 +28,17 @@ Description:
value comes from an ACPI _PXM method or a similar firmware
source. Initial users for this file would be devices like
arm smmu which are populated by arm64 acpi_iort.
+
+What: /sys/bus/platform/devices/.../msi_irqs/
+Date: August 2021
+Contact: Barry Song <song.bao.hua@hisilicon.com>
+Description:
+ The /sys/devices/.../msi_irqs directory contains a variable set
+ of files, with each file being named after a corresponding msi
+ irq vector allocated to that device.
+
+What: /sys/bus/platform/devices/.../msi_irqs/<N>
+Date: August 2021
+Contact: Barry Song <song.bao.hua@hisilicon.com>
+Description:
+ This attribute will show "msi" if <N> is a valid msi irq
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index 0b72b13..a3bf910 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -23,6 +23,7 @@
struct platform_msi_priv_data {
struct device *dev;
void *host_data;
+ const struct attribute_group **msi_irq_groups;
msi_alloc_info_t arg;
irq_write_msi_msg_t write_msg;
int devid;
@@ -272,8 +273,16 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec,
if (err)
goto out_free_desc;
+ priv_data->msi_irq_groups = msi_populate_sysfs(dev);
+ if (IS_ERR(priv_data->msi_irq_groups)) {
+ err = PTR_ERR(priv_data->msi_irq_groups);
+ goto out_free_irqs;
+ }
+
return 0;
+out_free_irqs:
+ msi_domain_free_irqs(dev->msi_domain, dev);
out_free_desc:
platform_msi_free_descs(dev, 0, nvec);
out_free_priv_data:
@@ -293,6 +302,7 @@ void platform_msi_domain_free_irqs(struct device *dev)
struct msi_desc *desc;
desc = first_msi_entry(dev);
+ msi_destroy_sysfs(dev, desc->platform.msi_priv_data->msi_irq_groups);
platform_msi_free_priv_data(desc->platform.msi_priv_data);
}
--
1.8.3.1
^ permalink raw reply related [flat|nested] 6+ messages in thread
* Re: [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to platform devices
2021-08-13 3:56 [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to platform devices Barry Song
2021-08-13 3:56 ` [PATCH v3 1/2] genirq/msi: Extract common sysfs populate entries to MSI core from PCI Barry Song
2021-08-13 3:56 ` [PATCH v3 2/2] platform-msi: Add ABI to show msi_irqs of platform devices Barry Song
@ 2021-08-14 9:45 ` Marc Zyngier
2 siblings, 0 replies; 6+ messages in thread
From: Marc Zyngier @ 2021-08-14 9:45 UTC (permalink / raw)
To: Barry Song
Cc: linux-kernel, tglx, bhelgaas, dwmw, gregkh, linux-pci, linuxarm,
lorenzo.pieralisi, rafael, robin.murphy, song.bao.hua, will
On Fri, 13 Aug 2021 04:56:26 +0100,
Barry Song <21cnbao@gmail.com> wrote:
>
> From: Barry Song <song.bao.hua@hisilicon.com>
>
> -v3:
> added Acked-by of Greg Kroah-Hartman and Bjorn Helgaas, thanks!
> refined commit log with respect to Bjorn's comments
>
> Hi Thomas & Marc,
> Would you please handle this series? Thanks!
>
> -v2:
> extract common code for msi_irqs sysfs populate/destory from PCI to MSI core,
> platform_device can directly reuse common code;
> https://lore.kernel.org/lkml/20210812105341.51657-1-21cnbao@gmail.com/
>
> -v1:
> https://lore.kernel.org/lkml/20210811105020.12980-1-song.bao.hua@hisilicon.com/
>
> Just like pci devices have msi_irqs which can be used by userspace irq affinity
> tools or applications to bind irqs, platform devices also widely support msi
> irqs.
> For platform devices, for example ARM SMMU, userspaces also care about its msi
> irqs as applications can know the mapping between devices and irqs and then
> make smarter decision on handling irq affinity. For example, for SVA mode,
> it is better to pin io page fault to the numa node applications are running
> on. Otherwise, io page fault will get a remote page from the node iopf happens
> rather than from the node applications are running on.
>
> The first patch extracts the sysfs populate/destory code from PCI to
> MSI core. The 2nd patch lets platform-msi export msi_irqs entry so that
> userspace can know the mapping between devices and irqs for platform
> devices.
Although I'm not in a position to test it right now, this looks
reasonable to me:
Acked-by: Marc Zyngier <maz@kernel.org>
Thomas, do you want to queue this one, given that you already carry a
ton of MSI-related patches?
Thanks,
M.
--
Without deviation from the norm, progress is not possible.
^ permalink raw reply [flat|nested] 6+ messages in thread
* [tip: irq/core] platform-msi: Add ABI to show msi_irqs of platform devices
2021-08-13 3:56 ` [PATCH v3 2/2] platform-msi: Add ABI to show msi_irqs of platform devices Barry Song
@ 2021-08-24 7:20 ` tip-bot2 for Barry Song
0 siblings, 0 replies; 6+ messages in thread
From: tip-bot2 for Barry Song @ 2021-08-24 7:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Barry Song, Thomas Gleixner, Greg Kroah-Hartman, Bjorn Helgaas,
Marc Zyngier, x86, linux-kernel
The following commit has been merged into the irq/core branch of tip:
Commit-ID: 00ed1401a0058e8cca4cc1b6ba14b893e5df746e
Gitweb: https://git.kernel.org/tip/00ed1401a0058e8cca4cc1b6ba14b893e5df746e
Author: Barry Song <song.bao.hua@hisilicon.com>
AuthorDate: Fri, 13 Aug 2021 15:56:28 +12:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 24 Aug 2021 09:16:20 +02:00
platform-msi: Add ABI to show msi_irqs of platform devices
PCI devices expose the associated MSI interrupts via sysfs, but platform
devices which utilize MSI interrupts do not. This information is important
for user space tools to optimize affinity settings.
Utilize the generic MSI sysfs facility to expose this information for
platform MSI.
Signed-off-by: Barry Song <song.bao.hua@hisilicon.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Acked-by: Bjorn Helgaas <bhelgaas@google.com>
Acked-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20210813035628.6844-3-21cnbao@gmail.com
---
Documentation/ABI/testing/sysfs-bus-platform | 14 +++++++++++++-
drivers/base/platform-msi.c | 20 ++++++++++++++-----
2 files changed, 29 insertions(+), 5 deletions(-)
diff --git a/Documentation/ABI/testing/sysfs-bus-platform b/Documentation/ABI/testing/sysfs-bus-platform
index 194ca70..ff30728 100644
--- a/Documentation/ABI/testing/sysfs-bus-platform
+++ b/Documentation/ABI/testing/sysfs-bus-platform
@@ -28,3 +28,17 @@ Description:
value comes from an ACPI _PXM method or a similar firmware
source. Initial users for this file would be devices like
arm smmu which are populated by arm64 acpi_iort.
+
+What: /sys/bus/platform/devices/.../msi_irqs/
+Date: August 2021
+Contact: Barry Song <song.bao.hua@hisilicon.com>
+Description:
+ The /sys/devices/.../msi_irqs directory contains a variable set
+ of files, with each file being named after a corresponding msi
+ irq vector allocated to that device.
+
+What: /sys/bus/platform/devices/.../msi_irqs/<N>
+Date: August 2021
+Contact: Barry Song <song.bao.hua@hisilicon.com>
+Description:
+ This attribute will show "msi" if <N> is a valid msi irq
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index 0b72b13..3d6c8f9 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -21,11 +21,12 @@
* and the callback to write the MSI message.
*/
struct platform_msi_priv_data {
- struct device *dev;
- void *host_data;
- msi_alloc_info_t arg;
- irq_write_msi_msg_t write_msg;
- int devid;
+ struct device *dev;
+ void *host_data;
+ const struct attribute_group **msi_irq_groups;
+ msi_alloc_info_t arg;
+ irq_write_msi_msg_t write_msg;
+ int devid;
};
/* The devid allocator */
@@ -272,8 +273,16 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec,
if (err)
goto out_free_desc;
+ priv_data->msi_irq_groups = msi_populate_sysfs(dev);
+ if (IS_ERR(priv_data->msi_irq_groups)) {
+ err = PTR_ERR(priv_data->msi_irq_groups);
+ goto out_free_irqs;
+ }
+
return 0;
+out_free_irqs:
+ msi_domain_free_irqs(dev->msi_domain, dev);
out_free_desc:
platform_msi_free_descs(dev, 0, nvec);
out_free_priv_data:
@@ -293,6 +302,7 @@ void platform_msi_domain_free_irqs(struct device *dev)
struct msi_desc *desc;
desc = first_msi_entry(dev);
+ msi_destroy_sysfs(dev, desc->platform.msi_priv_data->msi_irq_groups);
platform_msi_free_priv_data(desc->platform.msi_priv_data);
}
^ permalink raw reply related [flat|nested] 6+ messages in thread
* [tip: irq/core] genirq/msi: Move MSI sysfs handling from PCI to MSI core
2021-08-13 3:56 ` [PATCH v3 1/2] genirq/msi: Extract common sysfs populate entries to MSI core from PCI Barry Song
@ 2021-08-24 7:20 ` tip-bot2 for Barry Song
0 siblings, 0 replies; 6+ messages in thread
From: tip-bot2 for Barry Song @ 2021-08-24 7:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Barry Song, Thomas Gleixner, Greg Kroah-Hartman, Bjorn Helgaas,
Marc Zyngier, x86, linux-kernel
The following commit has been merged into the irq/core branch of tip:
Commit-ID: 2f170814bdd26289e9daaa4ae359290f854e5dcf
Gitweb: https://git.kernel.org/tip/2f170814bdd26289e9daaa4ae359290f854e5dcf
Author: Barry Song <song.bao.hua@hisilicon.com>
AuthorDate: Fri, 13 Aug 2021 15:56:27 +12:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 24 Aug 2021 09:16:20 +02:00
genirq/msi: Move MSI sysfs handling from PCI to MSI core
Move PCI's MSI sysfs code to the irq core so that other busses such as
platform can reuse it.
Signed-off-by: Barry Song <song.bao.hua@hisilicon.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Acked-by: Bjorn Helgaas <bhelgaas@google.com>
Acked-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20210813035628.6844-2-21cnbao@gmail.com
---
drivers/pci/msi.c | 125 +++-------------------------------------
include/linux/msi.h | 4 +-
kernel/irq/msi.c | 134 +++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 148 insertions(+), 115 deletions(-)
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index ce841f3..6eb0ae3 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -363,9 +363,7 @@ static void free_msi_irqs(struct pci_dev *dev)
{
struct list_head *msi_list = dev_to_msi_list(&dev->dev);
struct msi_desc *entry, *tmp;
- struct attribute **msi_attrs;
- struct device_attribute *dev_attr;
- int i, count = 0;
+ int i;
for_each_pci_msi_entry(entry, dev)
if (entry->irq)
@@ -385,18 +383,7 @@ static void free_msi_irqs(struct pci_dev *dev)
}
if (dev->msi_irq_groups) {
- sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
- msi_attrs = dev->msi_irq_groups[0]->attrs;
- while (msi_attrs[count]) {
- dev_attr = container_of(msi_attrs[count],
- struct device_attribute, attr);
- kfree(dev_attr->attr.name);
- kfree(dev_attr);
- ++count;
- }
- kfree(msi_attrs);
- kfree(dev->msi_irq_groups[0]);
- kfree(dev->msi_irq_groups);
+ msi_destroy_sysfs(&dev->dev, dev->msi_irq_groups);
dev->msi_irq_groups = NULL;
}
}
@@ -476,102 +463,6 @@ void pci_restore_msi_state(struct pci_dev *dev)
}
EXPORT_SYMBOL_GPL(pci_restore_msi_state);
-static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
- char *buf)
-{
- struct msi_desc *entry;
- unsigned long irq;
- int retval;
-
- retval = kstrtoul(attr->attr.name, 10, &irq);
- if (retval)
- return retval;
-
- entry = irq_get_msi_desc(irq);
- if (!entry)
- return -ENODEV;
-
- return sysfs_emit(buf, "%s\n",
- entry->msi_attrib.is_msix ? "msix" : "msi");
-}
-
-static int populate_msi_sysfs(struct pci_dev *pdev)
-{
- struct attribute **msi_attrs;
- struct attribute *msi_attr;
- struct device_attribute *msi_dev_attr;
- struct attribute_group *msi_irq_group;
- const struct attribute_group **msi_irq_groups;
- struct msi_desc *entry;
- int ret = -ENOMEM;
- int num_msi = 0;
- int count = 0;
- int i;
-
- /* Determine how many msi entries we have */
- for_each_pci_msi_entry(entry, pdev)
- num_msi += entry->nvec_used;
- if (!num_msi)
- return 0;
-
- /* Dynamically create the MSI attributes for the PCI device */
- msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL);
- if (!msi_attrs)
- return -ENOMEM;
- for_each_pci_msi_entry(entry, pdev) {
- for (i = 0; i < entry->nvec_used; i++) {
- msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
- if (!msi_dev_attr)
- goto error_attrs;
- msi_attrs[count] = &msi_dev_attr->attr;
-
- sysfs_attr_init(&msi_dev_attr->attr);
- msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
- entry->irq + i);
- if (!msi_dev_attr->attr.name)
- goto error_attrs;
- msi_dev_attr->attr.mode = S_IRUGO;
- msi_dev_attr->show = msi_mode_show;
- ++count;
- }
- }
-
- msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
- if (!msi_irq_group)
- goto error_attrs;
- msi_irq_group->name = "msi_irqs";
- msi_irq_group->attrs = msi_attrs;
-
- msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL);
- if (!msi_irq_groups)
- goto error_irq_group;
- msi_irq_groups[0] = msi_irq_group;
-
- ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
- if (ret)
- goto error_irq_groups;
- pdev->msi_irq_groups = msi_irq_groups;
-
- return 0;
-
-error_irq_groups:
- kfree(msi_irq_groups);
-error_irq_group:
- kfree(msi_irq_group);
-error_attrs:
- count = 0;
- msi_attr = msi_attrs[count];
- while (msi_attr) {
- msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
- kfree(msi_attr->name);
- kfree(msi_dev_attr);
- ++count;
- msi_attr = msi_attrs[count];
- }
- kfree(msi_attrs);
- return ret;
-}
-
static struct msi_desc *
msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
{
@@ -667,9 +558,11 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
if (ret)
goto err;
- ret = populate_msi_sysfs(dev);
- if (ret)
+ dev->msi_irq_groups = msi_populate_sysfs(&dev->dev);
+ if (IS_ERR(dev->msi_irq_groups)) {
+ ret = PTR_ERR(dev->msi_irq_groups);
goto err;
+ }
/* Set MSI enabled bits */
pci_intx_for_msi(dev, 0);
@@ -834,9 +727,11 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
msix_update_entries(dev, entries);
- ret = populate_msi_sysfs(dev);
- if (ret)
+ dev->msi_irq_groups = msi_populate_sysfs(&dev->dev);
+ if (IS_ERR(dev->msi_irq_groups)) {
+ ret = PTR_ERR(dev->msi_irq_groups);
goto out_free;
+ }
/* Set MSI-X enabled bits and unmask the function */
pci_intx_for_msi(dev, 0);
diff --git a/include/linux/msi.h b/include/linux/msi.h
index a20dc66..49cf6eb 100644
--- a/include/linux/msi.h
+++ b/include/linux/msi.h
@@ -239,6 +239,10 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg);
void pci_msi_mask_irq(struct irq_data *data);
void pci_msi_unmask_irq(struct irq_data *data);
+const struct attribute_group **msi_populate_sysfs(struct device *dev);
+void msi_destroy_sysfs(struct device *dev,
+ const struct attribute_group **msi_irq_groups);
+
/*
* The arch hooks to setup up msi irqs. Default functions are implemented
* as weak symbols so that they /can/ be overriden by architecture specific
diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c
index bb18040..48ef144 100644
--- a/kernel/irq/msi.c
+++ b/kernel/irq/msi.c
@@ -14,6 +14,7 @@
#include <linux/irqdomain.h>
#include <linux/msi.h>
#include <linux/slab.h>
+#include <linux/pci.h>
#include "internals.h"
@@ -71,6 +72,139 @@ void get_cached_msi_msg(unsigned int irq, struct msi_msg *msg)
}
EXPORT_SYMBOL_GPL(get_cached_msi_msg);
+static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ struct msi_desc *entry;
+ bool is_msix = false;
+ unsigned long irq;
+ int retval;
+
+ retval = kstrtoul(attr->attr.name, 10, &irq);
+ if (retval)
+ return retval;
+
+ entry = irq_get_msi_desc(irq);
+ if (!entry)
+ return -ENODEV;
+
+ if (dev_is_pci(dev))
+ is_msix = entry->msi_attrib.is_msix;
+
+ return sysfs_emit(buf, "%s\n", is_msix ? "msix" : "msi");
+}
+
+/**
+ * msi_populate_sysfs - Populate msi_irqs sysfs entries for devices
+ * @dev: The device(PCI, platform etc) who will get sysfs entries
+ *
+ * Return attribute_group ** so that specific bus MSI can save it to
+ * somewhere during initilizing msi irqs. If devices has no MSI irq,
+ * return NULL; if it fails to populate sysfs, return ERR_PTR
+ */
+const struct attribute_group **msi_populate_sysfs(struct device *dev)
+{
+ const struct attribute_group **msi_irq_groups;
+ struct attribute **msi_attrs, *msi_attr;
+ struct device_attribute *msi_dev_attr;
+ struct attribute_group *msi_irq_group;
+ struct msi_desc *entry;
+ int ret = -ENOMEM;
+ int num_msi = 0;
+ int count = 0;
+ int i;
+
+ /* Determine how many msi entries we have */
+ for_each_msi_entry(entry, dev)
+ num_msi += entry->nvec_used;
+ if (!num_msi)
+ return NULL;
+
+ /* Dynamically create the MSI attributes for the device */
+ msi_attrs = kcalloc(num_msi + 1, sizeof(void *), GFP_KERNEL);
+ if (!msi_attrs)
+ return ERR_PTR(-ENOMEM);
+
+ for_each_msi_entry(entry, dev) {
+ for (i = 0; i < entry->nvec_used; i++) {
+ msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
+ if (!msi_dev_attr)
+ goto error_attrs;
+ msi_attrs[count] = &msi_dev_attr->attr;
+
+ sysfs_attr_init(&msi_dev_attr->attr);
+ msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
+ entry->irq + i);
+ if (!msi_dev_attr->attr.name)
+ goto error_attrs;
+ msi_dev_attr->attr.mode = 0444;
+ msi_dev_attr->show = msi_mode_show;
+ ++count;
+ }
+ }
+
+ msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
+ if (!msi_irq_group)
+ goto error_attrs;
+ msi_irq_group->name = "msi_irqs";
+ msi_irq_group->attrs = msi_attrs;
+
+ msi_irq_groups = kcalloc(2, sizeof(void *), GFP_KERNEL);
+ if (!msi_irq_groups)
+ goto error_irq_group;
+ msi_irq_groups[0] = msi_irq_group;
+
+ ret = sysfs_create_groups(&dev->kobj, msi_irq_groups);
+ if (ret)
+ goto error_irq_groups;
+
+ return msi_irq_groups;
+
+error_irq_groups:
+ kfree(msi_irq_groups);
+error_irq_group:
+ kfree(msi_irq_group);
+error_attrs:
+ count = 0;
+ msi_attr = msi_attrs[count];
+ while (msi_attr) {
+ msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
+ kfree(msi_attr->name);
+ kfree(msi_dev_attr);
+ ++count;
+ msi_attr = msi_attrs[count];
+ }
+ kfree(msi_attrs);
+ return ERR_PTR(ret);
+}
+
+/**
+ * msi_destroy_sysfs - Destroy msi_irqs sysfs entries for devices
+ * @dev: The device(PCI, platform etc) who will remove sysfs entries
+ * @msi_irq_groups: attribute_group for device msi_irqs entries
+ */
+void msi_destroy_sysfs(struct device *dev, const struct attribute_group **msi_irq_groups)
+{
+ struct device_attribute *dev_attr;
+ struct attribute **msi_attrs;
+ int count = 0;
+
+ if (msi_irq_groups) {
+ sysfs_remove_groups(&dev->kobj, msi_irq_groups);
+ msi_attrs = msi_irq_groups[0]->attrs;
+ while (msi_attrs[count]) {
+ dev_attr = container_of(msi_attrs[count],
+ struct device_attribute, attr);
+ kfree(dev_attr->attr.name);
+ kfree(dev_attr);
+ ++count;
+ }
+ kfree(msi_attrs);
+ kfree(msi_irq_groups[0]);
+ kfree(msi_irq_groups);
+ }
+}
+
#ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN
static inline void irq_chip_write_msi_msg(struct irq_data *data,
struct msi_msg *msg)
^ permalink raw reply related [flat|nested] 6+ messages in thread
end of thread, other threads:[~2021-08-24 7:20 UTC | newest]
Thread overview: 6+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-08-13 3:56 [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to platform devices Barry Song
2021-08-13 3:56 ` [PATCH v3 1/2] genirq/msi: Extract common sysfs populate entries to MSI core from PCI Barry Song
2021-08-24 7:20 ` [tip: irq/core] genirq/msi: Move MSI sysfs handling from PCI to MSI core tip-bot2 for Barry Song
2021-08-13 3:56 ` [PATCH v3 2/2] platform-msi: Add ABI to show msi_irqs of platform devices Barry Song
2021-08-24 7:20 ` [tip: irq/core] " tip-bot2 for Barry Song
2021-08-14 9:45 ` [PATCH v3 0/2] msi: extend msi_irqs sysfs entries to " Marc Zyngier
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).