From: Eric Auger <eric.auger@redhat.com>
To: eric.auger.pro@gmail.com, eric.auger@redhat.com,
qemu-devel@nongnu.org, qemu-arm@nongnu.org,
alex.williamson@redhat.com
Cc: peter.maydell@linaro.org, jacob.jun.pan@linux.intel.com,
jean-philippe@linaro.org, tn@semihalf.com,
chenxiang66@hisilicon.com, jiangkunkun@huawei.com,
shameerali.kolothum.thodi@huawei.com, nicoleotsuka@gmail.com,
vivek.gautam@arm.com, vdumpa@nvidia.com, yi.l.liu@intel.com,
peterx@redhat.com, zhangfei.gao@gmail.com, yuzenghui@huawei.com,
zhukeqian1@huawei.com
Subject: [RFC v9 16/29] vfio: Pass stage 1 MSI bindings to the host
Date: Sun, 11 Apr 2021 14:08:59 +0200 [thread overview]
Message-ID: <20210411120912.15770-17-eric.auger@redhat.com> (raw)
In-Reply-To: <20210411120912.15770-1-eric.auger@redhat.com>
We register the stage1 MSI bindings when enabling the vectors
and we unregister them on msi disable.
Signed-off-by: Eric Auger <eric.auger@redhat.com>
---
v7 -> v8:
- add unregistration on msix_diable
- remove vfio_container_unbind_msis()
v4 -> v5:
- use VFIO_IOMMU_SET_MSI_BINDING
v2 -> v3:
- only register the notifier if the IOMMU translates MSIs
- record the msi bindings in a container list and unregister on
container release
---
include/hw/vfio/vfio-common.h | 12 ++++++
hw/vfio/common.c | 59 +++++++++++++++++++++++++++
hw/vfio/pci.c | 76 ++++++++++++++++++++++++++++++++++-
hw/vfio/trace-events | 2 +
4 files changed, 147 insertions(+), 2 deletions(-)
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index 6141162d7a..f30133b2a3 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -74,6 +74,14 @@ typedef struct VFIOAddressSpace {
QLIST_ENTRY(VFIOAddressSpace) list;
} VFIOAddressSpace;
+typedef struct VFIOMSIBinding {
+ int index;
+ hwaddr iova;
+ hwaddr gpa;
+ hwaddr size;
+ QLIST_ENTRY(VFIOMSIBinding) next;
+} VFIOMSIBinding;
+
struct VFIOGroup;
typedef struct VFIOContainer {
@@ -91,6 +99,7 @@ typedef struct VFIOContainer {
QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
QLIST_HEAD(, VFIOHostDMAWindow) hostwin_list;
QLIST_HEAD(, VFIOGroup) group_list;
+ QLIST_HEAD(, VFIOMSIBinding) msibinding_list;
QLIST_ENTRY(VFIOContainer) next;
} VFIOContainer;
@@ -200,6 +209,9 @@ VFIOGroup *vfio_get_group(int groupid, AddressSpace *as, Error **errp);
void vfio_put_group(VFIOGroup *group);
int vfio_get_device(VFIOGroup *group, const char *name,
VFIODevice *vbasedev, Error **errp);
+int vfio_iommu_set_msi_binding(VFIOContainer *container, int n,
+ IOMMUTLBEntry *entry);
+int vfio_iommu_unset_msi_binding(VFIOContainer *container, int n);
extern const MemoryRegionOps vfio_region_ops;
typedef QLIST_HEAD(VFIOGroupList, VFIOGroup) VFIOGroupList;
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index e369d451e7..970a5a7be7 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -662,6 +662,65 @@ static void vfio_iommu_unmap_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
}
}
+int vfio_iommu_set_msi_binding(VFIOContainer *container, int n,
+ IOMMUTLBEntry *iotlb)
+{
+ struct vfio_iommu_type1_set_msi_binding ustruct;
+ VFIOMSIBinding *binding;
+ int ret;
+
+ QLIST_FOREACH(binding, &container->msibinding_list, next) {
+ if (binding->index == n) {
+ return 0;
+ }
+ }
+
+ ustruct.argsz = sizeof(struct vfio_iommu_type1_set_msi_binding);
+ ustruct.iova = iotlb->iova;
+ ustruct.flags = VFIO_IOMMU_BIND_MSI;
+ ustruct.gpa = iotlb->translated_addr;
+ ustruct.size = iotlb->addr_mask + 1;
+ ret = ioctl(container->fd, VFIO_IOMMU_SET_MSI_BINDING , &ustruct);
+ if (ret) {
+ error_report("%s: failed to register the stage1 MSI binding (%m)",
+ __func__);
+ return ret;
+ }
+ binding = g_new0(VFIOMSIBinding, 1);
+ binding->iova = ustruct.iova;
+ binding->gpa = ustruct.gpa;
+ binding->size = ustruct.size;
+ binding->index = n;
+
+ QLIST_INSERT_HEAD(&container->msibinding_list, binding, next);
+ return 0;
+}
+
+int vfio_iommu_unset_msi_binding(VFIOContainer *container, int n)
+{
+ struct vfio_iommu_type1_set_msi_binding ustruct;
+ VFIOMSIBinding *binding, *tmp;
+ int ret;
+
+ ustruct.argsz = sizeof(struct vfio_iommu_type1_set_msi_binding);
+ QLIST_FOREACH_SAFE(binding, &container->msibinding_list, next, tmp) {
+ if (binding->index != n) {
+ continue;
+ }
+ ustruct.flags = VFIO_IOMMU_UNBIND_MSI;
+ ustruct.iova = binding->iova;
+ ret = ioctl(container->fd, VFIO_IOMMU_SET_MSI_BINDING , &ustruct);
+ if (ret) {
+ error_report("Failed to unregister the stage1 MSI binding "
+ "for iova=0x%"PRIx64" (%m)", binding->iova);
+ }
+ QLIST_REMOVE(binding, next);
+ g_free(binding);
+ return ret;
+ }
+ return 0;
+}
+
static void vfio_iommu_map_notify(IOMMUNotifier *n, IOMMUTLBEntry *iotlb)
{
VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n);
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index cad7deec71..a49029dfa4 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -366,6 +366,65 @@ static void vfio_msi_interrupt(void *opaque)
notify(&vdev->pdev, nr);
}
+static bool vfio_iommu_require_msi_binding(IOMMUMemoryRegion *iommu_mr)
+{
+ bool msi_translate = false, nested = false;
+
+ memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_MSI_TRANSLATE,
+ (void *)&msi_translate);
+ memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_VFIO_NESTED,
+ (void *)&nested);
+ if (!nested || !msi_translate) {
+ return false;
+ }
+ return true;
+}
+
+static int vfio_register_msi_binding(VFIOPCIDevice *vdev,
+ int vector_n, bool set)
+{
+ VFIOContainer *container = vdev->vbasedev.group->container;
+ PCIDevice *dev = &vdev->pdev;
+ AddressSpace *as = pci_device_iommu_address_space(dev);
+ IOMMUMemoryRegionClass *imrc;
+ IOMMUMemoryRegion *iommu_mr;
+ IOMMUTLBEntry entry;
+ MSIMessage msg;
+
+ if (as == &address_space_memory) {
+ return 0;
+ }
+
+ iommu_mr = IOMMU_MEMORY_REGION(as->root);
+ if (!vfio_iommu_require_msi_binding(iommu_mr)) {
+ return 0;
+ }
+
+ /* MSI doorbell address is translated by an IOMMU */
+
+ if (!set) { /* unregister */
+ trace_vfio_unregister_msi_binding(vdev->vbasedev.name, vector_n);
+
+ return vfio_iommu_unset_msi_binding(container, vector_n);
+ }
+
+ msg = pci_get_msi_message(dev, vector_n);
+ imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
+
+ rcu_read_lock();
+ entry = imrc->translate(iommu_mr, msg.address, IOMMU_WO, 0);
+ rcu_read_unlock();
+
+ if (entry.perm == IOMMU_NONE) {
+ return -ENOENT;
+ }
+
+ trace_vfio_register_msi_binding(vdev->vbasedev.name, vector_n,
+ msg.address, entry.translated_addr);
+
+ return vfio_iommu_set_msi_binding(container, vector_n, &entry);
+}
+
static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
{
struct vfio_irq_set *irq_set;
@@ -383,7 +442,7 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
fds = (int32_t *)&irq_set->data;
for (i = 0; i < vdev->nr_vectors; i++) {
- int fd = -1;
+ int ret, fd = -1;
/*
* MSI vs MSI-X - The guest has direct access to MSI mask and pending
@@ -392,6 +451,12 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
* KVM signaling path only when configured and unmasked.
*/
if (vdev->msi_vectors[i].use) {
+ ret = vfio_register_msi_binding(vdev, i, true);
+ if (ret) {
+ error_report("%s failed to register S1 MSI binding "
+ "for vector %d(%d)", vdev->vbasedev.name, i, ret);
+ goto out;
+ }
if (vdev->msi_vectors[i].virq < 0 ||
(msix && msix_is_masked(&vdev->pdev, i))) {
fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
@@ -405,6 +470,7 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
+out:
g_free(irq_set);
return ret;
@@ -719,7 +785,8 @@ static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
static void vfio_msix_disable(VFIOPCIDevice *vdev)
{
- int i;
+ int ret, i;
+
msix_unset_vector_notifiers(&vdev->pdev);
@@ -731,6 +798,11 @@ static void vfio_msix_disable(VFIOPCIDevice *vdev)
if (vdev->msi_vectors[i].use) {
vfio_msix_vector_release(&vdev->pdev, i);
msix_vector_unuse(&vdev->pdev, i);
+ ret = vfio_register_msi_binding(vdev, i, false);
+ if (ret) {
+ error_report("%s: failed to unregister S1 MSI binding "
+ "for vector %d(%d)", vdev->vbasedev.name, i, ret);
+ }
}
}
diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events
index 43696afc15..5c1b28d0d4 100644
--- a/hw/vfio/trace-events
+++ b/hw/vfio/trace-events
@@ -122,6 +122,8 @@ vfio_get_dev_region(const char *name, int index, uint32_t type, uint32_t subtype
vfio_dma_unmap_overflow_workaround(void) ""
vfio_iommu_addr_inv_iotlb(int asid, uint64_t addr, uint64_t size, uint64_t nb_granules, bool leaf) "nested IOTLB invalidate asid=%d, addr=0x%"PRIx64" granule_size=0x%"PRIx64" nb_granules=0x%"PRIx64" leaf=%d"
vfio_iommu_asid_inv_iotlb(int asid) "nested IOTLB invalidate asid=%d"
+vfio_register_msi_binding(const char *name, int vector, uint64_t giova, uint64_t gdb) "%s: register vector %d gIOVA=0x%"PRIx64 "-> gDB=0x%"PRIx64" stage 1 mapping"
+vfio_unregister_msi_binding(const char *name, int vector) "%s: unregister vector %d stage 1 mapping"
# platform.c
vfio_platform_base_device_init(char *name, int groupid) "%s belongs to group #%d"
--
2.26.3
next prev parent reply other threads:[~2021-04-11 12:34 UTC|newest]
Thread overview: 46+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-11 12:08 [RFC v9 00/29] vSMMUv3/pSMMUv3 2 stage VFIO integration Eric Auger
2021-04-11 12:08 ` [RFC v9 01/29] hw/vfio/common: trace vfio_connect_container operations Eric Auger
2021-04-11 12:08 ` [RFC v9 02/29] update-linux-headers: Import iommu.h Eric Auger
2021-04-11 12:08 ` [RFC v9 03/29] header update against 5.12-rc6 and IOMMU/VFIO nested stage APIs Eric Auger
2021-04-11 12:08 ` [RFC v9 04/29] memory: Add new fields in IOTLBEntry Eric Auger
2021-04-11 12:08 ` [RFC v9 05/29] hw/arm/smmuv3: Improve stage1 ASID invalidation Eric Auger
2021-04-11 12:08 ` [RFC v9 06/29] hw/arm/smmu-common: Allow domain invalidation for NH_ALL/NSNH_ALL Eric Auger
2021-04-11 12:08 ` [RFC v9 07/29] memory: Add IOMMU_ATTR_VFIO_NESTED IOMMU memory region attribute Eric Auger
2021-04-11 12:08 ` [RFC v9 08/29] memory: Add IOMMU_ATTR_MSI_TRANSLATE " Eric Auger
2021-04-11 12:08 ` [RFC v9 09/29] memory: Introduce IOMMU Memory Region inject_faults API Eric Auger
2021-04-11 12:08 ` [RFC v9 10/29] iommu: Introduce generic header Eric Auger
2021-04-11 12:08 ` [RFC v9 11/29] pci: introduce PCIPASIDOps to PCIDevice Eric Auger
2021-04-11 12:08 ` [RFC v9 12/29] vfio: Force nested if iommu requires it Eric Auger
2021-04-11 12:08 ` [RFC v9 13/29] vfio: Introduce hostwin_from_range helper Eric Auger
2021-04-11 12:08 ` [RFC v9 14/29] vfio: Introduce helpers to DMA map/unmap a RAM section Eric Auger
2021-04-27 14:05 ` Kunkun Jiang
2021-09-03 8:22 ` Kunkun Jiang
2021-04-11 12:08 ` [RFC v9 15/29] vfio: Set up nested stage mappings Eric Auger
2021-04-13 12:10 ` Kunkun Jiang
2021-04-13 12:57 ` Auger Eric
2021-04-14 1:45 ` Kunkun Jiang
2021-04-14 8:05 ` Auger Eric
2021-04-15 2:03 ` Kunkun Jiang
2021-04-26 19:16 ` Auger Eric
2021-04-28 9:51 ` Kunkun Jiang
2021-04-29 13:58 ` Auger Eric
2021-04-26 12:30 ` Auger Eric
2021-04-27 8:58 ` Kunkun Jiang
2021-10-07 16:58 ` Eric Auger
2021-10-08 2:13 ` Kunkun Jiang
2021-04-11 12:08 ` Eric Auger [this message]
2021-10-15 10:54 ` [RFC v9 16/29] vfio: Pass stage 1 MSI bindings to the host Shameerali Kolothum Thodi
2021-04-11 12:09 ` [RFC v9 17/29] vfio: Helper to get IRQ info including capabilities Eric Auger
2021-04-11 12:09 ` [RFC v9 18/29] vfio/pci: Register handler for iommu fault Eric Auger
2021-04-11 12:09 ` [RFC v9 19/29] vfio/pci: Set up the DMA FAULT region Eric Auger
2021-04-11 12:09 ` [RFC v9 20/29] vfio/pci: Implement the DMA fault handler Eric Auger
2021-04-11 12:09 ` [RFC v9 21/29] hw/arm/smmuv3: Advertise MSI_TRANSLATE attribute Eric Auger
2021-04-11 12:09 ` [RFC v9 22/29] hw/arm/smmuv3: Store the PASID table GPA in the translation config Eric Auger
2021-04-11 12:09 ` [RFC v9 23/29] hw/arm/smmuv3: Fill the IOTLBEntry arch_id on NH_VA invalidation Eric Auger
2021-04-11 12:09 ` [RFC v9 24/29] hw/arm/smmuv3: Fill the IOTLBEntry leaf field " Eric Auger
2021-05-13 7:09 ` Kunkun Jiang
2021-04-11 12:09 ` [RFC v9 25/29] hw/arm/smmuv3: Pass stage 1 configurations to the host Eric Auger
2021-04-11 12:09 ` [RFC v9 26/29] hw/arm/smmuv3: Implement fault injection Eric Auger
2021-04-11 12:09 ` [RFC v9 27/29] hw/arm/smmuv3: Allow MAP notifiers Eric Auger
2021-04-11 12:09 ` [RFC v9 28/29] pci: Add return_page_response pci ops Eric Auger
2021-04-11 12:09 ` [RFC v9 29/29] vfio/pci: Implement return_page_response page response callback Eric Auger
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210411120912.15770-17-eric.auger@redhat.com \
--to=eric.auger@redhat.com \
--cc=alex.williamson@redhat.com \
--cc=chenxiang66@hisilicon.com \
--cc=eric.auger.pro@gmail.com \
--cc=jacob.jun.pan@linux.intel.com \
--cc=jean-philippe@linaro.org \
--cc=jiangkunkun@huawei.com \
--cc=nicoleotsuka@gmail.com \
--cc=peter.maydell@linaro.org \
--cc=peterx@redhat.com \
--cc=qemu-arm@nongnu.org \
--cc=qemu-devel@nongnu.org \
--cc=shameerali.kolothum.thodi@huawei.com \
--cc=tn@semihalf.com \
--cc=vdumpa@nvidia.com \
--cc=vivek.gautam@arm.com \
--cc=yi.l.liu@intel.com \
--cc=yuzenghui@huawei.com \
--cc=zhangfei.gao@gmail.com \
--cc=zhukeqian1@huawei.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).