[Qemu-devel] [Resend RFC PATCH 0/4] VT-d: Inject fault event from IOMMU hardware

[Qemu-devel] [Resend RFC PATCH 0/4] VT-d: Inject fault event from IOMMU hardware

[Lan Tianyu]

Resend patchset due to wrong Qemu devel mail address. Sorry for noise.

This patchset proposes a solution for vIOMMU to get hardware IOMMU
fault event and info. Motivation is to make vIOMMU inject associated
fault event when pIOMMU reports fault event. vIOMMU is in charge of
transforming fault info and inject to guest. The feature is also very
important to support first level translation(Translation request with
PASID) in VM which requires vIOMMU to inject device page request to VM.

VFIO can get notification and read fault info via new VFIO cmds.
Add fault event handler in the memory IOMMU ops and Intel IOMMU 
device model needs to register its fault event callback. VFIO
will call the callback via memory wrapper function when get
fault notification.

This patches is prototype code and just passes build test. IOMMU driver
new interface is still in the design stage. This patches is to confirm
interface between Qemu and VFIO kernel driver is on the right way.

Very appreciate for comments. 

Lan Tianyu (4):
  VFIO: Set eventfd for IOMMU fault event via new vfio cmd
  Memory: Introduce IOMMU fault event callback
  Intel iommu: Add Intel IOMMU fault event callback
  VFIO: Read IOMMU fault info from kernel space when get fault event

 hw/i386/intel_iommu.c         | 25 ++++++++++++
 hw/vfio/common.c              | 88 +++++++++++++++++++++++++++++++++++++++++++
 include/exec/memory.h         | 19 ++++++++++
 include/hw/vfio/vfio-common.h |  3 ++
 linux-headers/linux/vfio.h    | 35 +++++++++++++++++
 memory.c                      |  8 ++++
 6 files changed, 178 insertions(+)

-- 
1.8.3.1

[Qemu-devel] [Resend RFC PATCH 1/4] VFIO: Set eventfd for IOMMU fault event via new vfio cmd

[Lan Tianyu]

This patch is to assign an event fd to VFIO IOMMU type1 driver
in order to get notification when IOMMU driver reports fault event.

Signed-off-by: Lan Tianyu <tianyu.lan@intel.com>
---
 hw/vfio/common.c              | 37 +++++++++++++++++++++++++++++++++++++
 include/hw/vfio/vfio-common.h |  3 +++
 linux-headers/linux/vfio.h    | 13 +++++++++++++
 3 files changed, 53 insertions(+)

diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 6b33b9f..628b424 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -33,6 +33,7 @@
 #include "qemu/error-report.h"
 #include "qemu/range.h"
 #include "sysemu/kvm.h"
+#include "sysemu/sysemu.h"
 #include "trace.h"
 #include "qapi/error.h"
 
@@ -294,6 +295,34 @@ static bool vfio_listener_skipped_section(MemoryRegionSection *section)
            section->offset_within_address_space & (1ULL << 63);
 }
 
+static void vfio_iommu_fault(void *opaque)
+{
+}
+
+static int vfio_set_iommu_fault_notifier(struct VFIOContainer *container)
+{
+    struct vfio_iommu_type1_set_fault_eventfd eventfd;
+    int ret;
+
+    ret = event_notifier_init(&container->fault_notifier, 0);
+    if (ret < 0) {
+        error_report("vfio: Failed to init notifier for IOMMU fault event");
+        return ret;
+    }
+
+    eventfd.fd = event_notifier_get_fd(&container->fault_notifier);
+    eventfd.argsz = sizeof(eventfd);
+
+    ret = ioctl(container->fd, VFIO_IOMMU_SET_FAULT_EVENTFD, &eventfd);
+    if (ret < 0) {
+        error_report("vfio: Failed to set notifier for IOMMU fault event");
+        return ret;
+    }
+
+    qemu_set_fd_handler(eventfd.fd, vfio_iommu_fault, NULL, container);
+    return 0;
+}
+
 /* Called with rcu_read_lock held.  */
 static bool vfio_get_vaddr(IOMMUTLBEntry *iotlb, void **vaddr,
                            bool *read_only)
@@ -1103,6 +1132,14 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
         goto listener_release_exit;
     }
 
+    if (memory_region_is_iommu(container->space->as->root)) {
+        if (vfio_set_iommu_fault_notifier(container)) {
+            error_setg_errno(errp, -ret,
+                "Fail to set IOMMU fault notifier");
+            goto listener_release_exit;
+        }
+    }
+
     container->initialized = true;
 
     QLIST_INIT(&container->group_list);
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
index c582de1..1b594c6 100644
--- a/include/hw/vfio/vfio-common.h
+++ b/include/hw/vfio/vfio-common.h
@@ -26,6 +26,7 @@
 #include "exec/memory.h"
 #include "qemu/queue.h"
 #include "qemu/notify.h"
+#include "qemu/event_notifier.h"
 #ifdef CONFIG_LINUX
 #include <linux/vfio.h>
 #endif
@@ -81,6 +82,8 @@ typedef struct VFIOContainer {
     unsigned iommu_type;
     int error;
     bool initialized;
+    EventNotifier fault_notifier;
+
     /*
      * This assumes the host IOMMU can support only a single
      * contiguous IOVA window.  We may need to generalize that in
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index 759b850..ca890ee 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -537,6 +537,19 @@ struct vfio_iommu_type1_dma_unmap {
 #define VFIO_IOMMU_ENABLE	_IO(VFIO_TYPE, VFIO_BASE + 15)
 #define VFIO_IOMMU_DISABLE	_IO(VFIO_TYPE, VFIO_BASE + 16)
 
+/*
+ * VFIO_IOMMU_SET_FAULT_EVENT_FD	_IO(VFIO_TYPE, VFIO_BASE + 17)
+ * 
+ * Receive eventfd from userspace to notify fault event from IOMMU.
+ */
+struct vfio_iommu_type1_set_fault_eventfd {
+	__u32	argsz;
+	__u32   flags;
+	__u32	fd;
+};
+
+#define VFIO_IOMMU_SET_FAULT_EVENTFD	_IO(VFIO_TYPE, VFIO_BASE + 17)
+
 /* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */
 
 /*
-- 
1.8.3.1

[Qemu-devel] [Resend RFC PATCH 3/4] Intel iommu: Add Intel IOMMU fault event callback

[Lan Tianyu]

This patch is to deal with fault event reported from IOMMU driver.

Signed-off-by: Lan Tianyu <tianyu.lan@intel.com>
---
 hw/i386/intel_iommu.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
index 9b1ba1b..79507d2 100644
--- a/hw/i386/intel_iommu.c
+++ b/hw/i386/intel_iommu.c
@@ -2286,6 +2286,30 @@ static void vtd_iommu_notify_flag_changed(MemoryRegion *iommu,
     }
 }
 
+static void vtd_iommu_notify_fault_event(MemoryRegion *iommu,
+                                         IOMMUFaultInfo *info)
+{
+    VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
+    IntelIOMMUState *s = vtd_as->iommu_state;
+    bool is_fpd_set = false;
+    uint8_t bus_num = pci_bus_num(vtd_as->bus);
+    uint8_t devfn = vtd_as->devfn;
+    VTDContextEntry ce;
+
+    /* Replace source id with device's vbdf */
+    info->sid = vtd_make_source_id(bus_num, devfn);
+
+    if (!vtd_dev_to_context_entry(s, bus_num, devfn, &ce)) {
+        is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
+        if (is_fpd_set) {
+            trace_vtd_fault_disabled();
+        } else {
+            vtd_report_dmar_fault(s, info->sid, info->addr,
+                                  info->fault_reason, info->is_write);
+        }
+    }
+}
+
 static const VMStateDescription vtd_vmstate = {
     .name = "iommu-intel",
     .version_id = 1,
@@ -2816,6 +2840,7 @@ static void vtd_init(IntelIOMMUState *s)
 
     s->iommu_ops.translate = vtd_iommu_translate;
     s->iommu_ops.notify_flag_changed = vtd_iommu_notify_flag_changed;
+    s->iommu_ops.notify_fault_event = vtd_iommu_notify_fault_event;
     s->iommu_ops.replay = vtd_iommu_replay;
     s->root = 0;
     s->root_extended = false;
-- 
1.8.3.1

[Qemu-devel] [Resend RFC PATCH 4/4] VFIO: Read IOMMU fault info from kernel space when get fault event

[Lan Tianyu]

This patch is to implement fault event handler with new vfio cmd to
get fault info and notify vIOMMU device model.

Signed-off-by: Lan Tianyu <tianyu.lan@intel.com>
---
 hw/vfio/common.c           | 51 ++++++++++++++++++++++++++++++++++++++++++++++
 linux-headers/linux/vfio.h | 22 ++++++++++++++++++++
 2 files changed, 73 insertions(+)

diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 628b424..4f76e26 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -297,6 +297,57 @@ static bool vfio_listener_skipped_section(MemoryRegionSection *section)
 
 static void vfio_iommu_fault(void *opaque)
 {
+    VFIOContainer *container = opaque;
+    struct vfio_iommu_type1_get_fault_info *info;
+    struct vfio_iommu_fault_info *fault_info;
+    MemoryRegion *mr = container->space->as->root;
+    int count = 0, i, ret;
+    IOMMUFaultInfo tmp;
+
+    if (!event_notifier_test_and_clear(&container->fault_notifier)) {
+        return;
+    }
+
+    info = g_malloc0(sizeof(*info));
+    if (!info) {
+        error_report("vfio: Fail to allocate memory");
+        return;
+    }
+
+    info->argsz = sizeof(*info);
+
+    ret = ioctl(container->fd, VFIO_IOMMU_GET_FAULT_INFO, info);
+    if (ret && ret != -ENOSPC) {
+        error_report("vfio: Can't get fault info");
+        goto err_exit;
+    }
+
+    count = info->count;
+    if (count <= 0) {
+        goto err_exit;
+    }
+
+    info = g_realloc(info, sizeof(*info) + count * sizeof(*fault_info));
+    info->argsz = sizeof(*info) + count * sizeof(*fault_info);
+    fault_info = info->fault_info;
+
+    ret = ioctl(container->fd, VFIO_IOMMU_GET_FAULT_INFO, info);
+    if (ret) {
+        error_report("vfio: Can't get fault info");
+        goto err_exit;
+    }
+
+    for (i = 0; i < info->count; i++) {
+        tmp.addr = fault_info[i].addr;
+        tmp.sid = fault_info[i].sid;
+        tmp.fault_reason = fault_info[i].fault_reason;
+        tmp.is_write = fault_info[i].is_write;
+
+        memory_region_iommu_fault_notify(mr, &tmp);
+    }
+
+err_exit:
+    g_free(info);
 }
 
 static int vfio_set_iommu_fault_notifier(struct VFIOContainer *container)
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
index ca890ee..8b172f5 100644
--- a/linux-headers/linux/vfio.h
+++ b/linux-headers/linux/vfio.h
@@ -550,6 +550,28 @@ struct vfio_iommu_type1_set_fault_eventfd {
 
 #define VFIO_IOMMU_SET_FAULT_EVENTFD	_IO(VFIO_TYPE, VFIO_BASE + 17)
 
+/*
+ * VFIO_IOMMU_GET_FAULT_INFO		_IO(VFIO_TYPE, VFIO_BASE + 18)
+ *
+ * Return IOMMU fault info to userspace.
+ */
+
+struct vfio_iommu_fault_info {
+	__u64	addr;
+	__u16   sid;
+	__u8    fault_reason;
+	__u8	is_write:1;
+};
+
+struct vfio_iommu_type1_get_fault_info {
+	__u32	argsz;
+	__u32   flags;
+	__u32	count;
+	struct vfio_iommu_fault_info fault_info[];
+};
+
+#define VFIO_IOMMU_GET_FAULT_INFO	_IO(VFIO_TYPE, VFIO_BASE + 18)
+
 /* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */
 
 /*
-- 
1.8.3.1

Re: [Qemu-devel] [Resend RFC PATCH 1/4] VFIO: Set eventfd for IOMMU fault event via new vfio cmd

[Alex Williamson]

On Mon, 20 Feb 2017 09:28:04 +0800
Lan Tianyu <tianyu.lan@intel.com> wrote:

> This patch is to assign an event fd to VFIO IOMMU type1 driver
> in order to get notification when IOMMU driver reports fault event.
> 
> Signed-off-by: Lan Tianyu <tianyu.lan@intel.com>
> ---
>  hw/vfio/common.c              | 37 +++++++++++++++++++++++++++++++++++++
>  include/hw/vfio/vfio-common.h |  3 +++
>  linux-headers/linux/vfio.h    | 13 +++++++++++++
>  3 files changed, 53 insertions(+)
> 
> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> index 6b33b9f..628b424 100644
> --- a/hw/vfio/common.c
> +++ b/hw/vfio/common.c
> @@ -33,6 +33,7 @@
>  #include "qemu/error-report.h"
>  #include "qemu/range.h"
>  #include "sysemu/kvm.h"
> +#include "sysemu/sysemu.h"
>  #include "trace.h"
>  #include "qapi/error.h"
>  
> @@ -294,6 +295,34 @@ static bool vfio_listener_skipped_section(MemoryRegionSection *section)
>             section->offset_within_address_space & (1ULL << 63);
>  }
>  
> +static void vfio_iommu_fault(void *opaque)
> +{
> +}
> +
> +static int vfio_set_iommu_fault_notifier(struct VFIOContainer *container)
> +{
> +    struct vfio_iommu_type1_set_fault_eventfd eventfd;
> +    int ret;
> +
> +    ret = event_notifier_init(&container->fault_notifier, 0);
> +    if (ret < 0) {
> +        error_report("vfio: Failed to init notifier for IOMMU fault event");
> +        return ret;
> +    }
> +
> +    eventfd.fd = event_notifier_get_fd(&container->fault_notifier);
> +    eventfd.argsz = sizeof(eventfd);
> +
> +    ret = ioctl(container->fd, VFIO_IOMMU_SET_FAULT_EVENTFD, &eventfd);
> +    if (ret < 0) {
> +        error_report("vfio: Failed to set notifier for IOMMU fault event");
> +        return ret;
> +    }
> +
> +    qemu_set_fd_handler(eventfd.fd, vfio_iommu_fault, NULL, container);
> +    return 0;
> +}
> +
>  /* Called with rcu_read_lock held.  */
>  static bool vfio_get_vaddr(IOMMUTLBEntry *iotlb, void **vaddr,
>                             bool *read_only)
> @@ -1103,6 +1132,14 @@ static int vfio_connect_container(VFIOGroup *group, AddressSpace *as,
>          goto listener_release_exit;
>      }
>  
> +    if (memory_region_is_iommu(container->space->as->root)) {
> +        if (vfio_set_iommu_fault_notifier(container)) {
> +            error_setg_errno(errp, -ret,
> +                "Fail to set IOMMU fault notifier");
> +            goto listener_release_exit;
> +        }
> +    }


Backwards compatibility needs to be supported, we can't invent a
feature and impose it on all users.  We can either opportunistically
enable it and be silent about it when it's not there, or we can add a
feature flag and complain and fail if the user requests the feature and
it's not available or errors.

> +
>      container->initialized = true;
>  
>      QLIST_INIT(&container->group_list);
> diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
> index c582de1..1b594c6 100644
> --- a/include/hw/vfio/vfio-common.h
> +++ b/include/hw/vfio/vfio-common.h
> @@ -26,6 +26,7 @@
>  #include "exec/memory.h"
>  #include "qemu/queue.h"
>  #include "qemu/notify.h"
> +#include "qemu/event_notifier.h"
>  #ifdef CONFIG_LINUX
>  #include <linux/vfio.h>
>  #endif
> @@ -81,6 +82,8 @@ typedef struct VFIOContainer {
>      unsigned iommu_type;
>      int error;
>      bool initialized;
> +    EventNotifier fault_notifier;
> +
>      /*
>       * This assumes the host IOMMU can support only a single
>       * contiguous IOVA window.  We may need to generalize that in
> diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
> index 759b850..ca890ee 100644
> --- a/linux-headers/linux/vfio.h
> +++ b/linux-headers/linux/vfio.h
> @@ -537,6 +537,19 @@ struct vfio_iommu_type1_dma_unmap {
>  #define VFIO_IOMMU_ENABLE	_IO(VFIO_TYPE, VFIO_BASE + 15)
>  #define VFIO_IOMMU_DISABLE	_IO(VFIO_TYPE, VFIO_BASE + 16)
>  
> +/*
> + * VFIO_IOMMU_SET_FAULT_EVENT_FD	_IO(VFIO_TYPE, VFIO_BASE + 17)
> + * 
> + * Receive eventfd from userspace to notify fault event from IOMMU.
> + */
> +struct vfio_iommu_type1_set_fault_eventfd {
> +	__u32	argsz;
> +	__u32   flags;
> +	__u32	fd;
> +};
> +
> +#define VFIO_IOMMU_SET_FAULT_EVENTFD	_IO(VFIO_TYPE, VFIO_BASE + 17)
> +
>  /* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */
>  
>  /*

Re: [Qemu-devel] [Resend RFC PATCH 3/4] Intel iommu: Add Intel IOMMU fault event callback

[Alex Williamson]

On Mon, 20 Feb 2017 09:28:06 +0800
Lan Tianyu <tianyu.lan@intel.com> wrote:

> This patch is to deal with fault event reported from IOMMU driver.
> 
> Signed-off-by: Lan Tianyu <tianyu.lan@intel.com>
> ---
>  hw/i386/intel_iommu.c | 25 +++++++++++++++++++++++++
>  1 file changed, 25 insertions(+)
> 
> diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
> index 9b1ba1b..79507d2 100644
> --- a/hw/i386/intel_iommu.c
> +++ b/hw/i386/intel_iommu.c
> @@ -2286,6 +2286,30 @@ static void vtd_iommu_notify_flag_changed(MemoryRegion *iommu,
>      }
>  }
>  
> +static void vtd_iommu_notify_fault_event(MemoryRegion *iommu,
> +                                         IOMMUFaultInfo *info)
> +{
> +    VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
> +    IntelIOMMUState *s = vtd_as->iommu_state;
> +    bool is_fpd_set = false;
> +    uint8_t bus_num = pci_bus_num(vtd_as->bus);
> +    uint8_t devfn = vtd_as->devfn;
> +    VTDContextEntry ce;
> +
> +    /* Replace source id with device's vbdf */
> +    info->sid = vtd_make_source_id(bus_num, devfn);
> +
> +    if (!vtd_dev_to_context_entry(s, bus_num, devfn, &ce)) {
> +        is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
> +        if (is_fpd_set) {
> +            trace_vtd_fault_disabled();
> +        } else {
> +            vtd_report_dmar_fault(s, info->sid, info->addr,
> +                                  info->fault_reason, info->is_write);

Convenient that the error structure exactly matches the function
parameters here.  Is fault_reason going to be standardized across all
IOMMU vendors?  If not, how does QEMU know whether the type1 container
is backed by VT-d, AMD-Vi, ARM SMMU, etc?

> +        }
> +    }
> +}
> +
>  static const VMStateDescription vtd_vmstate = {
>      .name = "iommu-intel",
>      .version_id = 1,
> @@ -2816,6 +2840,7 @@ static void vtd_init(IntelIOMMUState *s)
>  
>      s->iommu_ops.translate = vtd_iommu_translate;
>      s->iommu_ops.notify_flag_changed = vtd_iommu_notify_flag_changed;
> +    s->iommu_ops.notify_fault_event = vtd_iommu_notify_fault_event;
>      s->iommu_ops.replay = vtd_iommu_replay;
>      s->root = 0;
>      s->root_extended = false;

Re: [Qemu-devel] [Resend RFC PATCH 4/4] VFIO: Read IOMMU fault info from kernel space when get fault event

[Alex Williamson]

On Mon, 20 Feb 2017 09:28:07 +0800
Lan Tianyu <tianyu.lan@intel.com> wrote:

> This patch is to implement fault event handler with new vfio cmd to
> get fault info and notify vIOMMU device model.
> 
> Signed-off-by: Lan Tianyu <tianyu.lan@intel.com>
> ---
>  hw/vfio/common.c           | 51 ++++++++++++++++++++++++++++++++++++++++++++++
>  linux-headers/linux/vfio.h | 22 ++++++++++++++++++++
>  2 files changed, 73 insertions(+)
> 
> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> index 628b424..4f76e26 100644
> --- a/hw/vfio/common.c
> +++ b/hw/vfio/common.c
> @@ -297,6 +297,57 @@ static bool vfio_listener_skipped_section(MemoryRegionSection *section)
>  
>  static void vfio_iommu_fault(void *opaque)
>  {
> +    VFIOContainer *container = opaque;
> +    struct vfio_iommu_type1_get_fault_info *info;
> +    struct vfio_iommu_fault_info *fault_info;
> +    MemoryRegion *mr = container->space->as->root;
> +    int count = 0, i, ret;
> +    IOMMUFaultInfo tmp;
> +
> +    if (!event_notifier_test_and_clear(&container->fault_notifier)) {
> +        return;
> +    }
> +
> +    info = g_malloc0(sizeof(*info));
> +    if (!info) {
> +        error_report("vfio: Fail to allocate memory");
> +        return;
> +    }
> +
> +    info->argsz = sizeof(*info);
> +
> +    ret = ioctl(container->fd, VFIO_IOMMU_GET_FAULT_INFO, info);
> +    if (ret && ret != -ENOSPC) {
> +        error_report("vfio: Can't get fault info");
> +        goto err_exit;
> +    }
> +
> +    count = info->count;
> +    if (count <= 0) {
> +        goto err_exit;
> +    }
> +
> +    info = g_realloc(info, sizeof(*info) + count * sizeof(*fault_info));
> +    info->argsz = sizeof(*info) + count * sizeof(*fault_info);
> +    fault_info = info->fault_info;
> +
> +    ret = ioctl(container->fd, VFIO_IOMMU_GET_FAULT_INFO, info);
> +    if (ret) {
> +        error_report("vfio: Can't get fault info");
> +        goto err_exit;
> +    }
> +
> +    for (i = 0; i < info->count; i++) {
> +        tmp.addr = fault_info[i].addr;
> +        tmp.sid = fault_info[i].sid;
> +        tmp.fault_reason = fault_info[i].fault_reason;
> +        tmp.is_write = fault_info[i].is_write;
> +
> +        memory_region_iommu_fault_notify(mr, &tmp);
> +    }

Are there service requirements for handling these faults?  Can the
device wait indefinitely?  Can userspace handling of such faults meet
the device service requirements?  Is userspace handling sufficient for
the device's performance needs?  Do we get one eventfd per fault entry?
How do we know if the faults have overflowed?  Would an overflow be
fatal or would there be a retry mechanism?

> +
> +err_exit:
> +    g_free(info);
>  }
>  
>  static int vfio_set_iommu_fault_notifier(struct VFIOContainer *container)
> diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
> index ca890ee..8b172f5 100644
> --- a/linux-headers/linux/vfio.h
> +++ b/linux-headers/linux/vfio.h
> @@ -550,6 +550,28 @@ struct vfio_iommu_type1_set_fault_eventfd {
>  
>  #define VFIO_IOMMU_SET_FAULT_EVENTFD	_IO(VFIO_TYPE, VFIO_BASE + 17)
>  
> +/*
> + * VFIO_IOMMU_GET_FAULT_INFO		_IO(VFIO_TYPE, VFIO_BASE + 18)
> + *
> + * Return IOMMU fault info to userspace.
> + */
> +
> +struct vfio_iommu_fault_info {
> +	__u64	addr;
> +	__u16   sid;
> +	__u8    fault_reason;
> +	__u8	is_write:1;
> +};
> +
> +struct vfio_iommu_type1_get_fault_info {
> +	__u32	argsz;
> +	__u32   flags;
> +	__u32	count;
> +	struct vfio_iommu_fault_info fault_info[];
> +};
> +
> +#define VFIO_IOMMU_GET_FAULT_INFO	_IO(VFIO_TYPE, VFIO_BASE + 18)
> +
>  /* -------- Additional API for SPAPR TCE (Server POWERPC) IOMMU -------- */
>  
>  /*