* [PATCH 6/6 v5 updated] KVM: assigned dev: MSI-X mask support
2010-11-15 9:15 ` [PATCH 6/6] KVM: assigned dev: MSI-X mask support Sheng Yang
@ 2010-11-15 9:27 ` Sheng Yang
2010-11-16 19:45 ` [PATCH 6/6] " Marcelo Tosatti
2010-11-17 13:58 ` Avi Kivity
2 siblings, 0 replies; 26+ messages in thread
From: Sheng Yang @ 2010-11-15 9:27 UTC (permalink / raw)
To: Avi Kivity; +Cc: Marcelo Tosatti, Michael S. Tsirkin, kvm, Sheng Yang
This patch enable per-vector mask for assigned devices using MSI-X.
This patch provided two new APIs: one is for guest to specific device's MSI-X
table address in MMIO, the other is for userspace to get information about mask
bit.
All the mask bit operation are kept in kernel, in order to accelerate.
Userspace shouldn't access the device MMIO directly for the information,
instead it should uses provided API to do so.
Signed-off-by: Sheng Yang <sheng@linux.intel.com>
---
arch/x86/kvm/x86.c | 1 +
include/linux/kvm.h | 32 +++++
include/linux/kvm_host.h | 5 +
virt/kvm/assigned-dev.c | 323 +++++++++++++++++++++++++++++++++++++++++++++-
4 files changed, 360 insertions(+), 1 deletions(-)
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index fc29223..37602e2 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1966,6 +1966,7 @@ int kvm_dev_ioctl_check_extension(long ext)
case KVM_CAP_X86_ROBUST_SINGLESTEP:
case KVM_CAP_XSAVE:
case KVM_CAP_ASYNC_PF:
+ case KVM_CAP_MSIX_MASK:
r = 1;
break;
case KVM_CAP_COALESCED_MMIO:
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index ea2dc1a..b3e5ffe 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -541,6 +541,9 @@ struct kvm_ppc_pvinfo {
#define KVM_CAP_PPC_GET_PVINFO 57
#define KVM_CAP_PPC_IRQ_LEVEL 58
#define KVM_CAP_ASYNC_PF 59
+#ifdef __KVM_HAVE_MSIX
+#define KVM_CAP_MSIX_MASK 60
+#endif
#ifdef KVM_CAP_IRQ_ROUTING
@@ -672,6 +675,9 @@ struct kvm_clock_data {
#define KVM_XEN_HVM_CONFIG _IOW(KVMIO, 0x7a, struct kvm_xen_hvm_config)
#define KVM_SET_CLOCK _IOW(KVMIO, 0x7b, struct kvm_clock_data)
#define KVM_GET_CLOCK _IOR(KVMIO, 0x7c, struct kvm_clock_data)
+/* Available with KVM_CAP_MSIX_MASK */
+#define KVM_GET_MSIX_ENTRY _IOWR(KVMIO, 0x7d, struct kvm_msix_entry)
+#define KVM_UPDATE_MSIX_MMIO _IOW(KVMIO, 0x7e, struct kvm_msix_mmio)
/* Available with KVM_CAP_PIT_STATE2 */
#define KVM_GET_PIT2 _IOR(KVMIO, 0x9f, struct kvm_pit_state2)
#define KVM_SET_PIT2 _IOW(KVMIO, 0xa0, struct kvm_pit_state2)
@@ -795,4 +801,30 @@ struct kvm_assigned_msix_entry {
__u16 padding[3];
};
+#define KVM_MSIX_TYPE_ASSIGNED_DEV 1
+
+#define KVM_MSIX_FLAG_MASKBIT (1 << 0)
+#define KVM_MSIX_FLAG_QUERY_MASKBIT (1 << 0)
+
+struct kvm_msix_entry {
+ __u32 id;
+ __u32 type;
+ __u32 entry; /* The index of entry in the MSI-X table */
+ __u32 flags;
+ __u32 query_flags;
+ __u32 reserved[5];
+};
+
+#define KVM_MSIX_MMIO_FLAG_REGISTER (1 << 0)
+#define KVM_MSIX_MMIO_FLAG_UNREGISTER (1 << 1)
+
+struct kvm_msix_mmio {
+ __u32 id;
+ __u32 type;
+ __u64 base_addr;
+ __u32 max_entries_nr;
+ __u32 flags;
+ __u32 reserved[6];
+};
+
#endif /* __LINUX_KVM_H */
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index f09db87..57a437a 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -501,6 +501,7 @@ struct kvm_guest_msix_entry {
};
#define KVM_ASSIGNED_ENABLED_IOMMU (1 << 0)
+#define KVM_ASSIGNED_ENABLED_MSIX_MMIO (1 << 1)
struct kvm_assigned_dev_kernel {
struct kvm_irq_ack_notifier ack_notifier;
struct work_struct interrupt_work;
@@ -521,6 +522,10 @@ struct kvm_assigned_dev_kernel {
struct pci_dev *dev;
struct kvm *kvm;
spinlock_t assigned_dev_lock;
+ DECLARE_BITMAP(msix_mask_bitmap, KVM_MAX_MSIX_PER_DEV);
+ gpa_t msix_mmio_base;
+ struct kvm_io_device msix_mmio_dev;
+ int msix_max_entries_nr;
};
struct kvm_irq_mask_notifier {
diff --git a/virt/kvm/assigned-dev.c b/virt/kvm/assigned-dev.c
index 5c6b96d..a96a74d 100644
--- a/virt/kvm/assigned-dev.c
+++ b/virt/kvm/assigned-dev.c
@@ -226,12 +226,27 @@ static void kvm_free_assigned_irq(struct kvm *kvm,
kvm_deassign_irq(kvm, assigned_dev, assigned_dev->irq_requested_type);
}
+static void unregister_msix_mmio(struct kvm *kvm,
+ struct kvm_assigned_dev_kernel *adev)
+{
+ if (adev->flags & KVM_ASSIGNED_ENABLED_MSIX_MMIO) {
+ mutex_lock(&kvm->slots_lock);
+ kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
+ &adev->msix_mmio_dev);
+ mutex_unlock(&kvm->slots_lock);
+ adev->flags &= ~KVM_ASSIGNED_ENABLED_MSIX_MMIO;
+ }
+}
+
static void kvm_free_assigned_device(struct kvm *kvm,
struct kvm_assigned_dev_kernel
*assigned_dev)
{
kvm_free_assigned_irq(kvm, assigned_dev);
+#ifdef __KVM_HAVE_MSIX
+ unregister_msix_mmio(kvm, assigned_dev);
+#endif
pci_reset_function(assigned_dev->dev);
pci_release_regions(assigned_dev->dev);
@@ -504,7 +519,7 @@ out:
static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
struct kvm_assigned_pci_dev *assigned_dev)
{
- int r = 0, idx;
+ int r = 0, idx, i;
struct kvm_assigned_dev_kernel *match;
struct pci_dev *dev;
@@ -564,6 +579,10 @@ static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
list_add(&match->list, &kvm->arch.assigned_dev_head);
+ /* The state after reset of MSI-X table is all masked */
+ for (i = 0; i < KVM_MAX_MSIX_PER_DEV; i++)
+ set_bit(i, match->msix_mask_bitmap);
+
if (assigned_dev->flags & KVM_ASSIGNED_ENABLED_IOMMU) {
if (!kvm->arch.iommu_domain) {
r = kvm_iommu_map_guest(kvm);
@@ -667,6 +686,43 @@ msix_nr_out:
return r;
}
+static void update_msix_mask(struct kvm_assigned_dev_kernel *adev,
+ int idx, bool new_mask_flag)
+{
+ int irq;
+ bool old_mask_flag, need_flush = false;
+
+ spin_lock_irq(&adev->assigned_dev_lock);
+
+ if (!adev->dev->msix_enabled ||
+ !(adev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX))
+ goto out;
+
+ old_mask_flag = test_bit(adev->guest_msix_entries[idx].entry,
+ adev->msix_mask_bitmap);
+ if (old_mask_flag == new_mask_flag)
+ goto out;
+
+ irq = adev->host_msix_entries[idx].vector;
+ BUG_ON(irq == 0);
+
+ if (new_mask_flag) {
+ set_bit(adev->guest_msix_entries[idx].entry,
+ adev->msix_mask_bitmap);
+ disable_irq_nosync(irq);
+ need_flush = true;
+ } else {
+ clear_bit(adev->guest_msix_entries[idx].entry,
+ adev->msix_mask_bitmap);
+ enable_irq(irq);
+ }
+out:
+ spin_unlock_irq(&adev->assigned_dev_lock);
+
+ if (need_flush)
+ flush_work(&adev->interrupt_work);
+}
+
static int kvm_vm_ioctl_set_msix_entry(struct kvm *kvm,
struct kvm_assigned_msix_entry *entry)
{
@@ -701,6 +757,240 @@ msix_entry_out:
return r;
}
+
+static int kvm_vm_ioctl_get_msix_entry(struct kvm *kvm,
+ struct kvm_msix_entry *entry)
+{
+ int r = 0;
+ struct kvm_assigned_dev_kernel *adev;
+
+ if (entry->type != KVM_MSIX_TYPE_ASSIGNED_DEV)
+ return -EINVAL;
+
+ if (!entry->query_flags)
+ return -EINVAL;
+
+ mutex_lock(&kvm->lock);
+
+ adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
+ entry->id);
+
+ if (!adev) {
+ r = -EINVAL;
+ goto out;
+ }
+
+ if (entry->entry >= adev->msix_max_entries_nr) {
+ r = -ENOSPC;
+ goto out;
+ }
+
+ if (entry->query_flags & KVM_MSIX_FLAG_QUERY_MASKBIT) {
+ if (test_bit(entry->entry, adev->msix_mask_bitmap))
+ entry->flags |= KVM_MSIX_FLAG_MASKBIT;
+ else
+ entry->flags &= ~KVM_MSIX_FLAG_MASKBIT;
+ }
+
+out:
+ mutex_unlock(&kvm->lock);
+
+ return r;
+}
+
+static bool msix_mmio_in_range(struct kvm_assigned_dev_kernel *adev,
+ gpa_t addr, int len)
+{
+ gpa_t start, end;
+
+ BUG_ON(!(adev->flags & KVM_ASSIGNED_ENABLED_MSIX_MMIO));
+ start = adev->msix_mmio_base;
+ end = adev->msix_mmio_base + PCI_MSIX_ENTRY_SIZE *
+ adev->msix_max_entries_nr;
+ if (addr >= start && addr + len <= end)
+ return true;
+
+ return false;
+}
+
+static int msix_get_enabled_idx(struct kvm_assigned_dev_kernel *adev,
+ gpa_t addr, int len)
+{
+ int i, index = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
+
+ for (i = 0; i < adev->entries_nr; i++)
+ if (adev->guest_msix_entries[i].entry == index)
+ return i;
+
+ return -EINVAL;
+}
+
+static int msix_mmio_read(struct kvm_io_device *this, gpa_t addr, int len,
+ void *val)
+{
+ struct kvm_assigned_dev_kernel *adev =
+ container_of(this, struct kvm_assigned_dev_kernel,
+ msix_mmio_dev);
+ int idx, r = 0;
+ u32 entry[4];
+ struct kvm_kernel_irq_routing_entry e;
+
+ /* TODO: Get big-endian machine work */
+ mutex_lock(&adev->kvm->lock);
+ if (!msix_mmio_in_range(adev, addr, len)) {
+ r = -EOPNOTSUPP;
+ goto out;
+ }
+ if ((addr & 0x3) || len != 4) {
+ r = -EOPNOTSUPP;
+ goto out;
+ }
+
+ idx = msix_get_enabled_idx(adev, addr, len);
+ if (idx < 0) {
+ idx = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
+ if ((addr % PCI_MSIX_ENTRY_SIZE) ==
+ PCI_MSIX_ENTRY_VECTOR_CTRL)
+ *(unsigned long *)val =
+ test_bit(idx, adev->msix_mask_bitmap) ?
+ PCI_MSIX_ENTRY_CTRL_MASKBIT : 0;
+ else
+ r = -EOPNOTSUPP;
+ goto out;
+ }
+
+ r = kvm_get_irq_routing_entry(adev->kvm,
+ adev->guest_msix_entries[idx].vector, &e);
+ if (r || e.type != KVM_IRQ_ROUTING_MSI) {
+ r = -EOPNOTSUPP;
+ goto out;
+ }
+ entry[0] = e.msi.address_lo;
+ entry[1] = e.msi.address_hi;
+ entry[2] = e.msi.data;
+ entry[3] = test_bit(adev->guest_msix_entries[idx].entry,
+ adev->msix_mask_bitmap);
+ memcpy(val, &entry[addr % PCI_MSIX_ENTRY_SIZE / sizeof *entry], len);
+
+out:
+ mutex_unlock(&adev->kvm->lock);
+ return r;
+}
+
+static int msix_mmio_write(struct kvm_io_device *this, gpa_t addr, int len,
+ const void *val)
+{
+ struct kvm_assigned_dev_kernel *adev =
+ container_of(this, struct kvm_assigned_dev_kernel,
+ msix_mmio_dev);
+ int idx, r = 0;
+ unsigned long new_val;
+
+ /* TODO: Get big-endian machine work */
+ mutex_lock(&adev->kvm->lock);
+ if (!msix_mmio_in_range(adev, addr, len)) {
+ r = -EOPNOTSUPP;
+ goto out;
+ }
+ if ((addr & 0x3) || len != 4) {
+ r = -EOPNOTSUPP;
+ goto out;
+ }
+
+ new_val = *(unsigned long *)val;
+ idx = msix_get_enabled_idx(adev, addr, len);
+ if (idx < 0) {
+ idx = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
+ if (((addr % PCI_MSIX_ENTRY_SIZE) ==
+ PCI_MSIX_ENTRY_VECTOR_CTRL)) {
+ if (new_val & ~PCI_MSIX_ENTRY_CTRL_MASKBIT)
+ goto out;
+ if (new_val & PCI_MSIX_ENTRY_CTRL_MASKBIT)
+ set_bit(idx, adev->msix_mask_bitmap);
+ else
+ clear_bit(idx, adev->msix_mask_bitmap);
+ /* It's possible that we need re-enable MSI-X, so go
+ * back to userspace */
+ }
+ /* Userspace would handle other MMIO writing */
+ r = -EOPNOTSUPP;
+ goto out;
+ }
+ if (addr % PCI_MSIX_ENTRY_SIZE != PCI_MSIX_ENTRY_VECTOR_CTRL) {
+ r = -EOPNOTSUPP;
+ goto out;
+ }
+ if (new_val & ~PCI_MSIX_ENTRY_CTRL_MASKBIT)
+ goto out;
+ update_msix_mask(adev, idx, !!(new_val & PCI_MSIX_ENTRY_CTRL_MASKBIT));
+out:
+ mutex_unlock(&adev->kvm->lock);
+
+ return r;
+}
+
+static const struct kvm_io_device_ops msix_mmio_ops = {
+ .read = msix_mmio_read,
+ .write = msix_mmio_write,
+};
+
+static int kvm_vm_ioctl_update_msix_mmio(struct kvm *kvm,
+ struct kvm_msix_mmio *msix_mmio)
+{
+ int r = 0;
+ struct kvm_assigned_dev_kernel *adev;
+
+ if (msix_mmio->type != KVM_MSIX_TYPE_ASSIGNED_DEV)
+ return -EINVAL;
+
+ if (!msix_mmio->flags)
+ return -EINVAL;
+
+ mutex_lock(&kvm->lock);
+ adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
+ msix_mmio->id);
+ if (!adev) {
+ r = -EINVAL;
+ goto out;
+ }
+ if (msix_mmio->base_addr == 0) {
+ r = -EINVAL;
+ goto out;
+ }
+ if (msix_mmio->max_entries_nr == 0 ||
+ msix_mmio->max_entries_nr > KVM_MAX_MSIX_PER_DEV) {
+ r = -EINVAL;
+ goto out;
+ }
+
+ if ((msix_mmio->flags & KVM_MSIX_MMIO_FLAG_REGISTER) &&
+ (msix_mmio->flags & KVM_MSIX_MMIO_FLAG_UNREGISTER)) {
+ r = -EINVAL;
+ goto out;
+ }
+
+ if (msix_mmio->flags & KVM_MSIX_MMIO_FLAG_REGISTER) {
+ if (!(adev->flags & KVM_ASSIGNED_ENABLED_MSIX_MMIO)) {
+ mutex_lock(&kvm->slots_lock);
+ kvm_iodevice_init(&adev->msix_mmio_dev,
+ &msix_mmio_ops);
+ r = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
+ &adev->msix_mmio_dev);
+ if (!r)
+ adev->flags |= KVM_ASSIGNED_ENABLED_MSIX_MMIO;
+ mutex_unlock(&kvm->slots_lock);
+ }
+ if (!r) {
+ adev->msix_mmio_base = msix_mmio->base_addr;
+ adev->msix_max_entries_nr = msix_mmio->max_entries_nr;
+ }
+ } else if (msix_mmio->flags & KVM_MSIX_MMIO_FLAG_UNREGISTER)
+ unregister_msix_mmio(kvm, adev);
+out:
+ mutex_unlock(&kvm->lock);
+
+ return r;
+}
#endif
long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
@@ -813,6 +1103,37 @@ long kvm_vm_ioctl_assigned_device(struct kvm *kvm, unsigned ioctl,
goto out;
break;
}
+ case KVM_GET_MSIX_ENTRY: {
+ struct kvm_msix_entry entry;
+ r = -EFAULT;
+ if (copy_from_user(&entry, argp, sizeof entry))
+ goto out;
+ r = kvm_vm_ioctl_get_msix_entry(kvm, &entry);
+ if (r)
+ goto out;
+ r = -EFAULT;
+ if (copy_to_user(argp, &entry, sizeof entry))
+ goto out;
+ r = 0;
+ break;
+ }
+ case KVM_UPDATE_MSIX_MMIO: {
+ struct kvm_msix_mmio msix_mmio;
+
+ r = -EFAULT;
+ if (copy_from_user(&msix_mmio, argp, sizeof(msix_mmio)))
+ goto out;
+
+ r = -EINVAL;
+ if (find_first_bit((unsigned long *)msix_mmio.reserved,
+ sizeof(msix_mmio.reserved)) < sizeof(msix_mmio.reserved))
+ goto out;
+
+ r = kvm_vm_ioctl_update_msix_mmio(kvm, &msix_mmio);
+ if (r)
+ goto out;
+ break;
+ }
#endif
}
out:
--
1.7.0.1
^ permalink raw reply related [flat|nested] 26+ messages in thread* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-15 9:15 ` [PATCH 6/6] KVM: assigned dev: MSI-X mask support Sheng Yang
2010-11-15 9:27 ` [PATCH 6/6 v5 updated] " Sheng Yang
@ 2010-11-16 19:45 ` Marcelo Tosatti
2010-11-17 1:29 ` Sheng Yang
2010-11-17 13:58 ` Avi Kivity
2 siblings, 1 reply; 26+ messages in thread
From: Marcelo Tosatti @ 2010-11-16 19:45 UTC (permalink / raw)
To: Sheng Yang; +Cc: Avi Kivity, Michael S. Tsirkin, kvm
On Mon, Nov 15, 2010 at 05:15:32PM +0800, Sheng Yang wrote:
> This patch enable per-vector mask for assigned devices using MSI-X.
>
> This patch provided two new APIs: one is for guest to specific device's MSI-X
> table address in MMIO, the other is for userspace to get information about mask
> bit.
>
> All the mask bit operation are kept in kernel, in order to accelerate.
> Userspace shouldn't access the device MMIO directly for the information,
> instead it should uses provided API to do so.
>
> Signed-off-by: Sheng Yang <sheng@linux.intel.com>
> ---
> arch/x86/kvm/x86.c | 1 +
> include/linux/kvm.h | 32 +++++
> include/linux/kvm_host.h | 5 +
> virt/kvm/assigned-dev.c | 318 +++++++++++++++++++++++++++++++++++++++++++++-
> 4 files changed, 355 insertions(+), 1 deletions(-)
>
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index fc29223..37602e2 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -1966,6 +1966,7 @@ int kvm_dev_ioctl_check_extension(long ext)
> case KVM_CAP_X86_ROBUST_SINGLESTEP:
> case KVM_CAP_XSAVE:
> case KVM_CAP_ASYNC_PF:
> + case KVM_CAP_MSIX_MASK:
> r = 1;
> break;
> case KVM_CAP_COALESCED_MMIO:
> diff --git a/include/linux/kvm.h b/include/linux/kvm.h
> index ea2dc1a..b3e5ffe 100644
> --- a/include/linux/kvm.h
> +++ b/include/linux/kvm.h
> @@ -541,6 +541,9 @@ struct kvm_ppc_pvinfo {
> #define KVM_CAP_PPC_GET_PVINFO 57
> #define KVM_CAP_PPC_IRQ_LEVEL 58
> #define KVM_CAP_ASYNC_PF 59
> +#ifdef __KVM_HAVE_MSIX
> +#define KVM_CAP_MSIX_MASK 60
> +#endif
>
> #ifdef KVM_CAP_IRQ_ROUTING
>
> @@ -672,6 +675,9 @@ struct kvm_clock_data {
> #define KVM_XEN_HVM_CONFIG _IOW(KVMIO, 0x7a, struct kvm_xen_hvm_config)
> #define KVM_SET_CLOCK _IOW(KVMIO, 0x7b, struct kvm_clock_data)
> #define KVM_GET_CLOCK _IOR(KVMIO, 0x7c, struct kvm_clock_data)
> +/* Available with KVM_CAP_MSIX_MASK */
> +#define KVM_GET_MSIX_ENTRY _IOWR(KVMIO, 0x7d, struct kvm_msix_entry)
> +#define KVM_UPDATE_MSIX_MMIO _IOW(KVMIO, 0x7e, struct kvm_msix_mmio)
> /* Available with KVM_CAP_PIT_STATE2 */
> #define KVM_GET_PIT2 _IOR(KVMIO, 0x9f, struct kvm_pit_state2)
> #define KVM_SET_PIT2 _IOW(KVMIO, 0xa0, struct kvm_pit_state2)
> @@ -795,4 +801,30 @@ struct kvm_assigned_msix_entry {
> __u16 padding[3];
> };
>
> +#define KVM_MSIX_TYPE_ASSIGNED_DEV 1
> +
> +#define KVM_MSIX_FLAG_MASKBIT (1 << 0)
> +#define KVM_MSIX_FLAG_QUERY_MASKBIT (1 << 0)
> +
> +struct kvm_msix_entry {
> + __u32 id;
> + __u32 type;
Is type really necessary? Will it ever differ from
KVM_MSIX_TYPE_ASSIGNED_DEV?
> + __u32 entry; /* The index of entry in the MSI-X table */
> + __u32 flags;
> + __u32 query_flags;
> + __u32 reserved[5];
> +};
> +
> +#define KVM_MSIX_MMIO_FLAG_REGISTER (1 << 0)
> +#define KVM_MSIX_MMIO_FLAG_UNREGISTER (1 << 1)
> +
> +struct kvm_msix_mmio {
> + __u32 id;
> + __u32 type;
> + __u64 base_addr;
> + __u32 max_entries_nr;
> + __u32 flags;
> + __u32 reserved[6];
> +};
> +
> #endif /* __LINUX_KVM_H */
> diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> index f09db87..57a437a 100644
> --- a/include/linux/kvm_host.h
> +++ b/include/linux/kvm_host.h
> @@ -501,6 +501,7 @@ struct kvm_guest_msix_entry {
> };
>
> #define KVM_ASSIGNED_ENABLED_IOMMU (1 << 0)
> +#define KVM_ASSIGNED_ENABLED_MSIX_MMIO (1 << 1)
> struct kvm_assigned_dev_kernel {
> struct kvm_irq_ack_notifier ack_notifier;
> struct work_struct interrupt_work;
> @@ -521,6 +522,10 @@ struct kvm_assigned_dev_kernel {
> struct pci_dev *dev;
> struct kvm *kvm;
> spinlock_t assigned_dev_lock;
> + DECLARE_BITMAP(msix_mask_bitmap, KVM_MAX_MSIX_PER_DEV);
> + gpa_t msix_mmio_base;
> + struct kvm_io_device msix_mmio_dev;
> + int msix_max_entries_nr;
> };
>
> struct kvm_irq_mask_notifier {
> diff --git a/virt/kvm/assigned-dev.c b/virt/kvm/assigned-dev.c
> index 5c6b96d..76a1f12 100644
> --- a/virt/kvm/assigned-dev.c
> +++ b/virt/kvm/assigned-dev.c
> @@ -226,12 +226,27 @@ static void kvm_free_assigned_irq(struct kvm *kvm,
> kvm_deassign_irq(kvm, assigned_dev, assigned_dev->irq_requested_type);
> }
>
> +static void unregister_msix_mmio(struct kvm *kvm,
> + struct kvm_assigned_dev_kernel *adev)
> +{
> + if (adev->flags & KVM_ASSIGNED_ENABLED_MSIX_MMIO) {
> + mutex_lock(&kvm->slots_lock);
> + kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
> + &adev->msix_mmio_dev);
> + mutex_unlock(&kvm->slots_lock);
> + adev->flags &= ~KVM_ASSIGNED_ENABLED_MSIX_MMIO;
> + }
> +}
> +
> static void kvm_free_assigned_device(struct kvm *kvm,
> struct kvm_assigned_dev_kernel
> *assigned_dev)
> {
> kvm_free_assigned_irq(kvm, assigned_dev);
>
> +#ifdef __KVM_HAVE_MSIX
> + unregister_msix_mmio(kvm, assigned_dev);
> +#endif
> pci_reset_function(assigned_dev->dev);
>
> pci_release_regions(assigned_dev->dev);
> @@ -504,7 +519,7 @@ out:
> static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
> struct kvm_assigned_pci_dev *assigned_dev)
> {
> - int r = 0, idx;
> + int r = 0, idx, i;
> struct kvm_assigned_dev_kernel *match;
> struct pci_dev *dev;
>
> @@ -564,6 +579,10 @@ static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
>
> list_add(&match->list, &kvm->arch.assigned_dev_head);
>
> + /* The state after reset of MSI-X table is all masked */
> + for (i = 0; i < KVM_MAX_MSIX_PER_DEV; i++)
> + set_bit(i, match->msix_mask_bitmap);
> +
> if (assigned_dev->flags & KVM_ASSIGNED_ENABLED_IOMMU) {
> if (!kvm->arch.iommu_domain) {
> r = kvm_iommu_map_guest(kvm);
> @@ -667,6 +686,43 @@ msix_nr_out:
> return r;
> }
>
> +static void update_msix_mask(struct kvm_assigned_dev_kernel *adev,
> + int idx, bool new_mask_flag)
> +{
> + int irq;
> + bool old_mask_flag, need_flush = false;
> +
> + spin_lock_irq(&adev->assigned_dev_lock);
> +
> + if (!adev->dev->msix_enabled ||
> + !(adev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX))
> + goto out;
> +
> + old_mask_flag = test_bit(adev->guest_msix_entries[idx].entry,
> + adev->msix_mask_bitmap);
> + if (old_mask_flag == new_mask_flag)
> + goto out;
> +
> + irq = adev->host_msix_entries[idx].vector;
> + BUG_ON(irq == 0);
> +
> + if (new_mask_flag) {
> + set_bit(adev->guest_msix_entries[idx].entry,
> + adev->msix_mask_bitmap);
> + disable_irq_nosync(irq);
> + need_flush = true;
> + } else {
> + clear_bit(adev->guest_msix_entries[idx].entry,
> + adev->msix_mask_bitmap);
> + enable_irq(irq);
> + }
> +out:
> + spin_unlock_irq(&adev->assigned_dev_lock);
> +
> + if (need_flush)
> + flush_work(&adev->interrupt_work);
> +}
> +
> static int kvm_vm_ioctl_set_msix_entry(struct kvm *kvm,
> struct kvm_assigned_msix_entry *entry)
> {
> @@ -701,6 +757,235 @@ msix_entry_out:
>
> return r;
> }
> +
> +static int kvm_vm_ioctl_get_msix_entry(struct kvm *kvm,
> + struct kvm_msix_entry *entry)
> +{
> + int r = 0;
> + struct kvm_assigned_dev_kernel *adev;
> +
> + if (entry->type != KVM_MSIX_TYPE_ASSIGNED_DEV)
> + return -EINVAL;
> +
> + if (!entry->query_flags)
> + return -EINVAL;
> +
> + mutex_lock(&kvm->lock);
> +
> + adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
> + entry->id);
> +
> + if (!adev) {
> + r = -EINVAL;
> + goto out;
> + }
> +
> + if (entry->entry >= adev->msix_max_entries_nr) {
> + r = -ENOSPC;
> + goto out;
> + }
> +
> + if (entry->query_flags & KVM_MSIX_FLAG_QUERY_MASKBIT) {
> + if (test_bit(entry->entry, adev->msix_mask_bitmap))
> + entry->flags |= KVM_MSIX_FLAG_MASKBIT;
> + else
> + entry->flags &= ~KVM_MSIX_FLAG_MASKBIT;
> + }
> +
> +out:
> + mutex_unlock(&kvm->lock);
> +
> + return r;
> +}
> +
> +static bool msix_mmio_in_range(struct kvm_assigned_dev_kernel *adev,
> + gpa_t addr, int len)
> +{
> + gpa_t start, end;
> +
> + BUG_ON(!(adev->flags & KVM_ASSIGNED_ENABLED_MSIX_MMIO));
> + start = adev->msix_mmio_base;
> + end = adev->msix_mmio_base + PCI_MSIX_ENTRY_SIZE *
> + adev->msix_max_entries_nr;
> + if (addr >= start && addr + len <= end)
> + return true;
> +
> + return false;
> +}
> +
> +static int msix_get_enabled_idx(struct kvm_assigned_dev_kernel *adev,
> + gpa_t addr, int len)
> +{
> + int i, index = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
> +
> + for (i = 0; i < adev->entries_nr; i++)
> + if (adev->guest_msix_entries[i].entry == index)
> + return i;
> +
> + return -EINVAL;
> +}
> +
> +static int msix_mmio_read(struct kvm_io_device *this, gpa_t addr, int len,
> + void *val)
> +{
> + struct kvm_assigned_dev_kernel *adev =
> + container_of(this, struct kvm_assigned_dev_kernel,
> + msix_mmio_dev);
> + int idx, r = 0;
> + u32 entry[4];
> + struct kvm_kernel_irq_routing_entry e;
> +
> + /* TODO: Get big-endian machine work */
> + mutex_lock(&adev->kvm->lock);
> + if (!msix_mmio_in_range(adev, addr, len)) {
> + r = -EOPNOTSUPP;
> + goto out;
> + }
The unregister path does:
mutex_lock(kvm->lock)
kvm_io_bus_unregister_dev()
synchronize_srcu()
If an instance of msix_mmio_read/msix_mmio_write is waiting on
kvm->lock, synchronize_srcu will never complete.
You should use a separate lock for the in range check (and have it mind
that reads/writes can trigger after kvm_io_bus_register_dev, so all
state accessible in the r/w handlers should be complete by that time).
> + if ((addr & 0x3) || len != 4)
> + goto out;
> +
> + idx = msix_get_enabled_idx(adev, addr, len);
> + if (idx < 0) {
> + idx = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
> + if ((addr % PCI_MSIX_ENTRY_SIZE) ==
> + PCI_MSIX_ENTRY_VECTOR_CTRL)
> + *(unsigned long *)val =
> + test_bit(idx, adev->msix_mask_bitmap) ?
> + PCI_MSIX_ENTRY_CTRL_MASKBIT : 0;
> + else
> + r = -EOPNOTSUPP;
> + goto out;
> + }
> +
> + r = kvm_get_irq_routing_entry(adev->kvm,
> + adev->guest_msix_entries[idx].vector, &e);
> + if (r || e.type != KVM_IRQ_ROUTING_MSI) {
> + r = -EOPNOTSUPP;
> + goto out;
> + }
> + entry[0] = e.msi.address_lo;
> + entry[1] = e.msi.address_hi;
> + entry[2] = e.msi.data;
> + entry[3] = test_bit(adev->guest_msix_entries[idx].entry,
> + adev->msix_mask_bitmap);
> + memcpy(val, &entry[addr % PCI_MSIX_ENTRY_SIZE / sizeof *entry], len);
Division by zero?
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-16 19:45 ` [PATCH 6/6] " Marcelo Tosatti
@ 2010-11-17 1:29 ` Sheng Yang
2010-11-17 13:35 ` Marcelo Tosatti
2010-11-18 9:43 ` Michael S. Tsirkin
0 siblings, 2 replies; 26+ messages in thread
From: Sheng Yang @ 2010-11-17 1:29 UTC (permalink / raw)
To: Marcelo Tosatti; +Cc: Avi Kivity, Michael S. Tsirkin, kvm
On Wednesday 17 November 2010 03:45:22 Marcelo Tosatti wrote:
> On Mon, Nov 15, 2010 at 05:15:32PM +0800, Sheng Yang wrote:
> > This patch enable per-vector mask for assigned devices using MSI-X.
> >
> > This patch provided two new APIs: one is for guest to specific device's
> > MSI-X table address in MMIO, the other is for userspace to get
> > information about mask bit.
> >
> > All the mask bit operation are kept in kernel, in order to accelerate.
> > Userspace shouldn't access the device MMIO directly for the information,
> > instead it should uses provided API to do so.
> >
> > Signed-off-by: Sheng Yang <sheng@linux.intel.com>
> > ---
> >
> > arch/x86/kvm/x86.c | 1 +
> > include/linux/kvm.h | 32 +++++
> > include/linux/kvm_host.h | 5 +
> > virt/kvm/assigned-dev.c | 318
> > +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed,
355
> > insertions(+), 1 deletions(-)
> >
> > diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> > index fc29223..37602e2 100644
> > --- a/arch/x86/kvm/x86.c
> > +++ b/arch/x86/kvm/x86.c
> > @@ -1966,6 +1966,7 @@ int kvm_dev_ioctl_check_extension(long ext)
> >
> > case KVM_CAP_X86_ROBUST_SINGLESTEP:
> > case KVM_CAP_XSAVE:
> >
> > case KVM_CAP_ASYNC_PF:
> > + case KVM_CAP_MSIX_MASK:
> > r = 1;
> > break;
> >
> > case KVM_CAP_COALESCED_MMIO:
> > diff --git a/include/linux/kvm.h b/include/linux/kvm.h
> > index ea2dc1a..b3e5ffe 100644
> > --- a/include/linux/kvm.h
> > +++ b/include/linux/kvm.h
> > @@ -541,6 +541,9 @@ struct kvm_ppc_pvinfo {
> >
> > #define KVM_CAP_PPC_GET_PVINFO 57
> > #define KVM_CAP_PPC_IRQ_LEVEL 58
> > #define KVM_CAP_ASYNC_PF 59
> >
> > +#ifdef __KVM_HAVE_MSIX
> > +#define KVM_CAP_MSIX_MASK 60
> > +#endif
> >
> > #ifdef KVM_CAP_IRQ_ROUTING
> >
> > @@ -672,6 +675,9 @@ struct kvm_clock_data {
> >
> > #define KVM_XEN_HVM_CONFIG _IOW(KVMIO, 0x7a, struct
> > kvm_xen_hvm_config) #define KVM_SET_CLOCK _IOW(KVMIO,
> > 0x7b, struct kvm_clock_data) #define KVM_GET_CLOCK
> > _IOR(KVMIO, 0x7c, struct kvm_clock_data)
> >
> > +/* Available with KVM_CAP_MSIX_MASK */
> > +#define KVM_GET_MSIX_ENTRY _IOWR(KVMIO, 0x7d, struct
> > kvm_msix_entry) +#define KVM_UPDATE_MSIX_MMIO _IOW(KVMIO, 0x7e,
> > struct kvm_msix_mmio)
> >
> > /* Available with KVM_CAP_PIT_STATE2 */
> > #define KVM_GET_PIT2 _IOR(KVMIO, 0x9f, struct
> > kvm_pit_state2) #define KVM_SET_PIT2 _IOW(KVMIO, 0xa0,
> > struct kvm_pit_state2)
> >
> > @@ -795,4 +801,30 @@ struct kvm_assigned_msix_entry {
> >
> > __u16 padding[3];
> >
> > };
> >
> > +#define KVM_MSIX_TYPE_ASSIGNED_DEV 1
> > +
> > +#define KVM_MSIX_FLAG_MASKBIT (1 << 0)
> > +#define KVM_MSIX_FLAG_QUERY_MASKBIT (1 << 0)
> > +
> > +struct kvm_msix_entry {
> > + __u32 id;
> > + __u32 type;
>
> Is type really necessary? Will it ever differ from
> KVM_MSIX_TYPE_ASSIGNED_DEV?
This is the suggestion from Michael. He want it to be reused by emulated/pv
devices. So I add the type field here.
>
> > + __u32 entry; /* The index of entry in the MSI-X table */
> > + __u32 flags;
> > + __u32 query_flags;
> > + __u32 reserved[5];
> > +};
> > +
> > +#define KVM_MSIX_MMIO_FLAG_REGISTER (1 << 0)
> > +#define KVM_MSIX_MMIO_FLAG_UNREGISTER (1 << 1)
> > +
> > +struct kvm_msix_mmio {
> > + __u32 id;
> > + __u32 type;
> > + __u64 base_addr;
> > + __u32 max_entries_nr;
> > + __u32 flags;
> > + __u32 reserved[6];
> > +};
> > +
> >
> > #endif /* __LINUX_KVM_H */
> >
> > diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
> > index f09db87..57a437a 100644
> > --- a/include/linux/kvm_host.h
> > +++ b/include/linux/kvm_host.h
> > @@ -501,6 +501,7 @@ struct kvm_guest_msix_entry {
> >
> > };
> >
> > #define KVM_ASSIGNED_ENABLED_IOMMU (1 << 0)
> >
> > +#define KVM_ASSIGNED_ENABLED_MSIX_MMIO (1 << 1)
> >
> > struct kvm_assigned_dev_kernel {
> >
> > struct kvm_irq_ack_notifier ack_notifier;
> > struct work_struct interrupt_work;
> >
> > @@ -521,6 +522,10 @@ struct kvm_assigned_dev_kernel {
> >
> > struct pci_dev *dev;
> > struct kvm *kvm;
> > spinlock_t assigned_dev_lock;
> >
> > + DECLARE_BITMAP(msix_mask_bitmap, KVM_MAX_MSIX_PER_DEV);
> > + gpa_t msix_mmio_base;
> > + struct kvm_io_device msix_mmio_dev;
> > + int msix_max_entries_nr;
> >
> > };
> >
> > struct kvm_irq_mask_notifier {
> >
> > diff --git a/virt/kvm/assigned-dev.c b/virt/kvm/assigned-dev.c
> > index 5c6b96d..76a1f12 100644
> > --- a/virt/kvm/assigned-dev.c
> > +++ b/virt/kvm/assigned-dev.c
> > @@ -226,12 +226,27 @@ static void kvm_free_assigned_irq(struct kvm *kvm,
> >
> > kvm_deassign_irq(kvm, assigned_dev, assigned_dev->irq_requested_type);
> >
> > }
> >
> > +static void unregister_msix_mmio(struct kvm *kvm,
> > + struct kvm_assigned_dev_kernel *adev)
> > +{
> > + if (adev->flags & KVM_ASSIGNED_ENABLED_MSIX_MMIO) {
> > + mutex_lock(&kvm->slots_lock);
> > + kvm_io_bus_unregister_dev(kvm, KVM_MMIO_BUS,
> > + &adev->msix_mmio_dev);
> > + mutex_unlock(&kvm->slots_lock);
> > + adev->flags &= ~KVM_ASSIGNED_ENABLED_MSIX_MMIO;
> > + }
> > +}
> > +
> >
> > static void kvm_free_assigned_device(struct kvm *kvm,
> >
> > struct kvm_assigned_dev_kernel
> > *assigned_dev)
> >
> > {
> >
> > kvm_free_assigned_irq(kvm, assigned_dev);
> >
> > +#ifdef __KVM_HAVE_MSIX
> > + unregister_msix_mmio(kvm, assigned_dev);
> > +#endif
> >
> > pci_reset_function(assigned_dev->dev);
> >
> > pci_release_regions(assigned_dev->dev);
> >
> > @@ -504,7 +519,7 @@ out:
> > static int kvm_vm_ioctl_assign_device(struct kvm *kvm,
> >
> > struct kvm_assigned_pci_dev *assigned_dev)
> >
> > {
> >
> > - int r = 0, idx;
> > + int r = 0, idx, i;
> >
> > struct kvm_assigned_dev_kernel *match;
> > struct pci_dev *dev;
> >
> > @@ -564,6 +579,10 @@ static int kvm_vm_ioctl_assign_device(struct kvm
> > *kvm,
> >
> > list_add(&match->list, &kvm->arch.assigned_dev_head);
> >
> > + /* The state after reset of MSI-X table is all masked */
> > + for (i = 0; i < KVM_MAX_MSIX_PER_DEV; i++)
> > + set_bit(i, match->msix_mask_bitmap);
> > +
> >
> > if (assigned_dev->flags & KVM_ASSIGNED_ENABLED_IOMMU) {
> >
> > if (!kvm->arch.iommu_domain) {
> >
> > r = kvm_iommu_map_guest(kvm);
> >
> > @@ -667,6 +686,43 @@ msix_nr_out:
> > return r;
> >
> > }
> >
> > +static void update_msix_mask(struct kvm_assigned_dev_kernel *adev,
> > + int idx, bool new_mask_flag)
> > +{
> > + int irq;
> > + bool old_mask_flag, need_flush = false;
> > +
> > + spin_lock_irq(&adev->assigned_dev_lock);
> > +
> > + if (!adev->dev->msix_enabled ||
> > + !(adev->irq_requested_type & KVM_DEV_IRQ_HOST_MSIX))
> > + goto out;
> > +
> > + old_mask_flag = test_bit(adev->guest_msix_entries[idx].entry,
> > + adev->msix_mask_bitmap);
> > + if (old_mask_flag == new_mask_flag)
> > + goto out;
> > +
> > + irq = adev->host_msix_entries[idx].vector;
> > + BUG_ON(irq == 0);
> > +
> > + if (new_mask_flag) {
> > + set_bit(adev->guest_msix_entries[idx].entry,
> > + adev->msix_mask_bitmap);
> > + disable_irq_nosync(irq);
> > + need_flush = true;
> > + } else {
> > + clear_bit(adev->guest_msix_entries[idx].entry,
> > + adev->msix_mask_bitmap);
> > + enable_irq(irq);
> > + }
> > +out:
> > + spin_unlock_irq(&adev->assigned_dev_lock);
> > +
> > + if (need_flush)
> > + flush_work(&adev->interrupt_work);
> > +}
> > +
> >
> > static int kvm_vm_ioctl_set_msix_entry(struct kvm *kvm,
> >
> > struct kvm_assigned_msix_entry *entry)
> >
> > {
> >
> > @@ -701,6 +757,235 @@ msix_entry_out:
> > return r;
> >
> > }
> >
> > +
> > +static int kvm_vm_ioctl_get_msix_entry(struct kvm *kvm,
> > + struct kvm_msix_entry *entry)
> > +{
> > + int r = 0;
> > + struct kvm_assigned_dev_kernel *adev;
> > +
> > + if (entry->type != KVM_MSIX_TYPE_ASSIGNED_DEV)
> > + return -EINVAL;
> > +
> > + if (!entry->query_flags)
> > + return -EINVAL;
> > +
> > + mutex_lock(&kvm->lock);
> > +
> > + adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
> > + entry->id);
> > +
> > + if (!adev) {
> > + r = -EINVAL;
> > + goto out;
> > + }
> > +
> > + if (entry->entry >= adev->msix_max_entries_nr) {
> > + r = -ENOSPC;
> > + goto out;
> > + }
> > +
> > + if (entry->query_flags & KVM_MSIX_FLAG_QUERY_MASKBIT) {
> > + if (test_bit(entry->entry, adev->msix_mask_bitmap))
> > + entry->flags |= KVM_MSIX_FLAG_MASKBIT;
> > + else
> > + entry->flags &= ~KVM_MSIX_FLAG_MASKBIT;
> > + }
> > +
> > +out:
> > + mutex_unlock(&kvm->lock);
> > +
> > + return r;
> > +}
> > +
> > +static bool msix_mmio_in_range(struct kvm_assigned_dev_kernel *adev,
> > + gpa_t addr, int len)
> > +{
> > + gpa_t start, end;
> > +
> > + BUG_ON(!(adev->flags & KVM_ASSIGNED_ENABLED_MSIX_MMIO));
> > + start = adev->msix_mmio_base;
> > + end = adev->msix_mmio_base + PCI_MSIX_ENTRY_SIZE *
> > + adev->msix_max_entries_nr;
> > + if (addr >= start && addr + len <= end)
> > + return true;
> > +
> > + return false;
> > +}
> > +
> > +static int msix_get_enabled_idx(struct kvm_assigned_dev_kernel *adev,
> > + gpa_t addr, int len)
> > +{
> > + int i, index = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
> > +
> > + for (i = 0; i < adev->entries_nr; i++)
> > + if (adev->guest_msix_entries[i].entry == index)
> > + return i;
> > +
> > + return -EINVAL;
> > +}
> > +
> > +static int msix_mmio_read(struct kvm_io_device *this, gpa_t addr, int
> > len, + void *val)
> > +{
> > + struct kvm_assigned_dev_kernel *adev =
> > + container_of(this, struct kvm_assigned_dev_kernel,
> > + msix_mmio_dev);
> > + int idx, r = 0;
> > + u32 entry[4];
> > + struct kvm_kernel_irq_routing_entry e;
> > +
> > + /* TODO: Get big-endian machine work */
> > + mutex_lock(&adev->kvm->lock);
> > + if (!msix_mmio_in_range(adev, addr, len)) {
> > + r = -EOPNOTSUPP;
> > + goto out;
> > + }
>
> The unregister path does:
>
> mutex_lock(kvm->lock)
> kvm_io_bus_unregister_dev()
> synchronize_srcu()
>
> If an instance of msix_mmio_read/msix_mmio_write is waiting on
> kvm->lock, synchronize_srcu will never complete.
>
> You should use a separate lock for the in range check (and have it mind
> that reads/writes can trigger after kvm_io_bus_register_dev, so all
> state accessible in the r/w handlers should be complete by that time).
Good point! Would update it.
>
> > + if ((addr & 0x3) || len != 4)
> > + goto out;
> > +
> > + idx = msix_get_enabled_idx(adev, addr, len);
> > + if (idx < 0) {
> > + idx = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
> > + if ((addr % PCI_MSIX_ENTRY_SIZE) ==
> > + PCI_MSIX_ENTRY_VECTOR_CTRL)
> > + *(unsigned long *)val =
> > + test_bit(idx, adev->msix_mask_bitmap) ?
> > + PCI_MSIX_ENTRY_CTRL_MASKBIT : 0;
> > + else
> > + r = -EOPNOTSUPP;
> > + goto out;
> > + }
> > +
> > + r = kvm_get_irq_routing_entry(adev->kvm,
> > + adev->guest_msix_entries[idx].vector, &e);
> > + if (r || e.type != KVM_IRQ_ROUTING_MSI) {
> > + r = -EOPNOTSUPP;
> > + goto out;
> > + }
> > + entry[0] = e.msi.address_lo;
> > + entry[1] = e.msi.address_hi;
> > + entry[2] = e.msi.data;
> > + entry[3] = test_bit(adev->guest_msix_entries[idx].entry,
> > + adev->msix_mask_bitmap);
> > + memcpy(val, &entry[addr % PCI_MSIX_ENTRY_SIZE / sizeof *entry], len);
>
> Division by zero?
Not quite understand. You mean sizeof *entry or PCI_MSIX_ENTRY_SIZE? Both of them
should be positive integer I think... Maybe I should use sizeof u32 here?
--
regards
Yang, Sheng
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-17 1:29 ` Sheng Yang
@ 2010-11-17 13:35 ` Marcelo Tosatti
2010-11-18 9:43 ` Michael S. Tsirkin
1 sibling, 0 replies; 26+ messages in thread
From: Marcelo Tosatti @ 2010-11-17 13:35 UTC (permalink / raw)
To: Sheng Yang; +Cc: Avi Kivity, Michael S. Tsirkin, kvm
On Wed, Nov 17, 2010 at 09:29:22AM +0800, Sheng Yang wrote:
> > > + adev->msix_mask_bitmap);
> > > + memcpy(val, &entry[addr % PCI_MSIX_ENTRY_SIZE / sizeof *entry], len);
> >
> > Division by zero?
>
> Not quite understand. You mean sizeof *entry or PCI_MSIX_ENTRY_SIZE? Both of them
> should be positive integer I think... Maybe I should use sizeof u32 here?
You're right, nevermind.
^ permalink raw reply [flat|nested] 26+ messages in thread
* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-17 1:29 ` Sheng Yang
2010-11-17 13:35 ` Marcelo Tosatti
@ 2010-11-18 9:43 ` Michael S. Tsirkin
1 sibling, 0 replies; 26+ messages in thread
From: Michael S. Tsirkin @ 2010-11-18 9:43 UTC (permalink / raw)
To: Sheng Yang; +Cc: Marcelo Tosatti, Avi Kivity, kvm
On Wed, Nov 17, 2010 at 09:29:22AM +0800, Sheng Yang wrote:
> > > +#define KVM_MSIX_TYPE_ASSIGNED_DEV 1
> > > +
> > > +#define KVM_MSIX_FLAG_MASKBIT (1 << 0)
> > > +#define KVM_MSIX_FLAG_QUERY_MASKBIT (1 << 0)
> > > +
> > > +struct kvm_msix_entry {
> > > + __u32 id;
> > > + __u32 type;
> >
> > Is type really necessary? Will it ever differ from
> > KVM_MSIX_TYPE_ASSIGNED_DEV?
>
> This is the suggestion from Michael. He want it to be reused by emulated/pv
> devices. So I add the type field here.
Maybe id field can be reused for this somehow.
--
MST
^ permalink raw reply [flat|nested] 26+ messages in thread
* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-15 9:15 ` [PATCH 6/6] KVM: assigned dev: MSI-X mask support Sheng Yang
2010-11-15 9:27 ` [PATCH 6/6 v5 updated] " Sheng Yang
2010-11-16 19:45 ` [PATCH 6/6] " Marcelo Tosatti
@ 2010-11-17 13:58 ` Avi Kivity
2010-11-18 1:58 ` Sheng Yang
2 siblings, 1 reply; 26+ messages in thread
From: Avi Kivity @ 2010-11-17 13:58 UTC (permalink / raw)
To: Sheng Yang; +Cc: Marcelo Tosatti, Michael S. Tsirkin, kvm, Alex Williamson
On 11/15/2010 11:15 AM, Sheng Yang wrote:
> This patch enable per-vector mask for assigned devices using MSI-X.
>
> This patch provided two new APIs: one is for guest to specific device's MSI-X
> table address in MMIO, the other is for userspace to get information about mask
> bit.
>
> All the mask bit operation are kept in kernel, in order to accelerate.
> Userspace shouldn't access the device MMIO directly for the information,
> instead it should uses provided API to do so.
>
> Signed-off-by: Sheng Yang<sheng@linux.intel.com>
> ---
> arch/x86/kvm/x86.c | 1 +
> include/linux/kvm.h | 32 +++++
> include/linux/kvm_host.h | 5 +
> virt/kvm/assigned-dev.c | 318 +++++++++++++++++++++++++++++++++++++++++++++-
> 4 files changed, 355 insertions(+), 1 deletions(-)
>
Documentation?
> +static int msix_mmio_read(struct kvm_io_device *this, gpa_t addr, int len,
> + void *val)
> +{
> + struct kvm_assigned_dev_kernel *adev =
> + container_of(this, struct kvm_assigned_dev_kernel,
> + msix_mmio_dev);
> + int idx, r = 0;
> + u32 entry[4];
> + struct kvm_kernel_irq_routing_entry e;
> +
> + /* TODO: Get big-endian machine work */
> + mutex_lock(&adev->kvm->lock);
> + if (!msix_mmio_in_range(adev, addr, len)) {
> + r = -EOPNOTSUPP;
> + goto out;
> + }
> + if ((addr& 0x3) || len != 4)
> + goto out;
> +
> + idx = msix_get_enabled_idx(adev, addr, len);
> + if (idx< 0) {
> + idx = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
> + if ((addr % PCI_MSIX_ENTRY_SIZE) ==
> + PCI_MSIX_ENTRY_VECTOR_CTRL)
> + *(unsigned long *)val =
> + test_bit(idx, adev->msix_mask_bitmap) ?
> + PCI_MSIX_ENTRY_CTRL_MASKBIT : 0;
> + else
> + r = -EOPNOTSUPP;
> + goto out;
> + }
> +
> + r = kvm_get_irq_routing_entry(adev->kvm,
> + adev->guest_msix_entries[idx].vector,&e);
> + if (r || e.type != KVM_IRQ_ROUTING_MSI) {
> + r = -EOPNOTSUPP;
> + goto out;
> + }
> + entry[0] = e.msi.address_lo;
> + entry[1] = e.msi.address_hi;
> + entry[2] = e.msi.data;
> + entry[3] = test_bit(adev->guest_msix_entries[idx].entry,
> + adev->msix_mask_bitmap);
> + memcpy(val,&entry[addr % PCI_MSIX_ENTRY_SIZE / sizeof *entry], len);
> +
> +out:
> + mutex_unlock(&adev->kvm->lock);
> + return r;
> +}
> +
> +static int msix_mmio_write(struct kvm_io_device *this, gpa_t addr, int len,
> + const void *val)
> +{
> + struct kvm_assigned_dev_kernel *adev =
> + container_of(this, struct kvm_assigned_dev_kernel,
> + msix_mmio_dev);
> + int idx, r = 0;
> + unsigned long new_val = *(unsigned long *)val;
What if it's a 64-bit write on a 32-bit host?
Are we sure the trailing bytes of val are zero?
> +
> + /* TODO: Get big-endian machine work */
BUILD_BUG_ON(something)
> + mutex_lock(&adev->kvm->lock);
> + if (!msix_mmio_in_range(adev, addr, len)) {
> + r = -EOPNOTSUPP;
> + goto out;
> + }
Why is this needed? Didn't the iodev check already do this?
> + if ((addr& 0x3) || len != 4)
> + goto out;
What if len == 8? I think mst said it was legal.
> +
> + idx = msix_get_enabled_idx(adev, addr, len);
> + if (idx< 0) {
> + idx = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
> + if (((addr % PCI_MSIX_ENTRY_SIZE) ==
> + PCI_MSIX_ENTRY_VECTOR_CTRL)) {
> + if (new_val& ~PCI_MSIX_ENTRY_CTRL_MASKBIT)
> + goto out;
> + if (new_val& PCI_MSIX_ENTRY_CTRL_MASKBIT)
> + set_bit(idx, adev->msix_mask_bitmap);
> + else
> + clear_bit(idx, adev->msix_mask_bitmap);
> + /* It's possible that we need re-enable MSI-X, so go
> + * back to userspace */
> + }
> + /* Userspace would handle other MMIO writing */
> + r = -EOPNOTSUPP;
That's not very good. We should do the entire thing in the kernel or in
userspace. We can have a new EXIT_REASON to let userspace know an msix
entry changed, and it should read it from the kernel.
> + goto out;
> + }
> + if (addr % PCI_MSIX_ENTRY_SIZE != PCI_MSIX_ENTRY_VECTOR_CTRL) {
> + r = -EOPNOTSUPP;
> + goto out;
> + }
> + if (new_val& ~PCI_MSIX_ENTRY_CTRL_MASKBIT)
> + goto out;
> + update_msix_mask(adev, idx, !!(new_val& PCI_MSIX_ENTRY_CTRL_MASKBIT));
> +out:
> + mutex_unlock(&adev->kvm->lock);
> +
> + return r;
> +}
> +
> +static int kvm_vm_ioctl_update_msix_mmio(struct kvm *kvm,
> + struct kvm_msix_mmio *msix_mmio)
> +{
> + int r = 0;
> + struct kvm_assigned_dev_kernel *adev;
> +
> + if (msix_mmio->type != KVM_MSIX_TYPE_ASSIGNED_DEV)
> + return -EINVAL;
> +
> + if (!msix_mmio->flags)
> + return -EINVAL;
Need to check flags for undefined bits too.
> +
> + mutex_lock(&kvm->lock);
> + adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
> + msix_mmio->id);
> + if (!adev) {
> + r = -EINVAL;
> + goto out;
> + }
> + if (msix_mmio->base_addr == 0) {
> + r = -EINVAL;
> + goto out;
> + }
> + if (msix_mmio->max_entries_nr == 0 ||
> + msix_mmio->max_entries_nr> KVM_MAX_MSIX_PER_DEV) {
> + r = -EINVAL;
> + goto out;
> + }
> +
> + if ((msix_mmio->flags& KVM_MSIX_MMIO_FLAG_REGISTER)&&
> + (msix_mmio->flags& KVM_MSIX_MMIO_FLAG_UNREGISTER)) {
> + r = -EINVAL;
> + goto out;
> + }
> +
> + if (msix_mmio->flags& KVM_MSIX_MMIO_FLAG_REGISTER) {
> + if (!(adev->flags& KVM_ASSIGNED_ENABLED_MSIX_MMIO)) {
> + mutex_lock(&kvm->slots_lock);
> + kvm_iodevice_init(&adev->msix_mmio_dev,
> + &msix_mmio_ops);
> + r = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
> + &adev->msix_mmio_dev);
> + if (!r)
> + adev->flags |= KVM_ASSIGNED_ENABLED_MSIX_MMIO;
> + mutex_unlock(&kvm->slots_lock);
> + }
> + if (!r) {
> + adev->msix_mmio_base = msix_mmio->base_addr;
> + adev->msix_max_entries_nr = msix_mmio->max_entries_nr;
> + }
> + } else if (msix_mmio->flags& KVM_MSIX_MMIO_FLAG_UNREGISTER)
> + unregister_msix_mmio(kvm, adev);
> +out:
> + mutex_unlock(&kvm->lock);
> +
> + return r;
> +}
> #endif
Question: how do we do this with vfio? One idea is to have
ioctl(vmfd, KVM_IO_PIPE, { KVM_IO_MMIO /* or KVM_IO_PIO */, range,
pipefd1, pipefd2 })
and have kvm convert mmio or pio in that range to commands and responses
sent over that pipe. We could have qemu threads, other userspace
processes, or kernel components like vfio implement the other side of
the channel.
--
error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-17 13:58 ` Avi Kivity
@ 2010-11-18 1:58 ` Sheng Yang
2010-11-18 6:21 ` Michael S. Tsirkin
2010-11-18 9:28 ` Avi Kivity
0 siblings, 2 replies; 26+ messages in thread
From: Sheng Yang @ 2010-11-18 1:58 UTC (permalink / raw)
To: Avi Kivity; +Cc: Marcelo Tosatti, Michael S. Tsirkin, kvm, Alex Williamson
On Wednesday 17 November 2010 21:58:00 Avi Kivity wrote:
> On 11/15/2010 11:15 AM, Sheng Yang wrote:
> > This patch enable per-vector mask for assigned devices using MSI-X.
> >
> > This patch provided two new APIs: one is for guest to specific device's
> > MSI-X table address in MMIO, the other is for userspace to get
> > information about mask bit.
> >
> > All the mask bit operation are kept in kernel, in order to accelerate.
> > Userspace shouldn't access the device MMIO directly for the information,
> > instead it should uses provided API to do so.
> >
> > Signed-off-by: Sheng Yang<sheng@linux.intel.com>
> > ---
> >
> > arch/x86/kvm/x86.c | 1 +
> > include/linux/kvm.h | 32 +++++
> > include/linux/kvm_host.h | 5 +
> > virt/kvm/assigned-dev.c | 318
> > +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed,
355
> > insertions(+), 1 deletions(-)
>
> Documentation?
For we are keeping changing the API for last several versions, I'd like to settle
down the API first. Would bring back the document after API was agreed.
>
> > +static int msix_mmio_read(struct kvm_io_device *this, gpa_t addr, int
> > len, + void *val)
> > +{
> > + struct kvm_assigned_dev_kernel *adev =
> > + container_of(this, struct kvm_assigned_dev_kernel,
> > + msix_mmio_dev);
> > + int idx, r = 0;
> > + u32 entry[4];
> > + struct kvm_kernel_irq_routing_entry e;
> > +
> > + /* TODO: Get big-endian machine work */
> > + mutex_lock(&adev->kvm->lock);
> > + if (!msix_mmio_in_range(adev, addr, len)) {
> > + r = -EOPNOTSUPP;
> > + goto out;
> > + }
> > + if ((addr& 0x3) || len != 4)
> > + goto out;
> > +
> > + idx = msix_get_enabled_idx(adev, addr, len);
> > + if (idx< 0) {
> > + idx = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
> > + if ((addr % PCI_MSIX_ENTRY_SIZE) ==
> > + PCI_MSIX_ENTRY_VECTOR_CTRL)
> > + *(unsigned long *)val =
> > + test_bit(idx, adev->msix_mask_bitmap) ?
> > + PCI_MSIX_ENTRY_CTRL_MASKBIT : 0;
> > + else
> > + r = -EOPNOTSUPP;
> > + goto out;
> > + }
> > +
> > + r = kvm_get_irq_routing_entry(adev->kvm,
> > + adev->guest_msix_entries[idx].vector,&e);
> > + if (r || e.type != KVM_IRQ_ROUTING_MSI) {
> > + r = -EOPNOTSUPP;
> > + goto out;
> > + }
> > + entry[0] = e.msi.address_lo;
> > + entry[1] = e.msi.address_hi;
> > + entry[2] = e.msi.data;
> > + entry[3] = test_bit(adev->guest_msix_entries[idx].entry,
> > + adev->msix_mask_bitmap);
> > + memcpy(val,&entry[addr % PCI_MSIX_ENTRY_SIZE / sizeof *entry], len);
> > +
> > +out:
> > + mutex_unlock(&adev->kvm->lock);
> > + return r;
> > +}
> > +
> > +static int msix_mmio_write(struct kvm_io_device *this, gpa_t addr, int
> > len, + const void *val)
> > +{
> > + struct kvm_assigned_dev_kernel *adev =
> > + container_of(this, struct kvm_assigned_dev_kernel,
> > + msix_mmio_dev);
> > + int idx, r = 0;
> > + unsigned long new_val = *(unsigned long *)val;
>
> What if it's a 64-bit write on a 32-bit host?
In fact we haven't support QWORD(64bit) accessing now. The reason is we haven't
seen any OS is using it in this way now, so I think we can leave it later.
Also seems QEmu doesn't got the way to handle 64bit MMIO.
>
> Are we sure the trailing bytes of val are zero?
>
> > +
> > + /* TODO: Get big-endian machine work */
>
> BUILD_BUG_ON(something)
Good idea!
>
> > + mutex_lock(&adev->kvm->lock);
> > + if (!msix_mmio_in_range(adev, addr, len)) {
> > + r = -EOPNOTSUPP;
> > + goto out;
> > + }
>
> Why is this needed? Didn't the iodev check already do this?
Well, kvm_io_device_ops() hasn't got "in_range" callback yet...
>
> > + if ((addr& 0x3) || len != 4)
> > + goto out;
>
> What if len == 8? I think mst said it was legal.
Since we haven't seen anyone is using it in this way, so I think we can leave it
later.
>
> > +
> > + idx = msix_get_enabled_idx(adev, addr, len);
> > + if (idx< 0) {
> > + idx = (addr - adev->msix_mmio_base) / PCI_MSIX_ENTRY_SIZE;
> > + if (((addr % PCI_MSIX_ENTRY_SIZE) ==
> > + PCI_MSIX_ENTRY_VECTOR_CTRL)) {
> > + if (new_val& ~PCI_MSIX_ENTRY_CTRL_MASKBIT)
> > + goto out;
> > + if (new_val& PCI_MSIX_ENTRY_CTRL_MASKBIT)
> > + set_bit(idx, adev->msix_mask_bitmap);
> > + else
> > + clear_bit(idx, adev->msix_mask_bitmap);
> > + /* It's possible that we need re-enable MSI-X, so go
> > + * back to userspace */
> > + }
> > + /* Userspace would handle other MMIO writing */
> > + r = -EOPNOTSUPP;
>
> That's not very good. We should do the entire thing in the kernel or in
> userspace. We can have a new EXIT_REASON to let userspace know an msix
> entry changed, and it should read it from the kernel.
If you look it in this way:
1. Mask bit owned by kernel.
2. Routing owned by userspace.
3. Read the routing in kernel is an speed up for normal operation - because kernel
can read from them.
So I think the logic here is clear to understand.
But if we can modify the routing in kernel, it would be raise some sync issues due
to both kernel and userspace own routing. So maybe the better solution is move the
routing to kernel.
We can do something like that later, because this patch won't be need much change.
>
> > + goto out;
> > + }
> > + if (addr % PCI_MSIX_ENTRY_SIZE != PCI_MSIX_ENTRY_VECTOR_CTRL) {
> > + r = -EOPNOTSUPP;
> > + goto out;
> > + }
> > + if (new_val& ~PCI_MSIX_ENTRY_CTRL_MASKBIT)
> > + goto out;
> > + update_msix_mask(adev, idx, !!(new_val& PCI_MSIX_ENTRY_CTRL_MASKBIT));
> > +out:
> > + mutex_unlock(&adev->kvm->lock);
> > +
> > + return r;
> > +}
> > +
> > +static int kvm_vm_ioctl_update_msix_mmio(struct kvm *kvm,
> > + struct kvm_msix_mmio *msix_mmio)
> > +{
> > + int r = 0;
> > + struct kvm_assigned_dev_kernel *adev;
> > +
> > + if (msix_mmio->type != KVM_MSIX_TYPE_ASSIGNED_DEV)
> > + return -EINVAL;
> > +
> > + if (!msix_mmio->flags)
> > + return -EINVAL;
>
> Need to check flags for undefined bits too.
There is a checking later, I can move it earlier.
--
regards
Yang, Sheng
>
> > +
> > + mutex_lock(&kvm->lock);
> > + adev = kvm_find_assigned_dev(&kvm->arch.assigned_dev_head,
> > + msix_mmio->id);
> > + if (!adev) {
> > + r = -EINVAL;
> > + goto out;
> > + }
> > + if (msix_mmio->base_addr == 0) {
> > + r = -EINVAL;
> > + goto out;
> > + }
> > + if (msix_mmio->max_entries_nr == 0 ||
> > + msix_mmio->max_entries_nr> KVM_MAX_MSIX_PER_DEV) {
> > + r = -EINVAL;
> > + goto out;
> > + }
> > +
> > + if ((msix_mmio->flags& KVM_MSIX_MMIO_FLAG_REGISTER)&&
> > + (msix_mmio->flags& KVM_MSIX_MMIO_FLAG_UNREGISTER)) {
> > + r = -EINVAL;
> > + goto out;
> > + }
> > +
> > + if (msix_mmio->flags& KVM_MSIX_MMIO_FLAG_REGISTER) {
> > + if (!(adev->flags& KVM_ASSIGNED_ENABLED_MSIX_MMIO)) {
> > + mutex_lock(&kvm->slots_lock);
> > + kvm_iodevice_init(&adev->msix_mmio_dev,
> > + &msix_mmio_ops);
> > + r = kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS,
> > + &adev->msix_mmio_dev);
> > + if (!r)
> > + adev->flags |= KVM_ASSIGNED_ENABLED_MSIX_MMIO;
> > + mutex_unlock(&kvm->slots_lock);
> > + }
> > + if (!r) {
> > + adev->msix_mmio_base = msix_mmio->base_addr;
> > + adev->msix_max_entries_nr = msix_mmio->max_entries_nr;
> > + }
> > + } else if (msix_mmio->flags& KVM_MSIX_MMIO_FLAG_UNREGISTER)
> > + unregister_msix_mmio(kvm, adev);
> > +out:
> > + mutex_unlock(&kvm->lock);
> > +
> > + return r;
> > +}
> >
> > #endif
>
> Question: how do we do this with vfio? One idea is to have
>
> ioctl(vmfd, KVM_IO_PIPE, { KVM_IO_MMIO /* or KVM_IO_PIO */, range,
> pipefd1, pipefd2 })
>
> and have kvm convert mmio or pio in that range to commands and responses
> sent over that pipe. We could have qemu threads, other userspace
> processes, or kernel components like vfio implement the other side of
> the channel.
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-18 1:58 ` Sheng Yang
@ 2010-11-18 6:21 ` Michael S. Tsirkin
2010-11-18 6:39 ` Sheng Yang
2010-11-18 9:28 ` Avi Kivity
1 sibling, 1 reply; 26+ messages in thread
From: Michael S. Tsirkin @ 2010-11-18 6:21 UTC (permalink / raw)
To: Sheng Yang; +Cc: Avi Kivity, Marcelo Tosatti, kvm, Alex Williamson
On Thu, Nov 18, 2010 at 09:58:55AM +0800, Sheng Yang wrote:
> > > +static int msix_mmio_write(struct kvm_io_device *this, gpa_t addr, int
> > > len, + const void *val)
> > > +{
> > > + struct kvm_assigned_dev_kernel *adev =
> > > + container_of(this, struct kvm_assigned_dev_kernel,
> > > + msix_mmio_dev);
> > > + int idx, r = 0;
> > > + unsigned long new_val = *(unsigned long *)val;
> >
> > What if it's a 64-bit write on a 32-bit host?
>
> In fact we haven't support QWORD(64bit) accessing now. The reason is we haven't
> seen any OS is using it in this way now, so I think we can leave it later.
>
> Also seems QEmu doesn't got the way to handle 64bit MMIO.
I think it does. I think it simply splits these to 32-bit transactions
and handles as such. That seems to be spec-compilant. I wouldn't want us
to regress.
--
MST
^ permalink raw reply [flat|nested] 26+ messages in thread* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-18 6:21 ` Michael S. Tsirkin
@ 2010-11-18 6:39 ` Sheng Yang
0 siblings, 0 replies; 26+ messages in thread
From: Sheng Yang @ 2010-11-18 6:39 UTC (permalink / raw)
To: Michael S. Tsirkin; +Cc: Avi Kivity, Marcelo Tosatti, kvm, Alex Williamson
On Thursday 18 November 2010 14:21:40 Michael S. Tsirkin wrote:
> On Thu, Nov 18, 2010 at 09:58:55AM +0800, Sheng Yang wrote:
> > > > +static int msix_mmio_write(struct kvm_io_device *this, gpa_t addr,
> > > > int len, + const void *val)
> > > > +{
> > > > + struct kvm_assigned_dev_kernel *adev =
> > > > + container_of(this, struct kvm_assigned_dev_kernel,
> > > > + msix_mmio_dev);
> > > > + int idx, r = 0;
> > > > + unsigned long new_val = *(unsigned long *)val;
> > >
> > > What if it's a 64-bit write on a 32-bit host?
> >
> > In fact we haven't support QWORD(64bit) accessing now. The reason is we
> > haven't seen any OS is using it in this way now, so I think we can leave
> > it later.
> >
> > Also seems QEmu doesn't got the way to handle 64bit MMIO.
>
> I think it does. I think it simply splits these to 32-bit transactions
> and handles as such. That seems to be spec-compilant. I wouldn't want us
> to regress.
Yes, you're right...
I think I have to add it. :shrug:
--
regards
Yang, Sheng
^ permalink raw reply [flat|nested] 26+ messages in thread
* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-18 1:58 ` Sheng Yang
2010-11-18 6:21 ` Michael S. Tsirkin
@ 2010-11-18 9:28 ` Avi Kivity
2010-11-18 9:37 ` Michael S. Tsirkin
2010-11-18 12:08 ` Sheng Yang
1 sibling, 2 replies; 26+ messages in thread
From: Avi Kivity @ 2010-11-18 9:28 UTC (permalink / raw)
To: Sheng Yang; +Cc: Marcelo Tosatti, Michael S. Tsirkin, kvm, Alex Williamson
On 11/18/2010 03:58 AM, Sheng Yang wrote:
> On Wednesday 17 November 2010 21:58:00 Avi Kivity wrote:
> > On 11/15/2010 11:15 AM, Sheng Yang wrote:
> > > This patch enable per-vector mask for assigned devices using MSI-X.
> > >
> > > This patch provided two new APIs: one is for guest to specific device's
> > > MSI-X table address in MMIO, the other is for userspace to get
> > > information about mask bit.
> > >
> > > All the mask bit operation are kept in kernel, in order to accelerate.
> > > Userspace shouldn't access the device MMIO directly for the information,
> > > instead it should uses provided API to do so.
> > >
> > > Signed-off-by: Sheng Yang<sheng@linux.intel.com>
> > > ---
> > >
> > > arch/x86/kvm/x86.c | 1 +
> > > include/linux/kvm.h | 32 +++++
> > > include/linux/kvm_host.h | 5 +
> > > virt/kvm/assigned-dev.c | 318
> > > +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed,
> 355
> > > insertions(+), 1 deletions(-)
> >
> > Documentation?
>
> For we are keeping changing the API for last several versions, I'd like to settle
> down the API first. Would bring back the document after API was agreed.
Maybe for APIs we should start with only the documentation patch, agree
on that, and move on to the implementation.
> > What if it's a 64-bit write on a 32-bit host?
>
> In fact we haven't support QWORD(64bit) accessing now. The reason is we haven't
> seen any OS is using it in this way now, so I think we can leave it later.
>
> Also seems QEmu doesn't got the way to handle 64bit MMIO.
There's a difference, if the API doesn't support it, we can't add it
later without changing both kernel and userspace.
> >
> > That's not very good. We should do the entire thing in the kernel or in
> > userspace. We can have a new EXIT_REASON to let userspace know an msix
> > entry changed, and it should read it from the kernel.
>
> If you look it in this way:
> 1. Mask bit owned by kernel.
> 2. Routing owned by userspace.
> 3. Read the routing in kernel is an speed up for normal operation - because kernel
> can read from them.
>
> So I think the logic here is clear to understand.
Still, it's complicated and the state is split across multiple components.
> But if we can modify the routing in kernel, it would be raise some sync issues due
> to both kernel and userspace own routing. So maybe the better solution is move the
> routing to kernel.
That may work, but I don't think we can do this for vfio.
--
error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 26+ messages in thread
* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-18 9:28 ` Avi Kivity
@ 2010-11-18 9:37 ` Michael S. Tsirkin
2010-11-18 12:08 ` Sheng Yang
1 sibling, 0 replies; 26+ messages in thread
From: Michael S. Tsirkin @ 2010-11-18 9:37 UTC (permalink / raw)
To: Avi Kivity; +Cc: Sheng Yang, Marcelo Tosatti, kvm, Alex Williamson
On Thu, Nov 18, 2010 at 11:28:02AM +0200, Avi Kivity wrote:
> On 11/18/2010 03:58 AM, Sheng Yang wrote:
> >On Wednesday 17 November 2010 21:58:00 Avi Kivity wrote:
> >> On 11/15/2010 11:15 AM, Sheng Yang wrote:
> >> > This patch enable per-vector mask for assigned devices using MSI-X.
> >> >
> >> > This patch provided two new APIs: one is for guest to specific device's
> >> > MSI-X table address in MMIO, the other is for userspace to get
> >> > information about mask bit.
> >> >
> >> > All the mask bit operation are kept in kernel, in order to accelerate.
> >> > Userspace shouldn't access the device MMIO directly for the information,
> >> > instead it should uses provided API to do so.
> >> >
> >> > Signed-off-by: Sheng Yang<sheng@linux.intel.com>
> >> > ---
> >> >
> >> > arch/x86/kvm/x86.c | 1 +
> >> > include/linux/kvm.h | 32 +++++
> >> > include/linux/kvm_host.h | 5 +
> >> > virt/kvm/assigned-dev.c | 318
> >> > +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed,
> >355
> >> > insertions(+), 1 deletions(-)
> >>
> >> Documentation?
> >
> >For we are keeping changing the API for last several versions, I'd like to settle
> >down the API first. Would bring back the document after API was agreed.
>
> Maybe for APIs we should start with only the documentation patch,
> agree on that, and move on to the implementation.
>
> >> What if it's a 64-bit write on a 32-bit host?
> >
> >In fact we haven't support QWORD(64bit) accessing now. The reason is we haven't
> >seen any OS is using it in this way now, so I think we can leave it later.
> >
> >Also seems QEmu doesn't got the way to handle 64bit MMIO.
>
> There's a difference, if the API doesn't support it, we can't add it
> later without changing both kernel and userspace.
>
> >>
> >> That's not very good. We should do the entire thing in the kernel or in
> >> userspace. We can have a new EXIT_REASON to let userspace know an msix
> >> entry changed, and it should read it from the kernel.
> >
> >If you look it in this way:
> >1. Mask bit owned by kernel.
> >2. Routing owned by userspace.
> >3. Read the routing in kernel is an speed up for normal operation - because kernel
> >can read from them.
> >
> >So I think the logic here is clear to understand.
>
> Still, it's complicated and the state is split across multiple components.
>
> >But if we can modify the routing in kernel, it would be raise some sync issues due
> >to both kernel and userspace own routing. So maybe the better solution is move the
> >routing to kernel.
>
> That may work, but I don't think we can do this for vfio.
Actually, if done right it might work for VFIO: we would need
2 eventfds to notify it that it has to mask/unmask entries.
The interface would need to be careful to keep programming of the guest
side emulation and the masking in the backend device completely
separate.
> --
> error compiling committee.c: too many arguments to function
^ permalink raw reply [flat|nested] 26+ messages in thread
* Re: [PATCH 6/6] KVM: assigned dev: MSI-X mask support
2010-11-18 9:28 ` Avi Kivity
2010-11-18 9:37 ` Michael S. Tsirkin
@ 2010-11-18 12:08 ` Sheng Yang
1 sibling, 0 replies; 26+ messages in thread
From: Sheng Yang @ 2010-11-18 12:08 UTC (permalink / raw)
To: Avi Kivity
Cc: Sheng Yang, Marcelo Tosatti, Michael S. Tsirkin, kvm, Alex Williamson
On Thu, Nov 18, 2010 at 5:28 PM, Avi Kivity <avi@redhat.com> wrote:
> On 11/18/2010 03:58 AM, Sheng Yang wrote:
>>
>> On Wednesday 17 November 2010 21:58:00 Avi Kivity wrote:
>> > On 11/15/2010 11:15 AM, Sheng Yang wrote:
>> > > This patch enable per-vector mask for assigned devices using MSI-X.
>> > >
>> > > This patch provided two new APIs: one is for guest to specific
>> > device's
>> > > MSI-X table address in MMIO, the other is for userspace to get
>> > > information about mask bit.
>> > >
>> > > All the mask bit operation are kept in kernel, in order to
>> > accelerate.
>> > > Userspace shouldn't access the device MMIO directly for the
>> > information,
>> > > instead it should uses provided API to do so.
>> > >
>> > > Signed-off-by: Sheng Yang<sheng@linux.intel.com>
>> > > ---
>> > >
>> > > arch/x86/kvm/x86.c | 1 +
>> > > include/linux/kvm.h | 32 +++++
>> > > include/linux/kvm_host.h | 5 +
>> > > virt/kvm/assigned-dev.c | 318
>> > > +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed,
>> 355
>> > > insertions(+), 1 deletions(-)
>> >
>> > Documentation?
>>
>> For we are keeping changing the API for last several versions, I'd like to
>> settle
>> down the API first. Would bring back the document after API was agreed.
>
> Maybe for APIs we should start with only the documentation patch, agree on
> that, and move on to the implementation.
Yes, would follow it next time. And I would bring back the documents
in the next edition, for Michael and I have reached agreement on API.
>
>> > What if it's a 64-bit write on a 32-bit host?
>>
>> In fact we haven't support QWORD(64bit) accessing now. The reason is we
>> haven't
>> seen any OS is using it in this way now, so I think we can leave it later.
>>
>> Also seems QEmu doesn't got the way to handle 64bit MMIO.
>
> There's a difference, if the API doesn't support it, we can't add it later
> without changing both kernel and userspace.
Um... Which API you're talking about? I think userspace API(set msix
mmio, and get mask bit status) is unrelated here?
>
>> >
>> > That's not very good. We should do the entire thing in the kernel or
>> > in
>> > userspace. We can have a new EXIT_REASON to let userspace know an msix
>> > entry changed, and it should read it from the kernel.
>>
>> If you look it in this way:
>> 1. Mask bit owned by kernel.
>> 2. Routing owned by userspace.
>> 3. Read the routing in kernel is an speed up for normal operation -
>> because kernel
>> can read from them.
>>
>> So I think the logic here is clear to understand.
>
> Still, it's complicated and the state is split across multiple components.
So how about removing the reading acceleration part in the patch
temporarily? Kernel owns mask bit and userspace owns others. That
should be better. I can add the reading part later when we can find an
elegant way to do so.
>
>> But if we can modify the routing in kernel, it would be raise some sync
>> issues due
>> to both kernel and userspace own routing. So maybe the better solution is
>> move the
>> routing to kernel.
>
> That may work, but I don't think we can do this for vfio.
--
regards,
Yang, Sheng
>
> --
> error compiling committee.c: too many arguments to function
>
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
^ permalink raw reply [flat|nested] 26+ messages in thread