From: Yan Zhao <yan.y.zhao@intel.com>
To: alex.williamson@redhat.com
Cc: kvm@vger.kernel.org, linux-kernel@vger.kernel.org,
libvir-list@redhat.com, qemu-devel@nongnu.org, cohuck@redhat.com,
zhenyuw@linux.intel.com, zhi.a.wang@intel.com,
kevin.tian@intel.com, shaopeng.he@intel.com,
Yan Zhao <yan.y.zhao@intel.com>
Subject: [RFC PATCH 9/9] i40e/vf_migration: support dynamic trap of bar0
Date: Wed, 4 Dec 2019 22:27:49 -0500 [thread overview]
Message-ID: <20191205032749.30030-1-yan.y.zhao@intel.com> (raw)
In-Reply-To: <20191205032419.29606-1-yan.y.zhao@intel.com>
mediate dynamic_trap_info region to dynamically trap bar0.
bar0 is sparsely mmaped into 5 sub-regions, of which only two need to be
dynamically trapped.
By mediating dynamic_trap_info region and telling QEMU this information,
the two sub-regions of bar0 can be trapped when migration starts and put
to passthrough again when migration fails
Cc: Shaopeng He <shaopeng.he@intel.com>
Signed-off-by: Yan Zhao <yan.y.zhao@intel.com>
---
.../ethernet/intel/i40e/i40e_vf_migration.c | 140 +++++++++++++++++-
.../ethernet/intel/i40e/i40e_vf_migration.h | 12 ++
2 files changed, 147 insertions(+), 5 deletions(-)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_vf_migration.c b/drivers/net/ethernet/intel/i40e/i40e_vf_migration.c
index 5bb509fed66e..0b9d5be85049 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_vf_migration.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_vf_migration.c
@@ -29,6 +29,21 @@ static bool is_handle_valid(int handle)
return true;
}
+static
+void i40e_vf_migration_dynamic_trap_bar(struct i40e_vf_migration *i40e_vf_dev)
+{
+ if (i40e_vf_dev->dt_trigger)
+ eventfd_signal(i40e_vf_dev->dt_trigger, 1);
+}
+
+static void i40e_vf_trap_bar0(struct i40e_vf_migration *i40e_vf_dev, bool trap)
+{
+ if (i40e_vf_dev->trap_bar0 != trap) {
+ i40e_vf_dev->trap_bar0 = trap;
+ i40e_vf_migration_dynamic_trap_bar(i40e_vf_dev);
+ }
+}
+
static size_t set_device_state(struct i40e_vf_migration *i40e_vf_dev, u32 state)
{
int ret = 0;
@@ -39,8 +54,10 @@ static size_t set_device_state(struct i40e_vf_migration *i40e_vf_dev, u32 state)
switch (state) {
case VFIO_DEVICE_STATE_RUNNING:
+ i40e_vf_trap_bar0(i40e_vf_dev, false);
break;
case VFIO_DEVICE_STATE_SAVING | VFIO_DEVICE_STATE_RUNNING:
+ i40e_vf_trap_bar0(i40e_vf_dev, true);
// alloc dirty page tracking resources and
// do the first round dirty page scanning
break;
@@ -137,16 +154,22 @@ int i40e_vf_migration_open(struct pci_dev *pdev, u64 *caps, u32 *dm_handle)
MIGRATION_DIRTY_BITMAP_SIZE;
i40e_vf_dev->migration_region_index = -1;
+ i40e_vf_dev->dt_region_index = -1;
+ i40e_vf_dev->trap_bar0 = false;
+
vf = &pf->vf[vf_id];
i40e_vf_dev_array[handle] = i40e_vf_dev;
set_bit(handle, open_device_bits);
+
*dm_handle = handle;
*caps |= VFIO_PCI_DEVICE_CAP_MIGRATION;
+ *caps |= VFIO_PCI_DEVICE_CAP_DYNAMIC_TRAP_BAR;
pr_info("%s: device %x %x, vf id %d, handle=%x\n",
__func__, pdev->vendor, pdev->device, vf_id, handle);
+
error:
mutex_unlock(&device_bit_lock);
@@ -188,6 +211,10 @@ void i40e_vf_migration_release(int handle)
kfree(i40e_vf_dev->mig_ctl);
vfree(i40e_vf_dev->dirty_bitmap);
+
+ if (i40e_vf_dev->dt_trigger)
+ eventfd_ctx_put(i40e_vf_dev->dt_trigger);
+
kfree(i40e_vf_dev);
module_put(THIS_MODULE);
@@ -216,6 +243,47 @@ static void migration_region_sparse_mmap_cap(struct vfio_info_cap *caps)
kfree(sparse);
}
+static void bar0_sparse_mmap_cap(struct vfio_region_info *info,
+ struct vfio_info_cap *caps)
+{
+ struct vfio_region_info_cap_sparse_mmap *sparse;
+ size_t size;
+ int nr_areas = 5;
+
+ size = sizeof(*sparse) + (nr_areas * sizeof(*sparse->areas));
+
+ sparse = kzalloc(size, GFP_KERNEL);
+ if (!sparse)
+ return;
+
+ sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
+ sparse->header.version = 1;
+ sparse->nr_areas = nr_areas;
+
+ sparse->areas[0].offset = 0;
+ sparse->areas[0].size = IAVF_VF_TAIL_START;
+ sparse->areas[0].disablable = 0;//able to get toggled
+
+ sparse->areas[1].offset = IAVF_VF_TAIL_START;
+ sparse->areas[1].size = PAGE_SIZE;
+ sparse->areas[1].disablable = 1;//able to get toggled
+
+ sparse->areas[2].offset = IAVF_VF_TAIL_START + PAGE_SIZE;
+ sparse->areas[2].size = IAVF_VF_ARQH1 - sparse->areas[2].offset;
+ sparse->areas[2].disablable = 0;//able to get toggled
+
+ sparse->areas[3].offset = IAVF_VF_ARQT1;
+ sparse->areas[3].size = PAGE_SIZE;
+ sparse->areas[3].disablable = 1;//able to get toggled
+
+ sparse->areas[4].offset = IAVF_VF_ARQT1 + PAGE_SIZE;
+ sparse->areas[4].size = info->size - sparse->areas[4].offset;
+ sparse->areas[4].disablable = 0;//able to get toggled
+
+ vfio_info_add_capability(caps, &sparse->header, size);
+ kfree(sparse);
+}
+
static void
i40e_vf_migration_get_region_info(int handle,
struct vfio_region_info *info,
@@ -227,9 +295,8 @@ i40e_vf_migration_get_region_info(int handle,
switch (info->index) {
case VFIO_PCI_BAR0_REGION_INDEX:
- info->flags = VFIO_REGION_INFO_FLAG_READ |
- VFIO_REGION_INFO_FLAG_WRITE;
-
+ info->flags |= VFIO_REGION_INFO_FLAG_MMAP;
+ bar0_sparse_mmap_cap(info, caps);
break;
case VFIO_PCI_BAR1_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
case VFIO_PCI_CONFIG_REGION_INDEX:
@@ -237,7 +304,20 @@ i40e_vf_migration_get_region_info(int handle,
case VFIO_PCI_VGA_REGION_INDEX:
break;
default:
- if (cap_type->type == VFIO_REGION_TYPE_MIGRATION &&
+ if (cap_type->type ==
+ VFIO_REGION_TYPE_DYNAMIC_TRAP_BAR_INFO &&
+ cap_type->subtype ==
+ VFIO_REGION_SUBTYPE_DYNAMIC_TRAP_BAR_INFO) {
+ struct i40e_vf_migration *i40e_vf_dev;
+
+ i40e_vf_dev = i40e_vf_dev_array[handle];
+ i40e_vf_dev->dt_region_index = info->index;
+ info->size =
+ sizeof(struct vfio_device_dt_bar_info_region);
+ } else if ((cap_type->type == VFIO_REGION_TYPE_MIGRATION) &&
+ (cap_type->subtype ==
+ VFIO_REGION_SUBTYPE_MIGRATION)) {
+ } else if (cap_type->type == VFIO_REGION_TYPE_MIGRATION &&
cap_type->subtype == VFIO_REGION_SUBTYPE_MIGRATION) {
struct i40e_vf_migration *i40e_vf_dev;
@@ -254,6 +334,53 @@ i40e_vf_migration_get_region_info(int handle,
}
}
+static ssize_t i40e_vf_dt_region_rw(struct i40e_vf_migration *i40e_vf_dev,
+ char __user *buf, size_t count,
+ loff_t *ppos, bool iswrite, bool *pt)
+{
+#define DT_REGION_OFFSET(x) offsetof(struct vfio_device_dt_bar_info_region, x)
+ u64 pos = *ppos & VFIO_PCI_OFFSET_MASK;
+ ssize_t ret = 0;
+
+ *pt = false;
+ switch (pos) {
+ case DT_REGION_OFFSET(dt_fd):
+ if (iswrite) {
+ u32 dt_fd;
+ struct eventfd_ctx *trigger;
+
+ if (copy_from_user(&dt_fd, buf, sizeof(dt_fd)))
+ return -EFAULT;
+
+ trigger = eventfd_ctx_fdget(dt_fd);
+ if (IS_ERR(trigger)) {
+ pr_err("i40e_vf_migration_rw, dt trigger fd set error\n");
+ return -EINVAL;
+ }
+ i40e_vf_dev->dt_trigger = trigger;
+ ret = sizeof(dt_fd);
+ } else {
+ ret = -EFAULT;
+ }
+ break;
+
+ case DT_REGION_OFFSET(trap):
+ if (iswrite)
+ ret = copy_from_user(&i40e_vf_dev->trap_bar0,
+ buf, count) ? -EFAULT : count;
+ else
+ ret = copy_to_user(buf,
+ &i40e_vf_dev->trap_bar0,
+ sizeof(u32)) ?
+ -EFAULT : sizeof(u32);
+ break;
+ default:
+ ret = -EFAULT;
+ break;
+ }
+ return ret;
+}
+
static
ssize_t i40e_vf_migration_region_rw(struct i40e_vf_migration *i40e_vf_dev,
char __user *buf, size_t count,
@@ -420,7 +547,10 @@ static ssize_t i40e_vf_migration_rw(int handle, char __user *buf,
case VFIO_PCI_VGA_REGION_INDEX:
break;
default:
- if (index == i40e_vf_dev->migration_region_index) {
+ if (index == i40e_vf_dev->dt_region_index) {
+ return i40e_vf_dt_region_rw(i40e_vf_dev, buf, count,
+ ppos, iswrite, pt);
+ } else if (index == i40e_vf_dev->migration_region_index) {
return i40e_vf_migration_region_rw(i40e_vf_dev, buf,
count, ppos, iswrite, pt);
}
diff --git a/drivers/net/ethernet/intel/i40e/i40e_vf_migration.h b/drivers/net/ethernet/intel/i40e/i40e_vf_migration.h
index b31b500b3cd6..dfad4cc7e46f 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_vf_migration.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_vf_migration.h
@@ -21,6 +21,14 @@
PAGE_ALIGN(sizeof(struct vfio_device_migration_info))
#define MIGRATION_DIRTY_BITMAP_SIZE (64 * 1024UL)
+#define IAVF_VF_ARQBAH1 0x00006000 /* Reset: EMPR */
+#define IAVF_VF_ARQBAL1 0x00006C00 /* Reset: EMPR */
+#define IAVF_VF_ARQH1 0x00007400 /* Reset: EMPR */
+#define IAVF_VF_ARQT1 0x00007000 /* Reset: EMPR */
+#define IAVF_VF_ARQLEN1 0x00008000 /* Reset: EMPR */
+#define IAVF_VF_TAIL_START 0x00002000 /* Start of tail register region */
+#define IAVF_VF_TAIL_END 0x00002400 /* End of tail register region */
+
/* Single Root I/O Virtualization */
struct pci_sriov {
int pos; /* Capability position */
@@ -56,6 +64,10 @@ struct i40e_vf_migration {
struct pci_dev *vf_dev;
int vf_id;
+ __u64 dt_region_index;
+ struct eventfd_ctx *dt_trigger;
+ bool trap_bar0;
+
__u64 migration_region_index;
__u64 migration_region_size;
--
2.17.1
next prev parent reply other threads:[~2019-12-05 3:36 UTC|newest]
Thread overview: 46+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-12-05 3:24 [RFC PATCH 0/9] Introduce mediate ops in vfio-pci Yan Zhao
2019-12-05 3:25 ` [RFC PATCH 1/9] vfio/pci: introduce mediate ops to intercept vfio-pci ops Yan Zhao
2019-12-05 23:55 ` Alex Williamson
2019-12-06 7:56 ` Yan Zhao
2019-12-06 21:22 ` Alex Williamson
2019-12-09 3:42 ` Yan Zhao
2019-12-10 0:03 ` Alex Williamson
2019-12-10 2:44 ` Yan Zhao
2019-12-10 16:58 ` Alex Williamson
2019-12-11 1:19 ` Yan Zhao
2019-12-06 23:13 ` Eric Blake
2019-12-09 3:17 ` Yan Zhao
2019-12-05 3:25 ` [RFC PATCH 2/9] vfio/pci: test existence before calling region->ops Yan Zhao
2019-12-05 3:26 ` [RFC PATCH 3/9] vfio/pci: register a default migration region Yan Zhao
2019-12-05 23:55 ` Alex Williamson
2019-12-06 5:50 ` Yan Zhao
2019-12-05 3:26 ` [RFC PATCH 4/9] vfio-pci: register default dynamic-trap-bar-info region Yan Zhao
2019-12-05 23:55 ` Alex Williamson
2019-12-06 6:04 ` Yan Zhao
2019-12-06 15:20 ` Alex Williamson
2019-12-09 6:22 ` Yan Zhao
2019-12-09 21:16 ` Alex Williamson
2019-12-10 7:44 ` Yan Zhao
2019-12-10 16:38 ` Alex Williamson
2019-12-11 6:25 ` Yan Zhao
2019-12-11 18:56 ` Alex Williamson
2019-12-12 2:02 ` Yan Zhao
2019-12-12 3:07 ` Alex Williamson
2019-12-12 3:11 ` Yan Zhao
2019-12-05 3:27 ` [RFC PATCH 5/9] samples/vfio-pci/igd_dt: sample driver to mediate a passthrough IGD Yan Zhao
2019-12-05 3:27 ` [RFC PATCH 6/9] sample/vfio-pci/igd_dt: dynamically trap/untrap subregion of IGD bar0 Yan Zhao
2019-12-05 3:27 ` [RFC PATCH 7/9] i40e/vf_migration: register mediate_ops to vfio-pci Yan Zhao
2019-12-05 3:27 ` [RFC PATCH 8/9] i40e/vf_migration: mediate migration region Yan Zhao
2019-12-05 3:27 ` Yan Zhao [this message]
2019-12-05 6:33 ` [RFC PATCH 0/9] Introduce mediate ops in vfio-pci Jason Wang
2019-12-05 8:51 ` Yan Zhao
2019-12-05 13:05 ` Jason Wang
2019-12-06 8:22 ` Yan Zhao
2019-12-06 9:40 ` Jason Wang
2019-12-06 12:49 ` Yan Zhao
2019-12-12 3:48 ` Jason Wang
2019-12-12 5:47 ` Yan Zhao
2019-12-18 2:36 ` Jason Wang
2019-12-06 17:42 ` Alex Williamson
2019-12-12 4:09 ` Jason Wang
2019-12-12 18:39 ` Alex Williamson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20191205032749.30030-1-yan.y.zhao@intel.com \
--to=yan.y.zhao@intel.com \
--cc=alex.williamson@redhat.com \
--cc=cohuck@redhat.com \
--cc=kevin.tian@intel.com \
--cc=kvm@vger.kernel.org \
--cc=libvir-list@redhat.com \
--cc=linux-kernel@vger.kernel.org \
--cc=qemu-devel@nongnu.org \
--cc=shaopeng.he@intel.com \
--cc=zhenyuw@linux.intel.com \
--cc=zhi.a.wang@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).