[PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Sherry Sun]

Changes in V5:
1. Reorganize the vop_mmap function code in patch 1, which is done by Christoph. 
2. Completely remove the unnecessary code related to reassign the used ring for
card in patch 2.

The original vop driver only supports dma coherent device, as it allocates and
maps vring by _get_free_pages and dma_map_single, but not use 
dma_sync_single_for_cpu/device to sync the updates of device_page/vring between
EP and RC, which will cause memory synchronization problem for device don't
support hardware dma coherent.

And allocate vrings use dma_alloc_coherent is a common way in kernel, as the
memory interacted between two systems should use consistent memory to avoid
caching effects. So here add noncoherent platform support for vop driver.
Also add some related dma changes to make sure noncoherent platform works
well.

Sherry Sun (2):
  misc: vop: change the way of allocating vrings and device page
  misc: vop: do not allocate and reassign the used ring

 drivers/misc/mic/bus/vop_bus.h     |   2 +
 drivers/misc/mic/host/mic_boot.c   |   9 ++
 drivers/misc/mic/host/mic_main.c   |  43 ++------
 drivers/misc/mic/vop/vop_debugfs.c |   4 -
 drivers/misc/mic/vop/vop_main.c    |  70 +-----------
 drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
 include/uapi/linux/mic_common.h    |   9 +-
 7 files changed, 85 insertions(+), 218 deletions(-)

-- 
2.17.1

[PATCH V5 1/2] misc: vop: change the way of allocating vrings and device page

[Sherry Sun]

Allocate vrings use dma_alloc_coherent is a common way in kernel. As the
memory interacted between two systems should use consistent memory to
avoid caching effects, same as device page memory.

The orginal way use __get_free_pages and dma_map_single to allocate and
map vring, but not use dma_sync_single_for_cpu/device api to sync the
changes of vring between EP and RC, which will cause memory
synchronization problem for those devices which don't support hardware
dma coherent.

Also change to use dma_mmap_coherent for mmap callback to map the device
page and vring memory to userspace.

Signed-off-by: Christoph Hellwig <hch@infradead.org>
Signed-off-by: Sherry Sun <sherry.sun@nxp.com>
Signed-off-by: Joakim Zhang <qiangqing.zhang@nxp.com>
---
 drivers/misc/mic/bus/vop_bus.h    |   2 +
 drivers/misc/mic/host/mic_boot.c  |   9 ++
 drivers/misc/mic/host/mic_main.c  |  43 +++-------
 drivers/misc/mic/vop/vop_vringh.c | 135 ++++++++++++------------------
 4 files changed, 77 insertions(+), 112 deletions(-)

diff --git a/drivers/misc/mic/bus/vop_bus.h b/drivers/misc/mic/bus/vop_bus.h
index 4fa02808c1e2..e21c06aeda7a 100644
--- a/drivers/misc/mic/bus/vop_bus.h
+++ b/drivers/misc/mic/bus/vop_bus.h
@@ -75,6 +75,7 @@ struct vop_driver {
  *                 node to add/remove/configure virtio devices.
  * @get_dp: Get access to the virtio device page used by the self
  *          node to add/remove/configure virtio devices.
+ * @dp_mmap: Map the virtio device page to userspace.
  * @send_intr: Send an interrupt to the peer node on a specified doorbell.
  * @remap: Map a buffer with the specified DMA address and length.
  * @unmap: Unmap a buffer previously mapped.
@@ -92,6 +93,7 @@ struct vop_hw_ops {
 	void (*ack_interrupt)(struct vop_device *vpdev, int num);
 	void __iomem * (*get_remote_dp)(struct vop_device *vpdev);
 	void * (*get_dp)(struct vop_device *vpdev);
+	int (*dp_mmap)(struct vop_device *vpdev, struct vm_area_struct *vma);
 	void (*send_intr)(struct vop_device *vpdev, int db);
 	void __iomem * (*remap)(struct vop_device *vpdev,
 				  dma_addr_t pa, size_t len);
diff --git a/drivers/misc/mic/host/mic_boot.c b/drivers/misc/mic/host/mic_boot.c
index 8cb85b8b3e19..44ed918b49b4 100644
--- a/drivers/misc/mic/host/mic_boot.c
+++ b/drivers/misc/mic/host/mic_boot.c
@@ -89,6 +89,14 @@ static void *__mic_get_dp(struct vop_device *vpdev)
 	return mdev->dp;
 }
 
+static int __mic_dp_mmap(struct vop_device *vpdev, struct vm_area_struct *vma)
+{
+	struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
+
+	return dma_mmap_coherent(&mdev->pdev->dev, vma, mdev->dp,
+				 mdev->dp_dma_addr, MIC_DP_SIZE);
+}
+
 static void __iomem *__mic_get_remote_dp(struct vop_device *vpdev)
 {
 	return NULL;
@@ -120,6 +128,7 @@ static struct vop_hw_ops vop_hw_ops = {
 	.ack_interrupt = __mic_ack_interrupt,
 	.next_db = __mic_next_db,
 	.get_dp = __mic_get_dp,
+	.dp_mmap = __mic_dp_mmap,
 	.get_remote_dp = __mic_get_remote_dp,
 	.send_intr = __mic_send_intr,
 	.remap = __mic_ioremap,
diff --git a/drivers/misc/mic/host/mic_main.c b/drivers/misc/mic/host/mic_main.c
index ea4608527ea0..fab87a72a9a4 100644
--- a/drivers/misc/mic/host/mic_main.c
+++ b/drivers/misc/mic/host/mic_main.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/poll.h>
+#include <linux/dma-mapping.h>
 
 #include <linux/mic_common.h>
 #include "../common/mic_dev.h"
@@ -45,33 +46,6 @@ MODULE_DEVICE_TABLE(pci, mic_pci_tbl);
 /* ID allocator for MIC devices */
 static struct ida g_mic_ida;
 
-/* Initialize the device page */
-static int mic_dp_init(struct mic_device *mdev)
-{
-	mdev->dp = kzalloc(MIC_DP_SIZE, GFP_KERNEL);
-	if (!mdev->dp)
-		return -ENOMEM;
-
-	mdev->dp_dma_addr = mic_map_single(mdev,
-		mdev->dp, MIC_DP_SIZE);
-	if (mic_map_error(mdev->dp_dma_addr)) {
-		kfree(mdev->dp);
-		dev_err(&mdev->pdev->dev, "%s %d err %d\n",
-			__func__, __LINE__, -ENOMEM);
-		return -ENOMEM;
-	}
-	mdev->ops->write_spad(mdev, MIC_DPLO_SPAD, mdev->dp_dma_addr);
-	mdev->ops->write_spad(mdev, MIC_DPHI_SPAD, mdev->dp_dma_addr >> 32);
-	return 0;
-}
-
-/* Uninitialize the device page */
-static void mic_dp_uninit(struct mic_device *mdev)
-{
-	mic_unmap_single(mdev, mdev->dp_dma_addr, MIC_DP_SIZE);
-	kfree(mdev->dp);
-}
-
 /**
  * mic_ops_init: Initialize HW specific operation tables.
  *
@@ -227,11 +201,16 @@ static int mic_probe(struct pci_dev *pdev,
 
 	pci_set_drvdata(pdev, mdev);
 
-	rc = mic_dp_init(mdev);
-	if (rc) {
-		dev_err(&pdev->dev, "mic_dp_init failed rc %d\n", rc);
+	mdev->dp = dma_alloc_coherent(&pdev->dev, MIC_DP_SIZE,
+				      &mdev->dp_dma_addr, GFP_KERNEL);
+	if (!mdev->dp) {
+		dev_err(&pdev->dev, "failed to allocate device page\n");
 		goto smpt_uninit;
 	}
+
+	mdev->ops->write_spad(mdev, MIC_DPLO_SPAD, mdev->dp_dma_addr);
+	mdev->ops->write_spad(mdev, MIC_DPHI_SPAD, mdev->dp_dma_addr >> 32);
+
 	mic_bootparam_init(mdev);
 	mic_create_debug_dir(mdev);
 
@@ -244,7 +223,7 @@ static int mic_probe(struct pci_dev *pdev,
 	return 0;
 cleanup_debug_dir:
 	mic_delete_debug_dir(mdev);
-	mic_dp_uninit(mdev);
+	dma_free_coherent(&pdev->dev, MIC_DP_SIZE, mdev->dp, mdev->dp_dma_addr);
 smpt_uninit:
 	mic_smpt_uninit(mdev);
 free_interrupts:
@@ -283,7 +262,7 @@ static void mic_remove(struct pci_dev *pdev)
 
 	cosm_unregister_device(mdev->cosm_dev);
 	mic_delete_debug_dir(mdev);
-	mic_dp_uninit(mdev);
+	dma_free_coherent(&pdev->dev, MIC_DP_SIZE, mdev->dp, mdev->dp_dma_addr);
 	mic_smpt_uninit(mdev);
 	mic_free_interrupts(mdev, pdev);
 	iounmap(mdev->aper.va);
diff --git a/drivers/misc/mic/vop/vop_vringh.c b/drivers/misc/mic/vop/vop_vringh.c
index 7014ffe88632..39d34b164ede 100644
--- a/drivers/misc/mic/vop/vop_vringh.c
+++ b/drivers/misc/mic/vop/vop_vringh.c
@@ -298,9 +298,8 @@ static int vop_virtio_add_device(struct vop_vdev *vdev,
 		mutex_init(&vvr->vr_mutex);
 		vr_size = PAGE_ALIGN(round_up(vring_size(num, MIC_VIRTIO_RING_ALIGN), 4) +
 			sizeof(struct _mic_vring_info));
-		vr->va = (void *)
-			__get_free_pages(GFP_KERNEL | __GFP_ZERO,
-					 get_order(vr_size));
+		vr->va = dma_alloc_coherent(vop_dev(vdev), vr_size, &vr_addr,
+					    GFP_KERNEL);
 		if (!vr->va) {
 			ret = -ENOMEM;
 			dev_err(vop_dev(vdev), "%s %d err %d\n",
@@ -310,15 +309,6 @@ static int vop_virtio_add_device(struct vop_vdev *vdev,
 		vr->len = vr_size;
 		vr->info = vr->va + round_up(vring_size(num, MIC_VIRTIO_RING_ALIGN), 4);
 		vr->info->magic = cpu_to_le32(MIC_MAGIC + vdev->virtio_id + i);
-		vr_addr = dma_map_single(&vpdev->dev, vr->va, vr_size,
-					 DMA_BIDIRECTIONAL);
-		if (dma_mapping_error(&vpdev->dev, vr_addr)) {
-			free_pages((unsigned long)vr->va, get_order(vr_size));
-			ret = -ENOMEM;
-			dev_err(vop_dev(vdev), "%s %d err %d\n",
-				__func__, __LINE__, ret);
-			goto err;
-		}
 		vqconfig[i].address = cpu_to_le64(vr_addr);
 
 		vring_init(&vr->vr, num, vr->va, MIC_VIRTIO_RING_ALIGN);
@@ -339,11 +329,9 @@ static int vop_virtio_add_device(struct vop_vdev *vdev,
 		dev_dbg(&vpdev->dev,
 			"%s %d index %d va %p info %p vr_size 0x%x\n",
 			__func__, __LINE__, i, vr->va, vr->info, vr_size);
-		vvr->buf = (void *)__get_free_pages(GFP_KERNEL,
-					get_order(VOP_INT_DMA_BUF_SIZE));
-		vvr->buf_da = dma_map_single(&vpdev->dev,
-					  vvr->buf, VOP_INT_DMA_BUF_SIZE,
-					  DMA_BIDIRECTIONAL);
+		vvr->buf = dma_alloc_coherent(vop_dev(vdev),
+					      VOP_INT_DMA_BUF_SIZE,
+					      &vvr->buf_da, GFP_KERNEL);
 	}
 
 	snprintf(irqname, sizeof(irqname), "vop%dvirtio%d", vpdev->index,
@@ -382,10 +370,8 @@ static int vop_virtio_add_device(struct vop_vdev *vdev,
 	for (j = 0; j < i; j++) {
 		struct vop_vringh *vvr = &vdev->vvr[j];
 
-		dma_unmap_single(&vpdev->dev, le64_to_cpu(vqconfig[j].address),
-				 vvr->vring.len, DMA_BIDIRECTIONAL);
-		free_pages((unsigned long)vvr->vring.va,
-			   get_order(vvr->vring.len));
+		dma_free_coherent(vop_dev(vdev), vvr->vring.len, vvr->vring.va,
+				  le64_to_cpu(vqconfig[j].address));
 	}
 	return ret;
 }
@@ -433,17 +419,12 @@ static void vop_virtio_del_device(struct vop_vdev *vdev)
 	for (i = 0; i < vdev->dd->num_vq; i++) {
 		struct vop_vringh *vvr = &vdev->vvr[i];
 
-		dma_unmap_single(&vpdev->dev,
-				 vvr->buf_da, VOP_INT_DMA_BUF_SIZE,
-				 DMA_BIDIRECTIONAL);
-		free_pages((unsigned long)vvr->buf,
-			   get_order(VOP_INT_DMA_BUF_SIZE));
+		dma_free_coherent(vop_dev(vdev), VOP_INT_DMA_BUF_SIZE,
+				  vvr->buf, vvr->buf_da);
 		vringh_kiov_cleanup(&vvr->riov);
 		vringh_kiov_cleanup(&vvr->wiov);
-		dma_unmap_single(&vpdev->dev, le64_to_cpu(vqconfig[i].address),
-				 vvr->vring.len, DMA_BIDIRECTIONAL);
-		free_pages((unsigned long)vvr->vring.va,
-			   get_order(vvr->vring.len));
+		dma_free_coherent(vop_dev(vdev), vvr->vring.len, vvr->vring.va,
+				  le64_to_cpu(vqconfig[i].address));
 	}
 	/*
 	 * Order the type update with previous stores. This write barrier
@@ -1042,13 +1023,27 @@ static __poll_t vop_poll(struct file *f, poll_table *wait)
 	return mask;
 }
 
-static inline int
-vop_query_offset(struct vop_vdev *vdev, unsigned long offset,
-		 unsigned long *size, unsigned long *pa)
+/*
+ * Maps the device page and virtio rings to user space for readonly access.
+ */
+static int vop_mmap(struct file *f, struct vm_area_struct *vma)
 {
-	struct vop_device *vpdev = vdev->vpdev;
-	unsigned long start = MIC_DP_SIZE;
-	int i;
+	struct vop_vdev *vdev = f->private_data;
+	struct mic_vqconfig *vqconfig = mic_vq_config(vdev->dd);
+	unsigned long orig_start = vma->vm_start;
+	unsigned long orig_end = vma->vm_end;
+	int err, i;
+
+	if (!vdev->vpdev->hw_ops->dp_mmap)
+		return -EINVAL;
+	if (vma->vm_pgoff)
+		return -EINVAL;
+	if (vma->vm_flags & VM_WRITE)
+		return -EACCES;
+
+	err = vop_vdev_inited(vdev);
+	if (err)
+		return err;
 
 	/*
 	 * MMAP interface is as follows:
@@ -1057,58 +1052,38 @@ vop_query_offset(struct vop_vdev *vdev, unsigned long offset,
 	 * 0x1000				first vring
 	 * 0x1000 + size of 1st vring		second vring
 	 * ....
+	 *
+	 * We manipulate vm_start/vm_end to trick dma_mmap_coherent into
+	 * performing partial mappings, which is a bit of a hack, but safe
+	 * while we are under mmap_lock().  Eventually this needs to be
+	 * replaced by a proper DMA layer API.
 	 */
-	if (!offset) {
-		*pa = virt_to_phys(vpdev->hw_ops->get_dp(vpdev));
-		*size = MIC_DP_SIZE;
-		return 0;
-	}
+	vma->vm_end = vma->vm_start + MIC_DP_SIZE;
+	err = vdev->vpdev->hw_ops->dp_mmap(vdev->vpdev, vma);
+	if (err)
+		goto out;
 
 	for (i = 0; i < vdev->dd->num_vq; i++) {
 		struct vop_vringh *vvr = &vdev->vvr[i];
 
-		if (offset == start) {
-			*pa = virt_to_phys(vvr->vring.va);
-			*size = vvr->vring.len;
-			return 0;
-		}
-		start += vvr->vring.len;
-	}
-	return -1;
-}
+		vma->vm_start = vma->vm_end;
+		vma->vm_end += vvr->vring.len;
 
-/*
- * Maps the device page and virtio rings to user space for readonly access.
- */
-static int vop_mmap(struct file *f, struct vm_area_struct *vma)
-{
-	struct vop_vdev *vdev = f->private_data;
-	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
-	unsigned long pa, size = vma->vm_end - vma->vm_start, size_rem = size;
-	int i, err;
-
-	err = vop_vdev_inited(vdev);
-	if (err)
-		goto ret;
-	if (vma->vm_flags & VM_WRITE) {
-		err = -EACCES;
-		goto ret;
-	}
-	while (size_rem) {
-		i = vop_query_offset(vdev, offset, &size, &pa);
-		if (i < 0) {
-			err = -EINVAL;
-			goto ret;
-		}
-		err = remap_pfn_range(vma, vma->vm_start + offset,
-				      pa >> PAGE_SHIFT, size,
-				      vma->vm_page_prot);
+		err = -EINVAL;
+		if (vma->vm_end > orig_end)
+			goto out;
+		err = dma_mmap_coherent(vop_dev(vdev), vma, vvr->vring.va,
+					le64_to_cpu(vqconfig[i].address),
+					vvr->vring.len);
 		if (err)
-			goto ret;
-		size_rem -= size;
-		offset += size;
+			goto out;
 	}
-ret:
+out:
+	/*
+	 * Restore the original vma parameters.
+	 */
+	vma->vm_start = orig_start;
+	vma->vm_end = orig_end;
 	return err;
 }
 
-- 
2.17.1

[PATCH V5 2/2] misc: vop: do not allocate and reassign the used ring

[Sherry Sun]

We don't need to allocate and reassign the used ring here and remove the
used_address_updated flag. Since RC has allocated the entire vring,
including the used ring. Simply add the corresponding offset can get the
used ring address.

If following the orginal way to reassign the used ring, will encounter a
problem. When host finished with descriptor, it will update the used
ring with putused_kern api, if reassign used ring at EP side, used
ring will be io device memory for RC, use memcpy in putused_kern will
cause kernel panic.

Signed-off-by: Sherry Sun <sherry.sun@nxp.com>
Signed-off-by: Joakim Zhang <qiangqing.zhang@nxp.com>
---
 drivers/misc/mic/vop/vop_debugfs.c |  4 --
 drivers/misc/mic/vop/vop_main.c    | 70 ++----------------------------
 drivers/misc/mic/vop/vop_vringh.c  | 31 -------------
 include/uapi/linux/mic_common.h    |  9 ++--
 4 files changed, 8 insertions(+), 106 deletions(-)

diff --git a/drivers/misc/mic/vop/vop_debugfs.c b/drivers/misc/mic/vop/vop_debugfs.c
index 9d4f175f4dd1..27373b6dcea2 100644
--- a/drivers/misc/mic/vop/vop_debugfs.c
+++ b/drivers/misc/mic/vop/vop_debugfs.c
@@ -62,8 +62,6 @@ static int vop_dp_show(struct seq_file *s, void *pos)
 			seq_printf(s, "address 0x%llx ",
 				   vqconfig->address);
 			seq_printf(s, "num %d ", vqconfig->num);
-			seq_printf(s, "used address 0x%llx\n",
-				   vqconfig->used_address);
 		}
 
 		features = (__u32 *)mic_vq_features(d);
@@ -79,8 +77,6 @@ static int vop_dp_show(struct seq_file *s, void *pos)
 		seq_printf(s, "Vdev reset %d\n", dc->vdev_reset);
 		seq_printf(s, "Guest Ack %d ", dc->guest_ack);
 		seq_printf(s, "Host ack %d\n", dc->host_ack);
-		seq_printf(s, "Used address updated %d ",
-			   dc->used_address_updated);
 		seq_printf(s, "Vdev 0x%llx\n", dc->vdev);
 		seq_printf(s, "c2h doorbell %d ", dc->c2h_vdev_db);
 		seq_printf(s, "h2c doorbell %d\n", dc->h2c_vdev_db);
diff --git a/drivers/misc/mic/vop/vop_main.c b/drivers/misc/mic/vop/vop_main.c
index 714b94f42d38..226f462ab6a9 100644
--- a/drivers/misc/mic/vop/vop_main.c
+++ b/drivers/misc/mic/vop/vop_main.c
@@ -32,9 +32,6 @@
  * @dc: Virtio device control
  * @vpdev: VOP device which is the parent for this virtio device
  * @vr: Buffer for accessing the VRING
- * @used_virt: Virtual address of used ring
- * @used: DMA address of used ring
- * @used_size: Size of the used buffer
  * @reset_done: Track whether VOP reset is complete
  * @virtio_cookie: Cookie returned upon requesting a interrupt
  * @c2h_vdev_db: The doorbell used by the guest to interrupt the host
@@ -47,9 +44,6 @@ struct _vop_vdev {
 	struct mic_device_ctrl __iomem *dc;
 	struct vop_device *vpdev;
 	void __iomem *vr[VOP_MAX_VRINGS];
-	void *used_virt[VOP_MAX_VRINGS];
-	dma_addr_t used[VOP_MAX_VRINGS];
-	int used_size[VOP_MAX_VRINGS];
 	struct completion reset_done;
 	struct mic_irq *virtio_cookie;
 	int c2h_vdev_db;
@@ -250,10 +244,6 @@ static void vop_del_vq(struct virtqueue *vq, int n)
 	struct _vop_vdev *vdev = to_vopvdev(vq->vdev);
 	struct vop_device *vpdev = vdev->vpdev;
 
-	dma_unmap_single(&vpdev->dev, vdev->used[n],
-			 vdev->used_size[n], DMA_BIDIRECTIONAL);
-	free_pages((unsigned long)vdev->used_virt[n],
-		   get_order(vdev->used_size[n]));
 	vring_del_virtqueue(vq);
 	vpdev->hw_ops->unmap(vpdev, vdev->vr[n]);
 	vdev->vr[n] = NULL;
@@ -278,14 +268,12 @@ static struct virtqueue *vop_new_virtqueue(unsigned int index,
 				      void *pages,
 				      bool (*notify)(struct virtqueue *vq),
 				      void (*callback)(struct virtqueue *vq),
-				      const char *name,
-				      void *used)
+				      const char *name)
 {
 	bool weak_barriers = false;
 	struct vring vring;
 
 	vring_init(&vring, num, pages, MIC_VIRTIO_RING_ALIGN);
-	vring.used = used;
 
 	return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
 				     notify, callback, name);
@@ -308,7 +296,6 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev,
 	struct virtqueue *vq;
 	void __iomem *va;
 	struct _mic_vring_info __iomem *info;
-	void *used;
 	int vr_size, _vr_size, err, magic;
 	u8 type = ioread8(&vdev->desc->type);
 
@@ -337,45 +324,16 @@ static struct virtqueue *vop_find_vq(struct virtio_device *dev,
 		goto unmap;
 	}
 
-	vdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
-					     sizeof(struct vring_used_elem) *
-					     le16_to_cpu(config.num));
-	used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
-					get_order(vdev->used_size[index]));
-	vdev->used_virt[index] = used;
-	if (!used) {
-		err = -ENOMEM;
-		dev_err(_vop_dev(vdev), "%s %d err %d\n",
-			__func__, __LINE__, err);
-		goto unmap;
-	}
-
 	vq = vop_new_virtqueue(index, le16_to_cpu(config.num), dev, ctx,
 			       (void __force *)va, vop_notify, callback,
-			       name, used);
+			       name);
 	if (!vq) {
 		err = -ENOMEM;
-		goto free_used;
-	}
-
-	vdev->used[index] = dma_map_single(&vpdev->dev, used,
-					    vdev->used_size[index],
-					    DMA_BIDIRECTIONAL);
-	if (dma_mapping_error(&vpdev->dev, vdev->used[index])) {
-		err = -ENOMEM;
-		dev_err(_vop_dev(vdev), "%s %d err %d\n",
-			__func__, __LINE__, err);
-		goto del_vq;
+		goto unmap;
 	}
-	writeq(vdev->used[index], &vqconfig->used_address);
 
 	vq->priv = vdev;
 	return vq;
-del_vq:
-	vring_del_virtqueue(vq);
-free_used:
-	free_pages((unsigned long)used,
-		   get_order(vdev->used_size[index]));
 unmap:
 	vpdev->hw_ops->unmap(vpdev, vdev->vr[index]);
 	return ERR_PTR(err);
@@ -388,9 +346,7 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
 			struct irq_affinity *desc)
 {
 	struct _vop_vdev *vdev = to_vopvdev(dev);
-	struct vop_device *vpdev = vdev->vpdev;
-	struct mic_device_ctrl __iomem *dc = vdev->dc;
-	int i, err, retry, queue_idx = 0;
+	int i, err, queue_idx = 0;
 
 	/* We must have this many virtqueues. */
 	if (nvqs > ioread8(&vdev->desc->num_vq))
@@ -412,24 +368,6 @@ static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
 		}
 	}
 
-	iowrite8(1, &dc->used_address_updated);
-	/*
-	 * Send an interrupt to the host to inform it that used
-	 * rings have been re-assigned.
-	 */
-	vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
-	for (retry = 100; --retry;) {
-		if (!ioread8(&dc->used_address_updated))
-			break;
-		msleep(100);
-	}
-
-	dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
-	if (!retry) {
-		err = -ENODEV;
-		goto error;
-	}
-
 	return 0;
 error:
 	vop_del_vqs(dev);
diff --git a/drivers/misc/mic/vop/vop_vringh.c b/drivers/misc/mic/vop/vop_vringh.c
index 39d34b164ede..fac46f3ca5fe 100644
--- a/drivers/misc/mic/vop/vop_vringh.c
+++ b/drivers/misc/mic/vop/vop_vringh.c
@@ -53,33 +53,6 @@ static void _vop_notify(struct vringh *vrh)
 		vpdev->hw_ops->send_intr(vpdev, db);
 }
 
-static void vop_virtio_init_post(struct vop_vdev *vdev)
-{
-	struct mic_vqconfig *vqconfig = mic_vq_config(vdev->dd);
-	struct vop_device *vpdev = vdev->vpdev;
-	int i, used_size;
-
-	for (i = 0; i < vdev->dd->num_vq; i++) {
-		used_size = PAGE_ALIGN(sizeof(u16) * 3 +
-				sizeof(struct vring_used_elem) *
-				le16_to_cpu(vqconfig->num));
-		if (!le64_to_cpu(vqconfig[i].used_address)) {
-			dev_warn(vop_dev(vdev), "used_address zero??\n");
-			continue;
-		}
-		vdev->vvr[i].vrh.vring.used =
-			(void __force *)vpdev->hw_ops->remap(
-			vpdev,
-			le64_to_cpu(vqconfig[i].used_address),
-			used_size);
-	}
-
-	vdev->dc->used_address_updated = 0;
-
-	dev_info(vop_dev(vdev), "%s: device type %d LINKUP\n",
-		 __func__, vdev->virtio_id);
-}
-
 static inline void vop_virtio_device_reset(struct vop_vdev *vdev)
 {
 	int i;
@@ -130,9 +103,6 @@ static void vop_bh_handler(struct work_struct *work)
 	struct vop_vdev *vdev = container_of(work, struct vop_vdev,
 			virtio_bh_work);
 
-	if (vdev->dc->used_address_updated)
-		vop_virtio_init_post(vdev);
-
 	if (vdev->dc->vdev_reset)
 		vop_virtio_device_reset(vdev);
 
@@ -250,7 +220,6 @@ static void vop_init_device_ctrl(struct vop_vdev *vdev,
 	dc->guest_ack = 0;
 	dc->vdev_reset = 0;
 	dc->host_ack = 0;
-	dc->used_address_updated = 0;
 	dc->c2h_vdev_db = -1;
 	dc->h2c_vdev_db = -1;
 	vdev->dc = dc;
diff --git a/include/uapi/linux/mic_common.h b/include/uapi/linux/mic_common.h
index 504e523f702c..9560fb033a7f 100644
--- a/include/uapi/linux/mic_common.h
+++ b/include/uapi/linux/mic_common.h
@@ -56,8 +56,7 @@ struct mic_device_desc {
  * @vdev_reset: Set to 1 by guest to indicate virtio device has been reset.
  * @guest_ack: Set to 1 by guest to ack a command.
  * @host_ack: Set to 1 by host to ack a command.
- * @used_address_updated: Set to 1 by guest when the used address should be
- * updated.
+ * @must_be_zero: Reserved because this bit is no longer needed.
  * @c2h_vdev_db: The doorbell number to be used by guest. Set by host.
  * @h2c_vdev_db: The doorbell number to be used by host. Set by guest.
  */
@@ -67,7 +66,7 @@ struct mic_device_ctrl {
 	__u8 vdev_reset;
 	__u8 guest_ack;
 	__u8 host_ack;
-	__u8 used_address_updated;
+	__u8 must_be_zero;
 	__s8 c2h_vdev_db;
 	__s8 h2c_vdev_db;
 } __attribute__ ((aligned(8)));
@@ -110,12 +109,12 @@ struct mic_device_page {
  *
  * @address: Guest/MIC physical address of the virtio ring
  * (avail and desc rings)
- * @used_address: Guest/MIC physical address of the used ring
+ * @must_be_zero: Reserved because this bit is no longer needed.
  * @num: The number of entries in the virtio_ring
  */
 struct mic_vqconfig {
 	__le64 address;
-	__le64 used_address;
+	__le64 must_be_zero;
 	__le16 num;
 } __attribute__ ((aligned(8)));
 
-- 
2.17.1

Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Greg KH]

On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> Changes in V5:
> 1. Reorganize the vop_mmap function code in patch 1, which is done by Christoph. 
> 2. Completely remove the unnecessary code related to reassign the used ring for
> card in patch 2.
> 
> The original vop driver only supports dma coherent device, as it allocates and
> maps vring by _get_free_pages and dma_map_single, but not use 
> dma_sync_single_for_cpu/device to sync the updates of device_page/vring between
> EP and RC, which will cause memory synchronization problem for device don't
> support hardware dma coherent.
> 
> And allocate vrings use dma_alloc_coherent is a common way in kernel, as the
> memory interacted between two systems should use consistent memory to avoid
> caching effects. So here add noncoherent platform support for vop driver.
> Also add some related dma changes to make sure noncoherent platform works
> well.
> 
> Sherry Sun (2):
>   misc: vop: change the way of allocating vrings and device page
>   misc: vop: do not allocate and reassign the used ring
> 
>  drivers/misc/mic/bus/vop_bus.h     |   2 +
>  drivers/misc/mic/host/mic_boot.c   |   9 ++
>  drivers/misc/mic/host/mic_main.c   |  43 ++------
>  drivers/misc/mic/vop/vop_debugfs.c |   4 -
>  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
>  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
>  include/uapi/linux/mic_common.h    |   9 +-
>  7 files changed, 85 insertions(+), 218 deletions(-)

Have you all seen:
	https://lore.kernel.org/r/8c1443136563de34699d2c084df478181c205db4.1603854416.git.sudeep.dutt@intel.com

Looks like this code is asking to just be deleted, is that ok with you?

thanks,

greg k-h

RE: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Sherry Sun]

Hi Greg,

> Subject: Re: [PATCH V5 0/2] Change vring space from nomal memory to dma
> coherent memory
> 
> On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> > Changes in V5:
> > 1. Reorganize the vop_mmap function code in patch 1, which is done by
> Christoph.
> > 2. Completely remove the unnecessary code related to reassign the used
> > ring for card in patch 2.
> >
> > The original vop driver only supports dma coherent device, as it
> > allocates and maps vring by _get_free_pages and dma_map_single, but
> > not use dma_sync_single_for_cpu/device to sync the updates of
> > device_page/vring between EP and RC, which will cause memory
> > synchronization problem for device don't support hardware dma coherent.
> >
> > And allocate vrings use dma_alloc_coherent is a common way in kernel,
> > as the memory interacted between two systems should use consistent
> > memory to avoid caching effects. So here add noncoherent platform
> support for vop driver.
> > Also add some related dma changes to make sure noncoherent platform
> > works well.
> >
> > Sherry Sun (2):
> >   misc: vop: change the way of allocating vrings and device page
> >   misc: vop: do not allocate and reassign the used ring
> >
> >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> >  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
> >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
> >  include/uapi/linux/mic_common.h    |   9 +-
> >  7 files changed, 85 insertions(+), 218 deletions(-)
> 
> Have you all seen:
> 	https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%
> 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> 03854416.git.sudeep.dutt%40intel.com&data=04%7C01%7Csherry.sun%
> 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> VCI6Mn0%3D%7C1000&sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> 9zDyv4NVN4TpC%2FU%3D&reserved=0
> 
> Looks like this code is asking to just be deleted, is that ok with you?

Yes, I saw that patch. I'm ok with it.

Best regards
Sherry

> 
> thanks,
> 
> greg k-h

Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Greg KH]

On Wed, Oct 28, 2020 at 06:05:28AM +0000, Sherry Sun wrote:
> Hi Greg,
> 
> > Subject: Re: [PATCH V5 0/2] Change vring space from nomal memory to dma
> > coherent memory
> > 
> > On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> > > Changes in V5:
> > > 1. Reorganize the vop_mmap function code in patch 1, which is done by
> > Christoph.
> > > 2. Completely remove the unnecessary code related to reassign the used
> > > ring for card in patch 2.
> > >
> > > The original vop driver only supports dma coherent device, as it
> > > allocates and maps vring by _get_free_pages and dma_map_single, but
> > > not use dma_sync_single_for_cpu/device to sync the updates of
> > > device_page/vring between EP and RC, which will cause memory
> > > synchronization problem for device don't support hardware dma coherent.
> > >
> > > And allocate vrings use dma_alloc_coherent is a common way in kernel,
> > > as the memory interacted between two systems should use consistent
> > > memory to avoid caching effects. So here add noncoherent platform
> > support for vop driver.
> > > Also add some related dma changes to make sure noncoherent platform
> > > works well.
> > >
> > > Sherry Sun (2):
> > >   misc: vop: change the way of allocating vrings and device page
> > >   misc: vop: do not allocate and reassign the used ring
> > >
> > >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> > >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> > >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> > >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> > >  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
> > >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
> > >  include/uapi/linux/mic_common.h    |   9 +-
> > >  7 files changed, 85 insertions(+), 218 deletions(-)
> > 
> > Have you all seen:
> > 	https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%
> > 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> > 03854416.git.sudeep.dutt%40intel.com&data=04%7C01%7Csherry.sun%
> > 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> > a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> > FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > VCI6Mn0%3D%7C1000&sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> > 9zDyv4NVN4TpC%2FU%3D&reserved=0
> > 
> > Looks like this code is asking to just be deleted, is that ok with you?
> 
> Yes, I saw that patch. I'm ok with it.

Great, can you please provide a "Reviewed-by:" or "Acked-by:" for it?

thanks,

greg k-h

RE: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Sherry Sun]

> Subject: Re: [PATCH V5 0/2] Change vring space from nomal memory to dma
> coherent memory
> 
> On Wed, Oct 28, 2020 at 06:05:28AM +0000, Sherry Sun wrote:
> > Hi Greg,
> >
> > > Subject: Re: [PATCH V5 0/2] Change vring space from nomal memory to
> > > dma coherent memory
> > >
> > > On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> > > > Changes in V5:
> > > > 1. Reorganize the vop_mmap function code in patch 1, which is done
> > > > by
> > > Christoph.
> > > > 2. Completely remove the unnecessary code related to reassign the
> > > > used ring for card in patch 2.
> > > >
> > > > The original vop driver only supports dma coherent device, as it
> > > > allocates and maps vring by _get_free_pages and dma_map_single,
> > > > but not use dma_sync_single_for_cpu/device to sync the updates of
> > > > device_page/vring between EP and RC, which will cause memory
> > > > synchronization problem for device don't support hardware dma
> coherent.
> > > >
> > > > And allocate vrings use dma_alloc_coherent is a common way in
> > > > kernel, as the memory interacted between two systems should use
> > > > consistent memory to avoid caching effects. So here add
> > > > noncoherent platform
> > > support for vop driver.
> > > > Also add some related dma changes to make sure noncoherent
> > > > platform works well.
> > > >
> > > > Sherry Sun (2):
> > > >   misc: vop: change the way of allocating vrings and device page
> > > >   misc: vop: do not allocate and reassign the used ring
> > > >
> > > >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> > > >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> > > >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> > > >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> > > >  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
> > > >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
> > > >  include/uapi/linux/mic_common.h    |   9 +-
> > > >  7 files changed, 85 insertions(+), 218 deletions(-)
> > >
> > > Have you all seen:
> > >
> 	https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%
> 25
> > >
> 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> > >
> 03854416.git.sudeep.dutt%40intel.com&data=04%7C01%7Csherry.sun%
> > >
> 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> > >
> a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> > >
> FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > >
> VCI6Mn0%3D%7C1000&sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> > > 9zDyv4NVN4TpC%2FU%3D&reserved=0
> > >
> > > Looks like this code is asking to just be deleted, is that ok with you?
> >
> > Yes, I saw that patch. I'm ok with it.
> 
> Great, can you please provide a "Reviewed-by:" or "Acked-by:" for it?
> 

Sure.

Best regards
Sherry

> thanks,
> 
> greg k-h

Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Vincent Whitchurch]

On Wed, Oct 28, 2020 at 06:58:36AM +0100, Greg KH wrote:
> Have you all seen:
> 	https://lore.kernel.org/r/8c1443136563de34699d2c084df478181c205db4.1603854416.git.sudeep.dutt@intel.com

No, that link doesn't work and I can't find that email from Sudeep in
any of the archives:

 https://lore.kernel.org/lkml/?q=f%3Asudeep.dutt%40intel.com

Sudeep, perhaps you could try to resend the patch?  Thank you.

RE: [EXT] Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Andy Duan]

From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday, October 28, 2020 3:07 PM
> On Wed, Oct 28, 2020 at 06:05:28AM +0000, Sherry Sun wrote:
> > Hi Greg,
> >
> > > Subject: Re: [PATCH V5 0/2] Change vring space from nomal memory to
> > > dma coherent memory
> > >
> > > On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> > > > Changes in V5:
> > > > 1. Reorganize the vop_mmap function code in patch 1, which is done
> > > > by
> > > Christoph.
> > > > 2. Completely remove the unnecessary code related to reassign the
> > > > used ring for card in patch 2.
> > > >
> > > > The original vop driver only supports dma coherent device, as it
> > > > allocates and maps vring by _get_free_pages and dma_map_single,
> > > > but not use dma_sync_single_for_cpu/device to sync the updates of
> > > > device_page/vring between EP and RC, which will cause memory
> > > > synchronization problem for device don't support hardware dma coherent.
> > > >
> > > > And allocate vrings use dma_alloc_coherent is a common way in
> > > > kernel, as the memory interacted between two systems should use
> > > > consistent memory to avoid caching effects. So here add
> > > > noncoherent platform
> > > support for vop driver.
> > > > Also add some related dma changes to make sure noncoherent
> > > > platform works well.
> > > >
> > > > Sherry Sun (2):
> > > >   misc: vop: change the way of allocating vrings and device page
> > > >   misc: vop: do not allocate and reassign the used ring
> > > >
> > > >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> > > >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> > > >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> > > >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> > > >  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
> > > >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
> > > >  include/uapi/linux/mic_common.h    |   9 +-
> > > >  7 files changed, 85 insertions(+), 218 deletions(-)
> > >
> > > Have you all seen:
> > >
> > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%25
> > >
> 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> > >
> 03854416.git.sudeep.dutt%40intel.com&amp;data=04%7C01%7Csherry.sun%
> > >
> 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> > >
> a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> > >
> FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > >
> VCI6Mn0%3D%7C1000&amp;sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> > > 9zDyv4NVN4TpC%2FU%3D&amp;reserved=0
> > >
> > > Looks like this code is asking to just be deleted, is that ok with you?
> >
> > Yes, I saw that patch. I'm ok with it.
> 
> Great, can you please provide a "Reviewed-by:" or "Acked-by:" for it?
> 
> thanks,
> 
> greg k-h

Sherry took much effort on the features support on i.MX series like i.MX8QM/i.MX8QXP/i.MX8MM.

Now it is a pity to delete the vop code.

One question, 
can we resubmit vop code by clean up, now only for i.MX series as Dutt's suggestion ?
Or we have to drop the design and switch to select other solutions ?

Thanks,
Andy

Re: [EXT] Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Greg KH]

On Wed, Oct 28, 2020 at 10:17:39AM +0000, Andy Duan wrote:
> From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday, October 28, 2020 3:07 PM
> > On Wed, Oct 28, 2020 at 06:05:28AM +0000, Sherry Sun wrote:
> > > Hi Greg,
> > >
> > > > Subject: Re: [PATCH V5 0/2] Change vring space from nomal memory to
> > > > dma coherent memory
> > > >
> > > > On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> > > > > Changes in V5:
> > > > > 1. Reorganize the vop_mmap function code in patch 1, which is done
> > > > > by
> > > > Christoph.
> > > > > 2. Completely remove the unnecessary code related to reassign the
> > > > > used ring for card in patch 2.
> > > > >
> > > > > The original vop driver only supports dma coherent device, as it
> > > > > allocates and maps vring by _get_free_pages and dma_map_single,
> > > > > but not use dma_sync_single_for_cpu/device to sync the updates of
> > > > > device_page/vring between EP and RC, which will cause memory
> > > > > synchronization problem for device don't support hardware dma coherent.
> > > > >
> > > > > And allocate vrings use dma_alloc_coherent is a common way in
> > > > > kernel, as the memory interacted between two systems should use
> > > > > consistent memory to avoid caching effects. So here add
> > > > > noncoherent platform
> > > > support for vop driver.
> > > > > Also add some related dma changes to make sure noncoherent
> > > > > platform works well.
> > > > >
> > > > > Sherry Sun (2):
> > > > >   misc: vop: change the way of allocating vrings and device page
> > > > >   misc: vop: do not allocate and reassign the used ring
> > > > >
> > > > >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> > > > >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> > > > >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> > > > >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> > > > >  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
> > > > >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
> > > > >  include/uapi/linux/mic_common.h    |   9 +-
> > > > >  7 files changed, 85 insertions(+), 218 deletions(-)
> > > >
> > > > Have you all seen:
> > > >
> > > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%25
> > > >
> > 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> > > >
> > 03854416.git.sudeep.dutt%40intel.com&amp;data=04%7C01%7Csherry.sun%
> > > >
> > 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> > > >
> > a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> > > >
> > FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > > >
> > VCI6Mn0%3D%7C1000&amp;sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> > > > 9zDyv4NVN4TpC%2FU%3D&amp;reserved=0
> > > >
> > > > Looks like this code is asking to just be deleted, is that ok with you?
> > >
> > > Yes, I saw that patch. I'm ok with it.
> > 
> > Great, can you please provide a "Reviewed-by:" or "Acked-by:" for it?
> > 
> > thanks,
> > 
> > greg k-h
> 
> Sherry took much effort on the features support on i.MX series like i.MX8QM/i.MX8QXP/i.MX8MM.
> 
> Now it is a pity to delete the vop code.
> 
> One question, 
> can we resubmit vop code by clean up, now only for i.MX series as Dutt's suggestion ?
> Or we have to drop the design and switch to select other solutions ?

If this whole subsystem is being deleted because it is not used and
never shipped, yes, please use a different solution.

I don't understand why you were trying to piggy-back on this codebase if
the hardware was totally different, for some reason I thought this was
the same hardware.  What exactly is this?

thanks,

greg k-h

RE: [EXT] Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Andy Duan]

From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday, October 28, 2020 7:14 PM
> On Wed, Oct 28, 2020 at 10:17:39AM +0000, Andy Duan wrote:
> > From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday, October
> > 28, 2020 3:07 PM
> > > On Wed, Oct 28, 2020 at 06:05:28AM +0000, Sherry Sun wrote:
> > > > Hi Greg,
> > > >
> > > > > Subject: Re: [PATCH V5 0/2] Change vring space from nomal memory
> > > > > to dma coherent memory
> > > > >
> > > > > On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> > > > > > Changes in V5:
> > > > > > 1. Reorganize the vop_mmap function code in patch 1, which is
> > > > > > done by
> > > > > Christoph.
> > > > > > 2. Completely remove the unnecessary code related to reassign
> > > > > > the used ring for card in patch 2.
> > > > > >
> > > > > > The original vop driver only supports dma coherent device, as
> > > > > > it allocates and maps vring by _get_free_pages and
> > > > > > dma_map_single, but not use dma_sync_single_for_cpu/device to
> > > > > > sync the updates of device_page/vring between EP and RC, which
> > > > > > will cause memory synchronization problem for device don't support
> hardware dma coherent.
> > > > > >
> > > > > > And allocate vrings use dma_alloc_coherent is a common way in
> > > > > > kernel, as the memory interacted between two systems should
> > > > > > use consistent memory to avoid caching effects. So here add
> > > > > > noncoherent platform
> > > > > support for vop driver.
> > > > > > Also add some related dma changes to make sure noncoherent
> > > > > > platform works well.
> > > > > >
> > > > > > Sherry Sun (2):
> > > > > >   misc: vop: change the way of allocating vrings and device page
> > > > > >   misc: vop: do not allocate and reassign the used ring
> > > > > >
> > > > > >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> > > > > >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> > > > > >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> > > > > >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> > > > > >  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
> > > > > >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
> > > > > >  include/uapi/linux/mic_common.h    |   9 +-
> > > > > >  7 files changed, 85 insertions(+), 218 deletions(-)
> > > > >
> > > > > Have you all seen:
> > > > >
> > > > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%
> > > > > 25
> > > > >
> > >
> 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> > > > >
> > >
> 03854416.git.sudeep.dutt%40intel.com&amp;data=04%7C01%7Csherry.sun%
> > > > >
> > >
> 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> > > > >
> > >
> a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> > > > >
> > >
> FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > > > >
> > >
> VCI6Mn0%3D%7C1000&amp;sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> > > > > 9zDyv4NVN4TpC%2FU%3D&amp;reserved=0
> > > > >
> > > > > Looks like this code is asking to just be deleted, is that ok with you?
> > > >
> > > > Yes, I saw that patch. I'm ok with it.
> > >
> > > Great, can you please provide a "Reviewed-by:" or "Acked-by:" for it?
> > >
> > > thanks,
> > >
> > > greg k-h
> >
> > Sherry took much effort on the features support on i.MX series like
> i.MX8QM/i.MX8QXP/i.MX8MM.
> >
> > Now it is a pity to delete the vop code.
> >
> > One question,
> > can we resubmit vop code by clean up, now only for i.MX series as Dutt's
> suggestion ?
> > Or we have to drop the design and switch to select other solutions ?
>

Okay, we plan to switch to NTB solution.
 
> If this whole subsystem is being deleted because it is not used and never shipped,
> yes, please use a different solution.
> 
> I don't understand why you were trying to piggy-back on this codebase if the
> hardware was totally different, for some reason I thought this was the same
> hardware.  What exactly is this?

Not the whole codebase, just the vop framework.

> 
> thanks,
> 
> greg k-h

Re: [EXT] Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Greg KH]

On Wed, Oct 28, 2020 at 03:11:15PM +0000, Andy Duan wrote:
> From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday, October 28, 2020 7:14 PM
> > On Wed, Oct 28, 2020 at 10:17:39AM +0000, Andy Duan wrote:
> > > From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday, October
> > > 28, 2020 3:07 PM
> > > > On Wed, Oct 28, 2020 at 06:05:28AM +0000, Sherry Sun wrote:
> > > > > Hi Greg,
> > > > >
> > > > > > Subject: Re: [PATCH V5 0/2] Change vring space from nomal memory
> > > > > > to dma coherent memory
> > > > > >
> > > > > > On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> > > > > > > Changes in V5:
> > > > > > > 1. Reorganize the vop_mmap function code in patch 1, which is
> > > > > > > done by
> > > > > > Christoph.
> > > > > > > 2. Completely remove the unnecessary code related to reassign
> > > > > > > the used ring for card in patch 2.
> > > > > > >
> > > > > > > The original vop driver only supports dma coherent device, as
> > > > > > > it allocates and maps vring by _get_free_pages and
> > > > > > > dma_map_single, but not use dma_sync_single_for_cpu/device to
> > > > > > > sync the updates of device_page/vring between EP and RC, which
> > > > > > > will cause memory synchronization problem for device don't support
> > hardware dma coherent.
> > > > > > >
> > > > > > > And allocate vrings use dma_alloc_coherent is a common way in
> > > > > > > kernel, as the memory interacted between two systems should
> > > > > > > use consistent memory to avoid caching effects. So here add
> > > > > > > noncoherent platform
> > > > > > support for vop driver.
> > > > > > > Also add some related dma changes to make sure noncoherent
> > > > > > > platform works well.
> > > > > > >
> > > > > > > Sherry Sun (2):
> > > > > > >   misc: vop: change the way of allocating vrings and device page
> > > > > > >   misc: vop: do not allocate and reassign the used ring
> > > > > > >
> > > > > > >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> > > > > > >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> > > > > > >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> > > > > > >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> > > > > > >  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
> > > > > > >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------------
> > > > > > >  include/uapi/linux/mic_common.h    |   9 +-
> > > > > > >  7 files changed, 85 insertions(+), 218 deletions(-)
> > > > > >
> > > > > > Have you all seen:
> > > > > >
> > > > > > https://eur01.safelinks.protection.outlook.com/?url=https%3A%2F%
> > > > > > 25
> > > > > >
> > > >
> > 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> > > > > >
> > > >
> > 03854416.git.sudeep.dutt%40intel.com&amp;data=04%7C01%7Csherry.sun%
> > > > > >
> > > >
> > 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> > > > > >
> > > >
> > a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> > > > > >
> > > >
> > FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > > > > >
> > > >
> > VCI6Mn0%3D%7C1000&amp;sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> > > > > > 9zDyv4NVN4TpC%2FU%3D&amp;reserved=0
> > > > > >
> > > > > > Looks like this code is asking to just be deleted, is that ok with you?
> > > > >
> > > > > Yes, I saw that patch. I'm ok with it.
> > > >
> > > > Great, can you please provide a "Reviewed-by:" or "Acked-by:" for it?
> > > >
> > > > thanks,
> > > >
> > > > greg k-h
> > >
> > > Sherry took much effort on the features support on i.MX series like
> > i.MX8QM/i.MX8QXP/i.MX8MM.
> > >
> > > Now it is a pity to delete the vop code.
> > >
> > > One question,
> > > can we resubmit vop code by clean up, now only for i.MX series as Dutt's
> > suggestion ?
> > > Or we have to drop the design and switch to select other solutions ?
> >
> 
> Okay, we plan to switch to NTB solution.

What is a "NTB solution" exactly?

>  
> > If this whole subsystem is being deleted because it is not used and never shipped,
> > yes, please use a different solution.
> > 
> > I don't understand why you were trying to piggy-back on this codebase if the
> > hardware was totally different, for some reason I thought this was the same
> > hardware.  What exactly is this?
> 
> Not the whole codebase, just the vop framework.

That didn't answer the question at all, what are you all trying to do
here, with what hardware, that the VOP code seemed like a good fit?

thanks,

greg k-h

RE: [EXT] Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Sherry Sun]

Hi Greg,

> Subject: Re: [EXT] Re: [PATCH V5 0/2] Change vring space from nomal
> memory to dma coherent memory
> 
> On Wed, Oct 28, 2020 at 03:11:15PM +0000, Andy Duan wrote:
> > From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday, October
> > 28, 2020 7:14 PM
> > > On Wed, Oct 28, 2020 at 10:17:39AM +0000, Andy Duan wrote:
> > > > From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday,
> > > > October 28, 2020 3:07 PM
> > > > > On Wed, Oct 28, 2020 at 06:05:28AM +0000, Sherry Sun wrote:
> > > > > > Hi Greg,
> > > > > >
> > > > > > > Subject: Re: [PATCH V5 0/2] Change vring space from nomal
> > > > > > > memory to dma coherent memory
> > > > > > >
> > > > > > > On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun wrote:
> > > > > > > > Changes in V5:
> > > > > > > > 1. Reorganize the vop_mmap function code in patch 1, which
> > > > > > > > is done by
> > > > > > > Christoph.
> > > > > > > > 2. Completely remove the unnecessary code related to
> > > > > > > > reassign the used ring for card in patch 2.
> > > > > > > >
> > > > > > > > The original vop driver only supports dma coherent device,
> > > > > > > > as it allocates and maps vring by _get_free_pages and
> > > > > > > > dma_map_single, but not use dma_sync_single_for_cpu/device
> > > > > > > > to sync the updates of device_page/vring between EP and
> > > > > > > > RC, which will cause memory synchronization problem for
> > > > > > > > device don't support
> > > hardware dma coherent.
> > > > > > > >
> > > > > > > > And allocate vrings use dma_alloc_coherent is a common way
> > > > > > > > in kernel, as the memory interacted between two systems
> > > > > > > > should use consistent memory to avoid caching effects. So
> > > > > > > > here add noncoherent platform
> > > > > > > support for vop driver.
> > > > > > > > Also add some related dma changes to make sure noncoherent
> > > > > > > > platform works well.
> > > > > > > >
> > > > > > > > Sherry Sun (2):
> > > > > > > >   misc: vop: change the way of allocating vrings and device page
> > > > > > > >   misc: vop: do not allocate and reassign the used ring
> > > > > > > >
> > > > > > > >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> > > > > > > >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> > > > > > > >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> > > > > > > >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> > > > > > > >  drivers/misc/mic/vop/vop_main.c    |  70 +-----------
> > > > > > > >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-------------
> ------
> > > > > > > >  include/uapi/linux/mic_common.h    |   9 +-
> > > > > > > >  7 files changed, 85 insertions(+), 218 deletions(-)
> > > > > > >
> > > > > > > Have you all seen:
> > > > > > >
> > > > > > > https://eur01.safelinks.protection.outlook.com/?url=https%3A
> > > > > > > %2F%25
> > > > > > > 25
> > > > > > >
> > > > >
> > >
> 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> > > > > > >
> > > > >
> > >
> 03854416.git.sudeep.dutt%40intel.com&amp;data=04%7C01%7Csherry.sun%
> > > > > > >
> > > > >
> > >
> 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> > > > > > >
> > > > >
> > >
> a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> > > > > > >
> > > > >
> > >
> FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > > > > > >
> > > > >
> > >
> VCI6Mn0%3D%7C1000&amp;sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> > > > > > > 9zDyv4NVN4TpC%2FU%3D&amp;reserved=0
> > > > > > >
> > > > > > > Looks like this code is asking to just be deleted, is that ok with you?
> > > > > >
> > > > > > Yes, I saw that patch. I'm ok with it.
> > > > >
> > > > > Great, can you please provide a "Reviewed-by:" or "Acked-by:" for it?
> > > > >
> > > > > thanks,
> > > > >
> > > > > greg k-h
> > > >
> > > > Sherry took much effort on the features support on i.MX series
> > > > like
> > > i.MX8QM/i.MX8QXP/i.MX8MM.
> > > >
> > > > Now it is a pity to delete the vop code.
> > > >
> > > > One question,
> > > > can we resubmit vop code by clean up, now only for i.MX series as
> > > > Dutt's
> > > suggestion ?
> > > > Or we have to drop the design and switch to select other solutions ?
> > >
> >
> > Okay, we plan to switch to NTB solution.
> 
> What is a "NTB solution" exactly?

The driver located at drivers/ntb/, it also can setup a point-to-point PCI-E bus connecting between two systems.
But we haven't got a deep look of this driver yet, so we are not sure whether it can replace the vop framework.

> 
> >
> > > If this whole subsystem is being deleted because it is not used and
> > > never shipped, yes, please use a different solution.
> > >
> > > I don't understand why you were trying to piggy-back on this
> > > codebase if the hardware was totally different, for some reason I
> > > thought this was the same hardware.  What exactly is this?
> >
> > Not the whole codebase, just the vop framework.
> 
> That didn't answer the question at all, what are you all trying to do here, with
> what hardware, that the VOP code seemed like a good fit?

Vop is a common framework which is independent of the Intel MIC hardware.
We planed to reuse vop framework on two arm64 architecture devices, to setup the connection between two systems based on virtio over PCIE.

Best regards
Sherry

> 
> thanks,
> 
> greg k-h

Re: [EXT] Re: [PATCH V5 0/2] Change vring space from nomal memory to dma coherent memory

[Dutt, Sudeep]

On Thu, 2020-10-29 at 01:51 +0000, Sherry Sun wrote:
> Hi Greg,
> 
> > Subject: Re: [EXT] Re: [PATCH V5 0/2] Change vring space from nomal
> > memory to dma coherent memory
> > 
> > On Wed, Oct 28, 2020 at 03:11:15PM +0000, Andy Duan wrote:
> > > From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday,
> > > October
> > > 28, 2020 7:14 PM
> > > > On Wed, Oct 28, 2020 at 10:17:39AM +0000, Andy Duan wrote:
> > > > > From: Greg KH <gregkh@linuxfoundation.org> Sent: Wednesday,
> > > > > October 28, 2020 3:07 PM
> > > > > > On Wed, Oct 28, 2020 at 06:05:28AM +0000, Sherry Sun wrote:
> > > > > > > Hi Greg,
> > > > > > > 
> > > > > > > > Subject: Re: [PATCH V5 0/2] Change vring space from
> > > > > > > > nomal
> > > > > > > > memory to dma coherent memory
> > > > > > > > 
> > > > > > > > On Wed, Oct 28, 2020 at 10:03:03AM +0800, Sherry Sun
> > > > > > > > wrote:
> > > > > > > > > Changes in V5:
> > > > > > > > > 1. Reorganize the vop_mmap function code in patch 1,
> > > > > > > > > which
> > > > > > > > > is done by
> > > > > > > > 
> > > > > > > > Christoph.
> > > > > > > > > 2. Completely remove the unnecessary code related to
> > > > > > > > > reassign the used ring for card in patch 2.
> > > > > > > > > 
> > > > > > > > > The original vop driver only supports dma coherent
> > > > > > > > > device,
> > > > > > > > > as it allocates and maps vring by _get_free_pages and
> > > > > > > > > dma_map_single, but not use
> > > > > > > > > dma_sync_single_for_cpu/device
> > > > > > > > > to sync the updates of device_page/vring between EP
> > > > > > > > > and
> > > > > > > > > RC, which will cause memory synchronization problem
> > > > > > > > > for
> > > > > > > > > device don't support
> > > > 
> > > > hardware dma coherent.
> > > > > > > > > 
> > > > > > > > > And allocate vrings use dma_alloc_coherent is a
> > > > > > > > > common way
> > > > > > > > > in kernel, as the memory interacted between two
> > > > > > > > > systems
> > > > > > > > > should use consistent memory to avoid caching
> > > > > > > > > effects. So
> > > > > > > > > here add noncoherent platform
> > > > > > > > 
> > > > > > > > support for vop driver.
> > > > > > > > > Also add some related dma changes to make sure
> > > > > > > > > noncoherent
> > > > > > > > > platform works well.
> > > > > > > > > 
> > > > > > > > > Sherry Sun (2):
> > > > > > > > >   misc: vop: change the way of allocating vrings and
> > > > > > > > > device page
> > > > > > > > >   misc: vop: do not allocate and reassign the used
> > > > > > > > > ring
> > > > > > > > > 
> > > > > > > > >  drivers/misc/mic/bus/vop_bus.h     |   2 +
> > > > > > > > >  drivers/misc/mic/host/mic_boot.c   |   9 ++
> > > > > > > > >  drivers/misc/mic/host/mic_main.c   |  43 ++------
> > > > > > > > >  drivers/misc/mic/vop/vop_debugfs.c |   4 -
> > > > > > > > >  drivers/misc/mic/vop/vop_main.c    |  70 +--------
> > > > > > > > > ---
> > > > > > > > >  drivers/misc/mic/vop/vop_vringh.c  | 166 ++++++++++-
> > > > > > > > > ------------
> > 
> > ------
> > > > > > > > >  include/uapi/linux/mic_common.h    |   9 +-
> > > > > > > > >  7 files changed, 85 insertions(+), 218 deletions(-)
> > > > > > > > 
> > > > > > > > Have you all seen:
> > > > > > > > 
> > > > > > > > 
https://eur01.safelinks.protection.outlook.com/?url=https%3A
> > > > > > > > %2F%25
> > > > > > > > 25
> > > > > > > > 
> > 
> > 2Flore.kernel.org%2Fr%2F8c1443136563de34699d2c084df478181c205db4.16
> > > > > > > > 
> > 
> > 03854416.git.sudeep.dutt%40intel.com&amp;data=04%7C01%7Csherry.sun%
> > > > > > > > 
> > 
> > 40nxp.com%7Cc19c987667434969847e08d87b0685e8%7C686ea1d3bc2b4c6f
> > > > > > > > 
> > 
> > a92cd99c5c301635%7C0%7C0%7C637394615238940323%7CUnknown%7CTW
> > > > > > > > 
> > 
> > FpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
> > > > > > > > 
> > 
> > VCI6Mn0%3D%7C1000&amp;sdata=Zq%2FtHWTq%2BuIVBYXFGoeBmq0JJzYd
> > > > > > > > 9zDyv4NVN4TpC%2FU%3D&amp;reserved=0
> > > > > > > > 
> > > > > > > > Looks like this code is asking to just be deleted, is
> > > > > > > > that ok with you?
> > > > > > > 
> > > > > > > Yes, I saw that patch. I'm ok with it.
> > > > > > 
> > > > > > Great, can you please provide a "Reviewed-by:" or "Acked-
> > > > > > by:" for it?
> > > > > > 
> > > > > > thanks,
> > > > > > 
> > > > > > greg k-h
> > > > > 
> > > > > Sherry took much effort on the features support on i.MX
> > > > > series
> > > > > like
> > > > 
> > > > i.MX8QM/i.MX8QXP/i.MX8MM.
> > > > > 
> > > > > Now it is a pity to delete the vop code.
> > > > > 
> > > > > One question,
> > > > > can we resubmit vop code by clean up, now only for i.MX
> > > > > series as
> > > > > Dutt's
> > > > 
> > > > suggestion ?

Resubmitting the VOP code with cleanups tailored for i.MX makes sense
to me.

> > > > > Or we have to drop the design and switch to select other
> > > > > solutions ?
> > > 
> > > Okay, we plan to switch to NTB solution.
> > 
> > What is a "NTB solution" exactly?
> 
> The driver located at drivers/ntb/, it also can setup a point-to-
> point PCI-E bus connecting between two systems.
> But we haven't got a deep look of this driver yet, so we are not sure
> whether it can replace the vop framework.
> 
> > 
> > > 
> > > > If this whole subsystem is being deleted because it is not used
> > > > and
> > > > never shipped, yes, please use a different solution.
> > > > 
> > > > I don't understand why you were trying to piggy-back on this
> > > > codebase if the hardware was totally different, for some reason
> > > > I
> > > > thought this was the same hardware.  What exactly is this?
> > > 
> > > Not the whole codebase, just the vop framework.
> > 
> > That didn't answer the question at all, what are you all trying to
> > do here, with
> > what hardware, that the VOP code seemed like a good fit?
> 
> Vop is a common framework which is independent of the Intel MIC
> hardware.
> We planed to reuse vop framework on two arm64 architecture devices,
> to setup the connection between two systems based on virtio over
> PCIE.

Yes, we wanted Virtio Over PCIe (VOP) to be independent of the hardware
as much as possible. It did end up under the mic/ driver subsystem
though so it would be good to attempt placing it in a generic folder
which is not tied to a specific hardware layer this time around.

Regards,
Sudeep Dutt