[PATCH 1/6] dma-buf: add peer2peer flag

[PATCH 1/6] dma-buf: add peer2peer flag

[Christian König]

Add a peer2peer flag noting that the importer can deal with device
resources which are not backed by pages.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/dma-buf/dma-buf.c |  2 ++
 include/linux/dma-buf.h   | 10 ++++++++++
 2 files changed, 12 insertions(+)

diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index ccc9eda1bc28..570c923023e6 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
 
 	attach->dev = dev;
 	attach->dmabuf = dmabuf;
+	if (importer_ops)
+		attach->peer2peer = importer_ops->allow_peer2peer;
 	attach->importer_ops = importer_ops;
 	attach->importer_priv = importer_priv;
 
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 1ade486fc2bb..82e0a4a64601 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -334,6 +334,14 @@ struct dma_buf {
  * Attachment operations implemented by the importer.
  */
 struct dma_buf_attach_ops {
+	/**
+	 * @allow_peer2peer:
+	 *
+	 * If this is set to true the importer must be able to handle peer
+	 * resources without struct pages.
+	 */
+	bool allow_peer2peer;
+
 	/**
 	 * @move_notify
 	 *
@@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
  * @node: list of dma_buf_attachment, protected by dma_resv lock of the dmabuf.
  * @sgt: cached mapping.
  * @dir: direction of cached mapping.
+ * @peer2peer: true if the importer can handle peer resources without pages.
  * @priv: exporter specific attachment data.
  * @importer_ops: importer operations for this attachment, if provided
  * dma_buf_map/unmap_attachment() must be called with the dma_resv lock held.
@@ -382,6 +391,7 @@ struct dma_buf_attachment {
 	struct list_head node;
 	struct sg_table *sgt;
 	enum dma_data_direction dir;
+	bool peer2peer;
 	const struct dma_buf_attach_ops *importer_ops;
 	void *importer_priv;
 	void *priv;
-- 
2.17.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 1/6] dma-buf: add peer2peer flag

[Christian König]

Add a peer2peer flag noting that the importer can deal with device
resources which are not backed by pages.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/dma-buf/dma-buf.c |  2 ++
 include/linux/dma-buf.h   | 10 ++++++++++
 2 files changed, 12 insertions(+)

diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index ccc9eda1bc28..570c923023e6 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
 
 	attach->dev = dev;
 	attach->dmabuf = dmabuf;
+	if (importer_ops)
+		attach->peer2peer = importer_ops->allow_peer2peer;
 	attach->importer_ops = importer_ops;
 	attach->importer_priv = importer_priv;
 
diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
index 1ade486fc2bb..82e0a4a64601 100644
--- a/include/linux/dma-buf.h
+++ b/include/linux/dma-buf.h
@@ -334,6 +334,14 @@ struct dma_buf {
  * Attachment operations implemented by the importer.
  */
 struct dma_buf_attach_ops {
+	/**
+	 * @allow_peer2peer:
+	 *
+	 * If this is set to true the importer must be able to handle peer
+	 * resources without struct pages.
+	 */
+	bool allow_peer2peer;
+
 	/**
 	 * @move_notify
 	 *
@@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
  * @node: list of dma_buf_attachment, protected by dma_resv lock of the dmabuf.
  * @sgt: cached mapping.
  * @dir: direction of cached mapping.
+ * @peer2peer: true if the importer can handle peer resources without pages.
  * @priv: exporter specific attachment data.
  * @importer_ops: importer operations for this attachment, if provided
  * dma_buf_map/unmap_attachment() must be called with the dma_resv lock held.
@@ -382,6 +391,7 @@ struct dma_buf_attachment {
 	struct list_head node;
 	struct sg_table *sgt;
 	enum dma_data_direction dir;
+	bool peer2peer;
 	const struct dma_buf_attach_ops *importer_ops;
 	void *importer_priv;
 	void *priv;
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 2/6] drm/ttm: lock resv object during destruction

[Christian König]

Calling ttm_bo_cleanup_memtype_use() destroys the TT object
which in turn could result in warnings without this.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/ttm/ttm_bo.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 9e07c3f75156..f73b81c2576e 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -588,7 +588,8 @@ static void ttm_bo_release(struct kref *kref)
 		ttm_mem_io_unlock(man);
 	}
 
-	if (!dma_resv_test_signaled_rcu(bo->base.resv, true)) {
+	if (!dma_resv_test_signaled_rcu(bo->base.resv, true) ||
+	    !dma_resv_trylock(bo->base.resv)) {
 		/* The BO is not idle, resurrect it for delayed destroy */
 		ttm_bo_flush_all_fences(bo);
 		bo->deleted = true;
@@ -621,6 +622,7 @@ static void ttm_bo_release(struct kref *kref)
 	spin_unlock(&ttm_bo_glob.lru_lock);
 
 	ttm_bo_cleanup_memtype_use(bo);
+	dma_resv_unlock(bo->base.resv);
 
 	BUG_ON(bo->mem.mm_node != NULL);
 	atomic_dec(&ttm_bo_glob.bo_count);
-- 
2.17.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 2/6] drm/ttm: lock resv object during destruction

[Christian König]

Calling ttm_bo_cleanup_memtype_use() destroys the TT object
which in turn could result in warnings without this.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/ttm/ttm_bo.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 9e07c3f75156..f73b81c2576e 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -588,7 +588,8 @@ static void ttm_bo_release(struct kref *kref)
 		ttm_mem_io_unlock(man);
 	}
 
-	if (!dma_resv_test_signaled_rcu(bo->base.resv, true)) {
+	if (!dma_resv_test_signaled_rcu(bo->base.resv, true) ||
+	    !dma_resv_trylock(bo->base.resv)) {
 		/* The BO is not idle, resurrect it for delayed destroy */
 		ttm_bo_flush_all_fences(bo);
 		bo->deleted = true;
@@ -621,6 +622,7 @@ static void ttm_bo_release(struct kref *kref)
 	spin_unlock(&ttm_bo_glob.lru_lock);
 
 	ttm_bo_cleanup_memtype_use(bo);
+	dma_resv_unlock(bo->base.resv);
 
 	BUG_ON(bo->mem.mm_node != NULL);
 	atomic_dec(&ttm_bo_glob.bo_count);
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 3/6] drm/amdgpu: note that we can handle peer2peer DMA-buf

[Christian König]

Importing should work out of the box.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index ffeb20f11c07..aef12ee2f1e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -514,6 +514,7 @@ amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach)
 }
 
 static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops = {
+	.allow_peer2peer = true,
 	.move_notify = amdgpu_dma_buf_move_notify
 };
 
-- 
2.17.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 3/6] drm/amdgpu: note that we can handle peer2peer DMA-buf

[Christian König]

Importing should work out of the box.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index ffeb20f11c07..aef12ee2f1e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -514,6 +514,7 @@ amdgpu_dma_buf_move_notify(struct dma_buf_attachment *attach)
 }
 
 static const struct dma_buf_attach_ops amdgpu_dma_buf_attach_ops = {
+	.allow_peer2peer = true,
 	.move_notify = amdgpu_dma_buf_move_notify
 };
 
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Christian König]

Check if we can do peer2peer on the PCIe bus.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index aef12ee2f1e3..bbf67800c8a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -38,6 +38,7 @@
 #include <drm/amdgpu_drm.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-fence-array.h>
+#include <linux/pci-p2pdma.h>
 
 /**
  * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
@@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	int r;
 
+	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
+		attach->peer2peer = false;
+
 	if (attach->dev->driver == adev->dev->driver)
 		return 0;
 
-- 
2.17.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Christian König]

Check if we can do peer2peer on the PCIe bus.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index aef12ee2f1e3..bbf67800c8a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -38,6 +38,7 @@
 #include <drm/amdgpu_drm.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-fence-array.h>
+#include <linux/pci-p2pdma.h>
 
 /**
  * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
@@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	int r;
 
+	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
+		attach->peer2peer = false;
+
 	if (attach->dev->driver == adev->dev->driver)
 		return 0;
 
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 5/6] drm/amdgpu: add support for exporting VRAM using DMA-buf v3

[Christian König]

We should be able to do this now after checking all the prerequisites.

v2: fix entrie count in the sgt
v3: manually construct the sg

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c  | 56 ++++++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h      | 12 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 99 ++++++++++++++++++++
 3 files changed, 153 insertions(+), 14 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index bbf67800c8a6..43d8ed7dbd00 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -276,14 +276,21 @@ static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach,
 	struct dma_buf *dma_buf = attach->dmabuf;
 	struct drm_gem_object *obj = dma_buf->priv;
 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct sg_table *sgt;
 	long r;
 
 	if (!bo->pin_count) {
-		/* move buffer into GTT */
+		/* move buffer into GTT or VRAM */
 		struct ttm_operation_ctx ctx = { false, false };
+		unsigned domains = AMDGPU_GEM_DOMAIN_GTT;
 
-		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+		if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM &&
+		    attach->peer2peer) {
+			bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+			domains |= AMDGPU_GEM_DOMAIN_VRAM;
+		}
+		amdgpu_bo_placement_from_domain(bo, domains);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		if (r)
 			return ERR_PTR(r);
@@ -293,20 +300,34 @@ static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach,
 		return ERR_PTR(-EBUSY);
 	}
 
-	sgt = drm_prime_pages_to_sg(bo->tbo.ttm->pages, bo->tbo.num_pages);
-	if (IS_ERR(sgt))
-		return sgt;
-
-	if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir,
-			      DMA_ATTR_SKIP_CPU_SYNC))
-		goto error_free;
+	switch (bo->tbo.mem.mem_type) {
+	case TTM_PL_TT:
+		sgt = drm_prime_pages_to_sg(bo->tbo.ttm->pages,
+					    bo->tbo.num_pages);
+		if (IS_ERR(sgt))
+			return sgt;
+
+		if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir,
+				      DMA_ATTR_SKIP_CPU_SYNC))
+			goto error_free;
+		break;
+
+	case TTM_PL_VRAM:
+		r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, attach->dev,
+					      dir, &sgt);
+		if (r)
+			return ERR_PTR(r);
+		break;
+	default:
+		return ERR_PTR(-EINVAL);
+	}
 
 	return sgt;
 
 error_free:
 	sg_free_table(sgt);
 	kfree(sgt);
-	return ERR_PTR(-ENOMEM);
+	return ERR_PTR(-EBUSY);
 }
 
 /**
@@ -322,9 +343,18 @@ static void amdgpu_dma_buf_unmap(struct dma_buf_attachment *attach,
 				 struct sg_table *sgt,
 				 enum dma_data_direction dir)
 {
-	dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
-	sg_free_table(sgt);
-	kfree(sgt);
+	struct dma_buf *dma_buf = attach->dmabuf;
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+
+	if (sgt->sgl->page_link) {
+		dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
+		sg_free_table(sgt);
+		kfree(sgt);
+	} else {
+		amdgpu_vram_mgr_free_sgt(adev, attach->dev, dir, sgt);
+	}
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index bd05bbb4878d..6b22dc41ef13 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -24,8 +24,9 @@
 #ifndef __AMDGPU_TTM_H__
 #define __AMDGPU_TTM_H__
 
-#include "amdgpu.h"
+#include <linux/dma-direction.h>
 #include <drm/gpu_scheduler.h>
+#include "amdgpu.h"
 
 #define AMDGPU_PL_GDS		(TTM_PL_PRIV + 0)
 #define AMDGPU_PL_GWS		(TTM_PL_PRIV + 1)
@@ -74,6 +75,15 @@ uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man);
 int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man);
 
 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo);
+int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
+			      struct ttm_mem_reg *mem,
+			      struct device *dev,
+			      enum dma_data_direction dir,
+			      struct sg_table **sgt);
+void amdgpu_vram_mgr_free_sgt(struct amdgpu_device *adev,
+			      struct device *dev,
+			      enum dma_data_direction dir,
+			      struct sg_table *sgt);
 uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man);
 uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 82a3299e53c0..128a667ed8fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -22,6 +22,7 @@
  * Authors: Christian König
  */
 
+#include <linux/dma-mapping.h>
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
 #include "amdgpu_atomfirmware.h"
@@ -458,6 +459,104 @@ static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
 	mem->mm_node = NULL;
 }
 
+/**
+ * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
+ *
+ * @adev: amdgpu device pointer
+ * @mem: TTM memory object
+ * @dev: the other device
+ * @dir: dma direction
+ * @sgt: resulting sg table
+ *
+ * Allocate and fill a sg table from a VRAM allocation.
+ */
+int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
+			      struct ttm_mem_reg *mem,
+			      struct device *dev,
+			      enum dma_data_direction dir,
+			      struct sg_table **sgt)
+{
+	struct drm_mm_node *node;
+	struct scatterlist *sg;
+	int num_entries = 0;
+	unsigned int pages;
+	int i, r;
+
+	*sgt = kmalloc(sizeof(*sg), GFP_KERNEL);
+	if (!*sgt)
+		return -ENOMEM;
+
+	for (pages = mem->num_pages, node = mem->mm_node;
+	     pages; pages -= node->size, ++node)
+		++num_entries;
+
+	r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
+	if (r)
+		goto error_free;
+
+	for_each_sg((*sgt)->sgl, sg, num_entries, i)
+		sg->length = 0;
+
+	node = mem->mm_node;
+	for_each_sg((*sgt)->sgl, sg, num_entries, i) {
+		phys_addr_t phys = (node->start << PAGE_SHIFT) +
+			adev->gmc.aper_base;
+		size_t size = node->size << PAGE_SHIFT;
+		dma_addr_t addr;
+
+		++node;
+		addr = dma_map_resource(dev, phys, size, dir,
+					DMA_ATTR_SKIP_CPU_SYNC);
+		r = dma_mapping_error(dev, addr);
+		if (r)
+			goto error_unmap;
+
+		sg_set_page(sg, NULL, size, 0);
+		sg_dma_address(sg) = addr;
+		sg_dma_len(sg) = size;
+	}
+	return 0;
+
+error_unmap:
+	for_each_sg((*sgt)->sgl, sg, num_entries, i) {
+		if (!sg->length)
+			continue;
+
+		dma_unmap_resource(dev, sg->dma_address,
+				   sg->length, dir,
+				   DMA_ATTR_SKIP_CPU_SYNC);
+	}
+	sg_free_table(*sgt);
+
+error_free:
+	kfree(*sgt);
+	return r;
+}
+
+/**
+ * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
+ *
+ * @adev: amdgpu device pointer
+ * @sgt: sg table to free
+ *
+ * Free a previously allocate sg table.
+ */
+void amdgpu_vram_mgr_free_sgt(struct amdgpu_device *adev,
+			      struct device *dev,
+			      enum dma_data_direction dir,
+			      struct sg_table *sgt)
+{
+	struct scatterlist *sg;
+	int i;
+
+	for_each_sg(sgt->sgl, sg, sgt->nents, i)
+		dma_unmap_resource(dev, sg->dma_address,
+				   sg->length, dir,
+				   DMA_ATTR_SKIP_CPU_SYNC);
+	sg_free_table(sgt);
+	kfree(sgt);
+}
+
 /**
  * amdgpu_vram_mgr_usage - how many bytes are used in this domain
  *
-- 
2.17.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 5/6] drm/amdgpu: add support for exporting VRAM using DMA-buf v3

[Christian König]

We should be able to do this now after checking all the prerequisites.

v2: fix entrie count in the sgt
v3: manually construct the sg

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c  | 56 ++++++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h      | 12 ++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c | 99 ++++++++++++++++++++
 3 files changed, 153 insertions(+), 14 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index bbf67800c8a6..43d8ed7dbd00 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -276,14 +276,21 @@ static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach,
 	struct dma_buf *dma_buf = attach->dmabuf;
 	struct drm_gem_object *obj = dma_buf->priv;
 	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct sg_table *sgt;
 	long r;
 
 	if (!bo->pin_count) {
-		/* move buffer into GTT */
+		/* move buffer into GTT or VRAM */
 		struct ttm_operation_ctx ctx = { false, false };
+		unsigned domains = AMDGPU_GEM_DOMAIN_GTT;
 
-		amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+		if (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM &&
+		    attach->peer2peer) {
+			bo->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
+			domains |= AMDGPU_GEM_DOMAIN_VRAM;
+		}
+		amdgpu_bo_placement_from_domain(bo, domains);
 		r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 		if (r)
 			return ERR_PTR(r);
@@ -293,20 +300,34 @@ static struct sg_table *amdgpu_dma_buf_map(struct dma_buf_attachment *attach,
 		return ERR_PTR(-EBUSY);
 	}
 
-	sgt = drm_prime_pages_to_sg(bo->tbo.ttm->pages, bo->tbo.num_pages);
-	if (IS_ERR(sgt))
-		return sgt;
-
-	if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir,
-			      DMA_ATTR_SKIP_CPU_SYNC))
-		goto error_free;
+	switch (bo->tbo.mem.mem_type) {
+	case TTM_PL_TT:
+		sgt = drm_prime_pages_to_sg(bo->tbo.ttm->pages,
+					    bo->tbo.num_pages);
+		if (IS_ERR(sgt))
+			return sgt;
+
+		if (!dma_map_sg_attrs(attach->dev, sgt->sgl, sgt->nents, dir,
+				      DMA_ATTR_SKIP_CPU_SYNC))
+			goto error_free;
+		break;
+
+	case TTM_PL_VRAM:
+		r = amdgpu_vram_mgr_alloc_sgt(adev, &bo->tbo.mem, attach->dev,
+					      dir, &sgt);
+		if (r)
+			return ERR_PTR(r);
+		break;
+	default:
+		return ERR_PTR(-EINVAL);
+	}
 
 	return sgt;
 
 error_free:
 	sg_free_table(sgt);
 	kfree(sgt);
-	return ERR_PTR(-ENOMEM);
+	return ERR_PTR(-EBUSY);
 }
 
 /**
@@ -322,9 +343,18 @@ static void amdgpu_dma_buf_unmap(struct dma_buf_attachment *attach,
 				 struct sg_table *sgt,
 				 enum dma_data_direction dir)
 {
-	dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
-	sg_free_table(sgt);
-	kfree(sgt);
+	struct dma_buf *dma_buf = attach->dmabuf;
+	struct drm_gem_object *obj = dma_buf->priv;
+	struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
+	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
+
+	if (sgt->sgl->page_link) {
+		dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
+		sg_free_table(sgt);
+		kfree(sgt);
+	} else {
+		amdgpu_vram_mgr_free_sgt(adev, attach->dev, dir, sgt);
+	}
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index bd05bbb4878d..6b22dc41ef13 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -24,8 +24,9 @@
 #ifndef __AMDGPU_TTM_H__
 #define __AMDGPU_TTM_H__
 
-#include "amdgpu.h"
+#include <linux/dma-direction.h>
 #include <drm/gpu_scheduler.h>
+#include "amdgpu.h"
 
 #define AMDGPU_PL_GDS		(TTM_PL_PRIV + 0)
 #define AMDGPU_PL_GWS		(TTM_PL_PRIV + 1)
@@ -74,6 +75,15 @@ uint64_t amdgpu_gtt_mgr_usage(struct ttm_mem_type_manager *man);
 int amdgpu_gtt_mgr_recover(struct ttm_mem_type_manager *man);
 
 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo);
+int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
+			      struct ttm_mem_reg *mem,
+			      struct device *dev,
+			      enum dma_data_direction dir,
+			      struct sg_table **sgt);
+void amdgpu_vram_mgr_free_sgt(struct amdgpu_device *adev,
+			      struct device *dev,
+			      enum dma_data_direction dir,
+			      struct sg_table *sgt);
 uint64_t amdgpu_vram_mgr_usage(struct ttm_mem_type_manager *man);
 uint64_t amdgpu_vram_mgr_vis_usage(struct ttm_mem_type_manager *man);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 82a3299e53c0..128a667ed8fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -22,6 +22,7 @@
  * Authors: Christian König
  */
 
+#include <linux/dma-mapping.h>
 #include "amdgpu.h"
 #include "amdgpu_vm.h"
 #include "amdgpu_atomfirmware.h"
@@ -458,6 +459,104 @@ static void amdgpu_vram_mgr_del(struct ttm_mem_type_manager *man,
 	mem->mm_node = NULL;
 }
 
+/**
+ * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
+ *
+ * @adev: amdgpu device pointer
+ * @mem: TTM memory object
+ * @dev: the other device
+ * @dir: dma direction
+ * @sgt: resulting sg table
+ *
+ * Allocate and fill a sg table from a VRAM allocation.
+ */
+int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
+			      struct ttm_mem_reg *mem,
+			      struct device *dev,
+			      enum dma_data_direction dir,
+			      struct sg_table **sgt)
+{
+	struct drm_mm_node *node;
+	struct scatterlist *sg;
+	int num_entries = 0;
+	unsigned int pages;
+	int i, r;
+
+	*sgt = kmalloc(sizeof(*sg), GFP_KERNEL);
+	if (!*sgt)
+		return -ENOMEM;
+
+	for (pages = mem->num_pages, node = mem->mm_node;
+	     pages; pages -= node->size, ++node)
+		++num_entries;
+
+	r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
+	if (r)
+		goto error_free;
+
+	for_each_sg((*sgt)->sgl, sg, num_entries, i)
+		sg->length = 0;
+
+	node = mem->mm_node;
+	for_each_sg((*sgt)->sgl, sg, num_entries, i) {
+		phys_addr_t phys = (node->start << PAGE_SHIFT) +
+			adev->gmc.aper_base;
+		size_t size = node->size << PAGE_SHIFT;
+		dma_addr_t addr;
+
+		++node;
+		addr = dma_map_resource(dev, phys, size, dir,
+					DMA_ATTR_SKIP_CPU_SYNC);
+		r = dma_mapping_error(dev, addr);
+		if (r)
+			goto error_unmap;
+
+		sg_set_page(sg, NULL, size, 0);
+		sg_dma_address(sg) = addr;
+		sg_dma_len(sg) = size;
+	}
+	return 0;
+
+error_unmap:
+	for_each_sg((*sgt)->sgl, sg, num_entries, i) {
+		if (!sg->length)
+			continue;
+
+		dma_unmap_resource(dev, sg->dma_address,
+				   sg->length, dir,
+				   DMA_ATTR_SKIP_CPU_SYNC);
+	}
+	sg_free_table(*sgt);
+
+error_free:
+	kfree(*sgt);
+	return r;
+}
+
+/**
+ * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
+ *
+ * @adev: amdgpu device pointer
+ * @sgt: sg table to free
+ *
+ * Free a previously allocate sg table.
+ */
+void amdgpu_vram_mgr_free_sgt(struct amdgpu_device *adev,
+			      struct device *dev,
+			      enum dma_data_direction dir,
+			      struct sg_table *sgt)
+{
+	struct scatterlist *sg;
+	int i;
+
+	for_each_sg(sgt->sgl, sg, sgt->nents, i)
+		dma_unmap_resource(dev, sg->dma_address,
+				   sg->length, dir,
+				   DMA_ATTR_SKIP_CPU_SYNC);
+	sg_free_table(sgt);
+	kfree(sgt);
+}
+
 /**
  * amdgpu_vram_mgr_usage - how many bytes are used in this domain
  *
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 6/6] drm/amdgpu: improve amdgpu_gem_info debugfs file

[Christian König]

Note if a buffer was imported using peer2peer.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 4277125a79ee..e42608115c99 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -29,6 +29,7 @@
 #include <linux/module.h>
 #include <linux/pagemap.h>
 #include <linux/pci.h>
+#include <linux/dma-buf.h>
 
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_debugfs.h>
@@ -854,7 +855,8 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
 	attachment = READ_ONCE(bo->tbo.base.import_attach);
 
 	if (attachment)
-		seq_printf(m, " imported from %p", dma_buf);
+		seq_printf(m, " imported from %p%s", dma_buf,
+			   attachment->peer2peer ? " P2P" : "");
 	else if (dma_buf)
 		seq_printf(m, " exported as %p", dma_buf);
 
-- 
2.17.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

[PATCH 6/6] drm/amdgpu: improve amdgpu_gem_info debugfs file

[Christian König]

Note if a buffer was imported using peer2peer.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 4277125a79ee..e42608115c99 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -29,6 +29,7 @@
 #include <linux/module.h>
 #include <linux/pagemap.h>
 #include <linux/pci.h>
+#include <linux/dma-buf.h>
 
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_debugfs.h>
@@ -854,7 +855,8 @@ static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data)
 	attachment = READ_ONCE(bo->tbo.base.import_attach);
 
 	if (attachment)
-		seq_printf(m, " imported from %p", dma_buf);
+		seq_printf(m, " imported from %p%s", dma_buf,
+			   attachment->peer2peer ? " P2P" : "");
 	else if (dma_buf)
 		seq_printf(m, " exported as %p", dma_buf);
 
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 1/6] dma-buf: add peer2peer flag

[Daniel Vetter]

On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
> Add a peer2peer flag noting that the importer can deal with device
> resources which are not backed by pages.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

On the series:

Acked-by: Daniel Vetter <daniel.vetter@ffwll.ch>
> ---
>  drivers/dma-buf/dma-buf.c |  2 ++
>  include/linux/dma-buf.h   | 10 ++++++++++
>  2 files changed, 12 insertions(+)
> 
> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> index ccc9eda1bc28..570c923023e6 100644
> --- a/drivers/dma-buf/dma-buf.c
> +++ b/drivers/dma-buf/dma-buf.c
> @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
>  
>  	attach->dev = dev;
>  	attach->dmabuf = dmabuf;
> +	if (importer_ops)
> +		attach->peer2peer = importer_ops->allow_peer2peer;
>  	attach->importer_ops = importer_ops;
>  	attach->importer_priv = importer_priv;
>  
> diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
> index 1ade486fc2bb..82e0a4a64601 100644
> --- a/include/linux/dma-buf.h
> +++ b/include/linux/dma-buf.h
> @@ -334,6 +334,14 @@ struct dma_buf {
>   * Attachment operations implemented by the importer.
>   */
>  struct dma_buf_attach_ops {
> +	/**
> +	 * @allow_peer2peer:
> +	 *
> +	 * If this is set to true the importer must be able to handle peer
> +	 * resources without struct pages.
> +	 */
> +	bool allow_peer2peer;
> +
>  	/**
>  	 * @move_notify
>  	 *
> @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
>   * @node: list of dma_buf_attachment, protected by dma_resv lock of the dmabuf.
>   * @sgt: cached mapping.
>   * @dir: direction of cached mapping.
> + * @peer2peer: true if the importer can handle peer resources without pages.
>   * @priv: exporter specific attachment data.
>   * @importer_ops: importer operations for this attachment, if provided
>   * dma_buf_map/unmap_attachment() must be called with the dma_resv lock held.
> @@ -382,6 +391,7 @@ struct dma_buf_attachment {
>  	struct list_head node;
>  	struct sg_table *sgt;
>  	enum dma_data_direction dir;
> +	bool peer2peer;
>  	const struct dma_buf_attach_ops *importer_ops;
>  	void *importer_priv;
>  	void *priv;
> -- 
> 2.17.1
> 

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 1/6] dma-buf: add peer2peer flag

[Daniel Vetter]

On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
> Add a peer2peer flag noting that the importer can deal with device
> resources which are not backed by pages.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>

On the series:

Acked-by: Daniel Vetter <daniel.vetter@ffwll.ch>
> ---
>  drivers/dma-buf/dma-buf.c |  2 ++
>  include/linux/dma-buf.h   | 10 ++++++++++
>  2 files changed, 12 insertions(+)
> 
> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> index ccc9eda1bc28..570c923023e6 100644
> --- a/drivers/dma-buf/dma-buf.c
> +++ b/drivers/dma-buf/dma-buf.c
> @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
>  
>  	attach->dev = dev;
>  	attach->dmabuf = dmabuf;
> +	if (importer_ops)
> +		attach->peer2peer = importer_ops->allow_peer2peer;
>  	attach->importer_ops = importer_ops;
>  	attach->importer_priv = importer_priv;
>  
> diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
> index 1ade486fc2bb..82e0a4a64601 100644
> --- a/include/linux/dma-buf.h
> +++ b/include/linux/dma-buf.h
> @@ -334,6 +334,14 @@ struct dma_buf {
>   * Attachment operations implemented by the importer.
>   */
>  struct dma_buf_attach_ops {
> +	/**
> +	 * @allow_peer2peer:
> +	 *
> +	 * If this is set to true the importer must be able to handle peer
> +	 * resources without struct pages.
> +	 */
> +	bool allow_peer2peer;
> +
>  	/**
>  	 * @move_notify
>  	 *
> @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
>   * @node: list of dma_buf_attachment, protected by dma_resv lock of the dmabuf.
>   * @sgt: cached mapping.
>   * @dir: direction of cached mapping.
> + * @peer2peer: true if the importer can handle peer resources without pages.
>   * @priv: exporter specific attachment data.
>   * @importer_ops: importer operations for this attachment, if provided
>   * dma_buf_map/unmap_attachment() must be called with the dma_resv lock held.
> @@ -382,6 +391,7 @@ struct dma_buf_attachment {
>  	struct list_head node;
>  	struct sg_table *sgt;
>  	enum dma_data_direction dir;
> +	bool peer2peer;
>  	const struct dma_buf_attach_ops *importer_ops;
>  	void *importer_priv;
>  	void *priv;
> -- 
> 2.17.1
> 

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 1/6] dma-buf: add peer2peer flag

[Sumit Semwal]

[-- Attachment #1.1: Type: text/plain, Size: 2930 bytes --]

Hi Christian,

On Tue, 31 Mar 2020, 14:16 Daniel Vetter, <daniel@ffwll.ch> wrote:

> On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
> > Add a peer2peer flag noting that the importer can deal with device
> > resources which are not backed by pages.
> >
> > Signed-off-by: Christian König <christian.koenig@amd.com>
>
> On the series:
>
> Acked-by: Daniel Vetter <daniel.vetter@ffwll.ch>
>
Fwiw, for the series,
Acked-by: Sumit Semwal <sumit.semwal@linaro.org>

> ---
> >  drivers/dma-buf/dma-buf.c |  2 ++
> >  include/linux/dma-buf.h   | 10 ++++++++++
> >  2 files changed, 12 insertions(+)
> >
> > diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> > index ccc9eda1bc28..570c923023e6 100644
> > --- a/drivers/dma-buf/dma-buf.c
> > +++ b/drivers/dma-buf/dma-buf.c
> > @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf,
> struct device *dev,
> >
> >       attach->dev = dev;
> >       attach->dmabuf = dmabuf;
> > +     if (importer_ops)
> > +             attach->peer2peer = importer_ops->allow_peer2peer;
> >       attach->importer_ops = importer_ops;
> >       attach->importer_priv = importer_priv;
> >
> > diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
> > index 1ade486fc2bb..82e0a4a64601 100644
> > --- a/include/linux/dma-buf.h
> > +++ b/include/linux/dma-buf.h
> > @@ -334,6 +334,14 @@ struct dma_buf {
> >   * Attachment operations implemented by the importer.
> >   */
> >  struct dma_buf_attach_ops {
> > +     /**
> > +      * @allow_peer2peer:
> > +      *
> > +      * If this is set to true the importer must be able to handle peer
> > +      * resources without struct pages.
> > +      */
> > +     bool allow_peer2peer;
> > +
> >       /**
> >        * @move_notify
> >        *
> > @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
> >   * @node: list of dma_buf_attachment, protected by dma_resv lock of the
> dmabuf.
> >   * @sgt: cached mapping.
> >   * @dir: direction of cached mapping.
> > + * @peer2peer: true if the importer can handle peer resources without
> pages.
> >   * @priv: exporter specific attachment data.
> >   * @importer_ops: importer operations for this attachment, if provided
> >   * dma_buf_map/unmap_attachment() must be called with the dma_resv lock
> held.
> > @@ -382,6 +391,7 @@ struct dma_buf_attachment {
> >       struct list_head node;
> >       struct sg_table *sgt;
> >       enum dma_data_direction dir;
> > +     bool peer2peer;
> >       const struct dma_buf_attach_ops *importer_ops;
> >       void *importer_priv;
> >       void *priv;
> > --
> > 2.17.1
> >
>
> --
> Daniel Vetter
> Software Engineer, Intel Corporation
> http://blog.ffwll.ch
> _______________________________________________
> dri-devel mailing list
> dri-devel@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>

[-- Attachment #1.2: Type: text/html, Size: 4382 bytes --]

[-- Attachment #2: Type: text/plain, Size: 160 bytes --]

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 1/6] dma-buf: add peer2peer flag

[Sumit Semwal]

[-- Attachment #1.1: Type: text/plain, Size: 2930 bytes --]

Hi Christian,

On Tue, 31 Mar 2020, 14:16 Daniel Vetter, <daniel@ffwll.ch> wrote:

> On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
> > Add a peer2peer flag noting that the importer can deal with device
> > resources which are not backed by pages.
> >
> > Signed-off-by: Christian König <christian.koenig@amd.com>
>
> On the series:
>
> Acked-by: Daniel Vetter <daniel.vetter@ffwll.ch>
>
Fwiw, for the series,
Acked-by: Sumit Semwal <sumit.semwal@linaro.org>

> ---
> >  drivers/dma-buf/dma-buf.c |  2 ++
> >  include/linux/dma-buf.h   | 10 ++++++++++
> >  2 files changed, 12 insertions(+)
> >
> > diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> > index ccc9eda1bc28..570c923023e6 100644
> > --- a/drivers/dma-buf/dma-buf.c
> > +++ b/drivers/dma-buf/dma-buf.c
> > @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf,
> struct device *dev,
> >
> >       attach->dev = dev;
> >       attach->dmabuf = dmabuf;
> > +     if (importer_ops)
> > +             attach->peer2peer = importer_ops->allow_peer2peer;
> >       attach->importer_ops = importer_ops;
> >       attach->importer_priv = importer_priv;
> >
> > diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
> > index 1ade486fc2bb..82e0a4a64601 100644
> > --- a/include/linux/dma-buf.h
> > +++ b/include/linux/dma-buf.h
> > @@ -334,6 +334,14 @@ struct dma_buf {
> >   * Attachment operations implemented by the importer.
> >   */
> >  struct dma_buf_attach_ops {
> > +     /**
> > +      * @allow_peer2peer:
> > +      *
> > +      * If this is set to true the importer must be able to handle peer
> > +      * resources without struct pages.
> > +      */
> > +     bool allow_peer2peer;
> > +
> >       /**
> >        * @move_notify
> >        *
> > @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
> >   * @node: list of dma_buf_attachment, protected by dma_resv lock of the
> dmabuf.
> >   * @sgt: cached mapping.
> >   * @dir: direction of cached mapping.
> > + * @peer2peer: true if the importer can handle peer resources without
> pages.
> >   * @priv: exporter specific attachment data.
> >   * @importer_ops: importer operations for this attachment, if provided
> >   * dma_buf_map/unmap_attachment() must be called with the dma_resv lock
> held.
> > @@ -382,6 +391,7 @@ struct dma_buf_attachment {
> >       struct list_head node;
> >       struct sg_table *sgt;
> >       enum dma_data_direction dir;
> > +     bool peer2peer;
> >       const struct dma_buf_attach_ops *importer_ops;
> >       void *importer_priv;
> >       void *priv;
> > --
> > 2.17.1
> >
>
> --
> Daniel Vetter
> Software Engineer, Intel Corporation
> http://blog.ffwll.ch
> _______________________________________________
> dri-devel mailing list
> dri-devel@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/dri-devel
>

[-- Attachment #1.2: Type: text/html, Size: 4382 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 1/6] dma-buf: add peer2peer flag

[Daniel Vetter]

On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
> Add a peer2peer flag noting that the importer can deal with device
> resources which are not backed by pages.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/dma-buf/dma-buf.c |  2 ++
>  include/linux/dma-buf.h   | 10 ++++++++++
>  2 files changed, 12 insertions(+)
> 
> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> index ccc9eda1bc28..570c923023e6 100644
> --- a/drivers/dma-buf/dma-buf.c
> +++ b/drivers/dma-buf/dma-buf.c
> @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
>  
>  	attach->dev = dev;
>  	attach->dmabuf = dmabuf;
> +	if (importer_ops)
> +		attach->peer2peer = importer_ops->allow_peer2peer;

So an idea that crossed my mind to validate this, since we need quite some
bad amounts of bad luck if someone accidentally introduces and access to
struct_page in sg lists in some slowpath.

On map_sg, if ->peer2peer is set, we could mangle the struct_page
pointers, e.g. swap high bits for low bits (so that NULL stays NULL). On
unmap_sg we obviously need to undo that, in case the exporter needs those
pointers for its own book-keeping for some reason. I was also pondering
just setting them all to NULL, but that might break some exporters. With
the pointer mangling trick (especially if we flip high for low bits on 64
where this should result in invalid addresses in almost all cases) we
should be able to catch buggy p2p importers quite quickly.

Thoughts? Maybe add as a follow-up patch for testing?
-Daniel
>  	attach->importer_ops = importer_ops;
>  	attach->importer_priv = importer_priv;
>  
> diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
> index 1ade486fc2bb..82e0a4a64601 100644
> --- a/include/linux/dma-buf.h
> +++ b/include/linux/dma-buf.h
> @@ -334,6 +334,14 @@ struct dma_buf {
>   * Attachment operations implemented by the importer.
>   */
>  struct dma_buf_attach_ops {
> +	/**
> +	 * @allow_peer2peer:
> +	 *
> +	 * If this is set to true the importer must be able to handle peer
> +	 * resources without struct pages.
> +	 */
> +	bool allow_peer2peer;
> +
>  	/**
>  	 * @move_notify
>  	 *
> @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
>   * @node: list of dma_buf_attachment, protected by dma_resv lock of the dmabuf.
>   * @sgt: cached mapping.
>   * @dir: direction of cached mapping.
> + * @peer2peer: true if the importer can handle peer resources without pages.
>   * @priv: exporter specific attachment data.
>   * @importer_ops: importer operations for this attachment, if provided
>   * dma_buf_map/unmap_attachment() must be called with the dma_resv lock held.
> @@ -382,6 +391,7 @@ struct dma_buf_attachment {
>  	struct list_head node;
>  	struct sg_table *sgt;
>  	enum dma_data_direction dir;
> +	bool peer2peer;
>  	const struct dma_buf_attach_ops *importer_ops;
>  	void *importer_priv;
>  	void *priv;
> -- 
> 2.17.1
> 

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 1/6] dma-buf: add peer2peer flag

[Daniel Vetter]

On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
> Add a peer2peer flag noting that the importer can deal with device
> resources which are not backed by pages.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
> ---
>  drivers/dma-buf/dma-buf.c |  2 ++
>  include/linux/dma-buf.h   | 10 ++++++++++
>  2 files changed, 12 insertions(+)
> 
> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> index ccc9eda1bc28..570c923023e6 100644
> --- a/drivers/dma-buf/dma-buf.c
> +++ b/drivers/dma-buf/dma-buf.c
> @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf, struct device *dev,
>  
>  	attach->dev = dev;
>  	attach->dmabuf = dmabuf;
> +	if (importer_ops)
> +		attach->peer2peer = importer_ops->allow_peer2peer;

So an idea that crossed my mind to validate this, since we need quite some
bad amounts of bad luck if someone accidentally introduces and access to
struct_page in sg lists in some slowpath.

On map_sg, if ->peer2peer is set, we could mangle the struct_page
pointers, e.g. swap high bits for low bits (so that NULL stays NULL). On
unmap_sg we obviously need to undo that, in case the exporter needs those
pointers for its own book-keeping for some reason. I was also pondering
just setting them all to NULL, but that might break some exporters. With
the pointer mangling trick (especially if we flip high for low bits on 64
where this should result in invalid addresses in almost all cases) we
should be able to catch buggy p2p importers quite quickly.

Thoughts? Maybe add as a follow-up patch for testing?
-Daniel
>  	attach->importer_ops = importer_ops;
>  	attach->importer_priv = importer_priv;
>  
> diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
> index 1ade486fc2bb..82e0a4a64601 100644
> --- a/include/linux/dma-buf.h
> +++ b/include/linux/dma-buf.h
> @@ -334,6 +334,14 @@ struct dma_buf {
>   * Attachment operations implemented by the importer.
>   */
>  struct dma_buf_attach_ops {
> +	/**
> +	 * @allow_peer2peer:
> +	 *
> +	 * If this is set to true the importer must be able to handle peer
> +	 * resources without struct pages.
> +	 */
> +	bool allow_peer2peer;
> +
>  	/**
>  	 * @move_notify
>  	 *
> @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
>   * @node: list of dma_buf_attachment, protected by dma_resv lock of the dmabuf.
>   * @sgt: cached mapping.
>   * @dir: direction of cached mapping.
> + * @peer2peer: true if the importer can handle peer resources without pages.
>   * @priv: exporter specific attachment data.
>   * @importer_ops: importer operations for this attachment, if provided
>   * dma_buf_map/unmap_attachment() must be called with the dma_resv lock held.
> @@ -382,6 +391,7 @@ struct dma_buf_attachment {
>  	struct list_head node;
>  	struct sg_table *sgt;
>  	enum dma_data_direction dir;
> +	bool peer2peer;
>  	const struct dma_buf_attach_ops *importer_ops;
>  	void *importer_priv;
>  	void *priv;
> -- 
> 2.17.1
> 

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [PATCH 1/6] dma-buf: add peer2peer flag

[Ruhl, Michael J]

>-----Original Message-----
>From: dri-devel <dri-devel-bounces@lists.freedesktop.org> On Behalf Of
>Daniel Vetter
>Sent: Wednesday, April 1, 2020 7:35 AM
>To: Christian König <ckoenig.leichtzumerken@gmail.com>
>Cc: amd-gfx@lists.freedesktop.org; dri-devel@lists.freedesktop.org
>Subject: Re: [PATCH 1/6] dma-buf: add peer2peer flag
>
>On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
>> Add a peer2peer flag noting that the importer can deal with device
>> resources which are not backed by pages.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>  drivers/dma-buf/dma-buf.c |  2 ++
>>  include/linux/dma-buf.h   | 10 ++++++++++
>>  2 files changed, 12 insertions(+)
>>
>> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
>> index ccc9eda1bc28..570c923023e6 100644
>> --- a/drivers/dma-buf/dma-buf.c
>> +++ b/drivers/dma-buf/dma-buf.c
>> @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf,
>struct device *dev,
>>
>>  	attach->dev = dev;
>>  	attach->dmabuf = dmabuf;
>> +	if (importer_ops)
>> +		attach->peer2peer = importer_ops->allow_peer2peer;
>
>So an idea that crossed my mind to validate this, since we need quite some
>bad amounts of bad luck if someone accidentally introduces and access to
>struct_page in sg lists in some slowpath.
>
>On map_sg, if ->peer2peer is set, we could mangle the struct_page
>pointers, e.g. swap high bits for low bits (so that NULL stays NULL). On
>unmap_sg we obviously need to undo that, in case the exporter needs those
>pointers for its own book-keeping for some reason. I was also pondering
>just setting them all to NULL, but that might break some exporters. With
>the pointer mangling trick (especially if we flip high for low bits on 64
>where this should result in invalid addresses in almost all cases) we
>should be able to catch buggy p2p importers quite quickly.

The scatter list usage of the struct page pointer has other information in the
lower bits for keeping track of linking and other stuff.  Swizzling the page
pointers will probably make the scatter list unusable.

Mike

>Thoughts? Maybe add as a follow-up patch for testing?
>-Daniel
>>  	attach->importer_ops = importer_ops;
>>  	attach->importer_priv = importer_priv;
>>
>> diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
>> index 1ade486fc2bb..82e0a4a64601 100644
>> --- a/include/linux/dma-buf.h
>> +++ b/include/linux/dma-buf.h
>> @@ -334,6 +334,14 @@ struct dma_buf {
>>   * Attachment operations implemented by the importer.
>>   */
>>  struct dma_buf_attach_ops {
>> +	/**
>> +	 * @allow_peer2peer:
>> +	 *
>> +	 * If this is set to true the importer must be able to handle peer
>> +	 * resources without struct pages.
>> +	 */
>> +	bool allow_peer2peer;
>> +
>>  	/**
>>  	 * @move_notify
>>  	 *
>> @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
>>   * @node: list of dma_buf_attachment, protected by dma_resv lock of the
>dmabuf.
>>   * @sgt: cached mapping.
>>   * @dir: direction of cached mapping.
>> + * @peer2peer: true if the importer can handle peer resources without
>pages.
>>   * @priv: exporter specific attachment data.
>>   * @importer_ops: importer operations for this attachment, if provided
>>   * dma_buf_map/unmap_attachment() must be called with the dma_resv
>lock held.
>> @@ -382,6 +391,7 @@ struct dma_buf_attachment {
>>  	struct list_head node;
>>  	struct sg_table *sgt;
>>  	enum dma_data_direction dir;
>> +	bool peer2peer;
>>  	const struct dma_buf_attach_ops *importer_ops;
>>  	void *importer_priv;
>>  	void *priv;
>> --
>> 2.17.1
>>
>
>--
>Daniel Vetter
>Software Engineer, Intel Corporation
>http://blog.ffwll.ch
>_______________________________________________
>dri-devel mailing list
>dri-devel@lists.freedesktop.org
>https://lists.freedesktop.org/mailman/listinfo/dri-devel
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

RE: [PATCH 1/6] dma-buf: add peer2peer flag

[Ruhl, Michael J]

>-----Original Message-----
>From: dri-devel <dri-devel-bounces@lists.freedesktop.org> On Behalf Of
>Daniel Vetter
>Sent: Wednesday, April 1, 2020 7:35 AM
>To: Christian König <ckoenig.leichtzumerken@gmail.com>
>Cc: amd-gfx@lists.freedesktop.org; dri-devel@lists.freedesktop.org
>Subject: Re: [PATCH 1/6] dma-buf: add peer2peer flag
>
>On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
>> Add a peer2peer flag noting that the importer can deal with device
>> resources which are not backed by pages.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>> ---
>>  drivers/dma-buf/dma-buf.c |  2 ++
>>  include/linux/dma-buf.h   | 10 ++++++++++
>>  2 files changed, 12 insertions(+)
>>
>> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
>> index ccc9eda1bc28..570c923023e6 100644
>> --- a/drivers/dma-buf/dma-buf.c
>> +++ b/drivers/dma-buf/dma-buf.c
>> @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf,
>struct device *dev,
>>
>>  	attach->dev = dev;
>>  	attach->dmabuf = dmabuf;
>> +	if (importer_ops)
>> +		attach->peer2peer = importer_ops->allow_peer2peer;
>
>So an idea that crossed my mind to validate this, since we need quite some
>bad amounts of bad luck if someone accidentally introduces and access to
>struct_page in sg lists in some slowpath.
>
>On map_sg, if ->peer2peer is set, we could mangle the struct_page
>pointers, e.g. swap high bits for low bits (so that NULL stays NULL). On
>unmap_sg we obviously need to undo that, in case the exporter needs those
>pointers for its own book-keeping for some reason. I was also pondering
>just setting them all to NULL, but that might break some exporters. With
>the pointer mangling trick (especially if we flip high for low bits on 64
>where this should result in invalid addresses in almost all cases) we
>should be able to catch buggy p2p importers quite quickly.

The scatter list usage of the struct page pointer has other information in the
lower bits for keeping track of linking and other stuff.  Swizzling the page
pointers will probably make the scatter list unusable.

Mike

>Thoughts? Maybe add as a follow-up patch for testing?
>-Daniel
>>  	attach->importer_ops = importer_ops;
>>  	attach->importer_priv = importer_priv;
>>
>> diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
>> index 1ade486fc2bb..82e0a4a64601 100644
>> --- a/include/linux/dma-buf.h
>> +++ b/include/linux/dma-buf.h
>> @@ -334,6 +334,14 @@ struct dma_buf {
>>   * Attachment operations implemented by the importer.
>>   */
>>  struct dma_buf_attach_ops {
>> +	/**
>> +	 * @allow_peer2peer:
>> +	 *
>> +	 * If this is set to true the importer must be able to handle peer
>> +	 * resources without struct pages.
>> +	 */
>> +	bool allow_peer2peer;
>> +
>>  	/**
>>  	 * @move_notify
>>  	 *
>> @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
>>   * @node: list of dma_buf_attachment, protected by dma_resv lock of the
>dmabuf.
>>   * @sgt: cached mapping.
>>   * @dir: direction of cached mapping.
>> + * @peer2peer: true if the importer can handle peer resources without
>pages.
>>   * @priv: exporter specific attachment data.
>>   * @importer_ops: importer operations for this attachment, if provided
>>   * dma_buf_map/unmap_attachment() must be called with the dma_resv
>lock held.
>> @@ -382,6 +391,7 @@ struct dma_buf_attachment {
>>  	struct list_head node;
>>  	struct sg_table *sgt;
>>  	enum dma_data_direction dir;
>> +	bool peer2peer;
>>  	const struct dma_buf_attach_ops *importer_ops;
>>  	void *importer_priv;
>>  	void *priv;
>> --
>> 2.17.1
>>
>
>--
>Daniel Vetter
>Software Engineer, Intel Corporation
>http://blog.ffwll.ch
>_______________________________________________
>dri-devel mailing list
>dri-devel@lists.freedesktop.org
>https://lists.freedesktop.org/mailman/listinfo/dri-devel
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 1/6] dma-buf: add peer2peer flag

[Daniel Vetter]

On Wed, Apr 01, 2020 at 04:04:14PM +0000, Ruhl, Michael J wrote:
> >-----Original Message-----
> >From: dri-devel <dri-devel-bounces@lists.freedesktop.org> On Behalf Of
> >Daniel Vetter
> >Sent: Wednesday, April 1, 2020 7:35 AM
> >To: Christian König <ckoenig.leichtzumerken@gmail.com>
> >Cc: amd-gfx@lists.freedesktop.org; dri-devel@lists.freedesktop.org
> >Subject: Re: [PATCH 1/6] dma-buf: add peer2peer flag
> >
> >On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
> >> Add a peer2peer flag noting that the importer can deal with device
> >> resources which are not backed by pages.
> >>
> >> Signed-off-by: Christian König <christian.koenig@amd.com>
> >> ---
> >>  drivers/dma-buf/dma-buf.c |  2 ++
> >>  include/linux/dma-buf.h   | 10 ++++++++++
> >>  2 files changed, 12 insertions(+)
> >>
> >> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> >> index ccc9eda1bc28..570c923023e6 100644
> >> --- a/drivers/dma-buf/dma-buf.c
> >> +++ b/drivers/dma-buf/dma-buf.c
> >> @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf,
> >struct device *dev,
> >>
> >>  	attach->dev = dev;
> >>  	attach->dmabuf = dmabuf;
> >> +	if (importer_ops)
> >> +		attach->peer2peer = importer_ops->allow_peer2peer;
> >
> >So an idea that crossed my mind to validate this, since we need quite some
> >bad amounts of bad luck if someone accidentally introduces and access to
> >struct_page in sg lists in some slowpath.
> >
> >On map_sg, if ->peer2peer is set, we could mangle the struct_page
> >pointers, e.g. swap high bits for low bits (so that NULL stays NULL). On
> >unmap_sg we obviously need to undo that, in case the exporter needs those
> >pointers for its own book-keeping for some reason. I was also pondering
> >just setting them all to NULL, but that might break some exporters. With
> >the pointer mangling trick (especially if we flip high for low bits on 64
> >where this should result in invalid addresses in almost all cases) we
> >should be able to catch buggy p2p importers quite quickly.
> 
> The scatter list usage of the struct page pointer has other information in the
> lower bits for keeping track of linking and other stuff.  Swizzling the page
> pointers will probably make the scatter list unusable.

We'd need to swizzle only the pointers that are actual struct page
pointers. Plus keep the low bits as-is, and maybe only flip the top-most
60 bits or so. Doesn't break the idea fundamentally I think.
-Daniel

> 
> Mike
> 
> >Thoughts? Maybe add as a follow-up patch for testing?
> >-Daniel
> >>  	attach->importer_ops = importer_ops;
> >>  	attach->importer_priv = importer_priv;
> >>
> >> diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
> >> index 1ade486fc2bb..82e0a4a64601 100644
> >> --- a/include/linux/dma-buf.h
> >> +++ b/include/linux/dma-buf.h
> >> @@ -334,6 +334,14 @@ struct dma_buf {
> >>   * Attachment operations implemented by the importer.
> >>   */
> >>  struct dma_buf_attach_ops {
> >> +	/**
> >> +	 * @allow_peer2peer:
> >> +	 *
> >> +	 * If this is set to true the importer must be able to handle peer
> >> +	 * resources without struct pages.
> >> +	 */
> >> +	bool allow_peer2peer;
> >> +
> >>  	/**
> >>  	 * @move_notify
> >>  	 *
> >> @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
> >>   * @node: list of dma_buf_attachment, protected by dma_resv lock of the
> >dmabuf.
> >>   * @sgt: cached mapping.
> >>   * @dir: direction of cached mapping.
> >> + * @peer2peer: true if the importer can handle peer resources without
> >pages.
> >>   * @priv: exporter specific attachment data.
> >>   * @importer_ops: importer operations for this attachment, if provided
> >>   * dma_buf_map/unmap_attachment() must be called with the dma_resv
> >lock held.
> >> @@ -382,6 +391,7 @@ struct dma_buf_attachment {
> >>  	struct list_head node;
> >>  	struct sg_table *sgt;
> >>  	enum dma_data_direction dir;
> >> +	bool peer2peer;
> >>  	const struct dma_buf_attach_ops *importer_ops;
> >>  	void *importer_priv;
> >>  	void *priv;
> >> --
> >> 2.17.1
> >>
> >
> >--
> >Daniel Vetter
> >Software Engineer, Intel Corporation
> >http://blog.ffwll.ch
> >_______________________________________________
> >dri-devel mailing list
> >dri-devel@lists.freedesktop.org
> >https://lists.freedesktop.org/mailman/listinfo/dri-devel

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 1/6] dma-buf: add peer2peer flag

[Daniel Vetter]

On Wed, Apr 01, 2020 at 04:04:14PM +0000, Ruhl, Michael J wrote:
> >-----Original Message-----
> >From: dri-devel <dri-devel-bounces@lists.freedesktop.org> On Behalf Of
> >Daniel Vetter
> >Sent: Wednesday, April 1, 2020 7:35 AM
> >To: Christian König <ckoenig.leichtzumerken@gmail.com>
> >Cc: amd-gfx@lists.freedesktop.org; dri-devel@lists.freedesktop.org
> >Subject: Re: [PATCH 1/6] dma-buf: add peer2peer flag
> >
> >On Mon, Mar 30, 2020 at 03:55:31PM +0200, Christian König wrote:
> >> Add a peer2peer flag noting that the importer can deal with device
> >> resources which are not backed by pages.
> >>
> >> Signed-off-by: Christian König <christian.koenig@amd.com>
> >> ---
> >>  drivers/dma-buf/dma-buf.c |  2 ++
> >>  include/linux/dma-buf.h   | 10 ++++++++++
> >>  2 files changed, 12 insertions(+)
> >>
> >> diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
> >> index ccc9eda1bc28..570c923023e6 100644
> >> --- a/drivers/dma-buf/dma-buf.c
> >> +++ b/drivers/dma-buf/dma-buf.c
> >> @@ -690,6 +690,8 @@ dma_buf_dynamic_attach(struct dma_buf *dmabuf,
> >struct device *dev,
> >>
> >>  	attach->dev = dev;
> >>  	attach->dmabuf = dmabuf;
> >> +	if (importer_ops)
> >> +		attach->peer2peer = importer_ops->allow_peer2peer;
> >
> >So an idea that crossed my mind to validate this, since we need quite some
> >bad amounts of bad luck if someone accidentally introduces and access to
> >struct_page in sg lists in some slowpath.
> >
> >On map_sg, if ->peer2peer is set, we could mangle the struct_page
> >pointers, e.g. swap high bits for low bits (so that NULL stays NULL). On
> >unmap_sg we obviously need to undo that, in case the exporter needs those
> >pointers for its own book-keeping for some reason. I was also pondering
> >just setting them all to NULL, but that might break some exporters. With
> >the pointer mangling trick (especially if we flip high for low bits on 64
> >where this should result in invalid addresses in almost all cases) we
> >should be able to catch buggy p2p importers quite quickly.
> 
> The scatter list usage of the struct page pointer has other information in the
> lower bits for keeping track of linking and other stuff.  Swizzling the page
> pointers will probably make the scatter list unusable.

We'd need to swizzle only the pointers that are actual struct page
pointers. Plus keep the low bits as-is, and maybe only flip the top-most
60 bits or so. Doesn't break the idea fundamentally I think.
-Daniel

> 
> Mike
> 
> >Thoughts? Maybe add as a follow-up patch for testing?
> >-Daniel
> >>  	attach->importer_ops = importer_ops;
> >>  	attach->importer_priv = importer_priv;
> >>
> >> diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h
> >> index 1ade486fc2bb..82e0a4a64601 100644
> >> --- a/include/linux/dma-buf.h
> >> +++ b/include/linux/dma-buf.h
> >> @@ -334,6 +334,14 @@ struct dma_buf {
> >>   * Attachment operations implemented by the importer.
> >>   */
> >>  struct dma_buf_attach_ops {
> >> +	/**
> >> +	 * @allow_peer2peer:
> >> +	 *
> >> +	 * If this is set to true the importer must be able to handle peer
> >> +	 * resources without struct pages.
> >> +	 */
> >> +	bool allow_peer2peer;
> >> +
> >>  	/**
> >>  	 * @move_notify
> >>  	 *
> >> @@ -362,6 +370,7 @@ struct dma_buf_attach_ops {
> >>   * @node: list of dma_buf_attachment, protected by dma_resv lock of the
> >dmabuf.
> >>   * @sgt: cached mapping.
> >>   * @dir: direction of cached mapping.
> >> + * @peer2peer: true if the importer can handle peer resources without
> >pages.
> >>   * @priv: exporter specific attachment data.
> >>   * @importer_ops: importer operations for this attachment, if provided
> >>   * dma_buf_map/unmap_attachment() must be called with the dma_resv
> >lock held.
> >> @@ -382,6 +391,7 @@ struct dma_buf_attachment {
> >>  	struct list_head node;
> >>  	struct sg_table *sgt;
> >>  	enum dma_data_direction dir;
> >> +	bool peer2peer;
> >>  	const struct dma_buf_attach_ops *importer_ops;
> >>  	void *importer_priv;
> >>  	void *priv;
> >> --
> >> 2.17.1
> >>
> >
> >--
> >Daniel Vetter
> >Software Engineer, Intel Corporation
> >http://blog.ffwll.ch
> >_______________________________________________
> >dri-devel mailing list
> >dri-devel@lists.freedesktop.org
> >https://lists.freedesktop.org/mailman/listinfo/dri-devel

-- 
Daniel Vetter
Software Engineer, Intel Corporation
http://blog.ffwll.ch
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

P2P for DMA-buf

[Christian König]

This is the third and final part of my series to start supporting P2P with DMA-buf.

The implementation is straight forward, apart from a helper to aid constructing scatterlists without having struct pages we only add a new flag indicating that an DMA-buf importer can handle peer2peer.

The exporter can then check if P2P is general possible using the pci_p2pdma_distance_many() function and if necessary can also clear the flag.

The rest is an example how to implementing the necessary functionality into the amdgpu driver to setup scatterlists pointing to device memory.

Please review and comment,
Christian.

[PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Christian König]

Check if we can do peer2peer on the PCIe bus.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index aef12ee2f1e3..bbf67800c8a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -38,6 +38,7 @@
 #include <drm/amdgpu_drm.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-fence-array.h>
+#include <linux/pci-p2pdma.h>
 
 /**
  * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
@@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	int r;
 
+	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
+		attach->peer2peer = false;
+
 	if (attach->dev->driver == adev->dev->driver)
 		return 0;
 
-- 
2.17.1

[PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Christian König]

Check if we can do peer2peer on the PCIe bus.

Signed-off-by: Christian König <christian.koenig@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index aef12ee2f1e3..bbf67800c8a6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -38,6 +38,7 @@
 #include <drm/amdgpu_drm.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-fence-array.h>
+#include <linux/pci-p2pdma.h>
 
 /**
  * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
@@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	int r;
 
+	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
+		attach->peer2peer = false;
+
 	if (attach->dev->driver == adev->dev->driver)
 		return 0;
 
-- 
2.17.1

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Jason Gunthorpe]

On Wed, Mar 11, 2020 at 02:51:56PM +0100, Christian König wrote:
> Check if we can do peer2peer on the PCIe bus.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
>  drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
> index aef12ee2f1e3..bbf67800c8a6 100644
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
> @@ -38,6 +38,7 @@
>  #include <drm/amdgpu_drm.h>
>  #include <linux/dma-buf.h>
>  #include <linux/dma-fence-array.h>
> +#include <linux/pci-p2pdma.h>
>  
>  /**
>   * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
> @@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
>  	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
>  	int r;
>  
> +	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
> +		attach->peer2peer = false;
> +

Are there other related patches than this series?

p2p dma mapping needs to be done in common code, in p2pdma.c - ie this
open coding is missing the bus_offset stuff, at least. 

I really do not want to see drivers open code this stuff.

We already have a p2pdma API for handling the struct page case, so I
suggest adding some new p2pdma API to handle this for non-struct page
cases.

ie some thing like:

int 'p2pdma map bar'(
   struct pci_device *source,
   unsigned int source_bar_number, 
   struct pci_device *dest, 
   physaddr&len *array_of_offsets & length pairs into source bar,
   struct scatterlist *output_sgl)

Jason

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Jason Gunthorpe]

On Wed, Mar 11, 2020 at 02:51:56PM +0100, Christian König wrote:
> Check if we can do peer2peer on the PCIe bus.
> 
> Signed-off-by: Christian König <christian.koenig@amd.com>
>  drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
>  1 file changed, 4 insertions(+)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
> index aef12ee2f1e3..bbf67800c8a6 100644
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
> @@ -38,6 +38,7 @@
>  #include <drm/amdgpu_drm.h>
>  #include <linux/dma-buf.h>
>  #include <linux/dma-fence-array.h>
> +#include <linux/pci-p2pdma.h>
>  
>  /**
>   * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
> @@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
>  	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
>  	int r;
>  
> +	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
> +		attach->peer2peer = false;
> +

Are there other related patches than this series?

p2p dma mapping needs to be done in common code, in p2pdma.c - ie this
open coding is missing the bus_offset stuff, at least. 

I really do not want to see drivers open code this stuff.

We already have a p2pdma API for handling the struct page case, so I
suggest adding some new p2pdma API to handle this for non-struct page
cases.

ie some thing like:

int 'p2pdma map bar'(
   struct pci_device *source,
   unsigned int source_bar_number, 
   struct pci_device *dest, 
   physaddr&len *array_of_offsets & length pairs into source bar,
   struct scatterlist *output_sgl)

Jason
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Christian König]

Am 11.03.20 um 15:04 schrieb Jason Gunthorpe:
> On Wed, Mar 11, 2020 at 02:51:56PM +0100, Christian König wrote:
>> Check if we can do peer2peer on the PCIe bus.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
>>   1 file changed, 4 insertions(+)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
>> index aef12ee2f1e3..bbf67800c8a6 100644
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
>> @@ -38,6 +38,7 @@
>>   #include <drm/amdgpu_drm.h>
>>   #include <linux/dma-buf.h>
>>   #include <linux/dma-fence-array.h>
>> +#include <linux/pci-p2pdma.h>
>>   
>>   /**
>>    * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
>> @@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
>>   	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
>>   	int r;
>>   
>> +	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
>> +		attach->peer2peer = false;
>> +
> Are there other related patches than this series?
>
> p2p dma mapping needs to be done in common code, in p2pdma.c - ie this
> open coding is missing the bus_offset stuff, at least.

Yeah, I'm aware of this. But I couldn't find a better way for now.

> I really do not want to see drivers open code this stuff.
>
> We already have a p2pdma API for handling the struct page case, so I
> suggest adding some new p2pdma API to handle this for non-struct page
> cases.
>
> ie some thing like:
>
> int 'p2pdma map bar'(
>     struct pci_device *source,
>     unsigned int source_bar_number,
>     struct pci_device *dest,
>     physaddr&len *array_of_offsets & length pairs into source bar,
>     struct scatterlist *output_sgl)

Well that's exactly what I have to avoid since I don't have the array of 
offsets around and want to avoid constructing it.

Similar problem for dma_map_resource(). My example does this on demand, 
but essentially we also have use cases where this is done only once.

Ideally we would have some function to create an sgl based on some 
arbitrary collection of offsets and length inside a BAR.

Regards,
Christian.

>
> Jason

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Christian König]

Am 11.03.20 um 15:04 schrieb Jason Gunthorpe:
> On Wed, Mar 11, 2020 at 02:51:56PM +0100, Christian König wrote:
>> Check if we can do peer2peer on the PCIe bus.
>>
>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
>>   1 file changed, 4 insertions(+)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
>> index aef12ee2f1e3..bbf67800c8a6 100644
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
>> @@ -38,6 +38,7 @@
>>   #include <drm/amdgpu_drm.h>
>>   #include <linux/dma-buf.h>
>>   #include <linux/dma-fence-array.h>
>> +#include <linux/pci-p2pdma.h>
>>   
>>   /**
>>    * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
>> @@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
>>   	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
>>   	int r;
>>   
>> +	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
>> +		attach->peer2peer = false;
>> +
> Are there other related patches than this series?
>
> p2p dma mapping needs to be done in common code, in p2pdma.c - ie this
> open coding is missing the bus_offset stuff, at least.

Yeah, I'm aware of this. But I couldn't find a better way for now.

> I really do not want to see drivers open code this stuff.
>
> We already have a p2pdma API for handling the struct page case, so I
> suggest adding some new p2pdma API to handle this for non-struct page
> cases.
>
> ie some thing like:
>
> int 'p2pdma map bar'(
>     struct pci_device *source,
>     unsigned int source_bar_number,
>     struct pci_device *dest,
>     physaddr&len *array_of_offsets & length pairs into source bar,
>     struct scatterlist *output_sgl)

Well that's exactly what I have to avoid since I don't have the array of 
offsets around and want to avoid constructing it.

Similar problem for dma_map_resource(). My example does this on demand, 
but essentially we also have use cases where this is done only once.

Ideally we would have some function to create an sgl based on some 
arbitrary collection of offsets and length inside a BAR.

Regards,
Christian.

>
> Jason

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Jason Gunthorpe]

On Wed, Mar 11, 2020 at 03:33:01PM +0100, Christian König wrote:
> Am 11.03.20 um 15:04 schrieb Jason Gunthorpe:
> > On Wed, Mar 11, 2020 at 02:51:56PM +0100, Christian König wrote:
> > > Check if we can do peer2peer on the PCIe bus.
> > > 
> > > Signed-off-by: Christian König <christian.koenig@amd.com>
> > >   drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
> > >   1 file changed, 4 insertions(+)
> > > 
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
> > > index aef12ee2f1e3..bbf67800c8a6 100644
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
> > > @@ -38,6 +38,7 @@
> > >   #include <drm/amdgpu_drm.h>
> > >   #include <linux/dma-buf.h>
> > >   #include <linux/dma-fence-array.h>
> > > +#include <linux/pci-p2pdma.h>
> > >   /**
> > >    * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
> > > @@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
> > >   	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
> > >   	int r;
> > > +	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
> > > +		attach->peer2peer = false;
> > > +
> > Are there other related patches than this series?
> > 
> > p2p dma mapping needs to be done in common code, in p2pdma.c - ie this
> > open coding is missing the bus_offset stuff, at least.
> 
> Yeah, I'm aware of this. But I couldn't find a better way for now.

Well, it isn't optional :)
 
> > I really do not want to see drivers open code this stuff.
> > 
> > We already have a p2pdma API for handling the struct page case, so I
> > suggest adding some new p2pdma API to handle this for non-struct page
> > cases.
> > 
> > ie some thing like:
> > 
> > int 'p2pdma map bar'(
> >     struct pci_device *source,
> >     unsigned int source_bar_number,
> >     struct pci_device *dest,
> >     physaddr&len *array_of_offsets & length pairs into source bar,
> >     struct scatterlist *output_sgl)
> 
> Well that's exactly what I have to avoid since I don't have the array of
> offsets around and want to avoid constructing it.

Maybe it doesn't need an array of offsets - just a single offset and
callers can iterate the API?

> Similar problem for dma_map_resource(). My example does this on demand, but
> essentially we also have use cases where this is done only once.

I'm not sure if this is portable. Does any IOMMU HW need to know P2P
is happening to setup successfully? We currently support such a narrow
scope of HW for P2P..

> Ideally we would have some function to create an sgl based on some arbitrary
> collection of offsets and length inside a BAR.

Isn't that what I just proposed above ?

Jason

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Jason Gunthorpe]

On Wed, Mar 11, 2020 at 03:33:01PM +0100, Christian König wrote:
> Am 11.03.20 um 15:04 schrieb Jason Gunthorpe:
> > On Wed, Mar 11, 2020 at 02:51:56PM +0100, Christian König wrote:
> > > Check if we can do peer2peer on the PCIe bus.
> > > 
> > > Signed-off-by: Christian König <christian.koenig@amd.com>
> > >   drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
> > >   1 file changed, 4 insertions(+)
> > > 
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
> > > index aef12ee2f1e3..bbf67800c8a6 100644
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
> > > @@ -38,6 +38,7 @@
> > >   #include <drm/amdgpu_drm.h>
> > >   #include <linux/dma-buf.h>
> > >   #include <linux/dma-fence-array.h>
> > > +#include <linux/pci-p2pdma.h>
> > >   /**
> > >    * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
> > > @@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
> > >   	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
> > >   	int r;
> > > +	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
> > > +		attach->peer2peer = false;
> > > +
> > Are there other related patches than this series?
> > 
> > p2p dma mapping needs to be done in common code, in p2pdma.c - ie this
> > open coding is missing the bus_offset stuff, at least.
> 
> Yeah, I'm aware of this. But I couldn't find a better way for now.

Well, it isn't optional :)
 
> > I really do not want to see drivers open code this stuff.
> > 
> > We already have a p2pdma API for handling the struct page case, so I
> > suggest adding some new p2pdma API to handle this for non-struct page
> > cases.
> > 
> > ie some thing like:
> > 
> > int 'p2pdma map bar'(
> >     struct pci_device *source,
> >     unsigned int source_bar_number,
> >     struct pci_device *dest,
> >     physaddr&len *array_of_offsets & length pairs into source bar,
> >     struct scatterlist *output_sgl)
> 
> Well that's exactly what I have to avoid since I don't have the array of
> offsets around and want to avoid constructing it.

Maybe it doesn't need an array of offsets - just a single offset and
callers can iterate the API?

> Similar problem for dma_map_resource(). My example does this on demand, but
> essentially we also have use cases where this is done only once.

I'm not sure if this is portable. Does any IOMMU HW need to know P2P
is happening to setup successfully? We currently support such a narrow
scope of HW for P2P..

> Ideally we would have some function to create an sgl based on some arbitrary
> collection of offsets and length inside a BAR.

Isn't that what I just proposed above ?

Jason
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Christian König]

Am 11.03.20 um 15:38 schrieb Jason Gunthorpe:
> On Wed, Mar 11, 2020 at 03:33:01PM +0100, Christian König wrote:
>> Am 11.03.20 um 15:04 schrieb Jason Gunthorpe:
>>> On Wed, Mar 11, 2020 at 02:51:56PM +0100, Christian König wrote:
>>>> Check if we can do peer2peer on the PCIe bus.
>>>>
>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
>>>>    1 file changed, 4 insertions(+)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
>>>> index aef12ee2f1e3..bbf67800c8a6 100644
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
>>>> @@ -38,6 +38,7 @@
>>>>    #include <drm/amdgpu_drm.h>
>>>>    #include <linux/dma-buf.h>
>>>>    #include <linux/dma-fence-array.h>
>>>> +#include <linux/pci-p2pdma.h>
>>>>    /**
>>>>     * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
>>>> @@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
>>>>    	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
>>>>    	int r;
>>>> +	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
>>>> +		attach->peer2peer = false;
>>>> +
>>> Are there other related patches than this series?
>>>
>>> p2p dma mapping needs to be done in common code, in p2pdma.c - ie this
>>> open coding is missing the bus_offset stuff, at least.
>> Yeah, I'm aware of this. But I couldn't find a better way for now.
> Well, it isn't optional :)
>   
>>> I really do not want to see drivers open code this stuff.
>>>
>>> We already have a p2pdma API for handling the struct page case, so I
>>> suggest adding some new p2pdma API to handle this for non-struct page
>>> cases.
>>>
>>> ie some thing like:
>>>
>>> int 'p2pdma map bar'(
>>>      struct pci_device *source,
>>>      unsigned int source_bar_number,
>>>      struct pci_device *dest,
>>>      physaddr&len *array_of_offsets & length pairs into source bar,
>>>      struct scatterlist *output_sgl)
>> Well that's exactly what I have to avoid since I don't have the array of
>> offsets around and want to avoid constructing it.
> Maybe it doesn't need an array of offsets - just a single offset and
> callers can iterate the API?

Yes, that would of course work as well.

But I was assuming that p2pdma_map_bar() needs some state between those 
calls.

>
>> Similar problem for dma_map_resource(). My example does this on demand, but
>> essentially we also have use cases where this is done only once.
> I'm not sure if this is portable. Does any IOMMU HW need to know P2P
> is happening to setup successfully? We currently support such a narrow
> scope of HW for P2P..

On the AMD hardware I'm testing this calling dma_map_resource() already 
seems to work with IOMMU enabled. (Well at least it seemed so 6month ago 
when I last tested this).

>> Ideally we would have some function to create an sgl based on some arbitrary
>> collection of offsets and length inside a BAR.
> Isn't that what I just proposed above ?

Yes, just didn't thought that this would easily possible. I will double 
check the p2pdma code again.

Thanks,
Christian.

>
> Jason

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Christian König]

Am 11.03.20 um 15:38 schrieb Jason Gunthorpe:
> On Wed, Mar 11, 2020 at 03:33:01PM +0100, Christian König wrote:
>> Am 11.03.20 um 15:04 schrieb Jason Gunthorpe:
>>> On Wed, Mar 11, 2020 at 02:51:56PM +0100, Christian König wrote:
>>>> Check if we can do peer2peer on the PCIe bus.
>>>>
>>>> Signed-off-by: Christian König <christian.koenig@amd.com>
>>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c | 4 ++++
>>>>    1 file changed, 4 insertions(+)
>>>>
>>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
>>>> index aef12ee2f1e3..bbf67800c8a6 100644
>>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
>>>> @@ -38,6 +38,7 @@
>>>>    #include <drm/amdgpu_drm.h>
>>>>    #include <linux/dma-buf.h>
>>>>    #include <linux/dma-fence-array.h>
>>>> +#include <linux/pci-p2pdma.h>
>>>>    /**
>>>>     * amdgpu_gem_prime_vmap - &dma_buf_ops.vmap implementation
>>>> @@ -179,6 +180,9 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
>>>>    	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
>>>>    	int r;
>>>> +	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
>>>> +		attach->peer2peer = false;
>>>> +
>>> Are there other related patches than this series?
>>>
>>> p2p dma mapping needs to be done in common code, in p2pdma.c - ie this
>>> open coding is missing the bus_offset stuff, at least.
>> Yeah, I'm aware of this. But I couldn't find a better way for now.
> Well, it isn't optional :)
>   
>>> I really do not want to see drivers open code this stuff.
>>>
>>> We already have a p2pdma API for handling the struct page case, so I
>>> suggest adding some new p2pdma API to handle this for non-struct page
>>> cases.
>>>
>>> ie some thing like:
>>>
>>> int 'p2pdma map bar'(
>>>      struct pci_device *source,
>>>      unsigned int source_bar_number,
>>>      struct pci_device *dest,
>>>      physaddr&len *array_of_offsets & length pairs into source bar,
>>>      struct scatterlist *output_sgl)
>> Well that's exactly what I have to avoid since I don't have the array of
>> offsets around and want to avoid constructing it.
> Maybe it doesn't need an array of offsets - just a single offset and
> callers can iterate the API?

Yes, that would of course work as well.

But I was assuming that p2pdma_map_bar() needs some state between those 
calls.

>
>> Similar problem for dma_map_resource(). My example does this on demand, but
>> essentially we also have use cases where this is done only once.
> I'm not sure if this is portable. Does any IOMMU HW need to know P2P
> is happening to setup successfully? We currently support such a narrow
> scope of HW for P2P..

On the AMD hardware I'm testing this calling dma_map_resource() already 
seems to work with IOMMU enabled. (Well at least it seemed so 6month ago 
when I last tested this).

>> Ideally we would have some function to create an sgl based on some arbitrary
>> collection of offsets and length inside a BAR.
> Isn't that what I just proposed above ?

Yes, just didn't thought that this would easily possible. I will double 
check the p2pdma code again.

Thanks,
Christian.

>
> Jason

_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Jason Gunthorpe]

On Wed, Mar 11, 2020 at 03:43:03PM +0100, Christian König wrote:
> > > > int 'p2pdma map bar'(
> > > >      struct pci_device *source,
> > > >      unsigned int source_bar_number,
> > > >      struct pci_device *dest,
> > > >      physaddr&len *array_of_offsets & length pairs into source bar,
> > > >      struct scatterlist *output_sgl)
> > > Well that's exactly what I have to avoid since I don't have the array of
> > > offsets around and want to avoid constructing it.
> > Maybe it doesn't need an array of offsets - just a single offset and
> > callers can iterate the API?
> 
> Yes, that would of course work as well.
> 
> But I was assuming that p2pdma_map_bar() needs some state between those
> calls.

It might be able to run faster if some state is held. We've had APIs
before where the caller can provide a cache for expensive state for
APIs. Maybe that would be an appropriate pattern here?

IIRC the distance calculation is the expensive bit, that would be easy
enough to cache.

Jason

Re: [PATCH 4/6] drm/amdgpu: add checks if DMA-buf P2P is supported

[Jason Gunthorpe]

On Wed, Mar 11, 2020 at 03:43:03PM +0100, Christian König wrote:
> > > > int 'p2pdma map bar'(
> > > >      struct pci_device *source,
> > > >      unsigned int source_bar_number,
> > > >      struct pci_device *dest,
> > > >      physaddr&len *array_of_offsets & length pairs into source bar,
> > > >      struct scatterlist *output_sgl)
> > > Well that's exactly what I have to avoid since I don't have the array of
> > > offsets around and want to avoid constructing it.
> > Maybe it doesn't need an array of offsets - just a single offset and
> > callers can iterate the API?
> 
> Yes, that would of course work as well.
> 
> But I was assuming that p2pdma_map_bar() needs some state between those
> calls.

It might be able to run faster if some state is held. We've had APIs
before where the caller can provide a cache for expensive state for
APIs. Maybe that would be an appropriate pattern here?

IIRC the distance calculation is the expensive bit, that would be easy
enough to cache.

Jason
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel