* [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
@ 2022-06-03 10:52 Ramesh Errabolu
2022-06-03 22:26 ` Felix Kuehling
0 siblings, 1 reply; 14+ messages in thread
From: Ramesh Errabolu @ 2022-06-03 10:52 UTC (permalink / raw)
To: amd-gfx; +Cc: Ramesh Errabolu
Add support for peer-to-peer communication among AMD GPUs over PCIe
bus. Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 312 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 33 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
4 files changed, 294 insertions(+), 60 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 054e4a76ae2e..8e37eae26e49 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+ struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it instead
+ * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without regard to
+ * device ownership. This is because access requests for MMIO go through PCIe root
+ * complex.
+ *
+ * This is accomplished in two steps:
+ * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
+ * in updating requesting device's page table
+ * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
+ * accessible. This allows an update of requesting device's page table
+ * with entries associated with DOOREBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * - Mapping of DOORBELL or MMIO BO of same or peer device
+ * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
+ *
+ * Return: ZERO if successful, NON-ZERO otherwise
+ */
+static int
+kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+ dma_addr_t dma_addr;
+ bool mmio;
+ int ret;
+
+ /* Expect SG Table of dmapmap BO to be NULL */
+ mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
+ if (unlikely(ttm->sg)) {
+ pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
+ return -EINVAL;
+ }
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_addr = mem->bo->tbo.sg->sgl->dma_address;
+ pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
+ pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
+ dma_addr = dma_map_resource(adev->dev, dma_addr,
+ mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ ret = dma_mapping_error(adev->dev, dma_addr);
+ if (unlikely(ret))
+ return ret;
+ pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
+
+ ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
+ if (unlikely(!ttm->sg)) {
+ ret = -ENOMEM;
+ goto unmap_sg;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+ ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+ if (unlikely(ret))
+ goto free_sg;
+
+ return ret;
+
+free_sg:
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+unmap_sg:
+ dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
+ dir, DMA_ATTR_SKIP_CPU_SYNC);
+ return ret;
+}
+
static int
kfd_mem_dmamap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
return kfd_mem_dmamap_userptr(mem, attachment);
case KFD_MEM_ATT_DMABUF:
return kfd_mem_dmamap_dmabuf(attachment);
+ case KFD_MEM_ATT_SG:
+ return kfd_mem_dmamap_sg_bo(mem, attachment);
default:
WARN_ON_ONCE(1);
}
@@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * The method performs following steps:
+ * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
+ * - Free SG Table that is used to encapsulate DMA mapped memory of
+ * peer device's DOORBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
+ * Eviction of DOOREBELL or MMIO BO on device having access to its memory
+ *
+ * Return: void
+ */
+static void
+kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+
+ if (unlikely(!ttm->sg)) {
+ pr_err("SG Table of BO is UNEXPECTEDLY NULL");
+ return;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+ ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
+ ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+ bo->tbo.sg = NULL;
+}
+
static void
kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
case KFD_MEM_ATT_DMABUF:
kfd_mem_dmaunmap_dmabuf(attachment);
break;
+ case KFD_MEM_ATT_SG:
+ kfd_mem_dmaunmap_sg_bo(mem, attachment);
+ break;
default:
WARN_ON_ONCE(1);
}
}
-static int
-kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
- struct amdgpu_bo **bo)
-{
- unsigned long bo_size = mem->bo->tbo.base.size;
- struct drm_gem_object *gobj;
- int ret;
-
- ret = amdgpu_bo_reserve(mem->bo, false);
- if (ret)
- return ret;
-
- ret = amdgpu_gem_object_create(adev, bo_size, 1,
- AMDGPU_GEM_DOMAIN_CPU,
- AMDGPU_GEM_CREATE_PREEMPTIBLE,
- ttm_bo_type_sg, mem->bo->tbo.base.resv,
- &gobj);
- amdgpu_bo_unreserve(mem->bo);
- if (ret)
- return ret;
-
- *bo = gem_to_amdgpu_bo(gobj);
- (*bo)->parent = amdgpu_bo_ref(mem->bo);
-
- return 0;
-}
-
static int
kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
struct amdgpu_bo **bo)
@@ -691,6 +862,8 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
uint64_t va = mem->va;
struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
struct amdgpu_bo *bo[2] = {NULL, NULL};
+ bool same_hive = false;
+ bool accessible = false;
int i, ret;
if (!va) {
@@ -698,6 +871,31 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
return -EINVAL;
}
+ /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
+ *
+ * The access path of MMIO and DOORBELL BOs of is always over PCIe.
+ * In contrast the access path of VRAM BOs depens upon the type of
+ * link that connects the peer device. Access over PCIe is allowed
+ * if peer device has large BAR. In contrast, access over xGMI is
+ * allowed for both small and large BAR configurations of peer device
+ */
+
+ if ((adev != bo_adev) &&
+ ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
+ accessible = amdgpu_device_is_peer_accessible(bo_adev, adev);
+ if ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))
+ if (!accessible)
+ return -EINVAL;
+ if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
+ same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
+ if (!same_hive && !accessible)
+ return -EINVAL;
+ }
+ }
+
for (i = 0; i <= is_aql; i++) {
attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
if (unlikely(!attachment[i])) {
@@ -708,9 +906,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
va + bo_size, vm);
- if (adev == bo_adev ||
- (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
- (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
+ if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
+ (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
+ same_hive) {
/* Mappings on the local GPU, or VRAM mappings in the
* local hive, or userptr mapping IOMMU direct map mode
* share the original BO
@@ -726,26 +924,38 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
} else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
/* Create an SG BO to DMA-map userptrs on other GPUs */
attachment[i]->type = KFD_MEM_ATT_USERPTR;
- ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
if (ret)
goto unwind;
} else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
mem->bo->tbo.type != ttm_bo_type_sg) {
- /* GTT BOs use DMA-mapping ability of dynamic-attach
- * DMA bufs. TODO: The same should work for VRAM on
- * large-BAR GPUs.
- */
+ /* GTT BOs use DMA-mapping ability of dynamic-attach DMA bufs */
+ attachment[i]->type = KFD_MEM_ATT_DMABUF;
+ ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
+ if (ret)
+ goto unwind;
+ /* Enable acces to VRAM BOs of peer devices */
+#if defined(CONFIG_HSA_AMD_P2P)
+ } else if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM &&
+ mem->bo->tbo.type == ttm_bo_type_device) {
attachment[i]->type = KFD_MEM_ATT_DMABUF;
ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
if (ret)
goto unwind;
+ pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
+#endif
+ /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
+ } else if ((mem->bo->tbo.type == ttm_bo_type_sg) &&
+ ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
+ attachment[i]->type = KFD_MEM_ATT_SG;
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
+ if (ret)
+ goto unwind;
} else {
- /* FIXME: Need to DMA-map other BO types:
- * large-BAR VRAM, doorbells, MMIO remap
- */
- attachment[i]->type = KFD_MEM_ATT_SHARED;
- bo[i] = mem->bo;
- drm_gem_object_get(&bo[i]->tbo.base);
+ WARN_ONCE(true, "Handling invalid ATTACH request");
+ ret = -EINVAL;
+ goto unwind;
}
/* Add BO to VM internal data structures */
@@ -1146,24 +1356,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
return ret;
}
-static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
-{
- struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
-
- if (!sg)
- return NULL;
- if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
- kfree(sg);
- return NULL;
- }
- sg->sgl->dma_address = addr;
- sg->sgl->length = size;
-#ifdef CONFIG_NEED_SG_DMA_LENGTH
- sg->sgl->dma_length = size;
-#endif
- return sg;
-}
-
static int process_validate_vms(struct amdkfd_process_info *process_info)
{
struct amdgpu_vm *peer_vm;
@@ -1532,7 +1724,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
bo_type = ttm_bo_type_sg;
if (size > UINT_MAX)
return -EINVAL;
- sg = create_doorbell_sg(*offset, size);
+ sg = create_sg_table(*offset, size);
if (!sg)
return -ENOMEM;
} else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index b5ee0eb984ee..acb9e934adc9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -32,6 +32,7 @@
#include <linux/slab.h>
#include <linux/iommu.h>
#include <linux/pci.h>
+#include <linux/pci-p2pdma.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_probe_helper.h>
@@ -127,6 +128,8 @@ const char *amdgpu_asic_name[] = {
"LAST",
};
+extern bool pcie_p2p;
+
/**
* DOC: pcie_replay_count
*
@@ -5434,6 +5437,36 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
}
}
+/**
+ * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
+ *
+ * @adev: amdgpu_device pointer
+ * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
+ *
+ * Return true if @peer_adev can access (DMA) @adev through the PCIe
+ * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
+ * @peer_adev.
+ */
+bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev)
+{
+#ifdef CONFIG_HSA_AMD_P2P
+ bool p2p_access = false;
+ uint64_t address_mask = peer_adev->dev->dma_mask ?
+ ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
+ resource_size_t aper_limit =
+ adev->gmc.aper_base + adev->gmc.aper_size - 1;
+ p2p_access = !(pci_p2pdma_distance_many(adev->pdev, &peer_adev->dev, 1, true) < 0);
+
+ return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
+ adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
+ !(adev->gmc.aper_base & address_mask ||
+ aper_limit & address_mask));
+#else
+ return false;
+#endif
+}
+
int amdgpu_device_baco_enter(struct drm_device *dev)
{
struct amdgpu_device *adev = drm_to_adev(dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index bed4ed88951f..d1c82a9e8569 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
#endif
+/**
+ * DOC: pcie_p2p (bool)
+ * Enable PCIe P2P (requires large-BAR). Default value: true (on)
+ */
+bool pcie_p2p = true;
+module_param(pcie_p2p, bool, 0444);
+MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
+
/**
* DOC: dcfeaturemask (uint)
* Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
--
2.35.1
^ permalink raw reply related [flat|nested] 14+ messages in thread
* Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-06-03 10:52 [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs Ramesh Errabolu
@ 2022-06-03 22:26 ` Felix Kuehling
2022-06-04 10:23 ` Errabolu, Ramesh
0 siblings, 1 reply; 14+ messages in thread
From: Felix Kuehling @ 2022-06-03 22:26 UTC (permalink / raw)
To: Ramesh Errabolu, amd-gfx
On 2022-06-03 06:52, Ramesh Errabolu wrote:
> Add support for peer-to-peer communication among AMD GPUs over PCIe
> bus. Support REQUIRES enablement of config HSA_AMD_P2P.
>
> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 312 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 33 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
> 4 files changed, 294 insertions(+), 60 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index f8b9f27adcf5..5c00ea1df21c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
> KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
> KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
> KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
> + KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
> };
>
> struct kfd_mem_attachment {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 054e4a76ae2e..8e37eae26e49 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
> kfree(bo->kfd_bo);
> }
>
> +/**
> + * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
> + * about USERPTR or DOOREBELL or MMIO BO.
> + * @adev: Device for which dmamap BO is being created
> + * @mem: BO of peer device that is being DMA mapped. Provides parameters
> + * in building the dmamap BO
> + * @bo_out: Output parameter updated with handle of dmamap BO
> + */
> +static int
> +create_dmamap_sg_bo(struct amdgpu_device *adev,
> + struct kgd_mem *mem, struct amdgpu_bo **bo_out)
> +{
> + struct drm_gem_object *gem_obj;
> + int ret, align;
> +
> + ret = amdgpu_bo_reserve(mem->bo, false);
> + if (ret)
> + return ret;
> +
> + align = 1;
> + ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
> + AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
> + ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
> +
> + amdgpu_bo_unreserve(mem->bo);
> +
> + if (ret) {
> + pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
> + return -EINVAL;
> + }
> +
> + *bo_out = gem_to_amdgpu_bo(gem_obj);
> + (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
> + return ret;
> +}
> +
> /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
> * reservation object.
> *
> @@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
> return pte_flags;
> }
>
> +/**
> + * create_sg_table() - Create an sg_table for a contiguous DMA addr range
> + * @addr: The starting address to point to
> + * @size: Size of memory area in bytes being pointed to
> + *
> + * Allocates an instance of sg_table and initializes it to point to memory
> + * area specified by input parameters. The address used to build is assumed
> + * to be DMA mapped, if needed.
> + *
> + * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
> + * because they are physically contiguous.
> + *
> + * Return: Initialized instance of SG Table or NULL
> + */
> +static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
> +{
> + struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> +
> + if (!sg)
> + return NULL;
> + if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> + kfree(sg);
> + return NULL;
> + }
> + sg_dma_address(sg->sgl) = addr;
> + sg->sgl->length = size;
> +#ifdef CONFIG_NEED_SG_DMA_LENGTH
> + sg->sgl->dma_length = size;
> +#endif
> + return sg;
> +}
> +
> static int
> kfd_mem_dmamap_userptr(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
> return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing device
> + *
> + * An access request from the device that owns DOORBELL does not require DMA mapping.
> + * This is because the request doesn't go through PCIe root complex i.e. it instead
> + * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
> + *
> + * In contrast, all access requests for MMIO need to be DMA mapped without regard to
> + * device ownership. This is because access requests for MMIO go through PCIe root
> + * complex.
> + *
> + * This is accomplished in two steps:
> + * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
> + * in updating requesting device's page table
> + * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
> + * accessible. This allows an update of requesting device's page table
> + * with entries associated with DOOREBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * - Mapping of DOORBELL or MMIO BO of same or peer device
> + * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
> + *
> + * Return: ZERO if successful, NON-ZERO otherwise
> + */
> +static int
> +kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment)
> +{
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> + dma_addr_t dma_addr;
> + bool mmio;
> + int ret;
> +
> + /* Expect SG Table of dmapmap BO to be NULL */
> + mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
> + if (unlikely(ttm->sg)) {
> + pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
> + return -EINVAL;
> + }
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_addr = mem->bo->tbo.sg->sgl->dma_address;
> + pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
> + pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
> + dma_addr = dma_map_resource(adev->dev, dma_addr,
> + mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + ret = dma_mapping_error(adev->dev, dma_addr);
> + if (unlikely(ret))
> + return ret;
> + pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
> +
> + ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
> + if (unlikely(!ttm->sg)) {
> + ret = -ENOMEM;
> + goto unmap_sg;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
> + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> + if (unlikely(ret))
> + goto free_sg;
> +
> + return ret;
> +
> +free_sg:
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> +unmap_sg:
> + dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
> + dir, DMA_ATTR_SKIP_CPU_SYNC);
> + return ret;
> +}
> +
> static int
> kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> return kfd_mem_dmamap_userptr(mem, attachment);
> case KFD_MEM_ATT_DMABUF:
> return kfd_mem_dmamap_dmabuf(attachment);
> + case KFD_MEM_ATT_SG:
> + return kfd_mem_dmamap_sg_bo(mem, attachment);
> default:
> WARN_ON_ONCE(1);
> }
> @@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
> ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing device
> + *
> + * The method performs following steps:
> + * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
> + * - Free SG Table that is used to encapsulate DMA mapped memory of
> + * peer device's DOORBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
> + * Eviction of DOOREBELL or MMIO BO on device having access to its memory
> + *
> + * Return: void
> + */
> +static void
> +kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment)
> +{
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> +
> + if (unlikely(!ttm->sg)) {
> + pr_err("SG Table of BO is UNEXPECTEDLY NULL");
> + return;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
> + ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
> + ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> + bo->tbo.sg = NULL;
> +}
> +
> static void
> kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> case KFD_MEM_ATT_DMABUF:
> kfd_mem_dmaunmap_dmabuf(attachment);
> break;
> + case KFD_MEM_ATT_SG:
> + kfd_mem_dmaunmap_sg_bo(mem, attachment);
> + break;
> default:
> WARN_ON_ONCE(1);
> }
> }
>
> -static int
> -kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
> - struct amdgpu_bo **bo)
> -{
> - unsigned long bo_size = mem->bo->tbo.base.size;
> - struct drm_gem_object *gobj;
> - int ret;
> -
> - ret = amdgpu_bo_reserve(mem->bo, false);
> - if (ret)
> - return ret;
> -
> - ret = amdgpu_gem_object_create(adev, bo_size, 1,
> - AMDGPU_GEM_DOMAIN_CPU,
> - AMDGPU_GEM_CREATE_PREEMPTIBLE,
> - ttm_bo_type_sg, mem->bo->tbo.base.resv,
> - &gobj);
> - amdgpu_bo_unreserve(mem->bo);
> - if (ret)
> - return ret;
> -
> - *bo = gem_to_amdgpu_bo(gobj);
> - (*bo)->parent = amdgpu_bo_ref(mem->bo);
> -
> - return 0;
> -}
> -
> static int
> kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
> struct amdgpu_bo **bo)
> @@ -691,6 +862,8 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> uint64_t va = mem->va;
> struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
> struct amdgpu_bo *bo[2] = {NULL, NULL};
> + bool same_hive = false;
> + bool accessible = false;
> int i, ret;
>
> if (!va) {
> @@ -698,6 +871,31 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> return -EINVAL;
> }
>
> + /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
> + *
> + * The access path of MMIO and DOORBELL BOs of is always over PCIe.
> + * In contrast the access path of VRAM BOs depens upon the type of
> + * link that connects the peer device. Access over PCIe is allowed
> + * if peer device has large BAR. In contrast, access over xGMI is
> + * allowed for both small and large BAR configurations of peer device
> + */
> +
> + if ((adev != bo_adev) &&
> + ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
The parentheses around == and != are unnecessary.
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
> + accessible = amdgpu_device_is_peer_accessible(bo_adev, adev);
This can still call amdgpu_device_is_peer_accessible unnecessarily on
XGMI connected GPUs.
This whole block could be simplified like this
if (adev != bo_adev &&
(mem->domain == AMDGPU_GEM_DOMAIN_VRAM ||
(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
return -EINVAL;
}
The short-circuit of the && operator make sure
amdgpu_device_is_peer_accessible is only called if necessary. And you
don't need the bool accessible variable any more.
> + if ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))
> + if (!accessible)
> + return -EINVAL;
> + if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
> + same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
> + if (!same_hive && !accessible)
> + return -EINVAL;
> + }
> + }
> +
> for (i = 0; i <= is_aql; i++) {
> attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
> if (unlikely(!attachment[i])) {
> @@ -708,9 +906,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
> va + bo_size, vm);
>
> - if (adev == bo_adev ||
> - (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> - (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
> + if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
> + (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> + same_hive) {
> /* Mappings on the local GPU, or VRAM mappings in the
> * local hive, or userptr mapping IOMMU direct map mode
> * share the original BO
> @@ -726,26 +924,38 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
> /* Create an SG BO to DMA-map userptrs on other GPUs */
> attachment[i]->type = KFD_MEM_ATT_USERPTR;
> - ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
> mem->bo->tbo.type != ttm_bo_type_sg) {
> - /* GTT BOs use DMA-mapping ability of dynamic-attach
> - * DMA bufs. TODO: The same should work for VRAM on
> - * large-BAR GPUs.
> - */
> + /* GTT BOs use DMA-mapping ability of dynamic-attach DMA bufs */
> + attachment[i]->type = KFD_MEM_ATT_DMABUF;
> + ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
> + if (ret)
> + goto unwind;
> + /* Enable acces to VRAM BOs of peer devices */
> +#if defined(CONFIG_HSA_AMD_P2P)
> + } else if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM &&
> + mem->bo->tbo.type == ttm_bo_type_device) {
> attachment[i]->type = KFD_MEM_ATT_DMABUF;
> ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
Now this looks identical to the GTT case. Since the CONFIG_HSA_AMD_P2P
check is already done in amdgpu_device_is_peer_accessible, you can
probably just merge the two if-cases into one now:
...
} else if ((mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
mem->bo->tbo.type != ttm_bo_type_sg) ||
mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
/* GTT and VRAM BOs use DMA-mapping ability of dynamic-attach
* DMA bufs.
*/
attachment[i]->type = KFD_MEM_ATT_DMABUF;
ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
if (ret)
goto unwind;
...
If you move this after the doorbell and MMIO case, you can even drop the
mem->bo->tbo.type != ttm_bo_type_sg check.
> if (ret)
> goto unwind;
> + pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
> +#endif
> + /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
> + } else if ((mem->bo->tbo.type == ttm_bo_type_sg) &&
> + ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
I think userptr, doorbell and MMIO BOs are the only ones that use SGs.
So you don't really need to check the mem->alloc_flags here. Except
maybe as a WARN_ONCE sanity check.
> + attachment[i]->type = KFD_MEM_ATT_SG;
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> + if (ret)
> + goto unwind;
> } else {
> - /* FIXME: Need to DMA-map other BO types:
> - * large-BAR VRAM, doorbells, MMIO remap
> - */
> - attachment[i]->type = KFD_MEM_ATT_SHARED;
> - bo[i] = mem->bo;
> - drm_gem_object_get(&bo[i]->tbo.base);
> + WARN_ONCE(true, "Handling invalid ATTACH request");
> + ret = -EINVAL;
> + goto unwind;
> }
>
> /* Add BO to VM internal data structures */
> @@ -1146,24 +1356,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
> return ret;
> }
>
> -static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
> -{
> - struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> -
> - if (!sg)
> - return NULL;
> - if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> - kfree(sg);
> - return NULL;
> - }
> - sg->sgl->dma_address = addr;
> - sg->sgl->length = size;
> -#ifdef CONFIG_NEED_SG_DMA_LENGTH
> - sg->sgl->dma_length = size;
> -#endif
> - return sg;
> -}
> -
> static int process_validate_vms(struct amdkfd_process_info *process_info)
> {
> struct amdgpu_vm *peer_vm;
> @@ -1532,7 +1724,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
> bo_type = ttm_bo_type_sg;
> if (size > UINT_MAX)
> return -EINVAL;
> - sg = create_doorbell_sg(*offset, size);
> + sg = create_sg_table(*offset, size);
> if (!sg)
> return -ENOMEM;
> } else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index b5ee0eb984ee..acb9e934adc9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -32,6 +32,7 @@
> #include <linux/slab.h>
> #include <linux/iommu.h>
> #include <linux/pci.h>
> +#include <linux/pci-p2pdma.h>
>
> #include <drm/drm_atomic_helper.h>
> #include <drm/drm_probe_helper.h>
> @@ -127,6 +128,8 @@ const char *amdgpu_asic_name[] = {
> "LAST",
> };
>
> +extern bool pcie_p2p;
> +
> /**
> * DOC: pcie_replay_count
> *
> @@ -5434,6 +5437,36 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
> }
> }
>
> +/**
> + * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
> + *
> + * @adev: amdgpu_device pointer
> + * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
> + *
> + * Return true if @peer_adev can access (DMA) @adev through the PCIe
> + * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
> + * @peer_adev.
> + */
> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev)
> +{
> +#ifdef CONFIG_HSA_AMD_P2P
> + bool p2p_access = false;
> + uint64_t address_mask = peer_adev->dev->dma_mask ?
> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
> + resource_size_t aper_limit =
> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
> + p2p_access = !(pci_p2pdma_distance_many(adev->pdev, &peer_adev->dev, 1, true) < 0);
This would give you a checkpatch warning. Please run checkpatch.
Why can't you initialize p2p_access in the declaration above?
Regards,
Felix
> +
> + return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
> + adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
> + !(adev->gmc.aper_base & address_mask ||
> + aper_limit & address_mask));
> +#else
> + return false;
> +#endif
> +}
> +
> int amdgpu_device_baco_enter(struct drm_device *dev)
> {
> struct amdgpu_device *adev = drm_to_adev(dev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index bed4ed88951f..d1c82a9e8569 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
> module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
> #endif
>
> +/**
> + * DOC: pcie_p2p (bool)
> + * Enable PCIe P2P (requires large-BAR). Default value: true (on)
> + */
> +bool pcie_p2p = true;
> +module_param(pcie_p2p, bool, 0444);
> +MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
> +
> /**
> * DOC: dcfeaturemask (uint)
> * Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
^ permalink raw reply [flat|nested] 14+ messages in thread
* RE: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-06-03 22:26 ` Felix Kuehling
@ 2022-06-04 10:23 ` Errabolu, Ramesh
2022-06-06 14:26 ` Felix Kuehling
0 siblings, 1 reply; 14+ messages in thread
From: Errabolu, Ramesh @ 2022-06-04 10:23 UTC (permalink / raw)
To: Kuehling, Felix, amd-gfx
[AMD Official Use Only - General]
Will post a new patch. My responses are inline
Regards,
Ramesh
-----Original Message-----
From: Kuehling, Felix <Felix.Kuehling@amd.com>
Sent: Saturday, June 4, 2022 3:57 AM
To: Errabolu, Ramesh <Ramesh.Errabolu@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
On 2022-06-03 06:52, Ramesh Errabolu wrote:
> Add support for peer-to-peer communication among AMD GPUs over PCIe
> bus. Support REQUIRES enablement of config HSA_AMD_P2P.
>
> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 312 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 33 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
> 4 files changed, 294 insertions(+), 60 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index f8b9f27adcf5..5c00ea1df21c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
> KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
> KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
> KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
> + KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
> };
>
> struct kfd_mem_attachment {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 054e4a76ae2e..8e37eae26e49 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
> kfree(bo->kfd_bo);
> }
>
> +/**
> + * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect
> +information
> + * about USERPTR or DOOREBELL or MMIO BO.
> + * @adev: Device for which dmamap BO is being created
> + * @mem: BO of peer device that is being DMA mapped. Provides parameters
> + * in building the dmamap BO
> + * @bo_out: Output parameter updated with handle of dmamap BO */
> +static int create_dmamap_sg_bo(struct amdgpu_device *adev,
> + struct kgd_mem *mem, struct amdgpu_bo **bo_out) {
> + struct drm_gem_object *gem_obj;
> + int ret, align;
> +
> + ret = amdgpu_bo_reserve(mem->bo, false);
> + if (ret)
> + return ret;
> +
> + align = 1;
> + ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
> + AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
> + ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
> +
> + amdgpu_bo_unreserve(mem->bo);
> +
> + if (ret) {
> + pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
> + return -EINVAL;
> + }
> +
> + *bo_out = gem_to_amdgpu_bo(gem_obj);
> + (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
> + return ret;
> +}
> +
> /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
> * reservation object.
> *
> @@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
> return pte_flags;
> }
>
> +/**
> + * create_sg_table() - Create an sg_table for a contiguous DMA addr
> +range
> + * @addr: The starting address to point to
> + * @size: Size of memory area in bytes being pointed to
> + *
> + * Allocates an instance of sg_table and initializes it to point to
> +memory
> + * area specified by input parameters. The address used to build is
> +assumed
> + * to be DMA mapped, if needed.
> + *
> + * DOORBELL or MMIO BOs use only one scatterlist node in their
> +sg_table
> + * because they are physically contiguous.
> + *
> + * Return: Initialized instance of SG Table or NULL */ static struct
> +sg_table *create_sg_table(uint64_t addr, uint32_t size) {
> + struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> +
> + if (!sg)
> + return NULL;
> + if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> + kfree(sg);
> + return NULL;
> + }
> + sg_dma_address(sg->sgl) = addr;
> + sg->sgl->length = size;
> +#ifdef CONFIG_NEED_SG_DMA_LENGTH
> + sg->sgl->dma_length = size;
> +#endif
> + return sg;
> +}
> +
> static int
> kfd_mem_dmamap_userptr(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment) @@ -545,6 +613,87 @@
> kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
> return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access
> +DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing
> +device
> + *
> + * An access request from the device that owns DOORBELL does not require DMA mapping.
> + * This is because the request doesn't go through PCIe root complex
> +i.e. it instead
> + * loops back. The need to DMA map arises only when accessing peer
> +device's DOORBELL
> + *
> + * In contrast, all access requests for MMIO need to be DMA mapped
> +without regard to
> + * device ownership. This is because access requests for MMIO go
> +through PCIe root
> + * complex.
> + *
> + * This is accomplished in two steps:
> + * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
> + * in updating requesting device's page table
> + * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
> + * accessible. This allows an update of requesting device's page table
> + * with entries associated with DOOREBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * - Mapping of DOORBELL or MMIO BO of same or peer device
> + * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
> + *
> + * Return: ZERO if successful, NON-ZERO otherwise */ static int
> +kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment) {
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> + dma_addr_t dma_addr;
> + bool mmio;
> + int ret;
> +
> + /* Expect SG Table of dmapmap BO to be NULL */
> + mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
> + if (unlikely(ttm->sg)) {
> + pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
> + return -EINVAL;
> + }
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_addr = mem->bo->tbo.sg->sgl->dma_address;
> + pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
> + pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
> + dma_addr = dma_map_resource(adev->dev, dma_addr,
> + mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + ret = dma_mapping_error(adev->dev, dma_addr);
> + if (unlikely(ret))
> + return ret;
> + pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
> +
> + ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
> + if (unlikely(!ttm->sg)) {
> + ret = -ENOMEM;
> + goto unmap_sg;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
> + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> + if (unlikely(ret))
> + goto free_sg;
> +
> + return ret;
> +
> +free_sg:
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> +unmap_sg:
> + dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
> + dir, DMA_ATTR_SKIP_CPU_SYNC);
> + return ret;
> +}
> +
> static int
> kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment) @@ -556,6 +705,8 @@
> kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> return kfd_mem_dmamap_userptr(mem, attachment);
> case KFD_MEM_ATT_DMABUF:
> return kfd_mem_dmamap_dmabuf(attachment);
> + case KFD_MEM_ATT_SG:
> + return kfd_mem_dmamap_sg_bo(mem, attachment);
> default:
> WARN_ON_ONCE(1);
> }
> @@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
> ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or
> +MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing
> +device
> + *
> + * The method performs following steps:
> + * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
> + * - Free SG Table that is used to encapsulate DMA mapped memory of
> + * peer device's DOORBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
> + * Eviction of DOOREBELL or MMIO BO on device having access to its memory
> + *
> + * Return: void
> + */
> +static void
> +kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment) {
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> +
> + if (unlikely(!ttm->sg)) {
> + pr_err("SG Table of BO is UNEXPECTEDLY NULL");
> + return;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
> + ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
> + ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> + bo->tbo.sg = NULL;
> +}
> +
> static void
> kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment) @@ -609,38 +804,14 @@
> kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> case KFD_MEM_ATT_DMABUF:
> kfd_mem_dmaunmap_dmabuf(attachment);
> break;
> + case KFD_MEM_ATT_SG:
> + kfd_mem_dmaunmap_sg_bo(mem, attachment);
> + break;
> default:
> WARN_ON_ONCE(1);
> }
> }
>
> -static int
> -kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
> - struct amdgpu_bo **bo)
> -{
> - unsigned long bo_size = mem->bo->tbo.base.size;
> - struct drm_gem_object *gobj;
> - int ret;
> -
> - ret = amdgpu_bo_reserve(mem->bo, false);
> - if (ret)
> - return ret;
> -
> - ret = amdgpu_gem_object_create(adev, bo_size, 1,
> - AMDGPU_GEM_DOMAIN_CPU,
> - AMDGPU_GEM_CREATE_PREEMPTIBLE,
> - ttm_bo_type_sg, mem->bo->tbo.base.resv,
> - &gobj);
> - amdgpu_bo_unreserve(mem->bo);
> - if (ret)
> - return ret;
> -
> - *bo = gem_to_amdgpu_bo(gobj);
> - (*bo)->parent = amdgpu_bo_ref(mem->bo);
> -
> - return 0;
> -}
> -
> static int
> kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
> struct amdgpu_bo **bo)
> @@ -691,6 +862,8 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> uint64_t va = mem->va;
> struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
> struct amdgpu_bo *bo[2] = {NULL, NULL};
> + bool same_hive = false;
> + bool accessible = false;
> int i, ret;
>
> if (!va) {
> @@ -698,6 +871,31 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> return -EINVAL;
> }
>
> + /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
> + *
> + * The access path of MMIO and DOORBELL BOs of is always over PCIe.
> + * In contrast the access path of VRAM BOs depens upon the type of
> + * link that connects the peer device. Access over PCIe is allowed
> + * if peer device has large BAR. In contrast, access over xGMI is
> + * allowed for both small and large BAR configurations of peer device
> + */
> +
> + if ((adev != bo_adev) &&
> + ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
The parentheses around == and != are unnecessary.
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
> + accessible = amdgpu_device_is_peer_accessible(bo_adev, adev);
This can still call amdgpu_device_is_peer_accessible unnecessarily on XGMI connected GPUs.
This whole block could be simplified like this
if (adev != bo_adev &&
(mem->domain == AMDGPU_GEM_DOMAIN_VRAM ||
(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
return -EINVAL;
}
The short-circuit of the && operator make sure amdgpu_device_is_peer_accessible is only called if necessary. And you don't need the bool accessible variable any more.
> + if ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))
> + if (!accessible)
> + return -EINVAL;
> + if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
> + same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
> + if (!same_hive && !accessible)
> + return -EINVAL;
> + }
> + }
> +
> for (i = 0; i <= is_aql; i++) {
> attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
> if (unlikely(!attachment[i])) {
> @@ -708,9 +906,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
> va + bo_size, vm);
>
> - if (adev == bo_adev ||
> - (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> - (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
> + if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
> + (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> + same_hive) {
> /* Mappings on the local GPU, or VRAM mappings in the
> * local hive, or userptr mapping IOMMU direct map mode
> * share the original BO
> @@ -726,26 +924,38 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
> /* Create an SG BO to DMA-map userptrs on other GPUs */
> attachment[i]->type = KFD_MEM_ATT_USERPTR;
> - ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
> mem->bo->tbo.type != ttm_bo_type_sg) {
> - /* GTT BOs use DMA-mapping ability of dynamic-attach
> - * DMA bufs. TODO: The same should work for VRAM on
> - * large-BAR GPUs.
> - */
> + /* GTT BOs use DMA-mapping ability of dynamic-attach DMA bufs */
> + attachment[i]->type = KFD_MEM_ATT_DMABUF;
> + ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
> + if (ret)
> + goto unwind;
> + /* Enable acces to VRAM BOs of peer devices */ #if
> +defined(CONFIG_HSA_AMD_P2P)
> + } else if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM &&
> + mem->bo->tbo.type == ttm_bo_type_device) {
> attachment[i]->type = KFD_MEM_ATT_DMABUF;
> ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
Now this looks identical to the GTT case. Since the CONFIG_HSA_AMD_P2P check is already done in amdgpu_device_is_peer_accessible, you can probably just merge the two if-cases into one now:
...
} else if ((mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
mem->bo->tbo.type != ttm_bo_type_sg) ||
mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
/* GTT and VRAM BOs use DMA-mapping ability of dynamic-attach
* DMA bufs.
*/
attachment[i]->type = KFD_MEM_ATT_DMABUF;
ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
if (ret)
goto unwind;
...
If you move this after the doorbell and MMIO case, you can even drop the
mem->bo->tbo.type != ttm_bo_type_sg check.
Ramesh: Done
> if (ret)
> goto unwind;
> + pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
> +#endif
> + /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
> + } else if ((mem->bo->tbo.type == ttm_bo_type_sg) &&
> + ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
I think userptr, doorbell and MMIO BOs are the only ones that use SGs.
So you don't really need to check the mem->alloc_flags here. Except maybe as a WARN_ONCE sanity check.
Ramesh: Done
> + attachment[i]->type = KFD_MEM_ATT_SG;
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> + if (ret)
> + goto unwind;
> } else {
> - /* FIXME: Need to DMA-map other BO types:
> - * large-BAR VRAM, doorbells, MMIO remap
> - */
> - attachment[i]->type = KFD_MEM_ATT_SHARED;
> - bo[i] = mem->bo;
> - drm_gem_object_get(&bo[i]->tbo.base);
> + WARN_ONCE(true, "Handling invalid ATTACH request");
> + ret = -EINVAL;
> + goto unwind;
> }
>
> /* Add BO to VM internal data structures */
> @@ -1146,24 +1356,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
> return ret;
> }
>
> -static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
> -{
> - struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> -
> - if (!sg)
> - return NULL;
> - if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> - kfree(sg);
> - return NULL;
> - }
> - sg->sgl->dma_address = addr;
> - sg->sgl->length = size;
> -#ifdef CONFIG_NEED_SG_DMA_LENGTH
> - sg->sgl->dma_length = size;
> -#endif
> - return sg;
> -}
> -
> static int process_validate_vms(struct amdkfd_process_info *process_info)
> {
> struct amdgpu_vm *peer_vm;
> @@ -1532,7 +1724,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
> bo_type = ttm_bo_type_sg;
> if (size > UINT_MAX)
> return -EINVAL;
> - sg = create_doorbell_sg(*offset, size);
> + sg = create_sg_table(*offset, size);
> if (!sg)
> return -ENOMEM;
> } else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index b5ee0eb984ee..acb9e934adc9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -32,6 +32,7 @@
> #include <linux/slab.h>
> #include <linux/iommu.h>
> #include <linux/pci.h>
> +#include <linux/pci-p2pdma.h>
>
> #include <drm/drm_atomic_helper.h>
> #include <drm/drm_probe_helper.h>
> @@ -127,6 +128,8 @@ const char *amdgpu_asic_name[] = {
> "LAST",
> };
>
> +extern bool pcie_p2p;
> +
> /**
> * DOC: pcie_replay_count
> *
> @@ -5434,6 +5437,36 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
> }
> }
>
> +/**
> + * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
> + *
> + * @adev: amdgpu_device pointer
> + * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
> + *
> + * Return true if @peer_adev can access (DMA) @adev through the PCIe
> + * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
> + * @peer_adev.
> + */
> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev)
> +{
> +#ifdef CONFIG_HSA_AMD_P2P
> + bool p2p_access = false;
> + uint64_t address_mask = peer_adev->dev->dma_mask ?
> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
> + resource_size_t aper_limit =
> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
> + p2p_access = !(pci_p2pdma_distance_many(adev->pdev, &peer_adev->dev, 1, true) < 0);
This would give you a checkpatch warning. Please run checkpatch.
Why can't you initialize p2p_access in the declaration above?
Ramesh: I did run checkpatch script, and it didn't complain about the variable being not initialized. Being a variable of extern class, it is initialized at load time to default value of ZERO. This is then overridden in amdgpu_drv.c to true. So initializing it to true does not accomplish anything.
Regards,
Felix
> +
> + return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
> + adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
> + !(adev->gmc.aper_base & address_mask ||
> + aper_limit & address_mask));
> +#else
> + return false;
> +#endif
> +}
> +
> int amdgpu_device_baco_enter(struct drm_device *dev)
> {
> struct amdgpu_device *adev = drm_to_adev(dev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index bed4ed88951f..d1c82a9e8569 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
> module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
> #endif
>
> +/**
> + * DOC: pcie_p2p (bool)
> + * Enable PCIe P2P (requires large-BAR). Default value: true (on)
> + */
> +bool pcie_p2p = true;
> +module_param(pcie_p2p, bool, 0444);
> +MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
> +
> /**
> * DOC: dcfeaturemask (uint)
> * Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-06-04 10:23 ` Errabolu, Ramesh
@ 2022-06-06 14:26 ` Felix Kuehling
2022-06-06 18:11 ` Errabolu, Ramesh
0 siblings, 1 reply; 14+ messages in thread
From: Felix Kuehling @ 2022-06-06 14:26 UTC (permalink / raw)
To: Errabolu, Ramesh, amd-gfx list
Am 2022-06-04 um 06:23 schrieb Errabolu, Ramesh:
>> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
>> + struct amdgpu_device *peer_adev)
>> +{
>> +#ifdef CONFIG_HSA_AMD_P2P
>> + bool p2p_access = false;
>> + uint64_t address_mask = peer_adev->dev->dma_mask ?
>> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
>> + resource_size_t aper_limit =
>> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
>> + p2p_access = !(pci_p2pdma_distance_many(adev->pdev, &peer_adev->dev, 1, true) < 0);
> This would give you a checkpatch warning. Please run checkpatch.
>
> Why can't you initialize p2p_access in the declaration above?
>
> Ramesh: I did run checkpatch script, and it didn't complain about the variable being not initialized. Being a variable of extern class, it is initialized at load time to default value of ZERO. This is then overridden in amdgpu_drv.c to true. So initializing it to true does not accomplish anything.
p2p_access is a local variable. Are you talking about pcie_p2p?
Checkpatch is not a static analyzer, it would not complain about
uninitialized variables. It's more of a coding style checker. It usually
complains when there is no blank line between variable declarations and
the function body. That's why I suggested initializing p2p_access with
its final value where it's defined 3 lines above, and removing the extra
assignment that violates the coding style.
Regards,
Felix
^ permalink raw reply [flat|nested] 14+ messages in thread
* RE: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-06-06 14:26 ` Felix Kuehling
@ 2022-06-06 18:11 ` Errabolu, Ramesh
0 siblings, 0 replies; 14+ messages in thread
From: Errabolu, Ramesh @ 2022-06-06 18:11 UTC (permalink / raw)
To: Kuehling, Felix, amd-gfx list
[AMD Official Use Only - General]
Just posted updated patch addressing the comment
-----Original Message-----
From: Kuehling, Felix <Felix.Kuehling@amd.com>
Sent: Monday, June 6, 2022 7:57 PM
To: Errabolu, Ramesh <Ramesh.Errabolu@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org>
Subject: Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
Am 2022-06-04 um 06:23 schrieb Errabolu, Ramesh:
>> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
>> + struct amdgpu_device *peer_adev) { #ifdef
>> +CONFIG_HSA_AMD_P2P
>> + bool p2p_access = false;
>> + uint64_t address_mask = peer_adev->dev->dma_mask ?
>> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
>> + resource_size_t aper_limit =
>> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
>> + p2p_access = !(pci_p2pdma_distance_many(adev->pdev,
>> +&peer_adev->dev, 1, true) < 0);
> This would give you a checkpatch warning. Please run checkpatch.
>
> Why can't you initialize p2p_access in the declaration above?
>
> Ramesh: I did run checkpatch script, and it didn't complain about the variable being not initialized. Being a variable of extern class, it is initialized at load time to default value of ZERO. This is then overridden in amdgpu_drv.c to true. So initializing it to true does not accomplish anything.
p2p_access is a local variable. Are you talking about pcie_p2p?
Checkpatch is not a static analyzer, it would not complain about uninitialized variables. It's more of a coding style checker. It usually complains when there is no blank line between variable declarations and the function body. That's why I suggested initializing p2p_access with its final value where it's defined 3 lines above, and removing the extra assignment that violates the coding style.
Ramesh: I was misunderstanding your comment as being for pcie_p2p module parameter
Regards,
Felix
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-06-07 8:23 Ramesh Errabolu
@ 2022-06-07 16:10 ` Felix Kuehling
0 siblings, 0 replies; 14+ messages in thread
From: Felix Kuehling @ 2022-06-07 16:10 UTC (permalink / raw)
To: Ramesh Errabolu, amd-gfx
Am 2022-06-07 um 04:23 schrieb Ramesh Errabolu:
> Add support for peer-to-peer communication among AMD GPUs over PCIe
> bus. Support REQUIRES enablement of config HSA_AMD_P2P.
>
> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 300 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 31 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 10 +
> 5 files changed, 283 insertions(+), 62 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 70a2a0ffc007..d3b2245972b4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -223,6 +223,9 @@ static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS;
> static const bool __maybe_unused debug_evictions; /* = false */
> static const bool __maybe_unused no_system_mem_limit;
> #endif
> +#ifdef CONFIG_HSA_AMD_P2P
> +extern bool pcie_p2p;
> +#endif
>
> extern int amdgpu_tmz;
> extern int amdgpu_reset_method;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index f8b9f27adcf5..5c00ea1df21c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
> KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
> KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
> KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
> + KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
> };
>
> struct kfd_mem_attachment {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 054e4a76ae2e..ce012a148ed7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
> kfree(bo->kfd_bo);
> }
>
> +/**
> + * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
> + * about USERPTR or DOOREBELL or MMIO BO.
> + * @adev: Device for which dmamap BO is being created
> + * @mem: BO of peer device that is being DMA mapped. Provides parameters
> + * in building the dmamap BO
> + * @bo_out: Output parameter updated with handle of dmamap BO
> + */
> +static int
> +create_dmamap_sg_bo(struct amdgpu_device *adev,
> + struct kgd_mem *mem, struct amdgpu_bo **bo_out)
> +{
> + struct drm_gem_object *gem_obj;
> + int ret, align;
> +
> + ret = amdgpu_bo_reserve(mem->bo, false);
> + if (ret)
> + return ret;
> +
> + align = 1;
> + ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
> + AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
> + ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
> +
> + amdgpu_bo_unreserve(mem->bo);
> +
> + if (ret) {
> + pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
> + return -EINVAL;
> + }
> +
> + *bo_out = gem_to_amdgpu_bo(gem_obj);
> + (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
> + return ret;
> +}
> +
> /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
> * reservation object.
> *
> @@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
> return pte_flags;
> }
>
> +/**
> + * create_sg_table() - Create an sg_table for a contiguous DMA addr range
> + * @addr: The starting address to point to
> + * @size: Size of memory area in bytes being pointed to
> + *
> + * Allocates an instance of sg_table and initializes it to point to memory
> + * area specified by input parameters. The address used to build is assumed
> + * to be DMA mapped, if needed.
> + *
> + * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
> + * because they are physically contiguous.
> + *
> + * Return: Initialized instance of SG Table or NULL
> + */
> +static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
> +{
> + struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> +
> + if (!sg)
> + return NULL;
> + if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> + kfree(sg);
> + return NULL;
> + }
> + sg_dma_address(sg->sgl) = addr;
> + sg->sgl->length = size;
> +#ifdef CONFIG_NEED_SG_DMA_LENGTH
> + sg->sgl->dma_length = size;
> +#endif
> + return sg;
> +}
> +
> static int
> kfd_mem_dmamap_userptr(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
> return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing device
> + *
> + * An access request from the device that owns DOORBELL does not require DMA mapping.
> + * This is because the request doesn't go through PCIe root complex i.e. it instead
> + * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
> + *
> + * In contrast, all access requests for MMIO need to be DMA mapped without regard to
> + * device ownership. This is because access requests for MMIO go through PCIe root
> + * complex.
> + *
> + * This is accomplished in two steps:
> + * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
> + * in updating requesting device's page table
> + * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
> + * accessible. This allows an update of requesting device's page table
> + * with entries associated with DOOREBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * - Mapping of DOORBELL or MMIO BO of same or peer device
> + * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
> + *
> + * Return: ZERO if successful, NON-ZERO otherwise
> + */
> +static int
> +kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment)
> +{
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> + dma_addr_t dma_addr;
> + bool mmio;
> + int ret;
> +
> + /* Expect SG Table of dmapmap BO to be NULL */
> + mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
> + if (unlikely(ttm->sg)) {
> + pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
> + return -EINVAL;
> + }
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_addr = mem->bo->tbo.sg->sgl->dma_address;
> + pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
> + pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
> + dma_addr = dma_map_resource(adev->dev, dma_addr,
> + mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + ret = dma_mapping_error(adev->dev, dma_addr);
> + if (unlikely(ret))
> + return ret;
> + pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
> +
> + ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
> + if (unlikely(!ttm->sg)) {
> + ret = -ENOMEM;
> + goto unmap_sg;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
> + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> + if (unlikely(ret))
> + goto free_sg;
> +
> + return ret;
> +
> +free_sg:
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> +unmap_sg:
> + dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
> + dir, DMA_ATTR_SKIP_CPU_SYNC);
> + return ret;
> +}
> +
> static int
> kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> return kfd_mem_dmamap_userptr(mem, attachment);
> case KFD_MEM_ATT_DMABUF:
> return kfd_mem_dmamap_dmabuf(attachment);
> + case KFD_MEM_ATT_SG:
> + return kfd_mem_dmamap_sg_bo(mem, attachment);
> default:
> WARN_ON_ONCE(1);
> }
> @@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
> ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing device
> + *
> + * The method performs following steps:
> + * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
> + * - Free SG Table that is used to encapsulate DMA mapped memory of
> + * peer device's DOORBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
> + * Eviction of DOOREBELL or MMIO BO on device having access to its memory
> + *
> + * Return: void
> + */
> +static void
> +kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment)
> +{
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> +
> + if (unlikely(!ttm->sg)) {
> + pr_err("SG Table of BO is UNEXPECTEDLY NULL");
> + return;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
> + ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
> + ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> + bo->tbo.sg = NULL;
> +}
> +
> static void
> kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> case KFD_MEM_ATT_DMABUF:
> kfd_mem_dmaunmap_dmabuf(attachment);
> break;
> + case KFD_MEM_ATT_SG:
> + kfd_mem_dmaunmap_sg_bo(mem, attachment);
> + break;
> default:
> WARN_ON_ONCE(1);
> }
> }
>
> -static int
> -kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
> - struct amdgpu_bo **bo)
> -{
> - unsigned long bo_size = mem->bo->tbo.base.size;
> - struct drm_gem_object *gobj;
> - int ret;
> -
> - ret = amdgpu_bo_reserve(mem->bo, false);
> - if (ret)
> - return ret;
> -
> - ret = amdgpu_gem_object_create(adev, bo_size, 1,
> - AMDGPU_GEM_DOMAIN_CPU,
> - AMDGPU_GEM_CREATE_PREEMPTIBLE,
> - ttm_bo_type_sg, mem->bo->tbo.base.resv,
> - &gobj);
> - amdgpu_bo_unreserve(mem->bo);
> - if (ret)
> - return ret;
> -
> - *bo = gem_to_amdgpu_bo(gobj);
> - (*bo)->parent = amdgpu_bo_ref(mem->bo);
> -
> - return 0;
> -}
> -
> static int
> kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
> struct amdgpu_bo **bo)
> @@ -691,6 +862,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> uint64_t va = mem->va;
> struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
> struct amdgpu_bo *bo[2] = {NULL, NULL};
> + bool same_hive = false;
> int i, ret;
>
> if (!va) {
> @@ -698,6 +870,24 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> return -EINVAL;
> }
>
> + /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
> + *
> + * The access path of MMIO and DOORBELL BOs of is always over PCIe.
> + * In contrast the access path of VRAM BOs depens upon the type of
> + * link that connects the peer device. Access over PCIe is allowed
> + * if peer device has large BAR. In contrast, access over xGMI is
> + * allowed for both small and large BAR configurations of peer device
> + */
> + if ((adev != bo_adev) &&
> + ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
> + if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
> + same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
> + if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
> + return -EINVAL;
> + }
> +
> for (i = 0; i <= is_aql; i++) {
> attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
> if (unlikely(!attachment[i])) {
> @@ -708,9 +898,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
> va + bo_size, vm);
>
> - if (adev == bo_adev ||
> - (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> - (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
> + if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
> + (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> + same_hive) {
> /* Mappings on the local GPU, or VRAM mappings in the
> * local hive, or userptr mapping IOMMU direct map mode
> * share the original BO
> @@ -726,26 +916,30 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
> /* Create an SG BO to DMA-map userptrs on other GPUs */
> attachment[i]->type = KFD_MEM_ATT_USERPTR;
> - ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> - } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
> - mem->bo->tbo.type != ttm_bo_type_sg) {
> - /* GTT BOs use DMA-mapping ability of dynamic-attach
> - * DMA bufs. TODO: The same should work for VRAM on
> - * large-BAR GPUs.
> - */
> + /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
> + } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
> + WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
> + mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
> + "Handing invalid SG BO in ATTACH request");
> + attachment[i]->type = KFD_MEM_ATT_SG;
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> + if (ret)
> + goto unwind;
> + /* Enable acces to GTT and VRAM BOs of peer devices */
> + } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
> + mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
> attachment[i]->type = KFD_MEM_ATT_DMABUF;
> ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> + pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
> } else {
> - /* FIXME: Need to DMA-map other BO types:
> - * large-BAR VRAM, doorbells, MMIO remap
> - */
> - attachment[i]->type = KFD_MEM_ATT_SHARED;
> - bo[i] = mem->bo;
> - drm_gem_object_get(&bo[i]->tbo.base);
> + WARN_ONCE(true, "Handling invalid ATTACH request");
> + ret = -EINVAL;
> + goto unwind;
> }
>
> /* Add BO to VM internal data structures */
> @@ -1146,24 +1340,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
> return ret;
> }
>
> -static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
> -{
> - struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> -
> - if (!sg)
> - return NULL;
> - if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> - kfree(sg);
> - return NULL;
> - }
> - sg->sgl->dma_address = addr;
> - sg->sgl->length = size;
> -#ifdef CONFIG_NEED_SG_DMA_LENGTH
> - sg->sgl->dma_length = size;
> -#endif
> - return sg;
> -}
> -
> static int process_validate_vms(struct amdkfd_process_info *process_info)
> {
> struct amdgpu_vm *peer_vm;
> @@ -1532,7 +1708,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
> bo_type = ttm_bo_type_sg;
> if (size > UINT_MAX)
> return -EINVAL;
> - sg = create_doorbell_sg(*offset, size);
> + sg = create_sg_table(*offset, size);
> if (!sg)
> return -ENOMEM;
> } else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index be0baacc5942..803c7a8345da 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -34,6 +34,7 @@
> #include <linux/pci.h>
> #include <linux/devcoredump.h>
> #include <generated/utsrelease.h>
> +#include <linux/pci-p2pdma.h>
>
> #include <drm/drm_atomic_helper.h>
> #include <drm/drm_probe_helper.h>
> @@ -5498,6 +5499,36 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
> }
> }
>
> +/**
> + * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
> + *
> + * @adev: amdgpu_device pointer
> + * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
> + *
> + * Return true if @peer_adev can access (DMA) @adev through the PCIe
> + * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
> + * @peer_adev.
> + */
> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev)
> +{
> +#ifdef CONFIG_HSA_AMD_P2P
> + uint64_t address_mask = peer_adev->dev->dma_mask ?
> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
> + resource_size_t aper_limit =
> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
> + bool p2p_access = !(pci_p2pdma_distance_many(adev->pdev,
> + &peer_adev->dev, 1, true) < 0);
> +
> + return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
> + adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
> + !(adev->gmc.aper_base & address_mask ||
> + aper_limit & address_mask));
> +#else
> + return false;
> +#endif
> +}
> +
> int amdgpu_device_baco_enter(struct drm_device *dev)
> {
> struct amdgpu_device *adev = drm_to_adev(dev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index bed4ed88951f..1cc9260e75de 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -802,6 +802,16 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
> module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
> #endif
>
> +/**
> + * DOC: pcie_p2p (bool)
> + * Enable PCIe P2P (requires large-BAR). Default value: true (on)
> + */
> +#ifdef CONFIG_HSA_AMD_P2P
> +bool pcie_p2p = true;
> +module_param(pcie_p2p, bool, 0444);
> +MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
> +#endif
> +
> /**
> * DOC: dcfeaturemask (uint)
> * Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
@ 2022-06-07 8:23 Ramesh Errabolu
2022-06-07 16:10 ` Felix Kuehling
0 siblings, 1 reply; 14+ messages in thread
From: Ramesh Errabolu @ 2022-06-07 8:23 UTC (permalink / raw)
To: amd-gfx; +Cc: Ramesh Errabolu
Add support for peer-to-peer communication among AMD GPUs over PCIe
bus. Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu.h | 3 +
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 300 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 31 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 10 +
5 files changed, 283 insertions(+), 62 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 70a2a0ffc007..d3b2245972b4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -223,6 +223,9 @@ static const int __maybe_unused sched_policy = KFD_SCHED_POLICY_HWS;
static const bool __maybe_unused debug_evictions; /* = false */
static const bool __maybe_unused no_system_mem_limit;
#endif
+#ifdef CONFIG_HSA_AMD_P2P
+extern bool pcie_p2p;
+#endif
extern int amdgpu_tmz;
extern int amdgpu_reset_method;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 054e4a76ae2e..ce012a148ed7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+ struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it instead
+ * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without regard to
+ * device ownership. This is because access requests for MMIO go through PCIe root
+ * complex.
+ *
+ * This is accomplished in two steps:
+ * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
+ * in updating requesting device's page table
+ * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
+ * accessible. This allows an update of requesting device's page table
+ * with entries associated with DOOREBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * - Mapping of DOORBELL or MMIO BO of same or peer device
+ * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
+ *
+ * Return: ZERO if successful, NON-ZERO otherwise
+ */
+static int
+kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+ dma_addr_t dma_addr;
+ bool mmio;
+ int ret;
+
+ /* Expect SG Table of dmapmap BO to be NULL */
+ mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
+ if (unlikely(ttm->sg)) {
+ pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
+ return -EINVAL;
+ }
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_addr = mem->bo->tbo.sg->sgl->dma_address;
+ pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
+ pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
+ dma_addr = dma_map_resource(adev->dev, dma_addr,
+ mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ ret = dma_mapping_error(adev->dev, dma_addr);
+ if (unlikely(ret))
+ return ret;
+ pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
+
+ ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
+ if (unlikely(!ttm->sg)) {
+ ret = -ENOMEM;
+ goto unmap_sg;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+ ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+ if (unlikely(ret))
+ goto free_sg;
+
+ return ret;
+
+free_sg:
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+unmap_sg:
+ dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
+ dir, DMA_ATTR_SKIP_CPU_SYNC);
+ return ret;
+}
+
static int
kfd_mem_dmamap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
return kfd_mem_dmamap_userptr(mem, attachment);
case KFD_MEM_ATT_DMABUF:
return kfd_mem_dmamap_dmabuf(attachment);
+ case KFD_MEM_ATT_SG:
+ return kfd_mem_dmamap_sg_bo(mem, attachment);
default:
WARN_ON_ONCE(1);
}
@@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * The method performs following steps:
+ * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
+ * - Free SG Table that is used to encapsulate DMA mapped memory of
+ * peer device's DOORBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
+ * Eviction of DOOREBELL or MMIO BO on device having access to its memory
+ *
+ * Return: void
+ */
+static void
+kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+
+ if (unlikely(!ttm->sg)) {
+ pr_err("SG Table of BO is UNEXPECTEDLY NULL");
+ return;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+ ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
+ ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+ bo->tbo.sg = NULL;
+}
+
static void
kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
case KFD_MEM_ATT_DMABUF:
kfd_mem_dmaunmap_dmabuf(attachment);
break;
+ case KFD_MEM_ATT_SG:
+ kfd_mem_dmaunmap_sg_bo(mem, attachment);
+ break;
default:
WARN_ON_ONCE(1);
}
}
-static int
-kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
- struct amdgpu_bo **bo)
-{
- unsigned long bo_size = mem->bo->tbo.base.size;
- struct drm_gem_object *gobj;
- int ret;
-
- ret = amdgpu_bo_reserve(mem->bo, false);
- if (ret)
- return ret;
-
- ret = amdgpu_gem_object_create(adev, bo_size, 1,
- AMDGPU_GEM_DOMAIN_CPU,
- AMDGPU_GEM_CREATE_PREEMPTIBLE,
- ttm_bo_type_sg, mem->bo->tbo.base.resv,
- &gobj);
- amdgpu_bo_unreserve(mem->bo);
- if (ret)
- return ret;
-
- *bo = gem_to_amdgpu_bo(gobj);
- (*bo)->parent = amdgpu_bo_ref(mem->bo);
-
- return 0;
-}
-
static int
kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
struct amdgpu_bo **bo)
@@ -691,6 +862,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
uint64_t va = mem->va;
struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
struct amdgpu_bo *bo[2] = {NULL, NULL};
+ bool same_hive = false;
int i, ret;
if (!va) {
@@ -698,6 +870,24 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
return -EINVAL;
}
+ /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
+ *
+ * The access path of MMIO and DOORBELL BOs of is always over PCIe.
+ * In contrast the access path of VRAM BOs depens upon the type of
+ * link that connects the peer device. Access over PCIe is allowed
+ * if peer device has large BAR. In contrast, access over xGMI is
+ * allowed for both small and large BAR configurations of peer device
+ */
+ if ((adev != bo_adev) &&
+ ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
+ if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
+ same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
+ if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
+ return -EINVAL;
+ }
+
for (i = 0; i <= is_aql; i++) {
attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
if (unlikely(!attachment[i])) {
@@ -708,9 +898,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
va + bo_size, vm);
- if (adev == bo_adev ||
- (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
- (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
+ if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
+ (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
+ same_hive) {
/* Mappings on the local GPU, or VRAM mappings in the
* local hive, or userptr mapping IOMMU direct map mode
* share the original BO
@@ -726,26 +916,30 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
} else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
/* Create an SG BO to DMA-map userptrs on other GPUs */
attachment[i]->type = KFD_MEM_ATT_USERPTR;
- ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
if (ret)
goto unwind;
- } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
- mem->bo->tbo.type != ttm_bo_type_sg) {
- /* GTT BOs use DMA-mapping ability of dynamic-attach
- * DMA bufs. TODO: The same should work for VRAM on
- * large-BAR GPUs.
- */
+ /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
+ } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
+ WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
+ mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
+ "Handing invalid SG BO in ATTACH request");
+ attachment[i]->type = KFD_MEM_ATT_SG;
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
+ if (ret)
+ goto unwind;
+ /* Enable acces to GTT and VRAM BOs of peer devices */
+ } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
+ mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
attachment[i]->type = KFD_MEM_ATT_DMABUF;
ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
if (ret)
goto unwind;
+ pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
} else {
- /* FIXME: Need to DMA-map other BO types:
- * large-BAR VRAM, doorbells, MMIO remap
- */
- attachment[i]->type = KFD_MEM_ATT_SHARED;
- bo[i] = mem->bo;
- drm_gem_object_get(&bo[i]->tbo.base);
+ WARN_ONCE(true, "Handling invalid ATTACH request");
+ ret = -EINVAL;
+ goto unwind;
}
/* Add BO to VM internal data structures */
@@ -1146,24 +1340,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
return ret;
}
-static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
-{
- struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
-
- if (!sg)
- return NULL;
- if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
- kfree(sg);
- return NULL;
- }
- sg->sgl->dma_address = addr;
- sg->sgl->length = size;
-#ifdef CONFIG_NEED_SG_DMA_LENGTH
- sg->sgl->dma_length = size;
-#endif
- return sg;
-}
-
static int process_validate_vms(struct amdkfd_process_info *process_info)
{
struct amdgpu_vm *peer_vm;
@@ -1532,7 +1708,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
bo_type = ttm_bo_type_sg;
if (size > UINT_MAX)
return -EINVAL;
- sg = create_doorbell_sg(*offset, size);
+ sg = create_sg_table(*offset, size);
if (!sg)
return -ENOMEM;
} else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index be0baacc5942..803c7a8345da 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -34,6 +34,7 @@
#include <linux/pci.h>
#include <linux/devcoredump.h>
#include <generated/utsrelease.h>
+#include <linux/pci-p2pdma.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_probe_helper.h>
@@ -5498,6 +5499,36 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
}
}
+/**
+ * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
+ *
+ * @adev: amdgpu_device pointer
+ * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
+ *
+ * Return true if @peer_adev can access (DMA) @adev through the PCIe
+ * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
+ * @peer_adev.
+ */
+bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev)
+{
+#ifdef CONFIG_HSA_AMD_P2P
+ uint64_t address_mask = peer_adev->dev->dma_mask ?
+ ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
+ resource_size_t aper_limit =
+ adev->gmc.aper_base + adev->gmc.aper_size - 1;
+ bool p2p_access = !(pci_p2pdma_distance_many(adev->pdev,
+ &peer_adev->dev, 1, true) < 0);
+
+ return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
+ adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
+ !(adev->gmc.aper_base & address_mask ||
+ aper_limit & address_mask));
+#else
+ return false;
+#endif
+}
+
int amdgpu_device_baco_enter(struct drm_device *dev)
{
struct amdgpu_device *adev = drm_to_adev(dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index bed4ed88951f..1cc9260e75de 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -802,6 +802,16 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
#endif
+/**
+ * DOC: pcie_p2p (bool)
+ * Enable PCIe P2P (requires large-BAR). Default value: true (on)
+ */
+#ifdef CONFIG_HSA_AMD_P2P
+bool pcie_p2p = true;
+module_param(pcie_p2p, bool, 0444);
+MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
+#endif
+
/**
* DOC: dcfeaturemask (uint)
* Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
--
2.35.1
^ permalink raw reply related [flat|nested] 14+ messages in thread
* Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-06-06 20:04 ` Felix Kuehling
@ 2022-06-06 20:22 ` Felix Kuehling
0 siblings, 0 replies; 14+ messages in thread
From: Felix Kuehling @ 2022-06-06 20:22 UTC (permalink / raw)
To: Ramesh Errabolu, amd-gfx
Am 2022-06-06 um 16:04 schrieb Felix Kuehling:
> Am 2022-06-06 um 14:07 schrieb Ramesh Errabolu:
>> Add support for peer-to-peer communication among AMD GPUs over PCIe
>> bus. Support REQUIRES enablement of config HSA_AMD_P2P.
>>
>> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
>
> Sorry, one more nit-pick inline. With that fixed, the patch is
>
> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
Really inline now ...
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index be0baacc5942..8e88e7c88d26 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -34,6 +34,7 @@
>> #include <linux/pci.h>
>> #include <linux/devcoredump.h>
>> #include <generated/utsrelease.h>
>> +#include <linux/pci-p2pdma.h>
>> #include <drm/drm_atomic_helper.h>
>> #include <drm/drm_probe_helper.h>
>> @@ -129,6 +130,8 @@ const char *amdgpu_asic_name[] = {
>> "LAST",
>> };
>> +extern bool pcie_p2p;
>> +
This should be declared in amdgpu.h along with other module parameters.
>> /**
>> * DOC: pcie_replay_count
>> *
>> @@ -5498,6 +5501,36 @@ static void amdgpu_device_get_pcie_info(struct
>> amdgpu_device *adev)
>> }
>> }
>> +/**
>> + * amdgpu_device_is_peer_accessible - Check peer access through PCIe
>> BAR
>> + *
>> + * @adev: amdgpu_device pointer
>> + * @peer_adev: amdgpu_device pointer for peer device trying to
>> access @adev
>> + *
>> + * Return true if @peer_adev can access (DMA) @adev through the PCIe
>> + * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
>> + * @peer_adev.
>> + */
>> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
>> + struct amdgpu_device *peer_adev)
>> +{
>> +#ifdef CONFIG_HSA_AMD_P2P
>> + uint64_t address_mask = peer_adev->dev->dma_mask ?
>> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
>> + resource_size_t aper_limit =
>> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
>> + bool p2p_access = !(pci_p2pdma_distance_many(adev->pdev,
>> + &peer_adev->dev, 1, true) < 0);
>> +
>> + return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
>> + adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
>> + !(adev->gmc.aper_base & address_mask ||
>> + aper_limit & address_mask));
>> +#else
>> + return false;
>> +#endif
>> +}
>> +
>> int amdgpu_device_baco_enter(struct drm_device *dev)
>> {
>> struct amdgpu_device *adev = drm_to_adev(dev);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> index bed4ed88951f..d1c82a9e8569 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
>> @@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault,
>> "No queue eviction on VM fault (
>> module_param_named(no_queue_eviction_on_vm_fault,
>> amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
>> #endif
>> +/**
>> + * DOC: pcie_p2p (bool)
>> + * Enable PCIe P2P (requires large-BAR). Default value: true (on)
>> + */
>> +bool pcie_p2p = true;
>> +module_param(pcie_p2p, bool, 0444);
>> +MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N
>> = off, Y = on(default))");
>> +
>> /**
>> * DOC: dcfeaturemask (uint)
>> * Override display features enabled. See enum DC_FEATURE_MASK in
>> drivers/gpu/drm/amd/include/amd_shared.h.
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-06-06 18:07 Ramesh Errabolu
@ 2022-06-06 20:04 ` Felix Kuehling
2022-06-06 20:22 ` Felix Kuehling
0 siblings, 1 reply; 14+ messages in thread
From: Felix Kuehling @ 2022-06-06 20:04 UTC (permalink / raw)
To: Ramesh Errabolu, amd-gfx
Am 2022-06-06 um 14:07 schrieb Ramesh Errabolu:
> Add support for peer-to-peer communication among AMD GPUs over PCIe
> bus. Support REQUIRES enablement of config HSA_AMD_P2P.
>
> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
Sorry, one more nit-pick inline. With that fixed, the patch is
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 300 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 33 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
> 4 files changed, 280 insertions(+), 62 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index f8b9f27adcf5..5c00ea1df21c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
> KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
> KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
> KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
> + KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
> };
>
> struct kfd_mem_attachment {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 054e4a76ae2e..ce012a148ed7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
> kfree(bo->kfd_bo);
> }
>
> +/**
> + * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
> + * about USERPTR or DOOREBELL or MMIO BO.
> + * @adev: Device for which dmamap BO is being created
> + * @mem: BO of peer device that is being DMA mapped. Provides parameters
> + * in building the dmamap BO
> + * @bo_out: Output parameter updated with handle of dmamap BO
> + */
> +static int
> +create_dmamap_sg_bo(struct amdgpu_device *adev,
> + struct kgd_mem *mem, struct amdgpu_bo **bo_out)
> +{
> + struct drm_gem_object *gem_obj;
> + int ret, align;
> +
> + ret = amdgpu_bo_reserve(mem->bo, false);
> + if (ret)
> + return ret;
> +
> + align = 1;
> + ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
> + AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
> + ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
> +
> + amdgpu_bo_unreserve(mem->bo);
> +
> + if (ret) {
> + pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
> + return -EINVAL;
> + }
> +
> + *bo_out = gem_to_amdgpu_bo(gem_obj);
> + (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
> + return ret;
> +}
> +
> /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
> * reservation object.
> *
> @@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
> return pte_flags;
> }
>
> +/**
> + * create_sg_table() - Create an sg_table for a contiguous DMA addr range
> + * @addr: The starting address to point to
> + * @size: Size of memory area in bytes being pointed to
> + *
> + * Allocates an instance of sg_table and initializes it to point to memory
> + * area specified by input parameters. The address used to build is assumed
> + * to be DMA mapped, if needed.
> + *
> + * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
> + * because they are physically contiguous.
> + *
> + * Return: Initialized instance of SG Table or NULL
> + */
> +static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
> +{
> + struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> +
> + if (!sg)
> + return NULL;
> + if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> + kfree(sg);
> + return NULL;
> + }
> + sg_dma_address(sg->sgl) = addr;
> + sg->sgl->length = size;
> +#ifdef CONFIG_NEED_SG_DMA_LENGTH
> + sg->sgl->dma_length = size;
> +#endif
> + return sg;
> +}
> +
> static int
> kfd_mem_dmamap_userptr(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
> return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing device
> + *
> + * An access request from the device that owns DOORBELL does not require DMA mapping.
> + * This is because the request doesn't go through PCIe root complex i.e. it instead
> + * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
> + *
> + * In contrast, all access requests for MMIO need to be DMA mapped without regard to
> + * device ownership. This is because access requests for MMIO go through PCIe root
> + * complex.
> + *
> + * This is accomplished in two steps:
> + * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
> + * in updating requesting device's page table
> + * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
> + * accessible. This allows an update of requesting device's page table
> + * with entries associated with DOOREBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * - Mapping of DOORBELL or MMIO BO of same or peer device
> + * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
> + *
> + * Return: ZERO if successful, NON-ZERO otherwise
> + */
> +static int
> +kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment)
> +{
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> + dma_addr_t dma_addr;
> + bool mmio;
> + int ret;
> +
> + /* Expect SG Table of dmapmap BO to be NULL */
> + mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
> + if (unlikely(ttm->sg)) {
> + pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
> + return -EINVAL;
> + }
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_addr = mem->bo->tbo.sg->sgl->dma_address;
> + pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
> + pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
> + dma_addr = dma_map_resource(adev->dev, dma_addr,
> + mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + ret = dma_mapping_error(adev->dev, dma_addr);
> + if (unlikely(ret))
> + return ret;
> + pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
> +
> + ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
> + if (unlikely(!ttm->sg)) {
> + ret = -ENOMEM;
> + goto unmap_sg;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
> + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> + if (unlikely(ret))
> + goto free_sg;
> +
> + return ret;
> +
> +free_sg:
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> +unmap_sg:
> + dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
> + dir, DMA_ATTR_SKIP_CPU_SYNC);
> + return ret;
> +}
> +
> static int
> kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> return kfd_mem_dmamap_userptr(mem, attachment);
> case KFD_MEM_ATT_DMABUF:
> return kfd_mem_dmamap_dmabuf(attachment);
> + case KFD_MEM_ATT_SG:
> + return kfd_mem_dmamap_sg_bo(mem, attachment);
> default:
> WARN_ON_ONCE(1);
> }
> @@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
> ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing device
> + *
> + * The method performs following steps:
> + * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
> + * - Free SG Table that is used to encapsulate DMA mapped memory of
> + * peer device's DOORBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
> + * Eviction of DOOREBELL or MMIO BO on device having access to its memory
> + *
> + * Return: void
> + */
> +static void
> +kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment)
> +{
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> +
> + if (unlikely(!ttm->sg)) {
> + pr_err("SG Table of BO is UNEXPECTEDLY NULL");
> + return;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
> + ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
> + ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> + bo->tbo.sg = NULL;
> +}
> +
> static void
> kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> case KFD_MEM_ATT_DMABUF:
> kfd_mem_dmaunmap_dmabuf(attachment);
> break;
> + case KFD_MEM_ATT_SG:
> + kfd_mem_dmaunmap_sg_bo(mem, attachment);
> + break;
> default:
> WARN_ON_ONCE(1);
> }
> }
>
> -static int
> -kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
> - struct amdgpu_bo **bo)
> -{
> - unsigned long bo_size = mem->bo->tbo.base.size;
> - struct drm_gem_object *gobj;
> - int ret;
> -
> - ret = amdgpu_bo_reserve(mem->bo, false);
> - if (ret)
> - return ret;
> -
> - ret = amdgpu_gem_object_create(adev, bo_size, 1,
> - AMDGPU_GEM_DOMAIN_CPU,
> - AMDGPU_GEM_CREATE_PREEMPTIBLE,
> - ttm_bo_type_sg, mem->bo->tbo.base.resv,
> - &gobj);
> - amdgpu_bo_unreserve(mem->bo);
> - if (ret)
> - return ret;
> -
> - *bo = gem_to_amdgpu_bo(gobj);
> - (*bo)->parent = amdgpu_bo_ref(mem->bo);
> -
> - return 0;
> -}
> -
> static int
> kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
> struct amdgpu_bo **bo)
> @@ -691,6 +862,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> uint64_t va = mem->va;
> struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
> struct amdgpu_bo *bo[2] = {NULL, NULL};
> + bool same_hive = false;
> int i, ret;
>
> if (!va) {
> @@ -698,6 +870,24 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> return -EINVAL;
> }
>
> + /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
> + *
> + * The access path of MMIO and DOORBELL BOs of is always over PCIe.
> + * In contrast the access path of VRAM BOs depens upon the type of
> + * link that connects the peer device. Access over PCIe is allowed
> + * if peer device has large BAR. In contrast, access over xGMI is
> + * allowed for both small and large BAR configurations of peer device
> + */
> + if ((adev != bo_adev) &&
> + ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
> + if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
> + same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
> + if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
> + return -EINVAL;
> + }
> +
> for (i = 0; i <= is_aql; i++) {
> attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
> if (unlikely(!attachment[i])) {
> @@ -708,9 +898,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
> va + bo_size, vm);
>
> - if (adev == bo_adev ||
> - (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> - (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
> + if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
> + (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> + same_hive) {
> /* Mappings on the local GPU, or VRAM mappings in the
> * local hive, or userptr mapping IOMMU direct map mode
> * share the original BO
> @@ -726,26 +916,30 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
> /* Create an SG BO to DMA-map userptrs on other GPUs */
> attachment[i]->type = KFD_MEM_ATT_USERPTR;
> - ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> - } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
> - mem->bo->tbo.type != ttm_bo_type_sg) {
> - /* GTT BOs use DMA-mapping ability of dynamic-attach
> - * DMA bufs. TODO: The same should work for VRAM on
> - * large-BAR GPUs.
> - */
> + /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
> + } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
> + WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
> + mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
> + "Handing invalid SG BO in ATTACH request");
> + attachment[i]->type = KFD_MEM_ATT_SG;
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> + if (ret)
> + goto unwind;
> + /* Enable acces to GTT and VRAM BOs of peer devices */
> + } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
> + mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
> attachment[i]->type = KFD_MEM_ATT_DMABUF;
> ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> + pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
> } else {
> - /* FIXME: Need to DMA-map other BO types:
> - * large-BAR VRAM, doorbells, MMIO remap
> - */
> - attachment[i]->type = KFD_MEM_ATT_SHARED;
> - bo[i] = mem->bo;
> - drm_gem_object_get(&bo[i]->tbo.base);
> + WARN_ONCE(true, "Handling invalid ATTACH request");
> + ret = -EINVAL;
> + goto unwind;
> }
>
> /* Add BO to VM internal data structures */
> @@ -1146,24 +1340,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
> return ret;
> }
>
> -static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
> -{
> - struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> -
> - if (!sg)
> - return NULL;
> - if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> - kfree(sg);
> - return NULL;
> - }
> - sg->sgl->dma_address = addr;
> - sg->sgl->length = size;
> -#ifdef CONFIG_NEED_SG_DMA_LENGTH
> - sg->sgl->dma_length = size;
> -#endif
> - return sg;
> -}
> -
> static int process_validate_vms(struct amdkfd_process_info *process_info)
> {
> struct amdgpu_vm *peer_vm;
> @@ -1532,7 +1708,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
> bo_type = ttm_bo_type_sg;
> if (size > UINT_MAX)
> return -EINVAL;
> - sg = create_doorbell_sg(*offset, size);
> + sg = create_sg_table(*offset, size);
> if (!sg)
> return -ENOMEM;
> } else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index be0baacc5942..8e88e7c88d26 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -34,6 +34,7 @@
> #include <linux/pci.h>
> #include <linux/devcoredump.h>
> #include <generated/utsrelease.h>
> +#include <linux/pci-p2pdma.h>
>
> #include <drm/drm_atomic_helper.h>
> #include <drm/drm_probe_helper.h>
> @@ -129,6 +130,8 @@ const char *amdgpu_asic_name[] = {
> "LAST",
> };
>
> +extern bool pcie_p2p;
> +
> /**
> * DOC: pcie_replay_count
> *
> @@ -5498,6 +5501,36 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
> }
> }
>
> +/**
> + * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
> + *
> + * @adev: amdgpu_device pointer
> + * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
> + *
> + * Return true if @peer_adev can access (DMA) @adev through the PCIe
> + * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
> + * @peer_adev.
> + */
> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev)
> +{
> +#ifdef CONFIG_HSA_AMD_P2P
> + uint64_t address_mask = peer_adev->dev->dma_mask ?
> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
> + resource_size_t aper_limit =
> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
> + bool p2p_access = !(pci_p2pdma_distance_many(adev->pdev,
> + &peer_adev->dev, 1, true) < 0);
> +
> + return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
> + adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
> + !(adev->gmc.aper_base & address_mask ||
> + aper_limit & address_mask));
> +#else
> + return false;
> +#endif
> +}
> +
> int amdgpu_device_baco_enter(struct drm_device *dev)
> {
> struct amdgpu_device *adev = drm_to_adev(dev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index bed4ed88951f..d1c82a9e8569 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
> module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
> #endif
>
> +/**
> + * DOC: pcie_p2p (bool)
> + * Enable PCIe P2P (requires large-BAR). Default value: true (on)
> + */
> +bool pcie_p2p = true;
> +module_param(pcie_p2p, bool, 0444);
> +MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
> +
> /**
> * DOC: dcfeaturemask (uint)
> * Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
@ 2022-06-06 18:07 Ramesh Errabolu
2022-06-06 20:04 ` Felix Kuehling
0 siblings, 1 reply; 14+ messages in thread
From: Ramesh Errabolu @ 2022-06-06 18:07 UTC (permalink / raw)
To: amd-gfx; +Cc: Ramesh Errabolu
Add support for peer-to-peer communication among AMD GPUs over PCIe
bus. Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 300 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 33 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
4 files changed, 280 insertions(+), 62 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 054e4a76ae2e..ce012a148ed7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+ struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it instead
+ * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without regard to
+ * device ownership. This is because access requests for MMIO go through PCIe root
+ * complex.
+ *
+ * This is accomplished in two steps:
+ * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
+ * in updating requesting device's page table
+ * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
+ * accessible. This allows an update of requesting device's page table
+ * with entries associated with DOOREBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * - Mapping of DOORBELL or MMIO BO of same or peer device
+ * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
+ *
+ * Return: ZERO if successful, NON-ZERO otherwise
+ */
+static int
+kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+ dma_addr_t dma_addr;
+ bool mmio;
+ int ret;
+
+ /* Expect SG Table of dmapmap BO to be NULL */
+ mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
+ if (unlikely(ttm->sg)) {
+ pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
+ return -EINVAL;
+ }
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_addr = mem->bo->tbo.sg->sgl->dma_address;
+ pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
+ pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
+ dma_addr = dma_map_resource(adev->dev, dma_addr,
+ mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ ret = dma_mapping_error(adev->dev, dma_addr);
+ if (unlikely(ret))
+ return ret;
+ pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
+
+ ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
+ if (unlikely(!ttm->sg)) {
+ ret = -ENOMEM;
+ goto unmap_sg;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+ ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+ if (unlikely(ret))
+ goto free_sg;
+
+ return ret;
+
+free_sg:
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+unmap_sg:
+ dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
+ dir, DMA_ATTR_SKIP_CPU_SYNC);
+ return ret;
+}
+
static int
kfd_mem_dmamap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
return kfd_mem_dmamap_userptr(mem, attachment);
case KFD_MEM_ATT_DMABUF:
return kfd_mem_dmamap_dmabuf(attachment);
+ case KFD_MEM_ATT_SG:
+ return kfd_mem_dmamap_sg_bo(mem, attachment);
default:
WARN_ON_ONCE(1);
}
@@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * The method performs following steps:
+ * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
+ * - Free SG Table that is used to encapsulate DMA mapped memory of
+ * peer device's DOORBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
+ * Eviction of DOOREBELL or MMIO BO on device having access to its memory
+ *
+ * Return: void
+ */
+static void
+kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+
+ if (unlikely(!ttm->sg)) {
+ pr_err("SG Table of BO is UNEXPECTEDLY NULL");
+ return;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+ ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
+ ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+ bo->tbo.sg = NULL;
+}
+
static void
kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
case KFD_MEM_ATT_DMABUF:
kfd_mem_dmaunmap_dmabuf(attachment);
break;
+ case KFD_MEM_ATT_SG:
+ kfd_mem_dmaunmap_sg_bo(mem, attachment);
+ break;
default:
WARN_ON_ONCE(1);
}
}
-static int
-kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
- struct amdgpu_bo **bo)
-{
- unsigned long bo_size = mem->bo->tbo.base.size;
- struct drm_gem_object *gobj;
- int ret;
-
- ret = amdgpu_bo_reserve(mem->bo, false);
- if (ret)
- return ret;
-
- ret = amdgpu_gem_object_create(adev, bo_size, 1,
- AMDGPU_GEM_DOMAIN_CPU,
- AMDGPU_GEM_CREATE_PREEMPTIBLE,
- ttm_bo_type_sg, mem->bo->tbo.base.resv,
- &gobj);
- amdgpu_bo_unreserve(mem->bo);
- if (ret)
- return ret;
-
- *bo = gem_to_amdgpu_bo(gobj);
- (*bo)->parent = amdgpu_bo_ref(mem->bo);
-
- return 0;
-}
-
static int
kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
struct amdgpu_bo **bo)
@@ -691,6 +862,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
uint64_t va = mem->va;
struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
struct amdgpu_bo *bo[2] = {NULL, NULL};
+ bool same_hive = false;
int i, ret;
if (!va) {
@@ -698,6 +870,24 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
return -EINVAL;
}
+ /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
+ *
+ * The access path of MMIO and DOORBELL BOs of is always over PCIe.
+ * In contrast the access path of VRAM BOs depens upon the type of
+ * link that connects the peer device. Access over PCIe is allowed
+ * if peer device has large BAR. In contrast, access over xGMI is
+ * allowed for both small and large BAR configurations of peer device
+ */
+ if ((adev != bo_adev) &&
+ ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
+ if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
+ same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
+ if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
+ return -EINVAL;
+ }
+
for (i = 0; i <= is_aql; i++) {
attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
if (unlikely(!attachment[i])) {
@@ -708,9 +898,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
va + bo_size, vm);
- if (adev == bo_adev ||
- (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
- (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
+ if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
+ (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
+ same_hive) {
/* Mappings on the local GPU, or VRAM mappings in the
* local hive, or userptr mapping IOMMU direct map mode
* share the original BO
@@ -726,26 +916,30 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
} else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
/* Create an SG BO to DMA-map userptrs on other GPUs */
attachment[i]->type = KFD_MEM_ATT_USERPTR;
- ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
if (ret)
goto unwind;
- } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
- mem->bo->tbo.type != ttm_bo_type_sg) {
- /* GTT BOs use DMA-mapping ability of dynamic-attach
- * DMA bufs. TODO: The same should work for VRAM on
- * large-BAR GPUs.
- */
+ /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
+ } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
+ WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
+ mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
+ "Handing invalid SG BO in ATTACH request");
+ attachment[i]->type = KFD_MEM_ATT_SG;
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
+ if (ret)
+ goto unwind;
+ /* Enable acces to GTT and VRAM BOs of peer devices */
+ } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
+ mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
attachment[i]->type = KFD_MEM_ATT_DMABUF;
ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
if (ret)
goto unwind;
+ pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
} else {
- /* FIXME: Need to DMA-map other BO types:
- * large-BAR VRAM, doorbells, MMIO remap
- */
- attachment[i]->type = KFD_MEM_ATT_SHARED;
- bo[i] = mem->bo;
- drm_gem_object_get(&bo[i]->tbo.base);
+ WARN_ONCE(true, "Handling invalid ATTACH request");
+ ret = -EINVAL;
+ goto unwind;
}
/* Add BO to VM internal data structures */
@@ -1146,24 +1340,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
return ret;
}
-static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
-{
- struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
-
- if (!sg)
- return NULL;
- if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
- kfree(sg);
- return NULL;
- }
- sg->sgl->dma_address = addr;
- sg->sgl->length = size;
-#ifdef CONFIG_NEED_SG_DMA_LENGTH
- sg->sgl->dma_length = size;
-#endif
- return sg;
-}
-
static int process_validate_vms(struct amdkfd_process_info *process_info)
{
struct amdgpu_vm *peer_vm;
@@ -1532,7 +1708,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
bo_type = ttm_bo_type_sg;
if (size > UINT_MAX)
return -EINVAL;
- sg = create_doorbell_sg(*offset, size);
+ sg = create_sg_table(*offset, size);
if (!sg)
return -ENOMEM;
} else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index be0baacc5942..8e88e7c88d26 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -34,6 +34,7 @@
#include <linux/pci.h>
#include <linux/devcoredump.h>
#include <generated/utsrelease.h>
+#include <linux/pci-p2pdma.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_probe_helper.h>
@@ -129,6 +130,8 @@ const char *amdgpu_asic_name[] = {
"LAST",
};
+extern bool pcie_p2p;
+
/**
* DOC: pcie_replay_count
*
@@ -5498,6 +5501,36 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
}
}
+/**
+ * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
+ *
+ * @adev: amdgpu_device pointer
+ * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
+ *
+ * Return true if @peer_adev can access (DMA) @adev through the PCIe
+ * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
+ * @peer_adev.
+ */
+bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev)
+{
+#ifdef CONFIG_HSA_AMD_P2P
+ uint64_t address_mask = peer_adev->dev->dma_mask ?
+ ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
+ resource_size_t aper_limit =
+ adev->gmc.aper_base + adev->gmc.aper_size - 1;
+ bool p2p_access = !(pci_p2pdma_distance_many(adev->pdev,
+ &peer_adev->dev, 1, true) < 0);
+
+ return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
+ adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
+ !(adev->gmc.aper_base & address_mask ||
+ aper_limit & address_mask));
+#else
+ return false;
+#endif
+}
+
int amdgpu_device_baco_enter(struct drm_device *dev)
{
struct amdgpu_device *adev = drm_to_adev(dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index bed4ed88951f..d1c82a9e8569 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
#endif
+/**
+ * DOC: pcie_p2p (bool)
+ * Enable PCIe P2P (requires large-BAR). Default value: true (on)
+ */
+bool pcie_p2p = true;
+module_param(pcie_p2p, bool, 0444);
+MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
+
/**
* DOC: dcfeaturemask (uint)
* Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
--
2.35.1
^ permalink raw reply related [flat|nested] 14+ messages in thread
* [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
@ 2022-06-04 10:23 Ramesh Errabolu
0 siblings, 0 replies; 14+ messages in thread
From: Ramesh Errabolu @ 2022-06-04 10:23 UTC (permalink / raw)
To: amd-gfx; +Cc: Ramesh Errabolu
Add support for peer-to-peer communication among AMD GPUs over PCIe
bus. Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 300 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 33 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
4 files changed, 280 insertions(+), 62 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 054e4a76ae2e..ce012a148ed7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+ struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it instead
+ * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without regard to
+ * device ownership. This is because access requests for MMIO go through PCIe root
+ * complex.
+ *
+ * This is accomplished in two steps:
+ * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
+ * in updating requesting device's page table
+ * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
+ * accessible. This allows an update of requesting device's page table
+ * with entries associated with DOOREBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * - Mapping of DOORBELL or MMIO BO of same or peer device
+ * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
+ *
+ * Return: ZERO if successful, NON-ZERO otherwise
+ */
+static int
+kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+ dma_addr_t dma_addr;
+ bool mmio;
+ int ret;
+
+ /* Expect SG Table of dmapmap BO to be NULL */
+ mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
+ if (unlikely(ttm->sg)) {
+ pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
+ return -EINVAL;
+ }
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_addr = mem->bo->tbo.sg->sgl->dma_address;
+ pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
+ pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
+ dma_addr = dma_map_resource(adev->dev, dma_addr,
+ mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ ret = dma_mapping_error(adev->dev, dma_addr);
+ if (unlikely(ret))
+ return ret;
+ pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
+
+ ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
+ if (unlikely(!ttm->sg)) {
+ ret = -ENOMEM;
+ goto unmap_sg;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+ ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+ if (unlikely(ret))
+ goto free_sg;
+
+ return ret;
+
+free_sg:
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+unmap_sg:
+ dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
+ dir, DMA_ATTR_SKIP_CPU_SYNC);
+ return ret;
+}
+
static int
kfd_mem_dmamap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
return kfd_mem_dmamap_userptr(mem, attachment);
case KFD_MEM_ATT_DMABUF:
return kfd_mem_dmamap_dmabuf(attachment);
+ case KFD_MEM_ATT_SG:
+ return kfd_mem_dmamap_sg_bo(mem, attachment);
default:
WARN_ON_ONCE(1);
}
@@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * The method performs following steps:
+ * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
+ * - Free SG Table that is used to encapsulate DMA mapped memory of
+ * peer device's DOORBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
+ * Eviction of DOOREBELL or MMIO BO on device having access to its memory
+ *
+ * Return: void
+ */
+static void
+kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+
+ if (unlikely(!ttm->sg)) {
+ pr_err("SG Table of BO is UNEXPECTEDLY NULL");
+ return;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+ ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
+ ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+ bo->tbo.sg = NULL;
+}
+
static void
kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
case KFD_MEM_ATT_DMABUF:
kfd_mem_dmaunmap_dmabuf(attachment);
break;
+ case KFD_MEM_ATT_SG:
+ kfd_mem_dmaunmap_sg_bo(mem, attachment);
+ break;
default:
WARN_ON_ONCE(1);
}
}
-static int
-kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
- struct amdgpu_bo **bo)
-{
- unsigned long bo_size = mem->bo->tbo.base.size;
- struct drm_gem_object *gobj;
- int ret;
-
- ret = amdgpu_bo_reserve(mem->bo, false);
- if (ret)
- return ret;
-
- ret = amdgpu_gem_object_create(adev, bo_size, 1,
- AMDGPU_GEM_DOMAIN_CPU,
- AMDGPU_GEM_CREATE_PREEMPTIBLE,
- ttm_bo_type_sg, mem->bo->tbo.base.resv,
- &gobj);
- amdgpu_bo_unreserve(mem->bo);
- if (ret)
- return ret;
-
- *bo = gem_to_amdgpu_bo(gobj);
- (*bo)->parent = amdgpu_bo_ref(mem->bo);
-
- return 0;
-}
-
static int
kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
struct amdgpu_bo **bo)
@@ -691,6 +862,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
uint64_t va = mem->va;
struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
struct amdgpu_bo *bo[2] = {NULL, NULL};
+ bool same_hive = false;
int i, ret;
if (!va) {
@@ -698,6 +870,24 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
return -EINVAL;
}
+ /* Determine access to VRAM, MMIO and DOORBELL BOs of peer devices
+ *
+ * The access path of MMIO and DOORBELL BOs of is always over PCIe.
+ * In contrast the access path of VRAM BOs depens upon the type of
+ * link that connects the peer device. Access over PCIe is allowed
+ * if peer device has large BAR. In contrast, access over xGMI is
+ * allowed for both small and large BAR configurations of peer device
+ */
+ if ((adev != bo_adev) &&
+ ((mem->domain == AMDGPU_GEM_DOMAIN_VRAM) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
+ if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM)
+ same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
+ if (!same_hive && !amdgpu_device_is_peer_accessible(bo_adev, adev))
+ return -EINVAL;
+ }
+
for (i = 0; i <= is_aql; i++) {
attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
if (unlikely(!attachment[i])) {
@@ -708,9 +898,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
va + bo_size, vm);
- if (adev == bo_adev ||
- (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
- (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
+ if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
+ (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
+ same_hive) {
/* Mappings on the local GPU, or VRAM mappings in the
* local hive, or userptr mapping IOMMU direct map mode
* share the original BO
@@ -726,26 +916,30 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
} else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
/* Create an SG BO to DMA-map userptrs on other GPUs */
attachment[i]->type = KFD_MEM_ATT_USERPTR;
- ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
if (ret)
goto unwind;
- } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
- mem->bo->tbo.type != ttm_bo_type_sg) {
- /* GTT BOs use DMA-mapping ability of dynamic-attach
- * DMA bufs. TODO: The same should work for VRAM on
- * large-BAR GPUs.
- */
+ /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
+ } else if (mem->bo->tbo.type == ttm_bo_type_sg) {
+ WARN_ONCE(!(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL ||
+ mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP),
+ "Handing invalid SG BO in ATTACH request");
+ attachment[i]->type = KFD_MEM_ATT_SG;
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
+ if (ret)
+ goto unwind;
+ /* Enable acces to GTT and VRAM BOs of peer devices */
+ } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT ||
+ mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
attachment[i]->type = KFD_MEM_ATT_DMABUF;
ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
if (ret)
goto unwind;
+ pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
} else {
- /* FIXME: Need to DMA-map other BO types:
- * large-BAR VRAM, doorbells, MMIO remap
- */
- attachment[i]->type = KFD_MEM_ATT_SHARED;
- bo[i] = mem->bo;
- drm_gem_object_get(&bo[i]->tbo.base);
+ WARN_ONCE(true, "Handling invalid ATTACH request");
+ ret = -EINVAL;
+ goto unwind;
}
/* Add BO to VM internal data structures */
@@ -1146,24 +1340,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
return ret;
}
-static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
-{
- struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
-
- if (!sg)
- return NULL;
- if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
- kfree(sg);
- return NULL;
- }
- sg->sgl->dma_address = addr;
- sg->sgl->length = size;
-#ifdef CONFIG_NEED_SG_DMA_LENGTH
- sg->sgl->dma_length = size;
-#endif
- return sg;
-}
-
static int process_validate_vms(struct amdkfd_process_info *process_info)
{
struct amdgpu_vm *peer_vm;
@@ -1532,7 +1708,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
bo_type = ttm_bo_type_sg;
if (size > UINT_MAX)
return -EINVAL;
- sg = create_doorbell_sg(*offset, size);
+ sg = create_sg_table(*offset, size);
if (!sg)
return -ENOMEM;
} else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index b5ee0eb984ee..acb9e934adc9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -32,6 +32,7 @@
#include <linux/slab.h>
#include <linux/iommu.h>
#include <linux/pci.h>
+#include <linux/pci-p2pdma.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_probe_helper.h>
@@ -127,6 +128,8 @@ const char *amdgpu_asic_name[] = {
"LAST",
};
+extern bool pcie_p2p;
+
/**
* DOC: pcie_replay_count
*
@@ -5434,6 +5437,36 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
}
}
+/**
+ * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
+ *
+ * @adev: amdgpu_device pointer
+ * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
+ *
+ * Return true if @peer_adev can access (DMA) @adev through the PCIe
+ * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
+ * @peer_adev.
+ */
+bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev)
+{
+#ifdef CONFIG_HSA_AMD_P2P
+ bool p2p_access = false;
+ uint64_t address_mask = peer_adev->dev->dma_mask ?
+ ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
+ resource_size_t aper_limit =
+ adev->gmc.aper_base + adev->gmc.aper_size - 1;
+ p2p_access = !(pci_p2pdma_distance_many(adev->pdev, &peer_adev->dev, 1, true) < 0);
+
+ return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
+ adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
+ !(adev->gmc.aper_base & address_mask ||
+ aper_limit & address_mask));
+#else
+ return false;
+#endif
+}
+
int amdgpu_device_baco_enter(struct drm_device *dev)
{
struct amdgpu_device *adev = drm_to_adev(dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index bed4ed88951f..d1c82a9e8569 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
#endif
+/**
+ * DOC: pcie_p2p (bool)
+ * Enable PCIe P2P (requires large-BAR). Default value: true (on)
+ */
+bool pcie_p2p = true;
+module_param(pcie_p2p, bool, 0444);
+MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
+
/**
* DOC: dcfeaturemask (uint)
* Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
--
2.35.1
^ permalink raw reply related [flat|nested] 14+ messages in thread
* RE: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-06-01 16:12 ` Felix Kuehling
@ 2022-06-03 10:41 ` Errabolu, Ramesh
0 siblings, 0 replies; 14+ messages in thread
From: Errabolu, Ramesh @ 2022-06-03 10:41 UTC (permalink / raw)
To: Kuehling, Felix, amd-gfx
[AMD Official Use Only - General]
Applied review comments. Please find responses inline
Regards,
Ramesh
-----Original Message-----
From: Kuehling, Felix <Felix.Kuehling@amd.com>
Sent: Wednesday, June 1, 2022 9:42 PM
To: Errabolu, Ramesh <Ramesh.Errabolu@amd.com>; amd-gfx@lists.freedesktop.org
Subject: Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
Am 2022-05-31 um 13:01 schrieb Ramesh Errabolu:
> Add support for peer-to-peer communication, in both data and control
> planes, among AMD GPUs that are connected PCIe and have large BAR vBIOS.
Please don't use the "control plane", "data plane" terminology here.
This is not common usage in this context. Also the reference to large-BAR BIOSes is incorrect because BARs can be resized.
More comments inline ...
> Support REQUIRES enablement of config HSA_AMD_P2P.
>
> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 328 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 30 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
> 4 files changed, 307 insertions(+), 60 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index f8b9f27adcf5..5c00ea1df21c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
> KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
> KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
> KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
> + KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
> };
>
> struct kfd_mem_attachment {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 34ba9e776521..c2af82317a03 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
> kfree(bo->kfd_bo);
> }
>
> +/**
> + * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect
> +information
> + * about USERPTR or DOOREBELL or MMIO BO.
> + * @adev: Device for which dmamap BO is being created
> + * @mem: BO of peer device that is being DMA mapped. Provides parameters
> + * in building the dmamap BO
> + * @bo_out: Output parameter updated with handle of dmamap BO */
> +static int create_dmamap_sg_bo(struct amdgpu_device *adev,
> + struct kgd_mem *mem, struct amdgpu_bo **bo_out) {
> + struct drm_gem_object *gem_obj;
> + int ret, align;
> +
> + ret = amdgpu_bo_reserve(mem->bo, false);
> + if (ret)
> + return ret;
> +
> + align = 1;
> + ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
> + AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
> + ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
> +
> + amdgpu_bo_unreserve(mem->bo);
> +
> + if (ret) {
> + pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
> + return -EINVAL;
> + }
> +
> + *bo_out = gem_to_amdgpu_bo(gem_obj);
> + (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
> + return ret;
> +}
> +
> /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
> * reservation object.
> *
> @@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
> return pte_flags;
> }
>
> +/**
> + * create_sg_table() - Create an sg_table for a contiguous DMA addr
> +range
> + * @addr: The starting address to point to
> + * @size: Size of memory area in bytes being pointed to
> + *
> + * Allocates an instance of sg_table and initializes it to point to
> +memory
> + * area specified by input parameters. The address used to build is
> +assumed
> + * to be DMA mapped, if needed.
> + *
> + * DOORBELL or MMIO BOs use only one scatterlist node in their
> +sg_table
> + * because they are physically contiguous.
> + *
> + * Return: Initialized instance of SG Table or NULL */ static struct
> +sg_table *create_sg_table(uint64_t addr, uint32_t size) {
> + struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> +
> + if (!sg)
> + return NULL;
> + if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> + kfree(sg);
> + return NULL;
> + }
> + sg_dma_address(sg->sgl) = addr;
> + sg->sgl->length = size;
> +#ifdef CONFIG_NEED_SG_DMA_LENGTH
> + sg->sgl->dma_length = size;
> +#endif
> + return sg;
> +}
> +
> static int
> kfd_mem_dmamap_userptr(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment) @@ -545,6 +613,87 @@
> kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
> return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access
> +DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing
> +device
> + *
> + * An access request from the device that owns DOORBELL does not require DMA mapping.
> + * This is because the request doesn't go through PCIe root complex
> +i.e. it instead
> + * loops back. The need to DMA map arises only when accessing peer
> +device's DOORBELL
> + *
> + * In contrast, all access requests for MMIO need to be DMA mapped
> +without regard to
> + * device ownership. This is because access requests for MMIO go
> +through PCIe root
> + * complex.
> + *
> + * This is accomplished in two steps:
> + * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
> + * in updating requesting device's page table
> + * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
> + * accessible. This allows an update of requesting device's page table
> + * with entries associated with DOOREBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * - Mapping of DOORBELL or MMIO BO of same or peer device
> + * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
> + *
> + * Return: ZERO if successful, NON-ZERO otherwise */ static int
> +kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment) {
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> + dma_addr_t dma_addr;
> + bool mmio;
> + int ret;
> +
> + /* Expect SG Table of dmapmap BO to be NULL */
> + mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
> + if (unlikely(ttm->sg)) {
> + pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
> + return -EINVAL;
> + }
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_addr = mem->bo->tbo.sg->sgl->dma_address;
> + pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
> + pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
> + dma_addr = dma_map_resource(adev->dev, dma_addr,
> + mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + ret = dma_mapping_error(adev->dev, dma_addr);
> + if (unlikely(ret))
> + return ret;
> + pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
> +
> + ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
> + if (unlikely(!ttm->sg)) {
> + ret = -ENOMEM;
> + goto unmap_sg;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
> + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> + if (unlikely(ret))
> + goto free_sg;
> +
> + return ret;
> +
> +free_sg:
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> +unmap_sg:
> + dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
> + dir, DMA_ATTR_SKIP_CPU_SYNC);
> + return ret;
> +}
> +
> static int
> kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment) @@ -556,6 +705,8 @@
> kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> return kfd_mem_dmamap_userptr(mem, attachment);
> case KFD_MEM_ATT_DMABUF:
> return kfd_mem_dmamap_dmabuf(attachment);
> + case KFD_MEM_ATT_SG:
> + return kfd_mem_dmamap_sg_bo(mem, attachment);
> default:
> WARN_ON_ONCE(1);
> }
> @@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
> ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or
> +MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing
> +device
> + *
> + * The method performs following steps:
> + * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
> + * - Free SG Table that is used to encapsulate DMA mapped memory of
> + * peer device's DOORBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
> + * Eviction of DOOREBELL or MMIO BO on device having access to its memory
> + *
> + * Return: void
> + */
> +static void
> +kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment) {
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> +
> + if (unlikely(!ttm->sg)) {
> + pr_err("SG Table of BO is UNEXPECTEDLY NULL");
> + return;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
> + ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
> + ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> + bo->tbo.sg = NULL;
> +}
> +
> static void
> kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment) @@ -609,38 +804,14 @@
> kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> case KFD_MEM_ATT_DMABUF:
> kfd_mem_dmaunmap_dmabuf(attachment);
> break;
> + case KFD_MEM_ATT_SG:
> + kfd_mem_dmaunmap_sg_bo(mem, attachment);
> + break;
> default:
> WARN_ON_ONCE(1);
> }
> }
>
> -static int
> -kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
> - struct amdgpu_bo **bo)
> -{
> - unsigned long bo_size = mem->bo->tbo.base.size;
> - struct drm_gem_object *gobj;
> - int ret;
> -
> - ret = amdgpu_bo_reserve(mem->bo, false);
> - if (ret)
> - return ret;
> -
> - ret = amdgpu_gem_object_create(adev, bo_size, 1,
> - AMDGPU_GEM_DOMAIN_CPU,
> - AMDGPU_GEM_CREATE_PREEMPTIBLE,
> - ttm_bo_type_sg, mem->bo->tbo.base.resv,
> - &gobj);
> - amdgpu_bo_unreserve(mem->bo);
> - if (ret)
> - return ret;
> -
> - *bo = gem_to_amdgpu_bo(gobj);
> - (*bo)->parent = amdgpu_bo_ref(mem->bo);
> -
> - return 0;
> -}
> -
> static int
> kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
> struct amdgpu_bo **bo)
> @@ -670,6 +841,38 @@ kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
> return 0;
> }
>
> +/**
> + * @kfd_mem_attach_vram_bo: Acquires the handle of a VRAM BO that
> +could
> + * be used to enable a peer GPU access it
> + *
> + * Implementation determines if access to VRAM BO would employ DMABUF
> + * or Shared BO mechanism. Employ DMABUF mechanism if kernel has
> +config
> + * option HSA_AMD_P2P enabled. Employ Shared BO mechanism if above
> + * config option is not set. It is important to note that a Shared BO
> + * cannot be used to enable peer acces if system has IOMMU enabled
> + *
> + * @TODO: ADD Check to ensure IOMMU is not enabled. Should this check
> + * be somewhere as this is information could be useful in other
> +places */ static int kfd_mem_attach_vram_bo(struct amdgpu_device
> +*adev,
> + struct kgd_mem *mem, struct amdgpu_bo **bo,
> + struct kfd_mem_attachment *attachment) {
> + int ret = 0;
> +
> +#if defined(CONFIG_HSA_AMD_P2P)
> + attachment->type = KFD_MEM_ATT_DMABUF;
> + ret = kfd_mem_attach_dmabuf(adev, mem, bo);
> + pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
> +#else
> + *bo = mem->bo;
> + attachment->type = KFD_MEM_ATT_SHARED;
> + drm_gem_object_get(&(*bo)->tbo.base);
> + pr_debug("Employ Shared BO mechanim to enable peer GPU access\n");
This is something we cannot do in the upstream driver. If P2P is disabled, we have to fail any attempt to map peer memory.
Therefore I think this helper function is not needed. You can just call kfd_mem_attach_dmabuf directly in kfd_mem_attach.
Ramesh: Goo catch. Will remove the helper method as suggested.
> +#endif
> + return ret;
> +}
> +
> /* kfd_mem_attach - Add a BO to a VM
> *
> * Everything that needs to bo done only once when a BO is first added
> @@ -691,6 +894,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> uint64_t va = mem->va;
> struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
> struct amdgpu_bo *bo[2] = {NULL, NULL};
> + bool same_hive = false;
> int i, ret;
>
> if (!va) {
> @@ -698,6 +902,19 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> return -EINVAL;
> }
>
> + /* Determine if the mapping of VRAM BO to a peer device is valid
> + * It is possible that the peer device is connected via PCIe or
> + * xGMI link. Access over PCIe is allowed if device owning VRAM BO
> + * has large BAR. In contrast, access over xGMI is allowed for both
> + * small and large BAR configurations of device owning the VRAM BO
> + */
> + if (adev != bo_adev && mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
> + same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
> + if (!same_hive &&
> + !amdgpu_device_is_peer_accessible(bo_adev, adev))
> + return -EINVAL;
> + }
> +
> for (i = 0; i <= is_aql; i++) {
> attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
> if (unlikely(!attachment[i])) {
> @@ -708,9 +925,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
> va + bo_size, vm);
>
> - if (adev == bo_adev ||
> - (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> - (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
> + if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
> + (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> + same_hive) {
> /* Mappings on the local GPU, or VRAM mappings in the
> * local hive, or userptr mapping IOMMU direct map mode
> * share the original BO
> @@ -726,26 +943,35 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
> /* Create an SG BO to DMA-map userptrs on other GPUs */
> attachment[i]->type = KFD_MEM_ATT_USERPTR;
> - ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
> mem->bo->tbo.type != ttm_bo_type_sg) {
> - /* GTT BOs use DMA-mapping ability of dynamic-attach
> - * DMA bufs. TODO: The same should work for VRAM on
> - * large-BAR GPUs.
> - */
> + /* GTT BOs use DMA-mapping ability of dynamic-attach DMA bufs */
> attachment[i]->type = KFD_MEM_ATT_DMABUF;
> ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> + /* Enable acces to VRAM BOs of peer devices */
> + } else if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM &&
> + mem->bo->tbo.type == ttm_bo_type_device) {
> + ret = kfd_mem_attach_vram_bo(adev, mem,
> + &bo[i], attachment[i]);
You can just call kfd_mem_attach_dmabuf directly here. Wrap this whole
else-if block (and the following block for doorbells and MMIO) in #ifdef
CONFIG_HSA_AMD_P2P.
Ramesh: Agreed, the thread should fall into the "else" branch that prints a warning
> + if (ret)
> + goto unwind;
> + /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
> + } else if ((mem->bo->tbo.type == ttm_bo_type_sg) &&
> + ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
I think we need an amdgpu_device_is_peer_accessible check here, except
maybe for local MMIO mappings. Alternatively extend the
peer-accessibility check at the start of this function to also handle
doorbell and MMIO BOs.
Ramesh: Can't wrap this with HSA_AMD_P2P macro. The block should handle attaching local MMIO BOs. Added logic at the top of the method to determine if MMIO and DOORBELL BOs of peer devices should be allowed to attach. Return if they are not supported.
> + attachment[i]->type = KFD_MEM_ATT_SG;
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> + if (ret)
> + goto unwind;
> } else {
> - /* FIXME: Need to DMA-map other BO types:
> - * large-BAR VRAM, doorbells, MMIO remap
> - */
> - attachment[i]->type = KFD_MEM_ATT_SHARED;
> - bo[i] = mem->bo;
> - drm_gem_object_get(&bo[i]->tbo.base);
> + WARN_ONCE(true, "Handling invalid ATTACH request");
> + ret = -EINVAL;
> + goto unwind;
> }
>
> /* Add BO to VM internal data structures */
> @@ -1146,24 +1372,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
> return ret;
> }
>
> -static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
> -{
> - struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> -
> - if (!sg)
> - return NULL;
> - if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> - kfree(sg);
> - return NULL;
> - }
> - sg->sgl->dma_address = addr;
> - sg->sgl->length = size;
> -#ifdef CONFIG_NEED_SG_DMA_LENGTH
> - sg->sgl->dma_length = size;
> -#endif
> - return sg;
> -}
> -
> static int process_validate_vms(struct amdkfd_process_info *process_info)
> {
> struct amdgpu_vm *peer_vm;
> @@ -1532,7 +1740,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
> bo_type = ttm_bo_type_sg;
> if (size > UINT_MAX)
> return -EINVAL;
> - sg = create_doorbell_sg(*offset, size);
> + sg = create_sg_table(*offset, size);
> if (!sg)
> return -ENOMEM;
> } else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index f16f105a737b..3dfac07cf37c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -127,6 +127,8 @@ const char *amdgpu_asic_name[] = {
> "LAST",
> };
>
> +extern bool pcie_p2p;
> +
> /**
> * DOC: pcie_replay_count
> *
> @@ -5463,6 +5465,34 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
> }
> }
>
> +/**
> + * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
> + *
> + * @adev: amdgpu_device pointer
> + * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
> + *
> + * Return true if @peer_adev can access (DMA) @adev through the PCIe
> + * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
> + * @peer_adev.
> + */
> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev)
> +{
> +#ifdef CONFIG_HSA_AMD_P2P
I was expecting a call to pci_p2pdma_distance_many somewhere here.
Ramesh: Added a call to suggested method and use its return value to determine access.
Regards,
Felix
> + uint64_t address_mask = peer_adev->dev->dma_mask ?
> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
> + resource_size_t aper_limit =
> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
> +
> + return pcie_p2p && (adev->gmc.visible_vram_size &&
> + adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
> + !(adev->gmc.aper_base & address_mask ||
> + aper_limit & address_mask));
> +#else
> + return false;
> +#endif
> +}
> +
> int amdgpu_device_baco_enter(struct drm_device *dev)
> {
> struct amdgpu_device *adev = drm_to_adev(dev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index bed4ed88951f..d1c82a9e8569 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
> module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
> #endif
>
> +/**
> + * DOC: pcie_p2p (bool)
> + * Enable PCIe P2P (requires large-BAR). Default value: true (on)
> + */
> +bool pcie_p2p = true;
> +module_param(pcie_p2p, bool, 0444);
> +MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
> +
> /**
> * DOC: dcfeaturemask (uint)
> * Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
^ permalink raw reply [flat|nested] 14+ messages in thread
* Re: [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
2022-05-31 17:01 Ramesh Errabolu
@ 2022-06-01 16:12 ` Felix Kuehling
2022-06-03 10:41 ` Errabolu, Ramesh
0 siblings, 1 reply; 14+ messages in thread
From: Felix Kuehling @ 2022-06-01 16:12 UTC (permalink / raw)
To: Ramesh Errabolu, amd-gfx
Am 2022-05-31 um 13:01 schrieb Ramesh Errabolu:
> Add support for peer-to-peer communication, in both data and control
> planes, among AMD GPUs that are connected PCIe and have large BAR vBIOS.
Please don't use the "control plane", "data plane" terminology here.
This is not common usage in this context. Also the reference to
large-BAR BIOSes is incorrect because BARs can be resized.
More comments inline ...
> Support REQUIRES enablement of config HSA_AMD_P2P.
>
> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 328 ++++++++++++++----
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 30 ++
> drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
> 4 files changed, 307 insertions(+), 60 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index f8b9f27adcf5..5c00ea1df21c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
> KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
> KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
> KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
> + KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
> };
>
> struct kfd_mem_attachment {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 34ba9e776521..c2af82317a03 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
> kfree(bo->kfd_bo);
> }
>
> +/**
> + * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
> + * about USERPTR or DOOREBELL or MMIO BO.
> + * @adev: Device for which dmamap BO is being created
> + * @mem: BO of peer device that is being DMA mapped. Provides parameters
> + * in building the dmamap BO
> + * @bo_out: Output parameter updated with handle of dmamap BO
> + */
> +static int
> +create_dmamap_sg_bo(struct amdgpu_device *adev,
> + struct kgd_mem *mem, struct amdgpu_bo **bo_out)
> +{
> + struct drm_gem_object *gem_obj;
> + int ret, align;
> +
> + ret = amdgpu_bo_reserve(mem->bo, false);
> + if (ret)
> + return ret;
> +
> + align = 1;
> + ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
> + AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
> + ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
> +
> + amdgpu_bo_unreserve(mem->bo);
> +
> + if (ret) {
> + pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
> + return -EINVAL;
> + }
> +
> + *bo_out = gem_to_amdgpu_bo(gem_obj);
> + (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
> + return ret;
> +}
> +
> /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
> * reservation object.
> *
> @@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
> return pte_flags;
> }
>
> +/**
> + * create_sg_table() - Create an sg_table for a contiguous DMA addr range
> + * @addr: The starting address to point to
> + * @size: Size of memory area in bytes being pointed to
> + *
> + * Allocates an instance of sg_table and initializes it to point to memory
> + * area specified by input parameters. The address used to build is assumed
> + * to be DMA mapped, if needed.
> + *
> + * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
> + * because they are physically contiguous.
> + *
> + * Return: Initialized instance of SG Table or NULL
> + */
> +static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
> +{
> + struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> +
> + if (!sg)
> + return NULL;
> + if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> + kfree(sg);
> + return NULL;
> + }
> + sg_dma_address(sg->sgl) = addr;
> + sg->sgl->length = size;
> +#ifdef CONFIG_NEED_SG_DMA_LENGTH
> + sg->sgl->dma_length = size;
> +#endif
> + return sg;
> +}
> +
> static int
> kfd_mem_dmamap_userptr(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
> return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing device
> + *
> + * An access request from the device that owns DOORBELL does not require DMA mapping.
> + * This is because the request doesn't go through PCIe root complex i.e. it instead
> + * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
> + *
> + * In contrast, all access requests for MMIO need to be DMA mapped without regard to
> + * device ownership. This is because access requests for MMIO go through PCIe root
> + * complex.
> + *
> + * This is accomplished in two steps:
> + * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
> + * in updating requesting device's page table
> + * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
> + * accessible. This allows an update of requesting device's page table
> + * with entries associated with DOOREBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * - Mapping of DOORBELL or MMIO BO of same or peer device
> + * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
> + *
> + * Return: ZERO if successful, NON-ZERO otherwise
> + */
> +static int
> +kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment)
> +{
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> + dma_addr_t dma_addr;
> + bool mmio;
> + int ret;
> +
> + /* Expect SG Table of dmapmap BO to be NULL */
> + mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
> + if (unlikely(ttm->sg)) {
> + pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
> + return -EINVAL;
> + }
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_addr = mem->bo->tbo.sg->sgl->dma_address;
> + pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
> + pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
> + dma_addr = dma_map_resource(adev->dev, dma_addr,
> + mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + ret = dma_mapping_error(adev->dev, dma_addr);
> + if (unlikely(ret))
> + return ret;
> + pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
> +
> + ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
> + if (unlikely(!ttm->sg)) {
> + ret = -ENOMEM;
> + goto unmap_sg;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
> + ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> + if (unlikely(ret))
> + goto free_sg;
> +
> + return ret;
> +
> +free_sg:
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> +unmap_sg:
> + dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
> + dir, DMA_ATTR_SKIP_CPU_SYNC);
> + return ret;
> +}
> +
> static int
> kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
> return kfd_mem_dmamap_userptr(mem, attachment);
> case KFD_MEM_ATT_DMABUF:
> return kfd_mem_dmamap_dmabuf(attachment);
> + case KFD_MEM_ATT_SG:
> + return kfd_mem_dmamap_sg_bo(mem, attachment);
> default:
> WARN_ON_ONCE(1);
> }
> @@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
> ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> }
>
> +/**
> + * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
> + * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
> + * @attachment: Virtual address attachment of the BO on accessing device
> + *
> + * The method performs following steps:
> + * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
> + * - Free SG Table that is used to encapsulate DMA mapped memory of
> + * peer device's DOORBELL or MMIO memory
> + *
> + * This method is invoked in the following contexts:
> + * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
> + * Eviction of DOOREBELL or MMIO BO on device having access to its memory
> + *
> + * Return: void
> + */
> +static void
> +kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
> + struct kfd_mem_attachment *attachment)
> +{
> + struct ttm_operation_ctx ctx = {.interruptible = true};
> + struct amdgpu_bo *bo = attachment->bo_va->base.bo;
> + struct amdgpu_device *adev = attachment->adev;
> + struct ttm_tt *ttm = bo->tbo.ttm;
> + enum dma_data_direction dir;
> +
> + if (unlikely(!ttm->sg)) {
> + pr_err("SG Table of BO is UNEXPECTEDLY NULL");
> + return;
> + }
> +
> + amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
> + ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
> +
> + dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
> + DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
> + dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
> + ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
> + sg_free_table(ttm->sg);
> + kfree(ttm->sg);
> + ttm->sg = NULL;
> + bo->tbo.sg = NULL;
> +}
> +
> static void
> kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> struct kfd_mem_attachment *attachment)
> @@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
> case KFD_MEM_ATT_DMABUF:
> kfd_mem_dmaunmap_dmabuf(attachment);
> break;
> + case KFD_MEM_ATT_SG:
> + kfd_mem_dmaunmap_sg_bo(mem, attachment);
> + break;
> default:
> WARN_ON_ONCE(1);
> }
> }
>
> -static int
> -kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
> - struct amdgpu_bo **bo)
> -{
> - unsigned long bo_size = mem->bo->tbo.base.size;
> - struct drm_gem_object *gobj;
> - int ret;
> -
> - ret = amdgpu_bo_reserve(mem->bo, false);
> - if (ret)
> - return ret;
> -
> - ret = amdgpu_gem_object_create(adev, bo_size, 1,
> - AMDGPU_GEM_DOMAIN_CPU,
> - AMDGPU_GEM_CREATE_PREEMPTIBLE,
> - ttm_bo_type_sg, mem->bo->tbo.base.resv,
> - &gobj);
> - amdgpu_bo_unreserve(mem->bo);
> - if (ret)
> - return ret;
> -
> - *bo = gem_to_amdgpu_bo(gobj);
> - (*bo)->parent = amdgpu_bo_ref(mem->bo);
> -
> - return 0;
> -}
> -
> static int
> kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
> struct amdgpu_bo **bo)
> @@ -670,6 +841,38 @@ kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
> return 0;
> }
>
> +/**
> + * @kfd_mem_attach_vram_bo: Acquires the handle of a VRAM BO that could
> + * be used to enable a peer GPU access it
> + *
> + * Implementation determines if access to VRAM BO would employ DMABUF
> + * or Shared BO mechanism. Employ DMABUF mechanism if kernel has config
> + * option HSA_AMD_P2P enabled. Employ Shared BO mechanism if above
> + * config option is not set. It is important to note that a Shared BO
> + * cannot be used to enable peer acces if system has IOMMU enabled
> + *
> + * @TODO: ADD Check to ensure IOMMU is not enabled. Should this check
> + * be somewhere as this is information could be useful in other places
> + */
> +static int kfd_mem_attach_vram_bo(struct amdgpu_device *adev,
> + struct kgd_mem *mem, struct amdgpu_bo **bo,
> + struct kfd_mem_attachment *attachment)
> +{
> + int ret = 0;
> +
> +#if defined(CONFIG_HSA_AMD_P2P)
> + attachment->type = KFD_MEM_ATT_DMABUF;
> + ret = kfd_mem_attach_dmabuf(adev, mem, bo);
> + pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
> +#else
> + *bo = mem->bo;
> + attachment->type = KFD_MEM_ATT_SHARED;
> + drm_gem_object_get(&(*bo)->tbo.base);
> + pr_debug("Employ Shared BO mechanim to enable peer GPU access\n");
This is something we cannot do in the upstream driver. If P2P is
disabled, we have to fail any attempt to map peer memory.
Therefore I think this helper function is not needed. You can just call
kfd_mem_attach_dmabuf directly in kfd_mem_attach.
> +#endif
> + return ret;
> +}
> +
> /* kfd_mem_attach - Add a BO to a VM
> *
> * Everything that needs to bo done only once when a BO is first added
> @@ -691,6 +894,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> uint64_t va = mem->va;
> struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
> struct amdgpu_bo *bo[2] = {NULL, NULL};
> + bool same_hive = false;
> int i, ret;
>
> if (!va) {
> @@ -698,6 +902,19 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> return -EINVAL;
> }
>
> + /* Determine if the mapping of VRAM BO to a peer device is valid
> + * It is possible that the peer device is connected via PCIe or
> + * xGMI link. Access over PCIe is allowed if device owning VRAM BO
> + * has large BAR. In contrast, access over xGMI is allowed for both
> + * small and large BAR configurations of device owning the VRAM BO
> + */
> + if (adev != bo_adev && mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
> + same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
> + if (!same_hive &&
> + !amdgpu_device_is_peer_accessible(bo_adev, adev))
> + return -EINVAL;
> + }
> +
> for (i = 0; i <= is_aql; i++) {
> attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
> if (unlikely(!attachment[i])) {
> @@ -708,9 +925,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
> va + bo_size, vm);
>
> - if (adev == bo_adev ||
> - (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> - (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
> + if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
> + (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
> + same_hive) {
> /* Mappings on the local GPU, or VRAM mappings in the
> * local hive, or userptr mapping IOMMU direct map mode
> * share the original BO
> @@ -726,26 +943,35 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
> } else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
> /* Create an SG BO to DMA-map userptrs on other GPUs */
> attachment[i]->type = KFD_MEM_ATT_USERPTR;
> - ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> } else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
> mem->bo->tbo.type != ttm_bo_type_sg) {
> - /* GTT BOs use DMA-mapping ability of dynamic-attach
> - * DMA bufs. TODO: The same should work for VRAM on
> - * large-BAR GPUs.
> - */
> + /* GTT BOs use DMA-mapping ability of dynamic-attach DMA bufs */
> attachment[i]->type = KFD_MEM_ATT_DMABUF;
> ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
> if (ret)
> goto unwind;
> + /* Enable acces to VRAM BOs of peer devices */
> + } else if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM &&
> + mem->bo->tbo.type == ttm_bo_type_device) {
> + ret = kfd_mem_attach_vram_bo(adev, mem,
> + &bo[i], attachment[i]);
You can just call kfd_mem_attach_dmabuf directly here. Wrap this whole
else-if block (and the following block for doorbells and MMIO) in #ifdef
CONFIG_HSA_AMD_P2P.
> + if (ret)
> + goto unwind;
> + /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
> + } else if ((mem->bo->tbo.type == ttm_bo_type_sg) &&
> + ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
> + (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
I think we need an amdgpu_device_is_peer_accessible check here, except
maybe for local MMIO mappings. Alternatively extend the
peer-accessibility check at the start of this function to also handle
doorbell and MMIO BOs.
> + attachment[i]->type = KFD_MEM_ATT_SG;
> + ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
> + if (ret)
> + goto unwind;
> } else {
> - /* FIXME: Need to DMA-map other BO types:
> - * large-BAR VRAM, doorbells, MMIO remap
> - */
> - attachment[i]->type = KFD_MEM_ATT_SHARED;
> - bo[i] = mem->bo;
> - drm_gem_object_get(&bo[i]->tbo.base);
> + WARN_ONCE(true, "Handling invalid ATTACH request");
> + ret = -EINVAL;
> + goto unwind;
> }
>
> /* Add BO to VM internal data structures */
> @@ -1146,24 +1372,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
> return ret;
> }
>
> -static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
> -{
> - struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
> -
> - if (!sg)
> - return NULL;
> - if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
> - kfree(sg);
> - return NULL;
> - }
> - sg->sgl->dma_address = addr;
> - sg->sgl->length = size;
> -#ifdef CONFIG_NEED_SG_DMA_LENGTH
> - sg->sgl->dma_length = size;
> -#endif
> - return sg;
> -}
> -
> static int process_validate_vms(struct amdkfd_process_info *process_info)
> {
> struct amdgpu_vm *peer_vm;
> @@ -1532,7 +1740,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
> bo_type = ttm_bo_type_sg;
> if (size > UINT_MAX)
> return -EINVAL;
> - sg = create_doorbell_sg(*offset, size);
> + sg = create_sg_table(*offset, size);
> if (!sg)
> return -ENOMEM;
> } else {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index f16f105a737b..3dfac07cf37c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -127,6 +127,8 @@ const char *amdgpu_asic_name[] = {
> "LAST",
> };
>
> +extern bool pcie_p2p;
> +
> /**
> * DOC: pcie_replay_count
> *
> @@ -5463,6 +5465,34 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
> }
> }
>
> +/**
> + * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
> + *
> + * @adev: amdgpu_device pointer
> + * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
> + *
> + * Return true if @peer_adev can access (DMA) @adev through the PCIe
> + * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
> + * @peer_adev.
> + */
> +bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev)
> +{
> +#ifdef CONFIG_HSA_AMD_P2P
I was expecting a call to pci_p2pdma_distance_many somewhere here.
Regards,
Felix
> + uint64_t address_mask = peer_adev->dev->dma_mask ?
> + ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
> + resource_size_t aper_limit =
> + adev->gmc.aper_base + adev->gmc.aper_size - 1;
> +
> + return pcie_p2p && (adev->gmc.visible_vram_size &&
> + adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
> + !(adev->gmc.aper_base & address_mask ||
> + aper_limit & address_mask));
> +#else
> + return false;
> +#endif
> +}
> +
> int amdgpu_device_baco_enter(struct drm_device *dev)
> {
> struct amdgpu_device *adev = drm_to_adev(dev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index bed4ed88951f..d1c82a9e8569 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
> module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
> #endif
>
> +/**
> + * DOC: pcie_p2p (bool)
> + * Enable PCIe P2P (requires large-BAR). Default value: true (on)
> + */
> +bool pcie_p2p = true;
> +module_param(pcie_p2p, bool, 0444);
> +MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
> +
> /**
> * DOC: dcfeaturemask (uint)
> * Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
^ permalink raw reply [flat|nested] 14+ messages in thread
* [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs
@ 2022-05-31 17:01 Ramesh Errabolu
2022-06-01 16:12 ` Felix Kuehling
0 siblings, 1 reply; 14+ messages in thread
From: Ramesh Errabolu @ 2022-05-31 17:01 UTC (permalink / raw)
To: amd-gfx; +Cc: Ramesh Errabolu
Add support for peer-to-peer communication, in both data and control
planes, among AMD GPUs that are connected PCIe and have large BAR vBIOS.
Support REQUIRES enablement of config HSA_AMD_P2P.
Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
.../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 328 ++++++++++++++----
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 30 ++
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 8 +
4 files changed, 307 insertions(+), 60 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index f8b9f27adcf5..5c00ea1df21c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -48,6 +48,7 @@ enum kfd_mem_attachment_type {
KFD_MEM_ATT_SHARED, /* Share kgd_mem->bo or another attachment's */
KFD_MEM_ATT_USERPTR, /* SG bo to DMA map pages from a userptr bo */
KFD_MEM_ATT_DMABUF, /* DMAbuf to DMA map TTM BOs */
+ KFD_MEM_ATT_SG /* Tag to DMA map SG BOs */
};
struct kfd_mem_attachment {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 34ba9e776521..c2af82317a03 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -241,6 +241,42 @@ void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo)
kfree(bo->kfd_bo);
}
+/**
+ * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information
+ * about USERPTR or DOOREBELL or MMIO BO.
+ * @adev: Device for which dmamap BO is being created
+ * @mem: BO of peer device that is being DMA mapped. Provides parameters
+ * in building the dmamap BO
+ * @bo_out: Output parameter updated with handle of dmamap BO
+ */
+static int
+create_dmamap_sg_bo(struct amdgpu_device *adev,
+ struct kgd_mem *mem, struct amdgpu_bo **bo_out)
+{
+ struct drm_gem_object *gem_obj;
+ int ret, align;
+
+ ret = amdgpu_bo_reserve(mem->bo, false);
+ if (ret)
+ return ret;
+
+ align = 1;
+ ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align,
+ AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE,
+ ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj);
+
+ amdgpu_bo_unreserve(mem->bo);
+
+ if (ret) {
+ pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret);
+ return -EINVAL;
+ }
+
+ *bo_out = gem_to_amdgpu_bo(gem_obj);
+ (*bo_out)->parent = amdgpu_bo_ref(mem->bo);
+ return ret;
+}
+
/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's
* reservation object.
*
@@ -481,6 +517,38 @@ static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem)
return pte_flags;
}
+/**
+ * create_sg_table() - Create an sg_table for a contiguous DMA addr range
+ * @addr: The starting address to point to
+ * @size: Size of memory area in bytes being pointed to
+ *
+ * Allocates an instance of sg_table and initializes it to point to memory
+ * area specified by input parameters. The address used to build is assumed
+ * to be DMA mapped, if needed.
+ *
+ * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table
+ * because they are physically contiguous.
+ *
+ * Return: Initialized instance of SG Table or NULL
+ */
+static struct sg_table *create_sg_table(uint64_t addr, uint32_t size)
+{
+ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
+
+ if (!sg)
+ return NULL;
+ if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
+ kfree(sg);
+ return NULL;
+ }
+ sg_dma_address(sg->sgl) = addr;
+ sg->sgl->length = size;
+#ifdef CONFIG_NEED_SG_DMA_LENGTH
+ sg->sgl->dma_length = size;
+#endif
+ return sg;
+}
+
static int
kfd_mem_dmamap_userptr(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -545,6 +613,87 @@ kfd_mem_dmamap_dmabuf(struct kfd_mem_attachment *attachment)
return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmamap_sg_bo() - Create DMA mapped sg_table to access DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * An access request from the device that owns DOORBELL does not require DMA mapping.
+ * This is because the request doesn't go through PCIe root complex i.e. it instead
+ * loops back. The need to DMA map arises only when accessing peer device's DOORBELL
+ *
+ * In contrast, all access requests for MMIO need to be DMA mapped without regard to
+ * device ownership. This is because access requests for MMIO go through PCIe root
+ * complex.
+ *
+ * This is accomplished in two steps:
+ * - Obtain DMA mapped address of DOORBELL or MMIO memory that could be used
+ * in updating requesting device's page table
+ * - Signal TTM to mark memory pointed to by requesting device's BO as GPU
+ * accessible. This allows an update of requesting device's page table
+ * with entries associated with DOOREBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * - Mapping of DOORBELL or MMIO BO of same or peer device
+ * - Validating an evicted DOOREBELL or MMIO BO on device seeking access
+ *
+ * Return: ZERO if successful, NON-ZERO otherwise
+ */
+static int
+kfd_mem_dmamap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+ dma_addr_t dma_addr;
+ bool mmio;
+ int ret;
+
+ /* Expect SG Table of dmapmap BO to be NULL */
+ mmio = (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP);
+ if (unlikely(ttm->sg)) {
+ pr_err("SG Table of %d BO for peer device is UNEXPECTEDLY NON-NULL", mmio);
+ return -EINVAL;
+ }
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_addr = mem->bo->tbo.sg->sgl->dma_address;
+ pr_debug("%d BO size: %d\n", mmio, mem->bo->tbo.sg->sgl->length);
+ pr_debug("%d BO address before DMA mapping: %llx\n", mmio, dma_addr);
+ dma_addr = dma_map_resource(adev->dev, dma_addr,
+ mem->bo->tbo.sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ ret = dma_mapping_error(adev->dev, dma_addr);
+ if (unlikely(ret))
+ return ret;
+ pr_debug("%d BO address after DMA mapping: %llx\n", mmio, dma_addr);
+
+ ttm->sg = create_sg_table(dma_addr, mem->bo->tbo.sg->sgl->length);
+ if (unlikely(!ttm->sg)) {
+ ret = -ENOMEM;
+ goto unmap_sg;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT);
+ ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+ if (unlikely(ret))
+ goto free_sg;
+
+ return ret;
+
+free_sg:
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+unmap_sg:
+ dma_unmap_resource(adev->dev, dma_addr, mem->bo->tbo.sg->sgl->length,
+ dir, DMA_ATTR_SKIP_CPU_SYNC);
+ return ret;
+}
+
static int
kfd_mem_dmamap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -556,6 +705,8 @@ kfd_mem_dmamap_attachment(struct kgd_mem *mem,
return kfd_mem_dmamap_userptr(mem, attachment);
case KFD_MEM_ATT_DMABUF:
return kfd_mem_dmamap_dmabuf(attachment);
+ case KFD_MEM_ATT_SG:
+ return kfd_mem_dmamap_sg_bo(mem, attachment);
default:
WARN_ON_ONCE(1);
}
@@ -596,6 +747,50 @@ kfd_mem_dmaunmap_dmabuf(struct kfd_mem_attachment *attachment)
ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
}
+/**
+ * kfd_mem_dmaunmap_sg_bo() - Free DMA mapped sg_table of DOORBELL or MMIO BO
+ * @mem: SG BO of the DOORBELL or MMIO resource on the owning device
+ * @attachment: Virtual address attachment of the BO on accessing device
+ *
+ * The method performs following steps:
+ * - Signal TTM to mark memory pointed to by BO as GPU inaccessible
+ * - Free SG Table that is used to encapsulate DMA mapped memory of
+ * peer device's DOORBELL or MMIO memory
+ *
+ * This method is invoked in the following contexts:
+ * UNMapping of DOORBELL or MMIO BO on a device having access to its memory
+ * Eviction of DOOREBELL or MMIO BO on device having access to its memory
+ *
+ * Return: void
+ */
+static void
+kfd_mem_dmaunmap_sg_bo(struct kgd_mem *mem,
+ struct kfd_mem_attachment *attachment)
+{
+ struct ttm_operation_ctx ctx = {.interruptible = true};
+ struct amdgpu_bo *bo = attachment->bo_va->base.bo;
+ struct amdgpu_device *adev = attachment->adev;
+ struct ttm_tt *ttm = bo->tbo.ttm;
+ enum dma_data_direction dir;
+
+ if (unlikely(!ttm->sg)) {
+ pr_err("SG Table of BO is UNEXPECTEDLY NULL");
+ return;
+ }
+
+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
+ ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
+
+ dir = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ?
+ DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
+ dma_unmap_resource(adev->dev, ttm->sg->sgl->dma_address,
+ ttm->sg->sgl->length, dir, DMA_ATTR_SKIP_CPU_SYNC);
+ sg_free_table(ttm->sg);
+ kfree(ttm->sg);
+ ttm->sg = NULL;
+ bo->tbo.sg = NULL;
+}
+
static void
kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
struct kfd_mem_attachment *attachment)
@@ -609,38 +804,14 @@ kfd_mem_dmaunmap_attachment(struct kgd_mem *mem,
case KFD_MEM_ATT_DMABUF:
kfd_mem_dmaunmap_dmabuf(attachment);
break;
+ case KFD_MEM_ATT_SG:
+ kfd_mem_dmaunmap_sg_bo(mem, attachment);
+ break;
default:
WARN_ON_ONCE(1);
}
}
-static int
-kfd_mem_attach_userptr(struct amdgpu_device *adev, struct kgd_mem *mem,
- struct amdgpu_bo **bo)
-{
- unsigned long bo_size = mem->bo->tbo.base.size;
- struct drm_gem_object *gobj;
- int ret;
-
- ret = amdgpu_bo_reserve(mem->bo, false);
- if (ret)
- return ret;
-
- ret = amdgpu_gem_object_create(adev, bo_size, 1,
- AMDGPU_GEM_DOMAIN_CPU,
- AMDGPU_GEM_CREATE_PREEMPTIBLE,
- ttm_bo_type_sg, mem->bo->tbo.base.resv,
- &gobj);
- amdgpu_bo_unreserve(mem->bo);
- if (ret)
- return ret;
-
- *bo = gem_to_amdgpu_bo(gobj);
- (*bo)->parent = amdgpu_bo_ref(mem->bo);
-
- return 0;
-}
-
static int
kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
struct amdgpu_bo **bo)
@@ -670,6 +841,38 @@ kfd_mem_attach_dmabuf(struct amdgpu_device *adev, struct kgd_mem *mem,
return 0;
}
+/**
+ * @kfd_mem_attach_vram_bo: Acquires the handle of a VRAM BO that could
+ * be used to enable a peer GPU access it
+ *
+ * Implementation determines if access to VRAM BO would employ DMABUF
+ * or Shared BO mechanism. Employ DMABUF mechanism if kernel has config
+ * option HSA_AMD_P2P enabled. Employ Shared BO mechanism if above
+ * config option is not set. It is important to note that a Shared BO
+ * cannot be used to enable peer acces if system has IOMMU enabled
+ *
+ * @TODO: ADD Check to ensure IOMMU is not enabled. Should this check
+ * be somewhere as this is information could be useful in other places
+ */
+static int kfd_mem_attach_vram_bo(struct amdgpu_device *adev,
+ struct kgd_mem *mem, struct amdgpu_bo **bo,
+ struct kfd_mem_attachment *attachment)
+{
+ int ret = 0;
+
+#if defined(CONFIG_HSA_AMD_P2P)
+ attachment->type = KFD_MEM_ATT_DMABUF;
+ ret = kfd_mem_attach_dmabuf(adev, mem, bo);
+ pr_debug("Employ DMABUF mechanim to enable peer GPU access\n");
+#else
+ *bo = mem->bo;
+ attachment->type = KFD_MEM_ATT_SHARED;
+ drm_gem_object_get(&(*bo)->tbo.base);
+ pr_debug("Employ Shared BO mechanim to enable peer GPU access\n");
+#endif
+ return ret;
+}
+
/* kfd_mem_attach - Add a BO to a VM
*
* Everything that needs to bo done only once when a BO is first added
@@ -691,6 +894,7 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
uint64_t va = mem->va;
struct kfd_mem_attachment *attachment[2] = {NULL, NULL};
struct amdgpu_bo *bo[2] = {NULL, NULL};
+ bool same_hive = false;
int i, ret;
if (!va) {
@@ -698,6 +902,19 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
return -EINVAL;
}
+ /* Determine if the mapping of VRAM BO to a peer device is valid
+ * It is possible that the peer device is connected via PCIe or
+ * xGMI link. Access over PCIe is allowed if device owning VRAM BO
+ * has large BAR. In contrast, access over xGMI is allowed for both
+ * small and large BAR configurations of device owning the VRAM BO
+ */
+ if (adev != bo_adev && mem->domain == AMDGPU_GEM_DOMAIN_VRAM) {
+ same_hive = amdgpu_xgmi_same_hive(adev, bo_adev);
+ if (!same_hive &&
+ !amdgpu_device_is_peer_accessible(bo_adev, adev))
+ return -EINVAL;
+ }
+
for (i = 0; i <= is_aql; i++) {
attachment[i] = kzalloc(sizeof(*attachment[i]), GFP_KERNEL);
if (unlikely(!attachment[i])) {
@@ -708,9 +925,9 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
va + bo_size, vm);
- if (adev == bo_adev ||
- (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
- (mem->domain == AMDGPU_GEM_DOMAIN_VRAM && amdgpu_xgmi_same_hive(adev, bo_adev))) {
+ if ((adev == bo_adev && !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) ||
+ (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && adev->ram_is_direct_mapped) ||
+ same_hive) {
/* Mappings on the local GPU, or VRAM mappings in the
* local hive, or userptr mapping IOMMU direct map mode
* share the original BO
@@ -726,26 +943,35 @@ static int kfd_mem_attach(struct amdgpu_device *adev, struct kgd_mem *mem,
} else if (amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm)) {
/* Create an SG BO to DMA-map userptrs on other GPUs */
attachment[i]->type = KFD_MEM_ATT_USERPTR;
- ret = kfd_mem_attach_userptr(adev, mem, &bo[i]);
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
if (ret)
goto unwind;
} else if (mem->domain == AMDGPU_GEM_DOMAIN_GTT &&
mem->bo->tbo.type != ttm_bo_type_sg) {
- /* GTT BOs use DMA-mapping ability of dynamic-attach
- * DMA bufs. TODO: The same should work for VRAM on
- * large-BAR GPUs.
- */
+ /* GTT BOs use DMA-mapping ability of dynamic-attach DMA bufs */
attachment[i]->type = KFD_MEM_ATT_DMABUF;
ret = kfd_mem_attach_dmabuf(adev, mem, &bo[i]);
if (ret)
goto unwind;
+ /* Enable acces to VRAM BOs of peer devices */
+ } else if (mem->domain == AMDGPU_GEM_DOMAIN_VRAM &&
+ mem->bo->tbo.type == ttm_bo_type_device) {
+ ret = kfd_mem_attach_vram_bo(adev, mem,
+ &bo[i], attachment[i]);
+ if (ret)
+ goto unwind;
+ /* Handle DOORBELL BOs of peer devices and MMIO BOs of local and peer devices */
+ } else if ((mem->bo->tbo.type == ttm_bo_type_sg) &&
+ ((mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) ||
+ (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) {
+ attachment[i]->type = KFD_MEM_ATT_SG;
+ ret = create_dmamap_sg_bo(adev, mem, &bo[i]);
+ if (ret)
+ goto unwind;
} else {
- /* FIXME: Need to DMA-map other BO types:
- * large-BAR VRAM, doorbells, MMIO remap
- */
- attachment[i]->type = KFD_MEM_ATT_SHARED;
- bo[i] = mem->bo;
- drm_gem_object_get(&bo[i]->tbo.base);
+ WARN_ONCE(true, "Handling invalid ATTACH request");
+ ret = -EINVAL;
+ goto unwind;
}
/* Add BO to VM internal data structures */
@@ -1146,24 +1372,6 @@ static int map_bo_to_gpuvm(struct kgd_mem *mem,
return ret;
}
-static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
-{
- struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
-
- if (!sg)
- return NULL;
- if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
- kfree(sg);
- return NULL;
- }
- sg->sgl->dma_address = addr;
- sg->sgl->length = size;
-#ifdef CONFIG_NEED_SG_DMA_LENGTH
- sg->sgl->dma_length = size;
-#endif
- return sg;
-}
-
static int process_validate_vms(struct amdkfd_process_info *process_info)
{
struct amdgpu_vm *peer_vm;
@@ -1532,7 +1740,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
bo_type = ttm_bo_type_sg;
if (size > UINT_MAX)
return -EINVAL;
- sg = create_doorbell_sg(*offset, size);
+ sg = create_sg_table(*offset, size);
if (!sg)
return -ENOMEM;
} else {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index f16f105a737b..3dfac07cf37c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -127,6 +127,8 @@ const char *amdgpu_asic_name[] = {
"LAST",
};
+extern bool pcie_p2p;
+
/**
* DOC: pcie_replay_count
*
@@ -5463,6 +5465,34 @@ static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
}
}
+/**
+ * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
+ *
+ * @adev: amdgpu_device pointer
+ * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
+ *
+ * Return true if @peer_adev can access (DMA) @adev through the PCIe
+ * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
+ * @peer_adev.
+ */
+bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev)
+{
+#ifdef CONFIG_HSA_AMD_P2P
+ uint64_t address_mask = peer_adev->dev->dma_mask ?
+ ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
+ resource_size_t aper_limit =
+ adev->gmc.aper_base + adev->gmc.aper_size - 1;
+
+ return pcie_p2p && (adev->gmc.visible_vram_size &&
+ adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
+ !(adev->gmc.aper_base & address_mask ||
+ aper_limit & address_mask));
+#else
+ return false;
+#endif
+}
+
int amdgpu_device_baco_enter(struct drm_device *dev)
{
struct amdgpu_device *adev = drm_to_adev(dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index bed4ed88951f..d1c82a9e8569 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -802,6 +802,14 @@ MODULE_PARM_DESC(no_queue_eviction_on_vm_fault, "No queue eviction on VM fault (
module_param_named(no_queue_eviction_on_vm_fault, amdgpu_no_queue_eviction_on_vm_fault, int, 0444);
#endif
+/**
+ * DOC: pcie_p2p (bool)
+ * Enable PCIe P2P (requires large-BAR). Default value: true (on)
+ */
+bool pcie_p2p = true;
+module_param(pcie_p2p, bool, 0444);
+MODULE_PARM_DESC(pcie_p2p, "Enable PCIe P2P (requires large-BAR). (N = off, Y = on(default))");
+
/**
* DOC: dcfeaturemask (uint)
* Override display features enabled. See enum DC_FEATURE_MASK in drivers/gpu/drm/amd/include/amd_shared.h.
--
2.35.1
^ permalink raw reply related [flat|nested] 14+ messages in thread
end of thread, other threads:[~2022-06-07 16:10 UTC | newest]
Thread overview: 14+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-06-03 10:52 [PATCH 2/3] drm/amdgpu: Add peer-to-peer support among PCIe connected AMD GPUs Ramesh Errabolu
2022-06-03 22:26 ` Felix Kuehling
2022-06-04 10:23 ` Errabolu, Ramesh
2022-06-06 14:26 ` Felix Kuehling
2022-06-06 18:11 ` Errabolu, Ramesh
-- strict thread matches above, loose matches on Subject: below --
2022-06-07 8:23 Ramesh Errabolu
2022-06-07 16:10 ` Felix Kuehling
2022-06-06 18:07 Ramesh Errabolu
2022-06-06 20:04 ` Felix Kuehling
2022-06-06 20:22 ` Felix Kuehling
2022-06-04 10:23 Ramesh Errabolu
2022-05-31 17:01 Ramesh Errabolu
2022-06-01 16:12 ` Felix Kuehling
2022-06-03 10:41 ` Errabolu, Ramesh
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.