All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH] drm/amdkfd: fix no atomics settings in the kfd topology
@ 2021-05-03 19:49 Jonathan Kim
  2021-05-03 19:56 ` Felix Kuehling
  2021-05-04  2:06 ` Zeng, Oak
  0 siblings, 2 replies; 5+ messages in thread
From: Jonathan Kim @ 2021-05-03 19:49 UTC (permalink / raw)
  To: amd-gfx, Felix.Kuehling; +Cc: Jonathan Kim

To account for various PCIe and xGMI setups, check the no atomics settings
for a device in relation to every direct peer.

v2: apply suggested clean ups in main loop.

Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 61 ++++++++++++++---------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 30430aefcfc7..fb4f718a1148 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1192,47 +1192,60 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
 		mem->mem_clk_max = local_mem_info.mem_clk_max;
 }
 
-static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
+static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
+					struct kfd_topology_device *target_gpu_dev,
+					struct kfd_iolink_properties *link)
 {
-	struct kfd_iolink_properties *link, *cpu_link;
-	struct kfd_topology_device *cpu_dev;
-	struct amdgpu_device *adev;
-	uint32_t cap;
-	uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
-	uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
-
-	if (!dev || !dev->gpu)
+	/* xgmi always supports atomics between links. */
+	if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
 		return;
 
-	adev = (struct amdgpu_device *)(dev->gpu->kgd);
-	if (!adev->gmc.xgmi.connected_to_cpu) {
-		pcie_capability_read_dword(dev->gpu->pdev,
+	/* check pcie support to set cpu(dev) flags for target_gpu_dev link. */
+	if (target_gpu_dev) {
+		uint32_t cap;
+
+		pcie_capability_read_dword(target_gpu_dev->gpu->pdev,
 				PCI_EXP_DEVCAP2, &cap);
 
 		if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
 			     PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
-			cpu_flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
+			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
 				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
-	}
-
-	if (!adev->gmc.xgmi.num_physical_nodes) {
+	/* set gpu (dev) flags. */
+	} else {
 		if (!dev->gpu->pci_atomic_requested ||
 				dev->gpu->device_info->asic_family ==
 							CHIP_HAWAII)
-			flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
+			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
 				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
 	}
+}
+
+static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
+{
+	struct kfd_iolink_properties *link, *inbound_link;
+	struct kfd_topology_device *peer_dev;
+
+	if (!dev || !dev->gpu)
+		return;
 
 	/* GPU only creates direct links so apply flags setting to all */
 	list_for_each_entry(link, &dev->io_link_props, list) {
-		link->flags = flag;
-		cpu_dev = kfd_topology_device_by_proximity_domain(
+		link->flags = CRAT_IOLINK_FLAGS_ENABLED;
+		kfd_set_iolink_no_atomics(dev, NULL, link);
+		peer_dev = kfd_topology_device_by_proximity_domain(
 				link->node_to);
-		if (cpu_dev) {
-			list_for_each_entry(cpu_link,
-					    &cpu_dev->io_link_props, list)
-				if (cpu_link->node_to == link->node_from)
-					cpu_link->flags = cpu_flag;
+
+		if (!peer_dev)
+			continue;
+
+		list_for_each_entry(inbound_link, &peer_dev->io_link_props,
+									list) {
+			if (inbound_link->node_to != link->node_from)
+				continue;
+
+			inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED;
+			kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
 		}
 	}
 }
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 5+ messages in thread
* Re: [PATCH] drm/amdkfd: fix no atomics settings in the kfd topology
  2021-05-03 19:49 [PATCH] drm/amdkfd: fix no atomics settings in the kfd topology Jonathan Kim
@ 2021-05-03 19:56 ` Felix Kuehling
  2021-05-04  2:06 ` Zeng, Oak
  1 sibling, 0 replies; 5+ messages in thread
From: Felix Kuehling @ 2021-05-03 19:56 UTC (permalink / raw)
  To: Jonathan Kim, amd-gfx

Am 2021-05-03 um 3:49 p.m. schrieb Jonathan Kim:
> To account for various PCIe and xGMI setups, check the no atomics settings
> for a device in relation to every direct peer.
>
> v2: apply suggested clean ups in main loop.
>
> Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>

Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>


> ---
>  drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 61 ++++++++++++++---------
>  1 file changed, 37 insertions(+), 24 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> index 30430aefcfc7..fb4f718a1148 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> @@ -1192,47 +1192,60 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
>  		mem->mem_clk_max = local_mem_info.mem_clk_max;
>  }
>  
> -static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
> +static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
> +					struct kfd_topology_device *target_gpu_dev,
> +					struct kfd_iolink_properties *link)
>  {
> -	struct kfd_iolink_properties *link, *cpu_link;
> -	struct kfd_topology_device *cpu_dev;
> -	struct amdgpu_device *adev;
> -	uint32_t cap;
> -	uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
> -	uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
> -
> -	if (!dev || !dev->gpu)
> +	/* xgmi always supports atomics between links. */
> +	if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
>  		return;
>  
> -	adev = (struct amdgpu_device *)(dev->gpu->kgd);
> -	if (!adev->gmc.xgmi.connected_to_cpu) {
> -		pcie_capability_read_dword(dev->gpu->pdev,
> +	/* check pcie support to set cpu(dev) flags for target_gpu_dev link. */
> +	if (target_gpu_dev) {
> +		uint32_t cap;
> +
> +		pcie_capability_read_dword(target_gpu_dev->gpu->pdev,
>  				PCI_EXP_DEVCAP2, &cap);
>  
>  		if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
>  			     PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
> -			cpu_flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
> +			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
>  				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
> -	}
> -
> -	if (!adev->gmc.xgmi.num_physical_nodes) {
> +	/* set gpu (dev) flags. */
> +	} else {
>  		if (!dev->gpu->pci_atomic_requested ||
>  				dev->gpu->device_info->asic_family ==
>  							CHIP_HAWAII)
> -			flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
> +			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
>  				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
>  	}
> +}
> +
> +static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
> +{
> +	struct kfd_iolink_properties *link, *inbound_link;
> +	struct kfd_topology_device *peer_dev;
> +
> +	if (!dev || !dev->gpu)
> +		return;
>  
>  	/* GPU only creates direct links so apply flags setting to all */
>  	list_for_each_entry(link, &dev->io_link_props, list) {
> -		link->flags = flag;
> -		cpu_dev = kfd_topology_device_by_proximity_domain(
> +		link->flags = CRAT_IOLINK_FLAGS_ENABLED;
> +		kfd_set_iolink_no_atomics(dev, NULL, link);
> +		peer_dev = kfd_topology_device_by_proximity_domain(
>  				link->node_to);
> -		if (cpu_dev) {
> -			list_for_each_entry(cpu_link,
> -					    &cpu_dev->io_link_props, list)
> -				if (cpu_link->node_to == link->node_from)
> -					cpu_link->flags = cpu_flag;
> +
> +		if (!peer_dev)
> +			continue;
> +
> +		list_for_each_entry(inbound_link, &peer_dev->io_link_props,
> +									list) {
> +			if (inbound_link->node_to != link->node_from)
> +				continue;
> +
> +			inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED;
> +			kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
>  		}
>  	}
>  }
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 5+ messages in thread
* Re: [PATCH] drm/amdkfd: fix no atomics settings in the kfd topology
  2021-05-03 19:49 [PATCH] drm/amdkfd: fix no atomics settings in the kfd topology Jonathan Kim
  2021-05-03 19:56 ` Felix Kuehling
@ 2021-05-04  2:06 ` Zeng, Oak
  1 sibling, 0 replies; 5+ messages in thread
From: Zeng, Oak @ 2021-05-04  2:06 UTC (permalink / raw)
  To: Kim, Jonathan, amd-gfx, Kuehling, Felix

Reviewed-by: Oak Zeng <Oak.Zeng@amd.com>

Regards,
Oak 

 

On 2021-05-03, 3:49 PM, "amd-gfx on behalf of Jonathan Kim" <amd-gfx-bounces@lists.freedesktop.org on behalf of Jonathan.Kim@amd.com> wrote:

    To account for various PCIe and xGMI setups, check the no atomics settings
    for a device in relation to every direct peer.

    v2: apply suggested clean ups in main loop.

    Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
    ---
     drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 61 ++++++++++++++---------
     1 file changed, 37 insertions(+), 24 deletions(-)

    diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
    index 30430aefcfc7..fb4f718a1148 100644
    --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
    +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
    @@ -1192,47 +1192,60 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
     		mem->mem_clk_max = local_mem_info.mem_clk_max;
     }

    -static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
    +static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
    +					struct kfd_topology_device *target_gpu_dev,
    +					struct kfd_iolink_properties *link)
     {
    -	struct kfd_iolink_properties *link, *cpu_link;
    -	struct kfd_topology_device *cpu_dev;
    -	struct amdgpu_device *adev;
    -	uint32_t cap;
    -	uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
    -	uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
    -
    -	if (!dev || !dev->gpu)
    +	/* xgmi always supports atomics between links. */
    +	if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
     		return;

    -	adev = (struct amdgpu_device *)(dev->gpu->kgd);
    -	if (!adev->gmc.xgmi.connected_to_cpu) {
    -		pcie_capability_read_dword(dev->gpu->pdev,
    +	/* check pcie support to set cpu(dev) flags for target_gpu_dev link. */
    +	if (target_gpu_dev) {
    +		uint32_t cap;
    +
    +		pcie_capability_read_dword(target_gpu_dev->gpu->pdev,
     				PCI_EXP_DEVCAP2, &cap);

     		if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
     			     PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
    -			cpu_flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
    +			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
     				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
    -	}
    -
    -	if (!adev->gmc.xgmi.num_physical_nodes) {
    +	/* set gpu (dev) flags. */
    +	} else {
     		if (!dev->gpu->pci_atomic_requested ||
     				dev->gpu->device_info->asic_family ==
     							CHIP_HAWAII)
    -			flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
    +			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
     				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
     	}
    +}
    +
    +static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
    +{
    +	struct kfd_iolink_properties *link, *inbound_link;
    +	struct kfd_topology_device *peer_dev;
    +
    +	if (!dev || !dev->gpu)
    +		return;

     	/* GPU only creates direct links so apply flags setting to all */
     	list_for_each_entry(link, &dev->io_link_props, list) {
    -		link->flags = flag;
    -		cpu_dev = kfd_topology_device_by_proximity_domain(
    +		link->flags = CRAT_IOLINK_FLAGS_ENABLED;
    +		kfd_set_iolink_no_atomics(dev, NULL, link);
    +		peer_dev = kfd_topology_device_by_proximity_domain(
     				link->node_to);
    -		if (cpu_dev) {
    -			list_for_each_entry(cpu_link,
    -					    &cpu_dev->io_link_props, list)
    -				if (cpu_link->node_to == link->node_from)
    -					cpu_link->flags = cpu_flag;
    +
    +		if (!peer_dev)
    +			continue;
    +
    +		list_for_each_entry(inbound_link, &peer_dev->io_link_props,
    +									list) {
    +			if (inbound_link->node_to != link->node_from)
    +				continue;
    +
    +			inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED;
    +			kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
     		}
     	}
     }
    -- 
    2.17.1

    _______________________________________________
    amd-gfx mailing list
    amd-gfx@lists.freedesktop.org
    https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=04%7C01%7Coak.zeng%40amd.com%7Cb034fd7bd5d749a97cde08d90e6c9b1f%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637556681916702783%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&amp;sdata=kZ19ng5LPZ13qPOwoqcnraD9lJHniEWy42pRTWDf4oo%3D&amp;reserved=0

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 5+ messages in thread
* Re: [PATCH] drm/amdkfd: fix no atomics settings in the kfd topology
  2021-05-03 15:57 Jonathan Kim
@ 2021-05-03 18:09 ` Felix Kuehling
  0 siblings, 0 replies; 5+ messages in thread
From: Felix Kuehling @ 2021-05-03 18:09 UTC (permalink / raw)
  To: Jonathan Kim, amd-gfx

Am 2021-05-03 um 11:57 a.m. schrieb Jonathan Kim:
> To account for various PCIe and xGMI setups, check the no atomics settings
> for a device in relation to every direct peer.

Thanks, this looks reasonable. Some more nit-picks about naming and
coding style inline.


>
> Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
> ---
>  drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 55 ++++++++++++++---------
>  1 file changed, 34 insertions(+), 21 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> index 30430aefcfc7..40ac7fe2a499 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
> @@ -1192,47 +1192,60 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
>  		mem->mem_clk_max = local_mem_info.mem_clk_max;
>  }
>  
> -static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
> +static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
> +					struct kfd_topology_device *target_gpu_dev,
> +					struct kfd_iolink_properties *link)
>  {
> -	struct kfd_iolink_properties *link, *cpu_link;
> -	struct kfd_topology_device *cpu_dev;
> -	struct amdgpu_device *adev;
> -	uint32_t cap;
> -	uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
> -	uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
> -
> -	if (!dev || !dev->gpu)
> +	/* xgmi always supports atomics between links. */
> +	if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
>  		return;
>  
> -	adev = (struct amdgpu_device *)(dev->gpu->kgd);
> -	if (!adev->gmc.xgmi.connected_to_cpu) {
> -		pcie_capability_read_dword(dev->gpu->pdev,
> +	/* check pcie support to set cpu(dev) flags for target_get_dev link. */
> +	if (target_gpu_dev) {
> +		uint32_t cap;
> +
> +		pcie_capability_read_dword(target_gpu_dev->gpu->pdev,
>  				PCI_EXP_DEVCAP2, &cap);
>  
>  		if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
>  			     PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
> -			cpu_flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
> +			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
>  				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
> -	}
> -
> -	if (!adev->gmc.xgmi.num_physical_nodes) {
> +	/* set gpu (dev) flags. */
> +	} else {
>  		if (!dev->gpu->pci_atomic_requested ||
>  				dev->gpu->device_info->asic_family ==
>  							CHIP_HAWAII)
> -			flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
> +			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
>  				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
>  	}
> +}
> +
> +static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
> +{
> +	struct kfd_iolink_properties *link, *cpu_link;
> +	struct kfd_topology_device *cpu_dev;

The names cpu_link and cpu_dev are still misleading. When GPUs have XGMI
links these links may, in fact, be other GPUs and the XGMI links from
those GPUs. I think a better name would be "peer_dev" and "inbound_link".


> +	uint32_t flag_enable = CRAT_IOLINK_FLAGS_ENABLED;

You don't need this variable. Just use CRAT_IOLINK_FLAGS_ENABLED below.


> +
> +	if (!dev || !dev->gpu)
> +		return;
>  
>  	/* GPU only creates direct links so apply flags setting to all */
>  	list_for_each_entry(link, &dev->io_link_props, list) {
> -		link->flags = flag;
> +		link->flags = flag_enable;
> +		kfd_set_iolink_no_atomics(dev, NULL, link);
>  		cpu_dev = kfd_topology_device_by_proximity_domain(
>  				link->node_to);
> +
>  		if (cpu_dev) {

To minimize indentation, I'd do

	if (!peer_dev)
		continue;

>  			list_for_each_entry(cpu_link,
> -					    &cpu_dev->io_link_props, list)
> -				if (cpu_link->node_to == link->node_from)
> -					cpu_link->flags = cpu_flag;
> +					    &cpu_dev->io_link_props, list) {
> +				if (cpu_link->node_to == link->node_from) {
> +					cpu_link->flags = flag_enable;

To minimize indentation, and to avoid unnecessary loop iterations, I'd do

	if (inbound_link->node_to != link->node_from)
		continue;
	inbound_link->flags = KFD_IOLINK_CRAT_FLAGS_ENABLED;
	kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
	break;

Regards,
  Felix


> +					kfd_set_iolink_no_atomics(cpu_dev, dev,
> +								cpu_link);
> +				}
> +			}
>  		}
>  	}
>  }
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 5+ messages in thread
* [PATCH] drm/amdkfd: fix no atomics settings in the kfd topology
@ 2021-05-03 15:57 Jonathan Kim
  2021-05-03 18:09 ` Felix Kuehling
  0 siblings, 1 reply; 5+ messages in thread
From: Jonathan Kim @ 2021-05-03 15:57 UTC (permalink / raw)
  To: amd-gfx; +Cc: Felix.Kuehling, Jonathan Kim

To account for various PCIe and xGMI setups, check the no atomics settings
for a device in relation to every direct peer.

Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_topology.c | 55 ++++++++++++++---------
 1 file changed, 34 insertions(+), 21 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 30430aefcfc7..40ac7fe2a499 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -1192,47 +1192,60 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev)
 		mem->mem_clk_max = local_mem_info.mem_clk_max;
 }
 
-static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
+static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
+					struct kfd_topology_device *target_gpu_dev,
+					struct kfd_iolink_properties *link)
 {
-	struct kfd_iolink_properties *link, *cpu_link;
-	struct kfd_topology_device *cpu_dev;
-	struct amdgpu_device *adev;
-	uint32_t cap;
-	uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
-	uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
-
-	if (!dev || !dev->gpu)
+	/* xgmi always supports atomics between links. */
+	if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
 		return;
 
-	adev = (struct amdgpu_device *)(dev->gpu->kgd);
-	if (!adev->gmc.xgmi.connected_to_cpu) {
-		pcie_capability_read_dword(dev->gpu->pdev,
+	/* check pcie support to set cpu(dev) flags for target_get_dev link. */
+	if (target_gpu_dev) {
+		uint32_t cap;
+
+		pcie_capability_read_dword(target_gpu_dev->gpu->pdev,
 				PCI_EXP_DEVCAP2, &cap);
 
 		if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
 			     PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
-			cpu_flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
+			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
 				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
-	}
-
-	if (!adev->gmc.xgmi.num_physical_nodes) {
+	/* set gpu (dev) flags. */
+	} else {
 		if (!dev->gpu->pci_atomic_requested ||
 				dev->gpu->device_info->asic_family ==
 							CHIP_HAWAII)
-			flag |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
+			link->flags |= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT |
 				CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
 	}
+}
+
+static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
+{
+	struct kfd_iolink_properties *link, *cpu_link;
+	struct kfd_topology_device *cpu_dev;
+	uint32_t flag_enable = CRAT_IOLINK_FLAGS_ENABLED;
+
+	if (!dev || !dev->gpu)
+		return;
 
 	/* GPU only creates direct links so apply flags setting to all */
 	list_for_each_entry(link, &dev->io_link_props, list) {
-		link->flags = flag;
+		link->flags = flag_enable;
+		kfd_set_iolink_no_atomics(dev, NULL, link);
 		cpu_dev = kfd_topology_device_by_proximity_domain(
 				link->node_to);
+
 		if (cpu_dev) {
 			list_for_each_entry(cpu_link,
-					    &cpu_dev->io_link_props, list)
-				if (cpu_link->node_to == link->node_from)
-					cpu_link->flags = cpu_flag;
+					    &cpu_dev->io_link_props, list) {
+				if (cpu_link->node_to == link->node_from) {
+					cpu_link->flags = flag_enable;
+					kfd_set_iolink_no_atomics(cpu_dev, dev,
+								cpu_link);
+				}
+			}
 		}
 	}
 }
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 5+ messages in thread
end of thread, other threads:[~2021-05-04  2:06 UTC | newest]

Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-05-03 19:49 [PATCH] drm/amdkfd: fix no atomics settings in the kfd topology Jonathan Kim
2021-05-03 19:56 ` Felix Kuehling
2021-05-04  2:06 ` Zeng, Oak
  -- strict thread matches above, loose matches on Subject: below --
2021-05-03 15:57 Jonathan Kim
2021-05-03 18:09 ` Felix Kuehling

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.