* [PATCH 1/3] drm/amdgpu: add psp command to get num xgmi links between direct peers
@ 2021-07-16 16:43 Jonathan Kim
2021-07-16 16:43 ` [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd Jonathan Kim
2021-07-16 16:43 ` [PATCH 3/3] drm/amdkfd: report pcie bandwidth " Jonathan Kim
0 siblings, 2 replies; 9+ messages in thread
From: Jonathan Kim @ 2021-07-16 16:43 UTC (permalink / raw)
To: amd-gfx; +Cc: Felix.Kuehling, Jonathan Kim
The TA can now be invoked to provide the number of xgmi links connecting
a direct source and destination peer.
Non-direct peers will report zero links.
Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 23 +++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h | 1 +
drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h | 14 +++++++++++++-
3 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index dfb481a0780f..4d6752ec8adc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -1074,6 +1074,12 @@ int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
return 0;
}
+static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
+{
+ return psp->adev->asic_type == CHIP_ALDEBARAN &&
+ psp->ta_xgmi_ucode_version >= 0x2000000b;
+}
+
int psp_xgmi_get_topology_info(struct psp_context *psp,
int number_devices,
struct psp_xgmi_topology_info *topology)
@@ -1117,6 +1123,23 @@ int psp_xgmi_get_topology_info(struct psp_context *psp,
topology->nodes[i].sdma_engine = topology_info_output->nodes[i].sdma_engine;
}
+ /* Invoke xgmi ta again to get the link information */
+ if (psp_xgmi_peer_link_info_supported(psp)) {
+ struct ta_xgmi_cmd_get_peer_link_info_output *link_info_output;
+
+ xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
+
+ ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_PEER_LINKS);
+
+ if (ret)
+ return ret;
+
+ link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
+ for (i = 0; i < topology->num_nodes; i++)
+ topology->nodes[i].num_links =
+ link_info_output->nodes[i].num_links;
+ }
+
return 0;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 200d19139e73..f5b967b18e3f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -116,6 +116,7 @@ struct psp_xgmi_node_info {
uint8_t num_hops;
uint8_t is_sharing_enabled;
enum ta_xgmi_assigned_sdma_engine sdma_engine;
+ uint8_t num_links;
};
struct psp_xgmi_topology_info {
diff --git a/drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h b/drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h
index ac2c27b7630c..cce7127afeaa 100644
--- a/drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/ta_xgmi_if.h
@@ -33,7 +33,8 @@ enum ta_command_xgmi {
TA_COMMAND_XGMI__GET_NODE_ID = 0x01,
TA_COMMAND_XGMI__GET_HIVE_ID = 0x02,
TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO = 0x03,
- TA_COMMAND_XGMI__SET_TOPOLOGY_INFO = 0x04
+ TA_COMMAND_XGMI__SET_TOPOLOGY_INFO = 0x04,
+ TA_COMMAND_XGMI__GET_PEER_LINKS = 0x0B
};
/* XGMI related enumerations */
@@ -75,6 +76,11 @@ struct ta_xgmi_node_info {
enum ta_xgmi_assigned_sdma_engine sdma_engine;
};
+struct ta_xgmi_peer_link_info {
+ uint64_t node_id;
+ uint8_t num_links;
+};
+
struct ta_xgmi_cmd_initialize_output {
uint32_t status;
};
@@ -97,6 +103,11 @@ struct ta_xgmi_cmd_get_topology_info_output {
struct ta_xgmi_node_info nodes[TA_XGMI__MAX_CONNECTED_NODES];
};
+struct ta_xgmi_cmd_get_peer_link_info_output {
+ uint32_t num_nodes;
+ struct ta_xgmi_peer_link_info nodes[TA_XGMI__MAX_CONNECTED_NODES];
+};
+
struct ta_xgmi_cmd_set_topology_info_input {
uint32_t num_nodes;
struct ta_xgmi_node_info nodes[TA_XGMI__MAX_CONNECTED_NODES];
@@ -115,6 +126,7 @@ union ta_xgmi_cmd_output {
struct ta_xgmi_cmd_get_node_id_output get_node_id;
struct ta_xgmi_cmd_get_hive_id_output get_hive_id;
struct ta_xgmi_cmd_get_topology_info_output get_topology_info;
+ struct ta_xgmi_cmd_get_peer_link_info_output get_link_info;
};
/**********************************************************/
--
2.25.1
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd
2021-07-16 16:43 [PATCH 1/3] drm/amdgpu: add psp command to get num xgmi links between direct peers Jonathan Kim
@ 2021-07-16 16:43 ` Jonathan Kim
2021-07-17 5:46 ` Felix Kuehling
2021-07-19 7:21 ` Lazar, Lijo
2021-07-16 16:43 ` [PATCH 3/3] drm/amdkfd: report pcie bandwidth " Jonathan Kim
1 sibling, 2 replies; 9+ messages in thread
From: Jonathan Kim @ 2021-07-16 16:43 UTC (permalink / raw)
To: amd-gfx; +Cc: Felix.Kuehling, Jonathan Kim
Report the min/max bandwidth in megabytes to the kfd for direct
xgmi connections only.
v2: change reporting from num links to bandwidth
Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23 ++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 12 +++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 ++
drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 12 +++++++++++
5 files changed, 50 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index bfab2f9fdd17..3978578a1c49 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -553,6 +553,29 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
return (uint8_t)ret;
}
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)dst, *peer_adev;
+ int num_links;
+
+ if (adev->asic_type != CHIP_ALDEBARAN)
+ return 0;
+
+ if (src)
+ peer_adev = (struct amdgpu_device *)src;
+
+ num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev, peer_adev);
+ if (num_links < 0) {
+ DRM_ERROR("amdgpu: failed to get xgmi num links between node %d and %d. ret = %d\n",
+ adev->gmc.xgmi.physical_node_id,
+ peer_adev->gmc.xgmi.physical_node_id, num_links);
+ num_links = 0;
+ }
+
+ /* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for bandwidth. */
+ return (num_links * 16 * 25000)/BITS_PER_BYTE;
+}
+
uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
{
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index 81264517d532..e12fccb2d2c4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
+int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
/* Read user wptr from a specified user address space with page fault
* disabled. The memory must be pinned and mapped to the hardware when
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index 8567d5d77346..258cf86b32f6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
return -EINVAL;
}
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev)
+{
+ struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
+ int i;
+
+ for (i = 0 ; i < top->num_nodes; ++i)
+ if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
+ return top->nodes[i].num_links;
+ return -EINVAL;
+}
+
int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
{
struct psp_xgmi_topology_info *top_info;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 12969c0830d5..d2189bf7d428 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev);
int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
struct amdgpu_device *peer_adev);
+int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
+ struct amdgpu_device *peer_adev);
uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
uint64_t addr);
static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index c6b02aee4993..40ce6239c813 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
sub_type_hdr->num_hops_xgmi = 1;
+ if (adev->asic_type == CHIP_ALDEBARAN) {
+ sub_type_hdr->minimum_bandwidth_mbs =
+ amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
+ kdev->kgd, NULL, true);
+ sub_type_hdr->maximum_bandwidth_mbs =
+ sub_type_hdr->minimum_bandwidth_mbs;
+ }
} else {
sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
}
@@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
sub_type_hdr->proximity_domain_to = proximity_domain_to;
sub_type_hdr->num_hops_xgmi =
amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
+ sub_type_hdr->maximum_bandwidth_mbs =
+ amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, peer_kdev->kgd, false);
+ sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr->maximum_bandwidth_mbs ?
+ amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, NULL, true) : 0;
+
return 0;
}
--
2.25.1
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 9+ messages in thread
* [PATCH 3/3] drm/amdkfd: report pcie bandwidth to the kfd
2021-07-16 16:43 [PATCH 1/3] drm/amdgpu: add psp command to get num xgmi links between direct peers Jonathan Kim
2021-07-16 16:43 ` [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd Jonathan Kim
@ 2021-07-16 16:43 ` Jonathan Kim
2021-07-17 5:37 ` Felix Kuehling
1 sibling, 1 reply; 9+ messages in thread
From: Jonathan Kim @ 2021-07-16 16:43 UTC (permalink / raw)
To: amd-gfx; +Cc: Felix.Kuehling, Jonathan Kim
Similar to xGMI reporting the min/max bandwidth between direct peers, PCIe
will report the min/max bandwidth to the KFD.
v2: change to bandwidth
Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 61 ++++++++++++++++++++++
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 4 ++
3 files changed, 66 insertions(+)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 3978578a1c49..b7db52f1a9d1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -21,6 +21,7 @@
*/
#include "amdgpu_amdkfd.h"
+#include "amd_pcie.h"
#include "amd_shared.h"
#include "amdgpu.h"
@@ -576,6 +577,66 @@ int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev
return (num_links * 16 * 25000)/BITS_PER_BYTE;
}
+int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min)
+{
+ struct amdgpu_device *adev = (struct amdgpu_device *)dev;
+ int num_lanes_shift = is_min ? ffs(adev->pm.pcie_mlw_mask >>
+ CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1 :
+ fls(adev->pm.pcie_mlw_mask >>
+ CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
+ int gen_speed_shift = is_min ? ffs(adev->pm.pcie_gen_mask >>
+ CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1 :
+ fls(adev->pm.pcie_gen_mask >>
+ CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1;
+ uint32_t num_lanes_mask = (1UL << num_lanes_shift) << CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT;
+ uint32_t gen_speed_mask = (1UL << gen_speed_shift) << CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;
+ int num_lanes_factor = 0, gen_speed_mbits_factor = 0;
+
+ switch (num_lanes_mask) {
+ case CAIL_PCIE_LINK_WIDTH_SUPPORT_X1:
+ num_lanes_factor = 1;
+ break;
+ case CAIL_PCIE_LINK_WIDTH_SUPPORT_X2:
+ num_lanes_factor = 2;
+ break;
+ case CAIL_PCIE_LINK_WIDTH_SUPPORT_X4:
+ num_lanes_factor = 4;
+ break;
+ case CAIL_PCIE_LINK_WIDTH_SUPPORT_X8:
+ num_lanes_factor = 8;
+ break;
+ case CAIL_PCIE_LINK_WIDTH_SUPPORT_X12:
+ num_lanes_factor = 12;
+ break;
+ case CAIL_PCIE_LINK_WIDTH_SUPPORT_X16:
+ num_lanes_factor = 16;
+ break;
+ case CAIL_PCIE_LINK_WIDTH_SUPPORT_X32:
+ num_lanes_factor = 32;
+ break;
+ }
+
+ switch (gen_speed_mask) {
+ case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1:
+ gen_speed_mbits_factor = 2500;
+ break;
+ case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2:
+ gen_speed_mbits_factor = 5000;
+ break;
+ case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3:
+ gen_speed_mbits_factor = 8000;
+ break;
+ case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4:
+ gen_speed_mbits_factor = 16000;
+ break;
+ case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5:
+ gen_speed_mbits_factor = 32000;
+ break;
+ }
+
+ return (num_lanes_factor * gen_speed_mbits_factor)/BITS_PER_BYTE;
+}
+
uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
{
struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index e12fccb2d2c4..5d73f3108d13 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -227,6 +227,7 @@ uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
+int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min);
/* Read user wptr from a specified user address space with page fault
* disabled. The memory must be pinned and mapped to the hardware when
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 40ce6239c813..eada22b9ea69 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -1998,6 +1998,10 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
}
} else {
sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
+ sub_type_hdr->minimum_bandwidth_mbs =
+ amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, true);
+ sub_type_hdr->maximum_bandwidth_mbs =
+ amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, false);
}
sub_type_hdr->proximity_domain_from = proximity_domain;
--
2.25.1
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 9+ messages in thread
* Re: [PATCH 3/3] drm/amdkfd: report pcie bandwidth to the kfd
2021-07-16 16:43 ` [PATCH 3/3] drm/amdkfd: report pcie bandwidth " Jonathan Kim
@ 2021-07-17 5:37 ` Felix Kuehling
2021-07-19 15:48 ` Kim, Jonathan
0 siblings, 1 reply; 9+ messages in thread
From: Felix Kuehling @ 2021-07-17 5:37 UTC (permalink / raw)
To: Jonathan Kim, amd-gfx
Am 2021-07-16 um 12:43 p.m. schrieb Jonathan Kim:
> Similar to xGMI reporting the min/max bandwidth between direct peers, PCIe
> will report the min/max bandwidth to the KFD.
>
> v2: change to bandwidth
>
> Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 61 ++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 4 ++
> 3 files changed, 66 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> index 3978578a1c49..b7db52f1a9d1 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -21,6 +21,7 @@
> */
>
> #include "amdgpu_amdkfd.h"
> +#include "amd_pcie.h"
> #include "amd_shared.h"
>
> #include "amdgpu.h"
> @@ -576,6 +577,66 @@ int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev
> return (num_links * 16 * 25000)/BITS_PER_BYTE;
> }
>
> +int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min)
> +{
> + struct amdgpu_device *adev = (struct amdgpu_device *)dev;
> + int num_lanes_shift = is_min ? ffs(adev->pm.pcie_mlw_mask >>
> + CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1 :
> + fls(adev->pm.pcie_mlw_mask >>
> + CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
> + int gen_speed_shift = is_min ? ffs(adev->pm.pcie_gen_mask >>
> + CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1 :
> + fls(adev->pm.pcie_gen_mask >>
> + CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1;
The shifting is not necessary because you undo it below. I think this
would do the trick and be more readable:
int num_lanes_shift = (is_min ? ffs(adev->pm.pcie_mlw_mask) :
fls(adev->pm.pcie_mlw_mask)) - 1;
int gen_speed_shift = (is_min ? ffs(adev->pm.pcie_gen_mask) :
fls(adev->pm.pcie_gen_mask)) - 1;
uint32_t num_lanes_mask = 1 << num_lanes_shift;
uint32_t gen_speed_mask = 1 << gen_speed_shift;
With that fixed, this patch is
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
> + uint32_t num_lanes_mask = (1UL << num_lanes_shift) << CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT;
> + uint32_t gen_speed_mask = (1UL << gen_speed_shift) << CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;
> + int num_lanes_factor = 0, gen_speed_mbits_factor = 0;
> +
> + switch (num_lanes_mask) {
> + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X1:
> + num_lanes_factor = 1;
> + break;
> + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X2:
> + num_lanes_factor = 2;
> + break;
> + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X4:
> + num_lanes_factor = 4;
> + break;
> + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X8:
> + num_lanes_factor = 8;
> + break;
> + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X12:
> + num_lanes_factor = 12;
> + break;
> + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X16:
> + num_lanes_factor = 16;
> + break;
> + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X32:
> + num_lanes_factor = 32;
> + break;
> + }
> +
> + switch (gen_speed_mask) {
> + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1:
> + gen_speed_mbits_factor = 2500;
> + break;
> + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2:
> + gen_speed_mbits_factor = 5000;
> + break;
> + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3:
> + gen_speed_mbits_factor = 8000;
> + break;
> + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4:
> + gen_speed_mbits_factor = 16000;
> + break;
> + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5:
> + gen_speed_mbits_factor = 32000;
> + break;
> + }
> +
> + return (num_lanes_factor * gen_speed_mbits_factor)/BITS_PER_BYTE;
> +}
> +
> uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
> {
> struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index e12fccb2d2c4..5d73f3108d13 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -227,6 +227,7 @@ uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
> int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
> uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
> int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
> +int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev, bool is_min);
>
> /* Read user wptr from a specified user address space with page fault
> * disabled. The memory must be pinned and mapped to the hardware when
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> index 40ce6239c813..eada22b9ea69 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> @@ -1998,6 +1998,10 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
> }
> } else {
> sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
> + sub_type_hdr->minimum_bandwidth_mbs =
> + amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, true);
> + sub_type_hdr->maximum_bandwidth_mbs =
> + amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, false);
> }
>
> sub_type_hdr->proximity_domain_from = proximity_domain;
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd
2021-07-16 16:43 ` [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd Jonathan Kim
@ 2021-07-17 5:46 ` Felix Kuehling
2021-07-19 16:02 ` Kim, Jonathan
2021-07-19 7:21 ` Lazar, Lijo
1 sibling, 1 reply; 9+ messages in thread
From: Felix Kuehling @ 2021-07-17 5:46 UTC (permalink / raw)
To: Jonathan Kim, amd-gfx
Am 2021-07-16 um 12:43 p.m. schrieb Jonathan Kim:
> Report the min/max bandwidth in megabytes to the kfd for direct
> xgmi connections only.
By "direct XGMI connections", you mean this doesn't work for links with
more than one hop? Will that spew out DRM_ERROR messages for such links?
Then it's probably better to downgrade that to an INFO.
>
> v2: change reporting from num links to bandwidth
>
> Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
This patch is OK to provide bandwidth information on Aldebaran. What can
we do on older GPUs? Can we assume num_links = 1? Or maybe have some
hard-coded numbers depending on the number of nodes in the hive?
Either way, patch 1 and 2 are
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23 ++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 12 +++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 ++
> drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 12 +++++++++++
> 5 files changed, 50 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> index bfab2f9fdd17..3978578a1c49 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -553,6 +553,29 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
> return (uint8_t)ret;
> }
>
> +int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min)
> +{
> + struct amdgpu_device *adev = (struct amdgpu_device *)dst, *peer_adev;
> + int num_links;
> +
> + if (adev->asic_type != CHIP_ALDEBARAN)
> + return 0;
> +
> + if (src)
> + peer_adev = (struct amdgpu_device *)src;
> +
> + num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev, peer_adev);
> + if (num_links < 0) {
> + DRM_ERROR("amdgpu: failed to get xgmi num links between node %d and %d. ret = %d\n",
> + adev->gmc.xgmi.physical_node_id,
> + peer_adev->gmc.xgmi.physical_node_id, num_links);
> + num_links = 0;
> + }
> +
> + /* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for bandwidth. */
> + return (num_links * 16 * 25000)/BITS_PER_BYTE;
> +}
> +
> uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
> {
> struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index 81264517d532..e12fccb2d2c4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
> uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
> int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
> uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
> +int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
>
> /* Read user wptr from a specified user address space with page fault
> * disabled. The memory must be pinned and mapped to the hardware when
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> index 8567d5d77346..258cf86b32f6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> @@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
> return -EINVAL;
> }
>
> +int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev)
> +{
> + struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
> + int i;
> +
> + for (i = 0 ; i < top->num_nodes; ++i)
> + if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
> + return top->nodes[i].num_links;
> + return -EINVAL;
> +}
> +
> int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
> {
> struct psp_xgmi_topology_info *top_info;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> index 12969c0830d5..d2189bf7d428 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> @@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev);
> int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
> int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
> struct amdgpu_device *peer_adev);
> +int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev);
> uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
> uint64_t addr);
> static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> index c6b02aee4993..40ce6239c813 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> @@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
> sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
> sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
> sub_type_hdr->num_hops_xgmi = 1;
> + if (adev->asic_type == CHIP_ALDEBARAN) {
> + sub_type_hdr->minimum_bandwidth_mbs =
> + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
> + kdev->kgd, NULL, true);
> + sub_type_hdr->maximum_bandwidth_mbs =
> + sub_type_hdr->minimum_bandwidth_mbs;
> + }
> } else {
> sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
> }
> @@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
> sub_type_hdr->proximity_domain_to = proximity_domain_to;
> sub_type_hdr->num_hops_xgmi =
> amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
> + sub_type_hdr->maximum_bandwidth_mbs =
> + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, peer_kdev->kgd, false);
> + sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr->maximum_bandwidth_mbs ?
> + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, NULL, true) : 0;
> +
> return 0;
> }
>
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 9+ messages in thread
* Re: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd
2021-07-16 16:43 ` [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd Jonathan Kim
2021-07-17 5:46 ` Felix Kuehling
@ 2021-07-19 7:21 ` Lazar, Lijo
2021-07-19 15:50 ` Kim, Jonathan
1 sibling, 1 reply; 9+ messages in thread
From: Lazar, Lijo @ 2021-07-19 7:21 UTC (permalink / raw)
To: Jonathan Kim, amd-gfx; +Cc: Felix.Kuehling
On 7/16/2021 10:13 PM, Jonathan Kim wrote:
> Report the min/max bandwidth in megabytes to the kfd for direct
> xgmi connections only.
>
> v2: change reporting from num links to bandwidth
>
> Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23 ++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 12 +++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 ++
> drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 12 +++++++++++
> 5 files changed, 50 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> index bfab2f9fdd17..3978578a1c49 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> @@ -553,6 +553,29 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s
> return (uint8_t)ret;
> }
>
> +int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min)
> +{
> + struct amdgpu_device *adev = (struct amdgpu_device *)dst, *peer_adev;
> + int num_links;
> +
> + if (adev->asic_type != CHIP_ALDEBARAN)
> + return 0;
> +
> + if (src)
> + peer_adev = (struct amdgpu_device *)src;
> +
> + num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev, peer_adev);
> + if (num_links < 0) {
> + DRM_ERROR("amdgpu: failed to get xgmi num links between node %d and %d. ret = %d\n",
> + adev->gmc.xgmi.physical_node_id,
> + peer_adev->gmc.xgmi.physical_node_id, num_links);
> + num_links = 0;
> + }
> +
> + /* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for bandwidth. */
> + return (num_links * 16 * 25000)/BITS_PER_BYTE;
Instead of having ASIC family checks and bandwidth info in interface
functions, better to have this info come from base layer (amdgpu_xgmi or
xgmi ip). That will help to handle other ASICs.
Thanks,
Lijo
> uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd)
> {
> struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> index 81264517d532..e12fccb2d2c4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> @@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct kgd_dev *kgd);
> uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
> int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
> uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *src);
> +int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct kgd_dev *src, bool is_min);
>
> /* Read user wptr from a specified user address space with page fault
> * disabled. The memory must be pinned and mapped to the hardware when
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> index 8567d5d77346..258cf86b32f6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> @@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
> return -EINVAL;
> }
>
> +int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev)
> +{
> + struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info;
> + int i;
> +
> + for (i = 0 ; i < top->num_nodes; ++i)
> + if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
> + return top->nodes[i].num_links;
> + return -EINVAL;
> +}
> +
> int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
> {
> struct psp_xgmi_topology_info *top_info;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> index 12969c0830d5..d2189bf7d428 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> @@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev);
> int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
> int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
> struct amdgpu_device *peer_adev);
> +int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
> + struct amdgpu_device *peer_adev);
> uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev,
> uint64_t addr);
> static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> index c6b02aee4993..40ce6239c813 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> @@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
> sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
> sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
> sub_type_hdr->num_hops_xgmi = 1;
> + if (adev->asic_type == CHIP_ALDEBARAN) {
> + sub_type_hdr->minimum_bandwidth_mbs =
> + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
> + kdev->kgd, NULL, true);
> + sub_type_hdr->maximum_bandwidth_mbs =
> + sub_type_hdr->minimum_bandwidth_mbs;
> + }
> } else {
> sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
> }
> @@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
> sub_type_hdr->proximity_domain_to = proximity_domain_to;
> sub_type_hdr->num_hops_xgmi =
> amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd, peer_kdev->kgd);
> + sub_type_hdr->maximum_bandwidth_mbs =
> + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, peer_kdev->kgd, false);
> + sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr->maximum_bandwidth_mbs ?
> + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd, NULL, true) : 0;
> +
> return 0;
> }
>
>
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 9+ messages in thread
* RE: [PATCH 3/3] drm/amdkfd: report pcie bandwidth to the kfd
2021-07-17 5:37 ` Felix Kuehling
@ 2021-07-19 15:48 ` Kim, Jonathan
0 siblings, 0 replies; 9+ messages in thread
From: Kim, Jonathan @ 2021-07-19 15:48 UTC (permalink / raw)
To: Kuehling, Felix, amd-gfx
[AMD Official Use Only]
> -----Original Message-----
> From: Kuehling, Felix <Felix.Kuehling@amd.com>
> Sent: Saturday, July 17, 2021 1:37 AM
> To: Kim, Jonathan <Jonathan.Kim@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [PATCH 3/3] drm/amdkfd: report pcie bandwidth to the kfd
>
> Am 2021-07-16 um 12:43 p.m. schrieb Jonathan Kim:
> > Similar to xGMI reporting the min/max bandwidth between direct peers,
> > PCIe will report the min/max bandwidth to the KFD.
> >
> > v2: change to bandwidth
> >
> > Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
> > ---
> > drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 61
> > ++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> > drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 4 ++
> > 3 files changed, 66 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > index 3978578a1c49..b7db52f1a9d1 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > @@ -21,6 +21,7 @@
> > */
> >
> > #include "amdgpu_amdkfd.h"
> > +#include "amd_pcie.h"
> > #include "amd_shared.h"
> >
> > #include "amdgpu.h"
> > @@ -576,6 +577,66 @@ int
> amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst, struct
> kgd_dev
> > return (num_links * 16 * 25000)/BITS_PER_BYTE; }
> >
> > +int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev,
> bool
> > +is_min) {
> > + struct amdgpu_device *adev = (struct amdgpu_device *)dev;
> > + int num_lanes_shift = is_min ? ffs(adev->pm.pcie_mlw_mask >>
> > +
> CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1 :
> > + fls(adev->pm.pcie_mlw_mask >>
> > +
> CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT) - 1;
> > + int gen_speed_shift = is_min ? ffs(adev->pm.pcie_gen_mask >>
> > +
> CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1 :
> > + fls(adev->pm.pcie_gen_mask >>
> > +
> CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT) - 1;
>
> The shifting is not necessary because you undo it below. I think this would
> do the trick and be more readable:
>
> int num_lanes_shift = (is_min ? ffs(adev->pm.pcie_mlw_mask) :
> fls(adev->pm.pcie_mlw_mask)) - 1;
> int gen_speed_shift = (is_min ? ffs(adev->pm.pcie_gen_mask) :
> fls(adev->pm.pcie_gen_mask)) - 1;
Ok thanks for the review and suggestion. I've adjusted your suggestion by masking pcie_gen_mask with CAIL_PCIE_LINK_SPEED_SUPPORT_MASK as the mask sets some non-speed related lower bits.
Thanks,
Jon
> uint32_t num_lanes_mask = 1 << num_lanes_shift;
> uint32_t gen_speed_mask = 1 << gen_speed_shift;
>
> With that fixed, this patch is
>
> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
>
>
> > + uint32_t num_lanes_mask = (1UL << num_lanes_shift) <<
> CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT;
> > + uint32_t gen_speed_mask = (1UL << gen_speed_shift) <<
> CAIL_PCIE_LINK_SPEED_SUPPORT_SHIFT;
> > + int num_lanes_factor = 0, gen_speed_mbits_factor = 0;
> > +
> > + switch (num_lanes_mask) {
> > + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X1:
> > + num_lanes_factor = 1;
> > + break;
> > + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X2:
> > + num_lanes_factor = 2;
> > + break;
> > + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X4:
> > + num_lanes_factor = 4;
> > + break;
> > + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X8:
> > + num_lanes_factor = 8;
> > + break;
> > + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X12:
> > + num_lanes_factor = 12;
> > + break;
> > + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X16:
> > + num_lanes_factor = 16;
> > + break;
> > + case CAIL_PCIE_LINK_WIDTH_SUPPORT_X32:
> > + num_lanes_factor = 32;
> > + break;
> > + }
> > +
> > + switch (gen_speed_mask) {
> > + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1:
> > + gen_speed_mbits_factor = 2500;
> > + break;
> > + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2:
> > + gen_speed_mbits_factor = 5000;
> > + break;
> > + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3:
> > + gen_speed_mbits_factor = 8000;
> > + break;
> > + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4:
> > + gen_speed_mbits_factor = 16000;
> > + break;
> > + case CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5:
> > + gen_speed_mbits_factor = 32000;
> > + break;
> > + }
> > +
> > + return (num_lanes_factor *
> gen_speed_mbits_factor)/BITS_PER_BYTE;
> > +}
> > +
> > uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev
> *kgd)
> > {
> > struct amdgpu_device *adev = (struct amdgpu_device *)kgd; diff --git
> > a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > index e12fccb2d2c4..5d73f3108d13 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > @@ -227,6 +227,7 @@ uint32_t amdgpu_amdkfd_get_asic_rev_id(struct
> > kgd_dev *kgd); int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
> > uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct
> > kgd_dev *src); int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct
> > kgd_dev *dst, struct kgd_dev *src, bool is_min);
> > +int amdgpu_amdkfd_get_pcie_bandwidth_mbytes(struct kgd_dev *dev,
> bool
> > +is_min);
> >
> > /* Read user wptr from a specified user address space with page fault
> > * disabled. The memory must be pinned and mapped to the hardware
> > when diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > index 40ce6239c813..eada22b9ea69 100644
> > --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > @@ -1998,6 +1998,10 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int
> *avail_size,
> > }
> > } else {
> > sub_type_hdr->io_interface_type =
> CRAT_IOLINK_TYPE_PCIEXPRESS;
> > + sub_type_hdr->minimum_bandwidth_mbs =
> > +
> amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, true);
> > + sub_type_hdr->maximum_bandwidth_mbs =
> > +
> amdgpu_amdkfd_get_pcie_bandwidth_mbytes(kdev->kgd, false);
> > }
> >
> > sub_type_hdr->proximity_domain_from = proximity_domain;
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 9+ messages in thread
* RE: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd
2021-07-19 7:21 ` Lazar, Lijo
@ 2021-07-19 15:50 ` Kim, Jonathan
0 siblings, 0 replies; 9+ messages in thread
From: Kim, Jonathan @ 2021-07-19 15:50 UTC (permalink / raw)
To: Lazar, Lijo, amd-gfx; +Cc: Kuehling, Felix
[AMD Official Use Only]
> -----Original Message-----
> From: Lazar, Lijo <Lijo.Lazar@amd.com>
> Sent: Monday, July 19, 2021 3:22 AM
> To: Kim, Jonathan <Jonathan.Kim@amd.com>; amd-
> gfx@lists.freedesktop.org
> Cc: Kuehling, Felix <Felix.Kuehling@amd.com>
> Subject: Re: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between
> direct peers to the kfd
>
>
>
> On 7/16/2021 10:13 PM, Jonathan Kim wrote:
> > Report the min/max bandwidth in megabytes to the kfd for direct xgmi
> > connections only.
> >
> > v2: change reporting from num links to bandwidth
> >
> > Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
> > ---
> > drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23
> ++++++++++++++++++++++
> > drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> > drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 12 +++++++++++
> > drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 ++
> > drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 12 +++++++++++
> > 5 files changed, 50 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > index bfab2f9fdd17..3978578a1c49 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > @@ -553,6 +553,29 @@ uint8_t
> amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev
> *s
> > return (uint8_t)ret;
> > }
> >
> > +int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst,
> > +struct kgd_dev *src, bool is_min) {
> > + struct amdgpu_device *adev = (struct amdgpu_device *)dst,
> *peer_adev;
> > + int num_links;
> > +
> > + if (adev->asic_type != CHIP_ALDEBARAN)
> > + return 0;
> > +
> > + if (src)
> > + peer_adev = (struct amdgpu_device *)src;
> > +
> > + num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev,
> peer_adev);
> > + if (num_links < 0) {
> > + DRM_ERROR("amdgpu: failed to get xgmi num links between
> node %d and %d. ret = %d\n",
> > + adev->gmc.xgmi.physical_node_id,
> > + peer_adev->gmc.xgmi.physical_node_id, num_links);
> > + num_links = 0;
> > + }
> > +
> > + /* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for
> bandwidth. */
> > + return (num_links * 16 * 25000)/BITS_PER_BYTE;
>
> Instead of having ASIC family checks and bandwidth info in interface
> functions, better to have this info come from base layer (amdgpu_xgmi or
> xgmi ip). That will help to handle other ASICs.
Ok. We can revisit this as a follow up. Maybe the full solution is a link width/speed support mask analogous to pcie.
Thanks,
Jon
>
> Thanks,
> Lijo
>
> > uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev
> *kgd)
> > {
> > struct amdgpu_device *adev = (struct amdgpu_device *)kgd; diff
> > --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > index 81264517d532..e12fccb2d2c4 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > @@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct
> kgd_dev *kgd);
> > uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
> > int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd);
> > uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst,
> > struct kgd_dev *src);
> > +int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst,
> > +struct kgd_dev *src, bool is_min);
> >
> > /* Read user wptr from a specified user address space with page fault
> > * disabled. The memory must be pinned and mapped to the hardware
> > when diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> > index 8567d5d77346..258cf86b32f6 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> > @@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct
> amdgpu_device *adev,
> > return -EINVAL;
> > }
> >
> > +int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
> > + struct amdgpu_device *peer_adev)
> > +{
> > + struct psp_xgmi_topology_info *top = &adev-
> >psp.xgmi_context.top_info;
> > + int i;
> > +
> > + for (i = 0 ; i < top->num_nodes; ++i)
> > + if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
> > + return top->nodes[i].num_links;
> > + return -EINVAL;
> > +}
> > +
> > int amdgpu_xgmi_add_device(struct amdgpu_device *adev)
> > {
> > struct psp_xgmi_topology_info *top_info; diff --git
> > a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> > index 12969c0830d5..d2189bf7d428 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> > @@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct
> amdgpu_device *adev);
> > int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate);
> > int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
> > struct amdgpu_device *peer_adev);
> > +int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
> > + struct amdgpu_device *peer_adev);
> > uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device
> *adev,
> > uint64_t addr);
> > static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
> > diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > index c6b02aee4993..40ce6239c813 100644
> > --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > @@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int
> *avail_size,
> > sub_type_hdr->flags |=
> CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
> > sub_type_hdr->io_interface_type =
> CRAT_IOLINK_TYPE_XGMI;
> > sub_type_hdr->num_hops_xgmi = 1;
> > + if (adev->asic_type == CHIP_ALDEBARAN) {
> > + sub_type_hdr->minimum_bandwidth_mbs =
> > +
> amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
> > + kdev->kgd, NULL,
> true);
> > + sub_type_hdr->maximum_bandwidth_mbs =
> > + sub_type_hdr-
> >minimum_bandwidth_mbs;
> > + }
> > } else {
> > sub_type_hdr->io_interface_type =
> CRAT_IOLINK_TYPE_PCIEXPRESS;
> > }
> > @@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int
> *avail_size,
> > sub_type_hdr->proximity_domain_to = proximity_domain_to;
> > sub_type_hdr->num_hops_xgmi =
> > amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd,
> peer_kdev->kgd);
> > + sub_type_hdr->maximum_bandwidth_mbs =
> > + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd,
> peer_kdev->kgd, false);
> > + sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr-
> >maximum_bandwidth_mbs ?
> > + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd,
> NULL, true) : 0;
> > +
> > return 0;
> > }
> >
> >
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 9+ messages in thread
* RE: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd
2021-07-17 5:46 ` Felix Kuehling
@ 2021-07-19 16:02 ` Kim, Jonathan
0 siblings, 0 replies; 9+ messages in thread
From: Kim, Jonathan @ 2021-07-19 16:02 UTC (permalink / raw)
To: Kuehling, Felix, amd-gfx
[AMD Official Use Only]
> -----Original Message-----
> From: Kuehling, Felix <Felix.Kuehling@amd.com>
> Sent: Saturday, July 17, 2021 1:47 AM
> To: Kim, Jonathan <Jonathan.Kim@amd.com>; amd-
> gfx@lists.freedesktop.org
> Subject: Re: [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between
> direct peers to the kfd
>
> Am 2021-07-16 um 12:43 p.m. schrieb Jonathan Kim:
> > Report the min/max bandwidth in megabytes to the kfd for direct xgmi
> > connections only.
>
> By "direct XGMI connections", you mean this doesn't work for links with
> more than one hop? Will that spew out DRM_ERROR messages for such links?
> Then it's probably better to downgrade that to an INFO.
No DRM_ERROR only happens if psp fails on invoke.
I've added footnotes to the description and code to clear this up.
Non-adjacent peers return num_links as 0 since indirect route is unknown and linkage is asymmetrical.
>
>
> >
> > v2: change reporting from num links to bandwidth
> >
> > Signed-off-by: Jonathan Kim <jonathan.kim@amd.com>
>
> This patch is OK to provide bandwidth information on Aldebaran. What can
> we do on older GPUs? Can we assume num_links = 1? Or maybe have some
> hard-coded numbers depending on the number of nodes in the hive?
We could assume num_links = 1 but that wouldn't represent certain non-Aldebaran setups well.
For non-Aldebaran min/max bandwidth, we may be able to get away with setting non-zero values on non-adjacent peers since setup is symmetrical to date but that may raise questions on why Aldebaran indirect min/max-bandwidth is 0. For consistency, we'd have to use num_hops then to check directness.
Maybe it's worth making a bid to the FW team to support all other chips moving forward ...
Thanks,
Jon
>
> Either way, patch 1 and 2 are
>
> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
>
>
> > ---
> > drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 23
> > ++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 +
> > drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c | 12 +++++++++++
> > drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h | 2 ++
> > drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 12 +++++++++++
> > 5 files changed, 50 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > index bfab2f9fdd17..3978578a1c49 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
> > @@ -553,6 +553,29 @@ uint8_t
> amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev
> *s
> > return (uint8_t)ret;
> > }
> >
> > +int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst,
> > +struct kgd_dev *src, bool is_min) {
> > + struct amdgpu_device *adev = (struct amdgpu_device *)dst,
> *peer_adev;
> > + int num_links;
> > +
> > + if (adev->asic_type != CHIP_ALDEBARAN)
> > + return 0;
> > +
> > + if (src)
> > + peer_adev = (struct amdgpu_device *)src;
> > +
> > + num_links = is_min ? 1 : amdgpu_xgmi_get_num_links(adev,
> peer_adev);
> > + if (num_links < 0) {
> > + DRM_ERROR("amdgpu: failed to get xgmi num links between
> node %d and %d. ret = %d\n",
> > + adev->gmc.xgmi.physical_node_id,
> > + peer_adev->gmc.xgmi.physical_node_id, num_links);
> > + num_links = 0;
> > + }
> > +
> > + /* Aldebaran xGMI DPM is defeatured so assume x16 x 25Gbps for
> bandwidth. */
> > + return (num_links * 16 * 25000)/BITS_PER_BYTE; }
> > +
> > uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev
> *kgd)
> > {
> > struct amdgpu_device *adev = (struct amdgpu_device *)kgd; diff --git
> > a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > index 81264517d532..e12fccb2d2c4 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
> > @@ -226,6 +226,7 @@ uint32_t amdgpu_amdkfd_get_num_gws(struct
> kgd_dev
> > *kgd); uint32_t amdgpu_amdkfd_get_asic_rev_id(struct kgd_dev *kgd);
> > int amdgpu_amdkfd_get_noretry(struct kgd_dev *kgd); uint8_t
> > amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct
> kgd_dev
> > *src);
> > +int amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(struct kgd_dev *dst,
> > +struct kgd_dev *src, bool is_min);
> >
> > /* Read user wptr from a specified user address space with page fault
> > * disabled. The memory must be pinned and mapped to the hardware
> > when diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> > index 8567d5d77346..258cf86b32f6 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
> > @@ -486,6 +486,18 @@ int amdgpu_xgmi_get_hops_count(struct
> amdgpu_device *adev,
> > return -EINVAL;
> > }
> >
> > +int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
> > + struct amdgpu_device *peer_adev)
> > +{
> > + struct psp_xgmi_topology_info *top = &adev-
> >psp.xgmi_context.top_info;
> > + int i;
> > +
> > + for (i = 0 ; i < top->num_nodes; ++i)
> > + if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id)
> > + return top->nodes[i].num_links;
> > + return -EINVAL;
> > +}
> > +
> > int amdgpu_xgmi_add_device(struct amdgpu_device *adev) {
> > struct psp_xgmi_topology_info *top_info; diff --git
> > a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> > index 12969c0830d5..d2189bf7d428 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
> > @@ -59,6 +59,8 @@ int amdgpu_xgmi_remove_device(struct
> amdgpu_device
> > *adev); int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int
> > pstate); int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev,
> > struct amdgpu_device *peer_adev);
> > +int amdgpu_xgmi_get_num_links(struct amdgpu_device *adev,
> > + struct amdgpu_device *peer_adev);
> > uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device
> *adev,
> > uint64_t addr);
> > static inline bool amdgpu_xgmi_same_hive(struct amdgpu_device *adev,
> > diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > index c6b02aee4993..40ce6239c813 100644
> > --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> > @@ -1989,6 +1989,13 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int
> *avail_size,
> > sub_type_hdr->flags |=
> CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
> > sub_type_hdr->io_interface_type =
> CRAT_IOLINK_TYPE_XGMI;
> > sub_type_hdr->num_hops_xgmi = 1;
> > + if (adev->asic_type == CHIP_ALDEBARAN) {
> > + sub_type_hdr->minimum_bandwidth_mbs =
> > +
> amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(
> > + kdev->kgd, NULL,
> true);
> > + sub_type_hdr->maximum_bandwidth_mbs =
> > + sub_type_hdr-
> >minimum_bandwidth_mbs;
> > + }
> > } else {
> > sub_type_hdr->io_interface_type =
> CRAT_IOLINK_TYPE_PCIEXPRESS;
> > }
> > @@ -2033,6 +2040,11 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int
> *avail_size,
> > sub_type_hdr->proximity_domain_to = proximity_domain_to;
> > sub_type_hdr->num_hops_xgmi =
> > amdgpu_amdkfd_get_xgmi_hops_count(kdev->kgd,
> peer_kdev->kgd);
> > + sub_type_hdr->maximum_bandwidth_mbs =
> > + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd,
> peer_kdev->kgd, false);
> > + sub_type_hdr->minimum_bandwidth_mbs = sub_type_hdr-
> >maximum_bandwidth_mbs ?
> > + amdgpu_amdkfd_get_xgmi_bandwidth_mbytes(kdev->kgd,
> NULL, true) : 0;
> > +
> > return 0;
> > }
> >
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 9+ messages in thread
end of thread, other threads:[~2021-07-19 16:02 UTC | newest]
Thread overview: 9+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-07-16 16:43 [PATCH 1/3] drm/amdgpu: add psp command to get num xgmi links between direct peers Jonathan Kim
2021-07-16 16:43 ` [PATCH 2/3] drm/amdkfd: report xgmi bandwidth between direct peers to the kfd Jonathan Kim
2021-07-17 5:46 ` Felix Kuehling
2021-07-19 16:02 ` Kim, Jonathan
2021-07-19 7:21 ` Lazar, Lijo
2021-07-19 15:50 ` Kim, Jonathan
2021-07-16 16:43 ` [PATCH 3/3] drm/amdkfd: report pcie bandwidth " Jonathan Kim
2021-07-17 5:37 ` Felix Kuehling
2021-07-19 15:48 ` Kim, Jonathan
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.