* [PATCH] drm/amdkfd: Only add bi-directional iolink on GPU with XGMI or largebar
@ 2018-09-07 21:37 shaoyunl
[not found] ` <1536356236-2957-1-git-send-email-Shaoyun.Liu-5C7GfCeVMHo@public.gmane.org>
0 siblings, 1 reply; 2+ messages in thread
From: shaoyunl @ 2018-09-07 21:37 UTC (permalink / raw)
To: amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW; +Cc: shaoyunl
Change-Id: Ibb6a89ed878fffccb9a8bb4032b07a10ee298a99
Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
---
drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 15 +++++++++------
drivers/gpu/drm/amd/amdkfd/kfd_crat.h | 3 ++-
drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 1 +
3 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index 130db4d..d4560f1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -353,8 +353,8 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
id_from = iolink->proximity_domain_from;
id_to = iolink->proximity_domain_to;
- pr_debug("Found IO link entry in CRAT table with id_from=%d\n",
- id_from);
+ pr_debug("Found IO link entry in CRAT table with id_from=%d, id_to %d\n",
+ id_from, id_to);
list_for_each_entry(dev, device_list, list) {
if (id_from == dev->proximity_domain) {
props = kfd_alloc_struct(props);
@@ -391,12 +391,12 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
/* CPU topology is created before GPUs are detected, so CPU->GPU
* links are not built at that time. If a PCIe type is discovered, it
* means a GPU is detected and we are adding GPU->CPU to the topology.
- * At this time, also add the corresponded CPU->GPU link.
+ * At this time, also add the corresponded CPU->GPU link if GPU
+ * is large bar.
* For xGMI, we only added the link with one direction in the crat
* table, add corresponded reversed direction link now.
*/
- if (props && (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS ||
- props->iolink_type == CRAT_IOLINK_TYPE_XGMI)) {
+ if (props && (iolink->flags & CRAT_IOLINK_FLAGS_BI_DIRECTIONAL)) {
to_dev = kfd_topology_device_by_proximity_domain(id_to);
if (!to_dev)
return -ENODEV;
@@ -1057,6 +1057,8 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
+ if (kfd_dev_is_large_bar(kdev))
+ sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
/* Fill in IOLINK subtype.
* TODO: Fill-in other fields of iolink subtype
@@ -1088,7 +1090,8 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
- sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
+ sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED |
+ CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
sub_type_hdr->proximity_domain_from = proximity_domain_from;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
index 7a93aeb..7c3f192 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
@@ -232,7 +232,8 @@ struct crat_subtype_ccompute {
#define CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT (1 << 2)
#define CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT (1 << 3)
#define CRAT_IOLINK_FLAGS_NO_PEER_TO_PEER_DMA (1 << 4)
-#define CRAT_IOLINK_FLAGS_RESERVED_MASK 0xffffffe0
+#define CRAT_IOLINK_FLAGS_BI_DIRECTIONAL (1 << 31)
+#define CRAT_IOLINK_FLAGS_RESERVED_MASK 0x7fffffe0
/*
* IO interface types
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 6a5418f..05283c9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -696,6 +696,7 @@ struct amdkfd_ioctl_desc {
unsigned int cmd_drv;
const char *name;
};
+bool kfd_dev_is_large_bar(struct kfd_dev *dev);
int kfd_process_create_wq(void);
void kfd_process_destroy_wq(void);
--
2.7.4
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 2+ messages in thread
* Re: [PATCH] drm/amdkfd: Only add bi-directional iolink on GPU with XGMI or largebar
[not found] ` <1536356236-2957-1-git-send-email-Shaoyun.Liu-5C7GfCeVMHo@public.gmane.org>
@ 2018-09-07 23:16 ` Felix Kuehling
0 siblings, 0 replies; 2+ messages in thread
From: Felix Kuehling @ 2018-09-07 23:16 UTC (permalink / raw)
To: shaoyunl, amd-gfx-PD4FTy7X32lNgt0PjOBp9y5qC8QIuHrW, Huang, JinhuiEric
[+Eric, see my comment below about iolink atomic flags]
On 2018-09-07 05:37 PM, shaoyunl wrote:
> Change-Id: Ibb6a89ed878fffccb9a8bb4032b07a10ee298a99
> Signed-off-by: shaoyunl <Shaoyun.Liu@amd.com>
See one comment inline. It's not directly related to this change so this
is Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
> ---
> drivers/gpu/drm/amd/amdkfd/kfd_crat.c | 15 +++++++++------
> drivers/gpu/drm/amd/amdkfd/kfd_crat.h | 3 ++-
> drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 1 +
> 3 files changed, 12 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> index 130db4d..d4560f1 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
> @@ -353,8 +353,8 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
> id_from = iolink->proximity_domain_from;
> id_to = iolink->proximity_domain_to;
>
> - pr_debug("Found IO link entry in CRAT table with id_from=%d\n",
> - id_from);
> + pr_debug("Found IO link entry in CRAT table with id_from=%d, id_to %d\n",
> + id_from, id_to);
> list_for_each_entry(dev, device_list, list) {
> if (id_from == dev->proximity_domain) {
> props = kfd_alloc_struct(props);
> @@ -391,12 +391,12 @@ static int kfd_parse_subtype_iolink(struct crat_subtype_iolink *iolink,
> /* CPU topology is created before GPUs are detected, so CPU->GPU
> * links are not built at that time. If a PCIe type is discovered, it
> * means a GPU is detected and we are adding GPU->CPU to the topology.
> - * At this time, also add the corresponded CPU->GPU link.
> + * At this time, also add the corresponded CPU->GPU link if GPU
> + * is large bar.
> * For xGMI, we only added the link with one direction in the crat
> * table, add corresponded reversed direction link now.
> */
> - if (props && (props->iolink_type == CRAT_IOLINK_TYPE_PCIEXPRESS ||
> - props->iolink_type == CRAT_IOLINK_TYPE_XGMI)) {
> + if (props && (iolink->flags & CRAT_IOLINK_FLAGS_BI_DIRECTIONAL)) {
Hmm, looks like we're not populating props->flags at the moment. That's
probably an oversight we should correct. Then we could also move the
atomic flag code from kfd_fill_iolink_non_crat_info into
kfd_fill_gpu_direct_io_link for the GPU->CPU link and this function for
the reverse.
Regards,
Felix
> to_dev = kfd_topology_device_by_proximity_domain(id_to);
> if (!to_dev)
> return -ENODEV;
> @@ -1057,6 +1057,8 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size,
> sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
> sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
> sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
> + if (kfd_dev_is_large_bar(kdev))
> + sub_type_hdr->flags |= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
>
> /* Fill in IOLINK subtype.
> * TODO: Fill-in other fields of iolink subtype
> @@ -1088,7 +1090,8 @@ static int kfd_fill_gpu_xgmi_link_to_gpu(int *avail_size,
>
> sub_type_hdr->type = CRAT_SUBTYPE_IOLINK_AFFINITY;
> sub_type_hdr->length = sizeof(struct crat_subtype_iolink);
> - sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED;
> + sub_type_hdr->flags |= CRAT_SUBTYPE_FLAGS_ENABLED |
> + CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
>
> sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
> sub_type_hdr->proximity_domain_from = proximity_domain_from;
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
> index 7a93aeb..7c3f192 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.h
> @@ -232,7 +232,8 @@ struct crat_subtype_ccompute {
> #define CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT (1 << 2)
> #define CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT (1 << 3)
> #define CRAT_IOLINK_FLAGS_NO_PEER_TO_PEER_DMA (1 << 4)
> -#define CRAT_IOLINK_FLAGS_RESERVED_MASK 0xffffffe0
> +#define CRAT_IOLINK_FLAGS_BI_DIRECTIONAL (1 << 31)
> +#define CRAT_IOLINK_FLAGS_RESERVED_MASK 0x7fffffe0
>
> /*
> * IO interface types
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 6a5418f..05283c9 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -696,6 +696,7 @@ struct amdkfd_ioctl_desc {
> unsigned int cmd_drv;
> const char *name;
> };
> +bool kfd_dev_is_large_bar(struct kfd_dev *dev);
>
> int kfd_process_create_wq(void);
> void kfd_process_destroy_wq(void);
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2018-09-07 23:16 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2018-09-07 21:37 [PATCH] drm/amdkfd: Only add bi-directional iolink on GPU with XGMI or largebar shaoyunl
[not found] ` <1536356236-2957-1-git-send-email-Shaoyun.Liu-5C7GfCeVMHo@public.gmane.org>
2018-09-07 23:16 ` Felix Kuehling
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.