[PATCH 04/13] drm/amdgpu/gmc9: Adjust xgmi offset

[PATCH 04/13] drm/amdgpu/gmc9: Adjust xgmi offset

[shaoyunl]

From: Alex Deucher <alexander.deucher@amd.com>

On hives with xgmi enabled, the fb_location aperture is a size
which defines the total framebuffer size of all nodes in the
hive.  Each GPU in the hive has the same view via the fb_location
aperture.  GPU0 starts at offset (0 * segment size),
GPU1 starts at offset (1 * segment size), etc.

For access to local vram on each GPU, we need to take this offset into
account. This including on setting up GPUVM page table  and  GART table

Acked-by: Huang Rui <ray.huang@amd.com>
Acked-by: Slava Abramov <slava.abramov@amd.com>
Signed-off-by: Shaoyun Liu <Shaoyun.Liu@amd.com>
Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c  | 4 ++++
 drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c | 3 +++
 drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    | 6 ++++++
 3 files changed, 13 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 6acdeeb..a95b615 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -158,6 +158,10 @@ void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
 	if ((size_bf >= mc->gart_size && size_bf < size_af) ||
 	    (size_af < mc->gart_size))
 		mc->gart_start = 0;
+	else if (mc->xgmi.num_physical_nodes)
+		mc->gart_start = mc->vram_start +
+			(mc->xgmi.num_physical_nodes - mc->xgmi.physical_node_id)
+			* mc->xgmi.node_segment_size;
 	else
 		mc->gart_start = mc->mc_mask - mc->gart_size + 1;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
index d4170cb..5e9ab8e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
@@ -44,6 +44,9 @@ int gfxhub_v1_1_get_xgmi_info(struct amdgpu_device *adev)
 			REG_GET_FIELD(xgmi_lfb_cntl, MC_VM_XGMI_LFB_CNTL, PF_LFB_REGION);
 		if (adev->gmc.xgmi.physical_node_id > 3)
 			return -EINVAL;
+		adev->gmc.xgmi.node_segment_size = REG_GET_FIELD(
+			RREG32_SOC15(GC, 0, mmMC_VM_XGMI_LFB_SIZE),
+			MC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index 3529c55..0da89ba 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -770,12 +770,18 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
 	u64 base = 0;
 	if (!amdgpu_sriov_vf(adev))
 		base = mmhub_v1_0_get_fb_location(adev);
+	/* add the xgmi offset of the physical node */
+	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
 	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
 	amdgpu_gmc_gart_location(adev, mc);
 	if (!amdgpu_sriov_vf(adev))
 		amdgpu_gmc_agp_location(adev, mc);
 	/* base offset of vram pages */
 	adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
+
+	/* XXX: add the xgmi offset of the physical node? */
+	adev->vm_manager.vram_base_offset +=
+		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
 }
 
 /**
-- 
2.7.4

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 04/13] drm/amdgpu/gmc9: Adjust xgmi offset

[Christian König]

Am 05.09.2018 um 17:29 schrieb shaoyunl:
> From: Alex Deucher <alexander.deucher@amd.com>
>
> On hives with xgmi enabled, the fb_location aperture is a size
> which defines the total framebuffer size of all nodes in the
> hive.  Each GPU in the hive has the same view via the fb_location
> aperture.  GPU0 starts at offset (0 * segment size),
> GPU1 starts at offset (1 * segment size), etc.
>
> For access to local vram on each GPU, we need to take this offset into
> account. This including on setting up GPUVM page table  and  GART table
>
> Acked-by: Huang Rui <ray.huang@amd.com>
> Acked-by: Slava Abramov <slava.abramov@amd.com>
> Signed-off-by: Shaoyun Liu <Shaoyun.Liu@amd.com>
> Signed-off-by: Alex Deucher <alexander.deucher@amd.com>
> Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c  | 4 ++++
>   drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c | 3 +++
>   drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c    | 6 ++++++
>   3 files changed, 13 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 6acdeeb..a95b615 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -158,6 +158,10 @@ void amdgpu_gmc_gart_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc)
>   	if ((size_bf >= mc->gart_size && size_bf < size_af) ||
>   	    (size_af < mc->gart_size))
>   		mc->gart_start = 0;
> +	else if (mc->xgmi.num_physical_nodes)
> +		mc->gart_start = mc->vram_start +
> +			(mc->xgmi.num_physical_nodes - mc->xgmi.physical_node_id)
> +			* mc->xgmi.node_segment_size;

That looks incomplete to me. The calculation above as well as the AGP 
code needs adjustment as well.

Probably best if some fb_start/fb_end fields to struct amdgpu_gmc and 
calculate that only once in amdgpu_gmc_vram_location.

Christian.

>   	else
>   		mc->gart_start = mc->mc_mask - mc->gart_size + 1;
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
> index d4170cb..5e9ab8e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v1_1.c
> @@ -44,6 +44,9 @@ int gfxhub_v1_1_get_xgmi_info(struct amdgpu_device *adev)
>   			REG_GET_FIELD(xgmi_lfb_cntl, MC_VM_XGMI_LFB_CNTL, PF_LFB_REGION);
>   		if (adev->gmc.xgmi.physical_node_id > 3)
>   			return -EINVAL;
> +		adev->gmc.xgmi.node_segment_size = REG_GET_FIELD(
> +			RREG32_SOC15(GC, 0, mmMC_VM_XGMI_LFB_SIZE),
> +			MC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24;
>   	}
>   
>   	return 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> index 3529c55..0da89ba 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
> @@ -770,12 +770,18 @@ static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
>   	u64 base = 0;
>   	if (!amdgpu_sriov_vf(adev))
>   		base = mmhub_v1_0_get_fb_location(adev);
> +	/* add the xgmi offset of the physical node */
> +	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
>   	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
>   	amdgpu_gmc_gart_location(adev, mc);
>   	if (!amdgpu_sriov_vf(adev))
>   		amdgpu_gmc_agp_location(adev, mc);
>   	/* base offset of vram pages */
>   	adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
> +
> +	/* XXX: add the xgmi offset of the physical node? */
> +	adev->vm_manager.vram_base_offset +=
> +		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
>   }
>   
>   /**

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx