[PATCH] drm/amdgpu: fix modprobe failure of the secondary GPU when GDDR6 training enabled(V4)

[PATCH] drm/amdgpu: fix modprobe failure of the secondary GPU when GDDR6 training enabled(V4)

[Tianci Yin]

From: "Tianci.Yin" <tianci.yin@amd.com>

[why]
In dual GPUs scenario, stolen_size is assigned to zero on the secondary GPU,
since there is no pre-OS console using that memory. Then the bottom region of
VRAM was allocated as GTT, unfortunately a small region of bottom VRAM was
encroached by UMC firmware during GDDR6 BIST training, this cause page fault.

[how]
Forcing stolen_size to 3MB, then the bottom region of VRAM was
allocated as stolen memory, GTT corruption avoid.

Change-Id: I310a72ba0402994defbe50839842a8edb025a868
Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 27 ++++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index c91dd602d5f1..e4b2f9bcaeb7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -60,6 +60,11 @@
  */
 #define AMDGPU_GMC_FAULT_TIMEOUT	5000ULL
 
+/*
+ * Default stolen memory size, 1024 * 768 * 4
+ */
+#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE	0x300000ULL
+
 struct firmware;
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index 5ad89bb6f3ba..14961f1ebfab 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -566,7 +566,12 @@ static int gmc_v10_0_late_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int r;
 
-	amdgpu_bo_late_init(adev);
+	/*
+	 * Can't free the stolen VGA memory when it might be used for memory
+	 * training again.
+	 */
+	if (!adev->fw_vram_usage.mem_train_support)
+		amdgpu_bo_late_init(adev);
 
 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
 	if (r)
@@ -750,6 +755,19 @@ static int gmc_v10_0_sw_init(void *handle)
 
 	adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
 
+	/*
+	 * In dual GPUs scenario, stolen_size is assigned to zero on the
+	 * secondary GPU, since there is no pre-OS console using that memory.
+	 * Then the bottom region of VRAM was allocated as GTT, unfortunately a
+	 * small region of bottom VRAM was encroached by UMC firmware during
+	 * GDDR6 BIST training, this cause page fault.
+	 * The page fault can be fixed by forcing stolen_size to 3MB, then the
+	 * bottom region of VRAM was allocated as stolen memory, GTT corruption
+	 * avoid.
+	 */
+	adev->gmc.stolen_size = max(adev->gmc.stolen_size,
+				    AMDGPU_STOLEN_VGA_DEFAULT_SIZE);
+
 	/* Memory manager */
 	r = amdgpu_bo_init(adev);
 	if (r)
@@ -789,6 +807,13 @@ static void gmc_v10_0_gart_fini(struct amdgpu_device *adev)
 static int gmc_v10_0_sw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	void *stolen_vga_buf;
+
+	/*
+	 * Free the stolen memory if it wasn't already freed in late_init
+	 * because of memory training.
+	 */
+	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, &stolen_vga_buf);
 
 	amdgpu_vm_manager_fini(adev);
 	gmc_v10_0_gart_fini(adev);
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the secondary GPU when GDDR6 training enabled(V4)

[Christian König]

Am 10.01.20 um 06:30 schrieb Tianci Yin:
> From: "Tianci.Yin" <tianci.yin@amd.com>
>
> [why]
> In dual GPUs scenario, stolen_size is assigned to zero on the secondary GPU,
> since there is no pre-OS console using that memory. Then the bottom region of
> VRAM was allocated as GTT, unfortunately a small region of bottom VRAM was
> encroached by UMC firmware during GDDR6 BIST training, this cause page fault.
>
> [how]
> Forcing stolen_size to 3MB, then the bottom region of VRAM was
> allocated as stolen memory, GTT corruption avoid.
>
> Change-Id: I310a72ba0402994defbe50839842a8edb025a868
> Signed-off-by: Tianci.Yin <tianci.yin@amd.com>

Reviewed-by: Christian König <christian.koenig@amd.com>

> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
>   drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 27 ++++++++++++++++++++++++-
>   2 files changed, 31 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> index c91dd602d5f1..e4b2f9bcaeb7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> @@ -60,6 +60,11 @@
>    */
>   #define AMDGPU_GMC_FAULT_TIMEOUT	5000ULL
>   
> +/*
> + * Default stolen memory size, 1024 * 768 * 4
> + */
> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE	0x300000ULL
> +
>   struct firmware;
>   
>   /*
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> index 5ad89bb6f3ba..14961f1ebfab 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> @@ -566,7 +566,12 @@ static int gmc_v10_0_late_init(void *handle)
>   	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
>   	int r;
>   
> -	amdgpu_bo_late_init(adev);
> +	/*
> +	 * Can't free the stolen VGA memory when it might be used for memory
> +	 * training again.
> +	 */
> +	if (!adev->fw_vram_usage.mem_train_support)
> +		amdgpu_bo_late_init(adev);
>   
>   	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
>   	if (r)
> @@ -750,6 +755,19 @@ static int gmc_v10_0_sw_init(void *handle)
>   
>   	adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
>   
> +	/*
> +	 * In dual GPUs scenario, stolen_size is assigned to zero on the
> +	 * secondary GPU, since there is no pre-OS console using that memory.
> +	 * Then the bottom region of VRAM was allocated as GTT, unfortunately a
> +	 * small region of bottom VRAM was encroached by UMC firmware during
> +	 * GDDR6 BIST training, this cause page fault.
> +	 * The page fault can be fixed by forcing stolen_size to 3MB, then the
> +	 * bottom region of VRAM was allocated as stolen memory, GTT corruption
> +	 * avoid.
> +	 */
> +	adev->gmc.stolen_size = max(adev->gmc.stolen_size,
> +				    AMDGPU_STOLEN_VGA_DEFAULT_SIZE);
> +
>   	/* Memory manager */
>   	r = amdgpu_bo_init(adev);
>   	if (r)
> @@ -789,6 +807,13 @@ static void gmc_v10_0_gart_fini(struct amdgpu_device *adev)
>   static int gmc_v10_0_sw_fini(void *handle)
>   {
>   	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +	void *stolen_vga_buf;
> +
> +	/*
> +	 * Free the stolen memory if it wasn't already freed in late_init
> +	 * because of memory training.
> +	 */
> +	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, &stolen_vga_buf);
>   
>   	amdgpu_vm_manager_fini(adev);
>   	gmc_v10_0_gart_fini(adev);

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the secondary GPU when GDDR6 training enabled(V4)

[Alex Deucher]

On Fri, Jan 10, 2020 at 12:30 AM Tianci Yin <tianci.yin@amd.com> wrote:
>
> From: "Tianci.Yin" <tianci.yin@amd.com>
>
> [why]
> In dual GPUs scenario, stolen_size is assigned to zero on the secondary GPU,
> since there is no pre-OS console using that memory. Then the bottom region of
> VRAM was allocated as GTT, unfortunately a small region of bottom VRAM was
> encroached by UMC firmware during GDDR6 BIST training, this cause page fault.
>
> [how]
> Forcing stolen_size to 3MB, then the bottom region of VRAM was
> allocated as stolen memory, GTT corruption avoid.
>
> Change-Id: I310a72ba0402994defbe50839842a8edb025a868
> Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  5 +++++
>  drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 27 ++++++++++++++++++++++++-
>  2 files changed, 31 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> index c91dd602d5f1..e4b2f9bcaeb7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> @@ -60,6 +60,11 @@
>   */
>  #define AMDGPU_GMC_FAULT_TIMEOUT       5000ULL
>
> +/*
> + * Default stolen memory size, 1024 * 768 * 4
> + */
> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE 0x300000ULL

It would be better to call this
AMDGPU_STOLEN_BIST_TRAINING_DEFAULT_SIZE.  Also, I sounds like we are
probably going to have to assume that all of vram could be touched and
adjust our sequnce appropriately.  I think we are just getting lucky
with this size because most imporant kernel structures are in the
lower area of vram.

Alex

> +
>  struct firmware;
>
>  /*
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> index 5ad89bb6f3ba..14961f1ebfab 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> @@ -566,7 +566,12 @@ static int gmc_v10_0_late_init(void *handle)
>         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
>         int r;
>
> -       amdgpu_bo_late_init(adev);
> +       /*
> +        * Can't free the stolen VGA memory when it might be used for memory
> +        * training again.
> +        */
> +       if (!adev->fw_vram_usage.mem_train_support)
> +               amdgpu_bo_late_init(adev);
>
>         r = amdgpu_gmc_allocate_vm_inv_eng(adev);
>         if (r)
> @@ -750,6 +755,19 @@ static int gmc_v10_0_sw_init(void *handle)
>
>         adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
>
> +       /*
> +        * In dual GPUs scenario, stolen_size is assigned to zero on the
> +        * secondary GPU, since there is no pre-OS console using that memory.
> +        * Then the bottom region of VRAM was allocated as GTT, unfortunately a
> +        * small region of bottom VRAM was encroached by UMC firmware during
> +        * GDDR6 BIST training, this cause page fault.
> +        * The page fault can be fixed by forcing stolen_size to 3MB, then the
> +        * bottom region of VRAM was allocated as stolen memory, GTT corruption
> +        * avoid.
> +        */
> +       adev->gmc.stolen_size = max(adev->gmc.stolen_size,
> +                                   AMDGPU_STOLEN_VGA_DEFAULT_SIZE);
> +
>         /* Memory manager */
>         r = amdgpu_bo_init(adev);
>         if (r)
> @@ -789,6 +807,13 @@ static void gmc_v10_0_gart_fini(struct amdgpu_device *adev)
>  static int gmc_v10_0_sw_fini(void *handle)
>  {
>         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +       void *stolen_vga_buf;
> +
> +       /*
> +        * Free the stolen memory if it wasn't already freed in late_init
> +        * because of memory training.
> +        */
> +       amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, &stolen_vga_buf);
>
>         amdgpu_vm_manager_fini(adev);
>         gmc_v10_0_gart_fini(adev);
> --
> 2.17.1
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx