[PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Tianci Yin]

From: "Tianci.Yin" <tianci.yin@amd.com>

[why]
In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
then the bottom region of VRAM was allocated as GTT, unfortunately
a small region of bottom VRAM was encroached by UMC firmware during
GDDR6 BIST training, this cause pagefault.

[how]
Forcing stolen_size to 3MB, then the bottom region of VRAM was
allocated as stolen memory, GTT corruption avoid.
The stolen memory of the 2nd GPU will be free in late_init phase,
no memory wasted.

Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
 drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index c91dd602d5f1..440b793316df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -60,6 +60,11 @@
  */
 #define AMDGPU_GMC_FAULT_TIMEOUT	5000ULL
 
+/*
+ * Default stolen memory size, 1024 * 768 * 4
+ */
+#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE	0x300000
+
 struct firmware;
 
 /*
@@ -192,6 +197,7 @@ struct amdgpu_gmc {
 	uint32_t                srbm_soft_reset;
 	bool			prt_warning;
 	uint64_t		stolen_size;
+	bool			stolen_temp_reserved;
 	/* apertures */
 	u64			shared_aperture_start;
 	u64			shared_aperture_end;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index 7dc8c068c62a..0c96b67d6ca7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int r;
 
+	if (adev->gmc.stolen_temp_reserved) {
+		amdgpu_bo_late_init(adev);
+		adev->gmc.stolen_temp_reserved = false;
+	}
+
 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
 	if (r)
 		return r;
@@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
 		return r;
 
 	adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
+	/*
+	 * In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
+	 * then the bottom region of VRAM was allocated as GTT, unfortunately
+	 * a small region of bottom VRAM was encroached by UMC firmware during
+	 * GDDR6 BIST training, this cause pagefault.
+	 * The page fault can be fixed by forcing stolen_size to 3MB, then the bottom
+	 * region of VRAM was allocated as stolen memory, GTT corruption avoid.
+	 * The stolen memory of the 2nd GPU will be free in late_init phase,
+	 * no memory wasted.
+	 */
+	if (adev->fw_vram_usage.mem_train_support &&
+		adev->gmc.stolen_size == 0) {
+		adev->gmc.stolen_size = AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
+		adev->gmc.stolen_temp_reserved = true;
+	} else
+		adev->gmc.stolen_temp_reserved = false;
 
 	/* Memory manager */
 	r = amdgpu_bo_init(adev);
-- 
2.17.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Christian König]

Am 08.01.20 um 13:36 schrieb Tianci Yin:
> From: "Tianci.Yin" <tianci.yin@amd.com>
>
> [why]
> In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> then the bottom region of VRAM was allocated as GTT, unfortunately
> a small region of bottom VRAM was encroached by UMC firmware during
> GDDR6 BIST training, this cause pagefault.

What I'm missing here is why is the stolen size zero on the 2nd GPU?

Maybe we need to read the stolen size after posting the GPU instead?

Regards,
Christian.

>
> [how]
> Forcing stolen_size to 3MB, then the bottom region of VRAM was
> allocated as stolen memory, GTT corruption avoid.
> The stolen memory of the 2nd GPU will be free in late_init phase,
> no memory wasted.
>
> Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
> Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
>   drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
>   2 files changed, 27 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> index c91dd602d5f1..440b793316df 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> @@ -60,6 +60,11 @@
>    */
>   #define AMDGPU_GMC_FAULT_TIMEOUT	5000ULL
>   
> +/*
> + * Default stolen memory size, 1024 * 768 * 4
> + */
> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE	0x300000
> +
>   struct firmware;
>   
>   /*
> @@ -192,6 +197,7 @@ struct amdgpu_gmc {
>   	uint32_t                srbm_soft_reset;
>   	bool			prt_warning;
>   	uint64_t		stolen_size;
> +	bool			stolen_temp_reserved;
>   	/* apertures */
>   	u64			shared_aperture_start;
>   	u64			shared_aperture_end;
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> index 7dc8c068c62a..0c96b67d6ca7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> @@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
>   	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
>   	int r;
>   
> +	if (adev->gmc.stolen_temp_reserved) {
> +		amdgpu_bo_late_init(adev);
> +		adev->gmc.stolen_temp_reserved = false;
> +	}
> +
>   	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
>   	if (r)
>   		return r;
> @@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
>   		return r;
>   
>   	adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
> +	/*
> +	 * In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> +	 * then the bottom region of VRAM was allocated as GTT, unfortunately
> +	 * a small region of bottom VRAM was encroached by UMC firmware during
> +	 * GDDR6 BIST training, this cause pagefault.
> +	 * The page fault can be fixed by forcing stolen_size to 3MB, then the bottom
> +	 * region of VRAM was allocated as stolen memory, GTT corruption avoid.
> +	 * The stolen memory of the 2nd GPU will be free in late_init phase,
> +	 * no memory wasted.
> +	 */
> +	if (adev->fw_vram_usage.mem_train_support &&
> +		adev->gmc.stolen_size == 0) {
> +		adev->gmc.stolen_size = AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
> +		adev->gmc.stolen_temp_reserved = true;
> +	} else
> +		adev->gmc.stolen_temp_reserved = false;
>   
>   	/* Memory manager */
>   	r = amdgpu_bo_init(adev);

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Alex Deucher]

On Wed, Jan 8, 2020 at 7:52 AM Christian König <christian.koenig@amd.com> wrote:
>
> Am 08.01.20 um 13:36 schrieb Tianci Yin:
> > From: "Tianci.Yin" <tianci.yin@amd.com>
> >
> > [why]
> > In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> > then the bottom region of VRAM was allocated as GTT, unfortunately
> > a small region of bottom VRAM was encroached by UMC firmware during
> > GDDR6 BIST training, this cause pagefault.
>
> What I'm missing here is why is the stolen size zero on the 2nd GPU?
>
> Maybe we need to read the stolen size after posting the GPU instead?

There is no stolen memory on secondary GPUs because there is no pre-OS
console using that memory.

Alex

>
> Regards,
> Christian.
>
> >
> > [how]
> > Forcing stolen_size to 3MB, then the bottom region of VRAM was
> > allocated as stolen memory, GTT corruption avoid.
> > The stolen memory of the 2nd GPU will be free in late_init phase,
> > no memory wasted.
> >
> > Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
> > Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
> >   drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
> >   2 files changed, 27 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> > index c91dd602d5f1..440b793316df 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> > @@ -60,6 +60,11 @@
> >    */
> >   #define AMDGPU_GMC_FAULT_TIMEOUT    5000ULL
> >
> > +/*
> > + * Default stolen memory size, 1024 * 768 * 4
> > + */
> > +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE       0x300000
> > +
> >   struct firmware;
> >
> >   /*
> > @@ -192,6 +197,7 @@ struct amdgpu_gmc {
> >       uint32_t                srbm_soft_reset;
> >       bool                    prt_warning;
> >       uint64_t                stolen_size;
> > +     bool                    stolen_temp_reserved;
> >       /* apertures */
> >       u64                     shared_aperture_start;
> >       u64                     shared_aperture_end;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> > index 7dc8c068c62a..0c96b67d6ca7 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> > @@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
> >       struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> >       int r;
> >
> > +     if (adev->gmc.stolen_temp_reserved) {
> > +             amdgpu_bo_late_init(adev);
> > +             adev->gmc.stolen_temp_reserved = false;
> > +     }
> > +
> >       r = amdgpu_gmc_allocate_vm_inv_eng(adev);
> >       if (r)
> >               return r;
> > @@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
> >               return r;
> >
> >       adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
> > +     /*
> > +      * In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> > +      * then the bottom region of VRAM was allocated as GTT, unfortunately
> > +      * a small region of bottom VRAM was encroached by UMC firmware during
> > +      * GDDR6 BIST training, this cause pagefault.
> > +      * The page fault can be fixed by forcing stolen_size to 3MB, then the bottom
> > +      * region of VRAM was allocated as stolen memory, GTT corruption avoid.
> > +      * The stolen memory of the 2nd GPU will be free in late_init phase,
> > +      * no memory wasted.
> > +      */
> > +     if (adev->fw_vram_usage.mem_train_support &&
> > +             adev->gmc.stolen_size == 0) {
> > +             adev->gmc.stolen_size = AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
> > +             adev->gmc.stolen_temp_reserved = true;
> > +     } else
> > +             adev->gmc.stolen_temp_reserved = false;
> >
> >       /* Memory manager */
> >       r = amdgpu_bo_init(adev);
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Christian König]

Am 08.01.20 um 15:49 schrieb Alex Deucher:
> On Wed, Jan 8, 2020 at 7:52 AM Christian König <christian.koenig@amd.com> wrote:
>> Am 08.01.20 um 13:36 schrieb Tianci Yin:
>>> From: "Tianci.Yin" <tianci.yin@amd.com>
>>>
>>> [why]
>>> In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
>>> then the bottom region of VRAM was allocated as GTT, unfortunately
>>> a small region of bottom VRAM was encroached by UMC firmware during
>>> GDDR6 BIST training, this cause pagefault.
>> What I'm missing here is why is the stolen size zero on the 2nd GPU?
>>
>> Maybe we need to read the stolen size after posting the GPU instead?
> There is no stolen memory on secondary GPUs because there is no pre-OS
> console using that memory.

Ah! Yeah that makes sense.

But in this case I would say we should change the patch like the following:

adev->gmc.stolen_size = min(adev->gmc.stolen_size, 
AMDGPU_STOLEN_VGA_DEFAULT_SIZE);

And in amdgpu_ttm_late_init():

/* Can't free the stolen VGA memory when it might be used for memory 
training again. */
if (!adev->fw_vram_usage.mem_train_support)
     amdgpu_bo_free_kernel(....


Regards,
Christian.


>
> Alex
>
>> Regards,
>> Christian.
>>
>>> [how]
>>> Forcing stolen_size to 3MB, then the bottom region of VRAM was
>>> allocated as stolen memory, GTT corruption avoid.
>>> The stolen memory of the 2nd GPU will be free in late_init phase,
>>> no memory wasted.
>>>
>>> Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
>>> Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
>>>    drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
>>>    2 files changed, 27 insertions(+)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>>> index c91dd602d5f1..440b793316df 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>>> @@ -60,6 +60,11 @@
>>>     */
>>>    #define AMDGPU_GMC_FAULT_TIMEOUT    5000ULL
>>>
>>> +/*
>>> + * Default stolen memory size, 1024 * 768 * 4
>>> + */
>>> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE       0x300000
>>> +
>>>    struct firmware;
>>>
>>>    /*
>>> @@ -192,6 +197,7 @@ struct amdgpu_gmc {
>>>        uint32_t                srbm_soft_reset;
>>>        bool                    prt_warning;
>>>        uint64_t                stolen_size;
>>> +     bool                    stolen_temp_reserved;
>>>        /* apertures */
>>>        u64                     shared_aperture_start;
>>>        u64                     shared_aperture_end;
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
>>> index 7dc8c068c62a..0c96b67d6ca7 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
>>> @@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
>>>        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
>>>        int r;
>>>
>>> +     if (adev->gmc.stolen_temp_reserved) {
>>> +             amdgpu_bo_late_init(adev);
>>> +             adev->gmc.stolen_temp_reserved = false;
>>> +     }
>>> +
>>>        r = amdgpu_gmc_allocate_vm_inv_eng(adev);
>>>        if (r)
>>>                return r;
>>> @@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
>>>                return r;
>>>
>>>        adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
>>> +     /*
>>> +      * In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
>>> +      * then the bottom region of VRAM was allocated as GTT, unfortunately
>>> +      * a small region of bottom VRAM was encroached by UMC firmware during
>>> +      * GDDR6 BIST training, this cause pagefault.
>>> +      * The page fault can be fixed by forcing stolen_size to 3MB, then the bottom
>>> +      * region of VRAM was allocated as stolen memory, GTT corruption avoid.
>>> +      * The stolen memory of the 2nd GPU will be free in late_init phase,
>>> +      * no memory wasted.
>>> +      */
>>> +     if (adev->fw_vram_usage.mem_train_support &&
>>> +             adev->gmc.stolen_size == 0) {
>>> +             adev->gmc.stolen_size = AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
>>> +             adev->gmc.stolen_temp_reserved = true;
>>> +     } else
>>> +             adev->gmc.stolen_temp_reserved = false;
>>>
>>>        /* Memory manager */
>>>        r = amdgpu_bo_init(adev);
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Yin, Tianci (Rico)]

[-- Attachment #1.1: Type: text/plain, Size: 6527 bytes --]

[AMD Official Use Only - Internal Distribution Only]

Thanks Alex and Christian!

Hi Christian,

On ASICs with gmc v10, I find amdgpu_bo_late_init() is not invoked, so stolen memory never get free, on other ASICs with gmc v9/v8/v7/v6, stolen memory was freed in gmc_v*_0_late_init(). I don't know why, are there some special reasons or just missed by coding?

Thanks!

Rico

________________________________
From: Christian König <ckoenig.leichtzumerken@gmail.com>
Sent: Wednesday, January 8, 2020 23:04
To: Alex Deucher <alexdeucher@gmail.com>; Koenig, Christian <Christian.Koenig@amd.com>
Cc: Long, Gang <Gang.Long@amd.com>; Wang, Kevin(Yang) <Kevin1.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Yin, Tianci (Rico) <Tianci.Yin@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Yuan, Xiaojie <Xiaojie.Yuan@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>
Subject: Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

Am 08.01.20 um 15:49 schrieb Alex Deucher:
> On Wed, Jan 8, 2020 at 7:52 AM Christian König <christian.koenig@amd.com> wrote:
>> Am 08.01.20 um 13:36 schrieb Tianci Yin:
>>> From: "Tianci.Yin" <tianci.yin@amd.com>
>>>
>>> [why]
>>> In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
>>> then the bottom region of VRAM was allocated as GTT, unfortunately
>>> a small region of bottom VRAM was encroached by UMC firmware during
>>> GDDR6 BIST training, this cause pagefault.
>> What I'm missing here is why is the stolen size zero on the 2nd GPU?
>>
>> Maybe we need to read the stolen size after posting the GPU instead?
> There is no stolen memory on secondary GPUs because there is no pre-OS
> console using that memory.

Ah! Yeah that makes sense.

But in this case I would say we should change the patch like the following:

adev->gmc.stolen_size = min(adev->gmc.stolen_size,
AMDGPU_STOLEN_VGA_DEFAULT_SIZE);

And in amdgpu_ttm_late_init():

/* Can't free the stolen VGA memory when it might be used for memory
training again. */
if (!adev->fw_vram_usage.mem_train_support)
     amdgpu_bo_free_kernel(....


Regards,
Christian.


>
> Alex
>
>> Regards,
>> Christian.
>>
>>> [how]
>>> Forcing stolen_size to 3MB, then the bottom region of VRAM was
>>> allocated as stolen memory, GTT corruption avoid.
>>> The stolen memory of the 2nd GPU will be free in late_init phase,
>>> no memory wasted.
>>>
>>> Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
>>> Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
>>>    drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
>>>    2 files changed, 27 insertions(+)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>>> index c91dd602d5f1..440b793316df 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
>>> @@ -60,6 +60,11 @@
>>>     */
>>>    #define AMDGPU_GMC_FAULT_TIMEOUT    5000ULL
>>>
>>> +/*
>>> + * Default stolen memory size, 1024 * 768 * 4
>>> + */
>>> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE       0x300000
>>> +
>>>    struct firmware;
>>>
>>>    /*
>>> @@ -192,6 +197,7 @@ struct amdgpu_gmc {
>>>        uint32_t                srbm_soft_reset;
>>>        bool                    prt_warning;
>>>        uint64_t                stolen_size;
>>> +     bool                    stolen_temp_reserved;
>>>        /* apertures */
>>>        u64                     shared_aperture_start;
>>>        u64                     shared_aperture_end;
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
>>> index 7dc8c068c62a..0c96b67d6ca7 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
>>> @@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
>>>        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
>>>        int r;
>>>
>>> +     if (adev->gmc.stolen_temp_reserved) {
>>> +             amdgpu_bo_late_init(adev);
>>> +             adev->gmc.stolen_temp_reserved = false;
>>> +     }
>>> +
>>>        r = amdgpu_gmc_allocate_vm_inv_eng(adev);
>>>        if (r)
>>>                return r;
>>> @@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
>>>                return r;
>>>
>>>        adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
>>> +     /*
>>> +      * In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
>>> +      * then the bottom region of VRAM was allocated as GTT, unfortunately
>>> +      * a small region of bottom VRAM was encroached by UMC firmware during
>>> +      * GDDR6 BIST training, this cause pagefault.
>>> +      * The page fault can be fixed by forcing stolen_size to 3MB, then the bottom
>>> +      * region of VRAM was allocated as stolen memory, GTT corruption avoid.
>>> +      * The stolen memory of the 2nd GPU will be free in late_init phase,
>>> +      * no memory wasted.
>>> +      */
>>> +     if (adev->fw_vram_usage.mem_train_support &&
>>> +             adev->gmc.stolen_size == 0) {
>>> +             adev->gmc.stolen_size = AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
>>> +             adev->gmc.stolen_temp_reserved = true;
>>> +     } else
>>> +             adev->gmc.stolen_temp_reserved = false;
>>>
>>>        /* Memory manager */
>>>        r = amdgpu_bo_init(adev);
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7Ctianci.yin%40amd.com%7C1365f16aef4c418f0db308d7944c0f26%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637140926723776401&amp;sdata=ubt7GGRaAvLzv%2F%2BUXxaTW%2FgGbKkRKWv6%2BB50fHHW3Xc%3D&amp;reserved=0
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7Ctianci.yin%40amd.com%7C1365f16aef4c418f0db308d7944c0f26%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637140926723776401&amp;sdata=ubt7GGRaAvLzv%2F%2BUXxaTW%2FgGbKkRKWv6%2BB50fHHW3Xc%3D&amp;reserved=0


[-- Attachment #1.2: Type: text/html, Size: 13092 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Alex Deucher]

On Wed, Jan 8, 2020 at 10:07 PM Yin, Tianci (Rico) <Tianci.Yin@amd.com> wrote:
>
> [AMD Official Use Only - Internal Distribution Only]
>
>
> Thanks Alex and Christian!
>
> Hi Christian,
>
> On ASICs with gmc v10, I find amdgpu_bo_late_init() is not invoked, so stolen memory never get free, on other ASICs with gmc v9/v8/v7/v6, stolen memory was freed in gmc_v*_0_late_init(). I don't know why, are there some special reasons or just missed by coding?
>

Looks like it should be added.  Possibly got lost when we merged the
navi code from the topic branch.

Alex

> Thanks!
>
> Rico
>
> ________________________________
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Wednesday, January 8, 2020 23:04
> To: Alex Deucher <alexdeucher@gmail.com>; Koenig, Christian <Christian.Koenig@amd.com>
> Cc: Long, Gang <Gang.Long@amd.com>; Wang, Kevin(Yang) <Kevin1.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Yin, Tianci (Rico) <Tianci.Yin@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Yuan, Xiaojie <Xiaojie.Yuan@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled
>
> Am 08.01.20 um 15:49 schrieb Alex Deucher:
> > On Wed, Jan 8, 2020 at 7:52 AM Christian König <christian.koenig@amd.com> wrote:
> >> Am 08.01.20 um 13:36 schrieb Tianci Yin:
> >>> From: "Tianci.Yin" <tianci.yin@amd.com>
> >>>
> >>> [why]
> >>> In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> >>> then the bottom region of VRAM was allocated as GTT, unfortunately
> >>> a small region of bottom VRAM was encroached by UMC firmware during
> >>> GDDR6 BIST training, this cause pagefault.
> >> What I'm missing here is why is the stolen size zero on the 2nd GPU?
> >>
> >> Maybe we need to read the stolen size after posting the GPU instead?
> > There is no stolen memory on secondary GPUs because there is no pre-OS
> > console using that memory.
>
> Ah! Yeah that makes sense.
>
> But in this case I would say we should change the patch like the following:
>
> adev->gmc.stolen_size = min(adev->gmc.stolen_size,
> AMDGPU_STOLEN_VGA_DEFAULT_SIZE);
>
> And in amdgpu_ttm_late_init():
>
> /* Can't free the stolen VGA memory when it might be used for memory
> training again. */
> if (!adev->fw_vram_usage.mem_train_support)
>      amdgpu_bo_free_kernel(....
>
>
> Regards,
> Christian.
>
>
> >
> > Alex
> >
> >> Regards,
> >> Christian.
> >>
> >>> [how]
> >>> Forcing stolen_size to 3MB, then the bottom region of VRAM was
> >>> allocated as stolen memory, GTT corruption avoid.
> >>> The stolen memory of the 2nd GPU will be free in late_init phase,
> >>> no memory wasted.
> >>>
> >>> Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
> >>> Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
> >>> ---
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
> >>>    drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
> >>>    2 files changed, 27 insertions(+)
> >>>
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> index c91dd602d5f1..440b793316df 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> @@ -60,6 +60,11 @@
> >>>     */
> >>>    #define AMDGPU_GMC_FAULT_TIMEOUT    5000ULL
> >>>
> >>> +/*
> >>> + * Default stolen memory size, 1024 * 768 * 4
> >>> + */
> >>> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE       0x300000
> >>> +
> >>>    struct firmware;
> >>>
> >>>    /*
> >>> @@ -192,6 +197,7 @@ struct amdgpu_gmc {
> >>>        uint32_t                srbm_soft_reset;
> >>>        bool                    prt_warning;
> >>>        uint64_t                stolen_size;
> >>> +     bool                    stolen_temp_reserved;
> >>>        /* apertures */
> >>>        u64                     shared_aperture_start;
> >>>        u64                     shared_aperture_end;
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> index 7dc8c068c62a..0c96b67d6ca7 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> @@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
> >>>        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> >>>        int r;
> >>>
> >>> +     if (adev->gmc.stolen_temp_reserved) {
> >>> +             amdgpu_bo_late_init(adev);
> >>> +             adev->gmc.stolen_temp_reserved = false;
> >>> +     }
> >>> +
> >>>        r = amdgpu_gmc_allocate_vm_inv_eng(adev);
> >>>        if (r)
> >>>                return r;
> >>> @@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
> >>>                return r;
> >>>
> >>>        adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
> >>> +     /*
> >>> +      * In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> >>> +      * then the bottom region of VRAM was allocated as GTT, unfortunately
> >>> +      * a small region of bottom VRAM was encroached by UMC firmware during
> >>> +      * GDDR6 BIST training, this cause pagefault.
> >>> +      * The page fault can be fixed by forcing stolen_size to 3MB, then the bottom
> >>> +      * region of VRAM was allocated as stolen memory, GTT corruption avoid.
> >>> +      * The stolen memory of the 2nd GPU will be free in late_init phase,
> >>> +      * no memory wasted.
> >>> +      */
> >>> +     if (adev->fw_vram_usage.mem_train_support &&
> >>> +             adev->gmc.stolen_size == 0) {
> >>> +             adev->gmc.stolen_size = AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
> >>> +             adev->gmc.stolen_temp_reserved = true;
> >>> +     } else
> >>> +             adev->gmc.stolen_temp_reserved = false;
> >>>
> >>>        /* Memory manager */
> >>>        r = amdgpu_bo_init(adev);
> >> _______________________________________________
> >> amd-gfx mailing list
> >> amd-gfx@lists.freedesktop.org
> >> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7Ctianci.yin%40amd.com%7C1365f16aef4c418f0db308d7944c0f26%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637140926723776401&amp;sdata=ubt7GGRaAvLzv%2F%2BUXxaTW%2FgGbKkRKWv6%2BB50fHHW3Xc%3D&amp;reserved=0
> > _______________________________________________
> > amd-gfx mailing list
> > amd-gfx@lists.freedesktop.org
> > https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7Ctianci.yin%40amd.com%7C1365f16aef4c418f0db308d7944c0f26%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637140926723776401&amp;sdata=ubt7GGRaAvLzv%2F%2BUXxaTW%2FgGbKkRKWv6%2BB50fHHW3Xc%3D&amp;reserved=0
>
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Yin, Tianci (Rico)]

[-- Attachment #1.1: Type: text/plain, Size: 7751 bytes --]

[AMD Official Use Only - Internal Distribution Only]

Ok, thanks very much Alex!
________________________________
From: Alex Deucher <alexdeucher@gmail.com>
Sent: Thursday, January 9, 2020 11:12
To: Yin, Tianci (Rico) <Tianci.Yin@amd.com>
Cc: Christian König <ckoenig.leichtzumerken@gmail.com>; Koenig, Christian <Christian.Koenig@amd.com>; Long, Gang <Gang.Long@amd.com>; Wang, Kevin(Yang) <Kevin1.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Yuan, Xiaojie <Xiaojie.Yuan@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>
Subject: Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

On Wed, Jan 8, 2020 at 10:07 PM Yin, Tianci (Rico) <Tianci.Yin@amd.com> wrote:
>
> [AMD Official Use Only - Internal Distribution Only]
>
>
> Thanks Alex and Christian!
>
> Hi Christian,
>
> On ASICs with gmc v10, I find amdgpu_bo_late_init() is not invoked, so stolen memory never get free, on other ASICs with gmc v9/v8/v7/v6, stolen memory was freed in gmc_v*_0_late_init(). I don't know why, are there some special reasons or just missed by coding?
>

Looks like it should be added.  Possibly got lost when we merged the
navi code from the topic branch.

Alex

> Thanks!
>
> Rico
>
> ________________________________
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Wednesday, January 8, 2020 23:04
> To: Alex Deucher <alexdeucher@gmail.com>; Koenig, Christian <Christian.Koenig@amd.com>
> Cc: Long, Gang <Gang.Long@amd.com>; Wang, Kevin(Yang) <Kevin1.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Yin, Tianci (Rico) <Tianci.Yin@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Yuan, Xiaojie <Xiaojie.Yuan@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled
>
> Am 08.01.20 um 15:49 schrieb Alex Deucher:
> > On Wed, Jan 8, 2020 at 7:52 AM Christian König <christian.koenig@amd.com> wrote:
> >> Am 08.01.20 um 13:36 schrieb Tianci Yin:
> >>> From: "Tianci.Yin" <tianci.yin@amd.com>
> >>>
> >>> [why]
> >>> In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> >>> then the bottom region of VRAM was allocated as GTT, unfortunately
> >>> a small region of bottom VRAM was encroached by UMC firmware during
> >>> GDDR6 BIST training, this cause pagefault.
> >> What I'm missing here is why is the stolen size zero on the 2nd GPU?
> >>
> >> Maybe we need to read the stolen size after posting the GPU instead?
> > There is no stolen memory on secondary GPUs because there is no pre-OS
> > console using that memory.
>
> Ah! Yeah that makes sense.
>
> But in this case I would say we should change the patch like the following:
>
> adev->gmc.stolen_size = min(adev->gmc.stolen_size,
> AMDGPU_STOLEN_VGA_DEFAULT_SIZE);
>
> And in amdgpu_ttm_late_init():
>
> /* Can't free the stolen VGA memory when it might be used for memory
> training again. */
> if (!adev->fw_vram_usage.mem_train_support)
>      amdgpu_bo_free_kernel(....
>
>
> Regards,
> Christian.
>
>
> >
> > Alex
> >
> >> Regards,
> >> Christian.
> >>
> >>> [how]
> >>> Forcing stolen_size to 3MB, then the bottom region of VRAM was
> >>> allocated as stolen memory, GTT corruption avoid.
> >>> The stolen memory of the 2nd GPU will be free in late_init phase,
> >>> no memory wasted.
> >>>
> >>> Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
> >>> Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
> >>> ---
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
> >>>    drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
> >>>    2 files changed, 27 insertions(+)
> >>>
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> index c91dd602d5f1..440b793316df 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> @@ -60,6 +60,11 @@
> >>>     */
> >>>    #define AMDGPU_GMC_FAULT_TIMEOUT    5000ULL
> >>>
> >>> +/*
> >>> + * Default stolen memory size, 1024 * 768 * 4
> >>> + */
> >>> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE       0x300000
> >>> +
> >>>    struct firmware;
> >>>
> >>>    /*
> >>> @@ -192,6 +197,7 @@ struct amdgpu_gmc {
> >>>        uint32_t                srbm_soft_reset;
> >>>        bool                    prt_warning;
> >>>        uint64_t                stolen_size;
> >>> +     bool                    stolen_temp_reserved;
> >>>        /* apertures */
> >>>        u64                     shared_aperture_start;
> >>>        u64                     shared_aperture_end;
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> index 7dc8c068c62a..0c96b67d6ca7 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> @@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
> >>>        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> >>>        int r;
> >>>
> >>> +     if (adev->gmc.stolen_temp_reserved) {
> >>> +             amdgpu_bo_late_init(adev);
> >>> +             adev->gmc.stolen_temp_reserved = false;
> >>> +     }
> >>> +
> >>>        r = amdgpu_gmc_allocate_vm_inv_eng(adev);
> >>>        if (r)
> >>>                return r;
> >>> @@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
> >>>                return r;
> >>>
> >>>        adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
> >>> +     /*
> >>> +      * In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> >>> +      * then the bottom region of VRAM was allocated as GTT, unfortunately
> >>> +      * a small region of bottom VRAM was encroached by UMC firmware during
> >>> +      * GDDR6 BIST training, this cause pagefault.
> >>> +      * The page fault can be fixed by forcing stolen_size to 3MB, then the bottom
> >>> +      * region of VRAM was allocated as stolen memory, GTT corruption avoid.
> >>> +      * The stolen memory of the 2nd GPU will be free in late_init phase,
> >>> +      * no memory wasted.
> >>> +      */
> >>> +     if (adev->fw_vram_usage.mem_train_support &&
> >>> +             adev->gmc.stolen_size == 0) {
> >>> +             adev->gmc.stolen_size = AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
> >>> +             adev->gmc.stolen_temp_reserved = true;
> >>> +     } else
> >>> +             adev->gmc.stolen_temp_reserved = false;
> >>>
> >>>        /* Memory manager */
> >>>        r = amdgpu_bo_init(adev);
> >> _______________________________________________
> >> amd-gfx mailing list
> >> amd-gfx@lists.freedesktop.org
> >> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7CTianci.Yin%40amd.com%7Cb9b1622ed60e4ab36c6408d794b1d75e%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637141363862418597&amp;sdata=YT8zKlbLX0XdzqcLrZQaV6sKFWXS5nQTNMAMT9BMK70%3D&amp;reserved=0
> > _______________________________________________
> > amd-gfx mailing list
> > amd-gfx@lists.freedesktop.org
> > https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7CTianci.Yin%40amd.com%7Cb9b1622ed60e4ab36c6408d794b1d75e%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637141363862428589&amp;sdata=LLVRzFBxFKqTltpvsK%2F2l9CwnlUFzFKmDoPPHdO5e1I%3D&amp;reserved=0
>

[-- Attachment #1.2: Type: text/html, Size: 13749 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Christian König]

[-- Attachment #1.1: Type: text/plain, Size: 8587 bytes --]

Hi Rico,

maybe it is a good idea to look into the git history and/or google the 
mailing list history a bit more.

I do briefly remember that we disabled freeing up the stolen VGA memory 
on some hardware generations because somebody was accessing that memory 
even after VGA was turned off.

At that time we couldn't figure out what was going wrong, but it is 
perfectly possible that those memory training fixes your are working on 
here are related to that issue.

On the other hand if you can't find anything immediately feel free to go 
with Alex suggestion, if something goes wrong we can still revert the 
change.

Regards,
Christian.

Am 09.01.20 um 04:15 schrieb Yin, Tianci (Rico):
>
> [AMD Official Use Only - Internal Distribution Only]
>
>
> Ok, thanks very much Alex!
> ------------------------------------------------------------------------
> *From:* Alex Deucher <alexdeucher@gmail.com>
> *Sent:* Thursday, January 9, 2020 11:12
> *To:* Yin, Tianci (Rico) <Tianci.Yin@amd.com>
> *Cc:* Christian König <ckoenig.leichtzumerken@gmail.com>; Koenig, 
> Christian <Christian.Koenig@amd.com>; Long, Gang <Gang.Long@amd.com>; 
> Wang, Kevin(Yang) <Kevin1.Wang@amd.com>; Xu, Feifei 
> <Feifei.Xu@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org>; 
> Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander 
> <Alexander.Deucher@amd.com>; Yuan, Xiaojie <Xiaojie.Yuan@amd.com>; 
> Zhang, Hawking <Hawking.Zhang@amd.com>
> *Subject:* Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU 
> when GDDR6 training enabled
> On Wed, Jan 8, 2020 at 10:07 PM Yin, Tianci (Rico) 
> <Tianci.Yin@amd.com> wrote:
> >
> > [AMD Official Use Only - Internal Distribution Only]
> >
> >
> > Thanks Alex and Christian!
> >
> > Hi Christian,
> >
> > On ASICs with gmc v10, I find amdgpu_bo_late_init() is not invoked, 
> so stolen memory never get free, on other ASICs with gmc v9/v8/v7/v6, 
> stolen memory was freed in gmc_v*_0_late_init(). I don't know why, are 
> there some special reasons or just missed by coding?
> >
>
> Looks like it should be added.  Possibly got lost when we merged the
> navi code from the topic branch.
>
> Alex
>
> > Thanks!
> >
> > Rico
> >
> > ________________________________
> > From: Christian König <ckoenig.leichtzumerken@gmail.com>
> > Sent: Wednesday, January 8, 2020 23:04
> > To: Alex Deucher <alexdeucher@gmail.com>; Koenig, Christian 
> <Christian.Koenig@amd.com>
> > Cc: Long, Gang <Gang.Long@amd.com>; Wang, Kevin(Yang) 
> <Kevin1.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; Yin, Tianci 
> (Rico) <Tianci.Yin@amd.com>; amd-gfx list 
> <amd-gfx@lists.freedesktop.org>; Tuikov, Luben <Luben.Tuikov@amd.com>; 
> Deucher, Alexander <Alexander.Deucher@amd.com>; Yuan, Xiaojie 
> <Xiaojie.Yuan@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>
> > Subject: Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU 
> when GDDR6 training enabled
> >
> > Am 08.01.20 um 15:49 schrieb Alex Deucher:
> > > On Wed, Jan 8, 2020 at 7:52 AM Christian König 
> <christian.koenig@amd.com> wrote:
> > >> Am 08.01.20 um 13:36 schrieb Tianci Yin:
> > >>> From: "Tianci.Yin" <tianci.yin@amd.com>
> > >>>
> > >>> [why]
> > >>> In dual GPUs scenario, stolen_size is assigned to zero on the 
> 2nd GPU,
> > >>> then the bottom region of VRAM was allocated as GTT, unfortunately
> > >>> a small region of bottom VRAM was encroached by UMC firmware during
> > >>> GDDR6 BIST training, this cause pagefault.
> > >> What I'm missing here is why is the stolen size zero on the 2nd GPU?
> > >>
> > >> Maybe we need to read the stolen size after posting the GPU instead?
> > > There is no stolen memory on secondary GPUs because there is no pre-OS
> > > console using that memory.
> >
> > Ah! Yeah that makes sense.
> >
> > But in this case I would say we should change the patch like the 
> following:
> >
> > adev->gmc.stolen_size = min(adev->gmc.stolen_size,
> > AMDGPU_STOLEN_VGA_DEFAULT_SIZE);
> >
> > And in amdgpu_ttm_late_init():
> >
> > /* Can't free the stolen VGA memory when it might be used for memory
> > training again. */
> > if (!adev->fw_vram_usage.mem_train_support)
> >      amdgpu_bo_free_kernel(....
> >
> >
> > Regards,
> > Christian.
> >
> >
> > >
> > > Alex
> > >
> > >> Regards,
> > >> Christian.
> > >>
> > >>> [how]
> > >>> Forcing stolen_size to 3MB, then the bottom region of VRAM was
> > >>> allocated as stolen memory, GTT corruption avoid.
> > >>> The stolen memory of the 2nd GPU will be free in late_init phase,
> > >>> no memory wasted.
> > >>>
> > >>> Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
> > >>> Signed-off-by: Tianci.Yin <tianci.yin@amd.com>
> > >>> ---
> > >>> drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
> > >>> drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
> > >>>    2 files changed, 27 insertions(+)
> > >>>
> > >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h 
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> > >>> index c91dd602d5f1..440b793316df 100644
> > >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> > >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> > >>> @@ -60,6 +60,11 @@
> > >>>     */
> > >>>    #define AMDGPU_GMC_FAULT_TIMEOUT 5000ULL
> > >>>
> > >>> +/*
> > >>> + * Default stolen memory size, 1024 * 768 * 4
> > >>> + */
> > >>> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE       0x300000
> > >>> +
> > >>>    struct firmware;
> > >>>
> > >>>    /*
> > >>> @@ -192,6 +197,7 @@ struct amdgpu_gmc {
> > >>>        uint32_t srbm_soft_reset;
> > >>>        bool prt_warning;
> > >>>        uint64_t stolen_size;
> > >>> +     bool stolen_temp_reserved;
> > >>>        /* apertures */
> > >>>        u64 shared_aperture_start;
> > >>>        u64 shared_aperture_end;
> > >>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c 
> b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> > >>> index 7dc8c068c62a..0c96b67d6ca7 100644
> > >>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> > >>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> > >>> @@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
> > >>>        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> > >>>        int r;
> > >>>
> > >>> +     if (adev->gmc.stolen_temp_reserved) {
> > >>> + amdgpu_bo_late_init(adev);
> > >>> + adev->gmc.stolen_temp_reserved = false;
> > >>> +     }
> > >>> +
> > >>>        r = amdgpu_gmc_allocate_vm_inv_eng(adev);
> > >>>        if (r)
> > >>>                return r;
> > >>> @@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
> > >>>                return r;
> > >>>
> > >>>        adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
> > >>> +     /*
> > >>> +      * In dual GPUs scenario, stolen_size is assigned to zero 
> on the 2nd GPU,
> > >>> +      * then the bottom region of VRAM was allocated as GTT, 
> unfortunately
> > >>> +      * a small region of bottom VRAM was encroached by UMC 
> firmware during
> > >>> +      * GDDR6 BIST training, this cause pagefault.
> > >>> +      * The page fault can be fixed by forcing stolen_size to 
> 3MB, then the bottom
> > >>> +      * region of VRAM was allocated as stolen memory, GTT 
> corruption avoid.
> > >>> +      * The stolen memory of the 2nd GPU will be free in 
> late_init phase,
> > >>> +      * no memory wasted.
> > >>> +      */
> > >>> +     if (adev->fw_vram_usage.mem_train_support &&
> > >>> +             adev->gmc.stolen_size == 0) {
> > >>> +             adev->gmc.stolen_size = 
> AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
> > >>> + adev->gmc.stolen_temp_reserved = true;
> > >>> +     } else
> > >>> + adev->gmc.stolen_temp_reserved = false;
> > >>>
> > >>>        /* Memory manager */
> > >>>        r = amdgpu_bo_init(adev);
> > >> _______________________________________________
> > >> amd-gfx mailing list
> > >> amd-gfx@lists.freedesktop.org
> > >> 
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7CTianci.Yin%40amd.com%7Cb9b1622ed60e4ab36c6408d794b1d75e%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637141363862418597&amp;sdata=YT8zKlbLX0XdzqcLrZQaV6sKFWXS5nQTNMAMT9BMK70%3D&amp;reserved=0
> > > _______________________________________________
> > > amd-gfx mailing list
> > > amd-gfx@lists.freedesktop.org
> > > 
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7CTianci.Yin%40amd.com%7Cb9b1622ed60e4ab36c6408d794b1d75e%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637141363862428589&amp;sdata=LLVRzFBxFKqTltpvsK%2F2l9CwnlUFzFKmDoPPHdO5e1I%3D&amp;reserved=0
> >


[-- Attachment #1.2: Type: text/html, Size: 18828 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

[Yin, Tianci (Rico)]

[-- Attachment #1.1: Type: text/plain, Size: 11320 bytes --]

[AMD Official Use Only - Internal Distribution Only]

Hi Christian,

Thanks very much for your suggestions, I checked the original version of gmc_v10_0.c on branch origin/ruihuang/amd-temp-navi10-part1, amdgpu_bo_late_init() is not invoked by gmc_v10_0_late_init(), so I think it was missed from the beginning time.

Yes, it seems we all have ever met the VRAM overwritten issue, but it's not easy to figure out who did it. Are there some source that we can get this kind of information? If there is a place gathered this kind of information, it should save us much debug time.

Thanks!

Rico
________________________________
From: Christian König <ckoenig.leichtzumerken@gmail.com>
Sent: Thursday, January 9, 2020 18:29
To: Yin, Tianci (Rico) <Tianci.Yin@amd.com>; Alex Deucher <alexdeucher@gmail.com>
Cc: Koenig, Christian <Christian.Koenig@amd.com>; Long, Gang <Gang.Long@amd.com>; Wang, Kevin(Yang) <Kevin1.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org>; Tuikov, Luben <Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>; Yuan, Xiaojie <Xiaojie.Yuan@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com>
Subject: Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

Hi Rico,

maybe it is a good idea to look into the git history and/or google the mailing list history a bit more.

I do briefly remember that we disabled freeing up the stolen VGA memory on some hardware generations because somebody was accessing that memory even after VGA was turned off.

At that time we couldn't figure out what was going wrong, but it is perfectly possible that those memory training fixes your are working on here are related to that issue.

On the other hand if you can't find anything immediately feel free to go with Alex suggestion, if something goes wrong we can still revert the change.

Regards,
Christian.

Am 09.01.20 um 04:15 schrieb Yin, Tianci (Rico):

[AMD Official Use Only - Internal Distribution Only]

Ok, thanks very much Alex!
________________________________
From: Alex Deucher <alexdeucher@gmail.com><mailto:alexdeucher@gmail.com>
Sent: Thursday, January 9, 2020 11:12
To: Yin, Tianci (Rico) <Tianci.Yin@amd.com><mailto:Tianci.Yin@amd.com>
Cc: Christian König <ckoenig.leichtzumerken@gmail.com><mailto:ckoenig.leichtzumerken@gmail.com>; Koenig, Christian <Christian.Koenig@amd.com><mailto:Christian.Koenig@amd.com>; Long, Gang <Gang.Long@amd.com><mailto:Gang.Long@amd.com>; Wang, Kevin(Yang) <Kevin1.Wang@amd.com><mailto:Kevin1.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com><mailto:Feifei.Xu@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org><mailto:amd-gfx@lists.freedesktop.org>; Tuikov, Luben <Luben.Tuikov@amd.com><mailto:Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com><mailto:Alexander.Deucher@amd.com>; Yuan, Xiaojie <Xiaojie.Yuan@amd.com><mailto:Xiaojie.Yuan@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com><mailto:Hawking.Zhang@amd.com>
Subject: Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled

On Wed, Jan 8, 2020 at 10:07 PM Yin, Tianci (Rico) <Tianci.Yin@amd.com><mailto:Tianci.Yin@amd.com> wrote:
>
> [AMD Official Use Only - Internal Distribution Only]
>
>
> Thanks Alex and Christian!
>
> Hi Christian,
>
> On ASICs with gmc v10, I find amdgpu_bo_late_init() is not invoked, so stolen memory never get free, on other ASICs with gmc v9/v8/v7/v6, stolen memory was freed in gmc_v*_0_late_init(). I don't know why, are there some special reasons or just missed by coding?
>

Looks like it should be added.  Possibly got lost when we merged the
navi code from the topic branch.

Alex

> Thanks!
>
> Rico
>
> ________________________________
> From: Christian König <ckoenig.leichtzumerken@gmail.com><mailto:ckoenig.leichtzumerken@gmail.com>
> Sent: Wednesday, January 8, 2020 23:04
> To: Alex Deucher <alexdeucher@gmail.com><mailto:alexdeucher@gmail.com>; Koenig, Christian <Christian.Koenig@amd.com><mailto:Christian.Koenig@amd.com>
> Cc: Long, Gang <Gang.Long@amd.com><mailto:Gang.Long@amd.com>; Wang, Kevin(Yang) <Kevin1.Wang@amd.com><mailto:Kevin1.Wang@amd.com>; Xu, Feifei <Feifei.Xu@amd.com><mailto:Feifei.Xu@amd.com>; Yin, Tianci (Rico) <Tianci.Yin@amd.com><mailto:Tianci.Yin@amd.com>; amd-gfx list <amd-gfx@lists.freedesktop.org><mailto:amd-gfx@lists.freedesktop.org>; Tuikov, Luben <Luben.Tuikov@amd.com><mailto:Luben.Tuikov@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com><mailto:Alexander.Deucher@amd.com>; Yuan, Xiaojie <Xiaojie.Yuan@amd.com><mailto:Xiaojie.Yuan@amd.com>; Zhang, Hawking <Hawking.Zhang@amd.com><mailto:Hawking.Zhang@amd.com>
> Subject: Re: [PATCH] drm/amdgpu: fix modprobe failure of the 2nd GPU when GDDR6 training enabled
>
> Am 08.01.20 um 15:49 schrieb Alex Deucher:
> > On Wed, Jan 8, 2020 at 7:52 AM Christian König <christian.koenig@amd.com><mailto:christian.koenig@amd.com> wrote:
> >> Am 08.01.20 um 13:36 schrieb Tianci Yin:
> >>> From: "Tianci.Yin" <tianci.yin@amd.com><mailto:tianci.yin@amd.com>
> >>>
> >>> [why]
> >>> In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> >>> then the bottom region of VRAM was allocated as GTT, unfortunately
> >>> a small region of bottom VRAM was encroached by UMC firmware during
> >>> GDDR6 BIST training, this cause pagefault.
> >> What I'm missing here is why is the stolen size zero on the 2nd GPU?
> >>
> >> Maybe we need to read the stolen size after posting the GPU instead?
> > There is no stolen memory on secondary GPUs because there is no pre-OS
> > console using that memory.
>
> Ah! Yeah that makes sense.
>
> But in this case I would say we should change the patch like the following:
>
> adev->gmc.stolen_size = min(adev->gmc.stolen_size,
> AMDGPU_STOLEN_VGA_DEFAULT_SIZE);
>
> And in amdgpu_ttm_late_init():
>
> /* Can't free the stolen VGA memory when it might be used for memory
> training again. */
> if (!adev->fw_vram_usage.mem_train_support)
>      amdgpu_bo_free_kernel(....
>
>
> Regards,
> Christian.
>
>
> >
> > Alex
> >
> >> Regards,
> >> Christian.
> >>
> >>> [how]
> >>> Forcing stolen_size to 3MB, then the bottom region of VRAM was
> >>> allocated as stolen memory, GTT corruption avoid.
> >>> The stolen memory of the 2nd GPU will be free in late_init phase,
> >>> no memory wasted.
> >>>
> >>> Change-Id: Icd0ad7de41333282949bb1e3e676c6c307ddd081
> >>> Signed-off-by: Tianci.Yin <tianci.yin@amd.com><mailto:tianci.yin@amd.com>
> >>> ---
> >>>    drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h |  6 ++++++
> >>>    drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c  | 21 +++++++++++++++++++++
> >>>    2 files changed, 27 insertions(+)
> >>>
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> index c91dd602d5f1..440b793316df 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
> >>> @@ -60,6 +60,11 @@
> >>>     */
> >>>    #define AMDGPU_GMC_FAULT_TIMEOUT    5000ULL
> >>>
> >>> +/*
> >>> + * Default stolen memory size, 1024 * 768 * 4
> >>> + */
> >>> +#define AMDGPU_STOLEN_VGA_DEFAULT_SIZE       0x300000
> >>> +
> >>>    struct firmware;
> >>>
> >>>    /*
> >>> @@ -192,6 +197,7 @@ struct amdgpu_gmc {
> >>>        uint32_t                srbm_soft_reset;
> >>>        bool                    prt_warning;
> >>>        uint64_t                stolen_size;
> >>> +     bool                    stolen_temp_reserved;
> >>>        /* apertures */
> >>>        u64                     shared_aperture_start;
> >>>        u64                     shared_aperture_end;
> >>> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> index 7dc8c068c62a..0c96b67d6ca7 100644
> >>> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
> >>> @@ -566,6 +566,11 @@ static int gmc_v10_0_late_init(void *handle)
> >>>        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> >>>        int r;
> >>>
> >>> +     if (adev->gmc.stolen_temp_reserved) {
> >>> +             amdgpu_bo_late_init(adev);
> >>> +             adev->gmc.stolen_temp_reserved = false;
> >>> +     }
> >>> +
> >>>        r = amdgpu_gmc_allocate_vm_inv_eng(adev);
> >>>        if (r)
> >>>                return r;
> >>> @@ -756,6 +761,22 @@ static int gmc_v10_0_sw_init(void *handle)
> >>>                return r;
> >>>
> >>>        adev->gmc.stolen_size = gmc_v10_0_get_vbios_fb_size(adev);
> >>> +     /*
> >>> +      * In dual GPUs scenario, stolen_size is assigned to zero on the 2nd GPU,
> >>> +      * then the bottom region of VRAM was allocated as GTT, unfortunately
> >>> +      * a small region of bottom VRAM was encroached by UMC firmware during
> >>> +      * GDDR6 BIST training, this cause pagefault.
> >>> +      * The page fault can be fixed by forcing stolen_size to 3MB, then the bottom
> >>> +      * region of VRAM was allocated as stolen memory, GTT corruption avoid.
> >>> +      * The stolen memory of the 2nd GPU will be free in late_init phase,
> >>> +      * no memory wasted.
> >>> +      */
> >>> +     if (adev->fw_vram_usage.mem_train_support &&
> >>> +             adev->gmc.stolen_size == 0) {
> >>> +             adev->gmc.stolen_size = AMDGPU_STOLEN_VGA_DEFAULT_SIZE;
> >>> +             adev->gmc.stolen_temp_reserved = true;
> >>> +     } else
> >>> +             adev->gmc.stolen_temp_reserved = false;
> >>>
> >>>        /* Memory manager */
> >>>        r = amdgpu_bo_init(adev);
> >> _______________________________________________
> >> amd-gfx mailing list
> >> amd-gfx@lists.freedesktop.org<mailto:amd-gfx@lists.freedesktop.org>
> >> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7CTianci.Yin%40amd.com%7Cb9b1622ed60e4ab36c6408d794b1d75e%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637141363862418597&amp;sdata=YT8zKlbLX0XdzqcLrZQaV6sKFWXS5nQTNMAMT9BMK70%3D&amp;reserved=0<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=02%7C01%7CTianci.Yin%40amd.com%7Ca21d3576dde64ae42f0908d794eede11%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637141625988841768&sdata=j%2F7xYrXd%2F%2BUIke9EhCdHxknY8TmgspnCyxcz6cTbpYg%3D&reserved=0>
> > _______________________________________________
> > amd-gfx mailing list
> > amd-gfx@lists.freedesktop.org<mailto:amd-gfx@lists.freedesktop.org>
> > https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&amp;data=02%7C01%7CTianci.Yin%40amd.com%7Cb9b1622ed60e4ab36c6408d794b1d75e%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637141363862428589&amp;sdata=LLVRzFBxFKqTltpvsK%2F2l9CwnlUFzFKmDoPPHdO5e1I%3D&amp;reserved=0<https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=02%7C01%7CTianci.Yin%40amd.com%7Ca21d3576dde64ae42f0908d794eede11%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637141625988851763&sdata=0OGgqQKu%2F9G0lJGXgDy25P%2F9GEU%2FlvdeFd5gRtLhPyQ%3D&reserved=0>
>


[-- Attachment #1.2: Type: text/html, Size: 20310 bytes --]

[-- Attachment #2: Type: text/plain, Size: 154 bytes --]

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx