* [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
@ 2021-03-12 3:57 Jack Zhang
2021-03-12 4:13 ` Alex Deucher
0 siblings, 1 reply; 7+ messages in thread
From: Jack Zhang @ 2021-03-12 3:57 UTC (permalink / raw)
To: amd-gfx, Christian.Koenig, Andrey.Grodzovsky, Monk.Liu, Emily.Deng
Cc: Jack Zhang
re-insert Bailing jobs to avoid memory leak.
Signed-off-by: Jack Zhang <Jack.Zhang1@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 4 +++-
drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 8 ++++++--
drivers/gpu/drm/scheduler/sched_main.c | 8 +++++++-
include/drm/gpu_scheduler.h | 1 +
4 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 79b9cc73763f..86463b0f936e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4815,8 +4815,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
job ? job->base.id : -1);
/* even we skipped this reset, still need to set the job to guilty */
- if (job)
+ if (job) {
drm_sched_increase_karma(&job->base);
+ r = DRM_GPU_SCHED_STAT_BAILING;
+ }
goto skip_recovery;
}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 759b34799221..41390bdacd9e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -34,6 +34,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
struct amdgpu_job *job = to_amdgpu_job(s_job);
struct amdgpu_task_info ti;
struct amdgpu_device *adev = ring->adev;
+ int ret;
memset(&ti, 0, sizeof(struct amdgpu_task_info));
@@ -52,8 +53,11 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
ti.process_name, ti.tgid, ti.task_name, ti.pid);
if (amdgpu_device_should_recover_gpu(ring->adev)) {
- amdgpu_device_gpu_recover(ring->adev, job);
- return DRM_GPU_SCHED_STAT_NOMINAL;
+ ret = amdgpu_device_gpu_recover(ring->adev, job);
+ if (ret == DRM_GPU_SCHED_STAT_BAILING)
+ return DRM_GPU_SCHED_STAT_BAILING;
+ else
+ return DRM_GPU_SCHED_STAT_NOMINAL;
} else {
drm_sched_suspend_timeout(&ring->sched);
if (amdgpu_sriov_vf(adev))
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 92d8de24d0a1..a44f621fb5c4 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -314,6 +314,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
{
struct drm_gpu_scheduler *sched;
struct drm_sched_job *job;
+ int ret;
sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
@@ -331,8 +332,13 @@ static void drm_sched_job_timedout(struct work_struct *work)
list_del_init(&job->list);
spin_unlock(&sched->job_list_lock);
- job->sched->ops->timedout_job(job);
+ ret = job->sched->ops->timedout_job(job);
+ if (ret == DRM_GPU_SCHED_STAT_BAILING) {
+ spin_lock(&sched->job_list_lock);
+ list_add(&job->node, &sched->ring_mirror_list);
+ spin_unlock(&sched->job_list_lock);
+ }
/*
* Guilty job did complete and hence needs to be manually removed
* See drm_sched_stop doc.
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index 4ea8606d91fe..8093ac2427ef 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -210,6 +210,7 @@ enum drm_gpu_sched_stat {
DRM_GPU_SCHED_STAT_NONE, /* Reserve 0 */
DRM_GPU_SCHED_STAT_NOMINAL,
DRM_GPU_SCHED_STAT_ENODEV,
+ DRM_GPU_SCHED_STAT_BAILING,
};
/**
--
2.25.1
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply related [flat|nested] 7+ messages in thread
* Re: [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
2021-03-12 3:57 [PATCH] drm/scheduler re-insert Bailing job to avoid memleak Jack Zhang
@ 2021-03-12 4:13 ` Alex Deucher
0 siblings, 0 replies; 7+ messages in thread
From: Alex Deucher @ 2021-03-12 4:13 UTC (permalink / raw)
To: Jack Zhang, Maling list - DRI developers
Cc: monk.liu, Christian Koenig, amd-gfx list, Emily Deng
+dri-devel
Please be sure to cc dri-devel when you send out gpu scheduler patches.
On Thu, Mar 11, 2021 at 10:57 PM Jack Zhang <Jack.Zhang1@amd.com> wrote:
>
> re-insert Bailing jobs to avoid memory leak.
>
> Signed-off-by: Jack Zhang <Jack.Zhang1@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 4 +++-
> drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 8 ++++++--
> drivers/gpu/drm/scheduler/sched_main.c | 8 +++++++-
> include/drm/gpu_scheduler.h | 1 +
> 4 files changed, 17 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 79b9cc73763f..86463b0f936e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -4815,8 +4815,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
> job ? job->base.id : -1);
>
> /* even we skipped this reset, still need to set the job to guilty */
> - if (job)
> + if (job) {
> drm_sched_increase_karma(&job->base);
> + r = DRM_GPU_SCHED_STAT_BAILING;
> + }
> goto skip_recovery;
> }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> index 759b34799221..41390bdacd9e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> @@ -34,6 +34,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
> struct amdgpu_job *job = to_amdgpu_job(s_job);
> struct amdgpu_task_info ti;
> struct amdgpu_device *adev = ring->adev;
> + int ret;
>
> memset(&ti, 0, sizeof(struct amdgpu_task_info));
>
> @@ -52,8 +53,11 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
> ti.process_name, ti.tgid, ti.task_name, ti.pid);
>
> if (amdgpu_device_should_recover_gpu(ring->adev)) {
> - amdgpu_device_gpu_recover(ring->adev, job);
> - return DRM_GPU_SCHED_STAT_NOMINAL;
> + ret = amdgpu_device_gpu_recover(ring->adev, job);
> + if (ret == DRM_GPU_SCHED_STAT_BAILING)
> + return DRM_GPU_SCHED_STAT_BAILING;
> + else
> + return DRM_GPU_SCHED_STAT_NOMINAL;
> } else {
> drm_sched_suspend_timeout(&ring->sched);
> if (amdgpu_sriov_vf(adev))
> diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
> index 92d8de24d0a1..a44f621fb5c4 100644
> --- a/drivers/gpu/drm/scheduler/sched_main.c
> +++ b/drivers/gpu/drm/scheduler/sched_main.c
> @@ -314,6 +314,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
> {
> struct drm_gpu_scheduler *sched;
> struct drm_sched_job *job;
> + int ret;
>
> sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
>
> @@ -331,8 +332,13 @@ static void drm_sched_job_timedout(struct work_struct *work)
> list_del_init(&job->list);
> spin_unlock(&sched->job_list_lock);
>
> - job->sched->ops->timedout_job(job);
> + ret = job->sched->ops->timedout_job(job);
>
> + if (ret == DRM_GPU_SCHED_STAT_BAILING) {
> + spin_lock(&sched->job_list_lock);
> + list_add(&job->node, &sched->ring_mirror_list);
> + spin_unlock(&sched->job_list_lock);
> + }
> /*
> * Guilty job did complete and hence needs to be manually removed
> * See drm_sched_stop doc.
> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
> index 4ea8606d91fe..8093ac2427ef 100644
> --- a/include/drm/gpu_scheduler.h
> +++ b/include/drm/gpu_scheduler.h
> @@ -210,6 +210,7 @@ enum drm_gpu_sched_stat {
> DRM_GPU_SCHED_STAT_NONE, /* Reserve 0 */
> DRM_GPU_SCHED_STAT_NOMINAL,
> DRM_GPU_SCHED_STAT_ENODEV,
> + DRM_GPU_SCHED_STAT_BAILING,
> };
>
> /**
> --
> 2.25.1
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
@ 2021-03-12 4:13 ` Alex Deucher
0 siblings, 0 replies; 7+ messages in thread
From: Alex Deucher @ 2021-03-12 4:13 UTC (permalink / raw)
To: Jack Zhang, Maling list - DRI developers
Cc: monk.liu, Andrey Grodzovsky, Christian Koenig, amd-gfx list, Emily Deng
+dri-devel
Please be sure to cc dri-devel when you send out gpu scheduler patches.
On Thu, Mar 11, 2021 at 10:57 PM Jack Zhang <Jack.Zhang1@amd.com> wrote:
>
> re-insert Bailing jobs to avoid memory leak.
>
> Signed-off-by: Jack Zhang <Jack.Zhang1@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 4 +++-
> drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 8 ++++++--
> drivers/gpu/drm/scheduler/sched_main.c | 8 +++++++-
> include/drm/gpu_scheduler.h | 1 +
> 4 files changed, 17 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 79b9cc73763f..86463b0f936e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -4815,8 +4815,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
> job ? job->base.id : -1);
>
> /* even we skipped this reset, still need to set the job to guilty */
> - if (job)
> + if (job) {
> drm_sched_increase_karma(&job->base);
> + r = DRM_GPU_SCHED_STAT_BAILING;
> + }
> goto skip_recovery;
> }
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> index 759b34799221..41390bdacd9e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> @@ -34,6 +34,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
> struct amdgpu_job *job = to_amdgpu_job(s_job);
> struct amdgpu_task_info ti;
> struct amdgpu_device *adev = ring->adev;
> + int ret;
>
> memset(&ti, 0, sizeof(struct amdgpu_task_info));
>
> @@ -52,8 +53,11 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
> ti.process_name, ti.tgid, ti.task_name, ti.pid);
>
> if (amdgpu_device_should_recover_gpu(ring->adev)) {
> - amdgpu_device_gpu_recover(ring->adev, job);
> - return DRM_GPU_SCHED_STAT_NOMINAL;
> + ret = amdgpu_device_gpu_recover(ring->adev, job);
> + if (ret == DRM_GPU_SCHED_STAT_BAILING)
> + return DRM_GPU_SCHED_STAT_BAILING;
> + else
> + return DRM_GPU_SCHED_STAT_NOMINAL;
> } else {
> drm_sched_suspend_timeout(&ring->sched);
> if (amdgpu_sriov_vf(adev))
> diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
> index 92d8de24d0a1..a44f621fb5c4 100644
> --- a/drivers/gpu/drm/scheduler/sched_main.c
> +++ b/drivers/gpu/drm/scheduler/sched_main.c
> @@ -314,6 +314,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
> {
> struct drm_gpu_scheduler *sched;
> struct drm_sched_job *job;
> + int ret;
>
> sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
>
> @@ -331,8 +332,13 @@ static void drm_sched_job_timedout(struct work_struct *work)
> list_del_init(&job->list);
> spin_unlock(&sched->job_list_lock);
>
> - job->sched->ops->timedout_job(job);
> + ret = job->sched->ops->timedout_job(job);
>
> + if (ret == DRM_GPU_SCHED_STAT_BAILING) {
> + spin_lock(&sched->job_list_lock);
> + list_add(&job->node, &sched->ring_mirror_list);
> + spin_unlock(&sched->job_list_lock);
> + }
> /*
> * Guilty job did complete and hence needs to be manually removed
> * See drm_sched_stop doc.
> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
> index 4ea8606d91fe..8093ac2427ef 100644
> --- a/include/drm/gpu_scheduler.h
> +++ b/include/drm/gpu_scheduler.h
> @@ -210,6 +210,7 @@ enum drm_gpu_sched_stat {
> DRM_GPU_SCHED_STAT_NONE, /* Reserve 0 */
> DRM_GPU_SCHED_STAT_NOMINAL,
> DRM_GPU_SCHED_STAT_ENODEV,
> + DRM_GPU_SCHED_STAT_BAILING,
> };
>
> /**
> --
> 2.25.1
>
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/amd-gfx
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
2021-03-12 4:13 ` Alex Deucher
@ 2021-03-12 5:04 ` Andrey Grodzovsky
-1 siblings, 0 replies; 7+ messages in thread
From: Andrey Grodzovsky @ 2021-03-12 5:04 UTC (permalink / raw)
To: Alex Deucher, Jack Zhang, Maling list - DRI developers
Cc: monk.liu, Emily Deng, Christian Koenig, amd-gfx list
Check panfrost driver at panfrost_scheduler_stop,
and panfrost_job_timedout - they also terminate prematurely
in both places so probably worth adding this there too.
Andrey
On 2021-03-11 11:13 p.m., Alex Deucher wrote:
> +dri-devel
>
> Please be sure to cc dri-devel when you send out gpu scheduler patches.
>
> On Thu, Mar 11, 2021 at 10:57 PM Jack Zhang <Jack.Zhang1@amd.com> wrote:
>>
>> re-insert Bailing jobs to avoid memory leak.
>>
>> Signed-off-by: Jack Zhang <Jack.Zhang1@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 4 +++-
>> drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 8 ++++++--
>> drivers/gpu/drm/scheduler/sched_main.c | 8 +++++++-
>> include/drm/gpu_scheduler.h | 1 +
>> 4 files changed, 17 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index 79b9cc73763f..86463b0f936e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -4815,8 +4815,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
>> job ? job->base.id : -1);
>>
>> /* even we skipped this reset, still need to set the job to guilty */
>> - if (job)
>> + if (job) {
>> drm_sched_increase_karma(&job->base);
>> + r = DRM_GPU_SCHED_STAT_BAILING;
>> + }
>> goto skip_recovery;
>> }
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> index 759b34799221..41390bdacd9e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> @@ -34,6 +34,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>> struct amdgpu_job *job = to_amdgpu_job(s_job);
>> struct amdgpu_task_info ti;
>> struct amdgpu_device *adev = ring->adev;
>> + int ret;
>>
>> memset(&ti, 0, sizeof(struct amdgpu_task_info));
>>
>> @@ -52,8 +53,11 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>> ti.process_name, ti.tgid, ti.task_name, ti.pid);
>>
>> if (amdgpu_device_should_recover_gpu(ring->adev)) {
>> - amdgpu_device_gpu_recover(ring->adev, job);
>> - return DRM_GPU_SCHED_STAT_NOMINAL;
>> + ret = amdgpu_device_gpu_recover(ring->adev, job);
>> + if (ret == DRM_GPU_SCHED_STAT_BAILING)
>> + return DRM_GPU_SCHED_STAT_BAILING;
>> + else
>> + return DRM_GPU_SCHED_STAT_NOMINAL;
>> } else {
>> drm_sched_suspend_timeout(&ring->sched);
>> if (amdgpu_sriov_vf(adev))
>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
>> index 92d8de24d0a1..a44f621fb5c4 100644
>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>> @@ -314,6 +314,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
>> {
>> struct drm_gpu_scheduler *sched;
>> struct drm_sched_job *job;
>> + int ret;
>>
>> sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
>>
>> @@ -331,8 +332,13 @@ static void drm_sched_job_timedout(struct work_struct *work)
>> list_del_init(&job->list);
>> spin_unlock(&sched->job_list_lock);
>>
>> - job->sched->ops->timedout_job(job);
>> + ret = job->sched->ops->timedout_job(job);
>>
>> + if (ret == DRM_GPU_SCHED_STAT_BAILING) {
>> + spin_lock(&sched->job_list_lock);
>> + list_add(&job->node, &sched->ring_mirror_list);
>> + spin_unlock(&sched->job_list_lock);
>> + }
Problem here that since you already dropped the reset locks you are
racing here now against other recovery threads as they process the same
mirror list, and yet,I think this solution makes things better then
they are now with the leak but still, it's only temporary band-aid until
the full solution to be implemented. Probably then worth mentioning here
with a comment this it's a temporary fix and that races are possible.
Andrey
>> /*
>> * Guilty job did complete and hence needs to be manually removed
>> * See drm_sched_stop doc.
>> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
>> index 4ea8606d91fe..8093ac2427ef 100644
>> --- a/include/drm/gpu_scheduler.h
>> +++ b/include/drm/gpu_scheduler.h
>> @@ -210,6 +210,7 @@ enum drm_gpu_sched_stat {
>> DRM_GPU_SCHED_STAT_NONE, /* Reserve 0 */
>> DRM_GPU_SCHED_STAT_NOMINAL,
>> DRM_GPU_SCHED_STAT_ENODEV,
>> + DRM_GPU_SCHED_STAT_BAILING,
>> };
>>
>> /**
>> --
>> 2.25.1
>>
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=04%7C01%7CAndrey.Grodzovsky%40amd.com%7Cb8e9c87369a74a6b0f2a08d8e50d4707%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637511192474845823%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=FgOIeqk34Sk13L%2BbXYxbqja%2BDY8NSva4wDtUWBMS3q0%3D&reserved=0
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
@ 2021-03-12 5:04 ` Andrey Grodzovsky
0 siblings, 0 replies; 7+ messages in thread
From: Andrey Grodzovsky @ 2021-03-12 5:04 UTC (permalink / raw)
To: Alex Deucher, Jack Zhang, Maling list - DRI developers
Cc: monk.liu, Emily Deng, Christian Koenig, amd-gfx list
Check panfrost driver at panfrost_scheduler_stop,
and panfrost_job_timedout - they also terminate prematurely
in both places so probably worth adding this there too.
Andrey
On 2021-03-11 11:13 p.m., Alex Deucher wrote:
> +dri-devel
>
> Please be sure to cc dri-devel when you send out gpu scheduler patches.
>
> On Thu, Mar 11, 2021 at 10:57 PM Jack Zhang <Jack.Zhang1@amd.com> wrote:
>>
>> re-insert Bailing jobs to avoid memory leak.
>>
>> Signed-off-by: Jack Zhang <Jack.Zhang1@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 4 +++-
>> drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 8 ++++++--
>> drivers/gpu/drm/scheduler/sched_main.c | 8 +++++++-
>> include/drm/gpu_scheduler.h | 1 +
>> 4 files changed, 17 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index 79b9cc73763f..86463b0f936e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -4815,8 +4815,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
>> job ? job->base.id : -1);
>>
>> /* even we skipped this reset, still need to set the job to guilty */
>> - if (job)
>> + if (job) {
>> drm_sched_increase_karma(&job->base);
>> + r = DRM_GPU_SCHED_STAT_BAILING;
>> + }
>> goto skip_recovery;
>> }
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> index 759b34799221..41390bdacd9e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> @@ -34,6 +34,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>> struct amdgpu_job *job = to_amdgpu_job(s_job);
>> struct amdgpu_task_info ti;
>> struct amdgpu_device *adev = ring->adev;
>> + int ret;
>>
>> memset(&ti, 0, sizeof(struct amdgpu_task_info));
>>
>> @@ -52,8 +53,11 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>> ti.process_name, ti.tgid, ti.task_name, ti.pid);
>>
>> if (amdgpu_device_should_recover_gpu(ring->adev)) {
>> - amdgpu_device_gpu_recover(ring->adev, job);
>> - return DRM_GPU_SCHED_STAT_NOMINAL;
>> + ret = amdgpu_device_gpu_recover(ring->adev, job);
>> + if (ret == DRM_GPU_SCHED_STAT_BAILING)
>> + return DRM_GPU_SCHED_STAT_BAILING;
>> + else
>> + return DRM_GPU_SCHED_STAT_NOMINAL;
>> } else {
>> drm_sched_suspend_timeout(&ring->sched);
>> if (amdgpu_sriov_vf(adev))
>> diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
>> index 92d8de24d0a1..a44f621fb5c4 100644
>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>> @@ -314,6 +314,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
>> {
>> struct drm_gpu_scheduler *sched;
>> struct drm_sched_job *job;
>> + int ret;
>>
>> sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work);
>>
>> @@ -331,8 +332,13 @@ static void drm_sched_job_timedout(struct work_struct *work)
>> list_del_init(&job->list);
>> spin_unlock(&sched->job_list_lock);
>>
>> - job->sched->ops->timedout_job(job);
>> + ret = job->sched->ops->timedout_job(job);
>>
>> + if (ret == DRM_GPU_SCHED_STAT_BAILING) {
>> + spin_lock(&sched->job_list_lock);
>> + list_add(&job->node, &sched->ring_mirror_list);
>> + spin_unlock(&sched->job_list_lock);
>> + }
Problem here that since you already dropped the reset locks you are
racing here now against other recovery threads as they process the same
mirror list, and yet,I think this solution makes things better then
they are now with the leak but still, it's only temporary band-aid until
the full solution to be implemented. Probably then worth mentioning here
with a comment this it's a temporary fix and that races are possible.
Andrey
>> /*
>> * Guilty job did complete and hence needs to be manually removed
>> * See drm_sched_stop doc.
>> diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
>> index 4ea8606d91fe..8093ac2427ef 100644
>> --- a/include/drm/gpu_scheduler.h
>> +++ b/include/drm/gpu_scheduler.h
>> @@ -210,6 +210,7 @@ enum drm_gpu_sched_stat {
>> DRM_GPU_SCHED_STAT_NONE, /* Reserve 0 */
>> DRM_GPU_SCHED_STAT_NOMINAL,
>> DRM_GPU_SCHED_STAT_ENODEV,
>> + DRM_GPU_SCHED_STAT_BAILING,
>> };
>>
>> /**
>> --
>> 2.25.1
>>
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=04%7C01%7CAndrey.Grodzovsky%40amd.com%7Cb8e9c87369a74a6b0f2a08d8e50d4707%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637511192474845823%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=FgOIeqk34Sk13L%2BbXYxbqja%2BDY8NSva4wDtUWBMS3q0%3D&reserved=0
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 7+ messages in thread
* RE: [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
2021-03-12 5:04 ` Andrey Grodzovsky
@ 2021-03-12 6:51 ` Zhang, Jack (Jian)
-1 siblings, 0 replies; 7+ messages in thread
From: Zhang, Jack (Jian) @ 2021-03-12 6:51 UTC (permalink / raw)
To: Grodzovsky, Andrey, Alex Deucher, Maling list - DRI developers
Cc: Liu, Monk, Deng, Emily, Koenig, Christian, amd-gfx list
[AMD Official Use Only - Internal Distribution Only]
Hi, Andrey,
ok, I have changed it and uploaded V2 patch.
Thanks,
Jack
-----Original Message-----
From: Grodzovsky, Andrey <Andrey.Grodzovsky@amd.com>
Sent: Friday, March 12, 2021 1:04 PM
To: Alex Deucher <alexdeucher@gmail.com>; Zhang, Jack (Jian) <Jack.Zhang1@amd.com>; Maling list - DRI developers <dri-devel@lists.freedesktop.org>
Cc: amd-gfx list <amd-gfx@lists.freedesktop.org>; Koenig, Christian <Christian.Koenig@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Deng, Emily <Emily.Deng@amd.com>
Subject: Re: [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
Check panfrost driver at panfrost_scheduler_stop, and panfrost_job_timedout - they also terminate prematurely in both places so probably worth adding this there too.
Andrey
On 2021-03-11 11:13 p.m., Alex Deucher wrote:
> +dri-devel
>
> Please be sure to cc dri-devel when you send out gpu scheduler patches.
>
> On Thu, Mar 11, 2021 at 10:57 PM Jack Zhang <Jack.Zhang1@amd.com> wrote:
>>
>> re-insert Bailing jobs to avoid memory leak.
>>
>> Signed-off-by: Jack Zhang <Jack.Zhang1@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 4 +++-
>> drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 8 ++++++--
>> drivers/gpu/drm/scheduler/sched_main.c | 8 +++++++-
>> include/drm/gpu_scheduler.h | 1 +
>> 4 files changed, 17 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index 79b9cc73763f..86463b0f936e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -4815,8 +4815,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
>> job ? job->base.id : -1);
>>
>> /* even we skipped this reset, still need to set the job to guilty */
>> - if (job)
>> + if (job) {
>> drm_sched_increase_karma(&job->base);
>> + r = DRM_GPU_SCHED_STAT_BAILING;
>> + }
>> goto skip_recovery;
>> }
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> index 759b34799221..41390bdacd9e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> @@ -34,6 +34,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>> struct amdgpu_job *job = to_amdgpu_job(s_job);
>> struct amdgpu_task_info ti;
>> struct amdgpu_device *adev = ring->adev;
>> + int ret;
>>
>> memset(&ti, 0, sizeof(struct amdgpu_task_info));
>>
>> @@ -52,8 +53,11 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>> ti.process_name, ti.tgid, ti.task_name, ti.pid);
>>
>> if (amdgpu_device_should_recover_gpu(ring->adev)) {
>> - amdgpu_device_gpu_recover(ring->adev, job);
>> - return DRM_GPU_SCHED_STAT_NOMINAL;
>> + ret = amdgpu_device_gpu_recover(ring->adev, job);
>> + if (ret == DRM_GPU_SCHED_STAT_BAILING)
>> + return DRM_GPU_SCHED_STAT_BAILING;
>> + else
>> + return DRM_GPU_SCHED_STAT_NOMINAL;
>> } else {
>> drm_sched_suspend_timeout(&ring->sched);
>> if (amdgpu_sriov_vf(adev)) diff --git
>> a/drivers/gpu/drm/scheduler/sched_main.c
>> b/drivers/gpu/drm/scheduler/sched_main.c
>> index 92d8de24d0a1..a44f621fb5c4 100644
>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>> @@ -314,6 +314,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
>> {
>> struct drm_gpu_scheduler *sched;
>> struct drm_sched_job *job;
>> + int ret;
>>
>> sched = container_of(work, struct drm_gpu_scheduler,
>> work_tdr.work);
>>
>> @@ -331,8 +332,13 @@ static void drm_sched_job_timedout(struct work_struct *work)
>> list_del_init(&job->list);
>> spin_unlock(&sched->job_list_lock);
>>
>> - job->sched->ops->timedout_job(job);
>> + ret = job->sched->ops->timedout_job(job);
>>
>> + if (ret == DRM_GPU_SCHED_STAT_BAILING) {
>> + spin_lock(&sched->job_list_lock);
>> + list_add(&job->node, &sched->ring_mirror_list);
>> + spin_unlock(&sched->job_list_lock);
>> + }
Problem here that since you already dropped the reset locks you are racing here now against other recovery threads as they process the same mirror list, and yet,I think this solution makes things better then they are now with the leak but still, it's only temporary band-aid until the full solution to be implemented. Probably then worth mentioning here with a comment this it's a temporary fix and that races are possible.
Andrey
>> /*
>> * Guilty job did complete and hence needs to be manually removed
>> * See drm_sched_stop doc.
>> diff --git a/include/drm/gpu_scheduler.h
>> b/include/drm/gpu_scheduler.h index 4ea8606d91fe..8093ac2427ef 100644
>> --- a/include/drm/gpu_scheduler.h
>> +++ b/include/drm/gpu_scheduler.h
>> @@ -210,6 +210,7 @@ enum drm_gpu_sched_stat {
>> DRM_GPU_SCHED_STAT_NONE, /* Reserve 0 */
>> DRM_GPU_SCHED_STAT_NOMINAL,
>> DRM_GPU_SCHED_STAT_ENODEV,
>> + DRM_GPU_SCHED_STAT_BAILING,
>> };
>>
>> /**
>> --
>> 2.25.1
>>
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flis
>> ts.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=04%7C01%7C
>> Andrey.Grodzovsky%40amd.com%7Cb8e9c87369a74a6b0f2a08d8e50d4707%7C3dd8
>> 961fe4884e608e11a82d994e183d%7C0%7C0%7C637511192474845823%7CUnknown%7
>> CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
>> VCI6Mn0%3D%7C1000&sdata=FgOIeqk34Sk13L%2BbXYxbqja%2BDY8NSva4wDtUW
>> BMS3q0%3D&reserved=0
_______________________________________________
dri-devel mailing list
dri-devel@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/dri-devel
^ permalink raw reply [flat|nested] 7+ messages in thread
* RE: [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
@ 2021-03-12 6:51 ` Zhang, Jack (Jian)
0 siblings, 0 replies; 7+ messages in thread
From: Zhang, Jack (Jian) @ 2021-03-12 6:51 UTC (permalink / raw)
To: Grodzovsky, Andrey, Alex Deucher, Maling list - DRI developers
Cc: Liu, Monk, Deng, Emily, Koenig, Christian, amd-gfx list
[AMD Official Use Only - Internal Distribution Only]
Hi, Andrey,
ok, I have changed it and uploaded V2 patch.
Thanks,
Jack
-----Original Message-----
From: Grodzovsky, Andrey <Andrey.Grodzovsky@amd.com>
Sent: Friday, March 12, 2021 1:04 PM
To: Alex Deucher <alexdeucher@gmail.com>; Zhang, Jack (Jian) <Jack.Zhang1@amd.com>; Maling list - DRI developers <dri-devel@lists.freedesktop.org>
Cc: amd-gfx list <amd-gfx@lists.freedesktop.org>; Koenig, Christian <Christian.Koenig@amd.com>; Liu, Monk <Monk.Liu@amd.com>; Deng, Emily <Emily.Deng@amd.com>
Subject: Re: [PATCH] drm/scheduler re-insert Bailing job to avoid memleak
Check panfrost driver at panfrost_scheduler_stop, and panfrost_job_timedout - they also terminate prematurely in both places so probably worth adding this there too.
Andrey
On 2021-03-11 11:13 p.m., Alex Deucher wrote:
> +dri-devel
>
> Please be sure to cc dri-devel when you send out gpu scheduler patches.
>
> On Thu, Mar 11, 2021 at 10:57 PM Jack Zhang <Jack.Zhang1@amd.com> wrote:
>>
>> re-insert Bailing jobs to avoid memory leak.
>>
>> Signed-off-by: Jack Zhang <Jack.Zhang1@amd.com>
>> ---
>> drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 4 +++-
>> drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 8 ++++++--
>> drivers/gpu/drm/scheduler/sched_main.c | 8 +++++++-
>> include/drm/gpu_scheduler.h | 1 +
>> 4 files changed, 17 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index 79b9cc73763f..86463b0f936e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -4815,8 +4815,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
>> job ? job->base.id : -1);
>>
>> /* even we skipped this reset, still need to set the job to guilty */
>> - if (job)
>> + if (job) {
>> drm_sched_increase_karma(&job->base);
>> + r = DRM_GPU_SCHED_STAT_BAILING;
>> + }
>> goto skip_recovery;
>> }
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> index 759b34799221..41390bdacd9e 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> @@ -34,6 +34,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>> struct amdgpu_job *job = to_amdgpu_job(s_job);
>> struct amdgpu_task_info ti;
>> struct amdgpu_device *adev = ring->adev;
>> + int ret;
>>
>> memset(&ti, 0, sizeof(struct amdgpu_task_info));
>>
>> @@ -52,8 +53,11 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>> ti.process_name, ti.tgid, ti.task_name, ti.pid);
>>
>> if (amdgpu_device_should_recover_gpu(ring->adev)) {
>> - amdgpu_device_gpu_recover(ring->adev, job);
>> - return DRM_GPU_SCHED_STAT_NOMINAL;
>> + ret = amdgpu_device_gpu_recover(ring->adev, job);
>> + if (ret == DRM_GPU_SCHED_STAT_BAILING)
>> + return DRM_GPU_SCHED_STAT_BAILING;
>> + else
>> + return DRM_GPU_SCHED_STAT_NOMINAL;
>> } else {
>> drm_sched_suspend_timeout(&ring->sched);
>> if (amdgpu_sriov_vf(adev)) diff --git
>> a/drivers/gpu/drm/scheduler/sched_main.c
>> b/drivers/gpu/drm/scheduler/sched_main.c
>> index 92d8de24d0a1..a44f621fb5c4 100644
>> --- a/drivers/gpu/drm/scheduler/sched_main.c
>> +++ b/drivers/gpu/drm/scheduler/sched_main.c
>> @@ -314,6 +314,7 @@ static void drm_sched_job_timedout(struct work_struct *work)
>> {
>> struct drm_gpu_scheduler *sched;
>> struct drm_sched_job *job;
>> + int ret;
>>
>> sched = container_of(work, struct drm_gpu_scheduler,
>> work_tdr.work);
>>
>> @@ -331,8 +332,13 @@ static void drm_sched_job_timedout(struct work_struct *work)
>> list_del_init(&job->list);
>> spin_unlock(&sched->job_list_lock);
>>
>> - job->sched->ops->timedout_job(job);
>> + ret = job->sched->ops->timedout_job(job);
>>
>> + if (ret == DRM_GPU_SCHED_STAT_BAILING) {
>> + spin_lock(&sched->job_list_lock);
>> + list_add(&job->node, &sched->ring_mirror_list);
>> + spin_unlock(&sched->job_list_lock);
>> + }
Problem here that since you already dropped the reset locks you are racing here now against other recovery threads as they process the same mirror list, and yet,I think this solution makes things better then they are now with the leak but still, it's only temporary band-aid until the full solution to be implemented. Probably then worth mentioning here with a comment this it's a temporary fix and that races are possible.
Andrey
>> /*
>> * Guilty job did complete and hence needs to be manually removed
>> * See drm_sched_stop doc.
>> diff --git a/include/drm/gpu_scheduler.h
>> b/include/drm/gpu_scheduler.h index 4ea8606d91fe..8093ac2427ef 100644
>> --- a/include/drm/gpu_scheduler.h
>> +++ b/include/drm/gpu_scheduler.h
>> @@ -210,6 +210,7 @@ enum drm_gpu_sched_stat {
>> DRM_GPU_SCHED_STAT_NONE, /* Reserve 0 */
>> DRM_GPU_SCHED_STAT_NOMINAL,
>> DRM_GPU_SCHED_STAT_ENODEV,
>> + DRM_GPU_SCHED_STAT_BAILING,
>> };
>>
>> /**
>> --
>> 2.25.1
>>
>> _______________________________________________
>> amd-gfx mailing list
>> amd-gfx@lists.freedesktop.org
>> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flis
>> ts.freedesktop.org%2Fmailman%2Flistinfo%2Famd-gfx&data=04%7C01%7C
>> Andrey.Grodzovsky%40amd.com%7Cb8e9c87369a74a6b0f2a08d8e50d4707%7C3dd8
>> 961fe4884e608e11a82d994e183d%7C0%7C0%7C637511192474845823%7CUnknown%7
>> CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJX
>> VCI6Mn0%3D%7C1000&sdata=FgOIeqk34Sk13L%2BbXYxbqja%2BDY8NSva4wDtUW
>> BMS3q0%3D&reserved=0
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2021-03-12 6:51 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2021-03-12 3:57 [PATCH] drm/scheduler re-insert Bailing job to avoid memleak Jack Zhang
2021-03-12 4:13 ` Alex Deucher
2021-03-12 4:13 ` Alex Deucher
2021-03-12 5:04 ` Andrey Grodzovsky
2021-03-12 5:04 ` Andrey Grodzovsky
2021-03-12 6:51 ` Zhang, Jack (Jian)
2021-03-12 6:51 ` Zhang, Jack (Jian)
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.