From c7fad79b1790e5a4f43e95c390ebf4e638662fe4 Mon Sep 17 00:00:00 2001 From: Andrey Grodzovsky Date: Fri, 29 Jul 2022 12:26:50 -0400 Subject: drm/sced: Add FIFO sched policy to rq v7 Also add enblement flag. Add ordering based on TS Signed-off-by: Andrey Grodzovsky --- drivers/gpu/drm/scheduler/sched_entity.c | 6 ++ drivers/gpu/drm/scheduler/sched_main.c | 109 ++++++++++++++++++++++- include/drm/gpu_scheduler.h | 38 ++++++++ 3 files changed, 150 insertions(+), 3 deletions(-) diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c index 191c56064f19..6eb495dec09f 100644 --- a/drivers/gpu/drm/scheduler/sched_entity.c +++ b/drivers/gpu/drm/scheduler/sched_entity.c @@ -33,6 +33,8 @@ #define to_drm_sched_job(sched_job) \ container_of((sched_job), struct drm_sched_job, queue_node) +extern int drm_sched_policy; + /** * drm_sched_entity_init - Init a context entity used by scheduler when * submit to HW ring. @@ -73,6 +75,7 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, entity->priority = priority; entity->sched_list = num_sched_list > 1 ? sched_list : NULL; entity->last_scheduled = NULL; + RB_CLEAR_NODE(&entity->rb_tree_node); if(num_sched_list) entity->rq = &sched_list[0]->sched_rq[entity->priority]; @@ -443,6 +446,7 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) smp_wmb(); spsc_queue_pop(&entity->job_queue); + return sched_job; } @@ -507,6 +511,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) atomic_inc(entity->rq->sched->score); WRITE_ONCE(entity->last_user, current->group_leader); first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); + sched_job->submit_ts = ktime_get(); /* first job wakes up scheduler */ if (first) { @@ -518,6 +523,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) DRM_ERROR("Trying to push to a killed entity\n"); return; } + drm_sched_rq_add_entity(entity->rq, entity); spin_unlock(&entity->rq_lock); drm_sched_wakeup(entity->rq->sched); diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c index c5437ee03e3f..530f7f3e2aea 100644 --- a/drivers/gpu/drm/scheduler/sched_main.c +++ b/drivers/gpu/drm/scheduler/sched_main.c @@ -62,6 +62,42 @@ #define to_drm_sched_job(sched_job) \ container_of((sched_job), struct drm_sched_job, queue_node) +int drm_sched_policy = 1; + +/** + * DOC: sched_policy (int) + * Used to override default entites scheduling policy in a run queue. + */ +MODULE_PARM_DESC(sched_policy, + "specify schedule policy for entites on a runqueue (-1 = auto(default) value, 0 = Round Robin,1 = use FIFO"); +module_param_named(sched_policy, drm_sched_policy, int, 0444); + +static inline void drm_sched_rq_update_fifo_locked(struct drm_sched_rq *rq, + struct drm_sched_entity *entity, + bool remove_only) +{ + if (!RB_EMPTY_NODE(&entity->rb_tree_node)) { + rb_erase_cached(&entity->rb_tree_node, &rq->rb_tree_root); + RB_CLEAR_NODE(&entity->rb_tree_node); + } + + if (remove_only) + return; + + /* + * TODO - In case there is next job pending to run use it's TS to update + * the entity location in min heap. Otherwise just thow it to the 'back of + * the line' + */ + if (drm_sched_entity_is_ready(entity)) + entity->oldest_job_waiting = to_drm_sched_job(spsc_queue_peek(&tmp->job_queue)); + else + entity->oldest_job_waiting = ktime_get(); + + rb_add_cached(&entity->rb_tree_node, &rq->rb_tree_root, + drm_sched_entity_compare_earlier); +} + /** * drm_sched_rq_init - initialize a given run queue struct * @@ -75,6 +111,7 @@ static void drm_sched_rq_init(struct drm_gpu_scheduler *sched, { spin_lock_init(&rq->lock); INIT_LIST_HEAD(&rq->entities); + rq->rb_tree_root = RB_ROOT_CACHED; rq->current_entity = NULL; rq->sched = sched; } @@ -92,9 +129,15 @@ void drm_sched_rq_add_entity(struct drm_sched_rq *rq, { if (!list_empty(&entity->list)) return; + spin_lock(&rq->lock); + atomic_inc(rq->sched->score); list_add_tail(&entity->list, &rq->entities); + + if (drm_sched_policy == 1) + drm_sched_rq_update_fifo_locked(entity->rq, entity, false); + spin_unlock(&rq->lock); } @@ -111,23 +154,32 @@ void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, { if (list_empty(&entity->list)) return; + spin_lock(&rq->lock); + + atomic_dec(rq->sched->score); list_del_init(&entity->list); + if (rq->current_entity == entity) rq->current_entity = NULL; + + if (drm_sched_policy == 1) + drm_sched_rq_update_fifo_locked(entity->rq, entity, true); + spin_unlock(&rq->lock); } + /** - * drm_sched_rq_select_entity - Select an entity which could provide a job to run + * drm_sched_rq_select_entity_rr - Select an entity which could provide a job to run * * @rq: scheduler run queue to check. * * Try to find a ready entity, returns NULL if none found. */ static struct drm_sched_entity * -drm_sched_rq_select_entity(struct drm_sched_rq *rq) +drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq) { struct drm_sched_entity *entity; @@ -163,6 +215,54 @@ drm_sched_rq_select_entity(struct drm_sched_rq *rq) return NULL; } + +/** + * drm_sched_rq_select_entity_fifo - Select an entity which could provide a job to run + * + * @rq: scheduler run queue to check. + * + * Try to find a ready entity, returns NULL if none found. + */ +static struct drm_sched_entity * +drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq) +{ + struct drm_sched_entity *first, *entity = NULL; + struct rb_node *rb; + spin_lock(&rq->lock); + + rb = rb_first_cached(&rq->rb_tree_root); + if (!rb) + goto out; + + first = rb_entry((rb), struct drm_sched_entity, rb_tree_node); + entity = first; + + while(true){ + + /* Update entity's TS for the FIFO and update the FIFO accordingly */ + drm_sched_rq_update_fifo_locked(entity->rq, entity, false); + + if (drm_sched_entity_is_ready(entity)) { + rq->current_entity = entity; + reinit_completion(&entity->entity_idle); + break; + } + + rb = rb_first_cached(&rq->rb_tree_root); + entity = rb_entry((rb), struct drm_sched_entity, rb_tree_node); + + /* We completed full cycle */ + if (!drm_sched_entity_is_ready(entity) && entity == first) { + entity = NULL; + break; + } + } + + out: + spin_unlock(&rq->lock); + return entity; +} + /** * drm_sched_job_done - complete a job * @s_job: pointer to the job which is done @@ -592,6 +692,7 @@ int drm_sched_job_init(struct drm_sched_job *job, struct drm_sched_entity *entity, void *owner) { + drm_sched_entity_select_rq(entity); if (!entity->rq) return -ENOENT; @@ -801,7 +902,9 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) /* Kernel run queue has higher priority than normal run queue*/ for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { - entity = drm_sched_rq_select_entity(&sched->sched_rq[i]); + entity = drm_sched_policy != 1 ? + drm_sched_rq_select_entity_rr(&sched->sched_rq[i]) : + drm_sched_rq_select_entity_fifo(&sched->sched_rq[i]); if (entity) break; } diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h index 944f83ef9f2e..1c841a24b5c0 100644 --- a/include/drm/gpu_scheduler.h +++ b/include/drm/gpu_scheduler.h @@ -196,6 +196,21 @@ struct drm_sched_entity { * drm_sched_entity_fini(). */ struct completion entity_idle; + + /** + * @oldest_job_waiting: + * + * Marks earliest job waiting in SW queue + */ + ktime_t oldest_job_waiting; + + /** + * @rb_tree_node: + * + * To insert this entity into time based priority queue + */ + struct rb_node rb_tree_node; + }; /** @@ -205,6 +220,7 @@ struct drm_sched_entity { * @sched: the scheduler to which this rq belongs to. * @entities: list of the entities to be scheduled. * @current_entity: the entity which is to be scheduled. + * @rb_tree_root: root of time based priory queue of entites for FIFO scheduling * * Run queue is a set of entities scheduling command submissions for * one specific ring. It implements the scheduling policy that selects @@ -215,6 +231,7 @@ struct drm_sched_rq { struct drm_gpu_scheduler *sched; struct list_head entities; struct drm_sched_entity *current_entity; + struct rb_root_cached rb_tree_root; }; /** @@ -258,6 +275,14 @@ struct drm_sched_fence { * @owner: job owner for debugging */ void *owner; + + /** + * @submit_ts: + * + * Marks job submit time + */ + ktime_t submit_ts; + }; struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); @@ -501,6 +526,19 @@ void drm_sched_rq_add_entity(struct drm_sched_rq *rq, void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, struct drm_sched_entity *entity); +void drm_sched_rq_update_fifo(struct drm_sched_rq *rq, + struct drm_sched_entity *entity, + bool remove_only); + +static __always_inline bool drm_sched_entity_compare_earlier(struct rb_node *a, + const struct rb_node *b) +{ + struct drm_sched_entity *ent_a = rb_entry((a), struct drm_sched_entity, rb_tree_node); + struct drm_sched_entity *ent_b = rb_entry((b), struct drm_sched_entity, rb_tree_node); + + return ktime_before(ent_a->oldest_job_waiting, ent_b->oldest_job_waiting); +} + int drm_sched_entity_init(struct drm_sched_entity *entity, enum drm_sched_priority priority, struct drm_gpu_scheduler **sched_list, -- 2.25.1