[Xen-devel] [PATCH v3 27/47] xen/sched: move struct task_slice into struct sched_unit

[Juergen Gross <jgross@suse.com>]

In order to prepare for multiple vcpus per schedule unit move struct
task_slice in schedule() from the local stack into struct sched_unit
of the currently running unit. To make access easier for the single
schedulers add the pointer of the currently running unit as a parameter
of do_schedule().

While at it switch the tasklet_work_scheduled parameter of
do_schedule() from bool_t to bool.

As struct task_slice is only ever modified with the local schedule
lock held it is safe to directly set the different units in struct
sched_unit instead of using an on-stack copy for returning the data.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
V3:
- readd accidentally dropped call of continue_running() (Dario Faggioli)
---
 xen/common/sched_arinc653.c | 20 +++++++-------------
 xen/common/sched_credit.c   | 25 +++++++++++--------------
 xen/common/sched_credit2.c  | 21 +++++++++------------
 xen/common/sched_null.c     | 29 ++++++++++++++---------------
 xen/common/sched_rt.c       | 22 +++++++++++-----------
 xen/common/schedule.c       | 30 ++++++++++++++----------------
 xen/include/xen/sched-if.h  | 11 +++--------
 xen/include/xen/sched.h     |  6 ++++++
 8 files changed, 75 insertions(+), 89 deletions(-)

diff --git a/xen/common/sched_arinc653.c b/xen/common/sched_arinc653.c
index a4db34843b..96f3e844d2 100644
--- a/xen/common/sched_arinc653.c
+++ b/xen/common/sched_arinc653.c
@@ -497,18 +497,14 @@ a653sched_unit_wake(const struct scheduler *ops, struct sched_unit *unit)
  *
  * @param ops       Pointer to this instance of the scheduler structure
  * @param now       Current time
- *
- * @return          Address of the UNIT structure scheduled to be run next
- *                  Amount of time to execute the returned UNIT
- *                  Flag for whether the UNIT was migrated
  */
-static struct task_slice
+static void
 a653sched_do_schedule(
     const struct scheduler *ops,
+    struct sched_unit *prev,
     s_time_t now,
-    bool_t tasklet_work_scheduled)
+    bool tasklet_work_scheduled)
 {
-    struct task_slice ret;                      /* hold the chosen domain */
     struct sched_unit *new_task = NULL;
     static unsigned int sched_index = 0;
     static s_time_t next_switch_time;
@@ -586,13 +582,11 @@ a653sched_do_schedule(
      * Return the amount of time the next domain has to run and the address
      * of the selected task's UNIT structure.
      */
-    ret.time = next_switch_time - now;
-    ret.task = new_task;
-    ret.migrated = 0;
-
-    BUG_ON(ret.time <= 0);
+    prev->next_time = next_switch_time - now;
+    prev->next_task = new_task;
+    new_task->migrated = false;
 
-    return ret;
+    BUG_ON(prev->next_time <= 0);
 }
 
 /**
diff --git a/xen/common/sched_credit.c b/xen/common/sched_credit.c
index 203c5decef..5802a67784 100644
--- a/xen/common/sched_credit.c
+++ b/xen/common/sched_credit.c
@@ -1675,7 +1675,7 @@ csched_runq_steal(int peer_cpu, int cpu, int pri, int balance_step)
 
 static struct csched_unit *
 csched_load_balance(struct csched_private *prv, int cpu,
-    struct csched_unit *snext, bool_t *stolen)
+    struct csched_unit *snext, bool *stolen)
 {
     struct cpupool *c = per_cpu(cpupool, cpu);
     struct csched_unit *speer;
@@ -1791,7 +1791,7 @@ csched_load_balance(struct csched_private *prv, int cpu,
                 /* As soon as one unit is found, balancing ends */
                 if ( speer != NULL )
                 {
-                    *stolen = 1;
+                    *stolen = true;
                     /*
                      * Next time we'll look for work to steal on this node, we
                      * will start from the next pCPU, with respect to this one,
@@ -1821,19 +1821,18 @@ csched_load_balance(struct csched_private *prv, int cpu,
  * This function is in the critical path. It is designed to be simple and
  * fast for the common case.
  */
-static struct task_slice
-csched_schedule(
-    const struct scheduler *ops, s_time_t now, bool_t tasklet_work_scheduled)
+static void csched_schedule(
+    const struct scheduler *ops, struct sched_unit *unit, s_time_t now,
+    bool tasklet_work_scheduled)
 {
     const unsigned int cur_cpu = smp_processor_id();
     const unsigned int sched_cpu = sched_get_resource_cpu(cur_cpu);
     struct list_head * const runq = RUNQ(sched_cpu);
-    struct sched_unit *unit = current->sched_unit;
     struct csched_unit * const scurr = CSCHED_UNIT(unit);
     struct csched_private *prv = CSCHED_PRIV(ops);
     struct csched_unit *snext;
-    struct task_slice ret;
     s_time_t runtime, tslice;
+    bool migrated = false;
 
     SCHED_STAT_CRANK(schedule);
     CSCHED_UNIT_CHECK(unit);
@@ -1924,7 +1923,6 @@ csched_schedule(
                         (unsigned char *)&d);
         }
 
-        ret.migrated = 0;
         goto out;
     }
     tslice = prv->tslice;
@@ -1942,7 +1940,6 @@ csched_schedule(
     }
 
     snext = __runq_elem(runq->next);
-    ret.migrated = 0;
 
     /* Tasklet work (which runs in idle UNIT context) overrides all else. */
     if ( tasklet_work_scheduled )
@@ -1968,7 +1965,7 @@ csched_schedule(
     if ( snext->pri > CSCHED_PRI_TS_OVER )
         __runq_remove(snext);
     else
-        snext = csched_load_balance(prv, sched_cpu, snext, &ret.migrated);
+        snext = csched_load_balance(prv, sched_cpu, snext, &migrated);
 
     /*
      * Update idlers mask if necessary. When we're idling, other CPUs
@@ -1991,12 +1988,12 @@ out:
     /*
      * Return task to run next...
      */
-    ret.time = (is_idle_unit(snext->unit) ?
+    unit->next_time = (is_idle_unit(snext->unit) ?
                 -1 : tslice);
-    ret.task = snext->unit;
+    unit->next_task = snext->unit;
+    snext->unit->migrated = migrated;
 
-    CSCHED_UNIT_CHECK(ret.task);
-    return ret;
+    CSCHED_UNIT_CHECK(unit->next_task);
 }
 
 static void
diff --git a/xen/common/sched_credit2.c b/xen/common/sched_credit2.c
index cf0559e964..414ac8f5b6 100644
--- a/xen/common/sched_credit2.c
+++ b/xen/common/sched_credit2.c
@@ -3444,19 +3444,18 @@ runq_candidate(struct csched2_runqueue_data *rqd,
  * This function is in the critical path. It is designed to be simple and
  * fast for the common case.
  */
-static struct task_slice
-csched2_schedule(
-    const struct scheduler *ops, s_time_t now, bool tasklet_work_scheduled)
+static void csched2_schedule(
+    const struct scheduler *ops, struct sched_unit *currunit, s_time_t now,
+    bool tasklet_work_scheduled)
 {
     const unsigned int cur_cpu = smp_processor_id();
     const unsigned int sched_cpu = sched_get_resource_cpu(cur_cpu);
     struct csched2_runqueue_data *rqd;
-    struct sched_unit *currunit = current->sched_unit;
     struct csched2_unit * const scurr = csched2_unit(currunit);
     struct csched2_unit *snext = NULL;
     unsigned int skipped_units = 0;
-    struct task_slice ret;
     bool tickled;
+    bool migrated = false;
 
     SCHED_STAT_CRANK(schedule);
     CSCHED2_UNIT_CHECK(currunit);
@@ -3541,8 +3540,6 @@ csched2_schedule(
          && unit_runnable(currunit) )
         __set_bit(__CSFLAG_delayed_runq_add, &scurr->flags);
 
-    ret.migrated = 0;
-
     /* Accounting for non-idle tasks */
     if ( !is_idle_unit(snext->unit) )
     {
@@ -3592,7 +3589,7 @@ csched2_schedule(
             snext->credit += CSCHED2_MIGRATE_COMPENSATION;
             sched_set_res(snext->unit, get_sched_res(sched_cpu));
             SCHED_STAT_CRANK(migrated);
-            ret.migrated = 1;
+            migrated = true;
         }
     }
     else
@@ -3623,11 +3620,11 @@ csched2_schedule(
     /*
      * Return task to run next...
      */
-    ret.time = csched2_runtime(ops, sched_cpu, snext, now);
-    ret.task = snext->unit;
+    currunit->next_time = csched2_runtime(ops, sched_cpu, snext, now);
+    currunit->next_task = snext->unit;
+    snext->unit->migrated = migrated;
 
-    CSCHED2_UNIT_CHECK(ret.task);
-    return ret;
+    CSCHED2_UNIT_CHECK(currunit->next_task);
 }
 
 static void
diff --git a/xen/common/sched_null.c b/xen/common/sched_null.c
index 4131df0db3..d7deef07b8 100644
--- a/xen/common/sched_null.c
+++ b/xen/common/sched_null.c
@@ -779,16 +779,14 @@ static inline void null_unit_check(struct sched_unit *unit)
  *  - the unit assigned to the pCPU, if there's one and it can run;
  *  - the idle unit, otherwise.
  */
-static struct task_slice null_schedule(const struct scheduler *ops,
-                                       s_time_t now,
-                                       bool_t tasklet_work_scheduled)
+static void null_schedule(const struct scheduler *ops, struct sched_unit *prev,
+                          s_time_t now, bool tasklet_work_scheduled)
 {
     unsigned int bs;
     const unsigned int cur_cpu = smp_processor_id();
     const unsigned int sched_cpu = sched_get_resource_cpu(cur_cpu);
     struct null_private *prv = null_priv(ops);
     struct null_unit *wvc;
-    struct task_slice ret;
 
     SCHED_STAT_CRANK(schedule);
     NULL_UNIT_CHECK(current->sched_unit);
@@ -816,19 +814,18 @@ static struct task_slice null_schedule(const struct scheduler *ops,
     if ( tasklet_work_scheduled )
     {
         trace_var(TRC_SNULL_TASKLET, 1, 0, NULL);
-        ret.task = sched_idle_unit(sched_cpu);
+        prev->next_task = sched_idle_unit(sched_cpu);
     }
     else
-        ret.task = per_cpu(npc, sched_cpu).unit;
-    ret.migrated = 0;
-    ret.time = -1;
+        prev->next_task = per_cpu(npc, sched_cpu).unit;
+    prev->next_time = -1;
 
     /*
      * We may be new in the cpupool, or just coming back online. In which
      * case, there may be units in the waitqueue that we can assign to us
      * and run.
      */
-    if ( unlikely(ret.task == NULL) )
+    if ( unlikely(prev->next_task == NULL) )
     {
         spin_lock(&prv->waitq_lock);
 
@@ -854,7 +851,7 @@ static struct task_slice null_schedule(const struct scheduler *ops,
                 {
                     unit_assign(prv, wvc->unit, sched_cpu);
                     list_del_init(&wvc->waitq_elem);
-                    ret.task = wvc->unit;
+                    prev->next_task = wvc->unit;
                     goto unlock;
                 }
             }
@@ -862,15 +859,17 @@ static struct task_slice null_schedule(const struct scheduler *ops,
  unlock:
         spin_unlock(&prv->waitq_lock);
 
-        if ( ret.task == NULL && !cpumask_test_cpu(sched_cpu, &prv->cpus_free) )
+        if ( prev->next_task == NULL &&
+             !cpumask_test_cpu(sched_cpu, &prv->cpus_free) )
             cpumask_set_cpu(sched_cpu, &prv->cpus_free);
     }
 
-    if ( unlikely(ret.task == NULL || !unit_runnable(ret.task)) )
-        ret.task = sched_idle_unit(sched_cpu);
+    if ( unlikely(prev->next_task == NULL || !unit_runnable(prev->next_task)) )
+        prev->next_task = sched_idle_unit(sched_cpu);
 
-    NULL_UNIT_CHECK(ret.task);
-    return ret;
+    NULL_UNIT_CHECK(prev->next_task);
+
+    prev->next_task->migrated = false;
 }
 
 static inline void dump_unit(struct null_private *prv, struct null_unit *nvc)
diff --git a/xen/common/sched_rt.c b/xen/common/sched_rt.c
index ab48d4bb1d..91bc3d56fb 100644
--- a/xen/common/sched_rt.c
+++ b/xen/common/sched_rt.c
@@ -1054,16 +1054,16 @@ runq_pick(const struct scheduler *ops, const cpumask_t *mask)
  * schedule function for rt scheduler.
  * The lock is already grabbed in schedule.c, no need to lock here
  */
-static struct task_slice
-rt_schedule(const struct scheduler *ops, s_time_t now, bool_t tasklet_work_scheduled)
+static void
+rt_schedule(const struct scheduler *ops, struct sched_unit *currunit,
+            s_time_t now, bool tasklet_work_scheduled)
 {
     const unsigned int cur_cpu = smp_processor_id();
     const unsigned int sched_cpu = sched_get_resource_cpu(cur_cpu);
     struct rt_private *prv = rt_priv(ops);
-    struct rt_unit *const scurr = rt_unit(current->sched_unit);
+    struct rt_unit *const scurr = rt_unit(currunit);
     struct rt_unit *snext = NULL;
-    struct task_slice ret = { .migrated = 0 };
-    struct sched_unit *currunit = current->sched_unit;
+    bool migrated = false;
 
     /* TRACE */
     {
@@ -1111,7 +1111,7 @@ rt_schedule(const struct scheduler *ops, s_time_t now, bool_t tasklet_work_sched
         __set_bit(__RTDS_delayed_runq_add, &scurr->flags);
 
     snext->last_start = now;
-    ret.time =  -1; /* if an idle unit is picked */
+    currunit->next_time =  -1; /* if an idle unit is picked */
     if ( !is_idle_unit(snext->unit) )
     {
         if ( snext != scurr )
@@ -1122,13 +1122,13 @@ rt_schedule(const struct scheduler *ops, s_time_t now, bool_t tasklet_work_sched
         if ( sched_unit_cpu(snext->unit) != sched_cpu )
         {
             sched_set_res(snext->unit, get_sched_res(sched_cpu));
-            ret.migrated = 1;
+            migrated = true;
         }
-        ret.time = snext->cur_budget; /* invoke the scheduler next time */
+        /* Invoke the scheduler next time. */
+        currunit->next_time = snext->cur_budget;
     }
-    ret.task = snext->unit;
-
-    return ret;
+    currunit->next_task = snext->unit;
+    snext->unit->migrated = migrated;
 }
 
 /*
diff --git a/xen/common/schedule.c b/xen/common/schedule.c
index 70271cdea2..aad396ee54 100644
--- a/xen/common/schedule.c
+++ b/xen/common/schedule.c
@@ -108,15 +108,14 @@ sched_idle_free_vdata(const struct scheduler *ops, void *priv)
 {
 }
 
-static struct task_slice sched_idle_schedule(
-    const struct scheduler *ops, s_time_t now,
+static void sched_idle_schedule(
+    const struct scheduler *ops, struct sched_unit *unit, s_time_t now,
     bool tasklet_work_scheduled)
 {
     const unsigned int cpu = smp_processor_id();
-    struct task_slice ret = { .time = -1 };
 
-    ret.task = sched_idle_unit(cpu);
-    return ret;
+    unit->next_time = -1;
+    unit->next_task = sched_idle_unit(cpu);
 }
 
 static struct scheduler sched_idle_ops = {
@@ -1719,10 +1718,9 @@ static void schedule(void)
     s_time_t              now;
     struct scheduler     *sched;
     unsigned long        *tasklet_work = &this_cpu(tasklet_work_to_do);
-    bool_t                tasklet_work_scheduled = 0;
+    bool                  tasklet_work_scheduled = false;
     struct sched_resource *sd;
     spinlock_t           *lock;
-    struct task_slice     next_slice;
     int cpu = smp_processor_id();
 
     ASSERT_NOT_IN_ATOMIC();
@@ -1738,12 +1736,12 @@ static void schedule(void)
         set_bit(_TASKLET_scheduled, tasklet_work);
         /* fallthrough */
     case TASKLET_enqueued|TASKLET_scheduled:
-        tasklet_work_scheduled = 1;
+        tasklet_work_scheduled = true;
         break;
     case TASKLET_scheduled:
         clear_bit(_TASKLET_scheduled, tasklet_work);
     case 0:
-        /*tasklet_work_scheduled = 0;*/
+        /*tasklet_work_scheduled = false;*/
         break;
     default:
         BUG();
@@ -1757,14 +1755,14 @@ static void schedule(void)
 
     /* get policy-specific decision on scheduling... */
     sched = this_cpu(scheduler);
-    next_slice = sched->do_schedule(sched, now, tasklet_work_scheduled);
+    sched->do_schedule(sched, prev, now, tasklet_work_scheduled);
 
-    next = next_slice.task;
+    next = prev->next_task;
 
     sd->curr = next;
 
-    if ( next_slice.time >= 0 ) /* -ve means no limit */
-        set_timer(&sd->s_timer, now + next_slice.time);
+    if ( prev->next_time >= 0 ) /* -ve means no limit */
+        set_timer(&sd->s_timer, now + prev->next_time);
 
     if ( unlikely(prev == next) )
     {
@@ -1772,7 +1770,7 @@ static void schedule(void)
         TRACE_4D(TRC_SCHED_SWITCH_INFCONT,
                  next->domain->domain_id, next->unit_id,
                  now - prev->state_entry_time,
-                 next_slice.time);
+                 prev->next_time);
         trace_continue_running(next->vcpu_list);
         return continue_running(prev->vcpu_list);
     }
@@ -1784,7 +1782,7 @@ static void schedule(void)
              next->domain->domain_id, next->unit_id,
              (next->vcpu_list->runstate.state == RUNSTATE_runnable) ?
              (now - next->state_entry_time) : 0,
-             next_slice.time);
+             prev->next_time);
 
     ASSERT(prev->vcpu_list->runstate.state == RUNSTATE_running);
 
@@ -1813,7 +1811,7 @@ static void schedule(void)
 
     stop_timer(&prev->vcpu_list->periodic_timer);
 
-    if ( next_slice.migrated )
+    if ( next->migrated )
         vcpu_move_irqs(next->vcpu_list);
 
     vcpu_periodic_timer_work(next->vcpu_list);
diff --git a/xen/include/xen/sched-if.h b/xen/include/xen/sched-if.h
index 8429e9e287..4e89a1e640 100644
--- a/xen/include/xen/sched-if.h
+++ b/xen/include/xen/sched-if.h
@@ -196,12 +196,6 @@ static inline spinlock_t *pcpu_schedule_trylock(unsigned int cpu)
     return NULL;
 }
 
-struct task_slice {
-    struct sched_unit *task;
-    s_time_t           time;
-    bool_t             migrated;
-};
-
 struct scheduler {
     char *name;             /* full name for this scheduler      */
     char *opt_name;         /* option name for this scheduler    */
@@ -244,8 +238,9 @@ struct scheduler {
     void         (*context_saved)  (const struct scheduler *,
                                     struct sched_unit *);
 
-    struct task_slice (*do_schedule) (const struct scheduler *, s_time_t,
-                                      bool_t tasklet_work_scheduled);
+    void         (*do_schedule)    (const struct scheduler *,
+                                    struct sched_unit *, s_time_t,
+                                    bool tasklet_work_scheduled);
 
     struct sched_resource *(*pick_resource)(const struct scheduler *,
                                             const struct sched_unit *);
diff --git a/xen/include/xen/sched.h b/xen/include/xen/sched.h
index 36257a06fe..a8164d873c 100644
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -274,6 +274,8 @@ struct sched_unit {
     bool                   is_running;
     /* Does soft affinity actually play a role (given hard affinity)? */
     bool                   soft_aff_effective;
+    /* Item has been migrated to other cpu(s). */
+    bool                   migrated;
 
     /* Last time unit got (de-)scheduled. */
     uint64_t               state_entry_time;
@@ -286,6 +288,10 @@ struct sched_unit {
     cpumask_var_t          cpu_hard_affinity_saved;
     /* Bitmask of CPUs on which this VCPU prefers to run. */
     cpumask_var_t          cpu_soft_affinity;
+
+    /* Next unit to run. */
+    struct sched_unit      *next_task;
+    s_time_t                next_time;
 };
 
 #define for_each_sched_unit(d, e)                                         \
-- 
2.16.4


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xenproject.org
https://lists.xenproject.org/mailman/listinfo/xen-devel