[RFC PATCH 0/4] sched: support schedstat for RT sched class

[RFC PATCH 0/4] sched: support schedstat for RT sched class

[Yafang Shao]

We want to measure the latency of RT tasks in our production
environment with schedstat facility, but currently schedstat is only
supported for fair sched class. This patchset enable it for RT sched class
as well.

The schedstat statistics are defined in struct sched_entity, which is a
member of struct task_struct, so we can resue it for RT sched class.

The schedstat usage in RT sched class is similar with fair sched class,
for example,
		fair				RT
enqueue		update_stats_enqueue_fair	update_stats_enqueue_rt
dequeue		update_stats_dequeue_fair	update_stats_dequeue_rt
put_prev_task	update_stats_wait_start		update_stats_wait_start
set_next_task	update_stats_wait_end		update_stats_wait_end
show		/proc/[pid]/sched		/proc/[pid]/sched

The sched:sched_stats_* tracepoints can be used to trace RT tasks as
well after that patchset.

PATCH #1 ~ #3 are the preparation of PATCH #4.

Yafang Shao (4):
  sched: define task_of() as a common helper
  sched: make schedstats helpers not depend on cfs_rq
  sched: define update_stats_curr_start() as a common helper
  sched, rt: support schedstat for RT sched class

 kernel/sched/fair.c  | 173 ++-----------------------------------------
 kernel/sched/rt.c    |  63 +++++++++++++++-
 kernel/sched/sched.h |  23 ++++++
 kernel/sched/stats.c | 134 +++++++++++++++++++++++++++++++++
 kernel/sched/stats.h |  11 +++
 5 files changed, 236 insertions(+), 168 deletions(-)

-- 
2.18.4

[RFC PATCH 1/4] sched: define task_of() as a common helper

[Yafang Shao]

task_of() is used to get task_struct from sched_entity. As sched_entity
in struct task_struct can be used by all sched class, we'd better move
this macro into sched.h, then it can be used by all sched class.

Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
---
 kernel/sched/fair.c  | 11 -----------
 kernel/sched/sched.h |  9 +++++++++
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 8917d2d715ef..9d73e8e5ebec 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -258,12 +258,6 @@ const struct sched_class fair_sched_class;
  */
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static inline struct task_struct *task_of(struct sched_entity *se)
-{
-	SCHED_WARN_ON(!entity_is_task(se));
-	return container_of(se, struct task_struct, se);
-}
-
 /* Walk up scheduling entities hierarchy */
 #define for_each_sched_entity(se) \
 		for (; se; se = se->parent)
@@ -445,11 +439,6 @@ find_matching_se(struct sched_entity **se, struct sched_entity **pse)
 
 #else	/* !CONFIG_FAIR_GROUP_SCHED */
 
-static inline struct task_struct *task_of(struct sched_entity *se)
-{
-	return container_of(se, struct task_struct, se);
-}
-
 #define for_each_sched_entity(se) \
 		for (; se; se = NULL)
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index df80bfcea92e..e29f2375c4f5 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2633,3 +2633,12 @@ static inline bool is_per_cpu_kthread(struct task_struct *p)
 
 void swake_up_all_locked(struct swait_queue_head *q);
 void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
+
+static inline struct task_struct *task_of(struct sched_entity *se)
+{
+#ifdef CONFIG_FAIR_GROUP_SCHED
+	SCHED_WARN_ON(!entity_is_task(se));
+#endif
+	return container_of(se, struct task_struct, se);
+}
+
-- 
2.18.4

[RFC PATCH 2/4] sched: make schedstats helpers not depend on cfs_rq

[Yafang Shao]

The 'cfs_rq' in these helpers is only used to get the rq_clock, so we
can pass the rq_clock directly. After that, these helpers can be used by
all sched class.

Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
---
 kernel/sched/fair.c  | 148 ++-----------------------------------------
 kernel/sched/stats.c | 134 +++++++++++++++++++++++++++++++++++++++
 kernel/sched/stats.h |  11 ++++
 3 files changed, 150 insertions(+), 143 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 9d73e8e5ebec..aba21191283d 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -868,124 +868,6 @@ static void update_curr_fair(struct rq *rq)
 	update_curr(cfs_rq_of(&rq->curr->se));
 }
 
-static inline void
-update_stats_wait_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
-{
-	u64 wait_start, prev_wait_start;
-
-	if (!schedstat_enabled())
-		return;
-
-	wait_start = rq_clock(rq_of(cfs_rq));
-	prev_wait_start = schedstat_val(se->statistics.wait_start);
-
-	if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) &&
-	    likely(wait_start > prev_wait_start))
-		wait_start -= prev_wait_start;
-
-	__schedstat_set(se->statistics.wait_start, wait_start);
-}
-
-static inline void
-update_stats_wait_end(struct cfs_rq *cfs_rq, struct sched_entity *se)
-{
-	struct task_struct *p;
-	u64 delta;
-
-	if (!schedstat_enabled())
-		return;
-
-	delta = rq_clock(rq_of(cfs_rq)) - schedstat_val(se->statistics.wait_start);
-
-	if (entity_is_task(se)) {
-		p = task_of(se);
-		if (task_on_rq_migrating(p)) {
-			/*
-			 * Preserve migrating task's wait time so wait_start
-			 * time stamp can be adjusted to accumulate wait time
-			 * prior to migration.
-			 */
-			__schedstat_set(se->statistics.wait_start, delta);
-			return;
-		}
-		trace_sched_stat_wait(p, delta);
-	}
-
-	__schedstat_set(se->statistics.wait_max,
-		      max(schedstat_val(se->statistics.wait_max), delta));
-	__schedstat_inc(se->statistics.wait_count);
-	__schedstat_add(se->statistics.wait_sum, delta);
-	__schedstat_set(se->statistics.wait_start, 0);
-}
-
-static inline void
-update_stats_enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
-{
-	struct task_struct *tsk = NULL;
-	u64 sleep_start, block_start;
-
-	if (!schedstat_enabled())
-		return;
-
-	sleep_start = schedstat_val(se->statistics.sleep_start);
-	block_start = schedstat_val(se->statistics.block_start);
-
-	if (entity_is_task(se))
-		tsk = task_of(se);
-
-	if (sleep_start) {
-		u64 delta = rq_clock(rq_of(cfs_rq)) - sleep_start;
-
-		if ((s64)delta < 0)
-			delta = 0;
-
-		if (unlikely(delta > schedstat_val(se->statistics.sleep_max)))
-			__schedstat_set(se->statistics.sleep_max, delta);
-
-		__schedstat_set(se->statistics.sleep_start, 0);
-		__schedstat_add(se->statistics.sum_sleep_runtime, delta);
-
-		if (tsk) {
-			account_scheduler_latency(tsk, delta >> 10, 1);
-			trace_sched_stat_sleep(tsk, delta);
-		}
-	}
-	if (block_start) {
-		u64 delta = rq_clock(rq_of(cfs_rq)) - block_start;
-
-		if ((s64)delta < 0)
-			delta = 0;
-
-		if (unlikely(delta > schedstat_val(se->statistics.block_max)))
-			__schedstat_set(se->statistics.block_max, delta);
-
-		__schedstat_set(se->statistics.block_start, 0);
-		__schedstat_add(se->statistics.sum_sleep_runtime, delta);
-
-		if (tsk) {
-			if (tsk->in_iowait) {
-				__schedstat_add(se->statistics.iowait_sum, delta);
-				__schedstat_inc(se->statistics.iowait_count);
-				trace_sched_stat_iowait(tsk, delta);
-			}
-
-			trace_sched_stat_blocked(tsk, delta);
-
-			/*
-			 * Blocking time is in units of nanosecs, so shift by
-			 * 20 to get a milliseconds-range estimation of the
-			 * amount of time that the task spent sleeping:
-			 */
-			if (unlikely(prof_on == SLEEP_PROFILING)) {
-				profile_hits(SLEEP_PROFILING,
-						(void *)get_wchan(tsk),
-						delta >> 20);
-			}
-			account_scheduler_latency(tsk, delta >> 10, 0);
-		}
-	}
-}
-
 /*
  * Task is being enqueued - update stats:
  */
@@ -1000,10 +882,10 @@ update_stats_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	 * a dequeue/enqueue event is a NOP)
 	 */
 	if (se != cfs_rq->curr)
-		update_stats_wait_start(cfs_rq, se);
+		update_stats_wait_start(rq_of(cfs_rq), se);
 
 	if (flags & ENQUEUE_WAKEUP)
-		update_stats_enqueue_sleeper(cfs_rq, se);
+		update_stats_enqueue_sleeper(rq_of(cfs_rq), se);
 }
 
 static inline void
@@ -1018,7 +900,7 @@ update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	 * waiting task:
 	 */
 	if (se != cfs_rq->curr)
-		update_stats_wait_end(cfs_rq, se);
+		update_stats_wait_end(rq_of(cfs_rq), se);
 
 	if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) {
 		struct task_struct *tsk = task_of(se);
@@ -4127,26 +4009,6 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
 
 static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
 
-static inline void check_schedstat_required(void)
-{
-#ifdef CONFIG_SCHEDSTATS
-	if (schedstat_enabled())
-		return;
-
-	/* Force schedstat enabled if a dependent tracepoint is active */
-	if (trace_sched_stat_wait_enabled()    ||
-			trace_sched_stat_sleep_enabled()   ||
-			trace_sched_stat_iowait_enabled()  ||
-			trace_sched_stat_blocked_enabled() ||
-			trace_sched_stat_runtime_enabled())  {
-		printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, "
-			     "stat_blocked and stat_runtime require the "
-			     "kernel parameter schedstats=enable or "
-			     "kernel.sched_schedstats=1\n");
-	}
-#endif
-}
-
 static inline bool cfs_bandwidth_used(void);
 
 /*
@@ -4387,7 +4249,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 		 * a CPU. So account for the time it spent waiting on the
 		 * runqueue.
 		 */
-		update_stats_wait_end(cfs_rq, se);
+		update_stats_wait_end(rq_of(cfs_rq), se);
 		__dequeue_entity(cfs_rq, se);
 		update_load_avg(cfs_rq, se, UPDATE_TG);
 	}
@@ -4488,7 +4350,7 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
 	check_spread(cfs_rq, prev);
 
 	if (prev->on_rq) {
-		update_stats_wait_start(cfs_rq, prev);
+		update_stats_wait_start(rq_of(cfs_rq), prev);
 		/* Put 'current' back into the tree. */
 		__enqueue_entity(cfs_rq, prev);
 		/* in !on_rq case, update occurred at dequeue */
diff --git a/kernel/sched/stats.c b/kernel/sched/stats.c
index 750fb3c67eed..00ef7676ea36 100644
--- a/kernel/sched/stats.c
+++ b/kernel/sched/stats.c
@@ -4,6 +4,140 @@
  */
 #include "sched.h"
 
+void update_stats_wait_start(struct rq *rq, struct sched_entity *se)
+{
+	u64 wait_start, prev_wait_start;
+
+	if (!schedstat_enabled())
+		return;
+
+	wait_start = rq_clock(rq);
+	prev_wait_start = schedstat_val(se->statistics.wait_start);
+
+	if (entity_is_task(se) && task_on_rq_migrating(task_of(se)) &&
+	    likely(wait_start > prev_wait_start))
+		wait_start -= prev_wait_start;
+
+	__schedstat_set(se->statistics.wait_start, wait_start);
+}
+
+void update_stats_wait_end(struct rq *rq, struct sched_entity *se)
+{
+	struct task_struct *p;
+	u64 delta;
+
+	if (!schedstat_enabled())
+		return;
+
+	delta = rq_clock(rq) - schedstat_val(se->statistics.wait_start);
+
+	if (entity_is_task(se)) {
+		p = task_of(se);
+		if (task_on_rq_migrating(p)) {
+			/*
+			 * Preserve migrating task's wait time so wait_start
+			 * time stamp can be adjusted to accumulate wait time
+			 * prior to migration.
+			 */
+			__schedstat_set(se->statistics.wait_start, delta);
+			return;
+		}
+		trace_sched_stat_wait(p, delta);
+	}
+
+	__schedstat_set(se->statistics.wait_max,
+		      max(schedstat_val(se->statistics.wait_max), delta));
+	__schedstat_inc(se->statistics.wait_count);
+	__schedstat_add(se->statistics.wait_sum, delta);
+	__schedstat_set(se->statistics.wait_start, 0);
+}
+
+void update_stats_enqueue_sleeper(struct rq *rq, struct sched_entity *se)
+{
+	struct task_struct *tsk = NULL;
+	u64 sleep_start, block_start;
+
+	if (!schedstat_enabled())
+		return;
+
+	sleep_start = schedstat_val(se->statistics.sleep_start);
+	block_start = schedstat_val(se->statistics.block_start);
+
+	if (entity_is_task(se))
+		tsk = task_of(se);
+
+	if (sleep_start) {
+		u64 delta = rq_clock(rq) - sleep_start;
+
+		if ((s64)delta < 0)
+			delta = 0;
+
+		if (unlikely(delta > schedstat_val(se->statistics.sleep_max)))
+			__schedstat_set(se->statistics.sleep_max, delta);
+
+		__schedstat_set(se->statistics.sleep_start, 0);
+		__schedstat_add(se->statistics.sum_sleep_runtime, delta);
+
+		if (tsk) {
+			account_scheduler_latency(tsk, delta >> 10, 1);
+			trace_sched_stat_sleep(tsk, delta);
+		}
+	}
+
+	if (block_start) {
+		u64 delta = rq_clock(rq) - block_start;
+
+		if ((s64)delta < 0)
+			delta = 0;
+
+		if (unlikely(delta > schedstat_val(se->statistics.block_max)))
+			__schedstat_set(se->statistics.block_max, delta);
+
+		__schedstat_set(se->statistics.block_start, 0);
+		__schedstat_add(se->statistics.sum_sleep_runtime, delta);
+
+		if (tsk) {
+			if (tsk->in_iowait) {
+				__schedstat_add(se->statistics.iowait_sum, delta);
+				__schedstat_inc(se->statistics.iowait_count);
+				trace_sched_stat_iowait(tsk, delta);
+			}
+
+			trace_sched_stat_blocked(tsk, delta);
+
+			/*
+			 * Blocking time is in units of nanosecs, so shift by
+			 * 20 to get a milliseconds-range estimation of the
+			 * amount of time that the task spent sleeping:
+			 */
+			if (unlikely(prof_on == SLEEP_PROFILING)) {
+				profile_hits(SLEEP_PROFILING,
+						(void *)get_wchan(tsk),
+						delta >> 20);
+			}
+			account_scheduler_latency(tsk, delta >> 10, 0);
+		}
+	}
+}
+
+void check_schedstat_required(void)
+{
+	if (schedstat_enabled())
+		return;
+
+	/* Force schedstat enabled if a dependent tracepoint is active */
+	if (trace_sched_stat_wait_enabled()    ||
+			trace_sched_stat_sleep_enabled()   ||
+			trace_sched_stat_iowait_enabled()  ||
+			trace_sched_stat_blocked_enabled() ||
+			trace_sched_stat_runtime_enabled())  {
+		printk_deferred_once("Scheduler tracepoints stat_sleep, stat_iowait, "
+			     "stat_blocked and stat_runtime require the "
+			     "kernel parameter schedstats=enable or "
+			     "kernel.sched_schedstats=1\n");
+	}
+}
+
 /*
  * Current schedstat API version.
  *
diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h
index 33d0daf83842..b46612b83896 100644
--- a/kernel/sched/stats.h
+++ b/kernel/sched/stats.h
@@ -40,6 +40,11 @@ rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)
 #define   schedstat_val(var)		(var)
 #define   schedstat_val_or_zero(var)	((schedstat_enabled()) ? (var) : 0)
 
+void update_stats_wait_start(struct rq *rq, struct sched_entity *se);
+void update_stats_wait_end(struct rq *rq, struct sched_entity *se);
+void update_stats_enqueue_sleeper(struct rq *rq, struct sched_entity *se);
+void check_schedstat_required(void);
+
 #else /* !CONFIG_SCHEDSTATS: */
 static inline void rq_sched_info_arrive  (struct rq *rq, unsigned long long delta) { }
 static inline void rq_sched_info_dequeued(struct rq *rq, unsigned long long delta) { }
@@ -53,6 +58,12 @@ static inline void rq_sched_info_depart  (struct rq *rq, unsigned long long delt
 # define   schedstat_set(var, val)	do { } while (0)
 # define   schedstat_val(var)		0
 # define   schedstat_val_or_zero(var)	0
+
+# define update_stats_wait_start(rq, se)	do { } while (0)
+# define update_stats_wait_end(rq, se)		do { } while (0)
+# define update_stats_enqueue_sleeper(rq, se)	do { } while (0)
+# define check_schedstat_required()		do { } while (0)
+
 #endif /* CONFIG_SCHEDSTATS */
 
 #ifdef CONFIG_PSI
-- 
2.18.4

[RFC PATCH 3/4] sched: define update_stats_curr_start() as a common helper

[Yafang Shao]

update_stats_curr_start() is used to update the exec_start when we are
starting a new run period, which is used by all sched class. So we'd
better define it as a common helper.

Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
---
 kernel/sched/fair.c  | 14 +-------------
 kernel/sched/rt.c    |  2 +-
 kernel/sched/sched.h | 12 ++++++++++++
 3 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index aba21191283d..b762cc3e165c 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -914,18 +914,6 @@ update_stats_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	}
 }
 
-/*
- * We are picking a new current task - update its stats:
- */
-static inline void
-update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
-{
-	/*
-	 * We are starting a new run period:
-	 */
-	se->exec_start = rq_clock_task(rq_of(cfs_rq));
-}
-
 /**************************************************
  * Scheduling class queueing methods:
  */
@@ -4254,7 +4242,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 		update_load_avg(cfs_rq, se, UPDATE_TG);
 	}
 
-	update_stats_curr_start(cfs_rq, se);
+	update_stats_curr_start(rq_of(cfs_rq), se);
 	cfs_rq->curr = se;
 
 	/*
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 49ec096a8aa1..b9ec886702a1 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1574,7 +1574,7 @@ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flag
 
 static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
 {
-	p->se.exec_start = rq_clock_task(rq);
+	update_stats_curr_start(rq, &p->se);
 
 	/* The running task is never eligible for pushing */
 	dequeue_pushable_task(rq, p);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index e29f2375c4f5..28986736ced9 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2642,3 +2642,15 @@ static inline struct task_struct *task_of(struct sched_entity *se)
 	return container_of(se, struct task_struct, se);
 }
 
+/*
+ * We are picking a new current task - update its stats:
+ */
+static inline void
+update_stats_curr_start(struct rq *rq, struct sched_entity *se)
+{
+	/*
+	 * We are starting a new run period:
+	 */
+	se->exec_start = rq_clock_task(rq);
+}
+
-- 
2.18.4

[RFC PATCH 4/4] sched, rt: support schedstat for RT sched class

[Yafang Shao]

We want to measure the latency of RT tasks in our production
environment with schedstat facility, but currently schedstat is only
supported for fair sched class. This patch enable it for RT sched class
as well.

The schedstat statistics are define in struct sched_entity, which is a
member of struct task_struct, so we can resue it for RT sched class.

The schedstat usage in RT sched class is similar with fair sched class,
for example,
		fair				RT
enqueue		update_stats_enqueue_fair	update_stats_enqueue_rt
dequeue		update_stats_dequeue_fair	update_stats_dequeue_rt
put_prev_task	update_stats_wait_start		update_stats_wait_start
set_next_task	update_stats_wait_end		update_stats_wait_end
show		/proc/[pid]/sched		/proc/[pid]/sched

The sched:sched_stats_* tracepoints can be used to trace RT tasks as
well.

Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
---
 kernel/sched/rt.c    | 61 ++++++++++++++++++++++++++++++++++++++++++++
 kernel/sched/sched.h |  2 ++
 2 files changed, 63 insertions(+)

diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index b9ec886702a1..a318236b7166 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1246,6 +1246,46 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
 	dec_rt_group(rt_se, rt_rq);
 }
 
+static inline void
+update_stats_enqueue_rt(struct rq *rq, struct sched_entity *se,
+			struct sched_rt_entity *rt_se, int flags)
+{
+	struct rt_rq *rt_rq = &rq->rt;
+
+	if (!schedstat_enabled())
+		return;
+
+	if (rt_se != rt_rq->curr)
+		update_stats_wait_start(rq, se);
+
+	if (flags & ENQUEUE_WAKEUP)
+		update_stats_enqueue_sleeper(rq, se);
+}
+
+static inline void
+update_stats_dequeue_rt(struct rq *rq, struct sched_entity *se,
+			struct sched_rt_entity *rt_se, int flags)
+{
+	struct rt_rq *rt_rq = &rq->rt;
+
+	if (!schedstat_enabled())
+		return;
+
+	if (rt_se != rt_rq->curr)
+		update_stats_wait_end(rq, se);
+
+	if ((flags & DEQUEUE_SLEEP) && rt_entity_is_task(rt_se)) {
+		struct task_struct *tsk = rt_task_of(rt_se);
+
+		if (tsk->state & TASK_INTERRUPTIBLE)
+			__schedstat_set(se->statistics.sleep_start,
+					rq_clock(rq));
+		if (tsk->state & TASK_UNINTERRUPTIBLE)
+			__schedstat_set(se->statistics.block_start,
+					rq_clock(rq));
+	}
+}
+
 /*
  * Change rt_se->run_list location unless SAVE && !MOVE
  *
@@ -1275,6 +1315,7 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
 	struct rt_prio_array *array = &rt_rq->active;
 	struct rt_rq *group_rq = group_rt_rq(rt_se);
 	struct list_head *queue = array->queue + rt_se_prio(rt_se);
+	struct task_struct *task = rt_task_of(rt_se);
 
 	/*
 	 * Don't enqueue the group if its throttled, or when empty.
@@ -1288,6 +1329,8 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
 		return;
 	}
 
+	update_stats_enqueue_rt(rq_of_rt_rq(rt_rq), &task->se, rt_se, flags);
+
 	if (move_entity(flags)) {
 		WARN_ON_ONCE(rt_se->on_list);
 		if (flags & ENQUEUE_HEAD)
@@ -1307,7 +1350,9 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
 {
 	struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
 	struct rt_prio_array *array = &rt_rq->active;
+	struct task_struct *task = rt_task_of(rt_se);
 
+	update_stats_dequeue_rt(rq_of_rt_rq(rt_rq), &task->se, rt_se, flags);
 	if (move_entity(flags)) {
 		WARN_ON_ONCE(!rt_se->on_list);
 		__delist_rt_entity(rt_se, array);
@@ -1374,6 +1419,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
 	if (flags & ENQUEUE_WAKEUP)
 		rt_se->timeout = 0;
 
+	check_schedstat_required();
 	enqueue_rt_entity(rt_se, flags);
 
 	if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
@@ -1574,6 +1620,12 @@ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flag
 
 static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
 {
+	struct sched_rt_entity *rt_se = &p->rt;
+	struct rt_rq *rt_rq = &rq->rt;
+
+	if (on_rt_rq(&p->rt))
+		update_stats_wait_end(rq, &p->se);
+
 	update_stats_curr_start(rq, &p->se);
 
 	/* The running task is never eligible for pushing */
@@ -1591,6 +1643,8 @@ static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool f
 		update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
 
 	rt_queue_push_tasks(rq);
+
+	rt_rq->curr = rt_se;
 }
 
 static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
@@ -1638,6 +1692,11 @@ static struct task_struct *pick_next_task_rt(struct rq *rq)
 
 static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
 {
+	struct rt_rq *rt_rq = &rq->rt;
+
+	if (on_rt_rq(&p->rt))
+		update_stats_wait_start(rq, &p->se);
+
 	update_curr_rt(rq);
 
 	update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
@@ -1648,6 +1707,8 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
 	 */
 	if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
 		enqueue_pushable_task(rq, p);
+
+	rt_rq->curr = NULL;
 }
 
 #ifdef CONFIG_SMP
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 28986736ced9..7787afbd5723 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -649,6 +649,8 @@ struct rt_rq {
 	struct rq		*rq;
 	struct task_group	*tg;
 #endif
+
+	struct sched_rt_entity  *curr;
 };
 
 static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq)
-- 
2.18.4

Re: [RFC PATCH 2/4] sched: make schedstats helpers not depend on cfs_rq

[Mel Gorman]

On Thu, Nov 19, 2020 at 11:52:28AM +0800, Yafang Shao wrote:
> The 'cfs_rq' in these helpers is only used to get the rq_clock, so we
> can pass the rq_clock directly. After that, these helpers can be used by
> all sched class.
> 
> Signed-off-by: Yafang Shao <laoar.shao@gmail.com>

This introduces overhead in the general case even when schedstats is
disabled. Previously, update_stats_wait_start was a static inline so
function call overhead was avoided and schedstat_enabled() meant the
overhead was negligible. As it's now a function call, the cost of the
function entry/exit will be unconditionally hit regardless of intrest
in schedstat.

Regardless of the merit of adding schedstats for RT, the overhead of
schedstats when stats are disabled should remain the same with the
static branch check done in an inline function.

-- 
Mel Gorman
SUSE Labs

Re: [RFC PATCH 2/4] sched: make schedstats helpers not depend on cfs_rq

[Yafang Shao]

On Thu, Nov 19, 2020 at 3:45 PM Mel Gorman <mgorman@suse.de> wrote:
>
> On Thu, Nov 19, 2020 at 11:52:28AM +0800, Yafang Shao wrote:
> > The 'cfs_rq' in these helpers is only used to get the rq_clock, so we
> > can pass the rq_clock directly. After that, these helpers can be used by
> > all sched class.
> >
> > Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
>
> This introduces overhead in the general case even when schedstats is
> disabled. Previously, update_stats_wait_start was a static inline so
> function call overhead was avoided and schedstat_enabled() meant the
> overhead was negligible. As it's now a function call, the cost of the
> function entry/exit will be unconditionally hit regardless of intrest
> in schedstat.
>
> Regardless of the merit of adding schedstats for RT, the overhead of
> schedstats when stats are disabled should remain the same with the
> static branch check done in an inline function.
>

Thanks for the explanation.
I will make them inline in the next version.


-- 
Thanks
Yafang

Re: [RFC PATCH 4/4] sched, rt: support schedstat for RT sched class

[jun qian]

Yafang Shao <laoar.shao@gmail.com> 于2020年11月19日周四 上午11:55写道：
>
> We want to measure the latency of RT tasks in our production
> environment with schedstat facility, but currently schedstat is only
> supported for fair sched class. This patch enable it for RT sched class
> as well.
>
> The schedstat statistics are define in struct sched_entity, which is a
> member of struct task_struct, so we can resue it for RT sched class.
>
> The schedstat usage in RT sched class is similar with fair sched class,
> for example,
>                 fair                            RT
> enqueue         update_stats_enqueue_fair       update_stats_enqueue_rt
> dequeue         update_stats_dequeue_fair       update_stats_dequeue_rt
> put_prev_task   update_stats_wait_start         update_stats_wait_start
> set_next_task   update_stats_wait_end           update_stats_wait_end
> show            /proc/[pid]/sched               /proc/[pid]/sched
>
> The sched:sched_stats_* tracepoints can be used to trace RT tasks as
> well.
>
> Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
> ---
>  kernel/sched/rt.c    | 61 ++++++++++++++++++++++++++++++++++++++++++++
>  kernel/sched/sched.h |  2 ++
>  2 files changed, 63 insertions(+)
>
> diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
> index b9ec886702a1..a318236b7166 100644
> --- a/kernel/sched/rt.c
> +++ b/kernel/sched/rt.c
> @@ -1246,6 +1246,46 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
>         dec_rt_group(rt_se, rt_rq);
>  }
>

Does the deadline schedule class should be considered also?

thanks

> +static inline void
> +update_stats_enqueue_rt(struct rq *rq, struct sched_entity *se,
> +                       struct sched_rt_entity *rt_se, int flags)
> +{
> +       struct rt_rq *rt_rq = &rq->rt;
> +
> +       if (!schedstat_enabled())
> +               return;
> +
> +       if (rt_se != rt_rq->curr)
> +               update_stats_wait_start(rq, se);
> +
> +       if (flags & ENQUEUE_WAKEUP)
> +               update_stats_enqueue_sleeper(rq, se);
> +}
> +
> +static inline void
> +update_stats_dequeue_rt(struct rq *rq, struct sched_entity *se,
> +                       struct sched_rt_entity *rt_se, int flags)
> +{
> +       struct rt_rq *rt_rq = &rq->rt;
> +
> +       if (!schedstat_enabled())
> +               return;
> +
> +       if (rt_se != rt_rq->curr)
> +               update_stats_wait_end(rq, se);
> +
> +       if ((flags & DEQUEUE_SLEEP) && rt_entity_is_task(rt_se)) {
> +               struct task_struct *tsk = rt_task_of(rt_se);
> +
> +               if (tsk->state & TASK_INTERRUPTIBLE)
> +                       __schedstat_set(se->statistics.sleep_start,
> +                                       rq_clock(rq));
> +               if (tsk->state & TASK_UNINTERRUPTIBLE)
> +                       __schedstat_set(se->statistics.block_start,
> +                                       rq_clock(rq));
> +       }
> +}
> +
>  /*
>   * Change rt_se->run_list location unless SAVE && !MOVE
>   *
> @@ -1275,6 +1315,7 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
>         struct rt_prio_array *array = &rt_rq->active;
>         struct rt_rq *group_rq = group_rt_rq(rt_se);
>         struct list_head *queue = array->queue + rt_se_prio(rt_se);
> +       struct task_struct *task = rt_task_of(rt_se);
>
>         /*
>          * Don't enqueue the group if its throttled, or when empty.
> @@ -1288,6 +1329,8 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
>                 return;
>         }
>
> +       update_stats_enqueue_rt(rq_of_rt_rq(rt_rq), &task->se, rt_se, flags);
> +
>         if (move_entity(flags)) {
>                 WARN_ON_ONCE(rt_se->on_list);
>                 if (flags & ENQUEUE_HEAD)
> @@ -1307,7 +1350,9 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
>  {
>         struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
>         struct rt_prio_array *array = &rt_rq->active;
> +       struct task_struct *task = rt_task_of(rt_se);
>
> +       update_stats_dequeue_rt(rq_of_rt_rq(rt_rq), &task->se, rt_se, flags);
>         if (move_entity(flags)) {
>                 WARN_ON_ONCE(!rt_se->on_list);
>                 __delist_rt_entity(rt_se, array);
> @@ -1374,6 +1419,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
>         if (flags & ENQUEUE_WAKEUP)
>                 rt_se->timeout = 0;
>
> +       check_schedstat_required();
>         enqueue_rt_entity(rt_se, flags);
>
>         if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
> @@ -1574,6 +1620,12 @@ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flag
>
>  static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
>  {
> +       struct sched_rt_entity *rt_se = &p->rt;
> +       struct rt_rq *rt_rq = &rq->rt;
> +
> +       if (on_rt_rq(&p->rt))
> +               update_stats_wait_end(rq, &p->se);
> +
>         update_stats_curr_start(rq, &p->se);
>
>         /* The running task is never eligible for pushing */
> @@ -1591,6 +1643,8 @@ static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool f
>                 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
>
>         rt_queue_push_tasks(rq);
> +
> +       rt_rq->curr = rt_se;
>  }
>
>  static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
> @@ -1638,6 +1692,11 @@ static struct task_struct *pick_next_task_rt(struct rq *rq)
>
>  static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
>  {
> +       struct rt_rq *rt_rq = &rq->rt;
> +
> +       if (on_rt_rq(&p->rt))
> +               update_stats_wait_start(rq, &p->se);
> +
>         update_curr_rt(rq);
>
>         update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
> @@ -1648,6 +1707,8 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
>          */
>         if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
>                 enqueue_pushable_task(rq, p);
> +
> +       rt_rq->curr = NULL;
>  }
>
>  #ifdef CONFIG_SMP
> diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
> index 28986736ced9..7787afbd5723 100644
> --- a/kernel/sched/sched.h
> +++ b/kernel/sched/sched.h
> @@ -649,6 +649,8 @@ struct rt_rq {
>         struct rq               *rq;
>         struct task_group       *tg;
>  #endif
> +
> +       struct sched_rt_entity  *curr;
>  };
>
>  static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq)
> --
> 2.18.4
>

Re: [RFC PATCH 4/4] sched, rt: support schedstat for RT sched class

[Yafang Shao]

On Fri, Nov 20, 2020 at 10:39 AM jun qian <qianjun.kernel@gmail.com> wrote:
>
> Yafang Shao <laoar.shao@gmail.com> 于2020年11月19日周四 上午11:55写道：
> >
> > We want to measure the latency of RT tasks in our production
> > environment with schedstat facility, but currently schedstat is only
> > supported for fair sched class. This patch enable it for RT sched class
> > as well.
> >
> > The schedstat statistics are define in struct sched_entity, which is a
> > member of struct task_struct, so we can resue it for RT sched class.
> >
> > The schedstat usage in RT sched class is similar with fair sched class,
> > for example,
> >                 fair                            RT
> > enqueue         update_stats_enqueue_fair       update_stats_enqueue_rt
> > dequeue         update_stats_dequeue_fair       update_stats_dequeue_rt
> > put_prev_task   update_stats_wait_start         update_stats_wait_start
> > set_next_task   update_stats_wait_end           update_stats_wait_end
> > show            /proc/[pid]/sched               /proc/[pid]/sched
> >
> > The sched:sched_stats_* tracepoints can be used to trace RT tasks as
> > well.
> >
> > Signed-off-by: Yafang Shao <laoar.shao@gmail.com>
> > ---
> >  kernel/sched/rt.c    | 61 ++++++++++++++++++++++++++++++++++++++++++++
> >  kernel/sched/sched.h |  2 ++
> >  2 files changed, 63 insertions(+)
> >
> > diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
> > index b9ec886702a1..a318236b7166 100644
> > --- a/kernel/sched/rt.c
> > +++ b/kernel/sched/rt.c
> > @@ -1246,6 +1246,46 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
> >         dec_rt_group(rt_se, rt_rq);
> >  }
> >
>
> Does the deadline schedule class should be considered also?
>

deadline sched class can be supported as well per my understanding, I
think we can do it later.
This patchset only aims to support RT sched class.

> thanks
>
> > +static inline void
> > +update_stats_enqueue_rt(struct rq *rq, struct sched_entity *se,
> > +                       struct sched_rt_entity *rt_se, int flags)
> > +{
> > +       struct rt_rq *rt_rq = &rq->rt;
> > +
> > +       if (!schedstat_enabled())
> > +               return;
> > +
> > +       if (rt_se != rt_rq->curr)
> > +               update_stats_wait_start(rq, se);
> > +
> > +       if (flags & ENQUEUE_WAKEUP)
> > +               update_stats_enqueue_sleeper(rq, se);
> > +}
> > +
> > +static inline void
> > +update_stats_dequeue_rt(struct rq *rq, struct sched_entity *se,
> > +                       struct sched_rt_entity *rt_se, int flags)
> > +{
> > +       struct rt_rq *rt_rq = &rq->rt;
> > +
> > +       if (!schedstat_enabled())
> > +               return;
> > +
> > +       if (rt_se != rt_rq->curr)
> > +               update_stats_wait_end(rq, se);
> > +
> > +       if ((flags & DEQUEUE_SLEEP) && rt_entity_is_task(rt_se)) {
> > +               struct task_struct *tsk = rt_task_of(rt_se);
> > +
> > +               if (tsk->state & TASK_INTERRUPTIBLE)
> > +                       __schedstat_set(se->statistics.sleep_start,
> > +                                       rq_clock(rq));
> > +               if (tsk->state & TASK_UNINTERRUPTIBLE)
> > +                       __schedstat_set(se->statistics.block_start,
> > +                                       rq_clock(rq));
> > +       }
> > +}
> > +
> >  /*
> >   * Change rt_se->run_list location unless SAVE && !MOVE
> >   *
> > @@ -1275,6 +1315,7 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
> >         struct rt_prio_array *array = &rt_rq->active;
> >         struct rt_rq *group_rq = group_rt_rq(rt_se);
> >         struct list_head *queue = array->queue + rt_se_prio(rt_se);
> > +       struct task_struct *task = rt_task_of(rt_se);
> >
> >         /*
> >          * Don't enqueue the group if its throttled, or when empty.
> > @@ -1288,6 +1329,8 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
> >                 return;
> >         }
> >
> > +       update_stats_enqueue_rt(rq_of_rt_rq(rt_rq), &task->se, rt_se, flags);
> > +
> >         if (move_entity(flags)) {
> >                 WARN_ON_ONCE(rt_se->on_list);
> >                 if (flags & ENQUEUE_HEAD)
> > @@ -1307,7 +1350,9 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
> >  {
> >         struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
> >         struct rt_prio_array *array = &rt_rq->active;
> > +       struct task_struct *task = rt_task_of(rt_se);
> >
> > +       update_stats_dequeue_rt(rq_of_rt_rq(rt_rq), &task->se, rt_se, flags);
> >         if (move_entity(flags)) {
> >                 WARN_ON_ONCE(!rt_se->on_list);
> >                 __delist_rt_entity(rt_se, array);
> > @@ -1374,6 +1419,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
> >         if (flags & ENQUEUE_WAKEUP)
> >                 rt_se->timeout = 0;
> >
> > +       check_schedstat_required();
> >         enqueue_rt_entity(rt_se, flags);
> >
> >         if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
> > @@ -1574,6 +1620,12 @@ static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p, int flag
> >
> >  static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool first)
> >  {
> > +       struct sched_rt_entity *rt_se = &p->rt;
> > +       struct rt_rq *rt_rq = &rq->rt;
> > +
> > +       if (on_rt_rq(&p->rt))
> > +               update_stats_wait_end(rq, &p->se);
> > +
> >         update_stats_curr_start(rq, &p->se);
> >
> >         /* The running task is never eligible for pushing */
> > @@ -1591,6 +1643,8 @@ static inline void set_next_task_rt(struct rq *rq, struct task_struct *p, bool f
> >                 update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 0);
> >
> >         rt_queue_push_tasks(rq);
> > +
> > +       rt_rq->curr = rt_se;
> >  }
> >
> >  static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
> > @@ -1638,6 +1692,11 @@ static struct task_struct *pick_next_task_rt(struct rq *rq)
> >
> >  static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
> >  {
> > +       struct rt_rq *rt_rq = &rq->rt;
> > +
> > +       if (on_rt_rq(&p->rt))
> > +               update_stats_wait_start(rq, &p->se);
> > +
> >         update_curr_rt(rq);
> >
> >         update_rt_rq_load_avg(rq_clock_pelt(rq), rq, 1);
> > @@ -1648,6 +1707,8 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
> >          */
> >         if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
> >                 enqueue_pushable_task(rq, p);
> > +
> > +       rt_rq->curr = NULL;
> >  }
> >
> >  #ifdef CONFIG_SMP
> > diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
> > index 28986736ced9..7787afbd5723 100644
> > --- a/kernel/sched/sched.h
> > +++ b/kernel/sched/sched.h
> > @@ -649,6 +649,8 @@ struct rt_rq {
> >         struct rq               *rq;
> >         struct task_group       *tg;
> >  #endif
> > +
> > +       struct sched_rt_entity  *curr;
> >  };
> >
> >  static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq)
> > --
> > 2.18.4
> >



-- 
Thanks
Yafang