From: Song Liu <songliubraving@fb.com>
To: <linux-kernel@vger.kernel.org>, <cgroups@vger.kernel.org>
Cc: <mingo@redhat.com>, <peterz@infradead.org>,
<vincent.guittot@linaro.org>, <tglx@linutronix.de>,
<morten.rasmussen@arm.com>, <kernel-team@fb.com>,
Song Liu <songliubraving@fb.com>
Subject: [PATCH 5/7] sched/fair: global idleness counter for cpu.headroom
Date: Mon, 8 Apr 2019 14:45:37 -0700 [thread overview]
Message-ID: <20190408214539.2705660-6-songliubraving@fb.com> (raw)
In-Reply-To: <20190408214539.2705660-1-songliubraving@fb.com>
This patch introduces a global idleness counter in fair.c for the
cpu.headroom knob. This counter is based on per cpu get_idle_time().
The counter is used via function call:
unsigned long cfs_global_idleness_update(u64 now, u64 period);
The function returns global idleness in fixed-point percentage since
previous call of the function. If the time between previous call of the
function is called and @now is shorter than @period, the function will
return idleness calculated in previous call.
cfs_global_idleness_update() will be called from a non-preemptible
context, struct cfs_global_idleness uses raw_spin_lock instead of
spin_lock.
Signed-off-by: Song Liu <songliubraving@fb.com>
---
fs/proc/stat.c | 4 +--
include/linux/kernel_stat.h | 2 ++
kernel/sched/fair.c | 64 +++++++++++++++++++++++++++++++++++++
3 files changed, 68 insertions(+), 2 deletions(-)
diff --git a/fs/proc/stat.c b/fs/proc/stat.c
index 80c305f206bb..b327ffdb169f 100644
--- a/fs/proc/stat.c
+++ b/fs/proc/stat.c
@@ -23,7 +23,7 @@
#ifdef arch_idle_time
-static u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
+u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
{
u64 idle;
@@ -45,7 +45,7 @@ static u64 get_iowait_time(struct kernel_cpustat *kcs, int cpu)
#else
-static u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
+u64 get_idle_time(struct kernel_cpustat *kcs, int cpu)
{
u64 idle, idle_usecs = -1ULL;
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index 7ee2bb43b251..337135272391 100644
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -97,4 +97,6 @@ extern void account_process_tick(struct task_struct *, int user);
extern void account_idle_ticks(unsigned long ticks);
+u64 get_idle_time(struct kernel_cpustat *kcs, int cpu);
+
#endif /* _LINUX_KERNEL_STAT_H */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 65aa9d3b665f..49c68daffe7e 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -116,6 +116,62 @@ static unsigned int capacity_margin = 1280;
* (default: 5 msec, units: microseconds)
*/
unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
+
+/* tracking global idlenesss for cpu.headroom */
+struct cfs_global_idleness {
+ u64 prev_total_idle_time;
+ u64 prev_timestamp;
+ unsigned long idle_percent; /* fixed-point */
+ raw_spinlock_t lock;
+};
+
+static struct cfs_global_idleness global_idleness;
+
+/*
+ * Calculate global idleness in fixed-point percentage since previous call
+ * of the function. If the time between previous call of the function is
+ * called and @now is shorter than @period, return idleness calculated in
+ * previous call.
+ */
+static unsigned long cfs_global_idleness_update(u64 now, u64 period)
+{
+ u64 prev_timestamp, total_idle_time, delta_idle_time;
+ unsigned long idle_percent;
+ int cpu;
+
+ /*
+ * Fastpath: if idleness has been updated within the last period
+ * of time, just return previous idleness.
+ */
+ prev_timestamp = READ_ONCE(global_idleness.prev_timestamp);
+ if (prev_timestamp + period >= now)
+ return READ_ONCE(global_idleness.idle_percent);
+
+ raw_spin_lock_irq(&global_idleness.lock);
+ if (global_idleness.prev_timestamp + period >= now) {
+ idle_percent = global_idleness.idle_percent;
+ goto out;
+ }
+
+ /* Slowpath: calculate the average idleness since prev_timestamp */
+ total_idle_time = 0;
+ for_each_online_cpu(cpu)
+ total_idle_time += get_idle_time(&kcpustat_cpu(cpu), cpu);
+
+ delta_idle_time = total_idle_time -
+ global_idleness.prev_total_idle_time;
+
+ idle_percent = div64_u64((delta_idle_time << FSHIFT) * 100,
+ num_online_cpus() *
+ (now - global_idleness.prev_timestamp));
+
+ WRITE_ONCE(global_idleness.prev_total_idle_time, total_idle_time);
+ WRITE_ONCE(global_idleness.prev_timestamp, now);
+ WRITE_ONCE(global_idleness.idle_percent, idle_percent);
+out:
+ raw_spin_unlock_irq(&global_idleness.lock);
+ return idle_percent;
+}
#endif
static inline void update_load_add(struct load_weight *lw, unsigned long inc)
@@ -4293,6 +4349,11 @@ void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
cfs_b->runtime = cfs_b->quota;
cfs_b->runtime_expires = now + ktime_to_ns(cfs_b->period);
cfs_b->expires_seq++;
+
+ if (cfs_b->target_idle == 0)
+ return;
+
+ cfs_global_idleness_update(now, cfs_b->period);
}
static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
@@ -10676,4 +10737,7 @@ __init void init_sched_fair_class(void)
#endif
#endif /* SMP */
+#ifdef CONFIG_CFS_BANDWIDTH
+ raw_spin_lock_init(&global_idleness.lock);
+#endif
}
--
2.17.1
next prev parent reply other threads:[~2019-04-08 21:46 UTC|newest]
Thread overview: 26+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-04-08 21:45 [PATCH 0/7] introduce cpu.headroom knob to cpu controller Song Liu
2019-04-08 21:45 ` [PATCH 1/7] sched: refactor tg_set_cfs_bandwidth() Song Liu
2019-04-08 21:45 ` [PATCH 2/7] cgroup: introduce hook css_has_tasks_changed Song Liu
2019-04-08 21:45 ` [PATCH 3/7] cgroup: introduce cgroup_parse_percentage Song Liu
2019-04-08 21:45 ` [PATCH 4/7] sched, cgroup: add entry cpu.headroom Song Liu
2019-04-08 21:45 ` Song Liu [this message]
2019-04-08 21:45 ` [PATCH 6/7] sched/fair: throttle task runtime based on cpu.headroom Song Liu
2019-04-08 21:45 ` [PATCH 7/7] Documentation: cgroup-v2: add information for cpu.headroom Song Liu
2019-04-10 11:59 ` [PATCH 0/7] introduce cpu.headroom knob to cpu controller Morten Rasmussen
2019-04-10 19:43 ` Song Liu
2019-04-17 12:56 ` Vincent Guittot
2019-04-22 23:22 ` Song Liu
2019-04-28 19:47 ` Song Liu
2019-04-29 12:24 ` Vincent Guittot
2019-04-30 6:10 ` Song Liu
2019-04-30 16:20 ` Vincent Guittot
2019-04-30 16:54 ` Song Liu
2019-05-10 18:22 ` Song Liu
2019-05-14 20:58 ` Song Liu
2019-05-15 10:18 ` Vincent Guittot
2019-05-15 15:42 ` Song Liu
2019-05-21 13:47 ` Michal Koutný
2019-05-21 16:27 ` Song Liu
2019-06-26 8:26 ` Michal Koutný
2019-06-26 15:56 ` Song Liu
2019-04-15 16:48 ` Song Liu
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190408214539.2705660-6-songliubraving@fb.com \
--to=songliubraving@fb.com \
--cc=cgroups@vger.kernel.org \
--cc=kernel-team@fb.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=morten.rasmussen@arm.com \
--cc=peterz@infradead.org \
--cc=tglx@linutronix.de \
--cc=vincent.guittot@linaro.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).