From: Huacai Chen <chenhuacai@loongson.cn>
To: "Paul E . McKenney" <paulmck@kernel.org>,
Frederic Weisbecker <frederic@kernel.org>,
Neeraj Upadhyay <quic_neeraju@quicinc.com>,
Joel Fernandes <joel@joelfernandes.org>,
Josh Triplett <josh@joshtriplett.org>,
Boqun Feng <boqun.feng@gmail.com>,
Thomas Gleixner <tglx@linutronix.de>,
Ingo Molnar <mingo@kernel.org>, John Stultz <jstultz@google.com>,
Stephen Boyd <sboyd@kernel.org>
Cc: Steven Rostedt <rostedt@goodmis.org>,
Mathieu Desnoyers <mathieu.desnoyers@efficios.com>,
Lai Jiangshan <jiangshanlai@gmail.com>,
Zqiang <qiang.zhang1211@gmail.com>,
Sergey Senozhatsky <senozhatsky@chromium.org>,
chenhuacai@kernel.org, rcu@vger.kernel.org,
linux-kernel@vger.kernel.org,
Huacai Chen <chenhuacai@loongson.cn>
Subject: [PATCH V4 1/2] tick: Rename tick_do_update_jiffies64() and allow external usage
Date: Mon, 14 Aug 2023 10:00:44 +0800 [thread overview]
Message-ID: <20230814020045.51950-1-chenhuacai@loongson.cn> (raw)
Rename tick_do_update_jiffies64() to do_update_jiffies_64() and move it
to jiffies.c. This keeps the same naming style in jiffies.c and allow it
be used by external components. This patch is a preparation for the next
one which attempts to avoid necessary rcu stall warnings.
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
---
V2: Fix build.
V3: Fix build again.
V4: Define an empty version of do_update_jiffies_64() for periodic timer
ticks system.
include/linux/jiffies.h | 2 +
kernel/time/jiffies.c | 115 +++++++++++++++++++++++++++++++++++++-
kernel/time/tick-sched.c | 115 ++------------------------------------
kernel/time/timekeeping.h | 1 +
4 files changed, 120 insertions(+), 113 deletions(-)
diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h
index 5e13f801c902..48866314c68b 100644
--- a/include/linux/jiffies.h
+++ b/include/linux/jiffies.h
@@ -88,6 +88,8 @@ static inline u64 get_jiffies_64(void)
}
#endif
+void do_update_jiffies_64(s64 now); /* typedef s64 ktime_t */
+
/*
* These inlines deal with timer wrapping correctly. You are
* strongly encouraged to use them
diff --git a/kernel/time/jiffies.c b/kernel/time/jiffies.c
index bc4db9e5ab70..3c0a6092dbfb 100644
--- a/kernel/time/jiffies.c
+++ b/kernel/time/jiffies.c
@@ -5,14 +5,14 @@
* Copyright (C) 2004, 2005 IBM, John Stultz (johnstul@us.ibm.com)
*/
#include <linux/clocksource.h>
+#include <linux/init.h>
#include <linux/jiffies.h>
#include <linux/module.h>
-#include <linux/init.h>
+#include <linux/sched/loadavg.h>
#include "timekeeping.h"
#include "tick-internal.h"
-
static u64 jiffies_read(struct clocksource *cs)
{
return (u64) jiffies;
@@ -61,6 +61,117 @@ EXPORT_SYMBOL(get_jiffies_64);
EXPORT_SYMBOL(jiffies);
+/*
+ * The time, when the last jiffy update happened. Write access must hold
+ * jiffies_lock and jiffies_seq. Because tick_nohz_next_event() needs to
+ * get a consistent view of jiffies and last_jiffies_update.
+ */
+ktime_t last_jiffies_update;
+
+/*
+ * Must be called with interrupts disabled !
+ */
+#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
+void do_update_jiffies_64(ktime_t now)
+{
+ unsigned long ticks = 1;
+ ktime_t delta, nextp;
+
+ /*
+ * 64bit can do a quick check without holding jiffies lock and
+ * without looking at the sequence count. The smp_load_acquire()
+ * pairs with the update done later in this function.
+ *
+ * 32bit cannot do that because the store of tick_next_period
+ * consists of two 32bit stores and the first store could move it
+ * to a random point in the future.
+ */
+ if (IS_ENABLED(CONFIG_64BIT)) {
+ if (ktime_before(now, smp_load_acquire(&tick_next_period)))
+ return;
+ } else {
+ unsigned int seq;
+
+ /*
+ * Avoid contention on jiffies_lock and protect the quick
+ * check with the sequence count.
+ */
+ do {
+ seq = read_seqcount_begin(&jiffies_seq);
+ nextp = tick_next_period;
+ } while (read_seqcount_retry(&jiffies_seq, seq));
+
+ if (ktime_before(now, nextp))
+ return;
+ }
+
+ /* Quick check failed, i.e. update is required. */
+ raw_spin_lock(&jiffies_lock);
+ /*
+ * Reevaluate with the lock held. Another CPU might have done the
+ * update already.
+ */
+ if (ktime_before(now, tick_next_period)) {
+ raw_spin_unlock(&jiffies_lock);
+ return;
+ }
+
+ write_seqcount_begin(&jiffies_seq);
+
+ delta = ktime_sub(now, tick_next_period);
+ if (unlikely(delta >= TICK_NSEC)) {
+ /* Slow path for long idle sleep times */
+ s64 incr = TICK_NSEC;
+
+ ticks += ktime_divns(delta, incr);
+
+ last_jiffies_update = ktime_add_ns(last_jiffies_update,
+ incr * ticks);
+ } else {
+ last_jiffies_update = ktime_add_ns(last_jiffies_update,
+ TICK_NSEC);
+ }
+
+ /* Advance jiffies to complete the jiffies_seq protected job */
+ jiffies_64 += ticks;
+
+ /*
+ * Keep the tick_next_period variable up to date.
+ */
+ nextp = ktime_add_ns(last_jiffies_update, TICK_NSEC);
+
+ if (IS_ENABLED(CONFIG_64BIT)) {
+ /*
+ * Pairs with smp_load_acquire() in the lockless quick
+ * check above and ensures that the update to jiffies_64 is
+ * not reordered vs. the store to tick_next_period, neither
+ * by the compiler nor by the CPU.
+ */
+ smp_store_release(&tick_next_period, nextp);
+ } else {
+ /*
+ * A plain store is good enough on 32bit as the quick check
+ * above is protected by the sequence count.
+ */
+ tick_next_period = nextp;
+ }
+
+ /*
+ * Release the sequence count. calc_global_load() below is not
+ * protected by it, but jiffies_lock needs to be held to prevent
+ * concurrent invocations.
+ */
+ write_seqcount_end(&jiffies_seq);
+
+ calc_global_load();
+
+ raw_spin_unlock(&jiffies_lock);
+ update_wall_time();
+}
+#else
+void do_update_jiffies_64(ktime_t now) { }
+#endif
+
static int __init init_jiffies_clocksource(void)
{
return __clocksource_register(&clocksource_jiffies);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 4df14db4da49..e9dc4c81d69a 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -44,113 +44,6 @@ struct tick_sched *tick_get_tick_sched(int cpu)
}
#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
-/*
- * The time, when the last jiffy update happened. Write access must hold
- * jiffies_lock and jiffies_seq. tick_nohz_next_event() needs to get a
- * consistent view of jiffies and last_jiffies_update.
- */
-static ktime_t last_jiffies_update;
-
-/*
- * Must be called with interrupts disabled !
- */
-static void tick_do_update_jiffies64(ktime_t now)
-{
- unsigned long ticks = 1;
- ktime_t delta, nextp;
-
- /*
- * 64bit can do a quick check without holding jiffies lock and
- * without looking at the sequence count. The smp_load_acquire()
- * pairs with the update done later in this function.
- *
- * 32bit cannot do that because the store of tick_next_period
- * consists of two 32bit stores and the first store could move it
- * to a random point in the future.
- */
- if (IS_ENABLED(CONFIG_64BIT)) {
- if (ktime_before(now, smp_load_acquire(&tick_next_period)))
- return;
- } else {
- unsigned int seq;
-
- /*
- * Avoid contention on jiffies_lock and protect the quick
- * check with the sequence count.
- */
- do {
- seq = read_seqcount_begin(&jiffies_seq);
- nextp = tick_next_period;
- } while (read_seqcount_retry(&jiffies_seq, seq));
-
- if (ktime_before(now, nextp))
- return;
- }
-
- /* Quick check failed, i.e. update is required. */
- raw_spin_lock(&jiffies_lock);
- /*
- * Reevaluate with the lock held. Another CPU might have done the
- * update already.
- */
- if (ktime_before(now, tick_next_period)) {
- raw_spin_unlock(&jiffies_lock);
- return;
- }
-
- write_seqcount_begin(&jiffies_seq);
-
- delta = ktime_sub(now, tick_next_period);
- if (unlikely(delta >= TICK_NSEC)) {
- /* Slow path for long idle sleep times */
- s64 incr = TICK_NSEC;
-
- ticks += ktime_divns(delta, incr);
-
- last_jiffies_update = ktime_add_ns(last_jiffies_update,
- incr * ticks);
- } else {
- last_jiffies_update = ktime_add_ns(last_jiffies_update,
- TICK_NSEC);
- }
-
- /* Advance jiffies to complete the jiffies_seq protected job */
- jiffies_64 += ticks;
-
- /*
- * Keep the tick_next_period variable up to date.
- */
- nextp = ktime_add_ns(last_jiffies_update, TICK_NSEC);
-
- if (IS_ENABLED(CONFIG_64BIT)) {
- /*
- * Pairs with smp_load_acquire() in the lockless quick
- * check above and ensures that the update to jiffies_64 is
- * not reordered vs. the store to tick_next_period, neither
- * by the compiler nor by the CPU.
- */
- smp_store_release(&tick_next_period, nextp);
- } else {
- /*
- * A plain store is good enough on 32bit as the quick check
- * above is protected by the sequence count.
- */
- tick_next_period = nextp;
- }
-
- /*
- * Release the sequence count. calc_global_load() below is not
- * protected by it, but jiffies_lock needs to be held to prevent
- * concurrent invocations.
- */
- write_seqcount_end(&jiffies_seq);
-
- calc_global_load();
-
- raw_spin_unlock(&jiffies_lock);
- update_wall_time();
-}
-
/*
* Initialize and return retrieve the jiffies update.
*/
@@ -207,7 +100,7 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
/* Check, if the jiffies need an update */
if (tick_do_timer_cpu == cpu)
- tick_do_update_jiffies64(now);
+ do_update_jiffies_64(now);
/*
* If jiffies update stalled for too long (timekeeper in stop_machine()
@@ -218,7 +111,7 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
ts->last_tick_jiffies = READ_ONCE(jiffies);
} else {
if (++ts->stalled_jiffies == MAX_STALLED_JIFFIES) {
- tick_do_update_jiffies64(now);
+ do_update_jiffies_64(now);
ts->stalled_jiffies = 0;
ts->last_tick_jiffies = READ_ONCE(jiffies);
}
@@ -652,7 +545,7 @@ static void tick_nohz_update_jiffies(ktime_t now)
__this_cpu_write(tick_cpu_sched.idle_waketime, now);
local_irq_save(flags);
- tick_do_update_jiffies64(now);
+ do_update_jiffies_64(now);
local_irq_restore(flags);
touch_softlockup_watchdog_sched();
@@ -975,7 +868,7 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)
static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
{
/* Update jiffies first */
- tick_do_update_jiffies64(now);
+ do_update_jiffies_64(now);
/*
* Clear the timer idle flag, so we avoid IPIs on remote queueing and
diff --git a/kernel/time/timekeeping.h b/kernel/time/timekeeping.h
index 543beba096c7..21670f6c7421 100644
--- a/kernel/time/timekeeping.h
+++ b/kernel/time/timekeeping.h
@@ -28,6 +28,7 @@ extern void update_wall_time(void);
extern raw_spinlock_t jiffies_lock;
extern seqcount_raw_spinlock_t jiffies_seq;
+extern ktime_t last_jiffies_update;
#define CS_NAME_LEN 32
--
2.39.3
next reply other threads:[~2023-08-14 2:02 UTC|newest]
Thread overview: 56+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-08-14 2:00 Huacai Chen [this message]
2023-08-14 2:00 ` [PATCH V4 2/2] rcu: Update jiffies in rcu_cpu_stall_reset() Huacai Chen
2023-08-14 16:15 ` Paul E. McKenney
2023-08-15 6:05 ` Huacai Chen
2023-08-16 3:16 ` Z qiang
2023-08-16 4:53 ` Huacai Chen
2023-08-16 5:09 ` Z qiang
2023-08-16 9:33 ` Huacai Chen
2023-08-16 10:06 ` Z qiang
2023-08-16 12:28 ` Huacai Chen
2023-08-16 15:56 ` Alan Huang
2023-08-16 16:13 ` Huacai Chen
2023-08-16 16:52 ` Alan Huang
2023-08-17 4:04 ` Huacai Chen
2023-08-23 21:41 ` Thomas Gleixner
2023-08-16 19:27 ` Joel Fernandes
2023-08-17 8:06 ` Huacai Chen
2023-08-23 22:03 ` Thomas Gleixner
2023-08-23 22:41 ` Paul E. McKenney
2023-08-24 2:50 ` Huacai Chen
2023-08-24 11:40 ` Paul E. McKenney
2023-08-24 12:40 ` Huacai Chen
2023-08-24 13:24 ` Paul E. McKenney
2023-08-24 15:43 ` Huacai Chen
2023-08-24 18:28 ` Paul E. McKenney
2023-08-25 11:15 ` Huacai Chen
2023-08-25 23:28 ` Joel Fernandes
2023-08-27 3:27 ` Joel Fernandes
2023-08-27 5:50 ` Huacai Chen
2023-08-27 22:11 ` Joel Fernandes
2023-08-28 10:47 ` Paul E. McKenney
2023-08-28 11:30 ` Huacai Chen
2023-08-28 11:54 ` Paul E. McKenney
2023-08-28 13:33 ` Joel Fernandes
2023-08-28 14:02 ` Paul E. McKenney
2023-08-28 14:37 ` Huacai Chen
2023-08-28 14:50 ` Joel Fernandes
2023-08-28 15:12 ` Huacai Chen
2023-08-28 20:47 ` Joel Fernandes
2023-08-29 4:07 ` Huacai Chen
2023-08-29 14:46 ` Joel Fernandes
2023-08-30 4:25 ` Huacai Chen
2023-08-30 10:18 ` Joel Fernandes
2023-08-30 1:04 ` Joel Fernandes
2023-08-26 1:45 ` Paul E. McKenney
2023-08-24 13:09 ` Joel Fernandes
2023-08-24 13:28 ` Paul E. McKenney
2023-08-24 16:03 ` Huacai Chen
2023-08-24 16:32 ` Huacai Chen
2023-08-24 16:34 ` Paul E. McKenney
2023-08-24 2:47 ` Huacai Chen
2023-08-24 9:39 ` Thomas Gleixner
2023-08-24 13:21 ` Joel Fernandes
2023-08-24 13:29 ` Paul E. McKenney
2023-08-24 16:15 ` Huacai Chen
2023-08-23 21:36 ` Thomas Gleixner
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230814020045.51950-1-chenhuacai@loongson.cn \
--to=chenhuacai@loongson.cn \
--cc=boqun.feng@gmail.com \
--cc=chenhuacai@kernel.org \
--cc=frederic@kernel.org \
--cc=jiangshanlai@gmail.com \
--cc=joel@joelfernandes.org \
--cc=josh@joshtriplett.org \
--cc=jstultz@google.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=mingo@kernel.org \
--cc=paulmck@kernel.org \
--cc=qiang.zhang1211@gmail.com \
--cc=quic_neeraju@quicinc.com \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=sboyd@kernel.org \
--cc=senozhatsky@chromium.org \
--cc=tglx@linutronix.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).