From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
To: linux-kernel@vger.kernel.org
Cc: mingo@kernel.org, jiangshanlai@gmail.com, dipankar@in.ibm.com,
akpm@linux-foundation.org, mathieu.desnoyers@efficios.com,
josh@joshtriplett.org, tglx@linutronix.de, peterz@infradead.org,
rostedt@goodmis.org, dhowells@redhat.com, edumazet@google.com,
dvhart@linux.intel.com, fweisbec@gmail.com, oleg@redhat.com,
bobby.prani@gmail.com,
"Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Subject: [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations
Date: Sat, 14 Jan 2017 00:54:42 -0800 [thread overview]
Message-ID: <1484384085-19444-3-git-send-email-paulmck@linux.vnet.ibm.com> (raw)
In-Reply-To: <20170114085406.GA19052@linux.vnet.ibm.com>
This commit is the third step towards full abstraction of all accesses
to the ->dynticks counter, implementing the previously open-coded atomic
add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
This abstraction will ease changes to the ->dynticks counter operation.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
kernel/rcu/tree.c | 92 +++++++++++++++++++++++++++++++++++++++----------------
1 file changed, 66 insertions(+), 26 deletions(-)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 805d55ee0b2a..fc49e008963a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -282,6 +282,65 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
};
/*
+ * Record entry into an extended quiescent state. This is only to be
+ * called when not already in an extended quiescent state.
+ */
+static void rcu_dynticks_eqs_enter(void)
+{
+ struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+ /*
+ * CPUs seeing atomic_inc() must see prior RCU read-side critical
+ * sections, and we also must force ordering with the next idle
+ * sojourn.
+ */
+ smp_mb__before_atomic(); /* See above. */
+ atomic_inc(&rdtp->dynticks);
+ smp_mb__after_atomic(); /* See above. */
+ WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+ atomic_read(&rdtp->dynticks) & 0x1);
+}
+
+/*
+ * Record exit from an extended quiescent state. This is only to be
+ * called from an extended quiescent state.
+ */
+static void rcu_dynticks_eqs_exit(void)
+{
+ struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+ /*
+ * CPUs seeing atomic_inc() must see prior idle sojourns,
+ * and we also must force ordering with the next RCU read-side
+ * critical section.
+ */
+ smp_mb__before_atomic(); /* See above. */
+ atomic_inc(&rdtp->dynticks);
+ smp_mb__after_atomic(); /* See above. */
+ WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+ !(atomic_read(&rdtp->dynticks) & 0x1));
+}
+
+/*
+ * Reset the current CPU's ->dynticks counter to indicate that the
+ * newly onlined CPU is no longer in an extended quiescent state.
+ * This will either leave the counter unchanged, or increment it
+ * to the next non-quiescent value.
+ *
+ * The non-atomic test/increment sequence works because the upper bits
+ * of the ->dynticks counter are manipulated only by the corresponding CPU,
+ * or when the corresponding CPU is offline.
+ */
+static void rcu_dynticks_eqs_online(void)
+{
+ struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+ if (atomic_read(&rdtp->dynticks) & 0x1)
+ return;
+ atomic_add(0x1, &rdtp->dynticks);
+}
+
+/*
* Snapshot the ->dynticks counter with full ordering so as to allow
* stable comparison of this counter with past and future snapshots.
*/
@@ -693,7 +752,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
{
struct rcu_state *rsp;
struct rcu_data *rdp;
- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+ struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -712,12 +771,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
do_nocb_deferred_wakeup(rdp);
}
rcu_prepare_for_idle();
- /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
- smp_mb__before_atomic(); /* See above. */
- atomic_inc(&rdtp->dynticks);
- smp_mb__after_atomic(); /* Force ordering with next sojourn. */
- WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
- atomic_read(&rdtp->dynticks) & 0x1);
+ rcu_dynticks_eqs_enter();
rcu_dynticks_task_enter();
/*
@@ -846,15 +900,10 @@ void rcu_irq_exit_irqson(void)
*/
static void rcu_eqs_exit_common(long long oldval, int user)
{
- struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+ struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
rcu_dynticks_task_exit();
- smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */
- atomic_inc(&rdtp->dynticks);
- /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
- smp_mb__after_atomic(); /* See above. */
- WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
- !(atomic_read(&rdtp->dynticks) & 0x1));
+ rcu_dynticks_eqs_exit();
rcu_cleanup_after_idle();
trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -1001,11 +1050,7 @@ void rcu_nmi_enter(void)
* period (observation due to Andy Lutomirski).
*/
if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
- smp_mb__before_atomic(); /* Force delay from prior write. */
- atomic_inc(&rdtp->dynticks);
- /* atomic_inc() before later RCU read-side crit sects */
- smp_mb__after_atomic(); /* See above. */
- WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+ rcu_dynticks_eqs_exit();
incby = 1;
}
rdtp->dynticks_nmi_nesting += incby;
@@ -1043,11 +1088,7 @@ void rcu_nmi_exit(void)
/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
rdtp->dynticks_nmi_nesting = 0;
- /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
- smp_mb__before_atomic(); /* See above. */
- atomic_inc(&rdtp->dynticks);
- smp_mb__after_atomic(); /* Force delay to next write. */
- WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+ rcu_dynticks_eqs_enter();
}
/**
@@ -3800,8 +3841,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
init_callback_list(rdp); /* Re-enable callbacks on this CPU. */
rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
rcu_sysidle_init_percpu_data(rdp->dynticks);
- atomic_set(&rdp->dynticks->dynticks,
- (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
+ rcu_dynticks_eqs_online();
raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
/*
--
2.5.2
next prev parent reply other threads:[~2017-01-14 8:54 UTC|newest]
Thread overview: 35+ messages / expand[flat|nested] mbox.gz Atom feed top
2017-01-14 8:54 [PATCH tip/core/rcu 0/6] Dynticks updates for 4.11 Paul E. McKenney
2017-01-14 8:54 ` [PATCH tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation Paul E. McKenney
2017-01-16 7:39 ` Josh Triplett
2017-01-16 11:22 ` Paul E. McKenney
2017-01-16 18:57 ` Josh Triplett
2017-01-16 23:48 ` Paul E. McKenney
2017-01-14 8:54 ` [PATCH tip/core/rcu 2/6] rcu: Abstract the dynticks snapshot operation Paul E. McKenney
2017-01-14 8:54 ` Paul E. McKenney [this message]
2017-01-16 7:47 ` [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations Josh Triplett
2017-01-16 11:34 ` Paul E. McKenney
2017-01-16 19:25 ` Josh Triplett
2017-01-17 0:12 ` Paul E. McKenney
2017-01-16 16:43 ` Peter Zijlstra
2017-01-16 21:01 ` Paul E. McKenney
2017-01-17 0:06 ` Paul E. McKenney
2017-01-14 8:54 ` [PATCH tip/core/rcu 4/6] rcu: Abstract extended quiescent state determination Paul E. McKenney
2017-01-14 8:54 ` [PATCH tip/core/rcu 5/6] rcu: Check cond_resched_rcu_qs() state less often to reduce GP overhead Paul E. McKenney
2017-01-14 8:54 ` [PATCH tip/core/rcu 6/6] rcu: Adjust FQS offline checks for exact online-CPU detection Paul E. McKenney
2017-01-16 7:50 ` [PATCH tip/core/rcu 0/6] Dynticks updates for 4.11 Josh Triplett
2017-01-18 2:44 ` Paul E. McKenney
2017-01-18 2:45 ` [PATCH v2 tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation Paul E. McKenney
2017-01-18 2:45 ` [PATCH v2 tip/core/rcu 2/6] rcu: Abstract the dynticks snapshot operation Paul E. McKenney
2017-01-18 2:45 ` [PATCH v2 tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations Paul E. McKenney
2017-01-21 20:49 ` Josh Triplett
2017-01-23 19:45 ` Paul E. McKenney
2017-01-18 2:45 ` [PATCH v2 tip/core/rcu 4/6] rcu: Abstract extended quiescent state determination Paul E. McKenney
2017-01-18 2:45 ` [PATCH v2 tip/core/rcu 5/6] rcu: Check cond_resched_rcu_qs() state less often to reduce GP overhead Paul E. McKenney
2017-01-18 2:45 ` [PATCH v2 tip/core/rcu 6/6] rcu: Adjust FQS offline checks for exact online-CPU detection Paul E. McKenney
2017-01-24 21:46 ` [PATCH v3 tip/core/rcu 0/6] Dynticks updates for 4.11 Paul E. McKenney
2017-01-24 21:46 ` [PATCH v3 tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation Paul E. McKenney
2017-01-24 21:46 ` [PATCH v3 tip/core/rcu 2/6] rcu: Abstract the dynticks snapshot operation Paul E. McKenney
2017-01-24 21:46 ` [PATCH v3 tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations Paul E. McKenney
2017-01-24 21:46 ` [PATCH v3 tip/core/rcu 4/6] rcu: Abstract extended quiescent state determination Paul E. McKenney
2017-01-24 21:46 ` [PATCH v3 tip/core/rcu 5/6] rcu: Check cond_resched_rcu_qs() state less often to reduce GP overhead Paul E. McKenney
2017-01-24 21:46 ` [PATCH v3 tip/core/rcu 6/6] rcu: Adjust FQS offline checks for exact online-CPU detection Paul E. McKenney
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1484384085-19444-3-git-send-email-paulmck@linux.vnet.ibm.com \
--to=paulmck@linux.vnet.ibm.com \
--cc=akpm@linux-foundation.org \
--cc=bobby.prani@gmail.com \
--cc=dhowells@redhat.com \
--cc=dipankar@in.ibm.com \
--cc=dvhart@linux.intel.com \
--cc=edumazet@google.com \
--cc=fweisbec@gmail.com \
--cc=jiangshanlai@gmail.com \
--cc=josh@joshtriplett.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=mingo@kernel.org \
--cc=oleg@redhat.com \
--cc=peterz@infradead.org \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).