From: "Paul E. McKenney" <paulmck@linux.ibm.com>
To: rcu@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, mingo@kernel.org,
jiangshanlai@gmail.com, dipankar@in.ibm.com,
akpm@linux-foundation.org, mathieu.desnoyers@efficios.com,
josh@joshtriplett.org, tglx@linutronix.de, peterz@infradead.org,
rostedt@goodmis.org, dhowells@redhat.com, edumazet@google.com,
fweisbec@gmail.com, oleg@redhat.com, joel@joelfernandes.org,
"Paul E. McKenney" <paulmck@linux.ibm.com>
Subject: [PATCH tip/core/rcu 01/11] rcu/nocb: Rename rcu_data fields to prepare for forward-progress work
Date: Thu, 1 Aug 2019 15:50:18 -0700 [thread overview]
Message-ID: <20190801225028.18225-1-paulmck@linux.ibm.com> (raw)
In-Reply-To: <20190801225009.GA17155@linux.ibm.com>
This commit simply renames rcu_data fields to prepare for leader
nocb kthreads doing only grace-period work and callback shuffling.
This will mean the addition of replacement kthreads to invoke callbacks.
The "leader" and "follower" thus become less meaningful, so the commit
changes no-CB fields with these strings to "gp" and "cb", respectively.
Signed-off-by: Paul E. McKenney <paulmck@linux.ibm.com>
---
kernel/rcu/tree.h | 14 ++++----
kernel/rcu/tree_plugin.h | 78 ++++++++++++++++++++--------------------
2 files changed, 46 insertions(+), 46 deletions(-)
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 7acaf3a62d39..e4e59b627c5a 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -198,10 +198,10 @@ struct rcu_data {
struct rcu_head **nocb_tail;
atomic_long_t nocb_q_count; /* # CBs waiting for nocb */
atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */
- struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */
- struct rcu_head **nocb_follower_tail;
+ struct rcu_head *nocb_cb_head; /* CBs ready to invoke. */
+ struct rcu_head **nocb_cb_tail;
struct swait_queue_head nocb_wq; /* For nocb kthreads to sleep on. */
- struct task_struct *nocb_kthread;
+ struct task_struct *nocb_cb_kthread;
raw_spinlock_t nocb_lock; /* Guard following pair of fields. */
int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */
struct timer_list nocb_timer; /* Enforce finite deferral. */
@@ -210,12 +210,12 @@ struct rcu_data {
struct rcu_head *nocb_gp_head ____cacheline_internodealigned_in_smp;
/* CBs waiting for GP. */
struct rcu_head **nocb_gp_tail;
- bool nocb_leader_sleep; /* Is the nocb leader thread asleep? */
- struct rcu_data *nocb_next_follower;
- /* Next follower in wakeup chain. */
+ bool nocb_gp_sleep; /* Is the nocb leader thread asleep? */
+ struct rcu_data *nocb_next_cb_rdp;
+ /* Next rcu_data in wakeup chain. */
/* The following fields are used by the follower, hence new cachline. */
- struct rcu_data *nocb_leader ____cacheline_internodealigned_in_smp;
+ struct rcu_data *nocb_gp_rdp ____cacheline_internodealigned_in_smp;
/* Leader CPU takes GP-end wakeups. */
#endif /* #ifdef CONFIG_RCU_NOCB_CPU */
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 99e9d952827b..5ce1edd1c87f 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1528,19 +1528,19 @@ static void __wake_nocb_leader(struct rcu_data *rdp, bool force,
unsigned long flags)
__releases(rdp->nocb_lock)
{
- struct rcu_data *rdp_leader = rdp->nocb_leader;
+ struct rcu_data *rdp_leader = rdp->nocb_gp_rdp;
lockdep_assert_held(&rdp->nocb_lock);
- if (!READ_ONCE(rdp_leader->nocb_kthread)) {
+ if (!READ_ONCE(rdp_leader->nocb_cb_kthread)) {
raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
return;
}
- if (rdp_leader->nocb_leader_sleep || force) {
+ if (rdp_leader->nocb_gp_sleep || force) {
/* Prior smp_mb__after_atomic() orders against prior enqueue. */
- WRITE_ONCE(rdp_leader->nocb_leader_sleep, false);
+ WRITE_ONCE(rdp_leader->nocb_gp_sleep, false);
del_timer(&rdp->nocb_timer);
raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
- smp_mb(); /* ->nocb_leader_sleep before swake_up_one(). */
+ smp_mb(); /* ->nocb_gp_sleep before swake_up_one(). */
swake_up_one(&rdp_leader->nocb_wq);
} else {
raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
@@ -1604,10 +1604,10 @@ static bool rcu_nocb_cpu_needs_barrier(int cpu)
if (!rhp)
rhp = READ_ONCE(rdp->nocb_gp_head);
if (!rhp)
- rhp = READ_ONCE(rdp->nocb_follower_head);
+ rhp = READ_ONCE(rdp->nocb_cb_head);
/* Having no rcuo kthread but CBs after scheduler starts is bad! */
- if (!READ_ONCE(rdp->nocb_kthread) && rhp &&
+ if (!READ_ONCE(rdp->nocb_cb_kthread) && rhp &&
rcu_scheduler_fully_active) {
/* RCU callback enqueued before CPU first came online??? */
pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
@@ -1646,7 +1646,7 @@ static void __call_rcu_nocb_enqueue(struct rcu_data *rdp,
smp_mb__after_atomic(); /* Store *old_rhpp before _wake test. */
/* If we are not being polled and there is a kthread, awaken it ... */
- t = READ_ONCE(rdp->nocb_kthread);
+ t = READ_ONCE(rdp->nocb_cb_kthread);
if (rcu_nocb_poll || !t) {
trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
TPS("WakeNotPoll"));
@@ -1800,9 +1800,9 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
if (!rcu_nocb_poll) {
trace_rcu_nocb_wake(rcu_state.name, my_rdp->cpu, TPS("Sleep"));
swait_event_interruptible_exclusive(my_rdp->nocb_wq,
- !READ_ONCE(my_rdp->nocb_leader_sleep));
+ !READ_ONCE(my_rdp->nocb_gp_sleep));
raw_spin_lock_irqsave(&my_rdp->nocb_lock, flags);
- my_rdp->nocb_leader_sleep = true;
+ my_rdp->nocb_gp_sleep = true;
WRITE_ONCE(my_rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
del_timer(&my_rdp->nocb_timer);
raw_spin_unlock_irqrestore(&my_rdp->nocb_lock, flags);
@@ -1818,7 +1818,7 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
*/
gotcbs = false;
smp_mb(); /* wakeup and _sleep before ->nocb_head reads. */
- for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
+ for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) {
rdp->nocb_gp_head = READ_ONCE(rdp->nocb_head);
if (!rdp->nocb_gp_head)
continue; /* No CBs here, try next follower. */
@@ -1845,12 +1845,12 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
rcu_nocb_wait_gp(my_rdp);
/* Each pass through the following loop wakes a follower, if needed. */
- for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_follower) {
+ for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) {
if (!rcu_nocb_poll &&
READ_ONCE(rdp->nocb_head) &&
- READ_ONCE(my_rdp->nocb_leader_sleep)) {
+ READ_ONCE(my_rdp->nocb_gp_sleep)) {
raw_spin_lock_irqsave(&my_rdp->nocb_lock, flags);
- my_rdp->nocb_leader_sleep = false;/* No need to sleep.*/
+ my_rdp->nocb_gp_sleep = false;/* No need to sleep.*/
raw_spin_unlock_irqrestore(&my_rdp->nocb_lock, flags);
}
if (!rdp->nocb_gp_head)
@@ -1858,18 +1858,18 @@ static void nocb_leader_wait(struct rcu_data *my_rdp)
/* Append callbacks to follower's "done" list. */
raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
- tail = rdp->nocb_follower_tail;
- rdp->nocb_follower_tail = rdp->nocb_gp_tail;
+ tail = rdp->nocb_cb_tail;
+ rdp->nocb_cb_tail = rdp->nocb_gp_tail;
*tail = rdp->nocb_gp_head;
raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
- if (rdp != my_rdp && tail == &rdp->nocb_follower_head) {
+ if (rdp != my_rdp && tail == &rdp->nocb_cb_head) {
/* List was empty, so wake up the follower. */
swake_up_one(&rdp->nocb_wq);
}
}
/* If we (the leader) don't have CBs, go wait some more. */
- if (!my_rdp->nocb_follower_head)
+ if (!my_rdp->nocb_cb_head)
goto wait_again;
}
@@ -1882,8 +1882,8 @@ static void nocb_follower_wait(struct rcu_data *rdp)
for (;;) {
trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FollowerSleep"));
swait_event_interruptible_exclusive(rdp->nocb_wq,
- READ_ONCE(rdp->nocb_follower_head));
- if (smp_load_acquire(&rdp->nocb_follower_head)) {
+ READ_ONCE(rdp->nocb_cb_head));
+ if (smp_load_acquire(&rdp->nocb_cb_head)) {
/* ^^^ Ensure CB invocation follows _head test. */
return;
}
@@ -1910,17 +1910,17 @@ static int rcu_nocb_kthread(void *arg)
/* Each pass through this loop invokes one batch of callbacks */
for (;;) {
/* Wait for callbacks. */
- if (rdp->nocb_leader == rdp)
+ if (rdp->nocb_gp_rdp == rdp)
nocb_leader_wait(rdp);
else
nocb_follower_wait(rdp);
/* Pull the ready-to-invoke callbacks onto local list. */
raw_spin_lock_irqsave(&rdp->nocb_lock, flags);
- list = rdp->nocb_follower_head;
- rdp->nocb_follower_head = NULL;
- tail = rdp->nocb_follower_tail;
- rdp->nocb_follower_tail = &rdp->nocb_follower_head;
+ list = rdp->nocb_cb_head;
+ rdp->nocb_cb_head = NULL;
+ tail = rdp->nocb_cb_tail;
+ rdp->nocb_cb_tail = &rdp->nocb_cb_head;
raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
if (WARN_ON_ONCE(!list))
continue;
@@ -2048,7 +2048,7 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
{
rdp->nocb_tail = &rdp->nocb_head;
init_swait_queue_head(&rdp->nocb_wq);
- rdp->nocb_follower_tail = &rdp->nocb_follower_head;
+ rdp->nocb_cb_tail = &rdp->nocb_cb_head;
raw_spin_lock_init(&rdp->nocb_lock);
timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);
}
@@ -2070,27 +2070,27 @@ static void rcu_spawn_one_nocb_kthread(int cpu)
* If this isn't a no-CBs CPU or if it already has an rcuo kthread,
* then nothing to do.
*/
- if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
+ if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_cb_kthread)
return;
/* If we didn't spawn the leader first, reorganize! */
- rdp_old_leader = rdp_spawn->nocb_leader;
- if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
+ rdp_old_leader = rdp_spawn->nocb_gp_rdp;
+ if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_cb_kthread) {
rdp_last = NULL;
rdp = rdp_old_leader;
do {
- rdp->nocb_leader = rdp_spawn;
+ rdp->nocb_gp_rdp = rdp_spawn;
if (rdp_last && rdp != rdp_spawn)
- rdp_last->nocb_next_follower = rdp;
+ rdp_last->nocb_next_cb_rdp = rdp;
if (rdp == rdp_spawn) {
- rdp = rdp->nocb_next_follower;
+ rdp = rdp->nocb_next_cb_rdp;
} else {
rdp_last = rdp;
- rdp = rdp->nocb_next_follower;
- rdp_last->nocb_next_follower = NULL;
+ rdp = rdp->nocb_next_cb_rdp;
+ rdp_last->nocb_next_cb_rdp = NULL;
}
} while (rdp);
- rdp_spawn->nocb_next_follower = rdp_old_leader;
+ rdp_spawn->nocb_next_cb_rdp = rdp_old_leader;
}
/* Spawn the kthread for this CPU. */
@@ -2098,7 +2098,7 @@ static void rcu_spawn_one_nocb_kthread(int cpu)
"rcuo%c/%d", rcu_state.abbr, cpu);
if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo kthread, OOM is now expected behavior\n", __func__))
return;
- WRITE_ONCE(rdp_spawn->nocb_kthread, t);
+ WRITE_ONCE(rdp_spawn->nocb_cb_kthread, t);
}
/*
@@ -2158,12 +2158,12 @@ static void __init rcu_organize_nocb_kthreads(void)
if (rdp->cpu >= nl) {
/* New leader, set up for followers & next leader. */
nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
- rdp->nocb_leader = rdp;
+ rdp->nocb_gp_rdp = rdp;
rdp_leader = rdp;
} else {
/* Another follower, link to previous leader. */
- rdp->nocb_leader = rdp_leader;
- rdp_prev->nocb_next_follower = rdp;
+ rdp->nocb_gp_rdp = rdp_leader;
+ rdp_prev->nocb_next_cb_rdp = rdp;
}
rdp_prev = rdp;
}
--
2.17.1
next prev parent reply other threads:[~2019-08-01 22:51 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-08-01 22:50 [PATCH tip/core/rcu 0/11] No-CBs grace-period kthread updates for v5.3-rc2 Paul E. McKenney
2019-08-01 22:50 ` Paul E. McKenney [this message]
2019-08-01 22:50 ` [PATCH tip/core/rcu 02/11] rcu/nocb: Update comments to prepare for forward-progress work Paul E. McKenney
2019-08-01 22:50 ` [PATCH tip/core/rcu 03/11] rcu/nocb: Provide separate no-CBs grace-period kthreads Paul E. McKenney
2019-08-03 17:41 ` Joel Fernandes
2019-08-03 19:46 ` Paul E. McKenney
2019-08-04 19:24 ` Joel Fernandes
2019-08-01 22:50 ` [PATCH tip/core/rcu 04/11] rcu/nocb: Rename nocb_follower_wait() to nocb_cb_wait() Paul E. McKenney
2019-08-01 22:50 ` [PATCH tip/core/rcu 05/11] rcu/nocb: Rename wake_nocb_leader() to wake_nocb_gp() Paul E. McKenney
2019-08-01 22:50 ` [PATCH tip/core/rcu 06/11] rcu/nocb: Rename __wake_nocb_leader() to __wake_nocb_gp() Paul E. McKenney
2019-08-01 22:50 ` [PATCH tip/core/rcu 07/11] rcu/nocb: Rename wake_nocb_leader_defer() to wake_nocb_gp_defer() Paul E. McKenney
2019-08-01 22:50 ` [PATCH tip/core/rcu 08/11] rcu/nocb: Rename rcu_organize_nocb_kthreads() local variable Paul E. McKenney
2019-08-01 22:50 ` [PATCH tip/core/rcu 09/11] rcu/nocb: Rename and document no-CB CB kthread sleep trace event Paul E. McKenney
2019-08-01 22:50 ` [PATCH tip/core/rcu 10/11] rcu/nocb: Rename rcu_nocb_leader_stride kernel boot parameter Paul E. McKenney
2019-08-01 22:50 ` [PATCH tip/core/rcu 11/11] rcu/nocb: Print gp/cb kthread hierarchy if dump_tree Paul E. McKenney
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190801225028.18225-1-paulmck@linux.ibm.com \
--to=paulmck@linux.ibm.com \
--cc=akpm@linux-foundation.org \
--cc=dhowells@redhat.com \
--cc=dipankar@in.ibm.com \
--cc=edumazet@google.com \
--cc=fweisbec@gmail.com \
--cc=jiangshanlai@gmail.com \
--cc=joel@joelfernandes.org \
--cc=josh@joshtriplett.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=mingo@kernel.org \
--cc=oleg@redhat.com \
--cc=peterz@infradead.org \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=tglx@linutronix.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).