From: Frederic Weisbecker <frederic@kernel.org>
To: LKML <linux-kernel@vger.kernel.org>
Cc: Frederic Weisbecker <frederic@kernel.org>,
Peter Zijlstra <peterz@infradead.org>,
Phil Auld <pauld@redhat.com>, Alex Belits <abelits@marvell.com>,
Nicolas Saenz Julienne <nsaenz@kernel.org>,
Xiongfeng Wang <wangxiongfeng2@huawei.com>,
Neeraj Upadhyay <quic_neeraju@quicinc.com>,
Thomas Gleixner <tglx@linutronix.de>,
Yu Liao <liaoyu15@huawei.com>, Boqun Feng <boqun.feng@gmail.com>,
"Paul E . McKenney" <paulmck@kernel.org>,
Marcelo Tosatti <mtosatti@redhat.com>,
Paul Gortmaker <paul.gortmaker@windriver.com>,
Uladzislau Rezki <uladzislau.rezki@sony.com>,
Joel Fernandes <joel@joelfernandes.org>
Subject: [PATCH 13/20] rcu/context_tracking: Move dynticks_nesting to context tracking
Date: Wed, 8 Jun 2022 16:40:30 +0200 [thread overview]
Message-ID: <20220608144037.1765000-14-frederic@kernel.org> (raw)
In-Reply-To: <20220608144037.1765000-1-frederic@kernel.org>
The RCU eqs tracking is going to be performed by the context tracking
subsystem. The related nesting counters thus need to be moved to the
context tracking structure.
Acked-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Uladzislau Rezki <uladzislau.rezki@sony.com>
Cc: Joel Fernandes <joel@joelfernandes.org>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Nicolas Saenz Julienne <nsaenz@kernel.org>
Cc: Marcelo Tosatti <mtosatti@redhat.com>
Cc: Xiongfeng Wang <wangxiongfeng2@huawei.com>
Cc: Yu Liao<liaoyu15@huawei.com>
Cc: Phil Auld <pauld@redhat.com>
Cc: Paul Gortmaker<paul.gortmaker@windriver.com>
Cc: Alex Belits <abelits@marvell.com>
---
include/linux/context_tracking_state.h | 12 ++++++++++
kernel/context_tracking.c | 1 +
kernel/rcu/tree.c | 31 +++++++++++++-------------
kernel/rcu/tree.h | 1 -
kernel/rcu/tree_stall.h | 2 +-
5 files changed, 30 insertions(+), 17 deletions(-)
diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h
index 651bb9c353f0..164f4673ee74 100644
--- a/include/linux/context_tracking_state.h
+++ b/include/linux/context_tracking_state.h
@@ -25,6 +25,7 @@ struct context_tracking {
#endif
#ifdef CONFIG_CONTEXT_TRACKING_IDLE
atomic_t dynticks; /* Even value for idle, else odd. */
+ long dynticks_nesting; /* Track process nesting level. */
#endif
};
@@ -49,6 +50,17 @@ static __always_inline int ct_dynticks_cpu_acquire(int cpu)
struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
return atomic_read_acquire(&ct->dynticks);
}
+
+static __always_inline long ct_dynticks_nesting(void)
+{
+ return __this_cpu_read(context_tracking.dynticks_nesting);
+}
+
+static __always_inline long ct_dynticks_nesting_cpu(int cpu)
+{
+ struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
+ return ct->dynticks_nesting;
+}
#endif /* #ifdef CONFIG_CONTEXT_TRACKING_IDLE */
#ifdef CONFIG_CONTEXT_TRACKING_USER
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index 3475dbfa4d73..d5ff1de1a24a 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -25,6 +25,7 @@
DEFINE_PER_CPU(struct context_tracking, context_tracking) = {
#ifdef CONFIG_CONTEXT_TRACKING_IDLE
+ .dynticks_nesting = 1,
.dynticks = ATOMIC_INIT(1),
#endif
};
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 5c9ca5900540..5e7771c78162 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -75,7 +75,6 @@
/* Data structures. */
static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
- .dynticks_nesting = 1,
.dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
#ifdef CONFIG_RCU_NOCB_CPU
.cblist.flags = SEGCBLIST_RCU_CORE,
@@ -440,7 +439,7 @@ static int rcu_is_cpu_rrupt_from_idle(void)
lockdep_assert_irqs_disabled();
/* Check for counter underflows */
- RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0,
+ RCU_LOCKDEP_WARN(ct_dynticks_nesting() < 0,
"RCU dynticks_nesting counter underflow!");
RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0,
"RCU dynticks_nmi_nesting counter underflow/zero!");
@@ -456,7 +455,7 @@ static int rcu_is_cpu_rrupt_from_idle(void)
WARN_ON_ONCE(!nesting && !is_idle_task(current));
/* Does CPU appear to be idle from an RCU standpoint? */
- return __this_cpu_read(rcu_data.dynticks_nesting) == 0;
+ return ct_dynticks_nesting() == 0;
}
#define DEFAULT_RCU_BLIMIT (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ? 1000 : 10)
@@ -623,16 +622,16 @@ static noinstr void rcu_eqs_enter(bool user)
WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE);
WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
- rdp->dynticks_nesting == 0);
- if (rdp->dynticks_nesting != 1) {
+ ct_dynticks_nesting() == 0);
+ if (ct_dynticks_nesting() != 1) {
// RCU will still be watching, so just do accounting and leave.
- rdp->dynticks_nesting--;
+ ct->dynticks_nesting--;
return;
}
instrumentation_begin();
lockdep_assert_irqs_disabled();
- trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, ct_dynticks());
+ trace_rcu_dyntick(TPS("Start"), ct_dynticks_nesting(), 0, ct_dynticks());
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
rcu_preempt_deferred_qs(current);
@@ -640,7 +639,7 @@ static noinstr void rcu_eqs_enter(bool user)
instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks));
instrumentation_end();
- WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
+ WRITE_ONCE(ct->dynticks_nesting, 0); /* Avoid irq-access tearing. */
// RCU is watching here ...
rcu_dynticks_eqs_enter();
// ... but is no longer watching here.
@@ -797,7 +796,7 @@ void rcu_irq_exit_check_preempt(void)
{
lockdep_assert_irqs_disabled();
- RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) <= 0,
+ RCU_LOCKDEP_WARN(ct_dynticks_nesting() <= 0,
"RCU dynticks_nesting counter underflow/zero!");
RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) !=
DYNTICK_IRQ_NONIDLE,
@@ -823,11 +822,11 @@ static void noinstr rcu_eqs_exit(bool user)
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !raw_irqs_disabled());
rdp = this_cpu_ptr(&rcu_data);
- oldval = rdp->dynticks_nesting;
+ oldval = ct_dynticks_nesting();
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
if (oldval) {
// RCU was already watching, so just do accounting and leave.
- rdp->dynticks_nesting++;
+ ct->dynticks_nesting++;
return;
}
rcu_dynticks_task_exit();
@@ -839,9 +838,9 @@ static void noinstr rcu_eqs_exit(bool user)
// instrumentation for the noinstr rcu_dynticks_eqs_exit()
instrument_atomic_write(&ct->dynticks, sizeof(ct->dynticks));
- trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, ct_dynticks());
+ trace_rcu_dyntick(TPS("End"), ct_dynticks_nesting(), 1, ct_dynticks());
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
- WRITE_ONCE(rdp->dynticks_nesting, 1);
+ WRITE_ONCE(ct->dynticks_nesting, 1);
WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
instrumentation_end();
@@ -4229,12 +4228,13 @@ static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
static void __init
rcu_boot_init_percpu_data(int cpu)
{
+ struct context_tracking *ct = this_cpu_ptr(&context_tracking);
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
/* Set up local state, ensuring consistent view of global state. */
rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
INIT_WORK(&rdp->strict_work, strict_work_handler);
- WARN_ON_ONCE(rdp->dynticks_nesting != 1);
+ WARN_ON_ONCE(ct->dynticks_nesting != 1);
WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(cpu)));
rdp->barrier_seq_snap = rcu_state.barrier_sequence;
rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
@@ -4259,6 +4259,7 @@ rcu_boot_init_percpu_data(int cpu)
int rcutree_prepare_cpu(unsigned int cpu)
{
unsigned long flags;
+ struct context_tracking *ct = per_cpu_ptr(&context_tracking, cpu);
struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
struct rcu_node *rnp = rcu_get_root();
@@ -4267,7 +4268,7 @@ int rcutree_prepare_cpu(unsigned int cpu)
rdp->qlen_last_fqs_check = 0;
rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);
rdp->blimit = blimit;
- rdp->dynticks_nesting = 1; /* CPU not up, no tearing. */
+ ct->dynticks_nesting = 1; /* CPU not up, no tearing. */
raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
/*
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 2e46ee108d13..642b11cd8027 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -191,7 +191,6 @@ struct rcu_data {
/* 3) dynticks interface. */
int dynticks_snap; /* Per-GP tracking for dynticks. */
- long dynticks_nesting; /* Track process nesting level. */
long dynticks_nmi_nesting; /* Track irq/NMI nesting level. */
bool rcu_need_heavy_qs; /* GP old, so heavy quiescent state! */
bool rcu_urgent_qs; /* GP old need light quiescent state. */
diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h
index 594a3c5aded1..b5195bbd9ebc 100644
--- a/kernel/rcu/tree_stall.h
+++ b/kernel/rcu/tree_stall.h
@@ -479,7 +479,7 @@ static void print_cpu_stall_info(int cpu)
"!."[!delta],
ticks_value, ticks_title,
rcu_dynticks_snap(cpu) & 0xfff,
- rdp->dynticks_nesting, rdp->dynticks_nmi_nesting,
+ ct_dynticks_nesting_cpu(cpu), rdp->dynticks_nmi_nesting,
rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,
rcuc_starved ? buf : "",
--
2.25.1
next prev parent reply other threads:[~2022-06-08 14:44 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-06-08 14:40 [PATCH 00/20] rcu/context-tracking: Merge RCU eqs-dynticks counter to context tracking v4 Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 01/20] context_tracking: Remove unused context_tracking_in_user() Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 02/20] context_tracking: Add a note about noinstr VS unsafe context tracking functions Frederic Weisbecker
2022-06-09 10:50 ` Peter Zijlstra
2022-06-08 14:40 ` [PATCH 03/20] context_tracking: Rename __context_tracking_enter/exit() to __ct_user_enter/exit() Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 04/20] context_tracking: Rename context_tracking_user_enter/exit() to user_enter/exit_callable() Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 05/20] context_tracking: Rename context_tracking_enter/exit() to ct_user_enter/exit() Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 06/20] context_tracking: Rename context_tracking_cpu_set() to ct_cpu_track_user() Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 07/20] context_tracking: Split user tracking Kconfig Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 08/20] context_tracking: Take idle eqs entrypoints over RCU Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 09/20] context_tracking: Take IRQ " Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 10/20] context_tracking: Take NMI " Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 11/20] rcu/context-tracking: Remove rcu_irq_enter/exit() Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 12/20] rcu/context_tracking: Move dynticks counter to context tracking Frederic Weisbecker
2022-06-08 14:40 ` Frederic Weisbecker [this message]
2022-06-08 14:40 ` [PATCH 14/20] rcu/context_tracking: Move dynticks_nmi_nesting " Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 15/20] rcu/context-tracking: Move deferred nocb resched " Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 16/20] rcu/context-tracking: Move RCU-dynticks internal functions to context_tracking Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 17/20] rcu/context-tracking: Remove unused and/or unecessary middle functions Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 18/20] context_tracking: Convert state to atomic_t Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 19/20] rcu/context_tracking: Merge dynticks counter and context tracking states Frederic Weisbecker
2022-06-08 14:40 ` [PATCH 20/20] MAINTAINERS: Add Paul as context tracking maintainer Frederic Weisbecker
2022-06-20 10:14 ` [PATCH 00/20] rcu/context-tracking: Merge RCU eqs-dynticks counter to context tracking v4 nicolas saenz julienne
2022-06-20 17:16 ` Paul E. McKenney
2022-06-28 13:15 [PATCH 00/20] rcu/context-tracking: Merge RCU eqs-dynticks counter to context tracking v5 Frederic Weisbecker
2022-06-28 13:16 ` [PATCH 13/20] rcu/context_tracking: Move dynticks_nesting to context tracking Frederic Weisbecker
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20220608144037.1765000-14-frederic@kernel.org \
--to=frederic@kernel.org \
--cc=abelits@marvell.com \
--cc=boqun.feng@gmail.com \
--cc=joel@joelfernandes.org \
--cc=liaoyu15@huawei.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mtosatti@redhat.com \
--cc=nsaenz@kernel.org \
--cc=paul.gortmaker@windriver.com \
--cc=pauld@redhat.com \
--cc=paulmck@kernel.org \
--cc=peterz@infradead.org \
--cc=quic_neeraju@quicinc.com \
--cc=tglx@linutronix.de \
--cc=uladzislau.rezki@sony.com \
--cc=wangxiongfeng2@huawei.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).