From: Liu Jian <liujian56@huawei.com>
To: <corbet@lwn.net>, <paulmck@kernel.org>, <frederic@kernel.org>,
<quic_neeraju@quicinc.com>, <joel@joelfernandes.org>,
<josh@joshtriplett.org>, <boqun.feng@gmail.com>,
<rostedt@goodmis.org>, <mathieu.desnoyers@efficios.com>,
<jiangshanlai@gmail.com>, <qiang1.zhang@intel.com>,
<jstultz@google.com>, <tglx@linutronix.de>, <sboyd@kernel.org>,
<davem@davemloft.net>, <edumazet@google.com>, <kuba@kernel.org>,
<pabeni@redhat.com>, <peterz@infradead.org>,
<frankwoo@google.com>, <Rhinewuwu@google.com>
Cc: <liujian56@huawei.com>, <linux-doc@vger.kernel.org>,
<linux-kernel@vger.kernel.org>, <rcu@vger.kernel.org>,
<netdev@vger.kernel.org>
Subject: [PATCH 8/9] softirq,rcu: Use softirq_needs_break()
Date: Fri, 5 May 2023 19:33:14 +0800 [thread overview]
Message-ID: <20230505113315.3307723-9-liujian56@huawei.com> (raw)
In-Reply-To: <20230505113315.3307723-1-liujian56@huawei.com>
From: Peter Zijlstra <peterz@infradead.org>
SoftIRQs provide their own timeout/break code now, use that.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Liu Jian <liujian56@huawei.com>
---
kernel/rcu/tree.c | 29 +++++++----------------------
kernel/rcu/tree_nocb.h | 2 +-
2 files changed, 8 insertions(+), 23 deletions(-)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index f52ff7241041..1942e3db4145 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -397,10 +397,6 @@ static bool rcu_kick_kthreads;
static int rcu_divisor = 7;
module_param(rcu_divisor, int, 0644);
-/* Force an exit from rcu_do_batch() after 3 milliseconds. */
-static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
-module_param(rcu_resched_ns, long, 0644);
-
/*
* How long the grace period must be before we start recruiting
* quiescent-state help from rcu_note_context_switch().
@@ -2050,7 +2046,7 @@ rcu_check_quiescent_state(struct rcu_data *rdp)
* Invoke any RCU callbacks that have made it to the end of their grace
* period. Throttle as specified by rdp->blimit.
*/
-static void rcu_do_batch(struct rcu_data *rdp)
+static void rcu_do_batch(struct softirq_action *h, struct rcu_data *rdp)
{
int div;
bool __maybe_unused empty;
@@ -2058,7 +2054,7 @@ static void rcu_do_batch(struct rcu_data *rdp)
struct rcu_head *rhp;
struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
long bl, count = 0;
- long pending, tlimit = 0;
+ long pending;
/* If no callbacks are ready, just return. */
if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
@@ -2082,12 +2078,6 @@ static void rcu_do_batch(struct rcu_data *rdp)
div = READ_ONCE(rcu_divisor);
div = div < 0 ? 7 : div > sizeof(long) * 8 - 2 ? sizeof(long) * 8 - 2 : div;
bl = max(rdp->blimit, pending >> div);
- if (in_serving_softirq() && unlikely(bl > 100)) {
- long rrn = READ_ONCE(rcu_resched_ns);
-
- rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
- tlimit = local_clock() + rrn;
- }
trace_rcu_batch_start(rcu_state.name,
rcu_segcblist_n_cbs(&rdp->cblist), bl);
rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
@@ -2126,13 +2116,8 @@ static void rcu_do_batch(struct rcu_data *rdp)
* Make sure we don't spend too much time here and deprive other
* softirq vectors of CPU cycles.
*/
- if (unlikely(tlimit)) {
- /* only call local_clock() every 32 callbacks */
- if (likely((count & 31) || local_clock() < tlimit))
- continue;
- /* Exceeded the time limit, so leave. */
+ if (unlikely(!(count & 31)) && softirq_needs_break(h))
break;
- }
} else {
// In rcuoc context, so no worries about depriving
// other softirq vectors of CPU cycles.
@@ -2320,7 +2305,7 @@ static void strict_work_handler(struct work_struct *work)
}
/* Perform RCU core processing work for the current CPU. */
-static __latent_entropy void rcu_core(void)
+static __latent_entropy void rcu_core(struct softirq_action *h)
{
unsigned long flags;
struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
@@ -2374,7 +2359,7 @@ static __latent_entropy void rcu_core(void)
/* If there are callbacks ready, invoke them. */
if (do_batch && rcu_segcblist_ready_cbs(&rdp->cblist) &&
likely(READ_ONCE(rcu_scheduler_fully_active))) {
- rcu_do_batch(rdp);
+ rcu_do_batch(h, rdp);
/* Re-invoke RCU core processing if there are callbacks remaining. */
if (rcu_segcblist_ready_cbs(&rdp->cblist))
invoke_rcu_core();
@@ -2391,7 +2376,7 @@ static __latent_entropy void rcu_core(void)
static void rcu_core_si(struct softirq_action *h)
{
- rcu_core();
+ rcu_core(h);
}
static void rcu_wake_cond(struct task_struct *t, int status)
@@ -2462,7 +2447,7 @@ static void rcu_cpu_kthread(unsigned int cpu)
*workp = 0;
local_irq_enable();
if (work)
- rcu_core();
+ rcu_core(NULL);
local_bh_enable();
if (*workp == 0) {
trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index f2280616f9d5..44fc907fdb5e 100644
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -951,7 +951,7 @@ static void nocb_cb_wait(struct rcu_data *rdp)
* instances of this callback would execute concurrently.
*/
local_bh_disable();
- rcu_do_batch(rdp);
+ rcu_do_batch(NULL, rdp);
local_bh_enable();
lockdep_assert_irqs_enabled();
rcu_nocb_lock_irqsave(rdp, flags);
--
2.34.1
next prev parent reply other threads:[~2023-05-05 11:24 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-05-05 11:33 [PATCH 0/9] fix softlockup in run_timer_softirq Liu Jian
2023-05-05 11:33 ` [PATCH 1/9] softirq: Rewrite softirq processing loop Liu Jian
2023-05-05 11:33 ` [PATCH 2/9] softirq: Use sched_clock() based timeout Liu Jian
2023-05-08 4:08 ` Jason Xing
2023-05-08 17:51 ` Thomas Gleixner
2023-05-05 11:33 ` [PATCH 3/9] softirq: Factor loop termination condition Liu Jian
2023-05-05 11:33 ` [PATCH 4/9] softirq: Allow early break Liu Jian
2023-05-05 12:10 ` Eric Dumazet
2023-05-08 3:37 ` liujian (CE)
2023-05-05 11:33 ` [PATCH 5/9] softirq: Context aware timeout Liu Jian
2023-05-05 11:33 ` [PATCH 6/9] softirq: Provide a softirq_needs_break() API Liu Jian
2023-05-05 11:33 ` [PATCH 7/9] softirq,net: Use softirq_needs_break() Liu Jian
2023-05-08 6:21 ` Jason Xing
2023-05-05 11:33 ` Liu Jian [this message]
2023-05-05 11:33 ` [PATCH 9/9] softirq, timer: " Liu Jian
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230505113315.3307723-9-liujian56@huawei.com \
--to=liujian56@huawei.com \
--cc=Rhinewuwu@google.com \
--cc=boqun.feng@gmail.com \
--cc=corbet@lwn.net \
--cc=davem@davemloft.net \
--cc=edumazet@google.com \
--cc=frankwoo@google.com \
--cc=frederic@kernel.org \
--cc=jiangshanlai@gmail.com \
--cc=joel@joelfernandes.org \
--cc=josh@joshtriplett.org \
--cc=jstultz@google.com \
--cc=kuba@kernel.org \
--cc=linux-doc@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mathieu.desnoyers@efficios.com \
--cc=netdev@vger.kernel.org \
--cc=pabeni@redhat.com \
--cc=paulmck@kernel.org \
--cc=peterz@infradead.org \
--cc=qiang1.zhang@intel.com \
--cc=quic_neeraju@quicinc.com \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=sboyd@kernel.org \
--cc=tglx@linutronix.de \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).