[PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Byungchul Park]

When passing through irq or NMI contexts, the current code uses
->dynticks_nmi_nesting to detect if it's in the ourmost at the moment.

Here, the thing is that all the related functions, rcu_irq_enter(),
rcu_nmi_enter(), rcu_irq_exit() and rcu_nmi_exit() are carrying out
the check within each under the following call relationship so there
must be redundant conditional branches:

   rcu_irq_enter()
      /* A conditional branch with ->dynticks_nmi_nesting */
      rcu_nmi_enter()
         /* A conditional branch with ->dynticks_nmi_nesting */
      /* A conditional branch with ->dynticks_nmi_nesting */

   rcu_irq_exit()
      /* A conditional branch with ->dynticks_nmi_nesting */
      rcu_nmi_exit()
         /* A conditional branch with ->dynticks_nmi_nesting */
      /* A conditional branch with ->dynticks_nmi_nesting */

   rcu_nmi_enter()
      /* A conditional branch with ->dynticks_nmi_nesting */

   rcu_nmi_exit()
      /* A conditional branch with ->dynticks_nmi_nesting */

In order to make the code more readable and remove redundant
conditional branches, we need to refactor rcu_{nmi,irq}_{enter,exit}()
so they use a common function for each like:

   rcu_irq_enter() inlining rcu_nmi_enter_common(irq)
      /* A conditional branch with ->dynticks_nmi_nesting */

   rcu_irq_exit() inlining rcu_nmi_exit_common(irq)
      /* A conditional branch with ->dynticks_nmi_nesting */

   rcu_nmi_enter() inlining rcu_nmi_enter_common(nmi)
      /* A conditional branch with ->dynticks_nmi_nesting */

   rcu_nmi_exit() inlining rcu_nmi_exit_common(nmi)
      /* A conditional branch with ->dynticks_nmi_nesting */

Suggested-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Signed-off-by: Byungchul Park <byungchul.park@lge.com>
---
 kernel/rcu/tree.c | 59 ++++++++++++++++++++++++++++++++++++-------------------
 1 file changed, 39 insertions(+), 20 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index deb2508..cdae85f 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -764,17 +764,17 @@ void rcu_user_enter(void)
 #endif /* CONFIG_NO_HZ_FULL */
 
 /**
- * rcu_nmi_exit - inform RCU of exit from NMI context
+ * rcu_nmi_exit_common - inform RCU of exit from NMI context
  *
  * If we are returning from the outermost NMI handler that interrupted an
  * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
  * to let the RCU grace-period handling know that the CPU is back to
  * being RCU-idle.
  *
- * If you add or remove a call to rcu_nmi_exit(), be sure to test
+ * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
  * with CONFIG_RCU_EQS_DEBUG=y.
  */
-void rcu_nmi_exit(void)
+static __always_inline void rcu_nmi_exit_common(bool irq)
 {
 	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
 
@@ -800,7 +800,22 @@ void rcu_nmi_exit(void)
 	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
 	trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
 	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
+
+	if (irq)
+		rcu_prepare_for_idle();
+
 	rcu_dynticks_eqs_enter();
+
+	if (irq)
+		rcu_dynticks_task_enter();
+}
+
+/**
+ * rcu_nmi_exit - inform RCU of exit from NMI context
+ */
+void rcu_nmi_exit(void)
+{
+	rcu_nmi_exit_common(false);
 }
 
 /**
@@ -824,14 +839,8 @@ void rcu_nmi_exit(void)
  */
 void rcu_irq_exit(void)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-
 	lockdep_assert_irqs_disabled();
-	if (rdtp->dynticks_nmi_nesting == 1)
-		rcu_prepare_for_idle();
-	rcu_nmi_exit();
-	if (rdtp->dynticks_nmi_nesting == 0)
-		rcu_dynticks_task_enter();
+	rcu_nmi_exit_common(true);
 }
 
 /*
@@ -914,7 +923,7 @@ void rcu_user_exit(void)
 #endif /* CONFIG_NO_HZ_FULL */
 
 /**
- * rcu_nmi_enter - inform RCU of entry to NMI context
+ * rcu_nmi_enter_common - inform RCU of entry to NMI context
  *
  * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
  * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
@@ -922,10 +931,10 @@ void rcu_user_exit(void)
  * long as the nesting level does not overflow an int.  (You will probably
  * run out of stack space first.)
  *
- * If you add or remove a call to rcu_nmi_enter(), be sure to test
+ * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
  * with CONFIG_RCU_EQS_DEBUG=y.
  */
-void rcu_nmi_enter(void)
+static __always_inline void rcu_nmi_enter_common(bool irq)
 {
 	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
 	long incby = 2;
@@ -942,7 +951,15 @@ void rcu_nmi_enter(void)
 	 * period (observation due to Andy Lutomirski).
 	 */
 	if (rcu_dynticks_curr_cpu_in_eqs()) {
+
+		if (irq)
+			rcu_dynticks_task_exit();
+
 		rcu_dynticks_eqs_exit();
+
+		if (irq)
+			rcu_cleanup_after_idle();
+
 		incby = 1;
 	}
 	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
@@ -954,6 +971,14 @@ void rcu_nmi_enter(void)
 }
 
 /**
+ * rcu_nmi_enter - inform RCU of entry to NMI context
+ */
+void rcu_nmi_enter(void)
+{
+	rcu_nmi_enter_common(false);
+}
+
+/**
  * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
  *
  * Enter an interrupt handler, which might possibly result in exiting
@@ -977,14 +1002,8 @@ void rcu_nmi_enter(void)
  */
 void rcu_irq_enter(void)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-
 	lockdep_assert_irqs_disabled();
-	if (rdtp->dynticks_nmi_nesting == 0)
-		rcu_dynticks_task_exit();
-	rcu_nmi_enter();
-	if (rdtp->dynticks_nmi_nesting == 1)
-		rcu_cleanup_after_idle();
+	rcu_nmi_enter_common(true);
 }
 
 /*
-- 
1.9.1

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Byungchul Park]

On Fri, Jun 22, 2018 at 03:12:06PM +0900, Byungchul Park wrote:
> When passing through irq or NMI contexts, the current code uses
> ->dynticks_nmi_nesting to detect if it's in the ourmost at the moment.
> 
> Here, the thing is that all the related functions, rcu_irq_enter(),
> rcu_nmi_enter(), rcu_irq_exit() and rcu_nmi_exit() are carrying out
> the check within each under the following call relationship so there
> must be redundant conditional branches:
> 
>    rcu_irq_enter()
>       /* A conditional branch with ->dynticks_nmi_nesting */
>       rcu_nmi_enter()
>          /* A conditional branch with ->dynticks_nmi_nesting */
                                               ^
                                  Precisely, ->dynticks
>       /* A conditional branch with ->dynticks_nmi_nesting */
> 
>    rcu_irq_exit()
>       /* A conditional branch with ->dynticks_nmi_nesting */
>       rcu_nmi_exit()
>          /* A conditional branch with ->dynticks_nmi_nesting */
>       /* A conditional branch with ->dynticks_nmi_nesting */
> 
>    rcu_nmi_enter()
>       /* A conditional branch with ->dynticks_nmi_nesting */
                                               ^
                                  Precisely, ->dynticks
> 
>    rcu_nmi_exit()
>       /* A conditional branch with ->dynticks_nmi_nesting */
> 
> In order to make the code more readable and remove redundant
> conditional branches, we need to refactor rcu_{nmi,irq}_{enter,exit}()
> so they use a common function for each like:
> 
>    rcu_irq_enter() inlining rcu_nmi_enter_common(irq)
>       /* A conditional branch with ->dynticks_nmi_nesting */
                                               ^
                                  Precisely, ->dynticks
> 
>    rcu_irq_exit() inlining rcu_nmi_exit_common(irq)
>       /* A conditional branch with ->dynticks_nmi_nesting */
> 
>    rcu_nmi_enter() inlining rcu_nmi_enter_common(nmi)
>       /* A conditional branch with ->dynticks_nmi_nesting */
                                               ^
                                  Precisely, ->dynticks
> 
>    rcu_nmi_exit() inlining rcu_nmi_exit_common(nmi)
>       /* A conditional branch with ->dynticks_nmi_nesting */

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Paul E. McKenney]

On Fri, Jun 22, 2018 at 03:23:51PM +0900, Byungchul Park wrote:
> On Fri, Jun 22, 2018 at 03:12:06PM +0900, Byungchul Park wrote:
> > When passing through irq or NMI contexts, the current code uses
> > ->dynticks_nmi_nesting to detect if it's in the ourmost at the moment.
> > 
> > Here, the thing is that all the related functions, rcu_irq_enter(),
> > rcu_nmi_enter(), rcu_irq_exit() and rcu_nmi_exit() are carrying out
> > the check within each under the following call relationship so there
> > must be redundant conditional branches:
> > 
> >    rcu_irq_enter()
> >       /* A conditional branch with ->dynticks_nmi_nesting */
> >       rcu_nmi_enter()
> >          /* A conditional branch with ->dynticks_nmi_nesting */
>                                                ^
>                                   Precisely, ->dynticks
> >       /* A conditional branch with ->dynticks_nmi_nesting */
> > 
> >    rcu_irq_exit()
> >       /* A conditional branch with ->dynticks_nmi_nesting */
> >       rcu_nmi_exit()
> >          /* A conditional branch with ->dynticks_nmi_nesting */
> >       /* A conditional branch with ->dynticks_nmi_nesting */
> > 
> >    rcu_nmi_enter()
> >       /* A conditional branch with ->dynticks_nmi_nesting */
>                                                ^
>                                   Precisely, ->dynticks
> > 
> >    rcu_nmi_exit()
> >       /* A conditional branch with ->dynticks_nmi_nesting */
> > 
> > In order to make the code more readable and remove redundant
> > conditional branches, we need to refactor rcu_{nmi,irq}_{enter,exit}()
> > so they use a common function for each like:
> > 
> >    rcu_irq_enter() inlining rcu_nmi_enter_common(irq)
> >       /* A conditional branch with ->dynticks_nmi_nesting */
>                                                ^
>                                   Precisely, ->dynticks
> > 
> >    rcu_irq_exit() inlining rcu_nmi_exit_common(irq)
> >       /* A conditional branch with ->dynticks_nmi_nesting */
> > 
> >    rcu_nmi_enter() inlining rcu_nmi_enter_common(nmi)
> >       /* A conditional branch with ->dynticks_nmi_nesting */
>                                                ^
>                                   Precisely, ->dynticks
> > 
> >    rcu_nmi_exit() inlining rcu_nmi_exit_common(nmi)
> >       /* A conditional branch with ->dynticks_nmi_nesting */

I queued this for testing and further review, updating the commit log
as follows.  Please let me know if I messed up something.

							Thanx, Paul

------------------------------------------------------------------------

commit 5e5ea52645b197fb7ae2f59f7927079b91e91aa0
Author: Byungchul Park <byungchul.park@lge.com>
Date:   Fri Jun 22 15:12:06 2018 +0900

    rcu: Refactor rcu_{nmi,irq}_{enter,exit}()
    
    When entering or exiting irq or NMI handlers, the current code uses
    ->dynticks_nmi_nesting to detect if it is in the outermost handler,
    that is, the one interrupting or returning to an RCU-idle context (the
    idle loop or nohz_full usermode execution).  When entering the outermost
    handler via an interrupt (as opposed to NMI), it is necessary to invoke
    rcu_dynticks_task_exit() just before the CPU is marked non-idle from an
    RCU perspective and to invoke rcu_cleanup_after_idle() just after the
    CPU is marked non-idle.  Similarly, when exiting the outermost handler
    via an interrupt, it is necessary to invoke rcu_prepare_for_idle() just
    before marking the CPU idle and to invoke rcu_dynticks_task_enter()
    just after marking the CPU idle.
    
    The decision to execute these four functions is currently taken in
    rcu_irq_enter() and rcu_irq_exit() as follows:
    
       rcu_irq_enter()
          /* A conditional branch with ->dynticks_nmi_nesting */
          rcu_nmi_enter()
             /* A conditional branch with ->dynticks */
          /* A conditional branch with ->dynticks_nmi_nesting */
    
       rcu_irq_exit()
          /* A conditional branch with ->dynticks_nmi_nesting */
          rcu_nmi_exit()
             /* A conditional branch with ->dynticks_nmi_nesting */
          /* A conditional branch with ->dynticks_nmi_nesting */
    
       rcu_nmi_enter()
          /* A conditional branch with ->dynticks */
    
       rcu_nmi_exit()
          /* A conditional branch with ->dynticks_nmi_nesting */
    
    This works, but the conditional branches in rcu_irq_enter() and
    rcu_irq_exit() are redundant with those in rcu_nmi_enter() and
    rcu_nmi_exit(), respectively.  Redundant branches are not something
    we want in the to/from-idle fastpaths, so this commit refactors
    rcu_{nmi,irq}_{enter,exit}() so they use a common inlined function passed
    a constant argument as follows:
    
       rcu_irq_enter() inlining rcu_nmi_enter_common(irq=true)
          /* A conditional branch with ->dynticks */
    
       rcu_irq_exit() inlining rcu_nmi_exit_common(irq=true)
          /* A conditional branch with ->dynticks_nmi_nesting */
    
       rcu_nmi_enter() inlining rcu_nmi_enter_common(irq=false)
          /* A conditional branch with ->dynticks */
    
       rcu_nmi_exit() inlining rcu_nmi_exit_common(irq=false)
          /* A conditional branch with ->dynticks_nmi_nesting */
    
    The combination of the constant function argument and the inlining allows
    the compiler to discard the conditionals that previously controlled
    execution of rcu_dynticks_task_exit(), rcu_cleanup_after_idle(),
    rcu_prepare_for_idle(), and rcu_dynticks_task_enter().  This reduces both
    the to-idle and from-idle path lengths by two conditional branches each,
    and improves readability as well.
    
    Suggested-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
    Signed-off-by: Byungchul Park <byungchul.park@lge.com>
    Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 8788ddbc0d13..4ed74f10c852 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -773,17 +773,17 @@ void rcu_user_enter(void)
 #endif /* CONFIG_NO_HZ_FULL */
 
 /**
- * rcu_nmi_exit - inform RCU of exit from NMI context
+ * rcu_nmi_exit_common - inform RCU of exit from NMI context
  *
  * If we are returning from the outermost NMI handler that interrupted an
  * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
  * to let the RCU grace-period handling know that the CPU is back to
  * being RCU-idle.
  *
- * If you add or remove a call to rcu_nmi_exit(), be sure to test
+ * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
  * with CONFIG_RCU_EQS_DEBUG=y.
  */
-void rcu_nmi_exit(void)
+static __always_inline void rcu_nmi_exit_common(bool irq)
 {
 	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
 
@@ -809,7 +809,22 @@ void rcu_nmi_exit(void)
 	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
 	trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
 	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
+
+	if (irq)
+		rcu_prepare_for_idle();
+
 	rcu_dynticks_eqs_enter();
+
+	if (irq)
+		rcu_dynticks_task_enter();
+}
+
+/**
+ * rcu_nmi_exit - inform RCU of exit from NMI context
+ */
+void rcu_nmi_exit(void)
+{
+	rcu_nmi_exit_common(false);
 }
 
 /**
@@ -833,14 +848,8 @@ void rcu_nmi_exit(void)
  */
 void rcu_irq_exit(void)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-
 	lockdep_assert_irqs_disabled();
-	if (rdtp->dynticks_nmi_nesting == 1)
-		rcu_prepare_for_idle();
-	rcu_nmi_exit();
-	if (rdtp->dynticks_nmi_nesting == 0)
-		rcu_dynticks_task_enter();
+	rcu_nmi_exit_common(true);
 }
 
 /*
@@ -923,7 +932,7 @@ void rcu_user_exit(void)
 #endif /* CONFIG_NO_HZ_FULL */
 
 /**
- * rcu_nmi_enter - inform RCU of entry to NMI context
+ * rcu_nmi_enter_common - inform RCU of entry to NMI context
  *
  * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
  * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
@@ -931,10 +940,10 @@ void rcu_user_exit(void)
  * long as the nesting level does not overflow an int.  (You will probably
  * run out of stack space first.)
  *
- * If you add or remove a call to rcu_nmi_enter(), be sure to test
+ * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
  * with CONFIG_RCU_EQS_DEBUG=y.
  */
-void rcu_nmi_enter(void)
+static __always_inline void rcu_nmi_enter_common(bool irq)
 {
 	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
 	long incby = 2;
@@ -951,7 +960,15 @@ void rcu_nmi_enter(void)
 	 * period (observation due to Andy Lutomirski).
 	 */
 	if (rcu_dynticks_curr_cpu_in_eqs()) {
+
+		if (irq)
+			rcu_dynticks_task_exit();
+
 		rcu_dynticks_eqs_exit();
+
+		if (irq)
+			rcu_cleanup_after_idle();
+
 		incby = 1;
 	}
 	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
@@ -962,6 +979,14 @@ void rcu_nmi_enter(void)
 	barrier();
 }
 
+/**
+ * rcu_nmi_enter - inform RCU of entry to NMI context
+ */
+void rcu_nmi_enter(void)
+{
+	rcu_nmi_enter_common(false);
+}
+
 /**
  * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
  *
@@ -986,14 +1011,8 @@ void rcu_nmi_enter(void)
  */
 void rcu_irq_enter(void)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-
 	lockdep_assert_irqs_disabled();
-	if (rdtp->dynticks_nmi_nesting == 0)
-		rcu_dynticks_task_exit();
-	rcu_nmi_enter();
-	if (rdtp->dynticks_nmi_nesting == 1)
-		rcu_cleanup_after_idle();
+	rcu_nmi_enter_common(true);
 }
 
 /*

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Byungchul Park]

On Sat, Jun 23, 2018 at 10:49:54AM -0700, Paul E. McKenney wrote:
> On Fri, Jun 22, 2018 at 03:23:51PM +0900, Byungchul Park wrote:
> > On Fri, Jun 22, 2018 at 03:12:06PM +0900, Byungchul Park wrote:
> > > When passing through irq or NMI contexts, the current code uses
> > > ->dynticks_nmi_nesting to detect if it's in the ourmost at the moment.
> > > 
> > > Here, the thing is that all the related functions, rcu_irq_enter(),
> > > rcu_nmi_enter(), rcu_irq_exit() and rcu_nmi_exit() are carrying out
> > > the check within each under the following call relationship so there
> > > must be redundant conditional branches:
> > > 
> > >    rcu_irq_enter()
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> > >       rcu_nmi_enter()
> > >          /* A conditional branch with ->dynticks_nmi_nesting */
> >                                                ^
> >                                   Precisely, ->dynticks
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> > > 
> > >    rcu_irq_exit()
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> > >       rcu_nmi_exit()
> > >          /* A conditional branch with ->dynticks_nmi_nesting */
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> > > 
> > >    rcu_nmi_enter()
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> >                                                ^
> >                                   Precisely, ->dynticks
> > > 
> > >    rcu_nmi_exit()
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> > > 
> > > In order to make the code more readable and remove redundant
> > > conditional branches, we need to refactor rcu_{nmi,irq}_{enter,exit}()
> > > so they use a common function for each like:
> > > 
> > >    rcu_irq_enter() inlining rcu_nmi_enter_common(irq)
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> >                                                ^
> >                                   Precisely, ->dynticks
> > > 
> > >    rcu_irq_exit() inlining rcu_nmi_exit_common(irq)
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> > > 
> > >    rcu_nmi_enter() inlining rcu_nmi_enter_common(nmi)
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> >                                                ^
> >                                   Precisely, ->dynticks
> > > 
> > >    rcu_nmi_exit() inlining rcu_nmi_exit_common(nmi)
> > >       /* A conditional branch with ->dynticks_nmi_nesting */
> 
> I queued this for testing and further review, updating the commit log
> as follows.  Please let me know if I messed up something.

Much better. Thanks a lot.

> 							Thanx, Paul
> 
> ------------------------------------------------------------------------
> 
> commit 5e5ea52645b197fb7ae2f59f7927079b91e91aa0
> Author: Byungchul Park <byungchul.park@lge.com>
> Date:   Fri Jun 22 15:12:06 2018 +0900
> 
>     rcu: Refactor rcu_{nmi,irq}_{enter,exit}()
>     
>     When entering or exiting irq or NMI handlers, the current code uses
>     ->dynticks_nmi_nesting to detect if it is in the outermost handler,
>     that is, the one interrupting or returning to an RCU-idle context (the
>     idle loop or nohz_full usermode execution).  When entering the outermost
>     handler via an interrupt (as opposed to NMI), it is necessary to invoke
>     rcu_dynticks_task_exit() just before the CPU is marked non-idle from an
>     RCU perspective and to invoke rcu_cleanup_after_idle() just after the
>     CPU is marked non-idle.  Similarly, when exiting the outermost handler
>     via an interrupt, it is necessary to invoke rcu_prepare_for_idle() just
>     before marking the CPU idle and to invoke rcu_dynticks_task_enter()
>     just after marking the CPU idle.
>     
>     The decision to execute these four functions is currently taken in
>     rcu_irq_enter() and rcu_irq_exit() as follows:
>     
>        rcu_irq_enter()
>           /* A conditional branch with ->dynticks_nmi_nesting */
>           rcu_nmi_enter()
>              /* A conditional branch with ->dynticks */
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>        rcu_irq_exit()
>           /* A conditional branch with ->dynticks_nmi_nesting */
>           rcu_nmi_exit()
>              /* A conditional branch with ->dynticks_nmi_nesting */
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>        rcu_nmi_enter()
>           /* A conditional branch with ->dynticks */
>     
>        rcu_nmi_exit()
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>     This works, but the conditional branches in rcu_irq_enter() and
>     rcu_irq_exit() are redundant with those in rcu_nmi_enter() and
>     rcu_nmi_exit(), respectively.  Redundant branches are not something
>     we want in the to/from-idle fastpaths, so this commit refactors
>     rcu_{nmi,irq}_{enter,exit}() so they use a common inlined function passed
>     a constant argument as follows:
>     
>        rcu_irq_enter() inlining rcu_nmi_enter_common(irq=true)
>           /* A conditional branch with ->dynticks */
>     
>        rcu_irq_exit() inlining rcu_nmi_exit_common(irq=true)
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>        rcu_nmi_enter() inlining rcu_nmi_enter_common(irq=false)
>           /* A conditional branch with ->dynticks */
>     
>        rcu_nmi_exit() inlining rcu_nmi_exit_common(irq=false)
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>     The combination of the constant function argument and the inlining allows
>     the compiler to discard the conditionals that previously controlled
>     execution of rcu_dynticks_task_exit(), rcu_cleanup_after_idle(),
>     rcu_prepare_for_idle(), and rcu_dynticks_task_enter().  This reduces both
>     the to-idle and from-idle path lengths by two conditional branches each,
>     and improves readability as well.
>     
>     Suggested-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
>     Signed-off-by: Byungchul Park <byungchul.park@lge.com>
>     Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index 8788ddbc0d13..4ed74f10c852 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -773,17 +773,17 @@ void rcu_user_enter(void)
>  #endif /* CONFIG_NO_HZ_FULL */
>  
>  /**
> - * rcu_nmi_exit - inform RCU of exit from NMI context
> + * rcu_nmi_exit_common - inform RCU of exit from NMI context
>   *
>   * If we are returning from the outermost NMI handler that interrupted an
>   * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
>   * to let the RCU grace-period handling know that the CPU is back to
>   * being RCU-idle.
>   *
> - * If you add or remove a call to rcu_nmi_exit(), be sure to test
> + * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
>   * with CONFIG_RCU_EQS_DEBUG=y.
>   */
> -void rcu_nmi_exit(void)
> +static __always_inline void rcu_nmi_exit_common(bool irq)
>  {
>  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
>  
> @@ -809,7 +809,22 @@ void rcu_nmi_exit(void)
>  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
>  	trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
>  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
> +
> +	if (irq)
> +		rcu_prepare_for_idle();
> +
>  	rcu_dynticks_eqs_enter();
> +
> +	if (irq)
> +		rcu_dynticks_task_enter();
> +}
> +
> +/**
> + * rcu_nmi_exit - inform RCU of exit from NMI context
> + */
> +void rcu_nmi_exit(void)
> +{
> +	rcu_nmi_exit_common(false);
>  }
>  
>  /**
> @@ -833,14 +848,8 @@ void rcu_nmi_exit(void)
>   */
>  void rcu_irq_exit(void)
>  {
> -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> -
>  	lockdep_assert_irqs_disabled();
> -	if (rdtp->dynticks_nmi_nesting == 1)
> -		rcu_prepare_for_idle();
> -	rcu_nmi_exit();
> -	if (rdtp->dynticks_nmi_nesting == 0)
> -		rcu_dynticks_task_enter();
> +	rcu_nmi_exit_common(true);
>  }
>  
>  /*
> @@ -923,7 +932,7 @@ void rcu_user_exit(void)
>  #endif /* CONFIG_NO_HZ_FULL */
>  
>  /**
> - * rcu_nmi_enter - inform RCU of entry to NMI context
> + * rcu_nmi_enter_common - inform RCU of entry to NMI context
>   *
>   * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
>   * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
> @@ -931,10 +940,10 @@ void rcu_user_exit(void)
>   * long as the nesting level does not overflow an int.  (You will probably
>   * run out of stack space first.)
>   *
> - * If you add or remove a call to rcu_nmi_enter(), be sure to test
> + * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
>   * with CONFIG_RCU_EQS_DEBUG=y.
>   */
> -void rcu_nmi_enter(void)
> +static __always_inline void rcu_nmi_enter_common(bool irq)
>  {
>  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
>  	long incby = 2;
> @@ -951,7 +960,15 @@ void rcu_nmi_enter(void)
>  	 * period (observation due to Andy Lutomirski).
>  	 */
>  	if (rcu_dynticks_curr_cpu_in_eqs()) {
> +
> +		if (irq)
> +			rcu_dynticks_task_exit();
> +
>  		rcu_dynticks_eqs_exit();
> +
> +		if (irq)
> +			rcu_cleanup_after_idle();
> +
>  		incby = 1;
>  	}
>  	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
> @@ -962,6 +979,14 @@ void rcu_nmi_enter(void)
>  	barrier();
>  }
>  
> +/**
> + * rcu_nmi_enter - inform RCU of entry to NMI context
> + */
> +void rcu_nmi_enter(void)
> +{
> +	rcu_nmi_enter_common(false);
> +}
> +
>  /**
>   * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
>   *
> @@ -986,14 +1011,8 @@ void rcu_nmi_enter(void)
>   */
>  void rcu_irq_enter(void)
>  {
> -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> -
>  	lockdep_assert_irqs_disabled();
> -	if (rdtp->dynticks_nmi_nesting == 0)
> -		rcu_dynticks_task_exit();
> -	rcu_nmi_enter();
> -	if (rdtp->dynticks_nmi_nesting == 1)
> -		rcu_cleanup_after_idle();
> +	rcu_nmi_enter_common(true);
>  }
>  
>  /*

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Steven Rostedt]

On Sat, 23 Jun 2018 10:49:54 -0700
"Paul E. McKenney" <paulmck@linux.vnet.ibm.com> wrote:

> commit 5e5ea52645b197fb7ae2f59f7927079b91e91aa0
> Author: Byungchul Park <byungchul.park@lge.com>
> Date:   Fri Jun 22 15:12:06 2018 +0900
> 
>     rcu: Refactor rcu_{nmi,irq}_{enter,exit}()
>     
>     When entering or exiting irq or NMI handlers, the current code uses
>     ->dynticks_nmi_nesting to detect if it is in the outermost handler,  
>     that is, the one interrupting or returning to an RCU-idle context (the
>     idle loop or nohz_full usermode execution).  When entering the outermost
>     handler via an interrupt (as opposed to NMI), it is necessary to invoke
>     rcu_dynticks_task_exit() just before the CPU is marked non-idle from an
>     RCU perspective and to invoke rcu_cleanup_after_idle() just after the
>     CPU is marked non-idle.  Similarly, when exiting the outermost handler
>     via an interrupt, it is necessary to invoke rcu_prepare_for_idle() just
>     before marking the CPU idle and to invoke rcu_dynticks_task_enter()
>     just after marking the CPU idle.
>     
>     The decision to execute these four functions is currently taken in
>     rcu_irq_enter() and rcu_irq_exit() as follows:
>     
>        rcu_irq_enter()
>           /* A conditional branch with ->dynticks_nmi_nesting */
>           rcu_nmi_enter()
>              /* A conditional branch with ->dynticks */
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>        rcu_irq_exit()
>           /* A conditional branch with ->dynticks_nmi_nesting */
>           rcu_nmi_exit()
>              /* A conditional branch with ->dynticks_nmi_nesting */
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>        rcu_nmi_enter()
>           /* A conditional branch with ->dynticks */
>     
>        rcu_nmi_exit()
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>     This works, but the conditional branches in rcu_irq_enter() and
>     rcu_irq_exit() are redundant with those in rcu_nmi_enter() and
>     rcu_nmi_exit(), respectively.  Redundant branches are not something
>     we want in the to/from-idle fastpaths, so this commit refactors
>     rcu_{nmi,irq}_{enter,exit}() so they use a common inlined function passed
>     a constant argument as follows:
>     
>        rcu_irq_enter() inlining rcu_nmi_enter_common(irq=true)
>           /* A conditional branch with ->dynticks */
>     
>        rcu_irq_exit() inlining rcu_nmi_exit_common(irq=true)
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>        rcu_nmi_enter() inlining rcu_nmi_enter_common(irq=false)
>           /* A conditional branch with ->dynticks */
>     
>        rcu_nmi_exit() inlining rcu_nmi_exit_common(irq=false)
>           /* A conditional branch with ->dynticks_nmi_nesting */
>     
>     The combination of the constant function argument and the inlining allows
>     the compiler to discard the conditionals that previously controlled
>     execution of rcu_dynticks_task_exit(), rcu_cleanup_after_idle(),
>     rcu_prepare_for_idle(), and rcu_dynticks_task_enter().  This reduces both
>     the to-idle and from-idle path lengths by two conditional branches each,
>     and improves readability as well.

Nice write up. Makes a lot of sense.

>     
>     Suggested-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
>     Signed-off-by: Byungchul Park <byungchul.park@lge.com>
>     Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index 8788ddbc0d13..4ed74f10c852 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -773,17 +773,17 @@ void rcu_user_enter(void)
>  #endif /* CONFIG_NO_HZ_FULL */
>  
>  /**
> - * rcu_nmi_exit - inform RCU of exit from NMI context
> + * rcu_nmi_exit_common - inform RCU of exit from NMI context
>   *
>   * If we are returning from the outermost NMI handler that interrupted an
>   * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
>   * to let the RCU grace-period handling know that the CPU is back to
>   * being RCU-idle.
>   *
> - * If you add or remove a call to rcu_nmi_exit(), be sure to test
> + * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
>   * with CONFIG_RCU_EQS_DEBUG=y.
>   */
> -void rcu_nmi_exit(void)
> +static __always_inline void rcu_nmi_exit_common(bool irq)
>  {
>  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
>  
> @@ -809,7 +809,22 @@ void rcu_nmi_exit(void)
>  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
>  	trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
>  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
> +
> +	if (irq)
> +		rcu_prepare_for_idle();
> +
>  	rcu_dynticks_eqs_enter();
> +
> +	if (irq)
> +		rcu_dynticks_task_enter();
> +}
> +
> +/**
> + * rcu_nmi_exit - inform RCU of exit from NMI context
> + */
> +void rcu_nmi_exit(void)
> +{
> +	rcu_nmi_exit_common(false);
>  }
>  
>  /**
> @@ -833,14 +848,8 @@ void rcu_nmi_exit(void)
>   */
>  void rcu_irq_exit(void)
>  {
> -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> -
>  	lockdep_assert_irqs_disabled();
> -	if (rdtp->dynticks_nmi_nesting == 1)
> -		rcu_prepare_for_idle();
> -	rcu_nmi_exit();
> -	if (rdtp->dynticks_nmi_nesting == 0)
> -		rcu_dynticks_task_enter();
> +	rcu_nmi_exit_common(true);
>  }
>  
>  /*
> @@ -923,7 +932,7 @@ void rcu_user_exit(void)
>  #endif /* CONFIG_NO_HZ_FULL */
>  
>  /**
> - * rcu_nmi_enter - inform RCU of entry to NMI context
> + * rcu_nmi_enter_common - inform RCU of entry to NMI context
>   *
>   * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
>   * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
> @@ -931,10 +940,10 @@ void rcu_user_exit(void)
>   * long as the nesting level does not overflow an int.  (You will probably
>   * run out of stack space first.)
>   *
> - * If you add or remove a call to rcu_nmi_enter(), be sure to test
> + * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
>   * with CONFIG_RCU_EQS_DEBUG=y.
>   */
> -void rcu_nmi_enter(void)
> +static __always_inline void rcu_nmi_enter_common(bool irq)
>  {
>  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
>  	long incby = 2;
> @@ -951,7 +960,15 @@ void rcu_nmi_enter(void)
>  	 * period (observation due to Andy Lutomirski).
>  	 */
>  	if (rcu_dynticks_curr_cpu_in_eqs()) {
> +
> +		if (irq)
> +			rcu_dynticks_task_exit();
> +
>  		rcu_dynticks_eqs_exit();
> +
> +		if (irq)
> +			rcu_cleanup_after_idle();
> +
>  		incby = 1;
>  	}
>  	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),


There is a slight change here, although I don't think it is an issue,
but I want to bring it up just in case.

The old way had:

	rcu_dynticks_task_exit();
	rcu_dynticks_eqs_exit();
	trace_rcu_dyntick();
	rcu_cleanup_after_idle();

The new way has:

	rcu_dynticks_task_exit();
	rcu_dynticks_eqs_exit();
	rcu_cleanup_after_idle();
	trace_rcu_dyntick();

As that tracepoint will use RCU, will this cause any side effects?

My thought is that the new way is actually more correct, as I'm not
sure we wanted RCU usage before the rcu_cleanup_after_idle().

-- Steve


> @@ -962,6 +979,14 @@ void rcu_nmi_enter(void)
>  	barrier();
>  }
>  
> +/**
> + * rcu_nmi_enter - inform RCU of entry to NMI context
> + */
> +void rcu_nmi_enter(void)
> +{
> +	rcu_nmi_enter_common(false);
> +}
> +
>  /**
>   * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
>   *
> @@ -986,14 +1011,8 @@ void rcu_nmi_enter(void)
>   */
>  void rcu_irq_enter(void)
>  {
> -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> -
>  	lockdep_assert_irqs_disabled();
> -	if (rdtp->dynticks_nmi_nesting == 0)
> -		rcu_dynticks_task_exit();
> -	rcu_nmi_enter();
> -	if (rdtp->dynticks_nmi_nesting == 1)
> -		rcu_cleanup_after_idle();
> +	rcu_nmi_enter_common(true);
>  }
>  
>  /*

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Paul E. McKenney]

On Mon, Jun 25, 2018 at 10:07:08AM -0400, Steven Rostedt wrote:
> On Sat, 23 Jun 2018 10:49:54 -0700
> "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> wrote:
> 
> > commit 5e5ea52645b197fb7ae2f59f7927079b91e91aa0
> > Author: Byungchul Park <byungchul.park@lge.com>
> > Date:   Fri Jun 22 15:12:06 2018 +0900
> > 
> >     rcu: Refactor rcu_{nmi,irq}_{enter,exit}()
> >     
> >     When entering or exiting irq or NMI handlers, the current code uses
> >     ->dynticks_nmi_nesting to detect if it is in the outermost handler,  
> >     that is, the one interrupting or returning to an RCU-idle context (the
> >     idle loop or nohz_full usermode execution).  When entering the outermost
> >     handler via an interrupt (as opposed to NMI), it is necessary to invoke
> >     rcu_dynticks_task_exit() just before the CPU is marked non-idle from an
> >     RCU perspective and to invoke rcu_cleanup_after_idle() just after the
> >     CPU is marked non-idle.  Similarly, when exiting the outermost handler
> >     via an interrupt, it is necessary to invoke rcu_prepare_for_idle() just
> >     before marking the CPU idle and to invoke rcu_dynticks_task_enter()
> >     just after marking the CPU idle.
> >     
> >     The decision to execute these four functions is currently taken in
> >     rcu_irq_enter() and rcu_irq_exit() as follows:
> >     
> >        rcu_irq_enter()
> >           /* A conditional branch with ->dynticks_nmi_nesting */
> >           rcu_nmi_enter()
> >              /* A conditional branch with ->dynticks */
> >           /* A conditional branch with ->dynticks_nmi_nesting */
> >     
> >        rcu_irq_exit()
> >           /* A conditional branch with ->dynticks_nmi_nesting */
> >           rcu_nmi_exit()
> >              /* A conditional branch with ->dynticks_nmi_nesting */
> >           /* A conditional branch with ->dynticks_nmi_nesting */
> >     
> >        rcu_nmi_enter()
> >           /* A conditional branch with ->dynticks */
> >     
> >        rcu_nmi_exit()
> >           /* A conditional branch with ->dynticks_nmi_nesting */
> >     
> >     This works, but the conditional branches in rcu_irq_enter() and
> >     rcu_irq_exit() are redundant with those in rcu_nmi_enter() and
> >     rcu_nmi_exit(), respectively.  Redundant branches are not something
> >     we want in the to/from-idle fastpaths, so this commit refactors
> >     rcu_{nmi,irq}_{enter,exit}() so they use a common inlined function passed
> >     a constant argument as follows:
> >     
> >        rcu_irq_enter() inlining rcu_nmi_enter_common(irq=true)
> >           /* A conditional branch with ->dynticks */
> >     
> >        rcu_irq_exit() inlining rcu_nmi_exit_common(irq=true)
> >           /* A conditional branch with ->dynticks_nmi_nesting */
> >     
> >        rcu_nmi_enter() inlining rcu_nmi_enter_common(irq=false)
> >           /* A conditional branch with ->dynticks */
> >     
> >        rcu_nmi_exit() inlining rcu_nmi_exit_common(irq=false)
> >           /* A conditional branch with ->dynticks_nmi_nesting */
> >     
> >     The combination of the constant function argument and the inlining allows
> >     the compiler to discard the conditionals that previously controlled
> >     execution of rcu_dynticks_task_exit(), rcu_cleanup_after_idle(),
> >     rcu_prepare_for_idle(), and rcu_dynticks_task_enter().  This reduces both
> >     the to-idle and from-idle path lengths by two conditional branches each,
> >     and improves readability as well.
> 
> Nice write up. Makes a lot of sense.

On behalf of both Byungchul and myself, glad you like it!

> >     Suggested-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> >     Signed-off-by: Byungchul Park <byungchul.park@lge.com>
> >     Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> > 
> > diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> > index 8788ddbc0d13..4ed74f10c852 100644
> > --- a/kernel/rcu/tree.c
> > +++ b/kernel/rcu/tree.c
> > @@ -773,17 +773,17 @@ void rcu_user_enter(void)
> >  #endif /* CONFIG_NO_HZ_FULL */
> >  
> >  /**
> > - * rcu_nmi_exit - inform RCU of exit from NMI context
> > + * rcu_nmi_exit_common - inform RCU of exit from NMI context
> >   *
> >   * If we are returning from the outermost NMI handler that interrupted an
> >   * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
> >   * to let the RCU grace-period handling know that the CPU is back to
> >   * being RCU-idle.
> >   *
> > - * If you add or remove a call to rcu_nmi_exit(), be sure to test
> > + * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
> >   * with CONFIG_RCU_EQS_DEBUG=y.
> >   */
> > -void rcu_nmi_exit(void)
> > +static __always_inline void rcu_nmi_exit_common(bool irq)
> >  {
> >  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> >  
> > @@ -809,7 +809,22 @@ void rcu_nmi_exit(void)
> >  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
> >  	trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
> >  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
> > +
> > +	if (irq)
> > +		rcu_prepare_for_idle();
> > +
> >  	rcu_dynticks_eqs_enter();
> > +
> > +	if (irq)
> > +		rcu_dynticks_task_enter();
> > +}
> > +
> > +/**
> > + * rcu_nmi_exit - inform RCU of exit from NMI context
> > + */
> > +void rcu_nmi_exit(void)
> > +{
> > +	rcu_nmi_exit_common(false);
> >  }
> >  
> >  /**
> > @@ -833,14 +848,8 @@ void rcu_nmi_exit(void)
> >   */
> >  void rcu_irq_exit(void)
> >  {
> > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > -
> >  	lockdep_assert_irqs_disabled();
> > -	if (rdtp->dynticks_nmi_nesting == 1)
> > -		rcu_prepare_for_idle();
> > -	rcu_nmi_exit();
> > -	if (rdtp->dynticks_nmi_nesting == 0)
> > -		rcu_dynticks_task_enter();
> > +	rcu_nmi_exit_common(true);
> >  }
> >  
> >  /*
> > @@ -923,7 +932,7 @@ void rcu_user_exit(void)
> >  #endif /* CONFIG_NO_HZ_FULL */
> >  
> >  /**
> > - * rcu_nmi_enter - inform RCU of entry to NMI context
> > + * rcu_nmi_enter_common - inform RCU of entry to NMI context
> >   *
> >   * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
> >   * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
> > @@ -931,10 +940,10 @@ void rcu_user_exit(void)
> >   * long as the nesting level does not overflow an int.  (You will probably
> >   * run out of stack space first.)
> >   *
> > - * If you add or remove a call to rcu_nmi_enter(), be sure to test
> > + * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
> >   * with CONFIG_RCU_EQS_DEBUG=y.
> >   */
> > -void rcu_nmi_enter(void)
> > +static __always_inline void rcu_nmi_enter_common(bool irq)
> >  {
> >  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> >  	long incby = 2;
> > @@ -951,7 +960,15 @@ void rcu_nmi_enter(void)
> >  	 * period (observation due to Andy Lutomirski).
> >  	 */
> >  	if (rcu_dynticks_curr_cpu_in_eqs()) {
> > +
> > +		if (irq)
> > +			rcu_dynticks_task_exit();
> > +
> >  		rcu_dynticks_eqs_exit();
> > +
> > +		if (irq)
> > +			rcu_cleanup_after_idle();
> > +
> >  		incby = 1;
> >  	}
> >  	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
> 
> 
> There is a slight change here, although I don't think it is an issue,
> but I want to bring it up just in case.
> 
> The old way had:
> 
> 	rcu_dynticks_task_exit();
> 	rcu_dynticks_eqs_exit();
> 	trace_rcu_dyntick();
> 	rcu_cleanup_after_idle();
> 
> The new way has:
> 
> 	rcu_dynticks_task_exit();
> 	rcu_dynticks_eqs_exit();
> 	rcu_cleanup_after_idle();
> 	trace_rcu_dyntick();
> 
> As that tracepoint will use RCU, will this cause any side effects?
> 
> My thought is that the new way is actually more correct, as I'm not
> sure we wanted RCU usage before the rcu_cleanup_after_idle().

I believe that this is OK because is is the position of the call to
rcu_dynticks_eqs_exit() that really matters.  Before this call, RCU
is not yet watching, and after this call it is watching.  Reversing
the calls to rcu_cleanup_after_idle() and trace_rcu_dyntick() has them
both being invoked while RCU is watching.

All that rcu_cleanup_after_idle() does is to account for the time that
passed while the CPU was idle, for example, advancing callbacks to allow
for how ever many RCU grace periods completed during that idle period.

Or am I missing something subtle.

(At the very least, you would be quite right to ask that this be added
to the commit log!)

							Thanx, Paul

> -- Steve
> 
> 
> > @@ -962,6 +979,14 @@ void rcu_nmi_enter(void)
> >  	barrier();
> >  }
> >  
> > +/**
> > + * rcu_nmi_enter - inform RCU of entry to NMI context
> > + */
> > +void rcu_nmi_enter(void)
> > +{
> > +	rcu_nmi_enter_common(false);
> > +}
> > +
> >  /**
> >   * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
> >   *
> > @@ -986,14 +1011,8 @@ void rcu_nmi_enter(void)
> >   */
> >  void rcu_irq_enter(void)
> >  {
> > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > -
> >  	lockdep_assert_irqs_disabled();
> > -	if (rdtp->dynticks_nmi_nesting == 0)
> > -		rcu_dynticks_task_exit();
> > -	rcu_nmi_enter();
> > -	if (rdtp->dynticks_nmi_nesting == 1)
> > -		rcu_cleanup_after_idle();
> > +	rcu_nmi_enter_common(true);
> >  }
> >  
> >  /*
> 

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Steven Rostedt]

On Mon, 25 Jun 2018 07:48:49 -0700
"Paul E. McKenney" <paulmck@linux.vnet.ibm.com> wrote:

> > > @@ -923,7 +932,7 @@ void rcu_user_exit(void)
> > >  #endif /* CONFIG_NO_HZ_FULL */
> > >  
> > >  /**
> > > - * rcu_nmi_enter - inform RCU of entry to NMI context
> > > + * rcu_nmi_enter_common - inform RCU of entry to NMI context
> > >   *
> > >   * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
> > >   * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
> > > @@ -931,10 +940,10 @@ void rcu_user_exit(void)
> > >   * long as the nesting level does not overflow an int.  (You will probably
> > >   * run out of stack space first.)
> > >   *
> > > - * If you add or remove a call to rcu_nmi_enter(), be sure to test
> > > + * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
> > >   * with CONFIG_RCU_EQS_DEBUG=y.
> > >   */
> > > -void rcu_nmi_enter(void)
> > > +static __always_inline void rcu_nmi_enter_common(bool irq)
> > >  {
> > >  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > >  	long incby = 2;
> > > @@ -951,7 +960,15 @@ void rcu_nmi_enter(void)
> > >  	 * period (observation due to Andy Lutomirski).
> > >  	 */
> > >  	if (rcu_dynticks_curr_cpu_in_eqs()) {
> > > +
> > > +		if (irq)
> > > +			rcu_dynticks_task_exit();
> > > +
> > >  		rcu_dynticks_eqs_exit();
> > > +
> > > +		if (irq)
> > > +			rcu_cleanup_after_idle();
> > > +
> > >  		incby = 1;
> > >  	}
> > >  	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),  
> > 
> > 
> > There is a slight change here, although I don't think it is an issue,
> > but I want to bring it up just in case.
> > 
> > The old way had:
> > 
> > 	rcu_dynticks_task_exit();
> > 	rcu_dynticks_eqs_exit();
> > 	trace_rcu_dyntick();
> > 	rcu_cleanup_after_idle();
> > 
> > The new way has:
> > 
> > 	rcu_dynticks_task_exit();
> > 	rcu_dynticks_eqs_exit();
> > 	rcu_cleanup_after_idle();
> > 	trace_rcu_dyntick();
> > 
> > As that tracepoint will use RCU, will this cause any side effects?
> > 
> > My thought is that the new way is actually more correct, as I'm not
> > sure we wanted RCU usage before the rcu_cleanup_after_idle().  
> 
> I believe that this is OK because is is the position of the call to
> rcu_dynticks_eqs_exit() that really matters.  Before this call, RCU
> is not yet watching, and after this call it is watching.  Reversing
> the calls to rcu_cleanup_after_idle() and trace_rcu_dyntick() has them
> both being invoked while RCU is watching.
> 
> All that rcu_cleanup_after_idle() does is to account for the time that
> passed while the CPU was idle, for example, advancing callbacks to allow
> for how ever many RCU grace periods completed during that idle period.
> 
> Or am I missing something subtle.

As I stated above, I actually think the new way is more correct. That's
because the trace event is the first user of RCU here and it probably
wont be the last. It makes more sense to do it after the call to
rcu_cleanup_after_idle(), just because it keeps all the RCU users after
the RCU internal code for coming out of idle. Sure,
rcu_cleanup_after_idle() doesn't do anything now that could affect
this, but maybe it will in the future?

> 
> (At the very least, you would be quite right to ask that this be added
> to the commit log!)
> 

Yes, I agree. There should be a comment in the change log about this
simply because this is technically a functional change.

-- Steve

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Paul E. McKenney]

On Mon, Jun 25, 2018 at 11:02:48AM -0400, Steven Rostedt wrote:
> On Mon, 25 Jun 2018 07:48:49 -0700
> "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> wrote:
> 
> > > > @@ -923,7 +932,7 @@ void rcu_user_exit(void)
> > > >  #endif /* CONFIG_NO_HZ_FULL */
> > > >  
> > > >  /**
> > > > - * rcu_nmi_enter - inform RCU of entry to NMI context
> > > > + * rcu_nmi_enter_common - inform RCU of entry to NMI context
> > > >   *
> > > >   * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
> > > >   * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
> > > > @@ -931,10 +940,10 @@ void rcu_user_exit(void)
> > > >   * long as the nesting level does not overflow an int.  (You will probably
> > > >   * run out of stack space first.)
> > > >   *
> > > > - * If you add or remove a call to rcu_nmi_enter(), be sure to test
> > > > + * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
> > > >   * with CONFIG_RCU_EQS_DEBUG=y.
> > > >   */
> > > > -void rcu_nmi_enter(void)
> > > > +static __always_inline void rcu_nmi_enter_common(bool irq)
> > > >  {
> > > >  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > >  	long incby = 2;
> > > > @@ -951,7 +960,15 @@ void rcu_nmi_enter(void)
> > > >  	 * period (observation due to Andy Lutomirski).
> > > >  	 */
> > > >  	if (rcu_dynticks_curr_cpu_in_eqs()) {
> > > > +
> > > > +		if (irq)
> > > > +			rcu_dynticks_task_exit();
> > > > +
> > > >  		rcu_dynticks_eqs_exit();
> > > > +
> > > > +		if (irq)
> > > > +			rcu_cleanup_after_idle();
> > > > +
> > > >  		incby = 1;
> > > >  	}
> > > >  	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),  
> > > 
> > > 
> > > There is a slight change here, although I don't think it is an issue,
> > > but I want to bring it up just in case.
> > > 
> > > The old way had:
> > > 
> > > 	rcu_dynticks_task_exit();
> > > 	rcu_dynticks_eqs_exit();
> > > 	trace_rcu_dyntick();
> > > 	rcu_cleanup_after_idle();
> > > 
> > > The new way has:
> > > 
> > > 	rcu_dynticks_task_exit();
> > > 	rcu_dynticks_eqs_exit();
> > > 	rcu_cleanup_after_idle();
> > > 	trace_rcu_dyntick();
> > > 
> > > As that tracepoint will use RCU, will this cause any side effects?
> > > 
> > > My thought is that the new way is actually more correct, as I'm not
> > > sure we wanted RCU usage before the rcu_cleanup_after_idle().  
> > 
> > I believe that this is OK because is is the position of the call to
> > rcu_dynticks_eqs_exit() that really matters.  Before this call, RCU
> > is not yet watching, and after this call it is watching.  Reversing
> > the calls to rcu_cleanup_after_idle() and trace_rcu_dyntick() has them
> > both being invoked while RCU is watching.
> > 
> > All that rcu_cleanup_after_idle() does is to account for the time that
> > passed while the CPU was idle, for example, advancing callbacks to allow
> > for how ever many RCU grace periods completed during that idle period.
> > 
> > Or am I missing something subtle.
> 
> As I stated above, I actually think the new way is more correct. That's
> because the trace event is the first user of RCU here and it probably
> wont be the last. It makes more sense to do it after the call to
> rcu_cleanup_after_idle(), just because it keeps all the RCU users after
> the RCU internal code for coming out of idle. Sure,
> rcu_cleanup_after_idle() doesn't do anything now that could affect
> this, but maybe it will in the future?

If rcu_cleanup_after_idle() job changes, then yes, changes might be
needed here and perhaps elsewhere as well.  ;-)

> > (At the very least, you would be quite right to ask that this be added
> > to the commit log!)
> 
> Yes, I agree. There should be a comment in the change log about this
> simply because this is technically a functional change.

Very good, will do!

							Thanx, Paul

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 28835 bytes --]

Hi Byungchul,

Thank you for the patch! Perhaps something to improve:

[auto build test WARNING on rcu/rcu/next]
[also build test WARNING on v4.18-rc1 next-20180622]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Byungchul-Park/rcu-Refactor-rcu_-nmi-irq-_-enter-exit/20180622-141334
base:   https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git rcu/next
reproduce: make htmldocs

All warnings (new ones prefixed by >>):

   WARNING: convert(1) not found, for SVG to PDF conversion install ImageMagick (https://www.imagemagick.org)
>> kernel/rcu/tree.c:778: warning: Function parameter or member 'irq' not described in 'rcu_nmi_exit_common'
>> kernel/rcu/tree.c:938: warning: Function parameter or member 'irq' not described in 'rcu_nmi_enter_common'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.ibss' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.connect' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.keys' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.ie' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.ie_len' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.bssid' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.ssid' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.default_key' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.default_mgmt_key' not described in 'wireless_dev'
   include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.prev_bssid_valid' not described in 'wireless_dev'
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2282: warning: Function parameter or member 'radiotap_timestamp.units_pos' not described in 'ieee80211_hw'
   include/net/mac80211.h:2282: warning: Function parameter or member 'radiotap_timestamp.accuracy' not described in 'ieee80211_hw'
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:955: warning: Function parameter or member 'control.rates' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.rts_cts_rate_idx' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.use_rts' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.use_cts_prot' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.short_preamble' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.skip_table' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.jiffies' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.vif' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.hw_key' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.flags' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'control.enqueue_time' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'ack' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'ack.cookie' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'status.rates' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'status.ack_signal' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'status.ampdu_ack_len' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'status.ampdu_len' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'status.antenna' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'status.tx_time' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'status.is_valid_ack_signal' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'status.status_driver_data' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'driver_rates' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'pad' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:955: warning: Function parameter or member 'rate_driver_data' not described in 'ieee80211_tx_info'
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >
   include/net/mac80211.h:2083: warning: bad line: >

vim +778 kernel/rcu/tree.c

19dd1591f kernel/rcutree.c  Frederic Weisbecker 2012-06-04  765  
19dd1591f kernel/rcutree.c  Frederic Weisbecker 2012-06-04  766  /**
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  767   * rcu_nmi_exit_common - inform RCU of exit from NMI context
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  768   *
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  769   * If we are returning from the outermost NMI handler that interrupted an
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  770   * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  771   * to let the RCU grace-period handling know that the CPU is back to
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  772   * being RCU-idle.
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  773   *
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  774   * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  775   * with CONFIG_RCU_EQS_DEBUG=y.
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  776   */
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  777  static __always_inline void rcu_nmi_exit_common(bool irq)
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02 @778  {
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  779  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  780  
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  781  	/*
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  782  	 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  783  	 * (We are exiting an NMI handler, so RCU better be paying attention
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  784  	 * to us!)
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  785  	 */
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  786  	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  787  	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  788  
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  789  	/*
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  790  	 * If the nesting level is not 1, the CPU wasn't RCU-idle, so
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  791  	 * leave it in non-RCU-idle state.
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  792  	 */
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  793  	if (rdtp->dynticks_nmi_nesting != 1) {
dec98900e kernel/rcu/tree.c Paul E. McKenney    2017-10-04  794  		trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, rdtp->dynticks);
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  795  		WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  796  			   rdtp->dynticks_nmi_nesting - 2);
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  797  		return;
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  798  	}
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  799  
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  800  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
dec98900e kernel/rcu/tree.c Paul E. McKenney    2017-10-04  801  	trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  802  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  803  
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  804  	if (irq)
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  805  		rcu_prepare_for_idle();
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  806  
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  807  	rcu_dynticks_eqs_enter();
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  808  
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  809  	if (irq)
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  810  		rcu_dynticks_task_enter();
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  811  }
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  812  
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  813  /**
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  814   * rcu_nmi_exit - inform RCU of exit from NMI context
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  815   */
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  816  void rcu_nmi_exit(void)
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  817  {
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  818  	rcu_nmi_exit_common(false);
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  819  }
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  820  
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  821  /**
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  822   * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  823   *
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  824   * Exit from an interrupt handler, which might possibly result in entering
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  825   * idle mode, in other words, leaving the mode in which read-side critical
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  826   * sections can occur.  The caller must have disabled interrupts.
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  827   *
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  828   * This code assumes that the idle loop never does anything that might
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  829   * result in unbalanced calls to irq_enter() and irq_exit().  If your
58721f5da kernel/rcu/tree.c Paul E. McKenney    2017-10-03  830   * architecture's idle loop violates this assumption, RCU will give you what
58721f5da kernel/rcu/tree.c Paul E. McKenney    2017-10-03  831   * you deserve, good and hard.  But very infrequently and irreproducibly.
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  832   *
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  833   * Use things like work queues to work around this limitation.
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  834   *
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  835   * You have been warned.
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  836   *
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  837   * If you add or remove a call to rcu_irq_exit(), be sure to test with
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  838   * CONFIG_RCU_EQS_DEBUG=y.
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  839   */
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  840  void rcu_irq_exit(void)
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  841  {
b04db8e19 kernel/rcu/tree.c Frederic Weisbecker 2017-11-06  842  	lockdep_assert_irqs_disabled();
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  843  	rcu_nmi_exit_common(true);
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  844  }
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  845  
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  846  /*
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  847   * Wrapper for rcu_irq_exit() where interrupts are enabled.
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  848   *
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  849   * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  850   * with CONFIG_RCU_EQS_DEBUG=y.
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  851   */
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  852  void rcu_irq_exit_irqson(void)
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  853  {
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  854  	unsigned long flags;
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  855  
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  856  	local_irq_save(flags);
7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  857  	rcu_irq_exit();
23b5c8fa0 kernel/rcutree.c  Paul E. McKenney    2010-09-07  858  	local_irq_restore(flags);
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  859  }
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  860  
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  861  /*
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  862   * Exit an RCU extended quiescent state, which can be either the
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  863   * idle loop or adaptive-tickless usermode execution.
51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  864   *
51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  865   * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to
51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  866   * allow for the possibility of usermode upcalls messing up our count of
51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  867   * interrupt nesting level during the busy period that is just now starting.
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  868   */
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  869  static void rcu_eqs_exit(bool user)
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  870  {
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  871  	struct rcu_dynticks *rdtp;
84585aa8b kernel/rcu/tree.c Paul E. McKenney    2017-10-04  872  	long oldval;
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  873  
b04db8e19 kernel/rcu/tree.c Frederic Weisbecker 2017-11-06  874  	lockdep_assert_irqs_disabled();
c9d4b0af9 kernel/rcutree.c  Christoph Lameter   2013-08-31  875  	rdtp = this_cpu_ptr(&rcu_dynticks);
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  876  	oldval = rdtp->dynticks_nesting;
1ce46ee59 kernel/rcu/tree.c Paul E. McKenney    2015-05-05  877  	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  878  	if (oldval) {
51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  879  		rdtp->dynticks_nesting++;
9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  880  		return;
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  881  	}
9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  882  	rcu_dynticks_task_exit();
9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  883  	rcu_dynticks_eqs_exit();
9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  884  	rcu_cleanup_after_idle();
9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  885  	trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, rdtp->dynticks);
e68bbb266 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  886  	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
2342172fd kernel/rcu/tree.c Paul E. McKenney    2017-10-05  887  	WRITE_ONCE(rdtp->dynticks_nesting, 1);
6136d6e48 kernel/rcu/tree.c Paul E. McKenney    2017-10-03  888  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
3a5924052 kernel/rcu/tree.c Paul E. McKenney    2013-10-04  889  }
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  890  
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  891  /**
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  892   * rcu_idle_exit - inform RCU that current CPU is leaving idle
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  893   *
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  894   * Exit idle mode, in other words, -enter- the mode in which RCU
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  895   * read-side critical sections can occur.
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  896   *
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  897   * If you add or remove a call to rcu_idle_exit(), be sure to test with
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  898   * CONFIG_RCU_EQS_DEBUG=y.
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  899   */
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  900  void rcu_idle_exit(void)
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  901  {
c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  902  	unsigned long flags;
c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  903  
c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  904  	local_irq_save(flags);
cb349ca95 kernel/rcutree.c  Paul E. McKenney    2012-09-04  905  	rcu_eqs_exit(false);
c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  906  	local_irq_restore(flags);
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  907  }
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  908  
d1ec4c34c kernel/rcu/tree.c Paul E. McKenney    2015-05-13  909  #ifdef CONFIG_NO_HZ_FULL
9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  910  /**
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  911   * rcu_user_exit - inform RCU that we are exiting userspace.
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  912   *
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  913   * Exit RCU idle mode while entering the kernel because it can
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  914   * run a RCU read side critical section anytime.
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  915   *
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  916   * If you add or remove a call to rcu_user_exit(), be sure to test with
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  917   * CONFIG_RCU_EQS_DEBUG=y.
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  918   */
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  919  void rcu_user_exit(void)
adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  920  {
91d1aa43d kernel/rcutree.c  Frederic Weisbecker 2012-11-27  921  	rcu_eqs_exit(1);
c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  922  }
d1ec4c34c kernel/rcu/tree.c Paul E. McKenney    2015-05-13  923  #endif /* CONFIG_NO_HZ_FULL */
19dd1591f kernel/rcutree.c  Frederic Weisbecker 2012-06-04  924  
19dd1591f kernel/rcutree.c  Frederic Weisbecker 2012-06-04  925  /**
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  926   * rcu_nmi_enter_common - inform RCU of entry to NMI context
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  927   *
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  928   * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  929   * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  930   * that the CPU is active.  This implementation permits nested NMIs, as
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  931   * long as the nesting level does not overflow an int.  (You will probably
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  932   * run out of stack space first.)
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  933   *
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  934   * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  935   * with CONFIG_RCU_EQS_DEBUG=y.
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  936   */
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  937  static __always_inline void rcu_nmi_enter_common(bool irq)
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18 @938  {
c9d4b0af9 kernel/rcutree.c  Christoph Lameter   2013-08-31  939  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
84585aa8b kernel/rcu/tree.c Paul E. McKenney    2017-10-04  940  	long incby = 2;
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  941  
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  942  	/* Complain about underflow. */
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  943  	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  944  
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  945  	/*
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  946  	 * If idle from RCU viewpoint, atomically increment ->dynticks
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  947  	 * to mark non-idle and increment ->dynticks_nmi_nesting by one.
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  948  	 * Otherwise, increment ->dynticks_nmi_nesting by two.  This means
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  949  	 * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  950  	 * to be in the outermost NMI handler that interrupted an RCU-idle
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  951  	 * period (observation due to Andy Lutomirski).
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  952  	 */
02a5c550b kernel/rcu/tree.c Paul E. McKenney    2016-11-02  953  	if (rcu_dynticks_curr_cpu_in_eqs()) {
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  954  
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  955  		if (irq)
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  956  			rcu_dynticks_task_exit();
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  957  
2625d469b kernel/rcu/tree.c Paul E. McKenney    2016-11-02  958  		rcu_dynticks_eqs_exit();
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  959  
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  960  		if (irq)
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  961  			rcu_cleanup_after_idle();
1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  962  
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  963  		incby = 1;
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  964  	}
bd2b879a1 kernel/rcu/tree.c Paul E. McKenney    2017-10-04  965  	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
bd2b879a1 kernel/rcu/tree.c Paul E. McKenney    2017-10-04  966  			  rdtp->dynticks_nmi_nesting,
dec98900e kernel/rcu/tree.c Paul E. McKenney    2017-10-04  967  			  rdtp->dynticks_nmi_nesting + incby, rdtp->dynticks);
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  968  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */
fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  969  		   rdtp->dynticks_nmi_nesting + incby);
734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  970  	barrier();
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  971  }
64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  972  

:::::: The code at line 778 was first introduced by commit
:::::: fd581a91ac16187625ec509414d08d37827472c4 rcu: Move rcu_nmi_{enter,exit}() to prepare for consolidation

:::::: TO: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
:::::: CC: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 6353 bytes --]

Re: [PATCH] rcu: Refactor rcu_{nmi,irq}_{enter,exit}()

[Paul E. McKenney]

On Fri, Jun 22, 2018 at 04:34:48PM +0800, kbuild test robot wrote:
> Hi Byungchul,
> 
> Thank you for the patch! Perhaps something to improve:
> 
> [auto build test WARNING on rcu/rcu/next]
> [also build test WARNING on v4.18-rc1 next-20180622]
> [if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
> 
> url:    https://github.com/0day-ci/linux/commits/Byungchul-Park/rcu-Refactor-rcu_-nmi-irq-_-enter-exit/20180622-141334
> base:   https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git rcu/next
> reproduce: make htmldocs
> 
> All warnings (new ones prefixed by >>):
> 
>    WARNING: convert(1) not found, for SVG to PDF conversion install ImageMagick (https://www.imagemagick.org)
> >> kernel/rcu/tree.c:778: warning: Function parameter or member 'irq' not described in 'rcu_nmi_exit_common'
> >> kernel/rcu/tree.c:938: warning: Function parameter or member 'irq' not described in 'rcu_nmi_enter_common'

It is complaining about the docbook function-header comment.  It wants
the "irq" parameter described, something like this:

 * @irq: Is this call from rcu_irq_enter?

And similar for rcu_nmi_exit_common().

							Thanx, Paul

>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.ibss' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.connect' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.keys' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.ie' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.ie_len' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.bssid' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.ssid' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.default_key' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.default_mgmt_key' not described in 'wireless_dev'
>    include/net/cfg80211.h:4216: warning: Function parameter or member 'wext.prev_bssid_valid' not described in 'wireless_dev'
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2282: warning: Function parameter or member 'radiotap_timestamp.units_pos' not described in 'ieee80211_hw'
>    include/net/mac80211.h:2282: warning: Function parameter or member 'radiotap_timestamp.accuracy' not described in 'ieee80211_hw'
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.rates' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.rts_cts_rate_idx' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.use_rts' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.use_cts_prot' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.short_preamble' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.skip_table' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.jiffies' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.vif' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.hw_key' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.flags' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'control.enqueue_time' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'ack' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'ack.cookie' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'status.rates' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'status.ack_signal' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'status.ampdu_ack_len' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'status.ampdu_len' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'status.antenna' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'status.tx_time' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'status.is_valid_ack_signal' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'status.status_driver_data' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'driver_rates' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'pad' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:955: warning: Function parameter or member 'rate_driver_data' not described in 'ieee80211_tx_info'
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
>    include/net/mac80211.h:2083: warning: bad line: >
> 
> vim +778 kernel/rcu/tree.c
> 
> 19dd1591f kernel/rcutree.c  Frederic Weisbecker 2012-06-04  765  
> 19dd1591f kernel/rcutree.c  Frederic Weisbecker 2012-06-04  766  /**
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  767   * rcu_nmi_exit_common - inform RCU of exit from NMI context
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  768   *
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  769   * If we are returning from the outermost NMI handler that interrupted an
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  770   * RCU-idle period, update rdtp->dynticks and rdtp->dynticks_nmi_nesting
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  771   * to let the RCU grace-period handling know that the CPU is back to
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  772   * being RCU-idle.
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  773   *
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  774   * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  775   * with CONFIG_RCU_EQS_DEBUG=y.
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  776   */
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  777  static __always_inline void rcu_nmi_exit_common(bool irq)
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02 @778  {
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  779  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  780  
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  781  	/*
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  782  	 * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  783  	 * (We are exiting an NMI handler, so RCU better be paying attention
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  784  	 * to us!)
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  785  	 */
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  786  	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  787  	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  788  
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  789  	/*
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  790  	 * If the nesting level is not 1, the CPU wasn't RCU-idle, so
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  791  	 * leave it in non-RCU-idle state.
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  792  	 */
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  793  	if (rdtp->dynticks_nmi_nesting != 1) {
> dec98900e kernel/rcu/tree.c Paul E. McKenney    2017-10-04  794  		trace_rcu_dyntick(TPS("--="), rdtp->dynticks_nmi_nesting, rdtp->dynticks_nmi_nesting - 2, rdtp->dynticks);
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  795  		WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* No store tearing. */
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  796  			   rdtp->dynticks_nmi_nesting - 2);
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  797  		return;
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  798  	}
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  799  
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  800  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
> dec98900e kernel/rcu/tree.c Paul E. McKenney    2017-10-04  801  	trace_rcu_dyntick(TPS("Startirq"), rdtp->dynticks_nmi_nesting, 0, rdtp->dynticks);
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  802  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  803  
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  804  	if (irq)
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  805  		rcu_prepare_for_idle();
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  806  
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  807  	rcu_dynticks_eqs_enter();
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  808  
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  809  	if (irq)
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  810  		rcu_dynticks_task_enter();
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  811  }
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  812  
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  813  /**
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  814   * rcu_nmi_exit - inform RCU of exit from NMI context
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  815   */
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  816  void rcu_nmi_exit(void)
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  817  {
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  818  	rcu_nmi_exit_common(false);
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  819  }
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  820  
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  821  /**
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  822   * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  823   *
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  824   * Exit from an interrupt handler, which might possibly result in entering
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  825   * idle mode, in other words, leaving the mode in which read-side critical
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  826   * sections can occur.  The caller must have disabled interrupts.
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  827   *
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  828   * This code assumes that the idle loop never does anything that might
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  829   * result in unbalanced calls to irq_enter() and irq_exit().  If your
> 58721f5da kernel/rcu/tree.c Paul E. McKenney    2017-10-03  830   * architecture's idle loop violates this assumption, RCU will give you what
> 58721f5da kernel/rcu/tree.c Paul E. McKenney    2017-10-03  831   * you deserve, good and hard.  But very infrequently and irreproducibly.
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  832   *
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  833   * Use things like work queues to work around this limitation.
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  834   *
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  835   * You have been warned.
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  836   *
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  837   * If you add or remove a call to rcu_irq_exit(), be sure to test with
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  838   * CONFIG_RCU_EQS_DEBUG=y.
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  839   */
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  840  void rcu_irq_exit(void)
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  841  {
> b04db8e19 kernel/rcu/tree.c Frederic Weisbecker 2017-11-06  842  	lockdep_assert_irqs_disabled();
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  843  	rcu_nmi_exit_common(true);
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  844  }
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  845  
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  846  /*
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  847   * Wrapper for rcu_irq_exit() where interrupts are enabled.
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  848   *
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  849   * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  850   * with CONFIG_RCU_EQS_DEBUG=y.
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  851   */
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  852  void rcu_irq_exit_irqson(void)
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  853  {
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  854  	unsigned long flags;
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  855  
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  856  	local_irq_save(flags);
> 7c9906ca5 kernel/rcu/tree.c Paul E. McKenney    2015-10-31  857  	rcu_irq_exit();
> 23b5c8fa0 kernel/rcutree.c  Paul E. McKenney    2010-09-07  858  	local_irq_restore(flags);
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  859  }
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  860  
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  861  /*
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  862   * Exit an RCU extended quiescent state, which can be either the
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  863   * idle loop or adaptive-tickless usermode execution.
> 51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  864   *
> 51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  865   * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to
> 51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  866   * allow for the possibility of usermode upcalls messing up our count of
> 51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  867   * interrupt nesting level during the busy period that is just now starting.
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  868   */
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  869  static void rcu_eqs_exit(bool user)
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  870  {
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  871  	struct rcu_dynticks *rdtp;
> 84585aa8b kernel/rcu/tree.c Paul E. McKenney    2017-10-04  872  	long oldval;
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  873  
> b04db8e19 kernel/rcu/tree.c Frederic Weisbecker 2017-11-06  874  	lockdep_assert_irqs_disabled();
> c9d4b0af9 kernel/rcutree.c  Christoph Lameter   2013-08-31  875  	rdtp = this_cpu_ptr(&rcu_dynticks);
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  876  	oldval = rdtp->dynticks_nesting;
> 1ce46ee59 kernel/rcu/tree.c Paul E. McKenney    2015-05-05  877  	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
> 51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  878  	if (oldval) {
> 51a1fd30f kernel/rcu/tree.c Paul E. McKenney    2017-10-03  879  		rdtp->dynticks_nesting++;
> 9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  880  		return;
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  881  	}
> 9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  882  	rcu_dynticks_task_exit();
> 9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  883  	rcu_dynticks_eqs_exit();
> 9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  884  	rcu_cleanup_after_idle();
> 9dd238e28 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  885  	trace_rcu_dyntick(TPS("End"), rdtp->dynticks_nesting, 1, rdtp->dynticks);
> e68bbb266 kernel/rcu/tree.c Paul E. McKenney    2017-10-05  886  	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
> 2342172fd kernel/rcu/tree.c Paul E. McKenney    2017-10-05  887  	WRITE_ONCE(rdtp->dynticks_nesting, 1);
> 6136d6e48 kernel/rcu/tree.c Paul E. McKenney    2017-10-03  888  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
> 3a5924052 kernel/rcu/tree.c Paul E. McKenney    2013-10-04  889  }
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  890  
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  891  /**
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  892   * rcu_idle_exit - inform RCU that current CPU is leaving idle
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  893   *
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  894   * Exit idle mode, in other words, -enter- the mode in which RCU
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  895   * read-side critical sections can occur.
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  896   *
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  897   * If you add or remove a call to rcu_idle_exit(), be sure to test with
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  898   * CONFIG_RCU_EQS_DEBUG=y.
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  899   */
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  900  void rcu_idle_exit(void)
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  901  {
> c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  902  	unsigned long flags;
> c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  903  
> c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  904  	local_irq_save(flags);
> cb349ca95 kernel/rcutree.c  Paul E. McKenney    2012-09-04  905  	rcu_eqs_exit(false);
> c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  906  	local_irq_restore(flags);
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  907  }
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  908  
> d1ec4c34c kernel/rcu/tree.c Paul E. McKenney    2015-05-13  909  #ifdef CONFIG_NO_HZ_FULL
> 9b2e4f188 kernel/rcutree.c  Paul E. McKenney    2011-09-30  910  /**
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  911   * rcu_user_exit - inform RCU that we are exiting userspace.
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  912   *
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  913   * Exit RCU idle mode while entering the kernel because it can
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  914   * run a RCU read side critical section anytime.
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  915   *
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  916   * If you add or remove a call to rcu_user_exit(), be sure to test with
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  917   * CONFIG_RCU_EQS_DEBUG=y.
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  918   */
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  919  void rcu_user_exit(void)
> adf5091e6 kernel/rcutree.c  Frederic Weisbecker 2012-06-28  920  {
> 91d1aa43d kernel/rcutree.c  Frederic Weisbecker 2012-11-27  921  	rcu_eqs_exit(1);
> c5d900bf6 kernel/rcutree.c  Frederic Weisbecker 2012-07-11  922  }
> d1ec4c34c kernel/rcu/tree.c Paul E. McKenney    2015-05-13  923  #endif /* CONFIG_NO_HZ_FULL */
> 19dd1591f kernel/rcutree.c  Frederic Weisbecker 2012-06-04  924  
> 19dd1591f kernel/rcutree.c  Frederic Weisbecker 2012-06-04  925  /**
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  926   * rcu_nmi_enter_common - inform RCU of entry to NMI context
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  927   *
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  928   * If the CPU was idle from RCU's viewpoint, update rdtp->dynticks and
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  929   * rdtp->dynticks_nmi_nesting to let the RCU grace-period handling know
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  930   * that the CPU is active.  This implementation permits nested NMIs, as
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  931   * long as the nesting level does not overflow an int.  (You will probably
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  932   * run out of stack space first.)
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  933   *
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  934   * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
> c0da313e0 kernel/rcu/tree.c Paul E. McKenney    2017-09-22  935   * with CONFIG_RCU_EQS_DEBUG=y.
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  936   */
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  937  static __always_inline void rcu_nmi_enter_common(bool irq)
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18 @938  {
> c9d4b0af9 kernel/rcutree.c  Christoph Lameter   2013-08-31  939  	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> 84585aa8b kernel/rcu/tree.c Paul E. McKenney    2017-10-04  940  	long incby = 2;
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  941  
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  942  	/* Complain about underflow. */
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  943  	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting < 0);
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  944  
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  945  	/*
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  946  	 * If idle from RCU viewpoint, atomically increment ->dynticks
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  947  	 * to mark non-idle and increment ->dynticks_nmi_nesting by one.
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  948  	 * Otherwise, increment ->dynticks_nmi_nesting by two.  This means
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  949  	 * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  950  	 * to be in the outermost NMI handler that interrupted an RCU-idle
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  951  	 * period (observation due to Andy Lutomirski).
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  952  	 */
> 02a5c550b kernel/rcu/tree.c Paul E. McKenney    2016-11-02  953  	if (rcu_dynticks_curr_cpu_in_eqs()) {
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  954  
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  955  		if (irq)
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  956  			rcu_dynticks_task_exit();
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  957  
> 2625d469b kernel/rcu/tree.c Paul E. McKenney    2016-11-02  958  		rcu_dynticks_eqs_exit();
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  959  
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  960  		if (irq)
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  961  			rcu_cleanup_after_idle();
> 1e8a49734 kernel/rcu/tree.c Byungchul Park      2018-06-22  962  
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  963  		incby = 1;
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  964  	}
> bd2b879a1 kernel/rcu/tree.c Paul E. McKenney    2017-10-04  965  	trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
> bd2b879a1 kernel/rcu/tree.c Paul E. McKenney    2017-10-04  966  			  rdtp->dynticks_nmi_nesting,
> dec98900e kernel/rcu/tree.c Paul E. McKenney    2017-10-04  967  			  rdtp->dynticks_nmi_nesting + incby, rdtp->dynticks);
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  968  	WRITE_ONCE(rdtp->dynticks_nmi_nesting, /* Prevent store tearing. */
> fd581a91a kernel/rcu/tree.c Paul E. McKenney    2017-10-02  969  		   rdtp->dynticks_nmi_nesting + incby);
> 734d16801 kernel/rcu/tree.c Paul E. McKenney    2014-11-21  970  	barrier();
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  971  }
> 64db4cfff kernel/rcutree.c  Paul E. McKenney    2008-12-18  972  
> 
> :::::: The code at line 778 was first introduced by commit
> :::::: fd581a91ac16187625ec509414d08d37827472c4 rcu: Move rcu_nmi_{enter,exit}() to prepare for consolidation
> 
> :::::: TO: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> :::::: CC: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> ---
> 0-DAY kernel test infrastructure                Open Source Technology Center
> https://lists.01.org/pipermail/kbuild-all                   Intel Corporation