[PATCH tip/core/rcu 0/6] Dynticks updates for 4.11

[PATCH tip/core/rcu 0/6] Dynticks updates for 4.11

[Paul E. McKenney]

Hello!

This series provides dynticks updates:

1-4.	Abstract access to the dyntick counter, replacing the current
	open-coding of atomic operations.

5.	Check cond_resched_rcu_qs() state less often to reduce GP overhead.

6.	Adjust FQS offline checks for exact online-CPU detection. 

							Thanx, Paul

------------------------------------------------------------------------

 include/linux/rcutiny.h    |    6 +
 include/trace/events/rcu.h |   10 -
 kernel/rcu/tree.c          |  245 +++++++++++++++++++++++++++++++--------------
 kernel/rcu/tree.h          |    2 
 kernel/rcu/tree_exp.h      |   12 --
 kernel/rcu/tree_plugin.h   |    2 
 kernel/rcu/tree_trace.c    |    2 
 7 files changed, 190 insertions(+), 89 deletions(-)

[PATCH tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation

[Paul E. McKenney]

This commit is the first step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic add
of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
will ease changes to the ->dynticks counter operation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index cb4e2056ccf3..14e283c351f6 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 
+/*
+ * Do a double-increment of the ->dynticks counter to emulate a
+ * momentary idle-CPU quiescent state.
+ */
+static void rcu_dynticks_momentary_idle(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	int special = atomic_add_return(2, &rdtp->dynticks);
+
+	/* It is illegal to call this from idle state. */
+	WARN_ON_ONCE(!(special & 0x1));
+}
+
 DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
 EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 
@@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 static void rcu_momentary_dyntick_idle(void)
 {
 	struct rcu_data *rdp;
-	struct rcu_dynticks *rdtp;
 	int resched_mask;
 	struct rcu_state *rsp;
 
@@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
 		 * quiescent state, with no need for this CPU to do anything
 		 * further.
 		 */
-		rdtp = this_cpu_ptr(&rcu_dynticks);
-		smp_mb__before_atomic(); /* Earlier stuff before QS. */
-		atomic_add(2, &rdtp->dynticks);  /* QS. */
-		smp_mb__after_atomic(); /* Later stuff after QS. */
+		rcu_dynticks_momentary_idle();
 		break;
 	}
 }
-- 
2.5.2

[PATCH tip/core/rcu 2/6] rcu: Abstract the dynticks snapshot operation

[Paul E. McKenney]

This commit is the second step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic
add of zero in a new rcu_dynticks_snap() function.  This abstraction will
ease changes o the ->dynticks counter operation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c     | 19 ++++++++++++++++---
 kernel/rcu/tree_exp.h |  6 ++----
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 14e283c351f6..805d55ee0b2a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -282,6 +282,17 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 };
 
 /*
+ * Snapshot the ->dynticks counter with full ordering so as to allow
+ * stable comparison of this counter with past and future snapshots.
+ */
+static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+{
+	int snap = atomic_add_return(0, &rdtp->dynticks);
+
+	return snap;
+}
+
+/*
  * Do a double-increment of the ->dynticks counter to emulate a
  * momentary idle-CPU quiescent state.
  */
@@ -1049,7 +1060,9 @@ void rcu_nmi_exit(void)
  */
 bool notrace __rcu_is_watching(void)
 {
-	return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	return atomic_read(&rdtp->dynticks) & 0x1;
 }
 
 /**
@@ -1132,7 +1145,7 @@ static int rcu_is_cpu_rrupt_from_idle(void)
 static int dyntick_save_progress_counter(struct rcu_data *rdp,
 					 bool *isidle, unsigned long *maxj)
 {
-	rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
+	rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
 	rcu_sysidle_check_cpu(rdp, isidle, maxj);
 	if ((rdp->dynticks_snap & 0x1) == 0) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
@@ -1157,7 +1170,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	int *rcrmp;
 	unsigned int snap;
 
-	curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks);
+	curr = (unsigned int)rcu_dynticks_snap(rdp->dynticks);
 	snap = (unsigned int)rdp->dynticks_snap;
 
 	/*
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index e59e1849b89a..011f626b2fd8 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -356,10 +356,9 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 		mask_ofl_test = 0;
 		for_each_leaf_node_possible_cpu(rnp, cpu) {
 			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-			struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
 
 			rdp->exp_dynticks_snap =
-				atomic_add_return(0, &rdtp->dynticks);
+				rcu_dynticks_snap(rdp->dynticks);
 			if (raw_smp_processor_id() == cpu ||
 			    !(rdp->exp_dynticks_snap & 0x1) ||
 			    !(rnp->qsmaskinitnext & rdp->grpmask))
@@ -380,12 +379,11 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 		for_each_leaf_node_possible_cpu(rnp, cpu) {
 			unsigned long mask = leaf_node_cpu_bit(rnp, cpu);
 			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-			struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
 
 			if (!(mask_ofl_ipi & mask))
 				continue;
 retry_ipi:
-			if (atomic_add_return(0, &rdtp->dynticks) !=
+			if (rcu_dynticks_snap(rdp->dynticks) !=
 			    rdp->exp_dynticks_snap) {
 				mask_ofl_test |= mask;
 				continue;
-- 
2.5.2

[PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Paul E. McKenney]

This commit is the third step towards full abstraction of all accesses
to the ->dynticks counter, implementing the previously open-coded atomic
add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
This abstraction will ease changes to the ->dynticks counter operation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 92 +++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 66 insertions(+), 26 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 805d55ee0b2a..fc49e008963a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -282,6 +282,65 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 };
 
 /*
+ * Record entry into an extended quiescent state.  This is only to be
+ * called when not already in an extended quiescent state.
+ */
+static void rcu_dynticks_eqs_enter(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	/*
+	 * CPUs seeing atomic_inc() must see prior RCU read-side critical
+	 * sections, and we also must force ordering with the next idle
+	 * sojourn.
+	 */
+	smp_mb__before_atomic(); /* See above. */
+	atomic_inc(&rdtp->dynticks);
+	smp_mb__after_atomic(); /* See above. */
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+		     atomic_read(&rdtp->dynticks) & 0x1);
+}
+
+/*
+ * Record exit from an extended quiescent state.  This is only to be
+ * called from an extended quiescent state.
+ */
+static void rcu_dynticks_eqs_exit(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	/*
+	 * CPUs seeing atomic_inc() must see prior idle sojourns,
+	 * and we also must force ordering with the next RCU read-side
+	 * critical section.
+	 */
+	smp_mb__before_atomic(); /* See above. */
+	atomic_inc(&rdtp->dynticks);
+	smp_mb__after_atomic(); /* See above. */
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
+		     !(atomic_read(&rdtp->dynticks) & 0x1));
+}
+
+/*
+ * Reset the current CPU's ->dynticks counter to indicate that the
+ * newly onlined CPU is no longer in an extended quiescent state.
+ * This will either leave the counter unchanged, or increment it
+ * to the next non-quiescent value.
+ *
+ * The non-atomic test/increment sequence works because the upper bits
+ * of the ->dynticks counter are manipulated only by the corresponding CPU,
+ * or when the corresponding CPU is offline.
+ */
+static void rcu_dynticks_eqs_online(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	if (atomic_read(&rdtp->dynticks) & 0x1)
+		return;
+	atomic_add(0x1, &rdtp->dynticks);
+}
+
+/*
  * Snapshot the ->dynticks counter with full ordering so as to allow
  * stable comparison of this counter with past and future snapshots.
  */
@@ -693,7 +752,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
 {
 	struct rcu_state *rsp;
 	struct rcu_data *rdp;
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
 
 	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
 	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -712,12 +771,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
 		do_nocb_deferred_wakeup(rdp);
 	}
 	rcu_prepare_for_idle();
-	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
-	smp_mb__before_atomic();  /* See above. */
-	atomic_inc(&rdtp->dynticks);
-	smp_mb__after_atomic();  /* Force ordering with next sojourn. */
-	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
-		     atomic_read(&rdtp->dynticks) & 0x1);
+	rcu_dynticks_eqs_enter();
 	rcu_dynticks_task_enter();
 
 	/*
@@ -846,15 +900,10 @@ void rcu_irq_exit_irqson(void)
  */
 static void rcu_eqs_exit_common(long long oldval, int user)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
 
 	rcu_dynticks_task_exit();
-	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
-	atomic_inc(&rdtp->dynticks);
-	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
-	smp_mb__after_atomic();  /* See above. */
-	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
-		     !(atomic_read(&rdtp->dynticks) & 0x1));
+	rcu_dynticks_eqs_exit();
 	rcu_cleanup_after_idle();
 	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
 	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -1001,11 +1050,7 @@ void rcu_nmi_enter(void)
 	 * period (observation due to Andy Lutomirski).
 	 */
 	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
-		smp_mb__before_atomic();  /* Force delay from prior write. */
-		atomic_inc(&rdtp->dynticks);
-		/* atomic_inc() before later RCU read-side crit sects */
-		smp_mb__after_atomic();  /* See above. */
-		WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+		rcu_dynticks_eqs_exit();
 		incby = 1;
 	}
 	rdtp->dynticks_nmi_nesting += incby;
@@ -1043,11 +1088,7 @@ void rcu_nmi_exit(void)
 
 	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
 	rdtp->dynticks_nmi_nesting = 0;
-	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
-	smp_mb__before_atomic();  /* See above. */
-	atomic_inc(&rdtp->dynticks);
-	smp_mb__after_atomic();  /* Force delay to next write. */
-	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+	rcu_dynticks_eqs_enter();
 }
 
 /**
@@ -3800,8 +3841,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
 		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
 	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
 	rcu_sysidle_init_percpu_data(rdp->dynticks);
-	atomic_set(&rdp->dynticks->dynticks,
-		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
+	rcu_dynticks_eqs_online();
 	raw_spin_unlock_rcu_node(rnp);		/* irqs remain disabled. */
 
 	/*
-- 
2.5.2

[PATCH tip/core/rcu 4/6] rcu: Abstract extended quiescent state determination

[Paul E. McKenney]

This commit is the fourth step towards full abstraction of all accesses
to the ->dynticks counter, implementing previously open-coded checks and
comparisons in new rcu_dynticks_in_eqs() and rcu_dynticks_in_eqs_since()
functions.  This abstraction will ease changes to the ->dynticks counter
operation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 include/linux/rcutiny.h  |  6 ++++++
 kernel/rcu/tree.c        | 52 +++++++++++++++++++++++++++++++++++-------------
 kernel/rcu/tree.h        |  2 ++
 kernel/rcu/tree_exp.h    |  6 +++---
 kernel/rcu/tree_plugin.h |  2 +-
 kernel/rcu/tree_trace.c  |  2 +-
 6 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index ac81e4063b40..4f9b2fa2173d 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -27,6 +27,12 @@
 
 #include <linux/cache.h>
 
+struct rcu_dynticks;
+static inline int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+{
+	return 0;
+}
+
 static inline unsigned long get_state_synchronize_rcu(void)
 {
 	return 0;
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index fc49e008963a..5a4e7427f372 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -341,10 +341,22 @@ static void rcu_dynticks_eqs_online(void)
 }
 
 /*
+ * Is the current CPU in an extended quiescent state?
+ *
+ * No ordering, as we are sampling CPU-local information.
+ */
+bool rcu_dynticks_curr_cpu_in_eqs(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	return !(atomic_read(&rdtp->dynticks) & 0x1);
+}
+
+/*
  * Snapshot the ->dynticks counter with full ordering so as to allow
  * stable comparison of this counter with past and future snapshots.
  */
-static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
 {
 	int snap = atomic_add_return(0, &rdtp->dynticks);
 
@@ -352,6 +364,25 @@ static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
 }
 
 /*
+ * Return true if the snapshot returned from rcu_dynticks_snap()
+ * indicates that RCU is in an extended quiescent state.
+ */
+static bool rcu_dynticks_in_eqs(int snap)
+{
+	return !(snap & 0x1);
+}
+
+/*
+ * Return true if the CPU corresponding to the specified rcu_dynticks
+ * structure has spent some time in an extended quiescent state since
+ * rcu_dynticks_snap() returned the specified snapshot.
+ */
+static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap)
+{
+	return snap != rcu_dynticks_snap(rdtp);
+}
+
+/*
  * Do a double-increment of the ->dynticks counter to emulate a
  * momentary idle-CPU quiescent state.
  */
@@ -1049,7 +1080,7 @@ void rcu_nmi_enter(void)
 	 * to be in the outermost NMI handler that interrupted an RCU-idle
 	 * period (observation due to Andy Lutomirski).
 	 */
-	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
+	if (rcu_dynticks_curr_cpu_in_eqs()) {
 		rcu_dynticks_eqs_exit();
 		incby = 1;
 	}
@@ -1075,7 +1106,7 @@ void rcu_nmi_exit(void)
 	 * to us!)
 	 */
 	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
-	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
 
 	/*
 	 * If the nesting level is not 1, the CPU wasn't RCU-idle, so
@@ -1101,9 +1132,7 @@ void rcu_nmi_exit(void)
  */
 bool notrace __rcu_is_watching(void)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-
-	return atomic_read(&rdtp->dynticks) & 0x1;
+	return !rcu_dynticks_curr_cpu_in_eqs();
 }
 
 /**
@@ -1188,7 +1217,7 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 {
 	rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
 	rcu_sysidle_check_cpu(rdp, isidle, maxj);
-	if ((rdp->dynticks_snap & 0x1) == 0) {
+	if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
 		if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4,
 				 rdp->mynode->gpnum))
@@ -1207,12 +1236,7 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 				    bool *isidle, unsigned long *maxj)
 {
-	unsigned int curr;
 	int *rcrmp;
-	unsigned int snap;
-
-	curr = (unsigned int)rcu_dynticks_snap(rdp->dynticks);
-	snap = (unsigned int)rdp->dynticks_snap;
 
 	/*
 	 * If the CPU passed through or entered a dynticks idle phase with
@@ -1222,7 +1246,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	 * read-side critical section that started before the beginning
 	 * of the current RCU grace period.
 	 */
-	if ((curr & 0x1) == 0 || UINT_CMP_GE(curr, snap + 2)) {
+	if (rcu_dynticks_in_eqs_since(rdp->dynticks, rdp->dynticks_snap)) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
 		rdp->dynticks_fqs++;
 		return 1;
@@ -3811,7 +3835,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
 	rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
 	rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
 	WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
-	WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
+	WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks)));
 	rdp->cpu = cpu;
 	rdp->rsp = rsp;
 	rcu_boot_init_nocb_percpu_data(rdp);
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index fe98dd24adf8..3b953dcf6afc 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -595,6 +595,8 @@ extern struct rcu_state rcu_bh_state;
 extern struct rcu_state rcu_preempt_state;
 #endif /* #ifdef CONFIG_PREEMPT_RCU */
 
+int rcu_dynticks_snap(struct rcu_dynticks *rdtp);
+
 #ifdef CONFIG_RCU_BOOST
 DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
 DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu);
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 011f626b2fd8..e155a465cf84 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -360,7 +360,7 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 			rdp->exp_dynticks_snap =
 				rcu_dynticks_snap(rdp->dynticks);
 			if (raw_smp_processor_id() == cpu ||
-			    !(rdp->exp_dynticks_snap & 0x1) ||
+			    rcu_dynticks_in_eqs(rdp->exp_dynticks_snap) ||
 			    !(rnp->qsmaskinitnext & rdp->grpmask))
 				mask_ofl_test |= rdp->grpmask;
 		}
@@ -383,8 +383,8 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 			if (!(mask_ofl_ipi & mask))
 				continue;
 retry_ipi:
-			if (rcu_dynticks_snap(rdp->dynticks) !=
-			    rdp->exp_dynticks_snap) {
+			if (rcu_dynticks_in_eqs_since(rdp->dynticks,
+						      rdp->exp_dynticks_snap)) {
 				mask_ofl_test |= mask;
 				continue;
 			}
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 56583e764ebf..652209589adf 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1643,7 +1643,7 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
 	       "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
 	       "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
 	       ticks_value, ticks_title,
-	       atomic_read(&rdtp->dynticks) & 0xfff,
+	       rcu_dynticks_snap(rdtp) & 0xfff,
 	       rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
 	       rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
 	       READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c
index b1f28972872c..b833cd0a29e8 100644
--- a/kernel/rcu/tree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -124,7 +124,7 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
 		   rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu),
 		   rdp->core_needs_qs);
 	seq_printf(m, " dt=%d/%llx/%d df=%lu",
-		   atomic_read(&rdp->dynticks->dynticks),
+		   rcu_dynticks_snap(rdp->dynticks),
 		   rdp->dynticks->dynticks_nesting,
 		   rdp->dynticks->dynticks_nmi_nesting,
 		   rdp->dynticks_fqs);
-- 
2.5.2

[PATCH tip/core/rcu 5/6] rcu: Check cond_resched_rcu_qs() state less often to reduce GP overhead

[Paul E. McKenney]

Commit 4a81e8328d37 ("rcu: Reduce overhead of cond_resched() checks
for RCU") moved quiescent-state generation out of cond_resched()
and commit bde6c3aa9930 ("rcu: Provide cond_resched_rcu_qs() to force
quiescent states in long loops") introduced cond_resched_rcu_qs(), and
commit 5cd37193ce85 ("rcu: Make cond_resched_rcu_qs() apply to normal RCU
flavors") introduced the per-CPU rcu_qs_ctr variable, which is frequently
polled by the RCU core state machine.

This frequent polling can increase grace-period rate, which in turn
increases grace-period overhead, which is visible in some benchmarks
(for example, the "open1" benchmark in Anton Blanchard's "will it scale"
suite).  This commit therefore reduces the rate at which rcu_qs_ctr
is polled by moving that polling into the force-quiescent-state (FQS)
machinery, and by further polling it only after the grace period has
been in effect for at least jiffies_till_sched_qs jiffies.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 include/trace/events/rcu.h | 10 +++++-----
 kernel/rcu/tree.c          | 46 ++++++++++++++++++++++++++++++++++------------
 2 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index 9d4f9b3a2b7b..e3facb356838 100644
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -385,11 +385,11 @@ TRACE_EVENT(rcu_quiescent_state_report,
 
 /*
  * Tracepoint for quiescent states detected by force_quiescent_state().
- * These trace events include the type of RCU, the grace-period number
- * that was blocked by the CPU, the CPU itself, and the type of quiescent
- * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
- * or "kick" when kicking a CPU that has been in dyntick-idle mode for
- * too long.
+ * These trace events include the type of RCU, the grace-period number that
+ * was blocked by the CPU, the CPU itself, and the type of quiescent state,
+ * which can be "dti" for dyntick-idle mode, "ofl" for CPU offline, "kick"
+ * when kicking a CPU that has been in dyntick-idle mode for too long, or
+ * "rqc" if the CPU got a quiescent state via its rcu_qs_ctr.
  */
 TRACE_EVENT(rcu_fqs,
 
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 5a4e7427f372..c920ed3e158c 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1236,7 +1236,10 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 				    bool *isidle, unsigned long *maxj)
 {
+	unsigned long jtsq;
 	int *rcrmp;
+	unsigned long rjtsc;
+	struct rcu_node *rnp;
 
 	/*
 	 * If the CPU passed through or entered a dynticks idle phase with
@@ -1252,6 +1255,31 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 		return 1;
 	}
 
+	/* Compute and saturate jiffies_till_sched_qs. */
+	jtsq = jiffies_till_sched_qs;
+	rjtsc = rcu_jiffies_till_stall_check();
+	if (jtsq > rjtsc / 2) {
+		WRITE_ONCE(jiffies_till_sched_qs, rjtsc);
+		jtsq = rjtsc / 2;
+	} else if (jtsq < 1) {
+		WRITE_ONCE(jiffies_till_sched_qs, 1);
+		jtsq = 1;
+	}
+
+	/*
+	 * Has this CPU encountered a cond_resched_rcu_qs() since the
+	 * beginning of the grace period?  For this to be the case,
+	 * the CPU has to have noticed the current grace period.  This
+	 * might not be the case for nohz_full CPUs looping in the kernel.
+	 */
+	rnp = rdp->mynode;
+	if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
+	    READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_qs_ctr, rdp->cpu) &&
+	    READ_ONCE(rdp->gpnum) == rnp->gpnum && !rdp->gpwrap) {
+		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("rqc"));
+		return 1;
+	}
+
 	/*
 	 * Check for the CPU being offline, but only if the grace period
 	 * is old enough.  We don't need to worry about the CPU changing
@@ -1294,9 +1322,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	 * warning delay.
 	 */
 	rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu);
-	if (ULONG_CMP_GE(jiffies,
-			 rdp->rsp->gp_start + jiffies_till_sched_qs) ||
-	    ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
+	if (time_after(jiffies, rdp->rsp->gp_start + jtsq) ||
+	    time_after(jiffies, rdp->rsp->jiffies_resched)) {
 		if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) {
 			WRITE_ONCE(rdp->cond_resched_completed,
 				   READ_ONCE(rdp->mynode->completed));
@@ -2554,10 +2581,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
 
 	rnp = rdp->mynode;
 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
-	if ((rdp->cpu_no_qs.b.norm &&
-	     rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
-	    rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
-	    rdp->gpwrap) {
+	if (rdp->cpu_no_qs.b.norm || rdp->gpnum != rnp->gpnum ||
+	    rnp->completed == rnp->gpnum || rdp->gpwrap) {
 
 		/*
 		 * The grace period in which this quiescent state was
@@ -2612,8 +2637,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
 	 * Was there a quiescent state since the beginning of the grace
 	 * period? If no, then exit and wait for the next call.
 	 */
-	if (rdp->cpu_no_qs.b.norm &&
-	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
+	if (rdp->cpu_no_qs.b.norm)
 		return;
 
 	/*
@@ -3567,9 +3591,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
 	    rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
 	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
 		rdp->n_rp_core_needs_qs++;
-	} else if (rdp->core_needs_qs &&
-		   (!rdp->cpu_no_qs.b.norm ||
-		    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
+	} else if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm) {
 		rdp->n_rp_report_qs++;
 		return 1;
 	}
-- 
2.5.2

[PATCH tip/core/rcu 6/6] rcu: Adjust FQS offline checks for exact online-CPU detection

[Paul E. McKenney]

Commit 7ec99de36f40 ("rcu: Provide exact CPU-online tracking for RCU"),
as its title suggests, got rid of RCU's remaining CPU-hotplug timing
guesswork.  This commit therefore removes the one-jiffy kludge that was
used to paper over this guesswork.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index c920ed3e158c..ed65b1fc8eef 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1280,21 +1280,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 		return 1;
 	}
 
-	/*
-	 * Check for the CPU being offline, but only if the grace period
-	 * is old enough.  We don't need to worry about the CPU changing
-	 * state: If we see it offline even once, it has been through a
-	 * quiescent state.
-	 *
-	 * The reason for insisting that the grace period be at least
-	 * one jiffy old is that CPUs that are not quite online and that
-	 * have just gone offline can still execute RCU read-side critical
-	 * sections.
-	 */
-	if (ULONG_CMP_GE(rdp->rsp->gp_start + 2, jiffies))
-		return 0;  /* Grace period is not old enough. */
-	barrier();
-	if (cpu_is_offline(rdp->cpu)) {
+	/* Check for the CPU being offline. */
+	if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp))) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl"));
 		rdp->offline_fqs++;
 		return 1;
-- 
2.5.2

Re: [PATCH tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation

[Josh Triplett]

On Sat, Jan 14, 2017 at 12:54:40AM -0800, Paul E. McKenney wrote:
> This commit is the first step towards full abstraction of all accesses to
> the ->dynticks counter, implementing the previously open-coded atomic add
> of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
> will ease changes to the ->dynticks counter operation.
> 
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

This change has an additional effect not documented in the commit
message: it eliminates the smp_mb__before_atomic and
smp_mb__after_atomic calls.  Can you please document that in the commit
message, and explain why that doesn't cause a problem?

> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
>  #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
>  };
>  
> +/*
> + * Do a double-increment of the ->dynticks counter to emulate a
> + * momentary idle-CPU quiescent state.
> + */
> +static void rcu_dynticks_momentary_idle(void)
> +{
> +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +	int special = atomic_add_return(2, &rdtp->dynticks);
> +
> +	/* It is illegal to call this from idle state. */
> +	WARN_ON_ONCE(!(special & 0x1));
> +}
> +
>  DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
>  EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
>  
> @@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
>  static void rcu_momentary_dyntick_idle(void)
>  {
>  	struct rcu_data *rdp;
> -	struct rcu_dynticks *rdtp;
>  	int resched_mask;
>  	struct rcu_state *rsp;
>  
> @@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
>  		 * quiescent state, with no need for this CPU to do anything
>  		 * further.
>  		 */
> -		rdtp = this_cpu_ptr(&rcu_dynticks);
> -		smp_mb__before_atomic(); /* Earlier stuff before QS. */
> -		atomic_add(2, &rdtp->dynticks);  /* QS. */
> -		smp_mb__after_atomic(); /* Later stuff after QS. */
> +		rcu_dynticks_momentary_idle();
>  		break;
>  	}
>  }
> -- 
> 2.5.2
> 

Re: [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Josh Triplett]

On Sat, Jan 14, 2017 at 12:54:42AM -0800, Paul E. McKenney wrote:
> This commit is the third step towards full abstraction of all accesses
> to the ->dynticks counter, implementing the previously open-coded atomic
> add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
> the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
> This abstraction will ease changes to the ->dynticks counter operation.
> 
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

A couple of comments below.  With those addressed:
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

>  kernel/rcu/tree.c | 92 +++++++++++++++++++++++++++++++++++++++----------------
>  1 file changed, 66 insertions(+), 26 deletions(-)
> 
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index 805d55ee0b2a..fc49e008963a 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -282,6 +282,65 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
>  };
>  
>  /*
> + * Record entry into an extended quiescent state.  This is only to be
> + * called when not already in an extended quiescent state.
> + */
> +static void rcu_dynticks_eqs_enter(void)
> +{
> +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +
> +	/*
> +	 * CPUs seeing atomic_inc() must see prior RCU read-side critical
> +	 * sections, and we also must force ordering with the next idle
> +	 * sojourn.
> +	 */
> +	smp_mb__before_atomic(); /* See above. */
> +	atomic_inc(&rdtp->dynticks);
> +	smp_mb__after_atomic(); /* See above. */
> +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> +		     atomic_read(&rdtp->dynticks) & 0x1);
> +}
> +
> +/*
> + * Record exit from an extended quiescent state.  This is only to be
> + * called from an extended quiescent state.
> + */
> +static void rcu_dynticks_eqs_exit(void)
> +{
> +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +
> +	/*
> +	 * CPUs seeing atomic_inc() must see prior idle sojourns,
> +	 * and we also must force ordering with the next RCU read-side
> +	 * critical section.
> +	 */
> +	smp_mb__before_atomic(); /* See above. */
> +	atomic_inc(&rdtp->dynticks);
> +	smp_mb__after_atomic(); /* See above. */
> +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> +		     !(atomic_read(&rdtp->dynticks) & 0x1));
> +}
> +
> +/*
> + * Reset the current CPU's ->dynticks counter to indicate that the
> + * newly onlined CPU is no longer in an extended quiescent state.
> + * This will either leave the counter unchanged, or increment it
> + * to the next non-quiescent value.
> + *
> + * The non-atomic test/increment sequence works because the upper bits
> + * of the ->dynticks counter are manipulated only by the corresponding CPU,
> + * or when the corresponding CPU is offline.
> + */
> +static void rcu_dynticks_eqs_online(void)
> +{
> +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +
> +	if (atomic_read(&rdtp->dynticks) & 0x1)
> +		return;
> +	atomic_add(0x1, &rdtp->dynticks);
> +}
> +
> +/*
>   * Snapshot the ->dynticks counter with full ordering so as to allow
>   * stable comparison of this counter with past and future snapshots.
>   */
> @@ -693,7 +752,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
>  {
>  	struct rcu_state *rsp;
>  	struct rcu_data *rdp;
> -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);

Rather than marking a local variable as __maybe_unused (such that the
compiler can no longer help detect it as unused), could you move it into
the portion of the function that uses it, so that if reached, it'll
always get used?

>  	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
>  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> @@ -712,12 +771,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
>  		do_nocb_deferred_wakeup(rdp);
>  	}
>  	rcu_prepare_for_idle();
> -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> -	smp_mb__before_atomic();  /* See above. */
> -	atomic_inc(&rdtp->dynticks);
> -	smp_mb__after_atomic();  /* Force ordering with next sojourn. */
> -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> -		     atomic_read(&rdtp->dynticks) & 0x1);
> +	rcu_dynticks_eqs_enter();
>  	rcu_dynticks_task_enter();
>  
>  	/*
> @@ -846,15 +900,10 @@ void rcu_irq_exit_irqson(void)
>   */
>  static void rcu_eqs_exit_common(long long oldval, int user)
>  {
> -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);

Same comment as above.

>  	rcu_dynticks_task_exit();
> -	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
> -	atomic_inc(&rdtp->dynticks);
> -	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
> -	smp_mb__after_atomic();  /* See above. */
> -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> -		     !(atomic_read(&rdtp->dynticks) & 0x1));
> +	rcu_dynticks_eqs_exit();
>  	rcu_cleanup_after_idle();
>  	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
>  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> @@ -1001,11 +1050,7 @@ void rcu_nmi_enter(void)
>  	 * period (observation due to Andy Lutomirski).
>  	 */
>  	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
> -		smp_mb__before_atomic();  /* Force delay from prior write. */
> -		atomic_inc(&rdtp->dynticks);
> -		/* atomic_inc() before later RCU read-side crit sects */
> -		smp_mb__after_atomic();  /* See above. */
> -		WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
> +		rcu_dynticks_eqs_exit();
>  		incby = 1;
>  	}
>  	rdtp->dynticks_nmi_nesting += incby;
> @@ -1043,11 +1088,7 @@ void rcu_nmi_exit(void)
>  
>  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
>  	rdtp->dynticks_nmi_nesting = 0;
> -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> -	smp_mb__before_atomic();  /* See above. */
> -	atomic_inc(&rdtp->dynticks);
> -	smp_mb__after_atomic();  /* Force delay to next write. */
> -	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
> +	rcu_dynticks_eqs_enter();
>  }
>  
>  /**
> @@ -3800,8 +3841,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
>  		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
>  	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
>  	rcu_sysidle_init_percpu_data(rdp->dynticks);
> -	atomic_set(&rdp->dynticks->dynticks,
> -		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
> +	rcu_dynticks_eqs_online();
>  	raw_spin_unlock_rcu_node(rnp);		/* irqs remain disabled. */
>  
>  	/*
> -- 
> 2.5.2
> 

Re: [PATCH tip/core/rcu 0/6] Dynticks updates for 4.11

[Josh Triplett]

On Sat, Jan 14, 2017 at 12:54:06AM -0800, Paul E. McKenney wrote:
> Hello!
> 
> This series provides dynticks updates:
> 
> 1-4.	Abstract access to the dyntick counter, replacing the current
> 	open-coding of atomic operations.
> 
> 5.	Check cond_resched_rcu_qs() state less often to reduce GP overhead.
> 
> 6.	Adjust FQS offline checks for exact online-CPU detection. 

I replied to patches 1 and 3 with feedback.  For patches 2 and 4-6:
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

> 							Thanx, Paul
> 
> ------------------------------------------------------------------------
> 
>  include/linux/rcutiny.h    |    6 +
>  include/trace/events/rcu.h |   10 -
>  kernel/rcu/tree.c          |  245 +++++++++++++++++++++++++++++++--------------
>  kernel/rcu/tree.h          |    2 
>  kernel/rcu/tree_exp.h      |   12 --
>  kernel/rcu/tree_plugin.h   |    2 
>  kernel/rcu/tree_trace.c    |    2 
>  7 files changed, 190 insertions(+), 89 deletions(-)
> 

Re: [PATCH tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation

[Paul E. McKenney]

On Sun, Jan 15, 2017 at 11:39:51PM -0800, Josh Triplett wrote:
> On Sat, Jan 14, 2017 at 12:54:40AM -0800, Paul E. McKenney wrote:
> > This commit is the first step towards full abstraction of all accesses to
> > the ->dynticks counter, implementing the previously open-coded atomic add
> > of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
> > will ease changes to the ->dynticks counter operation.
> > 
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> This change has an additional effect not documented in the commit
> message: it eliminates the smp_mb__before_atomic and
> smp_mb__after_atomic calls.  Can you please document that in the commit
> message, and explain why that doesn't cause a problem?

The trick is that the old code used the non-value-returning atomic_add(),
which does not imply ordering, hence the smp_mb__before_atomic() and
smp_mb__after_atomic() calls.  The new code uses atomic_add_return(),
which does return a value, and therefore implies full ordering in and
of itself.

How would you like me to proceed?

							Thanx, Paul

> > --- a/kernel/rcu/tree.c
> > +++ b/kernel/rcu/tree.c
> > @@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
> >  #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
> >  };
> >  
> > +/*
> > + * Do a double-increment of the ->dynticks counter to emulate a
> > + * momentary idle-CPU quiescent state.
> > + */
> > +static void rcu_dynticks_momentary_idle(void)
> > +{
> > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +	int special = atomic_add_return(2, &rdtp->dynticks);
> > +
> > +	/* It is illegal to call this from idle state. */
> > +	WARN_ON_ONCE(!(special & 0x1));
> > +}
> > +
> >  DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
> >  EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
> >  
> > @@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
> >  static void rcu_momentary_dyntick_idle(void)
> >  {
> >  	struct rcu_data *rdp;
> > -	struct rcu_dynticks *rdtp;
> >  	int resched_mask;
> >  	struct rcu_state *rsp;
> >  
> > @@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
> >  		 * quiescent state, with no need for this CPU to do anything
> >  		 * further.
> >  		 */
> > -		rdtp = this_cpu_ptr(&rcu_dynticks);
> > -		smp_mb__before_atomic(); /* Earlier stuff before QS. */
> > -		atomic_add(2, &rdtp->dynticks);  /* QS. */
> > -		smp_mb__after_atomic(); /* Later stuff after QS. */
> > +		rcu_dynticks_momentary_idle();
> >  		break;
> >  	}
> >  }
> > -- 
> > 2.5.2
> > 
> 

Re: [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Paul E. McKenney]

On Sun, Jan 15, 2017 at 11:47:35PM -0800, Josh Triplett wrote:
> On Sat, Jan 14, 2017 at 12:54:42AM -0800, Paul E. McKenney wrote:
> > This commit is the third step towards full abstraction of all accesses
> > to the ->dynticks counter, implementing the previously open-coded atomic
> > add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
> > the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
> > This abstraction will ease changes to the ->dynticks counter operation.
> > 
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> A couple of comments below.  With those addressed:
> Reviewed-by: Josh Triplett <josh@joshtriplett.org>
> 
> >  kernel/rcu/tree.c | 92 +++++++++++++++++++++++++++++++++++++++----------------
> >  1 file changed, 66 insertions(+), 26 deletions(-)
> > 
> > diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> > index 805d55ee0b2a..fc49e008963a 100644
> > --- a/kernel/rcu/tree.c
> > +++ b/kernel/rcu/tree.c
> > @@ -282,6 +282,65 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
> >  };
> >  
> >  /*
> > + * Record entry into an extended quiescent state.  This is only to be
> > + * called when not already in an extended quiescent state.
> > + */
> > +static void rcu_dynticks_eqs_enter(void)
> > +{
> > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +
> > +	/*
> > +	 * CPUs seeing atomic_inc() must see prior RCU read-side critical
> > +	 * sections, and we also must force ordering with the next idle
> > +	 * sojourn.
> > +	 */
> > +	smp_mb__before_atomic(); /* See above. */
> > +	atomic_inc(&rdtp->dynticks);
> > +	smp_mb__after_atomic(); /* See above. */
> > +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > +		     atomic_read(&rdtp->dynticks) & 0x1);
> > +}
> > +
> > +/*
> > + * Record exit from an extended quiescent state.  This is only to be
> > + * called from an extended quiescent state.
> > + */
> > +static void rcu_dynticks_eqs_exit(void)
> > +{
> > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +
> > +	/*
> > +	 * CPUs seeing atomic_inc() must see prior idle sojourns,
> > +	 * and we also must force ordering with the next RCU read-side
> > +	 * critical section.
> > +	 */
> > +	smp_mb__before_atomic(); /* See above. */
> > +	atomic_inc(&rdtp->dynticks);
> > +	smp_mb__after_atomic(); /* See above. */
> > +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > +		     !(atomic_read(&rdtp->dynticks) & 0x1));
> > +}
> > +
> > +/*
> > + * Reset the current CPU's ->dynticks counter to indicate that the
> > + * newly onlined CPU is no longer in an extended quiescent state.
> > + * This will either leave the counter unchanged, or increment it
> > + * to the next non-quiescent value.
> > + *
> > + * The non-atomic test/increment sequence works because the upper bits
> > + * of the ->dynticks counter are manipulated only by the corresponding CPU,
> > + * or when the corresponding CPU is offline.
> > + */
> > +static void rcu_dynticks_eqs_online(void)
> > +{
> > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +
> > +	if (atomic_read(&rdtp->dynticks) & 0x1)
> > +		return;
> > +	atomic_add(0x1, &rdtp->dynticks);
> > +}
> > +
> > +/*
> >   * Snapshot the ->dynticks counter with full ordering so as to allow
> >   * stable comparison of this counter with past and future snapshots.
> >   */
> > @@ -693,7 +752,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
> >  {
> >  	struct rcu_state *rsp;
> >  	struct rcu_data *rdp;
> > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
> 
> Rather than marking a local variable as __maybe_unused (such that the
> compiler can no longer help detect it as unused), could you move it into
> the portion of the function that uses it, so that if reached, it'll
> always get used?
> 
> >  	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);

Its only use is in the above event trace, which can be disabled via
CONFIG_RCU_TRACE=n.  I could put the definition of rdtp under #ifdef,
but this seems ugly.  I could eliminate the variable, substituting
the initialization for rdtp in the event trace, but that would make
for a very long line, or an odd line break.

Or am I missing something here?

> >  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > @@ -712,12 +771,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
> >  		do_nocb_deferred_wakeup(rdp);
> >  	}
> >  	rcu_prepare_for_idle();
> > -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> > -	smp_mb__before_atomic();  /* See above. */
> > -	atomic_inc(&rdtp->dynticks);
> > -	smp_mb__after_atomic();  /* Force ordering with next sojourn. */
> > -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > -		     atomic_read(&rdtp->dynticks) & 0x1);
> > +	rcu_dynticks_eqs_enter();
> >  	rcu_dynticks_task_enter();
> >  
> >  	/*
> > @@ -846,15 +900,10 @@ void rcu_irq_exit_irqson(void)
> >   */
> >  static void rcu_eqs_exit_common(long long oldval, int user)
> >  {
> > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
> 
> Same comment as above.
> 
> >  	rcu_dynticks_task_exit();
> > -	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
> > -	atomic_inc(&rdtp->dynticks);
> > -	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
> > -	smp_mb__after_atomic();  /* See above. */
> > -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > -		     !(atomic_read(&rdtp->dynticks) & 0x1));
> > +	rcu_dynticks_eqs_exit();
> >  	rcu_cleanup_after_idle();
> >  	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);

This one is used here, at the top level of the function, and a few
lines down.  I see the same options available.  Thoughts?

							Thanx, Paul

> >  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > @@ -1001,11 +1050,7 @@ void rcu_nmi_enter(void)
> >  	 * period (observation due to Andy Lutomirski).
> >  	 */
> >  	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
> > -		smp_mb__before_atomic();  /* Force delay from prior write. */
> > -		atomic_inc(&rdtp->dynticks);
> > -		/* atomic_inc() before later RCU read-side crit sects */
> > -		smp_mb__after_atomic();  /* See above. */
> > -		WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
> > +		rcu_dynticks_eqs_exit();
> >  		incby = 1;
> >  	}
> >  	rdtp->dynticks_nmi_nesting += incby;
> > @@ -1043,11 +1088,7 @@ void rcu_nmi_exit(void)
> >  
> >  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
> >  	rdtp->dynticks_nmi_nesting = 0;
> > -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> > -	smp_mb__before_atomic();  /* See above. */
> > -	atomic_inc(&rdtp->dynticks);
> > -	smp_mb__after_atomic();  /* Force delay to next write. */
> > -	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
> > +	rcu_dynticks_eqs_enter();
> >  }
> >  
> >  /**
> > @@ -3800,8 +3841,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
> >  		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
> >  	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
> >  	rcu_sysidle_init_percpu_data(rdp->dynticks);
> > -	atomic_set(&rdp->dynticks->dynticks,
> > -		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
> > +	rcu_dynticks_eqs_online();
> >  	raw_spin_unlock_rcu_node(rnp);		/* irqs remain disabled. */
> >  
> >  	/*
> > -- 
> > 2.5.2
> > 
> 

Re: [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Peter Zijlstra]

On Sat, Jan 14, 2017 at 12:54:42AM -0800, Paul E. McKenney wrote:
>  /*
> + * Record entry into an extended quiescent state.  This is only to be
> + * called when not already in an extended quiescent state.
> + */
> +static void rcu_dynticks_eqs_enter(void)
> +{
> +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +
> +	/*
> +	 * CPUs seeing atomic_inc() must see prior RCU read-side critical
> +	 * sections, and we also must force ordering with the next idle
> +	 * sojourn.
> +	 */
> +	smp_mb__before_atomic(); /* See above. */
> +	atomic_inc(&rdtp->dynticks);
> +	smp_mb__after_atomic(); /* See above. */
> +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> +		     atomic_read(&rdtp->dynticks) & 0x1);
> +}

In an earlier patch you replaced things with atomic_add_return(), why
not use atomic_inc_return() here?

Re: [PATCH tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation

[Josh Triplett]

On Mon, Jan 16, 2017 at 03:22:39AM -0800, Paul E. McKenney wrote:
> On Sun, Jan 15, 2017 at 11:39:51PM -0800, Josh Triplett wrote:
> > On Sat, Jan 14, 2017 at 12:54:40AM -0800, Paul E. McKenney wrote:
> > > This commit is the first step towards full abstraction of all accesses to
> > > the ->dynticks counter, implementing the previously open-coded atomic add
> > > of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
> > > will ease changes to the ->dynticks counter operation.
> > > 
> > > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> > 
> > This change has an additional effect not documented in the commit
> > message: it eliminates the smp_mb__before_atomic and
> > smp_mb__after_atomic calls.  Can you please document that in the commit
> > message, and explain why that doesn't cause a problem?
> 
> The trick is that the old code used the non-value-returning atomic_add(),
> which does not imply ordering, hence the smp_mb__before_atomic() and
> smp_mb__after_atomic() calls.  The new code uses atomic_add_return(),
> which does return a value, and therefore implies full ordering in and
> of itself.
> 
> How would you like me to proceed?

With the above explanation added to the commit message:

Reviewed-by: Josh Triplett <josh@joshtriplett.org>

Re: [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Josh Triplett]

On Mon, Jan 16, 2017 at 03:34:20AM -0800, Paul E. McKenney wrote:
> On Sun, Jan 15, 2017 at 11:47:35PM -0800, Josh Triplett wrote:
> > On Sat, Jan 14, 2017 at 12:54:42AM -0800, Paul E. McKenney wrote:
> > > This commit is the third step towards full abstraction of all accesses
> > > to the ->dynticks counter, implementing the previously open-coded atomic
> > > add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
> > > the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
> > > This abstraction will ease changes to the ->dynticks counter operation.
> > > 
> > > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> > 
> > A couple of comments below.  With those addressed:
> > Reviewed-by: Josh Triplett <josh@joshtriplett.org>
> > 
> > >  kernel/rcu/tree.c | 92 +++++++++++++++++++++++++++++++++++++++----------------
> > >  1 file changed, 66 insertions(+), 26 deletions(-)
> > > 
> > > diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> > > index 805d55ee0b2a..fc49e008963a 100644
> > > --- a/kernel/rcu/tree.c
> > > +++ b/kernel/rcu/tree.c
> > > @@ -282,6 +282,65 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
> > >  };
> > >  
> > >  /*
> > > + * Record entry into an extended quiescent state.  This is only to be
> > > + * called when not already in an extended quiescent state.
> > > + */
> > > +static void rcu_dynticks_eqs_enter(void)
> > > +{
> > > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > +
> > > +	/*
> > > +	 * CPUs seeing atomic_inc() must see prior RCU read-side critical
> > > +	 * sections, and we also must force ordering with the next idle
> > > +	 * sojourn.
> > > +	 */
> > > +	smp_mb__before_atomic(); /* See above. */
> > > +	atomic_inc(&rdtp->dynticks);
> > > +	smp_mb__after_atomic(); /* See above. */
> > > +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > > +		     atomic_read(&rdtp->dynticks) & 0x1);
> > > +}
> > > +
> > > +/*
> > > + * Record exit from an extended quiescent state.  This is only to be
> > > + * called from an extended quiescent state.
> > > + */
> > > +static void rcu_dynticks_eqs_exit(void)
> > > +{
> > > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > +
> > > +	/*
> > > +	 * CPUs seeing atomic_inc() must see prior idle sojourns,
> > > +	 * and we also must force ordering with the next RCU read-side
> > > +	 * critical section.
> > > +	 */
> > > +	smp_mb__before_atomic(); /* See above. */
> > > +	atomic_inc(&rdtp->dynticks);
> > > +	smp_mb__after_atomic(); /* See above. */
> > > +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > > +		     !(atomic_read(&rdtp->dynticks) & 0x1));
> > > +}
> > > +
> > > +/*
> > > + * Reset the current CPU's ->dynticks counter to indicate that the
> > > + * newly onlined CPU is no longer in an extended quiescent state.
> > > + * This will either leave the counter unchanged, or increment it
> > > + * to the next non-quiescent value.
> > > + *
> > > + * The non-atomic test/increment sequence works because the upper bits
> > > + * of the ->dynticks counter are manipulated only by the corresponding CPU,
> > > + * or when the corresponding CPU is offline.
> > > + */
> > > +static void rcu_dynticks_eqs_online(void)
> > > +{
> > > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > +
> > > +	if (atomic_read(&rdtp->dynticks) & 0x1)
> > > +		return;
> > > +	atomic_add(0x1, &rdtp->dynticks);
> > > +}
> > > +
> > > +/*
> > >   * Snapshot the ->dynticks counter with full ordering so as to allow
> > >   * stable comparison of this counter with past and future snapshots.
> > >   */
> > > @@ -693,7 +752,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
> > >  {
> > >  	struct rcu_state *rsp;
> > >  	struct rcu_data *rdp;
> > > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > +	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
> > 
> > Rather than marking a local variable as __maybe_unused (such that the
> > compiler can no longer help detect it as unused), could you move it into
> > the portion of the function that uses it, so that if reached, it'll
> > always get used?
> > 
> > >  	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
> 
> Its only use is in the above event trace, which can be disabled via
> CONFIG_RCU_TRACE=n.  I could put the definition of rdtp under #ifdef,
> but this seems ugly.  I could eliminate the variable, substituting
> the initialization for rdtp in the event trace, but that would make
> for a very long line, or an odd line break.

For the trace_rcu_dyntick calls, you could create a small static inline
helper to do the tracing, eliminating the third argument.  Or you could
create a helper that returns
this_cpu_ptr(&rcu_dynticks)->dynticks_nesting.  Either way would work.

Or, if you prefer, you could wrap the variable
declaration/initialization in RCU_TRACE() rather than adding
__maybe_unused.

> > >  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > > @@ -712,12 +771,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
> > >  		do_nocb_deferred_wakeup(rdp);
> > >  	}
> > >  	rcu_prepare_for_idle();
> > > -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> > > -	smp_mb__before_atomic();  /* See above. */
> > > -	atomic_inc(&rdtp->dynticks);
> > > -	smp_mb__after_atomic();  /* Force ordering with next sojourn. */
> > > -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > > -		     atomic_read(&rdtp->dynticks) & 0x1);
> > > +	rcu_dynticks_eqs_enter();
> > >  	rcu_dynticks_task_enter();
> > >  
> > >  	/*
> > > @@ -846,15 +900,10 @@ void rcu_irq_exit_irqson(void)
> > >   */
> > >  static void rcu_eqs_exit_common(long long oldval, int user)
> > >  {
> > > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > +	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
> > 
> > Same comment as above.
> > 
> > >  	rcu_dynticks_task_exit();
> > > -	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
> > > -	atomic_inc(&rdtp->dynticks);
> > > -	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
> > > -	smp_mb__after_atomic();  /* See above. */
> > > -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > > -		     !(atomic_read(&rdtp->dynticks) & 0x1));
> > > +	rcu_dynticks_eqs_exit();
> > >  	rcu_cleanup_after_idle();
> > >  	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
> 
> This one is used here, at the top level of the function, and a few
> lines down.  I see the same options available.  Thoughts?

In the fourth patch, you introduce a helper that replaces the use in
atomic_read below.  That leaves only the tracing use, where the same
helper as above would apply.

> 							Thanx, Paul
> 
> > >  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > > @@ -1001,11 +1050,7 @@ void rcu_nmi_enter(void)
> > >  	 * period (observation due to Andy Lutomirski).
> > >  	 */
> > >  	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
> > > -		smp_mb__before_atomic();  /* Force delay from prior write. */
> > > -		atomic_inc(&rdtp->dynticks);
> > > -		/* atomic_inc() before later RCU read-side crit sects */
> > > -		smp_mb__after_atomic();  /* See above. */
> > > -		WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
> > > +		rcu_dynticks_eqs_exit();
> > >  		incby = 1;
> > >  	}
> > >  	rdtp->dynticks_nmi_nesting += incby;
> > > @@ -1043,11 +1088,7 @@ void rcu_nmi_exit(void)
> > >  
> > >  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
> > >  	rdtp->dynticks_nmi_nesting = 0;
> > > -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> > > -	smp_mb__before_atomic();  /* See above. */
> > > -	atomic_inc(&rdtp->dynticks);
> > > -	smp_mb__after_atomic();  /* Force delay to next write. */
> > > -	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
> > > +	rcu_dynticks_eqs_enter();
> > >  }
> > >  
> > >  /**
> > > @@ -3800,8 +3841,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
> > >  		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
> > >  	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
> > >  	rcu_sysidle_init_percpu_data(rdp->dynticks);
> > > -	atomic_set(&rdp->dynticks->dynticks,
> > > -		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
> > > +	rcu_dynticks_eqs_online();
> > >  	raw_spin_unlock_rcu_node(rnp);		/* irqs remain disabled. */
> > >  
> > >  	/*
> > > -- 
> > > 2.5.2
> > > 
> > 
> 

Re: [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Paul E. McKenney]

On Mon, Jan 16, 2017 at 05:43:37PM +0100, Peter Zijlstra wrote:
> On Sat, Jan 14, 2017 at 12:54:42AM -0800, Paul E. McKenney wrote:
> >  /*
> > + * Record entry into an extended quiescent state.  This is only to be
> > + * called when not already in an extended quiescent state.
> > + */
> > +static void rcu_dynticks_eqs_enter(void)
> > +{
> > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +
> > +	/*
> > +	 * CPUs seeing atomic_inc() must see prior RCU read-side critical
> > +	 * sections, and we also must force ordering with the next idle
> > +	 * sojourn.
> > +	 */
> > +	smp_mb__before_atomic(); /* See above. */
> > +	atomic_inc(&rdtp->dynticks);
> > +	smp_mb__after_atomic(); /* See above. */
> > +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > +		     atomic_read(&rdtp->dynticks) & 0x1);
> > +}
> 
> In an earlier patch you replaced things with atomic_add_return(), why
> not use atomic_inc_return() here?

The rationale is that the earlier patch uses the return value
unconditionally, while at this site, the return value would be used only
if CONFIG_RCU_EQS_DEBUG.  But mostly inertia.

							Thanx, Paul

Re: [PATCH tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation

[Paul E. McKenney]

On Mon, Jan 16, 2017 at 10:57:04AM -0800, Josh Triplett wrote:
> On Mon, Jan 16, 2017 at 03:22:39AM -0800, Paul E. McKenney wrote:
> > On Sun, Jan 15, 2017 at 11:39:51PM -0800, Josh Triplett wrote:
> > > On Sat, Jan 14, 2017 at 12:54:40AM -0800, Paul E. McKenney wrote:
> > > > This commit is the first step towards full abstraction of all accesses to
> > > > the ->dynticks counter, implementing the previously open-coded atomic add
> > > > of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
> > > > will ease changes to the ->dynticks counter operation.
> > > > 
> > > > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> > > 
> > > This change has an additional effect not documented in the commit
> > > message: it eliminates the smp_mb__before_atomic and
> > > smp_mb__after_atomic calls.  Can you please document that in the commit
> > > message, and explain why that doesn't cause a problem?
> > 
> > The trick is that the old code used the non-value-returning atomic_add(),
> > which does not imply ordering, hence the smp_mb__before_atomic() and
> > smp_mb__after_atomic() calls.  The new code uses atomic_add_return(),
> > which does return a value, and therefore implies full ordering in and
> > of itself.
> > 
> > How would you like me to proceed?
> 
> With the above explanation added to the commit message:
> 
> Reviewed-by: Josh Triplett <josh@joshtriplett.org>

Done, thank you!

							Thanx, Paul

Re: [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Paul E. McKenney]

On Mon, Jan 16, 2017 at 01:01:11PM -0800, Paul E. McKenney wrote:
> On Mon, Jan 16, 2017 at 05:43:37PM +0100, Peter Zijlstra wrote:
> > On Sat, Jan 14, 2017 at 12:54:42AM -0800, Paul E. McKenney wrote:
> > >  /*
> > > + * Record entry into an extended quiescent state.  This is only to be
> > > + * called when not already in an extended quiescent state.
> > > + */
> > > +static void rcu_dynticks_eqs_enter(void)
> > > +{
> > > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > +
> > > +	/*
> > > +	 * CPUs seeing atomic_inc() must see prior RCU read-side critical
> > > +	 * sections, and we also must force ordering with the next idle
> > > +	 * sojourn.
> > > +	 */
> > > +	smp_mb__before_atomic(); /* See above. */
> > > +	atomic_inc(&rdtp->dynticks);
> > > +	smp_mb__after_atomic(); /* See above. */
> > > +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > > +		     atomic_read(&rdtp->dynticks) & 0x1);
> > > +}
> > 
> > In an earlier patch you replaced things with atomic_add_return(), why
> > not use atomic_inc_return() here?
> 
> The rationale is that the earlier patch uses the return value
> unconditionally, while at this site, the return value would be used only
> if CONFIG_RCU_EQS_DEBUG.  But mostly inertia.

But saving two lines is saving two lines.  Changed to atomic_inc_return().

							Thanx, Paul

Re: [PATCH tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Paul E. McKenney]

On Mon, Jan 16, 2017 at 11:25:38AM -0800, Josh Triplett wrote:
> On Mon, Jan 16, 2017 at 03:34:20AM -0800, Paul E. McKenney wrote:
> > On Sun, Jan 15, 2017 at 11:47:35PM -0800, Josh Triplett wrote:
> > > On Sat, Jan 14, 2017 at 12:54:42AM -0800, Paul E. McKenney wrote:

[ . . . ]

> > > > @@ -693,7 +752,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
> > > >  {
> > > >  	struct rcu_state *rsp;
> > > >  	struct rcu_data *rdp;
> > > > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > > +	struct rcu_dynticks __maybe_unused *rdtp = this_cpu_ptr(&rcu_dynticks);
> > > 
> > > Rather than marking a local variable as __maybe_unused (such that the
> > > compiler can no longer help detect it as unused), could you move it into
> > > the portion of the function that uses it, so that if reached, it'll
> > > always get used?
> > > 
> > > >  	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
> > 
> > Its only use is in the above event trace, which can be disabled via
> > CONFIG_RCU_TRACE=n.  I could put the definition of rdtp under #ifdef,
> > but this seems ugly.  I could eliminate the variable, substituting
> > the initialization for rdtp in the event trace, but that would make
> > for a very long line, or an odd line break.
> 
> For the trace_rcu_dyntick calls, you could create a small static inline
> helper to do the tracing, eliminating the third argument.  Or you could
> create a helper that returns
> this_cpu_ptr(&rcu_dynticks)->dynticks_nesting.  Either way would work.
> 
> Or, if you prefer, you could wrap the variable
> declaration/initialization in RCU_TRACE() rather than adding
> __maybe_unused.

RCU_TRACE() it is in both cases, thank you!  (You would think I would
remember the code that I wrote...)

							Thanx, Paul

Re: [PATCH tip/core/rcu 0/6] Dynticks updates for 4.11

[Paul E. McKenney]

Hello!

This series provides v2 of the dynticks updates:

1-4.	Abstract access to the dyntick counter, replacing the current
	open-coding of atomic operations.

5.	Check cond_resched_rcu_qs() state less often to reduce GP overhead.

6.	Adjust FQS offline checks for exact online-CPU detection. 

Changes since v1:

o	Apply feedback from Josh Triplett and Peter Zijlstra.

							Thanx, Paul

------------------------------------------------------------------------

 include/linux/rcutiny.h    |    6 +
 include/trace/events/rcu.h |   10 -
 kernel/rcu/tree.c          |  241 ++++++++++++++++++++++++++++++---------------
 kernel/rcu/tree.h          |    2 
 kernel/rcu/tree_exp.h      |   12 --
 kernel/rcu/tree_plugin.h   |    2 
 kernel/rcu/tree_trace.c    |    2 
 7 files changed, 186 insertions(+), 89 deletions(-)

[PATCH v2 tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation

[Paul E. McKenney]

This commit is the first step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic add
of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
will ease changes to the ->dynticks counter operation.

Note that this commit gets rid of the smp_mb__before_atomic() and the
smp_mb__after_atomic() calls that were previously present.  The reason
that this is OK from a memory-ordering perspective is that the atomic
operation is now atomic_add_return(), which, as a value-returning atomic,
guarantees full ordering.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 kernel/rcu/tree.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index cb4e2056ccf3..14e283c351f6 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 
+/*
+ * Do a double-increment of the ->dynticks counter to emulate a
+ * momentary idle-CPU quiescent state.
+ */
+static void rcu_dynticks_momentary_idle(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	int special = atomic_add_return(2, &rdtp->dynticks);
+
+	/* It is illegal to call this from idle state. */
+	WARN_ON_ONCE(!(special & 0x1));
+}
+
 DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
 EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 
@@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 static void rcu_momentary_dyntick_idle(void)
 {
 	struct rcu_data *rdp;
-	struct rcu_dynticks *rdtp;
 	int resched_mask;
 	struct rcu_state *rsp;
 
@@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
 		 * quiescent state, with no need for this CPU to do anything
 		 * further.
 		 */
-		rdtp = this_cpu_ptr(&rcu_dynticks);
-		smp_mb__before_atomic(); /* Earlier stuff before QS. */
-		atomic_add(2, &rdtp->dynticks);  /* QS. */
-		smp_mb__after_atomic(); /* Later stuff after QS. */
+		rcu_dynticks_momentary_idle();
 		break;
 	}
 }
-- 
2.5.2

[PATCH v2 tip/core/rcu 2/6] rcu: Abstract the dynticks snapshot operation

[Paul E. McKenney]

This commit is the second step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic
add of zero in a new rcu_dynticks_snap() function.  This abstraction will
ease changes o the ->dynticks counter operation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 kernel/rcu/tree.c     | 19 ++++++++++++++++---
 kernel/rcu/tree_exp.h |  6 ++----
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 14e283c351f6..805d55ee0b2a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -282,6 +282,17 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 };
 
 /*
+ * Snapshot the ->dynticks counter with full ordering so as to allow
+ * stable comparison of this counter with past and future snapshots.
+ */
+static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+{
+	int snap = atomic_add_return(0, &rdtp->dynticks);
+
+	return snap;
+}
+
+/*
  * Do a double-increment of the ->dynticks counter to emulate a
  * momentary idle-CPU quiescent state.
  */
@@ -1049,7 +1060,9 @@ void rcu_nmi_exit(void)
  */
 bool notrace __rcu_is_watching(void)
 {
-	return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	return atomic_read(&rdtp->dynticks) & 0x1;
 }
 
 /**
@@ -1132,7 +1145,7 @@ static int rcu_is_cpu_rrupt_from_idle(void)
 static int dyntick_save_progress_counter(struct rcu_data *rdp,
 					 bool *isidle, unsigned long *maxj)
 {
-	rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
+	rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
 	rcu_sysidle_check_cpu(rdp, isidle, maxj);
 	if ((rdp->dynticks_snap & 0x1) == 0) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
@@ -1157,7 +1170,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	int *rcrmp;
 	unsigned int snap;
 
-	curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks);
+	curr = (unsigned int)rcu_dynticks_snap(rdp->dynticks);
 	snap = (unsigned int)rdp->dynticks_snap;
 
 	/*
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index e59e1849b89a..011f626b2fd8 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -356,10 +356,9 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 		mask_ofl_test = 0;
 		for_each_leaf_node_possible_cpu(rnp, cpu) {
 			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-			struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
 
 			rdp->exp_dynticks_snap =
-				atomic_add_return(0, &rdtp->dynticks);
+				rcu_dynticks_snap(rdp->dynticks);
 			if (raw_smp_processor_id() == cpu ||
 			    !(rdp->exp_dynticks_snap & 0x1) ||
 			    !(rnp->qsmaskinitnext & rdp->grpmask))
@@ -380,12 +379,11 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 		for_each_leaf_node_possible_cpu(rnp, cpu) {
 			unsigned long mask = leaf_node_cpu_bit(rnp, cpu);
 			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-			struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
 
 			if (!(mask_ofl_ipi & mask))
 				continue;
 retry_ipi:
-			if (atomic_add_return(0, &rdtp->dynticks) !=
+			if (rcu_dynticks_snap(rdp->dynticks) !=
 			    rdp->exp_dynticks_snap) {
 				mask_ofl_test |= mask;
 				continue;
-- 
2.5.2

[PATCH v2 tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Paul E. McKenney]

This commit is the third step towards full abstraction of all accesses
to the ->dynticks counter, implementing the previously open-coded atomic
add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
This abstraction will ease changes to the ->dynticks counter operation.

Note that this commit gets rid of the smp_mb__before_atomic() and the
smp_mb__after_atomic() calls that were previously present.  The reason
that this is OK from a memory-ordering perspective is that the atomic
operation is now atomic_add_return(), which, as a value-returning atomic,
guarantees full ordering.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 88 +++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 62 insertions(+), 26 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 805d55ee0b2a..70b01e1983e6 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -282,6 +282,61 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 };
 
 /*
+ * Record entry into an extended quiescent state.  This is only to be
+ * called when not already in an extended quiescent state.
+ */
+static void rcu_dynticks_eqs_enter(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	int special;
+
+	/*
+	 * CPUs seeing atomic_inc_return() must see prior RCU read-side
+	 * critical sections, and we also must force ordering with the
+	 * next idle sojourn.
+	 */
+	special = atomic_inc_return(&rdtp->dynticks);
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && special & 0x1);
+}
+
+/*
+ * Record exit from an extended quiescent state.  This is only to be
+ * called from an extended quiescent state.
+ */
+static void rcu_dynticks_eqs_exit(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	int special;
+
+	/*
+	 * CPUs seeing atomic_inc_return() must see prior idle sojourns,
+	 * and we also must force ordering with the next RCU read-side
+	 * critical section.
+	 */
+	special = atomic_inc_return(&rdtp->dynticks);
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(special & 0x1));
+}
+
+/*
+ * Reset the current CPU's ->dynticks counter to indicate that the
+ * newly onlined CPU is no longer in an extended quiescent state.
+ * This will either leave the counter unchanged, or increment it
+ * to the next non-quiescent value.
+ *
+ * The non-atomic test/increment sequence works because the upper bits
+ * of the ->dynticks counter are manipulated only by the corresponding CPU,
+ * or when the corresponding CPU is offline.
+ */
+static void rcu_dynticks_eqs_online(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	if (atomic_read(&rdtp->dynticks) & 0x1)
+		return;
+	atomic_add(0x1, &rdtp->dynticks);
+}
+
+/*
  * Snapshot the ->dynticks counter with full ordering so as to allow
  * stable comparison of this counter with past and future snapshots.
  */
@@ -693,7 +748,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
 {
 	struct rcu_state *rsp;
 	struct rcu_data *rdp;
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks));
 
 	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
 	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -712,12 +767,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
 		do_nocb_deferred_wakeup(rdp);
 	}
 	rcu_prepare_for_idle();
-	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
-	smp_mb__before_atomic();  /* See above. */
-	atomic_inc(&rdtp->dynticks);
-	smp_mb__after_atomic();  /* Force ordering with next sojourn. */
-	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
-		     atomic_read(&rdtp->dynticks) & 0x1);
+	rcu_dynticks_eqs_enter();
 	rcu_dynticks_task_enter();
 
 	/*
@@ -846,15 +896,10 @@ void rcu_irq_exit_irqson(void)
  */
 static void rcu_eqs_exit_common(long long oldval, int user)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks));
 
 	rcu_dynticks_task_exit();
-	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
-	atomic_inc(&rdtp->dynticks);
-	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
-	smp_mb__after_atomic();  /* See above. */
-	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
-		     !(atomic_read(&rdtp->dynticks) & 0x1));
+	rcu_dynticks_eqs_exit();
 	rcu_cleanup_after_idle();
 	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
 	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -1001,11 +1046,7 @@ void rcu_nmi_enter(void)
 	 * period (observation due to Andy Lutomirski).
 	 */
 	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
-		smp_mb__before_atomic();  /* Force delay from prior write. */
-		atomic_inc(&rdtp->dynticks);
-		/* atomic_inc() before later RCU read-side crit sects */
-		smp_mb__after_atomic();  /* See above. */
-		WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+		rcu_dynticks_eqs_exit();
 		incby = 1;
 	}
 	rdtp->dynticks_nmi_nesting += incby;
@@ -1043,11 +1084,7 @@ void rcu_nmi_exit(void)
 
 	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
 	rdtp->dynticks_nmi_nesting = 0;
-	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
-	smp_mb__before_atomic();  /* See above. */
-	atomic_inc(&rdtp->dynticks);
-	smp_mb__after_atomic();  /* Force delay to next write. */
-	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+	rcu_dynticks_eqs_enter();
 }
 
 /**
@@ -3800,8 +3837,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
 		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
 	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
 	rcu_sysidle_init_percpu_data(rdp->dynticks);
-	atomic_set(&rdp->dynticks->dynticks,
-		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
+	rcu_dynticks_eqs_online();
 	raw_spin_unlock_rcu_node(rnp);		/* irqs remain disabled. */
 
 	/*
-- 
2.5.2

[PATCH v2 tip/core/rcu 4/6] rcu: Abstract extended quiescent state determination

[Paul E. McKenney]

This commit is the fourth step towards full abstraction of all accesses
to the ->dynticks counter, implementing previously open-coded checks and
comparisons in new rcu_dynticks_in_eqs() and rcu_dynticks_in_eqs_since()
functions.  This abstraction will ease changes to the ->dynticks counter
operation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 include/linux/rcutiny.h  |  6 ++++++
 kernel/rcu/tree.c        | 52 +++++++++++++++++++++++++++++++++++-------------
 kernel/rcu/tree.h        |  2 ++
 kernel/rcu/tree_exp.h    |  6 +++---
 kernel/rcu/tree_plugin.h |  2 +-
 kernel/rcu/tree_trace.c  |  2 +-
 6 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index ac81e4063b40..4f9b2fa2173d 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -27,6 +27,12 @@
 
 #include <linux/cache.h>
 
+struct rcu_dynticks;
+static inline int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+{
+	return 0;
+}
+
 static inline unsigned long get_state_synchronize_rcu(void)
 {
 	return 0;
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 70b01e1983e6..c5362b6c7ed3 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -337,10 +337,22 @@ static void rcu_dynticks_eqs_online(void)
 }
 
 /*
+ * Is the current CPU in an extended quiescent state?
+ *
+ * No ordering, as we are sampling CPU-local information.
+ */
+bool rcu_dynticks_curr_cpu_in_eqs(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	return !(atomic_read(&rdtp->dynticks) & 0x1);
+}
+
+/*
  * Snapshot the ->dynticks counter with full ordering so as to allow
  * stable comparison of this counter with past and future snapshots.
  */
-static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
 {
 	int snap = atomic_add_return(0, &rdtp->dynticks);
 
@@ -348,6 +360,25 @@ static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
 }
 
 /*
+ * Return true if the snapshot returned from rcu_dynticks_snap()
+ * indicates that RCU is in an extended quiescent state.
+ */
+static bool rcu_dynticks_in_eqs(int snap)
+{
+	return !(snap & 0x1);
+}
+
+/*
+ * Return true if the CPU corresponding to the specified rcu_dynticks
+ * structure has spent some time in an extended quiescent state since
+ * rcu_dynticks_snap() returned the specified snapshot.
+ */
+static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap)
+{
+	return snap != rcu_dynticks_snap(rdtp);
+}
+
+/*
  * Do a double-increment of the ->dynticks counter to emulate a
  * momentary idle-CPU quiescent state.
  */
@@ -1045,7 +1076,7 @@ void rcu_nmi_enter(void)
 	 * to be in the outermost NMI handler that interrupted an RCU-idle
 	 * period (observation due to Andy Lutomirski).
 	 */
-	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
+	if (rcu_dynticks_curr_cpu_in_eqs()) {
 		rcu_dynticks_eqs_exit();
 		incby = 1;
 	}
@@ -1071,7 +1102,7 @@ void rcu_nmi_exit(void)
 	 * to us!)
 	 */
 	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
-	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
 
 	/*
 	 * If the nesting level is not 1, the CPU wasn't RCU-idle, so
@@ -1097,9 +1128,7 @@ void rcu_nmi_exit(void)
  */
 bool notrace __rcu_is_watching(void)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-
-	return atomic_read(&rdtp->dynticks) & 0x1;
+	return !rcu_dynticks_curr_cpu_in_eqs();
 }
 
 /**
@@ -1184,7 +1213,7 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 {
 	rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
 	rcu_sysidle_check_cpu(rdp, isidle, maxj);
-	if ((rdp->dynticks_snap & 0x1) == 0) {
+	if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
 		if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4,
 				 rdp->mynode->gpnum))
@@ -1203,12 +1232,7 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 				    bool *isidle, unsigned long *maxj)
 {
-	unsigned int curr;
 	int *rcrmp;
-	unsigned int snap;
-
-	curr = (unsigned int)rcu_dynticks_snap(rdp->dynticks);
-	snap = (unsigned int)rdp->dynticks_snap;
 
 	/*
 	 * If the CPU passed through or entered a dynticks idle phase with
@@ -1218,7 +1242,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	 * read-side critical section that started before the beginning
 	 * of the current RCU grace period.
 	 */
-	if ((curr & 0x1) == 0 || UINT_CMP_GE(curr, snap + 2)) {
+	if (rcu_dynticks_in_eqs_since(rdp->dynticks, rdp->dynticks_snap)) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
 		rdp->dynticks_fqs++;
 		return 1;
@@ -3807,7 +3831,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
 	rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
 	rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
 	WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
-	WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
+	WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks)));
 	rdp->cpu = cpu;
 	rdp->rsp = rsp;
 	rcu_boot_init_nocb_percpu_data(rdp);
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index fe98dd24adf8..3b953dcf6afc 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -595,6 +595,8 @@ extern struct rcu_state rcu_bh_state;
 extern struct rcu_state rcu_preempt_state;
 #endif /* #ifdef CONFIG_PREEMPT_RCU */
 
+int rcu_dynticks_snap(struct rcu_dynticks *rdtp);
+
 #ifdef CONFIG_RCU_BOOST
 DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
 DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu);
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 011f626b2fd8..e155a465cf84 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -360,7 +360,7 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 			rdp->exp_dynticks_snap =
 				rcu_dynticks_snap(rdp->dynticks);
 			if (raw_smp_processor_id() == cpu ||
-			    !(rdp->exp_dynticks_snap & 0x1) ||
+			    rcu_dynticks_in_eqs(rdp->exp_dynticks_snap) ||
 			    !(rnp->qsmaskinitnext & rdp->grpmask))
 				mask_ofl_test |= rdp->grpmask;
 		}
@@ -383,8 +383,8 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 			if (!(mask_ofl_ipi & mask))
 				continue;
 retry_ipi:
-			if (rcu_dynticks_snap(rdp->dynticks) !=
-			    rdp->exp_dynticks_snap) {
+			if (rcu_dynticks_in_eqs_since(rdp->dynticks,
+						      rdp->exp_dynticks_snap)) {
 				mask_ofl_test |= mask;
 				continue;
 			}
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 56583e764ebf..652209589adf 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1643,7 +1643,7 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
 	       "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
 	       "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
 	       ticks_value, ticks_title,
-	       atomic_read(&rdtp->dynticks) & 0xfff,
+	       rcu_dynticks_snap(rdtp) & 0xfff,
 	       rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
 	       rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
 	       READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c
index b1f28972872c..b833cd0a29e8 100644
--- a/kernel/rcu/tree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -124,7 +124,7 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
 		   rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu),
 		   rdp->core_needs_qs);
 	seq_printf(m, " dt=%d/%llx/%d df=%lu",
-		   atomic_read(&rdp->dynticks->dynticks),
+		   rcu_dynticks_snap(rdp->dynticks),
 		   rdp->dynticks->dynticks_nesting,
 		   rdp->dynticks->dynticks_nmi_nesting,
 		   rdp->dynticks_fqs);
-- 
2.5.2

[PATCH v2 tip/core/rcu 5/6] rcu: Check cond_resched_rcu_qs() state less often to reduce GP overhead

[Paul E. McKenney]

Commit 4a81e8328d37 ("rcu: Reduce overhead of cond_resched() checks
for RCU") moved quiescent-state generation out of cond_resched()
and commit bde6c3aa9930 ("rcu: Provide cond_resched_rcu_qs() to force
quiescent states in long loops") introduced cond_resched_rcu_qs(), and
commit 5cd37193ce85 ("rcu: Make cond_resched_rcu_qs() apply to normal RCU
flavors") introduced the per-CPU rcu_qs_ctr variable, which is frequently
polled by the RCU core state machine.

This frequent polling can increase grace-period rate, which in turn
increases grace-period overhead, which is visible in some benchmarks
(for example, the "open1" benchmark in Anton Blanchard's "will it scale"
suite).  This commit therefore reduces the rate at which rcu_qs_ctr
is polled by moving that polling into the force-quiescent-state (FQS)
machinery, and by further polling it only after the grace period has
been in effect for at least jiffies_till_sched_qs jiffies.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 include/trace/events/rcu.h | 10 +++++-----
 kernel/rcu/tree.c          | 46 ++++++++++++++++++++++++++++++++++------------
 2 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index 9d4f9b3a2b7b..e3facb356838 100644
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -385,11 +385,11 @@ TRACE_EVENT(rcu_quiescent_state_report,
 
 /*
  * Tracepoint for quiescent states detected by force_quiescent_state().
- * These trace events include the type of RCU, the grace-period number
- * that was blocked by the CPU, the CPU itself, and the type of quiescent
- * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
- * or "kick" when kicking a CPU that has been in dyntick-idle mode for
- * too long.
+ * These trace events include the type of RCU, the grace-period number that
+ * was blocked by the CPU, the CPU itself, and the type of quiescent state,
+ * which can be "dti" for dyntick-idle mode, "ofl" for CPU offline, "kick"
+ * when kicking a CPU that has been in dyntick-idle mode for too long, or
+ * "rqc" if the CPU got a quiescent state via its rcu_qs_ctr.
  */
 TRACE_EVENT(rcu_fqs,
 
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index c5362b6c7ed3..37a1d91f1ec7 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1232,7 +1232,10 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 				    bool *isidle, unsigned long *maxj)
 {
+	unsigned long jtsq;
 	int *rcrmp;
+	unsigned long rjtsc;
+	struct rcu_node *rnp;
 
 	/*
 	 * If the CPU passed through or entered a dynticks idle phase with
@@ -1248,6 +1251,31 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 		return 1;
 	}
 
+	/* Compute and saturate jiffies_till_sched_qs. */
+	jtsq = jiffies_till_sched_qs;
+	rjtsc = rcu_jiffies_till_stall_check();
+	if (jtsq > rjtsc / 2) {
+		WRITE_ONCE(jiffies_till_sched_qs, rjtsc);
+		jtsq = rjtsc / 2;
+	} else if (jtsq < 1) {
+		WRITE_ONCE(jiffies_till_sched_qs, 1);
+		jtsq = 1;
+	}
+
+	/*
+	 * Has this CPU encountered a cond_resched_rcu_qs() since the
+	 * beginning of the grace period?  For this to be the case,
+	 * the CPU has to have noticed the current grace period.  This
+	 * might not be the case for nohz_full CPUs looping in the kernel.
+	 */
+	rnp = rdp->mynode;
+	if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
+	    READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_qs_ctr, rdp->cpu) &&
+	    READ_ONCE(rdp->gpnum) == rnp->gpnum && !rdp->gpwrap) {
+		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("rqc"));
+		return 1;
+	}
+
 	/*
 	 * Check for the CPU being offline, but only if the grace period
 	 * is old enough.  We don't need to worry about the CPU changing
@@ -1290,9 +1318,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	 * warning delay.
 	 */
 	rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu);
-	if (ULONG_CMP_GE(jiffies,
-			 rdp->rsp->gp_start + jiffies_till_sched_qs) ||
-	    ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
+	if (time_after(jiffies, rdp->rsp->gp_start + jtsq) ||
+	    time_after(jiffies, rdp->rsp->jiffies_resched)) {
 		if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) {
 			WRITE_ONCE(rdp->cond_resched_completed,
 				   READ_ONCE(rdp->mynode->completed));
@@ -2550,10 +2577,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
 
 	rnp = rdp->mynode;
 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
-	if ((rdp->cpu_no_qs.b.norm &&
-	     rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
-	    rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
-	    rdp->gpwrap) {
+	if (rdp->cpu_no_qs.b.norm || rdp->gpnum != rnp->gpnum ||
+	    rnp->completed == rnp->gpnum || rdp->gpwrap) {
 
 		/*
 		 * The grace period in which this quiescent state was
@@ -2608,8 +2633,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
 	 * Was there a quiescent state since the beginning of the grace
 	 * period? If no, then exit and wait for the next call.
 	 */
-	if (rdp->cpu_no_qs.b.norm &&
-	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
+	if (rdp->cpu_no_qs.b.norm)
 		return;
 
 	/*
@@ -3563,9 +3587,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
 	    rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
 	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
 		rdp->n_rp_core_needs_qs++;
-	} else if (rdp->core_needs_qs &&
-		   (!rdp->cpu_no_qs.b.norm ||
-		    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
+	} else if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm) {
 		rdp->n_rp_report_qs++;
 		return 1;
 	}
-- 
2.5.2

[PATCH v2 tip/core/rcu 6/6] rcu: Adjust FQS offline checks for exact online-CPU detection

[Paul E. McKenney]

Commit 7ec99de36f40 ("rcu: Provide exact CPU-online tracking for RCU"),
as its title suggests, got rid of RCU's remaining CPU-hotplug timing
guesswork.  This commit therefore removes the one-jiffy kludge that was
used to paper over this guesswork.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 kernel/rcu/tree.c | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 37a1d91f1ec7..c49bb791aeca 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1276,21 +1276,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 		return 1;
 	}
 
-	/*
-	 * Check for the CPU being offline, but only if the grace period
-	 * is old enough.  We don't need to worry about the CPU changing
-	 * state: If we see it offline even once, it has been through a
-	 * quiescent state.
-	 *
-	 * The reason for insisting that the grace period be at least
-	 * one jiffy old is that CPUs that are not quite online and that
-	 * have just gone offline can still execute RCU read-side critical
-	 * sections.
-	 */
-	if (ULONG_CMP_GE(rdp->rsp->gp_start + 2, jiffies))
-		return 0;  /* Grace period is not old enough. */
-	barrier();
-	if (cpu_is_offline(rdp->cpu)) {
+	/* Check for the CPU being offline. */
+	if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp))) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl"));
 		rdp->offline_fqs++;
 		return 1;
-- 
2.5.2

Re: [PATCH v2 tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Josh Triplett]

On Tue, Jan 17, 2017 at 06:45:25PM -0800, Paul E. McKenney wrote:
> This commit is the third step towards full abstraction of all accesses
> to the ->dynticks counter, implementing the previously open-coded atomic
> add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
> the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
> This abstraction will ease changes to the ->dynticks counter operation.
> 
> Note that this commit gets rid of the smp_mb__before_atomic() and the
> smp_mb__after_atomic() calls that were previously present.  The reason
> that this is OK from a memory-ordering perspective is that the atomic
> operation is now atomic_add_return(), which, as a value-returning atomic,
> guarantees full ordering.
> 
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

A couple of issues with RCU_TRACE usage noted below.  With those fixed:
Reviewed-by: Josh Triplett <josh@joshtriplett.org>.

>  kernel/rcu/tree.c | 88 +++++++++++++++++++++++++++++++++++++++----------------
>  1 file changed, 62 insertions(+), 26 deletions(-)
> 
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index 805d55ee0b2a..70b01e1983e6 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -282,6 +282,61 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
>  };
>  
>  /*
> + * Record entry into an extended quiescent state.  This is only to be
> + * called when not already in an extended quiescent state.
> + */
> +static void rcu_dynticks_eqs_enter(void)
> +{
> +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +	int special;
> +
> +	/*
> +	 * CPUs seeing atomic_inc_return() must see prior RCU read-side
> +	 * critical sections, and we also must force ordering with the
> +	 * next idle sojourn.
> +	 */
> +	special = atomic_inc_return(&rdtp->dynticks);
> +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && special & 0x1);
> +}
> +
> +/*
> + * Record exit from an extended quiescent state.  This is only to be
> + * called from an extended quiescent state.
> + */
> +static void rcu_dynticks_eqs_exit(void)
> +{
> +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +	int special;
> +
> +	/*
> +	 * CPUs seeing atomic_inc_return() must see prior idle sojourns,
> +	 * and we also must force ordering with the next RCU read-side
> +	 * critical section.
> +	 */
> +	special = atomic_inc_return(&rdtp->dynticks);
> +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(special & 0x1));
> +}
> +
> +/*
> + * Reset the current CPU's ->dynticks counter to indicate that the
> + * newly onlined CPU is no longer in an extended quiescent state.
> + * This will either leave the counter unchanged, or increment it
> + * to the next non-quiescent value.
> + *
> + * The non-atomic test/increment sequence works because the upper bits
> + * of the ->dynticks counter are manipulated only by the corresponding CPU,
> + * or when the corresponding CPU is offline.
> + */
> +static void rcu_dynticks_eqs_online(void)
> +{
> +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +
> +	if (atomic_read(&rdtp->dynticks) & 0x1)
> +		return;
> +	atomic_add(0x1, &rdtp->dynticks);
> +}
> +
> +/*
>   * Snapshot the ->dynticks counter with full ordering so as to allow
>   * stable comparison of this counter with past and future snapshots.
>   */
> @@ -693,7 +748,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
>  {
>  	struct rcu_state *rsp;
>  	struct rcu_data *rdp;
> -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +	RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks));

RCU_TRACE doesn't act like a statement, and doesn't take a semicolon; it
just acts like a shorthand for an #ifdef.  The semicolon belongs inside
the macro parentheses, since it goes with the declaration.  Otherwise,
this compiles to the empty statement ';' without tracing.  Among other
things, that would lead to surprises for anyone who added a subsequent
declaration, because ';' followed by a declaration will produce a
-Wdeclaration-after-statement warning.

>  	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
>  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> @@ -712,12 +767,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
>  		do_nocb_deferred_wakeup(rdp);
>  	}
>  	rcu_prepare_for_idle();
> -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> -	smp_mb__before_atomic();  /* See above. */
> -	atomic_inc(&rdtp->dynticks);
> -	smp_mb__after_atomic();  /* Force ordering with next sojourn. */
> -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> -		     atomic_read(&rdtp->dynticks) & 0x1);
> +	rcu_dynticks_eqs_enter();
>  	rcu_dynticks_task_enter();
>  
>  	/*
> @@ -846,15 +896,10 @@ void rcu_irq_exit_irqson(void)
>   */
>  static void rcu_eqs_exit_common(long long oldval, int user)
>  {
> -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> +	RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks));

As above.

>  	rcu_dynticks_task_exit();
> -	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
> -	atomic_inc(&rdtp->dynticks);
> -	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
> -	smp_mb__after_atomic();  /* See above. */
> -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> -		     !(atomic_read(&rdtp->dynticks) & 0x1));
> +	rcu_dynticks_eqs_exit();
>  	rcu_cleanup_after_idle();
>  	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
>  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> @@ -1001,11 +1046,7 @@ void rcu_nmi_enter(void)
>  	 * period (observation due to Andy Lutomirski).
>  	 */
>  	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
> -		smp_mb__before_atomic();  /* Force delay from prior write. */
> -		atomic_inc(&rdtp->dynticks);
> -		/* atomic_inc() before later RCU read-side crit sects */
> -		smp_mb__after_atomic();  /* See above. */
> -		WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
> +		rcu_dynticks_eqs_exit();
>  		incby = 1;
>  	}
>  	rdtp->dynticks_nmi_nesting += incby;
> @@ -1043,11 +1084,7 @@ void rcu_nmi_exit(void)
>  
>  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
>  	rdtp->dynticks_nmi_nesting = 0;
> -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> -	smp_mb__before_atomic();  /* See above. */
> -	atomic_inc(&rdtp->dynticks);
> -	smp_mb__after_atomic();  /* Force delay to next write. */
> -	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
> +	rcu_dynticks_eqs_enter();
>  }
>  
>  /**
> @@ -3800,8 +3837,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
>  		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
>  	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
>  	rcu_sysidle_init_percpu_data(rdp->dynticks);
> -	atomic_set(&rdp->dynticks->dynticks,
> -		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
> +	rcu_dynticks_eqs_online();
>  	raw_spin_unlock_rcu_node(rnp);		/* irqs remain disabled. */
>  
>  	/*
> -- 
> 2.5.2
> 

Re: [PATCH v2 tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Paul E. McKenney]

On Sat, Jan 21, 2017 at 12:49:29PM -0800, Josh Triplett wrote:
> On Tue, Jan 17, 2017 at 06:45:25PM -0800, Paul E. McKenney wrote:
> > This commit is the third step towards full abstraction of all accesses
> > to the ->dynticks counter, implementing the previously open-coded atomic
> > add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
> > the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
> > This abstraction will ease changes to the ->dynticks counter operation.
> > 
> > Note that this commit gets rid of the smp_mb__before_atomic() and the
> > smp_mb__after_atomic() calls that were previously present.  The reason
> > that this is OK from a memory-ordering perspective is that the atomic
> > operation is now atomic_add_return(), which, as a value-returning atomic,
> > guarantees full ordering.
> > 
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> A couple of issues with RCU_TRACE usage noted below.  With those fixed:
> Reviewed-by: Josh Triplett <josh@joshtriplett.org>.
> 
> >  kernel/rcu/tree.c | 88 +++++++++++++++++++++++++++++++++++++++----------------
> >  1 file changed, 62 insertions(+), 26 deletions(-)
> > 
> > diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> > index 805d55ee0b2a..70b01e1983e6 100644
> > --- a/kernel/rcu/tree.c
> > +++ b/kernel/rcu/tree.c
> > @@ -282,6 +282,61 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
> >  };
> >  
> >  /*
> > + * Record entry into an extended quiescent state.  This is only to be
> > + * called when not already in an extended quiescent state.
> > + */
> > +static void rcu_dynticks_eqs_enter(void)
> > +{
> > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +	int special;
> > +
> > +	/*
> > +	 * CPUs seeing atomic_inc_return() must see prior RCU read-side
> > +	 * critical sections, and we also must force ordering with the
> > +	 * next idle sojourn.
> > +	 */
> > +	special = atomic_inc_return(&rdtp->dynticks);
> > +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && special & 0x1);
> > +}
> > +
> > +/*
> > + * Record exit from an extended quiescent state.  This is only to be
> > + * called from an extended quiescent state.
> > + */
> > +static void rcu_dynticks_eqs_exit(void)
> > +{
> > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +	int special;
> > +
> > +	/*
> > +	 * CPUs seeing atomic_inc_return() must see prior idle sojourns,
> > +	 * and we also must force ordering with the next RCU read-side
> > +	 * critical section.
> > +	 */
> > +	special = atomic_inc_return(&rdtp->dynticks);
> > +	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(special & 0x1));
> > +}
> > +
> > +/*
> > + * Reset the current CPU's ->dynticks counter to indicate that the
> > + * newly onlined CPU is no longer in an extended quiescent state.
> > + * This will either leave the counter unchanged, or increment it
> > + * to the next non-quiescent value.
> > + *
> > + * The non-atomic test/increment sequence works because the upper bits
> > + * of the ->dynticks counter are manipulated only by the corresponding CPU,
> > + * or when the corresponding CPU is offline.
> > + */
> > +static void rcu_dynticks_eqs_online(void)
> > +{
> > +	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +
> > +	if (atomic_read(&rdtp->dynticks) & 0x1)
> > +		return;
> > +	atomic_add(0x1, &rdtp->dynticks);
> > +}
> > +
> > +/*
> >   * Snapshot the ->dynticks counter with full ordering so as to allow
> >   * stable comparison of this counter with past and future snapshots.
> >   */
> > @@ -693,7 +748,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
> >  {
> >  	struct rcu_state *rsp;
> >  	struct rcu_data *rdp;
> > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +	RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks));
> 
> RCU_TRACE doesn't act like a statement, and doesn't take a semicolon; it
> just acts like a shorthand for an #ifdef.  The semicolon belongs inside
> the macro parentheses, since it goes with the declaration.  Otherwise,
> this compiles to the empty statement ';' without tracing.  Among other
> things, that would lead to surprises for anyone who added a subsequent
> declaration, because ';' followed by a declaration will produce a
> -Wdeclaration-after-statement warning.

I do need the ";" inside the RCU_TRACE(), don't I?  And I have this bug
well-represented in the RCU code.  I am fixing these two in this commit
and will queue commits to fix the others.  I also added your Reviewed-by,
thank you!

And good catch!!!

							Thanx, Paul

> >  	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
> >  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > @@ -712,12 +767,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
> >  		do_nocb_deferred_wakeup(rdp);
> >  	}
> >  	rcu_prepare_for_idle();
> > -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> > -	smp_mb__before_atomic();  /* See above. */
> > -	atomic_inc(&rdtp->dynticks);
> > -	smp_mb__after_atomic();  /* Force ordering with next sojourn. */
> > -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > -		     atomic_read(&rdtp->dynticks) & 0x1);
> > +	rcu_dynticks_eqs_enter();
> >  	rcu_dynticks_task_enter();
> >  
> >  	/*
> > @@ -846,15 +896,10 @@ void rcu_irq_exit_irqson(void)
> >   */
> >  static void rcu_eqs_exit_common(long long oldval, int user)
> >  {
> > -	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
> > +	RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks));
> 
> As above.
> 
> >  	rcu_dynticks_task_exit();
> > -	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
> > -	atomic_inc(&rdtp->dynticks);
> > -	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
> > -	smp_mb__after_atomic();  /* See above. */
> > -	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > -		     !(atomic_read(&rdtp->dynticks) & 0x1));
> > +	rcu_dynticks_eqs_exit();
> >  	rcu_cleanup_after_idle();
> >  	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
> >  	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
> > @@ -1001,11 +1046,7 @@ void rcu_nmi_enter(void)
> >  	 * period (observation due to Andy Lutomirski).
> >  	 */
> >  	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
> > -		smp_mb__before_atomic();  /* Force delay from prior write. */
> > -		atomic_inc(&rdtp->dynticks);
> > -		/* atomic_inc() before later RCU read-side crit sects */
> > -		smp_mb__after_atomic();  /* See above. */
> > -		WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
> > +		rcu_dynticks_eqs_exit();
> >  		incby = 1;
> >  	}
> >  	rdtp->dynticks_nmi_nesting += incby;
> > @@ -1043,11 +1084,7 @@ void rcu_nmi_exit(void)
> >  
> >  	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
> >  	rdtp->dynticks_nmi_nesting = 0;
> > -	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
> > -	smp_mb__before_atomic();  /* See above. */
> > -	atomic_inc(&rdtp->dynticks);
> > -	smp_mb__after_atomic();  /* Force delay to next write. */
> > -	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
> > +	rcu_dynticks_eqs_enter();
> >  }
> >  
> >  /**
> > @@ -3800,8 +3837,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
> >  		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
> >  	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
> >  	rcu_sysidle_init_percpu_data(rdp->dynticks);
> > -	atomic_set(&rdp->dynticks->dynticks,
> > -		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
> > +	rcu_dynticks_eqs_online();
> >  	raw_spin_unlock_rcu_node(rnp);		/* irqs remain disabled. */
> >  
> >  	/*
> > -- 
> > 2.5.2
> > 
> 

Re: [PATCH v3 tip/core/rcu 0/6] Dynticks updates for 4.11

[Paul E. McKenney]

Hello!

This series provides v3 of the dynticks updates:

1-4.	Abstract access to the dyntick counter, replacing the current
	open-coding of atomic operations.

5.	Check cond_resched_rcu_qs() state less often to reduce GP overhead.

6.	Adjust FQS offline checks for exact online-CPU detection. 

Changes since v2:

o	Fix RCU_TRACE() statements called out by Josh Triplett.

Changes since v1:

o	Apply feedback from Josh Triplett and Peter Zijlstra.

							Thanx, Paul

------------------------------------------------------------------------

 include/linux/rcutiny.h    |    6 +
 include/trace/events/rcu.h |   10 -
 kernel/rcu/tree.c          |  241 ++++++++++++++++++++++++++++++---------------
 kernel/rcu/tree.h          |    2 
 kernel/rcu/tree_exp.h      |   12 --
 kernel/rcu/tree_plugin.h   |    2 
 kernel/rcu/tree_trace.c    |    2 
 7 files changed, 186 insertions(+), 89 deletions(-)

[PATCH v3 tip/core/rcu 1/6] rcu: Abstract the dynticks momentary-idle operation

[Paul E. McKenney]

This commit is the first step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic add
of two in a new rcu_dynticks_momentary_idle() function.  This abstraction
will ease changes to the ->dynticks counter operation.

Note that this commit gets rid of the smp_mb__before_atomic() and the
smp_mb__after_atomic() calls that were previously present.  The reason
that this is OK from a memory-ordering perspective is that the atomic
operation is now atomic_add_return(), which, as a value-returning atomic,
guarantees full ordering.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 kernel/rcu/tree.c | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index cb4e2056ccf3..14e283c351f6 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -281,6 +281,19 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 #endif /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */
 };
 
+/*
+ * Do a double-increment of the ->dynticks counter to emulate a
+ * momentary idle-CPU quiescent state.
+ */
+static void rcu_dynticks_momentary_idle(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	int special = atomic_add_return(2, &rdtp->dynticks);
+
+	/* It is illegal to call this from idle state. */
+	WARN_ON_ONCE(!(special & 0x1));
+}
+
 DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, rcu_qs_ctr);
 EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 
@@ -300,7 +313,6 @@ EXPORT_PER_CPU_SYMBOL_GPL(rcu_qs_ctr);
 static void rcu_momentary_dyntick_idle(void)
 {
 	struct rcu_data *rdp;
-	struct rcu_dynticks *rdtp;
 	int resched_mask;
 	struct rcu_state *rsp;
 
@@ -327,10 +339,7 @@ static void rcu_momentary_dyntick_idle(void)
 		 * quiescent state, with no need for this CPU to do anything
 		 * further.
 		 */
-		rdtp = this_cpu_ptr(&rcu_dynticks);
-		smp_mb__before_atomic(); /* Earlier stuff before QS. */
-		atomic_add(2, &rdtp->dynticks);  /* QS. */
-		smp_mb__after_atomic(); /* Later stuff after QS. */
+		rcu_dynticks_momentary_idle();
 		break;
 	}
 }
-- 
2.5.2

[PATCH v3 tip/core/rcu 2/6] rcu: Abstract the dynticks snapshot operation

[Paul E. McKenney]

This commit is the second step towards full abstraction of all accesses to
the ->dynticks counter, implementing the previously open-coded atomic
add of zero in a new rcu_dynticks_snap() function.  This abstraction will
ease changes o the ->dynticks counter operation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 kernel/rcu/tree.c     | 19 ++++++++++++++++---
 kernel/rcu/tree_exp.h |  6 ++----
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 14e283c351f6..805d55ee0b2a 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -282,6 +282,17 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 };
 
 /*
+ * Snapshot the ->dynticks counter with full ordering so as to allow
+ * stable comparison of this counter with past and future snapshots.
+ */
+static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+{
+	int snap = atomic_add_return(0, &rdtp->dynticks);
+
+	return snap;
+}
+
+/*
  * Do a double-increment of the ->dynticks counter to emulate a
  * momentary idle-CPU quiescent state.
  */
@@ -1049,7 +1060,9 @@ void rcu_nmi_exit(void)
  */
 bool notrace __rcu_is_watching(void)
 {
-	return atomic_read(this_cpu_ptr(&rcu_dynticks.dynticks)) & 0x1;
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	return atomic_read(&rdtp->dynticks) & 0x1;
 }
 
 /**
@@ -1132,7 +1145,7 @@ static int rcu_is_cpu_rrupt_from_idle(void)
 static int dyntick_save_progress_counter(struct rcu_data *rdp,
 					 bool *isidle, unsigned long *maxj)
 {
-	rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
+	rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
 	rcu_sysidle_check_cpu(rdp, isidle, maxj);
 	if ((rdp->dynticks_snap & 0x1) == 0) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
@@ -1157,7 +1170,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	int *rcrmp;
 	unsigned int snap;
 
-	curr = (unsigned int)atomic_add_return(0, &rdp->dynticks->dynticks);
+	curr = (unsigned int)rcu_dynticks_snap(rdp->dynticks);
 	snap = (unsigned int)rdp->dynticks_snap;
 
 	/*
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index e59e1849b89a..011f626b2fd8 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -356,10 +356,9 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 		mask_ofl_test = 0;
 		for_each_leaf_node_possible_cpu(rnp, cpu) {
 			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-			struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
 
 			rdp->exp_dynticks_snap =
-				atomic_add_return(0, &rdtp->dynticks);
+				rcu_dynticks_snap(rdp->dynticks);
 			if (raw_smp_processor_id() == cpu ||
 			    !(rdp->exp_dynticks_snap & 0x1) ||
 			    !(rnp->qsmaskinitnext & rdp->grpmask))
@@ -380,12 +379,11 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 		for_each_leaf_node_possible_cpu(rnp, cpu) {
 			unsigned long mask = leaf_node_cpu_bit(rnp, cpu);
 			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-			struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
 
 			if (!(mask_ofl_ipi & mask))
 				continue;
 retry_ipi:
-			if (atomic_add_return(0, &rdtp->dynticks) !=
+			if (rcu_dynticks_snap(rdp->dynticks) !=
 			    rdp->exp_dynticks_snap) {
 				mask_ofl_test |= mask;
 				continue;
-- 
2.5.2

[PATCH v3 tip/core/rcu 3/6] rcu: Abstract dynticks extended quiescent state enter/exit operations

[Paul E. McKenney]

This commit is the third step towards full abstraction of all accesses
to the ->dynticks counter, implementing the previously open-coded atomic
add of 1 and entry checks in a new rcu_dynticks_eqs_enter() function, and
the same but with exit checks in a new rcu_dynticks_eqs_exit() function.
This abstraction will ease changes to the ->dynticks counter operation.

Note that this commit gets rid of the smp_mb__before_atomic() and the
smp_mb__after_atomic() calls that were previously present.  The reason
that this is OK from a memory-ordering perspective is that the atomic
operation is now atomic_add_return(), which, as a value-returning atomic,
guarantees full ordering.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
[ paulmck: Fixed RCU_TRACE() statements added by this commit. ]
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 kernel/rcu/tree.c | 88 +++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 62 insertions(+), 26 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 805d55ee0b2a..3169d5a21b55 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -282,6 +282,61 @@ static DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
 };
 
 /*
+ * Record entry into an extended quiescent state.  This is only to be
+ * called when not already in an extended quiescent state.
+ */
+static void rcu_dynticks_eqs_enter(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	int special;
+
+	/*
+	 * CPUs seeing atomic_inc_return() must see prior RCU read-side
+	 * critical sections, and we also must force ordering with the
+	 * next idle sojourn.
+	 */
+	special = atomic_inc_return(&rdtp->dynticks);
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && special & 0x1);
+}
+
+/*
+ * Record exit from an extended quiescent state.  This is only to be
+ * called from an extended quiescent state.
+ */
+static void rcu_dynticks_eqs_exit(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	int special;
+
+	/*
+	 * CPUs seeing atomic_inc_return() must see prior idle sojourns,
+	 * and we also must force ordering with the next RCU read-side
+	 * critical section.
+	 */
+	special = atomic_inc_return(&rdtp->dynticks);
+	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !(special & 0x1));
+}
+
+/*
+ * Reset the current CPU's ->dynticks counter to indicate that the
+ * newly onlined CPU is no longer in an extended quiescent state.
+ * This will either leave the counter unchanged, or increment it
+ * to the next non-quiescent value.
+ *
+ * The non-atomic test/increment sequence works because the upper bits
+ * of the ->dynticks counter are manipulated only by the corresponding CPU,
+ * or when the corresponding CPU is offline.
+ */
+static void rcu_dynticks_eqs_online(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	if (atomic_read(&rdtp->dynticks) & 0x1)
+		return;
+	atomic_add(0x1, &rdtp->dynticks);
+}
+
+/*
  * Snapshot the ->dynticks counter with full ordering so as to allow
  * stable comparison of this counter with past and future snapshots.
  */
@@ -693,7 +748,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
 {
 	struct rcu_state *rsp;
 	struct rcu_data *rdp;
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);)
 
 	trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting);
 	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -712,12 +767,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user)
 		do_nocb_deferred_wakeup(rdp);
 	}
 	rcu_prepare_for_idle();
-	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
-	smp_mb__before_atomic();  /* See above. */
-	atomic_inc(&rdtp->dynticks);
-	smp_mb__after_atomic();  /* Force ordering with next sojourn. */
-	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
-		     atomic_read(&rdtp->dynticks) & 0x1);
+	rcu_dynticks_eqs_enter();
 	rcu_dynticks_task_enter();
 
 	/*
@@ -846,15 +896,10 @@ void rcu_irq_exit_irqson(void)
  */
 static void rcu_eqs_exit_common(long long oldval, int user)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+	RCU_TRACE(struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);)
 
 	rcu_dynticks_task_exit();
-	smp_mb__before_atomic();  /* Force ordering w/previous sojourn. */
-	atomic_inc(&rdtp->dynticks);
-	/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
-	smp_mb__after_atomic();  /* See above. */
-	WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
-		     !(atomic_read(&rdtp->dynticks) & 0x1));
+	rcu_dynticks_eqs_exit();
 	rcu_cleanup_after_idle();
 	trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting);
 	if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
@@ -1001,11 +1046,7 @@ void rcu_nmi_enter(void)
 	 * period (observation due to Andy Lutomirski).
 	 */
 	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
-		smp_mb__before_atomic();  /* Force delay from prior write. */
-		atomic_inc(&rdtp->dynticks);
-		/* atomic_inc() before later RCU read-side crit sects */
-		smp_mb__after_atomic();  /* See above. */
-		WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+		rcu_dynticks_eqs_exit();
 		incby = 1;
 	}
 	rdtp->dynticks_nmi_nesting += incby;
@@ -1043,11 +1084,7 @@ void rcu_nmi_exit(void)
 
 	/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
 	rdtp->dynticks_nmi_nesting = 0;
-	/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
-	smp_mb__before_atomic();  /* See above. */
-	atomic_inc(&rdtp->dynticks);
-	smp_mb__after_atomic();  /* Force delay to next write. */
-	WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
+	rcu_dynticks_eqs_enter();
 }
 
 /**
@@ -3800,8 +3837,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
 		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
 	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
 	rcu_sysidle_init_percpu_data(rdp->dynticks);
-	atomic_set(&rdp->dynticks->dynticks,
-		   (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
+	rcu_dynticks_eqs_online();
 	raw_spin_unlock_rcu_node(rnp);		/* irqs remain disabled. */
 
 	/*
-- 
2.5.2

[PATCH v3 tip/core/rcu 4/6] rcu: Abstract extended quiescent state determination

[Paul E. McKenney]

This commit is the fourth step towards full abstraction of all accesses
to the ->dynticks counter, implementing previously open-coded checks and
comparisons in new rcu_dynticks_in_eqs() and rcu_dynticks_in_eqs_since()
functions.  This abstraction will ease changes to the ->dynticks counter
operation.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 include/linux/rcutiny.h  |  6 ++++++
 kernel/rcu/tree.c        | 52 +++++++++++++++++++++++++++++++++++-------------
 kernel/rcu/tree.h        |  2 ++
 kernel/rcu/tree_exp.h    |  6 +++---
 kernel/rcu/tree_plugin.h |  2 +-
 kernel/rcu/tree_trace.c  |  2 +-
 6 files changed, 51 insertions(+), 19 deletions(-)

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index ac81e4063b40..4f9b2fa2173d 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -27,6 +27,12 @@
 
 #include <linux/cache.h>
 
+struct rcu_dynticks;
+static inline int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+{
+	return 0;
+}
+
 static inline unsigned long get_state_synchronize_rcu(void)
 {
 	return 0;
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 3169d5a21b55..8b970319c75b 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -337,10 +337,22 @@ static void rcu_dynticks_eqs_online(void)
 }
 
 /*
+ * Is the current CPU in an extended quiescent state?
+ *
+ * No ordering, as we are sampling CPU-local information.
+ */
+bool rcu_dynticks_curr_cpu_in_eqs(void)
+{
+	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
+
+	return !(atomic_read(&rdtp->dynticks) & 0x1);
+}
+
+/*
  * Snapshot the ->dynticks counter with full ordering so as to allow
  * stable comparison of this counter with past and future snapshots.
  */
-static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
+int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
 {
 	int snap = atomic_add_return(0, &rdtp->dynticks);
 
@@ -348,6 +360,25 @@ static int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
 }
 
 /*
+ * Return true if the snapshot returned from rcu_dynticks_snap()
+ * indicates that RCU is in an extended quiescent state.
+ */
+static bool rcu_dynticks_in_eqs(int snap)
+{
+	return !(snap & 0x1);
+}
+
+/*
+ * Return true if the CPU corresponding to the specified rcu_dynticks
+ * structure has spent some time in an extended quiescent state since
+ * rcu_dynticks_snap() returned the specified snapshot.
+ */
+static bool rcu_dynticks_in_eqs_since(struct rcu_dynticks *rdtp, int snap)
+{
+	return snap != rcu_dynticks_snap(rdtp);
+}
+
+/*
  * Do a double-increment of the ->dynticks counter to emulate a
  * momentary idle-CPU quiescent state.
  */
@@ -1045,7 +1076,7 @@ void rcu_nmi_enter(void)
 	 * to be in the outermost NMI handler that interrupted an RCU-idle
 	 * period (observation due to Andy Lutomirski).
 	 */
-	if (!(atomic_read(&rdtp->dynticks) & 0x1)) {
+	if (rcu_dynticks_curr_cpu_in_eqs()) {
 		rcu_dynticks_eqs_exit();
 		incby = 1;
 	}
@@ -1071,7 +1102,7 @@ void rcu_nmi_exit(void)
 	 * to us!)
 	 */
 	WARN_ON_ONCE(rdtp->dynticks_nmi_nesting <= 0);
-	WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
+	WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
 
 	/*
 	 * If the nesting level is not 1, the CPU wasn't RCU-idle, so
@@ -1097,9 +1128,7 @@ void rcu_nmi_exit(void)
  */
 bool notrace __rcu_is_watching(void)
 {
-	struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks);
-
-	return atomic_read(&rdtp->dynticks) & 0x1;
+	return !rcu_dynticks_curr_cpu_in_eqs();
 }
 
 /**
@@ -1184,7 +1213,7 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 {
 	rdp->dynticks_snap = rcu_dynticks_snap(rdp->dynticks);
 	rcu_sysidle_check_cpu(rdp, isidle, maxj);
-	if ((rdp->dynticks_snap & 0x1) == 0) {
+	if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
 		if (ULONG_CMP_LT(READ_ONCE(rdp->gpnum) + ULONG_MAX / 4,
 				 rdp->mynode->gpnum))
@@ -1203,12 +1232,7 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 				    bool *isidle, unsigned long *maxj)
 {
-	unsigned int curr;
 	int *rcrmp;
-	unsigned int snap;
-
-	curr = (unsigned int)rcu_dynticks_snap(rdp->dynticks);
-	snap = (unsigned int)rdp->dynticks_snap;
 
 	/*
 	 * If the CPU passed through or entered a dynticks idle phase with
@@ -1218,7 +1242,7 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	 * read-side critical section that started before the beginning
 	 * of the current RCU grace period.
 	 */
-	if ((curr & 0x1) == 0 || UINT_CMP_GE(curr, snap + 2)) {
+	if (rcu_dynticks_in_eqs_since(rdp->dynticks, rdp->dynticks_snap)) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("dti"));
 		rdp->dynticks_fqs++;
 		return 1;
@@ -3807,7 +3831,7 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp)
 	rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
 	rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
 	WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
-	WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
+	WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp->dynticks)));
 	rdp->cpu = cpu;
 	rdp->rsp = rsp;
 	rcu_boot_init_nocb_percpu_data(rdp);
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index fe98dd24adf8..3b953dcf6afc 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -595,6 +595,8 @@ extern struct rcu_state rcu_bh_state;
 extern struct rcu_state rcu_preempt_state;
 #endif /* #ifdef CONFIG_PREEMPT_RCU */
 
+int rcu_dynticks_snap(struct rcu_dynticks *rdtp);
+
 #ifdef CONFIG_RCU_BOOST
 DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
 DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu);
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index 011f626b2fd8..e155a465cf84 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -360,7 +360,7 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 			rdp->exp_dynticks_snap =
 				rcu_dynticks_snap(rdp->dynticks);
 			if (raw_smp_processor_id() == cpu ||
-			    !(rdp->exp_dynticks_snap & 0x1) ||
+			    rcu_dynticks_in_eqs(rdp->exp_dynticks_snap) ||
 			    !(rnp->qsmaskinitnext & rdp->grpmask))
 				mask_ofl_test |= rdp->grpmask;
 		}
@@ -383,8 +383,8 @@ static void sync_rcu_exp_select_cpus(struct rcu_state *rsp,
 			if (!(mask_ofl_ipi & mask))
 				continue;
 retry_ipi:
-			if (rcu_dynticks_snap(rdp->dynticks) !=
-			    rdp->exp_dynticks_snap) {
+			if (rcu_dynticks_in_eqs_since(rdp->dynticks,
+						      rdp->exp_dynticks_snap)) {
 				mask_ofl_test |= mask;
 				continue;
 			}
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 56583e764ebf..652209589adf 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1643,7 +1643,7 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
 	       "o."[!!(rdp->grpmask & rdp->mynode->qsmaskinit)],
 	       "N."[!!(rdp->grpmask & rdp->mynode->qsmaskinitnext)],
 	       ticks_value, ticks_title,
-	       atomic_read(&rdtp->dynticks) & 0xfff,
+	       rcu_dynticks_snap(rdtp) & 0xfff,
 	       rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
 	       rdp->softirq_snap, kstat_softirqs_cpu(RCU_SOFTIRQ, cpu),
 	       READ_ONCE(rsp->n_force_qs) - rsp->n_force_qs_gpstart,
diff --git a/kernel/rcu/tree_trace.c b/kernel/rcu/tree_trace.c
index b1f28972872c..b833cd0a29e8 100644
--- a/kernel/rcu/tree_trace.c
+++ b/kernel/rcu/tree_trace.c
@@ -124,7 +124,7 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
 		   rdp->rcu_qs_ctr_snap == per_cpu(rcu_qs_ctr, rdp->cpu),
 		   rdp->core_needs_qs);
 	seq_printf(m, " dt=%d/%llx/%d df=%lu",
-		   atomic_read(&rdp->dynticks->dynticks),
+		   rcu_dynticks_snap(rdp->dynticks),
 		   rdp->dynticks->dynticks_nesting,
 		   rdp->dynticks->dynticks_nmi_nesting,
 		   rdp->dynticks_fqs);
-- 
2.5.2

[PATCH v3 tip/core/rcu 5/6] rcu: Check cond_resched_rcu_qs() state less often to reduce GP overhead

[Paul E. McKenney]

Commit 4a81e8328d37 ("rcu: Reduce overhead of cond_resched() checks
for RCU") moved quiescent-state generation out of cond_resched()
and commit bde6c3aa9930 ("rcu: Provide cond_resched_rcu_qs() to force
quiescent states in long loops") introduced cond_resched_rcu_qs(), and
commit 5cd37193ce85 ("rcu: Make cond_resched_rcu_qs() apply to normal RCU
flavors") introduced the per-CPU rcu_qs_ctr variable, which is frequently
polled by the RCU core state machine.

This frequent polling can increase grace-period rate, which in turn
increases grace-period overhead, which is visible in some benchmarks
(for example, the "open1" benchmark in Anton Blanchard's "will it scale"
suite).  This commit therefore reduces the rate at which rcu_qs_ctr
is polled by moving that polling into the force-quiescent-state (FQS)
machinery, and by further polling it only after the grace period has
been in effect for at least jiffies_till_sched_qs jiffies.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 include/trace/events/rcu.h | 10 +++++-----
 kernel/rcu/tree.c          | 46 ++++++++++++++++++++++++++++++++++------------
 2 files changed, 39 insertions(+), 17 deletions(-)

diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h
index 9d4f9b3a2b7b..e3facb356838 100644
--- a/include/trace/events/rcu.h
+++ b/include/trace/events/rcu.h
@@ -385,11 +385,11 @@ TRACE_EVENT(rcu_quiescent_state_report,
 
 /*
  * Tracepoint for quiescent states detected by force_quiescent_state().
- * These trace events include the type of RCU, the grace-period number
- * that was blocked by the CPU, the CPU itself, and the type of quiescent
- * state, which can be "dti" for dyntick-idle mode, "ofl" for CPU offline,
- * or "kick" when kicking a CPU that has been in dyntick-idle mode for
- * too long.
+ * These trace events include the type of RCU, the grace-period number that
+ * was blocked by the CPU, the CPU itself, and the type of quiescent state,
+ * which can be "dti" for dyntick-idle mode, "ofl" for CPU offline, "kick"
+ * when kicking a CPU that has been in dyntick-idle mode for too long, or
+ * "rqc" if the CPU got a quiescent state via its rcu_qs_ctr.
  */
 TRACE_EVENT(rcu_fqs,
 
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 8b970319c75b..d8245cbd08f9 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1232,7 +1232,10 @@ static int dyntick_save_progress_counter(struct rcu_data *rdp,
 static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 				    bool *isidle, unsigned long *maxj)
 {
+	unsigned long jtsq;
 	int *rcrmp;
+	unsigned long rjtsc;
+	struct rcu_node *rnp;
 
 	/*
 	 * If the CPU passed through or entered a dynticks idle phase with
@@ -1248,6 +1251,31 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 		return 1;
 	}
 
+	/* Compute and saturate jiffies_till_sched_qs. */
+	jtsq = jiffies_till_sched_qs;
+	rjtsc = rcu_jiffies_till_stall_check();
+	if (jtsq > rjtsc / 2) {
+		WRITE_ONCE(jiffies_till_sched_qs, rjtsc);
+		jtsq = rjtsc / 2;
+	} else if (jtsq < 1) {
+		WRITE_ONCE(jiffies_till_sched_qs, 1);
+		jtsq = 1;
+	}
+
+	/*
+	 * Has this CPU encountered a cond_resched_rcu_qs() since the
+	 * beginning of the grace period?  For this to be the case,
+	 * the CPU has to have noticed the current grace period.  This
+	 * might not be the case for nohz_full CPUs looping in the kernel.
+	 */
+	rnp = rdp->mynode;
+	if (time_after(jiffies, rdp->rsp->gp_start + jtsq) &&
+	    READ_ONCE(rdp->rcu_qs_ctr_snap) != per_cpu(rcu_qs_ctr, rdp->cpu) &&
+	    READ_ONCE(rdp->gpnum) == rnp->gpnum && !rdp->gpwrap) {
+		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("rqc"));
+		return 1;
+	}
+
 	/*
 	 * Check for the CPU being offline, but only if the grace period
 	 * is old enough.  We don't need to worry about the CPU changing
@@ -1290,9 +1318,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 	 * warning delay.
 	 */
 	rcrmp = &per_cpu(rcu_sched_qs_mask, rdp->cpu);
-	if (ULONG_CMP_GE(jiffies,
-			 rdp->rsp->gp_start + jiffies_till_sched_qs) ||
-	    ULONG_CMP_GE(jiffies, rdp->rsp->jiffies_resched)) {
+	if (time_after(jiffies, rdp->rsp->gp_start + jtsq) ||
+	    time_after(jiffies, rdp->rsp->jiffies_resched)) {
 		if (!(READ_ONCE(*rcrmp) & rdp->rsp->flavor_mask)) {
 			WRITE_ONCE(rdp->cond_resched_completed,
 				   READ_ONCE(rdp->mynode->completed));
@@ -2550,10 +2577,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_state *rsp, struct rcu_data *rdp)
 
 	rnp = rdp->mynode;
 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
-	if ((rdp->cpu_no_qs.b.norm &&
-	     rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) ||
-	    rdp->gpnum != rnp->gpnum || rnp->completed == rnp->gpnum ||
-	    rdp->gpwrap) {
+	if (rdp->cpu_no_qs.b.norm || rdp->gpnum != rnp->gpnum ||
+	    rnp->completed == rnp->gpnum || rdp->gpwrap) {
 
 		/*
 		 * The grace period in which this quiescent state was
@@ -2608,8 +2633,7 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
 	 * Was there a quiescent state since the beginning of the grace
 	 * period? If no, then exit and wait for the next call.
 	 */
-	if (rdp->cpu_no_qs.b.norm &&
-	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr))
+	if (rdp->cpu_no_qs.b.norm)
 		return;
 
 	/*
@@ -3563,9 +3587,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
 	    rdp->core_needs_qs && rdp->cpu_no_qs.b.norm &&
 	    rdp->rcu_qs_ctr_snap == __this_cpu_read(rcu_qs_ctr)) {
 		rdp->n_rp_core_needs_qs++;
-	} else if (rdp->core_needs_qs &&
-		   (!rdp->cpu_no_qs.b.norm ||
-		    rdp->rcu_qs_ctr_snap != __this_cpu_read(rcu_qs_ctr))) {
+	} else if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm) {
 		rdp->n_rp_report_qs++;
 		return 1;
 	}
-- 
2.5.2

[PATCH v3 tip/core/rcu 6/6] rcu: Adjust FQS offline checks for exact online-CPU detection

[Paul E. McKenney]

Commit 7ec99de36f40 ("rcu: Provide exact CPU-online tracking for RCU"),
as its title suggests, got rid of RCU's remaining CPU-hotplug timing
guesswork.  This commit therefore removes the one-jiffy kludge that was
used to paper over this guesswork.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
---
 kernel/rcu/tree.c | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index d8245cbd08f9..6e4b1dcebeb2 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1276,21 +1276,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp,
 		return 1;
 	}
 
-	/*
-	 * Check for the CPU being offline, but only if the grace period
-	 * is old enough.  We don't need to worry about the CPU changing
-	 * state: If we see it offline even once, it has been through a
-	 * quiescent state.
-	 *
-	 * The reason for insisting that the grace period be at least
-	 * one jiffy old is that CPUs that are not quite online and that
-	 * have just gone offline can still execute RCU read-side critical
-	 * sections.
-	 */
-	if (ULONG_CMP_GE(rdp->rsp->gp_start + 2, jiffies))
-		return 0;  /* Grace period is not old enough. */
-	barrier();
-	if (cpu_is_offline(rdp->cpu)) {
+	/* Check for the CPU being offline. */
+	if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp))) {
 		trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, TPS("ofl"));
 		rdp->offline_fqs++;
 		return 1;
-- 
2.5.2