[PATCH RFC tip/core/rcu 0/2] NOCB kthread economization patches

[PATCH RFC tip/core/rcu 0/2] NOCB kthread economization patches

[Paul E. McKenney]

Hello!

This series provides a couple of patches that reduce the number of rcuo
kthreads in cases where nr_cpu_ids is larger than the actual number of
online CPUs.

1.	Preparation patch that rationalizes RCU's creation of early-boot
	kthreads.

2.	Instead of creation rcuo kthreads for each possible CPU, create
	them only for those CPUs that have come online at least once
	since boot.

							Thanx, Paul

------------------------------------------------------------------------

 b/kernel/rcu/tree.c        |    8 ++-
 b/kernel/rcu/tree.h        |    5 +-
 b/kernel/rcu/tree_plugin.h |  105 ++++++++++++++++++++++++++++++++++++---------
 3 files changed, 95 insertions(+), 23 deletions(-)

[PATCH RFC tip/core/rcu 1/2] rcu: Rationalize kthread spawning

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

Currently, RCU spawns kthreads from several different early_initcall()
functions.  Although this has served RCU well for quite some time,
as more kthreads are added a more deterministic approach is required.
This commit therefore causes all of RCU's early-boot kthreads to be
spawned from a single early_initcall() function.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c        |  4 +++-
 kernel/rcu/tree.h        |  1 +
 kernel/rcu/tree_plugin.h | 12 +++---------
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 2719978ea018..eecdbe20a08e 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3491,7 +3491,7 @@ static int rcu_pm_notify(struct notifier_block *self,
 }
 
 /*
- * Spawn the kthread that handles this RCU flavor's grace periods.
+ * Spawn the kthreads that handle each RCU flavor's grace periods.
  */
 static int __init rcu_spawn_gp_kthread(void)
 {
@@ -3500,6 +3500,7 @@ static int __init rcu_spawn_gp_kthread(void)
 	struct rcu_state *rsp;
 	struct task_struct *t;
 
+	rcu_scheduler_fully_active = 1;
 	for_each_rcu_flavor(rsp) {
 		t = kthread_run(rcu_gp_kthread, rsp, "%s", rsp->name);
 		BUG_ON(IS_ERR(t));
@@ -3509,6 +3510,7 @@ static int __init rcu_spawn_gp_kthread(void)
 		raw_spin_unlock_irqrestore(&rnp->lock, flags);
 		rcu_spawn_nocb_kthreads(rsp);
 	}
+	rcu_spawn_boost_kthreads();
 	return 0;
 }
 early_initcall(rcu_spawn_gp_kthread);
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 71e64c718f75..bd7b63da9a8c 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -572,6 +572,7 @@ static void rcu_preempt_do_callbacks(void);
 static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
 						 struct rcu_node *rnp);
 #endif /* #ifdef CONFIG_RCU_BOOST */
+static void __init rcu_spawn_boost_kthreads(void);
 static void rcu_prepare_kthreads(int cpu);
 static void rcu_cleanup_after_idle(int cpu);
 static void rcu_prepare_for_idle(int cpu);
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 735dc48634e7..f5314cc20750 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1463,14 +1463,13 @@ static struct smp_hotplug_thread rcu_cpu_thread_spec = {
 };
 
 /*
- * Spawn all kthreads -- called as soon as the scheduler is running.
+ * Spawn boost kthreads -- called as soon as the scheduler is running.
  */
-static int __init rcu_spawn_kthreads(void)
+static void __init rcu_spawn_boost_kthreads(void)
 {
 	struct rcu_node *rnp;
 	int cpu;
 
-	rcu_scheduler_fully_active = 1;
 	for_each_possible_cpu(cpu)
 		per_cpu(rcu_cpu_has_work, cpu) = 0;
 	BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec));
@@ -1480,9 +1479,7 @@ static int __init rcu_spawn_kthreads(void)
 		rcu_for_each_leaf_node(rcu_state_p, rnp)
 			(void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp);
 	}
-	return 0;
 }
-early_initcall(rcu_spawn_kthreads);
 
 static void rcu_prepare_kthreads(int cpu)
 {
@@ -1520,12 +1517,9 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
 {
 }
 
-static int __init rcu_scheduler_really_started(void)
+static void __init rcu_spawn_boost_kthreads(void)
 {
-	rcu_scheduler_fully_active = 1;
-	return 0;
 }
-early_initcall(rcu_scheduler_really_started);
 
 static void rcu_prepare_kthreads(int cpu)
 {
-- 
1.8.1.5

[PATCH RFC tip/core/rcu 2/2] rcu: Create rcuo kthreads only for onlined CPUs

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

RCU currently uses for_each_possible_cpu() to spawn rcuo kthreads,
which can result in more rcuo kthreads than one would expect, for
example, derRichard reported 64 CPUs worth of rcuo kthreads on an
8-CPU image.  This commit therefore creates rcuo kthreads only for
those CPUs that actually come online.

Reported-by: derRichard
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c        |  4 ++-
 kernel/rcu/tree.h        |  4 ++-
 kernel/rcu/tree_plugin.h | 93 ++++++++++++++++++++++++++++++++++++++++++------
 3 files changed, 88 insertions(+), 13 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index eecdbe20a08e..a1abaa8abd49 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3444,6 +3444,7 @@ static int rcu_cpu_notify(struct notifier_block *self,
 	case CPU_UP_PREPARE_FROZEN:
 		rcu_prepare_cpu(cpu);
 		rcu_prepare_kthreads(cpu);
+		rcu_spawn_all_nocb_kthreads(cpu);
 		break;
 	case CPU_ONLINE:
 	case CPU_DOWN_FAILED:
@@ -3508,8 +3509,8 @@ static int __init rcu_spawn_gp_kthread(void)
 		raw_spin_lock_irqsave(&rnp->lock, flags);
 		rsp->gp_kthread = t;
 		raw_spin_unlock_irqrestore(&rnp->lock, flags);
-		rcu_spawn_nocb_kthreads(rsp);
 	}
+	rcu_spawn_nocb_kthreads();
 	rcu_spawn_boost_kthreads();
 	return 0;
 }
@@ -3643,6 +3644,7 @@ static void __init rcu_init_one(struct rcu_state *rsp,
 		rcu_boot_init_percpu_data(i, rsp);
 	}
 	list_add(&rsp->flavors, &rcu_struct_flavors);
+	rcu_organize_nocb_kthreads(rsp);
 }
 
 /*
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index bd7b63da9a8c..f703ea8b7836 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -593,7 +593,9 @@ static bool rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
 static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp);
 static void do_nocb_deferred_wakeup(struct rcu_data *rdp);
 static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
-static void rcu_spawn_nocb_kthreads(struct rcu_state *rsp);
+static void rcu_spawn_all_nocb_kthreads(int cpu);
+static void __init rcu_spawn_nocb_kthreads(void);
+static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp);
 static void __maybe_unused rcu_kick_nohz_cpu(int cpu);
 static bool init_nocb_callback_list(struct rcu_data *rdp);
 static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq);
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index f5314cc20750..cedb020f1f90 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2455,15 +2455,85 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
 	rdp->nocb_follower_tail = &rdp->nocb_follower_head;
 }
 
+/*
+ * If the specified CPU is a no-CBs CPU that does not already have its
+ * rcuo kthread for the specified RCU flavor, spawn it.  If the CPUs are
+ * brought online out of order, this can require require re-organizing
+ * the leader-follower relationships.
+ */
+static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
+{
+	struct rcu_data *rdp;
+	struct rcu_data *rdp_last;
+	struct rcu_data *rdp_old_leader;
+	struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
+	struct task_struct *t;
+
+	/*
+	 * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
+	 * then nothing to do.
+	 */
+	if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
+		return;
+
+	/* If we didn't spawn the leader first, reorganize! */
+	rdp_old_leader = rdp_spawn->nocb_leader;
+	if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
+		rdp_last = NULL;
+		rdp = rdp_old_leader;
+		do {
+			rdp->nocb_leader = rdp_spawn;
+			if (rdp_last && rdp != rdp_spawn)
+				rdp_last->nocb_next_follower = rdp;
+			rdp_last = rdp;
+			rdp = rdp->nocb_next_follower;
+			rdp_last->nocb_next_follower = NULL;
+		} while (rdp);
+		rdp_spawn->nocb_next_follower = rdp_old_leader;
+	}
+
+	/* Spawn the kthread for this CPU and RCU flavor. */
+	t = kthread_run(rcu_nocb_kthread, rdp_spawn,
+			"rcuo%c/%d", rsp->abbr, cpu);
+	BUG_ON(IS_ERR(t));
+	ACCESS_ONCE(rdp_spawn->nocb_kthread) = t;
+}
+
+/*
+ * If the specified CPU is a no-CBs CPU that does not already have its
+ * rcuo kthreads, spawn them.
+ */
+static void rcu_spawn_all_nocb_kthreads(int cpu)
+{
+	struct rcu_state *rsp;
+
+	if (rcu_scheduler_fully_active)
+		for_each_rcu_flavor(rsp)
+			rcu_spawn_one_nocb_kthread(rsp, cpu);
+}
+
+/*
+ * Once the scheduler is running, spawn rcuo kthreads for all online
+ * no-CBs CPUs.  This assumes that the early_initcall()s happen before
+ * non-boot CPUs come online -- if this changes, we will need to add
+ * some mutual exclusion.
+ */
+static void __init rcu_spawn_nocb_kthreads(void)
+{
+	int cpu;
+
+	for_each_online_cpu(cpu)
+		rcu_spawn_all_nocb_kthreads(cpu);
+}
+
 /* How many follower CPU IDs per leader?  Default of -1 for sqrt(nr_cpu_ids). */
 static int rcu_nocb_leader_stride = -1;
 module_param(rcu_nocb_leader_stride, int, 0444);
 
 /*
- * Create a kthread for each RCU flavor for each no-CBs CPU.
- * Also initialize leader-follower relationships.
+ * Initialize leader-follower relationships for all no-CBs CPU.
  */
-static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
+static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)
 {
 	int cpu;
 	int ls = rcu_nocb_leader_stride;
@@ -2471,7 +2541,6 @@ static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
 	struct rcu_data *rdp;
 	struct rcu_data *rdp_leader = NULL;  /* Suppress misguided gcc warn. */
 	struct rcu_data *rdp_prev = NULL;
-	struct task_struct *t;
 
 	if (rcu_nocb_mask == NULL)
 		return;
@@ -2500,12 +2569,6 @@ static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
 			rdp_prev->nocb_next_follower = rdp;
 		}
 		rdp_prev = rdp;
-
-		/* Spawn the kthread for this CPU. */
-		t = kthread_run(rcu_nocb_kthread, rdp,
-				"rcuo%c/%d", rsp->abbr, cpu);
-		BUG_ON(IS_ERR(t));
-		ACCESS_ONCE(rdp->nocb_kthread) = t;
 	}
 }
 
@@ -2559,7 +2622,15 @@ static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
 {
 }
 
-static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
+static void rcu_spawn_all_nocb_kthreads(int cpu)
+{
+}
+
+static void __init rcu_spawn_nocb_kthreads(void)
+{
+}
+
+static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp)
 {
 }
 
-- 
1.8.1.5

Re: [PATCH RFC tip/core/rcu 0/2] NOCB kthread economization patches

[Josh Triplett]

On Mon, Jul 14, 2014 at 03:06:04AM -0700, Paul E. McKenney wrote:
> This series provides a couple of patches that reduce the number of rcuo
> kthreads in cases where nr_cpu_ids is larger than the actual number of
> online CPUs.
> 
> 1.	Preparation patch that rationalizes RCU's creation of early-boot
> 	kthreads.
> 
> 2.	Instead of creation rcuo kthreads for each possible CPU, create
> 	them only for those CPUs that have come online at least once
> 	since boot.

For both:

Reviewed-by: Josh Triplett <josh@joshtriplett.org>

Re: [PATCH RFC tip/core/rcu 2/2] rcu: Create rcuo kthreads only for onlined CPUs

[Pranith Kumar]

On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
>
> RCU currently uses for_each_possible_cpu() to spawn rcuo kthreads,
> which can result in more rcuo kthreads than one would expect, for
> example, derRichard reported 64 CPUs worth of rcuo kthreads on an
> 8-CPU image.  This commit therefore creates rcuo kthreads only for
> those CPUs that actually come online.
>
> Reported-by: derRichard
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> ---
>  kernel/rcu/tree.c        |  4 ++-
>  kernel/rcu/tree.h        |  4 ++-
>  kernel/rcu/tree_plugin.h | 93 ++++++++++++++++++++++++++++++++++++++++++------
>  3 files changed, 88 insertions(+), 13 deletions(-)
>
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index eecdbe20a08e..a1abaa8abd49 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -3444,6 +3444,7 @@ static int rcu_cpu_notify(struct notifier_block *self,
>  	case CPU_UP_PREPARE_FROZEN:
>  		rcu_prepare_cpu(cpu);
>  		rcu_prepare_kthreads(cpu);
> +		rcu_spawn_all_nocb_kthreads(cpu);
>  		break;
>  	case CPU_ONLINE:
>  	case CPU_DOWN_FAILED:
> @@ -3508,8 +3509,8 @@ static int __init rcu_spawn_gp_kthread(void)
>  		raw_spin_lock_irqsave(&rnp->lock, flags);
>  		rsp->gp_kthread = t;
>  		raw_spin_unlock_irqrestore(&rnp->lock, flags);
> -		rcu_spawn_nocb_kthreads(rsp);
>  	}
> +	rcu_spawn_nocb_kthreads();
>  	rcu_spawn_boost_kthreads();
>  	return 0;
>  }
> @@ -3643,6 +3644,7 @@ static void __init rcu_init_one(struct rcu_state *rsp,
>  		rcu_boot_init_percpu_data(i, rsp);
>  	}
>  	list_add(&rsp->flavors, &rcu_struct_flavors);
> +	rcu_organize_nocb_kthreads(rsp);
>  }
>  
>  /*
> diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
> index bd7b63da9a8c..f703ea8b7836 100644
> --- a/kernel/rcu/tree.h
> +++ b/kernel/rcu/tree.h
> @@ -593,7 +593,9 @@ static bool rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
>  static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp);
>  static void do_nocb_deferred_wakeup(struct rcu_data *rdp);
>  static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
> -static void rcu_spawn_nocb_kthreads(struct rcu_state *rsp);
> +static void rcu_spawn_all_nocb_kthreads(int cpu);
> +static void __init rcu_spawn_nocb_kthreads(void);
> +static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp);
>  static void __maybe_unused rcu_kick_nohz_cpu(int cpu);
>  static bool init_nocb_callback_list(struct rcu_data *rdp);
>  static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq);
> diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
> index f5314cc20750..cedb020f1f90 100644
> --- a/kernel/rcu/tree_plugin.h
> +++ b/kernel/rcu/tree_plugin.h
> @@ -2455,15 +2455,85 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
>  	rdp->nocb_follower_tail = &rdp->nocb_follower_head;
>  }
>  
> +/*
> + * If the specified CPU is a no-CBs CPU that does not already have its
> + * rcuo kthread for the specified RCU flavor, spawn it.  If the CPUs are
> + * brought online out of order, this can require require re-organizing

double require, can remove one

> + * the leader-follower relationships.
> + */
> +static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
> +{
> +	struct rcu_data *rdp;
> +	struct rcu_data *rdp_last;
> +	struct rcu_data *rdp_old_leader;
> +	struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
> +	struct task_struct *t;
> +
> +	/*
> +	 * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
> +	 * then nothing to do.
> +	 */
> +	if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
> +		return;
> +
> +	/* If we didn't spawn the leader first, reorganize! */
> +	rdp_old_leader = rdp_spawn->nocb_leader;
> +	if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
> +		rdp_last = NULL;
> +		rdp = rdp_old_leader;
> +		do {
> +			rdp->nocb_leader = rdp_spawn;
> +			if (rdp_last && rdp != rdp_spawn)
> +				rdp_last->nocb_next_follower = rdp;
> +			rdp_last = rdp;
> +			rdp = rdp->nocb_next_follower;
> +			rdp_last->nocb_next_follower = NULL;
> +		} while (rdp);
> +		rdp_spawn->nocb_next_follower = rdp_old_leader;
> +	}
> +
> +	/* Spawn the kthread for this CPU and RCU flavor. */
> +	t = kthread_run(rcu_nocb_kthread, rdp_spawn,
> +			"rcuo%c/%d", rsp->abbr, cpu);
> +	BUG_ON(IS_ERR(t));
> +	ACCESS_ONCE(rdp_spawn->nocb_kthread) = t;
> +}
> +
> +/*
> + * If the specified CPU is a no-CBs CPU that does not already have its
> + * rcuo kthreads, spawn them.
> + */
> +static void rcu_spawn_all_nocb_kthreads(int cpu)
> +{
> +	struct rcu_state *rsp;
> +
> +	if (rcu_scheduler_fully_active)
> +		for_each_rcu_flavor(rsp)
> +			rcu_spawn_one_nocb_kthread(rsp, cpu);
> +}
> +
> +/*
> + * Once the scheduler is running, spawn rcuo kthreads for all online
> + * no-CBs CPUs.  This assumes that the early_initcall()s happen before
> + * non-boot CPUs come online -- if this changes, we will need to add
> + * some mutual exclusion.
> + */
> +static void __init rcu_spawn_nocb_kthreads(void)
> +{
> +	int cpu;
> +
> +	for_each_online_cpu(cpu)
> +		rcu_spawn_all_nocb_kthreads(cpu);
> +}

Minor nit but how about checking that the cpu is a no-CB cpu here? You are currently doing it in rcu_spawn_one_nocb_kthread two levels down.


        for_each_online_cpu(cpu)
-               rcu_spawn_all_nocb_kthreads(cpu);
+               if (rcu_is_nocb_cpu(cpu))
+                       rcu_spawn_all_nocb_kthreads(cpu);
 }
 

--
Pranith

Re: [PATCH RFC tip/core/rcu 2/2] rcu: Create rcuo kthreads only for onlined CPUs

[Paul E. McKenney]

On Mon, Jul 14, 2014 at 12:48:54PM -0400, Pranith Kumar wrote:
> 
> On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
> > From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> >
> > RCU currently uses for_each_possible_cpu() to spawn rcuo kthreads,
> > which can result in more rcuo kthreads than one would expect, for
> > example, derRichard reported 64 CPUs worth of rcuo kthreads on an
> > 8-CPU image.  This commit therefore creates rcuo kthreads only for
> > those CPUs that actually come online.
> >
> > Reported-by: derRichard
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> > ---
> >  kernel/rcu/tree.c        |  4 ++-
> >  kernel/rcu/tree.h        |  4 ++-
> >  kernel/rcu/tree_plugin.h | 93 ++++++++++++++++++++++++++++++++++++++++++------
> >  3 files changed, 88 insertions(+), 13 deletions(-)
> >
> > diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> > index eecdbe20a08e..a1abaa8abd49 100644
> > --- a/kernel/rcu/tree.c
> > +++ b/kernel/rcu/tree.c
> > @@ -3444,6 +3444,7 @@ static int rcu_cpu_notify(struct notifier_block *self,
> >  	case CPU_UP_PREPARE_FROZEN:
> >  		rcu_prepare_cpu(cpu);
> >  		rcu_prepare_kthreads(cpu);
> > +		rcu_spawn_all_nocb_kthreads(cpu);
> >  		break;
> >  	case CPU_ONLINE:
> >  	case CPU_DOWN_FAILED:
> > @@ -3508,8 +3509,8 @@ static int __init rcu_spawn_gp_kthread(void)
> >  		raw_spin_lock_irqsave(&rnp->lock, flags);
> >  		rsp->gp_kthread = t;
> >  		raw_spin_unlock_irqrestore(&rnp->lock, flags);
> > -		rcu_spawn_nocb_kthreads(rsp);
> >  	}
> > +	rcu_spawn_nocb_kthreads();
> >  	rcu_spawn_boost_kthreads();
> >  	return 0;
> >  }
> > @@ -3643,6 +3644,7 @@ static void __init rcu_init_one(struct rcu_state *rsp,
> >  		rcu_boot_init_percpu_data(i, rsp);
> >  	}
> >  	list_add(&rsp->flavors, &rcu_struct_flavors);
> > +	rcu_organize_nocb_kthreads(rsp);
> >  }
> >  
> >  /*
> > diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
> > index bd7b63da9a8c..f703ea8b7836 100644
> > --- a/kernel/rcu/tree.h
> > +++ b/kernel/rcu/tree.h
> > @@ -593,7 +593,9 @@ static bool rcu_nocb_adopt_orphan_cbs(struct rcu_state *rsp,
> >  static bool rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp);
> >  static void do_nocb_deferred_wakeup(struct rcu_data *rdp);
> >  static void rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp);
> > -static void rcu_spawn_nocb_kthreads(struct rcu_state *rsp);
> > +static void rcu_spawn_all_nocb_kthreads(int cpu);
> > +static void __init rcu_spawn_nocb_kthreads(void);
> > +static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp);
> >  static void __maybe_unused rcu_kick_nohz_cpu(int cpu);
> >  static bool init_nocb_callback_list(struct rcu_data *rdp);
> >  static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq);
> > diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
> > index f5314cc20750..cedb020f1f90 100644
> > --- a/kernel/rcu/tree_plugin.h
> > +++ b/kernel/rcu/tree_plugin.h
> > @@ -2455,15 +2455,85 @@ static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
> >  	rdp->nocb_follower_tail = &rdp->nocb_follower_head;
> >  }
> >  
> > +/*
> > + * If the specified CPU is a no-CBs CPU that does not already have its
> > + * rcuo kthread for the specified RCU flavor, spawn it.  If the CPUs are
> > + * brought online out of order, this can require require re-organizing
> 
> double require, can remove one

Good good catch catch, fixed fixed!  ;-)

> > + * the leader-follower relationships.
> > + */
> > +static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu)
> > +{
> > +	struct rcu_data *rdp;
> > +	struct rcu_data *rdp_last;
> > +	struct rcu_data *rdp_old_leader;
> > +	struct rcu_data *rdp_spawn = per_cpu_ptr(rsp->rda, cpu);
> > +	struct task_struct *t;
> > +
> > +	/*
> > +	 * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
> > +	 * then nothing to do.
> > +	 */
> > +	if (!rcu_is_nocb_cpu(cpu) || rdp_spawn->nocb_kthread)
> > +		return;
> > +
> > +	/* If we didn't spawn the leader first, reorganize! */
> > +	rdp_old_leader = rdp_spawn->nocb_leader;
> > +	if (rdp_old_leader != rdp_spawn && !rdp_old_leader->nocb_kthread) {
> > +		rdp_last = NULL;
> > +		rdp = rdp_old_leader;
> > +		do {
> > +			rdp->nocb_leader = rdp_spawn;
> > +			if (rdp_last && rdp != rdp_spawn)
> > +				rdp_last->nocb_next_follower = rdp;
> > +			rdp_last = rdp;
> > +			rdp = rdp->nocb_next_follower;
> > +			rdp_last->nocb_next_follower = NULL;
> > +		} while (rdp);
> > +		rdp_spawn->nocb_next_follower = rdp_old_leader;
> > +	}
> > +
> > +	/* Spawn the kthread for this CPU and RCU flavor. */
> > +	t = kthread_run(rcu_nocb_kthread, rdp_spawn,
> > +			"rcuo%c/%d", rsp->abbr, cpu);
> > +	BUG_ON(IS_ERR(t));
> > +	ACCESS_ONCE(rdp_spawn->nocb_kthread) = t;
> > +}
> > +
> > +/*
> > + * If the specified CPU is a no-CBs CPU that does not already have its
> > + * rcuo kthreads, spawn them.
> > + */
> > +static void rcu_spawn_all_nocb_kthreads(int cpu)
> > +{
> > +	struct rcu_state *rsp;
> > +
> > +	if (rcu_scheduler_fully_active)
> > +		for_each_rcu_flavor(rsp)
> > +			rcu_spawn_one_nocb_kthread(rsp, cpu);
> > +}
> > +
> > +/*
> > + * Once the scheduler is running, spawn rcuo kthreads for all online
> > + * no-CBs CPUs.  This assumes that the early_initcall()s happen before
> > + * non-boot CPUs come online -- if this changes, we will need to add
> > + * some mutual exclusion.
> > + */
> > +static void __init rcu_spawn_nocb_kthreads(void)
> > +{
> > +	int cpu;
> > +
> > +	for_each_online_cpu(cpu)
> > +		rcu_spawn_all_nocb_kthreads(cpu);
> > +}
> 
> Minor nit but how about checking that the cpu is a no-CB cpu here? You are currently doing it in rcu_spawn_one_nocb_kthread two levels down.

I need the check to be at least one level down because of the call to
rcu_spawn_all_nocb_kthreads() from rcu_cpu_notify(), so there is no
point in adding it here.  Please note that this code is called once at
boot, so there is no need to worry about this level of performance.
Furthermore, this function is marked __init, so some implementations
free up its memory once boot has progressed far enough.

So I need to leave this check where it is, two levels down.

								Thanx, Paul

>         for_each_online_cpu(cpu)
> -               rcu_spawn_all_nocb_kthreads(cpu);
> +               if (rcu_is_nocb_cpu(cpu))
> +                       rcu_spawn_all_nocb_kthreads(cpu);
>  }
> 
> 
> --
> Pranith
> 

Re: [PATCH RFC tip/core/rcu 1/2] rcu: Rationalize kthread spawning

[Sasha Levin]

On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> 
> Currently, RCU spawns kthreads from several different early_initcall()
> functions.  Although this has served RCU well for quite some time,
> as more kthreads are added a more deterministic approach is required.
> This commit therefore causes all of RCU's early-boot kthreads to be
> spawned from a single early_initcall() function.
> 
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

Hey Paul,

I've updated to today's -next and my VM started hanging on boot. Bisect
pointed out this patch.

I don't have any further info right now, but can provide whatever you'll
find useful.


Thanks,
Sasha

Re: [PATCH RFC tip/core/rcu 1/2] rcu: Rationalize kthread spawning

[Pranith Kumar]

On 07/16/2014 10:57 PM, Sasha Levin wrote:
> On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
>> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
>>
>> Currently, RCU spawns kthreads from several different early_initcall()
>> functions.  Although this has served RCU well for quite some time,
>> as more kthreads are added a more deterministic approach is required.
>> This commit therefore causes all of RCU's early-boot kthreads to be
>> spawned from a single early_initcall() function.
>>
>> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> Hey Paul,
> 
> I've updated to today's -next and my VM started hanging on boot. Bisect
> pointed out this patch.
> 
> I don't have any further info right now, but can provide whatever you'll
> find useful.

Hi Sasha,

Could you attach the config file which is failing for a start? 

Thanks,
--
Pranith

> 
> 
> Thanks,
> Sasha
> 

Re: [PATCH RFC tip/core/rcu 1/2] rcu: Rationalize kthread spawning

[Paul E. McKenney]

On Wed, Jul 16, 2014 at 10:57:39PM -0400, Sasha Levin wrote:
> On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
> > From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> > 
> > Currently, RCU spawns kthreads from several different early_initcall()
> > functions.  Although this has served RCU well for quite some time,
> > as more kthreads are added a more deterministic approach is required.
> > This commit therefore causes all of RCU's early-boot kthreads to be
> > spawned from a single early_initcall() function.
> > 
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> Hey Paul,
> 
> I've updated to today's -next and my VM started hanging on boot. Bisect
> pointed out this patch.
> 
> I don't have any further info right now, but can provide whatever you'll
> find useful.

Could you please add initcall_debug to qemu's -append list?  If sysrq-T
works that early, its output would be helpful as well.

Your .config would also be helpful.

							Thanx, Paul

Re: [PATCH RFC tip/core/rcu 1/2] rcu: Rationalize kthread spawning

[Paul E. McKenney]

On Thu, Jul 17, 2014 at 12:02:05PM -0400, Sasha Levin wrote:
> On 07/17/2014 01:33 AM, Paul E. McKenney wrote:
> > On Wed, Jul 16, 2014 at 10:57:39PM -0400, Sasha Levin wrote:
> >> On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
> >>> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> >>>
> >>> Currently, RCU spawns kthreads from several different early_initcall()
> >>> functions.  Although this has served RCU well for quite some time,
> >>> as more kthreads are added a more deterministic approach is required.
> >>> This commit therefore causes all of RCU's early-boot kthreads to be
> >>> spawned from a single early_initcall() function.
> >>>
> >>> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> >>
> >> Hey Paul,
> >>
> >> I've updated to today's -next and my VM started hanging on boot. Bisect
> >> pointed out this patch.
> >>
> >> I don't have any further info right now, but can provide whatever you'll
> >> find useful.
> > 
> > Could you please add initcall_debug to qemu's -append list?  If sysrq-T
> > works that early, its output would be helpful as well.
> > 
> > Your .config would also be helpful.
> 
> Hi Pranith, Paul,
> 
> I've attached my boot log with initcall_debug, as well as the .config.

Hello, Sasha,

Looks like you might have run into the same thing that Ingo ran into.
Could you please try the following patch?  (Or wait for it to show up
in the next -next, if you prefer.)

							Thanx, Paul

------------------------------------------------------------------------

rcu: Allow for NULL tick_nohz_full_mask when nohz_full= missing

If there isn't a nohz_full= kernel parameter specified, then
tick_nohz_full_mask can legitimately be NULL.  This can cause
problems when RCU's boot code tries to cpumask_or() this value into
rcu_nocb_mask.  In addition, if NO_HZ_FULL_ALL=y, there is no point
in doing the cpumask_or() in the first place because this will cause
RCU_NOCB_CPU_ALL=y, which in turn will have all bits already set in
rcu_nocb_mask.

This commit therefore avoids the cpumask_or() if NO_HZ_FULL_ALL=y
and checks for !tick_nohz_full_running otherwise, this latter check
catching cases when there was no nohz_full= kernel parameter specified.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index f62b7f2f6abd..00dc411e9676 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2479,9 +2479,10 @@ static void __init rcu_spawn_nocb_kthreads(struct rcu_state *rsp)
 
 	if (rcu_nocb_mask == NULL)
 		return;
-#ifdef CONFIG_NO_HZ_FULL
-	cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
-#endif /* #ifdef CONFIG_NO_HZ_FULL */
+#if defined(CONFIG_NO_HZ_FULL) && !defined(CONFIG_NO_HZ_FULL_ALL)
+	if (tick_nohz_full_running)
+		cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
+#endif /* #if defined(CONFIG_NO_HZ_FULL) && !defined(CONFIG_NO_HZ_FULL_ALL) */
 	if (ls == -1) {
 		ls = int_sqrt(nr_cpu_ids);
 		rcu_nocb_leader_stride = ls;

Re: [PATCH RFC tip/core/rcu 2/2] rcu: Create rcuo kthreads only for onlined CPUs

[Sasha Levin]

On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> 
> RCU currently uses for_each_possible_cpu() to spawn rcuo kthreads,
> which can result in more rcuo kthreads than one would expect, for
> example, derRichard reported 64 CPUs worth of rcuo kthreads on an
> 8-CPU image.  This commit therefore creates rcuo kthreads only for
> those CPUs that actually come online.
> 
> Reported-by: derRichard
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

Hey Paul,

Me again. :)

It seems that this patch moved thread initialization to a point way
too early during boot, before rest_init() which initializes kthreadd_task
runs, so creating a new kthread triggers a NULL ptr deref:

[    0.000000] BUG: unable to handle kernel NULL pointer dereference at           (null)
[    0.000000] IP: wake_up_process (kernel/sched/core.c:1768)
[    0.000000] PGD 0
[    0.000000] Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
[    0.000000] Modules linked in:
[    0.000000] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G        W      3.16.0-rc5-next-20140718-sasha-00046-g054cefc #898
[    0.000000] task: ffffffff9fa2d580 ti: ffffffff9fa29580 task.ti: ffffffff9fa29580
[    0.000000] RIP: wake_up_process (kernel/sched/core.c:1768)
[    0.000000] RSP: 0000:ffffffff9fa2d238  EFLAGS: 00010046
[    0.000000] RAX: ffffffff9fa2d580 RBX: 0000000000000000 RCX: 0000000000000000
[    0.000000] RDX: 0000000000000000 RSI: ffffffff9f2f4567 RDI: ffffffff9f28c581
[    0.000000] RBP: ffffffff9fa2d240 R08: 0000000000000001 R09: 0000000000000001
[    0.000000] R10: 0000000000000000 R11: 3d3d3d3d3d3d3d3d R12: ffffffff9fab3000
[    0.000000] R13: ffffffff99288bb0 R14: ffffffff9f18a72b R15: ffff8806f2050000
[    0.000000] FS:  0000000000000000(0000) GS:ffff8805f3a00000(0000) knlGS:0000000000000000
[    0.000000] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
[    0.000000] CR2: 0000000000000000 CR3: 000000001fa22000 CR4: 00000000000006b0
[    0.000000] Stack:
[    0.000000]  00000000ffffffff ffffffff9fa2d388 ffffffff991fd3e7 0000000000000002
[    0.000000]  ffffffff9fa2d580 ffffffffa13b04a0 ffffffff00000000 dead4ead00000000
[    0.000000]  ffffffffffffffff ffffffffffffffff ffffffffa13ade38 0000000000000000
[    0.000000] Call Trace:
[    0.000000]  <UNK>
[    0.000000] kthread_create_on_node (kernel/kthread.c:294)
[    0.000000] ? debug_smp_processor_id (lib/smp_processor_id.c:57)
[    0.000000] ? debug_smp_processor_id (lib/smp_processor_id.c:57)
[    0.000000] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63)
[    0.000000] ? trace_hardirqs_off_caller (kernel/locking/lockdep.c:2639 (discriminator 8))
[    0.000000] rcu_spawn_one_boost_kthread (kernel/rcu/tree_plugin.h:1348)
[    0.000000] rcu_cpu_notify (kernel/rcu/tree.c:3454)
[    0.000000] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63)
[    0.000000] ? trace_hardirqs_off_caller (kernel/locking/lockdep.c:2639 (discriminator 8))
[    0.000000] rcu_init (kernel/rcu/tree.c:3753)
[    0.000000] start_kernel (init/main.c:581)
[    0.000000] ? early_idt_handlers (arch/x86/kernel/head_64.S:344)
[    0.000000] x86_64_start_reservations (arch/x86/kernel/head64.c:194)
[    0.000000] x86_64_start_kernel (arch/x86/kernel/head64.c:183)
[ 0.000000] Code: 55 b0 8b 40 18 e9 2c fd ff ff 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 e8 1b 0b c9 04 55 48 89 e5 53 48 89 fb e8 ae 36 21 00 <48> 8b 03 a8 0c 75 17 48 89 df 31 d2 be 03 00 00 00 e8 98 f9 ff
All code
========
   0:   55                      push   %rbp
   1:   b0 8b                   mov    $0x8b,%al
   3:   40 18 e9                sbb    %bpl,%cl
   6:   2c fd                   sub    $0xfd,%al
   8:   ff                      (bad)
   9:   ff 66 66                jmpq   *0x66(%rsi)
   c:   66 66 66 66 2e 0f 1f    data32 data32 data32 nopw %cs:0x0(%rax,%rax,1)
  13:   84 00 00 00 00 00
  19:   e8 1b 0b c9 04          callq  0x4c90b39
  1e:   55                      push   %rbp
  1f:   48 89 e5                mov    %rsp,%rbp
  22:   53                      push   %rbx
  23:   48 89 fb                mov    %rdi,%rbx
  26:   e8 ae 36 21 00          callq  0x2136d9
  2b:*  48 8b 03                mov    (%rbx),%rax              <-- trapping instruction
  2e:   a8 0c                   test   $0xc,%al
  30:   75 17                   jne    0x49
  32:   48 89 df                mov    %rbx,%rdi
  35:   31 d2                   xor    %edx,%edx
  37:   be 03 00 00 00          mov    $0x3,%esi
  3c:   e8 98 f9 ff 00          callq  0xfff9d9

Code starting with the faulting instruction
===========================================
   0:   48 8b 03                mov    (%rbx),%rax
   3:   a8 0c                   test   $0xc,%al
   5:   75 17                   jne    0x1e
   7:   48 89 df                mov    %rbx,%rdi
   a:   31 d2                   xor    %edx,%edx
   c:   be 03 00 00 00          mov    $0x3,%esi
  11:   e8 98 f9 ff 00          callq  0xfff9ae
[    0.000000] RIP wake_up_process (kernel/sched/core.c:1768)
[    0.000000]  RSP <ffffffff9fa2d238>
[    0.000000] CR2: 0000000000000000


Thanks,
Sasha

Re: [PATCH RFC tip/core/rcu 2/2] rcu: Create rcuo kthreads only for onlined CPUs

[Paul E. McKenney]

On Fri, Jul 18, 2014 at 07:17:17AM -0400, Sasha Levin wrote:
> On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
> > From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> > 
> > RCU currently uses for_each_possible_cpu() to spawn rcuo kthreads,
> > which can result in more rcuo kthreads than one would expect, for
> > example, derRichard reported 64 CPUs worth of rcuo kthreads on an
> > 8-CPU image.  This commit therefore creates rcuo kthreads only for
> > those CPUs that actually come online.
> > 
> > Reported-by: derRichard
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> Hey Paul,
> 
> Me again. :)
> 
> It seems that this patch moved thread initialization to a point way
> too early during boot, before rest_init() which initializes kthreadd_task
> runs, so creating a new kthread triggers a NULL ptr deref:

This should be fixed by commit 918179699e4a in -rcu.  My guess is that
you are instead using commit c6e2955266d14, which, said to say, made it
into -next yesterday.  :-(

If my guess is wrong, please let me know!

							Thanx, Paul

> [    0.000000] BUG: unable to handle kernel NULL pointer dereference at           (null)
> [    0.000000] IP: wake_up_process (kernel/sched/core.c:1768)
> [    0.000000] PGD 0
> [    0.000000] Oops: 0000 [#1] PREEMPT SMP DEBUG_PAGEALLOC
> [    0.000000] Modules linked in:
> [    0.000000] CPU: 0 PID: 0 Comm: swapper/0 Tainted: G        W      3.16.0-rc5-next-20140718-sasha-00046-g054cefc #898
> [    0.000000] task: ffffffff9fa2d580 ti: ffffffff9fa29580 task.ti: ffffffff9fa29580
> [    0.000000] RIP: wake_up_process (kernel/sched/core.c:1768)
> [    0.000000] RSP: 0000:ffffffff9fa2d238  EFLAGS: 00010046
> [    0.000000] RAX: ffffffff9fa2d580 RBX: 0000000000000000 RCX: 0000000000000000
> [    0.000000] RDX: 0000000000000000 RSI: ffffffff9f2f4567 RDI: ffffffff9f28c581
> [    0.000000] RBP: ffffffff9fa2d240 R08: 0000000000000001 R09: 0000000000000001
> [    0.000000] R10: 0000000000000000 R11: 3d3d3d3d3d3d3d3d R12: ffffffff9fab3000
> [    0.000000] R13: ffffffff99288bb0 R14: ffffffff9f18a72b R15: ffff8806f2050000
> [    0.000000] FS:  0000000000000000(0000) GS:ffff8805f3a00000(0000) knlGS:0000000000000000
> [    0.000000] CS:  0010 DS: 0000 ES: 0000 CR0: 000000008005003b
> [    0.000000] CR2: 0000000000000000 CR3: 000000001fa22000 CR4: 00000000000006b0
> [    0.000000] Stack:
> [    0.000000]  00000000ffffffff ffffffff9fa2d388 ffffffff991fd3e7 0000000000000002
> [    0.000000]  ffffffff9fa2d580 ffffffffa13b04a0 ffffffff00000000 dead4ead00000000
> [    0.000000]  ffffffffffffffff ffffffffffffffff ffffffffa13ade38 0000000000000000
> [    0.000000] Call Trace:
> [    0.000000]  <UNK>
> [    0.000000] kthread_create_on_node (kernel/kthread.c:294)
> [    0.000000] ? debug_smp_processor_id (lib/smp_processor_id.c:57)
> [    0.000000] ? debug_smp_processor_id (lib/smp_processor_id.c:57)
> [    0.000000] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63)
> [    0.000000] ? trace_hardirqs_off_caller (kernel/locking/lockdep.c:2639 (discriminator 8))
> [    0.000000] rcu_spawn_one_boost_kthread (kernel/rcu/tree_plugin.h:1348)
> [    0.000000] rcu_cpu_notify (kernel/rcu/tree.c:3454)
> [    0.000000] ? __this_cpu_preempt_check (lib/smp_processor_id.c:63)
> [    0.000000] ? trace_hardirqs_off_caller (kernel/locking/lockdep.c:2639 (discriminator 8))
> [    0.000000] rcu_init (kernel/rcu/tree.c:3753)
> [    0.000000] start_kernel (init/main.c:581)
> [    0.000000] ? early_idt_handlers (arch/x86/kernel/head_64.S:344)
> [    0.000000] x86_64_start_reservations (arch/x86/kernel/head64.c:194)
> [    0.000000] x86_64_start_kernel (arch/x86/kernel/head64.c:183)
> [ 0.000000] Code: 55 b0 8b 40 18 e9 2c fd ff ff 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 e8 1b 0b c9 04 55 48 89 e5 53 48 89 fb e8 ae 36 21 00 <48> 8b 03 a8 0c 75 17 48 89 df 31 d2 be 03 00 00 00 e8 98 f9 ff
> All code
> ========
>    0:   55                      push   %rbp
>    1:   b0 8b                   mov    $0x8b,%al
>    3:   40 18 e9                sbb    %bpl,%cl
>    6:   2c fd                   sub    $0xfd,%al
>    8:   ff                      (bad)
>    9:   ff 66 66                jmpq   *0x66(%rsi)
>    c:   66 66 66 66 2e 0f 1f    data32 data32 data32 nopw %cs:0x0(%rax,%rax,1)
>   13:   84 00 00 00 00 00
>   19:   e8 1b 0b c9 04          callq  0x4c90b39
>   1e:   55                      push   %rbp
>   1f:   48 89 e5                mov    %rsp,%rbp
>   22:   53                      push   %rbx
>   23:   48 89 fb                mov    %rdi,%rbx
>   26:   e8 ae 36 21 00          callq  0x2136d9
>   2b:*  48 8b 03                mov    (%rbx),%rax              <-- trapping instruction
>   2e:   a8 0c                   test   $0xc,%al
>   30:   75 17                   jne    0x49
>   32:   48 89 df                mov    %rbx,%rdi
>   35:   31 d2                   xor    %edx,%edx
>   37:   be 03 00 00 00          mov    $0x3,%esi
>   3c:   e8 98 f9 ff 00          callq  0xfff9d9
> 
> Code starting with the faulting instruction
> ===========================================
>    0:   48 8b 03                mov    (%rbx),%rax
>    3:   a8 0c                   test   $0xc,%al
>    5:   75 17                   jne    0x1e
>    7:   48 89 df                mov    %rbx,%rdi
>    a:   31 d2                   xor    %edx,%edx
>    c:   be 03 00 00 00          mov    $0x3,%esi
>   11:   e8 98 f9 ff 00          callq  0xfff9ae
> [    0.000000] RIP wake_up_process (kernel/sched/core.c:1768)
> [    0.000000]  RSP <ffffffff9fa2d238>
> [    0.000000] CR2: 0000000000000000
> 
> 
> Thanks,
> Sasha
> 

Re: [PATCH RFC tip/core/rcu 2/2] rcu: Create rcuo kthreads only for onlined CPUs

[Sasha Levin]

On 07/18/2014 08:55 AM, Paul E. McKenney wrote:
> On Fri, Jul 18, 2014 at 07:17:17AM -0400, Sasha Levin wrote:
>> > On 07/14/2014 06:06 AM, Paul E. McKenney wrote:
>>> > > From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
>>> > > 
>>> > > RCU currently uses for_each_possible_cpu() to spawn rcuo kthreads,
>>> > > which can result in more rcuo kthreads than one would expect, for
>>> > > example, derRichard reported 64 CPUs worth of rcuo kthreads on an
>>> > > 8-CPU image.  This commit therefore creates rcuo kthreads only for
>>> > > those CPUs that actually come online.
>>> > > 
>>> > > Reported-by: derRichard
>>> > > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
>> > 
>> > Hey Paul,
>> > 
>> > Me again. :)
>> > 
>> > It seems that this patch moved thread initialization to a point way
>> > too early during boot, before rest_init() which initializes kthreadd_task
>> > runs, so creating a new kthread triggers a NULL ptr deref:
> This should be fixed by commit 918179699e4a in -rcu.  My guess is that
> you are instead using commit c6e2955266d14, which, said to say, made it
> into -next yesterday.  :-(
> 
> If my guess is wrong, please let me know!

Your guess is correct.

I'll wait for the next -next tree and retry.


Thanks,
Sasha