linux-kernel.vger.kernel.org archive mirror
 help / color / mirror / Atom feed
From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
To: Thomas Gleixner <tglx@linutronix.de>
Cc: LKML <linux-kernel@vger.kernel.org>,
	Ingo Molnar <mingo@kernel.org>,
	Peter Zijlstra <peterz@infradead.org>,
	Rusty Russell <rusty@rustcorp.com.au>,
	"Srivatsa S. Bhat" <srivatsa.bhat@linux.vnet.ibm.com>,
	Arjan van de Veen <arjan@infradead.org>,
	Paul Turner <pjt@google.com>,
	Richard Weinberger <rw@linutronix.de>,
	Magnus Damm <magnus.damm@gmail.com>
Subject: Re: [patch 32/40] rcu: Convert rcutree to hotplug state machine
Date: Mon, 11 Feb 2013 16:01:01 -0800	[thread overview]
Message-ID: <20130212000100.GB2666@linux.vnet.ibm.com> (raw)
In-Reply-To: <20130131120743.656655375@linutronix.de>

On Thu, Jan 31, 2013 at 12:11:38PM -0000, Thomas Gleixner wrote:
> Do we really need so many states here ?

Well, all that RCU does for CPU_DYING is to do tracing, which could be
ditched.  Required changes called out inline below.

All that the CPU_ONLINE and CPU_DOWN_PREPARE notifiers do is set
up affinity for the RCU-boost kthreads.  These are unfortunately not
per-CPU kthreads, but perhaps something similar could be set up.  This is
strictly a performance optimization, so the CPU_ONLINE notifier could
be replaced by having the kthread check which of its CPUs was online.
Unfortunately, the same is not true of CPU_DOWN_PREPARE because if the
kthread was too slow about it, the scheduler would get annoyed about a
kthread being runnable only on offlined CPUs.

It is not clear that this is worthwhile.  Thoughts on other ways to
get this done?

							Thanx, Paul

> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> ---
>  include/linux/cpuhotplug.h |   18 ++++++++
>  kernel/cpu.c               |   12 +++++
>  kernel/rcutree.c           |   95 ++++++++++++++++++++-------------------------
>  3 files changed, 73 insertions(+), 52 deletions(-)
> 
> Index: linux-2.6/include/linux/cpuhotplug.h
> ===================================================================
> --- linux-2.6.orig/include/linux/cpuhotplug.h
> +++ linux-2.6/include/linux/cpuhotplug.h
> @@ -12,6 +12,7 @@ enum cpuhp_states {
>  	CPUHP_PERF_PREPARE,
>  	CPUHP_SCHED_MIGRATE_PREP,
>  	CPUHP_WORKQUEUE_PREP,
> +	CPUHP_RCUTREE_PREPARE,
>  	CPUHP_NOTIFY_PREPARE,
>  	CPUHP_NOTIFY_DEAD,
>  	CPUHP_CPUFREQ_DEAD,
> @@ -27,6 +28,7 @@ enum cpuhp_states {
>  	CPUHP_AP_ARM64_TIMER_STARTING,
>  	CPUHP_AP_KVM_STARTING,
>  	CPUHP_AP_NOTIFY_DYING,
> +	CPUHP_AP_RCUTREE_DYING,

Drop this.

>  	CPUHP_AP_X86_TBOOT_DYING,
>  	CPUHP_AP_S390_VTIME_DYING,
>  	CPUHP_AP_SCHED_NOHZ_DYING,
> @@ -39,6 +41,7 @@ enum cpuhp_states {
>  	CPUHP_SCHED_MIGRATE_ONLINE,
>  	CPUHP_WORKQUEUE_ONLINE,
>  	CPUHP_CPUFREQ_ONLINE,
> +	CPUHP_RCUTREE_ONLINE,
>  	CPUHP_NOTIFY_ONLINE,
>  	CPUHP_NOTIFY_DOWN_PREPARE,
>  	CPUHP_PERF_X86_UNCORE_ONLINE,
> @@ -147,4 +150,19 @@ int workqueue_prepare_cpu(unsigned int c
>  int workqueue_online_cpu(unsigned int cpu);
>  int workqueue_offline_cpu(unsigned int cpu);
> 
> +/* RCUtree hotplug events */
> +#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
> +int rcutree_prepare_cpu(unsigned int cpu);
> +int rcutree_online_cpu(unsigned int cpu);
> +int rcutree_offline_cpu(unsigned int cpu);
> +int rcutree_dead_cpu(unsigned int cpu);
> +int rcutree_dying_cpu(unsigned int cpu);

And this...

> +#else
> +#define rcutree_prepare_cpu	NULL
> +#define rcutree_online_cpu	NULL
> +#define rcutree_offline_cpu	NULL
> +#define rcutree_dead_cpu	NULL
> +#define rcutree_dying_cpu	NULL

And of course this.

> +#endif
> +
>  #endif
> Index: linux-2.6/kernel/cpu.c
> ===================================================================
> --- linux-2.6.orig/kernel/cpu.c
> +++ linux-2.6/kernel/cpu.c
> @@ -755,6 +755,10 @@ static struct cpuhp_step cpuhp_bp_states
>  		.startup = workqueue_prepare_cpu,
>  		.teardown = NULL,
>  	},
> +	[CPUHP_RCUTREE_PREPARE] = {
> +		.startup = rcutree_prepare_cpu,
> +		.teardown = rcutree_dead_cpu,
> +	},
>  	[CPUHP_NOTIFY_PREPARE] = {
>  		.startup = notify_prepare,
>  		.teardown = NULL,
> @@ -787,6 +791,10 @@ static struct cpuhp_step cpuhp_bp_states
>  		.startup = workqueue_online_cpu,
>  		.teardown = workqueue_offline_cpu,
>  	},
> +	[CPUHP_RCUTREE_ONLINE] = {
> +		.startup = rcutree_online_cpu,
> +		.teardown = rcutree_offline_cpu,
> +	},
>  	[CPUHP_NOTIFY_ONLINE] = {
>  		.startup = notify_online,
>  		.teardown = NULL,
> @@ -813,6 +821,10 @@ static struct cpuhp_step cpuhp_ap_states
>  		.startup = NULL,
>  		.teardown = notify_dying,
>  	},
> +	[CPUHP_AP_RCUTREE_DYING] = {
> +		.startup = NULL,
> +		.teardown = rcutree_dying_cpu,
> +	},
>  	[CPUHP_AP_SCHED_NOHZ_DYING] = {
>  		.startup = NULL,
>  		.teardown = nohz_balance_exit_idle,
> Index: linux-2.6/kernel/rcutree.c
> ===================================================================
> --- linux-2.6.orig/kernel/rcutree.c
> +++ linux-2.6/kernel/rcutree.c
> @@ -2787,67 +2787,59 @@ rcu_init_percpu_data(int cpu, struct rcu
>  	mutex_unlock(&rsp->onoff_mutex);
>  }
> 
> -static void __cpuinit rcu_prepare_cpu(int cpu)
> +int __cpuinit rcutree_prepare_cpu(unsigned int cpu)
>  {
>  	struct rcu_state *rsp;
> 
>  	for_each_rcu_flavor(rsp)
>  		rcu_init_percpu_data(cpu, rsp,
>  				     strcmp(rsp->name, "rcu_preempt") == 0);
> +	rcu_prepare_kthreads(cpu);
> +	return 0;
>  }
> 
> -/*
> - * Handle CPU online/offline notification events.
> - */
> -static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
> -				    unsigned long action, void *hcpu)
> +int __cpuinit rcutree_dead_cpu(unsigned int cpu)
>  {
> -	long cpu = (long)hcpu;
> -	struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
> -	struct rcu_node *rnp = rdp->mynode;
>  	struct rcu_state *rsp;
> -	int ret = NOTIFY_OK;
> 
> -	trace_rcu_utilization("Start CPU hotplug");
> -	switch (action) {
> -	case CPU_UP_PREPARE:
> -	case CPU_UP_PREPARE_FROZEN:
> -		rcu_prepare_cpu(cpu);
> -		rcu_prepare_kthreads(cpu);
> -		break;
> -	case CPU_ONLINE:
> -	case CPU_DOWN_FAILED:
> -		rcu_boost_kthread_setaffinity(rnp, -1);
> -		break;
> -	case CPU_DOWN_PREPARE:
> -		if (nocb_cpu_expendable(cpu))
> -			rcu_boost_kthread_setaffinity(rnp, cpu);
> -		else
> -			ret = NOTIFY_BAD;
> -		break;
> -	case CPU_DYING:
> -	case CPU_DYING_FROZEN:
> -		/*
> -		 * The whole machine is "stopped" except this CPU, so we can
> -		 * touch any data without introducing corruption. We send the
> -		 * dying CPU's callbacks to an arbitrarily chosen online CPU.
> -		 */
> -		for_each_rcu_flavor(rsp)
> -			rcu_cleanup_dying_cpu(rsp);
> -		rcu_cleanup_after_idle(cpu);
> -		break;
> -	case CPU_DEAD:
> -	case CPU_DEAD_FROZEN:
> -	case CPU_UP_CANCELED:
> -	case CPU_UP_CANCELED_FROZEN:
> -		for_each_rcu_flavor(rsp)
> -			rcu_cleanup_dead_cpu(cpu, rsp);
> -		break;
> -	default:
> -		break;
> -	}
> -	trace_rcu_utilization("End CPU hotplug");
> -	return ret;
> +	for_each_rcu_flavor(rsp)
> +		rcu_cleanup_dead_cpu(cpu, rsp);
> +	return 0;
> +}
> +
> +static void __cpuinit rcutree_affinity_setting(unsigned int cpu, int outgoing)
> +{
> +	struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
> +
> +	rcu_boost_kthread_setaffinity(rdp->mynode, outgoing);
> +}
> +
> +int __cpuinit rcutree_online_cpu(unsigned int cpu)
> +{
> +	rcutree_affinity_setting(cpu, -1);
> +	return 0;
> +}
> +
> +int __cpuinit rcutree_offline_cpu(unsigned int cpu)
> +{
> +	if (!nocb_cpu_expendable(cpu))
> +		return -EINVAL;
> +	rcutree_affinity_setting(cpu, cpu);
> +	return 0;
> +}
> +
> +int __cpuinit rcutree_dying_cpu(unsigned int cpu)
> +{
> +	struct rcu_state *rsp;
> +	/*
> +	 * The whole machine is "stopped" except this CPU, so we can
> +	 * touch any data without introducing corruption. We send the
> +	 * dying CPU's callbacks to an arbitrarily chosen online CPU.
> +	 */
> +	for_each_rcu_flavor(rsp)
> +		rcu_cleanup_dying_cpu(rsp);
> +	rcu_cleanup_after_idle(cpu);
> +	return 0;
>  }

And rcu_dying_cpu() above, along with both definitions of
rcu_cleanup_dying_cpu().

>  /*
> @@ -3071,9 +3063,8 @@ void __init rcu_init(void)
>  	 * this is called early in boot, before either interrupts
>  	 * or the scheduler are operational.
>  	 */
> -	cpu_notifier(rcu_cpu_notify, 0);
>  	for_each_online_cpu(cpu)
> -		rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
> +		rcutree_prepare_cpu(cpu);
>  	check_cpu_stall_init();
>  }
> 
> 
> 


  reply	other threads:[~2013-02-12  0:01 UTC|newest]

Thread overview: 67+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2013-01-31 15:44 [patch 00/40] CPU hotplug rework - episode I Thomas Gleixner
2013-01-31 12:11 ` [patch 01/40] smpboot: Allow selfparking per cpu threads Thomas Gleixner
2013-02-09  0:29   ` Paul E. McKenney
2013-02-14 17:46   ` [tip:smp/hotplug] " tip-bot for Thomas Gleixner
2013-01-31 12:11 ` [patch 02/40] stop_machine: Store task reference in a separate per cpu variable Thomas Gleixner
2013-02-09  0:33   ` Paul E. McKenney
2013-02-14 17:47   ` [tip:smp/hotplug] " tip-bot for Thomas Gleixner
2013-01-31 12:11 ` [patch 03/40] stop_machine: Use smpboot threads Thomas Gleixner
2013-02-09  0:39   ` Paul E. McKenney
2013-02-14 17:49   ` [tip:smp/hotplug] " tip-bot for Thomas Gleixner
2013-01-31 12:11 ` [patch 04/40] cpu: Restructure FROZEN state handling Thomas Gleixner
2013-02-09  0:52   ` Paul E. McKenney
2014-10-09 16:53   ` Borislav Petkov
2013-01-31 12:11 ` [patch 05/40] cpu: Restructure cpu_down code Thomas Gleixner
2013-02-09  0:49   ` Paul E. McKenney
2014-10-09 17:05   ` Borislav Petkov
2013-01-31 12:11 ` [patch 06/40] cpu: hotplug: Split out cpu down functions Thomas Gleixner
2013-02-09  0:54   ` Paul E. McKenney
2013-01-31 12:11 ` [patch 07/40] cpu: hotplug: Convert to a state machine for the control processor Thomas Gleixner
2013-02-11 20:09   ` Paul E. McKenney
2013-01-31 12:11 ` [patch 08/40] cpu: hotplug: Convert the hotplugged processor work to a state machine Thomas Gleixner
2013-02-11 20:17   ` Paul E. McKenney
2013-01-31 12:11 ` [patch 10/40] sched: Convert to state machine callbacks Thomas Gleixner
2013-02-11 23:46   ` Paul E. McKenney
2013-01-31 12:11 ` [patch 09/40] cpu: hotplug: Implement setup/removal interface Thomas Gleixner
2013-02-01 13:44   ` Hillf Danton
2013-02-01 13:52     ` Thomas Gleixner
2013-01-31 12:11 ` [patch 11/40] x86: uncore: Move teardown callback to CPU_DEAD Thomas Gleixner
2013-01-31 12:11 ` [patch 12/40] x86: uncore: Convert to hotplug state machine Thomas Gleixner
2013-01-31 12:11 ` [patch 13/40] perf: " Thomas Gleixner
2013-01-31 12:11 ` [patch 14/40] x86: perf: Convert the core to the " Thomas Gleixner
2013-01-31 12:11 ` [patch 16/40] blackfin: perf: Convert hotplug notifier to " Thomas Gleixner
2013-01-31 12:11 ` [patch 15/40] x86: perf: Convert AMD IBS to hotplug " Thomas Gleixner
2013-01-31 12:11 ` [patch 17/40] powerpc: perf: Convert book3s notifier to state machine callbacks Thomas Gleixner
2013-01-31 12:11 ` [patch 18/40] s390: perf: Convert the hotplug " Thomas Gleixner
2013-01-31 12:11 ` [patch 19/40] sh: perf: Convert the hotplug notifiers " Thomas Gleixner
2013-01-31 12:11 ` [patch 21/40] sched: Convert the migration callback to hotplug states Thomas Gleixner
2013-01-31 12:11 ` [patch 20/40] perf: Remove perf cpu notifier code Thomas Gleixner
2013-01-31 12:11 ` [patch 22/40] workqueue: Convert to state machine callbacks Thomas Gleixner
2013-01-31 12:11 ` [patch 23/40] cpufreq: Convert to hotplug state machine Thomas Gleixner
2013-01-31 12:11 ` [patch 24/40] arm64: Convert generic timers " Thomas Gleixner
2013-01-31 12:11 ` [patch 25/40] arm: Convert VFP hotplug notifiers to " Thomas Gleixner
2013-01-31 12:11 ` [patch 26/40] arm: perf: Convert to hotplug " Thomas Gleixner
2013-01-31 12:11 ` [patch 27/40] virt: Convert kvm hotplug to " Thomas Gleixner
2013-01-31 12:11 ` [patch 28/40] cpuhotplug: Remove CPU_STARTING notifier Thomas Gleixner
2013-01-31 12:11 ` [patch 29/40] s390: Convert vtime to hotplug state machine Thomas Gleixner
2013-01-31 12:11 ` [patch 30/40] x86: tboot: Convert " Thomas Gleixner
2013-01-31 12:11 ` [patch 31/40] sched: Convert fair nohz balancer " Thomas Gleixner
2013-01-31 12:11 ` [patch 33/40] hrtimer: Convert " Thomas Gleixner
2013-01-31 12:11 ` [patch 32/40] rcu: Convert rcutree " Thomas Gleixner
2013-02-12  0:01   ` Paul E. McKenney [this message]
2013-02-12 15:50     ` Paul E. McKenney
2013-01-31 12:11 ` [patch 34/40] cpuhotplug: Remove CPU_DYING notifier Thomas Gleixner
2013-01-31 12:11 ` [patch 35/40] timers: Convert to hotplug state machine Thomas Gleixner
2013-01-31 12:11 ` [patch 36/40] profile: Convert ot " Thomas Gleixner
2013-01-31 12:11 ` [patch 37/40] x86: x2apic: Convert to cpu " Thomas Gleixner
2013-01-31 12:11 ` [patch 38/40] smp: Convert core to " Thomas Gleixner
2013-01-31 12:11 ` [patch 39/40] relayfs: Convert " Thomas Gleixner
2013-01-31 12:11 ` [patch 40/40] slab: " Thomas Gleixner
2013-01-31 20:23 ` [patch 00/40] CPU hotplug rework - episode I Andrew Morton
2013-01-31 21:48   ` Thomas Gleixner
2013-01-31 21:59     ` Linus Torvalds
2013-01-31 22:44       ` Thomas Gleixner
2013-01-31 22:55         ` Linus Torvalds
2013-02-01 10:51           ` Thomas Gleixner
2013-02-07  4:01             ` Rusty Russell
2013-02-09  0:28 ` Paul E. McKenney

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20130212000100.GB2666@linux.vnet.ibm.com \
    --to=paulmck@linux.vnet.ibm.com \
    --cc=arjan@infradead.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=magnus.damm@gmail.com \
    --cc=mingo@kernel.org \
    --cc=peterz@infradead.org \
    --cc=pjt@google.com \
    --cc=rusty@rustcorp.com.au \
    --cc=rw@linutronix.de \
    --cc=srivatsa.bhat@linux.vnet.ibm.com \
    --cc=tglx@linutronix.de \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).