From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S944991AbdDTLdx (ORCPT ); Thu, 20 Apr 2017 07:33:53 -0400 Received: from terminus.zytor.com ([65.50.211.136]:60899 "EHLO terminus.zytor.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S965859AbdDTLdM (ORCPT ); Thu, 20 Apr 2017 07:33:12 -0400 Date: Thu, 20 Apr 2017 04:30:37 -0700 From: tip-bot for Thomas Gleixner Message-ID: Cc: bigeasy@linutronix.de, rostedt@goodmis.org, peterz@infradead.org, linux-kernel@vger.kernel.org, mingo@kernel.org, hpa@zytor.com, tglx@linutronix.de Reply-To: tglx@linutronix.de, mingo@kernel.org, hpa@zytor.com, linux-kernel@vger.kernel.org, peterz@infradead.org, bigeasy@linutronix.de, rostedt@goodmis.org In-Reply-To: <20170418170554.382344438@linutronix.de> References: <20170418170554.382344438@linutronix.de> To: linux-tip-commits@vger.kernel.org Subject: [tip:smp/hotplug] cpu/hotplug: Convert hotplug locking to percpu rwsem Git-Commit-ID: d215aab82d81974f438bfbc80aa437132f3c37c3 X-Mailer: tip-git-log-daemon Robot-ID: Robot-Unsubscribe: Contact to get blacklisted from these emails MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Content-Type: text/plain; charset=UTF-8 Content-Disposition: inline Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Commit-ID: d215aab82d81974f438bfbc80aa437132f3c37c3 Gitweb: http://git.kernel.org/tip/d215aab82d81974f438bfbc80aa437132f3c37c3 Author: Thomas Gleixner AuthorDate: Tue, 18 Apr 2017 19:05:06 +0200 Committer: Thomas Gleixner CommitDate: Thu, 20 Apr 2017 13:08:58 +0200 cpu/hotplug: Convert hotplug locking to percpu rwsem There are no more (known) nested calls to get_online_cpus() so it's possible to remove the nested call magic and convert the mutex to a percpu-rwsem, which speeds up get/put_online_cpus() significantly for the uncontended case. The contended case (write locked for hotplug operations) is slow anyway, so the slightly more expensive down_write of the percpu rwsem does not matter. [ peterz: Add lockdep assertions ] Signed-off-by: Thomas Gleixner Cc: Peter Zijlstra Cc: Sebastian Siewior Cc: Steven Rostedt Link: http://lkml.kernel.org/r/20170418170554.382344438@linutronix.de --- include/linux/cpu.h | 2 + kernel/cpu.c | 110 ++++++++------------------------------------------ kernel/jump_label.c | 2 + kernel/padata.c | 1 + kernel/stop_machine.c | 2 + 5 files changed, 23 insertions(+), 94 deletions(-) diff --git a/include/linux/cpu.h b/include/linux/cpu.h index f920812..83010c3 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -105,6 +105,7 @@ extern void cpu_hotplug_begin(void); extern void cpu_hotplug_done(void); extern void get_online_cpus(void); extern void put_online_cpus(void); +extern void lockdep_assert_hotplug_held(void); extern void cpu_hotplug_disable(void); extern void cpu_hotplug_enable(void); void clear_tasks_mm_cpumask(int cpu); @@ -118,6 +119,7 @@ static inline void cpu_hotplug_done(void) {} #define put_online_cpus() do { } while (0) #define cpu_hotplug_disable() do { } while (0) #define cpu_hotplug_enable() do { } while (0) +static inline void lockdep_assert_hotplug_held(void) {} #endif /* CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_PM_SLEEP_SMP diff --git a/kernel/cpu.c b/kernel/cpu.c index f932e68..05341f7 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #define CREATE_TRACE_POINTS @@ -196,121 +197,41 @@ void cpu_maps_update_done(void) mutex_unlock(&cpu_add_remove_lock); } -/* If set, cpu_up and cpu_down will return -EBUSY and do nothing. +/* + * If set, cpu_up and cpu_down will return -EBUSY and do nothing. * Should always be manipulated under cpu_add_remove_lock */ static int cpu_hotplug_disabled; #ifdef CONFIG_HOTPLUG_CPU -static struct { - struct task_struct *active_writer; - /* wait queue to wake up the active_writer */ - wait_queue_head_t wq; - /* verifies that no writer will get active while readers are active */ - struct mutex lock; - /* - * Also blocks the new readers during - * an ongoing cpu hotplug operation. - */ - atomic_t refcount; - -#ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lockdep_map dep_map; -#endif -} cpu_hotplug = { - .active_writer = NULL, - .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), - .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), -#ifdef CONFIG_DEBUG_LOCK_ALLOC - .dep_map = STATIC_LOCKDEP_MAP_INIT("cpu_hotplug.dep_map", &cpu_hotplug.dep_map), -#endif -}; - -/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */ -#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map) -#define cpuhp_lock_acquire_tryread() \ - lock_map_acquire_tryread(&cpu_hotplug.dep_map) -#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) -#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) - +DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); void get_online_cpus(void) { - might_sleep(); - if (cpu_hotplug.active_writer == current) - return; - cpuhp_lock_acquire_read(); - mutex_lock(&cpu_hotplug.lock); - atomic_inc(&cpu_hotplug.refcount); - mutex_unlock(&cpu_hotplug.lock); + percpu_down_read(&cpu_hotplug_lock); } EXPORT_SYMBOL_GPL(get_online_cpus); void put_online_cpus(void) { - int refcount; - - if (cpu_hotplug.active_writer == current) - return; - - refcount = atomic_dec_return(&cpu_hotplug.refcount); - if (WARN_ON(refcount < 0)) /* try to fix things up */ - atomic_inc(&cpu_hotplug.refcount); - - if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq)) - wake_up(&cpu_hotplug.wq); - - cpuhp_lock_release(); - + percpu_up_read(&cpu_hotplug_lock); } EXPORT_SYMBOL_GPL(put_online_cpus); -/* - * This ensures that the hotplug operation can begin only when the - * refcount goes to zero. - * - * Note that during a cpu-hotplug operation, the new readers, if any, - * will be blocked by the cpu_hotplug.lock - * - * Since cpu_hotplug_begin() is always called after invoking - * cpu_maps_update_begin(), we can be sure that only one writer is active. - * - * Note that theoretically, there is a possibility of a livelock: - * - Refcount goes to zero, last reader wakes up the sleeping - * writer. - * - Last reader unlocks the cpu_hotplug.lock. - * - A new reader arrives at this moment, bumps up the refcount. - * - The writer acquires the cpu_hotplug.lock finds the refcount - * non zero and goes to sleep again. - * - * However, this is very difficult to achieve in practice since - * get_online_cpus() not an api which is called all that often. - * - */ void cpu_hotplug_begin(void) { - DEFINE_WAIT(wait); - - cpu_hotplug.active_writer = current; - cpuhp_lock_acquire(); - - for (;;) { - mutex_lock(&cpu_hotplug.lock); - prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE); - if (likely(!atomic_read(&cpu_hotplug.refcount))) - break; - mutex_unlock(&cpu_hotplug.lock); - schedule(); - } - finish_wait(&cpu_hotplug.wq, &wait); + percpu_down_write(&cpu_hotplug_lock); } void cpu_hotplug_done(void) { - cpu_hotplug.active_writer = NULL; - mutex_unlock(&cpu_hotplug.lock); - cpuhp_lock_release(); + percpu_up_write(&cpu_hotplug_lock); +} + +void lockdep_assert_hotplug_held(void) +{ + percpu_rwsem_assert_held(&cpu_hotplug_lock); } /* @@ -344,8 +265,6 @@ void cpu_hotplug_enable(void) EXPORT_SYMBOL_GPL(cpu_hotplug_enable); #endif /* CONFIG_HOTPLUG_CPU */ -/* Notifier wrappers for transitioning to state machine */ - static int bringup_wait_for_ap(unsigned int cpu) { struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); @@ -1482,6 +1401,8 @@ int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state, int cpu, ret = 0; bool dynstate; + lockdep_assert_hotplug_held(); + if (cpuhp_cb_check(state) || !name) return -EINVAL; @@ -1600,6 +1521,7 @@ void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke) int cpu; BUG_ON(cpuhp_cb_check(state)); + lockdep_assert_hotplug_held(); mutex_lock(&cpuhp_state_mutex); if (sp->multi_instance) { diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 308b12e..9455d2b 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -130,6 +130,7 @@ void __static_key_slow_inc(struct static_key *key) * the all CPUs, for that to be serialized against CPU hot-plug * we need to avoid CPUs coming online. */ + lockdep_assert_hotplug_held(); jump_label_lock(); if (atomic_read(&key->enabled) == 0) { atomic_set(&key->enabled, -1); @@ -158,6 +159,7 @@ EXPORT_SYMBOL_GPL(static_key_slow_inc_cpuslocked); static void __static_key_slow_dec(struct static_key *key, unsigned long rate_limit, struct delayed_work *work) { + lockdep_assert_hotplug_held(); /* * The negative count check is valid even when a negative * key->enabled is in use by static_key_slow_inc(); a diff --git a/kernel/padata.c b/kernel/padata.c index e5ff511..dc74ddba 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -1013,6 +1013,7 @@ err: */ struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq) { + lockdep_assert_hotplug_held(); return padata_alloc(wq, cpu_possible_mask, cpu_possible_mask); } EXPORT_SYMBOL(padata_alloc_possible); diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index a8c5636..2390340 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -562,6 +562,8 @@ int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data, .active_cpus = cpus, }; + lockdep_assert_hotplug_held(); + if (!stop_machine_initialized) { /* * Handle the case where stop_machine() is called