There are no more (known) nested calls to get_online_cpus() so it's possible to remove the nested call magic and convert the mutex to a percpu-rwsem, which speeds up get/put_online_cpus() significantly for the uncontended case. The contended case (write locked for hotplug operations) is slow anyway, so the slightly more expensive down_write of the percpu rwsem does not matter. [ peterz: Add lockdep assertions ] Signed-off-by: Thomas Gleixner --- include/linux/cpu.h | 2 kernel/cpu.c | 110 +++++++------------------------------------------- kernel/jump_label.c | 2 kernel/padata.c | 1 kernel/stop_machine.c | 2 5 files changed, 23 insertions(+), 94 deletions(-) --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -105,6 +105,7 @@ extern void cpu_hotplug_begin(void); extern void cpu_hotplug_done(void); extern void get_online_cpus(void); extern void put_online_cpus(void); +extern void lockdep_assert_hotplug_held(void); extern void cpu_hotplug_disable(void); extern void cpu_hotplug_enable(void); void clear_tasks_mm_cpumask(int cpu); @@ -118,6 +119,7 @@ static inline void cpu_hotplug_done(void #define put_online_cpus() do { } while (0) #define cpu_hotplug_disable() do { } while (0) #define cpu_hotplug_enable() do { } while (0) +static inline void lockdep_assert_hotplug_held(void) {} #endif /* CONFIG_HOTPLUG_CPU */ #ifdef CONFIG_PM_SLEEP_SMP --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #define CREATE_TRACE_POINTS @@ -196,121 +197,41 @@ void cpu_maps_update_done(void) mutex_unlock(&cpu_add_remove_lock); } -/* If set, cpu_up and cpu_down will return -EBUSY and do nothing. +/* + * If set, cpu_up and cpu_down will return -EBUSY and do nothing. * Should always be manipulated under cpu_add_remove_lock */ static int cpu_hotplug_disabled; #ifdef CONFIG_HOTPLUG_CPU -static struct { - struct task_struct *active_writer; - /* wait queue to wake up the active_writer */ - wait_queue_head_t wq; - /* verifies that no writer will get active while readers are active */ - struct mutex lock; - /* - * Also blocks the new readers during - * an ongoing cpu hotplug operation. - */ - atomic_t refcount; - -#ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lockdep_map dep_map; -#endif -} cpu_hotplug = { - .active_writer = NULL, - .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), - .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), -#ifdef CONFIG_DEBUG_LOCK_ALLOC - .dep_map = STATIC_LOCKDEP_MAP_INIT("cpu_hotplug.dep_map", &cpu_hotplug.dep_map), -#endif -}; - -/* Lockdep annotations for get/put_online_cpus() and cpu_hotplug_begin/end() */ -#define cpuhp_lock_acquire_read() lock_map_acquire_read(&cpu_hotplug.dep_map) -#define cpuhp_lock_acquire_tryread() \ - lock_map_acquire_tryread(&cpu_hotplug.dep_map) -#define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) -#define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) - +DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock); void get_online_cpus(void) { - might_sleep(); - if (cpu_hotplug.active_writer == current) - return; - cpuhp_lock_acquire_read(); - mutex_lock(&cpu_hotplug.lock); - atomic_inc(&cpu_hotplug.refcount); - mutex_unlock(&cpu_hotplug.lock); + percpu_down_read(&cpu_hotplug_lock); } EXPORT_SYMBOL_GPL(get_online_cpus); void put_online_cpus(void) { - int refcount; - - if (cpu_hotplug.active_writer == current) - return; - - refcount = atomic_dec_return(&cpu_hotplug.refcount); - if (WARN_ON(refcount < 0)) /* try to fix things up */ - atomic_inc(&cpu_hotplug.refcount); - - if (refcount <= 0 && waitqueue_active(&cpu_hotplug.wq)) - wake_up(&cpu_hotplug.wq); - - cpuhp_lock_release(); - + percpu_up_read(&cpu_hotplug_lock); } EXPORT_SYMBOL_GPL(put_online_cpus); -/* - * This ensures that the hotplug operation can begin only when the - * refcount goes to zero. - * - * Note that during a cpu-hotplug operation, the new readers, if any, - * will be blocked by the cpu_hotplug.lock - * - * Since cpu_hotplug_begin() is always called after invoking - * cpu_maps_update_begin(), we can be sure that only one writer is active. - * - * Note that theoretically, there is a possibility of a livelock: - * - Refcount goes to zero, last reader wakes up the sleeping - * writer. - * - Last reader unlocks the cpu_hotplug.lock. - * - A new reader arrives at this moment, bumps up the refcount. - * - The writer acquires the cpu_hotplug.lock finds the refcount - * non zero and goes to sleep again. - * - * However, this is very difficult to achieve in practice since - * get_online_cpus() not an api which is called all that often. - * - */ void cpu_hotplug_begin(void) { - DEFINE_WAIT(wait); - - cpu_hotplug.active_writer = current; - cpuhp_lock_acquire(); - - for (;;) { - mutex_lock(&cpu_hotplug.lock); - prepare_to_wait(&cpu_hotplug.wq, &wait, TASK_UNINTERRUPTIBLE); - if (likely(!atomic_read(&cpu_hotplug.refcount))) - break; - mutex_unlock(&cpu_hotplug.lock); - schedule(); - } - finish_wait(&cpu_hotplug.wq, &wait); + percpu_down_write(&cpu_hotplug_lock); } void cpu_hotplug_done(void) { - cpu_hotplug.active_writer = NULL; - mutex_unlock(&cpu_hotplug.lock); - cpuhp_lock_release(); + percpu_up_write(&cpu_hotplug_lock); +} + +void lockdep_assert_hotplug_held(void) +{ + percpu_rwsem_assert_held(&cpu_hotplug_lock); } /* @@ -344,8 +265,6 @@ void cpu_hotplug_enable(void) EXPORT_SYMBOL_GPL(cpu_hotplug_enable); #endif /* CONFIG_HOTPLUG_CPU */ -/* Notifier wrappers for transitioning to state machine */ - static int bringup_wait_for_ap(unsigned int cpu) { struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu); @@ -1482,6 +1401,8 @@ int __cpuhp_setup_state_cpuslocked(enum int cpu, ret = 0; bool dynstate; + lockdep_assert_hotplug_held(); + if (cpuhp_cb_check(state) || !name) return -EINVAL; @@ -1600,6 +1521,7 @@ void __cpuhp_remove_state_cpuslocked(enu int cpu; BUG_ON(cpuhp_cb_check(state)); + lockdep_assert_hotplug_held(); mutex_lock(&cpuhp_state_mutex); if (sp->multi_instance) { --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -130,6 +130,7 @@ void __static_key_slow_inc(struct static * the all CPUs, for that to be serialized against CPU hot-plug * we need to avoid CPUs coming online. */ + lockdep_assert_hotplug_held(); jump_label_lock(); if (atomic_read(&key->enabled) == 0) { atomic_set(&key->enabled, -1); @@ -158,6 +159,7 @@ EXPORT_SYMBOL_GPL(static_key_slow_inc_cp static void __static_key_slow_dec(struct static_key *key, unsigned long rate_limit, struct delayed_work *work) { + lockdep_assert_hotplug_held(); /* * The negative count check is valid even when a negative * key->enabled is in use by static_key_slow_inc(); a --- a/kernel/padata.c +++ b/kernel/padata.c @@ -1013,6 +1013,7 @@ static struct padata_instance *padata_al */ struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq) { + lockdep_assert_hotplug_held(); return padata_alloc(wq, cpu_possible_mask, cpu_possible_mask); } EXPORT_SYMBOL(padata_alloc_possible); --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -562,6 +562,8 @@ int stop_machine_cpuslocked(cpu_stop_fn_ .active_cpus = cpus, }; + lockdep_assert_hotplug_held(); + if (!stop_machine_initialized) { /* * Handle the case where stop_machine() is called