[PATCH v8 1/3] smpboot: allow excluding cpus from the smpboot threads

[Chris Metcalf <cmetcalf@ezchip.com>]

This change allows some cores to be excluded from running the
smp_hotplug_thread tasks.  The following commit to update
kernel/watchdog.c to use this functionality is the motivating
example, and more information on the motivation is provided there.

A new smp_hotplug_thread field is introduced, "cpumask", which
is an optional pointer to a cpumask that indicates whether
or not the given smp_hotplug_thread should run on that core; the
cpumask is checked when deciding whether to unpark the thread.

To change the cpumask after registering the thread, you must call
smpboot_update_cpumask_percpu_thread() with the new cpumask.

Signed-off-by: Chris Metcalf <cmetcalf@ezchip.com>
---
(Text repeated from v7 post since these are still open issues)

I don't know why it is necessary to explicitly unpark the threads in
smpboot_destroy_threads() before destroying them.  We can't even do it
in the same for_each_possible_cpu() loop as the kthread_stop() call;
it appears all the threads on online cores must be unparked prior to
trying to stop all the threads, or the system hangs.

In all honesty, I'm still fond of the model where we just do_exit(0)
the threads that we don't want, as soon as registration creates them.
(We can still support the watchdog_cpumask sysctl easily enough.)
This doesn't add any new semantics to smpboot (for good or for bad),
and more importantly it makes it clear that we don't have watchdog
tasks running on the nohz_full cores, which otherwise are going to
make people continually wonder what's going on until they carefully
read the code.  But I'm OK with the direction laid out in this patch
if it's the consensus preferred model.

v8: make cpumask only updated by smpboot subsystem [Frederic]

v7: change from valid_cpu() callback to optional cpumask field
    park smpboot threads rather than just not creating them

v6: change from an "exclude" data pointer to a more generic
    valid_cpu() callback [Frederic]

v5: switch from watchdog_exclude to watchdog_cpumask [Frederic]
    simplify the smp_hotplug_thread API to watchdog [Frederic]

 include/linux/smpboot.h |  6 ++++++
 kernel/smpboot.c        | 57 +++++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 61 insertions(+), 2 deletions(-)

diff --git a/include/linux/smpboot.h b/include/linux/smpboot.h
index d600afb21926..63271b19333e 100644
--- a/include/linux/smpboot.h
+++ b/include/linux/smpboot.h
@@ -27,6 +27,9 @@ struct smpboot_thread_data;
  * @pre_unpark:		Optional unpark function, called before the thread is
  *			unparked (cpu online). This is not guaranteed to be
  *			called on the target cpu of the thread. Careful!
+ * @cpumask:		Optional pointer to a set of possible cores to
+ *			allow threads to come unparked on.  To change it later
+ *			you must call smpboot_update_cpumask_percpu_thread().
  * @selfparking:	Thread is not parked by the park function.
  * @thread_comm:	The base name of the thread
  */
@@ -41,11 +44,14 @@ struct smp_hotplug_thread {
 	void				(*park)(unsigned int cpu);
 	void				(*unpark)(unsigned int cpu);
 	void				(*pre_unpark)(unsigned int cpu);
+	struct cpumask			*cpumask;
 	bool				selfparking;
 	const char			*thread_comm;
 };
 
 int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread);
 void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread);
+void smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread *plug_thread,
+					  const struct cpumask *);
 
 #endif
diff --git a/kernel/smpboot.c b/kernel/smpboot.c
index c697f73d82d6..c5d53a335387 100644
--- a/kernel/smpboot.c
+++ b/kernel/smpboot.c
@@ -92,6 +92,9 @@ enum {
 	HP_THREAD_PARKED,
 };
 
+/* Statically allocated and used under smpboot_threads_lock. */
+static struct cpumask tmp_mask;
+
 /**
  * smpboot_thread_fn - percpu hotplug thread loop function
  * @data:	thread data pointer
@@ -232,7 +235,8 @@ void smpboot_unpark_threads(unsigned int cpu)
 
 	mutex_lock(&smpboot_threads_lock);
 	list_for_each_entry(cur, &hotplug_threads, list)
-		smpboot_unpark_thread(cur, cpu);
+		if (cur->cpumask == NULL || cpumask_test_cpu(cpu, cur->cpumask))
+			smpboot_unpark_thread(cur, cpu);
 	mutex_unlock(&smpboot_threads_lock);
 }
 
@@ -258,6 +262,16 @@ static void smpboot_destroy_threads(struct smp_hotplug_thread *ht)
 {
 	unsigned int cpu;
 
+	/* Unpark any threads that were voluntarily parked. */
+	if (ht->cpumask) {
+		cpumask_andnot(&tmp_mask, cpu_online_mask, ht->cpumask);
+		for_each_cpu(cpu, &tmp_mask) {
+			struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
+			if (tsk)
+				kthread_unpark(tsk);
+		}
+	}
+
 	/* We need to destroy also the parked threads of offline cpus */
 	for_each_possible_cpu(cpu) {
 		struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu);
@@ -289,7 +303,9 @@ int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread)
 			smpboot_destroy_threads(plug_thread);
 			goto out;
 		}
-		smpboot_unpark_thread(plug_thread, cpu);
+		if (plug_thread->cpumask == NULL ||
+		    cpumask_test_cpu(cpu, plug_thread->cpumask))
+			smpboot_unpark_thread(plug_thread, cpu);
 	}
 	list_add(&plug_thread->list, &hotplug_threads);
 out:
@@ -316,6 +332,43 @@ void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread)
 }
 EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread);
 
+/**
+ * smpboot_update_cpumask_percpu_thread - Adjust which per_cpu hotplug threads stay parked
+ * @plug_thread:	Hotplug thread descriptor
+ * @new:		Revised mask to use
+ *
+ * The cpumask field in the smp_hotplug_thread must not be updated directly
+ * by the client, but only by calling this function.  A non-NULL cpumask must
+ * have been provided at registration time to be able to use this function.
+ */
+void smpboot_update_cpumask_percpu_thread(struct smp_hotplug_thread *plug_thread,
+					  const struct cpumask *new)
+{
+	unsigned int cpu;
+	struct cpumask *old = plug_thread->cpumask;
+
+	BUG_ON(old == NULL);
+
+	get_online_cpus();
+	mutex_lock(&smpboot_threads_lock);
+
+	/* Park threads that were exclusively enabled on the old mask. */
+	cpumask_andnot(&tmp_mask, old, new);
+	for_each_cpu_and(cpu, &tmp_mask, cpu_online_mask)
+		smpboot_park_thread(plug_thread, cpu);
+
+	/* Unpark threads that are exclusively enabled on the new mask. */
+	cpumask_andnot(&tmp_mask, new, old);
+	for_each_cpu_and(cpu, &tmp_mask, cpu_online_mask)
+		smpboot_unpark_thread(plug_thread, cpu);
+
+	cpumask_copy(old, new);
+
+	mutex_unlock(&smpboot_threads_lock);
+	put_online_cpus();
+}
+EXPORT_SYMBOL_GPL(smpboot_update_cpumask_percpu_thread);
+
 static DEFINE_PER_CPU(atomic_t, cpu_hotplug_state) = ATOMIC_INIT(CPU_POST_DEAD);
 
 /*
-- 
2.1.2