cpu hotplug: convert more drivers (batch #5)

cpu hotplug: convert more drivers (batch #5)

[Sebastian Andrzej Siewior]

Another small batch of drivers converted to the new hotplug state engine.
The whole series is also available at
  git://git.kernel.org/pub/scm/linux/kernel/git/bigeasy/hotplug-staging.git smp/hotplug

on top of tip/smp/hotplug (which includes last batch on top of v4.9-rc4).

Sebastian

[PATCH 01/20] x86/mce/therm_throt: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: x86@kernel.org
Cc: linux-edac@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/therm_throt.c | 53 ++++++++------------------------
 include/linux/cpuhotplug.h               |  1 +
 2 files changed, 13 insertions(+), 41 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 6b9dc4d18ccc..7f56620735ca 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -271,58 +271,29 @@ static void thermal_throttle_remove_dev(struct device *dev)
 }
 
 /* Get notified when a cpu comes on/off. Be hotplug friendly. */
-static int
-thermal_throttle_cpu_callback(struct notifier_block *nfb,
-			      unsigned long action,
-			      void *hcpu)
+static int thermal_throttle_prepare(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-	struct device *dev;
-	int err = 0;
+	struct device *dev = get_cpu_device(cpu);
 
-	dev = get_cpu_device(cpu);
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		err = thermal_throttle_add_dev(dev, cpu);
-		WARN_ON(err);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		thermal_throttle_remove_dev(dev);
-		break;
-	}
-	return notifier_from_errno(err);
+	return thermal_throttle_add_dev(dev, cpu);
 }
 
-static struct notifier_block thermal_throttle_cpu_notifier =
+static int thermal_throttle_dead(unsigned int cpu)
 {
-	.notifier_call = thermal_throttle_cpu_callback,
-};
+	struct device *dev = get_cpu_device(cpu);
+
+	thermal_throttle_remove_dev(dev);
+	return 0;
+}
 
 static __init int thermal_throttle_init_device(void)
 {
-	unsigned int cpu = 0;
-	int err;
-
 	if (!atomic_read(&therm_throt_en))
 		return 0;
 
-	cpu_notifier_register_begin();
-
-	/* connect live CPUs to sysfs */
-	for_each_online_cpu(cpu) {
-		err = thermal_throttle_add_dev(get_cpu_device(cpu), cpu);
-		WARN_ON(err);
-	}
-
-	__register_hotcpu_notifier(&thermal_throttle_cpu_notifier);
-	cpu_notifier_register_done();
-
-	return 0;
+	return cpuhp_setup_state(CPUHP_X86_THERM_PREPARE, "x86/therm:prepare",
+				 thermal_throttle_prepare,
+				 thermal_throttle_dead);
 }
 device_initcall(thermal_throttle_init_device);
 
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 79b96f647d64..aea6c6a63139 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -59,6 +59,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
+	CPUHP_X86_THERM_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
-- 
2.10.2

[tip:smp/hotplug] x86/mce/therm_throt: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  4705ddca1df1d462bdeeb2fbb38b5259ccf80edf
Gitweb:     http://git.kernel.org/tip/4705ddca1df1d462bdeeb2fbb38b5259ccf80edf
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:22 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:04 +0100

x86/mce/therm_throt: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linuxtronix.de
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161117183541.8588-2-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpu/mcheck/therm_throt.c | 55 ++++++++------------------------
 include/linux/cpuhotplug.h               |  1 +
 2 files changed, 14 insertions(+), 42 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 6b9dc4d..7f56620 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -271,58 +271,29 @@ static void thermal_throttle_remove_dev(struct device *dev)
 }
 
 /* Get notified when a cpu comes on/off. Be hotplug friendly. */
-static int
-thermal_throttle_cpu_callback(struct notifier_block *nfb,
-			      unsigned long action,
-			      void *hcpu)
+static int thermal_throttle_prepare(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-	struct device *dev;
-	int err = 0;
-
-	dev = get_cpu_device(cpu);
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		err = thermal_throttle_add_dev(dev, cpu);
-		WARN_ON(err);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		thermal_throttle_remove_dev(dev);
-		break;
-	}
-	return notifier_from_errno(err);
+	struct device *dev = get_cpu_device(cpu);
+
+	return thermal_throttle_add_dev(dev, cpu);
 }
 
-static struct notifier_block thermal_throttle_cpu_notifier =
+static int thermal_throttle_dead(unsigned int cpu)
 {
-	.notifier_call = thermal_throttle_cpu_callback,
-};
+	struct device *dev = get_cpu_device(cpu);
+
+	thermal_throttle_remove_dev(dev);
+	return 0;
+}
 
 static __init int thermal_throttle_init_device(void)
 {
-	unsigned int cpu = 0;
-	int err;
-
 	if (!atomic_read(&therm_throt_en))
 		return 0;
 
-	cpu_notifier_register_begin();
-
-	/* connect live CPUs to sysfs */
-	for_each_online_cpu(cpu) {
-		err = thermal_throttle_add_dev(get_cpu_device(cpu), cpu);
-		WARN_ON(err);
-	}
-
-	__register_hotcpu_notifier(&thermal_throttle_cpu_notifier);
-	cpu_notifier_register_done();
-
-	return 0;
+	return cpuhp_setup_state(CPUHP_X86_THERM_PREPARE, "x86/therm:prepare",
+				 thermal_throttle_prepare,
+				 thermal_throttle_dead);
 }
 device_initcall(thermal_throttle_init_device);
 
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 79b96f6..aea6c6a 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -59,6 +59,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
+	CPUHP_X86_THERM_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,

[tip:smp/hotplug] x86/mce/therm_throt: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  d6526e73dbbbc4c382c1b16942413eab77ed5e1a
Gitweb:     http://git.kernel.org/tip/d6526e73dbbbc4c382c1b16942413eab77ed5e1a
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:22 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:38 +0100

x86/mce/therm_throt: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linuxtronix.de
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161117183541.8588-2-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpu/mcheck/therm_throt.c | 55 ++++++++------------------------
 include/linux/cpuhotplug.h               |  1 +
 2 files changed, 14 insertions(+), 42 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/therm_throt.c b/arch/x86/kernel/cpu/mcheck/therm_throt.c
index 6b9dc4d..7f56620 100644
--- a/arch/x86/kernel/cpu/mcheck/therm_throt.c
+++ b/arch/x86/kernel/cpu/mcheck/therm_throt.c
@@ -271,58 +271,29 @@ static void thermal_throttle_remove_dev(struct device *dev)
 }
 
 /* Get notified when a cpu comes on/off. Be hotplug friendly. */
-static int
-thermal_throttle_cpu_callback(struct notifier_block *nfb,
-			      unsigned long action,
-			      void *hcpu)
+static int thermal_throttle_prepare(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-	struct device *dev;
-	int err = 0;
-
-	dev = get_cpu_device(cpu);
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		err = thermal_throttle_add_dev(dev, cpu);
-		WARN_ON(err);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		thermal_throttle_remove_dev(dev);
-		break;
-	}
-	return notifier_from_errno(err);
+	struct device *dev = get_cpu_device(cpu);
+
+	return thermal_throttle_add_dev(dev, cpu);
 }
 
-static struct notifier_block thermal_throttle_cpu_notifier =
+static int thermal_throttle_dead(unsigned int cpu)
 {
-	.notifier_call = thermal_throttle_cpu_callback,
-};
+	struct device *dev = get_cpu_device(cpu);
+
+	thermal_throttle_remove_dev(dev);
+	return 0;
+}
 
 static __init int thermal_throttle_init_device(void)
 {
-	unsigned int cpu = 0;
-	int err;
-
 	if (!atomic_read(&therm_throt_en))
 		return 0;
 
-	cpu_notifier_register_begin();
-
-	/* connect live CPUs to sysfs */
-	for_each_online_cpu(cpu) {
-		err = thermal_throttle_add_dev(get_cpu_device(cpu), cpu);
-		WARN_ON(err);
-	}
-
-	__register_hotcpu_notifier(&thermal_throttle_cpu_notifier);
-	cpu_notifier_register_done();
-
-	return 0;
+	return cpuhp_setup_state(CPUHP_X86_THERM_PREPARE, "x86/therm:prepare",
+				 thermal_throttle_prepare,
+				 thermal_throttle_dead);
 }
 device_initcall(thermal_throttle_init_device);
 
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 79b96f6..aea6c6a 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -59,6 +59,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
+	CPUHP_X86_THERM_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,

[PATCH 02/20] x86/cpuid: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpuid.c    | 54 ++++++++--------------------------------------
 include/linux/cpuhotplug.h |  1 +
 2 files changed, 10 insertions(+), 45 deletions(-)

diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index 2836de390f95..eaa272454fe4 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -115,7 +115,7 @@ static const struct file_operations cpuid_fops = {
 	.open = cpuid_open,
 };
 
-static int cpuid_device_create(int cpu)
+static int cpuid_device_create(unsigned int cpu)
 {
 	struct device *dev;
 
@@ -124,35 +124,12 @@ static int cpuid_device_create(int cpu)
 	return PTR_ERR_OR_ZERO(dev);
 }
 
-static void cpuid_device_destroy(int cpu)
+static int cpuid_device_destroy(unsigned int cpu)
 {
 	device_destroy(cpuid_class, MKDEV(CPUID_MAJOR, cpu));
+	return 0;
 }
 
-static int cpuid_class_cpu_callback(struct notifier_block *nfb,
-				    unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	int err = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		err = cpuid_device_create(cpu);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-		cpuid_device_destroy(cpu);
-		break;
-	}
-	return notifier_from_errno(err);
-}
-
-static struct notifier_block cpuid_class_cpu_notifier =
-{
-	.notifier_call = cpuid_class_cpu_callback,
-};
-
 static char *cpuid_devnode(struct device *dev, umode_t *mode)
 {
 	return kasprintf(GFP_KERNEL, "cpu/%u/cpuid", MINOR(dev->devt));
@@ -177,24 +154,17 @@ static int __init cpuid_init(void)
 	}
 	cpuid_class->devnode = cpuid_devnode;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = cpuid_device_create(i);
-		if (err != 0)
-			goto out_class;
-	}
-	__register_hotcpu_notifier(&cpuid_class_cpu_notifier);
-	cpu_notifier_register_done();
+	err = cpuhp_setup_state(CPUHP_X86_CPUID_PREPARE, "x86/cpuid:prepare",
+				cpuid_device_create, cpuid_device_destroy);
+	if (err)
+		goto out_class;
 
 	err = 0;
 	goto out;
 
 out_class:
 	i = 0;
-	for_each_online_cpu(i) {
-		cpuid_device_destroy(i);
-	}
-	cpu_notifier_register_done();
+	cpuhp_remove_state(CPUHP_X86_CPUID_PREPARE);
 	class_destroy(cpuid_class);
 out_chrdev:
 	__unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid");
@@ -204,15 +174,9 @@ static int __init cpuid_init(void)
 
 static void __exit cpuid_exit(void)
 {
-	int cpu = 0;
-
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		cpuid_device_destroy(cpu);
+	cpuhp_remove_state(CPUHP_X86_CPUID_PREPARE);
 	class_destroy(cpuid_class);
 	__unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid");
-	__unregister_hotcpu_notifier(&cpuid_class_cpu_notifier);
-	cpu_notifier_register_done();
 }
 
 module_init(cpuid_init);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index aea6c6a63139..2f9e083791c4 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -60,6 +60,7 @@ enum cpuhp_state {
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
 	CPUHP_X86_THERM_PREPARE,
+	CPUHP_X86_CPUID_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
-- 
2.10.2

[tip:smp/hotplug] x86/cpuid: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  20ce2eb8e46cee14a3f608b201fbb117055a3d6f
Gitweb:     http://git.kernel.org/tip/20ce2eb8e46cee14a3f608b201fbb117055a3d6f
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:23 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:05 +0100

x86/cpuid: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-3-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpuid.c    | 64 ++++++++--------------------------------------
 include/linux/cpuhotplug.h |  1 +
 2 files changed, 12 insertions(+), 53 deletions(-)

diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index 2836de3..fd85e93 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -115,7 +115,7 @@ static const struct file_operations cpuid_fops = {
 	.open = cpuid_open,
 };
 
-static int cpuid_device_create(int cpu)
+static int cpuid_device_create(unsigned int cpu)
 {
 	struct device *dev;
 
@@ -124,35 +124,12 @@ static int cpuid_device_create(int cpu)
 	return PTR_ERR_OR_ZERO(dev);
 }
 
-static void cpuid_device_destroy(int cpu)
+static int cpuid_device_destroy(unsigned int cpu)
 {
 	device_destroy(cpuid_class, MKDEV(CPUID_MAJOR, cpu));
+	return 0;
 }
 
-static int cpuid_class_cpu_callback(struct notifier_block *nfb,
-				    unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	int err = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		err = cpuid_device_create(cpu);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-		cpuid_device_destroy(cpu);
-		break;
-	}
-	return notifier_from_errno(err);
-}
-
-static struct notifier_block cpuid_class_cpu_notifier =
-{
-	.notifier_call = cpuid_class_cpu_callback,
-};
-
 static char *cpuid_devnode(struct device *dev, umode_t *mode)
 {
 	return kasprintf(GFP_KERNEL, "cpu/%u/cpuid", MINOR(dev->devt));
@@ -160,15 +137,13 @@ static char *cpuid_devnode(struct device *dev, umode_t *mode)
 
 static int __init cpuid_init(void)
 {
-	int i, err = 0;
-	i = 0;
+	int err;
 
 	if (__register_chrdev(CPUID_MAJOR, 0, NR_CPUS,
 			      "cpu/cpuid", &cpuid_fops)) {
 		printk(KERN_ERR "cpuid: unable to get major %d for cpuid\n",
 		       CPUID_MAJOR);
-		err = -EBUSY;
-		goto out;
+		return -EBUSY;
 	}
 	cpuid_class = class_create(THIS_MODULE, "cpuid");
 	if (IS_ERR(cpuid_class)) {
@@ -177,42 +152,25 @@ static int __init cpuid_init(void)
 	}
 	cpuid_class->devnode = cpuid_devnode;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = cpuid_device_create(i);
-		if (err != 0)
-			goto out_class;
-	}
-	__register_hotcpu_notifier(&cpuid_class_cpu_notifier);
-	cpu_notifier_register_done();
+	err = cpuhp_setup_state(CPUHP_X86_CPUID_PREPARE, "x86/cpuid:prepare",
+				cpuid_device_create, cpuid_device_destroy);
+	if (err)
+		goto out_class;
 
-	err = 0;
-	goto out;
+	return 0;
 
 out_class:
-	i = 0;
-	for_each_online_cpu(i) {
-		cpuid_device_destroy(i);
-	}
-	cpu_notifier_register_done();
 	class_destroy(cpuid_class);
 out_chrdev:
 	__unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid");
-out:
 	return err;
 }
 
 static void __exit cpuid_exit(void)
 {
-	int cpu = 0;
-
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		cpuid_device_destroy(cpu);
+	cpuhp_remove_state(CPUHP_X86_CPUID_PREPARE);
 	class_destroy(cpuid_class);
 	__unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid");
-	__unregister_hotcpu_notifier(&cpuid_class_cpu_notifier);
-	cpu_notifier_register_done();
 }
 
 module_init(cpuid_init);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 79b96f6..bc340ef 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -59,6 +59,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
+	CPUHP_X86_CPUID_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,

[tip:smp/hotplug] x86/cpuid: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  8c07b494ab2859bc7efb27c40d6faff255f2d2ae
Gitweb:     http://git.kernel.org/tip/8c07b494ab2859bc7efb27c40d6faff255f2d2ae
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:23 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:39 +0100

x86/cpuid: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-3-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpuid.c    | 64 ++++++++--------------------------------------
 include/linux/cpuhotplug.h |  1 +
 2 files changed, 12 insertions(+), 53 deletions(-)

diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index 2836de3..fd85e93 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -115,7 +115,7 @@ static const struct file_operations cpuid_fops = {
 	.open = cpuid_open,
 };
 
-static int cpuid_device_create(int cpu)
+static int cpuid_device_create(unsigned int cpu)
 {
 	struct device *dev;
 
@@ -124,35 +124,12 @@ static int cpuid_device_create(int cpu)
 	return PTR_ERR_OR_ZERO(dev);
 }
 
-static void cpuid_device_destroy(int cpu)
+static int cpuid_device_destroy(unsigned int cpu)
 {
 	device_destroy(cpuid_class, MKDEV(CPUID_MAJOR, cpu));
+	return 0;
 }
 
-static int cpuid_class_cpu_callback(struct notifier_block *nfb,
-				    unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	int err = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		err = cpuid_device_create(cpu);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-		cpuid_device_destroy(cpu);
-		break;
-	}
-	return notifier_from_errno(err);
-}
-
-static struct notifier_block cpuid_class_cpu_notifier =
-{
-	.notifier_call = cpuid_class_cpu_callback,
-};
-
 static char *cpuid_devnode(struct device *dev, umode_t *mode)
 {
 	return kasprintf(GFP_KERNEL, "cpu/%u/cpuid", MINOR(dev->devt));
@@ -160,15 +137,13 @@ static char *cpuid_devnode(struct device *dev, umode_t *mode)
 
 static int __init cpuid_init(void)
 {
-	int i, err = 0;
-	i = 0;
+	int err;
 
 	if (__register_chrdev(CPUID_MAJOR, 0, NR_CPUS,
 			      "cpu/cpuid", &cpuid_fops)) {
 		printk(KERN_ERR "cpuid: unable to get major %d for cpuid\n",
 		       CPUID_MAJOR);
-		err = -EBUSY;
-		goto out;
+		return -EBUSY;
 	}
 	cpuid_class = class_create(THIS_MODULE, "cpuid");
 	if (IS_ERR(cpuid_class)) {
@@ -177,42 +152,25 @@ static int __init cpuid_init(void)
 	}
 	cpuid_class->devnode = cpuid_devnode;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = cpuid_device_create(i);
-		if (err != 0)
-			goto out_class;
-	}
-	__register_hotcpu_notifier(&cpuid_class_cpu_notifier);
-	cpu_notifier_register_done();
+	err = cpuhp_setup_state(CPUHP_X86_CPUID_PREPARE, "x86/cpuid:prepare",
+				cpuid_device_create, cpuid_device_destroy);
+	if (err)
+		goto out_class;
 
-	err = 0;
-	goto out;
+	return 0;
 
 out_class:
-	i = 0;
-	for_each_online_cpu(i) {
-		cpuid_device_destroy(i);
-	}
-	cpu_notifier_register_done();
 	class_destroy(cpuid_class);
 out_chrdev:
 	__unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid");
-out:
 	return err;
 }
 
 static void __exit cpuid_exit(void)
 {
-	int cpu = 0;
-
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		cpuid_device_destroy(cpu);
+	cpuhp_remove_state(CPUHP_X86_CPUID_PREPARE);
 	class_destroy(cpuid_class);
 	__unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid");
-	__unregister_hotcpu_notifier(&cpuid_class_cpu_notifier);
-	cpu_notifier_register_done();
 }
 
 module_init(cpuid_init);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 79b96f6..bc340ef 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -59,6 +59,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
+	CPUHP_X86_CPUID_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,

[PATCH 03/20] x86/msr: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/msr.c      | 56 +++++++++-------------------------------------
 include/linux/cpuhotplug.h |  1 +
 2 files changed, 11 insertions(+), 46 deletions(-)

diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 7f3550acde1b..43a125b5c5dc 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -180,7 +180,7 @@ static const struct file_operations msr_fops = {
 	.compat_ioctl = msr_ioctl,
 };
 
-static int msr_device_create(int cpu)
+static int msr_device_create(unsigned int cpu)
 {
 	struct device *dev;
 
@@ -189,34 +189,12 @@ static int msr_device_create(int cpu)
 	return PTR_ERR_OR_ZERO(dev);
 }
 
-static void msr_device_destroy(int cpu)
+static int msr_device_destroy(unsigned int cpu)
 {
 	device_destroy(msr_class, MKDEV(MSR_MAJOR, cpu));
+	return 0;
 }
 
-static int msr_class_cpu_callback(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	int err = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		err = msr_device_create(cpu);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-		msr_device_destroy(cpu);
-		break;
-	}
-	return notifier_from_errno(err);
-}
-
-static struct notifier_block __refdata msr_class_cpu_notifier = {
-	.notifier_call = msr_class_cpu_callback,
-};
-
 static char *msr_devnode(struct device *dev, umode_t *mode)
 {
 	return kasprintf(GFP_KERNEL, "cpu/%u/msr", MINOR(dev->devt));
@@ -224,8 +202,7 @@ static char *msr_devnode(struct device *dev, umode_t *mode)
 
 static int __init msr_init(void)
 {
-	int i, err = 0;
-	i = 0;
+	int err;
 
 	if (__register_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr", &msr_fops)) {
 		pr_err("unable to get major %d for msr\n", MSR_MAJOR);
@@ -239,23 +216,16 @@ static int __init msr_init(void)
 	}
 	msr_class->devnode = msr_devnode;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = msr_device_create(i);
-		if (err != 0)
-			goto out_class;
-	}
-	__register_hotcpu_notifier(&msr_class_cpu_notifier);
-	cpu_notifier_register_done();
+	err = cpuhp_setup_state(CPUHP_X86_MSR_PREPARE, "x86/msr:prepare",
+				msr_device_create, msr_device_destroy);
+	if (err)
+		goto out_class;
 
 	err = 0;
 	goto out;
 
 out_class:
-	i = 0;
-	for_each_online_cpu(i)
-		msr_device_destroy(i);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(CPUHP_X86_MSR_PREPARE);
 	class_destroy(msr_class);
 out_chrdev:
 	__unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr");
@@ -265,15 +235,9 @@ static int __init msr_init(void)
 
 static void __exit msr_exit(void)
 {
-	int cpu = 0;
-
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		msr_device_destroy(cpu);
 	class_destroy(msr_class);
 	__unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr");
-	__unregister_hotcpu_notifier(&msr_class_cpu_notifier);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(CPUHP_X86_MSR_PREPARE);
 }
 
 module_init(msr_init);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 2f9e083791c4..6506dce8343a 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -61,6 +61,7 @@ enum cpuhp_state {
 	CPUHP_TOPOLOGY_PREPARE,
 	CPUHP_X86_THERM_PREPARE,
 	CPUHP_X86_CPUID_PREPARE,
+	CPUHP_X86_MSR_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
-- 
2.10.2

[tip:smp/hotplug] x86/msr: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  216a5d1b866538a021af00bfb7c5dcd1da8ff884
Gitweb:     http://git.kernel.org/tip/216a5d1b866538a021af00bfb7c5dcd1da8ff884
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:24 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:05 +0100

x86/msr: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Move the callbacks to online/offline as there is no point in having the
files around before the cpu is online and until its completely gone.

[ tglx: Move the callbacks to online/offline ]

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-4-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/msr.c | 66 +++++++++++----------------------------------------
 1 file changed, 14 insertions(+), 52 deletions(-)

diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 7f3550a..7935815 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -44,6 +44,7 @@
 #include <asm/msr.h>
 
 static struct class *msr_class;
+static enum cpuhp_state cpuhp_msr_state;
 
 static ssize_t msr_read(struct file *file, char __user *buf,
 			size_t count, loff_t *ppos)
@@ -180,7 +181,7 @@ static const struct file_operations msr_fops = {
 	.compat_ioctl = msr_ioctl,
 };
 
-static int msr_device_create(int cpu)
+static int msr_device_create(unsigned int cpu)
 {
 	struct device *dev;
 
@@ -189,34 +190,12 @@ static int msr_device_create(int cpu)
 	return PTR_ERR_OR_ZERO(dev);
 }
 
-static void msr_device_destroy(int cpu)
+static int msr_device_destroy(unsigned int cpu)
 {
 	device_destroy(msr_class, MKDEV(MSR_MAJOR, cpu));
+	return 0;
 }
 
-static int msr_class_cpu_callback(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	int err = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		err = msr_device_create(cpu);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-		msr_device_destroy(cpu);
-		break;
-	}
-	return notifier_from_errno(err);
-}
-
-static struct notifier_block __refdata msr_class_cpu_notifier = {
-	.notifier_call = msr_class_cpu_callback,
-};
-
 static char *msr_devnode(struct device *dev, umode_t *mode)
 {
 	return kasprintf(GFP_KERNEL, "cpu/%u/msr", MINOR(dev->devt));
@@ -224,8 +203,7 @@ static char *msr_devnode(struct device *dev, umode_t *mode)
 
 static int __init msr_init(void)
 {
-	int i, err = 0;
-	i = 0;
+	int err;
 
 	if (__register_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr", &msr_fops)) {
 		pr_err("unable to get major %d for msr\n", MSR_MAJOR);
@@ -239,44 +217,28 @@ static int __init msr_init(void)
 	}
 	msr_class->devnode = msr_devnode;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = msr_device_create(i);
-		if (err != 0)
-			goto out_class;
-	}
-	__register_hotcpu_notifier(&msr_class_cpu_notifier);
-	cpu_notifier_register_done();
-
-	err = 0;
-	goto out;
+	err  = cpuhp_setup_state(CPUHP_X86_MSR_PREPARE, "x86/msr:online",
+				 msr_device_create, msr_device_destroy);
+	if (err < 0)
+		goto out_class;
+	cpuhp_msr_state = err;
+	return 0;
 
 out_class:
-	i = 0;
-	for_each_online_cpu(i)
-		msr_device_destroy(i);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(cpuhp_msr_state);
 	class_destroy(msr_class);
 out_chrdev:
 	__unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr");
-out:
 	return err;
 }
+module_init(msr_init);
 
 static void __exit msr_exit(void)
 {
-	int cpu = 0;
-
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		msr_device_destroy(cpu);
+	cpuhp_remove_state(cpuhp_msr_state);
 	class_destroy(msr_class);
 	__unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr");
-	__unregister_hotcpu_notifier(&msr_class_cpu_notifier);
-	cpu_notifier_register_done();
 }
-
-module_init(msr_init);
 module_exit(msr_exit)
 
 MODULE_AUTHOR("H. Peter Anvin <hpa@zytor.com>");

[tip:smp/hotplug] x86/msr: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  8fba38c937cd585bcf562fda8db3decb25509506
Gitweb:     http://git.kernel.org/tip/8fba38c937cd585bcf562fda8db3decb25509506
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:24 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:39 +0100

x86/msr: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Move the callbacks to online/offline as there is no point in having the
files around before the cpu is online and until its completely gone.

[ tglx: Move the callbacks to online/offline ]

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-4-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/msr.c | 69 +++++++++++----------------------------------------
 1 file changed, 15 insertions(+), 54 deletions(-)

diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index 7f3550a..f5e3ff8 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -44,6 +44,7 @@
 #include <asm/msr.h>
 
 static struct class *msr_class;
+static enum cpuhp_state cpuhp_msr_state;
 
 static ssize_t msr_read(struct file *file, char __user *buf,
 			size_t count, loff_t *ppos)
@@ -180,7 +181,7 @@ static const struct file_operations msr_fops = {
 	.compat_ioctl = msr_ioctl,
 };
 
-static int msr_device_create(int cpu)
+static int msr_device_create(unsigned int cpu)
 {
 	struct device *dev;
 
@@ -189,34 +190,12 @@ static int msr_device_create(int cpu)
 	return PTR_ERR_OR_ZERO(dev);
 }
 
-static void msr_device_destroy(int cpu)
+static int msr_device_destroy(unsigned int cpu)
 {
 	device_destroy(msr_class, MKDEV(MSR_MAJOR, cpu));
+	return 0;
 }
 
-static int msr_class_cpu_callback(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	int err = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		err = msr_device_create(cpu);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-		msr_device_destroy(cpu);
-		break;
-	}
-	return notifier_from_errno(err);
-}
-
-static struct notifier_block __refdata msr_class_cpu_notifier = {
-	.notifier_call = msr_class_cpu_callback,
-};
-
 static char *msr_devnode(struct device *dev, umode_t *mode)
 {
 	return kasprintf(GFP_KERNEL, "cpu/%u/msr", MINOR(dev->devt));
@@ -224,13 +203,11 @@ static char *msr_devnode(struct device *dev, umode_t *mode)
 
 static int __init msr_init(void)
 {
-	int i, err = 0;
-	i = 0;
+	int err;
 
 	if (__register_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr", &msr_fops)) {
 		pr_err("unable to get major %d for msr\n", MSR_MAJOR);
-		err = -EBUSY;
-		goto out;
+		return -EBUSY;
 	}
 	msr_class = class_create(THIS_MODULE, "msr");
 	if (IS_ERR(msr_class)) {
@@ -239,44 +216,28 @@ static int __init msr_init(void)
 	}
 	msr_class->devnode = msr_devnode;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = msr_device_create(i);
-		if (err != 0)
-			goto out_class;
-	}
-	__register_hotcpu_notifier(&msr_class_cpu_notifier);
-	cpu_notifier_register_done();
-
-	err = 0;
-	goto out;
+	err  = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/msr:online",
+				 msr_device_create, msr_device_destroy);
+	if (err < 0)
+		goto out_class;
+	cpuhp_msr_state = err;
+	return 0;
 
 out_class:
-	i = 0;
-	for_each_online_cpu(i)
-		msr_device_destroy(i);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(cpuhp_msr_state);
 	class_destroy(msr_class);
 out_chrdev:
 	__unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr");
-out:
 	return err;
 }
+module_init(msr_init);
 
 static void __exit msr_exit(void)
 {
-	int cpu = 0;
-
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		msr_device_destroy(cpu);
+	cpuhp_remove_state(cpuhp_msr_state);
 	class_destroy(msr_class);
 	__unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr");
-	__unregister_hotcpu_notifier(&msr_class_cpu_notifier);
-	cpu_notifier_register_done();
 }
-
-module_init(msr_init);
 module_exit(msr_exit)
 
 MODULE_AUTHOR("H. Peter Anvin <hpa@zytor.com>");

[PATCH 04/20] hwmon/coretemp: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine. Setup and teardown are handled
by the hotplug core.

Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Jean Delvare <jdelvare@suse.com>
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: linux-hwmon@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 drivers/hwmon/coretemp.c | 74 +++++++++++++++---------------------------------
 1 file changed, 23 insertions(+), 51 deletions(-)

diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
index 6a27eb2fed17..390038884f9e 100644
--- a/drivers/hwmon/coretemp.c
+++ b/drivers/hwmon/coretemp.c
@@ -676,7 +676,7 @@ static bool is_any_core_online(struct platform_data *pdata)
 	return false;
 }
 
-static void get_core_online(unsigned int cpu)
+static int coretemp_cpu_online(unsigned int cpu)
 {
 	struct cpuinfo_x86 *c = &cpu_data(cpu);
 	struct platform_device *pdev = coretemp_get_pdev(cpu);
@@ -688,12 +688,12 @@ static void get_core_online(unsigned int cpu)
 	 * without thermal sensors will be filtered out.
 	 */
 	if (!cpu_has(c, X86_FEATURE_DTHERM))
-		return;
+		return 0;
 
 	if (!pdev) {
 		/* Check the microcode version of the CPU */
 		if (chk_ucode_version(cpu))
-			return;
+			return 0;
 
 		/*
 		 * Alright, we have DTS support.
@@ -703,7 +703,7 @@ static void get_core_online(unsigned int cpu)
 		 */
 		err = coretemp_device_add(cpu);
 		if (err)
-			return;
+			return 0;
 		/*
 		 * Check whether pkgtemp support is available.
 		 * If so, add interfaces for pkgtemp.
@@ -716,9 +716,10 @@ static void get_core_online(unsigned int cpu)
 	 * So, just add interfaces for this core.
 	 */
 	coretemp_add_core(cpu, 0);
+	return 0;
 }
 
-static void put_core_offline(unsigned int cpu)
+static int coretemp_cpu_offline(unsigned int cpu)
 {
 	int i, indx;
 	struct platform_data *pdata;
@@ -726,7 +727,7 @@ static void put_core_offline(unsigned int cpu)
 
 	/* If the physical CPU device does not exist, just return */
 	if (!pdev)
-		return;
+		return 0;
 
 	pdata = platform_get_drvdata(pdev);
 
@@ -734,7 +735,7 @@ static void put_core_offline(unsigned int cpu)
 
 	/* The core id is too big, just return */
 	if (indx > MAX_CORE_DATA - 1)
-		return;
+		return 0;
 
 	if (pdata->core_data[indx] && pdata->core_data[indx]->cpu == cpu)
 		coretemp_remove_core(pdata, indx);
@@ -747,7 +748,7 @@ static void put_core_offline(unsigned int cpu)
 	 */
 	for_each_sibling(i, cpu) {
 		if (i != cpu) {
-			get_core_online(i);
+			coretemp_cpu_online(i);
 			/*
 			 * Display temperature sensor data for one HT sibling
 			 * per core only, so abort the loop after one such
@@ -764,38 +765,20 @@ static void put_core_offline(unsigned int cpu)
 	 */
 	if (!is_any_core_online(pdata))
 		coretemp_device_remove(cpu);
+	return 0;
 }
 
-static int coretemp_cpu_callback(struct notifier_block *nfb,
-				 unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long) hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		get_core_online(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		put_core_offline(cpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block coretemp_cpu_notifier __refdata = {
-	.notifier_call = coretemp_cpu_callback,
-};
-
 static const struct x86_cpu_id __initconst coretemp_ids[] = {
 	{ X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_DTHERM },
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, coretemp_ids);
 
+static enum cpuhp_state coretemp_hp_online;
+
 static int __init coretemp_init(void)
 {
-	int i, err;
+	int err;
 
 	/*
 	 * CPUID.06H.EAX[0] indicates whether the CPU has thermal
@@ -809,44 +792,33 @@ static int __init coretemp_init(void)
 	if (err)
 		goto exit;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i)
-		get_core_online(i);
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/coretemp:online",
+				coretemp_cpu_online, coretemp_cpu_offline);
+	if (err < 0)
+		goto exit_driver_unreg;
+	coretemp_hp_online = err;
 
 #ifndef CONFIG_HOTPLUG_CPU
 	if (list_empty(&pdev_list)) {
-		cpu_notifier_register_done();
 		err = -ENODEV;
-		goto exit_driver_unreg;
+		goto exit_hp_unreg;
 	}
 #endif
-
-	__register_hotcpu_notifier(&coretemp_cpu_notifier);
-	cpu_notifier_register_done();
 	return 0;
 
 #ifndef CONFIG_HOTPLUG_CPU
+exit_hp_unreg:
+	cpuhp_remove_state(coretemp_hp_online);
+#endif
 exit_driver_unreg:
 	platform_driver_unregister(&coretemp_driver);
-#endif
 exit:
 	return err;
 }
 
 static void __exit coretemp_exit(void)
 {
-	struct pdev_entry *p, *n;
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&coretemp_cpu_notifier);
-	mutex_lock(&pdev_list_mutex);
-	list_for_each_entry_safe(p, n, &pdev_list, list) {
-		platform_device_unregister(p->pdev);
-		list_del(&p->list);
-		kfree(p);
-	}
-	mutex_unlock(&pdev_list_mutex);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(coretemp_hp_online);
 	platform_driver_unregister(&coretemp_driver);
 }
 
-- 
2.10.2

Re: [04/20] hwmon/coretemp: Convert to hotplug state machine

[Guenter Roeck]

On Thu, Nov 17, 2016 at 07:35:25PM +0100, Sebastian Andrzej Siewior wrote:
> Install the callbacks via the state machine. Setup and teardown are handled
> by the hotplug core.
> 
> Cc: Fenghua Yu <fenghua.yu@intel.com>
> Cc: Jean Delvare <jdelvare@suse.com>
> Cc: Guenter Roeck <linux@roeck-us.net>
> Cc: linux-hwmon@vger.kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

Install, modprobe, modprobe -r, modprobe,

[24078.179118] ------------[ cut here ]------------
[24078.179124] WARNING: CPU: 0 PID: 14 at fs/sysfs/dir.c:31 sysfs_warn_dup+0x6e/0x80
[24078.179125] sysfs: cannot create duplicate filename
'/devices/platform/coretemp.0'

and many more of those.

I can not test the other hwmon drivers in this series, but I would assume
that they have the same problem.

Guenter

> ---
>  drivers/hwmon/coretemp.c | 74 +++++++++++++++---------------------------------
>  1 file changed, 23 insertions(+), 51 deletions(-)
> 
> diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
> index 6a27eb2fed17..390038884f9e 100644
> --- a/drivers/hwmon/coretemp.c
> +++ b/drivers/hwmon/coretemp.c
> @@ -676,7 +676,7 @@ static bool is_any_core_online(struct platform_data *pdata)
>  	return false;
>  }
>  
> -static void get_core_online(unsigned int cpu)
> +static int coretemp_cpu_online(unsigned int cpu)
>  {
>  	struct cpuinfo_x86 *c = &cpu_data(cpu);
>  	struct platform_device *pdev = coretemp_get_pdev(cpu);
> @@ -688,12 +688,12 @@ static void get_core_online(unsigned int cpu)
>  	 * without thermal sensors will be filtered out.
>  	 */
>  	if (!cpu_has(c, X86_FEATURE_DTHERM))
> -		return;
> +		return 0;
>  
>  	if (!pdev) {
>  		/* Check the microcode version of the CPU */
>  		if (chk_ucode_version(cpu))
> -			return;
> +			return 0;
>  
>  		/*
>  		 * Alright, we have DTS support.
> @@ -703,7 +703,7 @@ static void get_core_online(unsigned int cpu)
>  		 */
>  		err = coretemp_device_add(cpu);
>  		if (err)
> -			return;
> +			return 0;
>  		/*
>  		 * Check whether pkgtemp support is available.
>  		 * If so, add interfaces for pkgtemp.
> @@ -716,9 +716,10 @@ static void get_core_online(unsigned int cpu)
>  	 * So, just add interfaces for this core.
>  	 */
>  	coretemp_add_core(cpu, 0);
> +	return 0;
>  }
>  
> -static void put_core_offline(unsigned int cpu)
> +static int coretemp_cpu_offline(unsigned int cpu)
>  {
>  	int i, indx;
>  	struct platform_data *pdata;
> @@ -726,7 +727,7 @@ static void put_core_offline(unsigned int cpu)
>  
>  	/* If the physical CPU device does not exist, just return */
>  	if (!pdev)
> -		return;
> +		return 0;
>  
>  	pdata = platform_get_drvdata(pdev);
>  
> @@ -734,7 +735,7 @@ static void put_core_offline(unsigned int cpu)
>  
>  	/* The core id is too big, just return */
>  	if (indx > MAX_CORE_DATA - 1)
> -		return;
> +		return 0;
>  
>  	if (pdata->core_data[indx] && pdata->core_data[indx]->cpu == cpu)
>  		coretemp_remove_core(pdata, indx);
> @@ -747,7 +748,7 @@ static void put_core_offline(unsigned int cpu)
>  	 */
>  	for_each_sibling(i, cpu) {
>  		if (i != cpu) {
> -			get_core_online(i);
> +			coretemp_cpu_online(i);
>  			/*
>  			 * Display temperature sensor data for one HT sibling
>  			 * per core only, so abort the loop after one such
> @@ -764,38 +765,20 @@ static void put_core_offline(unsigned int cpu)
>  	 */
>  	if (!is_any_core_online(pdata))
>  		coretemp_device_remove(cpu);
> +	return 0;
>  }
>  
> -static int coretemp_cpu_callback(struct notifier_block *nfb,
> -				 unsigned long action, void *hcpu)
> -{
> -	unsigned int cpu = (unsigned long) hcpu;
> -
> -	switch (action) {
> -	case CPU_ONLINE:
> -	case CPU_DOWN_FAILED:
> -		get_core_online(cpu);
> -		break;
> -	case CPU_DOWN_PREPARE:
> -		put_core_offline(cpu);
> -		break;
> -	}
> -	return NOTIFY_OK;
> -}
> -
> -static struct notifier_block coretemp_cpu_notifier __refdata = {
> -	.notifier_call = coretemp_cpu_callback,
> -};
> -
>  static const struct x86_cpu_id __initconst coretemp_ids[] = {
>  	{ X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_DTHERM },
>  	{}
>  };
>  MODULE_DEVICE_TABLE(x86cpu, coretemp_ids);
>  
> +static enum cpuhp_state coretemp_hp_online;
> +
>  static int __init coretemp_init(void)
>  {
> -	int i, err;
> +	int err;
>  
>  	/*
>  	 * CPUID.06H.EAX[0] indicates whether the CPU has thermal
> @@ -809,44 +792,33 @@ static int __init coretemp_init(void)
>  	if (err)
>  		goto exit;
>  
> -	cpu_notifier_register_begin();
> -	for_each_online_cpu(i)
> -		get_core_online(i);
> +	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/coretemp:online",
> +				coretemp_cpu_online, coretemp_cpu_offline);
> +	if (err < 0)
> +		goto exit_driver_unreg;
> +	coretemp_hp_online = err;
>  
>  #ifndef CONFIG_HOTPLUG_CPU
>  	if (list_empty(&pdev_list)) {
> -		cpu_notifier_register_done();
>  		err = -ENODEV;
> -		goto exit_driver_unreg;
> +		goto exit_hp_unreg;
>  	}
>  #endif
> -
> -	__register_hotcpu_notifier(&coretemp_cpu_notifier);
> -	cpu_notifier_register_done();
>  	return 0;
>  
>  #ifndef CONFIG_HOTPLUG_CPU
> +exit_hp_unreg:
> +	cpuhp_remove_state(coretemp_hp_online);
> +#endif
>  exit_driver_unreg:
>  	platform_driver_unregister(&coretemp_driver);
> -#endif
>  exit:
>  	return err;
>  }
>  
>  static void __exit coretemp_exit(void)
>  {
> -	struct pdev_entry *p, *n;
> -
> -	cpu_notifier_register_begin();
> -	__unregister_hotcpu_notifier(&coretemp_cpu_notifier);
> -	mutex_lock(&pdev_list_mutex);
> -	list_for_each_entry_safe(p, n, &pdev_list, list) {
> -		platform_device_unregister(p->pdev);
> -		list_del(&p->list);
> -		kfree(p);
> -	}
> -	mutex_unlock(&pdev_list_mutex);
> -	cpu_notifier_register_done();
> +	cpuhp_remove_state(coretemp_hp_online);
>  	platform_driver_unregister(&coretemp_driver);
>  }
>  

Re: [04/20] hwmon/coretemp: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

On 2016-11-20 14:30:55 [-0800], Guenter Roeck wrote:
> Install, modprobe, modprobe -r, modprobe,
> 
> [24078.179118] ------------[ cut here ]------------
> [24078.179124] WARNING: CPU: 0 PID: 14 at fs/sysfs/dir.c:31 sysfs_warn_dup+0x6e/0x80
> [24078.179125] sysfs: cannot create duplicate filename
> '/devices/platform/coretemp.0'
> 
> and many more of those.
> 
> I can not test the other hwmon drivers in this series, but I would assume
> that they have the same problem.

I managed to trigger this. I will take a look.

> Guenter

Sebastian

[tip:smp/hotplug] hwmon/coretemp: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  73f4f6f5ab9a069037b8e98663193accaf18bd5c
Gitweb:     http://git.kernel.org/tip/73f4f6f5ab9a069037b8e98663193accaf18bd5c
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:25 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:10 +0100

hwmon/coretemp: Convert to hotplug state machine

Install the callbacks via the state machine. Setup and teardown are handled
by the hotplug core.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: linux-hwmon@vger.kernel.org
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Jean Delvare <jdelvare@suse.com>
Cc: rt@linuxtronix.de
Cc: Guenter Roeck <linux@roeck-us.net>
Link: http://lkml.kernel.org/r/20161117183541.8588-5-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/hwmon/coretemp.c | 74 +++++++++++++++---------------------------------
 1 file changed, 23 insertions(+), 51 deletions(-)

diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
index 6a27eb2..3900388 100644
--- a/drivers/hwmon/coretemp.c
+++ b/drivers/hwmon/coretemp.c
@@ -676,7 +676,7 @@ static bool is_any_core_online(struct platform_data *pdata)
 	return false;
 }
 
-static void get_core_online(unsigned int cpu)
+static int coretemp_cpu_online(unsigned int cpu)
 {
 	struct cpuinfo_x86 *c = &cpu_data(cpu);
 	struct platform_device *pdev = coretemp_get_pdev(cpu);
@@ -688,12 +688,12 @@ static void get_core_online(unsigned int cpu)
 	 * without thermal sensors will be filtered out.
 	 */
 	if (!cpu_has(c, X86_FEATURE_DTHERM))
-		return;
+		return 0;
 
 	if (!pdev) {
 		/* Check the microcode version of the CPU */
 		if (chk_ucode_version(cpu))
-			return;
+			return 0;
 
 		/*
 		 * Alright, we have DTS support.
@@ -703,7 +703,7 @@ static void get_core_online(unsigned int cpu)
 		 */
 		err = coretemp_device_add(cpu);
 		if (err)
-			return;
+			return 0;
 		/*
 		 * Check whether pkgtemp support is available.
 		 * If so, add interfaces for pkgtemp.
@@ -716,9 +716,10 @@ static void get_core_online(unsigned int cpu)
 	 * So, just add interfaces for this core.
 	 */
 	coretemp_add_core(cpu, 0);
+	return 0;
 }
 
-static void put_core_offline(unsigned int cpu)
+static int coretemp_cpu_offline(unsigned int cpu)
 {
 	int i, indx;
 	struct platform_data *pdata;
@@ -726,7 +727,7 @@ static void put_core_offline(unsigned int cpu)
 
 	/* If the physical CPU device does not exist, just return */
 	if (!pdev)
-		return;
+		return 0;
 
 	pdata = platform_get_drvdata(pdev);
 
@@ -734,7 +735,7 @@ static void put_core_offline(unsigned int cpu)
 
 	/* The core id is too big, just return */
 	if (indx > MAX_CORE_DATA - 1)
-		return;
+		return 0;
 
 	if (pdata->core_data[indx] && pdata->core_data[indx]->cpu == cpu)
 		coretemp_remove_core(pdata, indx);
@@ -747,7 +748,7 @@ static void put_core_offline(unsigned int cpu)
 	 */
 	for_each_sibling(i, cpu) {
 		if (i != cpu) {
-			get_core_online(i);
+			coretemp_cpu_online(i);
 			/*
 			 * Display temperature sensor data for one HT sibling
 			 * per core only, so abort the loop after one such
@@ -764,38 +765,20 @@ static void put_core_offline(unsigned int cpu)
 	 */
 	if (!is_any_core_online(pdata))
 		coretemp_device_remove(cpu);
+	return 0;
 }
 
-static int coretemp_cpu_callback(struct notifier_block *nfb,
-				 unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long) hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		get_core_online(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		put_core_offline(cpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block coretemp_cpu_notifier __refdata = {
-	.notifier_call = coretemp_cpu_callback,
-};
-
 static const struct x86_cpu_id __initconst coretemp_ids[] = {
 	{ X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_DTHERM },
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, coretemp_ids);
 
+static enum cpuhp_state coretemp_hp_online;
+
 static int __init coretemp_init(void)
 {
-	int i, err;
+	int err;
 
 	/*
 	 * CPUID.06H.EAX[0] indicates whether the CPU has thermal
@@ -809,44 +792,33 @@ static int __init coretemp_init(void)
 	if (err)
 		goto exit;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i)
-		get_core_online(i);
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/coretemp:online",
+				coretemp_cpu_online, coretemp_cpu_offline);
+	if (err < 0)
+		goto exit_driver_unreg;
+	coretemp_hp_online = err;
 
 #ifndef CONFIG_HOTPLUG_CPU
 	if (list_empty(&pdev_list)) {
-		cpu_notifier_register_done();
 		err = -ENODEV;
-		goto exit_driver_unreg;
+		goto exit_hp_unreg;
 	}
 #endif
-
-	__register_hotcpu_notifier(&coretemp_cpu_notifier);
-	cpu_notifier_register_done();
 	return 0;
 
 #ifndef CONFIG_HOTPLUG_CPU
+exit_hp_unreg:
+	cpuhp_remove_state(coretemp_hp_online);
+#endif
 exit_driver_unreg:
 	platform_driver_unregister(&coretemp_driver);
-#endif
 exit:
 	return err;
 }
 
 static void __exit coretemp_exit(void)
 {
-	struct pdev_entry *p, *n;
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&coretemp_cpu_notifier);
-	mutex_lock(&pdev_list_mutex);
-	list_for_each_entry_safe(p, n, &pdev_list, list) {
-		platform_device_unregister(p->pdev);
-		list_del(&p->list);
-		kfree(p);
-	}
-	mutex_unlock(&pdev_list_mutex);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(coretemp_hp_online);
 	platform_driver_unregister(&coretemp_driver);
 }
 

Re: [tip:smp/hotplug] hwmon/coretemp: Convert to hotplug state machine

[Guenter Roeck]

Just be aware that this patch is buggy.

Guenter

On Mon, Nov 21, 2016 at 07:56:02AM -0800, tip-bot for Sebastian Andrzej Siewior wrote:
> Commit-ID:  73f4f6f5ab9a069037b8e98663193accaf18bd5c
> Gitweb:     http://git.kernel.org/tip/73f4f6f5ab9a069037b8e98663193accaf18bd5c
> Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> AuthorDate: Thu, 17 Nov 2016 19:35:25 +0100
> Committer:  Thomas Gleixner <tglx@linutronix.de>
> CommitDate: Mon, 21 Nov 2016 16:37:10 +0100
> 
> hwmon/coretemp: Convert to hotplug state machine
> 
> Install the callbacks via the state machine. Setup and teardown are handled
> by the hotplug core.
> 
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Cc: linux-hwmon@vger.kernel.org
> Cc: Fenghua Yu <fenghua.yu@intel.com>
> Cc: Jean Delvare <jdelvare@suse.com>
> Cc: rt@linuxtronix.de
> Cc: Guenter Roeck <linux@roeck-us.net>
> Link: http://lkml.kernel.org/r/20161117183541.8588-5-bigeasy@linutronix.de
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> 
> ---
>  drivers/hwmon/coretemp.c | 74 +++++++++++++++---------------------------------
>  1 file changed, 23 insertions(+), 51 deletions(-)
> 
> diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
> index 6a27eb2..3900388 100644
> --- a/drivers/hwmon/coretemp.c
> +++ b/drivers/hwmon/coretemp.c
> @@ -676,7 +676,7 @@ static bool is_any_core_online(struct platform_data *pdata)
>  	return false;
>  }
>  
> -static void get_core_online(unsigned int cpu)
> +static int coretemp_cpu_online(unsigned int cpu)
>  {
>  	struct cpuinfo_x86 *c = &cpu_data(cpu);
>  	struct platform_device *pdev = coretemp_get_pdev(cpu);
> @@ -688,12 +688,12 @@ static void get_core_online(unsigned int cpu)
>  	 * without thermal sensors will be filtered out.
>  	 */
>  	if (!cpu_has(c, X86_FEATURE_DTHERM))
> -		return;
> +		return 0;
>  
>  	if (!pdev) {
>  		/* Check the microcode version of the CPU */
>  		if (chk_ucode_version(cpu))
> -			return;
> +			return 0;
>  
>  		/*
>  		 * Alright, we have DTS support.
> @@ -703,7 +703,7 @@ static void get_core_online(unsigned int cpu)
>  		 */
>  		err = coretemp_device_add(cpu);
>  		if (err)
> -			return;
> +			return 0;
>  		/*
>  		 * Check whether pkgtemp support is available.
>  		 * If so, add interfaces for pkgtemp.
> @@ -716,9 +716,10 @@ static void get_core_online(unsigned int cpu)
>  	 * So, just add interfaces for this core.
>  	 */
>  	coretemp_add_core(cpu, 0);
> +	return 0;
>  }
>  
> -static void put_core_offline(unsigned int cpu)
> +static int coretemp_cpu_offline(unsigned int cpu)
>  {
>  	int i, indx;
>  	struct platform_data *pdata;
> @@ -726,7 +727,7 @@ static void put_core_offline(unsigned int cpu)
>  
>  	/* If the physical CPU device does not exist, just return */
>  	if (!pdev)
> -		return;
> +		return 0;
>  
>  	pdata = platform_get_drvdata(pdev);
>  
> @@ -734,7 +735,7 @@ static void put_core_offline(unsigned int cpu)
>  
>  	/* The core id is too big, just return */
>  	if (indx > MAX_CORE_DATA - 1)
> -		return;
> +		return 0;
>  
>  	if (pdata->core_data[indx] && pdata->core_data[indx]->cpu == cpu)
>  		coretemp_remove_core(pdata, indx);
> @@ -747,7 +748,7 @@ static void put_core_offline(unsigned int cpu)
>  	 */
>  	for_each_sibling(i, cpu) {
>  		if (i != cpu) {
> -			get_core_online(i);
> +			coretemp_cpu_online(i);
>  			/*
>  			 * Display temperature sensor data for one HT sibling
>  			 * per core only, so abort the loop after one such
> @@ -764,38 +765,20 @@ static void put_core_offline(unsigned int cpu)
>  	 */
>  	if (!is_any_core_online(pdata))
>  		coretemp_device_remove(cpu);
> +	return 0;
>  }
>  
> -static int coretemp_cpu_callback(struct notifier_block *nfb,
> -				 unsigned long action, void *hcpu)
> -{
> -	unsigned int cpu = (unsigned long) hcpu;
> -
> -	switch (action) {
> -	case CPU_ONLINE:
> -	case CPU_DOWN_FAILED:
> -		get_core_online(cpu);
> -		break;
> -	case CPU_DOWN_PREPARE:
> -		put_core_offline(cpu);
> -		break;
> -	}
> -	return NOTIFY_OK;
> -}
> -
> -static struct notifier_block coretemp_cpu_notifier __refdata = {
> -	.notifier_call = coretemp_cpu_callback,
> -};
> -
>  static const struct x86_cpu_id __initconst coretemp_ids[] = {
>  	{ X86_VENDOR_INTEL, X86_FAMILY_ANY, X86_MODEL_ANY, X86_FEATURE_DTHERM },
>  	{}
>  };
>  MODULE_DEVICE_TABLE(x86cpu, coretemp_ids);
>  
> +static enum cpuhp_state coretemp_hp_online;
> +
>  static int __init coretemp_init(void)
>  {
> -	int i, err;
> +	int err;
>  
>  	/*
>  	 * CPUID.06H.EAX[0] indicates whether the CPU has thermal
> @@ -809,44 +792,33 @@ static int __init coretemp_init(void)
>  	if (err)
>  		goto exit;
>  
> -	cpu_notifier_register_begin();
> -	for_each_online_cpu(i)
> -		get_core_online(i);
> +	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/coretemp:online",
> +				coretemp_cpu_online, coretemp_cpu_offline);
> +	if (err < 0)
> +		goto exit_driver_unreg;
> +	coretemp_hp_online = err;
>  
>  #ifndef CONFIG_HOTPLUG_CPU
>  	if (list_empty(&pdev_list)) {
> -		cpu_notifier_register_done();
>  		err = -ENODEV;
> -		goto exit_driver_unreg;
> +		goto exit_hp_unreg;
>  	}
>  #endif
> -
> -	__register_hotcpu_notifier(&coretemp_cpu_notifier);
> -	cpu_notifier_register_done();
>  	return 0;
>  
>  #ifndef CONFIG_HOTPLUG_CPU
> +exit_hp_unreg:
> +	cpuhp_remove_state(coretemp_hp_online);
> +#endif
>  exit_driver_unreg:
>  	platform_driver_unregister(&coretemp_driver);
> -#endif
>  exit:
>  	return err;
>  }
>  
>  static void __exit coretemp_exit(void)
>  {
> -	struct pdev_entry *p, *n;
> -
> -	cpu_notifier_register_begin();
> -	__unregister_hotcpu_notifier(&coretemp_cpu_notifier);
> -	mutex_lock(&pdev_list_mutex);
> -	list_for_each_entry_safe(p, n, &pdev_list, list) {
> -		platform_device_unregister(p->pdev);
> -		list_del(&p->list);
> -		kfree(p);
> -	}
> -	mutex_unlock(&pdev_list_mutex);
> -	cpu_notifier_register_done();
> +	cpuhp_remove_state(coretemp_hp_online);
>  	platform_driver_unregister(&coretemp_driver);
>  }
>  

Re: [tip:smp/hotplug] hwmon/coretemp: Convert to hotplug state machine

[Thomas Gleixner]

On Mon, 21 Nov 2016, Guenter Roeck wrote:

> Just be aware that this patch is buggy.

Zapped it for now.

Thanks,

	tglx

[PATCH 05/20] hwmon/via-cputemp: Remove pointless CPU check on each CPU

[Sebastian Andrzej Siewior]

From: Thomas Gleixner <tglx@linutronix.de>

The check loop for the cpu type is pointless as we already have a cpu model
match before that. The only thing which is not covered by that check would
be a smp system with two different cores. Not likely to happen.

Cc: Jean Delvare <jdelvare@suse.com>
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: linux-hwmon@vger.kernel.org
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 drivers/hwmon/via-cputemp.c | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c
index ac91c07e3f90..5b9866b1b437 100644
--- a/drivers/hwmon/via-cputemp.c
+++ b/drivers/hwmon/via-cputemp.c
@@ -319,22 +319,8 @@ static int __init via_cputemp_init(void)
 		goto exit;
 
 	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		struct cpuinfo_x86 *c = &cpu_data(i);
-
-		if (c->x86 != 6)
-			continue;
-
-		if (c->x86_model < 0x0a)
-			continue;
-
-		if (c->x86_model > 0x0f) {
-			pr_warn("Unknown CPU model 0x%x\n", c->x86_model);
-			continue;
-		}
-
+	for_each_online_cpu(i)
 		via_cputemp_device_add(i);
-	}
 
 #ifndef CONFIG_HOTPLUG_CPU
 	if (list_empty(&pdev_list)) {
-- 
2.10.2

Re: [05/20] hwmon/via-cputemp: Remove pointless CPU check on each CPU

[Guenter Roeck]

On Thu, Nov 17, 2016 at 07:35:26PM +0100, Sebastian Andrzej Siewior wrote:
> From: Thomas Gleixner <tglx@linutronix.de>
> 
> The check loop for the cpu type is pointless as we already have a cpu model
> match before that. The only thing which is not covered by that check would
> be a smp system with two different cores. Not likely to happen.
> 
> Cc: Jean Delvare <jdelvare@suse.com>
> Cc: Guenter Roeck <linux@roeck-us.net>
> Cc: linux-hwmon@vger.kernel.org
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

Applied to -next.

Thanks,
Guenter

Re: [05/20] hwmon/via-cputemp: Remove pointless CPU check on each CPU

[Sebastian Andrzej Siewior]

On 2016-11-19 09:23:31 [-0800], Guenter Roeck wrote:
> Applied to -next.

Thanks. Since you took that one, could you also please consider to apply
|[PATCH 06/20 v2] hwmon/via-cputemp: Convert to hotplug state machine
? It depends on the 5th patch from the series which applied.

> Thanks,
> Guenter

Sebastian

Re: [05/20] hwmon/via-cputemp: Remove pointless CPU check on each CPU

[Guenter Roeck]

On 11/19/2016 02:53 PM, Sebastian Andrzej Siewior wrote:
> On 2016-11-19 09:23:31 [-0800], Guenter Roeck wrote:
>> Applied to -next.
>
> Thanks. Since you took that one, could you also please consider to apply
> |[PATCH 06/20 v2] hwmon/via-cputemp: Convert to hotplug state machine
> ? It depends on the 5th patch from the series which applied.
>

Problem is that I have no idea if any of the patches in this series really work.
I wasn't copied on all patches, meaning I don't have the infrastructure, meaning
I'll have to dig them up from patchwork for testing, and/or figure out if the
required infrastructure is already in the kernel, and that all takes time.
Just Acking or applying the the patches w/o testing doesn't seem appropriate,
given their level of intrusiveness. I did spend most of today working through
my backlog of hwmon patches, but that series simply was too much. I hope I'll
get to it tomorrow, but no promises.

Guenter

Re: [05/20] hwmon/via-cputemp: Remove pointless CPU check on each CPU

[Sebastian Andrzej Siewior]

On 2016-11-19 19:53:21 [-0800], Guenter Roeck wrote:
> Problem is that I have no idea if any of the patches in this series really work.
> I wasn't copied on all patches, meaning I don't have the infrastructure, meaning
> I'll have to dig them up from patchwork for testing, and/or figure out if the
> required infrastructure is already in the kernel, and that all takes time.
> Just Acking or applying the the patches w/o testing doesn't seem appropriate,
> given their level of intrusiveness. I did spend most of today working through
> my backlog of hwmon patches, but that series simply was too much. I hope I'll
> get to it tomorrow, but no promises.

You don't have to dig all the patches from the series. Patch four and five are
via-cputemp related and independent of the others and you were on Cc: on
both of them. The other 18 patches were converting other drivers for
instance:
  [PATCH 01/20] x86/mce/therm_throt: Convert to hotplug state machine                      
  [PATCH 02/20] x86/cpuid: Convert to hotplug state machine                                
  [PATCH 03/20] x86/msr: Convert to hotplug state machine     

Infrastructure. The DYN allocation is available since 5b7aa87e0482
("cpu/hotplug: Implement setup/removal interface") which is part of part
of v4.6-rc1. So testing them on v4.8 should be enough :)

> Guenter

Sebastian

[PATCH 06/20] hwmon/via-cputemp: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke the
callbacks on the already online CPUs. When the hotplug state is
unregistered the cleanup function is called for each cpu. So both cpu loops
in init() and exit() are not longer required.

Cc: Jean Delvare <jdelvare@suse.com>
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: linux-hwmon@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 drivers/hwmon/via-cputemp.c | 61 ++++++++++++---------------------------------
 1 file changed, 16 insertions(+), 45 deletions(-)

diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c
index 5b9866b1b437..ee5377ecd7a9 100644
--- a/drivers/hwmon/via-cputemp.c
+++ b/drivers/hwmon/via-cputemp.c
@@ -220,7 +220,7 @@ struct pdev_entry {
 static LIST_HEAD(pdev_list);
 static DEFINE_MUTEX(pdev_list_mutex);
 
-static int via_cputemp_device_add(unsigned int cpu)
+static int via_cputemp_online(unsigned int cpu)
 {
 	int err;
 	struct platform_device *pdev;
@@ -261,7 +261,7 @@ static int via_cputemp_device_add(unsigned int cpu)
 	return err;
 }
 
-static void via_cputemp_device_remove(unsigned int cpu)
+static int via_cputemp_down_prep(unsigned int cpu)
 {
 	struct pdev_entry *p;
 
@@ -272,33 +272,13 @@ static void via_cputemp_device_remove(unsigned int cpu)
 			list_del(&p->list);
 			mutex_unlock(&pdev_list_mutex);
 			kfree(p);
-			return;
+			return 0;
 		}
 	}
 	mutex_unlock(&pdev_list_mutex);
+	return 0;
 }
 
-static int via_cputemp_cpu_callback(struct notifier_block *nfb,
-				    unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long) hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		via_cputemp_device_add(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		via_cputemp_device_remove(cpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block via_cputemp_cpu_notifier __refdata = {
-	.notifier_call = via_cputemp_cpu_callback,
-};
-
 static const struct x86_cpu_id __initconst cputemp_ids[] = {
 	{ X86_VENDOR_CENTAUR, 6, 0xa, }, /* C7 A */
 	{ X86_VENDOR_CENTAUR, 6, 0xd, }, /* C7 D */
@@ -307,6 +287,8 @@ static const struct x86_cpu_id __initconst cputemp_ids[] = {
 };
 MODULE_DEVICE_TABLE(x86cpu, cputemp_ids);
 
+static enum cpuhp_state via_temp_online;
+
 static int __init via_cputemp_init(void)
 {
 	int i, err;
@@ -318,44 +300,33 @@ static int __init via_cputemp_init(void)
 	if (err)
 		goto exit;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i)
-		via_cputemp_device_add(i);
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/via:online",
+				via_cputemp_online, via_cputemp_down_prep);
+	if (err < 0)
+		goto exit_driver_unreg;
+	via_temp_online = err;
 
 #ifndef CONFIG_HOTPLUG_CPU
 	if (list_empty(&pdev_list)) {
-		cpu_notifier_register_done();
 		err = -ENODEV;
-		goto exit_driver_unreg;
+		goto exit_hp_unreg;
 	}
 #endif
-
-	__register_hotcpu_notifier(&via_cputemp_cpu_notifier);
-	cpu_notifier_register_done();
 	return 0;
 
 #ifndef CONFIG_HOTPLUG_CPU
+exit_hp_unreg:
+	cpuhp_remove_state_nocalls(via_temp_online);
+#endif
 exit_driver_unreg:
 	platform_driver_unregister(&via_cputemp_driver);
-#endif
 exit:
 	return err;
 }
 
 static void __exit via_cputemp_exit(void)
 {
-	struct pdev_entry *p, *n;
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&via_cputemp_cpu_notifier);
-	mutex_lock(&pdev_list_mutex);
-	list_for_each_entry_safe(p, n, &pdev_list, list) {
-		platform_device_unregister(p->pdev);
-		list_del(&p->list);
-		kfree(p);
-	}
-	mutex_unlock(&pdev_list_mutex);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(via_temp_online);
 	platform_driver_unregister(&via_cputemp_driver);
 }
 
-- 
2.10.2

[PATCH 06/20 v2] hwmon/via-cputemp: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke the
callbacks on the already online CPUs. When the hotplug state is
unregistered the cleanup function is called for each cpu. So both cpu loops
in init() and exit() are not longer required.

Cc: Jean Delvare <jdelvare@suse.com>
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: linux-hwmon@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
v1…v2: drop `i' in via_cputemp_init() because it is unused, reported by
       kbuild test robot <fengguang.wu@intel.com> 

 drivers/hwmon/via-cputemp.c | 63 ++++++++++++---------------------------------
 1 file changed, 17 insertions(+), 46 deletions(-)

diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c
index 5b9866b1b437..d1f209a5feac 100644
--- a/drivers/hwmon/via-cputemp.c
+++ b/drivers/hwmon/via-cputemp.c
@@ -220,7 +220,7 @@ struct pdev_entry {
 static LIST_HEAD(pdev_list);
 static DEFINE_MUTEX(pdev_list_mutex);
 
-static int via_cputemp_device_add(unsigned int cpu)
+static int via_cputemp_online(unsigned int cpu)
 {
 	int err;
 	struct platform_device *pdev;
@@ -261,7 +261,7 @@ static int via_cputemp_device_add(unsigned int cpu)
 	return err;
 }
 
-static void via_cputemp_device_remove(unsigned int cpu)
+static int via_cputemp_down_prep(unsigned int cpu)
 {
 	struct pdev_entry *p;
 
@@ -272,33 +272,13 @@ static void via_cputemp_device_remove(unsigned int cpu)
 			list_del(&p->list);
 			mutex_unlock(&pdev_list_mutex);
 			kfree(p);
-			return;
+			return 0;
 		}
 	}
 	mutex_unlock(&pdev_list_mutex);
+	return 0;
 }
 
-static int via_cputemp_cpu_callback(struct notifier_block *nfb,
-				    unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long) hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		via_cputemp_device_add(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-		via_cputemp_device_remove(cpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block via_cputemp_cpu_notifier __refdata = {
-	.notifier_call = via_cputemp_cpu_callback,
-};
-
 static const struct x86_cpu_id __initconst cputemp_ids[] = {
 	{ X86_VENDOR_CENTAUR, 6, 0xa, }, /* C7 A */
 	{ X86_VENDOR_CENTAUR, 6, 0xd, }, /* C7 D */
@@ -307,9 +287,11 @@ static const struct x86_cpu_id __initconst cputemp_ids[] = {
 };
 MODULE_DEVICE_TABLE(x86cpu, cputemp_ids);
 
+static enum cpuhp_state via_temp_online;
+
 static int __init via_cputemp_init(void)
 {
-	int i, err;
+	int err;
 
 	if (!x86_match_cpu(cputemp_ids))
 		return -ENODEV;
@@ -318,44 +300,33 @@ static int __init via_cputemp_init(void)
 	if (err)
 		goto exit;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i)
-		via_cputemp_device_add(i);
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/via:online",
+				via_cputemp_online, via_cputemp_down_prep);
+	if (err < 0)
+		goto exit_driver_unreg;
+	via_temp_online = err;
 
 #ifndef CONFIG_HOTPLUG_CPU
 	if (list_empty(&pdev_list)) {
-		cpu_notifier_register_done();
 		err = -ENODEV;
-		goto exit_driver_unreg;
+		goto exit_hp_unreg;
 	}
 #endif
-
-	__register_hotcpu_notifier(&via_cputemp_cpu_notifier);
-	cpu_notifier_register_done();
 	return 0;
 
 #ifndef CONFIG_HOTPLUG_CPU
+exit_hp_unreg:
+	cpuhp_remove_state_nocalls(via_temp_online);
+#endif
 exit_driver_unreg:
 	platform_driver_unregister(&via_cputemp_driver);
-#endif
 exit:
 	return err;
 }
 
 static void __exit via_cputemp_exit(void)
 {
-	struct pdev_entry *p, *n;
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&via_cputemp_cpu_notifier);
-	mutex_lock(&pdev_list_mutex);
-	list_for_each_entry_safe(p, n, &pdev_list, list) {
-		platform_device_unregister(p->pdev);
-		list_del(&p->list);
-		kfree(p);
-	}
-	mutex_unlock(&pdev_list_mutex);
-	cpu_notifier_register_done();
+	cpuhp_remove_state(via_temp_online);
 	platform_driver_unregister(&via_cputemp_driver);
 }
 
-- 
2.10.2

Re: [PATCH 06/20] hwmon/via-cputemp: Convert to hotplug state machine

[Guenter Roeck]

On 11/17/2016 10:35 AM, Sebastian Andrzej Siewior wrote:
> Install the callbacks via the state machine and let the core invoke the
> callbacks on the already online CPUs. When the hotplug state is
> unregistered the cleanup function is called for each cpu. So both cpu loops
> in init() and exit() are not longer required.
>
> Cc: Jean Delvare <jdelvare@suse.com>
> Cc: Guenter Roeck <linux@roeck-us.net>
> Cc: linux-hwmon@vger.kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

I would like to apply this patch, but I can not test it, and I want to make sure
it survives multiple removal/insertion sequences. Can someone give me a Tested-by: ?

Thanks,
Guenter

> ---
>  drivers/hwmon/via-cputemp.c | 61 ++++++++++++---------------------------------
>  1 file changed, 16 insertions(+), 45 deletions(-)
>
> diff --git a/drivers/hwmon/via-cputemp.c b/drivers/hwmon/via-cputemp.c
> index 5b9866b1b437..ee5377ecd7a9 100644
> --- a/drivers/hwmon/via-cputemp.c
> +++ b/drivers/hwmon/via-cputemp.c
> @@ -220,7 +220,7 @@ struct pdev_entry {
>  static LIST_HEAD(pdev_list);
>  static DEFINE_MUTEX(pdev_list_mutex);
>
> -static int via_cputemp_device_add(unsigned int cpu)
> +static int via_cputemp_online(unsigned int cpu)
>  {
>  	int err;
>  	struct platform_device *pdev;
> @@ -261,7 +261,7 @@ static int via_cputemp_device_add(unsigned int cpu)
>  	return err;
>  }
>
> -static void via_cputemp_device_remove(unsigned int cpu)
> +static int via_cputemp_down_prep(unsigned int cpu)
>  {
>  	struct pdev_entry *p;
>
> @@ -272,33 +272,13 @@ static void via_cputemp_device_remove(unsigned int cpu)
>  			list_del(&p->list);
>  			mutex_unlock(&pdev_list_mutex);
>  			kfree(p);
> -			return;
> +			return 0;
>  		}
>  	}
>  	mutex_unlock(&pdev_list_mutex);
> +	return 0;
>  }
>
> -static int via_cputemp_cpu_callback(struct notifier_block *nfb,
> -				    unsigned long action, void *hcpu)
> -{
> -	unsigned int cpu = (unsigned long) hcpu;
> -
> -	switch (action) {
> -	case CPU_ONLINE:
> -	case CPU_DOWN_FAILED:
> -		via_cputemp_device_add(cpu);
> -		break;
> -	case CPU_DOWN_PREPARE:
> -		via_cputemp_device_remove(cpu);
> -		break;
> -	}
> -	return NOTIFY_OK;
> -}
> -
> -static struct notifier_block via_cputemp_cpu_notifier __refdata = {
> -	.notifier_call = via_cputemp_cpu_callback,
> -};
> -
>  static const struct x86_cpu_id __initconst cputemp_ids[] = {
>  	{ X86_VENDOR_CENTAUR, 6, 0xa, }, /* C7 A */
>  	{ X86_VENDOR_CENTAUR, 6, 0xd, }, /* C7 D */
> @@ -307,6 +287,8 @@ static const struct x86_cpu_id __initconst cputemp_ids[] = {
>  };
>  MODULE_DEVICE_TABLE(x86cpu, cputemp_ids);
>
> +static enum cpuhp_state via_temp_online;
> +
>  static int __init via_cputemp_init(void)
>  {
>  	int i, err;
> @@ -318,44 +300,33 @@ static int __init via_cputemp_init(void)
>  	if (err)
>  		goto exit;
>
> -	cpu_notifier_register_begin();
> -	for_each_online_cpu(i)
> -		via_cputemp_device_add(i);
> +	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hwmon/via:online",
> +				via_cputemp_online, via_cputemp_down_prep);
> +	if (err < 0)
> +		goto exit_driver_unreg;
> +	via_temp_online = err;
>
>  #ifndef CONFIG_HOTPLUG_CPU
>  	if (list_empty(&pdev_list)) {
> -		cpu_notifier_register_done();
>  		err = -ENODEV;
> -		goto exit_driver_unreg;
> +		goto exit_hp_unreg;
>  	}
>  #endif
> -
> -	__register_hotcpu_notifier(&via_cputemp_cpu_notifier);
> -	cpu_notifier_register_done();
>  	return 0;
>
>  #ifndef CONFIG_HOTPLUG_CPU
> +exit_hp_unreg:
> +	cpuhp_remove_state_nocalls(via_temp_online);
> +#endif
>  exit_driver_unreg:
>  	platform_driver_unregister(&via_cputemp_driver);
> -#endif
>  exit:
>  	return err;
>  }
>
>  static void __exit via_cputemp_exit(void)
>  {
> -	struct pdev_entry *p, *n;
> -
> -	cpu_notifier_register_begin();
> -	__unregister_hotcpu_notifier(&via_cputemp_cpu_notifier);
> -	mutex_lock(&pdev_list_mutex);
> -	list_for_each_entry_safe(p, n, &pdev_list, list) {
> -		platform_device_unregister(p->pdev);
> -		list_del(&p->list);
> -		kfree(p);
> -	}
> -	mutex_unlock(&pdev_list_mutex);
> -	cpu_notifier_register_done();
> +	cpuhp_remove_state(via_temp_online);
>  	platform_driver_unregister(&via_cputemp_driver);
>  }
>
>

Re: [PATCH 06/20] hwmon/via-cputemp: Convert to hotplug state machine

[Thomas Gleixner]

Guenter,

On Thu, 17 Nov 2016, Sebastian Andrzej Siewior wrote:

> Install the callbacks via the state machine and let the core invoke the
> callbacks on the already online CPUs. When the hotplug state is
> unregistered the cleanup function is called for each cpu. So both cpu loops
> in init() and exit() are not longer required.

Can we please get those two VIA patches merged for 4.10? They are blocking
the final removal of the CPU hotplug notifier crap.

The first one which removes that loop is really harmless as there are no
multisocket VIAs. Heterogenous cores in a single die would be surprising
and the loop check would be the least of our worries in that case. IOW, it
would never get so far...

The one converting the notifier is not changing any of the functionality.

I cannot test on all VIA SMP variants either, but at least on the one I
have access to it just works.

Thanks,

	tglx

Re: [PATCH 06/20] hwmon/via-cputemp: Convert to hotplug state machine

[Guenter Roeck]

On Fri, Dec 09, 2016 at 12:53:30PM +0100, Thomas Gleixner wrote:
> Guenter,
> 
> On Thu, 17 Nov 2016, Sebastian Andrzej Siewior wrote:
> 
> > Install the callbacks via the state machine and let the core invoke the
> > callbacks on the already online CPUs. When the hotplug state is
> > unregistered the cleanup function is called for each cpu. So both cpu loops
> > in init() and exit() are not longer required.
> 
> Can we please get those two VIA patches merged for 4.10? They are blocking
> the final removal of the CPU hotplug notifier crap.
> 
> The first one which removes that loop is really harmless as there are no
> multisocket VIAs. Heterogenous cores in a single die would be surprising
> and the loop check would be the least of our worries in that case. IOW, it
> would never get so far...
> 
I had queued that one already.

> The one converting the notifier is not changing any of the functionality.
> 
> I cannot test on all VIA SMP variants either, but at least on the one I
> have access to it just works.
> 
I queued up the second patch as well. Hope it does not blow up on us.
Sorry, I got a bit nervous after the coretemp experience.

Thanks,
Guenter

Re: [PATCH 06/20] hwmon/via-cputemp: Convert to hotplug state machine

[Thomas Gleixner]

On Fri, 9 Dec 2016, Guenter Roeck wrote:
> I queued up the second patch as well. Hope it does not blow up on us.
> Sorry, I got a bit nervous after the coretemp experience.

Sorry for that, but we are watching out for blow ups and are ready to fix
any fallout.

Thanks

	tglx

[PATCH 07/20] pci/xgene-msi: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: Duc Dang <dhdang@apm.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: linux-pci@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 drivers/pci/host/pci-xgene-msi.c | 69 +++++++++++-----------------------------
 include/linux/cpuhotplug.h       |  1 +
 2 files changed, 20 insertions(+), 50 deletions(-)

diff --git a/drivers/pci/host/pci-xgene-msi.c b/drivers/pci/host/pci-xgene-msi.c
index a6456b578269..1f38d0836751 100644
--- a/drivers/pci/host/pci-xgene-msi.c
+++ b/drivers/pci/host/pci-xgene-msi.c
@@ -360,16 +360,16 @@ static void xgene_msi_isr(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
+static enum cpuhp_state pci_xgene_online;
+
 static int xgene_msi_remove(struct platform_device *pdev)
 {
-	int virq, i;
 	struct xgene_msi *msi = platform_get_drvdata(pdev);
 
-	for (i = 0; i < NR_HW_IRQS; i++) {
-		virq = msi->msi_groups[i].gic_irq;
-		if (virq != 0)
-			irq_set_chained_handler_and_data(virq, NULL, NULL);
-	}
+	if (pci_xgene_online)
+		cpuhp_remove_state(pci_xgene_online);
+	cpuhp_remove_state(CPUHP_PCI_XGENE_DEAD);
+
 	kfree(msi->msi_groups);
 
 	kfree(msi->bitmap);
@@ -427,7 +427,7 @@ static int xgene_msi_hwirq_alloc(unsigned int cpu)
 	return 0;
 }
 
-static void xgene_msi_hwirq_free(unsigned int cpu)
+static int xgene_msi_hwirq_free(unsigned int cpu)
 {
 	struct xgene_msi *msi = &xgene_msi_ctrl;
 	struct xgene_msi_group *msi_group;
@@ -441,33 +441,9 @@ static void xgene_msi_hwirq_free(unsigned int cpu)
 		irq_set_chained_handler_and_data(msi_group->gic_irq, NULL,
 						 NULL);
 	}
+	return 0;
 }
 
-static int xgene_msi_cpu_callback(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
-{
-	unsigned cpu = (unsigned long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		xgene_msi_hwirq_alloc(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		xgene_msi_hwirq_free(cpu);
-		break;
-	default:
-		break;
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block xgene_msi_cpu_notifier = {
-	.notifier_call = xgene_msi_cpu_callback,
-};
-
 static const struct of_device_id xgene_msi_match_table[] = {
 	{.compatible = "apm,xgene1-msi"},
 	{},
@@ -478,7 +454,6 @@ static int xgene_msi_probe(struct platform_device *pdev)
 	struct resource *res;
 	int rc, irq_index;
 	struct xgene_msi *xgene_msi;
-	unsigned int cpu;
 	int virt_msir;
 	u32 msi_val, msi_idx;
 
@@ -540,28 +515,22 @@ static int xgene_msi_probe(struct platform_device *pdev)
 		}
 	}
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu)
-		if (xgene_msi_hwirq_alloc(cpu)) {
-			dev_err(&pdev->dev, "failed to register MSI handlers\n");
-			cpu_notifier_register_done();
-			goto error;
-		}
-
-	rc = __register_hotcpu_notifier(&xgene_msi_cpu_notifier);
-	if (rc) {
-		dev_err(&pdev->dev, "failed to add CPU MSI notifier\n");
-		cpu_notifier_register_done();
-		goto error;
-	}
-
-	cpu_notifier_register_done();
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/xgene:online",
+			       xgene_msi_hwirq_alloc, NULL);
+	if (rc)
+		goto err_cpuhp;
+	pci_xgene_online = rc;
+	rc = cpuhp_setup_state(CPUHP_PCI_XGENE_DEAD, "pci/xgene:dead", NULL,
+			       xgene_msi_hwirq_free);
+	if (rc)
+		goto err_cpuhp;
 
 	dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n");
 
 	return 0;
 
+err_cpuhp:
+	dev_err(&pdev->dev, "failed to add CPU MSI notifier\n");
 error:
 	xgene_msi_remove(pdev);
 	return rc;
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 6506dce8343a..fd5598b8353a 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -38,6 +38,7 @@ enum cpuhp_state {
 	CPUHP_RADIX_DEAD,
 	CPUHP_PAGE_ALLOC_DEAD,
 	CPUHP_NET_DEV_DEAD,
+	CPUHP_PCI_XGENE_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
-- 
2.10.2

[tip:smp/hotplug] PCI/xgene-msi: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  93405c5d155576801b0f6e96db339858541262dd
Gitweb:     http://git.kernel.org/tip/93405c5d155576801b0f6e96db339858541262dd
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:28 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:06 +0100

PCI/xgene-msi: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: linux-pci@vger.kernel.org
Cc: Duc Dang <dhdang@apm.com>
Cc: rt@linuxtronix.de
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: linux-arm-kernel@lists.infradead.org
Link: http://lkml.kernel.org/r/20161117183541.8588-8-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/pci/host/pci-xgene-msi.c | 69 +++++++++++-----------------------------
 include/linux/cpuhotplug.h       |  1 +
 2 files changed, 20 insertions(+), 50 deletions(-)

diff --git a/drivers/pci/host/pci-xgene-msi.c b/drivers/pci/host/pci-xgene-msi.c
index a6456b5..1f38d08 100644
--- a/drivers/pci/host/pci-xgene-msi.c
+++ b/drivers/pci/host/pci-xgene-msi.c
@@ -360,16 +360,16 @@ static void xgene_msi_isr(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
+static enum cpuhp_state pci_xgene_online;
+
 static int xgene_msi_remove(struct platform_device *pdev)
 {
-	int virq, i;
 	struct xgene_msi *msi = platform_get_drvdata(pdev);
 
-	for (i = 0; i < NR_HW_IRQS; i++) {
-		virq = msi->msi_groups[i].gic_irq;
-		if (virq != 0)
-			irq_set_chained_handler_and_data(virq, NULL, NULL);
-	}
+	if (pci_xgene_online)
+		cpuhp_remove_state(pci_xgene_online);
+	cpuhp_remove_state(CPUHP_PCI_XGENE_DEAD);
+
 	kfree(msi->msi_groups);
 
 	kfree(msi->bitmap);
@@ -427,7 +427,7 @@ static int xgene_msi_hwirq_alloc(unsigned int cpu)
 	return 0;
 }
 
-static void xgene_msi_hwirq_free(unsigned int cpu)
+static int xgene_msi_hwirq_free(unsigned int cpu)
 {
 	struct xgene_msi *msi = &xgene_msi_ctrl;
 	struct xgene_msi_group *msi_group;
@@ -441,33 +441,9 @@ static void xgene_msi_hwirq_free(unsigned int cpu)
 		irq_set_chained_handler_and_data(msi_group->gic_irq, NULL,
 						 NULL);
 	}
+	return 0;
 }
 
-static int xgene_msi_cpu_callback(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
-{
-	unsigned cpu = (unsigned long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		xgene_msi_hwirq_alloc(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		xgene_msi_hwirq_free(cpu);
-		break;
-	default:
-		break;
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block xgene_msi_cpu_notifier = {
-	.notifier_call = xgene_msi_cpu_callback,
-};
-
 static const struct of_device_id xgene_msi_match_table[] = {
 	{.compatible = "apm,xgene1-msi"},
 	{},
@@ -478,7 +454,6 @@ static int xgene_msi_probe(struct platform_device *pdev)
 	struct resource *res;
 	int rc, irq_index;
 	struct xgene_msi *xgene_msi;
-	unsigned int cpu;
 	int virt_msir;
 	u32 msi_val, msi_idx;
 
@@ -540,28 +515,22 @@ static int xgene_msi_probe(struct platform_device *pdev)
 		}
 	}
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu)
-		if (xgene_msi_hwirq_alloc(cpu)) {
-			dev_err(&pdev->dev, "failed to register MSI handlers\n");
-			cpu_notifier_register_done();
-			goto error;
-		}
-
-	rc = __register_hotcpu_notifier(&xgene_msi_cpu_notifier);
-	if (rc) {
-		dev_err(&pdev->dev, "failed to add CPU MSI notifier\n");
-		cpu_notifier_register_done();
-		goto error;
-	}
-
-	cpu_notifier_register_done();
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/xgene:online",
+			       xgene_msi_hwirq_alloc, NULL);
+	if (rc)
+		goto err_cpuhp;
+	pci_xgene_online = rc;
+	rc = cpuhp_setup_state(CPUHP_PCI_XGENE_DEAD, "pci/xgene:dead", NULL,
+			       xgene_msi_hwirq_free);
+	if (rc)
+		goto err_cpuhp;
 
 	dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n");
 
 	return 0;
 
+err_cpuhp:
+	dev_err(&pdev->dev, "failed to add CPU MSI notifier\n");
 error:
 	xgene_msi_remove(pdev);
 	return rc;
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 79b96f6..94c6a18 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -38,6 +38,7 @@ enum cpuhp_state {
 	CPUHP_RADIX_DEAD,
 	CPUHP_PAGE_ALLOC_DEAD,
 	CPUHP_NET_DEV_DEAD,
+	CPUHP_PCI_XGENE_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,

[tip:smp/hotplug] PCI/xgene-msi: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  9c248f8896e6bf0c77abb98bfea8d69b5a7cd11d
Gitweb:     http://git.kernel.org/tip/9c248f8896e6bf0c77abb98bfea8d69b5a7cd11d
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:28 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:40 +0100

PCI/xgene-msi: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: linux-pci@vger.kernel.org
Cc: Duc Dang <dhdang@apm.com>
Cc: rt@linuxtronix.de
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: linux-arm-kernel@lists.infradead.org
Link: http://lkml.kernel.org/r/20161117183541.8588-8-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/pci/host/pci-xgene-msi.c | 69 +++++++++++-----------------------------
 include/linux/cpuhotplug.h       |  1 +
 2 files changed, 20 insertions(+), 50 deletions(-)

diff --git a/drivers/pci/host/pci-xgene-msi.c b/drivers/pci/host/pci-xgene-msi.c
index a6456b5..1f38d08 100644
--- a/drivers/pci/host/pci-xgene-msi.c
+++ b/drivers/pci/host/pci-xgene-msi.c
@@ -360,16 +360,16 @@ static void xgene_msi_isr(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
+static enum cpuhp_state pci_xgene_online;
+
 static int xgene_msi_remove(struct platform_device *pdev)
 {
-	int virq, i;
 	struct xgene_msi *msi = platform_get_drvdata(pdev);
 
-	for (i = 0; i < NR_HW_IRQS; i++) {
-		virq = msi->msi_groups[i].gic_irq;
-		if (virq != 0)
-			irq_set_chained_handler_and_data(virq, NULL, NULL);
-	}
+	if (pci_xgene_online)
+		cpuhp_remove_state(pci_xgene_online);
+	cpuhp_remove_state(CPUHP_PCI_XGENE_DEAD);
+
 	kfree(msi->msi_groups);
 
 	kfree(msi->bitmap);
@@ -427,7 +427,7 @@ static int xgene_msi_hwirq_alloc(unsigned int cpu)
 	return 0;
 }
 
-static void xgene_msi_hwirq_free(unsigned int cpu)
+static int xgene_msi_hwirq_free(unsigned int cpu)
 {
 	struct xgene_msi *msi = &xgene_msi_ctrl;
 	struct xgene_msi_group *msi_group;
@@ -441,33 +441,9 @@ static void xgene_msi_hwirq_free(unsigned int cpu)
 		irq_set_chained_handler_and_data(msi_group->gic_irq, NULL,
 						 NULL);
 	}
+	return 0;
 }
 
-static int xgene_msi_cpu_callback(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
-{
-	unsigned cpu = (unsigned long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		xgene_msi_hwirq_alloc(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		xgene_msi_hwirq_free(cpu);
-		break;
-	default:
-		break;
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block xgene_msi_cpu_notifier = {
-	.notifier_call = xgene_msi_cpu_callback,
-};
-
 static const struct of_device_id xgene_msi_match_table[] = {
 	{.compatible = "apm,xgene1-msi"},
 	{},
@@ -478,7 +454,6 @@ static int xgene_msi_probe(struct platform_device *pdev)
 	struct resource *res;
 	int rc, irq_index;
 	struct xgene_msi *xgene_msi;
-	unsigned int cpu;
 	int virt_msir;
 	u32 msi_val, msi_idx;
 
@@ -540,28 +515,22 @@ static int xgene_msi_probe(struct platform_device *pdev)
 		}
 	}
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu)
-		if (xgene_msi_hwirq_alloc(cpu)) {
-			dev_err(&pdev->dev, "failed to register MSI handlers\n");
-			cpu_notifier_register_done();
-			goto error;
-		}
-
-	rc = __register_hotcpu_notifier(&xgene_msi_cpu_notifier);
-	if (rc) {
-		dev_err(&pdev->dev, "failed to add CPU MSI notifier\n");
-		cpu_notifier_register_done();
-		goto error;
-	}
-
-	cpu_notifier_register_done();
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/xgene:online",
+			       xgene_msi_hwirq_alloc, NULL);
+	if (rc)
+		goto err_cpuhp;
+	pci_xgene_online = rc;
+	rc = cpuhp_setup_state(CPUHP_PCI_XGENE_DEAD, "pci/xgene:dead", NULL,
+			       xgene_msi_hwirq_free);
+	if (rc)
+		goto err_cpuhp;
 
 	dev_info(&pdev->dev, "APM X-Gene PCIe MSI driver loaded\n");
 
 	return 0;
 
+err_cpuhp:
+	dev_err(&pdev->dev, "failed to add CPU MSI notifier\n");
 error:
 	xgene_msi_remove(pdev);
 	return rc;
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 79b96f6..94c6a18 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -38,6 +38,7 @@ enum cpuhp_state {
 	CPUHP_RADIX_DEAD,
 	CPUHP_PAGE_ALLOC_DEAD,
 	CPUHP_NET_DEV_DEAD,
+	CPUHP_PCI_XGENE_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,

[PATCH 08/20] powercap/intel_rapl: Add missing domain data update on hotplug

[Sebastian Andrzej Siewior]

From: Thomas Gleixner <tglx@linutronix.de>

The domain data of packages is only updated at init time, but new packages
created by hotplug miss that treatment.

Add it there and remove the global update at init time, because it's now
obsolete.

Cc: "Rafael J. Wysocki" <rjw@rjwysocki.net>
Cc: linux-pm@vger.kernel.org
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 drivers/powercap/intel_rapl.c | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c
index 243b233ff31b..44b12e2e4aa7 100644
--- a/drivers/powercap/intel_rapl.c
+++ b/drivers/powercap/intel_rapl.c
@@ -1164,24 +1164,20 @@ static const struct x86_cpu_id rapl_ids[] __initconst = {
 };
 MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
 
-/* read once for all raw primitive data for all packages, domains */
-static void rapl_update_domain_data(void)
+/* Read once for all raw primitive data for domains */
+static void rapl_update_domain_data(struct rapl_package *rp)
 {
 	int dmn, prim;
 	u64 val;
-	struct rapl_package *rp;
 
-	list_for_each_entry(rp, &rapl_packages, plist) {
-		for (dmn = 0; dmn < rp->nr_domains; dmn++) {
-			pr_debug("update package %d domain %s data\n", rp->id,
-				rp->domains[dmn].name);
+	for (dmn = 0; dmn < rp->nr_domains; dmn++) {
+		pr_debug("update package %d domain %s data\n", rp->id,
+			 rp->domains[dmn].name);
 			/* exclude non-raw primitives */
-			for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++)
-				if (!rapl_read_data_raw(&rp->domains[dmn], prim,
-								rpi[prim].unit,
-								&val))
-					rp->domains[dmn].rdd.primitives[prim] =
-									val;
+		for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
+			if (!rapl_read_data_raw(&rp->domains[dmn], prim,
+						rpi[prim].unit,	&val))
+				rp->domains[dmn].rdd.primitives[prim] = val;
 		}
 	}
 
@@ -1239,6 +1235,9 @@ static int rapl_package_register_powercap(struct rapl_package *rp)
 	struct powercap_zone *power_zone = NULL;
 	int nr_pl;
 
+	/* Update the domain data of the new package */
+	rapl_update_domain_data(rp);
+
 	/* first we register package domain as the parent zone*/
 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
 		if (rd->id == RAPL_DOMAIN_PACKAGE) {
@@ -1357,8 +1356,7 @@ static int rapl_register_powercap(void)
 		pr_debug("failed to register powercap control_type.\n");
 		return PTR_ERR(control_type);
 	}
-	/* read the initial data */
-	rapl_update_domain_data();
+
 	list_for_each_entry(rp, &rapl_packages, plist)
 		if (rapl_package_register_powercap(rp))
 			goto err_cleanup_package;
-- 
2.10.2

[tip:smp/hotplug] powercap/intel_rapl: Add missing domain data update on hotplug

[tip-bot for Thomas Gleixner]

Commit-ID:  4a6162f9e1c140c7436818983065365168075b0c
Gitweb:     http://git.kernel.org/tip/4a6162f9e1c140c7436818983065365168075b0c
Author:     Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:29 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:06 +0100

powercap/intel_rapl: Add missing domain data update on hotplug

The domain data of packages is only updated at init time, but new packages
created by hotplug miss that treatment.

Add it there and remove the global update at init time, because it's now
obsolete.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: "Rafael J. Wysocki" <rjw@rjwysocki.net>
Cc: rt@linuxtronix.de
Cc: linux-pm@vger.kernel.org
Link: http://lkml.kernel.org/r/20161117183541.8588-9-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/powercap/intel_rapl.c | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c
index 243b233..44b12e2 100644
--- a/drivers/powercap/intel_rapl.c
+++ b/drivers/powercap/intel_rapl.c
@@ -1164,24 +1164,20 @@ static const struct x86_cpu_id rapl_ids[] __initconst = {
 };
 MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
 
-/* read once for all raw primitive data for all packages, domains */
-static void rapl_update_domain_data(void)
+/* Read once for all raw primitive data for domains */
+static void rapl_update_domain_data(struct rapl_package *rp)
 {
 	int dmn, prim;
 	u64 val;
-	struct rapl_package *rp;
 
-	list_for_each_entry(rp, &rapl_packages, plist) {
-		for (dmn = 0; dmn < rp->nr_domains; dmn++) {
-			pr_debug("update package %d domain %s data\n", rp->id,
-				rp->domains[dmn].name);
+	for (dmn = 0; dmn < rp->nr_domains; dmn++) {
+		pr_debug("update package %d domain %s data\n", rp->id,
+			 rp->domains[dmn].name);
 			/* exclude non-raw primitives */
-			for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++)
-				if (!rapl_read_data_raw(&rp->domains[dmn], prim,
-								rpi[prim].unit,
-								&val))
-					rp->domains[dmn].rdd.primitives[prim] =
-									val;
+		for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
+			if (!rapl_read_data_raw(&rp->domains[dmn], prim,
+						rpi[prim].unit,	&val))
+				rp->domains[dmn].rdd.primitives[prim] = val;
 		}
 	}
 
@@ -1239,6 +1235,9 @@ static int rapl_package_register_powercap(struct rapl_package *rp)
 	struct powercap_zone *power_zone = NULL;
 	int nr_pl;
 
+	/* Update the domain data of the new package */
+	rapl_update_domain_data(rp);
+
 	/* first we register package domain as the parent zone*/
 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
 		if (rd->id == RAPL_DOMAIN_PACKAGE) {
@@ -1357,8 +1356,7 @@ static int rapl_register_powercap(void)
 		pr_debug("failed to register powercap control_type.\n");
 		return PTR_ERR(control_type);
 	}
-	/* read the initial data */
-	rapl_update_domain_data();
+
 	list_for_each_entry(rp, &rapl_packages, plist)
 		if (rapl_package_register_powercap(rp))
 			goto err_cleanup_package;

[PATCH 09/20] powercap/intel rapl: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine as a first step. The init/exit
code is a duplicate of the hotplug code. This is cleaned up in a
consecutive patch.

Cc: "Rafael J. Wysocki" <rjw@rjwysocki.net>
Cc: linux-pm@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 drivers/powercap/intel_rapl.c | 94 ++++++++++++++++++++++---------------------
 1 file changed, 49 insertions(+), 45 deletions(-)

diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c
index 44b12e2e4aa7..a42dd3b7e8a1 100644
--- a/drivers/powercap/intel_rapl.c
+++ b/drivers/powercap/intel_rapl.c
@@ -1607,50 +1607,48 @@ static int rapl_add_package(int cpu)
  * associated domains. Cooling devices are handled accordingly at
  * per-domain level.
  */
-static int rapl_cpu_callback(struct notifier_block *nfb,
-				unsigned long action, void *hcpu)
+static int rapl_cpu_online(unsigned int cpu)
+{
+	int phy_package_id;
+	struct rapl_package *rp;
+
+	phy_package_id = topology_physical_package_id(cpu);
+
+	rp = find_package_by_id(phy_package_id);
+	if (rp)
+		++rp->nr_cpus;
+	else
+		rapl_add_package(cpu);
+	return 0;
+}
+
+static int rapl_cpu_down_prep(unsigned int cpu)
 {
-	unsigned long cpu = (unsigned long)hcpu;
 	int phy_package_id;
 	struct rapl_package *rp;
 	int lead_cpu;
 
 	phy_package_id = topology_physical_package_id(cpu);
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-		rp = find_package_by_id(phy_package_id);
-		if (rp)
-			++rp->nr_cpus;
-		else
-			rapl_add_package(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		rp = find_package_by_id(phy_package_id);
-		if (!rp)
-			break;
-		if (--rp->nr_cpus == 0)
-			rapl_remove_package(rp);
-		else if (cpu == rp->lead_cpu) {
-			/* choose another active cpu in the package */
-			lead_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu);
-			if (lead_cpu < nr_cpu_ids)
-				rp->lead_cpu = lead_cpu;
-			else /* should never go here */
-				pr_err("no active cpu available for package %d\n",
-					phy_package_id);
+	rp = find_package_by_id(phy_package_id);
+	if (!rp)
+		return 0;
+	if (--rp->nr_cpus == 0) {
+		rapl_remove_package(rp);
+	} else if (cpu == rp->lead_cpu) {
+		/* choose another active cpu in the package */
+		lead_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu);
+		if (lead_cpu < nr_cpu_ids) {
+			rp->lead_cpu = lead_cpu;
+		} else {
+			/* should never go here */
+			pr_err("no active cpu available for package %d\n",
+			       phy_package_id);
 		}
 	}
-
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block rapl_cpu_notifier = {
-	.notifier_call = rapl_cpu_callback,
-};
+static enum cpuhp_state pcap_rapl_online;
 
 static int __init rapl_init(void)
 {
@@ -1667,36 +1665,42 @@ static int __init rapl_init(void)
 
 	rapl_defaults = (struct rapl_defaults *)id->driver_data;
 
-	cpu_notifier_register_begin();
-
 	/* prevent CPU hotplug during detection */
 	get_online_cpus();
 	ret = rapl_detect_topology();
 	if (ret)
-		goto done;
+		goto err;
 
 	if (rapl_register_powercap()) {
-		rapl_cleanup_data();
 		ret = -ENODEV;
-		goto done;
+		goto err_cleanup;
 	}
-	__register_hotcpu_notifier(&rapl_cpu_notifier);
-done:
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"powercap/rapl:online",
+					rapl_cpu_online, rapl_cpu_down_prep);
+	if (ret < 0)
+		goto err_unreg;
+	pcap_rapl_online = ret;
 	put_online_cpus();
-	cpu_notifier_register_done();
+	return 0;
 
+err_unreg:
+	rapl_unregister_powercap();
+
+err_cleanup:
+	rapl_cleanup_data();
+err:
+	put_online_cpus();
 	return ret;
 }
 
 static void __exit rapl_exit(void)
 {
-	cpu_notifier_register_begin();
 	get_online_cpus();
-	__unregister_hotcpu_notifier(&rapl_cpu_notifier);
+	cpuhp_remove_state(pcap_rapl_online);
 	rapl_unregister_powercap();
 	rapl_cleanup_data();
 	put_online_cpus();
-	cpu_notifier_register_done();
 }
 
 module_init(rapl_init);
-- 
2.10.2

[tip:smp/hotplug] powercap/intel rapl: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  29d9013958a6a25f9810a00b0d4ce72d601682ad
Gitweb:     http://git.kernel.org/tip/29d9013958a6a25f9810a00b0d4ce72d601682ad
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:30 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:06 +0100

powercap/intel rapl: Convert to hotplug state machine

Install the callbacks via the state machine as a first step. The init/exit
code is a duplicate of the hotplug code. This is cleaned up in a
consecutive patch.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Cc: "Rafael J. Wysocki" <rjw@rjwysocki.net>
Cc: linux-pm@vger.kernel.org
Link: http://lkml.kernel.org/r/20161117183541.8588-10-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/powercap/intel_rapl.c | 94 ++++++++++++++++++++++---------------------
 1 file changed, 49 insertions(+), 45 deletions(-)

diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c
index 44b12e2..a42dd3b 100644
--- a/drivers/powercap/intel_rapl.c
+++ b/drivers/powercap/intel_rapl.c
@@ -1607,50 +1607,48 @@ err_free_package:
  * associated domains. Cooling devices are handled accordingly at
  * per-domain level.
  */
-static int rapl_cpu_callback(struct notifier_block *nfb,
-				unsigned long action, void *hcpu)
+static int rapl_cpu_online(unsigned int cpu)
+{
+	int phy_package_id;
+	struct rapl_package *rp;
+
+	phy_package_id = topology_physical_package_id(cpu);
+
+	rp = find_package_by_id(phy_package_id);
+	if (rp)
+		++rp->nr_cpus;
+	else
+		rapl_add_package(cpu);
+	return 0;
+}
+
+static int rapl_cpu_down_prep(unsigned int cpu)
 {
-	unsigned long cpu = (unsigned long)hcpu;
 	int phy_package_id;
 	struct rapl_package *rp;
 	int lead_cpu;
 
 	phy_package_id = topology_physical_package_id(cpu);
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-		rp = find_package_by_id(phy_package_id);
-		if (rp)
-			++rp->nr_cpus;
-		else
-			rapl_add_package(cpu);
-		break;
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		rp = find_package_by_id(phy_package_id);
-		if (!rp)
-			break;
-		if (--rp->nr_cpus == 0)
-			rapl_remove_package(rp);
-		else if (cpu == rp->lead_cpu) {
-			/* choose another active cpu in the package */
-			lead_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu);
-			if (lead_cpu < nr_cpu_ids)
-				rp->lead_cpu = lead_cpu;
-			else /* should never go here */
-				pr_err("no active cpu available for package %d\n",
-					phy_package_id);
+	rp = find_package_by_id(phy_package_id);
+	if (!rp)
+		return 0;
+	if (--rp->nr_cpus == 0) {
+		rapl_remove_package(rp);
+	} else if (cpu == rp->lead_cpu) {
+		/* choose another active cpu in the package */
+		lead_cpu = cpumask_any_but(topology_core_cpumask(cpu), cpu);
+		if (lead_cpu < nr_cpu_ids) {
+			rp->lead_cpu = lead_cpu;
+		} else {
+			/* should never go here */
+			pr_err("no active cpu available for package %d\n",
+			       phy_package_id);
 		}
 	}
-
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block rapl_cpu_notifier = {
-	.notifier_call = rapl_cpu_callback,
-};
+static enum cpuhp_state pcap_rapl_online;
 
 static int __init rapl_init(void)
 {
@@ -1667,36 +1665,42 @@ static int __init rapl_init(void)
 
 	rapl_defaults = (struct rapl_defaults *)id->driver_data;
 
-	cpu_notifier_register_begin();
-
 	/* prevent CPU hotplug during detection */
 	get_online_cpus();
 	ret = rapl_detect_topology();
 	if (ret)
-		goto done;
+		goto err;
 
 	if (rapl_register_powercap()) {
-		rapl_cleanup_data();
 		ret = -ENODEV;
-		goto done;
+		goto err_cleanup;
 	}
-	__register_hotcpu_notifier(&rapl_cpu_notifier);
-done:
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"powercap/rapl:online",
+					rapl_cpu_online, rapl_cpu_down_prep);
+	if (ret < 0)
+		goto err_unreg;
+	pcap_rapl_online = ret;
 	put_online_cpus();
-	cpu_notifier_register_done();
+	return 0;
+
+err_unreg:
+	rapl_unregister_powercap();
 
+err_cleanup:
+	rapl_cleanup_data();
+err:
+	put_online_cpus();
 	return ret;
 }
 
 static void __exit rapl_exit(void)
 {
-	cpu_notifier_register_begin();
 	get_online_cpus();
-	__unregister_hotcpu_notifier(&rapl_cpu_notifier);
+	cpuhp_remove_state(pcap_rapl_online);
 	rapl_unregister_powercap();
 	rapl_cleanup_data();
 	put_online_cpus();
-	cpu_notifier_register_done();
 }
 
 module_init(rapl_init);

[PATCH 10/20] powercap/intel_rapl: Cleanup duplicated init code

[Sebastian Andrzej Siewior]

From: Thomas Gleixner <tglx@linutronix.de>

The whole init/exit code is a duplicate of the cpuhotplug code. So we can
just let the hotplug code do the actual work of setting up and tearing down
the domains.

Cc: "Rafael J. Wysocki" <rjw@rjwysocki.net>
Cc: linux-pm@vger.kernel.org
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 drivers/powercap/intel_rapl.c | 173 +++++-------------------------------------
 1 file changed, 20 insertions(+), 153 deletions(-)

diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c
index a42dd3b7e8a1..6c2e415c55c1 100644
--- a/drivers/powercap/intel_rapl.c
+++ b/drivers/powercap/intel_rapl.c
@@ -275,18 +275,6 @@ static struct rapl_package *find_package_by_id(int id)
 	return NULL;
 }
 
-/* caller must hold cpu hotplug lock */
-static void rapl_cleanup_data(void)
-{
-	struct rapl_package *p, *tmp;
-
-	list_for_each_entry_safe(p, tmp, &rapl_packages, plist) {
-		kfree(p->domains);
-		list_del(&p->plist);
-		kfree(p);
-	}
-}
-
 static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw)
 {
 	struct rapl_domain *rd;
@@ -1173,58 +1161,24 @@ static void rapl_update_domain_data(struct rapl_package *rp)
 	for (dmn = 0; dmn < rp->nr_domains; dmn++) {
 		pr_debug("update package %d domain %s data\n", rp->id,
 			 rp->domains[dmn].name);
-			/* exclude non-raw primitives */
+		/* exclude non-raw primitives */
 		for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
 			if (!rapl_read_data_raw(&rp->domains[dmn], prim,
-						rpi[prim].unit,	&val))
-				rp->domains[dmn].rdd.primitives[prim] = val;
+						rpi[prim].unit, &val))
+				rp->domains[dmn].rdd.primitives[prim] =	val;
 		}
 	}
 
 }
 
-static int rapl_unregister_powercap(void)
+static void rapl_unregister_powercap(void)
 {
-	struct rapl_package *rp;
-	struct rapl_domain *rd, *rd_package = NULL;
-
-	/* unregister all active rapl packages from the powercap layer,
-	 * hotplug lock held
-	 */
-	list_for_each_entry(rp, &rapl_packages, plist) {
-		package_power_limit_irq_restore(rp);
-
-		for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
-		     rd++) {
-			pr_debug("remove package, undo power limit on %d: %s\n",
-				rp->id, rd->name);
-			rapl_write_data_raw(rd, PL1_ENABLE, 0);
-			rapl_write_data_raw(rd, PL1_CLAMP, 0);
-			if (find_nr_power_limit(rd) > 1) {
-				rapl_write_data_raw(rd, PL2_ENABLE, 0);
-				rapl_write_data_raw(rd, PL2_CLAMP, 0);
-			}
-			if (rd->id == RAPL_DOMAIN_PACKAGE) {
-				rd_package = rd;
-				continue;
-			}
-			powercap_unregister_zone(control_type, &rd->power_zone);
-		}
-		/* do the package zone last */
-		if (rd_package)
-			powercap_unregister_zone(control_type,
-						&rd_package->power_zone);
-	}
-
 	if (platform_rapl_domain) {
 		powercap_unregister_zone(control_type,
 					 &platform_rapl_domain->power_zone);
 		kfree(platform_rapl_domain);
 	}
-
 	powercap_unregister_control_type(control_type);
-
-	return 0;
 }
 
 static int rapl_package_register_powercap(struct rapl_package *rp)
@@ -1347,37 +1301,16 @@ static int rapl_register_psys(void)
 
 static int rapl_register_powercap(void)
 {
-	struct rapl_domain *rd;
-	struct rapl_package *rp;
-	int ret = 0;
-
 	control_type = powercap_register_control_type(NULL, "intel-rapl", NULL);
 	if (IS_ERR(control_type)) {
 		pr_debug("failed to register powercap control_type.\n");
 		return PTR_ERR(control_type);
 	}
 
-	list_for_each_entry(rp, &rapl_packages, plist)
-		if (rapl_package_register_powercap(rp))
-			goto err_cleanup_package;
-
 	/* Don't bail out if PSys is not supported */
 	rapl_register_psys();
 
-	return ret;
-
-err_cleanup_package:
-	/* clean up previously initialized packages */
-	list_for_each_entry_continue_reverse(rp, &rapl_packages, plist) {
-		for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
-		     rd++) {
-			pr_debug("unregister zone/package %d, %s domain\n",
-				rp->id, rd->name);
-			powercap_unregister_zone(control_type, &rd->power_zone);
-		}
-	}
-
-	return ret;
+	return 0;
 }
 
 static int rapl_check_domain(int cpu, int domain)
@@ -1486,76 +1419,26 @@ static int rapl_detect_domains(struct rapl_package *rp, int cpu)
 	return ret;
 }
 
-static bool is_package_new(int package)
-{
-	struct rapl_package *rp;
-
-	/* caller prevents cpu hotplug, there will be no new packages added
-	 * or deleted while traversing the package list, no need for locking.
-	 */
-	list_for_each_entry(rp, &rapl_packages, plist)
-		if (package == rp->id)
-			return false;
-
-	return true;
-}
-
-/* RAPL interface can be made of a two-level hierarchy: package level and domain
- * level. We first detect the number of packages then domains of each package.
- * We have to consider the possiblity of CPU online/offline due to hotplug and
- * other scenarios.
- */
-static int rapl_detect_topology(void)
-{
-	int i;
-	int phy_package_id;
-	struct rapl_package *new_package, *rp;
-
-	for_each_online_cpu(i) {
-		phy_package_id = topology_physical_package_id(i);
-		if (is_package_new(phy_package_id)) {
-			new_package = kzalloc(sizeof(*rp), GFP_KERNEL);
-			if (!new_package) {
-				rapl_cleanup_data();
-				return -ENOMEM;
-			}
-			/* add the new package to the list */
-			new_package->id = phy_package_id;
-			new_package->nr_cpus = 1;
-			/* use the first active cpu of the package to access */
-			new_package->lead_cpu = i;
-			/* check if the package contains valid domains */
-			if (rapl_detect_domains(new_package, i) ||
-				rapl_defaults->check_unit(new_package, i)) {
-				kfree(new_package->domains);
-				kfree(new_package);
-				/* free up the packages already initialized */
-				rapl_cleanup_data();
-				return -ENODEV;
-			}
-			INIT_LIST_HEAD(&new_package->plist);
-			list_add(&new_package->plist, &rapl_packages);
-		} else {
-			rp = find_package_by_id(phy_package_id);
-			if (rp)
-				++rp->nr_cpus;
-		}
-	}
-
-	return 0;
-}
-
 /* called from CPU hotplug notifier, hotplug lock held */
 static void rapl_remove_package(struct rapl_package *rp)
 {
 	struct rapl_domain *rd, *rd_package = NULL;
 
 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
+		package_power_limit_irq_restore(rp);
+
+		pr_debug("remove package, undo power limit on %d: %s\n",
+			 rp->id, rd->name);
+		rapl_write_data_raw(rd, PL1_ENABLE, 0);
+		rapl_write_data_raw(rd, PL1_CLAMP, 0);
+		if (find_nr_power_limit(rd) > 1) {
+			rapl_write_data_raw(rd, PL2_ENABLE, 0);
+			rapl_write_data_raw(rd, PL2_CLAMP, 0);
+		}
 		if (rd->id == RAPL_DOMAIN_PACKAGE) {
 			rd_package = rd;
 			continue;
 		}
-		pr_debug("remove package %d, %s domain\n", rp->id, rd->name);
 		powercap_unregister_zone(control_type, &rd->power_zone);
 	}
 	/* do parent zone last */
@@ -1652,8 +1535,8 @@ static enum cpuhp_state pcap_rapl_online;
 
 static int __init rapl_init(void)
 {
-	int ret = 0;
 	const struct x86_cpu_id *id;
+	int ret;
 
 	id = x86_match_cpu(rapl_ids);
 	if (!id) {
@@ -1665,42 +1548,26 @@ static int __init rapl_init(void)
 
 	rapl_defaults = (struct rapl_defaults *)id->driver_data;
 
-	/* prevent CPU hotplug during detection */
-	get_online_cpus();
-	ret = rapl_detect_topology();
+	ret = rapl_register_powercap();
 	if (ret)
-		goto err;
+		return ret;
 
-	if (rapl_register_powercap()) {
-		ret = -ENODEV;
-		goto err_cleanup;
-	}
-	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
-					"powercap/rapl:online",
-					rapl_cpu_online, rapl_cpu_down_prep);
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online",
+				rapl_cpu_online, rapl_cpu_down_prep);
 	if (ret < 0)
 		goto err_unreg;
 	pcap_rapl_online = ret;
-	put_online_cpus();
 	return 0;
 
 err_unreg:
 	rapl_unregister_powercap();
-
-err_cleanup:
-	rapl_cleanup_data();
-err:
-	put_online_cpus();
 	return ret;
 }
 
 static void __exit rapl_exit(void)
 {
-	get_online_cpus();
 	cpuhp_remove_state(pcap_rapl_online);
 	rapl_unregister_powercap();
-	rapl_cleanup_data();
-	put_online_cpus();
 }
 
 module_init(rapl_init);
-- 
2.10.2

[tip:smp/hotplug] powercap/intel_rapl: Cleanup duplicated init code

[tip-bot for Thomas Gleixner]

Commit-ID:  7998a592159a2e95db0aac78bfe9594097832e53
Gitweb:     http://git.kernel.org/tip/7998a592159a2e95db0aac78bfe9594097832e53
Author:     Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:31 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:06 +0100

powercap/intel_rapl: Cleanup duplicated init code

The whole init/exit code is a duplicate of the cpuhotplug code. So we can
just let the hotplug code do the actual work of setting up and tearing down
the domains.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: "Rafael J. Wysocki" <rjw@rjwysocki.net>
Cc: rt@linuxtronix.de
Cc: linux-pm@vger.kernel.org
Link: http://lkml.kernel.org/r/20161117183541.8588-11-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/powercap/intel_rapl.c | 173 +++++-------------------------------------
 1 file changed, 20 insertions(+), 153 deletions(-)

diff --git a/drivers/powercap/intel_rapl.c b/drivers/powercap/intel_rapl.c
index a42dd3b..6c2e415 100644
--- a/drivers/powercap/intel_rapl.c
+++ b/drivers/powercap/intel_rapl.c
@@ -275,18 +275,6 @@ static struct rapl_package *find_package_by_id(int id)
 	return NULL;
 }
 
-/* caller must hold cpu hotplug lock */
-static void rapl_cleanup_data(void)
-{
-	struct rapl_package *p, *tmp;
-
-	list_for_each_entry_safe(p, tmp, &rapl_packages, plist) {
-		kfree(p->domains);
-		list_del(&p->plist);
-		kfree(p);
-	}
-}
-
 static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw)
 {
 	struct rapl_domain *rd;
@@ -1173,58 +1161,24 @@ static void rapl_update_domain_data(struct rapl_package *rp)
 	for (dmn = 0; dmn < rp->nr_domains; dmn++) {
 		pr_debug("update package %d domain %s data\n", rp->id,
 			 rp->domains[dmn].name);
-			/* exclude non-raw primitives */
+		/* exclude non-raw primitives */
 		for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++) {
 			if (!rapl_read_data_raw(&rp->domains[dmn], prim,
-						rpi[prim].unit,	&val))
-				rp->domains[dmn].rdd.primitives[prim] = val;
+						rpi[prim].unit, &val))
+				rp->domains[dmn].rdd.primitives[prim] =	val;
 		}
 	}
 
 }
 
-static int rapl_unregister_powercap(void)
+static void rapl_unregister_powercap(void)
 {
-	struct rapl_package *rp;
-	struct rapl_domain *rd, *rd_package = NULL;
-
-	/* unregister all active rapl packages from the powercap layer,
-	 * hotplug lock held
-	 */
-	list_for_each_entry(rp, &rapl_packages, plist) {
-		package_power_limit_irq_restore(rp);
-
-		for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
-		     rd++) {
-			pr_debug("remove package, undo power limit on %d: %s\n",
-				rp->id, rd->name);
-			rapl_write_data_raw(rd, PL1_ENABLE, 0);
-			rapl_write_data_raw(rd, PL1_CLAMP, 0);
-			if (find_nr_power_limit(rd) > 1) {
-				rapl_write_data_raw(rd, PL2_ENABLE, 0);
-				rapl_write_data_raw(rd, PL2_CLAMP, 0);
-			}
-			if (rd->id == RAPL_DOMAIN_PACKAGE) {
-				rd_package = rd;
-				continue;
-			}
-			powercap_unregister_zone(control_type, &rd->power_zone);
-		}
-		/* do the package zone last */
-		if (rd_package)
-			powercap_unregister_zone(control_type,
-						&rd_package->power_zone);
-	}
-
 	if (platform_rapl_domain) {
 		powercap_unregister_zone(control_type,
 					 &platform_rapl_domain->power_zone);
 		kfree(platform_rapl_domain);
 	}
-
 	powercap_unregister_control_type(control_type);
-
-	return 0;
 }
 
 static int rapl_package_register_powercap(struct rapl_package *rp)
@@ -1347,37 +1301,16 @@ static int rapl_register_psys(void)
 
 static int rapl_register_powercap(void)
 {
-	struct rapl_domain *rd;
-	struct rapl_package *rp;
-	int ret = 0;
-
 	control_type = powercap_register_control_type(NULL, "intel-rapl", NULL);
 	if (IS_ERR(control_type)) {
 		pr_debug("failed to register powercap control_type.\n");
 		return PTR_ERR(control_type);
 	}
 
-	list_for_each_entry(rp, &rapl_packages, plist)
-		if (rapl_package_register_powercap(rp))
-			goto err_cleanup_package;
-
 	/* Don't bail out if PSys is not supported */
 	rapl_register_psys();
 
-	return ret;
-
-err_cleanup_package:
-	/* clean up previously initialized packages */
-	list_for_each_entry_continue_reverse(rp, &rapl_packages, plist) {
-		for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
-		     rd++) {
-			pr_debug("unregister zone/package %d, %s domain\n",
-				rp->id, rd->name);
-			powercap_unregister_zone(control_type, &rd->power_zone);
-		}
-	}
-
-	return ret;
+	return 0;
 }
 
 static int rapl_check_domain(int cpu, int domain)
@@ -1486,76 +1419,26 @@ done:
 	return ret;
 }
 
-static bool is_package_new(int package)
-{
-	struct rapl_package *rp;
-
-	/* caller prevents cpu hotplug, there will be no new packages added
-	 * or deleted while traversing the package list, no need for locking.
-	 */
-	list_for_each_entry(rp, &rapl_packages, plist)
-		if (package == rp->id)
-			return false;
-
-	return true;
-}
-
-/* RAPL interface can be made of a two-level hierarchy: package level and domain
- * level. We first detect the number of packages then domains of each package.
- * We have to consider the possiblity of CPU online/offline due to hotplug and
- * other scenarios.
- */
-static int rapl_detect_topology(void)
-{
-	int i;
-	int phy_package_id;
-	struct rapl_package *new_package, *rp;
-
-	for_each_online_cpu(i) {
-		phy_package_id = topology_physical_package_id(i);
-		if (is_package_new(phy_package_id)) {
-			new_package = kzalloc(sizeof(*rp), GFP_KERNEL);
-			if (!new_package) {
-				rapl_cleanup_data();
-				return -ENOMEM;
-			}
-			/* add the new package to the list */
-			new_package->id = phy_package_id;
-			new_package->nr_cpus = 1;
-			/* use the first active cpu of the package to access */
-			new_package->lead_cpu = i;
-			/* check if the package contains valid domains */
-			if (rapl_detect_domains(new_package, i) ||
-				rapl_defaults->check_unit(new_package, i)) {
-				kfree(new_package->domains);
-				kfree(new_package);
-				/* free up the packages already initialized */
-				rapl_cleanup_data();
-				return -ENODEV;
-			}
-			INIT_LIST_HEAD(&new_package->plist);
-			list_add(&new_package->plist, &rapl_packages);
-		} else {
-			rp = find_package_by_id(phy_package_id);
-			if (rp)
-				++rp->nr_cpus;
-		}
-	}
-
-	return 0;
-}
-
 /* called from CPU hotplug notifier, hotplug lock held */
 static void rapl_remove_package(struct rapl_package *rp)
 {
 	struct rapl_domain *rd, *rd_package = NULL;
 
 	for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
+		package_power_limit_irq_restore(rp);
+
+		pr_debug("remove package, undo power limit on %d: %s\n",
+			 rp->id, rd->name);
+		rapl_write_data_raw(rd, PL1_ENABLE, 0);
+		rapl_write_data_raw(rd, PL1_CLAMP, 0);
+		if (find_nr_power_limit(rd) > 1) {
+			rapl_write_data_raw(rd, PL2_ENABLE, 0);
+			rapl_write_data_raw(rd, PL2_CLAMP, 0);
+		}
 		if (rd->id == RAPL_DOMAIN_PACKAGE) {
 			rd_package = rd;
 			continue;
 		}
-		pr_debug("remove package %d, %s domain\n", rp->id, rd->name);
 		powercap_unregister_zone(control_type, &rd->power_zone);
 	}
 	/* do parent zone last */
@@ -1652,8 +1535,8 @@ static enum cpuhp_state pcap_rapl_online;
 
 static int __init rapl_init(void)
 {
-	int ret = 0;
 	const struct x86_cpu_id *id;
+	int ret;
 
 	id = x86_match_cpu(rapl_ids);
 	if (!id) {
@@ -1665,42 +1548,26 @@ static int __init rapl_init(void)
 
 	rapl_defaults = (struct rapl_defaults *)id->driver_data;
 
-	/* prevent CPU hotplug during detection */
-	get_online_cpus();
-	ret = rapl_detect_topology();
+	ret = rapl_register_powercap();
 	if (ret)
-		goto err;
+		return ret;
 
-	if (rapl_register_powercap()) {
-		ret = -ENODEV;
-		goto err_cleanup;
-	}
-	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
-					"powercap/rapl:online",
-					rapl_cpu_online, rapl_cpu_down_prep);
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powercap/rapl:online",
+				rapl_cpu_online, rapl_cpu_down_prep);
 	if (ret < 0)
 		goto err_unreg;
 	pcap_rapl_online = ret;
-	put_online_cpus();
 	return 0;
 
 err_unreg:
 	rapl_unregister_powercap();
-
-err_cleanup:
-	rapl_cleanup_data();
-err:
-	put_online_cpus();
 	return ret;
 }
 
 static void __exit rapl_exit(void)
 {
-	get_online_cpus();
 	cpuhp_remove_state(pcap_rapl_online);
 	rapl_unregister_powercap();
-	rapl_cleanup_data();
-	put_online_cpus();
 }
 
 module_init(rapl_init);

[PATCH 11/20] watchdog/octeon: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: Wim Van Sebroeck <wim@iguana.be>
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: linux-watchdog@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 drivers/watchdog/octeon-wdt-main.c | 62 +++++++++-----------------------------
 1 file changed, 15 insertions(+), 47 deletions(-)

diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c
index b55981f88a08..529182d7d8a7 100644
--- a/drivers/watchdog/octeon-wdt-main.c
+++ b/drivers/watchdog/octeon-wdt-main.c
@@ -374,7 +374,7 @@ void octeon_wdt_nmi_stage3(u64 reg[32])
 	octeon_wdt_write_string("*** Chip soft reset soon ***\r\n");
 }
 
-static void octeon_wdt_disable_interrupt(int cpu)
+static int octeon_wdt_cpu_pre_down(unsigned int cpu)
 {
 	unsigned int core;
 	unsigned int irq;
@@ -392,9 +392,10 @@ static void octeon_wdt_disable_interrupt(int cpu)
 	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
 
 	free_irq(irq, octeon_wdt_poke_irq);
+	return 0;
 }
 
-static void octeon_wdt_setup_interrupt(int cpu)
+static int octeon_wdt_cpu_online(unsigned int cpu)
 {
 	unsigned int core;
 	unsigned int irq;
@@ -424,25 +425,8 @@ static void octeon_wdt_setup_interrupt(int cpu)
 	ciu_wdog.s.len = timeout_cnt;
 	ciu_wdog.s.mode = 3;	/* 3 = Interrupt + NMI + Soft-Reset */
 	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
-}
 
-static int octeon_wdt_cpu_callback(struct notifier_block *nfb,
-					   unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		octeon_wdt_disable_interrupt(cpu);
-		break;
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		octeon_wdt_setup_interrupt(cpu);
-		break;
-	default:
-		break;
-	}
-	return NOTIFY_OK;
+	return 0;
 }
 
 static int octeon_wdt_ping(struct watchdog_device __always_unused *wdog)
@@ -531,10 +515,6 @@ static int octeon_wdt_stop(struct watchdog_device *wdog)
 	return 0;
 }
 
-static struct notifier_block octeon_wdt_cpu_notifier = {
-	.notifier_call = octeon_wdt_cpu_callback,
-};
-
 static const struct watchdog_info octeon_wdt_info = {
 	.options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING,
 	.identity = "OCTEON",
@@ -553,6 +533,7 @@ static struct watchdog_device octeon_wdt = {
 	.ops	= &octeon_wdt_ops,
 };
 
+static enum cpuhp_state octeon_wdt_online;
 /**
  * Module/ driver initialization.
  *
@@ -562,7 +543,6 @@ static int __init octeon_wdt_init(void)
 {
 	int i;
 	int ret;
-	int cpu;
 	u64 *ptr;
 
 	/*
@@ -610,14 +590,16 @@ static int __init octeon_wdt_init(void)
 
 	cpumask_clear(&irq_enabled_cpus);
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		octeon_wdt_setup_interrupt(cpu);
-
-	__register_hotcpu_notifier(&octeon_wdt_cpu_notifier);
-	cpu_notifier_register_done();
-
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "watchdog/octeon:online",
+				octeon_wdt_cpu_online, octeon_wdt_cpu_pre_down);
+	if (ret < 0)
+		goto err;
+	octeon_wdt_online = ret;
 	return 0;
+err:
+	cvmx_write_csr(CVMX_MIO_BOOT_LOC_CFGX(0), 0);
+	watchdog_unregister_device(&octeon_wdt);
+	return ret;
 }
 
 /**
@@ -625,22 +607,8 @@ static int __init octeon_wdt_init(void)
  */
 static void __exit octeon_wdt_cleanup(void)
 {
-	int cpu;
-
 	watchdog_unregister_device(&octeon_wdt);
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&octeon_wdt_cpu_notifier);
-
-	for_each_online_cpu(cpu) {
-		int core = cpu2core(cpu);
-		/* Disable the watchdog */
-		cvmx_write_csr(CVMX_CIU_WDOGX(core), 0);
-		/* Free the interrupt handler */
-		free_irq(OCTEON_IRQ_WDOG0 + core, octeon_wdt_poke_irq);
-	}
-
-	cpu_notifier_register_done();
+	cpuhp_remove_state(octeon_wdt_online);
 
 	/*
 	 * Disable the boot-bus memory, the code it points to is soon
-- 
2.10.2

[tip:smp/hotplug] watchdog/octeon: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  883fa6da66effbada2b1c26ebb752166083e50bf
Gitweb:     http://git.kernel.org/tip/883fa6da66effbada2b1c26ebb752166083e50bf
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:32 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:07 +0100

watchdog/octeon: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Wim Van Sebroeck <wim@iguana.be>
Cc: rt@linuxtronix.de
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: linux-watchdog@vger.kernel.org
Link: http://lkml.kernel.org/r/20161117183541.8588-12-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/watchdog/octeon-wdt-main.c | 62 +++++++++-----------------------------
 1 file changed, 15 insertions(+), 47 deletions(-)

diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c
index b55981f..529182d 100644
--- a/drivers/watchdog/octeon-wdt-main.c
+++ b/drivers/watchdog/octeon-wdt-main.c
@@ -374,7 +374,7 @@ void octeon_wdt_nmi_stage3(u64 reg[32])
 	octeon_wdt_write_string("*** Chip soft reset soon ***\r\n");
 }
 
-static void octeon_wdt_disable_interrupt(int cpu)
+static int octeon_wdt_cpu_pre_down(unsigned int cpu)
 {
 	unsigned int core;
 	unsigned int irq;
@@ -392,9 +392,10 @@ static void octeon_wdt_disable_interrupt(int cpu)
 	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
 
 	free_irq(irq, octeon_wdt_poke_irq);
+	return 0;
 }
 
-static void octeon_wdt_setup_interrupt(int cpu)
+static int octeon_wdt_cpu_online(unsigned int cpu)
 {
 	unsigned int core;
 	unsigned int irq;
@@ -424,25 +425,8 @@ static void octeon_wdt_setup_interrupt(int cpu)
 	ciu_wdog.s.len = timeout_cnt;
 	ciu_wdog.s.mode = 3;	/* 3 = Interrupt + NMI + Soft-Reset */
 	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
-}
 
-static int octeon_wdt_cpu_callback(struct notifier_block *nfb,
-					   unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		octeon_wdt_disable_interrupt(cpu);
-		break;
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		octeon_wdt_setup_interrupt(cpu);
-		break;
-	default:
-		break;
-	}
-	return NOTIFY_OK;
+	return 0;
 }
 
 static int octeon_wdt_ping(struct watchdog_device __always_unused *wdog)
@@ -531,10 +515,6 @@ static int octeon_wdt_stop(struct watchdog_device *wdog)
 	return 0;
 }
 
-static struct notifier_block octeon_wdt_cpu_notifier = {
-	.notifier_call = octeon_wdt_cpu_callback,
-};
-
 static const struct watchdog_info octeon_wdt_info = {
 	.options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING,
 	.identity = "OCTEON",
@@ -553,6 +533,7 @@ static struct watchdog_device octeon_wdt = {
 	.ops	= &octeon_wdt_ops,
 };
 
+static enum cpuhp_state octeon_wdt_online;
 /**
  * Module/ driver initialization.
  *
@@ -562,7 +543,6 @@ static int __init octeon_wdt_init(void)
 {
 	int i;
 	int ret;
-	int cpu;
 	u64 *ptr;
 
 	/*
@@ -610,14 +590,16 @@ static int __init octeon_wdt_init(void)
 
 	cpumask_clear(&irq_enabled_cpus);
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		octeon_wdt_setup_interrupt(cpu);
-
-	__register_hotcpu_notifier(&octeon_wdt_cpu_notifier);
-	cpu_notifier_register_done();
-
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "watchdog/octeon:online",
+				octeon_wdt_cpu_online, octeon_wdt_cpu_pre_down);
+	if (ret < 0)
+		goto err;
+	octeon_wdt_online = ret;
 	return 0;
+err:
+	cvmx_write_csr(CVMX_MIO_BOOT_LOC_CFGX(0), 0);
+	watchdog_unregister_device(&octeon_wdt);
+	return ret;
 }
 
 /**
@@ -625,22 +607,8 @@ static int __init octeon_wdt_init(void)
  */
 static void __exit octeon_wdt_cleanup(void)
 {
-	int cpu;
-
 	watchdog_unregister_device(&octeon_wdt);
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&octeon_wdt_cpu_notifier);
-
-	for_each_online_cpu(cpu) {
-		int core = cpu2core(cpu);
-		/* Disable the watchdog */
-		cvmx_write_csr(CVMX_CIU_WDOGX(core), 0);
-		/* Free the interrupt handler */
-		free_irq(OCTEON_IRQ_WDOG0 + core, octeon_wdt_poke_irq);
-	}
-
-	cpu_notifier_register_done();
+	cpuhp_remove_state(octeon_wdt_online);
 
 	/*
 	 * Disable the boot-bus memory, the code it points to is soon

[tip:smp/hotplug] watchdog/octeon: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  948b9c60cb3e044ed4a56fd170ef0d0617ab840f
Gitweb:     http://git.kernel.org/tip/948b9c60cb3e044ed4a56fd170ef0d0617ab840f
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:32 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:40 +0100

watchdog/octeon: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Wim Van Sebroeck <wim@iguana.be>
Cc: rt@linuxtronix.de
Cc: Guenter Roeck <linux@roeck-us.net>
Cc: linux-watchdog@vger.kernel.org
Link: http://lkml.kernel.org/r/20161117183541.8588-12-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/watchdog/octeon-wdt-main.c | 62 +++++++++-----------------------------
 1 file changed, 15 insertions(+), 47 deletions(-)

diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c
index b55981f..529182d 100644
--- a/drivers/watchdog/octeon-wdt-main.c
+++ b/drivers/watchdog/octeon-wdt-main.c
@@ -374,7 +374,7 @@ void octeon_wdt_nmi_stage3(u64 reg[32])
 	octeon_wdt_write_string("*** Chip soft reset soon ***\r\n");
 }
 
-static void octeon_wdt_disable_interrupt(int cpu)
+static int octeon_wdt_cpu_pre_down(unsigned int cpu)
 {
 	unsigned int core;
 	unsigned int irq;
@@ -392,9 +392,10 @@ static void octeon_wdt_disable_interrupt(int cpu)
 	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
 
 	free_irq(irq, octeon_wdt_poke_irq);
+	return 0;
 }
 
-static void octeon_wdt_setup_interrupt(int cpu)
+static int octeon_wdt_cpu_online(unsigned int cpu)
 {
 	unsigned int core;
 	unsigned int irq;
@@ -424,25 +425,8 @@ static void octeon_wdt_setup_interrupt(int cpu)
 	ciu_wdog.s.len = timeout_cnt;
 	ciu_wdog.s.mode = 3;	/* 3 = Interrupt + NMI + Soft-Reset */
 	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
-}
 
-static int octeon_wdt_cpu_callback(struct notifier_block *nfb,
-					   unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		octeon_wdt_disable_interrupt(cpu);
-		break;
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		octeon_wdt_setup_interrupt(cpu);
-		break;
-	default:
-		break;
-	}
-	return NOTIFY_OK;
+	return 0;
 }
 
 static int octeon_wdt_ping(struct watchdog_device __always_unused *wdog)
@@ -531,10 +515,6 @@ static int octeon_wdt_stop(struct watchdog_device *wdog)
 	return 0;
 }
 
-static struct notifier_block octeon_wdt_cpu_notifier = {
-	.notifier_call = octeon_wdt_cpu_callback,
-};
-
 static const struct watchdog_info octeon_wdt_info = {
 	.options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING,
 	.identity = "OCTEON",
@@ -553,6 +533,7 @@ static struct watchdog_device octeon_wdt = {
 	.ops	= &octeon_wdt_ops,
 };
 
+static enum cpuhp_state octeon_wdt_online;
 /**
  * Module/ driver initialization.
  *
@@ -562,7 +543,6 @@ static int __init octeon_wdt_init(void)
 {
 	int i;
 	int ret;
-	int cpu;
 	u64 *ptr;
 
 	/*
@@ -610,14 +590,16 @@ static int __init octeon_wdt_init(void)
 
 	cpumask_clear(&irq_enabled_cpus);
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		octeon_wdt_setup_interrupt(cpu);
-
-	__register_hotcpu_notifier(&octeon_wdt_cpu_notifier);
-	cpu_notifier_register_done();
-
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "watchdog/octeon:online",
+				octeon_wdt_cpu_online, octeon_wdt_cpu_pre_down);
+	if (ret < 0)
+		goto err;
+	octeon_wdt_online = ret;
 	return 0;
+err:
+	cvmx_write_csr(CVMX_MIO_BOOT_LOC_CFGX(0), 0);
+	watchdog_unregister_device(&octeon_wdt);
+	return ret;
 }
 
 /**
@@ -625,22 +607,8 @@ static int __init octeon_wdt_init(void)
  */
 static void __exit octeon_wdt_cleanup(void)
 {
-	int cpu;
-
 	watchdog_unregister_device(&octeon_wdt);
-
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&octeon_wdt_cpu_notifier);
-
-	for_each_online_cpu(cpu) {
-		int core = cpu2core(cpu);
-		/* Disable the watchdog */
-		cvmx_write_csr(CVMX_CIU_WDOGX(core), 0);
-		/* Free the interrupt handler */
-		free_irq(OCTEON_IRQ_WDOG0 + core, octeon_wdt_poke_irq);
-	}
-
-	cpu_notifier_register_done();
+	cpuhp_remove_state(octeon_wdt_online);
 
 	/*
 	 * Disable the boot-bus memory, the code it points to is soon

Re: [11/20] watchdog/octeon: Convert to hotplug state machine

[Guenter Roeck]

On Thu, Nov 17, 2016 at 07:35:32PM +0100, Sebastian Andrzej Siewior wrote:
> Install the callbacks via the state machine and let the core invoke
> the callbacks on the already online CPUs.
> 
> Cc: Wim Van Sebroeck <wim@iguana.be>
> Cc: Guenter Roeck <linux@roeck-us.net>
> Cc: linux-watchdog@vger.kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

I don't really feel comfortable signing this off without Tested-by:.

Can someone please confirm that this is working as intended ?

Thanks,
Guenter

> ---
>  drivers/watchdog/octeon-wdt-main.c | 62 +++++++++-----------------------------
>  1 file changed, 15 insertions(+), 47 deletions(-)
> 
> diff --git a/drivers/watchdog/octeon-wdt-main.c b/drivers/watchdog/octeon-wdt-main.c
> index b55981f88a08..529182d7d8a7 100644
> --- a/drivers/watchdog/octeon-wdt-main.c
> +++ b/drivers/watchdog/octeon-wdt-main.c
> @@ -374,7 +374,7 @@ void octeon_wdt_nmi_stage3(u64 reg[32])
>  	octeon_wdt_write_string("*** Chip soft reset soon ***\r\n");
>  }
>  
> -static void octeon_wdt_disable_interrupt(int cpu)
> +static int octeon_wdt_cpu_pre_down(unsigned int cpu)
>  {
>  	unsigned int core;
>  	unsigned int irq;
> @@ -392,9 +392,10 @@ static void octeon_wdt_disable_interrupt(int cpu)
>  	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
>  
>  	free_irq(irq, octeon_wdt_poke_irq);
> +	return 0;
>  }
>  
> -static void octeon_wdt_setup_interrupt(int cpu)
> +static int octeon_wdt_cpu_online(unsigned int cpu)
>  {
>  	unsigned int core;
>  	unsigned int irq;
> @@ -424,25 +425,8 @@ static void octeon_wdt_setup_interrupt(int cpu)
>  	ciu_wdog.s.len = timeout_cnt;
>  	ciu_wdog.s.mode = 3;	/* 3 = Interrupt + NMI + Soft-Reset */
>  	cvmx_write_csr(CVMX_CIU_WDOGX(core), ciu_wdog.u64);
> -}
>  
> -static int octeon_wdt_cpu_callback(struct notifier_block *nfb,
> -					   unsigned long action, void *hcpu)
> -{
> -	unsigned int cpu = (unsigned long)hcpu;
> -
> -	switch (action & ~CPU_TASKS_FROZEN) {
> -	case CPU_DOWN_PREPARE:
> -		octeon_wdt_disable_interrupt(cpu);
> -		break;
> -	case CPU_ONLINE:
> -	case CPU_DOWN_FAILED:
> -		octeon_wdt_setup_interrupt(cpu);
> -		break;
> -	default:
> -		break;
> -	}
> -	return NOTIFY_OK;
> +	return 0;
>  }
>  
>  static int octeon_wdt_ping(struct watchdog_device __always_unused *wdog)
> @@ -531,10 +515,6 @@ static int octeon_wdt_stop(struct watchdog_device *wdog)
>  	return 0;
>  }
>  
> -static struct notifier_block octeon_wdt_cpu_notifier = {
> -	.notifier_call = octeon_wdt_cpu_callback,
> -};
> -
>  static const struct watchdog_info octeon_wdt_info = {
>  	.options = WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE | WDIOF_KEEPALIVEPING,
>  	.identity = "OCTEON",
> @@ -553,6 +533,7 @@ static struct watchdog_device octeon_wdt = {
>  	.ops	= &octeon_wdt_ops,
>  };
>  
> +static enum cpuhp_state octeon_wdt_online;
>  /**
>   * Module/ driver initialization.
>   *
> @@ -562,7 +543,6 @@ static int __init octeon_wdt_init(void)
>  {
>  	int i;
>  	int ret;
> -	int cpu;
>  	u64 *ptr;
>  
>  	/*
> @@ -610,14 +590,16 @@ static int __init octeon_wdt_init(void)
>  
>  	cpumask_clear(&irq_enabled_cpus);
>  
> -	cpu_notifier_register_begin();
> -	for_each_online_cpu(cpu)
> -		octeon_wdt_setup_interrupt(cpu);
> -
> -	__register_hotcpu_notifier(&octeon_wdt_cpu_notifier);
> -	cpu_notifier_register_done();
> -
> +	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "watchdog/octeon:online",
> +				octeon_wdt_cpu_online, octeon_wdt_cpu_pre_down);
> +	if (ret < 0)
> +		goto err;
> +	octeon_wdt_online = ret;
>  	return 0;
> +err:
> +	cvmx_write_csr(CVMX_MIO_BOOT_LOC_CFGX(0), 0);
> +	watchdog_unregister_device(&octeon_wdt);
> +	return ret;
>  }
>  
>  /**
> @@ -625,22 +607,8 @@ static int __init octeon_wdt_init(void)
>   */
>  static void __exit octeon_wdt_cleanup(void)
>  {
> -	int cpu;
> -
>  	watchdog_unregister_device(&octeon_wdt);
> -
> -	cpu_notifier_register_begin();
> -	__unregister_hotcpu_notifier(&octeon_wdt_cpu_notifier);
> -
> -	for_each_online_cpu(cpu) {
> -		int core = cpu2core(cpu);
> -		/* Disable the watchdog */
> -		cvmx_write_csr(CVMX_CIU_WDOGX(core), 0);
> -		/* Free the interrupt handler */
> -		free_irq(OCTEON_IRQ_WDOG0 + core, octeon_wdt_poke_irq);
> -	}
> -
> -	cpu_notifier_register_done();
> +	cpuhp_remove_state(octeon_wdt_online);
>  
>  	/*
>  	 * Disable the boot-bus memory, the code it points to is soon

[PATCH 12/20] net/iucv: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke the
callbacks on the already online CPUs. The smp function calls in the
online/downprep callbacks are not required as the callback is guaranteed to
be invoked on the upcoming/outgoing cpu.

Cc: Ursula Braun <ubraun@linux.vnet.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: linux-s390@vger.kernel.org
Cc: netdev@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 include/linux/cpuhotplug.h |   1 +
 net/iucv/iucv.c            | 118 +++++++++++++++++----------------------------
 2 files changed, 45 insertions(+), 74 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index fd5598b8353a..69abf2c09f6c 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -63,6 +63,7 @@ enum cpuhp_state {
 	CPUHP_X86_THERM_PREPARE,
 	CPUHP_X86_CPUID_PREPARE,
 	CPUHP_X86_MSR_PREPARE,
+	CPUHP_NET_IUCV_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c
index 88a2a3ba4212..f0d6afc5d4a9 100644
--- a/net/iucv/iucv.c
+++ b/net/iucv/iucv.c
@@ -639,7 +639,7 @@ static void iucv_disable(void)
 	put_online_cpus();
 }
 
-static void free_iucv_data(int cpu)
+static int iucv_cpu_dead(unsigned int cpu)
 {
 	kfree(iucv_param_irq[cpu]);
 	iucv_param_irq[cpu] = NULL;
@@ -647,9 +647,10 @@ static void free_iucv_data(int cpu)
 	iucv_param[cpu] = NULL;
 	kfree(iucv_irq_data[cpu]);
 	iucv_irq_data[cpu] = NULL;
+	return 0;
 }
 
-static int alloc_iucv_data(int cpu)
+static int iucv_cpu_prepare(unsigned int cpu)
 {
 	/* Note: GFP_DMA used to get memory below 2G */
 	iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data),
@@ -671,58 +672,38 @@ static int alloc_iucv_data(int cpu)
 	return 0;
 
 out_free:
-	free_iucv_data(cpu);
+	iucv_cpu_dead(cpu);
 	return -ENOMEM;
 }
 
-static int iucv_cpu_notify(struct notifier_block *self,
-				     unsigned long action, void *hcpu)
+static int iucv_cpu_online(unsigned int cpu)
 {
-	cpumask_t cpumask;
-	long cpu = (long) hcpu;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		if (alloc_iucv_data(cpu))
-			return notifier_from_errno(-ENOMEM);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		free_iucv_data(cpu);
-		break;
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-		if (!iucv_path_table)
-			break;
-		smp_call_function_single(cpu, iucv_declare_cpu, NULL, 1);
-		break;
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		if (!iucv_path_table)
-			break;
-		cpumask_copy(&cpumask, &iucv_buffer_cpumask);
-		cpumask_clear_cpu(cpu, &cpumask);
-		if (cpumask_empty(&cpumask))
-			/* Can't offline last IUCV enabled cpu. */
-			return notifier_from_errno(-EINVAL);
-		smp_call_function_single(cpu, iucv_retrieve_cpu, NULL, 1);
-		if (cpumask_empty(&iucv_irq_cpumask))
-			smp_call_function_single(
-				cpumask_first(&iucv_buffer_cpumask),
-				iucv_allow_cpu, NULL, 1);
-		break;
-	}
-	return NOTIFY_OK;
+	if (!iucv_path_table)
+		return 0;
+	iucv_declare_cpu(NULL);
+	return 0;
 }
 
-static struct notifier_block __refdata iucv_cpu_notifier = {
-	.notifier_call = iucv_cpu_notify,
-};
+static int iucv_cpu_down_prep(unsigned int cpu)
+{
+	cpumask_t cpumask;
+
+	if (!iucv_path_table)
+		return 0;
+
+	cpumask_copy(&cpumask, &iucv_buffer_cpumask);
+	cpumask_clear_cpu(cpu, &cpumask);
+	if (cpumask_empty(&cpumask))
+		/* Can't offline last IUCV enabled cpu. */
+		return -EINVAL;
+
+	iucv_retrieve_cpu(NULL);
+	if (!cpumask_empty(&iucv_irq_cpumask))
+		return 0;
+	smp_call_function_single(cpumask_first(&iucv_buffer_cpumask),
+				 iucv_allow_cpu, NULL, 1);
+	return 0;
+}
 
 /**
  * iucv_sever_pathid
@@ -2027,6 +2008,7 @@ struct iucv_interface iucv_if = {
 };
 EXPORT_SYMBOL(iucv_if);
 
+static enum cpuhp_state iucv_online;
 /**
  * iucv_init
  *
@@ -2035,7 +2017,6 @@ EXPORT_SYMBOL(iucv_if);
 static int __init iucv_init(void)
 {
 	int rc;
-	int cpu;
 
 	if (!MACHINE_IS_VM) {
 		rc = -EPROTONOSUPPORT;
@@ -2054,23 +2035,19 @@ static int __init iucv_init(void)
 		goto out_int;
 	}
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu) {
-		if (alloc_iucv_data(cpu)) {
-			rc = -ENOMEM;
-			goto out_free;
-		}
-	}
-	rc = __register_hotcpu_notifier(&iucv_cpu_notifier);
+	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
+			       iucv_cpu_prepare, iucv_cpu_dead);
 	if (rc)
 		goto out_free;
-
-	cpu_notifier_register_done();
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
+			       iucv_cpu_online, iucv_cpu_down_prep);
+	if (rc < 0)
+		goto out_free;
+	iucv_online = rc;
 
 	rc = register_reboot_notifier(&iucv_reboot_notifier);
 	if (rc)
-		goto out_cpu;
+		goto out_free;
 	ASCEBC(iucv_error_no_listener, 16);
 	ASCEBC(iucv_error_no_memory, 16);
 	ASCEBC(iucv_error_pathid, 16);
@@ -2084,14 +2061,10 @@ static int __init iucv_init(void)
 
 out_reboot:
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-out_cpu:
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
 out_free:
-	for_each_possible_cpu(cpu)
-		free_iucv_data(cpu);
-
-	cpu_notifier_register_done();
+	if (iucv_online)
+		cpuhp_remove_state(iucv_online);
+	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
 
 	root_device_unregister(iucv_root);
 out_int:
@@ -2110,7 +2083,6 @@ static int __init iucv_init(void)
 static void __exit iucv_exit(void)
 {
 	struct iucv_irq_list *p, *n;
-	int cpu;
 
 	spin_lock_irq(&iucv_queue_lock);
 	list_for_each_entry_safe(p, n, &iucv_task_queue, list)
@@ -2119,11 +2091,9 @@ static void __exit iucv_exit(void)
 		kfree(p);
 	spin_unlock_irq(&iucv_queue_lock);
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
-	for_each_possible_cpu(cpu)
-		free_iucv_data(cpu);
-	cpu_notifier_register_done();
+
+	cpuhp_remove_state_nocalls(iucv_online);
+	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
 	root_device_unregister(iucv_root);
 	bus_unregister(&iucv_bus);
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);
-- 
2.10.2

[tip:smp/hotplug] net/iucv: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  bc30e3fc0d1476c933f3f4d88db1552de3515b44
Gitweb:     http://git.kernel.org/tip/bc30e3fc0d1476c933f3f4d88db1552de3515b44
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:33 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:07 +0100

net/iucv: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke the
callbacks on the already online CPUs. The smp function calls in the
online/downprep callbacks are not required as the callback is guaranteed to
be invoked on the upcoming/outgoing cpu.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: linux-s390@vger.kernel.org
Cc: netdev@vger.kernel.org
Cc: Ursula Braun <ubraun@linux.vnet.ibm.com>
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-13-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |   1 +
 net/iucv/iucv.c            | 118 +++++++++++++++++----------------------------
 2 files changed, 45 insertions(+), 74 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 94c6a18..12bbcf3 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -60,6 +60,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
+	CPUHP_NET_IUCV_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c
index 88a2a3b..f0d6afc 100644
--- a/net/iucv/iucv.c
+++ b/net/iucv/iucv.c
@@ -639,7 +639,7 @@ static void iucv_disable(void)
 	put_online_cpus();
 }
 
-static void free_iucv_data(int cpu)
+static int iucv_cpu_dead(unsigned int cpu)
 {
 	kfree(iucv_param_irq[cpu]);
 	iucv_param_irq[cpu] = NULL;
@@ -647,9 +647,10 @@ static void free_iucv_data(int cpu)
 	iucv_param[cpu] = NULL;
 	kfree(iucv_irq_data[cpu]);
 	iucv_irq_data[cpu] = NULL;
+	return 0;
 }
 
-static int alloc_iucv_data(int cpu)
+static int iucv_cpu_prepare(unsigned int cpu)
 {
 	/* Note: GFP_DMA used to get memory below 2G */
 	iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data),
@@ -671,58 +672,38 @@ static int alloc_iucv_data(int cpu)
 	return 0;
 
 out_free:
-	free_iucv_data(cpu);
+	iucv_cpu_dead(cpu);
 	return -ENOMEM;
 }
 
-static int iucv_cpu_notify(struct notifier_block *self,
-				     unsigned long action, void *hcpu)
+static int iucv_cpu_online(unsigned int cpu)
 {
-	cpumask_t cpumask;
-	long cpu = (long) hcpu;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		if (alloc_iucv_data(cpu))
-			return notifier_from_errno(-ENOMEM);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		free_iucv_data(cpu);
-		break;
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-		if (!iucv_path_table)
-			break;
-		smp_call_function_single(cpu, iucv_declare_cpu, NULL, 1);
-		break;
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		if (!iucv_path_table)
-			break;
-		cpumask_copy(&cpumask, &iucv_buffer_cpumask);
-		cpumask_clear_cpu(cpu, &cpumask);
-		if (cpumask_empty(&cpumask))
-			/* Can't offline last IUCV enabled cpu. */
-			return notifier_from_errno(-EINVAL);
-		smp_call_function_single(cpu, iucv_retrieve_cpu, NULL, 1);
-		if (cpumask_empty(&iucv_irq_cpumask))
-			smp_call_function_single(
-				cpumask_first(&iucv_buffer_cpumask),
-				iucv_allow_cpu, NULL, 1);
-		break;
-	}
-	return NOTIFY_OK;
+	if (!iucv_path_table)
+		return 0;
+	iucv_declare_cpu(NULL);
+	return 0;
 }
 
-static struct notifier_block __refdata iucv_cpu_notifier = {
-	.notifier_call = iucv_cpu_notify,
-};
+static int iucv_cpu_down_prep(unsigned int cpu)
+{
+	cpumask_t cpumask;
+
+	if (!iucv_path_table)
+		return 0;
+
+	cpumask_copy(&cpumask, &iucv_buffer_cpumask);
+	cpumask_clear_cpu(cpu, &cpumask);
+	if (cpumask_empty(&cpumask))
+		/* Can't offline last IUCV enabled cpu. */
+		return -EINVAL;
+
+	iucv_retrieve_cpu(NULL);
+	if (!cpumask_empty(&iucv_irq_cpumask))
+		return 0;
+	smp_call_function_single(cpumask_first(&iucv_buffer_cpumask),
+				 iucv_allow_cpu, NULL, 1);
+	return 0;
+}
 
 /**
  * iucv_sever_pathid
@@ -2027,6 +2008,7 @@ struct iucv_interface iucv_if = {
 };
 EXPORT_SYMBOL(iucv_if);
 
+static enum cpuhp_state iucv_online;
 /**
  * iucv_init
  *
@@ -2035,7 +2017,6 @@ EXPORT_SYMBOL(iucv_if);
 static int __init iucv_init(void)
 {
 	int rc;
-	int cpu;
 
 	if (!MACHINE_IS_VM) {
 		rc = -EPROTONOSUPPORT;
@@ -2054,23 +2035,19 @@ static int __init iucv_init(void)
 		goto out_int;
 	}
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu) {
-		if (alloc_iucv_data(cpu)) {
-			rc = -ENOMEM;
-			goto out_free;
-		}
-	}
-	rc = __register_hotcpu_notifier(&iucv_cpu_notifier);
+	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
+			       iucv_cpu_prepare, iucv_cpu_dead);
 	if (rc)
 		goto out_free;
-
-	cpu_notifier_register_done();
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
+			       iucv_cpu_online, iucv_cpu_down_prep);
+	if (rc < 0)
+		goto out_free;
+	iucv_online = rc;
 
 	rc = register_reboot_notifier(&iucv_reboot_notifier);
 	if (rc)
-		goto out_cpu;
+		goto out_free;
 	ASCEBC(iucv_error_no_listener, 16);
 	ASCEBC(iucv_error_no_memory, 16);
 	ASCEBC(iucv_error_pathid, 16);
@@ -2084,14 +2061,10 @@ static int __init iucv_init(void)
 
 out_reboot:
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-out_cpu:
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
 out_free:
-	for_each_possible_cpu(cpu)
-		free_iucv_data(cpu);
-
-	cpu_notifier_register_done();
+	if (iucv_online)
+		cpuhp_remove_state(iucv_online);
+	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
 
 	root_device_unregister(iucv_root);
 out_int:
@@ -2110,7 +2083,6 @@ out:
 static void __exit iucv_exit(void)
 {
 	struct iucv_irq_list *p, *n;
-	int cpu;
 
 	spin_lock_irq(&iucv_queue_lock);
 	list_for_each_entry_safe(p, n, &iucv_task_queue, list)
@@ -2119,11 +2091,9 @@ static void __exit iucv_exit(void)
 		kfree(p);
 	spin_unlock_irq(&iucv_queue_lock);
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
-	for_each_possible_cpu(cpu)
-		free_iucv_data(cpu);
-	cpu_notifier_register_done();
+
+	cpuhp_remove_state_nocalls(iucv_online);
+	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
 	root_device_unregister(iucv_root);
 	bus_unregister(&iucv_bus);
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);

[tip:smp/hotplug] net/iucv: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  38b482929e8f96dfe459d2ef757d0a5c3a74cea3
Gitweb:     http://git.kernel.org/tip/38b482929e8f96dfe459d2ef757d0a5c3a74cea3
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:33 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:40 +0100

net/iucv: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke the
callbacks on the already online CPUs. The smp function calls in the
online/downprep callbacks are not required as the callback is guaranteed to
be invoked on the upcoming/outgoing cpu.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: linux-s390@vger.kernel.org
Cc: netdev@vger.kernel.org
Cc: Ursula Braun <ubraun@linux.vnet.ibm.com>
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-13-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |   1 +
 net/iucv/iucv.c            | 118 +++++++++++++++++----------------------------
 2 files changed, 45 insertions(+), 74 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 94c6a18..12bbcf3 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -60,6 +60,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
+	CPUHP_NET_IUCV_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c
index 88a2a3b..f0d6afc 100644
--- a/net/iucv/iucv.c
+++ b/net/iucv/iucv.c
@@ -639,7 +639,7 @@ static void iucv_disable(void)
 	put_online_cpus();
 }
 
-static void free_iucv_data(int cpu)
+static int iucv_cpu_dead(unsigned int cpu)
 {
 	kfree(iucv_param_irq[cpu]);
 	iucv_param_irq[cpu] = NULL;
@@ -647,9 +647,10 @@ static void free_iucv_data(int cpu)
 	iucv_param[cpu] = NULL;
 	kfree(iucv_irq_data[cpu]);
 	iucv_irq_data[cpu] = NULL;
+	return 0;
 }
 
-static int alloc_iucv_data(int cpu)
+static int iucv_cpu_prepare(unsigned int cpu)
 {
 	/* Note: GFP_DMA used to get memory below 2G */
 	iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data),
@@ -671,58 +672,38 @@ static int alloc_iucv_data(int cpu)
 	return 0;
 
 out_free:
-	free_iucv_data(cpu);
+	iucv_cpu_dead(cpu);
 	return -ENOMEM;
 }
 
-static int iucv_cpu_notify(struct notifier_block *self,
-				     unsigned long action, void *hcpu)
+static int iucv_cpu_online(unsigned int cpu)
 {
-	cpumask_t cpumask;
-	long cpu = (long) hcpu;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		if (alloc_iucv_data(cpu))
-			return notifier_from_errno(-ENOMEM);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		free_iucv_data(cpu);
-		break;
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-		if (!iucv_path_table)
-			break;
-		smp_call_function_single(cpu, iucv_declare_cpu, NULL, 1);
-		break;
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		if (!iucv_path_table)
-			break;
-		cpumask_copy(&cpumask, &iucv_buffer_cpumask);
-		cpumask_clear_cpu(cpu, &cpumask);
-		if (cpumask_empty(&cpumask))
-			/* Can't offline last IUCV enabled cpu. */
-			return notifier_from_errno(-EINVAL);
-		smp_call_function_single(cpu, iucv_retrieve_cpu, NULL, 1);
-		if (cpumask_empty(&iucv_irq_cpumask))
-			smp_call_function_single(
-				cpumask_first(&iucv_buffer_cpumask),
-				iucv_allow_cpu, NULL, 1);
-		break;
-	}
-	return NOTIFY_OK;
+	if (!iucv_path_table)
+		return 0;
+	iucv_declare_cpu(NULL);
+	return 0;
 }
 
-static struct notifier_block __refdata iucv_cpu_notifier = {
-	.notifier_call = iucv_cpu_notify,
-};
+static int iucv_cpu_down_prep(unsigned int cpu)
+{
+	cpumask_t cpumask;
+
+	if (!iucv_path_table)
+		return 0;
+
+	cpumask_copy(&cpumask, &iucv_buffer_cpumask);
+	cpumask_clear_cpu(cpu, &cpumask);
+	if (cpumask_empty(&cpumask))
+		/* Can't offline last IUCV enabled cpu. */
+		return -EINVAL;
+
+	iucv_retrieve_cpu(NULL);
+	if (!cpumask_empty(&iucv_irq_cpumask))
+		return 0;
+	smp_call_function_single(cpumask_first(&iucv_buffer_cpumask),
+				 iucv_allow_cpu, NULL, 1);
+	return 0;
+}
 
 /**
  * iucv_sever_pathid
@@ -2027,6 +2008,7 @@ struct iucv_interface iucv_if = {
 };
 EXPORT_SYMBOL(iucv_if);
 
+static enum cpuhp_state iucv_online;
 /**
  * iucv_init
  *
@@ -2035,7 +2017,6 @@ EXPORT_SYMBOL(iucv_if);
 static int __init iucv_init(void)
 {
 	int rc;
-	int cpu;
 
 	if (!MACHINE_IS_VM) {
 		rc = -EPROTONOSUPPORT;
@@ -2054,23 +2035,19 @@ static int __init iucv_init(void)
 		goto out_int;
 	}
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu) {
-		if (alloc_iucv_data(cpu)) {
-			rc = -ENOMEM;
-			goto out_free;
-		}
-	}
-	rc = __register_hotcpu_notifier(&iucv_cpu_notifier);
+	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
+			       iucv_cpu_prepare, iucv_cpu_dead);
 	if (rc)
 		goto out_free;
-
-	cpu_notifier_register_done();
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
+			       iucv_cpu_online, iucv_cpu_down_prep);
+	if (rc < 0)
+		goto out_free;
+	iucv_online = rc;
 
 	rc = register_reboot_notifier(&iucv_reboot_notifier);
 	if (rc)
-		goto out_cpu;
+		goto out_free;
 	ASCEBC(iucv_error_no_listener, 16);
 	ASCEBC(iucv_error_no_memory, 16);
 	ASCEBC(iucv_error_pathid, 16);
@@ -2084,14 +2061,10 @@ static int __init iucv_init(void)
 
 out_reboot:
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-out_cpu:
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
 out_free:
-	for_each_possible_cpu(cpu)
-		free_iucv_data(cpu);
-
-	cpu_notifier_register_done();
+	if (iucv_online)
+		cpuhp_remove_state(iucv_online);
+	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
 
 	root_device_unregister(iucv_root);
 out_int:
@@ -2110,7 +2083,6 @@ out:
 static void __exit iucv_exit(void)
 {
 	struct iucv_irq_list *p, *n;
-	int cpu;
 
 	spin_lock_irq(&iucv_queue_lock);
 	list_for_each_entry_safe(p, n, &iucv_task_queue, list)
@@ -2119,11 +2091,9 @@ static void __exit iucv_exit(void)
 		kfree(p);
 	spin_unlock_irq(&iucv_queue_lock);
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
-	for_each_possible_cpu(cpu)
-		free_iucv_data(cpu);
-	cpu_notifier_register_done();
+
+	cpuhp_remove_state_nocalls(iucv_online);
+	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
 	root_device_unregister(iucv_root);
 	bus_unregister(&iucv_bus);
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);

Re: [PATCH 12/20] net/iucv: Convert to hotplug state machine

[Ursula Braun]

Sebastian,

your patch looks good to me. I run successfully some small tests with it.
I want to suggest a small change in iucv_init() to keep the uniform technique
of undo labels below. Do you agree?

Kind regards, Ursula

On 11/17/2016 07:35 PM, Sebastian Andrzej Siewior wrote:
> Install the callbacks via the state machine and let the core invoke the
> callbacks on the already online CPUs. The smp function calls in the
> online/downprep callbacks are not required as the callback is guaranteed to
> be invoked on the upcoming/outgoing cpu.
> 
> Cc: Ursula Braun <ubraun@linux.vnet.ibm.com>
> Cc: "David S. Miller" <davem@davemloft.net>
> Cc: linux-s390@vger.kernel.org
> Cc: netdev@vger.kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> ---
>  include/linux/cpuhotplug.h |   1 +
>  net/iucv/iucv.c            | 118 +++++++++++++++++----------------------------
>  2 files changed, 45 insertions(+), 74 deletions(-)
> 
> diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
> index fd5598b8353a..69abf2c09f6c 100644
> --- a/include/linux/cpuhotplug.h
> +++ b/include/linux/cpuhotplug.h
> @@ -63,6 +63,7 @@ enum cpuhp_state {
>  	CPUHP_X86_THERM_PREPARE,
>  	CPUHP_X86_CPUID_PREPARE,
>  	CPUHP_X86_MSR_PREPARE,
> +	CPUHP_NET_IUCV_PREPARE,
>  	CPUHP_TIMERS_DEAD,
>  	CPUHP_NOTF_ERR_INJ_PREPARE,
>  	CPUHP_MIPS_SOC_PREPARE,
> diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c
> index 88a2a3ba4212..f0d6afc5d4a9 100644
> --- a/net/iucv/iucv.c
> +++ b/net/iucv/iucv.c
> @@ -639,7 +639,7 @@ static void iucv_disable(void)
>  	put_online_cpus();
>  }
>  
> -static void free_iucv_data(int cpu)
> +static int iucv_cpu_dead(unsigned int cpu)
>  {
>  	kfree(iucv_param_irq[cpu]);
>  	iucv_param_irq[cpu] = NULL;
> @@ -647,9 +647,10 @@ static void free_iucv_data(int cpu)
>  	iucv_param[cpu] = NULL;
>  	kfree(iucv_irq_data[cpu]);
>  	iucv_irq_data[cpu] = NULL;
> +	return 0;
>  }
>  
> -static int alloc_iucv_data(int cpu)
> +static int iucv_cpu_prepare(unsigned int cpu)
>  {
>  	/* Note: GFP_DMA used to get memory below 2G */
>  	iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data),
> @@ -671,58 +672,38 @@ static int alloc_iucv_data(int cpu)
>  	return 0;
>  
>  out_free:
> -	free_iucv_data(cpu);
> +	iucv_cpu_dead(cpu);
>  	return -ENOMEM;
>  }
>  
> -static int iucv_cpu_notify(struct notifier_block *self,
> -				     unsigned long action, void *hcpu)
> +static int iucv_cpu_online(unsigned int cpu)
>  {
> -	cpumask_t cpumask;
> -	long cpu = (long) hcpu;
> -
> -	switch (action) {
> -	case CPU_UP_PREPARE:
> -	case CPU_UP_PREPARE_FROZEN:
> -		if (alloc_iucv_data(cpu))
> -			return notifier_from_errno(-ENOMEM);
> -		break;
> -	case CPU_UP_CANCELED:
> -	case CPU_UP_CANCELED_FROZEN:
> -	case CPU_DEAD:
> -	case CPU_DEAD_FROZEN:
> -		free_iucv_data(cpu);
> -		break;
> -	case CPU_ONLINE:
> -	case CPU_ONLINE_FROZEN:
> -	case CPU_DOWN_FAILED:
> -	case CPU_DOWN_FAILED_FROZEN:
> -		if (!iucv_path_table)
> -			break;
> -		smp_call_function_single(cpu, iucv_declare_cpu, NULL, 1);
> -		break;
> -	case CPU_DOWN_PREPARE:
> -	case CPU_DOWN_PREPARE_FROZEN:
> -		if (!iucv_path_table)
> -			break;
> -		cpumask_copy(&cpumask, &iucv_buffer_cpumask);
> -		cpumask_clear_cpu(cpu, &cpumask);
> -		if (cpumask_empty(&cpumask))
> -			/* Can't offline last IUCV enabled cpu. */
> -			return notifier_from_errno(-EINVAL);
> -		smp_call_function_single(cpu, iucv_retrieve_cpu, NULL, 1);
> -		if (cpumask_empty(&iucv_irq_cpumask))
> -			smp_call_function_single(
> -				cpumask_first(&iucv_buffer_cpumask),
> -				iucv_allow_cpu, NULL, 1);
> -		break;
> -	}
> -	return NOTIFY_OK;
> +	if (!iucv_path_table)
> +		return 0;
> +	iucv_declare_cpu(NULL);
> +	return 0;
>  }
>  
> -static struct notifier_block __refdata iucv_cpu_notifier = {
> -	.notifier_call = iucv_cpu_notify,
> -};
> +static int iucv_cpu_down_prep(unsigned int cpu)
> +{
> +	cpumask_t cpumask;
> +
> +	if (!iucv_path_table)
> +		return 0;
> +
> +	cpumask_copy(&cpumask, &iucv_buffer_cpumask);
> +	cpumask_clear_cpu(cpu, &cpumask);
> +	if (cpumask_empty(&cpumask))
> +		/* Can't offline last IUCV enabled cpu. */
> +		return -EINVAL;
> +
> +	iucv_retrieve_cpu(NULL);
> +	if (!cpumask_empty(&iucv_irq_cpumask))
> +		return 0;
> +	smp_call_function_single(cpumask_first(&iucv_buffer_cpumask),
> +				 iucv_allow_cpu, NULL, 1);
> +	return 0;
> +}
>  
>  /**
>   * iucv_sever_pathid
> @@ -2027,6 +2008,7 @@ struct iucv_interface iucv_if = {
>  };
>  EXPORT_SYMBOL(iucv_if);
>  
> +static enum cpuhp_state iucv_online;
>  /**
>   * iucv_init
>   *
> @@ -2035,7 +2017,6 @@ EXPORT_SYMBOL(iucv_if);
>  static int __init iucv_init(void)
>  {
>  	int rc;
> -	int cpu;
>  
>  	if (!MACHINE_IS_VM) {
>  		rc = -EPROTONOSUPPORT;
> @@ -2054,23 +2035,19 @@ static int __init iucv_init(void)
>  		goto out_int;
>  	}
>  
> -	cpu_notifier_register_begin();
> -
> -	for_each_online_cpu(cpu) {
> -		if (alloc_iucv_data(cpu)) {
> -			rc = -ENOMEM;
> -			goto out_free;
> -		}
> -	}
> -	rc = __register_hotcpu_notifier(&iucv_cpu_notifier);
> +	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
> +			       iucv_cpu_prepare, iucv_cpu_dead);
>  	if (rc)
>  		goto out_free;
> -
> -	cpu_notifier_register_done();
> +	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
> +			       iucv_cpu_online, iucv_cpu_down_prep);
> +	if (rc < 0)
> +		goto out_free;
> +	iucv_online = rc;
>  
>  	rc = register_reboot_notifier(&iucv_reboot_notifier);
>  	if (rc)
> -		goto out_cpu;
> +		goto out_free;
>  	ASCEBC(iucv_error_no_listener, 16);
>  	ASCEBC(iucv_error_no_memory, 16);
>  	ASCEBC(iucv_error_pathid, 16);
> @@ -2084,14 +2061,10 @@ static int __init iucv_init(void)
>  
>  out_reboot:
>  	unregister_reboot_notifier(&iucv_reboot_notifier);
> -out_cpu:
> -	cpu_notifier_register_begin();
> -	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
>  out_free:
> -	for_each_possible_cpu(cpu)
> -		free_iucv_data(cpu);
> -
> -	cpu_notifier_register_done();
> +	if (iucv_online)
> +		cpuhp_remove_state(iucv_online);
> +	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
>  
>  	root_device_unregister(iucv_root);
>  out_int:
I prefer to keep the technique of cascaded undo labels here, like this:
@@ -2054,23 +2035,19 @@ static int __init iucv_init(void)
                goto out_int;
        }
 
-       cpu_notifier_register_begin();
-
-       for_each_online_cpu(cpu) {
-               if (alloc_iucv_data(cpu)) {
-                       rc = -ENOMEM;
-                       goto out_free;
-               }
-       }
-       rc = __register_hotcpu_notifier(&iucv_cpu_notifier);
+       rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
+                              iucv_cpu_prepare, iucv_cpu_dead);
        if (rc)
-               goto out_free;
-
-       cpu_notifier_register_done();
+               goto out_dev;
+       rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
+                              iucv_cpu_online, iucv_cpu_down_prep);
+       if (rc < 0)
+               goto out_prep;
+       iucv_online = rc;
 
        rc = register_reboot_notifier(&iucv_reboot_notifier);
        if (rc)
-               goto out_cpu;
+               goto out_remove;
        ASCEBC(iucv_error_no_listener, 16);
        ASCEBC(iucv_error_no_memory, 16);
        ASCEBC(iucv_error_pathid, 16);
@@ -2084,15 +2061,12 @@ static int __init iucv_init(void)
 
 out_reboot:
        unregister_reboot_notifier(&iucv_reboot_notifier);
-out_cpu:
-       cpu_notifier_register_begin();
-       __unregister_hotcpu_notifier(&iucv_cpu_notifier);
-out_free:
-       for_each_possible_cpu(cpu)
-               free_iucv_data(cpu);
-
-       cpu_notifier_register_done();
-
+out_remove:
+       if (iucv_online)
+               cpuhp_remove_state(iucv_online);
+out_prep:
+       cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
+out_dev:
        root_device_unregister(iucv_root);
 out_int:
        unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);

> @@ -2110,7 +2083,6 @@ static int __init iucv_init(void)
>  static void __exit iucv_exit(void)
>  {
>  	struct iucv_irq_list *p, *n;
> -	int cpu;
>  
>  	spin_lock_irq(&iucv_queue_lock);
>  	list_for_each_entry_safe(p, n, &iucv_task_queue, list)
> @@ -2119,11 +2091,9 @@ static void __exit iucv_exit(void)
>  		kfree(p);
>  	spin_unlock_irq(&iucv_queue_lock);
>  	unregister_reboot_notifier(&iucv_reboot_notifier);
> -	cpu_notifier_register_begin();
> -	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
> -	for_each_possible_cpu(cpu)
> -		free_iucv_data(cpu);
> -	cpu_notifier_register_done();
> +
> +	cpuhp_remove_state_nocalls(iucv_online);
> +	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
>  	root_device_unregister(iucv_root);
>  	bus_unregister(&iucv_bus);
>  	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);
> 

Re: [PATCH 12/20] net/iucv: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

On 2016-11-23 19:04:16 [+0100], Ursula Braun wrote:
> Sebastian,
Hallo Ursula,

> your patch looks good to me. I run successfully some small tests with it.
> I want to suggest a small change in iucv_init() to keep the uniform technique
> of undo labels below. Do you agree?

So what you ask for is:

diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c
index f0d6afc5d4a9..8f7ef167c45a 100644
--- a/net/iucv/iucv.c
+++ b/net/iucv/iucv.c
@@ -2038,16 +2038,16 @@ static int __init iucv_init(void)
 	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
 			       iucv_cpu_prepare, iucv_cpu_dead);
 	if (rc)
-		goto out_free;
+		goto out_dev;
 	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
 			       iucv_cpu_online, iucv_cpu_down_prep);
 	if (rc < 0)
-		goto out_free;
+		goto out_prep;
 	iucv_online = rc;
 
 	rc = register_reboot_notifier(&iucv_reboot_notifier);
 	if (rc)
-		goto out_free;
+		goto out_remove_hp;
 	ASCEBC(iucv_error_no_listener, 16);
 	ASCEBC(iucv_error_no_memory, 16);
 	ASCEBC(iucv_error_pathid, 16);
@@ -2061,11 +2061,11 @@ static int __init iucv_init(void)
 
 out_reboot:
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-out_free:
-	if (iucv_online)
-		cpuhp_remove_state(iucv_online);
+out_remove_hp:
+	cpuhp_remove_state(iucv_online);
+out_prep:
 	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
-
+out_dev:
 	root_device_unregister(iucv_root);
 out_int:
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);

This is your change including the removal of the `if' check in the
`out_remove' label which got renamed to `out_remove_hp'.
Of course, I agree with this change and a proper patch will follow in a
few hours if nobody objects.

> Kind regards, Ursula

Sebastian

[PATCH 12/20 v2] net/iucv: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke the
callbacks on the already online CPUs. The smp function calls in the
online/downprep callbacks are not required as the callback is guaranteed to
be invoked on the upcoming/outgoing cpu.

Cc: Ursula Braun <ubraun@linux.vnet.ibm.com>
Cc: "David S. Miller" <davem@davemloft.net>
Cc: linux-s390@vger.kernel.org
Cc: netdev@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
v1…v2: Use explicit labels for clean up in iucv_init() as suggested by
       Ursula.

 include/linux/cpuhotplug.h |    1 
 net/iucv/iucv.c            |  122 ++++++++++++++++-----------------------------
 2 files changed, 47 insertions(+), 76 deletions(-)

--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -63,6 +63,7 @@ enum cpuhp_state {
 	CPUHP_X86_THERM_PREPARE,
 	CPUHP_X86_CPUID_PREPARE,
 	CPUHP_X86_MSR_PREPARE,
+	CPUHP_NET_IUCV_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
--- a/net/iucv/iucv.c
+++ b/net/iucv/iucv.c
@@ -639,7 +639,7 @@ static void iucv_disable(void)
 	put_online_cpus();
 }
 
-static void free_iucv_data(int cpu)
+static int iucv_cpu_dead(unsigned int cpu)
 {
 	kfree(iucv_param_irq[cpu]);
 	iucv_param_irq[cpu] = NULL;
@@ -647,9 +647,10 @@ static void free_iucv_data(int cpu)
 	iucv_param[cpu] = NULL;
 	kfree(iucv_irq_data[cpu]);
 	iucv_irq_data[cpu] = NULL;
+	return 0;
 }
 
-static int alloc_iucv_data(int cpu)
+static int iucv_cpu_prepare(unsigned int cpu)
 {
 	/* Note: GFP_DMA used to get memory below 2G */
 	iucv_irq_data[cpu] = kmalloc_node(sizeof(struct iucv_irq_data),
@@ -671,58 +672,38 @@ static int alloc_iucv_data(int cpu)
 	return 0;
 
 out_free:
-	free_iucv_data(cpu);
+	iucv_cpu_dead(cpu);
 	return -ENOMEM;
 }
 
-static int iucv_cpu_notify(struct notifier_block *self,
-				     unsigned long action, void *hcpu)
+static int iucv_cpu_online(unsigned int cpu)
+{
+	if (!iucv_path_table)
+		return 0;
+	iucv_declare_cpu(NULL);
+	return 0;
+}
+
+static int iucv_cpu_down_prep(unsigned int cpu)
 {
 	cpumask_t cpumask;
-	long cpu = (long) hcpu;
 
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		if (alloc_iucv_data(cpu))
-			return notifier_from_errno(-ENOMEM);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		free_iucv_data(cpu);
-		break;
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-	case CPU_DOWN_FAILED:
-	case CPU_DOWN_FAILED_FROZEN:
-		if (!iucv_path_table)
-			break;
-		smp_call_function_single(cpu, iucv_declare_cpu, NULL, 1);
-		break;
-	case CPU_DOWN_PREPARE:
-	case CPU_DOWN_PREPARE_FROZEN:
-		if (!iucv_path_table)
-			break;
-		cpumask_copy(&cpumask, &iucv_buffer_cpumask);
-		cpumask_clear_cpu(cpu, &cpumask);
-		if (cpumask_empty(&cpumask))
-			/* Can't offline last IUCV enabled cpu. */
-			return notifier_from_errno(-EINVAL);
-		smp_call_function_single(cpu, iucv_retrieve_cpu, NULL, 1);
-		if (cpumask_empty(&iucv_irq_cpumask))
-			smp_call_function_single(
-				cpumask_first(&iucv_buffer_cpumask),
-				iucv_allow_cpu, NULL, 1);
-		break;
-	}
-	return NOTIFY_OK;
-}
+	if (!iucv_path_table)
+		return 0;
 
-static struct notifier_block __refdata iucv_cpu_notifier = {
-	.notifier_call = iucv_cpu_notify,
-};
+	cpumask_copy(&cpumask, &iucv_buffer_cpumask);
+	cpumask_clear_cpu(cpu, &cpumask);
+	if (cpumask_empty(&cpumask))
+		/* Can't offline last IUCV enabled cpu. */
+		return -EINVAL;
+
+	iucv_retrieve_cpu(NULL);
+	if (!cpumask_empty(&iucv_irq_cpumask))
+		return 0;
+	smp_call_function_single(cpumask_first(&iucv_buffer_cpumask),
+				 iucv_allow_cpu, NULL, 1);
+	return 0;
+}
 
 /**
  * iucv_sever_pathid
@@ -2027,6 +2008,7 @@ struct iucv_interface iucv_if = {
 };
 EXPORT_SYMBOL(iucv_if);
 
+static enum cpuhp_state iucv_online;
 /**
  * iucv_init
  *
@@ -2035,7 +2017,6 @@ EXPORT_SYMBOL(iucv_if);
 static int __init iucv_init(void)
 {
 	int rc;
-	int cpu;
 
 	if (!MACHINE_IS_VM) {
 		rc = -EPROTONOSUPPORT;
@@ -2054,23 +2035,19 @@ static int __init iucv_init(void)
 		goto out_int;
 	}
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu) {
-		if (alloc_iucv_data(cpu)) {
-			rc = -ENOMEM;
-			goto out_free;
-		}
-	}
-	rc = __register_hotcpu_notifier(&iucv_cpu_notifier);
+	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
+			       iucv_cpu_prepare, iucv_cpu_dead);
 	if (rc)
-		goto out_free;
-
-	cpu_notifier_register_done();
+		goto out_dev;
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
+			       iucv_cpu_online, iucv_cpu_down_prep);
+	if (rc < 0)
+		goto out_prep;
+	iucv_online = rc;
 
 	rc = register_reboot_notifier(&iucv_reboot_notifier);
 	if (rc)
-		goto out_cpu;
+		goto out_remove_hp;
 	ASCEBC(iucv_error_no_listener, 16);
 	ASCEBC(iucv_error_no_memory, 16);
 	ASCEBC(iucv_error_pathid, 16);
@@ -2084,15 +2061,11 @@ static int __init iucv_init(void)
 
 out_reboot:
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-out_cpu:
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
-out_free:
-	for_each_possible_cpu(cpu)
-		free_iucv_data(cpu);
-
-	cpu_notifier_register_done();
-
+out_remove_hp:
+	cpuhp_remove_state(iucv_online);
+out_prep:
+	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
+out_dev:
 	root_device_unregister(iucv_root);
 out_int:
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);
@@ -2110,7 +2083,6 @@ static int __init iucv_init(void)
 static void __exit iucv_exit(void)
 {
 	struct iucv_irq_list *p, *n;
-	int cpu;
 
 	spin_lock_irq(&iucv_queue_lock);
 	list_for_each_entry_safe(p, n, &iucv_task_queue, list)
@@ -2119,11 +2091,9 @@ static void __exit iucv_exit(void)
 		kfree(p);
 	spin_unlock_irq(&iucv_queue_lock);
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-	cpu_notifier_register_begin();
-	__unregister_hotcpu_notifier(&iucv_cpu_notifier);
-	for_each_possible_cpu(cpu)
-		free_iucv_data(cpu);
-	cpu_notifier_register_done();
+
+	cpuhp_remove_state_nocalls(iucv_online);
+	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
 	root_device_unregister(iucv_root);
 	bus_unregister(&iucv_bus);
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);

[PATCH] net/iucv: use explicit clean up labels in iucv_init()

[Sebastian Andrzej Siewior]

Ursula suggested to use explicit labels for clean up in the error path
instead of one `out_free' label which handles multiple exits.
Since the previous patch got already applied, here is a follow up patch.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 net/iucv/iucv.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c
index f0d6afc5d4a9..8f7ef167c45a 100644
--- a/net/iucv/iucv.c
+++ b/net/iucv/iucv.c
@@ -2038,16 +2038,16 @@ static int __init iucv_init(void)
 	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
 			       iucv_cpu_prepare, iucv_cpu_dead);
 	if (rc)
-		goto out_free;
+		goto out_dev;
 	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
 			       iucv_cpu_online, iucv_cpu_down_prep);
 	if (rc < 0)
-		goto out_free;
+		goto out_prep;
 	iucv_online = rc;
 
 	rc = register_reboot_notifier(&iucv_reboot_notifier);
 	if (rc)
-		goto out_free;
+		goto out_remove_hp;
 	ASCEBC(iucv_error_no_listener, 16);
 	ASCEBC(iucv_error_no_memory, 16);
 	ASCEBC(iucv_error_pathid, 16);
@@ -2061,11 +2061,11 @@ static int __init iucv_init(void)
 
 out_reboot:
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-out_free:
-	if (iucv_online)
-		cpuhp_remove_state(iucv_online);
+out_remove_hp:
+	cpuhp_remove_state(iucv_online);
+out_prep:
 	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
-
+out_dev:
 	root_device_unregister(iucv_root);
 out_int:
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);
-- 
2.10.2

[tip:smp/hotplug] net/iucv: Use explicit clean up labels in iucv_init()

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  97023a53c5f907e061d6bfa90462e36541f0ae65
Gitweb:     http://git.kernel.org/tip/97023a53c5f907e061d6bfa90462e36541f0ae65
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 24 Nov 2016 17:10:13 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Thu, 24 Nov 2016 20:48:51 +0100

net/iucv: Use explicit clean up labels in iucv_init()

Ursula suggested to use explicit labels for clean up in the error path
instead of one `out_free' label which handles multiple exits.
Since the previous patch got already applied, here is a follow up patch.

Suggested-by: Ursula Braun <ubraun@linux.vnet.ibm.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: linux-s390@vger.kernel.org
Cc: netdev@vger.kernel.org
Cc: rt@linutronix.de
Cc: "David S. Miller" <davem@davemloft.net>
Link: http://lkml.kernel.org/r/20161124161013.dukr42y2nwscosk6@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 net/iucv/iucv.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c
index f0d6afc..8f7ef16 100644
--- a/net/iucv/iucv.c
+++ b/net/iucv/iucv.c
@@ -2038,16 +2038,16 @@ static int __init iucv_init(void)
 	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
 			       iucv_cpu_prepare, iucv_cpu_dead);
 	if (rc)
-		goto out_free;
+		goto out_dev;
 	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
 			       iucv_cpu_online, iucv_cpu_down_prep);
 	if (rc < 0)
-		goto out_free;
+		goto out_prep;
 	iucv_online = rc;
 
 	rc = register_reboot_notifier(&iucv_reboot_notifier);
 	if (rc)
-		goto out_free;
+		goto out_remove_hp;
 	ASCEBC(iucv_error_no_listener, 16);
 	ASCEBC(iucv_error_no_memory, 16);
 	ASCEBC(iucv_error_pathid, 16);
@@ -2061,11 +2061,11 @@ static int __init iucv_init(void)
 
 out_reboot:
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-out_free:
-	if (iucv_online)
-		cpuhp_remove_state(iucv_online);
+out_remove_hp:
+	cpuhp_remove_state(iucv_online);
+out_prep:
 	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
-
+out_dev:
 	root_device_unregister(iucv_root);
 out_int:
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);

Re: [PATCH] net/iucv: use explicit clean up labels in iucv_init()

[David Miller]

From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Date: Thu, 24 Nov 2016 17:10:13 +0100

> Ursula suggested to use explicit labels for clean up in the error path
> instead of one `out_free' label which handles multiple exits.
> Since the previous patch got already applied, here is a follow up patch.
> 
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

"Previous patch" doesn't tell readers anything specific enough to identify
the change you are referring to.  This will be even more true years down
the line if someone tries to read this commit and figure out what you
are referring to.

We have a standard mechanism to refer to commits, via SHA1_ID and commit
header line text, please use it.

Thank you.

Re: [PATCH] net/iucv: use explicit clean up labels in iucv_init()

[Thomas Gleixner]

On Mon, 28 Nov 2016, David Miller wrote:
> From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Date: Thu, 24 Nov 2016 17:10:13 +0100
> 
> > Ursula suggested to use explicit labels for clean up in the error path
> > instead of one `out_free' label which handles multiple exits.
> > Since the previous patch got already applied, here is a follow up patch.
> > 
> > Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> 
> "Previous patch" doesn't tell readers anything specific enough to identify
> the change you are referring to.  This will be even more true years down
> the line if someone tries to read this commit and figure out what you
> are referring to.
> 
> We have a standard mechanism to refer to commits, via SHA1_ID and commit
> header line text, please use it.

I amended the commit message.

Thanks,

	tglx

[tip:smp/hotplug] net/iucv: Use explicit clean up labels in iucv_init()

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  9c6bafab03dec222237b6eb8b5adf5c18ec76264
Gitweb:     http://git.kernel.org/tip/9c6bafab03dec222237b6eb8b5adf5c18ec76264
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 24 Nov 2016 17:10:13 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 28 Nov 2016 17:29:04 +0100

net/iucv: Use explicit clean up labels in iucv_init()

Ursula suggested to use explicit labels for clean up in the error path
instead of one `out_free' label, which handles multiple exits, introduced
in commit 38b482929e8f ("net/iucv: Convert to hotplug state machine").

Suggested-by: Ursula Braun <ubraun@linux.vnet.ibm.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: linux-s390@vger.kernel.org
Cc: netdev@vger.kernel.org
Cc: rt@linutronix.de
Cc: "David S. Miller" <davem@davemloft.net>
Link: http://lkml.kernel.org/r/20161124161013.dukr42y2nwscosk6@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 net/iucv/iucv.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c
index f0d6afc..8f7ef16 100644
--- a/net/iucv/iucv.c
+++ b/net/iucv/iucv.c
@@ -2038,16 +2038,16 @@ static int __init iucv_init(void)
 	rc = cpuhp_setup_state(CPUHP_NET_IUCV_PREPARE, "net/iucv:prepare",
 			       iucv_cpu_prepare, iucv_cpu_dead);
 	if (rc)
-		goto out_free;
+		goto out_dev;
 	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "net/iucv:online",
 			       iucv_cpu_online, iucv_cpu_down_prep);
 	if (rc < 0)
-		goto out_free;
+		goto out_prep;
 	iucv_online = rc;
 
 	rc = register_reboot_notifier(&iucv_reboot_notifier);
 	if (rc)
-		goto out_free;
+		goto out_remove_hp;
 	ASCEBC(iucv_error_no_listener, 16);
 	ASCEBC(iucv_error_no_memory, 16);
 	ASCEBC(iucv_error_pathid, 16);
@@ -2061,11 +2061,11 @@ static int __init iucv_init(void)
 
 out_reboot:
 	unregister_reboot_notifier(&iucv_reboot_notifier);
-out_free:
-	if (iucv_online)
-		cpuhp_remove_state(iucv_online);
+out_remove_hp:
+	cpuhp_remove_state(iucv_online);
+out_prep:
 	cpuhp_remove_state(CPUHP_NET_IUCV_PREPARE);
-
+out_dev:
 	root_device_unregister(iucv_root);
 out_int:
 	unregister_external_irq(EXT_IRQ_IUCV, iucv_external_interrupt);

[PATCH 13/20] sched/nohz: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 kernel/time/tick-sched.c | 33 ++++++++++++++-------------------
 1 file changed, 14 insertions(+), 19 deletions(-)

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 3bcb61b52f6c..420da0749792 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -390,24 +390,16 @@ static int __init tick_nohz_full_setup(char *str)
 }
 __setup("nohz_full=", tick_nohz_full_setup);
 
-static int tick_nohz_cpu_down_callback(struct notifier_block *nfb,
-				       unsigned long action,
-				       void *hcpu)
+static int tick_nohz_cpu_down(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		/*
-		 * The boot CPU handles housekeeping duty (unbound timers,
-		 * workqueues, timekeeping, ...) on behalf of full dynticks
-		 * CPUs. It must remain online when nohz full is enabled.
-		 */
-		if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
-			return NOTIFY_BAD;
-		break;
-	}
-	return NOTIFY_OK;
+	/*
+	 * The boot CPU handles housekeeping duty (unbound timers,
+	 * workqueues, timekeeping, ...) on behalf of full dynticks
+	 * CPUs. It must remain online when nohz full is enabled.
+	 */
+	if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
+		return -EINVAL;
+	return 0;
 }
 
 static int tick_nohz_init_all(void)
@@ -428,7 +420,7 @@ static int tick_nohz_init_all(void)
 
 void __init tick_nohz_init(void)
 {
-	int cpu;
+	int cpu, ret;
 
 	if (!tick_nohz_full_running) {
 		if (tick_nohz_init_all() < 0)
@@ -469,7 +461,10 @@ void __init tick_nohz_init(void)
 	for_each_cpu(cpu, tick_nohz_full_mask)
 		context_tracking_cpu_set(cpu);
 
-	cpu_notifier(tick_nohz_cpu_down_callback, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"kernel/nohz:predown", NULL,
+					tick_nohz_cpu_down);
+	WARN_ON(ret < 0);
 	pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
 		cpumask_pr_args(tick_nohz_full_mask));
 
-- 
2.10.2

[tip:smp/hotplug] sched/nohz: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  703c96e4971c3d2383d9bf248fa656f154250edc
Gitweb:     http://git.kernel.org/tip/703c96e4971c3d2383d9bf248fa656f154250edc
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:34 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:07 +0100

sched/nohz: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-14-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 kernel/time/tick-sched.c | 33 ++++++++++++++-------------------
 1 file changed, 14 insertions(+), 19 deletions(-)

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 3bcb61b..71496a2 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -390,24 +390,16 @@ static int __init tick_nohz_full_setup(char *str)
 }
 __setup("nohz_full=", tick_nohz_full_setup);
 
-static int tick_nohz_cpu_down_callback(struct notifier_block *nfb,
-				       unsigned long action,
-				       void *hcpu)
+static int tick_nohz_cpu_down(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		/*
-		 * The boot CPU handles housekeeping duty (unbound timers,
-		 * workqueues, timekeeping, ...) on behalf of full dynticks
-		 * CPUs. It must remain online when nohz full is enabled.
-		 */
-		if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
-			return NOTIFY_BAD;
-		break;
-	}
-	return NOTIFY_OK;
+	/*
+	 * The boot CPU handles housekeeping duty (unbound timers,
+	 * workqueues, timekeeping, ...) on behalf of full dynticks
+	 * CPUs. It must remain online when nohz full is enabled.
+	 */
+	if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
+		return -EBUSY;
+	return 0;
 }
 
 static int tick_nohz_init_all(void)
@@ -428,7 +420,7 @@ static int tick_nohz_init_all(void)
 
 void __init tick_nohz_init(void)
 {
-	int cpu;
+	int cpu, ret;
 
 	if (!tick_nohz_full_running) {
 		if (tick_nohz_init_all() < 0)
@@ -469,7 +461,10 @@ void __init tick_nohz_init(void)
 	for_each_cpu(cpu, tick_nohz_full_mask)
 		context_tracking_cpu_set(cpu);
 
-	cpu_notifier(tick_nohz_cpu_down_callback, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"kernel/nohz:predown", NULL,
+					tick_nohz_cpu_down);
+	WARN_ON(ret < 0);
 	pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
 		cpumask_pr_args(tick_nohz_full_mask));
 

[tip:smp/hotplug] sched/nohz: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  31eff2434db542763a00074a8368d7bd78d14ea1
Gitweb:     http://git.kernel.org/tip/31eff2434db542763a00074a8368d7bd78d14ea1
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:34 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:41 +0100

sched/nohz: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-14-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 kernel/time/tick-sched.c | 33 ++++++++++++++-------------------
 1 file changed, 14 insertions(+), 19 deletions(-)

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 3bcb61b..71496a2 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -390,24 +390,16 @@ static int __init tick_nohz_full_setup(char *str)
 }
 __setup("nohz_full=", tick_nohz_full_setup);
 
-static int tick_nohz_cpu_down_callback(struct notifier_block *nfb,
-				       unsigned long action,
-				       void *hcpu)
+static int tick_nohz_cpu_down(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_PREPARE:
-		/*
-		 * The boot CPU handles housekeeping duty (unbound timers,
-		 * workqueues, timekeeping, ...) on behalf of full dynticks
-		 * CPUs. It must remain online when nohz full is enabled.
-		 */
-		if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
-			return NOTIFY_BAD;
-		break;
-	}
-	return NOTIFY_OK;
+	/*
+	 * The boot CPU handles housekeeping duty (unbound timers,
+	 * workqueues, timekeeping, ...) on behalf of full dynticks
+	 * CPUs. It must remain online when nohz full is enabled.
+	 */
+	if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
+		return -EBUSY;
+	return 0;
 }
 
 static int tick_nohz_init_all(void)
@@ -428,7 +420,7 @@ static int tick_nohz_init_all(void)
 
 void __init tick_nohz_init(void)
 {
-	int cpu;
+	int cpu, ret;
 
 	if (!tick_nohz_full_running) {
 		if (tick_nohz_init_all() < 0)
@@ -469,7 +461,10 @@ void __init tick_nohz_init(void)
 	for_each_cpu(cpu, tick_nohz_full_mask)
 		context_tracking_cpu_set(cpu);
 
-	cpu_notifier(tick_nohz_cpu_down_callback, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"kernel/nohz:predown", NULL,
+					tick_nohz_cpu_down);
+	WARN_ON(ret < 0);
 	pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
 		cpumask_pr_args(tick_nohz_full_mask));
 

[PATCH 14/20] arm/bL_switcher: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Cc: Russell King <linux@armlinux.org.uk>
Cc: linux-arm-kernel@lists.infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/arm/common/bL_switcher.c | 34 ++++++++++++++++++++--------------
 include/linux/cpuhotplug.h    |  1 +
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c
index 37dc0fe1093f..46730017b3c5 100644
--- a/arch/arm/common/bL_switcher.c
+++ b/arch/arm/common/bL_switcher.c
@@ -757,19 +757,18 @@ EXPORT_SYMBOL_GPL(bL_switcher_put_enabled);
  * while the switcher is active.
  * We're just not ready to deal with that given the trickery involved.
  */
-static int bL_switcher_hotplug_callback(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
+static int bL_switcher_cpu_pre(unsigned int cpu)
 {
-	if (bL_switcher_active) {
-		int pairing = bL_switcher_cpu_pairing[(unsigned long)hcpu];
-		switch (action & 0xf) {
-		case CPU_UP_PREPARE:
-		case CPU_DOWN_PREPARE:
-			if (pairing == -1)
-				return NOTIFY_BAD;
-		}
-	}
-	return NOTIFY_DONE;
+	int pairing;
+
+	if (!bL_switcher_active)
+		return 0;
+
+	pairing = bL_switcher_cpu_pairing[cpu];
+
+	if (pairing == -1)
+		return -EINVAL;
+	return 0;
 }
 
 static bool no_bL_switcher;
@@ -782,8 +781,15 @@ static int __init bL_switcher_init(void)
 	if (!mcpm_is_available())
 		return -ENODEV;
 
-	cpu_notifier(bL_switcher_hotplug_callback, 0);
-
+	cpuhp_setup_state_nocalls(CPUHP_ARM_BL_PREPARE, "arm/bl:prepare",
+				  bL_switcher_cpu_pre, NULL);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/bl:predown",
+					NULL, bL_switcher_cpu_pre);
+	if (ret < 0) {
+		cpuhp_remove_state_nocalls(CPUHP_ARM_BL_PREPARE);
+		pr_err("bL_switcher: Failed to allocate a hotplug state\n");
+		return ret;
+	}
 	if (!no_bL_switcher) {
 		ret = bL_switcher_enable();
 		if (ret)
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 69abf2c09f6c..4b99b79c16e5 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -64,6 +64,7 @@ enum cpuhp_state {
 	CPUHP_X86_CPUID_PREPARE,
 	CPUHP_X86_MSR_PREPARE,
 	CPUHP_NET_IUCV_PREPARE,
+	CPUHP_ARM_BL_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
-- 
2.10.2

[tip:smp/hotplug] arm/bL_switcher: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  5175233eccc753a9d660e08b75084de250f032ad
Gitweb:     http://git.kernel.org/tip/5175233eccc753a9d660e08b75084de250f032ad
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:35 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:08 +0100

arm/bL_switcher: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Cc: linux-arm-kernel@lists.infradead.org
Cc: Russell King <linux@armlinux.org.uk>
Link: http://lkml.kernel.org/r/20161117183541.8588-15-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/arm/common/bL_switcher.c | 34 ++++++++++++++++++++--------------
 include/linux/cpuhotplug.h    |  1 +
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c
index 37dc0fe..4673001 100644
--- a/arch/arm/common/bL_switcher.c
+++ b/arch/arm/common/bL_switcher.c
@@ -757,19 +757,18 @@ EXPORT_SYMBOL_GPL(bL_switcher_put_enabled);
  * while the switcher is active.
  * We're just not ready to deal with that given the trickery involved.
  */
-static int bL_switcher_hotplug_callback(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
+static int bL_switcher_cpu_pre(unsigned int cpu)
 {
-	if (bL_switcher_active) {
-		int pairing = bL_switcher_cpu_pairing[(unsigned long)hcpu];
-		switch (action & 0xf) {
-		case CPU_UP_PREPARE:
-		case CPU_DOWN_PREPARE:
-			if (pairing == -1)
-				return NOTIFY_BAD;
-		}
-	}
-	return NOTIFY_DONE;
+	int pairing;
+
+	if (!bL_switcher_active)
+		return 0;
+
+	pairing = bL_switcher_cpu_pairing[cpu];
+
+	if (pairing == -1)
+		return -EINVAL;
+	return 0;
 }
 
 static bool no_bL_switcher;
@@ -782,8 +781,15 @@ static int __init bL_switcher_init(void)
 	if (!mcpm_is_available())
 		return -ENODEV;
 
-	cpu_notifier(bL_switcher_hotplug_callback, 0);
-
+	cpuhp_setup_state_nocalls(CPUHP_ARM_BL_PREPARE, "arm/bl:prepare",
+				  bL_switcher_cpu_pre, NULL);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/bl:predown",
+					NULL, bL_switcher_cpu_pre);
+	if (ret < 0) {
+		cpuhp_remove_state_nocalls(CPUHP_ARM_BL_PREPARE);
+		pr_err("bL_switcher: Failed to allocate a hotplug state\n");
+		return ret;
+	}
 	if (!no_bL_switcher) {
 		ret = bL_switcher_enable();
 		if (ret)
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 12bbcf3..e3771fb 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -61,6 +61,7 @@ enum cpuhp_state {
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
 	CPUHP_NET_IUCV_PREPARE,
+	CPUHP_ARM_BL_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,

[tip:smp/hotplug] arm/bL_switcher: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  a3c9b14f6f151ee4c2a119fab14f9a60d1684d60
Gitweb:     http://git.kernel.org/tip/a3c9b14f6f151ee4c2a119fab14f9a60d1684d60
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:35 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:41 +0100

arm/bL_switcher: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: rt@linuxtronix.de
Cc: linux-arm-kernel@lists.infradead.org
Cc: Russell King <linux@armlinux.org.uk>
Link: http://lkml.kernel.org/r/20161117183541.8588-15-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/arm/common/bL_switcher.c | 34 ++++++++++++++++++++--------------
 include/linux/cpuhotplug.h    |  1 +
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/arch/arm/common/bL_switcher.c b/arch/arm/common/bL_switcher.c
index 37dc0fe..4673001 100644
--- a/arch/arm/common/bL_switcher.c
+++ b/arch/arm/common/bL_switcher.c
@@ -757,19 +757,18 @@ EXPORT_SYMBOL_GPL(bL_switcher_put_enabled);
  * while the switcher is active.
  * We're just not ready to deal with that given the trickery involved.
  */
-static int bL_switcher_hotplug_callback(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
+static int bL_switcher_cpu_pre(unsigned int cpu)
 {
-	if (bL_switcher_active) {
-		int pairing = bL_switcher_cpu_pairing[(unsigned long)hcpu];
-		switch (action & 0xf) {
-		case CPU_UP_PREPARE:
-		case CPU_DOWN_PREPARE:
-			if (pairing == -1)
-				return NOTIFY_BAD;
-		}
-	}
-	return NOTIFY_DONE;
+	int pairing;
+
+	if (!bL_switcher_active)
+		return 0;
+
+	pairing = bL_switcher_cpu_pairing[cpu];
+
+	if (pairing == -1)
+		return -EINVAL;
+	return 0;
 }
 
 static bool no_bL_switcher;
@@ -782,8 +781,15 @@ static int __init bL_switcher_init(void)
 	if (!mcpm_is_available())
 		return -ENODEV;
 
-	cpu_notifier(bL_switcher_hotplug_callback, 0);
-
+	cpuhp_setup_state_nocalls(CPUHP_ARM_BL_PREPARE, "arm/bl:prepare",
+				  bL_switcher_cpu_pre, NULL);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "arm/bl:predown",
+					NULL, bL_switcher_cpu_pre);
+	if (ret < 0) {
+		cpuhp_remove_state_nocalls(CPUHP_ARM_BL_PREPARE);
+		pr_err("bL_switcher: Failed to allocate a hotplug state\n");
+		return ret;
+	}
 	if (!no_bL_switcher) {
 		ret = bL_switcher_enable();
 		if (ret)
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 12bbcf3..e3771fb 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -61,6 +61,7 @@ enum cpuhp_state {
 	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TOPOLOGY_PREPARE,
 	CPUHP_NET_IUCV_PREPARE,
+	CPUHP_ARM_BL_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,

[PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

smp_call_function_single() has been removed because the function is already
invoked on the target CPU.

Cc: Will Deacon <will.deacon@arm.com>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: linux-arm-kernel@lists.infradead.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/arm/kernel/hw_breakpoint.c | 44 ++++++++++++++++++-----------------------
 1 file changed, 19 insertions(+), 25 deletions(-)

diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
index b8df45883cf7..51cff5a8feff 100644
--- a/arch/arm/kernel/hw_breakpoint.c
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -925,9 +925,9 @@ static bool core_has_os_save_restore(void)
 	}
 }
 
-static void reset_ctrl_regs(void *unused)
+static void reset_ctrl_regs(unsigned int cpu)
 {
-	int i, raw_num_brps, err = 0, cpu = smp_processor_id();
+	int i, raw_num_brps, err = 0;
 	u32 val;
 
 	/*
@@ -1020,25 +1020,20 @@ static void reset_ctrl_regs(void *unused)
 		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
 }
 
-static int dbg_reset_notify(struct notifier_block *self,
-				      unsigned long action, void *cpu)
+static int dbg_reset_online(unsigned int cpu)
 {
-	if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE)
-		smp_call_function_single((int)cpu, reset_ctrl_regs, NULL, 1);
-
-	return NOTIFY_OK;
+	local_irq_disable();
+	reset_ctrl_regs(cpu);
+	local_irq_enable();
+	return 0;
 }
 
-static struct notifier_block dbg_reset_nb = {
-	.notifier_call = dbg_reset_notify,
-};
-
 #ifdef CONFIG_CPU_PM
 static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action,
 			     void *v)
 {
 	if (action == CPU_PM_EXIT)
-		reset_ctrl_regs(NULL);
+		reset_ctrl_regs(smp_processor_id());
 
 	return NOTIFY_OK;
 }
@@ -1059,6 +1054,8 @@ static inline void pm_init(void)
 
 static int __init arch_hw_breakpoint_init(void)
 {
+	int ret;
+
 	debug_arch = get_debug_arch();
 
 	if (!debug_arch_supported()) {
@@ -1072,8 +1069,6 @@ static int __init arch_hw_breakpoint_init(void)
 	core_num_brps = get_num_brps();
 	core_num_wrps = get_num_wrps();
 
-	cpu_notifier_register_begin();
-
 	/*
 	 * We need to tread carefully here because DBGSWENABLE may be
 	 * driven low on this core and there isn't an architected way to
@@ -1082,15 +1077,18 @@ static int __init arch_hw_breakpoint_init(void)
 	register_undef_hook(&debug_reg_hook);
 
 	/*
-	 * Reset the breakpoint resources. We assume that a halting
-	 * debugger will leave the world in a nice state for us.
+	 * Register CPU notifier which resets the breakpoint resources. We
+	 * assume that a halting debugger will leave the world in a nice state
+	 * for us.
 	 */
-	on_each_cpu(reset_ctrl_regs, NULL, 1);
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm/hw_breakpoint:online",
+				dbg_reset_online, NULL);
 	unregister_undef_hook(&debug_reg_hook);
-	if (!cpumask_empty(&debug_err_mask)) {
+	if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) {
 		core_num_brps = 0;
 		core_num_wrps = 0;
-		cpu_notifier_register_done();
+		if (ret > 0)
+			cpuhp_remove_state_nocalls(ret);
 		return 0;
 	}
 
@@ -1109,11 +1107,7 @@ static int __init arch_hw_breakpoint_init(void)
 	hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
 			TRAP_HWBKPT, "breakpoint debug exception");
 
-	/* Register hotplug and PM notifiers. */
-	__register_cpu_notifier(&dbg_reset_nb);
-
-	cpu_notifier_register_done();
-
+	/* Register PM notifiers. */
 	pm_init();
 	return 0;
 }
-- 
2.10.2

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Will Deacon]

On Thu, Nov 17, 2016 at 07:35:36PM +0100, Sebastian Andrzej Siewior wrote:
> Install the callbacks via the state machine and let the core invoke
> the callbacks on the already online CPUs.
> 
> smp_call_function_single() has been removed because the function is already
> invoked on the target CPU.
> 
> Cc: Will Deacon <will.deacon@arm.com>
> Cc: Mark Rutland <mark.rutland@arm.com>
> Cc: Russell King <linux@armlinux.org.uk>
> Cc: linux-arm-kernel@lists.infradead.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> ---
>  arch/arm/kernel/hw_breakpoint.c | 44 ++++++++++++++++++-----------------------
>  1 file changed, 19 insertions(+), 25 deletions(-)

[...]

>  static int __init arch_hw_breakpoint_init(void)
>  {
> +	int ret;
> +
>  	debug_arch = get_debug_arch();
>  
>  	if (!debug_arch_supported()) {
> @@ -1072,8 +1069,6 @@ static int __init arch_hw_breakpoint_init(void)
>  	core_num_brps = get_num_brps();
>  	core_num_wrps = get_num_wrps();
>  
> -	cpu_notifier_register_begin();
> -
>  	/*
>  	 * We need to tread carefully here because DBGSWENABLE may be
>  	 * driven low on this core and there isn't an architected way to
> @@ -1082,15 +1077,18 @@ static int __init arch_hw_breakpoint_init(void)
>  	register_undef_hook(&debug_reg_hook);
>  
>  	/*
> -	 * Reset the breakpoint resources. We assume that a halting
> -	 * debugger will leave the world in a nice state for us.
> +	 * Register CPU notifier which resets the breakpoint resources. We
> +	 * assume that a halting debugger will leave the world in a nice state
> +	 * for us.
>  	 */
> -	on_each_cpu(reset_ctrl_regs, NULL, 1);
> +	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm/hw_breakpoint:online",
> +				dbg_reset_online, NULL);

I'm slightly unsure about this. The dbg_reset_online callback can execute
undefined instructions (unfortunately, there's no way to probe the presence
of some of the debug registers), so it absolutely has to run within the 
register_undef_hook/unregister_undef_hook calls that are in this function.

With this patch, I worry that the callback can be postponed to ONLINE time
for other CPUs, and then the kernel will panic.

Will

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Thomas Gleixner]

On Fri, 18 Nov 2016, Will Deacon wrote:
> On Thu, Nov 17, 2016 at 07:35:36PM +0100, Sebastian Andrzej Siewior wrote:
> > Install the callbacks via the state machine and let the core invoke
> > the callbacks on the already online CPUs.
> > 
> > smp_call_function_single() has been removed because the function is already
> > invoked on the target CPU.
> > 
> > Cc: Will Deacon <will.deacon@arm.com>
> > Cc: Mark Rutland <mark.rutland@arm.com>
> > Cc: Russell King <linux@armlinux.org.uk>
> > Cc: linux-arm-kernel@lists.infradead.org
> > Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> > Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> > ---
> >  arch/arm/kernel/hw_breakpoint.c | 44 ++++++++++++++++++-----------------------
> >  1 file changed, 19 insertions(+), 25 deletions(-)
> 
> [...]
> 
> >  static int __init arch_hw_breakpoint_init(void)
> >  {
> > +	int ret;
> > +
> >  	debug_arch = get_debug_arch();
> >  
> >  	if (!debug_arch_supported()) {
> > @@ -1072,8 +1069,6 @@ static int __init arch_hw_breakpoint_init(void)
> >  	core_num_brps = get_num_brps();
> >  	core_num_wrps = get_num_wrps();
> >  
> > -	cpu_notifier_register_begin();
> > -
> >  	/*
> >  	 * We need to tread carefully here because DBGSWENABLE may be
> >  	 * driven low on this core and there isn't an architected way to
> > @@ -1082,15 +1077,18 @@ static int __init arch_hw_breakpoint_init(void)
> >  	register_undef_hook(&debug_reg_hook);
> >  
> >  	/*
> > -	 * Reset the breakpoint resources. We assume that a halting
> > -	 * debugger will leave the world in a nice state for us.
> > +	 * Register CPU notifier which resets the breakpoint resources. We
> > +	 * assume that a halting debugger will leave the world in a nice state
> > +	 * for us.
> >  	 */
> > -	on_each_cpu(reset_ctrl_regs, NULL, 1);
> > +	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm/hw_breakpoint:online",
> > +				dbg_reset_online, NULL);
> 
> I'm slightly unsure about this. The dbg_reset_online callback can execute
> undefined instructions (unfortunately, there's no way to probe the presence
> of some of the debug registers), so it absolutely has to run within the 
> register_undef_hook/unregister_undef_hook calls that are in this function.
> 
> With this patch, I worry that the callback can be postponed to ONLINE time
> for other CPUs, and then the kernel will panic.

No. The flow is the following:

  	register_undef_hook(&debug_reg_hook);

	ret = cpuhp_setup_state(.., dbg_reset_online, NULL);
	      {
		for_each_online_cpu(cpu) {
			ret = call_on_cpu(cpu, dbg_reset_online);
			if (ret)
			      return ret:
		}
	      }

  	unregister_undef_hook(&debug_reg_hook);
	
The only difference to the current code is that the call is not invoked via
a smp function call (on_each_cpu), it's pushed to the hotplug thread
context of each cpu and executed there.

But it's guaranteed that cpuhp_setup_state() will not return before the
callback has been invoked on each online cpu.

If cpus are not yet online when that code is invoked, then it's the same
behaviour as before. It will be invoked when the cpu comes online.

Thanks,

	tglx

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Will Deacon]

On Fri, Nov 18, 2016 at 02:11:58PM +0100, Thomas Gleixner wrote:
> On Fri, 18 Nov 2016, Will Deacon wrote:
> > On Thu, Nov 17, 2016 at 07:35:36PM +0100, Sebastian Andrzej Siewior wrote:
> > > @@ -1082,15 +1077,18 @@ static int __init arch_hw_breakpoint_init(void)
> > >  	register_undef_hook(&debug_reg_hook);
> > >  
> > >  	/*
> > > -	 * Reset the breakpoint resources. We assume that a halting
> > > -	 * debugger will leave the world in a nice state for us.
> > > +	 * Register CPU notifier which resets the breakpoint resources. We
> > > +	 * assume that a halting debugger will leave the world in a nice state
> > > +	 * for us.
> > >  	 */
> > > -	on_each_cpu(reset_ctrl_regs, NULL, 1);
> > > +	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm/hw_breakpoint:online",
> > > +				dbg_reset_online, NULL);
> > 
> > I'm slightly unsure about this. The dbg_reset_online callback can execute
> > undefined instructions (unfortunately, there's no way to probe the presence
> > of some of the debug registers), so it absolutely has to run within the 
> > register_undef_hook/unregister_undef_hook calls that are in this function.
> > 
> > With this patch, I worry that the callback can be postponed to ONLINE time
> > for other CPUs, and then the kernel will panic.
> 
> No. The flow is the following:
> 
>   	register_undef_hook(&debug_reg_hook);
> 
> 	ret = cpuhp_setup_state(.., dbg_reset_online, NULL);
> 	      {
> 		for_each_online_cpu(cpu) {
> 			ret = call_on_cpu(cpu, dbg_reset_online);
> 			if (ret)
> 			      return ret:
> 		}
> 	      }
> 
>   	unregister_undef_hook(&debug_reg_hook);
> 	
> The only difference to the current code is that the call is not invoked via
> a smp function call (on_each_cpu), it's pushed to the hotplug thread
> context of each cpu and executed there.
> 
> But it's guaranteed that cpuhp_setup_state() will not return before the
> callback has been invoked on each online cpu.

Ok, that's good.

> If cpus are not yet online when that code is invoked, then it's the same
> behaviour as before. It will be invoked when the cpu comes online.

Just to check, but what stops a CPU from coming online between the call
to cpuhp_setup_state and the call to cpuhp_remove_state_nocalls in the
case of failure (debug_err_mask isn't empty)?

Will

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Thomas Gleixner]

On Fri, 18 Nov 2016, Will Deacon wrote:
> On Fri, Nov 18, 2016 at 02:11:58PM +0100, Thomas Gleixner wrote:
> > But it's guaranteed that cpuhp_setup_state() will not return before the
> > callback has been invoked on each online cpu.
> 
> Ok, that's good.
> 
> > If cpus are not yet online when that code is invoked, then it's the same
> > behaviour as before. It will be invoked when the cpu comes online.
> 
> Just to check, but what stops a CPU from coming online between the call
> to cpuhp_setup_state and the call to cpuhp_remove_state_nocalls in the
> case of failure (debug_err_mask isn't empty)?

Indeed! I missed that part. So we still need a get/put_online_cpus()
protection around all of this.

Just for curiosity sake. Wouldn't it be simpler and less error prone to
make the ARM_DBG_READ/WRITE macros use the exception table and handle that
in the undefined instruction handler to avoid this hook dance?

Thanks,

	tglx

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Will Deacon]

On Fri, Nov 18, 2016 at 02:42:15PM +0100, Thomas Gleixner wrote:
> On Fri, 18 Nov 2016, Will Deacon wrote:
> > On Fri, Nov 18, 2016 at 02:11:58PM +0100, Thomas Gleixner wrote:
> > > But it's guaranteed that cpuhp_setup_state() will not return before the
> > > callback has been invoked on each online cpu.
> > 
> > Ok, that's good.
> > 
> > > If cpus are not yet online when that code is invoked, then it's the same
> > > behaviour as before. It will be invoked when the cpu comes online.
> > 
> > Just to check, but what stops a CPU from coming online between the call
> > to cpuhp_setup_state and the call to cpuhp_remove_state_nocalls in the
> > case of failure (debug_err_mask isn't empty)?
> 
> Indeed! I missed that part. So we still need a get/put_online_cpus()
> protection around all of this.

Yes, that should do it.

> Just for curiosity sake. Wouldn't it be simpler and less error prone to
> make the ARM_DBG_READ/WRITE macros use the exception table and handle that
> in the undefined instruction handler to avoid this hook dance?

That would be an option, but it's only the reset sequence that could
generate this fault so it's simpler to isolate it there. We'd also have
to take into account SMP if we toggle the handler in the READ/WRITE
accessors, since the fault handler framework is system-wide as opposed
to per-cpu. The whole thing is grotty as hell.

Will

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Thomas Gleixner]

On Fri, 18 Nov 2016, Will Deacon wrote:
> On Fri, Nov 18, 2016 at 02:42:15PM +0100, Thomas Gleixner wrote:
> > On Fri, 18 Nov 2016, Will Deacon wrote:
> > > On Fri, Nov 18, 2016 at 02:11:58PM +0100, Thomas Gleixner wrote:
> > > > But it's guaranteed that cpuhp_setup_state() will not return before the
> > > > callback has been invoked on each online cpu.
> > > 
> > > Ok, that's good.
> > > 
> > > > If cpus are not yet online when that code is invoked, then it's the same
> > > > behaviour as before. It will be invoked when the cpu comes online.
> > > 
> > > Just to check, but what stops a CPU from coming online between the call
> > > to cpuhp_setup_state and the call to cpuhp_remove_state_nocalls in the
> > > case of failure (debug_err_mask isn't empty)?
> > 
> > Indeed! I missed that part. So we still need a get/put_online_cpus()
> > protection around all of this.
> 
> Yes, that should do it.
> 
> > Just for curiosity sake. Wouldn't it be simpler and less error prone to
> > make the ARM_DBG_READ/WRITE macros use the exception table and handle that
> > in the undefined instruction handler to avoid this hook dance?
> 
> That would be an option, but it's only the reset sequence that could
> generate this fault so it's simpler to isolate it there. 

ARM_DBG_READ/WRITE_SAFE() then for reset_ctrl_regs()

> We'd also have to take into account SMP if we toggle the handler in the
> READ/WRITE accessors, since the fault handler framework is system-wide as
> opposed to per-cpu. The whole thing is grotty as hell.

The exception table is not toggling anything. It's just providing an entry
in the exception tables, which is scanned by fixup_exception(), which then
moves PC to the exception code. See __get_user_asm().

So the whole thing becomes:

static int reset_ctrl_regs(unsigned cpu)
{
	....
	if (ARM_DBG_READ_SAFE(c1, c5, 4, val))
		return -ENODEV;
	....
	return 0;
}

All you need is the extra

    	if (fixup_exception(regs))
		return;

in do_undefinstr() like it is there in do_kernel_fault(). No hooks, no
scope issues, just works.

I just mention this because that's how x86 implements rdmsr/wrmsr_safe() so
it can probe msr access. The difference though it that this results in a
#GP and not in #UD, but that's not a show stopper :)

Thanks,

	tglx

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Will Deacon]

On Fri, Nov 18, 2016 at 02:59:04PM +0100, Thomas Gleixner wrote:
> On Fri, 18 Nov 2016, Will Deacon wrote:
> > On Fri, Nov 18, 2016 at 02:42:15PM +0100, Thomas Gleixner wrote:
> > > On Fri, 18 Nov 2016, Will Deacon wrote:
> > > > On Fri, Nov 18, 2016 at 02:11:58PM +0100, Thomas Gleixner wrote:
> > > > > But it's guaranteed that cpuhp_setup_state() will not return before the
> > > > > callback has been invoked on each online cpu.
> > > > 
> > > > Ok, that's good.
> > > > 
> > > > > If cpus are not yet online when that code is invoked, then it's the same
> > > > > behaviour as before. It will be invoked when the cpu comes online.
> > > > 
> > > > Just to check, but what stops a CPU from coming online between the call
> > > > to cpuhp_setup_state and the call to cpuhp_remove_state_nocalls in the
> > > > case of failure (debug_err_mask isn't empty)?
> > > 
> > > Indeed! I missed that part. So we still need a get/put_online_cpus()
> > > protection around all of this.
> > 
> > Yes, that should do it.
> > 
> > > Just for curiosity sake. Wouldn't it be simpler and less error prone to
> > > make the ARM_DBG_READ/WRITE macros use the exception table and handle that
> > > in the undefined instruction handler to avoid this hook dance?
> > 
> > That would be an option, but it's only the reset sequence that could
> > generate this fault so it's simpler to isolate it there. 
> 
> ARM_DBG_READ/WRITE_SAFE() then for reset_ctrl_regs()
> 
> > We'd also have to take into account SMP if we toggle the handler in the
> > READ/WRITE accessors, since the fault handler framework is system-wide as
> > opposed to per-cpu. The whole thing is grotty as hell.
> 
> The exception table is not toggling anything. It's just providing an entry
> in the exception tables, which is scanned by fixup_exception(), which then
> moves PC to the exception code. See __get_user_asm().

Oooh, now I see what you mean. I thought you were on about toggling
using register_undef_hook, but you're actually on about the extable stuff
that we already use for handling faults on user addresses in the kernel.

That's not a bad idea at all.

Will

[tip:smp/hotplug] ARM/hw_breakpoint: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  3b341295030a327d7c763630f865fbf20a1df1c8
Gitweb:     http://git.kernel.org/tip/3b341295030a327d7c763630f865fbf20a1df1c8
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:36 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:08 +0100

ARM/hw_breakpoint: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

smp_call_function_single() has been removed because the function is already
invoked on the target CPU.

[ tglx: Added protection agaist hotplug back according to discussion with Will ]

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: rt@linuxtronix.de
Cc: Will Deacon <will.deacon@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: linux-arm-kernel@lists.infradead.org
Link: http://lkml.kernel.org/r/20161117183541.8588-16-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/arm/kernel/hw_breakpoint.c | 47 +++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 25 deletions(-)

diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
index b8df458..188180b 100644
--- a/arch/arm/kernel/hw_breakpoint.c
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -925,9 +925,9 @@ static bool core_has_os_save_restore(void)
 	}
 }
 
-static void reset_ctrl_regs(void *unused)
+static void reset_ctrl_regs(unsigned int cpu)
 {
-	int i, raw_num_brps, err = 0, cpu = smp_processor_id();
+	int i, raw_num_brps, err = 0;
 	u32 val;
 
 	/*
@@ -1020,25 +1020,20 @@ out_mdbgen:
 		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
 }
 
-static int dbg_reset_notify(struct notifier_block *self,
-				      unsigned long action, void *cpu)
+static int dbg_reset_online(unsigned int cpu)
 {
-	if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE)
-		smp_call_function_single((int)cpu, reset_ctrl_regs, NULL, 1);
-
-	return NOTIFY_OK;
+	local_irq_disable();
+	reset_ctrl_regs(cpu);
+	local_irq_enable();
+	return 0;
 }
 
-static struct notifier_block dbg_reset_nb = {
-	.notifier_call = dbg_reset_notify,
-};
-
 #ifdef CONFIG_CPU_PM
 static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action,
 			     void *v)
 {
 	if (action == CPU_PM_EXIT)
-		reset_ctrl_regs(NULL);
+		reset_ctrl_regs(smp_processor_id());
 
 	return NOTIFY_OK;
 }
@@ -1059,6 +1054,8 @@ static inline void pm_init(void)
 
 static int __init arch_hw_breakpoint_init(void)
 {
+	int ret;
+
 	debug_arch = get_debug_arch();
 
 	if (!debug_arch_supported()) {
@@ -1072,25 +1069,28 @@ static int __init arch_hw_breakpoint_init(void)
 	core_num_brps = get_num_brps();
 	core_num_wrps = get_num_wrps();
 
-	cpu_notifier_register_begin();
-
 	/*
 	 * We need to tread carefully here because DBGSWENABLE may be
 	 * driven low on this core and there isn't an architected way to
 	 * determine that.
 	 */
+	get_online_cpus();
 	register_undef_hook(&debug_reg_hook);
 
 	/*
-	 * Reset the breakpoint resources. We assume that a halting
-	 * debugger will leave the world in a nice state for us.
+	 * Register CPU notifier which resets the breakpoint resources. We
+	 * assume that a halting debugger will leave the world in a nice state
+	 * for us.
 	 */
-	on_each_cpu(reset_ctrl_regs, NULL, 1);
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm/hw_breakpoint:online",
+				dbg_reset_online, NULL);
 	unregister_undef_hook(&debug_reg_hook);
-	if (!cpumask_empty(&debug_err_mask)) {
+	if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) {
 		core_num_brps = 0;
 		core_num_wrps = 0;
-		cpu_notifier_register_done();
+		if (ret > 0)
+			cpuhp_remove_state_nocalls(ret);
+		put_online_cpus();
 		return 0;
 	}
 
@@ -1108,12 +1108,9 @@ static int __init arch_hw_breakpoint_init(void)
 			TRAP_HWBKPT, "watchpoint debug exception");
 	hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
 			TRAP_HWBKPT, "breakpoint debug exception");
+	put_online_cpus();
 
-	/* Register hotplug and PM notifiers. */
-	__register_cpu_notifier(&dbg_reset_nb);
-
-	cpu_notifier_register_done();
-
+	/* Register PM notifiers. */
 	pm_init();
 	return 0;
 }

[tip:smp/hotplug] ARM/hw_breakpoint: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  9b377e217f0bd07f972d89ed0963df92818beffd
Gitweb:     http://git.kernel.org/tip/9b377e217f0bd07f972d89ed0963df92818beffd
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:36 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:41 +0100

ARM/hw_breakpoint: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

smp_call_function_single() has been removed because the function is already
invoked on the target CPU.

[ tglx: Added protection agaist hotplug back according to discussion with Will ]

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Mark Rutland <mark.rutland@arm.com>
Cc: rt@linuxtronix.de
Cc: Will Deacon <will.deacon@arm.com>
Cc: Russell King <linux@armlinux.org.uk>
Cc: linux-arm-kernel@lists.infradead.org
Link: http://lkml.kernel.org/r/20161117183541.8588-16-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/arm/kernel/hw_breakpoint.c | 47 +++++++++++++++++++----------------------
 1 file changed, 22 insertions(+), 25 deletions(-)

diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
index b8df458..188180b 100644
--- a/arch/arm/kernel/hw_breakpoint.c
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -925,9 +925,9 @@ static bool core_has_os_save_restore(void)
 	}
 }
 
-static void reset_ctrl_regs(void *unused)
+static void reset_ctrl_regs(unsigned int cpu)
 {
-	int i, raw_num_brps, err = 0, cpu = smp_processor_id();
+	int i, raw_num_brps, err = 0;
 	u32 val;
 
 	/*
@@ -1020,25 +1020,20 @@ out_mdbgen:
 		cpumask_or(&debug_err_mask, &debug_err_mask, cpumask_of(cpu));
 }
 
-static int dbg_reset_notify(struct notifier_block *self,
-				      unsigned long action, void *cpu)
+static int dbg_reset_online(unsigned int cpu)
 {
-	if ((action & ~CPU_TASKS_FROZEN) == CPU_ONLINE)
-		smp_call_function_single((int)cpu, reset_ctrl_regs, NULL, 1);
-
-	return NOTIFY_OK;
+	local_irq_disable();
+	reset_ctrl_regs(cpu);
+	local_irq_enable();
+	return 0;
 }
 
-static struct notifier_block dbg_reset_nb = {
-	.notifier_call = dbg_reset_notify,
-};
-
 #ifdef CONFIG_CPU_PM
 static int dbg_cpu_pm_notify(struct notifier_block *self, unsigned long action,
 			     void *v)
 {
 	if (action == CPU_PM_EXIT)
-		reset_ctrl_regs(NULL);
+		reset_ctrl_regs(smp_processor_id());
 
 	return NOTIFY_OK;
 }
@@ -1059,6 +1054,8 @@ static inline void pm_init(void)
 
 static int __init arch_hw_breakpoint_init(void)
 {
+	int ret;
+
 	debug_arch = get_debug_arch();
 
 	if (!debug_arch_supported()) {
@@ -1072,25 +1069,28 @@ static int __init arch_hw_breakpoint_init(void)
 	core_num_brps = get_num_brps();
 	core_num_wrps = get_num_wrps();
 
-	cpu_notifier_register_begin();
-
 	/*
 	 * We need to tread carefully here because DBGSWENABLE may be
 	 * driven low on this core and there isn't an architected way to
 	 * determine that.
 	 */
+	get_online_cpus();
 	register_undef_hook(&debug_reg_hook);
 
 	/*
-	 * Reset the breakpoint resources. We assume that a halting
-	 * debugger will leave the world in a nice state for us.
+	 * Register CPU notifier which resets the breakpoint resources. We
+	 * assume that a halting debugger will leave the world in a nice state
+	 * for us.
 	 */
-	on_each_cpu(reset_ctrl_regs, NULL, 1);
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "arm/hw_breakpoint:online",
+				dbg_reset_online, NULL);
 	unregister_undef_hook(&debug_reg_hook);
-	if (!cpumask_empty(&debug_err_mask)) {
+	if (WARN_ON(ret < 0) || !cpumask_empty(&debug_err_mask)) {
 		core_num_brps = 0;
 		core_num_wrps = 0;
-		cpu_notifier_register_done();
+		if (ret > 0)
+			cpuhp_remove_state_nocalls(ret);
+		put_online_cpus();
 		return 0;
 	}
 
@@ -1108,12 +1108,9 @@ static int __init arch_hw_breakpoint_init(void)
 			TRAP_HWBKPT, "watchpoint debug exception");
 	hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
 			TRAP_HWBKPT, "breakpoint debug exception");
+	put_online_cpus();
 
-	/* Register hotplug and PM notifiers. */
-	__register_cpu_notifier(&dbg_reset_nb);
-
-	cpu_notifier_register_done();
-
+	/* Register PM notifiers. */
 	pm_init();
 	return 0;
 }

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Linus Walleij]

On Thu, Nov 17, 2016 at 7:35 PM, Sebastian Andrzej Siewior
<bigeasy@linutronix.de> wrote:

> Install the callbacks via the state machine and let the core invoke
> the callbacks on the already online CPUs.
>
> smp_call_function_single() has been removed because the function is already
> invoked on the target CPU.
>
> Cc: Will Deacon <will.deacon@arm.com>
> Cc: Mark Rutland <mark.rutland@arm.com>
> Cc: Russell King <linux@armlinux.org.uk>
> Cc: linux-arm-kernel@lists.infradead.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

This patch causes a regression on my Qualcomm APQ8060 DragonBoard.

[    0.000000] CPU: ARMv7 Processor [510f02d2] revision 2 (ARMv7), cr=10c5787d
[    0.000000] CPU: PIPT / VIPT nonaliasing data cache, VIVT ASID
tagged instruction cache
[    0.000000] OF: fdt:Machine model: Qualcomm APQ8060 Dragonboard

The board hangs in very early boot. Reverting the commit does not work
because of dependencies, but I bisected down to this commit.

With earlypints the crash looks like this:

 NET: Registered protocol family 16
 DMA: preallocated 256 KiB pool for atomic coherent allocations
 cpuidle: using governor menu
 hw-breakpoint: found 3 (+1 reserved) breakpoint and 1 watchpoint registers.
 Internal error: Oops - undefined instruction: 0 [#1] PREEMPT SMP ARM
 Modules linked in: c
 CPU: 0 PID: 1 Comm: swapper/0 Not tainted 4.10.0-rc2-00001-g100b2fb6bf9c #19
 Hardware name: Generic DT based system
 task: c02a0000 task.stack: c029a000
PC is at write_wb_reg+0x20c/0x330
LR is at arch_hw_breakpoint_init+0x1dc/0x27c
pc : [<c03101dc>]    lr : [<c0c0551c>]    psr: 80000013
sp : c029bef0  ip : 00000000  fp : dfffcd80
r10: c0c4f83c  r9 : 00000004  r8 : 000000af
r7 : c0c4f828  r6 : c10a631c  r5 : c10a631c  r4 : 00001fe0
r3 : 00000020  r2 : 00001fe0  r1 : 00000000  r0 : 00000060
Flags: Nzcv  IRQs on  FIQs on  Mode SVC_32  ISA ARM  Segment none
Control: 10c5787d  Table: 4020406a  DAC: 00000051
 Process swapper/0 (pid: 1, stack limit = 0xc029a210)
 Stack: (0xc029bef0 to 0xc029c000)
 bee0:                                     00000000 00000000 dfffcd80 07f80000
 bf00: ffffe000 c0c05340 00000000 c030185c c0b7b7d4 dfffcded c0931100 c033b75c
 bf20: 60000013 00000003 00000001 c0ab8424 c0b7aa28 00000000 c1099a20 00000003
 bf40: 00000003 c0ac1440 c100c588 c10a6000 c10a6000 00000003 c10a6000 c10a6000
 bf60: c0c5a30c 000000af 00000004 c0c00e04 00000003 00000003 00000000 c0c005c4
 bf80: c08d66c0 00000000 c08d66c0 00000000 00000000 00000000 00000000 00000000
 bfa0: 00000000 c08d66c8 00000000 c0308518 00000000 00000000 00000000 00000000
 bfc0: 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
 bfe0: 00000000 00000000 00000000 00000000 00000013 00000000 00000000 00000000
[<c03101dc>] (write_wb_reg) from [<00000000>] (  (null))
 Code: ee001ed2 eaffffc3 ee001ed1 eaffffc1 (ee001ed0)
 ---[ end trace da08286cccfd900e ]---
 Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b

 CPU1: stopping
 CPU: 1 PID: 0 Comm: swapper/1 Tainted: G      D
4.10.0-rc2-00001-g100b2fb6bf9c #19
 Hardware name: Generic DT based system
[<c030f914>] (unwind_backtrace) from [<c030c7b0>] (show_stack+0x10/0x14)
[<c030c7b0>] (show_stack) from [<c05eb2a0>] (dump_stack+0x78/0x8c)
[<c05eb2a0>] (dump_stack) from [<c030ea14>] (handle_IPI+0x358/0x368)
[<c030ea14>] (handle_IPI) from [<c03014a4>] (gic_handle_irq+0x88/0x8c)
[<c03014a4>] (gic_handle_irq) from [<c08dce8c>] (__irq_svc+0x6c/0xa8)
Exception stack(0xc02c5f70 to 0xc02c5fb8)
5f60:                                     00000001 00000000 00000000 c0318580
5f80: c02c4000 c1003c78 c1003c2c c0fa1f20 c0afea4c c02c5fc8 00000000 00000000
5fa0: 00000008 c02c5fc0 c0308f98 c0308f9c 60000013 ffffffff
[<c08dce8c>] (__irq_svc) from [<c0308f9c>] (arch_cpu_idle+0x38/0x3c)
[<c0308f9c>] (arch_cpu_idle) from [<c035d24c>] (do_idle+0x170/0x204)
[<c035d24c>] (do_idle) from [<c035d598>] (cpu_startup_entry+0x18/0x1c)
[<c035d598>] (cpu_startup_entry) from [<4030154c>] (0x4030154c)
---[ end Kernel panic - not syncing: Attempted to kill init! exitcode=0x0000000b

Something hits a brick wall when initializing the hardware breakpoints.

This is pretty tricksy for me to debug, hints welcome...

Yours,
Linus Walleij

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Linus Walleij]

On Mon, Jan 2, 2017 at 3:15 PM, Linus Walleij <linus.walleij@linaro.org> wrote:

> Something hits a brick wall when initializing the hardware breakpoints.

Should be noted that since I'm not using the breakpoints, the dirtyfix is to put

return 0;

in the first line of arch_hw_breakpoint_init() in
arch/arm/kernel/hw_breakpoint.c

I suspect that is not an accepable solution ...

It hangs at PC is at write_wb_reg+0x20c/0x330
Which is c03101dc, and looks like this in objdump -d:

c031020c:       ee001eba        mcr     14, 0, r1, cr0, cr10, {5}
c0310210:       eaffffb3        b       c03100e4 <write_wb_reg+0x114>
c0310214:       ee001eb9        mcr     14, 0, r1, cr0, cr9, {5}
c0310218:       eaffffb1        b       c03100e4 <write_wb_reg+0x114>
c031021c:       ee001eb8        mcr     14, 0, r1, cr0, cr8, {5}
c0310220:       eaffffaf        b       c03100e4 <write_wb_reg+0x114>
c0310224:       ee001eb7        mcr     14, 0, r1, cr0, cr7, {5}
c0310228:       eaffffad        b       c03100e4 <write_wb_reg+0x114>
c031022c:       ee001eb6        mcr     14, 0, r1, cr0, cr6, {5}
c0310230:       eaffffab        b       c03100e4 <write_wb_reg+0x114>
c0310234:       ee001eb5        mcr     14, 0, r1, cr0, cr5, {5}
c0310238:       eaffffa9        b       c03100e4 <write_wb_reg+0x114>

No idea if this helps.

Yours,
Linus Walleij

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Russell King - ARM Linux]

On Mon, Jan 02, 2017 at 03:34:32PM +0100, Linus Walleij wrote:
> in the first line of arch_hw_breakpoint_init() in
> arch/arm/kernel/hw_breakpoint.c
> 
> I suspect that is not an accepable solution ...
> 
> It hangs at PC is at write_wb_reg+0x20c/0x330
> Which is c03101dc, and looks like this in objdump -d:
> 
> c031020c:       ee001eba        mcr     14, 0, r1, cr0, cr10, {5}
> c0310210:       eaffffb3        b       c03100e4 <write_wb_reg+0x114>

... and this is several instructions after the address you mention above.
Presumably c03101dc is accessing a higher numbered register?

-- 
RMK's Patch system: http://www.armlinux.org.uk/developer/patches/
FTTC broadband for 0.8mile line: currently at 9.6Mbps down 400kbps up
according to speedtest.net.

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Linus Walleij]

On Mon, Jan 2, 2017 at 4:00 PM, Russell King - ARM Linux
<linux@armlinux.org.uk> wrote:
> On Mon, Jan 02, 2017 at 03:34:32PM +0100, Linus Walleij wrote:
>> in the first line of arch_hw_breakpoint_init() in
>> arch/arm/kernel/hw_breakpoint.c
>>
>> I suspect that is not an accepable solution ...
>>
>> It hangs at PC is at write_wb_reg+0x20c/0x330
>> Which is c03101dc, and looks like this in objdump -d:
>>
>> c031020c:       ee001eba        mcr     14, 0, r1, cr0, cr10, {5}
>> c0310210:       eaffffb3        b       c03100e4 <write_wb_reg+0x114>
>
> ... and this is several instructions after the address you mention above.
> Presumably c03101dc is accessing a higher numbered register?

Ah sorry. It looks like this:

c03101dc:       ee001ed0        mcr     14, 0, r1, cr0, cr0, {6}
c03101e0:       eaffffbf        b       c03100e4 <write_wb_reg+0x114>
c03101e4:       ee001ebf        mcr     14, 0, r1, cr0, cr15, {5}
c03101e8:       eaffffbd        b       c03100e4 <write_wb_reg+0x114>
c03101ec:       ee001ebe        mcr     14, 0, r1, cr0, cr14, {5}
c03101f0:       eaffffbb        b       c03100e4 <write_wb_reg+0x114>
c03101f4:       ee001ebd        mcr     14, 0, r1, cr0, cr13, {5}
c03101f8:       eaffffb9        b       c03100e4 <write_wb_reg+0x114>

Yours,
Linus Walleij

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Mark Rutland]

Hi,

On Mon, Jan 02, 2017 at 09:15:29PM +0100, Linus Walleij wrote:
> On Mon, Jan 2, 2017 at 4:00 PM, Russell King - ARM Linux
> <linux@armlinux.org.uk> wrote:
> > On Mon, Jan 02, 2017 at 03:34:32PM +0100, Linus Walleij wrote:
> >> in the first line of arch_hw_breakpoint_init() in
> >> arch/arm/kernel/hw_breakpoint.c
> >>
> >> I suspect that is not an accepable solution ...
> >>
> >> It hangs at PC is at write_wb_reg+0x20c/0x330
> >> Which is c03101dc, and looks like this in objdump -d:
> >>
> >> c031020c:       ee001eba        mcr     14, 0, r1, cr0, cr10, {5}
> >> c0310210:       eaffffb3        b       c03100e4 <write_wb_reg+0x114>
> >
> > ... and this is several instructions after the address you mention above.
> > Presumably c03101dc is accessing a higher numbered register?
> 
> Ah sorry. It looks like this:
> 
> c03101dc:       ee001ed0        mcr     14, 0, r1, cr0, cr0, {6}
> c03101e0:       eaffffbf        b       c03100e4 <write_wb_reg+0x114>
> c03101e4:       ee001ebf        mcr     14, 0, r1, cr0, cr15, {5}
> c03101e8:       eaffffbd        b       c03100e4 <write_wb_reg+0x114>
> c03101ec:       ee001ebe        mcr     14, 0, r1, cr0, cr14, {5}
> c03101f0:       eaffffbb        b       c03100e4 <write_wb_reg+0x114>
> c03101f4:       ee001ebd        mcr     14, 0, r1, cr0, cr13, {5}
> c03101f8:       eaffffb9        b       c03100e4 <write_wb_reg+0x114>

FWIW, I was tracking an issue in this area before the holiday.

It looked like DBGPRSR.SPD is set unexpectedly over the default idle
path (i.e. WFI), causing the (otherwise valid) register accesses above
to be handled as undefined.

I haven't looked at the patch in detail, but I guess that it allows idle
to occur between reset_ctrl_regs() and arch_hw_breakpoint_init().

Reading DBGPRSR should clear SPD; but I'm not sure if other debug state
is affected.

Thanks,
Mark.

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

On 2017-01-03 09:33:36 [+0000], Mark Rutland wrote:
> Hi,
Hi,

> It looked like DBGPRSR.SPD is set unexpectedly over the default idle
> path (i.e. WFI), causing the (otherwise valid) register accesses above
> to be handled as undefined.
> 
> I haven't looked at the patch in detail, but I guess that it allows idle
> to occur between reset_ctrl_regs() and arch_hw_breakpoint_init().

This could be possible. The callback is installed and the per-CPU thread
is woken up to handle it. It starts with the lowest CPU and loops for
all present CPUs. While waiting the thread to complete the callback, it
could go idle. See the for_each_present_cpu() loop in
__cpuhp_setup_state() and cpuhp_invoke_ap_callback() kicks the thread
(cpuhp_thread_fun()) and waits for its completion.
So the difference to on_each_cpu() is that the latter performs a busy
loop while waiting for function to complete on other CPUs.

> Reading DBGPRSR should clear SPD; but I'm not sure if other debug state
> is affected.
> 
> Thanks,
> Mark.

Sebastian

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Mark Rutland]

On Tue, Jan 03, 2017 at 09:33:36AM +0000, Mark Rutland wrote:
> Hi,
> 
> On Mon, Jan 02, 2017 at 09:15:29PM +0100, Linus Walleij wrote:
> > On Mon, Jan 2, 2017 at 4:00 PM, Russell King - ARM Linux
> > <linux@armlinux.org.uk> wrote:
> > > On Mon, Jan 02, 2017 at 03:34:32PM +0100, Linus Walleij wrote:
> > >> in the first line of arch_hw_breakpoint_init() in
> > >> arch/arm/kernel/hw_breakpoint.c
> > >>
> > >> I suspect that is not an accepable solution ...
> > >>
> > >> It hangs at PC is at write_wb_reg+0x20c/0x330
> > >> Which is c03101dc, and looks like this in objdump -d:
> > >>
> > >> c031020c:       ee001eba        mcr     14, 0, r1, cr0, cr10, {5}
> > >> c0310210:       eaffffb3        b       c03100e4 <write_wb_reg+0x114>
> > >
> > > ... and this is several instructions after the address you mention above.
> > > Presumably c03101dc is accessing a higher numbered register?
> > 
> > Ah sorry. It looks like this:
> > 
> > c03101dc:       ee001ed0        mcr     14, 0, r1, cr0, cr0, {6}
> > c03101e0:       eaffffbf        b       c03100e4 <write_wb_reg+0x114>
> > c03101e4:       ee001ebf        mcr     14, 0, r1, cr0, cr15, {5}
> > c03101e8:       eaffffbd        b       c03100e4 <write_wb_reg+0x114>
> > c03101ec:       ee001ebe        mcr     14, 0, r1, cr0, cr14, {5}
> > c03101f0:       eaffffbb        b       c03100e4 <write_wb_reg+0x114>
> > c03101f4:       ee001ebd        mcr     14, 0, r1, cr0, cr13, {5}
> > c03101f8:       eaffffb9        b       c03100e4 <write_wb_reg+0x114>
> 
> FWIW, I was tracking an issue in this area before the holiday.
> 
> It looked like DBGPRSR.SPD is set unexpectedly over the default idle
> path (i.e. WFI), causing the (otherwise valid) register accesses above
> to be handled as undefined.
> 
> I haven't looked at the patch in detail, but I guess that it allows idle
> to occur between reset_ctrl_regs() and arch_hw_breakpoint_init().

I've just reproduced this locally on my dragonboard APQ8060.

It looks like the write_wb_reg() call that's exploding is from
get_max_wp_len(), which we call immediately after registering the
dbg_reset_online callback. Clearing DBGPRSR.SPD before the write_wb_reg() hides
the problem, so I suspect we're seeing the issue I mentioned above -- it just
so happens that we go idle in a new place.

The below hack allows boot to continue, but is not a real fix. I'm not
immediately sure what to do.

Linus, I wasn't able to get ethernet working. Do I need anything on top
of v4.10-rc2 && multi_v7_defconfig?

Thanks,
Mark.

---->8----
diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
index 188180b..a0982ab 100644
--- a/arch/arm/kernel/hw_breakpoint.c
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -302,7 +302,7 @@ int hw_breakpoint_slots(int type)
  */
 static u8 get_max_wp_len(void)
 {
-       u32 ctrl_reg;
+       u32 ctrl_reg, val;
        struct arch_hw_breakpoint_ctrl ctrl;
        u8 size = 4;
 
@@ -313,6 +313,9 @@ static u8 get_max_wp_len(void)
        ctrl.len = ARM_BREAKPOINT_LEN_8;
        ctrl_reg = encode_ctrl_reg(ctrl);
 
+       /* HACK: CLEAR SPD */
+       ARM_DBG_READ(c1, c5, 4, val);
+
        write_wb_reg(ARM_BASE_WVR, 0);
        write_wb_reg(ARM_BASE_WCR, ctrl_reg);
        if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Will Deacon]

On Wed, Jan 04, 2017 at 01:56:44PM +0000, Mark Rutland wrote:
> On Tue, Jan 03, 2017 at 09:33:36AM +0000, Mark Rutland wrote:
> > On Mon, Jan 02, 2017 at 09:15:29PM +0100, Linus Walleij wrote:
> > > On Mon, Jan 2, 2017 at 4:00 PM, Russell King - ARM Linux
> > > <linux@armlinux.org.uk> wrote:
> > > > On Mon, Jan 02, 2017 at 03:34:32PM +0100, Linus Walleij wrote:
> > > >> in the first line of arch_hw_breakpoint_init() in
> > > >> arch/arm/kernel/hw_breakpoint.c
> > > >>
> > > >> I suspect that is not an accepable solution ...
> > > >>
> > > >> It hangs at PC is at write_wb_reg+0x20c/0x330
> > > >> Which is c03101dc, and looks like this in objdump -d:
> > > >>
> > > >> c031020c:       ee001eba        mcr     14, 0, r1, cr0, cr10, {5}
> > > >> c0310210:       eaffffb3        b       c03100e4 <write_wb_reg+0x114>
> > > >
> > > > ... and this is several instructions after the address you mention above.
> > > > Presumably c03101dc is accessing a higher numbered register?
> > > 
> > > Ah sorry. It looks like this:
> > > 
> > > c03101dc:       ee001ed0        mcr     14, 0, r1, cr0, cr0, {6}
> > > c03101e0:       eaffffbf        b       c03100e4 <write_wb_reg+0x114>
> > > c03101e4:       ee001ebf        mcr     14, 0, r1, cr0, cr15, {5}
> > > c03101e8:       eaffffbd        b       c03100e4 <write_wb_reg+0x114>
> > > c03101ec:       ee001ebe        mcr     14, 0, r1, cr0, cr14, {5}
> > > c03101f0:       eaffffbb        b       c03100e4 <write_wb_reg+0x114>
> > > c03101f4:       ee001ebd        mcr     14, 0, r1, cr0, cr13, {5}
> > > c03101f8:       eaffffb9        b       c03100e4 <write_wb_reg+0x114>
> > 
> > FWIW, I was tracking an issue in this area before the holiday.
> > 
> > It looked like DBGPRSR.SPD is set unexpectedly over the default idle
> > path (i.e. WFI), causing the (otherwise valid) register accesses above
> > to be handled as undefined.
> > 
> > I haven't looked at the patch in detail, but I guess that it allows idle
> > to occur between reset_ctrl_regs() and arch_hw_breakpoint_init().
> 
> I've just reproduced this locally on my dragonboard APQ8060.
> 
> It looks like the write_wb_reg() call that's exploding is from
> get_max_wp_len(), which we call immediately after registering the
> dbg_reset_online callback. Clearing DBGPRSR.SPD before the write_wb_reg() hides
> the problem, so I suspect we're seeing the issue I mentioned above -- it just
> so happens that we go idle in a new place.

When you say "go idle", are we just executing a WFI, or is the power
controller coming into play and we're actually powering down the non-debug
logic? In the case of the latter, the PM notifier should clear SPD in
reset_ctrl_regs, so this sounds like a hardware bug where the SPD bit is
set unconditionally on WFI.

In that case, this code has always been dodgy -- what happens if you try
to use hardware breakpoints in GDB in the face of WFI-based idle?

> The below hack allows boot to continue, but is not a real fix. I'm not
> immediately sure what to do.

If it's never worked, I suggest we blacklist the MIDR until somebody from
Qualcomm can help us further.

Will

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Mark Rutland]

On Wed, Jan 04, 2017 at 02:32:06PM +0000, Will Deacon wrote:
> On Wed, Jan 04, 2017 at 01:56:44PM +0000, Mark Rutland wrote:
> > On Tue, Jan 03, 2017 at 09:33:36AM +0000, Mark Rutland wrote:
> > > On Mon, Jan 02, 2017 at 09:15:29PM +0100, Linus Walleij wrote:
> > > > On Mon, Jan 2, 2017 at 4:00 PM, Russell King - ARM Linux
> > > > <linux@armlinux.org.uk> wrote:
> > > > > On Mon, Jan 02, 2017 at 03:34:32PM +0100, Linus Walleij wrote:
> > > > >> in the first line of arch_hw_breakpoint_init() in
> > > > >> arch/arm/kernel/hw_breakpoint.c
> > > > >>
> > > > >> I suspect that is not an accepable solution ...
> > > > >>
> > > > >> It hangs at PC is at write_wb_reg+0x20c/0x330
> > > > >> Which is c03101dc, and looks like this in objdump -d:
> > > > >>
> > > > >> c031020c:       ee001eba        mcr     14, 0, r1, cr0, cr10, {5}
> > > > >> c0310210:       eaffffb3        b       c03100e4 <write_wb_reg+0x114>
> > > > >
> > > > > ... and this is several instructions after the address you mention above.
> > > > > Presumably c03101dc is accessing a higher numbered register?
> > > > 
> > > > Ah sorry. It looks like this:
> > > > 
> > > > c03101dc:       ee001ed0        mcr     14, 0, r1, cr0, cr0, {6}
> > > > c03101e0:       eaffffbf        b       c03100e4 <write_wb_reg+0x114>
> > > > c03101e4:       ee001ebf        mcr     14, 0, r1, cr0, cr15, {5}
> > > > c03101e8:       eaffffbd        b       c03100e4 <write_wb_reg+0x114>
> > > > c03101ec:       ee001ebe        mcr     14, 0, r1, cr0, cr14, {5}
> > > > c03101f0:       eaffffbb        b       c03100e4 <write_wb_reg+0x114>
> > > > c03101f4:       ee001ebd        mcr     14, 0, r1, cr0, cr13, {5}
> > > > c03101f8:       eaffffb9        b       c03100e4 <write_wb_reg+0x114>
> > > 
> > > FWIW, I was tracking an issue in this area before the holiday.
> > > 
> > > It looked like DBGPRSR.SPD is set unexpectedly over the default idle
> > > path (i.e. WFI), causing the (otherwise valid) register accesses above
> > > to be handled as undefined.
> > > 
> > > I haven't looked at the patch in detail, but I guess that it allows idle
> > > to occur between reset_ctrl_regs() and arch_hw_breakpoint_init().
> > 
> > I've just reproduced this locally on my dragonboard APQ8060.
> > 
> > It looks like the write_wb_reg() call that's exploding is from
> > get_max_wp_len(), which we call immediately after registering the
> > dbg_reset_online callback. Clearing DBGPRSR.SPD before the write_wb_reg() hides
> > the problem, so I suspect we're seeing the issue I mentioned above -- it just
> > so happens that we go idle in a new place.
> 
> When you say "go idle", are we just executing a WFI, 

>From my prior experiments, just executing a WFI as we happen to do in
the default cpu_v7_do_idle. I tried passing cpuidle.off=1, but that
didn't help. NOPing the WFI in cpu_v7_do_idle did mask the issue, from
what I recall.

> or is the power controller coming into play and we're actually
> powering down the non-debug logic?

As far as I can see, that isn't happening. We don't save/restore any
other CPU state in the default idle path, and the kernel is otherwise
happy.

> In the case of the latter, the PM notifier should clear SPD in
> reset_ctrl_regs, so this sounds like a hardware bug where the SPD bit is
> set unconditionally on WFI.
> 
> In that case, this code has always been dodgy -- what happens if you try
> to use hardware breakpoints in GDB in the face of WFI-based idle?

The kernel blows up similiarly to Linus's original report when the
kernel tries to program the breakpoint registers.

I also believe this has always been dodgy.

> > The below hack allows boot to continue, but is not a real fix. I'm not
> > immediately sure what to do.
> 
> If it's never worked, I suggest we blacklist the MIDR until somebody from
> Qualcomm can help us further.

I'll see about putting a patch together.

Thanks,
Mark.

> 
> Will

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Linus Walleij]

On Wed, Jan 4, 2017 at 2:56 PM, Mark Rutland <mark.rutland@arm.com> wrote:

> I've just reproduced this locally on my dragonboard APQ8060.

Sweet!

> It looks like the write_wb_reg() call that's exploding is from
> get_max_wp_len(), which we call immediately after registering the
> dbg_reset_online callback. Clearing DBGPRSR.SPD before the write_wb_reg() hides
> the problem, so I suspect we're seeing the issue I mentioned above -- it just
> so happens that we go idle in a new place.
>
> The below hack allows boot to continue, but is not a real fix. I'm not
> immediately sure what to do.

Me neither. But Will's suggestion to simply blacklist this chip might be
best.

> Linus, I wasn't able to get ethernet working. Do I need anything on top
> of v4.10-rc2 && multi_v7_defconfig?

I haven't tried it with multi_v7 but I should probably try that and patch
up the defconfigs, those are probably the root of the problem.

I do this on top of qcom_defconfig:

scripts/config --file .config \
        --enable QCOM_EBI2 \
        --enable ETHERNET \
        --enable NET_VENDOR_SMSC \
        --enable SMSC911X

Maybe you are missing the EBI2 config?

Yours,
Linus Walleij

Re: [PATCH 15/20] ARM/hw_breakpoint: Convert to hotplug state machine

[Mark Rutland]

On Thu, Jan 05, 2017 at 04:26:45PM +0100, Linus Walleij wrote:
> On Wed, Jan 4, 2017 at 2:56 PM, Mark Rutland <mark.rutland@arm.com> wrote:
 
> > Linus, I wasn't able to get ethernet working. Do I need anything on top
> > of v4.10-rc2 && multi_v7_defconfig?
> 
> I haven't tried it with multi_v7 but I should probably try that and patch
> up the defconfigs, those are probably the root of the problem.
> 
> I do this on top of qcom_defconfig:
> 
> scripts/config --file .config \
>         --enable QCOM_EBI2 \
>         --enable ETHERNET \
>         --enable NET_VENDOR_SMSC \
>         --enable SMSC911X
> 
> Maybe you are missing the EBI2 config?

That was it, yes! With QCOM_EBI2 atop of multi_v7_defconfig (along with
a hack to the hw_breakpoint code), I can boot to an NFS filesystem.

Thanks,
Mark.

[PATCH 16/20] powerpc/sysfs: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The previous convention of keeping the files around until the CPU is dead
has not been preserved as there is no point to keep them available when the
cpu is going down. This makes the hotplug call symmetric.

Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: linuxppc-dev@lists.ozlabs.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/powerpc/kernel/sysfs.c | 50 +++++++++------------------------------------
 1 file changed, 10 insertions(+), 40 deletions(-)

diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c
index c4f1d1f7bae0..c1fb255a60d6 100644
--- a/arch/powerpc/kernel/sysfs.c
+++ b/arch/powerpc/kernel/sysfs.c
@@ -703,7 +703,7 @@ static struct device_attribute pa6t_attrs[] = {
 #endif /* HAS_PPC_PMC_PA6T */
 #endif /* HAS_PPC_PMC_CLASSIC */
 
-static void register_cpu_online(unsigned int cpu)
+static int register_cpu_online(unsigned int cpu)
 {
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
@@ -782,11 +782,12 @@ static void register_cpu_online(unsigned int cpu)
 	}
 #endif
 	cacheinfo_cpu_online(cpu);
+	return 0;
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-static void unregister_cpu_online(unsigned int cpu)
+static int unregister_cpu_online(unsigned int cpu)
 {
+#ifdef CONFIG_HOTPLUG_CPU
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
 	struct device_attribute *attrs, *pmc_attrs;
@@ -863,6 +864,8 @@ static void unregister_cpu_online(unsigned int cpu)
 	}
 #endif
 	cacheinfo_cpu_offline(cpu);
+#endif /* CONFIG_HOTPLUG_CPU */
+	return 0;
 }
 
 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
@@ -883,32 +886,6 @@ ssize_t arch_cpu_release(const char *buf, size_t count)
 }
 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
 
-#endif /* CONFIG_HOTPLUG_CPU */
-
-static int sysfs_cpu_notify(struct notifier_block *self,
-				      unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned int)(long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		register_cpu_online(cpu);
-		break;
-#ifdef CONFIG_HOTPLUG_CPU
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		unregister_cpu_online(cpu);
-		break;
-#endif
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block sysfs_cpu_nb = {
-	.notifier_call	= sysfs_cpu_notify,
-};
-
 static DEFINE_MUTEX(cpu_mutex);
 
 int cpu_add_dev_attr(struct device_attribute *attr)
@@ -1023,12 +1000,10 @@ static DEVICE_ATTR(physical_id, 0444, show_physical_id, NULL);
 
 static int __init topology_init(void)
 {
-	int cpu;
+	int cpu, r;
 
 	register_nodes();
 
-	cpu_notifier_register_begin();
-
 	for_each_possible_cpu(cpu) {
 		struct cpu *c = &per_cpu(cpu_devices, cpu);
 
@@ -1047,15 +1022,10 @@ static int __init topology_init(void)
 
 			device_create_file(&c->dev, &dev_attr_physical_id);
 		}
-
-		if (cpu_online(cpu))
-			register_cpu_online(cpu);
 	}
-
-	__register_cpu_notifier(&sysfs_cpu_nb);
-
-	cpu_notifier_register_done();
-
+	r = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powerpc/topology:online",
+			      register_cpu_online, unregister_cpu_online);
+	WARN_ON(r < 0);
 #ifdef CONFIG_PPC64
 	sysfs_create_dscr_default();
 #endif /* CONFIG_PPC64 */
-- 
2.10.2

[tip:smp/hotplug] powerpc/sysfs: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  2e261004c45e0641492053c7c8ec523569b73eb3
Gitweb:     http://git.kernel.org/tip/2e261004c45e0641492053c7c8ec523569b73eb3
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:37 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:08 +0100

powerpc/sysfs: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The previous convention of keeping the files around until the CPU is dead
has not been preserved as there is no point to keep them available when the
cpu is going down. This makes the hotplug call symmetric.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: rt@linuxtronix.de
Cc: Paul Mackerras <paulus@samba.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: linuxppc-dev@lists.ozlabs.org
Link: http://lkml.kernel.org/r/20161117183541.8588-17-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/powerpc/kernel/sysfs.c | 50 +++++++++------------------------------------
 1 file changed, 10 insertions(+), 40 deletions(-)

diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c
index c4f1d1f..c1fb255 100644
--- a/arch/powerpc/kernel/sysfs.c
+++ b/arch/powerpc/kernel/sysfs.c
@@ -703,7 +703,7 @@ static struct device_attribute pa6t_attrs[] = {
 #endif /* HAS_PPC_PMC_PA6T */
 #endif /* HAS_PPC_PMC_CLASSIC */
 
-static void register_cpu_online(unsigned int cpu)
+static int register_cpu_online(unsigned int cpu)
 {
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
@@ -782,11 +782,12 @@ static void register_cpu_online(unsigned int cpu)
 	}
 #endif
 	cacheinfo_cpu_online(cpu);
+	return 0;
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-static void unregister_cpu_online(unsigned int cpu)
+static int unregister_cpu_online(unsigned int cpu)
 {
+#ifdef CONFIG_HOTPLUG_CPU
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
 	struct device_attribute *attrs, *pmc_attrs;
@@ -863,6 +864,8 @@ static void unregister_cpu_online(unsigned int cpu)
 	}
 #endif
 	cacheinfo_cpu_offline(cpu);
+#endif /* CONFIG_HOTPLUG_CPU */
+	return 0;
 }
 
 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
@@ -883,32 +886,6 @@ ssize_t arch_cpu_release(const char *buf, size_t count)
 }
 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
 
-#endif /* CONFIG_HOTPLUG_CPU */
-
-static int sysfs_cpu_notify(struct notifier_block *self,
-				      unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned int)(long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		register_cpu_online(cpu);
-		break;
-#ifdef CONFIG_HOTPLUG_CPU
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		unregister_cpu_online(cpu);
-		break;
-#endif
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block sysfs_cpu_nb = {
-	.notifier_call	= sysfs_cpu_notify,
-};
-
 static DEFINE_MUTEX(cpu_mutex);
 
 int cpu_add_dev_attr(struct device_attribute *attr)
@@ -1023,12 +1000,10 @@ static DEVICE_ATTR(physical_id, 0444, show_physical_id, NULL);
 
 static int __init topology_init(void)
 {
-	int cpu;
+	int cpu, r;
 
 	register_nodes();
 
-	cpu_notifier_register_begin();
-
 	for_each_possible_cpu(cpu) {
 		struct cpu *c = &per_cpu(cpu_devices, cpu);
 
@@ -1047,15 +1022,10 @@ static int __init topology_init(void)
 
 			device_create_file(&c->dev, &dev_attr_physical_id);
 		}
-
-		if (cpu_online(cpu))
-			register_cpu_online(cpu);
 	}
-
-	__register_cpu_notifier(&sysfs_cpu_nb);
-
-	cpu_notifier_register_done();
-
+	r = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powerpc/topology:online",
+			      register_cpu_online, unregister_cpu_online);
+	WARN_ON(r < 0);
 #ifdef CONFIG_PPC64
 	sysfs_create_dscr_default();
 #endif /* CONFIG_PPC64 */

[tip:smp/hotplug] powerpc/sysfs: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  977ab257a2b327f161728ab08bc618d770cc92ad
Gitweb:     http://git.kernel.org/tip/977ab257a2b327f161728ab08bc618d770cc92ad
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:37 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:42 +0100

powerpc/sysfs: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The previous convention of keeping the files around until the CPU is dead
has not been preserved as there is no point to keep them available when the
cpu is going down. This makes the hotplug call symmetric.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: rt@linuxtronix.de
Cc: Paul Mackerras <paulus@samba.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: linuxppc-dev@lists.ozlabs.org
Link: http://lkml.kernel.org/r/20161117183541.8588-17-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/powerpc/kernel/sysfs.c | 50 +++++++++------------------------------------
 1 file changed, 10 insertions(+), 40 deletions(-)

diff --git a/arch/powerpc/kernel/sysfs.c b/arch/powerpc/kernel/sysfs.c
index c4f1d1f..c1fb255 100644
--- a/arch/powerpc/kernel/sysfs.c
+++ b/arch/powerpc/kernel/sysfs.c
@@ -703,7 +703,7 @@ static struct device_attribute pa6t_attrs[] = {
 #endif /* HAS_PPC_PMC_PA6T */
 #endif /* HAS_PPC_PMC_CLASSIC */
 
-static void register_cpu_online(unsigned int cpu)
+static int register_cpu_online(unsigned int cpu)
 {
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
@@ -782,11 +782,12 @@ static void register_cpu_online(unsigned int cpu)
 	}
 #endif
 	cacheinfo_cpu_online(cpu);
+	return 0;
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-static void unregister_cpu_online(unsigned int cpu)
+static int unregister_cpu_online(unsigned int cpu)
 {
+#ifdef CONFIG_HOTPLUG_CPU
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
 	struct device_attribute *attrs, *pmc_attrs;
@@ -863,6 +864,8 @@ static void unregister_cpu_online(unsigned int cpu)
 	}
 #endif
 	cacheinfo_cpu_offline(cpu);
+#endif /* CONFIG_HOTPLUG_CPU */
+	return 0;
 }
 
 #ifdef CONFIG_ARCH_CPU_PROBE_RELEASE
@@ -883,32 +886,6 @@ ssize_t arch_cpu_release(const char *buf, size_t count)
 }
 #endif /* CONFIG_ARCH_CPU_PROBE_RELEASE */
 
-#endif /* CONFIG_HOTPLUG_CPU */
-
-static int sysfs_cpu_notify(struct notifier_block *self,
-				      unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned int)(long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		register_cpu_online(cpu);
-		break;
-#ifdef CONFIG_HOTPLUG_CPU
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		unregister_cpu_online(cpu);
-		break;
-#endif
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block sysfs_cpu_nb = {
-	.notifier_call	= sysfs_cpu_notify,
-};
-
 static DEFINE_MUTEX(cpu_mutex);
 
 int cpu_add_dev_attr(struct device_attribute *attr)
@@ -1023,12 +1000,10 @@ static DEVICE_ATTR(physical_id, 0444, show_physical_id, NULL);
 
 static int __init topology_init(void)
 {
-	int cpu;
+	int cpu, r;
 
 	register_nodes();
 
-	cpu_notifier_register_begin();
-
 	for_each_possible_cpu(cpu) {
 		struct cpu *c = &per_cpu(cpu_devices, cpu);
 
@@ -1047,15 +1022,10 @@ static int __init topology_init(void)
 
 			device_create_file(&c->dev, &dev_attr_physical_id);
 		}
-
-		if (cpu_online(cpu))
-			register_cpu_online(cpu);
 	}
-
-	__register_cpu_notifier(&sysfs_cpu_nb);
-
-	cpu_notifier_register_done();
-
+	r = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "powerpc/topology:online",
+			      register_cpu_online, unregister_cpu_online);
+	WARN_ON(r < 0);
 #ifdef CONFIG_PPC64
 	sysfs_create_dscr_default();
 #endif /* CONFIG_PPC64 */

[PATCH 17/20] sparc/sysfs: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The previous convention of keeping the files around until the CPU is dead
has not been preserved as there is no point to keep them available when the
cpu is going down. This makes the hotplug call symmetric.

Cc: "David S. Miller" <davem@davemloft.net>
Cc: sparclinux@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/sparc/kernel/sysfs.c | 45 +++++++++------------------------------------
 1 file changed, 9 insertions(+), 36 deletions(-)

diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c
index fa8e21abb5e0..4808b6d23455 100644
--- a/arch/sparc/kernel/sysfs.c
+++ b/arch/sparc/kernel/sysfs.c
@@ -221,7 +221,7 @@ static struct device_attribute cpu_core_attrs[] = {
 
 static DEFINE_PER_CPU(struct cpu, cpu_devices);
 
-static void register_cpu_online(unsigned int cpu)
+static int register_cpu_online(unsigned int cpu)
 {
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
@@ -231,11 +231,12 @@ static void register_cpu_online(unsigned int cpu)
 		device_create_file(s, &cpu_core_attrs[i]);
 
 	register_mmu_stats(s);
+	return 0;
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-static void unregister_cpu_online(unsigned int cpu)
+static int unregister_cpu_online(unsigned int cpu)
 {
+#ifdef CONFIG_HOTPLUG_CPU
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
 	int i;
@@ -243,33 +244,10 @@ static void unregister_cpu_online(unsigned int cpu)
 	unregister_mmu_stats(s);
 	for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++)
 		device_remove_file(s, &cpu_core_attrs[i]);
-}
 #endif
-
-static int sysfs_cpu_notify(struct notifier_block *self,
-				      unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned int)(long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		register_cpu_online(cpu);
-		break;
-#ifdef CONFIG_HOTPLUG_CPU
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		unregister_cpu_online(cpu);
-		break;
-#endif
-	}
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block sysfs_cpu_nb = {
-	.notifier_call	= sysfs_cpu_notify,
-};
-
 static void __init check_mmu_stats(void)
 {
 	unsigned long dummy1, err;
@@ -294,26 +272,21 @@ static void register_nodes(void)
 
 static int __init topology_init(void)
 {
-	int cpu;
+	int cpu, ret;
 
 	register_nodes();
 
 	check_mmu_stats();
 
-	cpu_notifier_register_begin();
-
 	for_each_possible_cpu(cpu) {
 		struct cpu *c = &per_cpu(cpu_devices, cpu);
 
 		register_cpu(c, cpu);
-		if (cpu_online(cpu))
-			register_cpu_online(cpu);
 	}
 
-	__register_cpu_notifier(&sysfs_cpu_nb);
-
-	cpu_notifier_register_done();
-
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "sparc/topology:online",
+				register_cpu_online, unregister_cpu_online);
+	WARN_ON(ret < 0);
 	return 0;
 }
 
-- 
2.10.2

Re: [PATCH 17/20] sparc/sysfs: Convert to hotplug state machine

[David Miller]

From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Date: Thu, 17 Nov 2016 19:35:38 +0100

> Install the callbacks via the state machine and let the core invoke
> the callbacks on the already online CPUs.
> 
> The previous convention of keeping the files around until the CPU is dead
> has not been preserved as there is no point to keep them available when the
> cpu is going down. This makes the hotplug call symmetric.
> 
> Cc: "David S. Miller" <davem@davemloft.net>
> Cc: sparclinux@vger.kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

Acked-by: David S. Miller <davem@davemloft.net>

[tip:smp/hotplug] sparc/sysfs: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  2c327901e8c588a8ea5ead6e2732feafc8f05af4
Gitweb:     http://git.kernel.org/tip/2c327901e8c588a8ea5ead6e2732feafc8f05af4
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:38 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:09 +0100

sparc/sysfs: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The previous convention of keeping the files around until the CPU is dead
has not been preserved as there is no point to keep them available when the
cpu is going down. This makes the hotplug call symmetric.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: "David S. Miller" <davem@davemloft.net>
Cc: sparclinux@vger.kernel.org
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-18-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/sparc/kernel/sysfs.c | 45 +++++++++------------------------------------
 1 file changed, 9 insertions(+), 36 deletions(-)

diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c
index fa8e21a..4808b6d 100644
--- a/arch/sparc/kernel/sysfs.c
+++ b/arch/sparc/kernel/sysfs.c
@@ -221,7 +221,7 @@ static struct device_attribute cpu_core_attrs[] = {
 
 static DEFINE_PER_CPU(struct cpu, cpu_devices);
 
-static void register_cpu_online(unsigned int cpu)
+static int register_cpu_online(unsigned int cpu)
 {
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
@@ -231,11 +231,12 @@ static void register_cpu_online(unsigned int cpu)
 		device_create_file(s, &cpu_core_attrs[i]);
 
 	register_mmu_stats(s);
+	return 0;
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-static void unregister_cpu_online(unsigned int cpu)
+static int unregister_cpu_online(unsigned int cpu)
 {
+#ifdef CONFIG_HOTPLUG_CPU
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
 	int i;
@@ -243,33 +244,10 @@ static void unregister_cpu_online(unsigned int cpu)
 	unregister_mmu_stats(s);
 	for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++)
 		device_remove_file(s, &cpu_core_attrs[i]);
-}
-#endif
-
-static int sysfs_cpu_notify(struct notifier_block *self,
-				      unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned int)(long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		register_cpu_online(cpu);
-		break;
-#ifdef CONFIG_HOTPLUG_CPU
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		unregister_cpu_online(cpu);
-		break;
 #endif
-	}
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block sysfs_cpu_nb = {
-	.notifier_call	= sysfs_cpu_notify,
-};
-
 static void __init check_mmu_stats(void)
 {
 	unsigned long dummy1, err;
@@ -294,26 +272,21 @@ static void register_nodes(void)
 
 static int __init topology_init(void)
 {
-	int cpu;
+	int cpu, ret;
 
 	register_nodes();
 
 	check_mmu_stats();
 
-	cpu_notifier_register_begin();
-
 	for_each_possible_cpu(cpu) {
 		struct cpu *c = &per_cpu(cpu_devices, cpu);
 
 		register_cpu(c, cpu);
-		if (cpu_online(cpu))
-			register_cpu_online(cpu);
 	}
 
-	__register_cpu_notifier(&sysfs_cpu_nb);
-
-	cpu_notifier_register_done();
-
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "sparc/topology:online",
+				register_cpu_online, unregister_cpu_online);
+	WARN_ON(ret < 0);
 	return 0;
 }
 

[tip:smp/hotplug] sparc/sysfs: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  e5355cd6e78c096b66c26de8c9e5680ddba71c1f
Gitweb:     http://git.kernel.org/tip/e5355cd6e78c096b66c26de8c9e5680ddba71c1f
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:38 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:42 +0100

sparc/sysfs: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The previous convention of keeping the files around until the CPU is dead
has not been preserved as there is no point to keep them available when the
cpu is going down. This makes the hotplug call symmetric.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: "David S. Miller" <davem@davemloft.net>
Cc: sparclinux@vger.kernel.org
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-18-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/sparc/kernel/sysfs.c | 45 +++++++++------------------------------------
 1 file changed, 9 insertions(+), 36 deletions(-)

diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c
index fa8e21a..4808b6d 100644
--- a/arch/sparc/kernel/sysfs.c
+++ b/arch/sparc/kernel/sysfs.c
@@ -221,7 +221,7 @@ static struct device_attribute cpu_core_attrs[] = {
 
 static DEFINE_PER_CPU(struct cpu, cpu_devices);
 
-static void register_cpu_online(unsigned int cpu)
+static int register_cpu_online(unsigned int cpu)
 {
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
@@ -231,11 +231,12 @@ static void register_cpu_online(unsigned int cpu)
 		device_create_file(s, &cpu_core_attrs[i]);
 
 	register_mmu_stats(s);
+	return 0;
 }
 
-#ifdef CONFIG_HOTPLUG_CPU
-static void unregister_cpu_online(unsigned int cpu)
+static int unregister_cpu_online(unsigned int cpu)
 {
+#ifdef CONFIG_HOTPLUG_CPU
 	struct cpu *c = &per_cpu(cpu_devices, cpu);
 	struct device *s = &c->dev;
 	int i;
@@ -243,33 +244,10 @@ static void unregister_cpu_online(unsigned int cpu)
 	unregister_mmu_stats(s);
 	for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++)
 		device_remove_file(s, &cpu_core_attrs[i]);
-}
-#endif
-
-static int sysfs_cpu_notify(struct notifier_block *self,
-				      unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned int)(long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		register_cpu_online(cpu);
-		break;
-#ifdef CONFIG_HOTPLUG_CPU
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		unregister_cpu_online(cpu);
-		break;
 #endif
-	}
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block sysfs_cpu_nb = {
-	.notifier_call	= sysfs_cpu_notify,
-};
-
 static void __init check_mmu_stats(void)
 {
 	unsigned long dummy1, err;
@@ -294,26 +272,21 @@ static void register_nodes(void)
 
 static int __init topology_init(void)
 {
-	int cpu;
+	int cpu, ret;
 
 	register_nodes();
 
 	check_mmu_stats();
 
-	cpu_notifier_register_begin();
-
 	for_each_possible_cpu(cpu) {
 		struct cpu *c = &per_cpu(cpu_devices, cpu);
 
 		register_cpu(c, cpu);
-		if (cpu_online(cpu))
-			register_cpu_online(cpu);
 	}
 
-	__register_cpu_notifier(&sysfs_cpu_nb);
-
-	cpu_notifier_register_done();
-
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "sparc/topology:online",
+				register_cpu_online, unregister_cpu_online);
+	WARN_ON(ret < 0);
 	return 0;
 }
 

[PATCH 18/20] x86/oprofile/nmi: Remove superfluous smp_function_call_single()

[Sebastian Andrzej Siewior]

From: Anna-Maria Gleixner <anna-maria@linutronix.de>

Since commit 1cf4f629d9d2 ("cpu/hotplug: Move online calls to
hotplugged cpu") the CPU_ONLINE and CPU_DOWN_PREPARE notifiers are
always run on the hot plugged CPU, and as of commit 3b9d6da67e11
("cpu/hotplug: Fix rollback during error-out in __cpu_disable()")
the CPU_DOWN_FAILED notifier also runs on the hot plugged CPU.
This patch converts the SMP functional calls into direct calls.

smp_call_function_single() executes the function with interrupts
disabled. This calling convention is preserved.

Cc: Robert Richter <rric@kernel.org>
Cc: x86@kernel.org
Cc: oprofile-list@lists.sf.net
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/x86/oprofile/nmi_int.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 28c04123b6dd..c39172cd6c87 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -387,20 +387,24 @@ static void nmi_cpu_shutdown(void *dummy)
 	nmi_cpu_restore_registers(msrs);
 }
 
-static void nmi_cpu_up(void *dummy)
+static void nmi_cpu_up(void)
 {
+	local_irq_disable();
 	if (nmi_enabled)
-		nmi_cpu_setup(dummy);
+		nmi_cpu_setup(NULL);
 	if (ctr_running)
-		nmi_cpu_start(dummy);
+		nmi_cpu_start(NULL);
+	local_irq_enable();
 }
 
-static void nmi_cpu_down(void *dummy)
+static void nmi_cpu_down(void)
 {
+	local_irq_disable();
 	if (ctr_running)
-		nmi_cpu_stop(dummy);
+		nmi_cpu_stop(NULL);
 	if (nmi_enabled)
-		nmi_cpu_shutdown(dummy);
+		nmi_cpu_shutdown(NULL);
+	local_irq_enable();
 }
 
 static int nmi_create_files(struct dentry *root)
@@ -436,15 +440,13 @@ static int nmi_create_files(struct dentry *root)
 static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
 				 void *data)
 {
-	int cpu = (unsigned long)data;
-
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_DOWN_FAILED:
 	case CPU_ONLINE:
-		smp_call_function_single(cpu, nmi_cpu_up, NULL, 0);
+		nmi_cpu_up();
 		break;
 	case CPU_DOWN_PREPARE:
-		smp_call_function_single(cpu, nmi_cpu_down, NULL, 1);
+		nmi_cpu_down();
 		break;
 	}
 	return NOTIFY_DONE;
-- 
2.10.2

[tip:smp/hotplug] x86/oprofile/nmi: Remove superfluous smp_function_call_single()

[tip-bot for Anna-Maria Gleixner]

Commit-ID:  9cb37ce65de7a6c645de2ab536807a4ba244c83a
Gitweb:     http://git.kernel.org/tip/9cb37ce65de7a6c645de2ab536807a4ba244c83a
Author:     Anna-Maria Gleixner <anna-maria@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:39 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:09 +0100

x86/oprofile/nmi: Remove superfluous smp_function_call_single()

Since commit 1cf4f629d9d2 ("cpu/hotplug: Move online calls to
hotplugged cpu") the CPU_ONLINE and CPU_DOWN_PREPARE notifiers are
always run on the hot plugged CPU, and as of commit 3b9d6da67e11
("cpu/hotplug: Fix rollback during error-out in __cpu_disable()")
the CPU_DOWN_FAILED notifier also runs on the hot plugged CPU.
This patch converts the SMP functional calls into direct calls.

smp_call_function_single() executes the function with interrupts
disabled. This calling convention is preserved.

Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Robert Richter <rric@kernel.org>
Cc: rt@linuxtronix.de
Cc: oprofile-list@lists.sf.net
Link: http://lkml.kernel.org/r/20161117183541.8588-19-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/oprofile/nmi_int.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 28c0412..c39172c 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -387,20 +387,24 @@ static void nmi_cpu_shutdown(void *dummy)
 	nmi_cpu_restore_registers(msrs);
 }
 
-static void nmi_cpu_up(void *dummy)
+static void nmi_cpu_up(void)
 {
+	local_irq_disable();
 	if (nmi_enabled)
-		nmi_cpu_setup(dummy);
+		nmi_cpu_setup(NULL);
 	if (ctr_running)
-		nmi_cpu_start(dummy);
+		nmi_cpu_start(NULL);
+	local_irq_enable();
 }
 
-static void nmi_cpu_down(void *dummy)
+static void nmi_cpu_down(void)
 {
+	local_irq_disable();
 	if (ctr_running)
-		nmi_cpu_stop(dummy);
+		nmi_cpu_stop(NULL);
 	if (nmi_enabled)
-		nmi_cpu_shutdown(dummy);
+		nmi_cpu_shutdown(NULL);
+	local_irq_enable();
 }
 
 static int nmi_create_files(struct dentry *root)
@@ -436,15 +440,13 @@ static int nmi_create_files(struct dentry *root)
 static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
 				 void *data)
 {
-	int cpu = (unsigned long)data;
-
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_DOWN_FAILED:
 	case CPU_ONLINE:
-		smp_call_function_single(cpu, nmi_cpu_up, NULL, 0);
+		nmi_cpu_up();
 		break;
 	case CPU_DOWN_PREPARE:
-		smp_call_function_single(cpu, nmi_cpu_down, NULL, 1);
+		nmi_cpu_down();
 		break;
 	}
 	return NOTIFY_DONE;

[tip:smp/hotplug] x86/oprofile/nmi: Remove superfluous smp_function_call_single()

[tip-bot for Anna-Maria Gleixner]

Commit-ID:  08ed487c819d285c3f6ceafd156094102acdfec2
Gitweb:     http://git.kernel.org/tip/08ed487c819d285c3f6ceafd156094102acdfec2
Author:     Anna-Maria Gleixner <anna-maria@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:39 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:42 +0100

x86/oprofile/nmi: Remove superfluous smp_function_call_single()

Since commit 1cf4f629d9d2 ("cpu/hotplug: Move online calls to
hotplugged cpu") the CPU_ONLINE and CPU_DOWN_PREPARE notifiers are
always run on the hot plugged CPU, and as of commit 3b9d6da67e11
("cpu/hotplug: Fix rollback during error-out in __cpu_disable()")
the CPU_DOWN_FAILED notifier also runs on the hot plugged CPU.
This patch converts the SMP functional calls into direct calls.

smp_call_function_single() executes the function with interrupts
disabled. This calling convention is preserved.

Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Robert Richter <rric@kernel.org>
Cc: rt@linuxtronix.de
Cc: oprofile-list@lists.sf.net
Link: http://lkml.kernel.org/r/20161117183541.8588-19-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/oprofile/nmi_int.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index 28c0412..c39172c 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -387,20 +387,24 @@ static void nmi_cpu_shutdown(void *dummy)
 	nmi_cpu_restore_registers(msrs);
 }
 
-static void nmi_cpu_up(void *dummy)
+static void nmi_cpu_up(void)
 {
+	local_irq_disable();
 	if (nmi_enabled)
-		nmi_cpu_setup(dummy);
+		nmi_cpu_setup(NULL);
 	if (ctr_running)
-		nmi_cpu_start(dummy);
+		nmi_cpu_start(NULL);
+	local_irq_enable();
 }
 
-static void nmi_cpu_down(void *dummy)
+static void nmi_cpu_down(void)
 {
+	local_irq_disable();
 	if (ctr_running)
-		nmi_cpu_stop(dummy);
+		nmi_cpu_stop(NULL);
 	if (nmi_enabled)
-		nmi_cpu_shutdown(dummy);
+		nmi_cpu_shutdown(NULL);
+	local_irq_enable();
 }
 
 static int nmi_create_files(struct dentry *root)
@@ -436,15 +440,13 @@ static int nmi_create_files(struct dentry *root)
 static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
 				 void *data)
 {
-	int cpu = (unsigned long)data;
-
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_DOWN_FAILED:
 	case CPU_ONLINE:
-		smp_call_function_single(cpu, nmi_cpu_up, NULL, 0);
+		nmi_cpu_up();
 		break;
 	case CPU_DOWN_PREPARE:
-		smp_call_function_single(cpu, nmi_cpu_down, NULL, 1);
+		nmi_cpu_down();
 		break;
 	}
 	return NOTIFY_DONE;

[PATCH 19/20] x86/oprofile/nmi: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: Robert Richter <rric@kernel.org>
Cc: x86@kernel.org
Cc: oprofile-list@lists.sf.net
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/x86/oprofile/nmi_int.c | 61 +++++++++++++--------------------------------
 1 file changed, 18 insertions(+), 43 deletions(-)

diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index c39172cd6c87..47b4cbf8da68 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -339,10 +339,11 @@ static int allocate_msrs(void)
 	return 0;
 }
 
-static void nmi_cpu_setup(void *dummy)
+static void nmi_cpu_setup(void)
 {
 	int cpu = smp_processor_id();
 	struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
+
 	nmi_cpu_save_registers(msrs);
 	raw_spin_lock(&oprofilefs_lock);
 	model->setup_ctrs(model, msrs);
@@ -369,7 +370,7 @@ static void nmi_cpu_restore_registers(struct op_msrs *msrs)
 	}
 }
 
-static void nmi_cpu_shutdown(void *dummy)
+static void nmi_cpu_shutdown(void)
 {
 	unsigned int v;
 	int cpu = smp_processor_id();
@@ -387,24 +388,26 @@ static void nmi_cpu_shutdown(void *dummy)
 	nmi_cpu_restore_registers(msrs);
 }
 
-static void nmi_cpu_up(void)
+static int nmi_cpu_online(unsigned int cpu)
 {
 	local_irq_disable();
 	if (nmi_enabled)
-		nmi_cpu_setup(NULL);
+		nmi_cpu_setup();
 	if (ctr_running)
 		nmi_cpu_start(NULL);
 	local_irq_enable();
+	return 0;
 }
 
-static void nmi_cpu_down(void)
+static int nmi_cpu_down_prep(unsigned int cpu)
 {
 	local_irq_disable();
 	if (ctr_running)
 		nmi_cpu_stop(NULL);
 	if (nmi_enabled)
-		nmi_cpu_shutdown(NULL);
+		nmi_cpu_shutdown();
 	local_irq_enable();
+	return 0;
 }
 
 static int nmi_create_files(struct dentry *root)
@@ -437,24 +440,7 @@ static int nmi_create_files(struct dentry *root)
 	return 0;
 }
 
-static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
-				 void *data)
-{
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_FAILED:
-	case CPU_ONLINE:
-		nmi_cpu_up();
-		break;
-	case CPU_DOWN_PREPARE:
-		nmi_cpu_down();
-		break;
-	}
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block oprofile_cpu_nb = {
-	.notifier_call = oprofile_cpu_notifier
-};
+static enum cpuhp_state hp_online;
 
 static int nmi_setup(void)
 {
@@ -497,20 +483,17 @@ static int nmi_setup(void)
 	if (err)
 		goto fail;
 
-	cpu_notifier_register_begin();
-
-	/* Use get/put_online_cpus() to protect 'nmi_enabled' */
-	get_online_cpus();
 	nmi_enabled = 1;
 	/* make nmi_enabled visible to the nmi handler: */
 	smp_mb();
-	on_each_cpu(nmi_cpu_setup, NULL, 1);
-	__register_cpu_notifier(&oprofile_cpu_nb);
-	put_online_cpus();
-
-	cpu_notifier_register_done();
-
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/oprofile:online",
+				nmi_cpu_online, nmi_cpu_down_prep);
+	if (err < 0)
+		goto fail_nmi;
+	hp_online = err;
 	return 0;
+fail_nmi:
+	unregister_nmi_handler(NMI_LOCAL, "oprofile");
 fail:
 	free_msrs();
 	return err;
@@ -520,17 +503,9 @@ static void nmi_shutdown(void)
 {
 	struct op_msrs *msrs;
 
-	cpu_notifier_register_begin();
-
-	/* Use get/put_online_cpus() to protect 'nmi_enabled' & 'ctr_running' */
-	get_online_cpus();
-	on_each_cpu(nmi_cpu_shutdown, NULL, 1);
+	cpuhp_remove_state(hp_online);
 	nmi_enabled = 0;
 	ctr_running = 0;
-	__unregister_cpu_notifier(&oprofile_cpu_nb);
-	put_online_cpus();
-
-	cpu_notifier_register_done();
 
 	/* make variables visible to the nmi handler: */
 	smp_mb();
-- 
2.10.2

[tip:smp/hotplug] x86/oprofile/nmi: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  cff817393431c3cc6fc3eaaeb8511ee0d16116aa
Gitweb:     http://git.kernel.org/tip/cff817393431c3cc6fc3eaaeb8511ee0d16116aa
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:40 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:09 +0100

x86/oprofile/nmi: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Robert Richter <rric@kernel.org>
Cc: oprofile-list@lists.sf.net
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-20-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/oprofile/nmi_int.c | 61 +++++++++++++--------------------------------
 1 file changed, 18 insertions(+), 43 deletions(-)

diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index c39172c..ffdbc48 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -339,10 +339,11 @@ fail:
 	return 0;
 }
 
-static void nmi_cpu_setup(void *dummy)
+static void nmi_cpu_setup(void)
 {
 	int cpu = smp_processor_id();
 	struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
+
 	nmi_cpu_save_registers(msrs);
 	raw_spin_lock(&oprofilefs_lock);
 	model->setup_ctrs(model, msrs);
@@ -369,7 +370,7 @@ static void nmi_cpu_restore_registers(struct op_msrs *msrs)
 	}
 }
 
-static void nmi_cpu_shutdown(void *dummy)
+static void nmi_cpu_shutdown(void)
 {
 	unsigned int v;
 	int cpu = smp_processor_id();
@@ -387,24 +388,26 @@ static void nmi_cpu_shutdown(void *dummy)
 	nmi_cpu_restore_registers(msrs);
 }
 
-static void nmi_cpu_up(void)
+static int nmi_cpu_online(unsigned int cpu)
 {
 	local_irq_disable();
 	if (nmi_enabled)
-		nmi_cpu_setup(NULL);
+		nmi_cpu_setup();
 	if (ctr_running)
 		nmi_cpu_start(NULL);
 	local_irq_enable();
+	return 0;
 }
 
-static void nmi_cpu_down(void)
+static int nmi_cpu_down_prep(unsigned int cpu)
 {
 	local_irq_disable();
 	if (ctr_running)
 		nmi_cpu_stop(NULL);
 	if (nmi_enabled)
-		nmi_cpu_shutdown(NULL);
+		nmi_cpu_shutdown();
 	local_irq_enable();
+	return 0;
 }
 
 static int nmi_create_files(struct dentry *root)
@@ -437,24 +440,7 @@ static int nmi_create_files(struct dentry *root)
 	return 0;
 }
 
-static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
-				 void *data)
-{
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_FAILED:
-	case CPU_ONLINE:
-		nmi_cpu_up();
-		break;
-	case CPU_DOWN_PREPARE:
-		nmi_cpu_down();
-		break;
-	}
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block oprofile_cpu_nb = {
-	.notifier_call = oprofile_cpu_notifier
-};
+static enum cpuhp_state cpuhp_nmi_online;
 
 static int nmi_setup(void)
 {
@@ -497,20 +483,17 @@ static int nmi_setup(void)
 	if (err)
 		goto fail;
 
-	cpu_notifier_register_begin();
-
-	/* Use get/put_online_cpus() to protect 'nmi_enabled' */
-	get_online_cpus();
 	nmi_enabled = 1;
 	/* make nmi_enabled visible to the nmi handler: */
 	smp_mb();
-	on_each_cpu(nmi_cpu_setup, NULL, 1);
-	__register_cpu_notifier(&oprofile_cpu_nb);
-	put_online_cpus();
-
-	cpu_notifier_register_done();
-
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/oprofile:online",
+				nmi_cpu_online, nmi_cpu_down_prep);
+	if (err < 0)
+		goto fail_nmi;
+	cpuhp_nmi_online = err;
 	return 0;
+fail_nmi:
+	unregister_nmi_handler(NMI_LOCAL, "oprofile");
 fail:
 	free_msrs();
 	return err;
@@ -520,17 +503,9 @@ static void nmi_shutdown(void)
 {
 	struct op_msrs *msrs;
 
-	cpu_notifier_register_begin();
-
-	/* Use get/put_online_cpus() to protect 'nmi_enabled' & 'ctr_running' */
-	get_online_cpus();
-	on_each_cpu(nmi_cpu_shutdown, NULL, 1);
+	cpuhp_remove_state(cpuhp_nmi_online);
 	nmi_enabled = 0;
 	ctr_running = 0;
-	__unregister_cpu_notifier(&oprofile_cpu_nb);
-	put_online_cpus();
-
-	cpu_notifier_register_done();
 
 	/* make variables visible to the nmi handler: */
 	smp_mb();

[tip:smp/hotplug] x86/oprofile/nmi: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  89666c50472263fba97b7edbfd2a642d1d9d6f74
Gitweb:     http://git.kernel.org/tip/89666c50472263fba97b7edbfd2a642d1d9d6f74
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:40 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:43 +0100

x86/oprofile/nmi: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Robert Richter <rric@kernel.org>
Cc: oprofile-list@lists.sf.net
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-20-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/oprofile/nmi_int.c | 61 +++++++++++++--------------------------------
 1 file changed, 18 insertions(+), 43 deletions(-)

diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index c39172c..ffdbc48 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -339,10 +339,11 @@ fail:
 	return 0;
 }
 
-static void nmi_cpu_setup(void *dummy)
+static void nmi_cpu_setup(void)
 {
 	int cpu = smp_processor_id();
 	struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu);
+
 	nmi_cpu_save_registers(msrs);
 	raw_spin_lock(&oprofilefs_lock);
 	model->setup_ctrs(model, msrs);
@@ -369,7 +370,7 @@ static void nmi_cpu_restore_registers(struct op_msrs *msrs)
 	}
 }
 
-static void nmi_cpu_shutdown(void *dummy)
+static void nmi_cpu_shutdown(void)
 {
 	unsigned int v;
 	int cpu = smp_processor_id();
@@ -387,24 +388,26 @@ static void nmi_cpu_shutdown(void *dummy)
 	nmi_cpu_restore_registers(msrs);
 }
 
-static void nmi_cpu_up(void)
+static int nmi_cpu_online(unsigned int cpu)
 {
 	local_irq_disable();
 	if (nmi_enabled)
-		nmi_cpu_setup(NULL);
+		nmi_cpu_setup();
 	if (ctr_running)
 		nmi_cpu_start(NULL);
 	local_irq_enable();
+	return 0;
 }
 
-static void nmi_cpu_down(void)
+static int nmi_cpu_down_prep(unsigned int cpu)
 {
 	local_irq_disable();
 	if (ctr_running)
 		nmi_cpu_stop(NULL);
 	if (nmi_enabled)
-		nmi_cpu_shutdown(NULL);
+		nmi_cpu_shutdown();
 	local_irq_enable();
+	return 0;
 }
 
 static int nmi_create_files(struct dentry *root)
@@ -437,24 +440,7 @@ static int nmi_create_files(struct dentry *root)
 	return 0;
 }
 
-static int oprofile_cpu_notifier(struct notifier_block *b, unsigned long action,
-				 void *data)
-{
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DOWN_FAILED:
-	case CPU_ONLINE:
-		nmi_cpu_up();
-		break;
-	case CPU_DOWN_PREPARE:
-		nmi_cpu_down();
-		break;
-	}
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block oprofile_cpu_nb = {
-	.notifier_call = oprofile_cpu_notifier
-};
+static enum cpuhp_state cpuhp_nmi_online;
 
 static int nmi_setup(void)
 {
@@ -497,20 +483,17 @@ static int nmi_setup(void)
 	if (err)
 		goto fail;
 
-	cpu_notifier_register_begin();
-
-	/* Use get/put_online_cpus() to protect 'nmi_enabled' */
-	get_online_cpus();
 	nmi_enabled = 1;
 	/* make nmi_enabled visible to the nmi handler: */
 	smp_mb();
-	on_each_cpu(nmi_cpu_setup, NULL, 1);
-	__register_cpu_notifier(&oprofile_cpu_nb);
-	put_online_cpus();
-
-	cpu_notifier_register_done();
-
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/oprofile:online",
+				nmi_cpu_online, nmi_cpu_down_prep);
+	if (err < 0)
+		goto fail_nmi;
+	cpuhp_nmi_online = err;
 	return 0;
+fail_nmi:
+	unregister_nmi_handler(NMI_LOCAL, "oprofile");
 fail:
 	free_msrs();
 	return err;
@@ -520,17 +503,9 @@ static void nmi_shutdown(void)
 {
 	struct op_msrs *msrs;
 
-	cpu_notifier_register_begin();
-
-	/* Use get/put_online_cpus() to protect 'nmi_enabled' & 'ctr_running' */
-	get_online_cpus();
-	on_each_cpu(nmi_cpu_shutdown, NULL, 1);
+	cpuhp_remove_state(cpuhp_nmi_online);
 	nmi_enabled = 0;
 	ctr_running = 0;
-	__unregister_cpu_notifier(&oprofile_cpu_nb);
-	put_online_cpus();
-
-	cpu_notifier_register_done();
 
 	/* make variables visible to the nmi handler: */
 	smp_mb();

[PATCH 20/20] x86/pci/amd-bus: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The smp_call_function_single() is dropped because the ONLINE callback is
invoked on the target CPU since commit 1cf4f629d9d2 ("cpu/hotplug: Move
online calls to hotplugged cpu"). smp_call_function_single() invokes the
invoked function with interrupts disabled, but this calling convention is
not preserved as the MSR is not modified by anything else than this code.

Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: x86@kernel.org
Cc: linux-pci@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/x86/pci/amd_bus.c | 34 +++++++---------------------------
 1 file changed, 7 insertions(+), 27 deletions(-)

diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c
index c20d2cc7ef64..ae387e5ee6f7 100644
--- a/arch/x86/pci/amd_bus.c
+++ b/arch/x86/pci/amd_bus.c
@@ -327,35 +327,18 @@ static int __init early_root_info_init(void)
 
 #define ENABLE_CF8_EXT_CFG      (1ULL << 46)
 
-static void enable_pci_io_ecs(void *unused)
+static int amd_bus_cpu_online(unsigned int cpu)
 {
 	u64 reg;
+
 	rdmsrl(MSR_AMD64_NB_CFG, reg);
 	if (!(reg & ENABLE_CF8_EXT_CFG)) {
 		reg |= ENABLE_CF8_EXT_CFG;
 		wrmsrl(MSR_AMD64_NB_CFG, reg);
 	}
+	return 0;
 }
 
-static int amd_cpu_notify(struct notifier_block *self, unsigned long action,
-			  void *hcpu)
-{
-	int cpu = (long)hcpu;
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		smp_call_function_single(cpu, enable_pci_io_ecs, NULL, 0);
-		break;
-	default:
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block amd_cpu_notifier = {
-	.notifier_call	= amd_cpu_notify,
-};
-
 static void __init pci_enable_pci_io_ecs(void)
 {
 #ifdef CONFIG_AMD_NB
@@ -385,7 +368,7 @@ static void __init pci_enable_pci_io_ecs(void)
 
 static int __init pci_io_ecs_init(void)
 {
-	int cpu;
+	int ret;
 
 	/* assume all cpus from fam10h have IO ECS */
 	if (boot_cpu_data.x86 < 0x10)
@@ -395,12 +378,9 @@ static int __init pci_io_ecs_init(void)
 	if (early_pci_allowed())
 		pci_enable_pci_io_ecs();
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		amd_cpu_notify(&amd_cpu_notifier, (unsigned long)CPU_ONLINE,
-			       (void *)(long)cpu);
-	__register_cpu_notifier(&amd_cpu_notifier);
-	cpu_notifier_register_done();
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/amd_bus:online",
+				amd_bus_cpu_online, NULL);
+	WARN_ON(ret < 0);
 
 	pci_probe |= PCI_HAS_IO_ECS;
 
-- 
2.10.2

[tip:smp/hotplug] x86/pci/amd-bus: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  57b85d6eb29b289722d2c85c0ff025618c7fe700
Gitweb:     http://git.kernel.org/tip/57b85d6eb29b289722d2c85c0ff025618c7fe700
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:41 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Mon, 21 Nov 2016 16:37:09 +0100

x86/pci/amd-bus: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The smp_call_function_single() is dropped because the ONLINE callback is
invoked on the target CPU since commit 1cf4f629d9d2 ("cpu/hotplug: Move
online calls to hotplugged cpu"). smp_call_function_single() invokes the
invoked function with interrupts disabled, but this calling convention is
not preserved as the MSR is not modified by anything else than this code.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: linux-pci@vger.kernel.org
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-21-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/pci/amd_bus.c | 34 +++++++---------------------------
 1 file changed, 7 insertions(+), 27 deletions(-)

diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c
index c20d2cc..ae387e5 100644
--- a/arch/x86/pci/amd_bus.c
+++ b/arch/x86/pci/amd_bus.c
@@ -327,35 +327,18 @@ static int __init early_root_info_init(void)
 
 #define ENABLE_CF8_EXT_CFG      (1ULL << 46)
 
-static void enable_pci_io_ecs(void *unused)
+static int amd_bus_cpu_online(unsigned int cpu)
 {
 	u64 reg;
+
 	rdmsrl(MSR_AMD64_NB_CFG, reg);
 	if (!(reg & ENABLE_CF8_EXT_CFG)) {
 		reg |= ENABLE_CF8_EXT_CFG;
 		wrmsrl(MSR_AMD64_NB_CFG, reg);
 	}
+	return 0;
 }
 
-static int amd_cpu_notify(struct notifier_block *self, unsigned long action,
-			  void *hcpu)
-{
-	int cpu = (long)hcpu;
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		smp_call_function_single(cpu, enable_pci_io_ecs, NULL, 0);
-		break;
-	default:
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block amd_cpu_notifier = {
-	.notifier_call	= amd_cpu_notify,
-};
-
 static void __init pci_enable_pci_io_ecs(void)
 {
 #ifdef CONFIG_AMD_NB
@@ -385,7 +368,7 @@ static void __init pci_enable_pci_io_ecs(void)
 
 static int __init pci_io_ecs_init(void)
 {
-	int cpu;
+	int ret;
 
 	/* assume all cpus from fam10h have IO ECS */
 	if (boot_cpu_data.x86 < 0x10)
@@ -395,12 +378,9 @@ static int __init pci_io_ecs_init(void)
 	if (early_pci_allowed())
 		pci_enable_pci_io_ecs();
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		amd_cpu_notify(&amd_cpu_notifier, (unsigned long)CPU_ONLINE,
-			       (void *)(long)cpu);
-	__register_cpu_notifier(&amd_cpu_notifier);
-	cpu_notifier_register_done();
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/amd_bus:online",
+				amd_bus_cpu_online, NULL);
+	WARN_ON(ret < 0);
 
 	pci_probe |= PCI_HAS_IO_ECS;
 

[tip:smp/hotplug] x86/pci/amd-bus: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  c8b877a5e58a132c5efb05f0c404585b9789fe5c
Gitweb:     http://git.kernel.org/tip/c8b877a5e58a132c5efb05f0c404585b9789fe5c
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 17 Nov 2016 19:35:41 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Tue, 22 Nov 2016 23:34:43 +0100

x86/pci/amd-bus: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The smp_call_function_single() is dropped because the ONLINE callback is
invoked on the target CPU since commit 1cf4f629d9d2 ("cpu/hotplug: Move
online calls to hotplugged cpu"). smp_call_function_single() invokes the
invoked function with interrupts disabled, but this calling convention is
not preserved as the MSR is not modified by anything else than this code.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: linux-pci@vger.kernel.org
Cc: rt@linuxtronix.de
Link: http://lkml.kernel.org/r/20161117183541.8588-21-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/pci/amd_bus.c | 34 +++++++---------------------------
 1 file changed, 7 insertions(+), 27 deletions(-)

diff --git a/arch/x86/pci/amd_bus.c b/arch/x86/pci/amd_bus.c
index c20d2cc..ae387e5 100644
--- a/arch/x86/pci/amd_bus.c
+++ b/arch/x86/pci/amd_bus.c
@@ -327,35 +327,18 @@ static int __init early_root_info_init(void)
 
 #define ENABLE_CF8_EXT_CFG      (1ULL << 46)
 
-static void enable_pci_io_ecs(void *unused)
+static int amd_bus_cpu_online(unsigned int cpu)
 {
 	u64 reg;
+
 	rdmsrl(MSR_AMD64_NB_CFG, reg);
 	if (!(reg & ENABLE_CF8_EXT_CFG)) {
 		reg |= ENABLE_CF8_EXT_CFG;
 		wrmsrl(MSR_AMD64_NB_CFG, reg);
 	}
+	return 0;
 }
 
-static int amd_cpu_notify(struct notifier_block *self, unsigned long action,
-			  void *hcpu)
-{
-	int cpu = (long)hcpu;
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		smp_call_function_single(cpu, enable_pci_io_ecs, NULL, 0);
-		break;
-	default:
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block amd_cpu_notifier = {
-	.notifier_call	= amd_cpu_notify,
-};
-
 static void __init pci_enable_pci_io_ecs(void)
 {
 #ifdef CONFIG_AMD_NB
@@ -385,7 +368,7 @@ static void __init pci_enable_pci_io_ecs(void)
 
 static int __init pci_io_ecs_init(void)
 {
-	int cpu;
+	int ret;
 
 	/* assume all cpus from fam10h have IO ECS */
 	if (boot_cpu_data.x86 < 0x10)
@@ -395,12 +378,9 @@ static int __init pci_io_ecs_init(void)
 	if (early_pci_allowed())
 		pci_enable_pci_io_ecs();
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(cpu)
-		amd_cpu_notify(&amd_cpu_notifier, (unsigned long)CPU_ONLINE,
-			       (void *)(long)cpu);
-	__register_cpu_notifier(&amd_cpu_notifier);
-	cpu_notifier_register_done();
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "pci/amd_bus:online",
+				amd_bus_cpu_online, NULL);
+	WARN_ON(ret < 0);
 
 	pci_probe |= PCI_HAS_IO_ECS;