cpu hotplug: convert more drivers (batch #4)

cpu hotplug: convert more drivers (batch #4)

[Sebastian Andrzej Siewior]

Another small batch of drivers converted to the new hotplug state engine.
The whole series is also available at
  git://git.kernel.org/pub/scm/linux/kernel/git/bigeasy/hotplug-staging.git smp/hotplug

on top of v4.9-rc3.

Sebastian

[PATCH 01/25] fs/buffer: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: linux-fsdevel@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 fs/buffer.c                | 16 ++++++----------
 include/linux/cpuhotplug.h |  1 +
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/fs/buffer.c b/fs/buffer.c
index b205a629001d..1613656028d6 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -3403,7 +3403,7 @@ void free_buffer_head(struct buffer_head *bh)
 }
 EXPORT_SYMBOL(free_buffer_head);
 
-static void buffer_exit_cpu(int cpu)
+static int buffer_exit_cpu_dead(unsigned int cpu)
 {
 	int i;
 	struct bh_lru *b = &per_cpu(bh_lrus, cpu);
@@ -3414,14 +3414,7 @@ static void buffer_exit_cpu(int cpu)
 	}
 	this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
 	per_cpu(bh_accounting, cpu).nr = 0;
-}
-
-static int buffer_cpu_notify(struct notifier_block *self,
-			      unsigned long action, void *hcpu)
-{
-	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
-		buffer_exit_cpu((unsigned long)hcpu);
-	return NOTIFY_OK;
+	return 0;
 }
 
 /**
@@ -3471,6 +3464,7 @@ EXPORT_SYMBOL(bh_submit_read);
 void __init buffer_init(void)
 {
 	unsigned long nrpages;
+	int ret;
 
 	bh_cachep = kmem_cache_create("buffer_head",
 			sizeof(struct buffer_head), 0,
@@ -3483,5 +3477,7 @@ void __init buffer_init(void)
 	 */
 	nrpages = (nr_free_buffer_pages() * 10) / 100;
 	max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
-	hotcpu_notifier(buffer_cpu_notify, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
+					NULL, buffer_exit_cpu_dead);
+	WARN_ON(ret < 0);
 }
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index afe641c02dca..69b74fa0da60 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -30,6 +30,7 @@ enum cpuhp_state {
 	CPUHP_ACPI_CPUDRV_DEAD,
 	CPUHP_S390_PFAULT_DEAD,
 	CPUHP_BLK_MQ_DEAD,
+	CPUHP_FS_BUFF_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
-- 
2.10.2

[PATCH 02/25] kernel/printk: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpuhotplug.h |  1 +
 kernel/printk/printk.c     | 27 ++++++++++++---------------
 2 files changed, 13 insertions(+), 15 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 69b74fa0da60..4174083280d7 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -31,6 +31,7 @@ enum cpuhp_state {
 	CPUHP_S390_PFAULT_DEAD,
 	CPUHP_BLK_MQ_DEAD,
 	CPUHP_FS_BUFF_DEAD,
+	CPUHP_PRINTK_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index de08fc90baaf..4487ffcd42d5 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -2185,27 +2185,18 @@ void resume_console(void)
 
 /**
  * console_cpu_notify - print deferred console messages after CPU hotplug
- * @self: notifier struct
- * @action: CPU hotplug event
- * @hcpu: unused
+ * @cpu: unused
  *
  * If printk() is called from a CPU that is not online yet, the messages
  * will be spooled but will not show up on the console.  This function is
  * called when a new CPU comes online (or fails to come up), and ensures
  * that any such output gets printed.
  */
-static int console_cpu_notify(struct notifier_block *self,
-	unsigned long action, void *hcpu)
+static int console_cpu_notify(unsigned int cpu)
 {
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DEAD:
-	case CPU_DOWN_FAILED:
-	case CPU_UP_CANCELED:
-		console_lock();
-		console_unlock();
-	}
-	return NOTIFY_OK;
+	console_lock();
+	console_unlock();
+	return 0;
 }
 
 /**
@@ -2843,6 +2834,7 @@ EXPORT_SYMBOL(unregister_console);
 static int __init printk_late_init(void)
 {
 	struct console *con;
+	int ret;
 
 	for_each_console(con) {
 		if (!keep_bootcon && con->flags & CON_BOOT) {
@@ -2857,7 +2849,12 @@ static int __init printk_late_init(void)
 				unregister_console(con);
 		}
 	}
-	hotcpu_notifier(console_cpu_notify, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL,
+					console_cpu_notify);
+	WARN_ON(ret < 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "printk:online",
+					console_cpu_notify, NULL);
+	WARN_ON(ret < 0);
 	return 0;
 }
 late_initcall(printk_late_init);
-- 
2.10.2

[PATCH 03/25] mm/memcg: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: cgroups@vger.kernel.org
Cc: linux-mm@kvack.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpuhotplug.h |  1 +
 mm/memcontrol.c            | 24 ++++++++----------------
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 4174083280d7..c622ab349af3 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -32,6 +32,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_DEAD,
 	CPUHP_FS_BUFF_DEAD,
 	CPUHP_PRINTK_DEAD,
+	CPUHP_MM_MEMCQ_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 0f870ba43942..6c2043509fb5 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1816,22 +1816,13 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
 	mutex_unlock(&percpu_charge_mutex);
 }
 
-static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
-					unsigned long action,
-					void *hcpu)
+static int memcg_hotplug_cpu_dead(unsigned int cpu)
 {
-	int cpu = (unsigned long)hcpu;
 	struct memcg_stock_pcp *stock;
 
-	if (action == CPU_ONLINE)
-		return NOTIFY_OK;
-
-	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
-		return NOTIFY_OK;
-
 	stock = &per_cpu(memcg_stock, cpu);
 	drain_stock(stock);
-	return NOTIFY_OK;
+	return 0;
 }
 
 static void reclaim_high(struct mem_cgroup *memcg,
@@ -5774,16 +5765,17 @@ __setup("cgroup.memory=", cgroup_memory);
 /*
  * subsys_initcall() for memory controller.
  *
- * Some parts like hotcpu_notifier() have to be initialized from this context
- * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
- * everything that doesn't depend on a specific mem_cgroup structure should
- * be initialized from here.
+ * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
+ * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
+ * basically everything that doesn't depend on a specific mem_cgroup structure
+ * should be initialized from here.
  */
 static int __init mem_cgroup_init(void)
 {
 	int cpu, node;
 
-	hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
+	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
+				  memcg_hotplug_cpu_dead);
 
 	for_each_possible_cpu(cpu)
 		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
-- 
2.10.2

[PATCH 04/25] lib/percpu_counter: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpuhotplug.h |  1 +
 lib/percpu_counter.c       | 25 ++++++++++++++-----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index c622ab349af3..04e5f99ffc70 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -33,6 +33,7 @@ enum cpuhp_state {
 	CPUHP_FS_BUFF_DEAD,
 	CPUHP_PRINTK_DEAD,
 	CPUHP_MM_MEMCQ_DEAD,
+	CPUHP_PERCPU_CNT_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 72d36113ccaa..c8cebb137076 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -158,25 +158,21 @@ EXPORT_SYMBOL(percpu_counter_destroy);
 int percpu_counter_batch __read_mostly = 32;
 EXPORT_SYMBOL(percpu_counter_batch);
 
-static void compute_batch_value(void)
+static int compute_batch_value(unsigned int cpu)
 {
 	int nr = num_online_cpus();
 
 	percpu_counter_batch = max(32, nr*2);
+	return 0;
 }
 
-static int percpu_counter_hotcpu_callback(struct notifier_block *nb,
-					unsigned long action, void *hcpu)
+static int percpu_counter_cpu_dead(unsigned int cpu)
 {
 #ifdef CONFIG_HOTPLUG_CPU
-	unsigned int cpu;
 	struct percpu_counter *fbc;
 
-	compute_batch_value();
-	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
-		return NOTIFY_OK;
+	compute_batch_value(cpu);
 
-	cpu = (unsigned long)hcpu;
 	spin_lock_irq(&percpu_counters_lock);
 	list_for_each_entry(fbc, &percpu_counters, list) {
 		s32 *pcount;
@@ -190,7 +186,7 @@ static int percpu_counter_hotcpu_callback(struct notifier_block *nb,
 	}
 	spin_unlock_irq(&percpu_counters_lock);
 #endif
-	return NOTIFY_OK;
+	return 0;
 }
 
 /*
@@ -222,8 +218,15 @@ EXPORT_SYMBOL(__percpu_counter_compare);
 
 static int __init percpu_counter_startup(void)
 {
-	compute_batch_value();
-	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
+	int ret;
+
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online",
+				compute_batch_value, NULL);
+	WARN_ON(ret < 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD,
+					"lib/percpu_cnt:dead", NULL,
+					percpu_counter_cpu_dead);
+	WARN_ON(ret < 0);
 	return 0;
 }
 module_init(percpu_counter_startup);
-- 
2.10.2

[PATCH 05/25] lib/radix-tree: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Cc: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpuhotplug.h |  1 +
 lib/radix-tree.c           | 25 ++++++++++++-------------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 04e5f99ffc70..89310fb1031d 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -34,6 +34,7 @@ enum cpuhp_state {
 	CPUHP_PRINTK_DEAD,
 	CPUHP_MM_MEMCQ_DEAD,
 	CPUHP_PERCPU_CNT_DEAD,
+	CPUHP_RADIX_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 8e6d552c40dd..4b8bb3618b83 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -1642,32 +1642,31 @@ static __init void radix_tree_init_maxnodes(void)
 	}
 }
 
-static int radix_tree_callback(struct notifier_block *nfb,
-				unsigned long action, void *hcpu)
+static int radix_tree_cpu_dead(unsigned int cpu)
 {
-	int cpu = (long)hcpu;
 	struct radix_tree_preload *rtp;
 	struct radix_tree_node *node;
 
 	/* Free per-cpu pool of preloaded nodes */
-	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-		rtp = &per_cpu(radix_tree_preloads, cpu);
-		while (rtp->nr) {
-			node = rtp->nodes;
-			rtp->nodes = node->private_data;
-			kmem_cache_free(radix_tree_node_cachep, node);
-			rtp->nr--;
-		}
+	rtp = &per_cpu(radix_tree_preloads, cpu);
+	while (rtp->nr) {
+		node = rtp->nodes;
+		rtp->nodes = node->private_data;
+		kmem_cache_free(radix_tree_node_cachep, node);
+		rtp->nr--;
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
 void __init radix_tree_init(void)
 {
+	int ret;
 	radix_tree_node_cachep = kmem_cache_create("radix_tree_node",
 			sizeof(struct radix_tree_node), 0,
 			SLAB_PANIC | SLAB_RECLAIM_ACCOUNT,
 			radix_tree_node_ctor);
 	radix_tree_init_maxnodes();
-	hotcpu_notifier(radix_tree_callback, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_RADIX_DEAD, "lib/radix:dead",
+					NULL, radix_tree_cpu_dead);
+	WARN_ON(ret < 0);
 }
-- 
2.10.2

[PATCH 06/25] mm/page_alloc: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Cc: linux-mm@kvack.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpuhotplug.h |  1 +
 mm/page_alloc.c            | 49 +++++++++++++++++++++++-----------------------
 2 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 89310fb1031d..31c58f6ec3c6 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -35,6 +35,7 @@ enum cpuhp_state {
 	CPUHP_MM_MEMCQ_DEAD,
 	CPUHP_PERCPU_CNT_DEAD,
 	CPUHP_RADIX_DEAD,
+	CPUHP_PAGE_ALLOC_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 8fd42aa7c4bd..68873a164cc0 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -6491,38 +6491,39 @@ void __init free_area_init(unsigned long *zones_size)
 			__pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
 }
 
-static int page_alloc_cpu_notify(struct notifier_block *self,
-				 unsigned long action, void *hcpu)
+static int page_alloc_cpu_dead(unsigned int cpu)
 {
-	int cpu = (unsigned long)hcpu;
 
-	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-		lru_add_drain_cpu(cpu);
-		drain_pages(cpu);
+	lru_add_drain_cpu(cpu);
+	drain_pages(cpu);
 
-		/*
-		 * Spill the event counters of the dead processor
-		 * into the current processors event counters.
-		 * This artificially elevates the count of the current
-		 * processor.
-		 */
-		vm_events_fold_cpu(cpu);
+	/*
+	 * Spill the event counters of the dead processor
+	 * into the current processors event counters.
+	 * This artificially elevates the count of the current
+	 * processor.
+	 */
+	vm_events_fold_cpu(cpu);
 
-		/*
-		 * Zero the differential counters of the dead processor
-		 * so that the vm statistics are consistent.
-		 *
-		 * This is only okay since the processor is dead and cannot
-		 * race with what we are doing.
-		 */
-		cpu_vm_stats_fold(cpu);
-	}
-	return NOTIFY_OK;
+	/*
+	 * Zero the differential counters of the dead processor
+	 * so that the vm statistics are consistent.
+	 *
+	 * This is only okay since the processor is dead and cannot
+	 * race with what we are doing.
+	 */
+	cpu_vm_stats_fold(cpu);
+	return 0;
 }
 
 void __init page_alloc_init(void)
 {
-	hotcpu_notifier(page_alloc_cpu_notify, 0);
+	int ret;
+
+	ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
+					"mm/page_alloc:dead", NULL,
+					page_alloc_cpu_dead);
+	WARN_ON(ret < 0);
 }
 
 /*
-- 
2.10.2

[PATCH 07/25] mm/vmscan: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Cc: linux-mm@kvack.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 mm/vmscan.c | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 76fda2268148..b8404d32caf0 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3556,24 +3556,21 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
    not required for correctness.  So if the last cpu in a node goes
    away, we get changed to run anywhere: as the first one comes back,
    restore their cpu bindings. */
-static int cpu_callback(struct notifier_block *nfb, unsigned long action,
-			void *hcpu)
+static int kswapd_cpu_online(unsigned int cpu)
 {
 	int nid;
 
-	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
-		for_each_node_state(nid, N_MEMORY) {
-			pg_data_t *pgdat = NODE_DATA(nid);
-			const struct cpumask *mask;
+	for_each_node_state(nid, N_MEMORY) {
+		pg_data_t *pgdat = NODE_DATA(nid);
+		const struct cpumask *mask;
 
-			mask = cpumask_of_node(pgdat->node_id);
+		mask = cpumask_of_node(pgdat->node_id);
 
-			if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
-				/* One of our CPUs online: restore mask */
-				set_cpus_allowed_ptr(pgdat->kswapd, mask);
-		}
+		if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
+			/* One of our CPUs online: restore mask */
+			set_cpus_allowed_ptr(pgdat->kswapd, mask);
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
 /*
@@ -3615,12 +3612,15 @@ void kswapd_stop(int nid)
 
 static int __init kswapd_init(void)
 {
-	int nid;
+	int nid, ret;
 
 	swap_setup();
 	for_each_node_state(nid, N_MEMORY)
  		kswapd_run(nid);
-	hotcpu_notifier(cpu_callback, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"mm/vmscan:online", kswapd_cpu_online,
+					NULL);
+	WARN_ON(ret < 0);
 	return 0;
 }
 
-- 
2.10.2

[PATCH 08/25] net/dev: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine.

Cc: "David S. Miller" <davem@davemloft.net>
Cc: netdev@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpuhotplug.h |  1 +
 net/core/dev.c             | 16 ++++++----------
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 31c58f6ec3c6..394eb7ed53be 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -36,6 +36,7 @@ enum cpuhp_state {
 	CPUHP_PERCPU_CNT_DEAD,
 	CPUHP_RADIX_DEAD,
 	CPUHP_PAGE_ALLOC_DEAD,
+	CPUHP_NET_DEV_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/net/core/dev.c b/net/core/dev.c
index 4bc19a164ba5..71693729bdd5 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -7947,18 +7947,13 @@ int dev_change_net_namespace(struct net_device *dev, struct net *net, const char
 }
 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
 
-static int dev_cpu_callback(struct notifier_block *nfb,
-			    unsigned long action,
-			    void *ocpu)
+static int dev_cpu_dead(unsigned int oldcpu)
 {
 	struct sk_buff **list_skb;
 	struct sk_buff *skb;
-	unsigned int cpu, oldcpu = (unsigned long)ocpu;
+	unsigned int cpu;
 	struct softnet_data *sd, *oldsd;
 
-	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
-		return NOTIFY_OK;
-
 	local_irq_disable();
 	cpu = smp_processor_id();
 	sd = &per_cpu(softnet_data, cpu);
@@ -8008,10 +8003,9 @@ static int dev_cpu_callback(struct notifier_block *nfb,
 		input_queue_head_incr(oldsd);
 	}
 
-	return NOTIFY_OK;
+	return 0;
 }
 
-
 /**
  *	netdev_increment_features - increment feature set by one
  *	@all: current feature set
@@ -8345,7 +8339,9 @@ static int __init net_dev_init(void)
 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
 
-	hotcpu_notifier(dev_cpu_callback, 0);
+	rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
+				       NULL, dev_cpu_dead);
+	WARN_ON(rc < 0);
 	dst_subsys_init();
 	rc = 0;
 out:
-- 
2.10.2

[PATCH 09/25] net/flowcache: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine. Use multi state support to avoid
custom list handling for the multiple instances.

Cc: "David S. Miller" <davem@davemloft.net>
Cc: Steffen Klassert <steffen.klassert@secunet.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: netdev@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/cpuhotplug.h |  1 +
 include/net/flow.h         |  1 +
 include/net/flowcache.h    |  2 +-
 net/core/flow.c            | 60 ++++++++++++++++++++--------------------------
 net/xfrm/xfrm_policy.c     |  1 +
 5 files changed, 30 insertions(+), 35 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 394eb7ed53be..86b940f19df8 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -56,6 +56,7 @@ enum cpuhp_state {
 	CPUHP_ARM_SHMOBILE_SCU_PREPARE,
 	CPUHP_SH_SH3X_PREPARE,
 	CPUHP_BLK_MQ_PREPARE,
+	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
diff --git a/include/net/flow.h b/include/net/flow.h
index 035aa7716967..2e386bd6ee63 100644
--- a/include/net/flow.h
+++ b/include/net/flow.h
@@ -239,6 +239,7 @@ struct flow_cache_object *flow_cache_lookup(struct net *net,
 					    void *ctx);
 int flow_cache_init(struct net *net);
 void flow_cache_fini(struct net *net);
+void flow_cache_hp_init(void);
 
 void flow_cache_flush(struct net *net);
 void flow_cache_flush_deferred(struct net *net);
diff --git a/include/net/flowcache.h b/include/net/flowcache.h
index c8f665ec6e0d..9caf3bfc8d2d 100644
--- a/include/net/flowcache.h
+++ b/include/net/flowcache.h
@@ -17,7 +17,7 @@ struct flow_cache_percpu {
 struct flow_cache {
 	u32				hash_shift;
 	struct flow_cache_percpu __percpu *percpu;
-	struct notifier_block		hotcpu_notifier;
+	struct hlist_node		node;
 	int				low_watermark;
 	int				high_watermark;
 	struct timer_list		rnd_timer;
diff --git a/net/core/flow.c b/net/core/flow.c
index 3937b1b68d5b..841fd7f87b30 100644
--- a/net/core/flow.c
+++ b/net/core/flow.c
@@ -419,28 +419,20 @@ static int flow_cache_cpu_prepare(struct flow_cache *fc, int cpu)
 	return 0;
 }
 
-static int flow_cache_cpu(struct notifier_block *nfb,
-			  unsigned long action,
-			  void *hcpu)
+static int flow_cache_cpu_up_prep(unsigned int cpu, struct hlist_node *node)
 {
-	struct flow_cache *fc = container_of(nfb, struct flow_cache,
-						hotcpu_notifier);
-	int res, cpu = (unsigned long) hcpu;
+	struct flow_cache *fc = hlist_entry_safe(node, struct flow_cache, node);
+
+	return flow_cache_cpu_prepare(fc, cpu);
+}
+
+static int flow_cache_cpu_dead(unsigned int cpu, struct hlist_node *node)
+{
+	struct flow_cache *fc = hlist_entry_safe(node, struct flow_cache, node);
 	struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu);
 
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		res = flow_cache_cpu_prepare(fc, cpu);
-		if (res)
-			return notifier_from_errno(res);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		__flow_cache_shrink(fc, fcp, 0);
-		break;
-	}
-	return NOTIFY_OK;
+	__flow_cache_shrink(fc, fcp, 0);
+	return 0;
 }
 
 int flow_cache_init(struct net *net)
@@ -467,18 +459,8 @@ int flow_cache_init(struct net *net)
 	if (!fc->percpu)
 		return -ENOMEM;
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		if (flow_cache_cpu_prepare(fc, i))
-			goto err;
-	}
-	fc->hotcpu_notifier = (struct notifier_block){
-		.notifier_call = flow_cache_cpu,
-	};
-	__register_hotcpu_notifier(&fc->hotcpu_notifier);
-
-	cpu_notifier_register_done();
+	if (cpuhp_state_add_instance(CPUHP_NET_FLOW_PREPARE, &fc->node))
+		goto err;
 
 	setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd,
 		    (unsigned long) fc);
@@ -494,8 +476,6 @@ int flow_cache_init(struct net *net)
 		fcp->hash_table = NULL;
 	}
 
-	cpu_notifier_register_done();
-
 	free_percpu(fc->percpu);
 	fc->percpu = NULL;
 
@@ -509,7 +489,8 @@ void flow_cache_fini(struct net *net)
 	struct flow_cache *fc = &net->xfrm.flow_cache_global;
 
 	del_timer_sync(&fc->rnd_timer);
-	unregister_hotcpu_notifier(&fc->hotcpu_notifier);
+
+	cpuhp_state_remove_instance_nocalls(CPUHP_NET_FLOW_PREPARE, &fc->node);
 
 	for_each_possible_cpu(i) {
 		struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i);
@@ -521,3 +502,14 @@ void flow_cache_fini(struct net *net)
 	fc->percpu = NULL;
 }
 EXPORT_SYMBOL(flow_cache_fini);
+
+void __init flow_cache_hp_init(void)
+{
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_NET_FLOW_PREPARE,
+				      "net/flow:prepare",
+				      flow_cache_cpu_up_prep,
+				      flow_cache_cpu_dead);
+	WARN_ON(ret < 0);
+}
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index fd6986634e6f..4a8eff11bdad 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -3111,6 +3111,7 @@ static struct pernet_operations __net_initdata xfrm_net_ops = {
 
 void __init xfrm_init(void)
 {
+	flow_cache_hp_init();
 	register_pernet_subsys(&xfrm_net_ops);
 	seqcount_init(&xfrm_policy_hash_generation);
 	xfrm_input_init();
-- 
2.10.2

[PATCH 10/25] s390/smp: Make cpu notifier symetric

[Sebastian Andrzej Siewior]

From: Thomas Gleixner <tglx@linutronix.de>

There is no reason to remove the sysfs cpu files when the CPU is dead, they
can be removed when the cpu is prepared to go down. Doing it at
DOWN_PREPARE allows us to convert it to a symetric hotplug state in the
next step.

Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: linux-s390@vger.kernel.org
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/s390/kernel/smp.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 35531fe1c5ea..1a21e66c484a 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -1056,9 +1056,10 @@ static int smp_cpu_notify(struct notifier_block *self, unsigned long action,
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+	case CPU_DOWN_FAILED:
 		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
 		break;
-	case CPU_DEAD:
+	case CPU_DOWN_PREPARE::
 		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
 		break;
 	}
-- 
2.10.2

[PATCH 11/25] s390/smp: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

cpuhp_setup_state() invokes the startup callback on all online cpus with
the proper protection, so we can remove the cpu hotplug protection from the
init function and the creation of the per cpu files for online cpus in
smp_add_present_cpu(). smp_add_present_cpu() is called also called from
__smp_rescan_cpus(), but this callpath never adds an online cpu, it merily
adds newly present cpus, so the creation of the cpu files is not required.

Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: linux-s390@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/s390/kernel/smp.c | 38 ++++++++++++--------------------------
 1 file changed, 12 insertions(+), 26 deletions(-)

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 1a21e66c484a..b613aa989e40 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -1047,23 +1047,18 @@ static struct attribute_group cpu_online_attr_group = {
 	.attrs = cpu_online_attrs,
 };
 
-static int smp_cpu_notify(struct notifier_block *self, unsigned long action,
-			  void *hcpu)
+static int smp_cpu_online(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned int)(long)hcpu;
 	struct device *s = &per_cpu(cpu_device, cpu)->dev;
-	int err = 0;
 
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
-		break;
-	case CPU_DOWN_PREPARE::
-		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
-		break;
-	}
-	return notifier_from_errno(err);
+	return sysfs_create_group(&s->kobj, &cpu_online_attr_group);
+}
+static int smp_cpu_pre_down(unsigned int cpu)
+{
+	struct device *s = &per_cpu(cpu_device, cpu)->dev;
+
+	sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
+	return 0;
 }
 
 static int smp_add_present_cpu(int cpu)
@@ -1084,20 +1079,13 @@ static int smp_add_present_cpu(int cpu)
 	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
 	if (rc)
 		goto out_cpu;
-	if (cpu_online(cpu)) {
-		rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
-		if (rc)
-			goto out_online;
-	}
+
 	rc = topology_cpu_init(c);
 	if (rc)
 		goto out_topology;
 	return 0;
 
 out_topology:
-	if (cpu_online(cpu))
-		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
-out_online:
 	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
 out_cpu:
 #ifdef CONFIG_HOTPLUG_CPU
@@ -1150,17 +1138,15 @@ static int __init s390_smp_init(void)
 	if (rc)
 		return rc;
 #endif
-	cpu_notifier_register_begin();
 	for_each_present_cpu(cpu) {
 		rc = smp_add_present_cpu(cpu);
 		if (rc)
 			goto out;
 	}
 
-	__hotcpu_notifier(smp_cpu_notify, 0);
-
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online",
+			       smp_cpu_online, smp_cpu_pre_down);
 out:
-	cpu_notifier_register_done();
 	return rc;
 }
 subsys_initcall(s390_smp_init);
-- 
2.10.2

[PATCH 12/25] drivers base/cacheinfo: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs. No functional change.

Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 drivers/base/cacheinfo.c | 57 +++++++++++++-----------------------------------
 1 file changed, 15 insertions(+), 42 deletions(-)

diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index e9fd32e91668..47983a2ba621 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -498,57 +498,30 @@ static int cache_add_dev(unsigned int cpu)
 	return rc;
 }
 
-static void cache_remove_dev(unsigned int cpu)
+static int cacheinfo_cpu_online(unsigned int cpu)
 {
-	if (!cpumask_test_cpu(cpu, &cache_dev_map))
-		return;
-	cpumask_clear_cpu(cpu, &cache_dev_map);
+	int rc = detect_cache_attributes(cpu);
 
-	cpu_cache_sysfs_exit(cpu);
+	if (rc)
+		return rc;
+	rc = cache_add_dev(cpu);
+	if (rc)
+		free_cache_attributes(cpu);
+	return rc;
 }
 
-static int cacheinfo_cpu_callback(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
+static int cacheinfo_cpu_pre_down(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-	int rc = 0;
+	if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map))
+		cpu_cache_sysfs_exit(cpu);
 
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-		rc = detect_cache_attributes(cpu);
-		if (!rc)
-			rc = cache_add_dev(cpu);
-		break;
-	case CPU_DEAD:
-		cache_remove_dev(cpu);
-		free_cache_attributes(cpu);
-		break;
-	}
-	return notifier_from_errno(rc);
+	free_cache_attributes(cpu);
+	return 0;
 }
 
 static int __init cacheinfo_sysfs_init(void)
 {
-	int cpu, rc = 0;
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu) {
-		rc = detect_cache_attributes(cpu);
-		if (rc)
-			goto out;
-		rc = cache_add_dev(cpu);
-		if (rc) {
-			free_cache_attributes(cpu);
-			pr_err("error populating cacheinfo..cpu%d\n", cpu);
-			goto out;
-		}
-	}
-	__hotcpu_notifier(cacheinfo_cpu_callback, 0);
-
-out:
-	cpu_notifier_register_done();
-	return rc;
+	return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "base/cacheinfo:online",
+				 cacheinfo_cpu_online, cacheinfo_cpu_pre_down);
 }
-
 device_initcall(cacheinfo_sysfs_init);
-- 
2.10.2

[PATCH 13/25] drivers base/topology: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The removal of the files happens now in the prepare down stage as there is
no reason to keep them around until the cpu has actually died.

Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 drivers/base/topology.c    | 42 +++++-------------------------------------
 include/linux/cpuhotplug.h |  1 +
 2 files changed, 6 insertions(+), 37 deletions(-)

diff --git a/drivers/base/topology.c b/drivers/base/topology.c
index df3c97cb4c99..d6ec1c546f5b 100644
--- a/drivers/base/topology.c
+++ b/drivers/base/topology.c
@@ -118,51 +118,19 @@ static int topology_add_dev(unsigned int cpu)
 	return sysfs_create_group(&dev->kobj, &topology_attr_group);
 }
 
-static void topology_remove_dev(unsigned int cpu)
+static int topology_remove_dev(unsigned int cpu)
 {
 	struct device *dev = get_cpu_device(cpu);
 
 	sysfs_remove_group(&dev->kobj, &topology_attr_group);
-}
-
-static int topology_cpu_callback(struct notifier_block *nfb,
-				 unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	int rc = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		rc = topology_add_dev(cpu);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		topology_remove_dev(cpu);
-		break;
-	}
-	return notifier_from_errno(rc);
+	return 0;
 }
 
 static int topology_sysfs_init(void)
 {
-	int cpu;
-	int rc = 0;
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu) {
-		rc = topology_add_dev(cpu);
-		if (rc)
-			goto out;
-	}
-	__hotcpu_notifier(topology_cpu_callback, 0);
-
-out:
-	cpu_notifier_register_done();
-	return rc;
+	return cpuhp_setup_state(CPUHP_TOPOLOGY_PREPARE,
+				 "base/topology:prepare", topology_add_dev,
+				 topology_remove_dev);
 }
 
 device_initcall(topology_sysfs_init);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 86b940f19df8..3410d83cc2e2 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -57,6 +57,7 @@ enum cpuhp_state {
 	CPUHP_SH_SH3X_PREPARE,
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
+	CPUHP_TOPOLOGY_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
-- 
2.10.2

[PATCH 14/25] ia64/err-inject: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The removal of the files happens now in the prepare down stage as there is
no reason to keep them around until the cpu has actually died.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/ia64/kernel/err_inject.c | 78 +++++++++++--------------------------------
 1 file changed, 19 insertions(+), 59 deletions(-)

diff --git a/arch/ia64/kernel/err_inject.c b/arch/ia64/kernel/err_inject.c
index 5ed0ea92c5bf..85bba43e7d5d 100644
--- a/arch/ia64/kernel/err_inject.c
+++ b/arch/ia64/kernel/err_inject.c
@@ -224,85 +224,45 @@ static struct attribute_group err_inject_attr_group = {
 	.name = "err_inject"
 };
 /* Add/Remove err_inject interface for CPU device */
-static int err_inject_add_dev(struct device *sys_dev)
+static int err_inject_add_dev(unsigned int cpu)
 {
+	struct device *sys_dev = get_cpu_device(cpu);
+
 	return sysfs_create_group(&sys_dev->kobj, &err_inject_attr_group);
 }
 
-static int err_inject_remove_dev(struct device *sys_dev)
+static int err_inject_remove_dev(unsigned int cpu)
 {
+	struct device *sys_dev = get_cpu_device(cpu);
+
 	sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group);
 	return 0;
 }
-static int err_inject_cpu_callback(struct notifier_block *nfb,
-		unsigned long action, void *hcpu)
+
+static enum cpuhp_state hp_online;
+
+static int __init err_inject_init(void)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-	struct device *sys_dev;
-
-	sys_dev = get_cpu_device(cpu);
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		err_inject_add_dev(sys_dev);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		err_inject_remove_dev(sys_dev);
-		break;
-	}
-
-	return NOTIFY_OK;
-}
-
-static struct notifier_block err_inject_cpu_notifier =
-{
-	.notifier_call = err_inject_cpu_callback,
-};
-
-static int __init
-err_inject_init(void)
-{
-	int i;
-
+	int ret;
 #ifdef ERR_INJ_DEBUG
 	printk(KERN_INFO "Enter error injection driver.\n");
 #endif
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		err_inject_cpu_callback(&err_inject_cpu_notifier, CPU_ONLINE,
-				(void *)(long)i);
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/err_inj:online",
+				err_inject_add_dev, err_inject_remove_dev);
+	if (ret >= 0) {
+		hp_online = ret;
+		ret = 0;
 	}
-
-	__register_hotcpu_notifier(&err_inject_cpu_notifier);
-
-	cpu_notifier_register_done();
-
-	return 0;
+	return ret;
 }
 
-static void __exit
-err_inject_exit(void)
+static void __exit err_inject_exit(void)
 {
-	int i;
-	struct device *sys_dev;
-
 #ifdef ERR_INJ_DEBUG
 	printk(KERN_INFO "Exit error injection driver.\n");
 #endif
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		sys_dev = get_cpu_device(i);
-		sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group);
-	}
-
-	__unregister_hotcpu_notifier(&err_inject_cpu_notifier);
-
-	cpu_notifier_register_done();
+	cpuhp_remove_state(hp_online);
 }
 
 module_init(err_inject_init);
-- 
2.10.2

[PATCH 15/25] ia64/palinfo: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The removal of the files happens now in the prepare down stage as there is
no reason to keep them around until the cpu has actually died.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/ia64/kernel/palinfo.c | 60 +++++++++++++---------------------------------
 1 file changed, 16 insertions(+), 44 deletions(-)

diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c
index c39c3cd3ac34..b6e597860888 100644
--- a/arch/ia64/kernel/palinfo.c
+++ b/arch/ia64/kernel/palinfo.c
@@ -932,8 +932,7 @@ static const struct file_operations proc_palinfo_fops = {
 	.release	= single_release,
 };
 
-static void
-create_palinfo_proc_entries(unsigned int cpu)
+static int palinfo_add_proc(unsigned int cpu)
 {
 	pal_func_cpu_u_t f;
 	struct proc_dir_entry *cpu_dir;
@@ -943,7 +942,7 @@ create_palinfo_proc_entries(unsigned int cpu)
 
 	cpu_dir = proc_mkdir(cpustr, palinfo_dir);
 	if (!cpu_dir)
-		return;
+		return -EINVAL;
 
 	f.req_cpu = cpu;
 
@@ -952,42 +951,21 @@ create_palinfo_proc_entries(unsigned int cpu)
 		proc_create_data(palinfo_entries[j].name, 0, cpu_dir,
 				 &proc_palinfo_fops, (void *)f.value);
 	}
+	return 0;
 }
 
-static void
-remove_palinfo_proc_entries(unsigned int hcpu)
+static int palinfo_del_proc(unsigned int hcpu)
 {
 	char cpustr[3+4+1];	/* cpu numbers are up to 4095 on itanic */
+
 	sprintf(cpustr, "cpu%d", hcpu);
 	remove_proc_subtree(cpustr, palinfo_dir);
+	return 0;
 }
 
-static int palinfo_cpu_callback(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
-{
-	unsigned int hotcpu = (unsigned long)hcpu;
+static enum cpuhp_state hp_online;
 
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		create_palinfo_proc_entries(hotcpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		remove_palinfo_proc_entries(hotcpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block __refdata palinfo_cpu_notifier =
-{
-	.notifier_call = palinfo_cpu_callback,
-	.priority = 0,
-};
-
-static int __init
-palinfo_init(void)
+static int __init palinfo_init(void)
 {
 	int i = 0;
 
@@ -996,25 +974,19 @@ palinfo_init(void)
 	if (!palinfo_dir)
 		return -ENOMEM;
 
-	cpu_notifier_register_begin();
-
-	/* Create palinfo dirs in /proc for all online cpus */
-	for_each_online_cpu(i) {
-		create_palinfo_proc_entries(i);
+	i = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/palinfo:online",
+			      palinfo_add_proc, palinfo_del_proc);
+	if (i < 0) {
+		remove_proc_subtree("pal", NULL);
+		return i;
 	}
-
-	/* Register for future delivery via notify registration */
-	__register_hotcpu_notifier(&palinfo_cpu_notifier);
-
-	cpu_notifier_register_done();
-
+	hp_online = i;
 	return 0;
 }
 
-static void __exit
-palinfo_exit(void)
+static void __exit palinfo_exit(void)
 {
-	unregister_hotcpu_notifier(&palinfo_cpu_notifier);
+	cpuhp_remove_state(hp_online);
 	remove_proc_subtree("pal", NULL);
 }
 
-- 
2.10.2

[PATCH 16/25] ia64/salinfo: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/ia64/kernel/salinfo.c | 83 ++++++++++++++++++----------------------------
 1 file changed, 32 insertions(+), 51 deletions(-)

diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c
index 5313007d5423..bfbf07ed38dd 100644
--- a/arch/ia64/kernel/salinfo.c
+++ b/arch/ia64/kernel/salinfo.c
@@ -550,52 +550,40 @@ static const struct file_operations salinfo_data_fops = {
 	.llseek  = default_llseek,
 };
 
-static int
-salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu)
+static int salinfo_cpu_online(unsigned int cpu)
 {
-	unsigned int i, cpu = (unsigned long)hcpu;
-	unsigned long flags;
+	unsigned int i, end = ARRAY_SIZE(salinfo_data);
 	struct salinfo_data *data;
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		spin_lock_irqsave(&data_saved_lock, flags);
-		for (i = 0, data = salinfo_data;
-		     i < ARRAY_SIZE(salinfo_data);
-		     ++i, ++data) {
-			cpumask_set_cpu(cpu, &data->cpu_event);
-			wake_up_interruptible(&data->read_wait);
-		}
-		spin_unlock_irqrestore(&data_saved_lock, flags);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		spin_lock_irqsave(&data_saved_lock, flags);
-		for (i = 0, data = salinfo_data;
-		     i < ARRAY_SIZE(salinfo_data);
-		     ++i, ++data) {
-			struct salinfo_data_saved *data_saved;
-			int j;
-			for (j = ARRAY_SIZE(data->data_saved) - 1, data_saved = data->data_saved + j;
-			     j >= 0;
-			     --j, --data_saved) {
-				if (data_saved->buffer && data_saved->cpu == cpu) {
-					shift1_data_saved(data, j);
-				}
-			}
-			cpumask_clear_cpu(cpu, &data->cpu_event);
-		}
-		spin_unlock_irqrestore(&data_saved_lock, flags);
-		break;
+
+	spin_lock_irq(&data_saved_lock);
+	for (i = 0, data = salinfo_data; i < end; ++i, ++data) {
+		cpumask_set_cpu(cpu, &data->cpu_event);
+		wake_up_interruptible(&data->read_wait);
 	}
-	return NOTIFY_OK;
+	spin_unlock_irq(&data_saved_lock);
+	return 0;
 }
 
-static struct notifier_block salinfo_cpu_notifier =
+static int salinfo_cpu_pre_down(unsigned int cpu)
 {
-	.notifier_call = salinfo_cpu_callback,
-	.priority = 0,
-};
+	unsigned int i, end = ARRAY_SIZE(salinfo_data);
+	struct salinfo_data *data;
+
+	spin_lock_irqsave(&data_saved_lock, flags);
+	for (i = 0, data = salinfo_data; i < end; ++i, ++data) {
+		struct salinfo_data_saved *data_saved;
+		int j = ARRAY_SIZE(data->data_saved) - 1;
+
+		for (data_saved = data->data_saved + j; j >= 0;
+		     --j, --data_saved) {
+			if (data_saved->buffer && data_saved->cpu == cpu)
+				shift1_data_saved(data, j);
+		}
+		cpumask_clear_cpu(cpu, &data->cpu_event);
+	}
+	spin_unlock_irqrestore(&data_saved_lock, flags);
+	return 0;
+}
 
 static int __init
 salinfo_init(void)
@@ -604,7 +592,7 @@ salinfo_init(void)
 	struct proc_dir_entry **sdir = salinfo_proc_entries; /* keeps track of every entry */
 	struct proc_dir_entry *dir, *entry;
 	struct salinfo_data *data;
-	int i, j;
+	int i;
 
 	salinfo_dir = proc_mkdir("sal", NULL);
 	if (!salinfo_dir)
@@ -617,8 +605,6 @@ salinfo_init(void)
 					   (void *)salinfo_entries[i].feature);
 	}
 
-	cpu_notifier_register_begin();
-
 	for (i = 0; i < ARRAY_SIZE(salinfo_log_name); i++) {
 		data = salinfo_data + i;
 		data->type = i;
@@ -639,10 +625,6 @@ salinfo_init(void)
 			continue;
 		*sdir++ = entry;
 
-		/* we missed any events before now */
-		for_each_online_cpu(j)
-			cpumask_set_cpu(j, &data->cpu_event);
-
 		*sdir++ = dir;
 	}
 
@@ -653,10 +635,9 @@ salinfo_init(void)
 	salinfo_timer.function = &salinfo_timeout;
 	add_timer(&salinfo_timer);
 
-	__register_hotcpu_notifier(&salinfo_cpu_notifier);
-
-	cpu_notifier_register_done();
-
+	i = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/salinfo:online",
+			      salinfo_cpu_online, salinfo_cpu_pre_down);
+	WARN_ON(i < 0);
 	return 0;
 }
 
-- 
2.10.2

[PATCH 17/25] ia64/topology: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/ia64/kernel/topology.c | 55 +++++++--------------------------------------
 1 file changed, 8 insertions(+), 47 deletions(-)

diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c
index c01fe8991244..9102cad408b8 100644
--- a/arch/ia64/kernel/topology.c
+++ b/arch/ia64/kernel/topology.c
@@ -349,9 +349,9 @@ static int cpu_cache_sysfs_init(unsigned int cpu)
 }
 
 /* Add cache interface for CPU device */
-static int cache_add_dev(struct device *sys_dev)
+static int cache_add_dev(unsigned int cpu)
 {
-	unsigned int cpu = sys_dev->id;
+	struct device *sys_dev = get_cpu_device(cpu);
 	unsigned long i, j;
 	struct cache_info *this_object;
 	int retval = 0;
@@ -399,9 +399,9 @@ static int cache_add_dev(struct device *sys_dev)
 }
 
 /* Remove cache interface for CPU device */
-static int cache_remove_dev(struct device *sys_dev)
+static int cache_remove_dev(unsigned int cpu)
 {
-	unsigned int cpu = sys_dev->id;
+	struct device *sys_dev = get_cpu_device(cpu);
 	unsigned long i;
 
 	for (i = 0; i < all_cpu_cache_info[cpu].num_cache_leaves; i++)
@@ -419,52 +419,13 @@ static int cache_remove_dev(struct device *sys_dev)
 	return 0;
 }
 
-/*
- * When a cpu is hot-plugged, do a check and initiate
- * cache kobject if necessary
- */
-static int cache_cpu_callback(struct notifier_block *nfb,
-		unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	struct device *sys_dev;
-
-	sys_dev = get_cpu_device(cpu);
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		cache_add_dev(sys_dev);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		cache_remove_dev(sys_dev);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block cache_cpu_notifier =
-{
-	.notifier_call = cache_cpu_callback
-};
-
 static int __init cache_sysfs_init(void)
 {
-	int i;
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		struct device *sys_dev = get_cpu_device((unsigned int)i);
-		cache_add_dev(sys_dev);
-	}
-
-	__register_hotcpu_notifier(&cache_cpu_notifier);
-
-	cpu_notifier_register_done();
+	int ret;
 
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/topology:online",
+				cache_add_dev, cache_remove_dev);
+	WARN_ON(ret < 0);
 	return 0;
 }
-
 device_initcall(cache_sysfs_init);
-
-- 
2.10.2

[PATCH 18/25] x86/mcheck: Move threshold_create_device()

[Sebastian Andrzej Siewior]

Move the threshold_create_device() so it can use
threshold_remove_device() without a a declaration.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 50 ++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 9b5403462936..75a3e3eab81e 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1010,31 +1010,6 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank)
 	return err;
 }
 
-/* create dir/files for all valid threshold banks */
-static int threshold_create_device(unsigned int cpu)
-{
-	unsigned int bank;
-	struct threshold_bank **bp;
-	int err = 0;
-
-	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
-		     GFP_KERNEL);
-	if (!bp)
-		return -ENOMEM;
-
-	per_cpu(threshold_banks, cpu) = bp;
-
-	for (bank = 0; bank < mca_cfg.banks; ++bank) {
-		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
-			continue;
-		err = threshold_create_bank(cpu, bank);
-		if (err)
-			return err;
-	}
-
-	return err;
-}
-
 static void deallocate_threshold_block(unsigned int cpu,
 						 unsigned int bank)
 {
@@ -1114,6 +1089,31 @@ static void threshold_remove_device(unsigned int cpu)
 	kfree(per_cpu(threshold_banks, cpu));
 }
 
+/* create dir/files for all valid threshold banks */
+static int threshold_create_device(unsigned int cpu)
+{
+	unsigned int bank;
+	struct threshold_bank **bp;
+	int err = 0;
+
+	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
+		     GFP_KERNEL);
+	if (!bp)
+		return -ENOMEM;
+
+	per_cpu(threshold_banks, cpu) = bp;
+
+	for (bank = 0; bank < mca_cfg.banks; ++bank) {
+		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
+			continue;
+		err = threshold_create_bank(cpu, bank);
+		if (err)
+			return err;
+	}
+
+	return err;
+}
+
 /* get notified when a cpu comes on/off */
 static void
 amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
-- 
2.10.2

[PATCH 19/25] x86/mcheck: Explicit cleanup on failure in mce_amd

[Sebastian Andrzej Siewior]

If the ONLINE callback fails, the driver does not any clean up right
away instead it waits to get to the DEAD stage to do it. Yes, it waits.
Since we don't pass the error code back to the caller, no one knows.

Do the clean up right away so it does not look like a leak.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 75a3e3eab81e..55cd018bc1ae 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1087,6 +1087,7 @@ static void threshold_remove_device(unsigned int cpu)
 		threshold_remove_bank(cpu, bank);
 	}
 	kfree(per_cpu(threshold_banks, cpu));
+	per_cpu(threshold_banks, cpu) = NULL;
 }
 
 /* create dir/files for all valid threshold banks */
@@ -1108,9 +1109,11 @@ static int threshold_create_device(unsigned int cpu)
 			continue;
 		err = threshold_create_bank(cpu, bank);
 		if (err)
-			return err;
+			goto err;
 	}
-
+	return err;
+err:
+	threshold_remove_device(cpu);
 	return err;
 }
 
-- 
2.10.2

[PATCH 20/25] x86/mcheck: Be prepared for a rollback back to the ONLINE state

[Sebastian Andrzej Siewior]

If we try a CPU down and fail in the middle then we roll back to the
online state. This  means we would perform CPU_ONLINE()
without invoking CPU_DEAD() for the cleanup of what was allocated in
CPU_ONLINE.
Be prepared for this and don't allocate the struct if we have it
already.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce.c     | 4 ++++
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index a7fdf453d895..e9ffd6d9e32d 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2409,6 +2409,10 @@ static int mce_device_create(unsigned int cpu)
 	if (!mce_available(&boot_cpu_data))
 		return -EIO;
 
+	dev = per_cpu(mce_device, cpu);
+	if (dev)
+		return 0;
+
 	dev = kzalloc(sizeof *dev, GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 55cd018bc1ae..3e529fd747f8 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1097,6 +1097,9 @@ static int threshold_create_device(unsigned int cpu)
 	struct threshold_bank **bp;
 	int err = 0;
 
+	bp = per_cpu(threshold_banks, cpu);
+	if (bp)
+		return 0;
 	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
 		     GFP_KERNEL);
 	if (!bp)
-- 
2.10.2

[PATCH 21/25] x86/mcheck: Split threshold_cpu_callback into two callbacks

[Sebastian Andrzej Siewior]

The threshold_cpu_callback callbacks looks like one of the notifier and
its arguments are almost the same. Split this out and have one ONLINE
and one DEAD callback. This will come handy later once the main code
gets changed to use the callback mechanism.
Also, handle threshold_cpu_callback_online() return value so we don't
continue if the function fails.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/include/asm/mce.h           |  3 ++-
 arch/x86/kernel/cpu/mcheck/mce.c     | 18 +++++++++++++-----
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 24 ++++--------------------
 3 files changed, 19 insertions(+), 26 deletions(-)

diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 9bd7ff5ffbcc..978be74a12c1 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -295,7 +295,8 @@ void do_machine_check(struct pt_regs *, long);
  */
 
 extern void (*mce_threshold_vector)(void);
-extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
+extern int (*threshold_cpu_callback_online)(unsigned int cpu);
+extern int (*threshold_cpu_callback_dead)(unsigned int cpu);
 
 /* Deferred error interrupt handler */
 extern void (*deferred_error_int_vector)(void);
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index e9ffd6d9e32d..79b5ad9570ca 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2255,7 +2255,8 @@ static struct bus_type mce_subsys = {
 
 DEFINE_PER_CPU(struct device *, mce_device);
 
-void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
+int (*threshold_cpu_callback_online)(unsigned int cpu);
+int (*threshold_cpu_callback_dead)(unsigned int cpu);
 
 static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
 {
@@ -2513,12 +2514,19 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
 		mce_device_create(cpu);
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+		if (threshold_cpu_callback_online) {
+			int ret;
+
+			ret = threshold_cpu_callback_online(cpu);
+			if (ret) {
+				mce_device_remove(cpu);
+				return NOTIFY_BAD;
+			}
+		}
 		break;
 	case CPU_DEAD:
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+		if (threshold_cpu_callback_dead)
+			threshold_cpu_callback_dead(cpu);
 		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 3e529fd747f8..a99ae03f8c03 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1077,7 +1077,7 @@ static void threshold_remove_bank(unsigned int cpu, int bank)
 	per_cpu(threshold_banks, cpu)[bank] = NULL;
 }
 
-static void threshold_remove_device(unsigned int cpu)
+static int threshold_remove_device(unsigned int cpu)
 {
 	unsigned int bank;
 
@@ -1088,6 +1088,7 @@ static void threshold_remove_device(unsigned int cpu)
 	}
 	kfree(per_cpu(threshold_banks, cpu));
 	per_cpu(threshold_banks, cpu) = NULL;
+	return 0;
 }
 
 /* create dir/files for all valid threshold banks */
@@ -1120,24 +1121,6 @@ static int threshold_create_device(unsigned int cpu)
 	return err;
 }
 
-/* get notified when a cpu comes on/off */
-static void
-amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
-{
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		threshold_create_device(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		threshold_remove_device(cpu);
-		break;
-	default:
-		break;
-	}
-}
-
 static __init int threshold_init_device(void)
 {
 	unsigned lcpu = 0;
@@ -1149,7 +1132,8 @@ static __init int threshold_init_device(void)
 		if (err)
 			return err;
 	}
-	threshold_cpu_callback = amd_64_threshold_cpu_callback;
+	threshold_cpu_callback_online = threshold_create_device;
+	threshold_cpu_callback_dead = threshold_remove_device;
 
 	return 0;
 }
-- 
2.10.2

[PATCH 22/25] x86/mcheck: Do the init in one place

[Sebastian Andrzej Siewior]

Part of the init (memory allocation and so on) is done
in mcheck_cpu_init(). While moving the the allocation to
mcheck_init_device() (where the hotplug calls are initialized) it
becomes necessary to move the callback (mcheck_cpu_init()), too.

The callback is now removed from identify_cpu() and registered as a
hotplug event which is invoked as the very first one which is shortly
after the original point of invocation (look at smp_store_cpu_info() and
notify_cpu_starting() in smp_callin()).
One "visible" difference is that MCE for the boot CPU is not enabled at
identify_boot_cpu() time but at device_initcall_sync() time. Either way,
both times we had no userland around.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/include/asm/mce.h       |  2 --
 arch/x86/kernel/cpu/common.c     |  4 ---
 arch/x86/kernel/cpu/mcheck/mce.c | 77 ++++++++++++++++++++++++----------------
 include/linux/cpuhotplug.h       |  1 +
 4 files changed, 48 insertions(+), 36 deletions(-)

diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 978be74a12c1..ba8108166aec 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -203,12 +203,10 @@ extern int mce_p5_enabled;
 
 #ifdef CONFIG_X86_MCE
 int mcheck_init(void);
-void mcheck_cpu_init(struct cpuinfo_x86 *c);
 void mcheck_cpu_clear(struct cpuinfo_x86 *c);
 void mcheck_vendor_init_severity(void);
 #else
 static inline int mcheck_init(void) { return 0; }
-static inline void mcheck_cpu_init(struct cpuinfo_x86 *c) {}
 static inline void mcheck_cpu_clear(struct cpuinfo_x86 *c) {}
 static inline void mcheck_vendor_init_severity(void) {}
 #endif
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 9bd910a7dd0a..283b1fa64e69 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -39,7 +39,6 @@
 #include <asm/asm.h>
 #include <asm/bugs.h>
 #include <asm/cpu.h>
-#include <asm/mce.h>
 #include <asm/msr.h>
 #include <asm/pat.h>
 #include <asm/microcode.h>
@@ -1095,9 +1094,6 @@ static void identify_cpu(struct cpuinfo_x86 *c)
 			c->x86_capability[i] |= boot_cpu_data.x86_capability[i];
 	}
 
-	/* Init Machine Check Exception if available. */
-	mcheck_cpu_init(c);
-
 	select_idle_routine(c);
 
 #ifdef CONFIG_NUMA
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 79b5ad9570ca..72af9db8526d 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -1436,11 +1436,24 @@ EXPORT_SYMBOL_GPL(mce_notify_irq);
 static int __mcheck_cpu_mce_banks_init(void)
 {
 	int i;
-	u8 num_banks = mca_cfg.banks;
+	u64 cap;
+	unsigned int num_banks;
+
+	rdmsrl(MSR_IA32_MCG_CAP, cap);
+
+	num_banks = cap & MCG_BANKCNT_MASK;
+	pr_info("CPU supports %d MCE banks\n", num_banks);
+
+	if (num_banks > MAX_NR_BANKS) {
+		pr_warn("Using only %u machine check banks out of %u\n",
+			MAX_NR_BANKS, num_banks);
+		num_banks = MAX_NR_BANKS;
+	}
 
 	mce_banks = kzalloc(num_banks * sizeof(struct mce_bank), GFP_KERNEL);
 	if (!mce_banks)
 		return -ENOMEM;
+	mca_cfg.banks = num_banks;
 
 	for (i = 0; i < num_banks; i++) {
 		struct mce_bank *b = &mce_banks[i];
@@ -1462,25 +1475,11 @@ static int __mcheck_cpu_cap_init(void)
 	rdmsrl(MSR_IA32_MCG_CAP, cap);
 
 	b = cap & MCG_BANKCNT_MASK;
-	if (!mca_cfg.banks)
-		pr_info("CPU supports %d MCE banks\n", b);
-
-	if (b > MAX_NR_BANKS) {
-		pr_warn("Using only %u machine check banks out of %u\n",
-			MAX_NR_BANKS, b);
+	if (b > MAX_NR_BANKS)
 		b = MAX_NR_BANKS;
-	}
 
 	/* Don't support asymmetric configurations today */
-	WARN_ON(mca_cfg.banks != 0 && b != mca_cfg.banks);
-	mca_cfg.banks = b;
-
-	if (!mce_banks) {
-		int err = __mcheck_cpu_mce_banks_init();
-
-		if (err)
-			return err;
-	}
+	WARN_ON(b != mca_cfg.banks);
 
 	/* Use accurate RIP reporting if available. */
 	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
@@ -1769,26 +1768,22 @@ void (*machine_check_vector)(struct pt_regs *, long error_code) =
  * Called for each booted CPU to set up machine checks.
  * Must be called with preempt off:
  */
-void mcheck_cpu_init(struct cpuinfo_x86 *c)
+static int mcheck_cpu_starting(unsigned int cpu)
 {
+	struct cpuinfo_x86 *c = &cpu_data(cpu);
+
 	if (mca_cfg.disabled)
-		return;
+		return 0;
 
 	if (__mcheck_cpu_ancient_init(c))
-		return;
+		return 0;
 
 	if (!mce_available(c))
-		return;
+		return 0;
 
 	if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
 		mca_cfg.disabled = true;
-		return;
-	}
-
-	if (mce_gen_pool_init()) {
-		mca_cfg.disabled = true;
-		pr_emerg("Couldn't allocate MCE records pool!\n");
-		return;
+		return 0;
 	}
 
 	machine_check_vector = do_machine_check;
@@ -1797,6 +1792,7 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c)
 	__mcheck_cpu_init_vendor(c);
 	__mcheck_cpu_init_clear_banks();
 	__mcheck_cpu_init_timer();
+	return 0;
 }
 
 /*
@@ -2584,11 +2580,26 @@ static __init int mcheck_init_device(void)
 		goto err_out;
 	}
 
+	err = __mcheck_cpu_mce_banks_init();
+	if (err)
+		goto err_out_mem;
+
 	mce_init_banks();
 
+	err = mce_gen_pool_init();
+	if (err) {
+		pr_emerg("Couldn't allocate MCE records pool!\n");
+		goto err_init_pool;
+	}
+
 	err = subsys_system_register(&mce_subsys, NULL);
 	if (err)
-		goto err_out_mem;
+		goto err_init_pool;
+
+	err = cpuhp_setup_state(CPUHP_AP_X86_MCE_STARTING, "x86/mce:starting",
+				mcheck_cpu_starting, NULL);
+	if (err)
+		goto err_init_pool;
 
 	cpu_notifier_register_begin();
 	for_each_online_cpu(i) {
@@ -2630,12 +2641,18 @@ static __init int mcheck_init_device(void)
 	for_each_possible_cpu(i)
 		mce_device_remove(i);
 
+	cpuhp_remove_state(CPUHP_AP_X86_MCE_STARTING);
+
+err_init_pool:
+	mca_cfg.banks = 0;
+	kfree(mce_banks);
+	mce_banks = NULL;
+
 err_out_mem:
 	free_cpumask_var(mce_device_initialized);
 
 err_out:
 	pr_err("Unable to init device /dev/mcelog (rc: %d)\n", err);
-
 	return err;
 }
 device_initcall_sync(mcheck_init_device);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 3410d83cc2e2..4fe2ba418471 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -64,6 +64,7 @@ enum cpuhp_state {
 	CPUHP_BRINGUP_CPU,
 	CPUHP_AP_IDLE_DEAD,
 	CPUHP_AP_OFFLINE,
+	CPUHP_AP_X86_MCE_STARTING,
 	CPUHP_AP_SCHED_STARTING,
 	CPUHP_AP_RCUTREE_DYING,
 	CPUHP_AP_IRQ_GIC_STARTING,
-- 
2.10.2

[PATCH 23/25] x86/mcheck: Make CPU_DOWN_PREPARE the counter part of CPU_STARTING

[Sebastian Andrzej Siewior]

The previous patch moved mcheck_cpu_init() out of identify_cpu() and put
it as the first CPU hotplug callback which is invoked on the target CPU
during bring up. It enables MCE and starts the MCE timer. If a CPU goes
down then those two things have to be reverted and this happens
currently in CPU_DOWN_PREPARE. This is not symmetrical because
CPU_DOWN_PREPARE is the counterpart of CPU_ONLINE.
Usually CPU_DOWN_FAILED and CPU_ONLINE can do the same thing but not in
this case since here (in CPU_DOWN_FAILED) it tries to revert what was
done in CPU_DOWN_PREPARE.

To make this simpler make CPU_DOWN_PREPARE the counterpart of
mcheck_cpu_starting() and just disable MCE and stop the timer. With this
change the callback is symmetrical again and we don't need CPU_DOWN_FAILED
including mce_reenable_cpu(). smp_call_function_single() can be dropped
because it is already invoked on the proper CPU and interrupts are disabled
at this point.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce.c | 44 +++++++++++-----------------------------
 1 file changed, 12 insertions(+), 32 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 72af9db8526d..596a7128a46b 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2469,43 +2469,22 @@ static void mce_device_remove(unsigned int cpu)
 }
 
 /* Make sure there are no machine checks on offlined CPUs. */
-static void mce_disable_cpu(void *h)
+static void mce_disable_cpu(void)
 {
-	unsigned long action = *(unsigned long *)h;
-
 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
 		return;
 
-	if (!(action & CPU_TASKS_FROZEN))
+	if (!cpuhp_tasks_frozen)
 		cmci_clear();
 
 	vendor_disable_error_reporting();
 }
 
-static void mce_reenable_cpu(void *h)
-{
-	unsigned long action = *(unsigned long *)h;
-	int i;
-
-	if (!mce_available(raw_cpu_ptr(&cpu_info)))
-		return;
-
-	if (!(action & CPU_TASKS_FROZEN))
-		cmci_reenable();
-	for (i = 0; i < mca_cfg.banks; i++) {
-		struct mce_bank *b = &mce_banks[i];
-
-		if (b->init)
-			wrmsrl(msr_ops.ctl(i), b->ctl);
-	}
-}
-
 /* Get notified when a cpu comes on/off. Be hotplug friendly. */
 static int
 mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 {
 	unsigned int cpu = (unsigned long)hcpu;
-	struct timer_list *t = &per_cpu(mce_timer, cpu);
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
@@ -2530,19 +2509,20 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 		if (!(action & CPU_TASKS_FROZEN))
 			cmci_rediscover();
 		break;
-	case CPU_DOWN_PREPARE:
-		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
-		del_timer_sync(t);
-		break;
-	case CPU_DOWN_FAILED:
-		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
-		mce_start_timer(cpu, t);
-		break;
 	}
 
 	return NOTIFY_OK;
 }
 
+static int mce_cpu_down_dying(unsigned int cpu)
+{
+	struct timer_list *t = this_cpu_ptr(&mce_timer);
+
+	mce_disable_cpu();
+	del_timer_sync(t);
+	return 0;
+}
+
 static struct notifier_block mce_cpu_notifier = {
 	.notifier_call = mce_cpu_callback,
 };
@@ -2597,7 +2577,7 @@ static __init int mcheck_init_device(void)
 		goto err_init_pool;
 
 	err = cpuhp_setup_state(CPUHP_AP_X86_MCE_STARTING, "x86/mce:starting",
-				mcheck_cpu_starting, NULL);
+				mcheck_cpu_starting, mce_cpu_down_dying);
 	if (err)
 		goto err_init_pool;
 
-- 
2.10.2

[PATCH 24/25] x86/mcheck: Move CPU_ONLINE to hotplug state machine

[Sebastian Andrzej Siewior]

This callback is still partly asymmetrical since the counterpart of
mce_device_create is done in CPU_DEAD.

On failure we don't undo mce_device_create() doing _but_ it will happen
once we move CPU_DEAD to the state machine.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce.c | 61 ++++++++++++++++------------------------
 1 file changed, 24 insertions(+), 37 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 596a7128a46b..b1770ebcb8de 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2487,18 +2487,6 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 	unsigned int cpu = (unsigned long)hcpu;
 
 	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-		mce_device_create(cpu);
-		if (threshold_cpu_callback_online) {
-			int ret;
-
-			ret = threshold_cpu_callback_online(cpu);
-			if (ret) {
-				mce_device_remove(cpu);
-				return NOTIFY_BAD;
-			}
-		}
-		break;
 	case CPU_DEAD:
 		if (threshold_cpu_callback_dead)
 			threshold_cpu_callback_dead(cpu);
@@ -2514,6 +2502,22 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 	return NOTIFY_OK;
 }
 
+static int mce_cpu_online(unsigned int cpu)
+{
+	int ret;
+
+	mce_device_create(cpu);
+	if (!threshold_cpu_callback_online)
+		return 0;
+
+	ret = threshold_cpu_callback_online(cpu);
+	if (ret) {
+		mce_device_remove(cpu);
+		return ret;
+	}
+	return 0;
+}
+
 static int mce_cpu_down_dying(unsigned int cpu)
 {
 	struct timer_list *t = this_cpu_ptr(&mce_timer);
@@ -2547,8 +2551,8 @@ static __init void mce_init_banks(void)
 
 static __init int mcheck_init_device(void)
 {
+	enum cpuhp_state hp_online;
 	int err;
-	int i = 0;
 
 	if (!mce_available(&boot_cpu_data)) {
 		err = -EIO;
@@ -2580,22 +2584,13 @@ static __init int mcheck_init_device(void)
 				mcheck_cpu_starting, mce_cpu_down_dying);
 	if (err)
 		goto err_init_pool;
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
+				mce_cpu_online, NULL);
+	if (err < 0)
+		goto err_hp_online;
+	hp_online = err;
 
 	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = mce_device_create(i);
-		if (err) {
-			/*
-			 * Register notifier anyway (and do not unreg it) so
-			 * that we don't leave undeleted timers, see notifier
-			 * callback above.
-			 */
-			__register_hotcpu_notifier(&mce_cpu_notifier);
-			cpu_notifier_register_done();
-			goto err_device_create;
-		}
-	}
-
 	__register_hotcpu_notifier(&mce_cpu_notifier);
 	cpu_notifier_register_done();
 
@@ -2610,17 +2605,9 @@ static __init int mcheck_init_device(void)
 
 err_register:
 	unregister_syscore_ops(&mce_syscore_ops);
+	cpuhp_remove_state(hp_online);
 
-err_device_create:
-	/*
-	 * We didn't keep track of which devices were created above, but
-	 * even if we had, the set of online cpus might have changed.
-	 * Play safe and remove for every possible cpu, since
-	 * mce_device_remove() will do the right thing.
-	 */
-	for_each_possible_cpu(i)
-		mce_device_remove(i);
-
+err_hp_online:
 	cpuhp_remove_state(CPUHP_AP_X86_MCE_STARTING);
 
 err_init_pool:
-- 
2.10.2

[PATCH 25/25] x86/mcheck: Move CPU_DEAD to hotplug state machine

[Sebastian Andrzej Siewior]

This moves the last piece of the old hotplug notifier code in MCE to the
new hotplug state machine.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce.c | 53 +++++++++++++++++-----------------------
 include/linux/cpuhotplug.h       |  1 +
 2 files changed, 23 insertions(+), 31 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index b1770ebcb8de..1e2036e012e0 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2480,28 +2480,6 @@ static void mce_disable_cpu(void)
 	vendor_disable_error_reporting();
 }
 
-/* Get notified when a cpu comes on/off. Be hotplug friendly. */
-static int
-mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DEAD:
-		if (threshold_cpu_callback_dead)
-			threshold_cpu_callback_dead(cpu);
-		mce_device_remove(cpu);
-		mce_intel_hcpu_update(cpu);
-
-		/* intentionally ignoring frozen here */
-		if (!(action & CPU_TASKS_FROZEN))
-			cmci_rediscover();
-		break;
-	}
-
-	return NOTIFY_OK;
-}
-
 static int mce_cpu_online(unsigned int cpu)
 {
 	int ret;
@@ -2518,6 +2496,19 @@ static int mce_cpu_online(unsigned int cpu)
 	return 0;
 }
 
+static int mce_cpu_dead(unsigned int cpu)
+{
+	if (threshold_cpu_callback_dead)
+		threshold_cpu_callback_dead(cpu);
+	mce_device_remove(cpu);
+	mce_intel_hcpu_update(cpu);
+
+	/* intentionally ignoring frozen here */
+	if (!cpuhp_tasks_frozen)
+		cmci_rediscover();
+	return 0;
+}
+
 static int mce_cpu_down_dying(unsigned int cpu)
 {
 	struct timer_list *t = this_cpu_ptr(&mce_timer);
@@ -2527,10 +2518,6 @@ static int mce_cpu_down_dying(unsigned int cpu)
 	return 0;
 }
 
-static struct notifier_block mce_cpu_notifier = {
-	.notifier_call = mce_cpu_callback,
-};
-
 static __init void mce_init_banks(void)
 {
 	int i;
@@ -2580,20 +2567,21 @@ static __init int mcheck_init_device(void)
 	if (err)
 		goto err_init_pool;
 
+	err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
+				mce_cpu_dead);
+	if (err)
+		goto err_init_pool;
+
 	err = cpuhp_setup_state(CPUHP_AP_X86_MCE_STARTING, "x86/mce:starting",
 				mcheck_cpu_starting, mce_cpu_down_dying);
 	if (err)
-		goto err_init_pool;
+		goto err_hp_starting;
 	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
 				mce_cpu_online, NULL);
 	if (err < 0)
 		goto err_hp_online;
 	hp_online = err;
 
-	cpu_notifier_register_begin();
-	__register_hotcpu_notifier(&mce_cpu_notifier);
-	cpu_notifier_register_done();
-
 	register_syscore_ops(&mce_syscore_ops);
 
 	/* register character device /dev/mcelog */
@@ -2610,6 +2598,9 @@ static __init int mcheck_init_device(void)
 err_hp_online:
 	cpuhp_remove_state(CPUHP_AP_X86_MCE_STARTING);
 
+err_hp_starting:
+	cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
+
 err_init_pool:
 	mca_cfg.banks = 0;
 	kfree(mce_banks);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 4fe2ba418471..acb37a7f7831 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -16,6 +16,7 @@ enum cpuhp_state {
 	CPUHP_PERF_SUPERH,
 	CPUHP_X86_HPET_DEAD,
 	CPUHP_X86_APB_DEAD,
+	CPUHP_X86_MCE_DEAD,
 	CPUHP_VIRT_NET_DEAD,
 	CPUHP_SLUB_DEAD,
 	CPUHP_MM_WRITEBACK_DEAD,
-- 
2.10.2

Re: [PATCH 16/25] ia64/salinfo: Convert to hotplug state machine

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 4385 bytes --]

Hi Sebastian,

[auto build test ERROR on linus/master]
[also build test ERROR on v4.9-rc3]
[cannot apply to next-20161028]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Sebastian-Andrzej-Siewior/fs-buffer-Convert-to-hotplug-state-machine/20161103-230222
config: ia64-allmodconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 6.2.0
reproduce:
        wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=ia64 

All error/warnings (new ones prefixed by >>):

   In file included from include/linux/kernel.h:12:0,
                    from include/linux/list.h:8,
                    from include/linux/kobject.h:20,
                    from include/linux/device.h:17,
                    from include/linux/node.h:17,
                    from include/linux/cpu.h:16,
                    from arch/ia64/kernel/salinfo.c:40:
   arch/ia64/kernel/salinfo.c: In function 'salinfo_cpu_pre_down':
>> arch/ia64/kernel/salinfo.c:572:38: error: 'flags' undeclared (first use in this function)
     spin_lock_irqsave(&data_saved_lock, flags);
                                         ^
   include/linux/typecheck.h:10:9: note: in definition of macro 'typecheck'
     typeof(x) __dummy2; \
            ^
   include/linux/spinlock.h:337:2: note: in expansion of macro 'raw_spin_lock_irqsave'
     raw_spin_lock_irqsave(spinlock_check(lock), flags); \
     ^~~~~~~~~~~~~~~~~~~~~
>> arch/ia64/kernel/salinfo.c:572:2: note: in expansion of macro 'spin_lock_irqsave'
     spin_lock_irqsave(&data_saved_lock, flags);
     ^~~~~~~~~~~~~~~~~
   arch/ia64/kernel/salinfo.c:572:38: note: each undeclared identifier is reported only once for each function it appears in
     spin_lock_irqsave(&data_saved_lock, flags);
                                         ^
   include/linux/typecheck.h:10:9: note: in definition of macro 'typecheck'
     typeof(x) __dummy2; \
            ^
   include/linux/spinlock.h:337:2: note: in expansion of macro 'raw_spin_lock_irqsave'
     raw_spin_lock_irqsave(spinlock_check(lock), flags); \
     ^~~~~~~~~~~~~~~~~~~~~
>> arch/ia64/kernel/salinfo.c:572:2: note: in expansion of macro 'spin_lock_irqsave'
     spin_lock_irqsave(&data_saved_lock, flags);
     ^~~~~~~~~~~~~~~~~
>> include/linux/typecheck.h:11:18: warning: comparison of distinct pointer types lacks a cast
     (void)(&__dummy == &__dummy2); \
                     ^
>> include/linux/spinlock.h:207:3: note: in expansion of macro 'typecheck'
      typecheck(unsigned long, flags); \
      ^~~~~~~~~
   include/linux/spinlock.h:337:2: note: in expansion of macro 'raw_spin_lock_irqsave'
     raw_spin_lock_irqsave(spinlock_check(lock), flags); \
     ^~~~~~~~~~~~~~~~~~~~~
>> arch/ia64/kernel/salinfo.c:572:2: note: in expansion of macro 'spin_lock_irqsave'
     spin_lock_irqsave(&data_saved_lock, flags);
     ^~~~~~~~~~~~~~~~~

vim +/flags +572 arch/ia64/kernel/salinfo.c

f0c5b094 Sebastian Andrzej Siewior 2016-11-03  566  
f0c5b094 Sebastian Andrzej Siewior 2016-11-03  567  static int salinfo_cpu_pre_down(unsigned int cpu)
f0c5b094 Sebastian Andrzej Siewior 2016-11-03  568  {
f0c5b094 Sebastian Andrzej Siewior 2016-11-03  569  	unsigned int i, end = ARRAY_SIZE(salinfo_data);
f0c5b094 Sebastian Andrzej Siewior 2016-11-03  570  	struct salinfo_data *data;
f0c5b094 Sebastian Andrzej Siewior 2016-11-03  571  
e026cca0 Keith Owens               2006-01-06 @572  	spin_lock_irqsave(&data_saved_lock, flags);
f0c5b094 Sebastian Andrzej Siewior 2016-11-03  573  	for (i = 0, data = salinfo_data; i < end; ++i, ++data) {
e026cca0 Keith Owens               2006-01-06  574  		struct salinfo_data_saved *data_saved;
f0c5b094 Sebastian Andrzej Siewior 2016-11-03  575  		int j = ARRAY_SIZE(data->data_saved) - 1;

:::::: The code at line 572 was first introduced by commit
:::::: e026cca0f2c09c4c28c902db6384fd8a412671d6 [IA64] Add hotplug cpu to salinfo.c, replace semaphore with mutex

:::::: TO: Keith Owens <kaos@sgi.com>
:::::: CC: Tony Luck <tony.luck@intel.com>

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 45092 bytes --]

Re: [PATCH 17/25] ia64/topology: Convert to hotplug state machine

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 2075 bytes --]

Hi Sebastian,

[auto build test WARNING on linus/master]
[also build test WARNING on v4.9-rc3]
[cannot apply to next-20161028]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Sebastian-Andrzej-Siewior/fs-buffer-Convert-to-hotplug-state-machine/20161103-230222
config: ia64-allmodconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 6.2.0
reproduce:
        wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=ia64 

All warnings (new ones prefixed by >>):

   arch/ia64/kernel/topology.c: In function 'cache_remove_dev':
>> arch/ia64/kernel/topology.c:404:17: warning: unused variable 'sys_dev' [-Wunused-variable]
     struct device *sys_dev = get_cpu_device(cpu);
                    ^~~~~~~

vim +/sys_dev +404 arch/ia64/kernel/topology.c

   388				for (j = 0; j < i; j++) {
   389					kobject_put(&(LEAF_KOBJECT_PTR(cpu,j)->kobj));
   390				}
   391				kobject_put(&all_cpu_cache_info[cpu].kobj);
   392				cpu_cache_sysfs_exit(cpu);
   393				return retval;
   394			}
   395			kobject_uevent(&(this_object->kobj), KOBJ_ADD);
   396		}
   397		kobject_uevent(&all_cpu_cache_info[cpu].kobj, KOBJ_ADD);
   398		return retval;
   399	}
   400	
   401	/* Remove cache interface for CPU device */
   402	static int cache_remove_dev(unsigned int cpu)
   403	{
 > 404		struct device *sys_dev = get_cpu_device(cpu);
   405		unsigned long i;
   406	
   407		for (i = 0; i < all_cpu_cache_info[cpu].num_cache_leaves; i++)
   408			kobject_put(&(LEAF_KOBJECT_PTR(cpu,i)->kobj));
   409	
   410		if (all_cpu_cache_info[cpu].kobj.parent) {
   411			kobject_put(&all_cpu_cache_info[cpu].kobj);
   412			memset(&all_cpu_cache_info[cpu].kobj,

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 45092 bytes --]

Re: [PATCH 16/25] ia64/salinfo: Convert to hotplug state machine

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 6622 bytes --]

Hi Sebastian,

[auto build test WARNING on linus/master]
[also build test WARNING on v4.9-rc3]
[cannot apply to next-20161028]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Sebastian-Andrzej-Siewior/fs-buffer-Convert-to-hotplug-state-machine/20161103-230222
config: ia64-allnoconfig (attached as .config)
compiler: ia64-linux-gcc (GCC) 6.2.0
reproduce:
        wget https://git.kernel.org/cgit/linux/kernel/git/wfg/lkp-tests.git/plain/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=ia64 

All warnings (new ones prefixed by >>):

   In file included from include/linux/kernel.h:12:0,
                    from include/linux/list.h:8,
                    from include/linux/kobject.h:20,
                    from include/linux/device.h:17,
                    from include/linux/node.h:17,
                    from include/linux/cpu.h:16,
                    from arch/ia64/kernel/salinfo.c:40:
   arch/ia64/kernel/salinfo.c: In function 'salinfo_cpu_pre_down':
   arch/ia64/kernel/salinfo.c:572:38: error: 'flags' undeclared (first use in this function)
     spin_lock_irqsave(&data_saved_lock, flags);
                                         ^
   include/linux/typecheck.h:10:9: note: in definition of macro 'typecheck'
     typeof(x) __dummy2; \
            ^
   include/linux/spinlock.h:337:2: note: in expansion of macro 'raw_spin_lock_irqsave'
     raw_spin_lock_irqsave(spinlock_check(lock), flags); \
     ^~~~~~~~~~~~~~~~~~~~~
   arch/ia64/kernel/salinfo.c:572:2: note: in expansion of macro 'spin_lock_irqsave'
     spin_lock_irqsave(&data_saved_lock, flags);
     ^~~~~~~~~~~~~~~~~
   arch/ia64/kernel/salinfo.c:572:38: note: each undeclared identifier is reported only once for each function it appears in
     spin_lock_irqsave(&data_saved_lock, flags);
                                         ^
   include/linux/typecheck.h:10:9: note: in definition of macro 'typecheck'
     typeof(x) __dummy2; \
            ^
   include/linux/spinlock.h:337:2: note: in expansion of macro 'raw_spin_lock_irqsave'
     raw_spin_lock_irqsave(spinlock_check(lock), flags); \
     ^~~~~~~~~~~~~~~~~~~~~
   arch/ia64/kernel/salinfo.c:572:2: note: in expansion of macro 'spin_lock_irqsave'
     spin_lock_irqsave(&data_saved_lock, flags);
     ^~~~~~~~~~~~~~~~~
   include/linux/typecheck.h:11:18: warning: comparison of distinct pointer types lacks a cast
     (void)(&__dummy == &__dummy2); \
                     ^
   include/linux/spinlock.h:229:3: note: in expansion of macro 'typecheck'
      typecheck(unsigned long, flags); \
      ^~~~~~~~~
   include/linux/spinlock.h:337:2: note: in expansion of macro 'raw_spin_lock_irqsave'
     raw_spin_lock_irqsave(spinlock_check(lock), flags); \
     ^~~~~~~~~~~~~~~~~~~~~
   arch/ia64/kernel/salinfo.c:572:2: note: in expansion of macro 'spin_lock_irqsave'
     spin_lock_irqsave(&data_saved_lock, flags);
     ^~~~~~~~~~~~~~~~~
   include/linux/typecheck.h:11:18: warning: comparison of distinct pointer types lacks a cast
     (void)(&__dummy == &__dummy2); \
                     ^
>> include/linux/irqflags.h:63:3: note: in expansion of macro 'typecheck'
      typecheck(unsigned long, flags); \
      ^~~~~~~~~
   include/linux/irqflags.h:124:3: note: in expansion of macro 'raw_local_irq_save'
      raw_local_irq_save(flags);   \
      ^~~~~~~~~~~~~~~~~~
   include/linux/spinlock_api_up.h:40:8: note: in expansion of macro 'local_irq_save'
      do { local_irq_save(flags); __LOCK(lock); } while (0)
           ^~~~~~~~~~~~~~
   include/linux/spinlock_api_up.h:69:45: note: in expansion of macro '__LOCK_IRQSAVE'
    #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags)
                                                ^~~~~~~~~~~~~~
   include/linux/spinlock.h:230:3: note: in expansion of macro '_raw_spin_lock_irqsave'
      _raw_spin_lock_irqsave(lock, flags); \
      ^~~~~~~~~~~~~~~~~~~~~~
   include/linux/spinlock.h:337:2: note: in expansion of macro 'raw_spin_lock_irqsave'
     raw_spin_lock_irqsave(spinlock_check(lock), flags); \
     ^~~~~~~~~~~~~~~~~~~~~
   arch/ia64/kernel/salinfo.c:572:2: note: in expansion of macro 'spin_lock_irqsave'
     spin_lock_irqsave(&data_saved_lock, flags);
     ^~~~~~~~~~~~~~~~~

vim +/typecheck +63 include/linux/irqflags.h

6cd8a4bb Steven Rostedt 2008-05-12  47  #if defined(CONFIG_IRQSOFF_TRACER) || \
6cd8a4bb Steven Rostedt 2008-05-12  48  	defined(CONFIG_PREEMPT_TRACER)
81d68a96 Steven Rostedt 2008-05-12  49   extern void stop_critical_timings(void);
81d68a96 Steven Rostedt 2008-05-12  50   extern void start_critical_timings(void);
81d68a96 Steven Rostedt 2008-05-12  51  #else
81d68a96 Steven Rostedt 2008-05-12  52  # define stop_critical_timings() do { } while (0)
81d68a96 Steven Rostedt 2008-05-12  53  # define start_critical_timings() do { } while (0)
81d68a96 Steven Rostedt 2008-05-12  54  #endif
81d68a96 Steven Rostedt 2008-05-12  55  
df9ee292 David Howells  2010-10-07  56  /*
df9ee292 David Howells  2010-10-07  57   * Wrap the arch provided IRQ routines to provide appropriate checks.
df9ee292 David Howells  2010-10-07  58   */
df9ee292 David Howells  2010-10-07  59  #define raw_local_irq_disable()		arch_local_irq_disable()
df9ee292 David Howells  2010-10-07  60  #define raw_local_irq_enable()		arch_local_irq_enable()
df9ee292 David Howells  2010-10-07  61  #define raw_local_irq_save(flags)			\
df9ee292 David Howells  2010-10-07  62  	do {						\
df9ee292 David Howells  2010-10-07 @63  		typecheck(unsigned long, flags);	\
df9ee292 David Howells  2010-10-07  64  		flags = arch_local_irq_save();		\
df9ee292 David Howells  2010-10-07  65  	} while (0)
df9ee292 David Howells  2010-10-07  66  #define raw_local_irq_restore(flags)			\
df9ee292 David Howells  2010-10-07  67  	do {						\
df9ee292 David Howells  2010-10-07  68  		typecheck(unsigned long, flags);	\
df9ee292 David Howells  2010-10-07  69  		arch_local_irq_restore(flags);		\
df9ee292 David Howells  2010-10-07  70  	} while (0)
df9ee292 David Howells  2010-10-07  71  #define raw_local_save_flags(flags)			\

:::::: The code at line 63 was first introduced by commit
:::::: df9ee29270c11dba7d0fe0b83ce47a4d8e8d2101 Fix IRQ flag handling naming

:::::: TO: David Howells <dhowells@redhat.com>
:::::: CC: David Howells <dhowells@redhat.com>

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 5715 bytes --]

[PATCH 16/25 v2] ia64/salinfo: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
v1…v2: non the save version. Noticed by kbuild test robot

 arch/ia64/kernel/salinfo.c | 83 ++++++++++++++++++----------------------------
 1 file changed, 32 insertions(+), 51 deletions(-)

diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c
index 5313007d5423..aaf74f36cfa1 100644
--- a/arch/ia64/kernel/salinfo.c
+++ b/arch/ia64/kernel/salinfo.c
@@ -550,52 +550,40 @@ static const struct file_operations salinfo_data_fops = {
 	.llseek  = default_llseek,
 };
 
-static int
-salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu)
+static int salinfo_cpu_online(unsigned int cpu)
 {
-	unsigned int i, cpu = (unsigned long)hcpu;
-	unsigned long flags;
+	unsigned int i, end = ARRAY_SIZE(salinfo_data);
 	struct salinfo_data *data;
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		spin_lock_irqsave(&data_saved_lock, flags);
-		for (i = 0, data = salinfo_data;
-		     i < ARRAY_SIZE(salinfo_data);
-		     ++i, ++data) {
-			cpumask_set_cpu(cpu, &data->cpu_event);
-			wake_up_interruptible(&data->read_wait);
-		}
-		spin_unlock_irqrestore(&data_saved_lock, flags);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		spin_lock_irqsave(&data_saved_lock, flags);
-		for (i = 0, data = salinfo_data;
-		     i < ARRAY_SIZE(salinfo_data);
-		     ++i, ++data) {
-			struct salinfo_data_saved *data_saved;
-			int j;
-			for (j = ARRAY_SIZE(data->data_saved) - 1, data_saved = data->data_saved + j;
-			     j >= 0;
-			     --j, --data_saved) {
-				if (data_saved->buffer && data_saved->cpu == cpu) {
-					shift1_data_saved(data, j);
-				}
-			}
-			cpumask_clear_cpu(cpu, &data->cpu_event);
-		}
-		spin_unlock_irqrestore(&data_saved_lock, flags);
-		break;
+
+	spin_lock_irq(&data_saved_lock);
+	for (i = 0, data = salinfo_data; i < end; ++i, ++data) {
+		cpumask_set_cpu(cpu, &data->cpu_event);
+		wake_up_interruptible(&data->read_wait);
 	}
-	return NOTIFY_OK;
+	spin_unlock_irq(&data_saved_lock);
+	return 0;
 }
 
-static struct notifier_block salinfo_cpu_notifier =
+static int salinfo_cpu_pre_down(unsigned int cpu)
 {
-	.notifier_call = salinfo_cpu_callback,
-	.priority = 0,
-};
+	unsigned int i, end = ARRAY_SIZE(salinfo_data);
+	struct salinfo_data *data;
+
+	spin_lock_irq(&data_saved_lock);
+	for (i = 0, data = salinfo_data; i < end; ++i, ++data) {
+		struct salinfo_data_saved *data_saved;
+		int j = ARRAY_SIZE(data->data_saved) - 1;
+
+		for (data_saved = data->data_saved + j; j >= 0;
+		     --j, --data_saved) {
+			if (data_saved->buffer && data_saved->cpu == cpu)
+				shift1_data_saved(data, j);
+		}
+		cpumask_clear_cpu(cpu, &data->cpu_event);
+	}
+	spin_unlock_irq(&data_saved_lock);
+	return 0;
+}
 
 static int __init
 salinfo_init(void)
@@ -604,7 +592,7 @@ salinfo_init(void)
 	struct proc_dir_entry **sdir = salinfo_proc_entries; /* keeps track of every entry */
 	struct proc_dir_entry *dir, *entry;
 	struct salinfo_data *data;
-	int i, j;
+	int i;
 
 	salinfo_dir = proc_mkdir("sal", NULL);
 	if (!salinfo_dir)
@@ -617,8 +605,6 @@ salinfo_init(void)
 					   (void *)salinfo_entries[i].feature);
 	}
 
-	cpu_notifier_register_begin();
-
 	for (i = 0; i < ARRAY_SIZE(salinfo_log_name); i++) {
 		data = salinfo_data + i;
 		data->type = i;
@@ -639,10 +625,6 @@ salinfo_init(void)
 			continue;
 		*sdir++ = entry;
 
-		/* we missed any events before now */
-		for_each_online_cpu(j)
-			cpumask_set_cpu(j, &data->cpu_event);
-
 		*sdir++ = dir;
 	}
 
@@ -653,10 +635,9 @@ salinfo_init(void)
 	salinfo_timer.function = &salinfo_timeout;
 	add_timer(&salinfo_timer);
 
-	__register_hotcpu_notifier(&salinfo_cpu_notifier);
-
-	cpu_notifier_register_done();
-
+	i = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/salinfo:online",
+			      salinfo_cpu_online, salinfo_cpu_pre_down);
+	WARN_ON(i < 0);
 	return 0;
 }
 
-- 
2.10.2

[PATCH 17/25 v2] ia64/topology: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: linux-ia64@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
v1…v2: remove unused variable. Noticed by kbuild test robot

 arch/ia64/kernel/topology.c | 54 ++++++---------------------------------------
 1 file changed, 7 insertions(+), 47 deletions(-)

diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c
index c01fe8991244..1a68f012a6dc 100644
--- a/arch/ia64/kernel/topology.c
+++ b/arch/ia64/kernel/topology.c
@@ -349,9 +349,9 @@ static int cpu_cache_sysfs_init(unsigned int cpu)
 }
 
 /* Add cache interface for CPU device */
-static int cache_add_dev(struct device *sys_dev)
+static int cache_add_dev(unsigned int cpu)
 {
-	unsigned int cpu = sys_dev->id;
+	struct device *sys_dev = get_cpu_device(cpu);
 	unsigned long i, j;
 	struct cache_info *this_object;
 	int retval = 0;
@@ -399,9 +399,8 @@ static int cache_add_dev(struct device *sys_dev)
 }
 
 /* Remove cache interface for CPU device */
-static int cache_remove_dev(struct device *sys_dev)
+static int cache_remove_dev(unsigned int cpu)
 {
-	unsigned int cpu = sys_dev->id;
 	unsigned long i;
 
 	for (i = 0; i < all_cpu_cache_info[cpu].num_cache_leaves; i++)
@@ -419,52 +418,13 @@ static int cache_remove_dev(struct device *sys_dev)
 	return 0;
 }
 
-/*
- * When a cpu is hot-plugged, do a check and initiate
- * cache kobject if necessary
- */
-static int cache_cpu_callback(struct notifier_block *nfb,
-		unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	struct device *sys_dev;
-
-	sys_dev = get_cpu_device(cpu);
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		cache_add_dev(sys_dev);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		cache_remove_dev(sys_dev);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block cache_cpu_notifier =
-{
-	.notifier_call = cache_cpu_callback
-};
-
 static int __init cache_sysfs_init(void)
 {
-	int i;
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		struct device *sys_dev = get_cpu_device((unsigned int)i);
-		cache_add_dev(sys_dev);
-	}
-
-	__register_hotcpu_notifier(&cache_cpu_notifier);
-
-	cpu_notifier_register_done();
+	int ret;
 
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/topology:online",
+				cache_add_dev, cache_remove_dev);
+	WARN_ON(ret < 0);
 	return 0;
 }
-
 device_initcall(cache_sysfs_init);
-
-- 
2.10.2

Re: [PATCH 10/25] s390/smp: Make cpu notifier symetric

[Heiko Carstens]

On Thu, Nov 03, 2016 at 03:50:06PM +0100, Sebastian Andrzej Siewior wrote:
> From: Thomas Gleixner <tglx@linutronix.de>
> 
> There is no reason to remove the sysfs cpu files when the CPU is dead, they
> can be removed when the cpu is prepared to go down. Doing it at
> DOWN_PREPARE allows us to convert it to a symetric hotplug state in the
> next step.
> 
> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
> Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
> Cc: linux-s390@vger.kernel.org
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> ---
>  arch/s390/kernel/smp.c | 3 ++-
>  1 file changed, 2 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
> index 35531fe1c5ea..1a21e66c484a 100644
> --- a/arch/s390/kernel/smp.c
> +++ b/arch/s390/kernel/smp.c
> @@ -1056,9 +1056,10 @@ static int smp_cpu_notify(struct notifier_block *self, unsigned long action,
>  
>  	switch (action & ~CPU_TASKS_FROZEN) {
>  	case CPU_ONLINE:
> +	case CPU_DOWN_FAILED:
>  		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
>  		break;
> -	case CPU_DEAD:
> +	case CPU_DOWN_PREPARE::

This won't compile... even though it will be removed with the following
patch it would be good to fix this to keep this bisectable.

Re: [PATCH 11/25] s390/smp: Convert to hotplug state machine

[Heiko Carstens]

On Thu, Nov 03, 2016 at 03:50:07PM +0100, Sebastian Andrzej Siewior wrote:
> @@ -1084,20 +1079,13 @@ static int smp_add_present_cpu(int cpu)
>  	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
>  	if (rc)
>  		goto out_cpu;
> -	if (cpu_online(cpu)) {
> -		rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
> -		if (rc)
> -			goto out_online;
> -	}
> +
>  	rc = topology_cpu_init(c);

Please don't add whitespace :)

If you fix the minor nits both s390 related patches are
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com>

Thanks!

[PATCH 10/25 v2] s390/smp: Make cpu notifier symetric

[Sebastian Andrzej Siewior]

From: Thomas Gleixner <tglx@linutronix.de>

There is no reason to remove the sysfs cpu files when the CPU is dead, they
can be removed when the cpu is prepared to go down. Doing it at
DOWN_PREPARE allows us to convert it to a symetric hotplug state in the
next step.

Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: linux-s390@vger.kernel.org
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
v1…v2: s/::/

 arch/s390/kernel/smp.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -1056,9 +1056,10 @@ static int smp_cpu_notify(struct notifie
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+	case CPU_DOWN_FAILED:
 		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
 		break;
-	case CPU_DEAD:
+	case CPU_DOWN_PREPARE:
 		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
 		break;
 	}

[PATCH 11/25 v2] s390/smp: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

cpuhp_setup_state() invokes the startup callback on all online cpus with
the proper protection, so we can remove the cpu hotplug protection from the
init function and the creation of the per cpu files for online cpus in
smp_add_present_cpu(). smp_add_present_cpu() is called also called from
__smp_rescan_cpus(), but this callpath never adds an online cpu, it merily
adds newly present cpus, so the creation of the cpu files is not required.

Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: linux-s390@vger.kernel.org
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
v1…v2:
	- removed (or not adding) one empty line
	- rebase on top of the previous patch

 arch/s390/kernel/smp.c |   37 +++++++++++--------------------------
 1 file changed, 11 insertions(+), 26 deletions(-)

--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -1047,23 +1047,18 @@ static struct attribute_group cpu_online
 	.attrs = cpu_online_attrs,
 };
 
-static int smp_cpu_notify(struct notifier_block *self, unsigned long action,
-			  void *hcpu)
+static int smp_cpu_online(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned int)(long)hcpu;
 	struct device *s = &per_cpu(cpu_device, cpu)->dev;
-	int err = 0;
 
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
-		break;
-	case CPU_DOWN_PREPARE:
-		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
-		break;
-	}
-	return notifier_from_errno(err);
+	return sysfs_create_group(&s->kobj, &cpu_online_attr_group);
+}
+static int smp_cpu_pre_down(unsigned int cpu)
+{
+	struct device *s = &per_cpu(cpu_device, cpu)->dev;
+
+	sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
+	return 0;
 }
 
 static int smp_add_present_cpu(int cpu)
@@ -1084,20 +1079,12 @@ static int smp_add_present_cpu(int cpu)
 	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
 	if (rc)
 		goto out_cpu;
-	if (cpu_online(cpu)) {
-		rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
-		if (rc)
-			goto out_online;
-	}
 	rc = topology_cpu_init(c);
 	if (rc)
 		goto out_topology;
 	return 0;
 
 out_topology:
-	if (cpu_online(cpu))
-		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
-out_online:
 	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
 out_cpu:
 #ifdef CONFIG_HOTPLUG_CPU
@@ -1150,17 +1137,15 @@ static int __init s390_smp_init(void)
 	if (rc)
 		return rc;
 #endif
-	cpu_notifier_register_begin();
 	for_each_present_cpu(cpu) {
 		rc = smp_add_present_cpu(cpu);
 		if (rc)
 			goto out;
 	}
 
-	__hotcpu_notifier(smp_cpu_notify, 0);
-
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online",
+			       smp_cpu_online, smp_cpu_pre_down);
 out:
-	cpu_notifier_register_done();
 	return rc;
 }
 subsys_initcall(s390_smp_init);

Re: [PATCH 20/25] x86/mcheck: Be prepared for a rollback back to the ONLINE state

[Borislav Petkov]

On Thu, Nov 03, 2016 at 03:50:16PM +0100, Sebastian Andrzej Siewior wrote:
> If we try a CPU down and fail in the middle then we roll back to the
> online state. This  means we would perform CPU_ONLINE()
> without invoking CPU_DEAD() for the cleanup of what was allocated in

Are CPU_ONLINE() and CPU_DEAD() functions? Those are the states, right?

> CPU_ONLINE.
> Be prepared for this and don't allocate the struct if we have it
> already.
> 
> Cc: Tony Luck <tony.luck@intel.com>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: linux-edac@vger.kernel.org
> Cc: x86@kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> ---
>  arch/x86/kernel/cpu/mcheck/mce.c     | 4 ++++
>  arch/x86/kernel/cpu/mcheck/mce_amd.c | 3 +++
>  2 files changed, 7 insertions(+)
> 
> diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
> index a7fdf453d895..e9ffd6d9e32d 100644
> --- a/arch/x86/kernel/cpu/mcheck/mce.c
> +++ b/arch/x86/kernel/cpu/mcheck/mce.c
> @@ -2409,6 +2409,10 @@ static int mce_device_create(unsigned int cpu)
>  	if (!mce_available(&boot_cpu_data))
>  		return -EIO;
>  
> +	dev = per_cpu(mce_device, cpu);
> +	if (dev)
> +		return 0;
> +
>  	dev = kzalloc(sizeof *dev, GFP_KERNEL);
>  	if (!dev)
>  		return -ENOMEM;
> diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
> index 55cd018bc1ae..3e529fd747f8 100644
> --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
> +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
> @@ -1097,6 +1097,9 @@ static int threshold_create_device(unsigned int cpu)
>  	struct threshold_bank **bp;
>  	int err = 0;
>  
> +	bp = per_cpu(threshold_banks, cpu);
> +	if (bp)
> +		return 0;

<--- newline here.

>  	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
>  		     GFP_KERNEL);
>  	if (!bp)
> -- 
> 2.10.2
> 
> 

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 18/25] x86/mcheck: Move threshold_create_device()

[Borislav Petkov]

On Thu, Nov 03, 2016 at 03:50:14PM +0100, Sebastian Andrzej Siewior wrote:
> Move the threshold_create_device() so it can use
> threshold_remove_device() without a a declaration.

Please fixup when committing:

				... a forward declaration.

> Cc: Tony Luck <tony.luck@intel.com>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: linux-edac@vger.kernel.org
> Cc: x86@kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> ---
>  arch/x86/kernel/cpu/mcheck/mce_amd.c | 50 ++++++++++++++++++------------------
>  1 file changed, 25 insertions(+), 25 deletions(-)

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 20/25] x86/mcheck: Be prepared for a rollback back to the ONLINE state

[Sebastian Andrzej Siewior]

On 2016-11-07 11:32:19 [+0100], Borislav Petkov wrote:
> On Thu, Nov 03, 2016 at 03:50:16PM +0100, Sebastian Andrzej Siewior wrote:
> > If we try a CPU down and fail in the middle then we roll back to the
> > online state. This  means we would perform CPU_ONLINE()
> > without invoking CPU_DEAD() for the cleanup of what was allocated in
> 
> Are CPU_ONLINE() and CPU_DEAD() functions? Those are the states, right?

those are states. I meant here the driver specific function invoked in
those states.

> > index 55cd018bc1ae..3e529fd747f8 100644
> > --- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
> > +++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
> > @@ -1097,6 +1097,9 @@ static int threshold_create_device(unsigned int cpu)
> >  	struct threshold_bank **bp;
> >  	int err = 0;
> >  
> > +	bp = per_cpu(threshold_banks, cpu);
> > +	if (bp)
> > +		return 0;
> 
> <--- newline here.

okay.

Sebastian

Re: [PATCH 20/25] x86/mcheck: Be prepared for a rollback back to the ONLINE state

[Borislav Petkov]

On Mon, Nov 07, 2016 at 11:40:55AM +0100, Sebastian Andrzej Siewior wrote:
> On 2016-11-07 11:32:19 [+0100], Borislav Petkov wrote:
> > On Thu, Nov 03, 2016 at 03:50:16PM +0100, Sebastian Andrzej Siewior wrote:
> > > If we try a CPU down and fail in the middle then we roll back to the
> > > online state. This  means we would perform CPU_ONLINE()
> > > without invoking CPU_DEAD() for the cleanup of what was allocated in
> > 
> > Are CPU_ONLINE() and CPU_DEAD() functions? Those are the states, right?
> 
> those are states. I meant here the driver specific function invoked in
> those states.

Yeah, so say that please - slapping "()" after the state name doesn't
make it clear.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 21/25] x86/mcheck: Split threshold_cpu_callback into two callbacks

[Borislav Petkov]

On Thu, Nov 03, 2016 at 03:50:17PM +0100, Sebastian Andrzej Siewior wrote:
> The threshold_cpu_callback callbacks looks like one of the notifier and
> its arguments are almost the same. Split this out and have one ONLINE
> and one DEAD callback. This will come handy later once the main code
> gets changed to use the callback mechanism.
> Also, handle threshold_cpu_callback_online() return value so we don't
> continue if the function fails.
> 
> Cc: Tony Luck <tony.luck@intel.com>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: linux-edac@vger.kernel.org
> Cc: x86@kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> ---
>  arch/x86/include/asm/mce.h           |  3 ++-
>  arch/x86/kernel/cpu/mcheck/mce.c     | 18 +++++++++++++-----
>  arch/x86/kernel/cpu/mcheck/mce_amd.c | 24 ++++--------------------
>  3 files changed, 19 insertions(+), 26 deletions(-)

...

> -void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
> +int (*threshold_cpu_callback_online)(unsigned int cpu);
> +int (*threshold_cpu_callback_dead)(unsigned int cpu);

Let's get rid of those pointers and pointer testing and export the
create/remove functions directly.

How's that? It builds here.

---

Index: b/arch/x86/include/asm/mce.h
===================================================================
--- a/arch/x86/include/asm/mce.h	2016-11-07 14:06:40.246518729 +0100
+++ b/arch/x86/include/asm/mce.h	2016-11-07 14:17:36.770529205 +0100
@@ -292,9 +292,9 @@ void do_machine_check(struct pt_regs *,
 /*
  * Threshold handler
  */
-
 extern void (*mce_threshold_vector)(void);
-extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
+extern int mce_threshold_create_device(unsigned int cpu);
+extern int mce_threshold_remove_device(unsigned int cpu);
 
 /* Deferred error interrupt handler */
 extern void (*deferred_error_int_vector)(void);
Index: b/arch/x86/kernel/cpu/mcheck/mce.c
===================================================================
--- a/arch/x86/kernel/cpu/mcheck/mce.c	2016-11-07 14:06:40.246518729 +0100
+++ b/arch/x86/kernel/cpu/mcheck/mce.c	2016-11-07 14:17:06.186528717 +0100
@@ -2291,8 +2291,6 @@ static struct bus_type mce_subsys = {
 
 DEFINE_PER_CPU(struct device *, mce_device);
 
-void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
-
 static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
 {
 	return container_of(attr, struct mce_bank, attr);
@@ -2548,13 +2546,17 @@ mce_cpu_callback(struct notifier_block *
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+
 		mce_device_create(cpu);
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+
+		if (mce_threshold_create_device(cpu)) {
+			mce_device_remove(cpu);
+			return NOTIFY_BAD;
+		}
+
 		break;
 	case CPU_DEAD:
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+		mce_threshold_remove_device(cpu);
 		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
Index: b/arch/x86/kernel/cpu/mcheck/mce_amd.c
===================================================================
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c	2016-11-07 14:06:40.246518729 +0100
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c	2016-11-07 14:18:10.506529743 +0100
@@ -55,6 +55,8 @@
 /* Threshold LVT offset is at MSR0xC0000410[15:12] */
 #define SMCA_THR_LVT_OFF	0xF000
 
+static bool thresholding_en;
+
 static const char * const th_names[] = {
 	"load_store",
 	"insn_fetch",
@@ -1097,10 +1099,13 @@ free_out:
 	per_cpu(threshold_banks, cpu)[bank] = NULL;
 }
 
-static void threshold_remove_device(unsigned int cpu)
+int mce_threshold_remove_device(unsigned int cpu)
 {
 	unsigned int bank;
 
+	if (!thresholding_en)
+		return 0;
+
 	for (bank = 0; bank < mca_cfg.banks; ++bank) {
 		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 			continue;
@@ -1108,15 +1113,19 @@ static void threshold_remove_device(unsi
 	}
 	kfree(per_cpu(threshold_banks, cpu));
 	per_cpu(threshold_banks, cpu) = NULL;
+	return 0;
 }
 
 /* create dir/files for all valid threshold banks */
-static int threshold_create_device(unsigned int cpu)
+int mce_threshold_create_device(unsigned int cpu)
 {
 	unsigned int bank;
 	struct threshold_bank **bp;
 	int err = 0;
 
+	if (!thresholding_en)
+		return 0;
+
 	bp = per_cpu(threshold_banks, cpu);
 	if (bp)
 		return 0;
@@ -1136,40 +1145,23 @@ static int threshold_create_device(unsig
 	}
 	return err;
 err:
-	threshold_remove_device(cpu);
+	mce_threshold_remove_device(cpu);
 	return err;
 }
 
-/* get notified when a cpu comes on/off */
-static void
-amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
-{
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		threshold_create_device(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		threshold_remove_device(cpu);
-		break;
-	default:
-		break;
-	}
-}
-
 static __init int threshold_init_device(void)
 {
 	unsigned lcpu = 0;
 
 	/* to hit CPUs online before the notifier is up */
 	for_each_online_cpu(lcpu) {
-		int err = threshold_create_device(lcpu);
+		int err = mce_threshold_create_device(lcpu);
 
 		if (err)
 			return err;
 	}
-	threshold_cpu_callback = amd_64_threshold_cpu_callback;
+
+	thresholding_en = true;
 
 	return 0;
 }

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 21/25] x86/mcheck: Split threshold_cpu_callback into two callbacks

[Sebastian Andrzej Siewior]

On 2016-11-07 14:20:40 [+0100], Borislav Petkov wrote:
> Let's get rid of those pointers and pointer testing and export the
> create/remove functions directly.

This moves it 1:1 (well, more or less). Wouldn't you prefer doing it as
separate patch/change?

> How's that? It builds here.
CONFIG_X86_MCE_AMD is where the callback is implemented. Wouldn't that
be broken now?

Sebastian

Re: [PATCH 21/25] x86/mcheck: Split threshold_cpu_callback into two callbacks

[Borislav Petkov]

On Mon, Nov 07, 2016 at 02:25:01PM +0100, Sebastian Andrzej Siewior wrote:
> This moves it 1:1 (well, more or less). Wouldn't you prefer doing it as
> separate patch/change?

Ontop of all of yours so that you don't have to redo yours?

> CONFIG_X86_MCE_AMD is where the callback is implemented. Wouldn't that
> be broken now?

Yeah, fixed.

Thanks.

---
Index: b/arch/x86/include/asm/mce.h
===================================================================
--- a/arch/x86/include/asm/mce.h	2016-11-07 14:06:40.246518729 +0100
+++ b/arch/x86/include/asm/mce.h	2016-11-07 15:52:38.786620194 +0100
@@ -292,9 +292,7 @@ void do_machine_check(struct pt_regs *,
 /*
  * Threshold handler
  */
-
 extern void (*mce_threshold_vector)(void);
-extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
 
 /* Deferred error interrupt handler */
 extern void (*deferred_error_int_vector)(void);
@@ -372,6 +370,12 @@ struct smca_bank {
 extern struct smca_bank smca_banks[MAX_NR_BANKS];
 
 extern const char *smca_get_long_name(enum smca_bank_types t);
+extern int mce_threshold_create_device(unsigned int cpu);
+extern int mce_threshold_remove_device(unsigned int cpu);
+#else
+
+static inline int mce_threshold_create_device(unsigned int cpu) { return 0; };
+static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; };
 #endif
 
 #endif /* _ASM_X86_MCE_H */
Index: b/arch/x86/kernel/cpu/mcheck/mce.c
===================================================================
--- a/arch/x86/kernel/cpu/mcheck/mce.c	2016-11-07 14:06:40.246518729 +0100
+++ b/arch/x86/kernel/cpu/mcheck/mce.c	2016-11-07 14:17:06.186528717 +0100
@@ -2291,8 +2291,6 @@ static struct bus_type mce_subsys = {
 
 DEFINE_PER_CPU(struct device *, mce_device);
 
-void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
-
 static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
 {
 	return container_of(attr, struct mce_bank, attr);
@@ -2548,13 +2546,17 @@ mce_cpu_callback(struct notifier_block *
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+
 		mce_device_create(cpu);
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+
+		if (mce_threshold_create_device(cpu)) {
+			mce_device_remove(cpu);
+			return NOTIFY_BAD;
+		}
+
 		break;
 	case CPU_DEAD:
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+		mce_threshold_remove_device(cpu);
 		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
Index: b/arch/x86/kernel/cpu/mcheck/mce_amd.c
===================================================================
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c	2016-11-07 14:06:40.246518729 +0100
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c	2016-11-07 14:18:10.506529743 +0100
@@ -55,6 +55,8 @@
 /* Threshold LVT offset is at MSR0xC0000410[15:12] */
 #define SMCA_THR_LVT_OFF	0xF000
 
+static bool thresholding_en;
+
 static const char * const th_names[] = {
 	"load_store",
 	"insn_fetch",
@@ -1097,10 +1099,13 @@ free_out:
 	per_cpu(threshold_banks, cpu)[bank] = NULL;
 }
 
-static void threshold_remove_device(unsigned int cpu)
+int mce_threshold_remove_device(unsigned int cpu)
 {
 	unsigned int bank;
 
+	if (!thresholding_en)
+		return 0;
+
 	for (bank = 0; bank < mca_cfg.banks; ++bank) {
 		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 			continue;
@@ -1108,15 +1113,19 @@ static void threshold_remove_device(unsi
 	}
 	kfree(per_cpu(threshold_banks, cpu));
 	per_cpu(threshold_banks, cpu) = NULL;
+	return 0;
 }
 
 /* create dir/files for all valid threshold banks */
-static int threshold_create_device(unsigned int cpu)
+int mce_threshold_create_device(unsigned int cpu)
 {
 	unsigned int bank;
 	struct threshold_bank **bp;
 	int err = 0;
 
+	if (!thresholding_en)
+		return 0;
+
 	bp = per_cpu(threshold_banks, cpu);
 	if (bp)
 		return 0;
@@ -1136,40 +1145,23 @@ static int threshold_create_device(unsig
 	}
 	return err;
 err:
-	threshold_remove_device(cpu);
+	mce_threshold_remove_device(cpu);
 	return err;
 }
 
-/* get notified when a cpu comes on/off */
-static void
-amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
-{
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		threshold_create_device(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		threshold_remove_device(cpu);
-		break;
-	default:
-		break;
-	}
-}
-
 static __init int threshold_init_device(void)
 {
 	unsigned lcpu = 0;
 
 	/* to hit CPUs online before the notifier is up */
 	for_each_online_cpu(lcpu) {
-		int err = threshold_create_device(lcpu);
+		int err = mce_threshold_create_device(lcpu);
 
 		if (err)
 			return err;
 	}
-	threshold_cpu_callback = amd_64_threshold_cpu_callback;
+
+	thresholding_en = true;
 
 	return 0;
 }

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 21/25] x86/mcheck: Split threshold_cpu_callback into two callbacks

[Sebastian Andrzej Siewior]

On 2016-11-07 16:07:45 [+0100], Borislav Petkov wrote:
> On Mon, Nov 07, 2016 at 02:25:01PM +0100, Sebastian Andrzej Siewior wrote:
> > This moves it 1:1 (well, more or less). Wouldn't you prefer doing it as
> > separate patch/change?
> 
> Ontop of all of yours so that you don't have to redo yours?

No. It is a separate thing so I would prefer it in a separate patch.
However if you are the MCE person in charge here and nobody disagrees I
have no problem to suck that into the patch and update remaining queue
if necessary.

> Thanks.

Sebastian

Re: [PATCH 20/25] x86/mcheck: Be prepared for a rollback back to the ONLINE state

[Sebastian Andrzej Siewior]

On 2016-11-07 13:31:29 [+0100], Borislav Petkov wrote:
> Yeah, so say that please - slapping "()" after the state name doesn't
> make it clear.

Updated.

Sebastian

Re: [PATCH 21/25] x86/mcheck: Split threshold_cpu_callback into two callbacks

[Sebastian Andrzej Siewior]

On 2016-11-07 16:07:45 [+0100], Borislav Petkov wrote:
> On Mon, Nov 07, 2016 at 02:25:01PM +0100, Sebastian Andrzej Siewior wrote:
> > This moves it 1:1 (well, more or less). Wouldn't you prefer doing it as
> > separate patch/change?
> 
> Ontop of all of yours so that you don't have to redo yours?
> 
> > CONFIG_X86_MCE_AMD is where the callback is implemented. Wouldn't that
> > be broken now?
> 
> Yeah, fixed.

included (slightly adjusted to get fit on top of -rc4):

Subject: x86/mcheck: Split threshold_cpu_callback into two callbacks
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Date: Tue, 25 Oct 2016 18:04:25 +0200

The threshold_cpu_callback callbacks looks like one of the notifier and
its arguments are almost the same. Split this out and have one ONLINE
and one DEAD callback. This will come handy later once the main code
gets changed to use the callback mechanism.
Also, handle threshold_cpu_callback_online() return value so we don't
continue if the function fails.

Boris Petkov removed the callback pointer and replaced it with proper
functions.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -293,9 +293,7 @@ void do_machine_check(struct pt_regs *,
 /*
  * Threshold handler
  */
-
 extern void (*mce_threshold_vector)(void);
-extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
 
 /* Deferred error interrupt handler */
 extern void (*deferred_error_int_vector)(void);
@@ -377,7 +375,12 @@ struct smca_bank_info {
 };
 
 extern struct smca_bank_info smca_banks[MAX_NR_BANKS];
+extern int mce_threshold_create_device(unsigned int cpu);
+extern int mce_threshold_remove_device(unsigned int cpu);
+#else
 
+static inline int mce_threshold_create_device(unsigned int cpu) { return 0; };
+static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; };
 #endif
 
 #endif /* _ASM_X86_MCE_H */
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2255,8 +2255,6 @@ static struct bus_type mce_subsys = {
 
 DEFINE_PER_CPU(struct device *, mce_device);
 
-void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
-
 static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
 {
 	return container_of(attr, struct mce_bank, attr);
@@ -2512,13 +2510,17 @@ mce_cpu_callback(struct notifier_block *
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+
 		mce_device_create(cpu);
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+
+		if (mce_threshold_create_device(cpu)) {
+			mce_device_remove(cpu);
+			return NOTIFY_BAD;
+		}
+
 		break;
 	case CPU_DEAD:
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+		mce_threshold_remove_device(cpu);
 		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -55,6 +55,8 @@
 /* Threshold LVT offset is at MSR0xC0000410[15:12] */
 #define SMCA_THR_LVT_OFF	0xF000
 
+static bool thresholding_en;
+
 static const char * const th_names[] = {
 	"load_store",
 	"insn_fetch",
@@ -1077,10 +1079,13 @@ static void threshold_remove_bank(unsign
 	per_cpu(threshold_banks, cpu)[bank] = NULL;
 }
 
-static void threshold_remove_device(unsigned int cpu)
+int mce_threshold_remove_device(unsigned int cpu)
 {
 	unsigned int bank;
 
+	if (!thresholding_en)
+		return 0;
+
 	for (bank = 0; bank < mca_cfg.banks; ++bank) {
 		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 			continue;
@@ -1088,15 +1093,19 @@ static void threshold_remove_device(unsi
 	}
 	kfree(per_cpu(threshold_banks, cpu));
 	per_cpu(threshold_banks, cpu) = NULL;
+	return 0;
 }
 
 /* create dir/files for all valid threshold banks */
-static int threshold_create_device(unsigned int cpu)
+int mce_threshold_create_device(unsigned int cpu)
 {
 	unsigned int bank;
 	struct threshold_bank **bp;
 	int err = 0;
 
+	if (!thresholding_en)
+		return 0;
+
 	bp = per_cpu(threshold_banks, cpu);
 	if (bp)
 		return 0;
@@ -1117,40 +1126,23 @@ static int threshold_create_device(unsig
 	}
 	return err;
 err:
-	threshold_remove_device(cpu);
+	mce_threshold_remove_device(cpu);
 	return err;
 }
 
-/* get notified when a cpu comes on/off */
-static void
-amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
-{
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		threshold_create_device(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		threshold_remove_device(cpu);
-		break;
-	default:
-		break;
-	}
-}
-
 static __init int threshold_init_device(void)
 {
 	unsigned lcpu = 0;
 
 	/* to hit CPUs online before the notifier is up */
 	for_each_online_cpu(lcpu) {
-		int err = threshold_create_device(lcpu);
+		int err = mce_threshold_create_device(lcpu);
 
 		if (err)
 			return err;
 	}
-	threshold_cpu_callback = amd_64_threshold_cpu_callback;
+
+	thresholding_en = true;
 
 	return 0;
 }


Sebastian

Re: [PATCH 21/25] x86/mcheck: Split threshold_cpu_callback into two callbacks

[Borislav Petkov]

On Mon, Nov 07, 2016 at 06:26:17PM +0100, Sebastian Andrzej Siewior wrote:
> Subject: x86/mcheck: Split threshold_cpu_callback into two callbacks
> From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Date: Tue, 25 Oct 2016 18:04:25 +0200
> 
> The threshold_cpu_callback callbacks looks like one of the notifier and
> its arguments are almost the same. Split this out and have one ONLINE
> and one DEAD callback. This will come handy later once the main code
> gets changed to use the callback mechanism.
> Also, handle threshold_cpu_callback_online() return value so we don't
> continue if the function fails.
> 
> Boris Petkov removed the callback pointer and replaced it with proper
> functions.
> 
> Cc: Tony Luck <tony.luck@intel.com>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: linux-edac@vger.kernel.org
> Cc: x86@kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> --- a/arch/x86/include/asm/mce.h
> +++ b/arch/x86/include/asm/mce.h
> @@ -293,9 +293,7 @@ void do_machine_check(struct pt_regs *,
>  /*
>   * Threshold handler
>   */
> -
>  extern void (*mce_threshold_vector)(void);
> -extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
>  
>  /* Deferred error interrupt handler */
>  extern void (*deferred_error_int_vector)(void);
> @@ -377,7 +375,12 @@ struct smca_bank_info {
>  };
>  
>  extern struct smca_bank_info smca_banks[MAX_NR_BANKS];
> +extern int mce_threshold_create_device(unsigned int cpu);
> +extern int mce_threshold_remove_device(unsigned int cpu);
> +#else
>  
> +static inline int mce_threshold_create_device(unsigned int cpu) { return 0; };
> +static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; };
>  #endif
>  
>  #endif /* _ASM_X86_MCE_H */

Right, just a heads-up for tip guys: this second hunk will cause a merge
conflict with the other RAS changes:

Applying patch 21-x86-mcheck-split_threshold_cpu_callback_into_two_callbacks.patch
patching file arch/x86/include/asm/mce.h
Hunk #1 succeeded at 292 (offset -1 lines).
Hunk #2 FAILED at 375.
1 out of 2 hunks FAILED -- saving rejects to file arch/x86/include/asm/mce.h.rej

The fixup is easy though as it is only a contextual conflict due to
renaming of smca_bank_info to smca_bank:

Index: b/arch/x86/include/asm/mce.h
===================================================================
--- a/arch/x86/include/asm/mce.h        2016-11-07 19:12:14.309920731 +0100
+++ b/arch/x86/include/asm/mce.h        2016-11-07 19:13:47.521922219 +0100
@@ -292,9 +292,7 @@ void do_machine_check(struct pt_regs *,
 /*
  * Threshold handler
  */
-
 extern void (*mce_threshold_vector)(void);
-extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
 
 /* Deferred error interrupt handler */
 extern void (*deferred_error_int_vector)(void);
@@ -372,6 +370,11 @@ struct smca_bank {
 extern struct smca_bank smca_banks[MAX_NR_BANKS];
 
 extern const char *smca_get_long_name(enum smca_bank_types t);
+extern int mce_threshold_create_device(unsigned int cpu);
+extern int mce_threshold_remove_device(unsigned int cpu);
+#else
+static inline int mce_threshold_create_device(unsigned int cpu) { return 0; };
+static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; };
 #endif
 
 #endif /* _ASM_X86_MCE_H */
---

Thanks!

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Borislav Petkov]

On Thu, Nov 03, 2016 at 03:50:18PM +0100, Sebastian Andrzej Siewior wrote:
> Part of the init (memory allocation and so on) is done
> in mcheck_cpu_init(). While moving the the allocation to
> mcheck_init_device() (where the hotplug calls are initialized) it
> becomes necessary to move the callback (mcheck_cpu_init()), too.
> 
> The callback is now removed from identify_cpu() and registered as a
> hotplug event which is invoked as the very first one which is shortly
> after the original point of invocation (look at smp_store_cpu_info() and
> notify_cpu_starting() in smp_callin()).
> One "visible" difference is that MCE for the boot CPU is not enabled at
> identify_boot_cpu() time but at device_initcall_sync() time. Either way,
> both times we had no userland around.

Uh, hm, I'm not sure about this: so the issue I see with this is that
the more we're delaying the enabling or MCE reporting - and especially
setting CR4[MCE] - the more we're increasing the window where a MCE
during early boot will cause a shutdown. (This is what happens if
CR4[MCE]=0b).

Perhaps we should split the init into a very early init which doesn't
need to be part of hotplug and the rest, which can do mce_disable_cpu()
and mce_reenable_cpu().

Tony, how do you see this?

> Cc: Tony Luck <tony.luck@intel.com>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: linux-edac@vger.kernel.org
> Cc: x86@kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> ---

...


> @@ -2584,11 +2580,26 @@ static __init int mcheck_init_device(void)
>  		goto err_out;
>  	}
>  
> +	err = __mcheck_cpu_mce_banks_init();
		^^^^^^^^

I guess you can merge this one...

> +	if (err)
> +		goto err_out_mem;
> +
>  	mce_init_banks();
	^^^^^^^^

into this one now.

But let's sort out the bigger issue first.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Luck, Tony]

On Mon, Nov 07, 2016 at 07:45:32PM +0100, Borislav Petkov wrote:
> On Thu, Nov 03, 2016 at 03:50:18PM +0100, Sebastian Andrzej Siewior wrote:
> > Part of the init (memory allocation and so on) is done
> > in mcheck_cpu_init(). While moving the the allocation to
> > mcheck_init_device() (where the hotplug calls are initialized) it
> > becomes necessary to move the callback (mcheck_cpu_init()), too.
> > 
> > The callback is now removed from identify_cpu() and registered as a
> > hotplug event which is invoked as the very first one which is shortly
> > after the original point of invocation (look at smp_store_cpu_info() and
> > notify_cpu_starting() in smp_callin()).
> > One "visible" difference is that MCE for the boot CPU is not enabled at
> > identify_boot_cpu() time but at device_initcall_sync() time. Either way,
> > both times we had no userland around.
> 
> Uh, hm, I'm not sure about this: so the issue I see with this is that
> the more we're delaying the enabling or MCE reporting - and especially
> setting CR4[MCE] - the more we're increasing the window where a MCE
> during early boot will cause a shutdown. (This is what happens if
> CR4[MCE]=0b).
> 
> Perhaps we should split the init into a very early init which doesn't
> need to be part of hotplug and the rest, which can do mce_disable_cpu()
> and mce_reenable_cpu().
> 
> Tony, how do you see this?

I don't think that helps as much as you'd like it to help (at
least on Intel). A broadcast machine check that finds the boot
CPU has set CR4[MCE]=1 is still going to end up in reset if any
other CPU still has CR4[MCE]=0

-Tony

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Borislav Petkov]

On Mon, Nov 07, 2016 at 10:55:24AM -0800, Luck, Tony wrote:
> I don't think that helps as much as you'd like it to help (at
> least on Intel). A broadcast machine check that finds the boot
> CPU has set CR4[MCE]=1 is still going to end up in reset if any
> other CPU still has CR4[MCE]=0

By leaving/moving the setting of CR4 earlier on all cores, we'll
at least make the possible window for such potential resets a lot
smaller...

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Borislav Petkov]

On Mon, Nov 07, 2016 at 09:12:24PM +0100, Borislav Petkov wrote:
> On Mon, Nov 07, 2016 at 10:55:24AM -0800, Luck, Tony wrote:
> > I don't think that helps as much as you'd like it to help (at
> > least on Intel). A broadcast machine check that finds the boot
> > CPU has set CR4[MCE]=1 is still going to end up in reset if any
> > other CPU still has CR4[MCE]=0
> 
> By leaving/moving the setting of CR4 earlier on all cores, we'll
> at least make the possible window for such potential resets a lot
> smaller...

... and in general, I'm still unsure about *why* we need this change for
hotplug. bigeasy, can you please clarify first?

Thanks.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Sebastian Andrzej Siewior]

On 2016-11-08 10:23:02 [+0100], Borislav Petkov wrote:
> On Mon, Nov 07, 2016 at 09:12:24PM +0100, Borislav Petkov wrote:
> > On Mon, Nov 07, 2016 at 10:55:24AM -0800, Luck, Tony wrote:
> > > I don't think that helps as much as you'd like it to help (at
> > > least on Intel). A broadcast machine check that finds the boot
> > > CPU has set CR4[MCE]=1 is still going to end up in reset if any
> > > other CPU still has CR4[MCE]=0
> > 
> > By leaving/moving the setting of CR4 earlier on all cores, we'll
> > at least make the possible window for such potential resets a lot
> > smaller...
> 
> ... and in general, I'm still unsure about *why* we need this change for
> hotplug. bigeasy, can you please clarify first?

I want to get rid of non-symmetrical part and the arch hook which should
be part of the hp notifier itself. I wouldn't be too much afraid about
the when point in time when the notifier runs: It is the *first*
notifier that will be invoked on the target CPU. This is only a few
lines after the old hook. Nothing else long delaying should be invoked.

> Thanks.

Sebastian

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Borislav Petkov]

On Wed, Nov 09, 2016 at 03:22:21PM +0100, Sebastian Andrzej Siewior wrote:
> I want to get rid of non-symmetrical part and the arch hook which should
> be part of the hp notifier itself. I wouldn't be too much afraid about
> the when point in time when the notifier runs: It is the *first*
> notifier that will be invoked on the target CPU. This is only a few
> lines after the old hook. Nothing else long delaying should be invoked.

So I'm not sure *everything* mcheck_cpu_init() does should be in hotplug
and toggle on and off. For example, we should set CR4.MCE once during
init and that's it. Not turn it on and off. It would be pretty pointless
IMO.

That's why the hotplug callback mce_disable_cpu() doesn't fiddle with
CR4 - it only clears the bits in MCi_CTL. And I think we should remain
that way.

The initial, one-time enabling of MCE features per core should be done
only once per boot and that's it.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Sebastian Andrzej Siewior]

On 2016-11-09 16:38:07 [+0100], Borislav Petkov wrote:
> On Wed, Nov 09, 2016 at 03:22:21PM +0100, Sebastian Andrzej Siewior wrote:
> > I want to get rid of non-symmetrical part and the arch hook which should
> > be part of the hp notifier itself. I wouldn't be too much afraid about
> > the when point in time when the notifier runs: It is the *first*
> > notifier that will be invoked on the target CPU. This is only a few
> > lines after the old hook. Nothing else long delaying should be invoked.
> 
> So I'm not sure *everything* mcheck_cpu_init() does should be in hotplug
> and toggle on and off. For example, we should set CR4.MCE once during
> init and that's it. Not turn it on and off. It would be pretty pointless
> IMO.
> 
> That's why the hotplug callback mce_disable_cpu() doesn't fiddle with
> CR4 - it only clears the bits in MCi_CTL. And I think we should remain
> that way.

The behaviour was not changed - it was only reorganized to keep in one
spot.

> The initial, one-time enabling of MCE features per core should be done
> only once per boot and that's it.

It is. You still want this in your hotplug code in case someone replaces
(physically) your CPU (say a larger NUMA box).

Sebastian

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Borislav Petkov]

On Wed, Nov 09, 2016 at 05:24:51PM +0100, Sebastian Andrzej Siewior wrote:
> The behaviour was not changed - it was only reorganized to keep in one
> spot.

So there's the full CPU init path down cpu_up() -> ... -> identify_cpu()
where mcheck_cpu_init() is called and then there's also the hotplug
callbacks in mce_cpu_callback().

What you're proposing now is, merge the two.

But then the full path down identify_cpu() could still do
mheck_cpu_init() regardless where you move it.

IOW, I still don't see why this change is needed.

In more OW, why can't you simply do:

	err = cpuhp_setup_state(CPUHP_AP_X86_MCE_STARTING, "x86/mce:starting",
				mce_reenable_cpu, NULL);

and use the current notifier callback?

I still don't get the need for this churn.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Sebastian Andrzej Siewior]

On 2016-11-09 18:01:09 [+0100], Borislav Petkov wrote:
> On Wed, Nov 09, 2016 at 05:24:51PM +0100, Sebastian Andrzej Siewior wrote:
> > The behaviour was not changed - it was only reorganized to keep in one
> > spot.
> 
> So there's the full CPU init path down cpu_up() -> ... -> identify_cpu()
> where mcheck_cpu_init() is called and then there's also the hotplug
> callbacks in mce_cpu_callback().

correct. And it looks like mcheck_cpu_init() is part of the notifier. It
tried to allocate memory which it only did on boot CPU.

> What you're proposing now is, merge the two.

I wouldn't call it a merge. identify_cpu() is one of the first things a
CPU does on its boot. Slightly later invokes the `old' CPU_STARTING
notifier callbacks. I had nothing what would match CPU_STARTING except
mcheck_cpu_init() so I took it and renamed to mcheck_cpu_starting().
The remaining bring up states in mce_cpu_callback() are just CPU_ONLINE
which is also converted away.

> But then the full path down identify_cpu() could still do
> mheck_cpu_init() regardless where you move it.
> 
> IOW, I still don't see why this change is needed.

>From what it looks, is that mce_cpu_down_dying() is the counter part to
mcheck_cpu_starting()/mcheck_cpu_init(). One hint for it is that the
latter disable the timer which is started by the former. So if you
rollback to online after the DYNING state you need to do STARTING in
order to start the timer.

> In more OW, why can't you simply do:
> 
> 	err = cpuhp_setup_state(CPUHP_AP_X86_MCE_STARTING, "x86/mce:starting",
> 				mce_reenable_cpu, NULL);

A later patch does:
         err = cpuhp_setup_state(CPUHP_AP_X86_MCE_STARTING, "x86/mce:starting",
                                 mcheck_cpu_starting, mce_cpu_down_dying);

> and us> e the current notifier callback?
> 
> I still don't get the need for this churn.
I tried to clarify it with the timer. You claimed that the MCE feature
itself is not disabled if the CPU goes down. I assumed that this is the
case (somewhere in mce_disable_cpu()) because a CPU which is off should
not response to any MCE errors and then it would be enabled in the
STARTING callback again. This (and timer) was the motivation to get the
MCE stuff in one place and get symmetrical CPU down/up behaviour.

Sebastian

RE: [PATCH 22/25] x86/mcheck: Do the init in one place

[Luck, Tony]

> That's why the hotplug callback mce_disable_cpu() doesn't fiddle with
> CR4 - it only clears the bits in MCi_CTL. And I think we should remain
> that way.

N.B. See vendor_disable_error_reporting() ... on Intel we don't clear MCi_CTL.

-Tony

[tip:smp/hotplug] fs/buffer: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  fc4d24c9b47150245b3eb5bebc2ad4764c754ef4
Gitweb:     http://git.kernel.org/tip/fc4d24c9b47150245b3eb5bebc2ad4764c754ef4
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:49:57 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:25 +0100

fs/buffer: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-fsdevel@vger.kernel.org
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-2-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 fs/buffer.c                | 16 ++++++----------
 include/linux/cpuhotplug.h |  1 +
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/fs/buffer.c b/fs/buffer.c
index b205a62..1613656 100644
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -3403,7 +3403,7 @@ void free_buffer_head(struct buffer_head *bh)
 }
 EXPORT_SYMBOL(free_buffer_head);
 
-static void buffer_exit_cpu(int cpu)
+static int buffer_exit_cpu_dead(unsigned int cpu)
 {
 	int i;
 	struct bh_lru *b = &per_cpu(bh_lrus, cpu);
@@ -3414,14 +3414,7 @@ static void buffer_exit_cpu(int cpu)
 	}
 	this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
 	per_cpu(bh_accounting, cpu).nr = 0;
-}
-
-static int buffer_cpu_notify(struct notifier_block *self,
-			      unsigned long action, void *hcpu)
-{
-	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
-		buffer_exit_cpu((unsigned long)hcpu);
-	return NOTIFY_OK;
+	return 0;
 }
 
 /**
@@ -3471,6 +3464,7 @@ EXPORT_SYMBOL(bh_submit_read);
 void __init buffer_init(void)
 {
 	unsigned long nrpages;
+	int ret;
 
 	bh_cachep = kmem_cache_create("buffer_head",
 			sizeof(struct buffer_head), 0,
@@ -3483,5 +3477,7 @@ void __init buffer_init(void)
 	 */
 	nrpages = (nr_free_buffer_pages() * 10) / 100;
 	max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
-	hotcpu_notifier(buffer_cpu_notify, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
+					NULL, buffer_exit_cpu_dead);
+	WARN_ON(ret < 0);
 }
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index afe641c..69b74fa 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -30,6 +30,7 @@ enum cpuhp_state {
 	CPUHP_ACPI_CPUDRV_DEAD,
 	CPUHP_S390_PFAULT_DEAD,
 	CPUHP_BLK_MQ_DEAD,
+	CPUHP_FS_BUFF_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,

[tip:smp/hotplug] kernel/printk: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  90b14889d2f9b29d7e5b4b2d36251c13ce3dd13f
Gitweb:     http://git.kernel.org/tip/90b14889d2f9b29d7e5b4b2d36251c13ce3dd13f
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:49:58 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:26 +0100

kernel/printk: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-3-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |  1 +
 kernel/printk/printk.c     | 29 +++++++++++++----------------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 69b74fa..4174083 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -31,6 +31,7 @@ enum cpuhp_state {
 	CPUHP_S390_PFAULT_DEAD,
 	CPUHP_BLK_MQ_DEAD,
 	CPUHP_FS_BUFF_DEAD,
+	CPUHP_PRINTK_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c
index de08fc9..4487ffc 100644
--- a/kernel/printk/printk.c
+++ b/kernel/printk/printk.c
@@ -2185,27 +2185,18 @@ void resume_console(void)
 
 /**
  * console_cpu_notify - print deferred console messages after CPU hotplug
- * @self: notifier struct
- * @action: CPU hotplug event
- * @hcpu: unused
+ * @cpu: unused
  *
  * If printk() is called from a CPU that is not online yet, the messages
  * will be spooled but will not show up on the console.  This function is
  * called when a new CPU comes online (or fails to come up), and ensures
  * that any such output gets printed.
  */
-static int console_cpu_notify(struct notifier_block *self,
-	unsigned long action, void *hcpu)
-{
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_DEAD:
-	case CPU_DOWN_FAILED:
-	case CPU_UP_CANCELED:
-		console_lock();
-		console_unlock();
-	}
-	return NOTIFY_OK;
+static int console_cpu_notify(unsigned int cpu)
+{
+	console_lock();
+	console_unlock();
+	return 0;
 }
 
 /**
@@ -2843,6 +2834,7 @@ EXPORT_SYMBOL(unregister_console);
 static int __init printk_late_init(void)
 {
 	struct console *con;
+	int ret;
 
 	for_each_console(con) {
 		if (!keep_bootcon && con->flags & CON_BOOT) {
@@ -2857,7 +2849,12 @@ static int __init printk_late_init(void)
 				unregister_console(con);
 		}
 	}
-	hotcpu_notifier(console_cpu_notify, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_PRINTK_DEAD, "printk:dead", NULL,
+					console_cpu_notify);
+	WARN_ON(ret < 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "printk:online",
+					console_cpu_notify, NULL);
+	WARN_ON(ret < 0);
 	return 0;
 }
 late_initcall(printk_late_init);

[tip:smp/hotplug] mm/memcg: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  308167fcb330296fc80505a6b11ba0661f38a4cc
Gitweb:     http://git.kernel.org/tip/308167fcb330296fc80505a6b11ba0661f38a4cc
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:49:59 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:26 +0100

mm/memcg: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Michal Hocko <mhocko@kernel.org>
Cc: linux-mm@kvack.org
Cc: rt@linutronix.de
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: cgroups@vger.kernel.org
Link: http://lkml.kernel.org/r/20161103145021.28528-4-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |  1 +
 mm/memcontrol.c            | 24 ++++++++----------------
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 4174083..c622ab3 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -32,6 +32,7 @@ enum cpuhp_state {
 	CPUHP_BLK_MQ_DEAD,
 	CPUHP_FS_BUFF_DEAD,
 	CPUHP_PRINTK_DEAD,
+	CPUHP_MM_MEMCQ_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 0f870ba..6c20435 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1816,22 +1816,13 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
 	mutex_unlock(&percpu_charge_mutex);
 }
 
-static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
-					unsigned long action,
-					void *hcpu)
+static int memcg_hotplug_cpu_dead(unsigned int cpu)
 {
-	int cpu = (unsigned long)hcpu;
 	struct memcg_stock_pcp *stock;
 
-	if (action == CPU_ONLINE)
-		return NOTIFY_OK;
-
-	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
-		return NOTIFY_OK;
-
 	stock = &per_cpu(memcg_stock, cpu);
 	drain_stock(stock);
-	return NOTIFY_OK;
+	return 0;
 }
 
 static void reclaim_high(struct mem_cgroup *memcg,
@@ -5774,16 +5765,17 @@ __setup("cgroup.memory=", cgroup_memory);
 /*
  * subsys_initcall() for memory controller.
  *
- * Some parts like hotcpu_notifier() have to be initialized from this context
- * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
- * everything that doesn't depend on a specific mem_cgroup structure should
- * be initialized from here.
+ * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
+ * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
+ * basically everything that doesn't depend on a specific mem_cgroup structure
+ * should be initialized from here.
  */
 static int __init mem_cgroup_init(void)
 {
 	int cpu, node;
 
-	hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
+	cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
+				  memcg_hotplug_cpu_dead);
 
 	for_each_possible_cpu(cpu)
 		INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,

[tip:smp/hotplug] lib/percpu_counter: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  5588f5afb4cfc33eb377b751ba4b97184373e8d6
Gitweb:     http://git.kernel.org/tip/5588f5afb4cfc33eb377b751ba4b97184373e8d6
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:00 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:26 +0100

lib/percpu_counter: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-5-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |  1 +
 lib/percpu_counter.c       | 25 ++++++++++++++-----------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index c622ab3..04e5f99 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -33,6 +33,7 @@ enum cpuhp_state {
 	CPUHP_FS_BUFF_DEAD,
 	CPUHP_PRINTK_DEAD,
 	CPUHP_MM_MEMCQ_DEAD,
+	CPUHP_PERCPU_CNT_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c
index 72d3611..c8cebb1 100644
--- a/lib/percpu_counter.c
+++ b/lib/percpu_counter.c
@@ -158,25 +158,21 @@ EXPORT_SYMBOL(percpu_counter_destroy);
 int percpu_counter_batch __read_mostly = 32;
 EXPORT_SYMBOL(percpu_counter_batch);
 
-static void compute_batch_value(void)
+static int compute_batch_value(unsigned int cpu)
 {
 	int nr = num_online_cpus();
 
 	percpu_counter_batch = max(32, nr*2);
+	return 0;
 }
 
-static int percpu_counter_hotcpu_callback(struct notifier_block *nb,
-					unsigned long action, void *hcpu)
+static int percpu_counter_cpu_dead(unsigned int cpu)
 {
 #ifdef CONFIG_HOTPLUG_CPU
-	unsigned int cpu;
 	struct percpu_counter *fbc;
 
-	compute_batch_value();
-	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
-		return NOTIFY_OK;
+	compute_batch_value(cpu);
 
-	cpu = (unsigned long)hcpu;
 	spin_lock_irq(&percpu_counters_lock);
 	list_for_each_entry(fbc, &percpu_counters, list) {
 		s32 *pcount;
@@ -190,7 +186,7 @@ static int percpu_counter_hotcpu_callback(struct notifier_block *nb,
 	}
 	spin_unlock_irq(&percpu_counters_lock);
 #endif
-	return NOTIFY_OK;
+	return 0;
 }
 
 /*
@@ -222,8 +218,15 @@ EXPORT_SYMBOL(__percpu_counter_compare);
 
 static int __init percpu_counter_startup(void)
 {
-	compute_batch_value();
-	hotcpu_notifier(percpu_counter_hotcpu_callback, 0);
+	int ret;
+
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online",
+				compute_batch_value, NULL);
+	WARN_ON(ret < 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD,
+					"lib/percpu_cnt:dead", NULL,
+					percpu_counter_cpu_dead);
+	WARN_ON(ret < 0);
 	return 0;
 }
 module_init(percpu_counter_startup);

[tip:smp/hotplug] lib/radix-tree: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  d544abd5ff7d8b07c0c67682a63e4939c3c82914
Gitweb:     http://git.kernel.org/tip/d544abd5ff7d8b07c0c67682a63e4939c3c82914
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:01 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:27 +0100

lib/radix-tree: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-6-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |  1 +
 lib/radix-tree.c           | 25 ++++++++++++-------------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 04e5f99..89310fb 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -34,6 +34,7 @@ enum cpuhp_state {
 	CPUHP_PRINTK_DEAD,
 	CPUHP_MM_MEMCQ_DEAD,
 	CPUHP_PERCPU_CNT_DEAD,
+	CPUHP_RADIX_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 8e6d552..4b8bb36 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -1642,32 +1642,31 @@ static __init void radix_tree_init_maxnodes(void)
 	}
 }
 
-static int radix_tree_callback(struct notifier_block *nfb,
-				unsigned long action, void *hcpu)
+static int radix_tree_cpu_dead(unsigned int cpu)
 {
-	int cpu = (long)hcpu;
 	struct radix_tree_preload *rtp;
 	struct radix_tree_node *node;
 
 	/* Free per-cpu pool of preloaded nodes */
-	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-		rtp = &per_cpu(radix_tree_preloads, cpu);
-		while (rtp->nr) {
-			node = rtp->nodes;
-			rtp->nodes = node->private_data;
-			kmem_cache_free(radix_tree_node_cachep, node);
-			rtp->nr--;
-		}
+	rtp = &per_cpu(radix_tree_preloads, cpu);
+	while (rtp->nr) {
+		node = rtp->nodes;
+		rtp->nodes = node->private_data;
+		kmem_cache_free(radix_tree_node_cachep, node);
+		rtp->nr--;
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
 void __init radix_tree_init(void)
 {
+	int ret;
 	radix_tree_node_cachep = kmem_cache_create("radix_tree_node",
 			sizeof(struct radix_tree_node), 0,
 			SLAB_PANIC | SLAB_RECLAIM_ACCOUNT,
 			radix_tree_node_ctor);
 	radix_tree_init_maxnodes();
-	hotcpu_notifier(radix_tree_callback, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_RADIX_DEAD, "lib/radix:dead",
+					NULL, radix_tree_cpu_dead);
+	WARN_ON(ret < 0);
 }

[tip:smp/hotplug] mm/page_alloc: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  005fd4bbef168e9dea896085b001d64369e9834a
Gitweb:     http://git.kernel.org/tip/005fd4bbef168e9dea896085b001d64369e9834a
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:02 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:27 +0100

mm/page_alloc: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-mm@kvack.org
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-7-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |  1 +
 mm/page_alloc.c            | 49 +++++++++++++++++++++++-----------------------
 2 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 89310fb..31c58f6 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -35,6 +35,7 @@ enum cpuhp_state {
 	CPUHP_MM_MEMCQ_DEAD,
 	CPUHP_PERCPU_CNT_DEAD,
 	CPUHP_RADIX_DEAD,
+	CPUHP_PAGE_ALLOC_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 072d791..fc98c2b 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -6491,38 +6491,39 @@ void __init free_area_init(unsigned long *zones_size)
 			__pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
 }
 
-static int page_alloc_cpu_notify(struct notifier_block *self,
-				 unsigned long action, void *hcpu)
+static int page_alloc_cpu_dead(unsigned int cpu)
 {
-	int cpu = (unsigned long)hcpu;
 
-	if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-		lru_add_drain_cpu(cpu);
-		drain_pages(cpu);
+	lru_add_drain_cpu(cpu);
+	drain_pages(cpu);
 
-		/*
-		 * Spill the event counters of the dead processor
-		 * into the current processors event counters.
-		 * This artificially elevates the count of the current
-		 * processor.
-		 */
-		vm_events_fold_cpu(cpu);
+	/*
+	 * Spill the event counters of the dead processor
+	 * into the current processors event counters.
+	 * This artificially elevates the count of the current
+	 * processor.
+	 */
+	vm_events_fold_cpu(cpu);
 
-		/*
-		 * Zero the differential counters of the dead processor
-		 * so that the vm statistics are consistent.
-		 *
-		 * This is only okay since the processor is dead and cannot
-		 * race with what we are doing.
-		 */
-		cpu_vm_stats_fold(cpu);
-	}
-	return NOTIFY_OK;
+	/*
+	 * Zero the differential counters of the dead processor
+	 * so that the vm statistics are consistent.
+	 *
+	 * This is only okay since the processor is dead and cannot
+	 * race with what we are doing.
+	 */
+	cpu_vm_stats_fold(cpu);
+	return 0;
 }
 
 void __init page_alloc_init(void)
 {
-	hotcpu_notifier(page_alloc_cpu_notify, 0);
+	int ret;
+
+	ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC_DEAD,
+					"mm/page_alloc:dead", NULL,
+					page_alloc_cpu_dead);
+	WARN_ON(ret < 0);
 }
 
 /*

[tip:smp/hotplug] mm/vmscan: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  517bbed90635a17cf27450fb385af0e502413bea
Gitweb:     http://git.kernel.org/tip/517bbed90635a17cf27450fb385af0e502413bea
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:03 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:27 +0100

mm/vmscan: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-mm@kvack.org
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-8-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 mm/vmscan.c | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 76fda22..b8404d3 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -3556,24 +3556,21 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
    not required for correctness.  So if the last cpu in a node goes
    away, we get changed to run anywhere: as the first one comes back,
    restore their cpu bindings. */
-static int cpu_callback(struct notifier_block *nfb, unsigned long action,
-			void *hcpu)
+static int kswapd_cpu_online(unsigned int cpu)
 {
 	int nid;
 
-	if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
-		for_each_node_state(nid, N_MEMORY) {
-			pg_data_t *pgdat = NODE_DATA(nid);
-			const struct cpumask *mask;
+	for_each_node_state(nid, N_MEMORY) {
+		pg_data_t *pgdat = NODE_DATA(nid);
+		const struct cpumask *mask;
 
-			mask = cpumask_of_node(pgdat->node_id);
+		mask = cpumask_of_node(pgdat->node_id);
 
-			if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
-				/* One of our CPUs online: restore mask */
-				set_cpus_allowed_ptr(pgdat->kswapd, mask);
-		}
+		if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
+			/* One of our CPUs online: restore mask */
+			set_cpus_allowed_ptr(pgdat->kswapd, mask);
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
 /*
@@ -3615,12 +3612,15 @@ void kswapd_stop(int nid)
 
 static int __init kswapd_init(void)
 {
-	int nid;
+	int nid, ret;
 
 	swap_setup();
 	for_each_node_state(nid, N_MEMORY)
  		kswapd_run(nid);
-	hotcpu_notifier(cpu_callback, 0);
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					"mm/vmscan:online", kswapd_cpu_online,
+					NULL);
+	WARN_ON(ret < 0);
 	return 0;
 }
 

[tip:smp/hotplug] net/dev: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  f0bf90def3528cebed45ebd81d9b5d0fa17d7422
Gitweb:     http://git.kernel.org/tip/f0bf90def3528cebed45ebd81d9b5d0fa17d7422
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:04 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:28 +0100

net/dev: Convert to hotplug state machine

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: netdev@vger.kernel.org
Cc: "David S. Miller" <davem@davemloft.net>
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-9-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |  1 +
 net/core/dev.c             | 16 ++++++----------
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 31c58f6..394eb7e 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -36,6 +36,7 @@ enum cpuhp_state {
 	CPUHP_PERCPU_CNT_DEAD,
 	CPUHP_RADIX_DEAD,
 	CPUHP_PAGE_ALLOC_DEAD,
+	CPUHP_NET_DEV_DEAD,
 	CPUHP_WORKQUEUE_PREP,
 	CPUHP_POWER_NUMA_PREPARE,
 	CPUHP_HRTIMERS_PREPARE,
diff --git a/net/core/dev.c b/net/core/dev.c
index 820bac2..8e909b2 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -7953,18 +7953,13 @@ out:
 }
 EXPORT_SYMBOL_GPL(dev_change_net_namespace);
 
-static int dev_cpu_callback(struct notifier_block *nfb,
-			    unsigned long action,
-			    void *ocpu)
+static int dev_cpu_dead(unsigned int oldcpu)
 {
 	struct sk_buff **list_skb;
 	struct sk_buff *skb;
-	unsigned int cpu, oldcpu = (unsigned long)ocpu;
+	unsigned int cpu;
 	struct softnet_data *sd, *oldsd;
 
-	if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
-		return NOTIFY_OK;
-
 	local_irq_disable();
 	cpu = smp_processor_id();
 	sd = &per_cpu(softnet_data, cpu);
@@ -8014,10 +8009,9 @@ static int dev_cpu_callback(struct notifier_block *nfb,
 		input_queue_head_incr(oldsd);
 	}
 
-	return NOTIFY_OK;
+	return 0;
 }
 
-
 /**
  *	netdev_increment_features - increment feature set by one
  *	@all: current feature set
@@ -8351,7 +8345,9 @@ static int __init net_dev_init(void)
 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
 
-	hotcpu_notifier(dev_cpu_callback, 0);
+	rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
+				       NULL, dev_cpu_dead);
+	WARN_ON(rc < 0);
 	dst_subsys_init();
 	rc = 0;
 out:

[tip:smp/hotplug] net/flowcache: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  a4fc1bfc42062e8bc7b2271a90d17403b096ce5d
Gitweb:     http://git.kernel.org/tip/a4fc1bfc42062e8bc7b2271a90d17403b096ce5d
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:05 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:28 +0100

net/flowcache: Convert to hotplug state machine

Install the callbacks via the state machine. Use multi state support to avoid
custom list handling for the multiple instances.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Steffen Klassert <steffen.klassert@secunet.com>
Cc: Herbert Xu <herbert@gondor.apana.org.au>
Cc: netdev@vger.kernel.org
Cc: rt@linutronix.de
Cc: "David S. Miller" <davem@davemloft.net>
Link: http://lkml.kernel.org/r/20161103145021.28528-10-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 include/linux/cpuhotplug.h |  1 +
 include/net/flow.h         |  1 +
 include/net/flowcache.h    |  2 +-
 net/core/flow.c            | 60 ++++++++++++++++++++--------------------------
 net/xfrm/xfrm_policy.c     |  1 +
 5 files changed, 30 insertions(+), 35 deletions(-)

diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 394eb7e..86b940f 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -56,6 +56,7 @@ enum cpuhp_state {
 	CPUHP_ARM_SHMOBILE_SCU_PREPARE,
 	CPUHP_SH_SH3X_PREPARE,
 	CPUHP_BLK_MQ_PREPARE,
+	CPUHP_NET_FLOW_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,
diff --git a/include/net/flow.h b/include/net/flow.h
index 035aa77..2e386bd 100644
--- a/include/net/flow.h
+++ b/include/net/flow.h
@@ -239,6 +239,7 @@ struct flow_cache_object *flow_cache_lookup(struct net *net,
 					    void *ctx);
 int flow_cache_init(struct net *net);
 void flow_cache_fini(struct net *net);
+void flow_cache_hp_init(void);
 
 void flow_cache_flush(struct net *net);
 void flow_cache_flush_deferred(struct net *net);
diff --git a/include/net/flowcache.h b/include/net/flowcache.h
index c8f665e..9caf3bf 100644
--- a/include/net/flowcache.h
+++ b/include/net/flowcache.h
@@ -17,7 +17,7 @@ struct flow_cache_percpu {
 struct flow_cache {
 	u32				hash_shift;
 	struct flow_cache_percpu __percpu *percpu;
-	struct notifier_block		hotcpu_notifier;
+	struct hlist_node		node;
 	int				low_watermark;
 	int				high_watermark;
 	struct timer_list		rnd_timer;
diff --git a/net/core/flow.c b/net/core/flow.c
index 3937b1b..841fd7f 100644
--- a/net/core/flow.c
+++ b/net/core/flow.c
@@ -419,28 +419,20 @@ static int flow_cache_cpu_prepare(struct flow_cache *fc, int cpu)
 	return 0;
 }
 
-static int flow_cache_cpu(struct notifier_block *nfb,
-			  unsigned long action,
-			  void *hcpu)
+static int flow_cache_cpu_up_prep(unsigned int cpu, struct hlist_node *node)
 {
-	struct flow_cache *fc = container_of(nfb, struct flow_cache,
-						hotcpu_notifier);
-	int res, cpu = (unsigned long) hcpu;
+	struct flow_cache *fc = hlist_entry_safe(node, struct flow_cache, node);
+
+	return flow_cache_cpu_prepare(fc, cpu);
+}
+
+static int flow_cache_cpu_dead(unsigned int cpu, struct hlist_node *node)
+{
+	struct flow_cache *fc = hlist_entry_safe(node, struct flow_cache, node);
 	struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, cpu);
 
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		res = flow_cache_cpu_prepare(fc, cpu);
-		if (res)
-			return notifier_from_errno(res);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		__flow_cache_shrink(fc, fcp, 0);
-		break;
-	}
-	return NOTIFY_OK;
+	__flow_cache_shrink(fc, fcp, 0);
+	return 0;
 }
 
 int flow_cache_init(struct net *net)
@@ -467,18 +459,8 @@ int flow_cache_init(struct net *net)
 	if (!fc->percpu)
 		return -ENOMEM;
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		if (flow_cache_cpu_prepare(fc, i))
-			goto err;
-	}
-	fc->hotcpu_notifier = (struct notifier_block){
-		.notifier_call = flow_cache_cpu,
-	};
-	__register_hotcpu_notifier(&fc->hotcpu_notifier);
-
-	cpu_notifier_register_done();
+	if (cpuhp_state_add_instance(CPUHP_NET_FLOW_PREPARE, &fc->node))
+		goto err;
 
 	setup_timer(&fc->rnd_timer, flow_cache_new_hashrnd,
 		    (unsigned long) fc);
@@ -494,8 +476,6 @@ err:
 		fcp->hash_table = NULL;
 	}
 
-	cpu_notifier_register_done();
-
 	free_percpu(fc->percpu);
 	fc->percpu = NULL;
 
@@ -509,7 +489,8 @@ void flow_cache_fini(struct net *net)
 	struct flow_cache *fc = &net->xfrm.flow_cache_global;
 
 	del_timer_sync(&fc->rnd_timer);
-	unregister_hotcpu_notifier(&fc->hotcpu_notifier);
+
+	cpuhp_state_remove_instance_nocalls(CPUHP_NET_FLOW_PREPARE, &fc->node);
 
 	for_each_possible_cpu(i) {
 		struct flow_cache_percpu *fcp = per_cpu_ptr(fc->percpu, i);
@@ -521,3 +502,14 @@ void flow_cache_fini(struct net *net)
 	fc->percpu = NULL;
 }
 EXPORT_SYMBOL(flow_cache_fini);
+
+void __init flow_cache_hp_init(void)
+{
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_NET_FLOW_PREPARE,
+				      "net/flow:prepare",
+				      flow_cache_cpu_up_prep,
+				      flow_cache_cpu_dead);
+	WARN_ON(ret < 0);
+}
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index fd69866..4a8eff1 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -3111,6 +3111,7 @@ static struct pernet_operations __net_initdata xfrm_net_ops = {
 
 void __init xfrm_init(void)
 {
+	flow_cache_hp_init();
 	register_pernet_subsys(&xfrm_net_ops);
 	seqcount_init(&xfrm_policy_hash_generation);
 	xfrm_input_init();

[tip:smp/hotplug] s390/smp: Make cpu notifier symetric

[tip-bot for Thomas Gleixner]

Commit-ID:  ef65d45cbfbb438a5fbe8ef3fc424314ff1e8b7c
Gitweb:     http://git.kernel.org/tip/ef65d45cbfbb438a5fbe8ef3fc424314ff1e8b7c
Author:     Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Fri, 4 Nov 2016 15:41:41 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:28 +0100

s390/smp: Make cpu notifier symetric

There is no reason to remove the sysfs cpu files when the CPU is dead, they
can be removed when the cpu is prepared to go down. Doing it at
DOWN_PREPARE allows us to convert it to a symetric hotplug state in the
next step.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: linux-s390@vger.kernel.org
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161104144140.lcee6kwmwlx37m7g@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/s390/kernel/smp.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 35531fe..2a9c03d 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -1056,9 +1056,10 @@ static int smp_cpu_notify(struct notifier_block *self, unsigned long action,
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+	case CPU_DOWN_FAILED:
 		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
 		break;
-	case CPU_DEAD:
+	case CPU_DOWN_PREPARE:
 		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
 		break;
 	}

[tip:smp/hotplug] s390/smp: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  dfbbd86a0f1c3ceec15b64c8f2149a903806ed8c
Gitweb:     http://git.kernel.org/tip/dfbbd86a0f1c3ceec15b64c8f2149a903806ed8c
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Fri, 4 Nov 2016 15:45:03 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:28 +0100

s390/smp: Convert to hotplug state machine

cpuhp_setup_state() invokes the startup callback on all online cpus with
the proper protection, so we can remove the cpu hotplug protection from the
init function and the creation of the per cpu files for online cpus in
smp_add_present_cpu(). smp_add_present_cpu() is called also called from
__smp_rescan_cpus(), but this callpath never adds an online cpu, it merily
adds newly present cpus, so the creation of the cpu files is not required.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Cc: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: linux-s390@vger.kernel.org
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161104144502.7kd4bxz2rxqvtack@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/s390/kernel/smp.c | 37 +++++++++++--------------------------
 1 file changed, 11 insertions(+), 26 deletions(-)

diff --git a/arch/s390/kernel/smp.c b/arch/s390/kernel/smp.c
index 2a9c03d..870fb5a 100644
--- a/arch/s390/kernel/smp.c
+++ b/arch/s390/kernel/smp.c
@@ -1047,23 +1047,18 @@ static struct attribute_group cpu_online_attr_group = {
 	.attrs = cpu_online_attrs,
 };
 
-static int smp_cpu_notify(struct notifier_block *self, unsigned long action,
-			  void *hcpu)
+static int smp_cpu_online(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned int)(long)hcpu;
 	struct device *s = &per_cpu(cpu_device, cpu)->dev;
-	int err = 0;
 
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-		err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
-		break;
-	case CPU_DOWN_PREPARE:
-		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
-		break;
-	}
-	return notifier_from_errno(err);
+	return sysfs_create_group(&s->kobj, &cpu_online_attr_group);
+}
+static int smp_cpu_pre_down(unsigned int cpu)
+{
+	struct device *s = &per_cpu(cpu_device, cpu)->dev;
+
+	sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
+	return 0;
 }
 
 static int smp_add_present_cpu(int cpu)
@@ -1084,20 +1079,12 @@ static int smp_add_present_cpu(int cpu)
 	rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
 	if (rc)
 		goto out_cpu;
-	if (cpu_online(cpu)) {
-		rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
-		if (rc)
-			goto out_online;
-	}
 	rc = topology_cpu_init(c);
 	if (rc)
 		goto out_topology;
 	return 0;
 
 out_topology:
-	if (cpu_online(cpu))
-		sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
-out_online:
 	sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
 out_cpu:
 #ifdef CONFIG_HOTPLUG_CPU
@@ -1150,17 +1137,15 @@ static int __init s390_smp_init(void)
 	if (rc)
 		return rc;
 #endif
-	cpu_notifier_register_begin();
 	for_each_present_cpu(cpu) {
 		rc = smp_add_present_cpu(cpu);
 		if (rc)
 			goto out;
 	}
 
-	__hotcpu_notifier(smp_cpu_notify, 0);
-
+	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "s390/smp:online",
+			       smp_cpu_online, smp_cpu_pre_down);
 out:
-	cpu_notifier_register_done();
 	return rc;
 }
 subsys_initcall(s390_smp_init);

[tip:smp/hotplug] drivers base/cacheinfo: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  7cc277b489b4fe91f42eb596b282879c2d13152e
Gitweb:     http://git.kernel.org/tip/7cc277b489b4fe91f42eb596b282879c2d13152e
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:08 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:29 +0100

drivers base/cacheinfo: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs. No functional change.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-13-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/base/cacheinfo.c | 59 +++++++++++++-----------------------------------
 1 file changed, 16 insertions(+), 43 deletions(-)

diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index e9fd32e..47983a2 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -498,57 +498,30 @@ err:
 	return rc;
 }
 
-static void cache_remove_dev(unsigned int cpu)
+static int cacheinfo_cpu_online(unsigned int cpu)
 {
-	if (!cpumask_test_cpu(cpu, &cache_dev_map))
-		return;
-	cpumask_clear_cpu(cpu, &cache_dev_map);
+	int rc = detect_cache_attributes(cpu);
 
-	cpu_cache_sysfs_exit(cpu);
+	if (rc)
+		return rc;
+	rc = cache_add_dev(cpu);
+	if (rc)
+		free_cache_attributes(cpu);
+	return rc;
 }
 
-static int cacheinfo_cpu_callback(struct notifier_block *nfb,
-				  unsigned long action, void *hcpu)
+static int cacheinfo_cpu_pre_down(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-	int rc = 0;
-
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-		rc = detect_cache_attributes(cpu);
-		if (!rc)
-			rc = cache_add_dev(cpu);
-		break;
-	case CPU_DEAD:
-		cache_remove_dev(cpu);
-		free_cache_attributes(cpu);
-		break;
-	}
-	return notifier_from_errno(rc);
+	if (cpumask_test_and_clear_cpu(cpu, &cache_dev_map))
+		cpu_cache_sysfs_exit(cpu);
+
+	free_cache_attributes(cpu);
+	return 0;
 }
 
 static int __init cacheinfo_sysfs_init(void)
 {
-	int cpu, rc = 0;
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu) {
-		rc = detect_cache_attributes(cpu);
-		if (rc)
-			goto out;
-		rc = cache_add_dev(cpu);
-		if (rc) {
-			free_cache_attributes(cpu);
-			pr_err("error populating cacheinfo..cpu%d\n", cpu);
-			goto out;
-		}
-	}
-	__hotcpu_notifier(cacheinfo_cpu_callback, 0);
-
-out:
-	cpu_notifier_register_done();
-	return rc;
+	return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "base/cacheinfo:online",
+				 cacheinfo_cpu_online, cacheinfo_cpu_pre_down);
 }
-
 device_initcall(cacheinfo_sysfs_init);

[tip:smp/hotplug] drivers base/topology: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  38643a0e691ec947d311eb2db011b289cf95014e
Gitweb:     http://git.kernel.org/tip/38643a0e691ec947d311eb2db011b289cf95014e
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:09 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:29 +0100

drivers base/topology: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs. No functional change

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-14-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 drivers/base/topology.c    | 42 +++++-------------------------------------
 include/linux/cpuhotplug.h |  1 +
 2 files changed, 6 insertions(+), 37 deletions(-)

diff --git a/drivers/base/topology.c b/drivers/base/topology.c
index df3c97c..d6ec1c5 100644
--- a/drivers/base/topology.c
+++ b/drivers/base/topology.c
@@ -118,51 +118,19 @@ static int topology_add_dev(unsigned int cpu)
 	return sysfs_create_group(&dev->kobj, &topology_attr_group);
 }
 
-static void topology_remove_dev(unsigned int cpu)
+static int topology_remove_dev(unsigned int cpu)
 {
 	struct device *dev = get_cpu_device(cpu);
 
 	sysfs_remove_group(&dev->kobj, &topology_attr_group);
-}
-
-static int topology_cpu_callback(struct notifier_block *nfb,
-				 unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	int rc = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-	case CPU_UP_PREPARE_FROZEN:
-		rc = topology_add_dev(cpu);
-		break;
-	case CPU_UP_CANCELED:
-	case CPU_UP_CANCELED_FROZEN:
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		topology_remove_dev(cpu);
-		break;
-	}
-	return notifier_from_errno(rc);
+	return 0;
 }
 
 static int topology_sysfs_init(void)
 {
-	int cpu;
-	int rc = 0;
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(cpu) {
-		rc = topology_add_dev(cpu);
-		if (rc)
-			goto out;
-	}
-	__hotcpu_notifier(topology_cpu_callback, 0);
-
-out:
-	cpu_notifier_register_done();
-	return rc;
+	return cpuhp_setup_state(CPUHP_TOPOLOGY_PREPARE,
+				 "base/topology:prepare", topology_add_dev,
+				 topology_remove_dev);
 }
 
 device_initcall(topology_sysfs_init);
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 86b940f..3410d83 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -57,6 +57,7 @@ enum cpuhp_state {
 	CPUHP_SH_SH3X_PREPARE,
 	CPUHP_BLK_MQ_PREPARE,
 	CPUHP_NET_FLOW_PREPARE,
+	CPUHP_TOPOLOGY_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_NOTF_ERR_INJ_PREPARE,
 	CPUHP_MIPS_SOC_PREPARE,

[tip:smp/hotplug] ia64/err-inject: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  122231445f2517de1d3dd30acd28801b66d72cab
Gitweb:     http://git.kernel.org/tip/122231445f2517de1d3dd30acd28801b66d72cab
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:10 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:29 +0100

ia64/err-inject: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The removal of the files happens now in the prepare down stage as there is
no reason to keep them around until the cpu has actually died.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: linux-ia64@vger.kernel.org
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-15-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/ia64/kernel/err_inject.c | 74 ++++++++++---------------------------------
 1 file changed, 17 insertions(+), 57 deletions(-)

diff --git a/arch/ia64/kernel/err_inject.c b/arch/ia64/kernel/err_inject.c
index 5ed0ea9..85bba43 100644
--- a/arch/ia64/kernel/err_inject.c
+++ b/arch/ia64/kernel/err_inject.c
@@ -224,85 +224,45 @@ static struct attribute_group err_inject_attr_group = {
 	.name = "err_inject"
 };
 /* Add/Remove err_inject interface for CPU device */
-static int err_inject_add_dev(struct device *sys_dev)
+static int err_inject_add_dev(unsigned int cpu)
 {
+	struct device *sys_dev = get_cpu_device(cpu);
+
 	return sysfs_create_group(&sys_dev->kobj, &err_inject_attr_group);
 }
 
-static int err_inject_remove_dev(struct device *sys_dev)
+static int err_inject_remove_dev(unsigned int cpu)
 {
+	struct device *sys_dev = get_cpu_device(cpu);
+
 	sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group);
 	return 0;
 }
-static int err_inject_cpu_callback(struct notifier_block *nfb,
-		unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	struct device *sys_dev;
-
-	sys_dev = get_cpu_device(cpu);
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		err_inject_add_dev(sys_dev);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		err_inject_remove_dev(sys_dev);
-		break;
-	}
-
-	return NOTIFY_OK;
-}
 
-static struct notifier_block err_inject_cpu_notifier =
-{
-	.notifier_call = err_inject_cpu_callback,
-};
+static enum cpuhp_state hp_online;
 
-static int __init
-err_inject_init(void)
+static int __init err_inject_init(void)
 {
-	int i;
-
+	int ret;
 #ifdef ERR_INJ_DEBUG
 	printk(KERN_INFO "Enter error injection driver.\n");
 #endif
 
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		err_inject_cpu_callback(&err_inject_cpu_notifier, CPU_ONLINE,
-				(void *)(long)i);
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/err_inj:online",
+				err_inject_add_dev, err_inject_remove_dev);
+	if (ret >= 0) {
+		hp_online = ret;
+		ret = 0;
 	}
-
-	__register_hotcpu_notifier(&err_inject_cpu_notifier);
-
-	cpu_notifier_register_done();
-
-	return 0;
+	return ret;
 }
 
-static void __exit
-err_inject_exit(void)
+static void __exit err_inject_exit(void)
 {
-	int i;
-	struct device *sys_dev;
-
 #ifdef ERR_INJ_DEBUG
 	printk(KERN_INFO "Exit error injection driver.\n");
 #endif
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		sys_dev = get_cpu_device(i);
-		sysfs_remove_group(&sys_dev->kobj, &err_inject_attr_group);
-	}
-
-	__unregister_hotcpu_notifier(&err_inject_cpu_notifier);
-
-	cpu_notifier_register_done();
+	cpuhp_remove_state(hp_online);
 }
 
 module_init(err_inject_init);

[tip:smp/hotplug] ia64/palinfo: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  715c32116fa9cfa298170646be02ae5c0135ba0f
Gitweb:     http://git.kernel.org/tip/715c32116fa9cfa298170646be02ae5c0135ba0f
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 15:50:11 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:30 +0100

ia64/palinfo: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

The removal of the files happens now in the prepare down stage as there is
no reason to keep them around until the cpu has actually died.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: linux-ia64@vger.kernel.org
Cc: rt@linutronix.de
Link: http://lkml.kernel.org/r/20161103145021.28528-16-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/ia64/kernel/palinfo.c | 60 +++++++++++++---------------------------------
 1 file changed, 16 insertions(+), 44 deletions(-)

diff --git a/arch/ia64/kernel/palinfo.c b/arch/ia64/kernel/palinfo.c
index c39c3cd..b6e5978 100644
--- a/arch/ia64/kernel/palinfo.c
+++ b/arch/ia64/kernel/palinfo.c
@@ -932,8 +932,7 @@ static const struct file_operations proc_palinfo_fops = {
 	.release	= single_release,
 };
 
-static void
-create_palinfo_proc_entries(unsigned int cpu)
+static int palinfo_add_proc(unsigned int cpu)
 {
 	pal_func_cpu_u_t f;
 	struct proc_dir_entry *cpu_dir;
@@ -943,7 +942,7 @@ create_palinfo_proc_entries(unsigned int cpu)
 
 	cpu_dir = proc_mkdir(cpustr, palinfo_dir);
 	if (!cpu_dir)
-		return;
+		return -EINVAL;
 
 	f.req_cpu = cpu;
 
@@ -952,42 +951,21 @@ create_palinfo_proc_entries(unsigned int cpu)
 		proc_create_data(palinfo_entries[j].name, 0, cpu_dir,
 				 &proc_palinfo_fops, (void *)f.value);
 	}
+	return 0;
 }
 
-static void
-remove_palinfo_proc_entries(unsigned int hcpu)
+static int palinfo_del_proc(unsigned int hcpu)
 {
 	char cpustr[3+4+1];	/* cpu numbers are up to 4095 on itanic */
+
 	sprintf(cpustr, "cpu%d", hcpu);
 	remove_proc_subtree(cpustr, palinfo_dir);
+	return 0;
 }
 
-static int palinfo_cpu_callback(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
-{
-	unsigned int hotcpu = (unsigned long)hcpu;
-
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		create_palinfo_proc_entries(hotcpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		remove_palinfo_proc_entries(hotcpu);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block __refdata palinfo_cpu_notifier =
-{
-	.notifier_call = palinfo_cpu_callback,
-	.priority = 0,
-};
+static enum cpuhp_state hp_online;
 
-static int __init
-palinfo_init(void)
+static int __init palinfo_init(void)
 {
 	int i = 0;
 
@@ -996,25 +974,19 @@ palinfo_init(void)
 	if (!palinfo_dir)
 		return -ENOMEM;
 
-	cpu_notifier_register_begin();
-
-	/* Create palinfo dirs in /proc for all online cpus */
-	for_each_online_cpu(i) {
-		create_palinfo_proc_entries(i);
+	i = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/palinfo:online",
+			      palinfo_add_proc, palinfo_del_proc);
+	if (i < 0) {
+		remove_proc_subtree("pal", NULL);
+		return i;
 	}
-
-	/* Register for future delivery via notify registration */
-	__register_hotcpu_notifier(&palinfo_cpu_notifier);
-
-	cpu_notifier_register_done();
-
+	hp_online = i;
 	return 0;
 }
 
-static void __exit
-palinfo_exit(void)
+static void __exit palinfo_exit(void)
 {
-	unregister_hotcpu_notifier(&palinfo_cpu_notifier);
+	cpuhp_remove_state(hp_online);
 	remove_proc_subtree("pal", NULL);
 }
 

[tip:smp/hotplug] ia64/salinfo: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  aa17662bc26d2555e1f79e7bcd259043582f32ed
Gitweb:     http://git.kernel.org/tip/aa17662bc26d2555e1f79e7bcd259043582f32ed
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 18:31:29 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:30 +0100

ia64/salinfo: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: linux-ia64@vger.kernel.org
Cc: kbuild test robot <lkp@intel.com>
Cc: rt@linutronix.de
Cc: kbuild-all@01.org
Link: http://lkml.kernel.org/r/20161103173128.xuulg4nius46dng5@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/ia64/kernel/salinfo.c | 83 ++++++++++++++++++----------------------------
 1 file changed, 32 insertions(+), 51 deletions(-)

diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c
index 5313007..aaf74f3 100644
--- a/arch/ia64/kernel/salinfo.c
+++ b/arch/ia64/kernel/salinfo.c
@@ -550,52 +550,40 @@ static const struct file_operations salinfo_data_fops = {
 	.llseek  = default_llseek,
 };
 
-static int
-salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu)
+static int salinfo_cpu_online(unsigned int cpu)
 {
-	unsigned int i, cpu = (unsigned long)hcpu;
-	unsigned long flags;
+	unsigned int i, end = ARRAY_SIZE(salinfo_data);
 	struct salinfo_data *data;
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		spin_lock_irqsave(&data_saved_lock, flags);
-		for (i = 0, data = salinfo_data;
-		     i < ARRAY_SIZE(salinfo_data);
-		     ++i, ++data) {
-			cpumask_set_cpu(cpu, &data->cpu_event);
-			wake_up_interruptible(&data->read_wait);
-		}
-		spin_unlock_irqrestore(&data_saved_lock, flags);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		spin_lock_irqsave(&data_saved_lock, flags);
-		for (i = 0, data = salinfo_data;
-		     i < ARRAY_SIZE(salinfo_data);
-		     ++i, ++data) {
-			struct salinfo_data_saved *data_saved;
-			int j;
-			for (j = ARRAY_SIZE(data->data_saved) - 1, data_saved = data->data_saved + j;
-			     j >= 0;
-			     --j, --data_saved) {
-				if (data_saved->buffer && data_saved->cpu == cpu) {
-					shift1_data_saved(data, j);
-				}
-			}
-			cpumask_clear_cpu(cpu, &data->cpu_event);
-		}
-		spin_unlock_irqrestore(&data_saved_lock, flags);
-		break;
+
+	spin_lock_irq(&data_saved_lock);
+	for (i = 0, data = salinfo_data; i < end; ++i, ++data) {
+		cpumask_set_cpu(cpu, &data->cpu_event);
+		wake_up_interruptible(&data->read_wait);
 	}
-	return NOTIFY_OK;
+	spin_unlock_irq(&data_saved_lock);
+	return 0;
 }
 
-static struct notifier_block salinfo_cpu_notifier =
+static int salinfo_cpu_pre_down(unsigned int cpu)
 {
-	.notifier_call = salinfo_cpu_callback,
-	.priority = 0,
-};
+	unsigned int i, end = ARRAY_SIZE(salinfo_data);
+	struct salinfo_data *data;
+
+	spin_lock_irq(&data_saved_lock);
+	for (i = 0, data = salinfo_data; i < end; ++i, ++data) {
+		struct salinfo_data_saved *data_saved;
+		int j = ARRAY_SIZE(data->data_saved) - 1;
+
+		for (data_saved = data->data_saved + j; j >= 0;
+		     --j, --data_saved) {
+			if (data_saved->buffer && data_saved->cpu == cpu)
+				shift1_data_saved(data, j);
+		}
+		cpumask_clear_cpu(cpu, &data->cpu_event);
+	}
+	spin_unlock_irq(&data_saved_lock);
+	return 0;
+}
 
 static int __init
 salinfo_init(void)
@@ -604,7 +592,7 @@ salinfo_init(void)
 	struct proc_dir_entry **sdir = salinfo_proc_entries; /* keeps track of every entry */
 	struct proc_dir_entry *dir, *entry;
 	struct salinfo_data *data;
-	int i, j;
+	int i;
 
 	salinfo_dir = proc_mkdir("sal", NULL);
 	if (!salinfo_dir)
@@ -617,8 +605,6 @@ salinfo_init(void)
 					   (void *)salinfo_entries[i].feature);
 	}
 
-	cpu_notifier_register_begin();
-
 	for (i = 0; i < ARRAY_SIZE(salinfo_log_name); i++) {
 		data = salinfo_data + i;
 		data->type = i;
@@ -639,10 +625,6 @@ salinfo_init(void)
 			continue;
 		*sdir++ = entry;
 
-		/* we missed any events before now */
-		for_each_online_cpu(j)
-			cpumask_set_cpu(j, &data->cpu_event);
-
 		*sdir++ = dir;
 	}
 
@@ -653,10 +635,9 @@ salinfo_init(void)
 	salinfo_timer.function = &salinfo_timeout;
 	add_timer(&salinfo_timer);
 
-	__register_hotcpu_notifier(&salinfo_cpu_notifier);
-
-	cpu_notifier_register_done();
-
+	i = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/salinfo:online",
+			      salinfo_cpu_online, salinfo_cpu_pre_down);
+	WARN_ON(i < 0);
 	return 0;
 }
 

[tip:smp/hotplug] ia64/topology: Convert to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  5c584dd58ea878027ed067996b306416a9c356b8
Gitweb:     http://git.kernel.org/tip/5c584dd58ea878027ed067996b306416a9c356b8
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 3 Nov 2016 18:33:53 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 9 Nov 2016 23:45:30 +0100

ia64/topology: Convert to hotplug state machine

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Fenghua Yu <fenghua.yu@intel.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: linux-ia64@vger.kernel.org
Cc: kbuild test robot <lkp@intel.com>
Cc: rt@linutronix.de
Cc: kbuild-all@01.org
Link: http://lkml.kernel.org/r/20161103173353.dudhkpioitghd74x@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/ia64/kernel/topology.c | 54 ++++++---------------------------------------
 1 file changed, 7 insertions(+), 47 deletions(-)

diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c
index c01fe89..1a68f01 100644
--- a/arch/ia64/kernel/topology.c
+++ b/arch/ia64/kernel/topology.c
@@ -349,9 +349,9 @@ static int cpu_cache_sysfs_init(unsigned int cpu)
 }
 
 /* Add cache interface for CPU device */
-static int cache_add_dev(struct device *sys_dev)
+static int cache_add_dev(unsigned int cpu)
 {
-	unsigned int cpu = sys_dev->id;
+	struct device *sys_dev = get_cpu_device(cpu);
 	unsigned long i, j;
 	struct cache_info *this_object;
 	int retval = 0;
@@ -399,9 +399,8 @@ static int cache_add_dev(struct device *sys_dev)
 }
 
 /* Remove cache interface for CPU device */
-static int cache_remove_dev(struct device *sys_dev)
+static int cache_remove_dev(unsigned int cpu)
 {
-	unsigned int cpu = sys_dev->id;
 	unsigned long i;
 
 	for (i = 0; i < all_cpu_cache_info[cpu].num_cache_leaves; i++)
@@ -419,52 +418,13 @@ static int cache_remove_dev(struct device *sys_dev)
 	return 0;
 }
 
-/*
- * When a cpu is hot-plugged, do a check and initiate
- * cache kobject if necessary
- */
-static int cache_cpu_callback(struct notifier_block *nfb,
-		unsigned long action, void *hcpu)
-{
-	unsigned int cpu = (unsigned long)hcpu;
-	struct device *sys_dev;
-
-	sys_dev = get_cpu_device(cpu);
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		cache_add_dev(sys_dev);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		cache_remove_dev(sys_dev);
-		break;
-	}
-	return NOTIFY_OK;
-}
-
-static struct notifier_block cache_cpu_notifier =
-{
-	.notifier_call = cache_cpu_callback
-};
-
 static int __init cache_sysfs_init(void)
 {
-	int i;
-
-	cpu_notifier_register_begin();
-
-	for_each_online_cpu(i) {
-		struct device *sys_dev = get_cpu_device((unsigned int)i);
-		cache_add_dev(sys_dev);
-	}
-
-	__register_hotcpu_notifier(&cache_cpu_notifier);
-
-	cpu_notifier_register_done();
+	int ret;
 
+	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ia64/topology:online",
+				cache_add_dev, cache_remove_dev);
+	WARN_ON(ret < 0);
 	return 0;
 }
-
 device_initcall(cache_sysfs_init);
-

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Sebastian Andrzej Siewior]

On 2016-11-09 18:37:23 [+0000], Luck, Tony wrote:
> > That's why the hotplug callback mce_disable_cpu() doesn't fiddle with
> > CR4 - it only clears the bits in MCi_CTL. And I think we should remain
> > that way.
> 
> N.B. See vendor_disable_error_reporting() ... on Intel we don't clear MCi_CTL.

okay. So I had a discussion with Boris and we agreed that we don't do
the init part in one place only the timer stuff will be moved.
If I understand Tony correctly then we undo something in
vendor_disable_error_reporting() (on !INTEL CPUs) that we enable in
identify_cpu().

Boris, if my understanding is correct, what now? Do move the whole
hotplug init part to an earlier stage? Or do we drop
vendor_disable_error_reporting()?

> -Tony

Sebastian

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Borislav Petkov]

On Thu, Nov 10, 2016 at 10:00:38AM +0100, Sebastian Andrzej Siewior wrote:
> okay. So I had a discussion with Boris and we agreed that we don't do
> the init part in one place only the timer stuff will be moved.
> If I understand Tony correctly then we undo something in
> vendor_disable_error_reporting() (on !INTEL CPUs) that we enable in
> identify_cpu().
> 
> Boris, if my understanding is correct, what now? Do move the whole
> hotplug init part to an earlier stage? Or do we drop
> vendor_disable_error_reporting()?

Nah, we proceed as we discussed it - let's move the timer only. The
mce_gen_pool_init() allocation is not where it should be - it should
run somewhere only once on CPU0 but I haven't found a better place yet
so that the allocation succeeds - mind you, this is very early and
gen_pool_create() uses kmalloc_node() which might not be functioning
yet...

Tony's point is showing that the hotplug notifiers don't do any
bit clearing on Intel, i.e., basically we don't do anything on the
CPU_DOWN_PREPARE path on Intel.

Thanks.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Thomas Gleixner]

On Thu, 10 Nov 2016, Sebastian Andrzej Siewior wrote:

> On 2016-11-09 18:37:23 [+0000], Luck, Tony wrote:
> > > That's why the hotplug callback mce_disable_cpu() doesn't fiddle with
> > > CR4 - it only clears the bits in MCi_CTL. And I think we should remain
> > > that way.
> > 
> > N.B. See vendor_disable_error_reporting() ... on Intel we don't clear MCi_CTL.
> 
> okay. So I had a discussion with Boris and we agreed that we don't do
> the init part in one place only the timer stuff will be moved.
> If I understand Tony correctly then we undo something in
> vendor_disable_error_reporting() (on !INTEL CPUs) that we enable in
> identify_cpu().

Why on earth do we enable MCE stuff in identify_cpu() ?

Thanks,

	tglx

Re: [PATCH 22/25] x86/mcheck: Do the init in one place

[Sebastian Andrzej Siewior]

On 2016-11-10 11:22:22 [+0100], Thomas Gleixner wrote:
> 
> Why on earth do we enable MCE stuff in identify_cpu() ?
so it happens asap.

Boris' mail in this thread:
|Uh, hm, I'm not sure about this: so the issue I see with this is that
|the more we're delaying the enabling or MCE reporting - and especially
|setting CR4[MCE] - the more we're increasing the window where a MCE
|during early boot will cause a shutdown. (This is what happens if
|CR4[MCE]=0b).

> Thanks,
> 
> 	tglx

Sebastian

Re: [PATCH 01/25] fs/buffer: Convert to hotplug state machine

[Al Viro]

On Thu, Nov 03, 2016 at 03:49:57PM +0100, Sebastian Andrzej Siewior wrote:
> Install the callbacks via the state machine.

> diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
> index afe641c02dca..69b74fa0da60 100644
> --- a/include/linux/cpuhotplug.h
> +++ b/include/linux/cpuhotplug.h
> @@ -30,6 +30,7 @@ enum cpuhp_state {
>  	CPUHP_ACPI_CPUDRV_DEAD,
>  	CPUHP_S390_PFAULT_DEAD,
>  	CPUHP_BLK_MQ_DEAD,
> +	CPUHP_FS_BUFF_DEAD,
>  	CPUHP_WORKQUEUE_PREP,
>  	CPUHP_POWER_NUMA_PREPARE,
>  	CPUHP_HRTIMERS_PREPARE,

*ouch*

So we are getting a large list of things from unrelated subsystems, maintained
in a single place, all next to each other.  All examples of that sort of
thing I can recall had ended up biting our arses...

Re: [PATCH 01/25] fs/buffer: Convert to hotplug state machine

[Thomas Gleixner]

On Thu, 10 Nov 2016, Al Viro wrote:

> On Thu, Nov 03, 2016 at 03:49:57PM +0100, Sebastian Andrzej Siewior wrote:
> > Install the callbacks via the state machine.
> 
> > diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
> > index afe641c02dca..69b74fa0da60 100644
> > --- a/include/linux/cpuhotplug.h
> > +++ b/include/linux/cpuhotplug.h
> > @@ -30,6 +30,7 @@ enum cpuhp_state {
> >  	CPUHP_ACPI_CPUDRV_DEAD,
> >  	CPUHP_S390_PFAULT_DEAD,
> >  	CPUHP_BLK_MQ_DEAD,
> > +	CPUHP_FS_BUFF_DEAD,
> >  	CPUHP_WORKQUEUE_PREP,
> >  	CPUHP_POWER_NUMA_PREPARE,
> >  	CPUHP_HRTIMERS_PREPARE,
> 
> *ouch*
> 
> So we are getting a large list of things from unrelated subsystems, maintained
> in a single place, all next to each other.  All examples of that sort of
> thing I can recall had ended up biting our arses...

I rather have my arse bitten by a few merge conflicts than constantly
chasing why odering by chance, link order or magic constants in random
files make things explode or not work.

Thanks,

	tglx

x86/mcheck: convert to hotplug state engine, take #2

[Sebastian Andrzej Siewior]

The fixed up version of patch 1-4 was already posted and are here just for
completeness.
Patches 5-7 are new and address Boris's concern not to init the MCE feature too
late on the boot CPU during the boot process.

Sebastian

[PATCH 1/7] x86/mcheck: Move threshold_create_device()

[Sebastian Andrzej Siewior]

Move the threshold_create_device() so it can use
threshold_remove_device() without a forward declaration.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 50 ++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 9b5403462936..75a3e3eab81e 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1010,31 +1010,6 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank)
 	return err;
 }
 
-/* create dir/files for all valid threshold banks */
-static int threshold_create_device(unsigned int cpu)
-{
-	unsigned int bank;
-	struct threshold_bank **bp;
-	int err = 0;
-
-	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
-		     GFP_KERNEL);
-	if (!bp)
-		return -ENOMEM;
-
-	per_cpu(threshold_banks, cpu) = bp;
-
-	for (bank = 0; bank < mca_cfg.banks; ++bank) {
-		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
-			continue;
-		err = threshold_create_bank(cpu, bank);
-		if (err)
-			return err;
-	}
-
-	return err;
-}
-
 static void deallocate_threshold_block(unsigned int cpu,
 						 unsigned int bank)
 {
@@ -1114,6 +1089,31 @@ static void threshold_remove_device(unsigned int cpu)
 	kfree(per_cpu(threshold_banks, cpu));
 }
 
+/* create dir/files for all valid threshold banks */
+static int threshold_create_device(unsigned int cpu)
+{
+	unsigned int bank;
+	struct threshold_bank **bp;
+	int err = 0;
+
+	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
+		     GFP_KERNEL);
+	if (!bp)
+		return -ENOMEM;
+
+	per_cpu(threshold_banks, cpu) = bp;
+
+	for (bank = 0; bank < mca_cfg.banks; ++bank) {
+		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
+			continue;
+		err = threshold_create_bank(cpu, bank);
+		if (err)
+			return err;
+	}
+
+	return err;
+}
+
 /* get notified when a cpu comes on/off */
 static void
 amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
-- 
2.10.2

[PATCH 2/7] x86/mcheck: Explicit cleanup on failure in mce_amd

[Sebastian Andrzej Siewior]

If the ONLINE callback fails, the driver does not any clean up right
away instead it waits to get to the DEAD stage to do it. Yes, it waits.
Since we don't pass the error code back to the caller, no one knows.

Do the clean up right away so it does not look like a leak.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 75a3e3eab81e..55cd018bc1ae 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1087,6 +1087,7 @@ static void threshold_remove_device(unsigned int cpu)
 		threshold_remove_bank(cpu, bank);
 	}
 	kfree(per_cpu(threshold_banks, cpu));
+	per_cpu(threshold_banks, cpu) = NULL;
 }
 
 /* create dir/files for all valid threshold banks */
@@ -1108,9 +1109,11 @@ static int threshold_create_device(unsigned int cpu)
 			continue;
 		err = threshold_create_bank(cpu, bank);
 		if (err)
-			return err;
+			goto err;
 	}
-
+	return err;
+err:
+	threshold_remove_device(cpu);
 	return err;
 }
 
-- 
2.10.2

[PATCH 3/7] x86/mcheck: Be prepared for a rollback back to the ONLINE state

[Sebastian Andrzej Siewior]

If we try a CPU down and fail in the middle then we roll back to the
online state. This means we would perform CPU_ONLINE / mce_device_create()
without invoking CPU_DEAD / mce_device_remove() for the cleanup of what was
allocated in CPU_ONLINE.
Be prepared for this and don't allocate the struct if we have it
already.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce.c     | 4 ++++
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index a7fdf453d895..e9ffd6d9e32d 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2409,6 +2409,10 @@ static int mce_device_create(unsigned int cpu)
 	if (!mce_available(&boot_cpu_data))
 		return -EIO;
 
+	dev = per_cpu(mce_device, cpu);
+	if (dev)
+		return 0;
+
 	dev = kzalloc(sizeof *dev, GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 55cd018bc1ae..e93580c82ef0 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1097,6 +1097,10 @@ static int threshold_create_device(unsigned int cpu)
 	struct threshold_bank **bp;
 	int err = 0;
 
+	bp = per_cpu(threshold_banks, cpu);
+	if (bp)
+		return 0;
+
 	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
 		     GFP_KERNEL);
 	if (!bp)
-- 
2.10.2

[PATCH 4/7] x86/mcheck: Split threshold_cpu_callback into two callbacks

[Sebastian Andrzej Siewior]

The threshold_cpu_callback callbacks looks like one of the notifier and
its arguments are almost the same. Split this out and have one ONLINE
and one DEAD callback. This will come handy later once the main code
gets changed to use the callback mechanism.
Also, handle threshold_cpu_callback_online() return value so we don't
continue if the function fails.

Boris Petkov removed the callback pointer and replaced it with proper
functions.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 arch/x86/include/asm/mce.h           |  7 +++++--
 arch/x86/kernel/cpu/mcheck/mce.c     | 14 +++++++------
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 38 ++++++++++++++----------------------
 3 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 9bd7ff5ffbcc..caafad4a211e 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -293,9 +293,7 @@ void do_machine_check(struct pt_regs *, long);
 /*
  * Threshold handler
  */
-
 extern void (*mce_threshold_vector)(void);
-extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
 
 /* Deferred error interrupt handler */
 extern void (*deferred_error_int_vector)(void);
@@ -377,7 +375,12 @@ struct smca_bank_info {
 };
 
 extern struct smca_bank_info smca_banks[MAX_NR_BANKS];
+extern int mce_threshold_create_device(unsigned int cpu);
+extern int mce_threshold_remove_device(unsigned int cpu);
+#else
 
+static inline int mce_threshold_create_device(unsigned int cpu) { return 0; };
+static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; };
 #endif
 
 #endif /* _ASM_X86_MCE_H */
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index e9ffd6d9e32d..052b5e05c3c4 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2255,8 +2255,6 @@ static struct bus_type mce_subsys = {
 
 DEFINE_PER_CPU(struct device *, mce_device);
 
-void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
-
 static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
 {
 	return container_of(attr, struct mce_bank, attr);
@@ -2512,13 +2510,17 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+
 		mce_device_create(cpu);
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+
+		if (mce_threshold_create_device(cpu)) {
+			mce_device_remove(cpu);
+			return NOTIFY_BAD;
+		}
+
 		break;
 	case CPU_DEAD:
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+		mce_threshold_remove_device(cpu);
 		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index e93580c82ef0..c33a3ee2e383 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -55,6 +55,8 @@
 /* Threshold LVT offset is at MSR0xC0000410[15:12] */
 #define SMCA_THR_LVT_OFF	0xF000
 
+static bool thresholding_en;
+
 static const char * const th_names[] = {
 	"load_store",
 	"insn_fetch",
@@ -1077,10 +1079,13 @@ static void threshold_remove_bank(unsigned int cpu, int bank)
 	per_cpu(threshold_banks, cpu)[bank] = NULL;
 }
 
-static void threshold_remove_device(unsigned int cpu)
+int mce_threshold_remove_device(unsigned int cpu)
 {
 	unsigned int bank;
 
+	if (!thresholding_en)
+		return 0;
+
 	for (bank = 0; bank < mca_cfg.banks; ++bank) {
 		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 			continue;
@@ -1088,15 +1093,19 @@ static void threshold_remove_device(unsigned int cpu)
 	}
 	kfree(per_cpu(threshold_banks, cpu));
 	per_cpu(threshold_banks, cpu) = NULL;
+	return 0;
 }
 
 /* create dir/files for all valid threshold banks */
-static int threshold_create_device(unsigned int cpu)
+int mce_threshold_create_device(unsigned int cpu)
 {
 	unsigned int bank;
 	struct threshold_bank **bp;
 	int err = 0;
 
+	if (!thresholding_en)
+		return 0;
+
 	bp = per_cpu(threshold_banks, cpu);
 	if (bp)
 		return 0;
@@ -1117,40 +1126,23 @@ static int threshold_create_device(unsigned int cpu)
 	}
 	return err;
 err:
-	threshold_remove_device(cpu);
+	mce_threshold_remove_device(cpu);
 	return err;
 }
 
-/* get notified when a cpu comes on/off */
-static void
-amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
-{
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		threshold_create_device(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		threshold_remove_device(cpu);
-		break;
-	default:
-		break;
-	}
-}
-
 static __init int threshold_init_device(void)
 {
 	unsigned lcpu = 0;
 
 	/* to hit CPUs online before the notifier is up */
 	for_each_online_cpu(lcpu) {
-		int err = threshold_create_device(lcpu);
+		int err = mce_threshold_create_device(lcpu);
 
 		if (err)
 			return err;
 	}
-	threshold_cpu_callback = amd_64_threshold_cpu_callback;
+
+	thresholding_en = true;
 
 	return 0;
 }
-- 
2.10.2

[PATCH 5/7] x86/mcheck: reorganize the hotplug callbacks

[Sebastian Andrzej Siewior]

Initially I wanted to remove mcheck_cpu_init() from identify_cpu() and let it
become an independent early hotplug callback. The main problem here was that
the init on the boot CPU may happen too late
(device_initcall_sync(mcheck_init_device)) and nobody wanted to risk receiving
and MCE event at boot time leading to a shutdown (if the MCE feature is not yet
enabled).

Here is attempt two: the timming stays as-is but the ordering of the functions
is changed:
- mcheck_cpu_init() (which is run from identify_cpu()) will setup the timer
  struct but won't fire the timer. This is moved to CPU_ONLINE since its
  cleanup part is in CPU_DOWN_PREPARE. So if it is okay to stop the timer early
  in the shutdown phase, it should be okay to start it late in the bring up phase.

- CPU_DOWN_PREPARE disables the MCE feature flags for !INTEL CPUs in
  mce_disable_cpu(). If a failure occures it would be re-enabled on all vendor
  CPUs (including Intel where it was not disabled during shutdown). To keep this
  working I am moving it to CPU_ONLINE. smp_call_function_single() is dropped
  beause the notifier runs nowdays on the target CPU.

- CPU_ONLINE is invoking mce_device_create() + mce_threshold_create_device()
  but its cleanup part is in CPU_DEAD (mce_threshold_remove_device() and
  mce_device_remove()). In order to keep this symmetrical I am moving the clean
  up from CPU_DEAD to CPU_DOWN_PREPARE.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce.c | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 052b5e05c3c4..3da6fd94fa2e 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -1771,6 +1771,9 @@ void (*machine_check_vector)(struct pt_regs *, long error_code) =
  */
 void mcheck_cpu_init(struct cpuinfo_x86 *c)
 {
+	struct timer_list *t = this_cpu_ptr(&mce_timer);
+	unsigned int cpu = smp_processor_id();
+
 	if (mca_cfg.disabled)
 		return;
 
@@ -1796,7 +1799,7 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c)
 	__mcheck_cpu_init_generic();
 	__mcheck_cpu_init_vendor(c);
 	__mcheck_cpu_init_clear_banks();
-	__mcheck_cpu_init_timer();
+	setup_pinned_timer(t, mce_timer_fn, cpu);
 }
 
 /*
@@ -2470,28 +2473,25 @@ static void mce_device_remove(unsigned int cpu)
 }
 
 /* Make sure there are no machine checks on offlined CPUs. */
-static void mce_disable_cpu(void *h)
+static void mce_disable_cpu(void)
 {
-	unsigned long action = *(unsigned long *)h;
-
 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
 		return;
 
-	if (!(action & CPU_TASKS_FROZEN))
+	if (!cpuhp_tasks_frozen)
 		cmci_clear();
 
 	vendor_disable_error_reporting();
 }
 
-static void mce_reenable_cpu(void *h)
+static void mce_reenable_cpu(void)
 {
-	unsigned long action = *(unsigned long *)h;
 	int i;
 
 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
 		return;
 
-	if (!(action & CPU_TASKS_FROZEN))
+	if (!cpuhp_tasks_frozen)
 		cmci_reenable();
 	for (i = 0; i < mca_cfg.banks; i++) {
 		struct mce_bank *b = &mce_banks[i];
@@ -2510,6 +2510,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+	case CPU_DOWN_FAILED:
 
 		mce_device_create(cpu);
 
@@ -2517,11 +2518,10 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 			mce_device_remove(cpu);
 			return NOTIFY_BAD;
 		}
-
+		mce_reenable_cpu();
+		mce_start_timer(cpu, t);
 		break;
 	case CPU_DEAD:
-		mce_threshold_remove_device(cpu);
-		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
 		/* intentionally ignoring frozen here */
@@ -2529,12 +2529,11 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 			cmci_rediscover();
 		break;
 	case CPU_DOWN_PREPARE:
-		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
+		mce_disable_cpu();
 		del_timer_sync(t);
-		break;
-	case CPU_DOWN_FAILED:
-		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
-		mce_start_timer(cpu, t);
+
+		mce_threshold_remove_device(cpu);
+		mce_device_remove(cpu);
 		break;
 	}
 
-- 
2.10.2

[PATCH 6/7] x86/mcheck: Move CPU_ONLINE and CPU_DOWN_PREPARE to hotplug state machine

[Sebastian Andrzej Siewior]

The CPU_ONLINE and CPU_DOWN_PREPARE look fully symmetrical and could be move
to the hotplug state machine.
On a failure during registration we have the tear down callback invoked
(mce_cpu_pre_down()) so there should be no timer around and so no need to need
keep notifier installed (this was the reason according to the comment why the
notifier was registered despite of errors).

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce.c | 78 +++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 3da6fd94fa2e..444177cafb65 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2506,21 +2506,8 @@ static int
 mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 {
 	unsigned int cpu = (unsigned long)hcpu;
-	struct timer_list *t = &per_cpu(mce_timer, cpu);
 
 	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-
-		mce_device_create(cpu);
-
-		if (mce_threshold_create_device(cpu)) {
-			mce_device_remove(cpu);
-			return NOTIFY_BAD;
-		}
-		mce_reenable_cpu();
-		mce_start_timer(cpu, t);
-		break;
 	case CPU_DEAD:
 		mce_intel_hcpu_update(cpu);
 
@@ -2529,17 +2516,41 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 			cmci_rediscover();
 		break;
 	case CPU_DOWN_PREPARE:
-		mce_disable_cpu();
-		del_timer_sync(t);
 
-		mce_threshold_remove_device(cpu);
-		mce_device_remove(cpu);
 		break;
 	}
 
 	return NOTIFY_OK;
 }
 
+static int mce_cpu_online(unsigned int cpu)
+{
+	struct timer_list *t = &per_cpu(mce_timer, cpu);
+	int ret;
+
+	mce_device_create(cpu);
+
+	ret = mce_threshold_create_device(cpu);
+	if (ret) {
+		mce_device_remove(cpu);
+		return ret;
+	}
+	mce_reenable_cpu();
+	mce_start_timer(cpu, t);
+	return 0;
+}
+
+static int mce_cpu_pre_down(unsigned int cpu)
+{
+	struct timer_list *t = &per_cpu(mce_timer, cpu);
+
+	mce_disable_cpu();
+	del_timer_sync(t);
+	mce_threshold_remove_device(cpu);
+	mce_device_remove(cpu);
+	return 0;
+}
+
 static struct notifier_block mce_cpu_notifier = {
 	.notifier_call = mce_cpu_callback,
 };
@@ -2564,8 +2575,8 @@ static __init void mce_init_banks(void)
 
 static __init int mcheck_init_device(void)
 {
+	enum cpuhp_state hp_online;
 	int err;
-	int i = 0;
 
 	if (!mce_available(&boot_cpu_data)) {
 		err = -EIO;
@@ -2583,21 +2594,13 @@ static __init int mcheck_init_device(void)
 	if (err)
 		goto err_out_mem;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = mce_device_create(i);
-		if (err) {
-			/*
-			 * Register notifier anyway (and do not unreg it) so
-			 * that we don't leave undeleted timers, see notifier
-			 * callback above.
-			 */
-			__register_hotcpu_notifier(&mce_cpu_notifier);
-			cpu_notifier_register_done();
-			goto err_device_create;
-		}
-	}
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
+				mce_cpu_online, mce_cpu_pre_down);
+	if (err < 0)
+		goto err_out_mem;
+	hp_online = err;
 
+	cpu_notifier_register_begin();
 	__register_hotcpu_notifier(&mce_cpu_notifier);
 	cpu_notifier_register_done();
 
@@ -2612,16 +2615,7 @@ static __init int mcheck_init_device(void)
 
 err_register:
 	unregister_syscore_ops(&mce_syscore_ops);
-
-err_device_create:
-	/*
-	 * We didn't keep track of which devices were created above, but
-	 * even if we had, the set of online cpus might have changed.
-	 * Play safe and remove for every possible cpu, since
-	 * mce_device_remove() will do the right thing.
-	 */
-	for_each_possible_cpu(i)
-		mce_device_remove(i);
+	cpuhp_remove_state(hp_online);
 
 err_out_mem:
 	free_cpumask_var(mce_device_initialized);
-- 
2.10.2

[PATCH 7/7] x86/mcheck: Move CPU_DEAD to hotplug state machine

[Sebastian Andrzej Siewior]

This moves the last piece of the old hotplug notifier code in MCE to the
new hotplug state machine.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/x86/kernel/cpu/mcheck/mce.c | 42 ++++++++++++++--------------------------
 include/linux/cpuhotplug.h       |  1 +
 2 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 444177cafb65..d31e3a7d8866 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2501,26 +2501,14 @@ static void mce_reenable_cpu(void)
 	}
 }
 
-/* Get notified when a cpu comes on/off. Be hotplug friendly. */
-static int
-mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
+static int mce_cpu_dead(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
+	mce_intel_hcpu_update(cpu);
 
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DEAD:
-		mce_intel_hcpu_update(cpu);
-
-		/* intentionally ignoring frozen here */
-		if (!(action & CPU_TASKS_FROZEN))
-			cmci_rediscover();
-		break;
-	case CPU_DOWN_PREPARE:
-
-		break;
-	}
-
-	return NOTIFY_OK;
+	/* intentionally ignoring frozen here */
+	if (!cpuhp_tasks_frozen)
+		cmci_rediscover();
+	return 0;
 }
 
 static int mce_cpu_online(unsigned int cpu)
@@ -2551,10 +2539,6 @@ static int mce_cpu_pre_down(unsigned int cpu)
 	return 0;
 }
 
-static struct notifier_block mce_cpu_notifier = {
-	.notifier_call = mce_cpu_callback,
-};
-
 static __init void mce_init_banks(void)
 {
 	int i;
@@ -2594,16 +2578,17 @@ static __init int mcheck_init_device(void)
 	if (err)
 		goto err_out_mem;
 
+	err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
+				mce_cpu_dead);
+	if (err)
+		goto err_out_mem;
+
 	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
 				mce_cpu_online, mce_cpu_pre_down);
 	if (err < 0)
-		goto err_out_mem;
+		goto err_out_online;
 	hp_online = err;
 
-	cpu_notifier_register_begin();
-	__register_hotcpu_notifier(&mce_cpu_notifier);
-	cpu_notifier_register_done();
-
 	register_syscore_ops(&mce_syscore_ops);
 
 	/* register character device /dev/mcelog */
@@ -2617,6 +2602,9 @@ static __init int mcheck_init_device(void)
 	unregister_syscore_ops(&mce_syscore_ops);
 	cpuhp_remove_state(hp_online);
 
+err_out_online:
+	cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
+
 err_out_mem:
 	free_cpumask_var(mce_device_initialized);
 
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 3410d83cc2e2..79b96f647d64 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -16,6 +16,7 @@ enum cpuhp_state {
 	CPUHP_PERF_SUPERH,
 	CPUHP_X86_HPET_DEAD,
 	CPUHP_X86_APB_DEAD,
+	CPUHP_X86_MCE_DEAD,
 	CPUHP_VIRT_NET_DEAD,
 	CPUHP_SLUB_DEAD,
 	CPUHP_MM_WRITEBACK_DEAD,
-- 
2.10.2

Re: [PATCH 5/7] x86/mcheck: reorganize the hotplug callbacks

[Borislav Petkov]

On Thu, Nov 10, 2016 at 06:44:45PM +0100, Sebastian Andrzej Siewior wrote:
> Initially I wanted to remove mcheck_cpu_init() from identify_cpu() and let it
> become an independent early hotplug callback. The main problem here was that
> the init on the boot CPU may happen too late
> (device_initcall_sync(mcheck_init_device)) and nobody wanted to risk receiving
> and MCE event at boot time leading to a shutdown (if the MCE feature is not yet
> enabled).
> 
> Here is attempt two: the timming stays as-is but the ordering of the functions

			   timing

> is changed:
> - mcheck_cpu_init() (which is run from identify_cpu()) will setup the timer
>   struct but won't fire the timer. This is moved to CPU_ONLINE since its
>   cleanup part is in CPU_DOWN_PREPARE. So if it is okay to stop the timer early
>   in the shutdown phase, it should be okay to start it late in the bring up phase.
> 
> - CPU_DOWN_PREPARE disables the MCE feature flags for !INTEL CPUs in

		     disables the MCE error reporting...

>   mce_disable_cpu(). If a failure occures it would be re-enabled on all vendor

				    occurs

>   CPUs (including Intel where it was not disabled during shutdown). To keep this
>   working I am moving it to CPU_ONLINE. smp_call_function_single() is dropped
>   beause the notifier runs nowdays on the target CPU.

"... because the notifier runs on the target CPU now."

Please run your commit messages text through a spellchecker.

> - CPU_ONLINE is invoking mce_device_create() + mce_threshold_create_device()
>   but its cleanup part is in CPU_DEAD (mce_threshold_remove_device() and
>   mce_device_remove()). In order to keep this symmetrical I am moving the clean
>   up from CPU_DEAD to CPU_DOWN_PREPARE.

   cleanup

> 
> Cc: Tony Luck <tony.luck@intel.com>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: linux-edac@vger.kernel.org
> Cc: x86@kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> ---
>  arch/x86/kernel/cpu/mcheck/mce.c | 31 +++++++++++++++----------------
>  1 file changed, 15 insertions(+), 16 deletions(-)
> 
> diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
> index 052b5e05c3c4..3da6fd94fa2e 100644
> --- a/arch/x86/kernel/cpu/mcheck/mce.c
> +++ b/arch/x86/kernel/cpu/mcheck/mce.c
> @@ -1771,6 +1771,9 @@ void (*machine_check_vector)(struct pt_regs *, long error_code) =
>   */
>  void mcheck_cpu_init(struct cpuinfo_x86 *c)
>  {
> +	struct timer_list *t = this_cpu_ptr(&mce_timer);
> +	unsigned int cpu = smp_processor_id();
> +
>  	if (mca_cfg.disabled)
>  		return;
>  
> @@ -1796,7 +1799,7 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c)
>  	__mcheck_cpu_init_generic();
>  	__mcheck_cpu_init_vendor(c);
>  	__mcheck_cpu_init_clear_banks();
> -	__mcheck_cpu_init_timer();
> +	setup_pinned_timer(t, mce_timer_fn, cpu);

Why not leave all that setup stuff in __mcheck_cpu_init_timer() ?

...

> @@ -2517,11 +2518,10 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
>  			mce_device_remove(cpu);
>  			return NOTIFY_BAD;
>  		}
> -
> +		mce_reenable_cpu();
> +		mce_start_timer(cpu, t);
>  		break;
>  	case CPU_DEAD:
> -		mce_threshold_remove_device(cpu);
> -		mce_device_remove(cpu);
>  		mce_intel_hcpu_update(cpu);
>  
>  		/* intentionally ignoring frozen here */

There's another place for cpuhp_tasks_frozen replacement here:

                /* intentionally ignoring frozen here */
                if (!(action & CPU_TASKS_FROZEN))
                        cmci_rediscover();

into

		if (!cpuhp_tasks_frozen)
			cmci_rediscover();

> @@ -2529,12 +2529,11 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
>  			cmci_rediscover();
>  		break;
>  	case CPU_DOWN_PREPARE:
> -		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
> +		mce_disable_cpu();
>  		del_timer_sync(t);
> -		break;
> -	case CPU_DOWN_FAILED:
> -		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
> -		mce_start_timer(cpu, t);
> +
> +		mce_threshold_remove_device(cpu);
> +		mce_device_remove(cpu);
>  		break;
>  	}
>  
> -- 
> 2.10.2
> 
> 

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 5/7] x86/mcheck: reorganize the hotplug callbacks

[Sebastian Andrzej Siewior]

On 2016-11-11 19:44:53 [+0100], Borislav Petkov wrote:
> > ---
> >  arch/x86/kernel/cpu/mcheck/mce.c | 31 +++++++++++++++----------------
> >  1 file changed, 15 insertions(+), 16 deletions(-)
> > 
> > diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
> > index 052b5e05c3c4..3da6fd94fa2e 100644
> > --- a/arch/x86/kernel/cpu/mcheck/mce.c
> > +++ b/arch/x86/kernel/cpu/mcheck/mce.c
> > @@ -1771,6 +1771,9 @@ void (*machine_check_vector)(struct pt_regs *, long error_code) =
> >   */
> >  void mcheck_cpu_init(struct cpuinfo_x86 *c)
> >  {
> > +	struct timer_list *t = this_cpu_ptr(&mce_timer);
> > +	unsigned int cpu = smp_processor_id();
> > +
> >  	if (mca_cfg.disabled)
> >  		return;
> >  
> > @@ -1796,7 +1799,7 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c)
> >  	__mcheck_cpu_init_generic();
> >  	__mcheck_cpu_init_vendor(c);
> >  	__mcheck_cpu_init_clear_banks();
> > -	__mcheck_cpu_init_timer();
> > +	setup_pinned_timer(t, mce_timer_fn, cpu);
> 
> Why not leave all that setup stuff in __mcheck_cpu_init_timer() ?
> 
__mcheck_cpu_init_timer() does not only prepare the timer but also fires
it. And __mcheck_cpu_init_timer() is invoked from other places and the
caller probably expects the timer to be fired.

> > @@ -2517,11 +2518,10 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
> >  			mce_device_remove(cpu);
> >  			return NOTIFY_BAD;
> >  		}
> > -
> > +		mce_reenable_cpu();
> > +		mce_start_timer(cpu, t);
> >  		break;
> >  	case CPU_DEAD:
> > -		mce_threshold_remove_device(cpu);
> > -		mce_device_remove(cpu);
> >  		mce_intel_hcpu_update(cpu);
> >  
> >  		/* intentionally ignoring frozen here */
> 
> There's another place for cpuhp_tasks_frozen replacement here:
> 
>                 /* intentionally ignoring frozen here */
>                 if (!(action & CPU_TASKS_FROZEN))
>                         cmci_rediscover();
> 
> into
> 
> 		if (!cpuhp_tasks_frozen)
> 			cmci_rediscover();

Correct. This is done later since it still works. For functions like
mce_disable_cpu() which were converted here I replaced the check so I
keep them void and don't care about the argument.

Sebastian

Re: [PATCH 5/7] x86/mcheck: reorganize the hotplug callbacks

[Borislav Petkov]

On Fri, Nov 11, 2016 at 08:36:31PM +0100, Sebastian Andrzej Siewior wrote:
> __mcheck_cpu_init_timer() does not only prepare the timer but also fires
> it. And __mcheck_cpu_init_timer() is invoked from other places and the
> caller probably expects the timer to be fired.

Then please do it right:

static void __mcheck_cap_setup_timer(void)
{
        struct timer_list *t = this_cpu_ptr(&mce_timer);
        unsigned int cpu = smp_processor_id();

        setup_pinned_timer(t, mce_timer_fn, cpu);
}

and call that function then.

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

Re: [PATCH 7/7] x86/mcheck: Move CPU_DEAD to hotplug state machine

[Borislav Petkov]

On Thu, Nov 10, 2016 at 06:44:47PM +0100, Sebastian Andrzej Siewior wrote:
> This moves the last piece of the old hotplug notifier code in MCE to the
> new hotplug state machine.
> 
> Cc: Tony Luck <tony.luck@intel.com>
> Cc: Borislav Petkov <bp@alien8.de>
> Cc: linux-edac@vger.kernel.org
> Cc: x86@kernel.org
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> ---
>  arch/x86/kernel/cpu/mcheck/mce.c | 42 ++++++++++++++--------------------------
>  include/linux/cpuhotplug.h       |  1 +
>  2 files changed, 16 insertions(+), 27 deletions(-)

Rest looks ok.

Acked-by: Borislav Petkov <bp@suse.de>

Thanks!

-- 
Regards/Gruss,
    Boris.

Good mailing practices for 400: avoid top-posting and trim the reply.

[PATCH 5/7 v2] x86/mcheck: reorganize the hotplug callbacks

[Sebastian Andrzej Siewior]

Initially I wanted to remove mcheck_cpu_init() from identify_cpu() and let it
become an independent early hotplug callback. The main problem here was that
the init on the boot CPU may happen too late
(device_initcall_sync(mcheck_init_device)) and nobody wanted to risk receiving
and MCE event at boot time leading to a shutdown (if the MCE feature is not yet
enabled).

Here is attempt two: the timing stays as-is but the ordering of the functions
is changed:
- mcheck_cpu_init() (which is run from identify_cpu()) will setup the timer
  struct but won't fire the timer. This is moved to CPU_ONLINE since its
  cleanup part is in CPU_DOWN_PREPARE. So if it is okay to stop the timer early
  in the shutdown phase, it should be okay to start it late in the bring up phase.

- CPU_DOWN_PREPARE disables the MCE feature flags for !INTEL CPUs in
  mce_disable_cpu(). If a failure occurs it would be re-enabled on all vendor
  CPUs (including Intel where it was not disabled during shutdown). To keep this
  working I am moving it to CPU_ONLINE. smp_call_function_single() is dropped
  because the notifier runs on the target CPU (since core code rework).

- CPU_ONLINE is invoking mce_device_create() + mce_threshold_create_device()
  but its cleanup part is in CPU_DEAD (mce_threshold_remove_device() and
  mce_device_remove()). In order to keep this symmetrical I am moving the clean
  up from CPU_DEAD to CPU_DOWN_PREPARE.

Cc: Tony Luck <tony.luck@intel.com>
Cc: Borislav Petkov <bp@alien8.de>
Cc: linux-edac@vger.kernel.org
Cc: x86@kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
On 2016-11-11 20:57:37 [+0100], Borislav Petkov wrote:
> Then please do it right:
> 
> static void __mcheck_cap_setup_timer(void)
> {
>         struct timer_list *t = this_cpu_ptr(&mce_timer);
>         unsigned int cpu = smp_processor_id();
> 
>         setup_pinned_timer(t, mce_timer_fn, cpu);
> }
> 
> and call that function then.

I can't believe that you ask for this but here it is.

 arch/x86/kernel/cpu/mcheck/mce.c |   36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -1745,6 +1745,14 @@ static void mce_start_timer(unsigned int
 	add_timer_on(t, cpu);
 }
 
+static void __mcheck_cap_setup_timer(void)
+{
+	struct timer_list *t = this_cpu_ptr(&mce_timer);
+	unsigned int cpu = smp_processor_id();
+
+	setup_pinned_timer(t, mce_timer_fn, cpu);
+}
+
 static void __mcheck_cpu_init_timer(void)
 {
 	struct timer_list *t = this_cpu_ptr(&mce_timer);
@@ -1796,7 +1804,7 @@ void mcheck_cpu_init(struct cpuinfo_x86
 	__mcheck_cpu_init_generic();
 	__mcheck_cpu_init_vendor(c);
 	__mcheck_cpu_init_clear_banks();
-	__mcheck_cpu_init_timer();
+	__mcheck_cap_setup_timer();
 }
 
 /*
@@ -2470,28 +2478,25 @@ static void mce_device_remove(unsigned i
 }
 
 /* Make sure there are no machine checks on offlined CPUs. */
-static void mce_disable_cpu(void *h)
+static void mce_disable_cpu(void)
 {
-	unsigned long action = *(unsigned long *)h;
-
 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
 		return;
 
-	if (!(action & CPU_TASKS_FROZEN))
+	if (!cpuhp_tasks_frozen)
 		cmci_clear();
 
 	vendor_disable_error_reporting();
 }
 
-static void mce_reenable_cpu(void *h)
+static void mce_reenable_cpu(void)
 {
-	unsigned long action = *(unsigned long *)h;
 	int i;
 
 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
 		return;
 
-	if (!(action & CPU_TASKS_FROZEN))
+	if (!cpuhp_tasks_frozen)
 		cmci_reenable();
 	for (i = 0; i < mca_cfg.banks; i++) {
 		struct mce_bank *b = &mce_banks[i];
@@ -2510,6 +2515,7 @@ mce_cpu_callback(struct notifier_block *
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+	case CPU_DOWN_FAILED:
 
 		mce_device_create(cpu);
 
@@ -2517,11 +2523,10 @@ mce_cpu_callback(struct notifier_block *
 			mce_device_remove(cpu);
 			return NOTIFY_BAD;
 		}
-
+		mce_reenable_cpu();
+		mce_start_timer(cpu, t);
 		break;
 	case CPU_DEAD:
-		mce_threshold_remove_device(cpu);
-		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
 		/* intentionally ignoring frozen here */
@@ -2529,12 +2534,11 @@ mce_cpu_callback(struct notifier_block *
 			cmci_rediscover();
 		break;
 	case CPU_DOWN_PREPARE:
-		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
+		mce_disable_cpu();
 		del_timer_sync(t);
-		break;
-	case CPU_DOWN_FAILED:
-		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
-		mce_start_timer(cpu, t);
+
+		mce_threshold_remove_device(cpu);
+		mce_device_remove(cpu);
 		break;
 	}
 

[tip:smp/hotplug] x86/mcheck: Move threshold_create_device()

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  0943637293a05aa3d8fb47bc30c7c0104480829a
Gitweb:     http://git.kernel.org/tip/0943637293a05aa3d8fb47bc30c7c0104480829a
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 10 Nov 2016 18:44:41 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 16 Nov 2016 09:34:16 +0100

x86/mcheck: Move threshold_create_device()

Move the threshold_create_device() so it can use
threshold_remove_device() without a forward declaration.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linutronix.de
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161110174447.11848-2-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 50 ++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 25 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 9b54034..75a3e3e 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1010,31 +1010,6 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank)
 	return err;
 }
 
-/* create dir/files for all valid threshold banks */
-static int threshold_create_device(unsigned int cpu)
-{
-	unsigned int bank;
-	struct threshold_bank **bp;
-	int err = 0;
-
-	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
-		     GFP_KERNEL);
-	if (!bp)
-		return -ENOMEM;
-
-	per_cpu(threshold_banks, cpu) = bp;
-
-	for (bank = 0; bank < mca_cfg.banks; ++bank) {
-		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
-			continue;
-		err = threshold_create_bank(cpu, bank);
-		if (err)
-			return err;
-	}
-
-	return err;
-}
-
 static void deallocate_threshold_block(unsigned int cpu,
 						 unsigned int bank)
 {
@@ -1114,6 +1089,31 @@ static void threshold_remove_device(unsigned int cpu)
 	kfree(per_cpu(threshold_banks, cpu));
 }
 
+/* create dir/files for all valid threshold banks */
+static int threshold_create_device(unsigned int cpu)
+{
+	unsigned int bank;
+	struct threshold_bank **bp;
+	int err = 0;
+
+	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
+		     GFP_KERNEL);
+	if (!bp)
+		return -ENOMEM;
+
+	per_cpu(threshold_banks, cpu) = bp;
+
+	for (bank = 0; bank < mca_cfg.banks; ++bank) {
+		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
+			continue;
+		err = threshold_create_bank(cpu, bank);
+		if (err)
+			return err;
+	}
+
+	return err;
+}
+
 /* get notified when a cpu comes on/off */
 static void
 amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)

[tip:smp/hotplug] x86/mcheck: Explicit cleanup on failure in mce_amd

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  ec553abb318d75b1eafd275c71125569fa13b33b
Gitweb:     http://git.kernel.org/tip/ec553abb318d75b1eafd275c71125569fa13b33b
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 10 Nov 2016 18:44:42 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 16 Nov 2016 09:34:17 +0100

x86/mcheck: Explicit cleanup on failure in mce_amd

If the ONLINE callback fails, the driver does not any clean up right
away instead it waits to get to the DEAD stage to do it. Yes, it waits.
Since we don't pass the error code back to the caller, no one knows.

Do the clean up right away so it does not look like a leak.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linutronix.de
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161110174447.11848-3-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 75a3e3e..55cd018 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1087,6 +1087,7 @@ static void threshold_remove_device(unsigned int cpu)
 		threshold_remove_bank(cpu, bank);
 	}
 	kfree(per_cpu(threshold_banks, cpu));
+	per_cpu(threshold_banks, cpu) = NULL;
 }
 
 /* create dir/files for all valid threshold banks */
@@ -1108,9 +1109,11 @@ static int threshold_create_device(unsigned int cpu)
 			continue;
 		err = threshold_create_bank(cpu, bank);
 		if (err)
-			return err;
+			goto err;
 	}
-
+	return err;
+err:
+	threshold_remove_device(cpu);
 	return err;
 }
 

[tip:smp/hotplug] x86/mcheck: Be prepared for a rollback back to the ONLINE state

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  7f34b935e8bf2f5304fce273a8fa98c63886d686
Gitweb:     http://git.kernel.org/tip/7f34b935e8bf2f5304fce273a8fa98c63886d686
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 10 Nov 2016 18:44:43 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 16 Nov 2016 09:34:17 +0100

x86/mcheck: Be prepared for a rollback back to the ONLINE state

If we try a CPU down and fail in the middle then we roll back to the
online state. This means we would perform CPU_ONLINE / mce_device_create()
without invoking CPU_DEAD / mce_device_remove() for the cleanup of what was
allocated in CPU_ONLINE.

Be prepared for this and don't allocate the struct if we have it
already.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linutronix.de
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161110174447.11848-4-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpu/mcheck/mce.c     | 4 ++++
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index a7fdf45..e9ffd6d 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2409,6 +2409,10 @@ static int mce_device_create(unsigned int cpu)
 	if (!mce_available(&boot_cpu_data))
 		return -EIO;
 
+	dev = per_cpu(mce_device, cpu);
+	if (dev)
+		return 0;
+
 	dev = kzalloc(sizeof *dev, GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index 55cd018..e93580c 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -1097,6 +1097,10 @@ static int threshold_create_device(unsigned int cpu)
 	struct threshold_bank **bp;
 	int err = 0;
 
+	bp = per_cpu(threshold_banks, cpu);
+	if (bp)
+		return 0;
+
 	bp = kzalloc(sizeof(struct threshold_bank *) * mca_cfg.banks,
 		     GFP_KERNEL);
 	if (!bp)

[tip:smp/hotplug] x86/mcheck: Split threshold_cpu_callback into two callbacks

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  4d7b02d58c4000597d08930193d7aed81fba6b7c
Gitweb:     http://git.kernel.org/tip/4d7b02d58c4000597d08930193d7aed81fba6b7c
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 10 Nov 2016 18:44:44 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 16 Nov 2016 09:34:17 +0100

x86/mcheck: Split threshold_cpu_callback into two callbacks

The threshold_cpu_callback callbacks looks like one of the notifier and
its arguments are almost the same. Split this out and have one ONLINE
and one DEAD callback. This will come handy later once the main code
gets changed to use the callback mechanism.
Also, handle threshold_cpu_callback_online() return value so we don't
continue if the function fails.

Boris Petkov removed the callback pointer and replaced it with proper
functions.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linutronix.de
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161110174447.11848-5-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/include/asm/mce.h           |  7 +++++--
 arch/x86/kernel/cpu/mcheck/mce.c     | 14 +++++++------
 arch/x86/kernel/cpu/mcheck/mce_amd.c | 38 ++++++++++++++----------------------
 3 files changed, 28 insertions(+), 31 deletions(-)

diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index 9bd7ff5..caafad4 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -293,9 +293,7 @@ void do_machine_check(struct pt_regs *, long);
 /*
  * Threshold handler
  */
-
 extern void (*mce_threshold_vector)(void);
-extern void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
 
 /* Deferred error interrupt handler */
 extern void (*deferred_error_int_vector)(void);
@@ -377,7 +375,12 @@ struct smca_bank_info {
 };
 
 extern struct smca_bank_info smca_banks[MAX_NR_BANKS];
+extern int mce_threshold_create_device(unsigned int cpu);
+extern int mce_threshold_remove_device(unsigned int cpu);
+#else
 
+static inline int mce_threshold_create_device(unsigned int cpu) { return 0; };
+static inline int mce_threshold_remove_device(unsigned int cpu) { return 0; };
 #endif
 
 #endif /* _ASM_X86_MCE_H */
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index e9ffd6d..052b5e0 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2255,8 +2255,6 @@ static struct bus_type mce_subsys = {
 
 DEFINE_PER_CPU(struct device *, mce_device);
 
-void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
-
 static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
 {
 	return container_of(attr, struct mce_bank, attr);
@@ -2512,13 +2510,17 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+
 		mce_device_create(cpu);
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+
+		if (mce_threshold_create_device(cpu)) {
+			mce_device_remove(cpu);
+			return NOTIFY_BAD;
+		}
+
 		break;
 	case CPU_DEAD:
-		if (threshold_cpu_callback)
-			threshold_cpu_callback(action, cpu);
+		mce_threshold_remove_device(cpu);
 		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd.c b/arch/x86/kernel/cpu/mcheck/mce_amd.c
index e93580c..c33a3ee 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_amd.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_amd.c
@@ -55,6 +55,8 @@
 /* Threshold LVT offset is at MSR0xC0000410[15:12] */
 #define SMCA_THR_LVT_OFF	0xF000
 
+static bool thresholding_en;
+
 static const char * const th_names[] = {
 	"load_store",
 	"insn_fetch",
@@ -1077,10 +1079,13 @@ free_out:
 	per_cpu(threshold_banks, cpu)[bank] = NULL;
 }
 
-static void threshold_remove_device(unsigned int cpu)
+int mce_threshold_remove_device(unsigned int cpu)
 {
 	unsigned int bank;
 
+	if (!thresholding_en)
+		return 0;
+
 	for (bank = 0; bank < mca_cfg.banks; ++bank) {
 		if (!(per_cpu(bank_map, cpu) & (1 << bank)))
 			continue;
@@ -1088,15 +1093,19 @@ static void threshold_remove_device(unsigned int cpu)
 	}
 	kfree(per_cpu(threshold_banks, cpu));
 	per_cpu(threshold_banks, cpu) = NULL;
+	return 0;
 }
 
 /* create dir/files for all valid threshold banks */
-static int threshold_create_device(unsigned int cpu)
+int mce_threshold_create_device(unsigned int cpu)
 {
 	unsigned int bank;
 	struct threshold_bank **bp;
 	int err = 0;
 
+	if (!thresholding_en)
+		return 0;
+
 	bp = per_cpu(threshold_banks, cpu);
 	if (bp)
 		return 0;
@@ -1117,40 +1126,23 @@ static int threshold_create_device(unsigned int cpu)
 	}
 	return err;
 err:
-	threshold_remove_device(cpu);
+	mce_threshold_remove_device(cpu);
 	return err;
 }
 
-/* get notified when a cpu comes on/off */
-static void
-amd_64_threshold_cpu_callback(unsigned long action, unsigned int cpu)
-{
-	switch (action) {
-	case CPU_ONLINE:
-	case CPU_ONLINE_FROZEN:
-		threshold_create_device(cpu);
-		break;
-	case CPU_DEAD:
-	case CPU_DEAD_FROZEN:
-		threshold_remove_device(cpu);
-		break;
-	default:
-		break;
-	}
-}
-
 static __init int threshold_init_device(void)
 {
 	unsigned lcpu = 0;
 
 	/* to hit CPUs online before the notifier is up */
 	for_each_online_cpu(lcpu) {
-		int err = threshold_create_device(lcpu);
+		int err = mce_threshold_create_device(lcpu);
 
 		if (err)
 			return err;
 	}
-	threshold_cpu_callback = amd_64_threshold_cpu_callback;
+
+	thresholding_en = true;
 
 	return 0;
 }

[tip:smp/hotplug] x86/mcheck: Reorganize the hotplug callbacks

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  39f152ffbfedb42b57b6e0c896eeae51dbe83b7a
Gitweb:     http://git.kernel.org/tip/39f152ffbfedb42b57b6e0c896eeae51dbe83b7a
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 10 Nov 2016 18:44:45 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 16 Nov 2016 09:34:18 +0100

x86/mcheck: Reorganize the hotplug callbacks

Initially I wanted to remove mcheck_cpu_init() from identify_cpu() and let it
become an independent early hotplug callback. The main problem here was that
the init on the boot CPU may happen too late
(device_initcall_sync(mcheck_init_device)) and nobody wanted to risk receiving
and MCE event at boot time leading to a shutdown (if the MCE feature is not yet
enabled).

Here is attempt two: the timming stays as-is but the ordering of the functions
is changed:
- mcheck_cpu_init() (which is run from identify_cpu()) will setup the timer
  struct but won't fire the timer. This is moved to CPU_ONLINE since its
  cleanup part is in CPU_DOWN_PREPARE. So if it is okay to stop the timer early
  in the shutdown phase, it should be okay to start it late in the bring up phase.

- CPU_DOWN_PREPARE disables the MCE feature flags for !INTEL CPUs in
  mce_disable_cpu(). If a failure occures it would be re-enabled on all vendor
  CPUs (including Intel where it was not disabled during shutdown). To keep this
  working I am moving it to CPU_ONLINE. smp_call_function_single() is dropped
  beause the notifier runs nowdays on the target CPU.

- CPU_ONLINE is invoking mce_device_create() + mce_threshold_create_device()
  but its cleanup part is in CPU_DEAD (mce_threshold_remove_device() and
  mce_device_remove()). In order to keep this symmetrical I am moving the clean
  up from CPU_DEAD to CPU_DOWN_PREPARE.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linutronix.de
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161110174447.11848-6-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpu/mcheck/mce.c | 36 ++++++++++++++++++++----------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 052b5e0..a524faa 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -1745,6 +1745,14 @@ static void mce_start_timer(unsigned int cpu, struct timer_list *t)
 	add_timer_on(t, cpu);
 }
 
+static void __mcheck_cpu_setup_timer(void)
+{
+	struct timer_list *t = this_cpu_ptr(&mce_timer);
+	unsigned int cpu = smp_processor_id();
+
+	setup_pinned_timer(t, mce_timer_fn, cpu);
+}
+
 static void __mcheck_cpu_init_timer(void)
 {
 	struct timer_list *t = this_cpu_ptr(&mce_timer);
@@ -1796,7 +1804,7 @@ void mcheck_cpu_init(struct cpuinfo_x86 *c)
 	__mcheck_cpu_init_generic();
 	__mcheck_cpu_init_vendor(c);
 	__mcheck_cpu_init_clear_banks();
-	__mcheck_cpu_init_timer();
+	__mcheck_cpu_setup_timer();
 }
 
 /*
@@ -2470,28 +2478,25 @@ static void mce_device_remove(unsigned int cpu)
 }
 
 /* Make sure there are no machine checks on offlined CPUs. */
-static void mce_disable_cpu(void *h)
+static void mce_disable_cpu(void)
 {
-	unsigned long action = *(unsigned long *)h;
-
 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
 		return;
 
-	if (!(action & CPU_TASKS_FROZEN))
+	if (!cpuhp_tasks_frozen)
 		cmci_clear();
 
 	vendor_disable_error_reporting();
 }
 
-static void mce_reenable_cpu(void *h)
+static void mce_reenable_cpu(void)
 {
-	unsigned long action = *(unsigned long *)h;
 	int i;
 
 	if (!mce_available(raw_cpu_ptr(&cpu_info)))
 		return;
 
-	if (!(action & CPU_TASKS_FROZEN))
+	if (!cpuhp_tasks_frozen)
 		cmci_reenable();
 	for (i = 0; i < mca_cfg.banks; i++) {
 		struct mce_bank *b = &mce_banks[i];
@@ -2510,6 +2515,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 
 	switch (action & ~CPU_TASKS_FROZEN) {
 	case CPU_ONLINE:
+	case CPU_DOWN_FAILED:
 
 		mce_device_create(cpu);
 
@@ -2517,11 +2523,10 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 			mce_device_remove(cpu);
 			return NOTIFY_BAD;
 		}
-
+		mce_reenable_cpu();
+		mce_start_timer(cpu, t);
 		break;
 	case CPU_DEAD:
-		mce_threshold_remove_device(cpu);
-		mce_device_remove(cpu);
 		mce_intel_hcpu_update(cpu);
 
 		/* intentionally ignoring frozen here */
@@ -2529,12 +2534,11 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 			cmci_rediscover();
 		break;
 	case CPU_DOWN_PREPARE:
-		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
+		mce_disable_cpu();
 		del_timer_sync(t);
-		break;
-	case CPU_DOWN_FAILED:
-		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
-		mce_start_timer(cpu, t);
+
+		mce_threshold_remove_device(cpu);
+		mce_device_remove(cpu);
 		break;
 	}
 

[tip:smp/hotplug] x86/mcheck: Move CPU_ONLINE and CPU_DOWN_PREPARE to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  8c0eeac819c85e4c1143f7a874d87b4594739208
Gitweb:     http://git.kernel.org/tip/8c0eeac819c85e4c1143f7a874d87b4594739208
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 10 Nov 2016 18:44:46 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 16 Nov 2016 09:34:18 +0100

x86/mcheck: Move CPU_ONLINE and CPU_DOWN_PREPARE to hotplug state machine

The CPU_ONLINE and CPU_DOWN_PREPARE look fully symmetrical and could be move
to the hotplug state machine.
On a failure during registration we have the tear down callback invoked
(mce_cpu_pre_down()) so there should be no timer around and so no need to need
keep notifier installed (this was the reason according to the comment why the
notifier was registered despite of errors).

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linutronix.de
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161110174447.11848-7-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpu/mcheck/mce.c | 78 +++++++++++++++++++---------------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index a524faa..78955f5 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2511,21 +2511,8 @@ static int
 mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 {
 	unsigned int cpu = (unsigned long)hcpu;
-	struct timer_list *t = &per_cpu(mce_timer, cpu);
 
 	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_ONLINE:
-	case CPU_DOWN_FAILED:
-
-		mce_device_create(cpu);
-
-		if (mce_threshold_create_device(cpu)) {
-			mce_device_remove(cpu);
-			return NOTIFY_BAD;
-		}
-		mce_reenable_cpu();
-		mce_start_timer(cpu, t);
-		break;
 	case CPU_DEAD:
 		mce_intel_hcpu_update(cpu);
 
@@ -2534,17 +2521,41 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
 			cmci_rediscover();
 		break;
 	case CPU_DOWN_PREPARE:
-		mce_disable_cpu();
-		del_timer_sync(t);
 
-		mce_threshold_remove_device(cpu);
-		mce_device_remove(cpu);
 		break;
 	}
 
 	return NOTIFY_OK;
 }
 
+static int mce_cpu_online(unsigned int cpu)
+{
+	struct timer_list *t = &per_cpu(mce_timer, cpu);
+	int ret;
+
+	mce_device_create(cpu);
+
+	ret = mce_threshold_create_device(cpu);
+	if (ret) {
+		mce_device_remove(cpu);
+		return ret;
+	}
+	mce_reenable_cpu();
+	mce_start_timer(cpu, t);
+	return 0;
+}
+
+static int mce_cpu_pre_down(unsigned int cpu)
+{
+	struct timer_list *t = &per_cpu(mce_timer, cpu);
+
+	mce_disable_cpu();
+	del_timer_sync(t);
+	mce_threshold_remove_device(cpu);
+	mce_device_remove(cpu);
+	return 0;
+}
+
 static struct notifier_block mce_cpu_notifier = {
 	.notifier_call = mce_cpu_callback,
 };
@@ -2569,8 +2580,8 @@ static __init void mce_init_banks(void)
 
 static __init int mcheck_init_device(void)
 {
+	enum cpuhp_state hp_online;
 	int err;
-	int i = 0;
 
 	if (!mce_available(&boot_cpu_data)) {
 		err = -EIO;
@@ -2588,21 +2599,13 @@ static __init int mcheck_init_device(void)
 	if (err)
 		goto err_out_mem;
 
-	cpu_notifier_register_begin();
-	for_each_online_cpu(i) {
-		err = mce_device_create(i);
-		if (err) {
-			/*
-			 * Register notifier anyway (and do not unreg it) so
-			 * that we don't leave undeleted timers, see notifier
-			 * callback above.
-			 */
-			__register_hotcpu_notifier(&mce_cpu_notifier);
-			cpu_notifier_register_done();
-			goto err_device_create;
-		}
-	}
+	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
+				mce_cpu_online, mce_cpu_pre_down);
+	if (err < 0)
+		goto err_out_mem;
+	hp_online = err;
 
+	cpu_notifier_register_begin();
 	__register_hotcpu_notifier(&mce_cpu_notifier);
 	cpu_notifier_register_done();
 
@@ -2617,16 +2620,7 @@ static __init int mcheck_init_device(void)
 
 err_register:
 	unregister_syscore_ops(&mce_syscore_ops);
-
-err_device_create:
-	/*
-	 * We didn't keep track of which devices were created above, but
-	 * even if we had, the set of online cpus might have changed.
-	 * Play safe and remove for every possible cpu, since
-	 * mce_device_remove() will do the right thing.
-	 */
-	for_each_possible_cpu(i)
-		mce_device_remove(i);
+	cpuhp_remove_state(hp_online);
 
 err_out_mem:
 	free_cpumask_var(mce_device_initialized);

[tip:smp/hotplug] x86/mcheck: Move CPU_DEAD to hotplug state machine

[tip-bot for Sebastian Andrzej Siewior]

Commit-ID:  0e285d36bd2bfee0b95433ccc9065c878164f5b2
Gitweb:     http://git.kernel.org/tip/0e285d36bd2bfee0b95433ccc9065c878164f5b2
Author:     Sebastian Andrzej Siewior <bigeasy@linutronix.de>
AuthorDate: Thu, 10 Nov 2016 18:44:47 +0100
Committer:  Thomas Gleixner <tglx@linutronix.de>
CommitDate: Wed, 16 Nov 2016 09:34:18 +0100

x86/mcheck: Move CPU_DEAD to hotplug state machine

This moves the last piece of the old hotplug notifier code in MCE to the
new hotplug state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Acked-by: Borislav Petkov <bp@alien8.de>
Cc: Tony Luck <tony.luck@intel.com>
Cc: rt@linutronix.de
Cc: linux-edac@vger.kernel.org
Link: http://lkml.kernel.org/r/20161110174447.11848-8-bigeasy@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

---
 arch/x86/kernel/cpu/mcheck/mce.c | 42 ++++++++++++++--------------------------
 include/linux/cpuhotplug.h       |  1 +
 2 files changed, 16 insertions(+), 27 deletions(-)

diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 78955f5..b888e2f 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -2506,26 +2506,14 @@ static void mce_reenable_cpu(void)
 	}
 }
 
-/* Get notified when a cpu comes on/off. Be hotplug friendly. */
-static int
-mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
+static int mce_cpu_dead(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
+	mce_intel_hcpu_update(cpu);
 
-	switch (action & ~CPU_TASKS_FROZEN) {
-	case CPU_DEAD:
-		mce_intel_hcpu_update(cpu);
-
-		/* intentionally ignoring frozen here */
-		if (!(action & CPU_TASKS_FROZEN))
-			cmci_rediscover();
-		break;
-	case CPU_DOWN_PREPARE:
-
-		break;
-	}
-
-	return NOTIFY_OK;
+	/* intentionally ignoring frozen here */
+	if (!cpuhp_tasks_frozen)
+		cmci_rediscover();
+	return 0;
 }
 
 static int mce_cpu_online(unsigned int cpu)
@@ -2556,10 +2544,6 @@ static int mce_cpu_pre_down(unsigned int cpu)
 	return 0;
 }
 
-static struct notifier_block mce_cpu_notifier = {
-	.notifier_call = mce_cpu_callback,
-};
-
 static __init void mce_init_banks(void)
 {
 	int i;
@@ -2599,16 +2583,17 @@ static __init int mcheck_init_device(void)
 	if (err)
 		goto err_out_mem;
 
+	err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
+				mce_cpu_dead);
+	if (err)
+		goto err_out_mem;
+
 	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
 				mce_cpu_online, mce_cpu_pre_down);
 	if (err < 0)
-		goto err_out_mem;
+		goto err_out_online;
 	hp_online = err;
 
-	cpu_notifier_register_begin();
-	__register_hotcpu_notifier(&mce_cpu_notifier);
-	cpu_notifier_register_done();
-
 	register_syscore_ops(&mce_syscore_ops);
 
 	/* register character device /dev/mcelog */
@@ -2622,6 +2607,9 @@ err_register:
 	unregister_syscore_ops(&mce_syscore_ops);
 	cpuhp_remove_state(hp_online);
 
+err_out_online:
+	cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
+
 err_out_mem:
 	free_cpumask_var(mce_device_initialized);
 
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 3410d83..79b96f6 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -16,6 +16,7 @@ enum cpuhp_state {
 	CPUHP_PERF_SUPERH,
 	CPUHP_X86_HPET_DEAD,
 	CPUHP_X86_APB_DEAD,
+	CPUHP_X86_MCE_DEAD,
 	CPUHP_VIRT_NET_DEAD,
 	CPUHP_SLUB_DEAD,
 	CPUHP_MM_WRITEBACK_DEAD,