[PATCH v2 0/2] Convert to new CPU hotplug framework

[PATCH v2 0/2] Convert to new CPU hotplug framework

[Boris Ostrovsky]

New CPU hotplug framework was introduced recently. These patches convert Xen
CPU hotplug code to this infrastructure.

The patches (patch 1 specifically) will apply on top of
  https://lists.xenproject.org/archives/html/xen-devel/2016-08/msg00562.html

v2: Changes in patch 1 suggested by Sebastian

Boris Ostrovsky (2):
  xen/x86: Convert to hotplug state machine
  xen/events: Convert to hotplug state machine

 arch/x86/xen/enlighten.c         | 115 ++++++++++++++++++++++-----------------
 drivers/xen/events/events_fifo.c |  34 ++++--------
 include/linux/cpuhotplug.h       |   2 +
 3 files changed, 79 insertions(+), 72 deletions(-)

-- 
2.7.4

[PATCH v2 1/2] xen/x86: Convert to hotplug state machine

[Boris Ostrovsky]

Switch to new CPU hotplug infrastructure.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Suggested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
Changes in v2:
* Replace xen_cpu_up_cancel with xen_cpu_dead
* Use existing CPUHP_AP_ONLINE_DYN instead of introducing new state
* Be more careful with return value of cpuhp_setup_state_nocalls()
  as it may return a positive (non-error) number. (Which suggests
  that comment on top of __cpuhp_setup_state() is probably incorrect)

 arch/x86/xen/enlighten.c   | 115 +++++++++++++++++++++++++--------------------
 include/linux/cpuhotplug.h |   1 +
 2 files changed, 66 insertions(+), 50 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 78e1d06..e4c3399 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -140,7 +140,9 @@ RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
 __read_mostly int xen_have_vector_callback;
 EXPORT_SYMBOL_GPL(xen_have_vector_callback);
 
-static struct notifier_block xen_cpu_notifier;
+static int xen_cpu_up_prepare(unsigned int cpu);
+static int xen_cpu_up_online(unsigned int cpu);
+static int xen_cpu_dead(unsigned int cpu);
 
 /*
  * Point at some empty memory to start with. We map the real shared_info
@@ -1541,6 +1543,24 @@ static void __init xen_dom0_set_legacy_features(void)
 	x86_platform.legacy.rtc = 1;
 }
 
+static int xen_cpuhp_setup(void)
+{
+	int rc;
+
+	rc = cpuhp_setup_state_nocalls(CPUHP_XEN_PREPARE,
+				       "XEN_HVM_GUEST_PREPARE",
+				       xen_cpu_up_prepare, xen_cpu_dead);
+	if (rc >= 0) {
+		rc = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
+					       "XEN_HVM_GUEST_ONLINE",
+					       xen_cpu_up_online, NULL);
+		if (rc < 0)
+			cpuhp_remove_state_nocalls(CPUHP_XEN_PREPARE);
+	}
+
+	return rc >= 0 ? 0 : rc;
+}
+
 /* First C function to be called on Xen boot */
 asmlinkage __visible void __init xen_start_kernel(void)
 {
@@ -1629,7 +1649,7 @@ asmlinkage __visible void __init xen_start_kernel(void)
 	xen_initial_gdt = &per_cpu(gdt_page, 0);
 
 	xen_smp_init();
-	register_cpu_notifier(&xen_cpu_notifier);
+	WARN_ON(xen_cpuhp_setup());
 
 #ifdef CONFIG_ACPI_NUMA
 	/*
@@ -1823,63 +1843,58 @@ static void __init init_hvm_pv_info(void)
 	xen_domain_type = XEN_HVM_DOMAIN;
 }
 
-static int xen_cpu_notify(struct notifier_block *self, unsigned long action,
-			 void *hcpu)
+static int xen_cpu_up_prepare(unsigned int cpu)
 {
-	int cpu = (long)hcpu;
 	int rc;
 
-	switch (action) {
-	case CPU_UP_PREPARE:
-		if (xen_hvm_domain()) {
-			/*
-			 * This can happen if CPU was offlined earlier and
-			 * offlining timed out in common_cpu_die().
-			 */
-			if (cpu_report_state(cpu) == CPU_DEAD_FROZEN) {
-				xen_smp_intr_free(cpu);
-				xen_uninit_lock_cpu(cpu);
-			}
-
-			if (cpu_acpi_id(cpu) != U32_MAX)
-				per_cpu(xen_vcpu_id, cpu) = cpu_acpi_id(cpu);
-			else
-				per_cpu(xen_vcpu_id, cpu) = cpu;
-			xen_vcpu_setup(cpu);
+	if (xen_hvm_domain()) {
+		/*
+		 * This can happen if CPU was offlined earlier and
+		 * offlining timed out in common_cpu_die().
+		 */
+		if (cpu_report_state(cpu) == CPU_DEAD_FROZEN) {
+			xen_smp_intr_free(cpu);
+			xen_uninit_lock_cpu(cpu);
 		}
 
-		if (xen_pv_domain() ||
-		    (xen_have_vector_callback &&
-		     xen_feature(XENFEAT_hvm_safe_pvclock)))
-			xen_setup_timer(cpu);
+		if (cpu_acpi_id(cpu) != U32_MAX)
+			per_cpu(xen_vcpu_id, cpu) = cpu_acpi_id(cpu);
+		else
+			per_cpu(xen_vcpu_id, cpu) = cpu;
+		xen_vcpu_setup(cpu);
+	}
 
-		rc = xen_smp_intr_init(cpu);
-		if (rc) {
-			WARN(1, "xen_smp_intr_init() for CPU %d failed: %d\n",
-			     cpu, rc);
-			return NOTIFY_BAD;
-		}
+	if (xen_pv_domain() ||
+	    (xen_have_vector_callback &&
+	     xen_feature(XENFEAT_hvm_safe_pvclock)))
+		xen_setup_timer(cpu);
 
-		break;
-	case CPU_ONLINE:
-		xen_init_lock_cpu(cpu);
-		break;
-	case CPU_UP_CANCELED:
-		xen_smp_intr_free(cpu);
-		if (xen_pv_domain() ||
-		    (xen_have_vector_callback &&
-		     xen_feature(XENFEAT_hvm_safe_pvclock)))
-			xen_teardown_timer(cpu);
-		break;
-	default:
-		break;
+	rc = xen_smp_intr_init(cpu);
+	if (rc) {
+		WARN(1, "xen_smp_intr_init() for CPU %d failed: %d\n",
+		     cpu, rc);
+		return rc;
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block xen_cpu_notifier = {
-	.notifier_call	= xen_cpu_notify,
-};
+static int xen_cpu_dead(unsigned int cpu)
+{
+	xen_smp_intr_free(cpu);
+
+	if (xen_pv_domain() ||
+	    (xen_have_vector_callback &&
+	     xen_feature(XENFEAT_hvm_safe_pvclock)))
+		xen_teardown_timer(cpu);
+
+	return 0;
+}
+
+static int xen_cpu_up_online(unsigned int cpu)
+{
+	xen_init_lock_cpu(cpu);
+	return 0;
+}
 
 #ifdef CONFIG_KEXEC_CORE
 static void xen_hvm_shutdown(void)
@@ -1910,7 +1925,7 @@ static void __init xen_hvm_guest_init(void)
 	if (xen_feature(XENFEAT_hvm_callback_vector))
 		xen_have_vector_callback = 1;
 	xen_hvm_smp_init();
-	register_cpu_notifier(&xen_cpu_notifier);
+	WARN_ON(xen_cpuhp_setup());
 	xen_unplug_emulated_devices();
 	x86_init.irqs.intr_init = xen_init_IRQ;
 	xen_hvm_init_time_ops();
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 242bf53..33d352f 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -21,6 +21,7 @@ enum cpuhp_state {
 	CPUHP_X2APIC_PREPARE,
 	CPUHP_SMPCFD_PREPARE,
 	CPUHP_RCUTREE_PREP,
+	CPUHP_XEN_PREPARE,
 	CPUHP_NOTIFY_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_BRINGUP_CPU,
-- 
2.7.4

[PATCH v2 2/2] xen/events: Convert to hotplug state machine

[Boris Ostrovsky]

From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 drivers/xen/events/events_fifo.c | 34 ++++++++++++----------------------
 include/linux/cpuhotplug.h       |  1 +
 2 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c
index 266c2c7..7ef27c6 100644
--- a/drivers/xen/events/events_fifo.c
+++ b/drivers/xen/events/events_fifo.c
@@ -418,30 +418,18 @@ static int evtchn_fifo_alloc_control_block(unsigned cpu)
 	return ret;
 }
 
-static int evtchn_fifo_cpu_notification(struct notifier_block *self,
-						  unsigned long action,
-						  void *hcpu)
+static int xen_evtchn_cpu_prepare(unsigned int cpu)
 {
-	int cpu = (long)hcpu;
-	int ret = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		if (!per_cpu(cpu_control_block, cpu))
-			ret = evtchn_fifo_alloc_control_block(cpu);
-		break;
-	case CPU_DEAD:
-		__evtchn_fifo_handle_events(cpu, true);
-		break;
-	default:
-		break;
-	}
-	return ret < 0 ? NOTIFY_BAD : NOTIFY_OK;
+	if (!per_cpu(cpu_control_block, cpu))
+		return evtchn_fifo_alloc_control_block(cpu);
+	return 0;
 }
 
-static struct notifier_block evtchn_fifo_cpu_notifier = {
-	.notifier_call	= evtchn_fifo_cpu_notification,
-};
+static int xen_evtchn_cpu_dead(unsigned int cpu)
+{
+	__evtchn_fifo_handle_events(cpu, true);
+	return 0;
+}
 
 int __init xen_evtchn_fifo_init(void)
 {
@@ -456,7 +444,9 @@ int __init xen_evtchn_fifo_init(void)
 
 	evtchn_ops = &evtchn_ops_fifo;
 
-	register_cpu_notifier(&evtchn_fifo_cpu_notifier);
+	cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
+				  "CPUHP_XEN_EVTCHN_PREPARE",
+				  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
 out:
 	put_cpu();
 	return ret;
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 33d352f..5f60316 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -22,6 +22,7 @@ enum cpuhp_state {
 	CPUHP_SMPCFD_PREPARE,
 	CPUHP_RCUTREE_PREP,
 	CPUHP_XEN_PREPARE,
+	CPUHP_XEN_EVTCHN_PREPARE,
 	CPUHP_NOTIFY_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_BRINGUP_CPU,
-- 
2.7.4

Re: [PATCH v2 1/2] xen/x86: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

On 2016-09-07 13:19:00 [-0400], Boris Ostrovsky wrote:
> * Be more careful with return value of cpuhp_setup_state_nocalls()
>   as it may return a positive (non-error) number. (Which suggests
>   that comment on top of __cpuhp_setup_state() is probably incorrect)

Yes, we need to update that one.
The two patches look good.

Sebastian

Re: [Xen-devel] [PATCH v2 0/2] Convert to new CPU hotplug framework

[David Vrabel]

On 07/09/16 18:18, Boris Ostrovsky wrote:
> New CPU hotplug framework was introduced recently. These patches convert Xen
> CPU hotplug code to this infrastructure.

Applied to for-linus-4.9, thanks.

David