[PATCH 0/2] Convert to new CPU hotplug framework

[PATCH 0/2] Convert to new CPU hotplug framework

[Boris Ostrovsky]

New CPU hotplug framework was introduced recently. These patches convert Xen
CPU hotplug code to this infrastructure.

The patches (patch 1 specifically) will apply on top of
  https://lists.xenproject.org/archives/html/xen-devel/2016-08/msg00562.html

(I will be out of the office starting tomorrow so may not be able to respond
to comments right away)


Boris Ostrovsky (2):
  xen/x86: Convert to hotplug state machine
  xen/events: Convert to hotplug state machine

 arch/x86/xen/enlighten.c         |  115 +++++++++++++++++++++----------------
 drivers/xen/events/events_fifo.c |   34 ++++-------
 include/linux/cpuhotplug.h       |    3 +
 3 files changed, 80 insertions(+), 72 deletions(-)

[PATCH 1/2] xen/x86: Convert to hotplug state machine

[Boris Ostrovsky]

Switch to new CPU hotplug infrastructure.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
Suggested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
 arch/x86/xen/enlighten.c   |  115 +++++++++++++++++++++++++-------------------
 include/linux/cpuhotplug.h |    2 +
 2 files changed, 67 insertions(+), 50 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index c7f6b1f..2283976 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -140,7 +140,9 @@ RESERVE_BRK(shared_info_page_brk, PAGE_SIZE);
 __read_mostly int xen_have_vector_callback;
 EXPORT_SYMBOL_GPL(xen_have_vector_callback);
 
-static struct notifier_block xen_cpu_notifier;
+static int xen_cpu_up_prepare(unsigned int cpu);
+static int xen_cpu_up_online(unsigned int cpu);
+static int xen_cpu_up_cancel(unsigned int cpu);
 
 /*
  * Point at some empty memory to start with. We map the real shared_info
@@ -1541,6 +1543,24 @@ static void __init xen_dom0_set_legacy_features(void)
 	x86_platform.legacy.rtc = 1;
 }
 
+static int xen_cpuhp_setup(void)
+{
+	int rc;
+
+	rc = cpuhp_setup_state_nocalls(CPUHP_XEN_PREPARE,
+				       "XEN_HVM_GUEST_PREPARE",
+				       xen_cpu_up_prepare, xen_cpu_up_cancel);
+	if (!rc) {
+		rc = cpuhp_setup_state_nocalls(CPUHP_AP_XEN_ONLINE,
+					       "XEN_HVM_GUEST_PREPARE",
+					       xen_cpu_up_online, NULL);
+		if (rc)
+			cpuhp_remove_state_nocalls(CPUHP_XEN_PREPARE);
+	}
+
+	return rc;
+}
+
 /* First C function to be called on Xen boot */
 asmlinkage __visible void __init xen_start_kernel(void)
 {
@@ -1629,7 +1649,7 @@ asmlinkage __visible void __init xen_start_kernel(void)
 	xen_initial_gdt = &per_cpu(gdt_page, 0);
 
 	xen_smp_init();
-	register_cpu_notifier(&xen_cpu_notifier);
+	WARN_ON(xen_cpuhp_setup());
 
 #ifdef CONFIG_ACPI_NUMA
 	/*
@@ -1823,63 +1843,58 @@ static void __init init_hvm_pv_info(void)
 	xen_domain_type = XEN_HVM_DOMAIN;
 }
 
-static int xen_cpu_notify(struct notifier_block *self, unsigned long action,
-			 void *hcpu)
+static int xen_cpu_up_prepare(unsigned int cpu)
 {
-	int cpu = (long)hcpu;
 	int rc;
 
-	switch (action) {
-	case CPU_UP_PREPARE:
-		if (xen_hvm_domain()) {
-			/*
-			 * This can happen if CPU was offlined earlier and
-			 * offlining timed out in common_cpu_die().
-			 */
-			if (cpu_report_state(cpu) == CPU_DEAD_FROZEN) {
-				xen_smp_intr_free(cpu);
-				xen_uninit_lock_cpu(cpu);
-			}
-
-			if (cpu_acpi_id(cpu) != U32_MAX)
-				per_cpu(xen_vcpu_id, cpu) = cpu_acpi_id(cpu);
-			else
-				per_cpu(xen_vcpu_id, cpu) = cpu;
-			xen_vcpu_setup(cpu);
+	if (xen_hvm_domain()) {
+		/*
+		 * This can happen if CPU was offlined earlier and
+		 * offlining timed out in common_cpu_die().
+		 */
+		if (cpu_report_state(cpu) == CPU_DEAD_FROZEN) {
+			xen_smp_intr_free(cpu);
+			xen_uninit_lock_cpu(cpu);
 		}
 
-		if (xen_pv_domain() ||
-		    (xen_have_vector_callback &&
-		     xen_feature(XENFEAT_hvm_safe_pvclock)))
-			xen_setup_timer(cpu);
+		if (cpu_acpi_id(cpu) != U32_MAX)
+			per_cpu(xen_vcpu_id, cpu) = cpu_acpi_id(cpu);
+		else
+			per_cpu(xen_vcpu_id, cpu) = cpu;
+		xen_vcpu_setup(cpu);
+	}
 
-		rc = xen_smp_intr_init(cpu);
-		if (rc) {
-			WARN(1, "xen_smp_intr_init() for CPU %d failed: %d\n",
-			     cpu, rc);
-			return NOTIFY_BAD;
-		}
+	if (xen_pv_domain() ||
+	    (xen_have_vector_callback &&
+	     xen_feature(XENFEAT_hvm_safe_pvclock)))
+		xen_setup_timer(cpu);
 
-		break;
-	case CPU_ONLINE:
-		xen_init_lock_cpu(cpu);
-		break;
-	case CPU_UP_CANCELED:
-		xen_smp_intr_free(cpu);
-		if (xen_pv_domain() ||
-		    (xen_have_vector_callback &&
-		     xen_feature(XENFEAT_hvm_safe_pvclock)))
-			xen_teardown_timer(cpu);
-		break;
-	default:
-		break;
+	rc = xen_smp_intr_init(cpu);
+	if (rc) {
+		WARN(1, "xen_smp_intr_init() for CPU %d failed: %d\n",
+		     cpu, rc);
+		return rc;
 	}
-	return NOTIFY_OK;
+	return 0;
 }
 
-static struct notifier_block xen_cpu_notifier = {
-	.notifier_call	= xen_cpu_notify,
-};
+static int xen_cpu_up_cancel(unsigned int cpu)
+{
+	xen_smp_intr_free(cpu);
+
+	if (xen_pv_domain() ||
+	    (xen_have_vector_callback &&
+	     xen_feature(XENFEAT_hvm_safe_pvclock)))
+		xen_teardown_timer(cpu);
+
+	return 0;
+}
+
+static int xen_cpu_up_online(unsigned int cpu)
+{
+	xen_init_lock_cpu(cpu);
+	return 0;
+}
 
 #ifdef CONFIG_KEXEC_CORE
 static void xen_hvm_shutdown(void)
@@ -1910,7 +1925,7 @@ static void __init xen_hvm_guest_init(void)
 	if (xen_feature(XENFEAT_hvm_callback_vector))
 		xen_have_vector_callback = 1;
 	xen_hvm_smp_init();
-	register_cpu_notifier(&xen_cpu_notifier);
+	WARN_ON(xen_cpuhp_setup());
 	xen_unplug_emulated_devices();
 	x86_init.irqs.intr_init = xen_init_IRQ;
 	xen_hvm_init_time_ops();
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 242bf53..d6beeb9 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -21,6 +21,7 @@ enum cpuhp_state {
 	CPUHP_X2APIC_PREPARE,
 	CPUHP_SMPCFD_PREPARE,
 	CPUHP_RCUTREE_PREP,
+	CPUHP_XEN_PREPARE,
 	CPUHP_NOTIFY_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_BRINGUP_CPU,
@@ -93,6 +94,7 @@ enum cpuhp_state {
 	CPUHP_AP_ONLINE_DYN_END		= CPUHP_AP_ONLINE_DYN + 30,
 	CPUHP_AP_X86_HPET_ONLINE,
 	CPUHP_AP_X86_KVM_CLK_ONLINE,
+	CPUHP_AP_XEN_ONLINE,
 	CPUHP_AP_ACTIVE,
 	CPUHP_ONLINE,
 };
-- 
1.7.1

Re: [PATCH 1/2] xen/x86: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

On 2016-08-15 10:46:46 [-0400], Boris Ostrovsky wrote:
> Switch to new CPU hotplug infrastructure.
> 
> Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
> Suggested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> ---
>  arch/x86/xen/enlighten.c   |  115 +++++++++++++++++++++++++-------------------
>  include/linux/cpuhotplug.h |    2 +
>  2 files changed, 67 insertions(+), 50 deletions(-)
> 
> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
> index c7f6b1f..2283976 100644
> --- a/arch/x86/xen/enlighten.c
> +++ b/arch/x86/xen/enlighten.c
> @@ -1541,6 +1543,24 @@ static void __init xen_dom0_set_legacy_features(void)
>  	x86_platform.legacy.rtc = 1;
>  }
>  
> +static int xen_cpuhp_setup(void)
> +{
> +	int rc;
> +
> +	rc = cpuhp_setup_state_nocalls(CPUHP_XEN_PREPARE,
> +				       "XEN_HVM_GUEST_PREPARE",
> +				       xen_cpu_up_prepare, xen_cpu_up_cancel);

The old states UP_CANCEL is different from UP_PREPARE. The latter was
invoked only in the error case while your new callback is always
invoked. From looking at the code you free memory in
xen_cpu_up_cancel() which was allocated in xen_cpu_up_prepare() and
therefore I would name it xen_cpu_dead().
If you do find the time, you might manage to rework the code to avoid
using the _nocalls() function. If see this right, you use
xen_setup_vcpu_info_placement() for the init in the first place. This
uses for_each_possible_cpu macro. The cpuhp_setup_state() function would
perform the init for all CPUs before they come up.

> +	if (!rc) {
> +		rc = cpuhp_setup_state_nocalls(CPUHP_AP_XEN_ONLINE,

If there is no need to run this after KVM's CLK callback please use
CPUHP_AP_ONLINE_DYN. If there is such a need then please document it.

> +					       "XEN_HVM_GUEST_PREPARE",
> +					       xen_cpu_up_online, NULL);
> +		if (rc)
> +			cpuhp_remove_state_nocalls(CPUHP_XEN_PREPARE);
> +	}
> +
> +	return rc;
> +}
> +
>  /* First C function to be called on Xen boot */
>  asmlinkage __visible void __init xen_start_kernel(void)
>  {

Sebastian

Re: [PATCH 1/2] xen/x86: Convert to hotplug state machine

[Boris Ostrovsky]

On 08/17/2016 04:33 AM, Sebastian Andrzej Siewior wrote:
> On 2016-08-15 10:46:46 [-0400], Boris Ostrovsky wrote:
>> Switch to new CPU hotplug infrastructure.
>>
>> Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
>> Suggested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
>> ---
>>  arch/x86/xen/enlighten.c   |  115 +++++++++++++++++++++++++-------------------
>>  include/linux/cpuhotplug.h |    2 +
>>  2 files changed, 67 insertions(+), 50 deletions(-)
>>
>> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
>> index c7f6b1f..2283976 100644
>> --- a/arch/x86/xen/enlighten.c
>> +++ b/arch/x86/xen/enlighten.c
>> @@ -1541,6 +1543,24 @@ static void __init xen_dom0_set_legacy_features(void)
>>  	x86_platform.legacy.rtc = 1;
>>  }
>>  
>> +static int xen_cpuhp_setup(void)
>> +{
>> +	int rc;
>> +
>> +	rc = cpuhp_setup_state_nocalls(CPUHP_XEN_PREPARE,
>> +				       "XEN_HVM_GUEST_PREPARE",
>> +				       xen_cpu_up_prepare, xen_cpu_up_cancel);
> The old states UP_CANCEL is different from UP_PREPARE. The latter was
> invoked only in the error case while your new callback is always
> invoked. From looking at the code you free memory in
> xen_cpu_up_cancel() which was allocated in xen_cpu_up_prepare() and
> therefore I would name it xen_cpu_dead().

Yes, "cancel" is wrong term to use here.

> If you do find the time, you might manage to rework the code to avoid
> using the _nocalls() function. If see this right, you use
> xen_setup_vcpu_info_placement() for the init in the first place. This
> uses for_each_possible_cpu macro. The cpuhp_setup_state() function would
> perform the init for all CPUs before they come up.

I am not sure I see what this would buy us.

Besides, cpuhp_setup_state() uses for_each_present_cpu().

>
>> +	if (!rc) {
>> +		rc = cpuhp_setup_state_nocalls(CPUHP_AP_XEN_ONLINE,
> If there is no need to run this after KVM's CLK callback please use
> CPUHP_AP_ONLINE_DYN. If there is such a need then please document it.

OK.

-boris

>
>> +					       "XEN_HVM_GUEST_PREPARE",
>> +					       xen_cpu_up_online, NULL);
>> +		if (rc)
>> +			cpuhp_remove_state_nocalls(CPUHP_XEN_PREPARE);
>> +	}
>> +
>> +	return rc;
>> +}
>> +
>>  /* First C function to be called on Xen boot */
>>  asmlinkage __visible void __init xen_start_kernel(void)
>>  {
> Sebastian

Re: [PATCH 1/2] xen/x86: Convert to hotplug state machine

[Sebastian Andrzej Siewior]

On 2016-08-26 15:37:38 [-0400], Boris Ostrovsky wrote:
> > If you do find the time, you might manage to rework the code to avoid
> > using the _nocalls() function. If see this right, you use
> > xen_setup_vcpu_info_placement() for the init in the first place. This
> > uses for_each_possible_cpu macro. The cpuhp_setup_state() function would
> > perform the init for all CPUs before they come up.
> 
> I am not sure I see what this would buy us.
> 
> Besides, cpuhp_setup_state() uses for_each_present_cpu().

Correct. So you would avoid running the init code on CPUs which are
within the for_each_possible_cpu() set but not in for_each_present_cpu().

Assuming a NUMA box with two CPUs, 8 cores each gives you 32 CPUs in
Linux with hyper threading. BIOS may report 240 CPUs as the upper limit
(possible CPUs) but if you never deploy them you don't need to
initialize them… Should they be plugged physically then the
for_each_present_cpu() loop will cover them once they come up.

Sebastian

Re: [PATCH 1/2] xen/x86: Convert to hotplug state machine

[Boris Ostrovsky]

On 08/31/2016 12:15 PM, Sebastian Andrzej Siewior wrote:
> On 2016-08-26 15:37:38 [-0400], Boris Ostrovsky wrote:
>>> If you do find the time, you might manage to rework the code to avoid
>>> using the _nocalls() function. If see this right, you use
>>> xen_setup_vcpu_info_placement() for the init in the first place. This
>>> uses for_each_possible_cpu macro. The cpuhp_setup_state() function would
>>> perform the init for all CPUs before they come up.
>> I am not sure I see what this would buy us.
>>
>> Besides, cpuhp_setup_state() uses for_each_present_cpu().
> Correct. So you would avoid running the init code on CPUs which are
> within the for_each_possible_cpu() set but not in for_each_present_cpu().
>
> Assuming a NUMA box with two CPUs, 8 cores each gives you 32 CPUs in
> Linux with hyper threading. BIOS may report 240 CPUs as the upper limit
> (possible CPUs) but if you never deploy them you don't need to
> initialize them… Should they be plugged physically then the
> for_each_present_cpu() loop will cover them once they come up.
>

That's not going to help Xen guests: all possible CPUs are brought up
right away and then those that are not in use are unplugged.


-boris

[PATCH 2/2] xen/events: Convert to hotplug state machine

[Boris Ostrovsky]

From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

Install the callbacks via the state machine.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
Changes from Sebastian's original version:
* Renamed hotplug state from CPUHP_XEN_EV_PREPEARE to CPUHP_XEN_EVTCHN_PREPARE
* Dropped suggestion in the commit messages about moving evtchn_fifo_alloc_control_block
  call since I think it is in the right place right now.


 drivers/xen/events/events_fifo.c |   34 ++++++++++++----------------------
 include/linux/cpuhotplug.h       |    1 +
 2 files changed, 13 insertions(+), 22 deletions(-)

diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c
index 266c2c7..7ef27c6 100644
--- a/drivers/xen/events/events_fifo.c
+++ b/drivers/xen/events/events_fifo.c
@@ -418,30 +418,18 @@ static int evtchn_fifo_alloc_control_block(unsigned cpu)
 	return ret;
 }
 
-static int evtchn_fifo_cpu_notification(struct notifier_block *self,
-						  unsigned long action,
-						  void *hcpu)
+static int xen_evtchn_cpu_prepare(unsigned int cpu)
 {
-	int cpu = (long)hcpu;
-	int ret = 0;
-
-	switch (action) {
-	case CPU_UP_PREPARE:
-		if (!per_cpu(cpu_control_block, cpu))
-			ret = evtchn_fifo_alloc_control_block(cpu);
-		break;
-	case CPU_DEAD:
-		__evtchn_fifo_handle_events(cpu, true);
-		break;
-	default:
-		break;
-	}
-	return ret < 0 ? NOTIFY_BAD : NOTIFY_OK;
+	if (!per_cpu(cpu_control_block, cpu))
+		return evtchn_fifo_alloc_control_block(cpu);
+	return 0;
 }
 
-static struct notifier_block evtchn_fifo_cpu_notifier = {
-	.notifier_call	= evtchn_fifo_cpu_notification,
-};
+static int xen_evtchn_cpu_dead(unsigned int cpu)
+{
+	__evtchn_fifo_handle_events(cpu, true);
+	return 0;
+}
 
 int __init xen_evtchn_fifo_init(void)
 {
@@ -456,7 +444,9 @@ int __init xen_evtchn_fifo_init(void)
 
 	evtchn_ops = &evtchn_ops_fifo;
 
-	register_cpu_notifier(&evtchn_fifo_cpu_notifier);
+	cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
+				  "CPUHP_XEN_EVTCHN_PREPARE",
+				  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
 out:
 	put_cpu();
 	return ret;
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index d6beeb9..c60a17c 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -22,6 +22,7 @@ enum cpuhp_state {
 	CPUHP_SMPCFD_PREPARE,
 	CPUHP_RCUTREE_PREP,
 	CPUHP_XEN_PREPARE,
+	CPUHP_XEN_EVTCHN_PREPARE,
 	CPUHP_NOTIFY_PREPARE,
 	CPUHP_TIMERS_DEAD,
 	CPUHP_BRINGUP_CPU,
-- 
1.7.1

Re: [Xen-devel] [PATCH 2/2] xen/events: Convert to hotplug state machine

[David Vrabel]

On 15/08/16 15:46, Boris Ostrovsky wrote:
> From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> 
> Install the callbacks via the state machine.
[...]
> +static int xen_evtchn_cpu_dead(unsigned int cpu)
> +{
> +	__evtchn_fifo_handle_events(cpu, true);
> +	return 0;
> +}

I'm not familiar with the new state machine.  When this is called, what
state is the CPU in?

In particular, local interrupts must be disabled and all non-percpu irqs
must have been migrated to other CPUs.


>  int __init xen_evtchn_fifo_init(void)
>  {
> @@ -456,7 +444,9 @@ int __init xen_evtchn_fifo_init(void)
>  
>  	evtchn_ops = &evtchn_ops_fifo;
>  
> -	register_cpu_notifier(&evtchn_fifo_cpu_notifier);
> +	cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
> +				  "CPUHP_XEN_EVTCHN_PREPARE",
> +				  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
>  out:
>  	put_cpu();
>  	return ret;
> diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
> index d6beeb9..c60a17c 100644
> --- a/include/linux/cpuhotplug.h
> +++ b/include/linux/cpuhotplug.h
> @@ -22,6 +22,7 @@ enum cpuhp_state {
>  	CPUHP_SMPCFD_PREPARE,
>  	CPUHP_RCUTREE_PREP,
>  	CPUHP_XEN_PREPARE,
> +	CPUHP_XEN_EVTCHN_PREPARE,
>  	CPUHP_NOTIFY_PREPARE,
>  	CPUHP_TIMERS_DEAD,
>  	CPUHP_BRINGUP_CPU,
> 

Re: [Xen-devel] [PATCH 2/2] xen/events: Convert to hotplug state machine

[Boris Ostrovsky]

On 08/15/2016 11:06 AM, David Vrabel wrote:
> On 15/08/16 15:46, Boris Ostrovsky wrote:
>> From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
>>
>> Install the callbacks via the state machine.
> [...]
>> +static int xen_evtchn_cpu_dead(unsigned int cpu)
>> +{
>> +	__evtchn_fifo_handle_events(cpu, true);
>> +	return 0;
>> +}
> I'm not familiar with the new state machine.  When this is called, what
> state is the CPU in?
>
> In particular, local interrupts must be disabled and all non-percpu irqs
> must have been migrated to other CPUs.

This (xen_evtchn_cpu_dead()) is called immediately after notify_dead()
on the way down.

The state machine is walking cpuhp_state list and for each member of the
list it calls the callback that has been registered for that member.

So when we bring a CPU up first we call xen_evtchn_cpu_prepare() and
then in the next iteration of the state machine loop notify_prepare()
(because CPUHP_XEN_EVTCHN_PREPARE is immediately before
CPUHP_NOTIFY_PREPARE). On the way down it's done in reverse: first
notify_dead() and then xen_evtchn_cpu_dead().

In other words, the old notification scheme is part of new state machine:

	CPUHP_RCUTREE_PREP,
 	CPUHP_XEN_PREPARE,
+	CPUHP_XEN_EVTCHN_PREPARE,
 	CPUHP_NOTIFY_PREPARE,  <=== CPU notifiers callback
	CPUHP_TIMERS_DEAD,
	CPUHP_BRINGUP_CPU,


-boris


>
>
>>  int __init xen_evtchn_fifo_init(void)
>>  {
>> @@ -456,7 +444,9 @@ int __init xen_evtchn_fifo_init(void)
>>  
>>  	evtchn_ops = &evtchn_ops_fifo;
>>  
>> -	register_cpu_notifier(&evtchn_fifo_cpu_notifier);
>> +	cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
>> +				  "CPUHP_XEN_EVTCHN_PREPARE",
>> +				  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
>>  out:
>>  	put_cpu();
>>  	return ret;
>> diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
>> index d6beeb9..c60a17c 100644
>> --- a/include/linux/cpuhotplug.h
>> +++ b/include/linux/cpuhotplug.h
>> @@ -22,6 +22,7 @@ enum cpuhp_state {
>>  	CPUHP_SMPCFD_PREPARE,
>>  	CPUHP_RCUTREE_PREP,
>>  	CPUHP_XEN_PREPARE,
>> +	CPUHP_XEN_EVTCHN_PREPARE,
>>  	CPUHP_NOTIFY_PREPARE,
>>  	CPUHP_TIMERS_DEAD,
>>  	CPUHP_BRINGUP_CPU,
>>