[patch 31/66] x86/kvm/kvmclock: Convert to hotplug state machine

[patch 31/66] x86/kvm/kvmclock: Convert to hotplug state machine

[Anna-Maria Gleixner]

[-- Attachment #1: 0133-x86-kvm-kvmclock-Convert-to-hotplug-state-machine.patch --]
[-- Type: text/plain, Size: 3256 bytes --]

From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

We assumed that the priority ordering was ment to invoke the online
callback as the last step. In the original code this also invoked the
down prepare callback as the last step. With the symmetric state
machine the down prepare callback is now the first step.

Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Gleb Natapov <gleb@kernel.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: kvm@vger.kernel.org
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
---
 arch/x86/kvm/x86.c         |   35 ++++++++---------------------------
 include/linux/cpuhotplug.h |    1 +
 2 files changed, 9 insertions(+), 27 deletions(-)

--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5552,9 +5552,10 @@ int kvm_fast_pio_out(struct kvm_vcpu *vc
 }
 EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
 
-static void tsc_bad(void *info)
+static int kvmclock_cpu_down_prep(unsigned int cpu)
 {
 	__this_cpu_write(cpu_tsc_khz, 0);
+	return 0;
 }
 
 static void tsc_khz_changed(void *data)
@@ -5659,35 +5660,18 @@ static struct notifier_block kvmclock_cp
 	.notifier_call  = kvmclock_cpufreq_notifier
 };
 
-static int kvmclock_cpu_notifier(struct notifier_block *nfb,
-					unsigned long action, void *hcpu)
+static int kvmclock_cpu_online(unsigned int cpu)
 {
-	unsigned int cpu = (unsigned long)hcpu;
-
-	switch (action) {
-		case CPU_ONLINE:
-		case CPU_DOWN_FAILED:
-			tsc_khz_changed(NULL);
-			break;
-		case CPU_DOWN_PREPARE:
-			tsc_bad(NULL);
-			break;
-	}
-	return NOTIFY_OK;
+	tsc_khz_changed(NULL);
+	return 0;
 }
 
-static struct notifier_block kvmclock_cpu_notifier_block = {
-	.notifier_call  = kvmclock_cpu_notifier,
-	.priority = -INT_MAX
-};
-
 static void kvm_timer_init(void)
 {
 	int cpu;
 
 	max_tsc_khz = tsc_khz;
 
-	cpu_notifier_register_begin();
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
 #ifdef CONFIG_CPU_FREQ
 		struct cpufreq_policy policy;
@@ -5702,12 +5686,9 @@ static void kvm_timer_init(void)
 					  CPUFREQ_TRANSITION_NOTIFIER);
 	}
 	pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
-	for_each_online_cpu(cpu)
-		smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
-
-	__register_hotcpu_notifier(&kvmclock_cpu_notifier_block);
-	cpu_notifier_register_done();
 
+	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "AP_X86_KVM_CLK_ONLINE",
+			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
 }
 
 static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
@@ -5896,7 +5877,7 @@ void kvm_arch_exit(void)
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
 		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
 					    CPUFREQ_TRANSITION_NOTIFIER);
-	unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
+	cpuhp_remove_state_nocalls(CPUHP_X86_KVM_CLK_ONLINE);
 #ifdef CONFIG_X86_64
 	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
 #endif
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -56,6 +56,7 @@ enum cpuhp_state {
 	CPUHP_AP_ONLINE_DYN,
 	CPUHP_AP_ONLINE_DYN_END		= CPUHP_AP_ONLINE_DYN + 30,
 	CPUHP_AP_X86_HPET_ONLINE,
+	CPUHP_AP_X86_KVM_CLK_ONLINE,
 	CPUHP_AP_ACTIVE,
 	CPUHP_ONLINE,
 };

[patch 35/66] virt: Convert kvm hotplug to state machine

[Anna-Maria Gleixner]

[-- Attachment #1: 0029-virt-Convert-kvm-hotplug-to-state-machine.patch --]
[-- Type: text/plain, Size: 3451 bytes --]

From: Thomas Gleixner <tglx@linutronix.de>

Install the callbacks via the state machine. The core won't invoke the
callbacks on already online CPUs.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Gleb Natapov <gleb@kernel.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: kvm@vger.kernel.org
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>

---
 include/linux/cpuhotplug.h |    1 +
 virt/kvm/kvm_main.c        |   32 ++++++++------------------------
 2 files changed, 9 insertions(+), 24 deletions(-)

--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -37,6 +37,7 @@ enum cpuhp_state {
 	CPUHP_AP_PERF_XTENSA_STARTING,
 	CPUHP_AP_ARM_VFP_STARTING,
 	CPUHP_AP_PERF_ARM_STARTING,
+	CPUHP_AP_KVM_STARTING,
 	CPUHP_AP_NOTIFY_STARTING,
 	CPUHP_AP_ONLINE,
 	CPUHP_TEARDOWN_CPU,
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -3144,12 +3144,13 @@ static void hardware_enable_nolock(void
 	}
 }
 
-static void hardware_enable(void)
+static int kvm_starting_cpu(unsigned int cpu)
 {
 	raw_spin_lock(&kvm_count_lock);
 	if (kvm_usage_count)
 		hardware_enable_nolock(NULL);
 	raw_spin_unlock(&kvm_count_lock);
+	return 0;
 }
 
 static void hardware_disable_nolock(void *junk)
@@ -3162,12 +3163,13 @@ static void hardware_disable_nolock(void
 	kvm_arch_hardware_disable();
 }
 
-static void hardware_disable(void)
+static int kvm_dying_cpu(unsigned int cpu)
 {
 	raw_spin_lock(&kvm_count_lock);
 	if (kvm_usage_count)
 		hardware_disable_nolock(NULL);
 	raw_spin_unlock(&kvm_count_lock);
+	return 0;
 }
 
 static void hardware_disable_all_nolock(void)
@@ -3208,21 +3210,6 @@ static int hardware_enable_all(void)
 	return r;
 }
 
-static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
-			   void *v)
-{
-	val &= ~CPU_TASKS_FROZEN;
-	switch (val) {
-	case CPU_DYING:
-		hardware_disable();
-		break;
-	case CPU_STARTING:
-		hardware_enable();
-		break;
-	}
-	return NOTIFY_OK;
-}
-
 static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
 		      void *v)
 {
@@ -3489,10 +3476,6 @@ int kvm_io_bus_unregister_dev(struct kvm
 	return r;
 }
 
-static struct notifier_block kvm_cpu_notifier = {
-	.notifier_call = kvm_cpu_hotplug,
-};
-
 static int kvm_debugfs_open(struct inode *inode, struct file *file,
 			   int (*get)(void *, u64 *), int (*set)(void *, u64),
 			   const char *fmt)
@@ -3743,7 +3726,8 @@ int kvm_init(void *opaque, unsigned vcpu
 			goto out_free_1;
 	}
 
-	r = register_cpu_notifier(&kvm_cpu_notifier);
+	r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "AP_KVM_STARTING",
+				      kvm_starting_cpu, kvm_dying_cpu);
 	if (r)
 		goto out_free_2;
 	register_reboot_notifier(&kvm_reboot_notifier);
@@ -3797,7 +3781,7 @@ int kvm_init(void *opaque, unsigned vcpu
 	kmem_cache_destroy(kvm_vcpu_cache);
 out_free_3:
 	unregister_reboot_notifier(&kvm_reboot_notifier);
-	unregister_cpu_notifier(&kvm_cpu_notifier);
+	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
 out_free_2:
 out_free_1:
 	kvm_arch_hardware_unsetup();
@@ -3820,7 +3804,7 @@ void kvm_exit(void)
 	kvm_async_pf_deinit();
 	unregister_syscore_ops(&kvm_syscore_ops);
 	unregister_reboot_notifier(&kvm_reboot_notifier);
-	unregister_cpu_notifier(&kvm_cpu_notifier);
+	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
 	on_each_cpu(hardware_disable_nolock, NULL, 1);
 	kvm_arch_hardware_unsetup();
 	kvm_arch_exit();

[patch 47/66] arm: kvm: vgic: Convert to hotplug state machine

[Anna-Maria Gleixner]

[-- Attachment #1: 0047-arm-kvm-vgic-Convert-to-hotplug-state-machine.patch --]
[-- Type: text/plain, Size: 2531 bytes --]

From: Richard Cochran <rcochran@linutronix.de>

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.
The VGIC callback is run after KVM's main callback since it reflects the
makefile order.

Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
Reviewed-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Cc: Gleb Natapov <gleb@kernel.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: kvmarm@lists.cs.columbia.edu
Cc: kvm@vger.kernel.org
---
 include/linux/cpuhotplug.h |    1 +
 virt/kvm/arm/vgic.c        |   39 ++++++++-------------------------------
 2 files changed, 9 insertions(+), 31 deletions(-)

--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -47,6 +47,7 @@ enum cpuhp_state {
 	CPUHP_AP_MARCO_TIMER_STARTING,
 	CPUHP_AP_MIPS_GIC_TIMER_STARTING,
 	CPUHP_AP_KVM_STARTING,
+	CPUHP_AP_KVM_ARM_VGIC_STARTING,
 	CPUHP_AP_LEDTRIG_STARTING,
 	CPUHP_AP_NOTIFY_STARTING,
 	CPUHP_AP_ONLINE,
--- a/virt/kvm/arm/vgic.c
+++ b/virt/kvm/arm/vgic.c
@@ -2326,32 +2326,18 @@ int vgic_has_attr_regs(const struct vgic
 		return -ENXIO;
 }
 
-static void vgic_init_maintenance_interrupt(void *info)
+static int vgic_starting_cpu(unsigned int cpu)
 {
 	enable_percpu_irq(vgic->maint_irq, 0);
+	return 0;
 }
 
-static int vgic_cpu_notify(struct notifier_block *self,
-			   unsigned long action, void *cpu)
+static int vgic_dying_cpu(unsigned int cpu)
 {
-	switch (action) {
-	case CPU_STARTING:
-	case CPU_STARTING_FROZEN:
-		vgic_init_maintenance_interrupt(NULL);
-		break;
-	case CPU_DYING:
-	case CPU_DYING_FROZEN:
-		disable_percpu_irq(vgic->maint_irq);
-		break;
-	}
-
-	return NOTIFY_OK;
+	disable_percpu_irq(vgic->maint_irq);
+	return 0;
 }
 
-static struct notifier_block vgic_cpu_nb = {
-	.notifier_call = vgic_cpu_notify,
-};
-
 static int kvm_vgic_probe(void)
 {
 	const struct gic_kvm_info *gic_kvm_info;
@@ -2392,19 +2378,10 @@ int kvm_vgic_hyp_init(void)
 		return ret;
 	}
 
-	ret = __register_cpu_notifier(&vgic_cpu_nb);
-	if (ret) {
-		kvm_err("Cannot register vgic CPU notifier\n");
-		goto out_free_irq;
-	}
-
-	on_each_cpu(vgic_init_maintenance_interrupt, NULL, 1);
-
+	cpuhp_setup_state(CPUHP_AP_KVM_ARM_VGIC_STARTING,
+			  "AP_KVM_ARM_VGIC_STARTING", vgic_starting_cpu,
+			  vgic_dying_cpu);
 	return 0;
-
-out_free_irq:
-	free_percpu_irq(vgic->maint_irq, kvm_get_running_vcpus());
-	return ret;
 }
 
 int kvm_irq_map_gsi(struct kvm *kvm,

[patch 48/66] arm: kvm: arch_timer: Convert to hotplug state machine

[Anna-Maria Gleixner]

[-- Attachment #1: 0048-arm-kvm-arch_timer-Convert-to-hotplug-state-machine.patch --]
[-- Type: text/plain, Size: 2556 bytes --]

From: Richard Cochran <rcochran@linutronix.de>

Install the callbacks via the state machine and let the core invoke
the callbacks on the already online CPUs.

Signed-off-by: Richard Cochran <rcochran@linutronix.de>
Reviewed-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Cc: Christoffer Dall <christoffer.dall@linaro.org>
Cc: Marc Zyngier <marc.zyngier@arm.com>
Cc: Gleb Natapov <gleb@kernel.org>
Cc: Paolo Bonzini <pbonzini@redhat.com>
Cc: kvmarm@lists.cs.columbia.edu
Cc: kvm@vger.kernel.org
Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
---
 include/linux/cpuhotplug.h |    1 +
 virt/kvm/arm/arch_timer.c  |   35 +++++++++++------------------------
 2 files changed, 12 insertions(+), 24 deletions(-)

--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -48,6 +48,7 @@ enum cpuhp_state {
 	CPUHP_AP_MIPS_GIC_TIMER_STARTING,
 	CPUHP_AP_KVM_STARTING,
 	CPUHP_AP_KVM_ARM_VGIC_STARTING,
+	CPUHP_AP_KVM_ARM_TIMER_STARTING,
 	CPUHP_AP_LEDTRIG_STARTING,
 	CPUHP_AP_NOTIFY_STARTING,
 	CPUHP_AP_ONLINE,
--- a/virt/kvm/arm/arch_timer.c
+++ b/virt/kvm/arm/arch_timer.c
@@ -405,26 +405,17 @@ u64 kvm_arm_timer_get_reg(struct kvm_vcp
 	return (u64)-1;
 }
 
-static int kvm_timer_cpu_notify(struct notifier_block *self,
-				unsigned long action, void *cpu)
+static int kvm_timer_starting_cpu(unsigned int cpu)
 {
-	switch (action) {
-	case CPU_STARTING:
-	case CPU_STARTING_FROZEN:
-		kvm_timer_init_interrupt(NULL);
-		break;
-	case CPU_DYING:
-	case CPU_DYING_FROZEN:
-		disable_percpu_irq(host_vtimer_irq);
-		break;
-	}
-
-	return NOTIFY_OK;
+	kvm_timer_init_interrupt(NULL);
+	return 0;
 }
 
-static struct notifier_block kvm_timer_cpu_nb = {
-	.notifier_call = kvm_timer_cpu_notify,
-};
+static int kvm_timer_dying_cpu(unsigned int cpu)
+{
+	disable_percpu_irq(host_vtimer_irq);
+	return 0;
+}
 
 int kvm_timer_hyp_init(void)
 {
@@ -449,12 +440,6 @@ int kvm_timer_hyp_init(void)
 		goto out;
 	}
 
-	err = __register_cpu_notifier(&kvm_timer_cpu_nb);
-	if (err) {
-		kvm_err("Cannot register timer CPU notifier\n");
-		goto out_free;
-	}
-
 	wqueue = create_singlethread_workqueue("kvm_arch_timer");
 	if (!wqueue) {
 		err = -ENOMEM;
@@ -462,8 +447,10 @@ int kvm_timer_hyp_init(void)
 	}
 
 	kvm_info("virtual timer IRQ%d\n", host_vtimer_irq);
-	on_each_cpu(kvm_timer_init_interrupt, NULL, 1);
 
+	cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
+			  "AP_KVM_ARM_TIMER_STARTING", kvm_timer_starting_cpu,
+			  kvm_timer_dying_cpu);
 	goto out;
 out_free:
 	free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());

Re: [patch 31/66] x86/kvm/kvmclock: Convert to hotplug state machine

[Anna-Maria Gleixner]

On Mon, 11 Jul 2016, Anna-Maria Gleixner wrote:

> @@ -5896,7 +5877,7 @@ void kvm_arch_exit(void)
>  	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
>  		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
>  					    CPUFREQ_TRANSITION_NOTIFIER);
> -	unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
> +	cpuhp_remove_state_nocalls(CPUHP_X86_KVM_CLK_ONLINE);

I'm sorry, this is buggy and wants to be:

    cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);

See delta patch below.

    Anna-Maria

8<-----------------

--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5877,7 +5877,7 @@ void kvm_arch_exit(void)
 	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
 		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
 					    CPUFREQ_TRANSITION_NOTIFIER);
-	cpuhp_remove_state_nocalls(CPUHP_X86_KVM_CLK_ONLINE);
+	cpuhp_remove_state_nocalls(CPUHP_AP_X86_KVM_CLK_ONLINE);
 #ifdef CONFIG_X86_64
 	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
 #endif

Re: [patch 31/66] x86/kvm/kvmclock: Convert to hotplug state machine

[Paolo Bonzini]

On 11/07/2016 14:28, Anna-Maria Gleixner wrote:
> From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> 
> Install the callbacks via the state machine and let the core invoke
> the callbacks on the already online CPUs.
> 
> We assumed that the priority ordering was ment to invoke the online
> callback as the last step. In the original code this also invoked the
> down prepare callback as the last step. With the symmetric state
> machine the down prepare callback is now the first step.
> 
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Cc: Gleb Natapov <gleb@kernel.org>
> Cc: Paolo Bonzini <pbonzini@redhat.com>
> Cc: kvm@vger.kernel.org
> Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
> ---
>  arch/x86/kvm/x86.c         |   35 ++++++++---------------------------
>  include/linux/cpuhotplug.h |    1 +
>  2 files changed, 9 insertions(+), 27 deletions(-)
> 
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -5552,9 +5552,10 @@ int kvm_fast_pio_out(struct kvm_vcpu *vc
>  }
>  EXPORT_SYMBOL_GPL(kvm_fast_pio_out);
>  
> -static void tsc_bad(void *info)
> +static int kvmclock_cpu_down_prep(unsigned int cpu)
>  {
>  	__this_cpu_write(cpu_tsc_khz, 0);
> +	return 0;
>  }
>  
>  static void tsc_khz_changed(void *data)
> @@ -5659,35 +5660,18 @@ static struct notifier_block kvmclock_cp
>  	.notifier_call  = kvmclock_cpufreq_notifier
>  };
>  
> -static int kvmclock_cpu_notifier(struct notifier_block *nfb,
> -					unsigned long action, void *hcpu)
> +static int kvmclock_cpu_online(unsigned int cpu)
>  {
> -	unsigned int cpu = (unsigned long)hcpu;
> -
> -	switch (action) {
> -		case CPU_ONLINE:
> -		case CPU_DOWN_FAILED:
> -			tsc_khz_changed(NULL);
> -			break;
> -		case CPU_DOWN_PREPARE:
> -			tsc_bad(NULL);
> -			break;
> -	}
> -	return NOTIFY_OK;
> +	tsc_khz_changed(NULL);
> +	return 0;
>  }
>  
> -static struct notifier_block kvmclock_cpu_notifier_block = {
> -	.notifier_call  = kvmclock_cpu_notifier,
> -	.priority = -INT_MAX
> -};
> -
>  static void kvm_timer_init(void)
>  {
>  	int cpu;
>  
>  	max_tsc_khz = tsc_khz;
>  
> -	cpu_notifier_register_begin();
>  	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) {
>  #ifdef CONFIG_CPU_FREQ
>  		struct cpufreq_policy policy;
> @@ -5702,12 +5686,9 @@ static void kvm_timer_init(void)
>  					  CPUFREQ_TRANSITION_NOTIFIER);
>  	}
>  	pr_debug("kvm: max_tsc_khz = %ld\n", max_tsc_khz);
> -	for_each_online_cpu(cpu)
> -		smp_call_function_single(cpu, tsc_khz_changed, NULL, 1);
> -
> -	__register_hotcpu_notifier(&kvmclock_cpu_notifier_block);
> -	cpu_notifier_register_done();
>  
> +	cpuhp_setup_state(CPUHP_AP_X86_KVM_CLK_ONLINE, "AP_X86_KVM_CLK_ONLINE",
> +			  kvmclock_cpu_online, kvmclock_cpu_down_prep);
>  }
>  
>  static DEFINE_PER_CPU(struct kvm_vcpu *, current_vcpu);
> @@ -5896,7 +5877,7 @@ void kvm_arch_exit(void)
>  	if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC))
>  		cpufreq_unregister_notifier(&kvmclock_cpufreq_notifier_block,
>  					    CPUFREQ_TRANSITION_NOTIFIER);
> -	unregister_hotcpu_notifier(&kvmclock_cpu_notifier_block);
> +	cpuhp_remove_state_nocalls(CPUHP_X86_KVM_CLK_ONLINE);
>  #ifdef CONFIG_X86_64
>  	pvclock_gtod_unregister_notifier(&pvclock_gtod_notifier);
>  #endif
> --- a/include/linux/cpuhotplug.h
> +++ b/include/linux/cpuhotplug.h
> @@ -56,6 +56,7 @@ enum cpuhp_state {
>  	CPUHP_AP_ONLINE_DYN,
>  	CPUHP_AP_ONLINE_DYN_END		= CPUHP_AP_ONLINE_DYN + 30,
>  	CPUHP_AP_X86_HPET_ONLINE,
> +	CPUHP_AP_X86_KVM_CLK_ONLINE,
>  	CPUHP_AP_ACTIVE,
>  	CPUHP_ONLINE,
>  };
> 
> 
> 

Acked-by: Paolo Bonzini <pbonzini@redhat.com>

(I don't have problems with patches sent as attachments, but others might).

Re: [patch 35/66] virt: Convert kvm hotplug to state machine

[Paolo Bonzini]

On 11/07/2016 14:28, Anna-Maria Gleixner wrote:
> From: Thomas Gleixner <tglx@linutronix.de>
> 
> Install the callbacks via the state machine. The core won't invoke the
> callbacks on already online CPUs.
> 
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Reviewed-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> Cc: Gleb Natapov <gleb@kernel.org>
> Cc: Paolo Bonzini <pbonzini@redhat.com>
> Cc: kvm@vger.kernel.org
> Signed-off-by: Anna-Maria Gleixner <anna-maria@linutronix.de>
> 
> ---
>  include/linux/cpuhotplug.h |    1 +
>  virt/kvm/kvm_main.c        |   32 ++++++++------------------------
>  2 files changed, 9 insertions(+), 24 deletions(-)
> 
> --- a/include/linux/cpuhotplug.h
> +++ b/include/linux/cpuhotplug.h
> @@ -37,6 +37,7 @@ enum cpuhp_state {
>  	CPUHP_AP_PERF_XTENSA_STARTING,
>  	CPUHP_AP_ARM_VFP_STARTING,
>  	CPUHP_AP_PERF_ARM_STARTING,
> +	CPUHP_AP_KVM_STARTING,
>  	CPUHP_AP_NOTIFY_STARTING,
>  	CPUHP_AP_ONLINE,
>  	CPUHP_TEARDOWN_CPU,
> --- a/virt/kvm/kvm_main.c
> +++ b/virt/kvm/kvm_main.c
> @@ -3144,12 +3144,13 @@ static void hardware_enable_nolock(void
>  	}
>  }
>  
> -static void hardware_enable(void)
> +static int kvm_starting_cpu(unsigned int cpu)
>  {
>  	raw_spin_lock(&kvm_count_lock);
>  	if (kvm_usage_count)
>  		hardware_enable_nolock(NULL);
>  	raw_spin_unlock(&kvm_count_lock);
> +	return 0;
>  }
>  
>  static void hardware_disable_nolock(void *junk)
> @@ -3162,12 +3163,13 @@ static void hardware_disable_nolock(void
>  	kvm_arch_hardware_disable();
>  }
>  
> -static void hardware_disable(void)
> +static int kvm_dying_cpu(unsigned int cpu)
>  {
>  	raw_spin_lock(&kvm_count_lock);
>  	if (kvm_usage_count)
>  		hardware_disable_nolock(NULL);
>  	raw_spin_unlock(&kvm_count_lock);
> +	return 0;
>  }
>  
>  static void hardware_disable_all_nolock(void)
> @@ -3208,21 +3210,6 @@ static int hardware_enable_all(void)
>  	return r;
>  }
>  
> -static int kvm_cpu_hotplug(struct notifier_block *notifier, unsigned long val,
> -			   void *v)
> -{
> -	val &= ~CPU_TASKS_FROZEN;
> -	switch (val) {
> -	case CPU_DYING:
> -		hardware_disable();
> -		break;
> -	case CPU_STARTING:
> -		hardware_enable();
> -		break;
> -	}
> -	return NOTIFY_OK;
> -}
> -
>  static int kvm_reboot(struct notifier_block *notifier, unsigned long val,
>  		      void *v)
>  {
> @@ -3489,10 +3476,6 @@ int kvm_io_bus_unregister_dev(struct kvm
>  	return r;
>  }
>  
> -static struct notifier_block kvm_cpu_notifier = {
> -	.notifier_call = kvm_cpu_hotplug,
> -};
> -
>  static int kvm_debugfs_open(struct inode *inode, struct file *file,
>  			   int (*get)(void *, u64 *), int (*set)(void *, u64),
>  			   const char *fmt)
> @@ -3743,7 +3726,8 @@ int kvm_init(void *opaque, unsigned vcpu
>  			goto out_free_1;
>  	}
>  
> -	r = register_cpu_notifier(&kvm_cpu_notifier);
> +	r = cpuhp_setup_state_nocalls(CPUHP_AP_KVM_STARTING, "AP_KVM_STARTING",
> +				      kvm_starting_cpu, kvm_dying_cpu);
>  	if (r)
>  		goto out_free_2;
>  	register_reboot_notifier(&kvm_reboot_notifier);
> @@ -3797,7 +3781,7 @@ int kvm_init(void *opaque, unsigned vcpu
>  	kmem_cache_destroy(kvm_vcpu_cache);
>  out_free_3:
>  	unregister_reboot_notifier(&kvm_reboot_notifier);
> -	unregister_cpu_notifier(&kvm_cpu_notifier);
> +	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
>  out_free_2:
>  out_free_1:
>  	kvm_arch_hardware_unsetup();
> @@ -3820,7 +3804,7 @@ void kvm_exit(void)
>  	kvm_async_pf_deinit();
>  	unregister_syscore_ops(&kvm_syscore_ops);
>  	unregister_reboot_notifier(&kvm_reboot_notifier);
> -	unregister_cpu_notifier(&kvm_cpu_notifier);
> +	cpuhp_remove_state_nocalls(CPUHP_AP_KVM_STARTING);
>  	on_each_cpu(hardware_disable_nolock, NULL, 1);
>  	kvm_arch_hardware_unsetup();
>  	kvm_arch_exit();
> 
> 
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

Acked-by: Paolo Bonzini <pbonzini@redhat.com>