[PATCH v2 11/11] xen/hvmlite: Enable CPU on-/offlining

[Boris Ostrovsky <boris.ostrovsky@oracle.com>]

When offlining, we should properly clean up interrupts and wait
until hypervisor declares VCPU as down before cleaning up.

After VCPU that was previously offlined is brought back to life
we want to jump back to bare-metal entry points. It's a simple
jump on 64-bit but requires minor tweaking for 32-bit case.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/smp.c         |   35 +++++++++++++++++++++++++----------
 arch/x86/xen/xen-hvmlite.S |    8 ++++++++
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index a22cae2..d768c4e 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -143,7 +143,7 @@ static void xen_smp_intr_free(unsigned int cpu)
 		kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
 		per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
 	}
-	if (xen_hvm_domain())
+	if (xen_hvm_domain() && !xen_hvmlite)
 		return;
 
 	if (per_cpu(xen_irq_work, cpu).irq >= 0) {
@@ -585,7 +585,8 @@ static int xen_cpu_disable(void)
 
 static void xen_cpu_die(unsigned int cpu)
 {
-	while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
+	while ((xen_pv_domain() || xen_hvmlite) &&
+	       HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
 		__set_current_state(TASK_UNINTERRUPTIBLE);
 		schedule_timeout(HZ/10);
 	}
@@ -602,14 +603,28 @@ static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
 {
 	play_dead_common();
 	HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
-	cpu_bringup();
-	/*
-	 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
-	 * clears certain data that the cpu_idle loop (which called us
-	 * and that we return from) expects. The only way to get that
-	 * data back is to call:
-	 */
-	tick_nohz_idle_enter();
+
+	if (!xen_hvm_domain()) {
+		cpu_bringup();
+		/*
+		 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu
+		 * down) clears certain data that the cpu_idle loop (which
+		 * called us and that we return from) expects. The only way to
+		 * get that data back is to call:
+		 */
+		tick_nohz_idle_enter();
+	} else {
+		/*
+		 * For 64-bit we can jump directly to SMP entry point but for
+		 * 32-bit we need to disable paging and load boot GDT (just
+		 * like in cpu_initialize_context()).
+		 */
+#ifdef CONFIG_X86_64
+		asm("jmp secondary_startup_64");
+#else
+		asm("jmp hvmlite_smp_32_hp");
+#endif
+	}
 }
 
 #else /* !CONFIG_HOTPLUG_CPU */
diff --git a/arch/x86/xen/xen-hvmlite.S b/arch/x86/xen/xen-hvmlite.S
index 805e6a0..1db2e96 100644
--- a/arch/x86/xen/xen-hvmlite.S
+++ b/arch/x86/xen/xen-hvmlite.S
@@ -145,6 +145,14 @@ ENTRY(hvmlite_start_xen)
 #endif
 
 #ifdef CONFIG_X86_32
+ENTRY(hvmlite_smp_32_hp)
+	movl $_pa(initial_page_table), %eax
+	movl %eax, %cr3
+	ljmp $__KERNEL_CS,$_pa(5f)
+5:
+	movl $X86_CR0_PE, %eax
+	movl %eax, %cr0
+
 ENTRY(hvmlite_smp_32)
         mov $_pa(boot_gdt_descr), %eax
         lgdt (%eax)
-- 
1.7.1