[PATCH v2 1/6] xen/x86/pvh: Save %rbx in xen_pvh_early_cpu_init()

[Boris Ostrovsky <boris.ostrovsky@oracle.com>]

x86-64 ABI requires that functions preserve %rbx. When
xen_pvh_early_cpu_init() is executed on boot cpu it is invoked as a
function and 'cpuid' instruction will clobber %rbx. (This is not a
concern on secondary processors since there xen_pvh_early_cpu_init() is
the entry point and thus does not need to preserve registers.)

Since we cannot access stack on secondary processors (PTE's NX bit will
cause #GP on AMD --- see commit a2ef5dc2c7cb ("x86/xen: Set EFER.NX and
EFER.SCE in PVH guests")) we can't always save %rbx on the stack. We
work around this by creating a new entry point for those processors ---
xen_pvh_early_cpu_init_secondary(). Note that on secondary CPUs we will
load %rbx with garbage from the stack, which is OK.

As a side effect of these changes we don't need to save %rsi anymore,
since we can use %rbx instead of %rsi as a temporary register.

(We could store %rbx into another scratch register such as %r8 but since
we will need new entry point for 32-bit PVH anyway we go with this
approach for symmetry).

Konrad also sugested that with separate entry point for secondary
processors we don't need the second argument ('bool entry') to
xen_pvh_early_cpu_init[_secondary](). So it is dropped.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
Changes in v2:
* Dropped second argument to xen_pvh_early_cpu_init()

 arch/x86/xen/enlighten.c |  2 +-
 arch/x86/xen/smp.c       |  4 ++--
 arch/x86/xen/smp.h       |  8 ++++++--
 arch/x86/xen/xen-head.S  | 27 ++++++++++++++-------------
 4 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 0b95c9b..f8dc398 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1454,7 +1454,7 @@ static void __init xen_pvh_early_guest_init(void)
 
 	xen_have_vector_callback = 1;
 
-	xen_pvh_early_cpu_init(0, false);
+	xen_pvh_early_cpu_init(0);
 	xen_pvh_set_cr_flags(0);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 8648438..e53be3b 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -423,9 +423,9 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 		 * bit set. This means before DS/SS is touched, NX in
 		 * EFER must be set. Hence the following assembly glue code.
 		 */
-		ctxt->user_regs.eip = (unsigned long)xen_pvh_early_cpu_init;
+		ctxt->user_regs.eip =
+			(unsigned long)xen_pvh_early_cpu_init_secondary;
 		ctxt->user_regs.rdi = cpu;
-		ctxt->user_regs.rsi = true;  /* entry == true */
 	}
 #endif
 	ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
diff --git a/arch/x86/xen/smp.h b/arch/x86/xen/smp.h
index 963d62a..538c00d 100644
--- a/arch/x86/xen/smp.h
+++ b/arch/x86/xen/smp.h
@@ -9,9 +9,13 @@ extern void xen_send_IPI_all(int vector);
 extern void xen_send_IPI_self(int vector);
 
 #ifdef CONFIG_XEN_PVH
-extern void xen_pvh_early_cpu_init(int cpu, bool entry);
+extern void xen_pvh_early_cpu_init(int cpu);
+extern void xen_pvh_early_cpu_init_secondary(int cpu);
 #else
-static inline void xen_pvh_early_cpu_init(int cpu, bool entry)
+static inline void xen_pvh_early_cpu_init(int cpu)
+{
+}
+static inline void xen_pvh_early_cpu_init_secondary(int cpu)
 {
 }
 #endif
diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index 8afdfcc..944b08b 100644
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -53,31 +53,32 @@ ENTRY(startup_xen)
 /*
  * xen_pvh_early_cpu_init() - early PVH VCPU initialization
  * @cpu:   this cpu number (%rdi)
- * @entry: true if this is a secondary vcpu coming up on this entry
- *         point, false if this is the boot CPU being initialized for
- *         the first time (%rsi)
- *
- * Note: This is called as a function on the boot CPU, and is the entry point
- *       on the secondary CPU.
  */
 ENTRY(xen_pvh_early_cpu_init)
-	mov     %rsi, %r11
-
+	mov	%rbx, -8(%rsp)
+	xor	%esi, %esi
+	jmp	1f
+
+/* Entry point for secondary CPUs. Can't touch stack until NX is dealt with. */
+ENTRY(xen_pvh_early_cpu_init_secondary)
+	mov	$1, %esi
+1:
 	/* Gather features to see if NX implemented. */
 	mov     $0x80000001, %eax
 	cpuid
-	mov     %edx, %esi
+	mov     %edx, %ebx
 
 	mov     $MSR_EFER, %ecx
 	rdmsr
 	bts     $_EFER_SCE, %eax
 
-	bt      $20, %esi
-	jnc     1f      	/* No NX, skip setting it */
+	bt      $20, %ebx
+	jnc     2f      	/* No NX, skip setting it */
 	bts     $_EFER_NX, %eax
-1:	wrmsr
+2:	wrmsr
+	mov	-8(%rsp), %rbx
 #ifdef CONFIG_SMP
-	cmp     $0, %r11b
+	cmp     $0, %esi
 	jne     cpu_bringup_and_idle
 #endif
 	ret
-- 
1.8.1.4