[PATCH v1 00/12] HVMlite domU support

[PATCH v1 00/12] HVMlite domU support

[Boris Ostrovsky]

This series introduces HVMlite support for unprivileged guests.

It has been tested on Intel/AMD, both 32- and 64-bit, including CPU on- and
offlining and save/restore. (Restore will result in APIC write warnings
which exist now for 32-bit PV guests as well so I didn't address this in
this series)

Compile-tested on ARM

Boris Ostrovsky (12):
  x86/smp: Make start_secondary() and initial_pg_pmd visible globally
  xen/hvmlite: Factor out common kernel init code
  xen/hvmlite: Import hvmlite-related Xen public interfaces
  xen/hvmlite: Bootstrap HVMlite guest
  xen/hvmlite: Allow HVMlite guests delay initializing grant table
  xen/hvmlite: Initialize PCI
  xen/hvmlite: Prepare cpu_initialize_context() routine for HVMlite SMP
  xen/hvmlite: Initialize context for secondary VCPUs
  xen/hvmlite: Extend APIC operations for HVMlite guests
  xen/hvmlite: Use x86's default timer init for HVMlite guests
  xen/hvmlite: Boot secondary CPUs
  xen/hvmlite: Enable CPU on-/offlining

 arch/x86/include/asm/smp.h           |    1 +
 arch/x86/kernel/head_32.S            |    2 +-
 arch/x86/kernel/smpboot.c            |    2 +-
 arch/x86/pci/xen.c                   |    2 +-
 arch/x86/xen/Makefile                |    1 +
 arch/x86/xen/apic.c                  |   39 ++++-
 arch/x86/xen/enlighten.c             |  318 ++++++++++++++++++++++------------
 arch/x86/xen/grant-table.c           |    4 +-
 arch/x86/xen/platform-pci-unplug.c   |    4 +-
 arch/x86/xen/pmu.c                   |    4 +-
 arch/x86/xen/smp.c                   |  248 ++++++++++++++++++--------
 arch/x86/xen/smp.h                   |    4 +
 arch/x86/xen/time.c                  |    5 +-
 arch/x86/xen/xen-hvmlite.S           |  173 ++++++++++++++++++
 drivers/xen/grant-table.c            |    8 +-
 include/xen/interface/elfnote.h      |   12 ++-
 include/xen/interface/hvm/hvm_vcpu.h |  143 +++++++++++++++
 include/xen/interface/xen.h          |   24 +++
 include/xen/xen.h                    |    6 +
 19 files changed, 801 insertions(+), 199 deletions(-)
 create mode 100644 arch/x86/xen/xen-hvmlite.S
 create mode 100644 include/xen/interface/hvm/hvm_vcpu.h

[PATCH v1 01/12] x86/smp: Make start_secondary() and initial_pg_pmd visible globally

[Boris Ostrovsky]

Xen's HVMlite guests will want to use them.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/include/asm/smp.h |    1 +
 arch/x86/kernel/head_32.S  |    2 +-
 arch/x86/kernel/smpboot.c  |    2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index dfcf072..739abb0 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -147,6 +147,7 @@ void x86_idle_thread_init(unsigned int cpu, struct task_struct *idle);
 
 void smp_store_boot_cpu_info(void);
 void smp_store_cpu_info(int id);
+void start_secondary(void *unused);
 #define cpu_physical_id(cpu)	per_cpu(x86_cpu_to_apicid, cpu)
 
 #else /* !CONFIG_SMP */
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index 6bc9ae2..eeb5c42 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -666,7 +666,7 @@ ENTRY(setup_once_ref)
 __PAGE_ALIGNED_BSS
 	.align PAGE_SIZE
 #ifdef CONFIG_X86_PAE
-initial_pg_pmd:
+ENTRY(initial_pg_pmd)
 	.fill 1024*KPMDS,4,0
 #else
 ENTRY(initial_page_table)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 24d57f7..cba00bc 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -199,7 +199,7 @@ static int enable_start_cpu0;
 /*
  * Activate a secondary processor.
  */
-static void notrace start_secondary(void *unused)
+void notrace start_secondary(void *unused)
 {
 	/*
 	 * Don't put *anything* before cpu_init(), SMP booting is too
-- 
1.7.1

[PATCH v1 02/12] xen/hvmlite: Factor out common kernel init code

[Boris Ostrovsky]

HVMlite guests (to be introduced in subsequent patches) share most
of the kernel initialization code with PV(H).

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/enlighten.c |  225 ++++++++++++++++++++++++----------------------
 1 files changed, 119 insertions(+), 106 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index d09e4c9..2cf446a 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1505,19 +1505,14 @@ static void __init xen_pvh_early_guest_init(void)
 }
 #endif    /* CONFIG_XEN_PVH */
 
-/* First C function to be called on Xen boot */
-asmlinkage __visible void __init xen_start_kernel(void)
+
+static void __init xen_init_kernel(void)
 {
 	struct physdev_set_iopl set_iopl;
 	unsigned long initrd_start = 0;
 	u64 pat;
 	int rc;
 
-	if (!xen_start_info)
-		return;
-
-	xen_domain_type = XEN_PV_DOMAIN;
-
 	xen_setup_features();
 #ifdef CONFIG_XEN_PVH
 	xen_pvh_early_guest_init();
@@ -1529,48 +1524,9 @@ asmlinkage __visible void __init xen_start_kernel(void)
 	if (xen_initial_domain())
 		pv_info.features |= PV_SUPPORTED_RTC;
 	pv_init_ops = xen_init_ops;
-	if (!xen_pvh_domain()) {
-		pv_cpu_ops = xen_cpu_ops;
-
-		x86_platform.get_nmi_reason = xen_get_nmi_reason;
-	}
-
-	if (xen_feature(XENFEAT_auto_translated_physmap))
-		x86_init.resources.memory_setup = xen_auto_xlated_memory_setup;
-	else
-		x86_init.resources.memory_setup = xen_memory_setup;
-	x86_init.oem.arch_setup = xen_arch_setup;
-	x86_init.oem.banner = xen_banner;
 
 	xen_init_time_ops();
 
-	/*
-	 * Set up some pagetable state before starting to set any ptes.
-	 */
-
-	xen_init_mmu_ops();
-
-	/* Prevent unwanted bits from being set in PTEs. */
-	__supported_pte_mask &= ~_PAGE_GLOBAL;
-
-	/*
-	 * Prevent page tables from being allocated in highmem, even
-	 * if CONFIG_HIGHPTE is enabled.
-	 */
-	__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
-
-	/* Work out if we support NX */
-	x86_configure_nx();
-
-	/* Get mfn list */
-	xen_build_dynamic_phys_to_machine();
-
-	/*
-	 * Set up kernel GDT and segment registers, mainly so that
-	 * -fstack-protector code can be executed.
-	 */
-	xen_setup_gdt(0);
-
 	xen_init_irq_ops();
 	xen_init_cpuid_mask();
 
@@ -1580,21 +1536,8 @@ asmlinkage __visible void __init xen_start_kernel(void)
 	 */
 	xen_init_apic();
 #endif
-
-	if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
-		pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
-		pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
-	}
-
 	machine_ops = xen_machine_ops;
 
-	/*
-	 * The only reliable way to retain the initial address of the
-	 * percpu gdt_page is to remember it here, so we can go and
-	 * mark it RW later, when the initial percpu area is freed.
-	 */
-	xen_initial_gdt = &per_cpu(gdt_page, 0);
-
 	xen_smp_init();
 
 #ifdef CONFIG_ACPI_NUMA
@@ -1609,66 +1552,126 @@ asmlinkage __visible void __init xen_start_kernel(void)
 	   possible map and a non-dummy shared_info. */
 	per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
 
-	local_irq_disable();
-	early_boot_irqs_disabled = true;
+	if (!xen_hvm_domain()) {
+		if (!xen_pvh_domain()) {
+			pv_cpu_ops = xen_cpu_ops;
 
-	xen_raw_console_write("mapping kernel into physical memory\n");
-	xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base,
-				   xen_start_info->nr_pages);
-	xen_reserve_special_pages();
+			x86_platform.get_nmi_reason = xen_get_nmi_reason;
+		}
 
-	/*
-	 * Modify the cache mode translation tables to match Xen's PAT
-	 * configuration.
-	 */
-	rdmsrl(MSR_IA32_CR_PAT, pat);
-	pat_init_cache_modes(pat);
+		if (xen_feature(XENFEAT_auto_translated_physmap))
+			x86_init.resources.memory_setup =
+				xen_auto_xlated_memory_setup;
+		else
+			x86_init.resources.memory_setup = xen_memory_setup;
+		x86_init.oem.arch_setup = xen_arch_setup;
+		x86_init.oem.banner = xen_banner;
 
-	/* keep using Xen gdt for now; no urgent need to change it */
+		/*
+		 * Set up some pagetable state before starting to set any ptes.
+		 */
 
-#ifdef CONFIG_X86_32
-	pv_info.kernel_rpl = 1;
-	if (xen_feature(XENFEAT_supervisor_mode_kernel))
-		pv_info.kernel_rpl = 0;
-#else
-	pv_info.kernel_rpl = 0;
-#endif
-	/* set the limit of our address space */
-	xen_reserve_top();
+		xen_init_mmu_ops();
+
+		/* Prevent unwanted bits from being set in PTEs. */
+		__supported_pte_mask &= ~_PAGE_GLOBAL;
 
-	/* PVH: runs at default kernel iopl of 0 */
-	if (!xen_pvh_domain()) {
 		/*
-		 * We used to do this in xen_arch_setup, but that is too late
-		 * on AMD were early_cpu_init (run before ->arch_setup()) calls
-		 * early_amd_init which pokes 0xcf8 port.
+		 * Prevent page tables from being allocated in highmem, even
+		 * if CONFIG_HIGHPTE is enabled.
 		 */
-		set_iopl.iopl = 1;
-		rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
-		if (rc != 0)
-			xen_raw_printk("physdev_op failed %d\n", rc);
-	}
+		__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
+
+		/* Work out if we support NX */
+		x86_configure_nx();
+
+		/* Get mfn list */
+		xen_build_dynamic_phys_to_machine();
+
+		/*
+		 * Set up kernel GDT and segment registers, mainly so that
+		 * -fstack-protector code can be executed.
+		 */
+		xen_setup_gdt(0);
+
+		if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
+			pv_mmu_ops.ptep_modify_prot_start =
+				xen_ptep_modify_prot_start;
+			pv_mmu_ops.ptep_modify_prot_commit =
+				xen_ptep_modify_prot_commit;
+		}
+
+		/*
+		 * The only reliable way to retain the initial address of the
+		 * percpu gdt_page is to remember it here, so we can go and
+		 * mark it RW later, when the initial percpu area is freed.
+		 */
+		xen_initial_gdt = &per_cpu(gdt_page, 0);
+
+		xen_raw_console_write("mapping kernel into physical memory\n");
+		xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base,
+					   xen_start_info->nr_pages);
+		xen_reserve_special_pages();
+
+		/*
+		 * Modify the cache mode translation tables to match Xen's PAT
+		 * configuration.
+		 */
+		rdmsrl(MSR_IA32_CR_PAT, pat);
+		pat_init_cache_modes(pat);
+
+		/* keep using Xen gdt for now; no urgent need to change it */
 
 #ifdef CONFIG_X86_32
-	/* set up basic CPUID stuff */
-	cpu_detect(&new_cpu_data);
-	set_cpu_cap(&new_cpu_data, X86_FEATURE_FPU);
-	new_cpu_data.wp_works_ok = 1;
-	new_cpu_data.x86_capability[0] = cpuid_edx(1);
+		pv_info.kernel_rpl = 1;
+		if (xen_feature(XENFEAT_supervisor_mode_kernel))
+			pv_info.kernel_rpl = 0;
+#else
+		pv_info.kernel_rpl = 0;
+#endif
+		/* set the limit of our address space */
+		xen_reserve_top();
+
+		/* PVH: runs at default kernel iopl of 0 */
+		if (!xen_pvh_domain()) {
+			/*
+			 * We used to do this in xen_arch_setup, but that is
+			 * too late on AMD were early_cpu_init (run before
+			 * ->arch_setup()) calls early_amd_init which pokes
+			 * 0xcf8 port.
+			 */
+			set_iopl.iopl = 1;
+			rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl,
+						   &set_iopl);
+			if (rc != 0)
+				xen_raw_printk("physdev_op failed %d\n", rc);
+		}
+
+#ifdef CONFIG_X86_32
+		/* set up basic CPUID stuff */
+		cpu_detect(&new_cpu_data);
+		set_cpu_cap(&new_cpu_data, X86_FEATURE_FPU);
+		new_cpu_data.wp_works_ok = 1;
+		new_cpu_data.x86_capability[0] = cpuid_edx(1);
 #endif
 
-	if (xen_start_info->mod_start) {
-	    if (xen_start_info->flags & SIF_MOD_START_PFN)
-		initrd_start = PFN_PHYS(xen_start_info->mod_start);
-	    else
-		initrd_start = __pa(xen_start_info->mod_start);
+		if (xen_start_info->mod_start) {
+			if (xen_start_info->flags & SIF_MOD_START_PFN)
+				initrd_start =
+					PFN_PHYS(xen_start_info->mod_start);
+			else
+				initrd_start = __pa(xen_start_info->mod_start);
+		}
+
+		/* Poke various useful things into boot_params */
+		boot_params.hdr.type_of_loader = (9 << 4) | 0;
+		boot_params.hdr.ramdisk_image = initrd_start;
+		boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
+		boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
 	}
 
-	/* Poke various useful things into boot_params */
-	boot_params.hdr.type_of_loader = (9 << 4) | 0;
-	boot_params.hdr.ramdisk_image = initrd_start;
-	boot_params.hdr.ramdisk_size = xen_start_info->mod_len;
-	boot_params.hdr.cmd_line_ptr = __pa(xen_start_info->cmd_line);
+	local_irq_disable();
+	early_boot_irqs_disabled = true;
 
 	if (!xen_initial_domain()) {
 		add_preferred_console("xenboot", 0, NULL);
@@ -1713,6 +1716,16 @@ asmlinkage __visible void __init xen_start_kernel(void)
 	xen_setup_runstate_info(0);
 
 	xen_efi_init();
+}
+
+/* First C function to be called on Xen boot */
+asmlinkage __visible void __init xen_start_kernel(void)
+{
+	if (!xen_start_info)
+		return;
+	xen_domain_type = XEN_PV_DOMAIN;
+
+	xen_init_kernel();
 
 	/* Start the world */
 #ifdef CONFIG_X86_32
-- 
1.7.1

[PATCH v1 03/12] xen/hvmlite: Import hvmlite-related Xen public interfaces

[Boris Ostrovsky]

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 include/xen/interface/elfnote.h      |   12 +++-
 include/xen/interface/hvm/hvm_vcpu.h |  143 ++++++++++++++++++++++++++++++++++
 include/xen/interface/xen.h          |   24 ++++++
 3 files changed, 178 insertions(+), 1 deletions(-)
 create mode 100644 include/xen/interface/hvm/hvm_vcpu.h

diff --git a/include/xen/interface/elfnote.h b/include/xen/interface/elfnote.h
index f90b034..9e9f9bf 100644
--- a/include/xen/interface/elfnote.h
+++ b/include/xen/interface/elfnote.h
@@ -193,9 +193,19 @@
 #define XEN_ELFNOTE_SUPPORTED_FEATURES 17
 
 /*
+ * Physical entry point into the kernel.
+ *
+ * 32bit entry point into the kernel. When requested to launch the
+ * guest kernel in a HVM container, Xen will use this entry point to
+ * launch the guest in 32bit protected mode with paging disabled.
+ * Ignored otherwise.
+ */
+#define XEN_ELFNOTE_PHYS32_ENTRY 18
+
+/*
  * The number of the highest elfnote defined.
  */
-#define XEN_ELFNOTE_MAX XEN_ELFNOTE_SUPPORTED_FEATURES
+#define XEN_ELFNOTE_MAX XEN_ELFNOTE_PHYS32_ENTRY
 
 #endif /* __XEN_PUBLIC_ELFNOTE_H__ */
 
diff --git a/include/xen/interface/hvm/hvm_vcpu.h b/include/xen/interface/hvm/hvm_vcpu.h
new file mode 100644
index 0000000..32ca83e
--- /dev/null
+++ b/include/xen/interface/hvm/hvm_vcpu.h
@@ -0,0 +1,143 @@
+/*
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Copyright (c) 2015, Roger Pau Monne <roger.pau@citrix.com>
+ */
+
+#ifndef __XEN_PUBLIC_HVM_HVM_VCPU_H__
+#define __XEN_PUBLIC_HVM_HVM_VCPU_H__
+
+#include "../xen.h"
+
+struct vcpu_hvm_x86_32 {
+    uint32_t eax;
+    uint32_t ecx;
+    uint32_t edx;
+    uint32_t ebx;
+    uint32_t esp;
+    uint32_t ebp;
+    uint32_t esi;
+    uint32_t edi;
+    uint32_t eip;
+    uint32_t eflags;
+
+    uint32_t cr0;
+    uint32_t cr3;
+    uint32_t cr4;
+
+    uint32_t pad1;
+
+    /*
+     * EFER should only be used to set the NXE bit (if required)
+     * when starting a vCPU in 32bit mode with paging enabled or
+     * to set the LME/LMA bits in order to start the vCPU in
+     * compatibility mode.
+     */
+    uint64_t efer;
+
+    uint32_t cs_base;
+    uint32_t ds_base;
+    uint32_t ss_base;
+    uint32_t es_base;
+    uint32_t tr_base;
+    uint32_t cs_limit;
+    uint32_t ds_limit;
+    uint32_t ss_limit;
+    uint32_t es_limit;
+    uint32_t tr_limit;
+    uint16_t cs_ar;
+    uint16_t ds_ar;
+    uint16_t ss_ar;
+    uint16_t es_ar;
+    uint16_t tr_ar;
+
+    uint16_t pad2[3];
+};
+
+/*
+ * The layout of the _ar fields of the segment registers is the
+ * following:
+ *
+ * Bits   [0,3]: type (bits 40-43).
+ * Bit        4: s    (descriptor type, bit 44).
+ * Bit    [5,6]: dpl  (descriptor privilege level, bits 45-46).
+ * Bit        7: p    (segment-present, bit 47).
+ * Bit        8: avl  (available for system software, bit 52).
+ * Bit        9: l    (64-bit code segment, bit 53).
+ * Bit       10: db   (meaning depends on the segment, bit 54).
+ * Bit       11: g    (granularity, bit 55)
+ * Bits [12,15]: unused, must be blank.
+ *
+ * A more complete description of the meaning of this fields can be
+ * obtained from the Intel SDM, Volume 3, section 3.4.5.
+ */
+
+struct vcpu_hvm_x86_64 {
+    uint64_t rax;
+    uint64_t rcx;
+    uint64_t rdx;
+    uint64_t rbx;
+    uint64_t rsp;
+    uint64_t rbp;
+    uint64_t rsi;
+    uint64_t rdi;
+    uint64_t rip;
+    uint64_t rflags;
+
+    uint64_t cr0;
+    uint64_t cr3;
+    uint64_t cr4;
+    uint64_t efer;
+
+    /*
+     * Using VCPU_HVM_MODE_64B implies that the vCPU is launched
+     * directly in long mode, so the cached parts of the segment
+     * registers get set to match that environment.
+     *
+     * If the user wants to launch the vCPU in compatibility mode
+     * the 32-bit structure should be used instead.
+     */
+};
+
+struct vcpu_hvm_context {
+#define VCPU_HVM_MODE_32B 0  /* 32bit fields of the structure will be used. */
+#define VCPU_HVM_MODE_64B 1  /* 64bit fields of the structure will be used. */
+    uint32_t mode;
+
+    uint32_t pad;
+
+    /* CPU registers. */
+    union {
+        struct vcpu_hvm_x86_32 x86_32;
+        struct vcpu_hvm_x86_64 x86_64;
+    } cpu_regs;
+};
+typedef struct vcpu_hvm_context vcpu_hvm_context_t;
+
+#endif /* __XEN_PUBLIC_HVM_HVM_VCPU_H__ */
+
+/*
+ * Local variables:
+ * mode: C
+ * c-file-style: "BSD"
+ * c-basic-offset: 4
+ * tab-width: 4
+ * indent-tabs-mode: nil
+ * End:
+ */
diff --git a/include/xen/interface/xen.h b/include/xen/interface/xen.h
index d133112..61a25c4 100644
--- a/include/xen/interface/xen.h
+++ b/include/xen/interface/xen.h
@@ -645,6 +645,30 @@ struct start_info {
 	unsigned long nr_p2m_frames;/* # of pfns forming initial P->M table.  */
 };
 
+/*
+ * Start of day structure passed to PVH guests in %ebx.
+ *
+ * NOTE: nothing will be loaded at physical address 0, so
+ * a 0 value in any of the address fields should be treated
+ * as not present.
+ */
+struct hvm_start_info {
+#define HVM_START_MAGIC_VALUE 0x336ec578
+    uint32_t magic;             /* Contains the magic value 0x336ec578       */
+                                /* ("xEn3" with the 0x80 bit of the "E" set).*/
+    uint32_t flags;             /* SIF_xxx flags.                            */
+    uint32_t cmdline_paddr;     /* Physical address of the command line.     */
+    uint32_t nr_modules;        /* Number of modules passed to the kernel.   */
+    uint32_t modlist_paddr;     /* Physical address of an array of           */
+                                /* hvm_modlist_entry.                        */
+};
+
+struct hvm_modlist_entry {
+    uint32_t paddr;             /* Physical address of the module.           */
+    uint32_t size;              /* Size of the module in bytes.              */
+};
+
+
 /* These flags are passed in the 'flags' field of start_info_t. */
 #define SIF_PRIVILEGED      (1<<0)  /* Is the domain privileged? */
 #define SIF_INITDOMAIN      (1<<1)  /* Is this the initial control domain? */
-- 
1.7.1

[PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

Start HVMlite guest XEN_ELFNOTE_PHYS32_ENTRY address. Setup hypercall
page, initialize boot_params, enable early page tables.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/Makefile      |    1 +
 arch/x86/xen/enlighten.c   |   91 +++++++++++++++++++++++++-
 arch/x86/xen/xen-hvmlite.S |  158 ++++++++++++++++++++++++++++++++++++++++++++
 include/xen/xen.h          |    6 ++
 4 files changed, 255 insertions(+), 1 deletions(-)
 create mode 100644 arch/x86/xen/xen-hvmlite.S

diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile
index e47e527..1d913d7 100644
--- a/arch/x86/xen/Makefile
+++ b/arch/x86/xen/Makefile
@@ -23,3 +23,4 @@ obj-$(CONFIG_XEN_DEBUG_FS)	+= debugfs.o
 obj-$(CONFIG_XEN_DOM0)		+= vga.o
 obj-$(CONFIG_SWIOTLB_XEN)	+= pci-swiotlb-xen.o
 obj-$(CONFIG_XEN_EFI)		+= efi.o
+obj-$(CONFIG_XEN_PVHVM) 	+= xen-hvmlite.o
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2cf446a..2ed8b2b 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -118,7 +118,8 @@ DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
  */
 DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
 
-enum xen_domain_type xen_domain_type = XEN_NATIVE;
+enum xen_domain_type xen_domain_type
+	__attribute__((section(".data"))) = XEN_NATIVE;
 EXPORT_SYMBOL_GPL(xen_domain_type);
 
 unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
@@ -171,6 +172,17 @@ struct tls_descs {
  */
 static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc);
 
+#ifdef CONFIG_XEN_PVHVM
+/*
+ * HVMlite variables. These need to live in data segment since they are
+ * initialized before startup_{32|64}, which clear .bss, are invoked.
+ */
+int xen_hvmlite __attribute__((section(".data"))) = 0;
+struct hvm_start_info hvmlite_start_info __attribute__((section(".data")));
+uint hvmlite_start_info_sz = sizeof(hvmlite_start_info);
+struct boot_params xen_hvmlite_boot_params __attribute__((section(".data")));
+#endif
+
 static void clamp_max_cpus(void)
 {
 #ifdef CONFIG_SMP
@@ -1736,6 +1748,83 @@ asmlinkage __visible void __init xen_start_kernel(void)
 #endif
 }
 
+#ifdef CONFIG_XEN_PVHVM
+static void __init hvmlite_bootparams(void)
+{
+	struct xen_memory_map memmap;
+	int i;
+
+	memset(&xen_hvmlite_boot_params, 0, sizeof(xen_hvmlite_boot_params));
+
+	memmap.nr_entries = ARRAY_SIZE(xen_hvmlite_boot_params.e820_map);
+	set_xen_guest_handle(memmap.buffer, xen_hvmlite_boot_params.e820_map);
+	if (HYPERVISOR_memory_op(XENMEM_memory_map, &memmap)) {
+		xen_raw_console_write("XENMEM_memory_map failed\n");
+		BUG();
+	}
+
+	xen_hvmlite_boot_params.e820_map[memmap.nr_entries].addr =
+		ISA_START_ADDRESS;
+	xen_hvmlite_boot_params.e820_map[memmap.nr_entries].size =
+		ISA_END_ADDRESS - ISA_START_ADDRESS;
+	xen_hvmlite_boot_params.e820_map[memmap.nr_entries++].type =
+		E820_RESERVED;
+
+	sanitize_e820_map(xen_hvmlite_boot_params.e820_map,
+			  ARRAY_SIZE(xen_hvmlite_boot_params.e820_map),
+			  &memmap.nr_entries);
+
+	xen_hvmlite_boot_params.e820_entries = memmap.nr_entries;
+	for (i = 0; i < xen_hvmlite_boot_params.e820_entries; i++)
+		e820_add_region(xen_hvmlite_boot_params.e820_map[i].addr,
+				xen_hvmlite_boot_params.e820_map[i].size,
+				xen_hvmlite_boot_params.e820_map[i].type);
+
+	xen_hvmlite_boot_params.hdr.cmd_line_ptr =
+		hvmlite_start_info.cmdline_paddr;
+
+	/* The first module is always ramdisk */
+	if (hvmlite_start_info.nr_modules) {
+		struct hvm_modlist_entry *modaddr =
+			__va(hvmlite_start_info.modlist_paddr);
+		xen_hvmlite_boot_params.hdr.ramdisk_image = modaddr->paddr;
+		xen_hvmlite_boot_params.hdr.ramdisk_size = modaddr->size;
+	}
+
+	/*
+	 * See Documentation/x86/boot.txt.
+	 *
+	 * Version 2.12 supports Xen entry point but we will use default x86/PC
+	 * environment (i.e. hardware_subarch 0).
+	 */
+	xen_hvmlite_boot_params.hdr.version = 0x212;
+	xen_hvmlite_boot_params.hdr.type_of_loader = 9; /* Xen loader */
+}
+
+/*
+ * This routine (and those that it might call) should not use
+ * anything that lives in .bss since that segment will be cleared later
+ */
+void __init xen_prepare_hvmlite(void)
+{
+	u32 eax, ecx, edx, msr;
+	u64 pfn;
+
+	cpuid(xen_cpuid_base() + 2, &eax, &msr, &ecx, &edx);
+	pfn = __pa(hypercall_page);
+	wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));
+
+	pv_info.name = "Xen HVMlite";
+	xen_domain_type = XEN_HVM_DOMAIN;
+	xen_hvmlite = 1;
+
+	x86_init.oem.arch_setup = xen_init_kernel;
+	x86_init.oem.banner = xen_banner;
+
+	hvmlite_bootparams();
+}
+#endif
+
 void __ref xen_hvm_init_shared_info(void)
 {
 	int cpu;
diff --git a/arch/x86/xen/xen-hvmlite.S b/arch/x86/xen/xen-hvmlite.S
new file mode 100644
index 0000000..90f03d0
--- /dev/null
+++ b/arch/x86/xen/xen-hvmlite.S
@@ -0,0 +1,158 @@
+/*
+ * Copyright C 2016, Oracle and/or its affiliates. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+	.code32
+	.text
+#define _pa(x)          ((x) - __START_KERNEL_map)
+
+#include <linux/elfnote.h>
+#include <linux/init.h>
+#include <linux/linkage.h>
+#include <asm/segment.h>
+#include <asm/asm.h>
+#include <asm/boot.h>
+#include <asm/processor-flags.h>
+#include <asm/msr.h>
+#include <xen/interface/elfnote.h>
+
+	__HEAD
+	.code32
+
+/* Entry point for HVMlite guests */
+ENTRY(hvmlite_start_xen)
+	cli
+	cld
+
+	mov $_pa(gdt), %eax
+	lgdt (%eax)
+
+	movl $(__BOOT_DS),%eax
+	movl %eax,%ds
+	movl %eax,%es
+	movl %eax,%ss
+
+	/* Stash hvm_start_info */
+	mov $_pa(hvmlite_start_info), %edi
+	mov %ebx, %esi
+	mov $_pa(hvmlite_start_info_sz), %ecx
+	mov (%ecx), %ecx
+	rep
+	movsb
+
+	movl $_pa(early_stack_end), %eax
+	movl %eax, %esp
+
+	/* Enable PAE mode */
+	movl %cr4, %eax
+	orl $X86_CR4_PAE, %eax
+	movl %eax, %cr4
+
+#ifdef CONFIG_X86_64
+	/* Enable Long mode */
+	movl $MSR_EFER, %ecx
+	rdmsr
+	btsl $_EFER_LME, %eax
+	wrmsr
+
+	/* Enable pre-constructed page tables */
+	mov $_pa(init_level4_pgt), %eax
+	movl %eax, %cr3
+	movl $(X86_CR0_PG | X86_CR0_PE), %eax
+	movl %eax, %cr0
+
+	/* Jump to 64-bit mode. */
+	pushl $__KERNEL_CS
+	leal _pa(1f), %eax
+	pushl %eax
+	lret
+
+	/* 64-bit entry point */
+	.code64
+1:
+	call xen_prepare_hvmlite
+
+	/* startup_64 expects boot_params in %rsi */
+	mov $_pa(xen_hvmlite_boot_params), %rsi
+	movq $_pa(startup_64), %rax
+	jmp *%rax
+
+#else /* CONFIG_X86_64 */
+
+	/* Use initial_page table and set level 2 to map 2M pages */
+	movl $_pa(initial_pg_pmd), %edi
+	movl $(_PAGE_PSE | _PAGE_RW | _PAGE_PRESENT), %eax
+	movl $2048, %ecx
+2:
+	movl %eax, 0(%edi)
+	addl $0x00200000, %eax
+	addl $8, %edi
+	decl %ecx
+	jnz 2b
+
+	/* Enable the boot paging */
+	movl $_pa(initial_page_table), %eax
+	movl %eax, %cr3
+	movl %cr0, %eax
+	orl $(X86_CR0_PG | X86_CR0_PE), %eax
+	movl %eax, %cr0
+
+	ljmp $__BOOT_CS,$3f
+3:
+	call xen_prepare_hvmlite
+	mov $_pa(xen_hvmlite_boot_params), %esi
+
+	/* startup_32 doesn't expect paging and PAE to be on */
+	ljmp $__BOOT_CS,$_pa(4f)
+4:
+	movl %cr0, %eax
+	andl $~X86_CR0_PG, %eax
+	movl %eax, %cr0
+	movl %cr4, %eax
+	andl $~X86_CR4_PAE, %eax
+	movl %eax, %cr4
+
+	/* Restore initial_pg_pmd to its original (zero) state */
+	movl $_pa(initial_pg_pmd), %edi
+	xorl %eax, %eax
+	movl $(PAGE_SIZE/4), %ecx
+	rep stosl
+
+	ljmp    $0x10, $_pa(startup_32)
+#endif
+
+	.data
+gdt:
+	.word	gdt_end - gdt
+	.long	_pa(gdt)
+	.word	0
+	.quad	0x0000000000000000 /* NULL descriptor */
+#ifdef CONFIG_X86_64
+	.quad	0x00af9a000000ffff /* __KERNEL_CS */
+#else
+	.quad	0x00cf9a000000ffff /* __KERNEL_CS */
+#endif
+	.quad	0x00cf92000000ffff /* __KERNEL_DS */
+gdt_end:
+
+	.bss
+	.balign 4
+early_stack:
+	.fill 16, 1, 0
+early_stack_end:
+
+	ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY,
+	             _ASM_PTR (hvmlite_start_xen - __START_KERNEL_map))
diff --git a/include/xen/xen.h b/include/xen/xen.h
index 0c0e3ef..6a0d3f3 100644
--- a/include/xen/xen.h
+++ b/include/xen/xen.h
@@ -29,6 +29,12 @@ extern enum xen_domain_type xen_domain_type;
 #define xen_initial_domain()	(0)
 #endif	/* CONFIG_XEN_DOM0 */
 
+#ifdef CONFIG_XEN_PVHVM
+extern int xen_hvmlite;
+#else
+#define xen_hvmlite		(0)
+#endif
+
 #ifdef CONFIG_XEN_PVH
 /* This functionality exists only for x86. The XEN_PVHVM support exists
  * only in x86 world - hence on ARM it will be always disabled.
-- 
1.7.1

[PATCH v1 05/12] xen/hvmlite: Allow HVMlite guests delay initializing grant table

[Boris Ostrovsky]

.. just like we currently do for PVH guests

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/grant-table.c |    4 ++--
 drivers/xen/grant-table.c  |    8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/arch/x86/xen/grant-table.c b/arch/x86/xen/grant-table.c
index e079500..40ad9c2 100644
--- a/arch/x86/xen/grant-table.c
+++ b/arch/x86/xen/grant-table.c
@@ -110,7 +110,7 @@ int arch_gnttab_init(unsigned long nr_shared)
 	return arch_gnttab_valloc(&gnttab_shared_vm_area, nr_shared);
 }
 
-#ifdef CONFIG_XEN_PVH
+#ifdef CONFIG_XEN_PVHVM
 #include <xen/balloon.h>
 #include <xen/events.h>
 #include <linux/slab.h>
@@ -164,7 +164,7 @@ static int __init xlated_setup_gnttab_pages(void)
 
 static int __init xen_pvh_gnttab_setup(void)
 {
-	if (!xen_pvh_domain())
+	if (!xen_pvh_domain() && !xen_hvmlite)
 		return -ENODEV;
 
 	return xlated_setup_gnttab_pages();
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index effbaf9..8c6c0cf 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -1147,13 +1147,13 @@ EXPORT_SYMBOL_GPL(gnttab_init);
 
 static int __gnttab_init(void)
 {
+	if (!xen_domain())
+		return -ENODEV;
+
 	/* Delay grant-table initialization in the PV on HVM case */
-	if (xen_hvm_domain())
+	if (xen_hvm_domain() && !xen_hvmlite)
 		return 0;
 
-	if (!xen_pv_domain())
-		return -ENODEV;
-
 	return gnttab_init();
 }
 /* Starts after core_initcall so that xen_pvh_gnttab_setup can be called
-- 
1.7.1

[PATCH v1 06/12] xen/hvmlite: Initialize PCI

[Boris Ostrovsky]

HVMlite guests need PCI frontend and always have PV devices

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/pci/xen.c                 |    2 +-
 arch/x86/xen/platform-pci-unplug.c |    4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c
index ff31ab4..d847f7d 100644
--- a/arch/x86/pci/xen.c
+++ b/arch/x86/pci/xen.c
@@ -405,7 +405,7 @@ static void xen_teardown_msi_irq(unsigned int irq)
 
 int __init pci_xen_init(void)
 {
-	if (!xen_pv_domain() || xen_initial_domain())
+	if ((!xen_pv_domain() && !xen_hvmlite) || xen_initial_domain())
 		return -ENODEV;
 
 	printk(KERN_INFO "PCI: setting up Xen PCI frontend stub\n");
diff --git a/arch/x86/xen/platform-pci-unplug.c b/arch/x86/xen/platform-pci-unplug.c
index 9586ff3..802ec90 100644
--- a/arch/x86/xen/platform-pci-unplug.c
+++ b/arch/x86/xen/platform-pci-unplug.c
@@ -73,8 +73,8 @@ bool xen_has_pv_devices(void)
 	if (!xen_domain())
 		return false;
 
-	/* PV domains always have them. */
-	if (xen_pv_domain())
+	/* PV and HVMlite domains always have them. */
+	if (xen_pv_domain() || xen_hvmlite)
 		return true;
 
 	/* And user has xen_platform_pci=0 set in guest config as
-- 
1.7.1

[PATCH v1 07/12] xen/hvmlite: Prepare cpu_initialize_context() routine for HVMlite SMP

[Boris Ostrovsky]

Subsequent patch will add support for starting secondary VCPUs in
HVMlite guest. This patch exists to make review easier.

No functional changes (except for introduction of 'if (!xen_hvmlite)').

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/smp.c |  104 ++++++++++++++++++++++++++++-----------------------
 1 files changed, 57 insertions(+), 47 deletions(-)

diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 3f4ebf0..5fc4afb 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -390,70 +390,80 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 	if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
 		return 0;
 
-	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
-	if (ctxt == NULL)
-		return -ENOMEM;
+	if (!xen_hvmlite) {
 
-	gdt = get_cpu_gdt_table(cpu);
+		ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
+		if (ctxt == NULL)
+			return -ENOMEM;
+
+		gdt = get_cpu_gdt_table(cpu);
 
 #ifdef CONFIG_X86_32
-	/* Note: PVH is not yet supported on x86_32. */
-	ctxt->user_regs.fs = __KERNEL_PERCPU;
-	ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
+		/* Note: PVH is not yet supported on x86_32. */
+		ctxt->user_regs.fs = __KERNEL_PERCPU;
+		ctxt->user_regs.gs = __KERNEL_STACK_CANARY;
 #endif
-	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
+		memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
 
-	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
-		ctxt->user_regs.eip = (unsigned long)cpu_bringup_and_idle;
-		ctxt->flags = VGCF_IN_KERNEL;
-		ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
-		ctxt->user_regs.ds = __USER_DS;
-		ctxt->user_regs.es = __USER_DS;
-		ctxt->user_regs.ss = __KERNEL_DS;
+		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
+			ctxt->user_regs.eip =
+				(unsigned long)cpu_bringup_and_idle;
+			ctxt->flags = VGCF_IN_KERNEL;
+			ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
+			ctxt->user_regs.ds = __USER_DS;
+			ctxt->user_regs.es = __USER_DS;
+			ctxt->user_regs.ss = __KERNEL_DS;
 
-		xen_copy_trap_info(ctxt->trap_ctxt);
+			xen_copy_trap_info(ctxt->trap_ctxt);
 
-		ctxt->ldt_ents = 0;
+			ctxt->ldt_ents = 0;
 
-		BUG_ON((unsigned long)gdt & ~PAGE_MASK);
+			BUG_ON((unsigned long)gdt & ~PAGE_MASK);
 
-		gdt_mfn = arbitrary_virt_to_mfn(gdt);
-		make_lowmem_page_readonly(gdt);
-		make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
+			gdt_mfn = arbitrary_virt_to_mfn(gdt);
+			make_lowmem_page_readonly(gdt);
+			make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
 
-		ctxt->gdt_frames[0] = gdt_mfn;
-		ctxt->gdt_ents      = GDT_ENTRIES;
+			ctxt->gdt_frames[0] = gdt_mfn;
+			ctxt->gdt_ents      = GDT_ENTRIES;
 
-		ctxt->kernel_ss = __KERNEL_DS;
-		ctxt->kernel_sp = idle->thread.sp0;
+			ctxt->kernel_ss = __KERNEL_DS;
+			ctxt->kernel_sp = idle->thread.sp0;
 
 #ifdef CONFIG_X86_32
-		ctxt->event_callback_cs     = __KERNEL_CS;
-		ctxt->failsafe_callback_cs  = __KERNEL_CS;
+			ctxt->event_callback_cs     = __KERNEL_CS;
+			ctxt->failsafe_callback_cs  = __KERNEL_CS;
 #else
-		ctxt->gs_base_kernel = per_cpu_offset(cpu);
+			ctxt->gs_base_kernel = per_cpu_offset(cpu);
 #endif
-		ctxt->event_callback_eip    =
-					(unsigned long)xen_hypervisor_callback;
-		ctxt->failsafe_callback_eip =
-					(unsigned long)xen_failsafe_callback;
-		ctxt->user_regs.cs = __KERNEL_CS;
-		per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
-	}
+			ctxt->event_callback_eip    =
+				(unsigned long)xen_hypervisor_callback;
+			ctxt->failsafe_callback_eip =
+				(unsigned long)xen_failsafe_callback;
+			ctxt->user_regs.cs = __KERNEL_CS;
+			per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
+		}
 #ifdef CONFIG_XEN_PVH
-	else {
-		/*
-		 * The vcpu comes on kernel page tables which have the NX pte
-		 * bit set. This means before DS/SS is touched, NX in
-		 * EFER must be set. Hence the following assembly glue code.
-		 */
-		ctxt->user_regs.eip = (unsigned long)xen_pvh_early_cpu_init;
-		ctxt->user_regs.rdi = cpu;
-		ctxt->user_regs.rsi = true;  /* entry == true */
-	}
+		else {
+			/*
+			 * The vcpu comes on kernel page tables which have the
+			 * NX pte bit set. This means before DS/SS is touched,
+			 * NX in EFER must be set. Hence the following assembly
+			 * glue code.
+			 */
+			ctxt->user_regs.eip =
+				(unsigned long)xen_pvh_early_cpu_init;
+			ctxt->user_regs.rdi = cpu;
+			ctxt->user_regs.rsi = true;  /* entry == true */
+		}
 #endif
-	ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
-	ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
+		ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
+		ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
+	} else {
+		ctxt = NULL; /* To quiet down compiler */
+		BUG();
+	}
+
 	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
 		BUG();
 
-- 
1.7.1

[PATCH v1 08/12] xen/hvmlite: Initialize context for secondary VCPUs

[Boris Ostrovsky]

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/smp.c         |   57 ++++++++++++++++++++++++++++++++++++++++----
 arch/x86/xen/smp.h         |    4 +++
 arch/x86/xen/xen-hvmlite.S |    7 +++++
 3 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 5fc4afb..b265c4f 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -27,6 +27,7 @@
 #include <xen/interface/xen.h>
 #include <xen/interface/vcpu.h>
 #include <xen/interface/xenpmu.h>
+#include <xen/interface/hvm/hvm_vcpu.h>
 
 #include <asm/xen/interface.h>
 #include <asm/xen/hypercall.h>
@@ -384,6 +385,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 	struct vcpu_guest_context *ctxt;
 	struct desc_struct *gdt;
 	unsigned long gdt_mfn;
+	void *ctxt_arg;
 
 	/* used to tell cpu_init() that it can proceed with initialization */
 	cpumask_set_cpu(cpu, cpu_callout_mask);
@@ -392,7 +394,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 
 	if (!xen_hvmlite) {
 
-		ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
+		ctxt_arg = ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
 		if (ctxt == NULL)
 			return -ENOMEM;
 
@@ -460,14 +462,59 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
 		ctxt->user_regs.esp = idle->thread.sp0 - sizeof(struct pt_regs);
 		ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
 	} else {
-		ctxt = NULL; /* To quiet down compiler */
-		BUG();
+#ifdef CONFIG_XEN_PVHVM
+		struct vcpu_hvm_context *hctxt;
+
+		ctxt_arg = hctxt = kzalloc(sizeof(*hctxt), GFP_KERNEL);
+		if (hctxt == NULL)
+			return -ENOMEM;
+
+#ifdef CONFIG_X86_64
+		hctxt->mode = VCPU_HVM_MODE_64B;
+		hctxt->cpu_regs.x86_64.rip =
+			(unsigned long)secondary_startup_64;
+		hctxt->cpu_regs.x86_64.rsp = stack_start;
+
+		hctxt->cpu_regs.x86_64.cr0 =
+			X86_CR0_PG | X86_CR0_WP | X86_CR0_PE;
+		hctxt->cpu_regs.x86_64.cr4 = X86_CR4_PAE;
+		hctxt->cpu_regs.x86_64.cr3 =
+			xen_pfn_to_cr3(virt_to_mfn(init_level4_pgt));
+		hctxt->cpu_regs.x86_64.efer = EFER_LME | EFER_NX;
+#else
+		hctxt->mode = VCPU_HVM_MODE_32B;
+		/*
+		 * startup_32_smp expects GDT loaded so we can't jump
+		 * there directly.
+		 */
+		hctxt->cpu_regs.x86_32.eip =
+			(unsigned long)hvmlite_smp_32 - __START_KERNEL_map;
+
+		hctxt->cpu_regs.x86_32.cr0 = X86_CR0_PE;
+
+		hctxt->cpu_regs.x86_32.cs_base = 0;
+		hctxt->cpu_regs.x86_32.cs_limit = ~0u;
+		hctxt->cpu_regs.x86_32.cs_ar = 0xc9b;
+		hctxt->cpu_regs.x86_32.ds_base = 0;
+		hctxt->cpu_regs.x86_32.ds_limit = ~0u;
+		hctxt->cpu_regs.x86_32.ds_ar = 0xc93;
+		hctxt->cpu_regs.x86_32.es_base = 0;
+		hctxt->cpu_regs.x86_32.es_limit = ~0u;
+		hctxt->cpu_regs.x86_32.es_ar = 0xc93;
+		hctxt->cpu_regs.x86_32.ss_base = 0;
+		hctxt->cpu_regs.x86_32.ss_limit = ~0u;
+		hctxt->cpu_regs.x86_32.ss_ar = 0xc93;
+		hctxt->cpu_regs.x86_32.tr_base = 0;
+		hctxt->cpu_regs.x86_32.tr_limit = 0xff;
+		hctxt->cpu_regs.x86_32.tr_ar = 0x8b;
+#endif
+#endif
 	}
 
-	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt))
+	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, cpu, ctxt_arg))
 		BUG();
 
-	kfree(ctxt);
+	kfree(ctxt_arg);
 	return 0;
 }
 
diff --git a/arch/x86/xen/smp.h b/arch/x86/xen/smp.h
index 963d62a..b4a833c 100644
--- a/arch/x86/xen/smp.h
+++ b/arch/x86/xen/smp.h
@@ -8,6 +8,10 @@ extern void xen_send_IPI_allbutself(int vector);
 extern void xen_send_IPI_all(int vector);
 extern void xen_send_IPI_self(int vector);
 
+#ifdef CONFIG_X86_32
+extern void hvmlite_smp_32(void);
+#endif
+
 #ifdef CONFIG_XEN_PVH
 extern void xen_pvh_early_cpu_init(int cpu, bool entry);
 #else
diff --git a/arch/x86/xen/xen-hvmlite.S b/arch/x86/xen/xen-hvmlite.S
index 90f03d0..8d6a642 100644
--- a/arch/x86/xen/xen-hvmlite.S
+++ b/arch/x86/xen/xen-hvmlite.S
@@ -134,6 +134,13 @@ ENTRY(hvmlite_start_xen)
 	ljmp    $0x10, $_pa(startup_32)
 #endif
 
+#ifdef CONFIG_X86_32
+ENTRY(hvmlite_smp_32)
+        mov $_pa(boot_gdt_descr), %eax
+        lgdt (%eax)
+        jmp startup_32_smp
+#endif
+
 	.data
 gdt:
 	.word	gdt_end - gdt
-- 
1.7.1

[PATCH v1 09/12] xen/hvmlite: Extend APIC operations for HVMlite guests

[Boris Ostrovsky]

HVMlite guests need to be viewed as having APIC, otherwise smpboot code,
for example, will complain.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---

Not sure about xen_cpu_present_to_apicid() being an identity function, given
xen_x86_32_early_logical_apicid().

 arch/x86/xen/apic.c |   39 +++++++++++++++++++++++++++++++++++++--
 arch/x86/xen/smp.c  |    2 +-
 2 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
index abf4901..2a59c39 100644
--- a/arch/x86/xen/apic.c
+++ b/arch/x86/xen/apic.c
@@ -6,6 +6,7 @@
 
 #include <xen/xen.h>
 #include <xen/interface/physdev.h>
+#include <xen/interface/vcpu.h>
 #include "xen-ops.h"
 #include "pmu.h"
 #include "smp.h"
@@ -78,6 +79,21 @@ static void xen_apic_write(u32 reg, u32 val)
 		return;
 	}
 
+	if (xen_hvmlite) {
+		switch (reg) {
+		case APIC_TASKPRI:
+		case APIC_SPIV:
+		case APIC_ESR:
+		case APIC_LVTT:
+		case APIC_LVT0:
+		case APIC_LVT1:
+		case APIC_LVTERR:
+			pr_debug("Unimplemented APIC register %x,"
+				 " value: %x\n", reg, val);
+			return;
+		}
+	}
+
 	/* Warn to see if there's any stray references */
 	WARN(1,"register: %x, value: %x\n", reg, val);
 }
@@ -100,7 +116,7 @@ static u32 xen_safe_apic_wait_icr_idle(void)
 
 static int xen_apic_probe_pv(void)
 {
-	if (xen_pv_domain())
+	if (xen_pv_domain() || xen_hvmlite)
 		return 1;
 
 	return 0;
@@ -142,6 +158,19 @@ static void xen_silent_inquire(int apicid)
 {
 }
 
+static int xen_cpu_present_to_apicid(int cpu)
+{
+	return cpu;
+}
+
+static int xen_wakeup_secondary_cpu(int cpu, unsigned long start_eip)
+{
+	if (!xen_hvmlite)
+		return -EINVAL;
+
+	return HYPERVISOR_vcpu_op(VCPUOP_up, cpu, NULL);
+}
+
 static struct apic xen_pv_apic = {
 	.name 				= "Xen PV",
 	.probe 				= xen_apic_probe_pv,
@@ -162,7 +191,7 @@ static struct apic xen_pv_apic = {
 
 	.ioapic_phys_id_map		= default_ioapic_phys_id_map, /* Used on 32-bit */
 	.setup_apic_routing		= NULL,
-	.cpu_present_to_apicid		= default_cpu_present_to_apicid,
+	.cpu_present_to_apicid		= xen_cpu_present_to_apicid,
 	.apicid_to_cpu_present		= physid_set_mask_of_physid, /* Used on 32-bit */
 	.check_phys_apicid_present	= default_check_phys_apicid_present, /* smp_sanity_check needs it */
 	.phys_pkg_id			= xen_phys_pkg_id, /* detect_ht */
@@ -180,6 +209,9 @@ static struct apic xen_pv_apic = {
 	.send_IPI_all 			= xen_send_IPI_all,
 	.send_IPI_self 			= xen_send_IPI_self,
 #endif
+
+	.wakeup_secondary_cpu	= xen_wakeup_secondary_cpu,
+
 	/* .wait_for_init_deassert- used  by AP bootup - smp_callin which we don't use */
 	.inquire_remote_apic		= xen_silent_inquire,
 
@@ -216,5 +248,8 @@ void __init xen_init_apic(void)
 		apic = &xen_pv_apic;
 
 	x86_platform.apic_post_init = xen_apic_check;
+
+	if (xen_hvmlite)
+		setup_force_cpu_cap(X86_FEATURE_APIC);
 }
 apic_driver(xen_pv_apic);
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index b265c4f..fb085ef 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -214,7 +214,7 @@ static int xen_smp_intr_init(unsigned int cpu)
 	 * The IRQ worker on PVHVM goes through the native path and uses the
 	 * IPI mechanism.
 	 */
-	if (xen_hvm_domain())
+	if (xen_hvm_domain() && !xen_hvmlite)
 		return 0;
 
 	callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
-- 
1.7.1

[PATCH v1 10/12] xen/hvmlite: Use x86's default timer init for HVMlite guests

[Boris Ostrovsky]

xen_timer_init() will be called from apic_bsp_setup().

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/time.c |    5 ++++-
 1 files changed, 4 insertions(+), 1 deletions(-)

diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
index a0a4e55..93745e7 100644
--- a/arch/x86/xen/time.c
+++ b/arch/x86/xen/time.c
@@ -439,7 +439,10 @@ void __init xen_init_time_ops(void)
 {
 	pv_time_ops = xen_time_ops;
 
-	x86_init.timers.timer_init = xen_time_init;
+	if (!xen_hvmlite)
+		x86_init.timers.timer_init = xen_time_init;
+	else
+		x86_init.timers.timer_init = x86_init_noop;
 	x86_init.timers.setup_percpu_clockev = x86_init_noop;
 	x86_cpuinit.setup_percpu_clockev = x86_init_noop;
 
-- 
1.7.1

[PATCH v1 11/12] xen/hvmlite: Boot secondary CPUs

[Boris Ostrovsky]

HVMlite secondary VCPUs use baremetal bringup path (i.e. native_*
smp_ops) but need to do some preparation in PV code.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/enlighten.c |    2 +
 arch/x86/xen/pmu.c       |    4 +-
 arch/x86/xen/smp.c       |   60 +++++++++++++++++++++++++++++++++-------------
 3 files changed, 47 insertions(+), 19 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 2ed8b2b..850ce66 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1937,6 +1937,8 @@ static void __init xen_hvm_guest_init(void)
 		xen_have_vector_callback = 1;
 	xen_hvm_smp_init();
 	register_cpu_notifier(&xen_hvm_cpu_notifier);
+	if (xen_hvmlite)
+		smp_found_config = 1;
 	xen_unplug_emulated_devices();
 	x86_init.irqs.intr_init = xen_init_IRQ;
 	xen_hvm_init_time_ops();
diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c
index 724a087..7bc209b 100644
--- a/arch/x86/xen/pmu.c
+++ b/arch/x86/xen/pmu.c
@@ -518,7 +518,7 @@ void xen_pmu_init(int cpu)
 
 	BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
 
-	if (xen_hvm_domain())
+	if (xen_hvm_domain() && !xen_hvmlite)
 		return;
 
 	xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
@@ -556,7 +556,7 @@ void xen_pmu_finish(int cpu)
 {
 	struct xen_pmu_params xp;
 
-	if (xen_hvm_domain())
+	if (xen_hvm_domain() && !xen_hvmlite)
 		return;
 
 	xp.vcpu = cpu;
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index fb085ef..fbad829 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -348,26 +348,31 @@ static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
 	}
 	xen_init_lock_cpu(0);
 
-	smp_store_boot_cpu_info();
-	cpu_data(0).x86_max_cores = 1;
+	if (!xen_hvmlite) {
+		smp_store_boot_cpu_info();
+		cpu_data(0).x86_max_cores = 1;
+
+		for_each_possible_cpu(i) {
+			zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i),
+					   GFP_KERNEL);
+			zalloc_cpumask_var(&per_cpu(cpu_core_map, i),
+					   GFP_KERNEL);
+			zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i),
+					   GFP_KERNEL);
+		}
+		set_cpu_sibling_map(0);
 
-	for_each_possible_cpu(i) {
-		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
-		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
-		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
+		if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
+			panic("could not allocate xen_cpu_initialized_map\n");
+
+		cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
 	}
-	set_cpu_sibling_map(0);
 
 	xen_pmu_init(0);
 
 	if (xen_smp_intr_init(0))
 		BUG();
 
-	if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
-		panic("could not allocate xen_cpu_initialized_map\n");
-
-	cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
-
 	/* Restrict the possible_map according to max_cpus. */
 	while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
 		for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
@@ -375,8 +380,11 @@ static void __init xen_smp_prepare_cpus(unsigned int max_cpus)
 		set_cpu_possible(cpu, false);
 	}
 
-	for_each_possible_cpu(cpu)
+	for_each_possible_cpu(cpu) {
 		set_cpu_present(cpu, true);
+		if (xen_hvmlite)
+			physid_set(cpu, phys_cpu_present_map);
+	}
 }
 
 static int
@@ -810,10 +818,15 @@ void __init xen_smp_init(void)
 
 static void __init xen_hvm_smp_prepare_cpus(unsigned int max_cpus)
 {
+	if (xen_hvmlite)
+		xen_smp_prepare_cpus(max_cpus);
+
 	native_smp_prepare_cpus(max_cpus);
-	WARN_ON(xen_smp_intr_init(0));
 
-	xen_init_lock_cpu(0);
+	if (!xen_hvmlite) {
+		WARN_ON(xen_smp_intr_init(0));
+		xen_init_lock_cpu(0);
+	}
 }
 
 static int xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle)
@@ -836,8 +849,21 @@ static int xen_hvm_cpu_up(unsigned int cpu, struct task_struct *tidle)
 	*/
 	rc = xen_smp_intr_init(cpu);
 	WARN_ON(rc);
-	if (!rc)
-		rc =  native_cpu_up(cpu, tidle);
+
+	if (xen_hvmlite) {
+		rc = cpu_initialize_context(cpu, tidle);
+		if (rc) {
+			xen_smp_intr_free(cpu);
+			return rc;
+		}
+		xen_pmu_init(cpu);
+	}
+
+	if (!rc) {
+		rc = native_cpu_up(cpu, tidle);
+		if (rc && xen_hvmlite)
+			xen_pmu_finish(cpu);
+	}
 
 	/*
 	 * We must initialize the slowpath CPU kicker _after_ the native
-- 
1.7.1

[PATCH v1 12/12] xen/hvmlite: Enable CPU on-/offlining

[Boris Ostrovsky]

When offlining, we should properly clean up interrupts and wait
until hypervisor declares VCPU as down before cleaning up.

After VCPU that was previously offlined is brought back to life
we want to jump back to bare-metal entry points. It's a simple
jump on 64-bit but requires minor tweaking for 32-bit case.

Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
---
 arch/x86/xen/smp.c         |   35 +++++++++++++++++++++++++----------
 arch/x86/xen/xen-hvmlite.S |    8 ++++++++
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index fbad829..7e96a23 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -143,7 +143,7 @@ static void xen_smp_intr_free(unsigned int cpu)
 		kfree(per_cpu(xen_callfuncsingle_irq, cpu).name);
 		per_cpu(xen_callfuncsingle_irq, cpu).name = NULL;
 	}
-	if (xen_hvm_domain())
+	if (xen_hvm_domain() && !xen_hvmlite)
 		return;
 
 	if (per_cpu(xen_irq_work, cpu).irq >= 0) {
@@ -585,7 +585,8 @@ static int xen_cpu_disable(void)
 
 static void xen_cpu_die(unsigned int cpu)
 {
-	while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
+	while ((xen_pv_domain() || xen_hvmlite) &&
+	       HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
 		__set_current_state(TASK_UNINTERRUPTIBLE);
 		schedule_timeout(HZ/10);
 	}
@@ -602,14 +603,28 @@ static void xen_play_dead(void) /* used only with HOTPLUG_CPU */
 {
 	play_dead_common();
 	HYPERVISOR_vcpu_op(VCPUOP_down, smp_processor_id(), NULL);
-	cpu_bringup();
-	/*
-	 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
-	 * clears certain data that the cpu_idle loop (which called us
-	 * and that we return from) expects. The only way to get that
-	 * data back is to call:
-	 */
-	tick_nohz_idle_enter();
+
+	if (!xen_hvm_domain()) {
+		cpu_bringup();
+		/*
+		 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu
+		 * down) clears certain data that the cpu_idle loop (which
+		 * called us and that we return from) expects. The only way to
+		 * get that data back is to call:
+		 */
+		tick_nohz_idle_enter();
+	} else {
+		/*
+		 * For 64-bit we can jump directly to SMP entry point but for
+		 * 32-bit we need to disable paging and load boot GDT (just
+		 * like in cpu_initialize_context()).
+		 */
+#ifdef CONFIG_X86_64
+		asm("jmp secondary_startup_64");
+#else
+		asm("jmp hvmlite_smp_32_hp");
+#endif
+	}
 }
 
 #else /* !CONFIG_HOTPLUG_CPU */
diff --git a/arch/x86/xen/xen-hvmlite.S b/arch/x86/xen/xen-hvmlite.S
index 8d6a642..4edd6ef 100644
--- a/arch/x86/xen/xen-hvmlite.S
+++ b/arch/x86/xen/xen-hvmlite.S
@@ -135,6 +135,14 @@ ENTRY(hvmlite_start_xen)
 #endif
 
 #ifdef CONFIG_X86_32
+ENTRY(hvmlite_smp_32_hp)
+	movl $_pa(initial_page_table), %eax
+	movl %eax, %cr3
+	ljmp $__KERNEL_CS,$_pa(5f)
+5:
+	movl $X86_CR0_PE, %eax
+	movl %eax, %cr0
+
 ENTRY(hvmlite_smp_32)
         mov $_pa(boot_gdt_descr), %eax
         lgdt (%eax)
-- 
1.7.1

Re: [PATCH v1 02/12] xen/hvmlite: Factor out common kernel init code

[Luis R. Rodriguez]

On Fri, Jan 22, 2016 at 04:35:48PM -0500, Boris Ostrovsky wrote:
> HVMlite guests (to be introduced in subsequent patches) share most
> of the kernel initialization code with PV(H).
> 
> Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
> ---
>  arch/x86/xen/enlighten.c |  225 ++++++++++++++++++++++++----------------------
>  1 files changed, 119 insertions(+), 106 deletions(-)
> 
> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
> index d09e4c9..2cf446a 100644
> --- a/arch/x86/xen/enlighten.c
> +++ b/arch/x86/xen/enlighten.c
> @@ -1505,19 +1505,14 @@ static void __init xen_pvh_early_guest_init(void)
>  }
>  #endif    /* CONFIG_XEN_PVH */
>  
> -/* First C function to be called on Xen boot */
> -asmlinkage __visible void __init xen_start_kernel(void)
> +
> +static void __init xen_init_kernel(void)
>  {
>  	struct physdev_set_iopl set_iopl;
>  	unsigned long initrd_start = 0;
>  	u64 pat;
>  	int rc;
>  
> -	if (!xen_start_info)
> -		return;
> -
> -	xen_domain_type = XEN_PV_DOMAIN;
> -
>  	xen_setup_features();
>  #ifdef CONFIG_XEN_PVH
>  	xen_pvh_early_guest_init();
> @@ -1529,48 +1524,9 @@ asmlinkage __visible void __init xen_start_kernel(void)
>  	if (xen_initial_domain())
>  		pv_info.features |= PV_SUPPORTED_RTC;
>  	pv_init_ops = xen_init_ops;
> -	if (!xen_pvh_domain()) {
> -		pv_cpu_ops = xen_cpu_ops;
> -
> -		x86_platform.get_nmi_reason = xen_get_nmi_reason;
> -	}
> -
> -	if (xen_feature(XENFEAT_auto_translated_physmap))
> -		x86_init.resources.memory_setup = xen_auto_xlated_memory_setup;
> -	else
> -		x86_init.resources.memory_setup = xen_memory_setup;
> -	x86_init.oem.arch_setup = xen_arch_setup;
> -	x86_init.oem.banner = xen_banner;
>  
>  	xen_init_time_ops();
>  
> -	/*
> -	 * Set up some pagetable state before starting to set any ptes.
> -	 */
> -
> -	xen_init_mmu_ops();
> -
> -	/* Prevent unwanted bits from being set in PTEs. */
> -	__supported_pte_mask &= ~_PAGE_GLOBAL;
> -
> -	/*
> -	 * Prevent page tables from being allocated in highmem, even
> -	 * if CONFIG_HIGHPTE is enabled.
> -	 */
> -	__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
> -
> -	/* Work out if we support NX */
> -	x86_configure_nx();
> -
> -	/* Get mfn list */
> -	xen_build_dynamic_phys_to_machine();
> -
> -	/*
> -	 * Set up kernel GDT and segment registers, mainly so that
> -	 * -fstack-protector code can be executed.
> -	 */
> -	xen_setup_gdt(0);
> -
>  	xen_init_irq_ops();
>  	xen_init_cpuid_mask();
>  
> @@ -1580,21 +1536,8 @@ asmlinkage __visible void __init xen_start_kernel(void)
>  	 */
>  	xen_init_apic();
>  #endif
> -
> -	if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
> -		pv_mmu_ops.ptep_modify_prot_start = xen_ptep_modify_prot_start;
> -		pv_mmu_ops.ptep_modify_prot_commit = xen_ptep_modify_prot_commit;
> -	}
> -
>  	machine_ops = xen_machine_ops;
>  
> -	/*
> -	 * The only reliable way to retain the initial address of the
> -	 * percpu gdt_page is to remember it here, so we can go and
> -	 * mark it RW later, when the initial percpu area is freed.
> -	 */
> -	xen_initial_gdt = &per_cpu(gdt_page, 0);
> -
>  	xen_smp_init();
>  
>  #ifdef CONFIG_ACPI_NUMA
> @@ -1609,66 +1552,126 @@ asmlinkage __visible void __init xen_start_kernel(void)
>  	   possible map and a non-dummy shared_info. */
>  	per_cpu(xen_vcpu, 0) = &HYPERVISOR_shared_info->vcpu_info[0];
>  
> -	local_irq_disable();
> -	early_boot_irqs_disabled = true;
> +	if (!xen_hvm_domain()) {
> +		if (!xen_pvh_domain()) {
> +			pv_cpu_ops = xen_cpu_ops;
>  
> -	xen_raw_console_write("mapping kernel into physical memory\n");
> -	xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base,
> -				   xen_start_info->nr_pages);
> -	xen_reserve_special_pages();
> +			x86_platform.get_nmi_reason = xen_get_nmi_reason;
> +		}
>  
> -	/*
> -	 * Modify the cache mode translation tables to match Xen's PAT
> -	 * configuration.
> -	 */
> -	rdmsrl(MSR_IA32_CR_PAT, pat);
> -	pat_init_cache_modes(pat);
> +		if (xen_feature(XENFEAT_auto_translated_physmap))
> +			x86_init.resources.memory_setup =
> +				xen_auto_xlated_memory_setup;
> +		else
> +			x86_init.resources.memory_setup = xen_memory_setup;
> +		x86_init.oem.arch_setup = xen_arch_setup;
> +		x86_init.oem.banner = xen_banner;
>  
> -	/* keep using Xen gdt for now; no urgent need to change it */
> +		/*
> +		 * Set up some pagetable state before starting to set any ptes.
> +		 */
>  
> -#ifdef CONFIG_X86_32
> -	pv_info.kernel_rpl = 1;
> -	if (xen_feature(XENFEAT_supervisor_mode_kernel))
> -		pv_info.kernel_rpl = 0;
> -#else
> -	pv_info.kernel_rpl = 0;
> -#endif
> -	/* set the limit of our address space */
> -	xen_reserve_top();
> +		xen_init_mmu_ops();
> +
> +		/* Prevent unwanted bits from being set in PTEs. */
> +		__supported_pte_mask &= ~_PAGE_GLOBAL;
>  
> -	/* PVH: runs at default kernel iopl of 0 */
> -	if (!xen_pvh_domain()) {
>  		/*
> -		 * We used to do this in xen_arch_setup, but that is too late
> -		 * on AMD were early_cpu_init (run before ->arch_setup()) calls
> -		 * early_amd_init which pokes 0xcf8 port.
> +		 * Prevent page tables from being allocated in highmem, even
> +		 * if CONFIG_HIGHPTE is enabled.
>  		 */
> -		set_iopl.iopl = 1;
> -		rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl, &set_iopl);
> -		if (rc != 0)
> -			xen_raw_printk("physdev_op failed %d\n", rc);
> -	}
> +		__userpte_alloc_gfp &= ~__GFP_HIGHMEM;
> +
> +		/* Work out if we support NX */
> +		x86_configure_nx();
> +
> +		/* Get mfn list */
> +		xen_build_dynamic_phys_to_machine();
> +
> +		/*
> +		 * Set up kernel GDT and segment registers, mainly so that
> +		 * -fstack-protector code can be executed.
> +		 */
> +		xen_setup_gdt(0);
> +
> +		if (xen_feature(XENFEAT_mmu_pt_update_preserve_ad)) {
> +			pv_mmu_ops.ptep_modify_prot_start =
> +				xen_ptep_modify_prot_start;
> +			pv_mmu_ops.ptep_modify_prot_commit =
> +				xen_ptep_modify_prot_commit;
> +		}
> +
> +		/*
> +		 * The only reliable way to retain the initial address of the
> +		 * percpu gdt_page is to remember it here, so we can go and
> +		 * mark it RW later, when the initial percpu area is freed.
> +		 */
> +		xen_initial_gdt = &per_cpu(gdt_page, 0);
> +
> +		xen_raw_console_write("mapping kernel into physical memory\n");
> +		xen_setup_kernel_pagetable((pgd_t *)xen_start_info->pt_base,
> +					   xen_start_info->nr_pages);
> +		xen_reserve_special_pages();
> +
> +		/*
> +		 * Modify the cache mode translation tables to match Xen's PAT
> +		 * configuration.
> +		 */
> +		rdmsrl(MSR_IA32_CR_PAT, pat);
> +		pat_init_cache_modes(pat);
> +
> +		/* keep using Xen gdt for now; no urgent need to change it */
>  
>  #ifdef CONFIG_X86_32
> -	/* set up basic CPUID stuff */
> -	cpu_detect(&new_cpu_data);
> -	set_cpu_cap(&new_cpu_data, X86_FEATURE_FPU);
> -	new_cpu_data.wp_works_ok = 1;
> -	new_cpu_data.x86_capability[0] = cpuid_edx(1);
> +		pv_info.kernel_rpl = 1;
> +		if (xen_feature(XENFEAT_supervisor_mode_kernel))
> +			pv_info.kernel_rpl = 0;
> +#else
> +		pv_info.kernel_rpl = 0;
> +#endif
> +		/* set the limit of our address space */
> +		xen_reserve_top();
> +
> +		/* PVH: runs at default kernel iopl of 0 */
> +		if (!xen_pvh_domain()) {
> +			/*
> +			 * We used to do this in xen_arch_setup, but that is
> +			 * too late on AMD were early_cpu_init (run before
> +			 * ->arch_setup()) calls early_amd_init which pokes
> +			 * 0xcf8 port.
> +			 */
> +			set_iopl.iopl = 1;
> +			rc = HYPERVISOR_physdev_op(PHYSDEVOP_set_iopl,
> +						   &set_iopl);
> +			if (rc != 0)
> +				xen_raw_printk("physdev_op failed %d\n", rc);
> +		}
> +
> +#ifdef CONFIG_X86_32
> +		/* set up basic CPUID stuff */
> +		cpu_detect(&new_cpu_data);
> +		set_cpu_cap(&new_cpu_data, X86_FEATURE_FPU);
> +		new_cpu_data.wp_works_ok = 1;
> +		new_cpu_data.x86_capability[0] = cpuid_edx(1);
>  #endif

Whoa, I'm lost, its hard for me to tell what exactly stayed and what
got pulled into a helper, etc. Is there a possibility to split this
patch in 2 somehow to make the actual functional changes easier to
read? There are too many changes here and I just can't tell easily
what's going on.

  Luis

Re: [PATCH v1 02/12] xen/hvmlite: Factor out common kernel init code

[Boris Ostrovsky]

On 01/22/2016 06:01 PM, Luis R. Rodriguez wrote:
> On Fri, Jan 22, 2016 at 04:35:48PM -0500, Boris Ostrovsky wrote:
>> HVMlite guests (to be introduced in subsequent patches) share most
>> of the kernel initialization code with PV(H).
>>
>> Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
>> ---
>>   arch/x86/xen/enlighten.c |  225 ++++++++++++++++++++++++----------------------
>>   1 files changed, 119 insertions(+), 106 deletions(-)
>>
>> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
>> index d09e4c9..2cf446a 100644
>> --- a/arch/x86/xen/enlighten.c
>> +++ b/arch/x86/xen/enlighten.c
> Whoa, I'm lost, its hard for me to tell what exactly stayed and what
> got pulled into a helper, etc. Is there a possibility to split this
> patch in 2 somehow to make the actual functional changes easier to
> read? There are too many changes here and I just can't tell easily
> what's going on.


The only real changes that this patch introduces is it reorders some of 
the operations that used to be in xen_start_kernel(). This is done so 
that in the next patch when we add hvmlite we can easily put those 
specific to PV(H) inside 'if (!xen_hvm_domain())'. I probably should 
have said so in the commit message.

It is indeed difficult to review but I don't see how I can split this. 
Even if I just moved it (without reordering) it would still be hard to read.

-boris

Re: [PATCH v1 02/12] xen/hvmlite: Factor out common kernel init code

[Boris Ostrovsky]

On 01/22/2016 06:12 PM, Boris Ostrovsky wrote:
> On 01/22/2016 06:01 PM, Luis R. Rodriguez wrote:
>> On Fri, Jan 22, 2016 at 04:35:48PM -0500, Boris Ostrovsky wrote:
>>> HVMlite guests (to be introduced in subsequent patches) share most
>>> of the kernel initialization code with PV(H).
>>>
>>> Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
>>> ---
>>>   arch/x86/xen/enlighten.c |  225 
>>> ++++++++++++++++++++++++----------------------
>>>   1 files changed, 119 insertions(+), 106 deletions(-)
>>>
>>> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
>>> index d09e4c9..2cf446a 100644
>>> --- a/arch/x86/xen/enlighten.c
>>> +++ b/arch/x86/xen/enlighten.c
>> Whoa, I'm lost, its hard for me to tell what exactly stayed and what
>> got pulled into a helper, etc. Is there a possibility to split this
>> patch in 2 somehow to make the actual functional changes easier to
>> read? There are too many changes here and I just can't tell easily
>> what's going on.
>
>
> The only real changes that this patch introduces is it reorders some 
> of the operations that used to be in xen_start_kernel(). This is done 
> so that in the next patch when we add hvmlite we can easily put those 
> specific to PV(H) inside 'if (!xen_hvm_domain())'. I probably should 
> have said so in the commit message.


Actually, I forgot that I merged the 'if' clause into this patch already 
so I'll see if I can separate them again to make it easier on the eye.

-boris


>
> It is indeed difficult to review but I don't see how I can split this. 
> Even if I just moved it (without reordering) it would still be hard to 
> read.
>
> -boris

Re: [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Fri, Jan 22, 2016 at 04:35:50PM -0500, Boris Ostrovsky wrote:
> Start HVMlite guest XEN_ELFNOTE_PHYS32_ENTRY address. Setup hypercall
> page, initialize boot_params, enable early page tables.
> 
> Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
> ---
>  arch/x86/xen/Makefile      |    1 +
>  arch/x86/xen/enlighten.c   |   91 +++++++++++++++++++++++++-
>  arch/x86/xen/xen-hvmlite.S |  158 ++++++++++++++++++++++++++++++++++++++++++++
>  include/xen/xen.h          |    6 ++
>  4 files changed, 255 insertions(+), 1 deletions(-)
>  create mode 100644 arch/x86/xen/xen-hvmlite.S
> 
> diff --git a/arch/x86/xen/Makefile b/arch/x86/xen/Makefile
> index e47e527..1d913d7 100644
> --- a/arch/x86/xen/Makefile
> +++ b/arch/x86/xen/Makefile
> @@ -23,3 +23,4 @@ obj-$(CONFIG_XEN_DEBUG_FS)	+= debugfs.o
>  obj-$(CONFIG_XEN_DOM0)		+= vga.o
>  obj-$(CONFIG_SWIOTLB_XEN)	+= pci-swiotlb-xen.o
>  obj-$(CONFIG_XEN_EFI)		+= efi.o
> +obj-$(CONFIG_XEN_PVHVM) 	+= xen-hvmlite.o
> diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
> index 2cf446a..2ed8b2b 100644
> --- a/arch/x86/xen/enlighten.c
> +++ b/arch/x86/xen/enlighten.c
> @@ -118,7 +118,8 @@ DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu);
>   */
>  DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info);
>  
> -enum xen_domain_type xen_domain_type = XEN_NATIVE;
> +enum xen_domain_type xen_domain_type
> +	__attribute__((section(".data"))) = XEN_NATIVE;
>  EXPORT_SYMBOL_GPL(xen_domain_type);

But why? This is not explained.

>  
>  unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START;
> @@ -171,6 +172,17 @@ struct tls_descs {
>   */
>  static DEFINE_PER_CPU(struct tls_descs, shadow_tls_desc);
>  
> +#ifdef CONFIG_XEN_PVHVM
> +/*
> + * HVMlite variables. These need to live in data segment since they are
> + * initialized before startup_{32|64}, which clear .bss, are invoked.
> + */

I guess for this reason.. Perhaps a bit more clear:

/*
 * HVMlite variables. These need to live in data segment since they are 
 * before before startup_{32|64} is invoked, otherwise they would be cleared.
 */

> +int xen_hvmlite __attribute__((section(".data"))) = 0;
> +struct hvm_start_info hvmlite_start_info __attribute__((section(".data")));
> +uint hvmlite_start_info_sz = sizeof(hvmlite_start_info);
> +struct boot_params xen_hvmlite_boot_params __attribute__((section(".data")));
> +#endif
> +
>  static void clamp_max_cpus(void)
>  {
>  #ifdef CONFIG_SMP
> @@ -1736,6 +1748,83 @@ asmlinkage __visible void __init xen_start_kernel(void)
>  #endif
>  }
>  
> +#ifdef CONFIG_XEN_PVHVM
> +static void __init hvmlite_bootparams(void)
> +{

Hrm.

> +	struct xen_memory_map memmap;
> +	int i;
> +
> +	memset(&xen_hvmlite_boot_params, 0, sizeof(xen_hvmlite_boot_params));
> +
> +	memmap.nr_entries = ARRAY_SIZE(xen_hvmlite_boot_params.e820_map);
> +	set_xen_guest_handle(memmap.buffer, xen_hvmlite_boot_params.e820_map);
> +	if (HYPERVISOR_memory_op(XENMEM_memory_map, &memmap)) {
> +		xen_raw_console_write("XENMEM_memory_map failed\n");
> +		BUG();
> +	}
> +
> +	xen_hvmlite_boot_params.e820_map[memmap.nr_entries].addr =
> +		ISA_START_ADDRESS;
> +	xen_hvmlite_boot_params.e820_map[memmap.nr_entries].size =
> +		ISA_END_ADDRESS - ISA_START_ADDRESS;
> +	xen_hvmlite_boot_params.e820_map[memmap.nr_entries++].type =
> +		E820_RESERVED;
> +
> +	sanitize_e820_map(xen_hvmlite_boot_params.e820_map,
> +			  ARRAY_SIZE(xen_hvmlite_boot_params.e820_map),
> +			  &memmap.nr_entries);
> +
> +	xen_hvmlite_boot_params.e820_entries = memmap.nr_entries;
> +	for (i = 0; i < xen_hvmlite_boot_params.e820_entries; i++)
> +		e820_add_region(xen_hvmlite_boot_params.e820_map[i].addr,
> +				xen_hvmlite_boot_params.e820_map[i].size,
> +				xen_hvmlite_boot_params.e820_map[i].type);
> +
> +	xen_hvmlite_boot_params.hdr.cmd_line_ptr =
> +		hvmlite_start_info.cmdline_paddr;
> +
> +	/* The first module is always ramdisk */
> +	if (hvmlite_start_info.nr_modules) {
> +		struct hvm_modlist_entry *modaddr =
> +			__va(hvmlite_start_info.modlist_paddr);
> +		xen_hvmlite_boot_params.hdr.ramdisk_image = modaddr->paddr;
> +		xen_hvmlite_boot_params.hdr.ramdisk_size = modaddr->size;
> +	}
> +
> +	/*
> +	 * See Documentation/x86/boot.txt.
> +	 *
> +	 * Version 2.12 supports Xen entry point but we will use default x86/PC
> +	 * environment (i.e. hardware_subarch 0).
> +	 */
> +	xen_hvmlite_boot_params.hdr.version = 0x212;
> +	xen_hvmlite_boot_params.hdr.type_of_loader = 9; /* Xen loader */
> +}

I realize PV got away with setting up boot_params on C code but best
ask now that this new code is being introduced: why can't we just have
the Xen hypervisor fill this in? It'd save us all this code.

On this page I show the difference, as an example of what this would look
like in contrast to how lguest set things up as an example in a very
clean way:

http://www.do-not-panic.com/2015/12/xen-and-x86-linux-zero-page.html

> +/*
> + * This routine (and those that it might call) should not use
> + * anything that lives in .bss since that segment will be cleared later
> + */
> +void __init xen_prepare_hvmlite(void)
> +{
> +	u32 eax, ecx, edx, msr;
> +	u64 pfn;
> +
> +	cpuid(xen_cpuid_base() + 2, &eax, &msr, &ecx, &edx);
> +	pfn = __pa(hypercall_page);
> +	wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));
> +
> +	pv_info.name = "Xen HVMlite";
> +	xen_domain_type = XEN_HVM_DOMAIN;
> +	xen_hvmlite = 1;
> +
> +	x86_init.oem.arch_setup = xen_init_kernel;
> +	x86_init.oem.banner = xen_banner;
> +
> +	hvmlite_bootparams();
> +}
> +#endif

If the boot_params.hdr.hardware_subarch_data pointed to a custom
struct then the first C entry point for Xen could shuffle this and
set this too, by still using less asm entry helpers. We'd still
need this run but with the linker table I think we could have
a stub small stub for hvm run, it would not be a call from asm.

> +
>  void __ref xen_hvm_init_shared_info(void)
>  {
>  	int cpu;
> diff --git a/arch/x86/xen/xen-hvmlite.S b/arch/x86/xen/xen-hvmlite.S
> new file mode 100644
> index 0000000..90f03d0
> --- /dev/null
> +++ b/arch/x86/xen/xen-hvmlite.S
> @@ -0,0 +1,158 @@
> +/*
> + * Copyright C 2016, Oracle and/or its affiliates. All rights reserved.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program.  If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +	.code32
> +	.text
> +#define _pa(x)          ((x) - __START_KERNEL_map)
> +
> +#include <linux/elfnote.h>
> +#include <linux/init.h>
> +#include <linux/linkage.h>
> +#include <asm/segment.h>
> +#include <asm/asm.h>
> +#include <asm/boot.h>
> +#include <asm/processor-flags.h>
> +#include <asm/msr.h>
> +#include <xen/interface/elfnote.h>
> +
> +	__HEAD
> +	.code32
> +
> +/* Entry point for HVMlite guests */
> +ENTRY(hvmlite_start_xen)

Yet another asm entry point for Linux, could it perhaps someway be possible to
just share some of this asm entry code in a clean way ?

> +	cli
> +	cld
> +
> +	mov $_pa(gdt), %eax
> +	lgdt (%eax)
> +
> +	movl $(__BOOT_DS),%eax
> +	movl %eax,%ds
> +	movl %eax,%es
> +	movl %eax,%ss
> +
> +	/* Stash hvm_start_info */
> +	mov $_pa(hvmlite_start_info), %edi
> +	mov %ebx, %esi
> +	mov $_pa(hvmlite_start_info_sz), %ecx
> +	mov (%ecx), %ecx
> +	rep
> +	movsb
> +
> +	movl $_pa(early_stack_end), %eax
> +	movl %eax, %esp
> +
> +	/* Enable PAE mode */
> +	movl %cr4, %eax
> +	orl $X86_CR4_PAE, %eax
> +	movl %eax, %cr4
> +
> +#ifdef CONFIG_X86_64
> +	/* Enable Long mode */
> +	movl $MSR_EFER, %ecx
> +	rdmsr
> +	btsl $_EFER_LME, %eax
> +	wrmsr
> +
> +	/* Enable pre-constructed page tables */
> +	mov $_pa(init_level4_pgt), %eax
> +	movl %eax, %cr3
> +	movl $(X86_CR0_PG | X86_CR0_PE), %eax
> +	movl %eax, %cr0
> +
> +	/* Jump to 64-bit mode. */
> +	pushl $__KERNEL_CS
> +	leal _pa(1f), %eax
> +	pushl %eax
> +	lret
> +
> +	/* 64-bit entry point */
> +	.code64
> +1:
> +	call xen_prepare_hvmlite
> +
> +	/* startup_64 expects boot_params in %rsi */
> +	mov $_pa(xen_hvmlite_boot_params), %rsi
> +	movq $_pa(startup_64), %rax

Nice! But again why can't the Xen hypervisor just set the boot_params?
All other Linux loaders do it. Why is Xen special?

  Luis

> +	jmp *%rax
> +
> +#else /* CONFIG_X86_64 */
> +
> +	/* Use initial_page table and set level 2 to map 2M pages */
> +	movl $_pa(initial_pg_pmd), %edi
> +	movl $(_PAGE_PSE | _PAGE_RW | _PAGE_PRESENT), %eax
> +	movl $2048, %ecx
> +2:
> +	movl %eax, 0(%edi)
> +	addl $0x00200000, %eax
> +	addl $8, %edi
> +	decl %ecx
> +	jnz 2b
> +
> +	/* Enable the boot paging */
> +	movl $_pa(initial_page_table), %eax
> +	movl %eax, %cr3
> +	movl %cr0, %eax
> +	orl $(X86_CR0_PG | X86_CR0_PE), %eax
> +	movl %eax, %cr0
> +
> +	ljmp $__BOOT_CS,$3f
> +3:
> +	call xen_prepare_hvmlite
> +	mov $_pa(xen_hvmlite_boot_params), %esi
> +
> +	/* startup_32 doesn't expect paging and PAE to be on */
> +	ljmp $__BOOT_CS,$_pa(4f)
> +4:
> +	movl %cr0, %eax
> +	andl $~X86_CR0_PG, %eax
> +	movl %eax, %cr0
> +	movl %cr4, %eax
> +	andl $~X86_CR4_PAE, %eax
> +	movl %eax, %cr4
> +
> +	/* Restore initial_pg_pmd to its original (zero) state */
> +	movl $_pa(initial_pg_pmd), %edi
> +	xorl %eax, %eax
> +	movl $(PAGE_SIZE/4), %ecx
> +	rep stosl
> +
> +	ljmp    $0x10, $_pa(startup_32)
> +#endif
> +
> +	.data
> +gdt:
> +	.word	gdt_end - gdt
> +	.long	_pa(gdt)
> +	.word	0
> +	.quad	0x0000000000000000 /* NULL descriptor */
> +#ifdef CONFIG_X86_64
> +	.quad	0x00af9a000000ffff /* __KERNEL_CS */
> +#else
> +	.quad	0x00cf9a000000ffff /* __KERNEL_CS */
> +#endif
> +	.quad	0x00cf92000000ffff /* __KERNEL_DS */
> +gdt_end:
> +
> +	.bss
> +	.balign 4
> +early_stack:
> +	.fill 16, 1, 0
> +early_stack_end:
> +
> +	ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY,
> +	             _ASM_PTR (hvmlite_start_xen - __START_KERNEL_map))
> diff --git a/include/xen/xen.h b/include/xen/xen.h
> index 0c0e3ef..6a0d3f3 100644
> --- a/include/xen/xen.h
> +++ b/include/xen/xen.h
> @@ -29,6 +29,12 @@ extern enum xen_domain_type xen_domain_type;
>  #define xen_initial_domain()	(0)
>  #endif	/* CONFIG_XEN_DOM0 */
>  
> +#ifdef CONFIG_XEN_PVHVM
> +extern int xen_hvmlite;
> +#else
> +#define xen_hvmlite		(0)
> +#endif
> +
>  #ifdef CONFIG_XEN_PVH
>  /* This functionality exists only for x86. The XEN_PVHVM support exists
>   * only in x86 world - hence on ARM it will be always disabled.
> -- 
> 1.7.1
> 
> 

-- 
Luis Rodriguez, SUSE LINUX GmbH
Maxfeldstrasse 5; D-90409 Nuernberg

Re: [PATCH v1 02/12] xen/hvmlite: Factor out common kernel init code

[Luis R. Rodriguez]

On Fri, Jan 22, 2016 at 06:12:47PM -0500, Boris Ostrovsky wrote:
> On 01/22/2016 06:01 PM, Luis R. Rodriguez wrote:
> >On Fri, Jan 22, 2016 at 04:35:48PM -0500, Boris Ostrovsky wrote:
> >>HVMlite guests (to be introduced in subsequent patches) share most
> >>of the kernel initialization code with PV(H).
> >>
> >>Signed-off-by: Boris Ostrovsky <boris.ostrovsky@oracle.com>
> >>---
> >>  arch/x86/xen/enlighten.c |  225 ++++++++++++++++++++++++----------------------
> >>  1 files changed, 119 insertions(+), 106 deletions(-)
> >>
> >>diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
> >>index d09e4c9..2cf446a 100644
> >>--- a/arch/x86/xen/enlighten.c
> >>+++ b/arch/x86/xen/enlighten.c
> >Whoa, I'm lost, its hard for me to tell what exactly stayed and what
> >got pulled into a helper, etc. Is there a possibility to split this
> >patch in 2 somehow to make the actual functional changes easier to
> >read? There are too many changes here and I just can't tell easily
> >what's going on.
> 
> 
> The only real changes that this patch introduces is it reorders some
> of the operations that used to be in xen_start_kernel(). This is
> done so that in the next patch when we add hvmlite we can easily put
> those specific to PV(H) inside 'if (!xen_hvm_domain())'. I probably
> should have said so in the commit message.

Ah, I see thanks.

> It is indeed difficult to review but I don't see how I can split
> this. Even if I just moved it (without reordering) it would still be
> hard to read.

A code shuffle but yet introducing non-functional changes as you did
in some other patches might help if possible, but sure if you can say
this is non-functional here or if you can split this up.

  Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Andrew Cooper]

On 22/01/2016 23:32, Luis R. Rodriguez wrote:
> On Fri, Jan 22, 2016 at 04:35:50PM -0500, Boris Ostrovsky wrote:
>> +	/*
>> +	 * See Documentation/x86/boot.txt.
>> +	 *
>> +	 * Version 2.12 supports Xen entry point but we will use default x86/PC
>> +	 * environment (i.e. hardware_subarch 0).
>> +	 */
>> +	xen_hvmlite_boot_params.hdr.version = 0x212;
>> +	xen_hvmlite_boot_params.hdr.type_of_loader = 9; /* Xen loader */
>> +}
> I realize PV got away with setting up boot_params on C code but best
> ask now that this new code is being introduced: why can't we just have
> the Xen hypervisor fill this in? It'd save us all this code.

I agree that this looks to be a mess.  Having said that, the DMLite boot
protocol is OS agnostic, and will be staying that way.

It happens to look suspiciously like multiboot; a flat 32bit protected
mode entry (at a location chosen in an ELF note), with %ebx pointing to
an in-ram structure containing things like a command line and module list.

I would have though the correct way to do direct Linux support would be
to have a very small init stub which constructs an appropriate zero
page, and lets the native entry point get on with things.

This covers the usecase where people wish to boot a specific Linux
kernel straight out of the dom0 filesystem.

For the alternative usecase of general OS support, dom0 would boot
something such as grub2 as the DMLite "kernel", at which point all
stooging around in the guests filesystem is done from guest context,
rather than control context (mitigating a substantial attack surface).

~Andrew

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Fri, Jan 22, 2016 at 4:30 PM, Andrew Cooper
<andrew.cooper3@citrix.com> wrote:
> I would have though the correct way to do direct Linux support would be
> to have a very small init stub which constructs an appropriate zero
> page, and lets the native entry point get on with things.

As hpa noted recently in another thread [0] that is precisely what
hardware_subarch and hardware_subarch_data was meant to be used for,
and its what I'm alluding to.

The only thing though is that as far as we're concerned on x86 we had
expected use of hardware_subarch and hardware_subarch_data only for
PV, and not for HVM. This seems to be HVM related, but I think this is
just a rebranding of PVH to HVMLite, right, so I think the use case of
hardware_subarch and hardware_subarch_data are still welcomed as
expected in the original design.

[0] http://lkml.kernel.org/r/56A130B5.8060701@zytor.com

 Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Fri, Jan 22, 2016 at 4:30 PM, Andrew Cooper
<andrew.cooper3@citrix.com> wrote:
> the DMLite boot
> protocol is OS agnostic, and will be staying that way.

What's the DMLite boot protocol? Is that the protocol that is defined
by Xen to boot Xen guests and dom0? Is this well documented somewhere?

To be clear are you saying that by no means will Xen change to instead
of setting a, say zero-page, it would just want to always stuff a xen
struct, pass that to the boot entry, and then expect always the guest
kernel to always parse this?

If true, then by no means, no matter how hard we try, and no matter
what we do on the Linux front to help clean things up will we be able
to have a unified bare metal / Xen entry. I'm noting it could be
possible though provided we do just set the zero page, the subarch to
Xen and subarch_data to the Xen custom data structure.

 Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Andrew Cooper]

On 23/01/2016 00:55, Luis R. Rodriguez wrote:
> On Fri, Jan 22, 2016 at 4:30 PM, Andrew Cooper
> <andrew.cooper3@citrix.com> wrote:
>> the DMLite boot
>> protocol is OS agnostic, and will be staying that way.
> What's the DMLite boot protocol?

It is  a statement of the initial state of a DMLite container.

> Is that the protocol that is defined by Xen to boot Xen guests and dom0?

Technically it is toolstack which constructs this initial state, but
broadly yes.

>  Is this well documented somewhere?

There is a patch out on the list formalising the ABI in writing. (Roger:
ping?)

> To be clear are you saying that by no means will Xen change to instead
> of setting a, say zero-page, it would just want to always stuff a xen
> struct, pass that to the boot entry, and then expect always the guest
> kernel to always parse this?

Correct.  Why do you think we should lumber non-Linux guests with a
Linux-specific boot protocol?

Quite apart from the fact that Linux is second to the table here
(FreeBSD was first), it causes inappropriate linkage between the
toolstack and the version of Linux wishing to be booted.

>
> If true, then by no means, no matter how hard we try, and no matter
> what we do on the Linux front to help clean things up will we be able
> to have a unified bare metal / Xen entry. I'm noting it could be
> possible though provided we do just set the zero page, the subarch to
> Xen and subarch_data to the Xen custom data structure.

All you need is a very small stub which starts per the DMLite ABI, sets
up an appropriate zero_page, and jumps to the native entrypoint.

However, this stub belongs in Linux, not in the Xen toolstack.  That
way, when the Linux boot protocol is modified, both sides can be updated
accordingly.

~Andrew

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Konrad Rzeszutek Wilk]

>However, this stub belongs in Linux, not in the Xen toolstack.  That
>way, when the Linux boot protocol is modified, both sides can be
>updated
>accordingly.

I would add that this idea is borrowed from the EFI stub code that Linux has which also constructs the boot parameter structure when invoked (either from firmware or from EFI shell).

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[H. Peter Anvin]

On January 23, 2016 7:34:33 AM PST, Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> wrote:
>
>>However, this stub belongs in Linux, not in the Xen toolstack.  That
>>way, when the Linux boot protocol is modified, both sides can be
>>updated
>>accordingly.
>
>I would add that this idea is borrowed from the EFI stub code that
>Linux has which also constructs the boot parameter structure when
>invoked (either from firmware or from EFI shell).

There is a huge difference though: EFI is a widely used multivendor industry standard.  You are taking about something Xen-specific, and which in good Xen tradition isn't even documented, apparently (did we ever get documentation for the hypervisor ABI?)

Asking "why burden Xen with something Linux-specific" is a pretty extreme case of the tail wagging the dog.

That being said, before any code can be put anywhere, it needs to be written.  We can argue where to put it later.  We went through this process with the EFI stub, too: a standalone implementation (efilinux) first.

-- 
Sent from my Android device with K-9 Mail. Please excuse brevity and formatting.

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Konrad Rzeszutek Wilk]

On January 23, 2016 11:01:06 AM EST, "H. Peter Anvin" <hpa@zytor.com> wrote:
>On January 23, 2016 7:34:33 AM PST, Konrad Rzeszutek Wilk
><konrad.wilk@oracle.com> wrote:
>>
>>>However, this stub belongs in Linux, not in the Xen toolstack.  That
>>>way, when the Linux boot protocol is modified, both sides can be
>>>updated
>>>accordingly.
>>
>>I would add that this idea is borrowed from the EFI stub code that
>>Linux has which also constructs the boot parameter structure when
>>invoked (either from firmware or from EFI shell).
>
>There is a huge difference though: EFI is a widely used multivendor
>industry standard.  You are taking about something Xen-specific, and
>which in good Xen tradition isn't even documented, apparently (did we
>ever get documentation for the hypervisor ABI?)
>
>Asking "why burden Xen with something Linux-specific" is a pretty
>extreme case of the tail wagging the dog.
>
>That being said, before any code can be put anywhere, it needs to be
>written.  We can argue where to put it later.  We went through this
>process with the EFI stub, too: a standalone implementation (efilinux)
>first.

http://lists.xenproject.org/archives/html/xen-devel/2015-12/msg01793.html

I believe is the latest version. Roger (CCed) has probably an updated one.

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[H. Peter Anvin]

On January 23, 2016 8:12:23 AM PST, Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> wrote:
>On January 23, 2016 11:01:06 AM EST, "H. Peter Anvin" <hpa@zytor.com>
>wrote:
>>On January 23, 2016 7:34:33 AM PST, Konrad Rzeszutek Wilk
>><konrad.wilk@oracle.com> wrote:
>>>
>>>>However, this stub belongs in Linux, not in the Xen toolstack.  That
>>>>way, when the Linux boot protocol is modified, both sides can be
>>>>updated
>>>>accordingly.
>>>
>>>I would add that this idea is borrowed from the EFI stub code that
>>>Linux has which also constructs the boot parameter structure when
>>>invoked (either from firmware or from EFI shell).
>>
>>There is a huge difference though: EFI is a widely used multivendor
>>industry standard.  You are taking about something Xen-specific, and
>>which in good Xen tradition isn't even documented, apparently (did we
>>ever get documentation for the hypervisor ABI?)
>>
>>Asking "why burden Xen with something Linux-specific" is a pretty
>>extreme case of the tail wagging the dog.
>>
>>That being said, before any code can be put anywhere, it needs to be
>>written.  We can argue where to put it later.  We went through this
>>process with the EFI stub, too: a standalone implementation (efilinux)
>>first.
>
>http://lists.xenproject.org/archives/html/xen-devel/2015-12/msg01793.html
>
>I believe is the latest version. Roger (CCed) has probably an updated
>one.

I suspect you should write a noninteractive bootloader as a reference implementation, and then consider porting Grub2 and maybe Syslinux to your ABI for those that want a full featured interactive bootloader compatible with the normal management.
-- 
Sent from my Android device with K-9 Mail. Please excuse brevity and formatting.

Re: [PATCH v1 06/12] xen/hvmlite: Initialize PCI

[Roger Pau Monné]

El 22/01/16 a les 22.35, Boris Ostrovsky ha escrit:
> HVMlite guests need PCI frontend and always have PV devices

We still haven't discussed how to perform pci-passthrough for HVMlite
guests. I admit there's a big chance that we are going to use the PV
pcifront driver but there's no guarantee about it yet, so I would just
leave it out for the moment.

Roger.

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Roger Pau Monné]

El 23/01/16 a les 17.12, Konrad Rzeszutek Wilk ha escrit:
> On January 23, 2016 11:01:06 AM EST, "H. Peter Anvin" <hpa@zytor.com> wrote:
>> On January 23, 2016 7:34:33 AM PST, Konrad Rzeszutek Wilk
>> <konrad.wilk@oracle.com> wrote:
>>>
>>>> However, this stub belongs in Linux, not in the Xen toolstack.  That
>>>> way, when the Linux boot protocol is modified, both sides can be
>>>> updated
>>>> accordingly.
>>>
>>> I would add that this idea is borrowed from the EFI stub code that
>>> Linux has which also constructs the boot parameter structure when
>>> invoked (either from firmware or from EFI shell).
>>
>> There is a huge difference though: EFI is a widely used multivendor
>> industry standard.  You are taking about something Xen-specific, and
>> which in good Xen tradition isn't even documented, apparently (did we
>> ever get documentation for the hypervisor ABI?)
>>
>> Asking "why burden Xen with something Linux-specific" is a pretty
>> extreme case of the tail wagging the dog.
>>
>> That being said, before any code can be put anywhere, it needs to be
>> written.  We can argue where to put it later.  We went through this
>> process with the EFI stub, too: a standalone implementation (efilinux)
>> first.
> 
> http://lists.xenproject.org/archives/html/xen-devel/2015-12/msg01793.html
> 
> I believe is the latest version. Roger (CCed) has probably an updated one.

Yes, the technical side has not changed at all. I've just send an
updated version with some wording changes:

http://lists.xenproject.org/archives/html/xen-devel/2016-01/msg03029.html

Roger.

Re: [PATCH v1 00/12] HVMlite domU support

[David Vrabel]

On 22/01/16 21:35, Boris Ostrovsky wrote:
> This series introduces HVMlite support for unprivileged guests.
> 
> It has been tested on Intel/AMD, both 32- and 64-bit, including CPU on- and
> offlining and save/restore. (Restore will result in APIC write warnings
> which exist now for 32-bit PV guests as well so I didn't address this in
> this series)

Can you remove PVH support in this series as well?  We won't necessarily
remove PVH support immediately but I'd like to see the ultimate end result.

David

Re: [PATCH v1 01/12] x86/smp: Make start_secondary() and initial_pg_pmd visible globally

[David Vrabel]

On 22/01/16 21:35, Boris Ostrovsky wrote:
> Xen's HVMlite guests will want to use them.

Please explain in detail why they are needed.

David

Re: [Xen-devel] [PATCH v1 02/12] xen/hvmlite: Factor out common kernel init code

[David Vrabel]

On 22/01/16 21:35, Boris Ostrovsky wrote:
> HVMlite guests (to be introduced in subsequent patches) share most
> of the kernel initialization code with PV(H).

Where possible, HVMlite should share initialization with bare metal/HVM
and not PV(H).

Sharing any code with the existing PVH code isn't useful, since PVH
support will be removed.

David

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/22/2016 07:30 PM, Andrew Cooper wrote:
> On 22/01/2016 23:32, Luis R. Rodriguez wrote:
>> On Fri, Jan 22, 2016 at 04:35:50PM -0500, Boris Ostrovsky wrote:
>>> +	/*
>>> +	 * See Documentation/x86/boot.txt.
>>> +	 *
>>> +	 * Version 2.12 supports Xen entry point but we will use default x86/PC
>>> +	 * environment (i.e. hardware_subarch 0).
>>> +	 */
>>> +	xen_hvmlite_boot_params.hdr.version = 0x212;
>>> +	xen_hvmlite_boot_params.hdr.type_of_loader = 9; /* Xen loader */
>>> +}
>> I realize PV got away with setting up boot_params on C code but best
>> ask now that this new code is being introduced: why can't we just have
>> the Xen hypervisor fill this in? It'd save us all this code.
> I agree that this looks to be a mess.  Having said that, the DMLite boot
> protocol is OS agnostic, and will be staying that way.
>
> It happens to look suspiciously like multiboot; a flat 32bit protected
> mode entry (at a location chosen in an ELF note), with %ebx pointing to
> an in-ram structure containing things like a command line and module list.
>
> I would have though the correct way to do direct Linux support would be
> to have a very small init stub which constructs an appropriate zero
> page, and lets the native entry point get on with things.

Which is really what 
hvmlite_start_xen()->xen_prepare_hvmlite()->hvmlite_bootparams() is 
doing. Not much more than that (for 64-bit it also loads identity 
mapping because that's what startup_64 wants)

-boris

>
> This covers the usecase where people wish to boot a specific Linux
> kernel straight out of the dom0 filesystem.
>
> For the alternative usecase of general OS support, dom0 would boot
> something such as grub2 as the DMLite "kernel", at which point all
> stooging around in the guests filesystem is done from guest context,
> rather than control context (mitigating a substantial attack surface).
>
> ~Andrew

Re: [PATCH v1 00/12] HVMlite domU support

[Boris Ostrovsky]

On 01/25/2016 05:51 AM, David Vrabel wrote:
> On 22/01/16 21:35, Boris Ostrovsky wrote:
>> This series introduces HVMlite support for unprivileged guests.
>>
>> It has been tested on Intel/AMD, both 32- and 64-bit, including CPU on- and
>> offlining and save/restore. (Restore will result in APIC write warnings
>> which exist now for 32-bit PV guests as well so I didn't address this in
>> this series)
> Can you remove PVH support in this series as well?  We won't necessarily
> remove PVH support immediately but I'd like to see the ultimate end result.

I'd rather wait until we have HVMlite dom0 before dropping PVH. If 
nothing else it may help debugging.

BTW, I assume we are going to rename HVMlite to PVH once the original 
implementation is removed.


-boris

Re: [PATCH v1 01/12] x86/smp: Make start_secondary() and initial_pg_pmd visible globally

[Boris Ostrovsky]

On 01/25/2016 05:53 AM, David Vrabel wrote:
> On 22/01/16 21:35, Boris Ostrovsky wrote:
>> Xen's HVMlite guests will want to use them.
> Please explain in detail why they are needed.
>


Actually, start_secondary is not needed anymore, I removed its use at 
some point.

initial_pg_pmd (together with initial_page_table) are not really 
required  --- I just used them for temporary page tables in the hvmlite 
startup code instead of allocating dedicated pages for that. Perhaps 
there is other (Xen-specific) area that I can use for that. Once we jump 
to startup_32 these tables are no longer in use.

-boris

Re: [Xen-devel] [PATCH v1 02/12] xen/hvmlite: Factor out common kernel init code

[Boris Ostrovsky]

On 01/25/2016 06:04 AM, David Vrabel wrote:
> On 22/01/16 21:35, Boris Ostrovsky wrote:
>> HVMlite guests (to be introduced in subsequent patches) share most
>> of the kernel initialization code with PV(H).
> Where possible, HVMlite should share initialization with bare metal/HVM
> and not PV(H).

This is really platform initialization, for HVMlite it's invoked as 
x86_init.oem.arch_setup(). Things like xen_setup_features(), 
xen_init_apic() etc.

I suppose I can move it to xen_hvm_guest_init() but then we will have 
some amount of code duplication. There is also a chunk of code there (in 
xen_init_kernel()) that will probably be used for HVMlite dom0.

>
> Sharing any code with the existing PVH code isn't useful, since PVH
> support will be removed.

There is nothing PVH-specific, I said "(H)" mostly because that code is 
the same is PV.

-boris

Re: [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/22/2016 06:32 PM, Luis R. Rodriguez wrote:
> On Fri, Jan 22, 2016 at 04:35:50PM -0500, Boris Ostrovsky wrote:
>
>> +/*
>> + * This routine (and those that it might call) should not use
>> + * anything that lives in .bss since that segment will be cleared later
>> + */
>> +void __init xen_prepare_hvmlite(void)
>> +{
>> +	u32 eax, ecx, edx, msr;
>> +	u64 pfn;
>> +
>> +	cpuid(xen_cpuid_base() + 2, &eax, &msr, &ecx, &edx);
>> +	pfn = __pa(hypercall_page);
>> +	wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));
>> +
>> +	pv_info.name = "Xen HVMlite";
>> +	xen_domain_type = XEN_HVM_DOMAIN;
>> +	xen_hvmlite = 1;
>> +
>> +	x86_init.oem.arch_setup = xen_init_kernel;
>> +	x86_init.oem.banner = xen_banner;
>> +
>> +	hvmlite_bootparams();
>> +}
>> +#endif
> If the boot_params.hdr.hardware_subarch_data pointed to a custom
> struct then the first C entry point for Xen could shuffle this and
> set this too, by still using less asm entry helpers. We'd still
> need this run but with the linker table I think we could have
> a stub small stub for hvm run, it would not be a call from asm.

Perhaps, but someone would still have to set hardware_subarch. And it's 
hvmlite_bootparams() that does it.

And that's not sufficient, I think. There are still some things that 
trampoline code sets up (e.g. page tables for 64-bit).

-boris

Re: [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Mon, Jan 25, 2016 at 11:08:47AM -0500, Boris Ostrovsky wrote:
> On 01/22/2016 06:32 PM, Luis R. Rodriguez wrote:
> >On Fri, Jan 22, 2016 at 04:35:50PM -0500, Boris Ostrovsky wrote:
> >
> >>+/*
> >>+ * This routine (and those that it might call) should not use
> >>+ * anything that lives in .bss since that segment will be cleared later
> >>+ */
> >>+void __init xen_prepare_hvmlite(void)
> >>+{
> >>+	u32 eax, ecx, edx, msr;
> >>+	u64 pfn;
> >>+
> >>+	cpuid(xen_cpuid_base() + 2, &eax, &msr, &ecx, &edx);
> >>+	pfn = __pa(hypercall_page);
> >>+	wrmsr_safe(msr, (u32)pfn, (u32)(pfn >> 32));
> >>+
> >>+	pv_info.name = "Xen HVMlite";
> >>+	xen_domain_type = XEN_HVM_DOMAIN;
> >>+	xen_hvmlite = 1;
> >>+
> >>+	x86_init.oem.arch_setup = xen_init_kernel;
> >>+	x86_init.oem.banner = xen_banner;
> >>+
> >>+	hvmlite_bootparams();
> >>+}
> >>+#endif
> >If the boot_params.hdr.hardware_subarch_data pointed to a custom
> >struct then the first C entry point for Xen could shuffle this and
> >set this too, by still using less asm entry helpers. We'd still
> >need this run but with the linker table I think we could have
> >a stub small stub for hvm run, it would not be a call from asm.
> 
> Perhaps, but someone would still have to set hardware_subarch. And
> it's hvmlite_bootparams() that does it.

No, Xen would do it as well, essentially all of hvmlite_bootparams() could be
done in Xen.

> And that's not sufficient, I think. There are still some things that
> trampoline code sets up (e.g. page tables for 64-bit).

Sure, but only what is required, that should be rather smaller.

  Luis

Re: [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[H. Peter Anvin]

On 01/25/16 13:12, Luis R. Rodriguez wrote:
>>
>> Perhaps, but someone would still have to set hardware_subarch. And
>> it's hvmlite_bootparams() that does it.
> 
> No, Xen would do it as well, essentially all of hvmlite_bootparams() could be
> done in Xen.
> 

Or a stub code.

	-hpa

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Sat, Jan 23, 2016 at 02:49:36PM +0000, Andrew Cooper wrote:
> On 23/01/2016 00:55, Luis R. Rodriguez wrote:
> > On Fri, Jan 22, 2016 at 4:30 PM, Andrew Cooper
> > <andrew.cooper3@citrix.com> wrote:
> >> the DMLite boot
> >> protocol is OS agnostic, and will be staying that way.
> > What's the DMLite boot protocol?
> 
> It is  a statement of the initial state of a DMLite container.

I see.

> > Is that the protocol that is defined by Xen to boot Xen guests and dom0?
> 
> Technically it is toolstack which constructs this initial state, but
> broadly yes.

I see thanks.

> >  Is this well documented somewhere?
> 
> There is a patch out on the list formalising the ABI in writing. (Roger:
> ping?)

Thanks for this.

> > To be clear are you saying that by no means will Xen change to instead
> > of setting a, say zero-page, it would just want to always stuff a xen
> > struct, pass that to the boot entry, and then expect always the guest
> > kernel to always parse this?
> 
> Correct.  Why do you think we should lumber non-Linux guests with a
> Linux-specific boot protocol?

You should by no means do that, agreed, however one should have the
flexibility to support such types of OSes that use page specific
settings for the OS. The way I see it this is an OS feature and
obviously it can vary.

> Quite apart from the fact that Linux is second to the table here
> (FreeBSD was first),

This means little but just history.

> it causes inappropriate linkage between the
> toolstack and the version of Linux wishing to be booted.

There are ways to do it in such a way that it does not create dependency issues
on Linux specific code.


0) The Xen way and EFI way

A generic data structure is passed to the entry point on the kernel to
load the kernel. The kernel in turn must parse this generic specific struct
and interprets it and translate it to the kernel specific values.

1) The qemu way:

One way is to simply not refer to the boot_params data structures but IMHO that
produces really ugly code. The qemu folks did it this way, so qemu does not use
arch/x86/include/uapi/asm/bootparam.h in any way, instead it uses offsets from
a char *header. Refer to load_linux() in hw/i386/pc.c, for instance:

header[0x211] |= 0x80;  /* CAN_USE_HEAP */

2) The grub way:

Another way, which grub uses, is to define their own data structures based
on arch/x86/include/uapi/asm/bootparam.h, with their own data types, and refer
to them in code, for instance refer to grub_cmd_linux() on the grub file
grub-core/loader/i386/pc/linux.c and its copy definition of the header definition
in include/grub/i386/linux.h.

lh.loadflags |= GRUB_LINUX_FLAG_CAN_USE_HEAP;

The way lguest does it in the lguest launcher is IMHO the cleanest of course,
but it includes asm/bootparam.h and uses that in code:

/* Boot protocol version: 2.07 supports the fields for lguest. */       
boot->hdr.version = 0x207;  

But that does mean copying over or using the bootparam.h file and using
linux data types.

3) Merge of xen way and using subarch_data

Only set the subarch and subarch_data pointer, and have the kernel then
read the generic data structure and parse it as it used to, the benefit
is sharing a common entry point.

No one uses this yet. But I think its a reasonable compromise.

Perhaps there are other ways as well.

> > If true, then by no means, no matter how hard we try, and no matter
> > what we do on the Linux front to help clean things up will we be able
> > to have a unified bare metal / Xen entry. I'm noting it could be
> > possible though provided we do just set the zero page, the subarch to
> > Xen and subarch_data to the Xen custom data structure.
> 
> All you need is a very small stub which starts per the DMLite ABI, sets
> up an appropriate zero_page, and jumps to the native entrypoint.

Right.

> However, this stub belongs in Linux, not in the Xen toolstack.  That
> way, when the Linux boot protocol is modified, both sides can be updated
> accordingly.

Makes sense the different entry points just seems best avoided if possible.

  Luis

Re: [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/25/2016 04:21 PM, H. Peter Anvin wrote:
> On 01/25/16 13:12, Luis R. Rodriguez wrote:
>>> Perhaps, but someone would still have to set hardware_subarch. And
>>> it's hvmlite_bootparams() that does it.
>> No, Xen would do it as well, essentially all of hvmlite_bootparams() could be
>> done in Xen.
>>
> Or a stub code.

This patch in fact is the stub for Xen HVMlite guests, after we are done 
with it we jump to bare-metal startup code (i.e startup_32|64)

-boris

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/25/2016 05:19 PM, Luis R. Rodriguez wrote:
> On Sat, Jan 23, 2016 at 02:49:36PM +0000, Andrew Cooper wrote:
>
>
>> it causes inappropriate linkage between the
>> toolstack and the version of Linux wishing to be booted.
> There are ways to do it in such a way that it does not create dependency issues
> on Linux specific code.
>
>
> 0) The Xen way and EFI way
>
> A generic data structure is passed to the entry point on the kernel to
> load the kernel. The kernel in turn must parse this generic specific struct
> and interprets it and translate it to the kernel specific values.
>
> 1) The qemu way:
>
> One way is to simply not refer to the boot_params data structures but IMHO that
> produces really ugly code. The qemu folks did it this way, so qemu does not use
> arch/x86/include/uapi/asm/bootparam.h in any way, instead it uses offsets from
> a char *header. Refer to load_linux() in hw/i386/pc.c, for instance:
>
> header[0x211] |= 0x80;  /* CAN_USE_HEAP */
>
> 2) The grub way:
>
> Another way, which grub uses, is to define their own data structures based
> on arch/x86/include/uapi/asm/bootparam.h, with their own data types, and refer
> to them in code, for instance refer to grub_cmd_linux() on the grub file
> grub-core/loader/i386/pc/linux.c and its copy definition of the header definition
> in include/grub/i386/linux.h.
>
> lh.loadflags |= GRUB_LINUX_FLAG_CAN_USE_HEAP;
>
> The way lguest does it in the lguest launcher is IMHO the cleanest of course,
> but it includes asm/bootparam.h and uses that in code:
>
> /* Boot protocol version: 2.07 supports the fields for lguest. */
> boot->hdr.version = 0x207;
>
> But that does mean copying over or using the bootparam.h file and using
> linux data types.
>
> 3) Merge of xen way and using subarch_data
>
> Only set the subarch and subarch_data pointer, and have the kernel then
> read the generic data structure and parse it as it used to, the benefit
> is sharing a common entry point.
>
> No one uses this yet. But I think its a reasonable compromise.
>
> Perhaps there are other ways as well.
>
>>> If true, then by no means, no matter how hard we try, and no matter
>>> what we do on the Linux front to help clean things up will we be able
>>> to have a unified bare metal / Xen entry. I'm noting it could be
>>> possible though provided we do just set the zero page, the subarch to
>>> Xen and subarch_data to the Xen custom data structure.
>> All you need is a very small stub which starts per the DMLite ABI, sets
>> up an appropriate zero_page, and jumps to the native entrypoint.
> Right.


I am trying to understand what your objection is to what is proposed in 
this patch. It is the size of the stub? If yes -- what would you like to 
leave out to be done later?


-boris

>
>> However, this stub belongs in Linux, not in the Xen toolstack.  That
>> way, when the Linux boot protocol is modified, both sides can be updated
>> accordingly.
> Makes sense the different entry points just seems best avoided if possible.
>
>    Luis

Re: [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Mon, Jan 25, 2016 at 05:28:08PM -0500, Boris Ostrovsky wrote:
> On 01/25/2016 04:21 PM, H. Peter Anvin wrote:
> >On 01/25/16 13:12, Luis R. Rodriguez wrote:
> >>>Perhaps, but someone would still have to set hardware_subarch. And
> >>>it's hvmlite_bootparams() that does it.
> >>No, Xen would do it as well, essentially all of hvmlite_bootparams() could be
> >>done in Xen.
> >>
> >Or a stub code.
> 
> This patch in fact is the stub for Xen HVMlite guests, after we are
> done with it we jump to bare-metal startup code (i.e startup_32|64)

Right the point is the stub need not be in Linux, I'll explain in the other
thread where I provided more details on the different known approaches.

  Luis

Re: [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Andy Lutomirski]

On Tue, Jan 26, 2016 at 10:34 AM, Luis R. Rodriguez <mcgrof@suse.com> wrote:
> On Mon, Jan 25, 2016 at 05:28:08PM -0500, Boris Ostrovsky wrote:
>> On 01/25/2016 04:21 PM, H. Peter Anvin wrote:
>> >On 01/25/16 13:12, Luis R. Rodriguez wrote:
>> >>>Perhaps, but someone would still have to set hardware_subarch. And
>> >>>it's hvmlite_bootparams() that does it.
>> >>No, Xen would do it as well, essentially all of hvmlite_bootparams() could be
>> >>done in Xen.
>> >>
>> >Or a stub code.
>>
>> This patch in fact is the stub for Xen HVMlite guests, after we are
>> done with it we jump to bare-metal startup code (i.e startup_32|64)
>
> Right the point is the stub need not be in Linux, I'll explain in the other
> thread where I provided more details on the different known approaches.
>

ISTM if the Xen ABI-specified entry point has a different convention
than the Linux native entry, then the stub should live in Linux.  It
would be just a couple if lines of code, right?

The issue that caused headaches in the past isn't that there's code
that's executed only on native, it's that there are whole big
functions that are executed only on native for no good reason and that
aren't clearly marked.

If we had native_start_kernel and xen_start_kernel, and they both
called very quickly in to common_start_kernel, it would be very clear
what's going on.

--Andy

Re: [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/26/2016 01:46 PM, Andy Lutomirski wrote:
> On Tue, Jan 26, 2016 at 10:34 AM, Luis R. Rodriguez <mcgrof@suse.com> wrote:
>> On Mon, Jan 25, 2016 at 05:28:08PM -0500, Boris Ostrovsky wrote:
>>> On 01/25/2016 04:21 PM, H. Peter Anvin wrote:
>>>> On 01/25/16 13:12, Luis R. Rodriguez wrote:
>>>>>> Perhaps, but someone would still have to set hardware_subarch. And
>>>>>> it's hvmlite_bootparams() that does it.
>>>>> No, Xen would do it as well, essentially all of hvmlite_bootparams() could be
>>>>> done in Xen.
>>>>>
>>>> Or a stub code.
>>> This patch in fact is the stub for Xen HVMlite guests, after we are
>>> done with it we jump to bare-metal startup code (i.e startup_32|64)
>> Right the point is the stub need not be in Linux, I'll explain in the other
>> thread where I provided more details on the different known approaches.
>>
> ISTM if the Xen ABI-specified entry point has a different convention
> than the Linux native entry, then the stub should live in Linux.  It
> would be just a couple if lines of code, right?

It's not exactly a couple of lines but it's not large neither. It mainly 
sets up boot_params (similar to what make_boot_params() does for EFI). 
Plus, for 64-bit, it loads identity page tables and switches to long 
mode. And then jumps to bare-meta startup code.


>
> The issue that caused headaches in the past isn't that there's code
> that's executed only on native, it's that there are whole big
> functions that are executed only on native for no good reason and that
> aren't clearly marked.

This won't happen with HVMlite.

>
> If we had native_start_kernel and xen_start_kernel, and they both
> called very quickly in to common_start_kernel, it would be very clear
> what's going on.

What is now xen_start_kernel() is no longer the entry point for HVMlite. 
It is called as x86_init.oem.arch_setup() (or I may even move it to 
x86_hyper_xen.init_platform() or something like that).

-boris

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Tue, Jan 26, 2016 at 11:00 AM, Boris Ostrovsky
<boris.ostrovsky@oracle.com> wrote:
> On 01/26/2016 01:46 PM, Andy Lutomirski wrote:
>>
>> On Tue, Jan 26, 2016 at 10:34 AM, Luis R. Rodriguez <mcgrof@suse.com>
>> wrote:
>>>
>>> On Mon, Jan 25, 2016 at 05:28:08PM -0500, Boris Ostrovsky wrote:
>>>>
>>>> On 01/25/2016 04:21 PM, H. Peter Anvin wrote:
>>>>>
>>>>> On 01/25/16 13:12, Luis R. Rodriguez wrote:
>>>>>>>
>>>>>>> Perhaps, but someone would still have to set hardware_subarch. And
>>>>>>> it's hvmlite_bootparams() that does it.
>>>>>>
>>>>>> No, Xen would do it as well, essentially all of hvmlite_bootparams()
>>>>>> could be
>>>>>> done in Xen.
>>>>>>
>>>>> Or a stub code.
>>>>
>>>> This patch in fact is the stub for Xen HVMlite guests, after we are
>>>> done with it we jump to bare-metal startup code (i.e startup_32|64)
>>>
>>> Right the point is the stub need not be in Linux, I'll explain in the
>>> other
>>> thread where I provided more details on the different known approaches.
>>>
>> ISTM if the Xen ABI-specified entry point has a different convention
>> than the Linux native entry, then the stub should live in Linux.  It
>> would be just a couple if lines of code, right?
>
>
> It's not exactly a couple of lines but it's not large neither. It mainly
> sets up boot_params (similar to what make_boot_params() does for EFI). Plus,
> for 64-bit, it loads identity page tables and switches to long mode. And
> then jumps to bare-meta startup code.

This terse summary provides an example of the issue I'm highlighting.
Even though the stub is small we undermine its impact!

Although the stub is small the different entry point also enables
subtle additions which are required, although they are minimal they
are important and if not considered for new x86 features causes
regressions. I don't care what people tell me about "this should have
been caught by code review, and no one CC'd me -- whaa!" -- this is a
silly expectation and I think we should do better.

Case in point:

xen_start_kernel() has seen regressions now on both cr4_init_shadow()
which you forgot to add to Xen Andy -- and later Boris fixed. Another
example: the latest one is a kasan init -- which to this day remains
fucked up -- a Kasan enabled PV guest crashes, and fixing is no where
in sight. I flagged this a while ago, and I think we should put a
proactive measure in place for that.

The linker table stuff I'm doing was not for kicks -- its a means to
an end here, and although I can't yet read subarch at early init, if
we had it, then things link xen_start_kernel() could just be an x86
init stub, it'd be the first stub Xen runs. You can keep whatever
boot_params setup stub it does today, even though I think its cleaner
done in Xen, but at the very least it could at least instead just
*read* the Xen custom generic structure to parse and set boot_params
by using subarch_data pointer.

I also have been reading the history of code changes on the other
entry points in Linux and even though the code is small I see tons of
commits in there for minor but critical fixes all over, likewise
comments and recommendations and visions to unify / share code. I
refuse to accept that we should undermine the issues of a new entry
point or leaving the situation as-is with dual entry points for x86_64
 / xen if we can instead just cleanly unify even asm entry points.

>> The issue that caused headaches in the past isn't that there's code
>> that's executed only on native, it's that there are whole big
>> functions that are executed only on native for no good reason and that
>> aren't clearly marked.

Its more than that, as I noted.

> This won't happen with HVMlite.

And that's fine, its just I think we can avoid yet-another entry --
even if we still are coming into a common C entry later.

>> If we had native_start_kernel and xen_start_kernel, and they both
>> called very quickly in to common_start_kernel, it would be very clear
>> what's going on.
>
> What is now xen_start_kernel() is no longer the entry point for HVMlite. It
> is called as x86_init.oem.arch_setup() (or I may even move it to
> x86_hyper_xen.init_platform() or something like that).

And that's a huge win! Yet I invite us to consider other prospects to
even merge more and simplify more.

 Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Mon, Jan 25, 2016 at 05:55:02PM -0500, Boris Ostrovsky wrote:
> On 01/25/2016 05:19 PM, Luis R. Rodriguez wrote:
> >On Sat, Jan 23, 2016 at 02:49:36PM +0000, Andrew Cooper wrote:
> >
> >
> >>it causes inappropriate linkage between the
> >>toolstack and the version of Linux wishing to be booted.
> >There are ways to do it in such a way that it does not create dependency issues
> >on Linux specific code.
> >
> >
> >0) The Xen way and EFI way
> >
> >A generic data structure is passed to the entry point on the kernel to
> >load the kernel. The kernel in turn must parse this generic specific struct
> >and interprets it and translate it to the kernel specific values.
> >
> >1) The qemu way:
> >
> >One way is to simply not refer to the boot_params data structures but IMHO that
> >produces really ugly code. The qemu folks did it this way, so qemu does not use
> >arch/x86/include/uapi/asm/bootparam.h in any way, instead it uses offsets from
> >a char *header. Refer to load_linux() in hw/i386/pc.c, for instance:
> >
> >header[0x211] |= 0x80;  /* CAN_USE_HEAP */
> >
> >2) The grub way:
> >
> >Another way, which grub uses, is to define their own data structures based
> >on arch/x86/include/uapi/asm/bootparam.h, with their own data types, and refer
> >to them in code, for instance refer to grub_cmd_linux() on the grub file
> >grub-core/loader/i386/pc/linux.c and its copy definition of the header definition
> >in include/grub/i386/linux.h.
> >
> >lh.loadflags |= GRUB_LINUX_FLAG_CAN_USE_HEAP;
> >
> >The way lguest does it in the lguest launcher is IMHO the cleanest of course,
> >but it includes asm/bootparam.h and uses that in code:
> >
> >/* Boot protocol version: 2.07 supports the fields for lguest. */
> >boot->hdr.version = 0x207;
> >
> >But that does mean copying over or using the bootparam.h file and using
> >linux data types.
> >
> >3) Merge of xen way and using subarch_data
> >
> >Only set the subarch and subarch_data pointer, and have the kernel then
> >read the generic data structure and parse it as it used to, the benefit
> >is sharing a common entry point.
> >
> >No one uses this yet. But I think its a reasonable compromise.
> >
> >Perhaps there are other ways as well.
> >
> >>>If true, then by no means, no matter how hard we try, and no matter
> >>>what we do on the Linux front to help clean things up will we be able
> >>>to have a unified bare metal / Xen entry. I'm noting it could be
> >>>possible though provided we do just set the zero page, the subarch to
> >>>Xen and subarch_data to the Xen custom data structure.
> >>All you need is a very small stub which starts per the DMLite ABI, sets
> >>up an appropriate zero_page, and jumps to the native entrypoint.
> >Right.
> 
> 
> I am trying to understand what your objection is to what is proposed
> in this patch. It is the size of the stub? If yes -- what would you
> like to leave out to be done later?

I explained that here briefly:

> >>However, this stub belongs in Linux, not in the Xen toolstack.  That
> >>way, when the Linux boot protocol is modified, both sides can be updated
> >>accordingly.
>
> >Makes sense the different entry points just seems best avoided if possible.

But let me elaborate:

We want to strive towards avoiding new entry points on Linux at all costs,
there have been historic issues with having multiple entry points, some of these
may relate to Xen, some not so much but one should not ignore history. A few
related patches, this incldues EFI entry points since Konrad brings up Xen's
approach was inspired by that as well. Read from bottom up.

Kasan: not yet fixed, the feature completely ignored Xen PV guests as a
possible candidate for x86. Xen PV guests with Kasan enabled crash. The fix
is not so trivial. What I'd prefer is for kasan to have a feature to be disabled
at run time, and with that in place we can simply at early boot disable Kasan
early on boot on Xen until Kasan is properly implemented and supported on Xen.

5054daa285beaf706f051fbd395dc36c9f0f907f - x86/xen: Initialize cr4 shadow for 64-bit PV(H) guests

Due to xen's separate entry point and the addition of a cr4 shadow, only
native had its feature added, this made Xen crash.

f3670394c29ff3730638762c1760fd2f624e6d7b - Revert "x86/efi: Fixup GOT in all boot code paths"

Fails to boot LInus' Sony Vaio Pro 11

9cb0e394234d244fe5a97e743ec9dd7ddff7e64b - x86/efi: Fixup GOT in all boot code paths

Because of the multiple boot entry points the fixup for GOT needs to be done in
different places.

7e8213c1f3acc064aef37813a39f13cbfe7c3ce7 - x86/efi: Correct EFI boot stub use of code32_start

Title says it.

b8ff87a6158886771677e6dc8139bac6e3cba717 - x86/efi: Firmware agnostic handover entry points

Fixes the issue of a mismatch on 32-bit and 64-bit between firmware and kernel run.

99f857db8857aff691c51302f93648263ed07eb1 - x86, build: Dynamically find entry points in compressed startup code

Boot loaders abuse static entry point. Perhaps not directly related to Xen but its a
lesson to learn that regardless of what you document people may abuse the entry points
in very unexpected ways.

f791620fa7517e1045742c475a7f005db9a634b8 - x86, efi: Fix 32-bit EFI handover protocol entry point

After the EFI handover protocol was added 32-bit boots had an issue.

9ca8f72a9297f2052d806bd1111e176533aa69bd  - x86, efi: Handover Protocol

*Only* use an EFI boot stub for EFI capable to avoid duplicate code in
boot loaders to set up and boot Linux.

51b26ada79b605ed709ddcedbb6012e8f8e0ebed - x86: unify arch/x86/boot/compressed/vmlinux_*.lds

Linus unifies vmlinux_*.lds and calls to unify startup_32() and startup_64().
This hasn't been done yet.

--

As someone new reading all this code / commits all I can do is cringe when I see
yet-another-entry-point being added. I think we can do better, so we should strive
to that unless there are real technical impossibilities here.

What I'm proposing?

1) Lets see if we can put a proactive stop-gap to issues such as the cr4 shadow
bug and Kasan bugs from creeping in. This should not only help PV but perhaps
HVMLite. If you'd like to help with that refer to this thread:

http://lkml.kernel.org/r/CAB=NE6VTCRCazcNpCdJ7pN1eD3=x_fcGOdH37MzVpxkKEN5esw@mail.gmail.com

2) asm code sharing prospects - rebranding of PVH to HVMlite

It sounds like HVMlite is really just a clean PVH implementation so we'll
be doing this it seems according to this approach:

a) Since PVH brand is taken add new new Xen clean solution as "HVMLite implementation"
b) Drop PVH
c) Re-brand HVMLite as PVH

Is there really no asm code to share between PV and HVMlite ? How about between
PV and other Linux asm ? Specifically I'm talking about a huge new entry point
called hvmlite_start_xen() of asm code. How much sharing or duplication
is being avoided here?

3) C code sharing prospects: rebranding of PVH to HVMlite

This code seems to share a lot from PV guest. Can we combine?

4) hardware_subarch, hardware_subarch_data and future prospects

Your patch relies on a *new* Linux entry point. Sure, we had one
for EFI, and sure there is another for Xen PV, but since you're just
rebranding PVH to HVMlite and given historic issues with any new
Linux entry points I'd like for us to take a breather and evaluate
the extent our lofty unification goals, and how the x86 boot protocol
could help with this already.

Now, perhaps today it may seem as difficult to unify asm entry points
today all over, but if we can take baby steps towards that I think that
should be seriously evaluated.

For instance, we should consider on relying on hardware_subarch and
hardware_subarch_data to fetch the hvmlite_start_info by taking advantage of
the x86 boot protocol.  The hvmlite_start_info is what Xen uses to send us info
about the guest as your patch proposes (this matches the Xen PV style entry),
we stash it into a standard Linux boot_params variable called
xen_hvmlite_boot_params for the Xen guest in hvmlite_bootparam(). This
data structure and code seems to match very much the PV guest structure,
why not just use a union and differentiate on PV subtype ? If you want to avoid
a lot of PV calls for HVMlite it seems you could just take advantage of
subarch Xen type, and differentiate on the subarch_data within Xen code
to make that code just PV sepecific.

I only see gains by using the Xen subarch, so would like to know why PC is
being pushed.

  Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/26/2016 03:30 PM, Luis R. Rodriguez wrote:
>
> What I'm proposing?
>
> 1) Lets see if we can put a proactive stop-gap to issues such as the cr4 shadow
> bug and Kasan bugs from creeping in. This should not only help PV but perhaps
> HVMLite. If you'd like to help with that refer to this thread:
>
> http://lkml.kernel.org/r/CAB=NE6VTCRCazcNpCdJ7pN1eD3=x_fcGOdH37MzVpxkKEN5esw@mail.gmail.com
>
> 2) asm code sharing prospects - rebranding of PVH to HVMlite
>
> It sounds like HVMlite is really just a clean PVH implementation so we'll
> be doing this it seems according to this approach:
>
> a) Since PVH brand is taken add new new Xen clean solution as "HVMLite implementation"
> b) Drop PVH
> c) Re-brand HVMLite as PVH
>
> Is there really no asm code to share between PV and HVMlite ?

Not the early boot code.

>   How about between
> PV and other Linux asm ? Specifically I'm talking about a huge new entry point
> called hvmlite_start_xen() of asm code. How much sharing or duplication
> is being avoided here?

I don't see that we can share much here. Especially given that 
hvmlite_start() is a 32-bit entry point so we can't start a 64-bit PV guest.

I suppose we can find some common (C) code that sets boot_params but on 
the first sight they are quite different too.

(And whether it's huge is a matter of opinion ;-). It is more than a 
couple of instructions, I'll give you that)

>
> 3) C code sharing prospects: rebranding of PVH to HVMlite
>
> This code seems to share a lot from PV guest. Can we combine?

HVMlite closely follows HVM (and baremetal) code path. There is some 
common setup code with PV (which is what the second patch in this series 
is trying to factor out)

>
> 4) hardware_subarch, hardware_subarch_data and future prospects
>
> Your patch relies on a *new* Linux entry point. Sure, we had one
> for EFI, and sure there is another for Xen PV, but since you're just
> rebranding PVH to HVMlite and given historic issues with any new
> Linux entry points I'd like for us to take a breather and evaluate
> the extent our lofty unification goals, and how the x86 boot protocol
> could help with this already.

I am not sure I see how you can avoid having new entry point. For 
example, all HVMlite guests start in 32-bit mode. Who will switch to 
long mode?

>
> Now, perhaps today it may seem as difficult to unify asm entry points
> today all over, but if we can take baby steps towards that I think that
> should be seriously evaluated.
>
> For instance, we should consider on relying on hardware_subarch and
> hardware_subarch_data to fetch the hvmlite_start_info by taking advantage of
> the x86 boot protocol.  The hvmlite_start_info is what Xen uses to send us info
> about the guest as your patch proposes (this matches the Xen PV style entry),
> we stash it into a standard Linux boot_params variable called
> xen_hvmlite_boot_params for the Xen guest in hvmlite_bootparam(). This
> data structure and code seems to match very much the PV guest structure,

No, the two don't match at all.

> why not just use a union and differentiate on PV subtype ? If you want to avoid
> a lot of PV calls for HVMlite it seems you could just take advantage of
> subarch Xen type, and differentiate on the subarch_data within Xen code
> to make that code just PV sepecific.
>
> I only see gains by using the Xen subarch, so would like to know why PC is
> being pushed.

It's not that subarch 0 is being pushed here. It's that I don't see how 
it can be useful for this particular guest type. Maybe as we add new 
features (or discover something that we've missed up till now) we can 
switch to using it. If you think we should delay initializing 
boot_params until then --- I will disagree: boot_params are used before 
we look at subarch and I don't believe it makes sense to pick and choose 
what to initialize before and what can wait.

(And I am not sure it can be useful on PV neither, at least the way it 
is used now. You will not reach the point in the (32-bit) code where it 
is tested. You will die way earlier (I think on startup_32()'s fourth 
instruction).)


-boris

Re: [PATCH v1 01/12] x86/smp: Make start_secondary() and initial_pg_pmd visible globally

[Borislav Petkov]

On Mon, Jan 25, 2016 at 10:30:26AM -0500, Boris Ostrovsky wrote:
> initial_pg_pmd (together with initial_page_table) are not really required
> --- I just used them for temporary page tables in the hvmlite startup code
> instead of allocating dedicated pages for that. Perhaps there is other
> (Xen-specific) area that I can use for that. Once we jump to startup_32
> these tables are no longer in use.

Then please create and use your own. There's no need for this short
temporary dependency during boot, especially if one day we go and change
things and break xen in the process.

-- 
Regards/Gruss,
    Boris.

ECO tip #101: Trim your mails when you reply.

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Tue, Jan 26, 2016 at 04:51:38PM -0500, Boris Ostrovsky wrote:
> On 01/26/2016 03:30 PM, Luis R. Rodriguez wrote:

> hvmlite_start() is a 32-bit entry point [...]
>
> >4) hardware_subarch, hardware_subarch_data and future prospects
> >
> >Your patch relies on a *new* Linux entry point. Sure, we had one
> >for EFI, and sure there is another for Xen PV, but since you're just
> >rebranding PVH to HVMlite and given historic issues with any new
> >Linux entry points I'd like for us to take a breather and evaluate
> >the extent our lofty unification goals, and how the x86 boot protocol
> >could help with this already.
> 
> I am not sure I see how you can avoid having new entry point. For
> example, all HVMlite guests start in 32-bit mode. Who will switch to
> long mode?

x86 i386 entry points need to have code to do all that stuff, this can
happen for instance when you boot x86_64 from a 32-bit boot loader,
and I think other things as well are possible that trigger this as well.

> >why not just use a union and differentiate on PV subtype ? If you want to avoid
> >a lot of PV calls for HVMlite it seems you could just take advantage of
> >subarch Xen type, and differentiate on the subarch_data within Xen code
> >to make that code just PV sepecific.
> >
> >I only see gains by using the Xen subarch, so would like to know why PC is
> >being pushed.
> 
> It's not that subarch 0 is being pushed here. It's that I don't see
> how it can be useful for this particular guest type. Maybe as we add
> new features (or discover something that we've missed up till now)
> we can switch to using it. If you think we should delay initializing
> boot_params until then --- I will disagree: boot_params are used
> before we look at subarch and I don't believe it makes sense to pick
> and choose what to initialize before and what can wait.

subarch is part of boot_params, so not sure what I mean by an issue
in timing here. The question is if its set and then how early can
you possibly read the subarch from the boot_params.

> (And I am not sure it can be useful on PV neither, 

Well..

> at least the way it is used now. 

that is the issue.. If we get access to boot_params on early boot
then we can simply share the x86_64 entry point between PVH, PV,
and native x86_64 with what I'm proposing and some minor extensions.

> You will not reach the point in the (32-bit) code
> where it is tested. You will die way earlier (I think on
> startup_32()'s fourth instruction).)

Its a bit different requirement for the subarch for PV/PVH and
for HVMlite. Given what you have explained things are bit clearer
now and I see the issue.

You go:

hvmlite_start_xen() -->
	HVM stub
	startup_64() | (startup_32()

Note at the end of startup_32() though we have a neat asm
set of entries that depend on the subarch type. Perhaps we should
have a PV type early on startup_32() ? And/or I wonder if we can
work off of the EFI boot loader code. Half baked thoughts for now,
sorry have to go.

  Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Tue, Jan 26, 2016 at 4:04 PM, Luis R. Rodriguez <mcgrof@suse.com> wrote:
> You go:
>
> hvmlite_start_xen() -->
>         HVM stub
>         startup_64() | (startup_32()

Hrm, does HVMlite work well with load_ucode_bsp(), note the patches to
rebrand pv_enabled() to pv_legacy() or whatever, this PV type will not
be legacy or crap / old, so we'd need a way to catch it if we should
not use that code for this PV type. This begs the question, are you
also sure other callers in startup_32() or startup_64() might be OK as
well where previously guarded with pv_enabled() ?

 Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[David Vrabel]

On 27/01/16 14:42, Konrad Rzeszutek Wilk wrote:
> On Tue, Jan 26, 2016 at 08:54:56PM -0800, Luis R. Rodriguez wrote:
>> On Jan 26, 2016 6:16 PM, "Luis R. Rodriguez" <mcgrof@suse.com> wrote:
>>>
>>> On Tue, Jan 26, 2016 at 4:04 PM, Luis R. Rodriguez <mcgrof@suse.com>
>> wrote:
>>>> You go:
>>>>
>>>> hvmlite_start_xen() -->
>>>>         HVM stub
>>>>         startup_64() | (startup_32()
>>>
>>> Hrm, does HVMlite work well with load_ucode_bsp(), note the patches to
>>> rebrand pv_enabled() to pv_legacy() or whatever, this PV type will not
>>> be legacy or crap / old, so we'd need a way to catch it if we should
>>> not use that code for this PV type. This begs the question, are you
>>> also sure other callers in startup_32() or startup_64() might be OK as
>>> well where previously guarded with pv_enabled() ?
>>
>> Actually this call can't be used, and if early code used it prior to
>> setup_arch() it'd be a bug as its only properly set until later. Vetting
>> for correctness of all code call is still required though and perhaps we do
>> need something to catch now this PV type on early code such as this one if
>> we don't want it. From what I've gathered before on other bsp ucode we
>> don't want ucode loaded for PV guest types through these mechanisms.
> 
> It may help to not think of PVH/hvmlite as PV. It really is HVM with a lot
> of emulated devices removed.
> 
> How does early microcode work on EFI? Does the EFI stub code have an early
> microcode loading code ?

Surely the interesting comparison here is how is (early) microcode
loading disabled in KVM guests?  We should use the same mechanism for
HVMlite guests.

David

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/27/2016 09:50 AM, David Vrabel wrote:
> On 27/01/16 14:42, Konrad Rzeszutek Wilk wrote:
>> On Tue, Jan 26, 2016 at 08:54:56PM -0800, Luis R. Rodriguez wrote:
>>> On Jan 26, 2016 6:16 PM, "Luis R. Rodriguez" <mcgrof@suse.com> wrote:
>>>> On Tue, Jan 26, 2016 at 4:04 PM, Luis R. Rodriguez <mcgrof@suse.com>
>>> wrote:
>>>>> You go:
>>>>>
>>>>> hvmlite_start_xen() -->
>>>>>          HVM stub
>>>>>          startup_64() | (startup_32()
>>>> Hrm, does HVMlite work well with load_ucode_bsp(), note the patches to
>>>> rebrand pv_enabled() to pv_legacy() or whatever, this PV type will not
>>>> be legacy or crap / old, so we'd need a way to catch it if we should
>>>> not use that code for this PV type. This begs the question, are you
>>>> also sure other callers in startup_32() or startup_64() might be OK as
>>>> well where previously guarded with pv_enabled() ?
>>> Actually this call can't be used, and if early code used it prior to
>>> setup_arch() it'd be a bug as its only properly set until later. Vetting
>>> for correctness of all code call is still required though and perhaps we do
>>> need something to catch now this PV type on early code such as this one if
>>> we don't want it. From what I've gathered before on other bsp ucode we
>>> don't want ucode loaded for PV guest types through these mechanisms.
>> It may help to not think of PVH/hvmlite as PV. It really is HVM with a lot
>> of emulated devices removed.
>>
>> How does early microcode work on EFI? Does the EFI stub code have an early
>> microcode loading code ?
> Surely the interesting comparison here is how is (early) microcode
> loading disabled in KVM guests?  We should use the same mechanism for
> HVMlite guests.


Why would we ever want to have a guest load microcode during boot? I can 
see how a (privileged) guest may want to load microcode from a shell 
(via microcode driver).

-boris

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[David Vrabel]

On 27/01/16 15:06, Boris Ostrovsky wrote:
> On 01/27/2016 09:50 AM, David Vrabel wrote:
>> On 27/01/16 14:42, Konrad Rzeszutek Wilk wrote:
>>> On Tue, Jan 26, 2016 at 08:54:56PM -0800, Luis R. Rodriguez wrote:
>>>> On Jan 26, 2016 6:16 PM, "Luis R. Rodriguez" <mcgrof@suse.com> wrote:
>>>>> On Tue, Jan 26, 2016 at 4:04 PM, Luis R. Rodriguez <mcgrof@suse.com>
>>>> wrote:
>>>>>> You go:
>>>>>>
>>>>>> hvmlite_start_xen() -->
>>>>>>          HVM stub
>>>>>>          startup_64() | (startup_32()
>>>>> Hrm, does HVMlite work well with load_ucode_bsp(), note the patches to
>>>>> rebrand pv_enabled() to pv_legacy() or whatever, this PV type will not
>>>>> be legacy or crap / old, so we'd need a way to catch it if we should
>>>>> not use that code for this PV type. This begs the question, are you
>>>>> also sure other callers in startup_32() or startup_64() might be OK as
>>>>> well where previously guarded with pv_enabled() ?
>>>> Actually this call can't be used, and if early code used it prior to
>>>> setup_arch() it'd be a bug as its only properly set until later.
>>>> Vetting
>>>> for correctness of all code call is still required though and
>>>> perhaps we do
>>>> need something to catch now this PV type on early code such as this
>>>> one if
>>>> we don't want it. From what I've gathered before on other bsp ucode we
>>>> don't want ucode loaded for PV guest types through these mechanisms.
>>> It may help to not think of PVH/hvmlite as PV. It really is HVM with
>>> a lot
>>> of emulated devices removed.
>>>
>>> How does early microcode work on EFI? Does the EFI stub code have an
>>> early
>>> microcode loading code ?
>> Surely the interesting comparison here is how is (early) microcode
>> loading disabled in KVM guests?  We should use the same mechanism for
           ^^^^^^^^
>> HVMlite guests.
> 
> 
> Why would we ever want to have a guest load microcode during boot? I can
> see how a (privileged) guest may want to load microcode from a shell
> (via microcode driver).

I think you missed a word when you read my reply.

David

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/27/2016 10:09 AM, David Vrabel wrote:
> On 27/01/16 15:06, Boris Ostrovsky wrote:
>> On 01/27/2016 09:50 AM, David Vrabel wrote:
>>> On 27/01/16 14:42, Konrad Rzeszutek Wilk wrote:
>>>> On Tue, Jan 26, 2016 at 08:54:56PM -0800, Luis R. Rodriguez wrote:
>>>>> On Jan 26, 2016 6:16 PM, "Luis R. Rodriguez" <mcgrof@suse.com> wrote:
>>>>>> On Tue, Jan 26, 2016 at 4:04 PM, Luis R. Rodriguez <mcgrof@suse.com>
>>>>> wrote:
>>>>>>> You go:
>>>>>>>
>>>>>>> hvmlite_start_xen() -->
>>>>>>>           HVM stub
>>>>>>>           startup_64() | (startup_32()
>>>>>> Hrm, does HVMlite work well with load_ucode_bsp(), note the patches to
>>>>>> rebrand pv_enabled() to pv_legacy() or whatever, this PV type will not
>>>>>> be legacy or crap / old, so we'd need a way to catch it if we should
>>>>>> not use that code for this PV type. This begs the question, are you
>>>>>> also sure other callers in startup_32() or startup_64() might be OK as
>>>>>> well where previously guarded with pv_enabled() ?
>>>>> Actually this call can't be used, and if early code used it prior to
>>>>> setup_arch() it'd be a bug as its only properly set until later.
>>>>> Vetting
>>>>> for correctness of all code call is still required though and
>>>>> perhaps we do
>>>>> need something to catch now this PV type on early code such as this
>>>>> one if
>>>>> we don't want it. From what I've gathered before on other bsp ucode we
>>>>> don't want ucode loaded for PV guest types through these mechanisms.
>>>> It may help to not think of PVH/hvmlite as PV. It really is HVM with
>>>> a lot
>>>> of emulated devices removed.
>>>>
>>>> How does early microcode work on EFI? Does the EFI stub code have an
>>>> early
>>>> microcode loading code ?
>>> Surely the interesting comparison here is how is (early) microcode
>>> loading disabled in KVM guests?  We should use the same mechanism for
>             ^^^^^^^^
>>> HVMlite guests.
>>
>> Why would we ever want to have a guest load microcode during boot? I can
>> see how a (privileged) guest may want to load microcode from a shell
>> (via microcode driver).
> I think you missed a word when you read my reply.

Yes, I missed it ;-)

Why not continue relying on paravirt_enabled()? We are going to keep 
this in some form for HVMlite.

-boris

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Borislav Petkov]

On Wed, Jan 27, 2016 at 02:50:36PM +0000, David Vrabel wrote:
> Surely the interesting comparison here is how is (early) microcode
> loading disabled in KVM guests?

It isn't - kvm simply ignores the write to the microcode application
MSRs MSR_AMD64_PATCH_LOADER and MSR_IA32_UCODE_REV, respectively.

> We should use the same mechanism for HVMlite guests.

Good idea.

-- 
Regards/Gruss,
    Boris.

SUSE Linux GmbH, GF: Felix Imendörffer, Jane Smithard, Graham Norton, HRB 21284 (AG Nürnberg)
-- 

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/27/2016 10:29 AM, Konrad Rzeszutek Wilk wrote:
> On Wed, Jan 27, 2016 at 10:17:56AM -0500, Boris Ostrovsky wrote:
>> On 01/27/2016 10:09 AM, David Vrabel wrote:
>>> On 27/01/16 15:06, Boris Ostrovsky wrote:
>>>> On 01/27/2016 09:50 AM, David Vrabel wrote:
>>>>> On 27/01/16 14:42, Konrad Rzeszutek Wilk wrote:
>>>>>> On Tue, Jan 26, 2016 at 08:54:56PM -0800, Luis R. Rodriguez wrote:
>>>>>>> On Jan 26, 2016 6:16 PM, "Luis R. Rodriguez" <mcgrof@suse.com> wrote:
>>>>>>>> On Tue, Jan 26, 2016 at 4:04 PM, Luis R. Rodriguez <mcgrof@suse.com>
>>>>>>> wrote:
>>>>>>>>> You go:
>>>>>>>>>
>>>>>>>>> hvmlite_start_xen() -->
>>>>>>>>>           HVM stub
>>>>>>>>>           startup_64() | (startup_32()
>>>>>>>> Hrm, does HVMlite work well with load_ucode_bsp(), note the patches to
>>>>>>>> rebrand pv_enabled() to pv_legacy() or whatever, this PV type will not
>>>>>>>> be legacy or crap / old, so we'd need a way to catch it if we should
>>>>>>>> not use that code for this PV type. This begs the question, are you
>>>>>>>> also sure other callers in startup_32() or startup_64() might be OK as
>>>>>>>> well where previously guarded with pv_enabled() ?
>>>>>>> Actually this call can't be used, and if early code used it prior to
>>>>>>> setup_arch() it'd be a bug as its only properly set until later.
>>>>>>> Vetting
>>>>>>> for correctness of all code call is still required though and
>>>>>>> perhaps we do
>>>>>>> need something to catch now this PV type on early code such as this
>>>>>>> one if
>>>>>>> we don't want it. From what I've gathered before on other bsp ucode we
>>>>>>> don't want ucode loaded for PV guest types through these mechanisms.
>>>>>> It may help to not think of PVH/hvmlite as PV. It really is HVM with
>>>>>> a lot
>>>>>> of emulated devices removed.
>>>>>>
>>>>>> How does early microcode work on EFI? Does the EFI stub code have an
>>>>>> early
>>>>>> microcode loading code ?
>>>>> Surely the interesting comparison here is how is (early) microcode
>>>>> loading disabled in KVM guests?  We should use the same mechanism for
>>>             ^^^^^^^^
>>>>> HVMlite guests.
>>>> Why would we ever want to have a guest load microcode during boot? I can
>>>> see how a (privileged) guest may want to load microcode from a shell
>>>> (via microcode driver).
>>> I think you missed a word when you read my reply.
>> Yes, I missed it ;-)
>>
>> Why not continue relying on paravirt_enabled()? We are going to keep this in
>> some form for HVMlite.
> And this is where Luis comes in. He has posted an patchset which removes the
> paravirt_enabled with .. Here is the link https://lkml.org/lkml/2015/12/15/772

Yes, I saw that and this will be renamed as paravirt_legacy() (which I 
am not sure is really what it should be called.)

Another option is to have early microcode code query CPUID to see 
whether we are running on a hypervisor (this in fact is what we 
originally thought of doing before realizing that we have 
paravirt_enabled()).

But then how is HVMlite different from a regular HVM guest trying to 
load microcode?

(BTW, to answer David's question about what KVM is doing --- it is 
ignoring writes to microcode MSRs, see kvm_set_msr_common().)

-boris

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

Bleh moving forward please use mcgrof@kernel.org, that will be sent to
my employer and my personal address. More below.

On Wed, Jan 27, 2016 at 8:15 AM, Boris Ostrovsky
<boris.ostrovsky@oracle.com> wrote:
> On 01/27/2016 10:29 AM, Konrad Rzeszutek Wilk wrote:
>> On Wed, Jan 27, 2016 at 10:17:56AM -0500, Boris Ostrovsky wrote:
>>> On 01/27/2016 10:09 AM, David Vrabel wrote:
>>>> On 27/01/16 15:06, Boris Ostrovsky wrote:
>>> Why not continue relying on paravirt_enabled()? We are going to keep this
>>> in
>>> some form for HVMlite.
>>
>> And this is where Luis comes in. He has posted an patchset which removes
>> the
>> paravirt_enabled with .. Here is the link
>> https://lkml.org/lkml/2015/12/15/772
>
>
> Yes, I saw that and this will be renamed as paravirt_legacy() (which I am
> not sure is really what it should be called.)

Given Konrad's pointers about some folks pushing for BIOS to have a
"legacy free option" where all legacy crap (PS/2, PnP, serial port,
parallel port) are all disabled [0], I'm inclined to respin the rename
patch to use x86_legacy_free(). This can later then be extended
provided such BIOS check becomes available.

[0] http://lkml.kernel.org/r/20160120193241.GA4769@char.us.oracle.com

But -- as I also pointed out to hpa recently, there is an issue with
using things like cpu_has_hypervisor() (or in this case
paravirt_enabled()) for early code, given that it relies on
init_hypervisor_platform() having been called first which is called on
setup_arch() [1]. So paravirt_enabled() really can only be used prior
to setup_arch() correctly (specifically after,
init_hypervisor_platform()), and anything else would be a bug. I
learned the hard way while doing some linker table work and trying to
find a good use / solution to the dead code concerns I've been aiming
to address due to Xen's separate entry point and the nature of pvops
[2] [3]. hpa's response to this issue was that we cannot and should
not abuse the boot_params hardware_subarch for this purpose (in this
case I was suggesting perhaps KVM could use it if it in the future
needed it for a kvm check earlier than setup_arch()), but he noted
that:

  "If you have a genuine need for a "hypervisor type" then that is a
separate thing and should be
   treated separately from subarch.  However, you need to consider
that some hypervisors can
  emulate other hypervisors and you may have more than one hypervisor
API available."

This goes along with my suggestion here earlier where I mentioned that
subarch is already used nicely to pivot off some specific subarch
entry after startup_32(), if we want to avoid yet-another-entry for
HVMlite (the PVH rebrand done cleanly) and still use startup_32() for
it we could perhaps follow similar strategy as with subarch but
instead add a hypervisor type and use that for a stub call the
hypervisor type if needed early on in startup_32(). This could perhaps
in turn also be used as a generic hypervisor type / and more robust /
easier / not-so-convoluted check. For instance of what I think is a
convoluted check refer to snd_intel8x0_inside_vm() in
sound/pci/intel8x0.c with its use of kvm_para_available(), #ifdefery
over boot_cpu_has(X86_FEATURE_HYPERVISOR) (which is just
cpu_has_hypervisor()). If we had a hypervisor type easily accessible
it could both be used by early init code and perhaps driver code to
replace these convoluted checks.

If this seems a bit sensible the next question to ask if -- how we'd
*set* the hypervisor type, do we extend the x86 boot protocol and
enable a hypervisor type on the zero page? That would clearly only
make sense if:

1) Its reasonable to x86 maintainers (perhaps pending this
discussion's outcome? If its preemptively known to not reasonable an
alternative suggestion would be appreciated)
2) Xen is willing to actually set this (and perhaps a new
hypervisor_type_data for the private data structure), but not much
more would be needed, and incorporate this as part of the DmLite
protocol, or whatever its called as an option for some OSes
3) The kernel could get access to this value from the zero page really
early on, immediately following startup_32()

Worth mentioning here also is hpa's clarification on when subarch type
PC (0) should be used: [it should be used if the hardware is]
"enumerable using standard PC mechanisms (PCI, ACPI) and doesn't need
a special boot flow" -- does that fit HVMLite's description so far? If
so then The Xen subarch may need to be redefined as well to be clear
what it means. I don't think we need to be precise but at the very
least cover grounds to enable the definitions to meet its actual use
to not confuse users.

[1] http://lkml.kernel.org/r/CAB=NE6WoKsP8+KGnJEtigWYktCMjg6iherCOcq-jskxi4P2QqA@mail.gmail.com
[2] http://www.do-not-panic.com/2015/12/avoiding-dead-code-pvops-not-silver-bullet.html
[3] http://www.do-not-panic.com/2015/12/xen-and-x86-linux-zero-page.html
[4] http://lkml.kernel.org/r/56A17B01.9040708@zytor.com

> Another option is to have early microcode code query CPUID to see whether we
> are running on a hypervisor (this in fact is what we originally thought of
> doing before realizing that we have paravirt_enabled()).

Please keep in mind we want to consider use and setting of something
generic, not just a solution for Xen, and if we want to use this to
help avoid a new entry point then strive for something we actually
could get access to very early on the first x86 entry point, both for
32-bit and 64-bit. If its accessible for both 32-bit and 64-bit well
hell, we could unify not only the entry points for HVMLite thing, but
I'm pretty sure also unify the entry points for PV and x86 as well (if
we wanted that) !

  Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Luis R. Rodriguez]

On Wed, Jan 27, 2016 at 10:48 AM, Luis R. Rodriguez <mcgrof@kernel.org> wrote:
>
> Worth mentioning here also is hpa's clarification on when subarch type
> PC (0) should be used: [it should be used if the hardware is]
> "enumerable using standard PC mechanisms (PCI, ACPI) and doesn't need
> a special boot flow" -- does that fit HVMLite's description so far? If
> so then The Xen subarch may need to be redefined as well to be clear
> what it means. I don't think we need to be precise but at the very
> least cover grounds to enable the definitions to meet its actual use
> to not confuse users.

Another thing to consider for HVMlite is that if the 0 subarch (PC) is
used in light of my linker table work and x86's use of it with the
subarch and supported subarch bitmask, is that it would also mean
HVMLite would run all routines currently pegged as needing PC type
(the current KVM and bare metal path) and it would mean not running
anything only pegged with Xen subarch type (but note that today Xen
doesn't even set the subarch type). If there is nothing in common
between PV and HVMlite (no x86 init calls to share), and if HVMLite
*can* call *alllllllll* PC init calls, then by all means this is fine,
and if we just need to distinguish stuff between PC types that's fine,
it may still be possible to further extend hypervisor_type to the x86
init calls I'm adding as another supported_hyper_types to ensure even
though a subarch is being used, that we also check the supported
hypervisor type as well.

 Luis

Re: [Xen-devel] [PATCH v1 04/12] xen/hvmlite: Bootstrap HVMlite guest

[Boris Ostrovsky]

On 01/27/2016 02:00 PM, Luis R. Rodriguez wrote:
> On Wed, Jan 27, 2016 at 10:48 AM, Luis R. Rodriguez <mcgrof@kernel.org> wrote:
>> Worth mentioning here also is hpa's clarification on when subarch type
>> PC (0) should be used: [it should be used if the hardware is]
>> "enumerable using standard PC mechanisms (PCI, ACPI) and doesn't need
>> a special boot flow" -- does that fit HVMLite's description so far? If
>> so then The Xen subarch may need to be redefined as well to be clear
>> what it means. I don't think we need to be precise but at the very
>> least cover grounds to enable the definitions to meet its actual use
>> to not confuse users.
> Another thing to consider for HVMlite is that if the 0 subarch (PC) is
> used in light of my linker table work and x86's use of it with the
> subarch and supported subarch bitmask, is that it would also mean
> HVMLite would run all routines currently pegged as needing PC type
> (the current KVM and bare metal path) and it would mean not running
> anything only pegged with Xen subarch type (but note that today Xen
> doesn't even set the subarch type). If there is nothing in common
> between PV and HVMlite (no x86 init calls to share), and if HVMLite
> *can* call *alllllllll* PC init calls, then by all means this is fine,

Yes, that's the idea. HVMlite jumps to startup_32|64 from the stub and 
runs from there with subarch 0.

-boris

> and if we just need to distinguish stuff between PC types that's fine,
> it may still be possible to further extend hypervisor_type to the x86
> init calls I'm adding as another supported_hyper_types to ensure even
> though a subarch is being used, that we also check the supported
> hypervisor type as well.
>
>   Luis