[RFC] ACPI S3 and Xen (suprisingly small\!).

[RFC] ACPI S3 and Xen (suprisingly small\!).

[Konrad Rzeszutek Wilk]

>From Konrad Rzeszutek Wilk <konrad.wilk@oracle.com> # This line is ignored.
From: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Subject: [RFC] ACPI S3 and Xen (surprisingly small\!).
In-Reply-To: 

Hey,

Way back at the LinuxPlumbers 2012 I chatted with Len about the issue
with Xen and Linux not working very well when suspending. We did a bit
of brainstorming and armed with these ideas I started looking at this.

The end result is this is a nice set of patches where there is only
_one_ change in the x86 code (and it is just more of dealing with
error case) - and the rest are all done in Xen side.

The change is to check whether there is a TSS GDT descriptor before
trying to set it. On 64-bit Xen there is no TSS and we ignore any
calls to set such field.

The bulk of the Xen code is filling out some of the pvops calls
(some of them are already in v3.7-rc1), and one to "prep" the cr3
value so that the hypervisor is fine with it.

Note: These are the other patches that went in 3.7-rc1:
xen/bootup: allow {read|write}_cr8 pvops call [https://lkml.org/lkml/2012/10/10/339]
xen/bootup: allow read_tscp call for Xen PV guests. [https://lkml.org/lkml/2012/10/10/340]

Anyhow, with these patches ACPI S3 suspend/resume of Linux works great!

 arch/x86/power/cpu.c     |    7 +++++--
 arch/x86/xen/enlighten.c |   24 ++++++++++++++++++++++--
 drivers/xen/acpi.c       |   25 +++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 4 deletions(-)

[PATCH 1/4] x86/wakeup/sleep: Check whether the TSS GDT descriptor is empty before using it.

[Konrad Rzeszutek Wilk]

We check the TSS descriptor before we try to dereference it.
Also fix up the value to use the #defines.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/power/cpu.c |    7 +++++--
 1 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
index 218cdb1..c17370e 100644
--- a/arch/x86/power/cpu.c
+++ b/arch/x86/power/cpu.c
@@ -133,7 +133,9 @@ static void fix_processor_context(void)
 {
 	int cpu = smp_processor_id();
 	struct tss_struct *t = &per_cpu(init_tss, cpu);
-
+#ifdef CONFIG_X86_64
+	struct desc_struct *desc = get_cpu_gdt_table(cpu);
+#endif
 	set_tss_desc(cpu, t);	/*
 				 * This just modifies memory; should not be
 				 * necessary. But... This is necessary, because
@@ -142,7 +144,8 @@ static void fix_processor_context(void)
 				 */
 
 #ifdef CONFIG_X86_64
-	get_cpu_gdt_table(cpu)[GDT_ENTRY_TSS].type = 9;
+	if (!desc_empty(&desc[GDT_ENTRY_TSS]))
+		desc[GDT_ENTRY_TSS].type = DESC_TSS;
 
 	syscall_init();				/* This sets MSR_*STAR and related */
 #endif
-- 
1.7.7.6

[PATCH 2/4] xen/lowlevel: Implement pvop call for load_idt (sidt).

[Konrad Rzeszutek Wilk]

In the past it used to point to 'sidt' (native_store_idt) operation
which is a non-privileged operation. This resulted in the
'struct desc_ptr' value containing the address of Xen's IDT table,
instead of the IDT table that Linux thinks its using. The end result
is that doing:

  store_idt(&desc);
  load_idt(&desc);

would blow up b/c xen_load_idt would try to parse the IDT contents
(desc) and de-reference a virtual address that is outside Linux's
__va (it is in Xen's virtual address).

With this patch we are providing the last written IDT address.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/enlighten.c |    8 +++++++-
 1 files changed, 7 insertions(+), 1 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index e3497f2..f29d6d6 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -777,7 +777,13 @@ static void xen_load_idt(const struct desc_ptr *desc)
 
 	spin_unlock(&lock);
 }
+static void xen_store_idt(struct desc_ptr *dtr)
+{
+	const struct desc_ptr *desc = &__get_cpu_var(idt_desc);
 
+	dtr->address = desc->address;
+	dtr->size = desc->size;
+}
 /* Write a GDT descriptor entry.  Ignore LDT descriptors, since
    they're handled differently. */
 static void xen_write_gdt_entry(struct desc_struct *dt, int entry,
@@ -1200,7 +1206,7 @@ static const struct pv_cpu_ops xen_cpu_ops __initconst = {
 	.free_ldt = xen_free_ldt,
 
 	.store_gdt = native_store_gdt,
-	.store_idt = native_store_idt,
+	.store_idt = xen_store_idt,
 	.store_tr = xen_store_tr,
 
 	.write_ldt_entry = xen_write_ldt_entry,
-- 
1.7.7.6

[PATCH 3/4] xen/lowlevel: Implement pvop call for store_gdt (gidt)

[Konrad Rzeszutek Wilk]

In the past it used to point to 'sgdt' (native_store_gdt)
operation which is a non-privileged operation. This resulted
in the value of 'struct desc_ptr' pointing to an bogus address
0xffff820000000000, instead of the GDT table that Linux thinks
it is using. The end result is that doing:

      store_gdt(&desc);
      load_gdt(&desc);

would blow up b/c xen_load_gdt would try to parse the GDT contents
(desc) and de-reference an bogus virtual address.

With this patch we are providing the last written address and size
of the GDT.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/xen/enlighten.c |   16 +++++++++++++++-
 1 files changed, 15 insertions(+), 1 deletions(-)

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index f29d6d6..4a65138 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -471,6 +471,8 @@ static void xen_set_ldt(const void *addr, unsigned entries)
 	xen_mc_issue(PARAVIRT_LAZY_CPU);
 }
 
+static DEFINE_PER_CPU(struct desc_ptr, gdt_desc);
+
 static void xen_load_gdt(const struct desc_ptr *dtr)
 {
 	unsigned long va = dtr->address;
@@ -478,6 +480,7 @@ static void xen_load_gdt(const struct desc_ptr *dtr)
 	unsigned pages = (size + PAGE_SIZE - 1) / PAGE_SIZE;
 	unsigned long frames[pages];
 	int f;
+	struct desc_ptr *shadow;
 
 	/*
 	 * A GDT can be up to 64k in size, which corresponds to 8192
@@ -515,8 +518,19 @@ static void xen_load_gdt(const struct desc_ptr *dtr)
 
 	if (HYPERVISOR_set_gdt(frames, size / sizeof(struct desc_struct)))
 		BUG();
+
+	shadow = &__get_cpu_var(gdt_desc);
+	shadow->address = dtr->address;
+	shadow->size = size;
 }
 
+static void xen_store_gdt(struct desc_ptr *dtr)
+{
+	const struct desc_ptr *desc = &__get_cpu_var(gdt_desc);
+
+	dtr->address = desc->address;
+	dtr->size = desc->size;
+}
 /*
  * load_gdt for early boot, when the gdt is only mapped once
  */
@@ -1205,7 +1219,7 @@ static const struct pv_cpu_ops xen_cpu_ops __initconst = {
 	.alloc_ldt = xen_alloc_ldt,
 	.free_ldt = xen_free_ldt,
 
-	.store_gdt = native_store_gdt,
+	.store_gdt = xen_store_gdt,
 	.store_idt = xen_store_idt,
 	.store_tr = xen_store_tr,
 
-- 
1.7.7.6

[PATCH 4/4] xen/acpi: Prep saved_context cr3 values.

[Konrad Rzeszutek Wilk]

When save_processor_state is executed it executes a bunch of
pvops calls to save the CPU state values. When it comes back
from Xen's S3 (so acpi_enter_sleep_state, which ends up calling
xen_acpi_notify_hypervisor_state), it ends up back in the
assembler code in wakeup_[32|64].S. It skips the wakeup
calls (so wakeup_pmode_return and wakeup_long64) as that has
been done in the hypervisor. Instead it continues on in
the resume_point (64-bit) or ret_point (32-bit). Most of the
calls in there are safe to be executed except when it comes to
reloading of cr3 (which it only does on 64-bit kernels). Since
they are native assembler calls and Xen expects a PV kernel to
prep those to use machine address for cr3 that is what
we are going to do. Note: that it is not Machine Frame Numbers
(those are used in the MMUEXT_NEW_BASEPTR hypercall for cr3
installation) but the machine physical address.

When the assembler code executes this mov %ebx, cr3 it ends
end up trapped in the hypervisor (traps.c) which properly now
sets the cr3 value.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 drivers/xen/acpi.c |   25 +++++++++++++++++++++++++
 1 files changed, 25 insertions(+), 0 deletions(-)

diff --git a/drivers/xen/acpi.c b/drivers/xen/acpi.c
index 119d42a..25e612c 100644
--- a/drivers/xen/acpi.c
+++ b/drivers/xen/acpi.c
@@ -35,6 +35,13 @@
 #include <asm/xen/hypercall.h>
 #include <asm/xen/hypervisor.h>
 
+#include <xen/page.h>
+
+#ifdef CONFIG_X86_64
+#include <asm/suspend_64.h>
+extern struct saved_context saved_context;
+#endif
+
 int xen_acpi_notify_hypervisor_state(u8 sleep_state,
 				     u32 pm1a_cnt, u32 pm1b_cnt)
 {
@@ -56,7 +63,25 @@ int xen_acpi_notify_hypervisor_state(u8 sleep_state,
 		     pm1a_cnt, pm1b_cnt);
 		return -1;
 	}
+#ifdef CONFIG_X86_64
+	/* We do not need to anything for 32-bit kernels as the
+	 * low-level calls (write to cr3) is done in the wakup code
+	 * which we never execute when running under Xen.
+	 */
+	{
+		unsigned long mfn;
 
+		/* resume_point in wakeup_64.s barrels through and does:
+		 * movq    saved_context_cr3(%rax), %rbx
+		 * movq    %rbx, %cr3
+		 * so lets prep the values so they are OK with the
+		 * hypervisor. */
+		mfn = pfn_to_mfn(PFN_DOWN(saved_context.cr3));
+		/* and back to a physical address */
+		mfn = PFN_PHYS(mfn);
+		saved_context.cr3 = mfn;
+	}
+#endif
 	HYPERVISOR_dom0_op(&op);
 	return 1;
 }
-- 
1.7.7.6

Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[H. Peter Anvin]

On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
>
> Note: These are the other patches that went in 3.7-rc1:
> xen/bootup: allow {read|write}_cr8 pvops call [https://lkml.org/lkml/2012/10/10/339]
> xen/bootup: allow read_tscp call for Xen PV guests. [https://lkml.org/lkml/2012/10/10/340]
>

So WTF do we have a read_tscp PV call?  Again, if there isn't a user we 
should just axe it...

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 09:03:12AM -0700, H. Peter Anvin wrote:
> On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> >
> >Note: These are the other patches that went in 3.7-rc1:
> >xen/bootup: allow {read|write}_cr8 pvops call [https://lkml.org/lkml/2012/10/10/339]
> >xen/bootup: allow read_tscp call for Xen PV guests. [https://lkml.org/lkml/2012/10/10/340]
> >
> 
> So WTF do we have a read_tscp PV call?  Again, if there isn't a user
> we should just axe it...

Let me spin off a patch to see if that can be done.

Re: Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 12:10:36PM -0400, Konrad Rzeszutek Wilk wrote:
> On Wed, Oct 17, 2012 at 09:03:12AM -0700, H. Peter Anvin wrote:
> > On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> > >
> > >Note: These are the other patches that went in 3.7-rc1:
> > >xen/bootup: allow {read|write}_cr8 pvops call [https://lkml.org/lkml/2012/10/10/339]
> > >xen/bootup: allow read_tscp call for Xen PV guests. [https://lkml.org/lkml/2012/10/10/340]
> > >
> > 
> > So WTF do we have a read_tscp PV call?  Again, if there isn't a user
> > we should just axe it...
> 
> Let me spin off a patch to see if that can be done.

It can be done faily easy. That said  the only user that could
_potentially_ use this (if the read_tscp had some extra logic to
do 'readtsc' operations) would be the __vdso_getcpu.

Meaning in __vdso_getcpu we would modify it from native_read_tscp
to paravirt_read_tscp:

notrace long
__vdso_getcpu(unsigned *cpu, unsigned *node, struct getcpu_cache *unused)
{
	unsigned int p;

	if (VVAR(vgetcpu_mode) == VGETCPU_RDTSCP) {
		/* Load per CPU data from RDTSCP */
===>		native_read_tscp(&p);
	} else {
		/* Load per CPU data from GDT */
		asm("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG));
	}
	if (cpu)
		*cpu = p & 0xfff;
	if (node)
		*node = p >> 12;
	return 0;
}

but that line was added for a purpose, which was
in git commit 8f12dea6135d0a55b151dcb4c6bbe211f5f8d35d
Author: Glauber de Oliveira Costa <gcosta@redhat.com>
Date:   Wed Jan 30 13:31:06 2008 +0100

    x86: introduce native_read_tscp
    
    Targetting paravirt, this patch introduces native_read_tscp, in
    place of rdtscp() macro. When in a paravirt guest, this will
    involve a function call, and thus, cannot be done in the vdso area.
    These users then have to call the native version directly
    
    Signed-off-by: Glauber de Oliveira Costa <gcosta@redhat.com>
    Signed-off-by: Ingo Molnar <mingo@elte.hu>
    Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

which implies that it since it is a vDSO area it cannot do paravirt
calls anyhow.

In other words, I think I'm OK with axing it. Going to spin a patch
and ask for some other folks to review/double check.

Re: Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[H. Peter Anvin]

On 10/17/2012 09:10 AM, Konrad Rzeszutek Wilk wrote:
> On Wed, Oct 17, 2012 at 09:03:12AM -0700, H. Peter Anvin wrote:
>> On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
>>>
>>> Note: These are the other patches that went in 3.7-rc1:
>>> xen/bootup: allow {read|write}_cr8 pvops call [https://lkml.org/lkml/2012/10/10/339]
>>> xen/bootup: allow read_tscp call for Xen PV guests. [https://lkml.org/lkml/2012/10/10/340]
>>>
>>
>> So WTF do we have a read_tscp PV call?  Again, if there isn't a user
>> we should just axe it...
>
> Let me spin off a patch to see if that can be done.
>

Could you do an audit for other pvops calls that have no users?  If the 
*only* user is lguest, we should talk about it, too...

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[H. Peter Anvin]

On 10/17/2012 09:39 AM, Konrad Rzeszutek Wilk wrote:
>
> which implies that it since it is a vDSO area it cannot do paravirt
> calls anyhow.
>

Obviously.  The vdso is *user space*.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 09:50:11AM -0700, H. Peter Anvin wrote:
> On 10/17/2012 09:10 AM, Konrad Rzeszutek Wilk wrote:
> >On Wed, Oct 17, 2012 at 09:03:12AM -0700, H. Peter Anvin wrote:
> >>On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> >>>
> >>>Note: These are the other patches that went in 3.7-rc1:
> >>>xen/bootup: allow {read|write}_cr8 pvops call [https://lkml.org/lkml/2012/10/10/339]
> >>>xen/bootup: allow read_tscp call for Xen PV guests. [https://lkml.org/lkml/2012/10/10/340]
> >>>
> >>
> >>So WTF do we have a read_tscp PV call?  Again, if there isn't a user
> >>we should just axe it...
> >
> >Let me spin off a patch to see if that can be done.
> >
> 
> Could you do an audit for other pvops calls that have no users?  If
> the *only* user is lguest, we should talk about it, too...

I can do that - but I don't want to be hasty here. There is a bit of
danger here - for example the read_pmc (or read_tsc) is not in use right
now. But it might be when one starts looking at making perf be able to
analyze the hypervisor (hand-waving the implementation details). So while
removing read_pmc now sounds good, it might be needed in the future.

Or maybe not :-)

Let me do a candidate list and get some conversation going.

Re: Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[H. Peter Anvin]

On 10/17/2012 09:54 AM, Konrad Rzeszutek Wilk wrote:
>>
>> Could you do an audit for other pvops calls that have no users?  If
>> the *only* user is lguest, we should talk about it, too...
>
> I can do that - but I don't want to be hasty here. There is a bit of
> danger here - for example the read_pmc (or read_tsc) is not in use right
> now. But it might be when one starts looking at making perf be able to
> analyze the hypervisor (hand-waving the implementation details). So while
> removing read_pmc now sounds good, it might be needed in the future.
>

We do not keep a pvop around just because it "might be needed in the 
future".  That's just crazy.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 01:46:09PM -0400, Ben Guthro wrote:
> On Wed, Oct 17, 2012 at 9:49 AM, Konrad Rzeszutek Wilk
> <konrad.wilk@oracle.com> wrote:
> [...]
> 
> > The end result is this is a nice set of patches where there is only
> > _one_ change in the x86 code (and it is just more of dealing with
> > error case) - and the rest are all done in Xen side.
> 
> I'm sorry to report that this series doesn't seem to work in my setup
> against xen-unstable.
> 
> To verify that it was, in fact this patch series, and not another Xen
> regression - I swapped out the kernel with this patch series, with an
> identical one, replacing only this series with your acpi-s3.v9 branch
> - and everything worked fine.

Thanks for testing it!

I had tested it with Xen 4.1.3, which could be doing something different.
Will see what is up.

Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Ben Guthro]

On Wed, Oct 17, 2012 at 9:49 AM, Konrad Rzeszutek Wilk
<konrad.wilk@oracle.com> wrote:
[...]

> The end result is this is a nice set of patches where there is only
> _one_ change in the x86 code (and it is just more of dealing with
> error case) - and the rest are all done in Xen side.

I'm sorry to report that this series doesn't seem to work in my setup
against xen-unstable.

To verify that it was, in fact this patch series, and not another Xen
regression - I swapped out the kernel with this patch series, with an
identical one, replacing only this series with your acpi-s3.v9 branch
- and everything worked fine.

serial capture from bad run pasted below.



(XEN) Xen version 4.3-unstable (bguthro@) (gcc (Ubuntu/Linaro
4.6.3-1ubuntu5) 4.6.3) Wed Oct 17 08:53:46 EDT 2012
(XEN) Latest ChangeSet: unavailable
(XEN) Bootloader: GRUB 1.99-21ubuntu3+orc1
(XEN) Command line: com1=115200,8n1,pci console=com1
dom0_mem=auto:620M^30M cpufreq=xen cpuidle e820-verbose=1
dom0_vcpus_pin
(XEN) Video information:
(XEN)  VGA is text mode 80x25, font 8x16
(XEN)  VBE/DDC methods: V2; EDID transfer time: 1 seconds
(XEN) Disc information:
(XEN)  Found 1 MBR signatures
(XEN)  Found 1 EDD information structures
(XEN) Initial Xen-e820 RAM map:
(XEN)  0000000000000000 - 000000000009ac00 (usable)
(XEN)  000000000009ac00 - 00000000000a0000 (reserved)
(XEN)  00000000000e0000 - 0000000000100000 (reserved)
(XEN)  0000000000100000 - 0000000020000000 (usable)
(XEN)  0000000020000000 - 0000000020200000 (reserved)
(XEN)  0000000020200000 - 0000000040004000 (usable)
(XEN)  0000000040004000 - 0000000040005000 (reserved)
(XEN)  0000000040005000 - 00000000a8584000 (usable)
(XEN)  00000000a8584000 - 00000000a85c7000 (reserved)
(XEN)  00000000a85c7000 - 00000000aa58a000 (usable)
(XEN)  00000000aa58a000 - 00000000aad6a000 (reserved)
(XEN)  00000000aad6a000 - 00000000aafea000 (ACPI NVS)
(XEN)  00000000aafea000 - 00000000aafff000 (ACPI data)
(XEN)  00000000aafff000 - 00000000ab000000 (usable)
(XEN)  0000000100000000 - 000000014e600000 (usable)
(XEN)  00000000ab000000 - 00000000afa00000 (reserved)
(XEN)  00000000f8000000 - 00000000fc000000 (reserved)
(XEN)  00000000fec00000 - 00000000fec01000 (reserved)
(XEN)  00000000fed10000 - 00000000fed14000 (reserved)
(XEN)  00000000fed18000 - 00000000fed1a000 (reserved)
(XEN)  00000000fed1c000 - 00000000fed20000 (reserved)
(XEN)  00000000fee00000 - 00000000fee01000 (reserved)
(XEN)  00000000ff900000 - 00000000ffc00000 (reserved)
(XEN)  00000000ffd00000 - 0000000100000000 (reserved)
(XEN) Checking MTRR ranges...
(XEN)  MTRR cap: d0a type: c00
(XEN)  MTRR[0]: base 6 mask f80000800
(XEN)  MTRR[1]: base 80000006 mask fe0000800
(XEN)  MTRR[2]: base a0000006 mask ff8000800
(XEN)  MTRR[3]: base a8000006 mask ffe000800
(XEN)  MTRR[4]: base aa000006 mask fff000800
(XEN)  MTRR[5]: base 100000006 mask fc0000800
(XEN)  MTRR[6]: base 140000006 mask ff0000800
(XEN)  MTRR[7]: base 14f000000 mask fff000800
(XEN)  MTRR[8]: base 14e800000 mask fff800800
(XEN)  MTRR[9]: base 14e600000 mask fffe00800
(XEN) Xen-e820 RAM map:
(XEN)  0000000000000000 - 000000000009ac00 (usable)
(XEN)  000000000009ac00 - 00000000000a0000 (reserved)
(XEN)  00000000000e0000 - 0000000000100000 (reserved)
(XEN)  0000000000100000 - 0000000020000000 (usable)
(XEN)  0000000020000000 - 0000000020200000 (reserved)
(XEN)  0000000020200000 - 0000000040004000 (usable)
(XEN)  0000000040004000 - 0000000040005000 (reserved)
(XEN)  0000000040005000 - 00000000a8584000 (usable)
(XEN)  00000000a8584000 - 00000000a85c7000 (reserved)
(XEN)  00000000a85c7000 - 00000000aa58a000 (usable)
(XEN)  00000000aa58a000 - 00000000aad6a000 (reserved)
(XEN)  00000000aad6a000 - 00000000aafea000 (ACPI NVS)
(XEN)  00000000aafea000 - 00000000aafff000 (ACPI data)
(XEN)  00000000aafff000 - 00000000ab000000 (usable)
(XEN)  00000000ab000000 - 00000000afa00000 (reserved)
(XEN)  00000000f8000000 - 00000000fc000000 (reserved)
(XEN)  00000000fec00000 - 00000000fec01000 (reserved)
(XEN)  00000000fed10000 - 00000000fed14000 (reserved)
(XEN)  00000000fed18000 - 00000000fed1a000 (reserved)
(XEN)  00000000fed1c000 - 00000000fed20000 (reserved)
(XEN)  00000000fee00000 - 00000000fee01000 (reserved)
(XEN)  00000000ff900000 - 00000000ffc00000 (reserved)
(XEN)  00000000ffd00000 - 0000000100000000 (reserved)
(XEN)  0000000100000000 - 000000014e600000 (usable)
(XEN) ACPI: RSDP 000F0410, 0024 (r2  INTEL)
(XEN) ACPI: XSDT AAFFDE18, 0074 (r1  INTEL  IVB-CPT  6222004 MSFT    10013)
(XEN) ACPI: FACP AAFE3D98, 00F4 (r4  INTEL  IVB-CPT  6222004 MSFT    10013)
(XEN) ACPI: DSDT AAFBC018, 1082F (r2  INTEL  IVB-CPT        0 INTL 20110623)
(XEN) ACPI: FACS AAFE9E40, 0040
(XEN) ACPI: APIC AAFFCF18, 00CC (r2  INTEL  IVB-CPT  6222004 MSFT    10013)
(XEN) ACPI: HPET AAFE8F18, 0038 (r1 A M I   PCHHPET  6222004 AMI.        3)
(XEN) ACPI: SSDT AAFE5018, 10A8 (r1 TrmRef PtidDevc     1000 INTL 20110623)
(XEN) ACPI: SSDT AAFE4A18, 0461 (r1    AMI PerfTune     1000 INTL 20110623)
(XEN) ACPI: MCFG AAFE8E98, 003C (r1 INTEL  SNDYBRDG  6222004 MSFT       97)
(XEN) ACPI: SSDT AAFD1818, 079A (r1  PmRef  Cpu0Ist     3000 INTL 20110623)
(XEN) ACPI: SSDT AAFD0018, 0A92 (r1  PmRef    CpuPm     3000 INTL 20110623)
(XEN) ACPI: DMAR AAFE4F18, 00B8 (r1 INTEL      SNB         1 INTL        1)
(XEN) ACPI: FPDT AAFF4018, 0064 (r1  INTEL  IVB-CPT    10000 INTL 20111107)
(XEN) System RAM: 3976MB (4072324kB)
(XEN) No NUMA configuration found
(XEN) Faking a node at 0000000000000000-000000014e600000
(XEN) Domain heap initialised
(XEN) found SMP MP-table at 000fcaa0
(XEN) DMI 2.6 present.
(XEN) Using APIC driver default
(XEN) ACPI: PM-Timer IO Port: 0x408
(XEN) ACPI: ACPI SLEEP INFO: pm1x_cnt[404,0], pm1x_evt[400,0]
(XEN) ACPI: 32/64X FACS address mismatch in FADT -
aafe9e40/0000000000000000, using 32
(XEN) ACPI:                  wakeup_vec[aafe9e4c], vec_size[20]
(XEN) ACPI: Local APIC address 0xfee00000
(XEN) ACPI: LAPIC (acpi_id[0x01] lapic_id[0x00] enabled)
(XEN) Processor #0 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
(XEN) Processor #2 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x03] lapic_id[0x04] enabled)
(XEN) Processor #4 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x04] lapic_id[0x06] enabled)
(XEN) Processor #6 7:10 APIC version 21
(XEN) ACPI: LAPIC (acpi_id[0x05] lapic_id[0x04] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x06] lapic_id[0x05] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x07] lapic_id[0x06] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x08] lapic_id[0x07] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x09] lapic_id[0x08] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x09] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x0a] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x0c] lapic_id[0x0b] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x0d] lapic_id[0x0c] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x0e] lapic_id[0x0d] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x0f] lapic_id[0x0e] disabled)
(XEN) ACPI: LAPIC (acpi_id[0x10] lapic_id[0x0f] disabled)
(XEN) ACPI: IOAPIC (id[0x02] address[0xfec00000] gsi_base[0])
(XEN) IOAPIC[0]: apic_id 2, version 32, address 0xfec00000, GSI 0-23
(XEN) ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
(XEN) ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
(XEN) ACPI: IRQ0 used by override.
(XEN) ACPI: IRQ2 used by override.
(XEN) ACPI: IRQ9 used by override.
(XEN) Enabling APIC mode:  Flat.  Using 1 I/O APICs
(XEN) ACPI: HPET id: 0x8086a701 base: 0xfed00000
(XEN) Table is not found!
(XEN) Using ACPI (MADT) for SMP configuration information
(XEN) SMP: Allowing 16 CPUs (12 hotplug CPUs)
(XEN) IRQ limits: 24 GSI, 760 MSI/MSI-X
(XEN) Switched to APIC driver x2apic_cluster.
(XEN) Using scheduler: SMP Credit Scheduler (credit)
(XEN) Detected 2195.065 MHz processor.
(XEN) Initing memory sharing.
(XEN) mce_intel.c:721: MCA Capability: BCAST 1 SER 0 CMCI 1 firstbank
0 extended MCE MSR 0
(XEN) Intel machine check reporting enabled
(XEN) PCI: MCFG configuration 0: base f8000000 segment 0000 buses 00 - 3f
(XEN) PCI: MCFG area at f8000000 reserved in E820
(XEN) PCI: Using MCFG for segment 0000 bus 00-3f
(XEN) Intel VT-d supported page sizes: 4kB.
(XEN) Intel VT-d supported page sizes: 4kB.
(XEN) Intel VT-d Snoop Control not enabled.
(XEN) Intel VT-d Dom0 DMA Passthrough not enabled.
(XEN) Intel VT-d Queued Invalidation enabled.
(XEN) Intel VT-d Interrupt Remapping enabled.
(XEN) Intel VT-d Shared EPT tables not enabled.
(XEN) I/O virtualisation enabled
(XEN)  - Dom0 mode: Relaxed
(XEN) Enabled directed EOI with ioapic_ack_old on!
(XEN) ENABLING IO-APIC IRQs
(XEN)  -> Using old ACK method
(XEN) ..TIMER: vector=0xF0 apic1=0 pin1=2 apic2=-1 pin2=-1
(XEN) TSC deadline timer enabled
(XEN) Platform timer is 14.318MHz HPET
(XEN) Allocated console ring of 32 KiB.
(XEN) mwait-idle: MWAIT substates: 0x1120
(XEN) mwait-idle: v0.4 model 0x3a
(XEN) mwait-idle: lapic_timer_reliable_states 0xffffffff
(XEN) VMX: Supported advanced features:
(XEN)  - APIC MMIO access virtualisation
(XEN)  - APIC TPR shadow
(XEN)  - Extended Page Tables (EPT)
(XEN)  - Virtual-Processor Identifiers (VPID)
(XEN)  - Virtual NMI
(XEN)  - MSR direct-access bitmap
(XEN)  - Unrestricted Guest
(XEN) HVM: ASIDs enabled.
(XEN) HVM: VMX enabled
(XEN) HVM: Hardware Assisted Paging (HAP) detected
(XEN) HVM: HAP page sizes: 4kB, 2MB
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) Brought up 4 CPUs
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) ACPI sleep modes: S3
(XEN) mcheck_poll: Machine check polling timer started.
(XEN) *** LOADING DOMAIN 0 ***
(XEN) elf_parse_binary: phdr: paddr=0x1000000 memsz=0x8a2000
(XEN) elf_parse_binary: phdr: paddr=0x1a00000 memsz=0xb30f0
(XEN) elf_parse_binary: phdr: paddr=0x1ab4000 memsz=0x14600
(XEN) elf_parse_binary: phdr: paddr=0x1ac9000 memsz=0x62b000
(XEN) elf_parse_binary: memory: 0x1000000 -> 0x20f4000
(XEN) elf_xen_parse_note: GUEST_OS = "linux"
(XEN) elf_xen_parse_note: GUEST_VERSION = "2.6"
(XEN) elf_xen_parse_note: XEN_VERSION = "xen-3.0"
(XEN) elf_xen_parse_note: VIRT_BASE = 0xffffffff80000000
(XEN) elf_xen_parse_note: ENTRY = 0xffffffff81ac9210
(XEN) elf_xen_parse_note: HYPERCALL_PAGE = 0xffffffff81001000
(XEN) elf_xen_parse_note: FEATURES = "!writable_page_tables|pae_pgdir_above_4gb"
(XEN) elf_xen_parse_note: PAE_MODE = "yes"
(XEN) elf_xen_parse_note: LOADER = "generic"
(XEN) elf_xen_parse_note: unknown xen elf note (0xd)
(XEN) elf_xen_parse_note: SUSPEND_CANCEL = 0x1
(XEN) elf_xen_parse_note: HV_START_LOW = 0xffff800000000000
(XEN) elf_xen_parse_note: PADDR_OFFSET = 0x0
(XEN) elf_xen_addr_calc_check: addresses:
(XEN)     virt_base        = 0xffffffff80000000
(XEN)     elf_paddr_offset = 0x0
(XEN)     virt_offset      = 0xffffffff80000000
(XEN)     virt_kstart      = 0xffffffff81000000
(XEN)     virt_kend        = 0xffffffff820f4000
(XEN)     virt_entry       = 0xffffffff81ac9210
(XEN)     p2m_base         = 0xffffffffffffffff
(XEN)  Xen  kernel: 64-bit, lsb, compat32
(XEN)  Dom0 kernel: 64-bit, PAE, lsb, paddr 0x1000000 -> 0x20f4000
(XEN) PHYSICAL MEMORY ARRANGEMENT:
(XEN)  Dom0 alloc.:   0000000140000000->0000000144000000 (161339 pages
to be allocated)
(XEN)  Init. ramdisk: 000000014bd49000->000000014e5ffc00
(XEN) VIRTUAL MEMORY ARRANGEMENT:
(XEN)  Loaded kernel: ffffffff81000000->ffffffff820f4000
(XEN)  Init. ramdisk: ffffffff820f4000->ffffffff849aac00
(XEN)  Phys-Mach map: ffffffff849ab000->ffffffff84b1a790
(XEN)  Start info:    ffffffff84b1b000->ffffffff84b1b4b4
(XEN)  Page tables:   ffffffff84b1c000->ffffffff84b45000
(XEN)  Boot stack:    ffffffff84b45000->ffffffff84b46000
(XEN)  TOTAL:         ffffffff80000000->ffffffff84c00000
(XEN)  ENTRY ADDRESS: ffffffff81ac9210
(XEN) Dom0 has maximum 4 VCPUs
(XEN) elf_load_binary: phdr 0 at 0xffffffff81000000 -> 0xffffffff818a2000
(XEN) elf_load_binary: phdr 1 at 0xffffffff81a00000 -> 0xffffffff81ab30f0
(XEN) elf_load_binary: phdr 2 at 0xffffffff81ab4000 -> 0xffffffff81ac8600
(XEN) elf_load_binary: phdr 3 at 0xffffffff81ac9000 -> 0xffffffff81b9a000
(XEN) Scrubbing Free RAM: ................................done.
(XEN) Initial low memory virq threshold set at 0x4000 pages.
(XEN) Std. Loglevel: All
(XEN) Guest Loglevel: All
(XEN) *** Serial input -> DOM0 (type 'CTRL-a' three times to switch
input to Xen)
(XEN) Freed 268kB init memory.
mapping kernel into physical memory
about to get started...
[    0.000000] Initializing cgroup subsys cpuset
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Linux version 3.7.0-orc (bguthro@bguthro-desktop) (gcc
version 4.6.3 (Ubuntu/Linaro 4.6.3-1ubuntu5) ) #7 SMP Wed Oct 17
13:02:38 EDT 2012
[    0.000000] Command line:
root=/dev/mapper/NxVG--eb56f027--0aeb--4e9a--9233--678d57b9dc9e-NxDisk6
ro xencons=tty console=hvc0
[    0.000000] Freeing 9a-100 pfn range: 102 pages freed
[    0.000000] Freeing 20000-20200 pfn range: 512 pages freed
[    0.000000] Released 614 pages of unused memory
[    0.000000] Set 351519 page(s) to 1-1 mapping
[    0.000000] Populating 2def2-2e158 pfn range: 614 pages added
[    0.000000] e820: BIOS-provided physical RAM map:
[    0.000000] Xen: [mem 0x0000000000000000-0x0000000000099fff] usable
[    0.000000] Xen: [mem 0x000000000009ac00-0x00000000000fffff] reserved
[    0.000000] Xen: [mem 0x0000000000100000-0x000000001fffffff] usable
[    0.000000] Xen: [mem 0x0000000020000000-0x00000000201fffff] reserved
[    0.000000] Xen: [mem 0x0000000020200000-0x000000002e157fff] usable
[    0.000000] Xen: [mem 0x000000002e158000-0x0000000040003fff] unusable
[    0.000000] Xen: [mem 0x0000000040004000-0x0000000040004fff] reserved
[    0.000000] Xen: [mem 0x0000000040005000-0x00000000a8583fff] unusable
[    0.000000] Xen: [mem 0x00000000a8584000-0x00000000a85c6fff] reserved
[    0.000000] Xen: [mem 0x00000000a85c7000-0x00000000aa589fff] unusable
[    0.000000] Xen: [mem 0x00000000aa58a000-0x00000000aad69fff] reserved
[    0.000000] Xen: [mem 0x00000000aad6a000-0x00000000aafe9fff] ACPI NVS
[    0.000000] Xen: [mem 0x00000000aafea000-0x00000000aaffefff] ACPI data
[    0.000000] Xen: [mem 0x00000000aafff000-0x00000000aaffffff] unusable
[    0.000000] Xen: [mem 0x00000000ab000000-0x00000000af9fffff] reserved
[    0.000000] Xen: [mem 0x00000000f8000000-0x00000000fbffffff] reserved
[    0.000000] Xen: [mem 0x00000000fec00000-0x00000000fec00fff] reserved
[    0.000000] Xen: [mem 0x00000000fed10000-0x00000000fed13fff] reserved
[    0.000000] Xen: [mem 0x00000000fed18000-0x00000000fed19fff] reserved
[    0.000000] Xen: [mem 0x00000000fed1c000-0x00000000fed1ffff] reserved
[    0.000000] Xen: [mem 0x00000000fee00000-0x00000000fee00fff] reserved
[    0.000000] Xen: [mem 0x00000000ff900000-0x00000000ffbfffff] reserved
[    0.000000] Xen: [mem 0x00000000ffd00000-0x00000000ffffffff] reserved
[    0.000000] Xen: [mem 0x0000000100000000-0x000000014e5fffff] unusable
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] DMI 2.6 present.
[    0.000000] No AGP bridge found
[    0.000000] e820: last_pfn = 0x2e158 max_arch_pfn = 0x400000000
[    0.000000] x2apic enabled by BIOS, switching to x2apic ops
[    0.000000] init_memory_mapping: [mem 0x00000000-0x2e157fff]
[    0.000000] RAMDISK: [mem 0x020f4000-0x049aafff]
[    0.000000] ACPI: RSDP 00000000000f0410 00024 (v02  INTEL)
[    0.000000] ACPI: XSDT 00000000aaffde18 00074 (v01  INTEL  IVB-CPT
06222004 MSFT 00010013)
[    0.000000] ACPI: FACP 00000000aafe3d98 000F4 (v04  INTEL  IVB-CPT
06222004 MSFT 00010013)
[    0.000000] ACPI: DSDT 00000000aafbc018 1082F (v02  INTEL  IVB-CPT
00000000 INTL 20110623)
[    0.000000] ACPI: FACS 00000000aafe9e40 00040
[    0.000000] ACPI: APIC 00000000aaffcf18 000CC (v02  INTEL  IVB-CPT
06222004 MSFT 00010013)
[    0.000000] ACPI: HPET 00000000aafe8f18 00038 (v01 A M I   PCHHPET
06222004 AMI. 00000003)
[    0.000000] ACPI: SSDT 00000000aafe5018 010A8 (v01 TrmRef PtidDevc
00001000 INTL 20110623)
[    0.000000] ACPI: SSDT 00000000aafe4a18 00461 (v01    AMI PerfTune
00001000 INTL 20110623)
[    0.000000] ACPI: MCFG 00000000aafe8e98 0003C (v01 INTEL  SNDYBRDG
06222004 MSFT 00000097)
[    0.000000] ACPI: SSDT 00000000aafd1818 0079A (v01  PmRef  Cpu0Ist
00003000 INTL 20110623)
[    0.000000] ACPI: SSDT 00000000aafd0018 00A92 (v01  PmRef    CpuPm
00003000 INTL 20110623)
[    0.000000] ACPI: XMAR 00000000aafe4f18 000B8 (v01 INTEL      SNB
00000001 INTL 00000001)
[    0.000000] ACPI: FPDT 00000000aaff4018 00064 (v01  INTEL  IVB-CPT
00010000 INTL 20111107)
[    0.000000] Setting APIC routing to cluster x2apic.
[    0.000000] Zone ranges:
[    0.000000]   DMA      [mem 0x00010000-0x00ffffff]
[    0.000000]   DMA32    [mem 0x01000000-0xffffffff]
[    0.000000]   Normal   empty
[    0.000000] Movable zone start for each node
[    0.000000] Early memory node ranges
[    0.000000]   node   0: [mem 0x00010000-0x00099fff]
[    0.000000]   node   0: [mem 0x00100000-0x1fffffff]
[    0.000000]   node   0: [mem 0x20200000-0x2e157fff]
[    0.000000] ACPI: PM-Timer IO Port: 0x408
[    0.000000] ACPI: LAPIC (acpi_id[0x01] lapic_id[0x00] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x03] lapic_id[0x04] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x04] lapic_id[0x06] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x05] lapic_id[0x04] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x06] lapic_id[0x05] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x07] lapic_id[0x06] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x08] lapic_id[0x07] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x09] lapic_id[0x08] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0a] lapic_id[0x09] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0b] lapic_id[0x0a] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0c] lapic_id[0x0b] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0d] lapic_id[0x0c] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0e] lapic_id[0x0d] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x0f] lapic_id[0x0e] disabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x10] lapic_id[0x0f] disabled)
[    0.000000] ACPI: IOAPIC (id[0x02] address[0xfec00000] gsi_base[0])
[    0.000000] IOAPIC[0]: apic_id 2, version 32, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a701 base: 0xfed00000
[    0.000000] smpboot: Allowing 16 CPUs, 12 hotplug CPUs
[    0.000000] PM: Registered nosave memory: 000000000009a000 - 000000000009b000
[    0.000000] PM: Registered nosave memory: 000000000009b000 - 0000000000100000
[    0.000000] PM: Registered nosave memory: 0000000020000000 - 0000000020200000
[    0.000000] e820: [mem 0xafa00000-0xf7ffffff] available for PCI devices
[    0.000000] Booting paravirtualized kernel on Xen
[    0.000000] Xen version: 4.3-unstable (preserve-AD)
[    0.000000] setup_percpu: NR_CPUS:64 nr_cpumask_bits:64
nr_cpu_ids:16 nr_node_ids:1
[    0.000000] PERCPU: Embedded 28 pages/cpu @ffff88002da00000 s83456
r8192 d23040 u131072
[    1.854461] Built 1 zonelists in Zone order, mobility grouping on.
Total pages: 185174
[    1.854464] Kernel command line:
root=/dev/mapper/NxVG--eb56f027--0aeb--4e9a--9233--678d57b9dc9e-NxDisk6
ro xencons=tty console=hvc0
[    1.854903] PID hash table entries: 4096 (order: 3, 32768 bytes)
[    1.854969] Dentry cache hash table entries: 131072 (order: 8, 1048576 bytes)
[    1.855248] Inode-cache hash table entries: 65536 (order: 7, 524288 bytes)
[    1.855369] __ex_table already sorted, skipping sort
[    1.884622] software IO TLB [mem 0x28c00000-0x2cbfffff] (64MB)
mapped at [ffff880028c00000-ffff88002cbfffff]
[    1.886801] Memory: 612908k/755040k available (5817k kernel code,
2520k absent, 139612k reserved, 5136k data, 896k init)
[    1.886877] SLUB: Genslabs=15, HWalign=64, Order=0-3, MinObjects=0,
CPUs=4, Nodes=1
[    1.886905] Hierarchical RCU implementation.
[    1.886906] 	RCU dyntick-idle grace-period acceleration is enabled.
[    1.886907] 	RCU restricting CPUs from NR_CPUS=64 to nr_cpu_ids=4.
[    1.886916] NR_IRQS:4352 nr_irqs:712 16
[    1.886989] xen: sci override: global_irq=9 trigger=0 polarity=0
[    1.887024] xen: acpi sci 9
[    1.888406] Console: colour VGA+ 80x25
[    1.888704] console [hvc0] enabled
[    1.888728] installing Xen timer for CPU 0
[    1.888755] tsc: Detected 2195.064 MHz processor
[    1.888761] Calibrating delay loop (skipped), value calculated
using timer frequency.. 4390.12 BogoMIPS (lpj=21950640)
[    1.888768] pid_max: default: 32768 minimum: 301
[    1.888860] Mount-cache hash table entries: 256
[    1.889088] Initializing cgroup subsys cpuacct
[    1.889093] Initializing cgroup subsys devices
[    1.889096] Initializing cgroup subsys freezer
[    1.889100] Initializing cgroup subsys net_cls
[    1.889168] ENERGY_PERF_BIAS: Set to 'normal', was 'performance'
[    1.889168] ENERGY_PERF_BIAS: View and update with x86_energy_perf_policy(8)
[    1.889177] CPU: Physical Processor ID: 0
[    1.889179] CPU: Processor Core ID: 0
[    1.889591] mce: CPU supports 2 MCE banks
(XEN) traps.c:2485:d0 Domain attempted WRMSR 000000000000082f from
0x00000000000000f2 to 0x00000000000000f9.
[    1.889624] Last level iTLB entries: 4KB 512, 2MB 0, 4MB 0
[    1.889624] Last level dTLB entries: 4KB 512, 2MB 32, 4MB 32
[    1.889624] tlb_flushall_shift: 1
[    1.889739] Freeing SMP alternatives: 20k freed
[    1.891834] ACPI: Core revision 20120913
[    1.908378] ftrace: allocating 21833 entries in 86 pages
[    1.919224] cpu 0 spinlock event irq 41
[    1.919242] Performance Events: unsupported p6 CPU model 58 no PMU
driver, software events only.
[    1.920070] installing Xen timer for CPU 1
[    1.920082] cpu 1 spinlock event irq 48
(XEN) traps.c:2485:d0 Domain attempted WRMSR 000000000000082f from
0x00000000000000f2 to 0x00000000000000f9.
[    1.920805] installing Xen timer for CPU 2
[    1.920815] cpu 2 spinlock event irq 55
(XEN) traps.c:2485:d0 Domain attempted WRMSR 000000000000082f from
0x00000000000000f2 to 0x00000000000000f9.
[    1.921556] installing Xen timer for CPU 3
[    1.921566] cpu 3 spinlock event irq 62
(XEN) traps.c:2485:d0 Domain attempted WRMSR 000000000000082f from
0x00000000000000f2 to 0x00000000000000f9.
[    1.922256] Brought up 4 CPUs
[    1.922685] devtmpfs: initialized
[    1.922949] PM: Registering ACPI NVS region [mem
0xaad6a000-0xaafe9fff] (2621440 bytes)
[    1.923631] Grant tables using version 2 layout.
[    1.923646] Grant table initialized
[    1.923694] regulator-dummy: no parameters
[    1.923742] RTC time: 17:32:37, date: 10/17/12
[    1.923777] NET: Registered protocol family 16
[    1.924024] ACPI FADT declares the system doesn't support PCIe
ASPM, so disable it
[    1.924030] ACPI: bus type pci registered
[    1.924147] PCI: MMCONFIG for domain 0000 [bus 00-3f] at [mem
0xf8000000-0xfbffffff] (base 0xf8000000)
[    1.924153] PCI: MMCONFIG at [mem 0xf8000000-0xfbffffff] reserved in E820
[    1.937145] PCI: Using configuration type 1 for base access
[    1.937864] bio: create slab <bio-0> at 0
[    1.937979] ACPI: Added _OSI(Module Device)
[    1.937983] ACPI: Added _OSI(Processor Device)
[    1.937986] ACPI: Added _OSI(3.0 _SCP Extensions)
[    1.937990] ACPI: Added _OSI(Processor Aggregator Device)
[    1.943414] ACPI: Executed 1 blocks of module-level executable AML code
[    1.961910] ACPI: SSDT 00000000aaccd018 00A4F (v01  PmRef  Cpu0Cst
00003001 INTL 20110623)
[    1.962563] ACPI: Dynamic OEM Table Load:
[    1.962568] ACPI: SSDT           (null) 00A4F (v01  PmRef  Cpu0Cst
00003001 INTL 20110623)
[    2.001604] ACPI: SSDT 00000000aaccea98 00303 (v01  PmRef    ApIst
00003000 INTL 20110623)
[    2.002283] ACPI: Dynamic OEM Table Load:
[    2.002287] ACPI: SSDT           (null) 00303 (v01  PmRef    ApIst
00003000 INTL 20110623)
[    2.041649] ACPI: SSDT 00000000aacccd98 00119 (v01  PmRef    ApCst
00003000 INTL 20110623)
[    2.042297] ACPI: Dynamic OEM Table Load:
[    2.042302] ACPI: SSDT           (null) 00119 (v01  PmRef    ApCst
00003000 INTL 20110623)
[    2.082598] ACPI: Interpreter enabled
[    2.082605] ACPI: (supports S0 S1 S3 S4 S5)
[    2.082631] ACPI: Using IOAPIC for interrupt routing
[    2.088693] ACPI: Power Resource [FN00] (off)
[    2.088768] ACPI: Power Resource [FN01] (off)
[    2.088848] ACPI: Power Resource [FN02] (off)
[    2.088921] ACPI: Power Resource [FN03] (off)
[    2.088992] ACPI: Power Resource [FN04] (off)
[    2.089894] ACPI: ACPI Dock Station Driver: 1 docks/bays found
[    2.089901] PCI: Using host bridge windows from ACPI; if necessary,
use "pci=nocrs" and report a bug
[    2.090503] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-3e])
[    2.091264] PCI host bridge to bus 0000:00
[    2.091269] pci_bus 0000:00: root bus resource [bus 00-3e]
[    2.091273] pci_bus 0000:00: root bus resource [io  0x0000-0x0cf7]
[    2.091277] pci_bus 0000:00: root bus resource [io  0x0d00-0xffff]
[    2.091281] pci_bus 0000:00: root bus resource [mem 0x000a0000-0x000bffff]
[    2.091285] pci_bus 0000:00: root bus resource [mem 0x000d0000-0x000d3fff]
[    2.091289] pci_bus 0000:00: root bus resource [mem 0x000d4000-0x000d7fff]
[    2.091293] pci_bus 0000:00: root bus resource [mem 0x000d8000-0x000dbfff]
[    2.091297] pci_bus 0000:00: root bus resource [mem 0x000dc000-0x000dffff]
[    2.091301] pci_bus 0000:00: root bus resource [mem 0x000e0000-0x000e3fff]
[    2.091305] pci_bus 0000:00: root bus resource [mem 0x000e4000-0x000e7fff]
[    2.091309] pci_bus 0000:00: root bus resource [mem 0xafa00000-0xfeafffff]
[    2.095857] pci 0000:00:1c.0: PCI bridge to [bus 01]
[    2.096587] pci 0000:00:1c.6: PCI bridge to [bus 02]
[    2.097102] pci 0000:00:1c.7: PCI bridge to [bus 03-04]
[    2.097749] pci 0000:03:00.0: PCI bridge to [bus 04]
[    2.098158] pci 0000:00:1e.0: PCI bridge to [bus 05] (subtractive decode)
[    2.098677]  pci0000:00: ACPI _OSC support notification failed,
disabling PCIe ASPM
[    2.098682]  pci0000:00: Unable to request _OSC control (_OSC
support mask: 0x08)
(XEN) PCI add device 0000:00:00.0
(XEN) PCI add device 0000:00:02.0
(XEN) PCI add device 0000:00:14.0
(XEN) PCI add device 0000:00:16.0
(XEN) PCI add device 0000:00:16.3
(XEN) PCI add device 0000:00:19.0
(XEN) PCI add device 0000:00:1a.0
(XEN) PCI add device 0000:00:1b.0
(XEN) PCI add device 0000:00:1c.0
(XEN) PCI add device 0000:00:1c.6
(XEN) PCI add device 0000:00:1c.7
(XEN) PCI add device 0000:00:1d.0
(XEN) PCI add device 0000:00:1e.0
(XEN) PCI add device 0000:00:1f.0
(XEN) PCI add device 0000:00:1f.2
(XEN) PCI add device 0000:00:1f.3
(XEN) PCI add device 0000:02:00.0
(XEN) PCI add device 0000:03:00.0
(XEN) PCI add device 0000:04:00.0
(XEN) PCI add device 0000:05:01.0
[    2.104211] ACPI: PCI Interrupt Link [LNKA] (IRQs 3 4 5 6 10 *11 12 14 15)
[    2.104279] ACPI: PCI Interrupt Link [LNKB] (IRQs 3 4 5 6 10 11 12
14 15) *0, disabled.
[    2.104348] ACPI: PCI Interrupt Link [LNKC] (IRQs 3 *4 5 6 10 11 12 14 15)
[    2.104414] ACPI: PCI Interrupt Link [LNKD] (IRQs 3 4 5 6 *10 11 12 14 15)
[    2.104480] ACPI: PCI Interrupt Link [LNKE] (IRQs 3 4 *5 6 10 11 12 14 15)
[    2.104544] ACPI: PCI Interrupt Link [LNKF] (IRQs 3 4 5 6 10 11 12
14 15) *0, disabled.
[    2.104612] ACPI: PCI Interrupt Link [LNKG] (IRQs *3 4 5 6 10 11 12 14 15)
[    2.104676] ACPI: PCI Interrupt Link [LNKH] (IRQs 3 4 5 6 10 *11 12 14 15)
[    2.104716] xen/balloon: Initialising balloon driver.
[    2.104855] vgaarb: device added:
PCI:0000:00:02.0,decodes=io+mem,owns=io+mem,locks=none
[    2.104869] vgaarb: loaded
[    2.104871] vgaarb: bridge control possible 0000:00:02.0
[    2.104950] SCSI subsystem initialized
[    2.104954] ACPI: bus type scsi registered
[    2.105131] ACPI: bus type usb registered
[    2.105151] usbcore: registered new interface driver usbfs
[    2.105162] usbcore: registered new interface driver hub
[    2.105304] usbcore: registered new device driver usb
[    2.105513] wmi: Mapper loaded
[    2.105519] PCI: Using ACPI for IRQ routing
[    2.111086] Switching to clocksource xen
[    2.116428] pnp: PnP ACPI init
[    2.116439] ACPI: bus type pnp registered
[    2.117207] system 00:05: [io  0x0680-0x069f] has been reserved
[    2.117212] system 00:05: [io  0x1000-0x100f] has been reserved
[    2.117216] system 00:05: [io  0xffff] has been reserved
[    2.117219] system 00:05: [io  0xffff] has been reserved
[    2.117223] system 00:05: [io  0x0400-0x0453] has been reserved
[    2.117227] system 00:05: [io  0x0458-0x047f] has been reserved
[    2.117231] system 00:05: [io  0x0500-0x057f] has been reserved
[    2.117235] system 00:05: [io  0x164e-0x164f] has been reserved
[    2.117330] system 00:07: [io  0x0454-0x0457] has been reserved
[    2.117721] system 00:08: [mem 0xfed1c000-0xfed1ffff] has been reserved
[    2.117726] system 00:08: [mem 0xfed10000-0xfed17fff] could not be reserved
[    2.117730] system 00:08: [mem 0xfed18000-0xfed18fff] has been reserved
[    2.117735] system 00:08: [mem 0xfed19000-0xfed19fff] has been reserved
[    2.117739] system 00:08: [mem 0xf8000000-0xfbffffff] has been reserved
[    2.117743] system 00:08: [mem 0xfed20000-0xfed3ffff] has been reserved
[    2.117748] system 00:08: [mem 0xfed90000-0xfed93fff] has been reserved
[    2.117753] system 00:08: [mem 0xfed45000-0xfed8ffff] has been reserved
[    2.117757] system 00:08: [mem 0xff000000-0xffffffff] could not be reserved
[    2.117762] system 00:08: [mem 0xfee00000-0xfeefffff] could not be reserved
[    2.117766] system[    3.281195] usb 3-3: new low-speed USB device
number 2 using xhci_hcd
[    3.282603] bio: create slab <bio-1> at 1
[    3.304654] usb 3-3: ep 0x81 - rounding interval to 64 microframes,
ep desc says 80 microframes
[    3.311353] usbcore: registered new interface driver usbhid
[    3.311360] usbhid: USB HID core driver
[    3.421244] usb 3-4: new low-speed USB device number 3 using xhci_hcd
Begin: Running /scripts/local-premount ... [    3.448335] usb 3-4: ep
0x81 - rounding interval to 128 microframes, ep desc says 192
microframes
done.
[    3.523331] EXT4-fs (dm-3): mounted filesystem with ordered data
mode. Opts: (null)
Begin: Running /scripts/local-bottom ... done.
done.
Begin: Running /scripts/init-bottom ... udevd[290]:
inotify_add_watch(6, /dev/dm-6, 10) failed: Invalid argument

done.
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7

NxTop Engine (99.99.pre-121017-1309-bguthro) poison-ivy hvc0

poison-ivy login: root (automatic login)
Last login: Wed Oct 17 17:27:26 UTC 2012 from
bguthro-desktop.oldroadcomputing.net on pts/8
Welcome to NxTop Engine (GNU/Linux 3.7.0-orc x86_64)

root@poison-ivy:~#
root@poison-ivy:~#
root@poison-ivy:~# [   13.645342] irq 22: nobody cared (try booting
with the "irqpoll" option)
[   13.645442] handlers:
[   13.645446] [<ffffffff81394fe0>] serial8250_interrupt
[   13.645449] Disabling IRQ #22

root@poison-ivy:~#
root@poison-ivy:~#
root@poison-ivy:~# [   26.153266] irq 22: nobody cared (try booting
with the "irqpoll" option)
[   26.153341] handlers:
[   26.153345] [<ffffffff81394fe0>] serial8250_interrupt
[   26.153349] Disabling IRQ #22
(XEN) Preparing system for ACPI S3 state.
(XEN) Disabling non-boot CPUs ...
(XEN) Breaking vcpu affinity for domain 0 vcpu 2
(XEN) Breaking vcpu affinity for domain 0 vcpu 3
(XEN) Entering ACPI S3 state.
(XEN) mce_intel.c:721: MCA Capability: BCAST 1 SER 0 CMCI 1 firstbank
0 extended MCE MSR 0
(XEN) CPU0 CMCI LVT vector (0xf2) already installed
(XEN) Finishing wakeup from ACPI S3 state.
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) Enabling non-boot CPUs  ...
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) microcode: collect_cpu_info : sig=0x306a4, pf=0x2, rev=0x7
(XEN) mm.c:587:d0 Could not get page ref for pfn ffffffffffffffff
(XEN) mm.c:2571:d0 Error while installing new baseptr ffffffffffffffff
(XEN) mm.c:2265:d0 Bad type (saw 4400000000000001 != exp
7000000000000000) for mfn 13db97 (pfn 26eb2)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13db97 (pfn 26eb2) from L1 entry
800000013db97063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 3400000000000002 != exp
7000000000000000) for mfn 13edc4 (pfn 28085)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13edc4 (pfn 28085) from L1 entry
800000013edc4063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 2400000000000001 != exp
7000000000000000) for mfn 13e625 (pfn 27c24)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13e625 (pfn 27c24) from L1 entry
800000013e625063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 1400000000000001 != exp
7000000000000000) for mfn 13e886 (pfn 27dc3)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13e886 (pfn 27dc3) from L1 entry
800000013e886063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 1400000000000001 != exp
7000000000000000) for mfn 13ebfa (pfn 27e4f)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13ebfa (pfn 27e4f) from L1 entry
800000013ebfa063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 1400000000000001 != exp
7000000000000000) for mfn 14d52f (pfn 38da)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 14d52f (pfn 38da) from L1 entry
800000014d52f063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 3400000000000002 != exp
7000000000000000) for mfn 13e5a1 (pfn 278a8)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13e5a1 (pfn 278a8) from L1 entry
800000013e5a1063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 2400000000000001 != exp
7000000000000000) for mfn 14dba6 (pfn 3f51)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 14dba6 (pfn 3f51) from L1 entry
800000014dba6063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 1400000000000001 != exp
7000000000000000) for mfn 13da86 (pfn 26fc3)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13da86 (pfn 26fc3) from L1 entry
800000013da86063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 1400000000000001 != exp
7000000000000000) for mfn 14d50a (pfn 38b5)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 14d50a (pfn 38b5) from L1 entry
800000014d50a063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 1400000000000001 != exp
7000000000000000) for mfn 14d0a7 (pfn 3452)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 14d0a7 (pfn 3452) from L1 entry
800000014d0a7063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 1400000000000001 != exp
7000000000000000) for mfn 14d6e7 (pfn 3a92)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 14d6e7 (pfn 3a92) from L1 entry
800000014d6e7063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 3400000000000002 != exp
7000000000000000) for mfn 14dafd (pfn 3ea8)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 14dafd (pfn 3ea8) from L1 entry
800000014dafd063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 2400000000000001 != exp
7000000000000000) for mfn 13d598 (pfn 268b1)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13d598 (pfn 268b1) from L1 entry
800000013d598063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 1400000000000001 != exp
7000000000000000) for mfn 13daa1 (pfn 26fa8)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 13daa1 (pfn 26fa8) from L1 entry
800000013daa1063 for l1e_owner=0, pg_owner=0
(XEN) mm.c:2265:d0 Bad type (saw 4400000000000001 != exp
7000000000000000) for mfn 14d0d5 (pfn 3480)
(XEN) mm.c:812:d0 Could not get page type PGT_writable_page
(XEN) mm.c:864:d0 Error getting mfn 14d0d5 (pfn 3480) from L1 entry
800000014d0d5063 for l1e_owner=0, pg_owner=0

Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Ben Guthro]

I'm not sure it matters, but I'm testing against a changeset about a week old:
http://xenbits.xen.org/gitweb/?p=xen.git;a=commit;h=1c4a5b37b55c56e49135e65728137f54288d1fe6

Plus patches specific to XenClient Enterprise.


On Wed, Oct 17, 2012 at 1:43 PM, Konrad Rzeszutek Wilk
<konrad.wilk@oracle.com> wrote:
> On Wed, Oct 17, 2012 at 01:46:09PM -0400, Ben Guthro wrote:
>> On Wed, Oct 17, 2012 at 9:49 AM, Konrad Rzeszutek Wilk
>> <konrad.wilk@oracle.com> wrote:
>> [...]
>>
>> > The end result is this is a nice set of patches where there is only
>> > _one_ change in the x86 code (and it is just more of dealing with
>> > error case) - and the rest are all done in Xen side.
>>
>> I'm sorry to report that this series doesn't seem to work in my setup
>> against xen-unstable.
>>
>> To verify that it was, in fact this patch series, and not another Xen
>> regression - I swapped out the kernel with this patch series, with an
>> identical one, replacing only this series with your acpi-s3.v9 branch
>> - and everything worked fine.
>
> Thanks for testing it!
>
> I had tested it with Xen 4.1.3, which could be doing something different.
> Will see what is up.

Re: [PATCH 2/4] xen/lowlevel: Implement pvop call for load_idt (sidt).

[H. Peter Anvin]

On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> In the past it used to point to 'sidt' (native_store_idt) operation
> which is a non-privileged operation. This resulted in the
> 'struct desc_ptr' value containing the address of Xen's IDT table,
> instead of the IDT table that Linux thinks its using. The end result
> is that doing:
>
>    store_idt(&desc);
>    load_idt(&desc);
>
> would blow up b/c xen_load_idt would try to parse the IDT contents
> (desc) and de-reference a virtual address that is outside Linux's
> __va (it is in Xen's virtual address).
>
> With this patch we are providing the last written IDT address.
>

OK... this seems like another excellent set of pvops calls that should 
be nukable to Kingdom Come.  There is no reason, ever, to read the IDT 
and GDT from the kernel... the kernel already knows what they should be!

	-hpa


-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH 1/4] x86/wakeup/sleep: Check whether the TSS GDT descriptor is empty before using it.

[H. Peter Anvin]

On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> We check the TSS descriptor before we try to dereference it.
> Also fix up the value to use the #defines.
>
> Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> ---
>   arch/x86/power/cpu.c |    7 +++++--
>   1 files changed, 5 insertions(+), 2 deletions(-)
>
> diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
> index 218cdb1..c17370e 100644
> --- a/arch/x86/power/cpu.c
> +++ b/arch/x86/power/cpu.c
> @@ -133,7 +133,9 @@ static void fix_processor_context(void)
>   {
>   	int cpu = smp_processor_id();
>   	struct tss_struct *t = &per_cpu(init_tss, cpu);
> -
> +#ifdef CONFIG_X86_64
> +	struct desc_struct *desc = get_cpu_gdt_table(cpu);
> +#endif
>   	set_tss_desc(cpu, t);	/*
>   				 * This just modifies memory; should not be
>   				 * necessary. But... This is necessary, because
> @@ -142,7 +144,8 @@ static void fix_processor_context(void)
>   				 */
>
>   #ifdef CONFIG_X86_64
> -	get_cpu_gdt_table(cpu)[GDT_ENTRY_TSS].type = 9;
> +	if (!desc_empty(&desc[GDT_ENTRY_TSS]))
> +		desc[GDT_ENTRY_TSS].type = DESC_TSS;
>
>   	syscall_init();				/* This sets MSR_*STAR and related */
>   #endif
>

Why is this patch necessary?  Presumably there is something further down 
the line which depends on the TSS descriptor being empty, but if so, what?

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH 2/4] xen/lowlevel: Implement pvop call for load_idt (sidt).

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 04:51:17PM -0700, H. Peter Anvin wrote:
> On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> >In the past it used to point to 'sidt' (native_store_idt) operation
> >which is a non-privileged operation. This resulted in the
> >'struct desc_ptr' value containing the address of Xen's IDT table,
> >instead of the IDT table that Linux thinks its using. The end result
> >is that doing:
> >
> >   store_idt(&desc);
> >   load_idt(&desc);
> >
> >would blow up b/c xen_load_idt would try to parse the IDT contents
> >(desc) and de-reference a virtual address that is outside Linux's
> >__va (it is in Xen's virtual address).
> >
> >With this patch we are providing the last written IDT address.
> >
> 
> OK... this seems like another excellent set of pvops calls that
> should be nukable to Kingdom Come.  There is no reason, ever, to
> read the IDT and GDT from the kernel... the kernel already knows
> what they should be!

Even during suspend and resume cycle? There are the sequence of
sidt/lidt calls being done there. And we do need to filter at
least the sidt call - and in the case of ACPI suspend path,
the lidt.
> 
> 	-hpa
> 
> 
> -- 
> H. Peter Anvin, Intel Open Source Technology Center
> I work for Intel.  I don't speak on their behalf.
> 

Re: [PATCH 1/4] x86/wakeup/sleep: Check whether the TSS GDT descriptor is empty before using it.

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 05:03:09PM -0700, H. Peter Anvin wrote:
> On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> >We check the TSS descriptor before we try to dereference it.
> >Also fix up the value to use the #defines.
> >
> >Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> >---
> >  arch/x86/power/cpu.c |    7 +++++--
> >  1 files changed, 5 insertions(+), 2 deletions(-)
> >
> >diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
> >index 218cdb1..c17370e 100644
> >--- a/arch/x86/power/cpu.c
> >+++ b/arch/x86/power/cpu.c
> >@@ -133,7 +133,9 @@ static void fix_processor_context(void)
> >  {
> >  	int cpu = smp_processor_id();
> >  	struct tss_struct *t = &per_cpu(init_tss, cpu);
> >-
> >+#ifdef CONFIG_X86_64
> >+	struct desc_struct *desc = get_cpu_gdt_table(cpu);
> >+#endif
> >  	set_tss_desc(cpu, t);	/*
> >  				 * This just modifies memory; should not be
> >  				 * necessary. But... This is necessary, because
> >@@ -142,7 +144,8 @@ static void fix_processor_context(void)
> >  				 */
> >
> >  #ifdef CONFIG_X86_64
> >-	get_cpu_gdt_table(cpu)[GDT_ENTRY_TSS].type = 9;
> >+	if (!desc_empty(&desc[GDT_ENTRY_TSS]))
> >+		desc[GDT_ENTRY_TSS].type = DESC_TSS;
> >
> >  	syscall_init();				/* This sets MSR_*STAR and related */
> >  #endif
> >
> 
> Why is this patch necessary?  Presumably there is something further
> down the line which depends on the TSS descriptor being empty, but
> if so, what?

I could not find it. This write has been in the code since the initial
git history. Is the pre-git bitkeeper tree somewhere available?
> 
> 	-hpa
> 
> -- 
> H. Peter Anvin, Intel Open Source Technology Center
> I work for Intel.  I don't speak on their behalf.
> 

Re: [PATCH 1/4] x86/wakeup/sleep: Check whether the TSS GDT descriptor is empty before using it.

[H. Peter Anvin]

On 10/18/2012 07:47 AM, Konrad Rzeszutek Wilk wrote:
>>
>> Why is this patch necessary?  Presumably there is something further
>> down the line which depends on the TSS descriptor being empty, but
>> if so, what?
>
> I could not find it. This write has been in the code since the initial
> git history. Is the pre-git bitkeeper tree somewhere available?

I didn't ask why the write was necessary, but why you need it to be 
conditional.

I know why the write is necessary: it is (presumably) there to clear the 
TSS busy bit.

However, as for older records:

-bk era history (2.5.0-2.6.12-rc2):
git://git.kernel.org/pub/scm/linux/kernel/git/tglx/history.git

Ancient history (pre-2.4.0):
git://git.kernel.org/pub/scm/linux/kernel/git/davej/history.git

There doesn't seem to be any git trees known for the bridge between 
2.4.0 and 2.5.0.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH 2/4] xen/lowlevel: Implement pvop call for load_idt (sidt).

[H. Peter Anvin]

On 10/18/2012 07:45 AM, Konrad Rzeszutek Wilk wrote:
>>
>> OK... this seems like another excellent set of pvops calls that
>> should be nukable to Kingdom Come.  There is no reason, ever, to
>> read the IDT and GDT from the kernel... the kernel already knows
>> what they should be!
>
> Even during suspend and resume cycle? There are the sequence of
> sidt/lidt calls being done there. And we do need to filter at
> least the sidt call - and in the case of ACPI suspend path,
> the lidt.

Yes, I am pretty sure we can make static guarantees on the IDT and GDTs.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

RE: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Dan Magenheimer]

> From: H. Peter Anvin [mailto:hpa@zytor.com]
> Sent: Wednesday, October 17, 2012 11:35 AM
> To: Konrad Rzeszutek Wilk
> Cc: linux-acpi@vger.kernel.org; x86@kernel.org; xen-devel@lists.xensource.com; linux-
> kernel@vger.kernel.org; lenb@kernel.org
> Subject: Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly
> small\!).
> 
> On 10/17/2012 09:54 AM, Konrad Rzeszutek Wilk wrote:
> >>
> >> Could you do an audit for other pvops calls that have no users?  If
> >> the *only* user is lguest, we should talk about it, too...
> >
> > I can do that - but I don't want to be hasty here. There is a bit of
> > danger here - for example the read_pmc (or read_tsc) is not in use right
> > now. But it might be when one starts looking at making perf be able to
> > analyze the hypervisor (hand-waving the implementation details). So while
> > removing read_pmc now sounds good, it might be needed in the future.
> >
> 
> We do not keep a pvop around just because it "might be needed in the
> future".  That's just crazy.
> 
> 	-hpa

It's a bit more complicated than that.  The problem is that if
any patch is ever submitted to the kernel that uses the rdtscp
instruction *in kernel space* in some clever way, the resultant
kernel may not behave as expected (depending on how the instruction
is used) on a 32-bit[1] PV kernel running on Xen, up to and including
the possibility of data corruption.

I don't know how one would implement it, but it's like a
BUILD_BUG_ON is needed if any kernel developer uses rdtscp
(one that never gets invoked by vdso code), that prints:

"WARNING: Please do not use this instruction in the kernel
without notifying the Xen maintainer as there is a possibility
it may behave unpredictably in some Xen environments.
See Documentation/.../xen_pv_limitations for detail."

The other virtualization-unsafe instructions may have similar
problems.

Just FYI...

Dan

[1] I _think_ this is not a problem on 64-bit kernels but
am not certain.

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[H. Peter Anvin]

On 10/18/2012 08:22 AM, Dan Magenheimer wrote:
>
> It's a bit more complicated than that.  The problem is that if
> any patch is ever submitted to the kernel that uses the rdtscp
> instruction *in kernel space* in some clever way, the resultant
> kernel may not behave as expected (depending on how the instruction
> is used) on a 32-bit[1] PV kernel running on Xen, up to and including
> the possibility of data corruption.
>
> I don't know how one would implement it, but it's like a
> BUILD_BUG_ON is needed if any kernel developer uses rdtscp
> (one that never gets invoked by vdso code), that prints:
>
> "WARNING: Please do not use this instruction in the kernel
> without notifying the Xen maintainer as there is a possibility
> it may behave unpredictably in some Xen environments.
> See Documentation/.../xen_pv_limitations for detail."
>
> The other virtualization-unsafe instructions may have similar
> problems.
>

Good frakking God.  This is the sort of things that makes me think that 
Xen PV should just be thrown out of the kernel once and for all.

Do you notice that the document you just claimed doesn't even exist at 
this point, never mind being somehow enforced?  In other word, there is 
ABSOLUTELY NO WAY a mainline kernel developer can have any idea what 
amount of violence Xen does to the architecture that it is parasiting on.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

RE: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Dan Magenheimer]

> From: H. Peter Anvin [mailto:hpa@zytor.com]
> Subject: Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly
> small\!).
> 
> On 10/18/2012 08:22 AM, Dan Magenheimer wrote:
> >
> > It's a bit more complicated than that.  The problem is that if
> > any patch is ever submitted to the kernel that uses the rdtscp
> > instruction *in kernel space* in some clever way, the resultant
> > kernel may not behave as expected (depending on how the instruction
> > is used) on a 32-bit[1] PV kernel running on Xen, up to and including
> > the possibility of data corruption.
> >
> > I don't know how one would implement it, but it's like a
> > BUILD_BUG_ON is needed if any kernel developer uses rdtscp
> > (one that never gets invoked by vdso code), that prints:
> >
> > "WARNING: Please do not use this instruction in the kernel
> > without notifying the Xen maintainer as there is a possibility
> > it may behave unpredictably in some Xen environments.
> > See Documentation/.../xen_pv_limitations for detail."
> >
> > The other virtualization-unsafe instructions may have similar
> > problems.
> >
> 
> Good frakking God.  This is the sort of things that makes me think that
> Xen PV should just be thrown out of the kernel once and for all.

I agree the whole idea of paravirtualization is a hack, but it
is a hack to workaround some poor architectural design decisions
many years ago by Intel processor designers who should have known
better.  Go yell at them.

Worse, the rdtscp instruction was a poor design decision by
AMD processor designers to hack around tsc skew problems.
Go yell at them too.

And both Intel and AMD chose to perpetuate the problem
with a complicated VT/SVM implementation that will never
perform as well as native.  At least they tried ;-)
 
> Do you notice that the document you just claimed doesn't even exist at
> this point, never mind being somehow enforced?  In other word, there is
> ABSOLUTELY NO WAY a mainline kernel developer can have any idea what
> amount of violence Xen does to the architecture that it is parasiting on.

Of course I know it doesn't exist.  I probably should have
noted that in my email.  But it should exist because else
subtle issues like this will get lost in the mist of time.
And I have no clue how to enforce it (though some BUILD_BUG_ON
might help).

Like so many other things in the kernel (and computing in general),
paravirtualization is a tradeoff of maintainability for kernel/software
developers vs significant performance improvement for (a large
number of) users.  That tradeoff is above my pay grade.  But
it does provide job security.

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Borislav Petkov]

On Thu, Oct 18, 2012 at 08:56:40AM -0700, Dan Magenheimer wrote:
> I agree the whole idea of paravirtualization is a hack, but it is a
> hack to workaround some poor architectural design decisions many years
> ago by Intel processor designers who should have known better. Go yell
> at them.
>
> Worse, the rdtscp instruction was a poor design decision by AMD
> processor designers to hack around tsc skew problems. Go yell at them
> too.
>
> And both Intel and AMD chose to perpetuate the problem with a
> complicated VT/SVM implementation that will never perform as well as
> native. At least they tried ;-)

Looks like xen people seem to know better so maybe they should design
their own processor, add xen support for it and leave the linux kernel
alone so that both camps can finally get on with their lives.

-- 
Regards/Gruss,
Boris.

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[David Vrabel]

On 17/10/12 17:54, Konrad Rzeszutek Wilk wrote:
> On Wed, Oct 17, 2012 at 09:50:11AM -0700, H. Peter Anvin wrote:
>> On 10/17/2012 09:10 AM, Konrad Rzeszutek Wilk wrote:
>>> On Wed, Oct 17, 2012 at 09:03:12AM -0700, H. Peter Anvin wrote:
>>>> On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
>>>>>
>>>>> Note: These are the other patches that went in 3.7-rc1:
>>>>> xen/bootup: allow {read|write}_cr8 pvops call [https://lkml.org/lkml/2012/10/10/339]
>>>>> xen/bootup: allow read_tscp call for Xen PV guests. [https://lkml.org/lkml/2012/10/10/340]
>>>>>
>>>>
>>>> So WTF do we have a read_tscp PV call?  Again, if there isn't a user
>>>> we should just axe it...
>>>
>>> Let me spin off a patch to see if that can be done.
>>>
>>
>> Could you do an audit for other pvops calls that have no users?  If
>> the *only* user is lguest, we should talk about it, too...
> 
> I can do that - but I don't want to be hasty here. There is a bit of
> danger here - for example the read_pmc (or read_tsc) is not in use right
> now. But it might be when one starts looking at making perf be able to
> analyze the hypervisor (hand-waving the implementation details). So while
> removing read_pmc now sounds good, it might be needed in the future.

I don't see any reason why would ever need a PV-specific implementation
of either read_pmc or read_tsc.  And I certainly agree with hpa that
leaving them around 'just in case' isn't useful.

As for 'perf', since Xen already provides a virtual PMU for HVM guests
It's not clear why we would spend the effort to implement another
mechanism for PV guests (instead of using the virtual PMU from a PVH guest).

David

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[David Vrabel]

On 17/10/12 17:54, Konrad Rzeszutek Wilk wrote:
> On Wed, Oct 17, 2012 at 09:50:11AM -0700, H. Peter Anvin wrote:
>> On 10/17/2012 09:10 AM, Konrad Rzeszutek Wilk wrote:
>>> On Wed, Oct 17, 2012 at 09:03:12AM -0700, H. Peter Anvin wrote:
>>>> On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
>>>>>
>>>>> Note: These are the other patches that went in 3.7-rc1:
>>>>> xen/bootup: allow {read|write}_cr8 pvops call [https://lkml.org/lkml/2012/10/10/339]
>>>>> xen/bootup: allow read_tscp call for Xen PV guests. [https://lkml.org/lkml/2012/10/10/340]
>>>>>
>>>>
>>>> So WTF do we have a read_tscp PV call?  Again, if there isn't a user
>>>> we should just axe it...
>>>
>>> Let me spin off a patch to see if that can be done.
>>>
>>
>> Could you do an audit for other pvops calls that have no users?  If
>> the *only* user is lguest, we should talk about it, too...
> 
> I can do that - but I don't want to be hasty here. There is a bit of
> danger here - for example the read_pmc (or read_tsc) is not in use right
> now. But it might be when one starts looking at making perf be able to
> analyze the hypervisor (hand-waving the implementation details). So while
> removing read_pmc now sounds good, it might be needed in the future.

I don't see any reason why would ever need a PV-specific implementation
of either read_pmc or read_tsc.  And I certainly agree with hpa that
leaving them around 'just in case' isn't useful.

As for 'perf', since Xen already provides a virtual PMU for HVM guests
It's not clear why we would spend the effort to implement another
mechanism for PV guests (instead of using the virtual PMU from a PVH guest).

David

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[H. Peter Anvin]

On 10/18/2012 08:56 AM, Dan Magenheimer wrote:
>
>> Do you notice that the document you just claimed doesn't even exist at
>> this point, never mind being somehow enforced?  In other word, there is
>> ABSOLUTELY NO WAY a mainline kernel developer can have any idea what
>> amount of violence Xen does to the architecture that it is parasiting on.
>
> Of course I know it doesn't exist.  I probably should have
> noted that in my email.  But it should exist because else
> subtle issues like this will get lost in the mist of time.
> And I have no clue how to enforce it (though some BUILD_BUG_ON
> might help).
>

Do you know for how long I have been yelling at various Xen people for 
*not documenting their architecture*?  There are plenty of 
paravirtualized architectures out there which are perfectly well 
documented and supportable, but Xen has resisted doing that for years, 
and all we ever get are vague future promises.

	-hpa


-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Stefano Stabellini]

On Thu, 18 Oct 2012, Borislav Petkov wrote:
> On Thu, Oct 18, 2012 at 08:56:40AM -0700, Dan Magenheimer wrote:
> > I agree the whole idea of paravirtualization is a hack, but it is a
> > hack to workaround some poor architectural design decisions many years
> > ago by Intel processor designers who should have known better. Go yell
> > at them.
> >
> > Worse, the rdtscp instruction was a poor design decision by AMD
> > processor designers to hack around tsc skew problems. Go yell at them
> > too.
> >
> > And both Intel and AMD chose to perpetuate the problem with a
> > complicated VT/SVM implementation that will never perform as well as
> > native. At least they tried ;-)
> 
> Looks like xen people seem to know better so maybe they should design
> their own processor, add xen support for it and leave the linux kernel
> alone so that both camps can finally get on with their lives.

I know that it is obvious but it is worth stating it in clear letters:

these are Dan's personal opinions and by no means represent the position
of the Xen community as a whole on this topic.

I, for one, have no idea what he is talking about.

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[H. Peter Anvin]

On 10/18/2012 09:44 AM, Stefano Stabellini wrote:
>
> I know that it is obvious but it is worth stating it in clear letters:
>
> these are Dan's personal opinions and by no means represent the position
> of the Xen community as a whole on this topic.
>
> I, for one, have no idea what he is talking about.
>

He is referring to the non-self-virtualizability of the pre-VT-x x86 
architecture; search for "Popek and Goldberg Virtualization Criteria". 
However, his response is misguided, because the issue at hand isn't the 
paravirtualization itself but the lack of documentation.

Paravirtualization creates a new platform, and that platform needs to be 
documented as much as any hardware platform.  Once that documentation 
exists it is possible to make a reasoned judgement if that platform can 
be unified with an existing platform like native x86.

However, the Xen platform is not documented in any useful way at all, 
and having "fun" little bits like this coming out of nowhere is just 
plain unacceptable.

Either way, it doesn't change the starting point of this -- we don't 
keep around hooks that aren't even used.  End of story.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Konrad Rzeszutek Wilk]

> As for 'perf', since Xen already provides a virtual PMU for HVM guests
> It's not clear why we would spend the effort to implement another
> mechanism for PV guests (instead of using the virtual PMU from a PVH guest).

Would that allow one to evaluate the performance/bottlenecks that the
hypervisor might have?

Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[David Vrabel]

On 18/10/12 18:42, Konrad Rzeszutek Wilk wrote:
>> As for 'perf', since Xen already provides a virtual PMU for HVM guests
>> It's not clear why we would spend the effort to implement another
>> mechanism for PV guests (instead of using the virtual PMU from a PVH guest).
> 
> Would that allow one to evaluate the performance/bottlenecks that the
> hypervisor might have?

Not right now, no. But I don't so why it couldn't be possible.

David

Is: Xen architecture document. Was: Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Konrad Rzeszutek Wilk]

> paravirtualized architectures out there which are perfectly well
> documented and supportable, but Xen has resisted doing that for
> years, and all we ever get are vague future promises.

There is no resistance - and it is being done. Every month we document
various APIs, man-pages, etc so that knowledge won't be lost. The
end-result of that is to create a PDF similar to the PowerPC architecture
document.

I have not been pointing you to it, b/c as of right what we have is
raw data (various header files growing in size) that keep on growing.
I need to hire/cajole an editor to help us get from the "raw" brain
dump to a book and some sense of milestones/schedule or whatever one
does in the book-world.

And Peter, just in case it is not clear, every suggestion you make is
appreciated and taken seriously. If there are things that have been
dropped or forgotten by me - please remind me.

Re: Is: Xen architecture document. Was: Re: [Xen-devel] Is: axe read_tscp pvops call. Was: Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[H. Peter Anvin]

On 10/19/2012 08:48 AM, Konrad Rzeszutek Wilk wrote:
>> paravirtualized architectures out there which are perfectly well
>> documented and supportable, but Xen has resisted doing that for
>> years, and all we ever get are vague future promises.
> 
> There is no resistance - and it is being done. Every month we document
> various APIs, man-pages, etc so that knowledge won't be lost. The
> end-result of that is to create a PDF similar to the PowerPC architecture
> document.
> 
> I have not been pointing you to it, b/c as of right what we have is
> raw data (various header files growing in size) that keep on growing.
> I need to hire/cajole an editor to help us get from the "raw" brain
> dump to a book and some sense of milestones/schedule or whatever one
> does in the book-world.
> 
> And Peter, just in case it is not clear, every suggestion you make is
> appreciated and taken seriously. If there are things that have been
> dropped or forgotten by me - please remind me.
> 

Right... so for those who don't know, this is something I have discussed
with Konrad over the course of the last several years.  It is good to
know progress is happening.  It is late, of course -- this should have
happened before Xen was integrated -- but we can't change the past.

	-hpa

Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 02:00:29PM -0400, Ben Guthro wrote:
> I'm not sure it matters, but I'm testing against a changeset about a week old:
> http://xenbits.xen.org/gitweb/?p=xen.git;a=commit;h=1c4a5b37b55c56e49135e65728137f54288d1fe6

I was able to reproduce it with Xen 4.2 so found the culprit.

.. And then another issue :-(

> 
> Plus patches specific to XenClient Enterprise.
> 
> 
> On Wed, Oct 17, 2012 at 1:43 PM, Konrad Rzeszutek Wilk
> <konrad.wilk@oracle.com> wrote:
> > On Wed, Oct 17, 2012 at 01:46:09PM -0400, Ben Guthro wrote:
> >> On Wed, Oct 17, 2012 at 9:49 AM, Konrad Rzeszutek Wilk
> >> <konrad.wilk@oracle.com> wrote:
> >> [...]
> >>
> >> > The end result is this is a nice set of patches where there is only
> >> > _one_ change in the x86 code (and it is just more of dealing with
> >> > error case) - and the rest are all done in Xen side.
> >>
> >> I'm sorry to report that this series doesn't seem to work in my setup
> >> against xen-unstable.
> >>
> >> To verify that it was, in fact this patch series, and not another Xen
> >> regression - I swapped out the kernel with this patch series, with an
> >> identical one, replacing only this series with your acpi-s3.v9 branch
> >> - and everything worked fine.
> >
> > Thanks for testing it!
> >
> > I had tested it with Xen 4.1.3, which could be doing something different.
> > Will see what is up.

Re: [RFC] ACPI S3 and Xen (suprisingly small\!).

[Ben Guthro]

ack.
I'm happy to test a 2nd round, if you CC me on any fixed patches (just
in case I'm not monitoring lkml / xen-devel on that particular day)

On Fri, Oct 19, 2012 at 2:49 PM, Konrad Rzeszutek Wilk
<konrad.wilk@oracle.com> wrote:
> On Wed, Oct 17, 2012 at 02:00:29PM -0400, Ben Guthro wrote:
>> I'm not sure it matters, but I'm testing against a changeset about a week old:
>> http://xenbits.xen.org/gitweb/?p=xen.git;a=commit;h=1c4a5b37b55c56e49135e65728137f54288d1fe6
>
> I was able to reproduce it with Xen 4.2 so found the culprit.
>
> .. And then another issue :-(
>
>>
>> Plus patches specific to XenClient Enterprise.
>>
>>
>> On Wed, Oct 17, 2012 at 1:43 PM, Konrad Rzeszutek Wilk
>> <konrad.wilk@oracle.com> wrote:
>> > On Wed, Oct 17, 2012 at 01:46:09PM -0400, Ben Guthro wrote:
>> >> On Wed, Oct 17, 2012 at 9:49 AM, Konrad Rzeszutek Wilk
>> >> <konrad.wilk@oracle.com> wrote:
>> >> [...]
>> >>
>> >> > The end result is this is a nice set of patches where there is only
>> >> > _one_ change in the x86 code (and it is just more of dealing with
>> >> > error case) - and the rest are all done in Xen side.
>> >>
>> >> I'm sorry to report that this series doesn't seem to work in my setup
>> >> against xen-unstable.
>> >>
>> >> To verify that it was, in fact this patch series, and not another Xen
>> >> regression - I swapped out the kernel with this patch series, with an
>> >> identical one, replacing only this series with your acpi-s3.v9 branch
>> >> - and everything worked fine.
>> >
>> > Thanks for testing it!
>> >
>> > I had tested it with Xen 4.1.3, which could be doing something different.
>> > Will see what is up.

Re: [PATCH 2/4] xen/lowlevel: Implement pvop call for load_idt (sidt).

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 04:51:17PM -0700, H. Peter Anvin wrote:
> On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> >In the past it used to point to 'sidt' (native_store_idt) operation
> >which is a non-privileged operation. This resulted in the
> >'struct desc_ptr' value containing the address of Xen's IDT table,
> >instead of the IDT table that Linux thinks its using. The end result
> >is that doing:
> >
> >   store_idt(&desc);
> >   load_idt(&desc);
> >
> >would blow up b/c xen_load_idt would try to parse the IDT contents
> >(desc) and de-reference a virtual address that is outside Linux's
> >__va (it is in Xen's virtual address).
> >
> >With this patch we are providing the last written IDT address.
> >
> 
> OK... this seems like another excellent set of pvops calls that
> should be nukable to Kingdom Come.  There is no reason, ever, to
> read the IDT and GDT from the kernel... the kernel already knows
> what they should be!

The code that uses these is "__save_processor_state" and 
"__restore_processor_state". To test the viability of removing
them, I did a very simple patch (see attached) and found
out that omitting those calls on AMD machines at least (hadn't
tried Intel yet) crashes the machine.

Interestingly enough, skipping the cr3, cr2, and cr0 loads in
arch/x86/kernel/acpi/wakeup_64.S worked fine!?


>From 7a7b1ea335f852f2a5ced1eb7f305fa329f7615a Mon Sep 17 00:00:00 2001
From: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Date: Wed, 16 Jan 2013 16:16:46 -0500
Subject: [PATCH] x86: ignore_idt, ignore_gdt, tss, cr3 knobs.

Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
---
 arch/x86/include/asm/suspend_64.h |  1 +
 arch/x86/kernel/acpi/sleep.c      | 19 +++++++++++++++-
 arch/x86/kernel/acpi/wakeup_64.S  |  4 ++++
 arch/x86/kernel/asm-offsets_64.c  |  1 +
 arch/x86/power/cpu.c              | 47 ++++++++++++++++++++++++++++++---------
 arch/x86/xen/enlighten.c          |  4 +++-
 drivers/xen/acpi.c                |  6 ++++-
 7 files changed, 69 insertions(+), 13 deletions(-)

diff --git a/arch/x86/include/asm/suspend_64.h b/arch/x86/include/asm/suspend_64.h
index 09b0bf1..15603b1 100644
--- a/arch/x86/include/asm/suspend_64.h
+++ b/arch/x86/include/asm/suspend_64.h
@@ -36,6 +36,7 @@ struct saved_context {
 	unsigned long tr;
 	unsigned long safety;
 	unsigned long return_address;
+	u8 skip;
 } __attribute__((packed));
 
 #define loaddebug(thread,register) \
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index d5e0d71..13c71e6 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -24,6 +24,9 @@ unsigned long acpi_realmode_flags;
 #if defined(CONFIG_SMP) && defined(CONFIG_64BIT)
 static char temp_stack[4096];
 #endif
+#include <asm/suspend_64.h>
+bool skip_cr3;
+extern struct saved_context saved_context;
 
 /**
  * acpi_suspend_lowlevel - save kernel state
@@ -82,10 +85,16 @@ int acpi_suspend_lowlevel(void)
        saved_magic = 0x123456789abcdef0L;
 #endif /* CONFIG_64BIT */
 
+	if (skip_cr3) {
+		printk(KERN_INFO "Skipping cr3\n");
+		saved_context.skip = 1;
+	}
 	do_suspend_lowlevel();
 	return 0;
 }
-
+bool ignore_idt;
+bool ignore_gdt;
+bool alternative_tss;
 static int __init acpi_sleep_setup(char *str)
 {
 	while ((str != NULL) && (*str != '\0')) {
@@ -105,6 +114,14 @@ static int __init acpi_sleep_setup(char *str)
 			acpi_nvs_nosave_s3();
 		if (strncmp(str, "old_ordering", 12) == 0)
 			acpi_old_suspend_ordering();
+		if (strncmp(str, "ignore_idt", 10) == 0)
+			ignore_idt = true;
+		if (strncmp(str, "ignore_gdt", 10) == 0)
+			ignore_idt = true;
+		if (strncmp(str, "tss", 3) == 0)
+			alternative_tss = true;
+		if (strncmp(str, "cr3", 3) == 0)
+			skip_cr3 = true;
 		str = strchr(str, ',');
 		if (str != NULL)
 			str += strspn(str, ", \t");
diff --git a/arch/x86/kernel/acpi/wakeup_64.S b/arch/x86/kernel/acpi/wakeup_64.S
index 8ea5164..c7e41c5 100644
--- a/arch/x86/kernel/acpi/wakeup_64.S
+++ b/arch/x86/kernel/acpi/wakeup_64.S
@@ -83,12 +83,16 @@ resume_point:
 	movq	$saved_context, %rax
 	movq	saved_context_cr4(%rax), %rbx
 	movq	%rbx, %cr4
+
+	cmp	$0x0, saved_context_skip(%rax)
+	jne	skip
 	movq	saved_context_cr3(%rax), %rbx
 	movq	%rbx, %cr3
 	movq	saved_context_cr2(%rax), %rbx
 	movq	%rbx, %cr2
 	movq	saved_context_cr0(%rax), %rbx
 	movq	%rbx, %cr0
+skip:
 	pushq	pt_regs_flags(%rax)
 	popfq
 	movq	pt_regs_sp(%rax), %rsp
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c
index 1b4754f..b7b7808 100644
--- a/arch/x86/kernel/asm-offsets_64.c
+++ b/arch/x86/kernel/asm-offsets_64.c
@@ -73,6 +73,7 @@ int main(void)
 	ENTRY(cr3);
 	ENTRY(cr4);
 	ENTRY(cr8);
+	ENTRY(skip);
 	BLANK();
 #undef ENTRY
 
diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
index c8ece7d..b4b2436 100644
--- a/arch/x86/power/cpu.c
+++ b/arch/x86/power/cpu.c
@@ -35,6 +35,10 @@ unsigned long saved_context_eflags;
 struct saved_context saved_context;
 #endif
 
+extern bool ignore_idt;
+extern bool ignore_gdt;
+extern bool alternative_tss;
+
 /**
  *	__save_processor_state - save CPU registers before creating a
  *		hibernation image and before restoring the memory state from it
@@ -61,12 +65,16 @@ static void __save_processor_state(struct saved_context *ctxt)
 	 * descriptor tables
 	 */
 #ifdef CONFIG_X86_32
-	store_gdt(&ctxt->gdt);
-	store_idt(&ctxt->idt);
+	if (!ignore_gdt)
+		store_gdt(&ctxt->gdt);
+	if (!ignore_idt)
+		store_idt(&ctxt->idt);
 #else
 /* CONFIG_X86_64 */
-	store_gdt((struct desc_ptr *)&ctxt->gdt_limit);
-	store_idt((struct desc_ptr *)&ctxt->idt_limit);
+	if (!ignore_gdt)
+		store_gdt((struct desc_ptr *)&ctxt->gdt_limit);
+	if (!ignore_idt)
+		store_idt((struct desc_ptr *)&ctxt->idt_limit);
 #endif
 	store_tr(ctxt->tr);
 
@@ -136,6 +144,7 @@ static void fix_processor_context(void)
 	struct tss_struct *t = &per_cpu(init_tss, cpu);
 #ifdef CONFIG_X86_64
 	struct desc_struct *desc = get_cpu_gdt_table(cpu);
+	tss_desc tss;
 #endif
 	set_tss_desc(cpu, t);	/*
 				 * This just modifies memory; should not be
@@ -145,13 +154,23 @@ static void fix_processor_context(void)
 				 */
 
 #ifdef CONFIG_X86_64
-	if (!desc_empty(&desc[GDT_ENTRY_TSS]))
-		desc[GDT_ENTRY_TSS].type = DESC_TSS;
+	if (alternative_tss) {
+ 		memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
+		tss.type = DESC_TSS;
+		write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
+	} else {
+		if (!desc_empty(&desc[GDT_ENTRY_TSS])) {
+			desc[GDT_ENTRY_TSS].type = DESC_TSS;
+		}
+	}
 
 	syscall_init();				/* This sets MSR_*STAR and related */
 #endif
+	outb('G', 0xe9);
 	load_TR_desc();				/* This does ltr */
+	outb('H', 0xe9);
 	load_LDT(&current->active_mm->context);	/* This does lldt */
+	outb('I', 0xe9);
 }
 
 /**
@@ -185,12 +204,16 @@ static void __restore_processor_state(struct saved_context *ctxt)
 	 * ltr is done i fix_processor_context().
 	 */
 #ifdef CONFIG_X86_32
-	load_gdt(&ctxt->gdt);
-	load_idt(&ctxt->idt);
+	if (!ignore_gdt)
+		load_gdt(&ctxt->gdt);
+	if (!ignore_idt)
+		load_idt(&ctxt->idt);
 #else
 /* CONFIG_X86_64 */
-	load_gdt((const struct desc_ptr *)&ctxt->gdt_limit);
-	load_idt((const struct desc_ptr *)&ctxt->idt_limit);
+	if (!ignore_gdt)
+		load_gdt((const struct desc_ptr *)&ctxt->gdt_limit);
+	if (!ignore_idt)
+		load_idt((const struct desc_ptr *)&ctxt->idt_limit);
 #endif
 
 	/*
@@ -228,9 +251,13 @@ static void __restore_processor_state(struct saved_context *ctxt)
 
 	fix_processor_context();
 
+	outb('P', 0xe9);
 	do_fpu_end();
+	outb('Q', 0xe9);
 	x86_platform.restore_sched_clock_state();
+	outb('R', 0xe9);
 	mtrr_bp_restore();
+	outb('S', 0xe9);
 }
 
 /* Needed by apm.c */
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index dfe1332..8be90ea 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1035,6 +1035,7 @@ static void xen_write_cr4(unsigned long cr4)
 	native_write_cr4(cr4);
 }
 #ifdef CONFIG_X86_64
+extern bool skip_cr3;
 static inline unsigned long xen_read_cr8(void)
 {
 	return 0;
@@ -1094,7 +1095,8 @@ static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
  		 * xen_read_cr3 will try to find the cr3 for the user-space
  		 * case - and feed it to the hypercall (which would fail).
  		 */
-		xen_acpi_reload_cr3_value();
+		if (!skip_cr3)
+			xen_acpi_reload_cr3_value();
 #endif
 	default:
 		ret = native_write_msr_safe(msr, low, high);
diff --git a/drivers/xen/acpi.c b/drivers/xen/acpi.c
index a97414d..2ac4aac 100644
--- a/drivers/xen/acpi.c
+++ b/drivers/xen/acpi.c
@@ -56,6 +56,8 @@ void xen_acpi_reload_cr3_value(void)
 	saved_context.cr3 = read_cr3();
 }
 #endif
+extern bool skip_cr3;
+
 int xen_acpi_notify_hypervisor_state(u8 sleep_state,
 				     u32 pm1a_cnt, u32 pm1b_cnt)
 {
@@ -98,7 +100,9 @@ int xen_acpi_notify_hypervisor_state(u8 sleep_state,
 		mfn = pfn_to_mfn(PFN_DOWN(saved_context.cr3));
 		/* and back to a physical address */
 		mfn = PFN_PHYS(mfn);
-		saved_context.cr3 = mfn;
+
+		if (!skip_cr3)
+			saved_context.cr3 = mfn;
 
 		/* Sadly, this has the end result that if we the resume code
  		 * does the movq X, %cr3 and then later uses the X value to do
-- 
1.8.0.2

> 
> 	-hpa
> 
> 
> -- 
> H. Peter Anvin, Intel Open Source Technology Center
> I work for Intel.  I don't speak on their behalf.

Re: [PATCH 1/4] x86/wakeup/sleep: Check whether the TSS GDT descriptor is empty before using it.

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 05:03:09PM -0700, H. Peter Anvin wrote:
> On 10/17/2012 06:49 AM, Konrad Rzeszutek Wilk wrote:
> >We check the TSS descriptor before we try to dereference it.
> >Also fix up the value to use the #defines.
> >
> >Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
> >---
> >  arch/x86/power/cpu.c |    7 +++++--
> >  1 files changed, 5 insertions(+), 2 deletions(-)
> >
> >diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
> >index 218cdb1..c17370e 100644
> >--- a/arch/x86/power/cpu.c
> >+++ b/arch/x86/power/cpu.c
> >@@ -133,7 +133,9 @@ static void fix_processor_context(void)
> >  {
> >  	int cpu = smp_processor_id();
> >  	struct tss_struct *t = &per_cpu(init_tss, cpu);
> >-
> >+#ifdef CONFIG_X86_64
> >+	struct desc_struct *desc = get_cpu_gdt_table(cpu);
> >+#endif
> >  	set_tss_desc(cpu, t);	/*
> >  				 * This just modifies memory; should not be
> >  				 * necessary. But... This is necessary, because
> >@@ -142,7 +144,8 @@ static void fix_processor_context(void)
> >  				 */
> >
> >  #ifdef CONFIG_X86_64
> >-	get_cpu_gdt_table(cpu)[GDT_ENTRY_TSS].type = 9;
> >+	if (!desc_empty(&desc[GDT_ENTRY_TSS]))
> >+		desc[GDT_ENTRY_TSS].type = DESC_TSS;
> >
> >  	syscall_init();				/* This sets MSR_*STAR and related */
> >  #endif
> >
> 
> Why is this patch necessary?  Presumably there is something further
> down the line which depends on the TSS descriptor being empty, but
> if so, what?

Ah, so at least on Xen, that desc is marked as RO b/c it has been given
to the hypervisor. And the hypervisor adds its own entries. Lastly on
64-bit, the TSS for PV guests is not used - as the PV kernel and user-space
both run in the user-space ring. So there is nothing in that entry.

The issue I stumbled upon was just a page-fault b/c of trying to modify
a RO region. The other way of fixing this (without knowing that the
TSS entry is not set), was to use the functions/macros, as such:


        if (alternative_tss) {
                memcpy(&tss, &desc[GDT_ENTRY_TSS], sizeof(tss_desc));
                tss.type = DESC_TSS;
                write_gdt_entry(desc, GDT_ENTRY_TSS, &tss, DESC_TSS);
        } else {
                if (!desc_empty(&desc[GDT_ENTRY_TSS])) {
                        desc[GDT_ENTRY_TSS].type = DESC_TSS;
                }   
        }   

which works as well and does not throw the person from trying to figure
out why the TSS descriptor is empty (or not).

> 
> 	-hpa
> 
> -- 
> H. Peter Anvin, Intel Open Source Technology Center
> I work for Intel.  I don't speak on their behalf.

Re: [PATCH 4/4] xen/acpi: Prep saved_context cr3 values.

[Konrad Rzeszutek Wilk]

On Wed, Oct 17, 2012 at 09:49:46AM -0400, Konrad Rzeszutek Wilk wrote:
> When save_processor_state is executed it executes a bunch of
> pvops calls to save the CPU state values. When it comes back
> from Xen's S3 (so acpi_enter_sleep_state, which ends up calling
> xen_acpi_notify_hypervisor_state), it ends up back in the
> assembler code in wakeup_[32|64].S. It skips the wakeup
> calls (so wakeup_pmode_return and wakeup_long64) as that has
> been done in the hypervisor. Instead it continues on in
> the resume_point (64-bit) or ret_point (32-bit). Most of the
> calls in there are safe to be executed except when it comes to
> reloading of cr3 (which it only does on 64-bit kernels). Since
> they are native assembler calls and Xen expects a PV kernel to
> prep those to use machine address for cr3 that is what
> we are going to do. Note: that it is not Machine Frame Numbers
> (those are used in the MMUEXT_NEW_BASEPTR hypercall for cr3
> installation) but the machine physical address.
> 
> When the assembler code executes this mov %ebx, cr3 it ends
> end up trapped in the hypervisor (traps.c) which properly now
> sets the cr3 value.

And then I found out that after this nice
	 mov %ebx,cr3

in the assembler code (resume_point), it ends up calling
__restore_processor_state later on which does:

	write_cr3(ctxt->cr3);

and since that is a pvops call which expects physical address and in
this patch we  modified the ctxt->cr3 to be a machine address value, we end
up giving the hypervisor the wrong value.

This workaround makes it work.. but it is the ugly green-puke spotted
duct-tape variant.

So I think this idea of modifying the ctxt->cr3 to without modifying
the resume_point is a no-go. Len suggested at some point doing this
in the resume point:

-- a/arch/x86/kernel/acpi/wakeup_64.S
+++ b/arch/x86/kernel/acpi/wakeup_64.S
@@ -83,12 +83,16 @@ resume_point:
        movq    $saved_context, %rax
        movq    saved_context_cr4(%rax), %rbx
        movq    %rbx, %cr4
+
+       cmp     $0x0, saved_context_skip(%rax)
+       jne     skip
        movq    saved_context_cr3(%rax), %rbx
        movq    %rbx, %cr3
        movq    saved_context_cr2(%rax), %rbx
        movq    %rbx, %cr2
        movq    saved_context_cr0(%rax), %rbx
        movq    %rbx, %cr0
+skip:
        pushq   pt_regs_flags(%rax)
        popfq
        movq    pt_regs_sp(%rax), %rsp

and that makes it work too. Surprisingly it also makes it work on
baremetal! (Note: Only tested right now on AMD).

anyhow, here is the duct-tape:

commit 09194f091d1eaef7b1aac0289f46acd7cc8c0845
Author: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Date:   Fri Oct 19 13:41:10 2012 -0400

    xen/acpi: Workaround movq X, %cr3 and write_cr3 pvops call using same value.
    
    This is a quirk to work-around the discontinuity of the
    x86_lowlevel_suspend code on x86_64. In it, after calling
    acpi_suspend it calls resume_point, which restores the registers
    and one of them is a movq X, %cr3. The movq X in that case needs
    to be machine address. Later on it calls to  restore_processor_state()
    which uses the pvops variant (write_cr3) - which expects X to be
    physical address. This function restores the cr3 value to be a
    physical address to allow the pvops variant (write_cr3) to work.
    
    Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index fd1f3dd..dfe1332 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1082,7 +1082,20 @@ static int xen_write_msr_safe(unsigned int msr, unsigned low, unsigned high)
 		if (smp_processor_id() == 0)
 			xen_set_pat(((u64)high << 32) | low);
 		break;
-
+#if defined(CONFIG_X86_64)
+	case MSR_EFER:
+		/* Piggyback on the fact that in powers/cpu.c we do
+ 		 * a wrmsr before the paravirt write_cr3. The cr3 value at that
+ 		 * stage in saved_context.cr3 is a machine address instead of
+ 		 * the physical address (this is done in drivers/xen/acpi.c to
+ 		 * compensate for 'return_point' in wakeup_64.S doing an:
+ 		 * movw %ebx, cr3). Anyhow, we piggy back here to reload the
+ 		 * cr3 value of the saved_context. This is done b/c otherwise
+ 		 * xen_read_cr3 will try to find the cr3 for the user-space
+ 		 * case - and feed it to the hypercall (which would fail).
+ 		 */
+		xen_acpi_reload_cr3_value();
+#endif
 	default:
 		ret = native_write_msr_safe(msr, low, high);
 	}
diff --git a/drivers/xen/acpi.c b/drivers/xen/acpi.c
index 25e612c..a97414d 100644
--- a/drivers/xen/acpi.c
+++ b/drivers/xen/acpi.c
@@ -40,8 +40,22 @@
 #ifdef CONFIG_X86_64
 #include <asm/suspend_64.h>
 extern struct saved_context saved_context;
-#endif
 
+/*
+ * This is a quirk to work-around the discontinuity of the
+ * x86_lowlevel_suspend code on x86_64. In it, after calling
+ * acpi_suspend it calls resume_point, which restores the registers
+ * and one of them is a movq X, %cr3. The movq X in that case needs
+ * to be machine address. Later on it calls to  restore_processor_state()
+ * which uses the pvops variant (write_cr3) - which expects X to be
+ * physical address. This function restores the cr3 value to be a
+ * physical address to allow the pvops variant (write_cr3) to work.
+ */
+void xen_acpi_reload_cr3_value(void)
+{
+	saved_context.cr3 = read_cr3();
+}
+#endif
 int xen_acpi_notify_hypervisor_state(u8 sleep_state,
 				     u32 pm1a_cnt, u32 pm1b_cnt)
 {
@@ -71,6 +85,11 @@ int xen_acpi_notify_hypervisor_state(u8 sleep_state,
 	{
 		unsigned long mfn;
 
+		/* Flushes out any multi-calls. */
+		arch_flush_lazy_mmu_mode();
+
+		/* Re-reads the CR3 in case of pending multicalls */
+		saved_context.cr3 = read_cr3();
 		/* resume_point in wakeup_64.s barrels through and does:
 		 * movq    saved_context_cr3(%rax), %rbx
 		 * movq    %rbx, %cr3
@@ -80,6 +99,14 @@ int xen_acpi_notify_hypervisor_state(u8 sleep_state,
 		/* and back to a physical address */
 		mfn = PFN_PHYS(mfn);
 		saved_context.cr3 = mfn;
+
+		/* Sadly, this has the end result that if we the resume code
+ 		 * does the movq X, %cr3 and then later uses the X value to do
+ 		 * an pvops call (write_cr3), we have a discontinuity.
+ 		 * The movq expects a machine frame address while the pvops call
+ 		 * expects a physical frame address. We fix this up with
+ 		 * xen_acpi_reload_cr3_quirk which we put in wrmsr code.
+ 		 */
 	}
 #endif
 	HYPERVISOR_dom0_op(&op);
diff --git a/include/xen/acpi.h b/include/xen/acpi.h
index 48a9c01..ccf26f1 100644
--- a/include/xen/acpi.h
+++ b/include/xen/acpi.h
@@ -42,7 +42,9 @@
 
 int xen_acpi_notify_hypervisor_state(u8 sleep_state,
 				     u32 pm1a_cnt, u32 pm1b_cnd);
-
+#ifdef CONFIG_X86_64
+void xen_acpi_reload_cr3_value(void);
+#endif
 static inline void xen_acpi_sleep_register(void)
 {
 	if (xen_initial_domain())
@@ -53,6 +55,11 @@ static inline void xen_acpi_sleep_register(void)
 static inline void xen_acpi_sleep_register(void)
 {
 }
+#ifdef CONFIG_X86_64
+static inline void xen_acpi_reload_cr3_value(void)
+{
+}
+#endif
 #endif
 
 #endif	/* _XEN_ACPI_H */