All of lore.kernel.org
 help / color / mirror / Atom feed
* x86: kvmclock: abstract save/restore sched_clock_state
@ 2012-02-07 21:05 Marcelo Tosatti
  2012-02-08 10:53 ` Igor Mammedov
                   ` (3 more replies)
  0 siblings, 4 replies; 24+ messages in thread
From: Marcelo Tosatti @ 2012-02-07 21:05 UTC (permalink / raw)
  To: kvm; +Cc: tglx, mingo, hpa, x86, johnstul, riel, avi, imammedo


Upon resume from hibernation, CPU 0's hvclock area contains the old
values for system_time and tsc_timestamp. It is necessary for the
hypervisor to update these values with uptodate ones before the CPU uses
them.

Abstract TSC's save/restore sched_clock_state functions and use
restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.

Fixes suspend-to-disk with kvmclock.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index 15d9915..c91e8b9 100644
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -61,7 +61,7 @@ extern void check_tsc_sync_source(int cpu);
 extern void check_tsc_sync_target(void);
 
 extern int notsc_setup(char *);
-extern void save_sched_clock_state(void);
-extern void restore_sched_clock_state(void);
+extern void tsc_save_sched_clock_state(void);
+extern void tsc_restore_sched_clock_state(void);
 
 #endif /* _ASM_X86_TSC_H */
diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index 5d0afac..baaca8d 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -162,6 +162,8 @@ struct x86_cpuinit_ops {
  * @is_untracked_pat_range	exclude from PAT logic
  * @nmi_init			enable NMI on cpus
  * @i8042_detect		pre-detect if i8042 controller exists
+ * @save_sched_clock_state:	save state for sched_clock() on suspend
+ * @restore_sched_clock_state:	restore state for sched_clock() on resume
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -173,6 +175,8 @@ struct x86_platform_ops {
 	void (*nmi_init)(void);
 	unsigned char (*get_nmi_reason)(void);
 	int (*i8042_detect)(void);
+	void (*save_sched_clock_state)(void);
+	void (*restore_sched_clock_state)(void);
 };
 
 struct pci_dev;
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index ca4e735..57e6b78 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -136,6 +136,15 @@ int kvm_register_clock(char *txt)
 	return ret;
 }
 
+void kvm_save_sched_clock_state(void)
+{
+}
+
+void kvm_restore_sched_clock_state(void)
+{
+	kvm_register_clock("primary cpu clock, resume");
+}
+
 #ifdef CONFIG_X86_LOCAL_APIC
 static void __cpuinit kvm_setup_secondary_clock(void)
 {
@@ -195,6 +204,8 @@ void __init kvmclock_init(void)
 	x86_cpuinit.early_percpu_clock_init =
 		kvm_setup_secondary_clock;
 #endif
+	x86_platform.save_sched_clock_state = kvm_save_sched_clock_state;
+	x86_platform.restore_sched_clock_state = kvm_restore_sched_clock_state;
 	machine_ops.shutdown  = kvm_shutdown;
 #ifdef CONFIG_KEXEC
 	machine_ops.crash_shutdown  = kvm_crash_shutdown;
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index a62c201..aed2aa1 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -629,7 +629,7 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
 
 static unsigned long long cyc2ns_suspend;
 
-void save_sched_clock_state(void)
+void tsc_save_sched_clock_state(void)
 {
 	if (!sched_clock_stable)
 		return;
@@ -645,7 +645,7 @@ void save_sched_clock_state(void)
  * that sched_clock() continues from the point where it was left off during
  * suspend.
  */
-void restore_sched_clock_state(void)
+void tsc_restore_sched_clock_state(void)
 {
 	unsigned long long offset;
 	unsigned long flags;
diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
index 6f2ec53..e9f265f 100644
--- a/arch/x86/kernel/x86_init.c
+++ b/arch/x86/kernel/x86_init.c
@@ -108,7 +108,9 @@ struct x86_platform_ops x86_platform = {
 	.is_untracked_pat_range		= is_ISA_range,
 	.nmi_init			= default_nmi_init,
 	.get_nmi_reason			= default_get_nmi_reason,
-	.i8042_detect			= default_i8042_detect
+	.i8042_detect			= default_i8042_detect,
+	.save_sched_clock_state 	= tsc_save_sched_clock_state,
+	.restore_sched_clock_state 	= tsc_restore_sched_clock_state,
 };
 
 EXPORT_SYMBOL_GPL(x86_platform);
diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
index f10c0af..ca5f79a 100644
--- a/arch/x86/power/cpu.c
+++ b/arch/x86/power/cpu.c
@@ -114,7 +114,7 @@ static void __save_processor_state(struct saved_context *ctxt)
 void save_processor_state(void)
 {
 	__save_processor_state(&saved_context);
-	save_sched_clock_state();
+	x86_platform.save_sched_clock_state();
 }
 #ifdef CONFIG_X86_32
 EXPORT_SYMBOL(save_processor_state);
@@ -231,7 +231,7 @@ static void __restore_processor_state(struct saved_context *ctxt)
 void restore_processor_state(void)
 {
 	__restore_processor_state(&saved_context);
-	restore_sched_clock_state();
+	x86_platform.restore_sched_clock_state();
 }
 #ifdef CONFIG_X86_32
 EXPORT_SYMBOL(restore_processor_state);


^ permalink raw reply related	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-07 21:05 x86: kvmclock: abstract save/restore sched_clock_state Marcelo Tosatti
@ 2012-02-08 10:53 ` Igor Mammedov
  2012-02-09 12:27 ` Amit Shah
                   ` (2 subsequent siblings)
  3 siblings, 0 replies; 24+ messages in thread
From: Igor Mammedov @ 2012-02-08 10:53 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On 02/07/2012 10:05 PM, Marcelo Tosatti wrote:
>
> Upon resume from hibernation, CPU 0's hvclock area contains the old
> values for system_time and tsc_timestamp. It is necessary for the
> hypervisor to update these values with uptodate ones before the CPU uses
> them.
>
> Abstract TSC's save/restore sched_clock_state functions and use
> restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.
>
> Fixes suspend-to-disk with kvmclock.
>
> Signed-off-by: Marcelo Tosatti<mtosatti@redhat.com>
>

Acked-by: Igor Mammedov <imammedo@redhat.com>

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-07 21:05 x86: kvmclock: abstract save/restore sched_clock_state Marcelo Tosatti
  2012-02-08 10:53 ` Igor Mammedov
@ 2012-02-09 12:27 ` Amit Shah
  2012-02-09 15:13   ` Igor Mammedov
  2012-02-13 13:07 ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Marcelo Tosatti
  2012-03-01  9:58 ` x86: kvmclock: abstract save/restore sched_clock_state Thomas Gleixner
  3 siblings, 1 reply; 24+ messages in thread
From: Amit Shah @ 2012-02-09 12:27 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: kvm, tglx, mingo, hpa, x86, johnstul, riel, avi, imammedo

On (Tue) 07 Feb 2012 [19:05:42], Marcelo Tosatti wrote:
> 
> Upon resume from hibernation, CPU 0's hvclock area contains the old
> values for system_time and tsc_timestamp. It is necessary for the
> hypervisor to update these values with uptodate ones before the CPU uses
> them.
> 
> Abstract TSC's save/restore sched_clock_state functions and use
> restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.
> 
> Fixes suspend-to-disk with kvmclock.

There are stalls after resume, see trace below.

./x86_64-softmmu/qemu-system-x86_64 -kernel ~/src/linux/arch/x86/boot/bzImage  -append 'root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend' -drive file=/guests/f14-suspend.qcow2,if=none,id=dr0 -device virtio-blk-pci,drive=dr0 -net nic,model=virtio -net user  -serial stdio  -enable-kvm -m 512  -cpu host -smp 4

<The following is on resume:>

Disabling non-boot CPUs ...
CPU 1 is now offline
CPU 2 is now offline
CPU 3 is now offline
lockdep: fixing up alternatives.
kvm-clock: cpu 0, msr 0:1f1d19c1, primary cpu clock, resume
PM: Restoring platform NVS memory
Enabling non-boot CPUs ...
lockdep: fixing up alternatives.
Booting Node 0 Processor 1 APIC 0x1
Calibrating delay loop (skipped) already calibrated this CPU
INFO: rcu_sched detected stalls on CPUs/tasks: { 1} (detected by 0, t=314192 jiffies)
Pid: 662, comm: bash Not tainted 3.3.0-rc2+ #293
Call Trace:
 <IRQ>  [<ffffffff8108b6c9>] __rcu_pending+0x268/0x3c5
 [<ffffffff8108bef5>] rcu_check_callbacks+0xab/0x108
 [<ffffffff81031208>] update_process_times+0x3f/0x75
 [<ffffffff810649b5>] tick_sched_timer+0x6d/0x8c
 [<ffffffff81042e68>] __run_hrtimer+0xc9/0x15c
 [<ffffffff81064948>] ? tick_nohz_handler+0xd5/0xd5
 [<ffffffff8104359b>] hrtimer_interrupt+0xea/0x1b4
 [<ffffffff81316192>] smp_apic_timer_interrupt+0x76/0x89
 [<ffffffff81315833>] apic_timer_interrupt+0x73/0x80
 <EOI>  [<ffffffff8130b9ec>] ? arch_local_irq_restore+0x6/0xd
 [<ffffffff8130840c>] native_cpu_up+0x108/0x127
 [<ffffffff81308f2b>] _cpu_up+0x92/0xfc
 [<ffffffff812facf7>] enable_nonboot_cpus+0x4d/0xb2
 [<ffffffff81059836>] hibernation_snapshot+0x1f3/0x2a0
 [<ffffffff81044406>] ? cleanup_srcu_struct+0x52/0x58
 [<ffffffff81059e6d>] hibernate+0x97/0x196
 [<ffffffff81057eb0>] state_store+0x5c/0x106
 [<ffffffff811af49b>] kobj_attr_store+0x17/0x19
 [<ffffffff811368fa>] sysfs_write_file+0x10e/0x14a
 [<ffffffff810eb5f6>] vfs_write+0xab/0xd2
 [<ffffffff810ec955>] ? fget_light+0x3a/0xa1
 [<ffffffff810eb7ad>] sys_write+0x4d/0x74
 [<ffffffff81314ce9>] system_call_fastpath+0x16/0x1b
INFO: rcu_preempt detected stalls on CPUs/tasks: { 1} (detected by 0, t=314193 jiffies)
Pid: 662, comm: bash Not tainted 3.3.0-rc2+ #293
Call Trace:
 <IRQ>  [<ffffffff8108b6c9>] __rcu_pending+0x268/0x3c5
 [<ffffffff8108bf33>] rcu_check_callbacks+0xe9/0x108
 [<ffffffff81031208>] update_process_times+0x3f/0x75
 [<ffffffff810649b5>] tick_sched_timer+0x6d/0x8c
 [<ffffffff81042e68>] __run_hrtimer+0xc9/0x15c
 [<ffffffff81064948>] ? tick_nohz_handler+0xd5/0xd5
 [<ffffffff8104359b>] hrtimer_interrupt+0xea/0x1b4
 [<ffffffff81316192>] smp_apic_timer_interrupt+0x76/0x89
 [<ffffffff81315833>] apic_timer_interrupt+0x73/0x80
 <EOI>  [<ffffffff8130b9ec>] ? arch_local_irq_restore+0x6/0xd
 [<ffffffff8130840c>] native_cpu_up+0x108/0x127
 [<ffffffff81308f2b>] _cpu_up+0x92/0xfc
 [<ffffffff812facf7>] enable_nonboot_cpus+0x4d/0xb2
 [<ffffffff81059836>] hibernation_snapshot+0x1f3/0x2a0
 [<ffffffff81044406>] ? cleanup_srcu_struct+0x52/0x58
 [<ffffffff81059e6d>] hibernate+0x97/0x196
 [<ffffffff81057eb0>] state_store+0x5c/0x106
 [<ffffffff811af49b>] kobj_attr_store+0x17/0x19
 [<ffffffff811368fa>] sysfs_write_file+0x10e/0x14a
 [<ffffffff810eb5f6>] vfs_write+0xab/0xd2
 [<ffffffff810ec955>] ? fget_light+0x3a/0xa1
 [<ffffffff810eb7ad>] sys_write+0x4d/0x74
 [<ffffffff81314ce9>] system_call_fastpath+0x16/0x1b
INFO: rcu_sched detected stalls on CPUs/tasks: { 1} (detected by 0, t=1713579840232 jiffies)
Pid: 662, comm: bash Not tainted 3.3.0-rc2+ #293
Call Trace:
 <IRQ>  [<ffffffff8108b6c9>] __rcu_pending+0x268/0x3c5
 [<ffffffff8108bef5>] rcu_check_callbacks+0xab/0x108
 [<ffffffff81031208>] update_process_times+0x3f/0x75
 [<ffffffff810649b5>] tick_sched_timer+0x6d/0x8c
 [<ffffffff81042e68>] __run_hrtimer+0xc9/0x15c
 [<ffffffff81064948>] ? tick_nohz_handler+0xd5/0xd5
 [<ffffffff8104359b>] hrtimer_interrupt+0xea/0x1b4
 [<ffffffff81316192>] smp_apic_timer_interrupt+0x76/0x89
 [<ffffffff81315833>] apic_timer_interrupt+0x73/0x80
 <EOI>  [<ffffffff8130b9ec>] ? arch_local_irq_restore+0x6/0xd
 [<ffffffff8130840c>] native_cpu_up+0x108/0x127
 [<ffffffff81308f2b>] _cpu_up+0x92/0xfc
 [<ffffffff812facf7>] enable_nonboot_cpus+0x4d/0xb2
 [<ffffffff81059836>] hibernation_snapshot+0x1f3/0x2a0
 [<ffffffff81044406>] ? cleanup_srcu_struct+0x52/0x58
 [<ffffffff81059e6d>] hibernate+0x97/0x196
 [<ffffffff81057eb0>] state_store+0x5c/0x106
 [<ffffffff811af49b>] kobj_attr_store+0x17/0x19
 [<ffffffff811368fa>] sysfs_write_file+0x10e/0x14a
 [<ffffffff810eb5f6>] vfs_write+0xab/0xd2
 [<ffffffff810ec955>] ? fget_light+0x3a/0xa1
 [<ffffffff810eb7ad>] sys_write+0x4d/0x74
 [<ffffffff81314ce9>] system_call_fastpath+0x16/0x1b
INFO: rcu_preempt detected stalls on CPUs/tasks: { 1} (detected by 0, t=1713579840233 jiffies)
Pid: 662, comm: bash Not tainted 3.3.0-rc2+ #293
Call Trace:
 <IRQ>  [<ffffffff8108b6c9>] __rcu_pending+0x268/0x3c5
 [<ffffffff8108bf33>] rcu_check_callbacks+0xe9/0x108
 [<ffffffff81031208>] update_process_times+0x3f/0x75
 [<ffffffff810649b5>] tick_sched_timer+0x6d/0x8c
 [<ffffffff81042e68>] __run_hrtimer+0xc9/0x15c
 [<ffffffff81064948>] ? tick_nohz_handler+0xd5/0xd5
 [<ffffffff8104359b>] hrtimer_interrupt+0xea/0x1b4
 [<ffffffff81316192>] smp_apic_timer_interrupt+0x76/0x89
 [<ffffffff81315833>] apic_timer_interrupt+0x73/0x80
 <EOI>  [<ffffffff8130b9ec>] ? arch_local_irq_restore+0x6/0xd
 [<ffffffff8130840c>] native_cpu_up+0x108/0x127
 [<ffffffff81308f2b>] _cpu_up+0x92/0xfc
 [<ffffffff812facf7>] enable_nonboot_cpus+0x4d/0xb2
 [<ffffffff81059836>] hibernation_snapshot+0x1f3/0x2a0
 [<ffffffff81044406>] ? cleanup_srcu_struct+0x52/0x58
 [<ffffffff81059e6d>] hibernate+0x97/0x196
 [<ffffffff81057eb0>] state_store+0x5c/0x106
 [<ffffffff811af49b>] kobj_attr_store+0x17/0x19
 [<ffffffff811368fa>] sysfs_write_file+0x10e/0x14a
 [<ffffffff810eb5f6>] vfs_write+0xab/0xd2
 [<ffffffff810ec955>] ? fget_light+0x3a/0xa1
 [<ffffffff810eb7ad>] sys_write+0x4d/0x74
 [<ffffffff81314ce9>] system_call_fastpath+0x16/0x1b


		Amit

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-09 12:27 ` Amit Shah
@ 2012-02-09 15:13   ` Igor Mammedov
  2012-02-10 10:02     ` Amit Shah
  0 siblings, 1 reply; 24+ messages in thread
From: Igor Mammedov @ 2012-02-09 15:13 UTC (permalink / raw)
  To: Amit Shah
  Cc: Marcelo Tosatti, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On 02/09/2012 01:27 PM, Amit Shah wrote:
> On (Tue) 07 Feb 2012 [19:05:42], Marcelo Tosatti wrote:
>>
>> Upon resume from hibernation, CPU 0's hvclock area contains the old
>> values for system_time and tsc_timestamp. It is necessary for the
>> hypervisor to update these values with uptodate ones before the CPU uses
>> them.
>>
>> Abstract TSC's save/restore sched_clock_state functions and use
>> restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.
>>
>> Fixes suspend-to-disk with kvmclock.
>
> There are stalls after resume, see trace below.
>
> ./x86_64-softmmu/qemu-system-x86_64 -kernel ~/src/linux/arch/x86/boot/bzImage  -append 'root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend' -drive file=/guests/f14-suspend.qcow2,if=none,id=dr0 -device virtio-blk-pci,drive=dr0 -net nic,model=virtio -net user  -serial stdio  -enable-kvm -m 512  -cpu host -smp 4
>
> <The following is on resume:>
>
> Disabling non-boot CPUs ...
> CPU 1 is now offline
> CPU 2 is now offline
> CPU 3 is now offline
> lockdep: fixing up alternatives.
> kvm-clock: cpu 0, msr 0:1f1d19c1, primary cpu clock, resume
> PM: Restoring platform NVS memory
> Enabling non-boot CPUs ...
> lockdep: fixing up alternatives.
> Booting Node 0 Processor 1 APIC 0x1
> Calibrating delay loop (skipped) already calibrated this CPU
> INFO: rcu_sched detected stalls on CPUs/tasks: { 1} (detected by 0, t=314192 jiffies)
> Pid: 662, comm: bash Not tainted 3.3.0-rc2+ #293
> Call Trace:
>   <IRQ>   [<ffffffff8108b6c9>] __rcu_pending+0x268/0x3c5
>   [<ffffffff8108bef5>] rcu_check_callbacks+0xab/0x108
>   [<ffffffff81031208>] update_process_times+0x3f/0x75
>   [<ffffffff810649b5>] tick_sched_timer+0x6d/0x8c
>   [<ffffffff81042e68>] __run_hrtimer+0xc9/0x15c
>   [<ffffffff81064948>] ? tick_nohz_handler+0xd5/0xd5
>   [<ffffffff8104359b>] hrtimer_interrupt+0xea/0x1b4
>   [<ffffffff81316192>] smp_apic_timer_interrupt+0x76/0x89
>   [<ffffffff81315833>] apic_timer_interrupt+0x73/0x80
>   <EOI>   [<ffffffff8130b9ec>] ? arch_local_irq_restore+0x6/0xd
>   [<ffffffff8130840c>] native_cpu_up+0x108/0x127
>   [<ffffffff81308f2b>] _cpu_up+0x92/0xfc
>   [<ffffffff812facf7>] enable_nonboot_cpus+0x4d/0xb2
>   [<ffffffff81059836>] hibernation_snapshot+0x1f3/0x2a0
>   [<ffffffff81044406>] ? cleanup_srcu_struct+0x52/0x58
>   [<ffffffff81059e6d>] hibernate+0x97/0x196
>   [<ffffffff81057eb0>] state_store+0x5c/0x106
>   [<ffffffff811af49b>] kobj_attr_store+0x17/0x19
>   [<ffffffff811368fa>] sysfs_write_file+0x10e/0x14a
>   [<ffffffff810eb5f6>] vfs_write+0xab/0xd2
>   [<ffffffff810ec955>] ? fget_light+0x3a/0xa1
>   [<ffffffff810eb7ad>] sys_write+0x4d/0x74
>   [<ffffffff81314ce9>] system_call_fastpath+0x16/0x1b
> INFO: rcu_preempt detected stalls on CPUs/tasks: { 1} (detected by 0, t=314193 jiffies)
> Pid: 662, comm: bash Not tainted 3.3.0-rc2+ #293
> Call Trace:
>   <IRQ>   [<ffffffff8108b6c9>] __rcu_pending+0x268/0x3c5
>   [<ffffffff8108bf33>] rcu_check_callbacks+0xe9/0x108
>   [<ffffffff81031208>] update_process_times+0x3f/0x75
>   [<ffffffff810649b5>] tick_sched_timer+0x6d/0x8c
>   [<ffffffff81042e68>] __run_hrtimer+0xc9/0x15c
>   [<ffffffff81064948>] ? tick_nohz_handler+0xd5/0xd5
>   [<ffffffff8104359b>] hrtimer_interrupt+0xea/0x1b4
>   [<ffffffff81316192>] smp_apic_timer_interrupt+0x76/0x89
>   [<ffffffff81315833>] apic_timer_interrupt+0x73/0x80
>   <EOI>   [<ffffffff8130b9ec>] ? arch_local_irq_restore+0x6/0xd
>   [<ffffffff8130840c>] native_cpu_up+0x108/0x127
>   [<ffffffff81308f2b>] _cpu_up+0x92/0xfc
>   [<ffffffff812facf7>] enable_nonboot_cpus+0x4d/0xb2
>   [<ffffffff81059836>] hibernation_snapshot+0x1f3/0x2a0
>   [<ffffffff81044406>] ? cleanup_srcu_struct+0x52/0x58
>   [<ffffffff81059e6d>] hibernate+0x97/0x196
>   [<ffffffff81057eb0>] state_store+0x5c/0x106
>   [<ffffffff811af49b>] kobj_attr_store+0x17/0x19
>   [<ffffffff811368fa>] sysfs_write_file+0x10e/0x14a
>   [<ffffffff810eb5f6>] vfs_write+0xab/0xd2
>   [<ffffffff810ec955>] ? fget_light+0x3a/0xa1
>   [<ffffffff810eb7ad>] sys_write+0x4d/0x74
>   [<ffffffff81314ce9>] system_call_fastpath+0x16/0x1b
> INFO: rcu_sched detected stalls on CPUs/tasks: { 1} (detected by 0, t=1713579840232 jiffies)
> Pid: 662, comm: bash Not tainted 3.3.0-rc2+ #293
> Call Trace:
>   <IRQ>   [<ffffffff8108b6c9>] __rcu_pending+0x268/0x3c5
>   [<ffffffff8108bef5>] rcu_check_callbacks+0xab/0x108
>   [<ffffffff81031208>] update_process_times+0x3f/0x75
>   [<ffffffff810649b5>] tick_sched_timer+0x6d/0x8c
>   [<ffffffff81042e68>] __run_hrtimer+0xc9/0x15c
>   [<ffffffff81064948>] ? tick_nohz_handler+0xd5/0xd5
>   [<ffffffff8104359b>] hrtimer_interrupt+0xea/0x1b4
>   [<ffffffff81316192>] smp_apic_timer_interrupt+0x76/0x89
>   [<ffffffff81315833>] apic_timer_interrupt+0x73/0x80
>   <EOI>   [<ffffffff8130b9ec>] ? arch_local_irq_restore+0x6/0xd
>   [<ffffffff8130840c>] native_cpu_up+0x108/0x127
>   [<ffffffff81308f2b>] _cpu_up+0x92/0xfc
>   [<ffffffff812facf7>] enable_nonboot_cpus+0x4d/0xb2
>   [<ffffffff81059836>] hibernation_snapshot+0x1f3/0x2a0
>   [<ffffffff81044406>] ? cleanup_srcu_struct+0x52/0x58
>   [<ffffffff81059e6d>] hibernate+0x97/0x196
>   [<ffffffff81057eb0>] state_store+0x5c/0x106
>   [<ffffffff811af49b>] kobj_attr_store+0x17/0x19
>   [<ffffffff811368fa>] sysfs_write_file+0x10e/0x14a
>   [<ffffffff810eb5f6>] vfs_write+0xab/0xd2
>   [<ffffffff810ec955>] ? fget_light+0x3a/0xa1
>   [<ffffffff810eb7ad>] sys_write+0x4d/0x74
>   [<ffffffff81314ce9>] system_call_fastpath+0x16/0x1b
> INFO: rcu_preempt detected stalls on CPUs/tasks: { 1} (detected by 0, t=1713579840233 jiffies)
> Pid: 662, comm: bash Not tainted 3.3.0-rc2+ #293
> Call Trace:
>   <IRQ>   [<ffffffff8108b6c9>] __rcu_pending+0x268/0x3c5
>   [<ffffffff8108bf33>] rcu_check_callbacks+0xe9/0x108
>   [<ffffffff81031208>] update_process_times+0x3f/0x75
>   [<ffffffff810649b5>] tick_sched_timer+0x6d/0x8c
>   [<ffffffff81042e68>] __run_hrtimer+0xc9/0x15c
>   [<ffffffff81064948>] ? tick_nohz_handler+0xd5/0xd5
>   [<ffffffff8104359b>] hrtimer_interrupt+0xea/0x1b4
>   [<ffffffff81316192>] smp_apic_timer_interrupt+0x76/0x89
>   [<ffffffff81315833>] apic_timer_interrupt+0x73/0x80
>   <EOI>   [<ffffffff8130b9ec>] ? arch_local_irq_restore+0x6/0xd
>   [<ffffffff8130840c>] native_cpu_up+0x108/0x127
>   [<ffffffff81308f2b>] _cpu_up+0x92/0xfc
>   [<ffffffff812facf7>] enable_nonboot_cpus+0x4d/0xb2
>   [<ffffffff81059836>] hibernation_snapshot+0x1f3/0x2a0
>   [<ffffffff81044406>] ? cleanup_srcu_struct+0x52/0x58
>   [<ffffffff81059e6d>] hibernate+0x97/0x196
>   [<ffffffff81057eb0>] state_store+0x5c/0x106
>   [<ffffffff811af49b>] kobj_attr_store+0x17/0x19
>   [<ffffffff811368fa>] sysfs_write_file+0x10e/0x14a
>   [<ffffffff810eb5f6>] vfs_write+0xab/0xd2
>   [<ffffffff810ec955>] ? fget_light+0x3a/0xa1
>   [<ffffffff810eb7ad>] sys_write+0x4d/0x74
>   [<ffffffff81314ce9>] system_call_fastpath+0x16/0x1b
>
>
> 		Amit

Stalls are probably caused by uninitialized percpu hv_clock, with
following patch I don't see stalls. Although I might be just lucky.
http://git.kernel.org/?p=virt/kvm/kvm.git;a=commit;h=e2971ac7e1d186af059e088d305496c5cb47d487

However there is/are a warning/s on suspend path and with following patch:
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
index 66c74f4..c8e7587 100644
--- a/arch/x86/kernel/smp.c
+++ b/arch/x86/kernel/smp.c
@@ -117,6 +117,7 @@
  static void native_smp_send_reschedule(int cpu)
  {
         if (unlikely(cpu_is_offline(cpu))) {
+               printk ("curr_cpu: %d, for cpu: %d\n", smp_processor_id(), cpu);
                 WARN_ON(1);
                 return;
         }


[    3.101919] Unregister pv shared memory for cpu 1
[    3.207321] CPU 1 is now offline
[    3.210177] curr_cpu: 0, for cpu: 1
[    3.211172] ------------[ cut here ]------------
[    3.211172] WARNING: at arch/x86/kernel/smp.c:121 native_smp_send_reschedule+0x40/0x71()
[    3.211172] Hardware name: Bochs
[    3.211172] Modules linked in: floppy virtio_blk
[    3.211172] Pid: 264, comm: sh Not tainted 3.3.0-rc1+ #42
[    3.211172] Call Trace:
[    3.211172]  <IRQ>  [<ffffffff81045ecd>] warn_slowpath_common+0x83/0x9b
[    3.211172]  [<ffffffff81045eff>] warn_slowpath_null+0x1a/0x1c
[    3.211172]  [<ffffffff81026dba>] native_smp_send_reschedule+0x40/0x71
[    3.211172]  [<ffffffff810749d6>] trigger_load_balance+0x28e/0x2ba
[    3.211172]  [<ffffffff8106e7ec>] scheduler_tick+0xe9/0xf2
[    3.211172]  [<ffffffff81052db8>] update_process_times+0x65/0x76
[    3.211172]  [<ffffffff81088219>] tick_sched_timer+0x75/0x9f
[    3.211172]  [<ffffffff81064056>] __run_hrtimer+0xb1/0x155
[    3.211172]  [<ffffffff810881a4>] ? tick_nohz_handler+0xdd/0xdd
[    3.211172]  [<ffffffff8106476c>] hrtimer_interrupt+0xe0/0x19e
[    3.211172]  [<ffffffff814d847c>] smp_apic_timer_interrupt+0x77/0x8a
[    3.211172]  [<ffffffff814d7133>] apic_timer_interrupt+0x73/0x80
[    3.211172]  <EOI>  [<ffffffff814ba14f>] ? cpuid4_cache_sysfs_exit+0x94/0xad
[    3.211172]  [<ffffffff8111ebe3>] ? cmpxchg_double_slab+0x6/0xef
[    3.211172]  [<ffffffff814c9afc>] __slab_free+0xe6/0x1b7
[    3.211172]  [<ffffffff8111eed2>] kmem_cache_free+0x72/0xac
[    3.211172]  [<ffffffff814ba14f>] ? cpuid4_cache_sysfs_exit+0x94/0xad
[    3.211172]  [<ffffffff81186d23>] release_sysfs_dirent+0x9b/0xb9
[    3.211172]  [<ffffffff81186d59>] __sysfs_put+0x18/0x1a
[    3.211172]  [<ffffffff81187257>] sysfs_addrm_finish+0x8a/0xa2
[    3.211172]  [<ffffffff81185d2b>] sysfs_hash_and_remove+0x5c/0x71
[    3.211172]  [<ffffffff8118674d>] sysfs_remove_file+0x34/0x3b
[    3.211172]  [<ffffffff812ee9da>] device_remove_file+0x17/0x19
[    3.211172]  [<ffffffff814bcaff>] mce_cpu_callback+0xa3/0x186
[    3.211172]  [<ffffffff814d31c0>] notifier_call_chain+0x37/0x63
[    3.211172]  [<ffffffff810655b6>] __raw_notifier_call_chain+0xe/0x10
[    3.211172]  [<ffffffff81047758>] __cpu_notify+0x20/0x32
[    3.211172]  [<ffffffff81047863>] cpu_notify_nofail+0x13/0x1b
[    3.211172]  [<ffffffff814ae6b3>] _cpu_down+0x13a/0x21f
[    3.211172]  [<ffffffff81047915>] disable_nonboot_cpus+0x66/0x11e
[    3.211172]  [<ffffffff8107bff1>] hibernation_restore+0x78/0x106
[    3.211172]  [<ffffffff8107c25f>] software_resume+0x1e0/0x249
[    3.211172]  [<ffffffff8107c353>] resume_store+0x8b/0x97
[    3.211172]  [<ffffffff8122f23b>] kobj_attr_store+0x17/0x19
[    3.211172]  [<ffffffff81186183>] sysfs_write_file+0x101/0x13d
[    3.211172]  [<ffffffff8112f2e3>] vfs_write+0xac/0xf3
[    3.211172]  [<ffffffff810caf1d>] ? trace_hardirqs_on_caller+0xf/0x24
[    3.211172]  [<ffffffff8112f4d2>] sys_write+0x4a/0x6e
[    3.211172]  [<ffffffff814d6629>] system_call_fastpath+0x16/0x1b
[    3.211172] ---[ end trace 5aa7abdf47c70330 ]---
[    3.276748] Unregister pv shared memory for cpu 2
[    3.280771] CPU 2 is now offline
[    3.285070] Unregister pv shared memory for cpu 3
[    3.288206] CPU 3 is now offline
[   31.290110] kvm-clock: cpu 0, msr 0:1fc13341, primary cpu clock, resume
[   31.290110] Enabling non-boot CPUs ...
[   31.291319] Booting Node 0 Processor 1 APIC 0x1
[   31.293720] smpboot cpu 1: start_ip = 96000
[   31.172141] Calibrating delay loop (skipped) already calibrated this CPUcurr_cpu: 0, for cpu: 2
[   31.330037] ------------[ cut here ]------------
[   31.330094] WARNING: at arch/x86/kernel/smp.c:121 native_smp_send_reschedule+0x40/0x71()
[   31.330094] Hardware name: Bochs
[   31.330094] Modules linked in: lockd fcoe libfcoe 8021q libfc scsi_transport_fc scsi_tgt garp stp llc ip6t_REJECT nf_conntrack_ipv6 nf_defrag_ipv6 
nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack ip6table_filter ip6_tables e1000 pcspkr uinput i2c_piix4 i2c_core ppdev parport_pc parport 
microcode sunrpc floppy virtio_blk [last unloaded: scsi_wait_scan]
[   31.330094] Pid: 1016, comm: bash Tainted: G        W    3.3.0-rc1+ #42
[   31.330094] Call Trace:
[   31.330094]  <IRQ>  [<ffffffff81045ecd>] warn_slowpath_common+0x83/0x9b
[   31.330094]  [<ffffffff81045eff>] warn_slowpath_null+0x1a/0x1c
[   31.330094]  [<ffffffff81026dba>] native_smp_send_reschedule+0x40/0x71
[   31.330094]  [<ffffffff810749d6>] trigger_load_balance+0x28e/0x2ba
[   31.330094]  [<ffffffff8106e7ec>] scheduler_tick+0xe9/0xf2
[   31.330094]  [<ffffffff81052db8>] update_process_times+0x65/0x76
[   31.330094]  [<ffffffff81088219>] tick_sched_timer+0x75/0x9f
[   31.330094]  [<ffffffff81064056>] __run_hrtimer+0xb1/0x155
[   31.330094]  [<ffffffff810881a4>] ? tick_nohz_handler+0xdd/0xdd
[   31.330094]  [<ffffffff8106476c>] hrtimer_interrupt+0xe0/0x19e
[   31.330094]  [<ffffffff814d847c>] smp_apic_timer_interrupt+0x77/0x8a
[   31.330094]  [<ffffffff814d7133>] apic_timer_interrupt+0x73/0x80
[   31.330094]  <EOI>  [<ffffffff814cffe1>] ? retint_restore_args+0x13/0x13
[   31.330094]  [<ffffffff8108d42a>] ? arch_local_irq_restore+0x6/0xd
[   31.330094]  [<ffffffff814cfd88>] _raw_spin_unlock_irqrestore+0x3d/0x3f
[   31.330094]  [<ffffffff814b1bd3>] workqueue_cpu_callback+0x203/0x214
[   31.330094]  [<ffffffff814d31c0>] notifier_call_chain+0x37/0x63
[   31.330094]  [<ffffffff814c6b68>] ? arch_local_irq_restore+0x6/0xd
[   31.330094]  [<ffffffff810655b6>] __raw_notifier_call_chain+0xe/0x10
[   31.330094]  [<ffffffff81047758>] __cpu_notify+0x20/0x32
[   31.330094]  [<ffffffff814bfb99>] _cpu_up+0xcd/0xfc
[   31.330094]  [<ffffffff814ae88b>] enable_nonboot_cpus+0x57/0xc4
[   31.330094]  [<ffffffff8107becc>] hibernation_snapshot+0x1f3/0x2a0
[   31.330094]  [<ffffffff8107c503>] hibernate+0x97/0x198
[   31.330094]  [<ffffffff8107a613>] state_store+0x5c/0x106
[   31.330094]  [<ffffffff8122f23b>] kobj_attr_store+0x17/0x19
[   31.330094]  [<ffffffff81186183>] sysfs_write_file+0x101/0x13d
[   31.330094]  [<ffffffff8112f2e3>] vfs_write+0xac/0xf3
[   31.330094]  [<ffffffff810caf1d>] ? trace_hardirqs_on_caller+0xf/0x24
[   31.330094]  [<ffffffff8112f4d2>] sys_write+0x4a/0x6e
[   31.330094]  [<ffffffff814d6629>] system_call_fastpath+0x16/0x1b
[   31.330094] ---[ end trace bbd3f0319d3e2c44 ]---
[   31.402829] KVM setup async PF for cpu 1
[   31.402829] kvm-stealtime: cpu 1, msr 1fc8df00
[   31.656224] NMI watchdog disabled (cpu1): hardware events not enabled
[   31.659240] CPU1 is up
[   31.660402] Booting Node 0 Processor 2 APIC 0x2
[   31.661522] smpboot cpu 2: start_ip = 96000
[   31.278595] Calibrating delay loop (skipped) already calibrated this CPU
[   31.697981] NMI watchdog disabled (cpu2): hardware events not enabled
[   31.698047] KVM setup async PF for cpu 2
[   31.698051] kvm-stealtime: cpu 2, msr 1fd0df00
[   31.706326] CPU2 is up

however I think they are not related to subj patch.



^ permalink raw reply related	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-09 15:13   ` Igor Mammedov
@ 2012-02-10 10:02     ` Amit Shah
  2012-02-10 10:11       ` Igor Mammedov
  2012-02-10 12:32       ` Marcelo Tosatti
  0 siblings, 2 replies; 24+ messages in thread
From: Amit Shah @ 2012-02-10 10:02 UTC (permalink / raw)
  To: Igor Mammedov
  Cc: Marcelo Tosatti, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On (Thu) 09 Feb 2012 [16:13:29], Igor Mammedov wrote:

> Stalls are probably caused by uninitialized percpu hv_clock, with
> following patch I don't see stalls. Although I might be just lucky.
> http://git.kernel.org/?p=virt/kvm/kvm.git;a=commit;h=e2971ac7e1d186af059e088d305496c5cb47d487

Your commit does make things better, I don't see any stalls on the
first resume.

However, a subsequent s4 causes the stall to re-appear on resume, and
this time there are no stall messages; the kernel just sits there
spinning on something.  I've not found the solution to this one yet (I
had a commit similar to Marcelo's in the works, which got me to the
previous works-but-stalls behaviour).

> However there is/are a warning/s on suspend path and with following patch:

I didn't see this.

		Amit

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 10:02     ` Amit Shah
@ 2012-02-10 10:11       ` Igor Mammedov
  2012-02-10 10:23         ` Amit Shah
  2012-02-10 12:32       ` Marcelo Tosatti
  1 sibling, 1 reply; 24+ messages in thread
From: Igor Mammedov @ 2012-02-10 10:11 UTC (permalink / raw)
  To: Amit Shah
  Cc: Marcelo Tosatti, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

Could you send me your .config and commit id of kernel you are using?

----- Original Message -----
> From: "Amit Shah" <amit.shah@redhat.com>
> To: "Igor Mammedov" <imammedo@redhat.com>
> Cc: "Marcelo Tosatti" <mtosatti@redhat.com>, kvm@vger.kernel.org, tglx@linutronix.de, mingo@redhat.com,
> hpa@zytor.com, x86@kernel.org, johnstul@us.ibm.com, riel@redhat.com, avi@redhat.com
> Sent: Friday, February 10, 2012 11:02:11 AM
> Subject: Re: x86: kvmclock: abstract save/restore sched_clock_state
> 
> On (Thu) 09 Feb 2012 [16:13:29], Igor Mammedov wrote:
> 
> > Stalls are probably caused by uninitialized percpu hv_clock, with
> > following patch I don't see stalls. Although I might be just lucky.
> > http://git.kernel.org/?p=virt/kvm/kvm.git;a=commit;h=e2971ac7e1d186af059e088d305496c5cb47d487
> 
> Your commit does make things better, I don't see any stalls on the
> first resume.
> 
> However, a subsequent s4 causes the stall to re-appear on resume, and
> this time there are no stall messages; the kernel just sits there
> spinning on something.  I've not found the solution to this one yet
> (I
> had a commit similar to Marcelo's in the works, which got me to the
> previous works-but-stalls behaviour).
> 
> > However there is/are a warning/s on suspend path and with following
> > patch:
> 
> I didn't see this.
> 
> 		Amit
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 10:11       ` Igor Mammedov
@ 2012-02-10 10:23         ` Amit Shah
  0 siblings, 0 replies; 24+ messages in thread
From: Amit Shah @ 2012-02-10 10:23 UTC (permalink / raw)
  To: Igor Mammedov
  Cc: Marcelo Tosatti, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On (Fri) 10 Feb 2012 [05:11:00], Igor Mammedov wrote:
> Could you send me your .config and commit id of kernel you are using?

Kernel's based on bd3ce7d57c380af110c86d19e256115d0e7053ca plus your
commit + Marcelo's patch.

config is attached below.

#
# Automatically generated file; DO NOT EDIT.
# Linux/x86_64 3.3.0-rc2 Kernel Configuration
#
CONFIG_64BIT=y
# CONFIG_X86_32 is not set
CONFIG_X86_64=y
CONFIG_X86=y
CONFIG_INSTRUCTION_DECODER=y
CONFIG_OUTPUT_FORMAT="elf64-x86-64"
CONFIG_ARCH_DEFCONFIG="arch/x86/configs/x86_64_defconfig"
CONFIG_GENERIC_CMOS_UPDATE=y
CONFIG_CLOCKSOURCE_WATCHDOG=y
CONFIG_GENERIC_CLOCKEVENTS=y
CONFIG_ARCH_CLOCKSOURCE_DATA=y
CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y
CONFIG_LOCKDEP_SUPPORT=y
CONFIG_STACKTRACE_SUPPORT=y
CONFIG_HAVE_LATENCYTOP_SUPPORT=y
CONFIG_MMU=y
CONFIG_NEED_DMA_MAP_STATE=y
CONFIG_NEED_SG_DMA_LENGTH=y
CONFIG_GENERIC_ISA_DMA=y
CONFIG_GENERIC_BUG=y
CONFIG_GENERIC_BUG_RELATIVE_POINTERS=y
CONFIG_GENERIC_HWEIGHT=y
CONFIG_ARCH_MAY_HAVE_PC_FDC=y
# CONFIG_RWSEM_GENERIC_SPINLOCK is not set
CONFIG_RWSEM_XCHGADD_ALGORITHM=y
CONFIG_ARCH_HAS_CPU_IDLE_WAIT=y
CONFIG_GENERIC_CALIBRATE_DELAY=y
CONFIG_GENERIC_TIME_VSYSCALL=y
CONFIG_ARCH_HAS_CPU_RELAX=y
CONFIG_ARCH_HAS_DEFAULT_IDLE=y
CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
CONFIG_HAVE_SETUP_PER_CPU_AREA=y
CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK=y
CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK=y
CONFIG_ARCH_HIBERNATION_POSSIBLE=y
CONFIG_ARCH_SUSPEND_POSSIBLE=y
CONFIG_ZONE_DMA32=y
CONFIG_AUDIT_ARCH=y
CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y
CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
CONFIG_X86_64_SMP=y
CONFIG_X86_HT=y
CONFIG_ARCH_HWEIGHT_CFLAGS="-fcall-saved-rdi -fcall-saved-rsi -fcall-saved-rdx -fcall-saved-rcx -fcall-saved-r8 -fcall-saved-r9 -fcall-saved-r10 -fcall-saved-r11"
# CONFIG_KTIME_SCALAR is not set
CONFIG_ARCH_CPU_PROBE_RELEASE=y
CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
CONFIG_HAVE_IRQ_WORK=y
CONFIG_IRQ_WORK=y

#
# General setup
#
CONFIG_EXPERIMENTAL=y
CONFIG_INIT_ENV_ARG_LIMIT=32
CONFIG_CROSS_COMPILE=""
CONFIG_LOCALVERSION=""
# CONFIG_LOCALVERSION_AUTO is not set
CONFIG_HAVE_KERNEL_GZIP=y
CONFIG_HAVE_KERNEL_BZIP2=y
CONFIG_HAVE_KERNEL_LZMA=y
CONFIG_HAVE_KERNEL_XZ=y
CONFIG_HAVE_KERNEL_LZO=y
# CONFIG_KERNEL_GZIP is not set
# CONFIG_KERNEL_BZIP2 is not set
CONFIG_KERNEL_LZMA=y
# CONFIG_KERNEL_XZ is not set
# CONFIG_KERNEL_LZO is not set
CONFIG_DEFAULT_HOSTNAME="virthost"
CONFIG_SWAP=y
CONFIG_SYSVIPC=y
CONFIG_SYSVIPC_SYSCTL=y
CONFIG_POSIX_MQUEUE=y
CONFIG_POSIX_MQUEUE_SYSCTL=y
CONFIG_BSD_PROCESS_ACCT=y
# CONFIG_BSD_PROCESS_ACCT_V3 is not set
# CONFIG_FHANDLE is not set
# CONFIG_TASKSTATS is not set
CONFIG_AUDIT=y
CONFIG_AUDITSYSCALL=y
CONFIG_AUDIT_WATCH=y
CONFIG_AUDIT_TREE=y
# CONFIG_AUDIT_LOGINUID_IMMUTABLE is not set
CONFIG_HAVE_GENERIC_HARDIRQS=y

#
# IRQ subsystem
#
CONFIG_GENERIC_HARDIRQS=y
CONFIG_HAVE_SPARSE_IRQ=y
CONFIG_GENERIC_IRQ_PROBE=y
CONFIG_GENERIC_IRQ_SHOW=y
CONFIG_GENERIC_PENDING_IRQ=y
CONFIG_IRQ_FORCED_THREADING=y
CONFIG_SPARSE_IRQ=y

#
# RCU Subsystem
#
CONFIG_TREE_PREEMPT_RCU=y
CONFIG_PREEMPT_RCU=y
# CONFIG_RCU_TRACE is not set
CONFIG_RCU_FANOUT=64
# CONFIG_RCU_FANOUT_EXACT is not set
# CONFIG_RCU_FAST_NO_HZ is not set
# CONFIG_TREE_RCU_TRACE is not set
# CONFIG_RCU_BOOST is not set
# CONFIG_IKCONFIG is not set
CONFIG_LOG_BUF_SHIFT=17
CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y
CONFIG_CGROUPS=y
# CONFIG_CGROUP_DEBUG is not set
# CONFIG_CGROUP_FREEZER is not set
# CONFIG_CGROUP_DEVICE is not set
# CONFIG_CPUSETS is not set
# CONFIG_CGROUP_CPUACCT is not set
# CONFIG_RESOURCE_COUNTERS is not set
# CONFIG_CGROUP_PERF is not set
CONFIG_CGROUP_SCHED=y
CONFIG_FAIR_GROUP_SCHED=y
# CONFIG_CFS_BANDWIDTH is not set
# CONFIG_RT_GROUP_SCHED is not set
# CONFIG_BLK_CGROUP is not set
# CONFIG_CHECKPOINT_RESTORE is not set
CONFIG_NAMESPACES=y
CONFIG_UTS_NS=y
CONFIG_IPC_NS=y
CONFIG_USER_NS=y
CONFIG_PID_NS=y
CONFIG_NET_NS=y
CONFIG_SCHED_AUTOGROUP=y
# CONFIG_SYSFS_DEPRECATED is not set
CONFIG_RELAY=y
CONFIG_BLK_DEV_INITRD=y
CONFIG_INITRAMFS_SOURCE=""
CONFIG_RD_GZIP=y
CONFIG_RD_BZIP2=y
CONFIG_RD_LZMA=y
CONFIG_RD_XZ=y
CONFIG_RD_LZO=y
CONFIG_CC_OPTIMIZE_FOR_SIZE=y
CONFIG_SYSCTL=y
CONFIG_ANON_INODES=y
# CONFIG_EXPERT is not set
# CONFIG_SYSCTL_SYSCALL is not set
CONFIG_KALLSYMS=y
CONFIG_KALLSYMS_ALL=y
CONFIG_HOTPLUG=y
CONFIG_PRINTK=y
CONFIG_BUG=y
CONFIG_ELF_CORE=y
CONFIG_PCSPKR_PLATFORM=y
CONFIG_HAVE_PCSPKR_PLATFORM=y
CONFIG_BASE_FULL=y
CONFIG_FUTEX=y
CONFIG_EPOLL=y
CONFIG_SIGNALFD=y
CONFIG_TIMERFD=y
CONFIG_EVENTFD=y
CONFIG_SHMEM=y
CONFIG_AIO=y
# CONFIG_EMBEDDED is not set
CONFIG_HAVE_PERF_EVENTS=y

#
# Kernel Performance Events And Counters
#
CONFIG_PERF_EVENTS=y
# CONFIG_PERF_COUNTERS is not set
# CONFIG_DEBUG_PERF_USE_VMALLOC is not set
CONFIG_VM_EVENT_COUNTERS=y
CONFIG_PCI_QUIRKS=y
CONFIG_SLUB_DEBUG=y
# CONFIG_COMPAT_BRK is not set
# CONFIG_SLAB is not set
CONFIG_SLUB=y
# CONFIG_PROFILING is not set
CONFIG_TRACEPOINTS=y
CONFIG_HAVE_OPROFILE=y
CONFIG_OPROFILE_NMI_TIMER=y
# CONFIG_KPROBES is not set
# CONFIG_JUMP_LABEL is not set
CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y
CONFIG_HAVE_IOREMAP_PROT=y
CONFIG_HAVE_KPROBES=y
CONFIG_HAVE_KRETPROBES=y
CONFIG_HAVE_OPTPROBES=y
CONFIG_HAVE_ARCH_TRACEHOOK=y
CONFIG_HAVE_DMA_ATTRS=y
CONFIG_USE_GENERIC_SMP_HELPERS=y
CONFIG_HAVE_REGS_AND_STACK_ACCESS_API=y
CONFIG_HAVE_DMA_API_DEBUG=y
CONFIG_HAVE_HW_BREAKPOINT=y
CONFIG_HAVE_MIXED_BREAKPOINTS_REGS=y
CONFIG_HAVE_USER_RETURN_NOTIFIER=y
CONFIG_HAVE_PERF_EVENTS_NMI=y
CONFIG_HAVE_ARCH_JUMP_LABEL=y
CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG=y
CONFIG_HAVE_ALIGNED_STRUCT_PAGE=y
CONFIG_HAVE_CMPXCHG_LOCAL=y
CONFIG_HAVE_CMPXCHG_DOUBLE=y

#
# GCOV-based kernel profiling
#
# CONFIG_GCOV_KERNEL is not set
# CONFIG_HAVE_GENERIC_DMA_COHERENT is not set
CONFIG_SLABINFO=y
CONFIG_RT_MUTEXES=y
CONFIG_BASE_SMALL=0
CONFIG_MODULES=y
# CONFIG_MODULE_FORCE_LOAD is not set
CONFIG_MODULE_UNLOAD=y
# CONFIG_MODULE_FORCE_UNLOAD is not set
# CONFIG_MODVERSIONS is not set
CONFIG_MODULE_SRCVERSION_ALL=y
CONFIG_STOP_MACHINE=y
CONFIG_BLOCK=y
CONFIG_BLK_DEV_BSG=y
# CONFIG_BLK_DEV_BSGLIB is not set
CONFIG_BLK_DEV_INTEGRITY=y

#
# Partition Types
#
CONFIG_PARTITION_ADVANCED=y
# CONFIG_ACORN_PARTITION is not set
# CONFIG_OSF_PARTITION is not set
# CONFIG_AMIGA_PARTITION is not set
# CONFIG_ATARI_PARTITION is not set
# CONFIG_MAC_PARTITION is not set
CONFIG_MSDOS_PARTITION=y
CONFIG_BSD_DISKLABEL=y
CONFIG_MINIX_SUBPARTITION=y
CONFIG_SOLARIS_X86_PARTITION=y
CONFIG_UNIXWARE_DISKLABEL=y
# CONFIG_LDM_PARTITION is not set
# CONFIG_SGI_PARTITION is not set
# CONFIG_ULTRIX_PARTITION is not set
# CONFIG_SUN_PARTITION is not set
# CONFIG_KARMA_PARTITION is not set
CONFIG_EFI_PARTITION=y
# CONFIG_SYSV68_PARTITION is not set

#
# IO Schedulers
#
CONFIG_IOSCHED_NOOP=y
# CONFIG_IOSCHED_DEADLINE is not set
# CONFIG_IOSCHED_CFQ is not set
CONFIG_DEFAULT_NOOP=y
CONFIG_DEFAULT_IOSCHED="noop"
# CONFIG_INLINE_SPIN_TRYLOCK is not set
# CONFIG_INLINE_SPIN_TRYLOCK_BH is not set
# CONFIG_INLINE_SPIN_LOCK is not set
# CONFIG_INLINE_SPIN_LOCK_BH is not set
# CONFIG_INLINE_SPIN_LOCK_IRQ is not set
# CONFIG_INLINE_SPIN_LOCK_IRQSAVE is not set
# CONFIG_INLINE_SPIN_UNLOCK is not set
# CONFIG_INLINE_SPIN_UNLOCK_BH is not set
# CONFIG_INLINE_SPIN_UNLOCK_IRQ is not set
# CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE is not set
# CONFIG_INLINE_READ_TRYLOCK is not set
# CONFIG_INLINE_READ_LOCK is not set
# CONFIG_INLINE_READ_LOCK_BH is not set
# CONFIG_INLINE_READ_LOCK_IRQ is not set
# CONFIG_INLINE_READ_LOCK_IRQSAVE is not set
# CONFIG_INLINE_READ_UNLOCK is not set
# CONFIG_INLINE_READ_UNLOCK_BH is not set
# CONFIG_INLINE_READ_UNLOCK_IRQ is not set
# CONFIG_INLINE_READ_UNLOCK_IRQRESTORE is not set
# CONFIG_INLINE_WRITE_TRYLOCK is not set
# CONFIG_INLINE_WRITE_LOCK is not set
# CONFIG_INLINE_WRITE_LOCK_BH is not set
# CONFIG_INLINE_WRITE_LOCK_IRQ is not set
# CONFIG_INLINE_WRITE_LOCK_IRQSAVE is not set
# CONFIG_INLINE_WRITE_UNLOCK is not set
# CONFIG_INLINE_WRITE_UNLOCK_BH is not set
# CONFIG_INLINE_WRITE_UNLOCK_IRQ is not set
# CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE is not set
# CONFIG_MUTEX_SPIN_ON_OWNER is not set
CONFIG_FREEZER=y

#
# Processor type and features
#
CONFIG_ZONE_DMA=y
CONFIG_TICK_ONESHOT=y
CONFIG_NO_HZ=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_GENERIC_CLOCKEVENTS_BUILD=y
CONFIG_GENERIC_CLOCKEVENTS_MIN_ADJUST=y
CONFIG_SMP=y
# CONFIG_X86_MPPARSE is not set
# CONFIG_X86_EXTENDED_PLATFORM is not set
CONFIG_SCHED_OMIT_FRAME_POINTER=y
CONFIG_PARAVIRT_GUEST=y
# CONFIG_PARAVIRT_TIME_ACCOUNTING is not set
# CONFIG_XEN is not set
# CONFIG_XEN_PRIVILEGED_GUEST is not set
CONFIG_KVM_CLOCK=y
# CONFIG_KVM_GUEST is not set
CONFIG_PARAVIRT=y
# CONFIG_PARAVIRT_SPINLOCKS is not set
CONFIG_PARAVIRT_CLOCK=y
# CONFIG_PARAVIRT_DEBUG is not set
CONFIG_NO_BOOTMEM=y
# CONFIG_MEMTEST is not set
# CONFIG_MK8 is not set
# CONFIG_MPSC is not set
CONFIG_MCORE2=y
# CONFIG_MATOM is not set
# CONFIG_GENERIC_CPU is not set
CONFIG_X86_INTERNODE_CACHE_SHIFT=6
CONFIG_X86_CMPXCHG=y
CONFIG_X86_L1_CACHE_SHIFT=6
CONFIG_X86_XADD=y
CONFIG_X86_WP_WORKS_OK=y
CONFIG_X86_INTEL_USERCOPY=y
CONFIG_X86_USE_PPRO_CHECKSUM=y
CONFIG_X86_P6_NOP=y
CONFIG_X86_TSC=y
CONFIG_X86_CMPXCHG64=y
CONFIG_X86_CMOV=y
CONFIG_X86_MINIMUM_CPU_FAMILY=64
CONFIG_X86_DEBUGCTLMSR=y
CONFIG_CPU_SUP_INTEL=y
CONFIG_CPU_SUP_AMD=y
CONFIG_CPU_SUP_CENTAUR=y
CONFIG_HPET_TIMER=y
CONFIG_DMI=y
CONFIG_GART_IOMMU=y
# CONFIG_CALGARY_IOMMU is not set
CONFIG_SWIOTLB=y
CONFIG_IOMMU_HELPER=y
# CONFIG_MAXSMP is not set
CONFIG_NR_CPUS=4
# CONFIG_SCHED_SMT is not set
# CONFIG_SCHED_MC is not set
# CONFIG_IRQ_TIME_ACCOUNTING is not set
# CONFIG_PREEMPT_NONE is not set
# CONFIG_PREEMPT_VOLUNTARY is not set
CONFIG_PREEMPT=y
CONFIG_PREEMPT_COUNT=y
CONFIG_X86_LOCAL_APIC=y
CONFIG_X86_IO_APIC=y
CONFIG_X86_REROUTE_FOR_BROKEN_BOOT_IRQS=y
# CONFIG_X86_MCE is not set
# CONFIG_I8K is not set
# CONFIG_MICROCODE is not set
CONFIG_X86_MSR=y
CONFIG_X86_CPUID=y
CONFIG_ARCH_PHYS_ADDR_T_64BIT=y
CONFIG_ARCH_DMA_ADDR_T_64BIT=y
CONFIG_DIRECT_GBPAGES=y
# CONFIG_NUMA is not set
CONFIG_ARCH_SPARSEMEM_ENABLE=y
CONFIG_ARCH_SPARSEMEM_DEFAULT=y
CONFIG_ARCH_SELECT_MEMORY_MODEL=y
CONFIG_ARCH_PROC_KCORE_TEXT=y
CONFIG_ILLEGAL_POINTER_VALUE=0xdead000000000000
CONFIG_SELECT_MEMORY_MODEL=y
CONFIG_SPARSEMEM_MANUAL=y
CONFIG_SPARSEMEM=y
CONFIG_HAVE_MEMORY_PRESENT=y
CONFIG_SPARSEMEM_EXTREME=y
CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y
CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER=y
CONFIG_SPARSEMEM_VMEMMAP=y
CONFIG_HAVE_MEMBLOCK=y
CONFIG_HAVE_MEMBLOCK_NODE_MAP=y
CONFIG_ARCH_DISCARD_MEMBLOCK=y
# CONFIG_MEMORY_HOTPLUG is not set
CONFIG_PAGEFLAGS_EXTENDED=y
CONFIG_SPLIT_PTLOCK_CPUS=999999
CONFIG_COMPACTION=y
CONFIG_MIGRATION=y
CONFIG_PHYS_ADDR_T_64BIT=y
CONFIG_ZONE_DMA_FLAG=1
CONFIG_BOUNCE=y
CONFIG_VIRT_TO_BUS=y
# CONFIG_KSM is not set
CONFIG_DEFAULT_MMAP_MIN_ADDR=32768
CONFIG_TRANSPARENT_HUGEPAGE=y
CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS=y
# CONFIG_TRANSPARENT_HUGEPAGE_MADVISE is not set
# CONFIG_CLEANCACHE is not set
# CONFIG_X86_CHECK_BIOS_CORRUPTION is not set
CONFIG_X86_RESERVE_LOW=64
CONFIG_MTRR=y
CONFIG_MTRR_SANITIZER=y
CONFIG_MTRR_SANITIZER_ENABLE_DEFAULT=1
CONFIG_MTRR_SANITIZER_SPARE_REG_NR_DEFAULT=1
CONFIG_X86_PAT=y
CONFIG_ARCH_USES_PG_UNCACHED=y
CONFIG_ARCH_RANDOM=y
# CONFIG_EFI is not set
CONFIG_SECCOMP=y
# CONFIG_CC_STACKPROTECTOR is not set
# CONFIG_HZ_100 is not set
CONFIG_HZ_250=y
# CONFIG_HZ_300 is not set
# CONFIG_HZ_1000 is not set
CONFIG_HZ=250
CONFIG_SCHED_HRTICK=y
# CONFIG_KEXEC is not set
# CONFIG_CRASH_DUMP is not set
CONFIG_PHYSICAL_START=0x1000000
# CONFIG_RELOCATABLE is not set
CONFIG_PHYSICAL_ALIGN=0x1000000
CONFIG_HOTPLUG_CPU=y
# CONFIG_CMDLINE_BOOL is not set
CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y

#
# Power management and ACPI options
#
CONFIG_ARCH_HIBERNATION_HEADER=y
CONFIG_SUSPEND=y
CONFIG_SUSPEND_FREEZER=y
CONFIG_HIBERNATE_CALLBACKS=y
CONFIG_HIBERNATION=y
CONFIG_PM_STD_PARTITION=""
CONFIG_PM_SLEEP=y
CONFIG_PM_SLEEP_SMP=y
# CONFIG_PM_RUNTIME is not set
CONFIG_PM=y
CONFIG_PM_DEBUG=y
CONFIG_PM_ADVANCED_DEBUG=y
CONFIG_CAN_PM_TRACE=y
# CONFIG_PM_TRACE_RTC is not set
CONFIG_ACPI=y
CONFIG_ACPI_SLEEP=y
# CONFIG_ACPI_PROCFS is not set
# CONFIG_ACPI_PROCFS_POWER is not set
# CONFIG_ACPI_EC_DEBUGFS is not set
# CONFIG_ACPI_PROC_EVENT is not set
# CONFIG_ACPI_AC is not set
# CONFIG_ACPI_BATTERY is not set
# CONFIG_ACPI_BUTTON is not set
# CONFIG_ACPI_FAN is not set
CONFIG_ACPI_DOCK=y
CONFIG_ACPI_PROCESSOR=y
CONFIG_ACPI_HOTPLUG_CPU=y
# CONFIG_ACPI_PROCESSOR_AGGREGATOR is not set
# CONFIG_ACPI_THERMAL is not set
# CONFIG_ACPI_CUSTOM_DSDT is not set
CONFIG_ACPI_BLACKLIST_YEAR=0
# CONFIG_ACPI_DEBUG is not set
# CONFIG_ACPI_PCI_SLOT is not set
CONFIG_X86_PM_TIMER=y
CONFIG_ACPI_CONTAINER=y
# CONFIG_ACPI_SBS is not set
# CONFIG_ACPI_HED is not set
# CONFIG_ACPI_CUSTOM_METHOD is not set
# CONFIG_ACPI_APEI is not set
# CONFIG_SFI is not set

#
# CPU Frequency scaling
#
# CONFIG_CPU_FREQ is not set
CONFIG_CPU_IDLE=y
CONFIG_CPU_IDLE_GOV_LADDER=y
CONFIG_CPU_IDLE_GOV_MENU=y
CONFIG_INTEL_IDLE=y

#
# Memory power savings
#
# CONFIG_I7300_IDLE is not set

#
# Bus options (PCI etc.)
#
CONFIG_PCI=y
CONFIG_PCI_DIRECT=y
# CONFIG_PCI_MMCONFIG is not set
CONFIG_PCI_DOMAINS=y
# CONFIG_PCI_CNB20LE_QUIRK is not set
# CONFIG_PCIEPORTBUS is not set
CONFIG_ARCH_SUPPORTS_MSI=y
CONFIG_PCI_MSI=y
# CONFIG_PCI_DEBUG is not set
# CONFIG_PCI_STUB is not set
CONFIG_HT_IRQ=y
# CONFIG_PCI_IOV is not set
# CONFIG_PCI_PRI is not set
# CONFIG_PCI_PASID is not set
CONFIG_PCI_IOAPIC=y
CONFIG_PCI_LABEL=y
CONFIG_ISA_DMA_API=y
CONFIG_AMD_NB=y
# CONFIG_PCCARD is not set
CONFIG_HOTPLUG_PCI=y
CONFIG_HOTPLUG_PCI_FAKE=y
CONFIG_HOTPLUG_PCI_ACPI=y
# CONFIG_HOTPLUG_PCI_ACPI_IBM is not set
# CONFIG_HOTPLUG_PCI_CPCI is not set
# CONFIG_HOTPLUG_PCI_SHPC is not set
# CONFIG_RAPIDIO is not set

#
# Executable file formats / Emulations
#
CONFIG_BINFMT_ELF=y
CONFIG_ARCH_BINFMT_ELF_RANDOMIZE_PIE=y
CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS=y
# CONFIG_HAVE_AOUT is not set
# CONFIG_BINFMT_MISC is not set
# CONFIG_IA32_EMULATION is not set
# CONFIG_COMPAT_FOR_U64_ALIGNMENT is not set
CONFIG_HAVE_TEXT_POKE_SMP=y
CONFIG_NET=y

#
# Networking options
#
CONFIG_PACKET=y
CONFIG_UNIX=y
# CONFIG_UNIX_DIAG is not set
# CONFIG_NET_KEY is not set
CONFIG_INET=y
# CONFIG_IP_MULTICAST is not set
# CONFIG_IP_ADVANCED_ROUTER is not set
# CONFIG_IP_PNP is not set
# CONFIG_NET_IPIP is not set
# CONFIG_NET_IPGRE_DEMUX is not set
# CONFIG_ARPD is not set
# CONFIG_SYN_COOKIES is not set
# CONFIG_INET_AH is not set
# CONFIG_INET_ESP is not set
# CONFIG_INET_IPCOMP is not set
# CONFIG_INET_XFRM_TUNNEL is not set
# CONFIG_INET_TUNNEL is not set
# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
# CONFIG_INET_XFRM_MODE_TUNNEL is not set
# CONFIG_INET_XFRM_MODE_BEET is not set
# CONFIG_INET_LRO is not set
# CONFIG_INET_DIAG is not set
# CONFIG_TCP_CONG_ADVANCED is not set
CONFIG_TCP_CONG_CUBIC=y
CONFIG_DEFAULT_TCP_CONG="cubic"
# CONFIG_TCP_MD5SIG is not set
# CONFIG_IPV6 is not set
# CONFIG_NETWORK_SECMARK is not set
# CONFIG_NETWORK_PHY_TIMESTAMPING is not set
# CONFIG_NETFILTER is not set
# CONFIG_IP_DCCP is not set
# CONFIG_IP_SCTP is not set
# CONFIG_RDS is not set
# CONFIG_TIPC is not set
# CONFIG_ATM is not set
# CONFIG_L2TP is not set
# CONFIG_BRIDGE is not set
# CONFIG_NET_DSA is not set
# CONFIG_VLAN_8021Q is not set
# CONFIG_DECNET is not set
# CONFIG_LLC2 is not set
# CONFIG_IPX is not set
# CONFIG_ATALK is not set
# CONFIG_X25 is not set
# CONFIG_LAPB is not set
# CONFIG_ECONET is not set
# CONFIG_WAN_ROUTER is not set
# CONFIG_PHONET is not set
# CONFIG_IEEE802154 is not set
# CONFIG_NET_SCHED is not set
# CONFIG_DCB is not set
# CONFIG_BATMAN_ADV is not set
# CONFIG_OPENVSWITCH is not set
CONFIG_RPS=y
CONFIG_RFS_ACCEL=y
CONFIG_XPS=y
# CONFIG_NETPRIO_CGROUP is not set
CONFIG_BQL=y
CONFIG_HAVE_BPF_JIT=y
# CONFIG_BPF_JIT is not set

#
# Network testing
#
# CONFIG_NET_PKTGEN is not set
# CONFIG_NET_DROP_MONITOR is not set
# CONFIG_HAMRADIO is not set
# CONFIG_CAN is not set
# CONFIG_IRDA is not set
# CONFIG_BT is not set
# CONFIG_AF_RXRPC is not set
# CONFIG_WIRELESS is not set
# CONFIG_WIMAX is not set
# CONFIG_RFKILL is not set
# CONFIG_NET_9P is not set
# CONFIG_CAIF is not set
# CONFIG_CEPH_LIB is not set
# CONFIG_NFC is not set

#
# Device Drivers
#

#
# Generic Driver Options
#
CONFIG_UEVENT_HELPER_PATH=""
# CONFIG_DEVTMPFS is not set
CONFIG_STANDALONE=y
CONFIG_PREVENT_FIRMWARE_BUILD=y
CONFIG_FW_LOADER=y
# CONFIG_FIRMWARE_IN_KERNEL is not set
CONFIG_EXTRA_FIRMWARE=""
# CONFIG_DEBUG_DRIVER is not set
CONFIG_DEBUG_DEVRES=y
# CONFIG_SYS_HYPERVISOR is not set
# CONFIG_GENERIC_CPU_DEVICES is not set
# CONFIG_DMA_SHARED_BUFFER is not set
# CONFIG_CONNECTOR is not set
# CONFIG_MTD is not set
# CONFIG_PARPORT is not set
CONFIG_PNP=y
CONFIG_PNP_DEBUG_MESSAGES=y

#
# Protocols
#
CONFIG_PNPACPI=y
CONFIG_BLK_DEV=y
# CONFIG_BLK_DEV_FD is not set
# CONFIG_BLK_CPQ_DA is not set
# CONFIG_BLK_CPQ_CISS_DA is not set
# CONFIG_BLK_DEV_DAC960 is not set
# CONFIG_BLK_DEV_UMEM is not set
# CONFIG_BLK_DEV_COW_COMMON is not set
CONFIG_BLK_DEV_LOOP=y
CONFIG_BLK_DEV_LOOP_MIN_COUNT=8
# CONFIG_BLK_DEV_CRYPTOLOOP is not set

#
# DRBD disabled because PROC_FS, INET or CONNECTOR not selected
#
# CONFIG_BLK_DEV_NBD is not set
# CONFIG_BLK_DEV_NVME is not set
# CONFIG_BLK_DEV_SX8 is not set
CONFIG_BLK_DEV_RAM=y
CONFIG_BLK_DEV_RAM_COUNT=16
CONFIG_BLK_DEV_RAM_SIZE=16384
# CONFIG_BLK_DEV_XIP is not set
# CONFIG_CDROM_PKTCDVD is not set
# CONFIG_ATA_OVER_ETH is not set
CONFIG_VIRTIO_BLK=y
# CONFIG_BLK_DEV_HD is not set
# CONFIG_BLK_DEV_RBD is not set
# CONFIG_SENSORS_LIS3LV02D is not set
# CONFIG_MISC_DEVICES is not set
CONFIG_HAVE_IDE=y
# CONFIG_IDE is not set

#
# SCSI device support
#
CONFIG_SCSI_MOD=y
# CONFIG_RAID_ATTRS is not set
CONFIG_SCSI=y
CONFIG_SCSI_DMA=y
# CONFIG_SCSI_TGT is not set
# CONFIG_SCSI_NETLINK is not set
# CONFIG_SCSI_PROC_FS is not set

#
# SCSI support type (disk, tape, CD-ROM)
#
CONFIG_BLK_DEV_SD=y
# CONFIG_CHR_DEV_ST is not set
# CONFIG_CHR_DEV_OSST is not set
CONFIG_BLK_DEV_SR=y
# CONFIG_BLK_DEV_SR_VENDOR is not set
# CONFIG_CHR_DEV_SG is not set
# CONFIG_CHR_DEV_SCH is not set
# CONFIG_SCSI_MULTI_LUN is not set
CONFIG_SCSI_CONSTANTS=y
CONFIG_SCSI_LOGGING=y
# CONFIG_SCSI_SCAN_ASYNC is not set
CONFIG_SCSI_WAIT_SCAN=m

#
# SCSI Transports
#
# CONFIG_SCSI_SPI_ATTRS is not set
# CONFIG_SCSI_FC_ATTRS is not set
# CONFIG_SCSI_ISCSI_ATTRS is not set
# CONFIG_SCSI_SAS_ATTRS is not set
# CONFIG_SCSI_SAS_LIBSAS is not set
# CONFIG_SCSI_SRP_ATTRS is not set
# CONFIG_SCSI_LOWLEVEL is not set
# CONFIG_SCSI_DH is not set
# CONFIG_SCSI_OSD_INITIATOR is not set
CONFIG_ATA=y
# CONFIG_ATA_NONSTANDARD is not set
CONFIG_ATA_VERBOSE_ERROR=y
# CONFIG_ATA_ACPI is not set
# CONFIG_SATA_PMP is not set

#
# Controllers with non-SFF native interface
#
# CONFIG_SATA_AHCI is not set
# CONFIG_SATA_AHCI_PLATFORM is not set
# CONFIG_SATA_INIC162X is not set
# CONFIG_SATA_ACARD_AHCI is not set
# CONFIG_SATA_SIL24 is not set
CONFIG_ATA_SFF=y

#
# SFF controllers with custom DMA interface
#
# CONFIG_PDC_ADMA is not set
# CONFIG_SATA_QSTOR is not set
# CONFIG_SATA_SX4 is not set
CONFIG_ATA_BMDMA=y

#
# SATA SFF controllers with BMDMA
#
CONFIG_ATA_PIIX=y
# CONFIG_SATA_MV is not set
# CONFIG_SATA_NV is not set
# CONFIG_SATA_PROMISE is not set
# CONFIG_SATA_SIL is not set
# CONFIG_SATA_SIS is not set
# CONFIG_SATA_SVW is not set
# CONFIG_SATA_ULI is not set
# CONFIG_SATA_VIA is not set
# CONFIG_SATA_VITESSE is not set

#
# PATA SFF controllers with BMDMA
#
# CONFIG_PATA_ALI is not set
# CONFIG_PATA_AMD is not set
# CONFIG_PATA_ARTOP is not set
# CONFIG_PATA_ATIIXP is not set
# CONFIG_PATA_ATP867X is not set
# CONFIG_PATA_CMD64X is not set
# CONFIG_PATA_CS5520 is not set
# CONFIG_PATA_CS5530 is not set
# CONFIG_PATA_CS5536 is not set
# CONFIG_PATA_CYPRESS is not set
# CONFIG_PATA_EFAR is not set
# CONFIG_PATA_HPT366 is not set
# CONFIG_PATA_HPT37X is not set
# CONFIG_PATA_HPT3X2N is not set
# CONFIG_PATA_HPT3X3 is not set
# CONFIG_PATA_IT8213 is not set
# CONFIG_PATA_IT821X is not set
# CONFIG_PATA_JMICRON is not set
# CONFIG_PATA_MARVELL is not set
# CONFIG_PATA_NETCELL is not set
# CONFIG_PATA_NINJA32 is not set
# CONFIG_PATA_NS87415 is not set
# CONFIG_PATA_OLDPIIX is not set
# CONFIG_PATA_OPTIDMA is not set
# CONFIG_PATA_PDC2027X is not set
# CONFIG_PATA_PDC_OLD is not set
# CONFIG_PATA_RADISYS is not set
# CONFIG_PATA_RDC is not set
# CONFIG_PATA_SC1200 is not set
# CONFIG_PATA_SCH is not set
# CONFIG_PATA_SERVERWORKS is not set
# CONFIG_PATA_SIL680 is not set
# CONFIG_PATA_SIS is not set
# CONFIG_PATA_TOSHIBA is not set
# CONFIG_PATA_TRIFLEX is not set
# CONFIG_PATA_VIA is not set
# CONFIG_PATA_WINBOND is not set

#
# PIO-only SFF controllers
#
# CONFIG_PATA_CMD640_PCI is not set
# CONFIG_PATA_MPIIX is not set
# CONFIG_PATA_NS87410 is not set
# CONFIG_PATA_OPTI is not set
# CONFIG_PATA_RZ1000 is not set

#
# Generic fallback / legacy drivers
#
# CONFIG_ATA_GENERIC is not set
# CONFIG_PATA_LEGACY is not set
CONFIG_MD=y
# CONFIG_BLK_DEV_MD is not set
CONFIG_BLK_DEV_DM=y
# CONFIG_DM_DEBUG is not set
# CONFIG_DM_CRYPT is not set
# CONFIG_DM_SNAPSHOT is not set
# CONFIG_DM_THIN_PROVISIONING is not set
# CONFIG_DM_MIRROR is not set
# CONFIG_DM_RAID is not set
# CONFIG_DM_ZERO is not set
# CONFIG_DM_MULTIPATH is not set
# CONFIG_DM_DELAY is not set
# CONFIG_DM_UEVENT is not set
# CONFIG_DM_FLAKEY is not set
# CONFIG_TARGET_CORE is not set
# CONFIG_FUSION is not set

#
# IEEE 1394 (FireWire) support
#
# CONFIG_FIREWIRE is not set
# CONFIG_FIREWIRE_NOSY is not set
# CONFIG_I2O is not set
# CONFIG_MACINTOSH_DRIVERS is not set
CONFIG_NETDEVICES=y
CONFIG_NET_CORE=y
# CONFIG_BONDING is not set
# CONFIG_DUMMY is not set
# CONFIG_EQUALIZER is not set
# CONFIG_NET_FC is not set
CONFIG_MII=y
# CONFIG_NET_TEAM is not set
# CONFIG_MACVLAN is not set
# CONFIG_NETCONSOLE is not set
# CONFIG_NETPOLL is not set
# CONFIG_NET_POLL_CONTROLLER is not set
# CONFIG_TUN is not set
# CONFIG_VETH is not set
CONFIG_VIRTIO_NET=y
# CONFIG_ARCNET is not set

#
# CAIF transport drivers
#
CONFIG_ETHERNET=y
CONFIG_NET_VENDOR_3COM=y
# CONFIG_VORTEX is not set
# CONFIG_TYPHOON is not set
CONFIG_NET_VENDOR_ADAPTEC=y
# CONFIG_ADAPTEC_STARFIRE is not set
CONFIG_NET_VENDOR_ALTEON=y
# CONFIG_ACENIC is not set
CONFIG_NET_VENDOR_AMD=y
# CONFIG_AMD8111_ETH is not set
# CONFIG_PCNET32 is not set
CONFIG_NET_VENDOR_ATHEROS=y
# CONFIG_ATL2 is not set
# CONFIG_ATL1 is not set
# CONFIG_ATL1E is not set
# CONFIG_ATL1C is not set
CONFIG_NET_VENDOR_BROADCOM=y
# CONFIG_B44 is not set
# CONFIG_BNX2 is not set
# CONFIG_CNIC is not set
# CONFIG_TIGON3 is not set
# CONFIG_BNX2X is not set
CONFIG_NET_VENDOR_BROCADE=y
# CONFIG_BNA is not set
# CONFIG_NET_CALXEDA_XGMAC is not set
CONFIG_NET_VENDOR_CHELSIO=y
# CONFIG_CHELSIO_T1 is not set
# CONFIG_CHELSIO_T3 is not set
# CONFIG_CHELSIO_T4 is not set
# CONFIG_CHELSIO_T4VF is not set
CONFIG_NET_VENDOR_CISCO=y
# CONFIG_ENIC is not set
# CONFIG_DNET is not set
CONFIG_NET_VENDOR_DEC=y
# CONFIG_NET_TULIP is not set
CONFIG_NET_VENDOR_DLINK=y
# CONFIG_DL2K is not set
# CONFIG_SUNDANCE is not set
CONFIG_NET_VENDOR_EMULEX=y
# CONFIG_BE2NET is not set
CONFIG_NET_VENDOR_EXAR=y
# CONFIG_S2IO is not set
# CONFIG_VXGE is not set
CONFIG_NET_VENDOR_HP=y
# CONFIG_HP100 is not set
CONFIG_NET_VENDOR_INTEL=y
# CONFIG_E100 is not set
CONFIG_E1000=y
# CONFIG_E1000E is not set
# CONFIG_IGB is not set
# CONFIG_IGBVF is not set
# CONFIG_IXGB is not set
# CONFIG_IXGBE is not set
# CONFIG_IXGBEVF is not set
CONFIG_NET_VENDOR_I825XX=y
# CONFIG_ZNET is not set
# CONFIG_IP1000 is not set
# CONFIG_JME is not set
CONFIG_NET_VENDOR_MARVELL=y
# CONFIG_SKGE is not set
# CONFIG_SKY2 is not set
CONFIG_NET_VENDOR_MELLANOX=y
# CONFIG_MLX4_EN is not set
# CONFIG_MLX4_CORE is not set
CONFIG_NET_VENDOR_MICREL=y
# CONFIG_KS8851_MLL is not set
# CONFIG_KSZ884X_PCI is not set
CONFIG_NET_VENDOR_MYRI=y
# CONFIG_MYRI10GE is not set
# CONFIG_FEALNX is not set
CONFIG_NET_VENDOR_NATSEMI=y
# CONFIG_NATSEMI is not set
# CONFIG_NS83820 is not set
CONFIG_NET_VENDOR_8390=y
# CONFIG_NE2K_PCI is not set
CONFIG_NET_VENDOR_NVIDIA=y
# CONFIG_FORCEDETH is not set
CONFIG_NET_VENDOR_OKI=y
# CONFIG_PCH_GBE is not set
# CONFIG_ETHOC is not set
# CONFIG_NET_PACKET_ENGINE is not set
CONFIG_NET_VENDOR_QLOGIC=y
# CONFIG_QLA3XXX is not set
# CONFIG_QLCNIC is not set
# CONFIG_QLGE is not set
# CONFIG_NETXEN_NIC is not set
CONFIG_NET_VENDOR_REALTEK=y
# CONFIG_8139CP is not set
# CONFIG_8139TOO is not set
# CONFIG_R8169 is not set
CONFIG_NET_VENDOR_RDC=y
# CONFIG_R6040 is not set
CONFIG_NET_VENDOR_SEEQ=y
# CONFIG_SEEQ8005 is not set
CONFIG_NET_VENDOR_SILAN=y
# CONFIG_SC92031 is not set
CONFIG_NET_VENDOR_SIS=y
# CONFIG_SIS900 is not set
# CONFIG_SIS190 is not set
# CONFIG_SFC is not set
CONFIG_NET_VENDOR_SMSC=y
# CONFIG_EPIC100 is not set
# CONFIG_SMSC9420 is not set
CONFIG_NET_VENDOR_STMICRO=y
# CONFIG_STMMAC_ETH is not set
CONFIG_NET_VENDOR_SUN=y
# CONFIG_HAPPYMEAL is not set
# CONFIG_SUNGEM is not set
# CONFIG_CASSINI is not set
# CONFIG_NIU is not set
CONFIG_NET_VENDOR_TEHUTI=y
# CONFIG_TEHUTI is not set
CONFIG_NET_VENDOR_TI=y
# CONFIG_TLAN is not set
CONFIG_NET_VENDOR_VIA=y
# CONFIG_VIA_RHINE is not set
# CONFIG_VIA_VELOCITY is not set
# CONFIG_FDDI is not set
# CONFIG_HIPPI is not set
# CONFIG_NET_SB1000 is not set
# CONFIG_PHYLIB is not set
# CONFIG_PPP is not set
# CONFIG_SLIP is not set
# CONFIG_TR is not set
# CONFIG_WLAN is not set

#
# Enable WiMAX (Networking options) to see the WiMAX drivers
#
# CONFIG_WAN is not set
# CONFIG_VMXNET3 is not set
# CONFIG_ISDN is not set
# CONFIG_PHONE is not set

#
# Input device support
#
CONFIG_INPUT=y
# CONFIG_INPUT_FF_MEMLESS is not set
# CONFIG_INPUT_POLLDEV is not set
# CONFIG_INPUT_SPARSEKMAP is not set

#
# Userland interfaces
#
CONFIG_INPUT_MOUSEDEV=y
# CONFIG_INPUT_MOUSEDEV_PSAUX is not set
CONFIG_INPUT_MOUSEDEV_SCREEN_X=1024
CONFIG_INPUT_MOUSEDEV_SCREEN_Y=768
# CONFIG_INPUT_JOYDEV is not set
# CONFIG_INPUT_EVDEV is not set
# CONFIG_INPUT_EVBUG is not set

#
# Input Device Drivers
#
CONFIG_INPUT_KEYBOARD=y
CONFIG_KEYBOARD_ATKBD=y
# CONFIG_KEYBOARD_LKKBD is not set
# CONFIG_KEYBOARD_NEWTON is not set
# CONFIG_KEYBOARD_OPENCORES is not set
# CONFIG_KEYBOARD_STOWAWAY is not set
# CONFIG_KEYBOARD_SUNKBD is not set
# CONFIG_KEYBOARD_XTKBD is not set
# CONFIG_INPUT_MOUSE is not set
# CONFIG_INPUT_JOYSTICK is not set
# CONFIG_INPUT_TABLET is not set
# CONFIG_INPUT_TOUCHSCREEN is not set
# CONFIG_INPUT_MISC is not set

#
# Hardware I/O ports
#
CONFIG_SERIO=y
CONFIG_SERIO_I8042=y
CONFIG_SERIO_SERPORT=y
# CONFIG_SERIO_CT82C710 is not set
# CONFIG_SERIO_PCIPS2 is not set
CONFIG_SERIO_LIBPS2=y
CONFIG_SERIO_RAW=y
# CONFIG_SERIO_ALTERA_PS2 is not set
# CONFIG_SERIO_PS2MULT is not set
# CONFIG_GAMEPORT is not set

#
# Character devices
#
CONFIG_VT=y
CONFIG_CONSOLE_TRANSLATIONS=y
CONFIG_VT_CONSOLE=y
CONFIG_VT_CONSOLE_SLEEP=y
CONFIG_HW_CONSOLE=y
CONFIG_VT_HW_CONSOLE_BINDING=y
CONFIG_UNIX98_PTYS=y
CONFIG_DEVPTS_MULTIPLE_INSTANCES=y
# CONFIG_LEGACY_PTYS is not set
# CONFIG_SERIAL_NONSTANDARD is not set
# CONFIG_NOZOMI is not set
# CONFIG_N_GSM is not set
# CONFIG_TRACE_SINK is not set
# CONFIG_DEVKMEM is not set

#
# Serial drivers
#
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_FIX_EARLYCON_MEM=y
CONFIG_SERIAL_8250_PCI=y
CONFIG_SERIAL_8250_PNP=y
CONFIG_SERIAL_8250_NR_UARTS=32
CONFIG_SERIAL_8250_RUNTIME_UARTS=4
CONFIG_SERIAL_8250_EXTENDED=y
CONFIG_SERIAL_8250_MANY_PORTS=y
CONFIG_SERIAL_8250_SHARE_IRQ=y
CONFIG_SERIAL_8250_DETECT_IRQ=y
CONFIG_SERIAL_8250_RSA=y

#
# Non-8250 serial port support
#
# CONFIG_SERIAL_MFD_HSU is not set
CONFIG_SERIAL_CORE=y
CONFIG_SERIAL_CORE_CONSOLE=y
CONFIG_SERIAL_JSM=m
# CONFIG_SERIAL_TIMBERDALE is not set
# CONFIG_SERIAL_ALTERA_JTAGUART is not set
# CONFIG_SERIAL_ALTERA_UART is not set
# CONFIG_SERIAL_PCH_UART is not set
# CONFIG_SERIAL_XILINX_PS_UART is not set
CONFIG_HVC_DRIVER=y
CONFIG_VIRTIO_CONSOLE=y
# CONFIG_IPMI_HANDLER is not set
# CONFIG_HW_RANDOM is not set
CONFIG_NVRAM=y
# CONFIG_RTC is not set
# CONFIG_GEN_RTC is not set
# CONFIG_R3964 is not set
# CONFIG_APPLICOM is not set
# CONFIG_MWAVE is not set
CONFIG_RAW_DRIVER=y
CONFIG_MAX_RAW_DEVS=8192
# CONFIG_HPET is not set
# CONFIG_HANGCHECK_TIMER is not set
# CONFIG_TCG_TPM is not set
# CONFIG_TELCLOCK is not set
CONFIG_DEVPORT=y
# CONFIG_RAMOOPS is not set
# CONFIG_I2C is not set
# CONFIG_SPI is not set

#
# PPS support
#
# CONFIG_PPS is not set

#
# PPS generators support
#

#
# PTP clock support
#

#
# Enable Device Drivers -> PPS to see the PTP clock options.
#
CONFIG_ARCH_WANT_OPTIONAL_GPIOLIB=y
# CONFIG_GPIOLIB is not set
# CONFIG_W1 is not set
CONFIG_POWER_SUPPLY=y
# CONFIG_POWER_SUPPLY_DEBUG is not set
# CONFIG_PDA_POWER is not set
# CONFIG_TEST_POWER is not set
# CONFIG_BATTERY_DS2780 is not set
# CONFIG_BATTERY_BQ27x00 is not set
# CONFIG_CHARGER_MAX8903 is not set
# CONFIG_HWMON is not set
CONFIG_THERMAL=y
# CONFIG_WATCHDOG is not set
CONFIG_SSB_POSSIBLE=y

#
# Sonics Silicon Backplane
#
# CONFIG_SSB is not set
CONFIG_BCMA_POSSIBLE=y

#
# Broadcom specific AMBA
#
# CONFIG_BCMA is not set

#
# Multifunction device drivers
#
# CONFIG_MFD_CORE is not set
# CONFIG_MFD_SM501 is not set
# CONFIG_HTC_PASIC3 is not set
# CONFIG_MFD_TMIO is not set
# CONFIG_ABX500_CORE is not set
# CONFIG_MFD_CS5535 is not set
# CONFIG_LPC_SCH is not set
# CONFIG_MFD_RDC321X is not set
# CONFIG_MFD_JANZ_CMODIO is not set
# CONFIG_MFD_VX855 is not set
# CONFIG_REGULATOR is not set
# CONFIG_MEDIA_SUPPORT is not set

#
# Graphics support
#
CONFIG_AGP=y
CONFIG_AGP_AMD64=y
# CONFIG_AGP_INTEL is not set
# CONFIG_AGP_SIS is not set
# CONFIG_AGP_VIA is not set
CONFIG_VGA_ARB=y
CONFIG_VGA_ARB_MAX_GPUS=1
# CONFIG_VGA_SWITCHEROO is not set
# CONFIG_DRM is not set
# CONFIG_STUB_POULSBO is not set
# CONFIG_VGASTATE is not set
# CONFIG_VIDEO_OUTPUT_CONTROL is not set
# CONFIG_FB is not set
# CONFIG_BACKLIGHT_LCD_SUPPORT is not set

#
# Console display driver support
#
CONFIG_VGA_CONSOLE=y
CONFIG_VGACON_SOFT_SCROLLBACK=y
CONFIG_VGACON_SOFT_SCROLLBACK_SIZE=64
CONFIG_DUMMY_CONSOLE=y
# CONFIG_SOUND is not set
# CONFIG_HID_SUPPORT is not set
# CONFIG_USB_SUPPORT is not set
# CONFIG_UWB is not set
# CONFIG_MMC is not set
# CONFIG_MEMSTICK is not set
# CONFIG_NEW_LEDS is not set
# CONFIG_ACCESSIBILITY is not set
# CONFIG_INFINIBAND is not set
# CONFIG_EDAC is not set
# CONFIG_RTC_CLASS is not set
# CONFIG_DMADEVICES is not set
# CONFIG_AUXDISPLAY is not set
# CONFIG_UIO is not set
CONFIG_VIRTIO=y
CONFIG_VIRTIO_RING=y

#
# Virtio drivers
#
CONFIG_VIRTIO_PCI=y
CONFIG_VIRTIO_BALLOON=y
# CONFIG_VIRTIO_MMIO is not set

#
# Microsoft Hyper-V guest support
#
# CONFIG_HYPERV is not set
# CONFIG_STAGING is not set
# CONFIG_X86_PLATFORM_DEVICES is not set

#
# Hardware Spinlock drivers
#
CONFIG_CLKEVT_I8253=y
CONFIG_I8253_LOCK=y
CONFIG_CLKBLD_I8253=y
# CONFIG_IOMMU_SUPPORT is not set
CONFIG_VIRT_DRIVERS=y
# CONFIG_PM_DEVFREQ is not set

#
# Firmware Drivers
#
# CONFIG_EDD is not set
CONFIG_FIRMWARE_MEMMAP=y
# CONFIG_DELL_RBU is not set
# CONFIG_DCDBAS is not set
# CONFIG_DMIID is not set
# CONFIG_DMI_SYSFS is not set
# CONFIG_ISCSI_IBFT_FIND is not set
# CONFIG_GOOGLE_FIRMWARE is not set

#
# File systems
#
# CONFIG_EXT2_FS is not set
# CONFIG_EXT3_FS is not set
CONFIG_EXT4_FS=y
CONFIG_EXT4_USE_FOR_EXT23=y
CONFIG_EXT4_FS_XATTR=y
CONFIG_EXT4_FS_POSIX_ACL=y
CONFIG_EXT4_FS_SECURITY=y
# CONFIG_EXT4_DEBUG is not set
CONFIG_JBD2=y
CONFIG_JBD2_DEBUG=y
CONFIG_FS_MBCACHE=y
# CONFIG_REISERFS_FS is not set
# CONFIG_JFS_FS is not set
# CONFIG_XFS_FS is not set
# CONFIG_GFS2_FS is not set
# CONFIG_OCFS2_FS is not set
# CONFIG_BTRFS_FS is not set
# CONFIG_NILFS2_FS is not set
CONFIG_FS_POSIX_ACL=y
CONFIG_FILE_LOCKING=y
CONFIG_FSNOTIFY=y
CONFIG_DNOTIFY=y
CONFIG_INOTIFY_USER=y
CONFIG_FANOTIFY=y
# CONFIG_QUOTA is not set
# CONFIG_QUOTACTL is not set
# CONFIG_AUTOFS4_FS is not set
# CONFIG_FUSE_FS is not set
CONFIG_GENERIC_ACL=y

#
# Caches
#
# CONFIG_FSCACHE is not set

#
# CD-ROM/DVD Filesystems
#
CONFIG_ISO9660_FS=y
# CONFIG_JOLIET is not set
# CONFIG_ZISOFS is not set
CONFIG_UDF_FS=y
CONFIG_UDF_NLS=y

#
# DOS/FAT/NT Filesystems
#
# CONFIG_MSDOS_FS is not set
# CONFIG_VFAT_FS is not set
# CONFIG_NTFS_FS is not set

#
# Pseudo filesystems
#
CONFIG_PROC_FS=y
CONFIG_PROC_KCORE=y
CONFIG_PROC_SYSCTL=y
CONFIG_PROC_PAGE_MONITOR=y
CONFIG_SYSFS=y
CONFIG_TMPFS=y
CONFIG_TMPFS_POSIX_ACL=y
CONFIG_TMPFS_XATTR=y
CONFIG_HUGETLBFS=y
CONFIG_HUGETLB_PAGE=y
CONFIG_CONFIGFS_FS=y
# CONFIG_MISC_FILESYSTEMS is not set
# CONFIG_NETWORK_FILESYSTEMS is not set
CONFIG_NLS=y
CONFIG_NLS_DEFAULT="iso8859-1"
# CONFIG_NLS_CODEPAGE_437 is not set
# CONFIG_NLS_CODEPAGE_737 is not set
# CONFIG_NLS_CODEPAGE_775 is not set
# CONFIG_NLS_CODEPAGE_850 is not set
# CONFIG_NLS_CODEPAGE_852 is not set
# CONFIG_NLS_CODEPAGE_855 is not set
# CONFIG_NLS_CODEPAGE_857 is not set
# CONFIG_NLS_CODEPAGE_860 is not set
# CONFIG_NLS_CODEPAGE_861 is not set
# CONFIG_NLS_CODEPAGE_862 is not set
# CONFIG_NLS_CODEPAGE_863 is not set
# CONFIG_NLS_CODEPAGE_864 is not set
# CONFIG_NLS_CODEPAGE_865 is not set
# CONFIG_NLS_CODEPAGE_866 is not set
# CONFIG_NLS_CODEPAGE_869 is not set
# CONFIG_NLS_CODEPAGE_936 is not set
# CONFIG_NLS_CODEPAGE_950 is not set
# CONFIG_NLS_CODEPAGE_932 is not set
# CONFIG_NLS_CODEPAGE_949 is not set
# CONFIG_NLS_CODEPAGE_874 is not set
# CONFIG_NLS_ISO8859_8 is not set
# CONFIG_NLS_CODEPAGE_1250 is not set
# CONFIG_NLS_CODEPAGE_1251 is not set
# CONFIG_NLS_ASCII is not set
# CONFIG_NLS_ISO8859_1 is not set
# CONFIG_NLS_ISO8859_2 is not set
# CONFIG_NLS_ISO8859_3 is not set
# CONFIG_NLS_ISO8859_4 is not set
# CONFIG_NLS_ISO8859_5 is not set
# CONFIG_NLS_ISO8859_6 is not set
# CONFIG_NLS_ISO8859_7 is not set
# CONFIG_NLS_ISO8859_9 is not set
# CONFIG_NLS_ISO8859_13 is not set
# CONFIG_NLS_ISO8859_14 is not set
# CONFIG_NLS_ISO8859_15 is not set
# CONFIG_NLS_KOI8_R is not set
# CONFIG_NLS_KOI8_U is not set
# CONFIG_NLS_UTF8 is not set
# CONFIG_DLM is not set

#
# Kernel hacking
#
CONFIG_TRACE_IRQFLAGS_SUPPORT=y
# CONFIG_PRINTK_TIME is not set
CONFIG_DEFAULT_MESSAGE_LOGLEVEL=4
# CONFIG_ENABLE_WARN_DEPRECATED is not set
CONFIG_ENABLE_MUST_CHECK=y
CONFIG_FRAME_WARN=2048
CONFIG_MAGIC_SYSRQ=y
# CONFIG_STRIP_ASM_SYMS is not set
CONFIG_UNUSED_SYMBOLS=y
CONFIG_DEBUG_FS=y
CONFIG_HEADERS_CHECK=y
# CONFIG_DEBUG_SECTION_MISMATCH is not set
CONFIG_DEBUG_KERNEL=y
# CONFIG_DEBUG_SHIRQ is not set
# CONFIG_LOCKUP_DETECTOR is not set
# CONFIG_HARDLOCKUP_DETECTOR is not set
CONFIG_DETECT_HUNG_TASK=y
CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=120
# CONFIG_BOOTPARAM_HUNG_TASK_PANIC is not set
CONFIG_BOOTPARAM_HUNG_TASK_PANIC_VALUE=0
CONFIG_SCHED_DEBUG=y
CONFIG_SCHEDSTATS=y
CONFIG_TIMER_STATS=y
# CONFIG_DEBUG_OBJECTS is not set
CONFIG_SLUB_DEBUG_ON=y
# CONFIG_SLUB_STATS is not set
CONFIG_DEBUG_KMEMLEAK=y
CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE=400
# CONFIG_DEBUG_KMEMLEAK_TEST is not set
# CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF is not set
CONFIG_DEBUG_PREEMPT=y
CONFIG_DEBUG_RT_MUTEXES=y
CONFIG_DEBUG_PI_LIST=y
# CONFIG_RT_MUTEX_TESTER is not set
CONFIG_DEBUG_SPINLOCK=y
CONFIG_DEBUG_MUTEXES=y
CONFIG_DEBUG_LOCK_ALLOC=y
CONFIG_PROVE_LOCKING=y
# CONFIG_PROVE_RCU is not set
# CONFIG_SPARSE_RCU_POINTER is not set
CONFIG_LOCKDEP=y
CONFIG_LOCK_STAT=y
# CONFIG_DEBUG_LOCKDEP is not set
CONFIG_TRACE_IRQFLAGS=y
# CONFIG_DEBUG_ATOMIC_SLEEP is not set
# CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
CONFIG_STACKTRACE=y
# CONFIG_DEBUG_STACK_USAGE is not set
# CONFIG_DEBUG_KOBJECT is not set
CONFIG_DEBUG_BUGVERBOSE=y
CONFIG_DEBUG_INFO=y
# CONFIG_DEBUG_INFO_REDUCED is not set
# CONFIG_DEBUG_VM is not set
# CONFIG_DEBUG_VIRTUAL is not set
# CONFIG_DEBUG_WRITECOUNT is not set
CONFIG_DEBUG_MEMORY_INIT=y
CONFIG_DEBUG_LIST=y
# CONFIG_TEST_LIST_SORT is not set
CONFIG_DEBUG_SG=y
# CONFIG_DEBUG_NOTIFIERS is not set
# CONFIG_DEBUG_CREDENTIALS is not set
CONFIG_ARCH_WANT_FRAME_POINTERS=y
CONFIG_FRAME_POINTER=y
CONFIG_BOOT_PRINTK_DELAY=y
# CONFIG_RCU_TORTURE_TEST is not set
CONFIG_RCU_CPU_STALL_TIMEOUT=60
CONFIG_RCU_CPU_STALL_VERBOSE=y
# CONFIG_BACKTRACE_SELF_TEST is not set
# CONFIG_DEBUG_BLOCK_EXT_DEVT is not set
# CONFIG_DEBUG_FORCE_WEAK_PER_CPU is not set
# CONFIG_DEBUG_PER_CPU_MAPS is not set
# CONFIG_LKDTM is not set
# CONFIG_CPU_NOTIFIER_ERROR_INJECT is not set
# CONFIG_FAULT_INJECTION is not set
CONFIG_LATENCYTOP=y
# CONFIG_SYSCTL_SYSCALL_CHECK is not set
CONFIG_DEBUG_PAGEALLOC=y
CONFIG_WANT_PAGE_DEBUG_FLAGS=y
CONFIG_PAGE_GUARD=y
CONFIG_USER_STACKTRACE_SUPPORT=y
CONFIG_NOP_TRACER=y
CONFIG_HAVE_FTRACE_NMI_ENTER=y
CONFIG_HAVE_FUNCTION_TRACER=y
CONFIG_HAVE_FUNCTION_GRAPH_TRACER=y
CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST=y
CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST=y
CONFIG_HAVE_DYNAMIC_FTRACE=y
CONFIG_HAVE_FTRACE_MCOUNT_RECORD=y
CONFIG_HAVE_SYSCALL_TRACEPOINTS=y
CONFIG_HAVE_C_RECORDMCOUNT=y
CONFIG_RING_BUFFER=y
CONFIG_FTRACE_NMI_ENTER=y
CONFIG_EVENT_TRACING=y
CONFIG_EVENT_POWER_TRACING_DEPRECATED=y
CONFIG_CONTEXT_SWITCH_TRACER=y
CONFIG_TRACING=y
CONFIG_GENERIC_TRACER=y
CONFIG_TRACING_SUPPORT=y
CONFIG_FTRACE=y
CONFIG_FUNCTION_TRACER=y
CONFIG_FUNCTION_GRAPH_TRACER=y
# CONFIG_IRQSOFF_TRACER is not set
# CONFIG_PREEMPT_TRACER is not set
# CONFIG_SCHED_TRACER is not set
CONFIG_FTRACE_SYSCALLS=y
CONFIG_BRANCH_PROFILE_NONE=y
# CONFIG_PROFILE_ANNOTATED_BRANCHES is not set
# CONFIG_PROFILE_ALL_BRANCHES is not set
# CONFIG_STACK_TRACER is not set
# CONFIG_BLK_DEV_IO_TRACE is not set
CONFIG_DYNAMIC_FTRACE=y
CONFIG_FUNCTION_PROFILER=y
CONFIG_FTRACE_MCOUNT_RECORD=y
# CONFIG_FTRACE_STARTUP_TEST is not set
# CONFIG_MMIOTRACE is not set
# CONFIG_RING_BUFFER_BENCHMARK is not set
# CONFIG_PROVIDE_OHCI1394_DMA_INIT is not set
# CONFIG_BUILD_DOCSRC is not set
# CONFIG_DYNAMIC_DEBUG is not set
# CONFIG_DMA_API_DEBUG is not set
# CONFIG_ATOMIC64_SELFTEST is not set
# CONFIG_SAMPLES is not set
CONFIG_HAVE_ARCH_KGDB=y
# CONFIG_KGDB is not set
CONFIG_HAVE_ARCH_KMEMCHECK=y
# CONFIG_TEST_KSTRTOX is not set
CONFIG_STRICT_DEVMEM=y
# CONFIG_X86_VERBOSE_BOOTUP is not set
CONFIG_EARLY_PRINTK=y
# CONFIG_EARLY_PRINTK_DBGP is not set
CONFIG_DEBUG_STACKOVERFLOW=y
# CONFIG_X86_PTDUMP is not set
CONFIG_DEBUG_RODATA=y
CONFIG_DEBUG_RODATA_TEST=y
CONFIG_DEBUG_SET_MODULE_RONX=y
# CONFIG_DEBUG_NX_TEST is not set
# CONFIG_IOMMU_DEBUG is not set
# CONFIG_IOMMU_STRESS is not set
CONFIG_HAVE_MMIOTRACE_SUPPORT=y
CONFIG_IO_DELAY_TYPE_0X80=0
CONFIG_IO_DELAY_TYPE_0XED=1
CONFIG_IO_DELAY_TYPE_UDELAY=2
CONFIG_IO_DELAY_TYPE_NONE=3
CONFIG_IO_DELAY_0X80=y
# CONFIG_IO_DELAY_0XED is not set
# CONFIG_IO_DELAY_UDELAY is not set
# CONFIG_IO_DELAY_NONE is not set
CONFIG_DEFAULT_IO_DELAY_TYPE=0
CONFIG_DEBUG_BOOT_PARAMS=y
# CONFIG_CPA_DEBUG is not set
CONFIG_OPTIMIZE_INLINING=y
# CONFIG_DEBUG_STRICT_USER_COPY_CHECKS is not set
# CONFIG_DEBUG_NMI_SELFTEST is not set

#
# Security options
#
# CONFIG_KEYS is not set
# CONFIG_SECURITY_DMESG_RESTRICT is not set
# CONFIG_SECURITY is not set
# CONFIG_SECURITYFS is not set
CONFIG_DEFAULT_SECURITY_DAC=y
CONFIG_DEFAULT_SECURITY=""
# CONFIG_CRYPTO is not set
CONFIG_HAVE_KVM=y
CONFIG_VIRTUALIZATION=y
# CONFIG_KVM is not set
# CONFIG_VHOST_NET is not set
CONFIG_BINARY_PRINTF=y

#
# Library routines
#
CONFIG_BITREVERSE=y
CONFIG_GENERIC_FIND_FIRST_BIT=y
CONFIG_GENERIC_PCI_IOMAP=y
CONFIG_GENERIC_IOMAP=y
# CONFIG_CRC_CCITT is not set
CONFIG_CRC16=y
CONFIG_CRC_T10DIF=y
CONFIG_CRC_ITU_T=y
CONFIG_CRC32=y
# CONFIG_CRC7 is not set
# CONFIG_LIBCRC32C is not set
# CONFIG_CRC8 is not set
CONFIG_ZLIB_INFLATE=y
CONFIG_LZO_COMPRESS=y
CONFIG_LZO_DECOMPRESS=y
CONFIG_XZ_DEC=y
CONFIG_XZ_DEC_X86=y
CONFIG_XZ_DEC_POWERPC=y
CONFIG_XZ_DEC_IA64=y
CONFIG_XZ_DEC_ARM=y
CONFIG_XZ_DEC_ARMTHUMB=y
CONFIG_XZ_DEC_SPARC=y
CONFIG_XZ_DEC_BCJ=y
# CONFIG_XZ_DEC_TEST is not set
CONFIG_DECOMPRESS_GZIP=y
CONFIG_DECOMPRESS_BZIP2=y
CONFIG_DECOMPRESS_LZMA=y
CONFIG_DECOMPRESS_XZ=y
CONFIG_DECOMPRESS_LZO=y
CONFIG_HAS_IOMEM=y
CONFIG_HAS_IOPORT=y
CONFIG_HAS_DMA=y
CONFIG_CPU_RMAP=y
CONFIG_DQL=y
CONFIG_NLATTR=y
# CONFIG_AVERAGE is not set
# CONFIG_CORDIC is not set



		Amit

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 10:02     ` Amit Shah
  2012-02-10 10:11       ` Igor Mammedov
@ 2012-02-10 12:32       ` Marcelo Tosatti
  2012-02-10 12:33         ` Marcelo Tosatti
  2012-02-10 13:18         ` Amit Shah
  1 sibling, 2 replies; 24+ messages in thread
From: Marcelo Tosatti @ 2012-02-10 12:32 UTC (permalink / raw)
  To: Amit Shah; +Cc: Igor Mammedov, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On Fri, Feb 10, 2012 at 03:32:11PM +0530, Amit Shah wrote:
> On (Thu) 09 Feb 2012 [16:13:29], Igor Mammedov wrote:
> 
> > Stalls are probably caused by uninitialized percpu hv_clock, with
> > following patch I don't see stalls. Although I might be just lucky.
> > http://git.kernel.org/?p=virt/kvm/kvm.git;a=commit;h=e2971ac7e1d186af059e088d305496c5cb47d487
> 
> Your commit does make things better, I don't see any stalls on the
> first resume.
> 
> However, a subsequent s4 causes the stall to re-appear on resume, and
> this time there are no stall messages; the kernel just sits there
> spinning on something.  I've not found the solution to this one yet (I
> had a commit similar to Marcelo's in the works, which got me to the
> previous works-but-stalls behaviour).

I cannot reproduce it here. Suspend/resume are operating normally after
several iterations. Igor do you see anything similar?

Amit, can you please enable CONFIG_PRINTK_TIME=y and post a full dmesg 
(both during suspend and also the new kernel during resume).

Thanks.

> > However there is/are a warning/s on suspend path and with following patch:
> 
> I didn't see this.

This is unrelated.


^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 12:32       ` Marcelo Tosatti
@ 2012-02-10 12:33         ` Marcelo Tosatti
  2012-02-10 12:43           ` Igor Mammedov
  2012-02-13 12:56           ` Amit Shah
  2012-02-10 13:18         ` Amit Shah
  1 sibling, 2 replies; 24+ messages in thread
From: Marcelo Tosatti @ 2012-02-10 12:33 UTC (permalink / raw)
  To: Amit Shah; +Cc: Igor Mammedov, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On Fri, Feb 10, 2012 at 10:32:16AM -0200, Marcelo Tosatti wrote:
> On Fri, Feb 10, 2012 at 03:32:11PM +0530, Amit Shah wrote:
> > On (Thu) 09 Feb 2012 [16:13:29], Igor Mammedov wrote:
> > 
> > > Stalls are probably caused by uninitialized percpu hv_clock, with
> > > following patch I don't see stalls. Although I might be just lucky.
> > > http://git.kernel.org/?p=virt/kvm/kvm.git;a=commit;h=e2971ac7e1d186af059e088d305496c5cb47d487
> > 
> > Your commit does make things better, I don't see any stalls on the
> > first resume.
> > 
> > However, a subsequent s4 causes the stall to re-appear on resume, and
> > this time there are no stall messages; the kernel just sits there
> > spinning on something.  I've not found the solution to this one yet (I
> > had a commit similar to Marcelo's in the works, which got me to the
> > previous works-but-stalls behaviour).
> 
> I cannot reproduce it here. Suspend/resume are operating normally after
> several iterations. Igor do you see anything similar?
> 
> Amit, can you please enable CONFIG_PRINTK_TIME=y and post a full dmesg 
> (both during suspend and also the new kernel during resume).

Also is it reproducible with UP guest?


^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 12:33         ` Marcelo Tosatti
@ 2012-02-10 12:43           ` Igor Mammedov
  2012-02-13 12:46             ` Amit Shah
  2012-02-13 12:56           ` Amit Shah
  1 sibling, 1 reply; 24+ messages in thread
From: Igor Mammedov @ 2012-02-10 12:43 UTC (permalink / raw)
  To: Marcelo Tosatti
  Cc: Amit Shah, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On 02/10/2012 01:33 PM, Marcelo Tosatti wrote:
> On Fri, Feb 10, 2012 at 10:32:16AM -0200, Marcelo Tosatti wrote:
>> On Fri, Feb 10, 2012 at 03:32:11PM +0530, Amit Shah wrote:
>>> On (Thu) 09 Feb 2012 [16:13:29], Igor Mammedov wrote:
>>>
>>>> Stalls are probably caused by uninitialized percpu hv_clock, with
>>>> following patch I don't see stalls. Although I might be just lucky.
>>>> http://git.kernel.org/?p=virt/kvm/kvm.git;a=commit;h=e2971ac7e1d186af059e088d305496c5cb47d487
>>>
>>> Your commit does make things better, I don't see any stalls on the
>>> first resume.
>>>
>>> However, a subsequent s4 causes the stall to re-appear on resume, and
>>> this time there are no stall messages; the kernel just sits there
>>> spinning on something.  I've not found the solution to this one yet (I
>>> had a commit similar to Marcelo's in the works, which got me to the
>>> previous works-but-stalls behaviour).
>>
>> I cannot reproduce it here. Suspend/resume are operating normally after
>> several iterations. Igor do you see anything similar?

I wasn't able to reproduce it either but I haven't tried with Amit's config
yet.

>>
>> Amit, can you please enable CONFIG_PRINTK_TIME=y and post a full dmesg
>> (both during suspend and also the new kernel during resume).
>
> Also is it reproducible with UP guest?

Another thing is to try smp guest without kvmclock and see if it helps.
It might be just something else.

> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

-- 
Thanks,
  Igor

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 12:32       ` Marcelo Tosatti
  2012-02-10 12:33         ` Marcelo Tosatti
@ 2012-02-10 13:18         ` Amit Shah
  2012-02-10 20:58           ` Igor Mammedov
  1 sibling, 1 reply; 24+ messages in thread
From: Amit Shah @ 2012-02-10 13:18 UTC (permalink / raw)
  To: Marcelo Tosatti
  Cc: Igor Mammedov, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On (Fri) 10 Feb 2012 [10:32:16], Marcelo Tosatti wrote:
> On Fri, Feb 10, 2012 at 03:32:11PM +0530, Amit Shah wrote:
> > On (Thu) 09 Feb 2012 [16:13:29], Igor Mammedov wrote:
> > 
> > > Stalls are probably caused by uninitialized percpu hv_clock, with
> > > following patch I don't see stalls. Although I might be just lucky.
> > > http://git.kernel.org/?p=virt/kvm/kvm.git;a=commit;h=e2971ac7e1d186af059e088d305496c5cb47d487
> > 
> > Your commit does make things better, I don't see any stalls on the
> > first resume.
> > 
> > However, a subsequent s4 causes the stall to re-appear on resume, and
> > this time there are no stall messages; the kernel just sits there
> > spinning on something.  I've not found the solution to this one yet (I
> > had a commit similar to Marcelo's in the works, which got me to the
> > previous works-but-stalls behaviour).
> 
> I cannot reproduce it here. Suspend/resume are operating normally after
> several iterations. Igor do you see anything similar?
> 
> Amit, can you please enable CONFIG_PRINTK_TIME=y and post a full dmesg 
> (both during suspend and also the new kernel during resume).

In my case, I run a ping to the host (10.0.2.2) while the s4
suspend/resume operations are performed.

Complete dmesg, for all 3 invocations of the guest.  First one boots,
starts ping, starts s4.  Second one starts s4 after confirming ping is
working fine.  Third one just stays there, spinning.

$ ./x86_64-softmmu/qemu-system-x86_64 -kernel ~/src/linux/arch/x86/boot/bzImage  -append 'root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend' -drive file=/guests/f14-suspend.qcow2,if=none,id=dr0 -device virtio-blk-pci,drive=dr0 -net nic,model=virtio -net user  -serial stdio  -enable-kvm -m 512  -cpu host -smp 4
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Linux version 3.3.0-rc2+ (amit@amit.redhat.com) (gcc version 4.6.2 20111027 (Red Hat 4.6.2-1) (GCC) ) #295 SMP PREEMPT Fri Feb 10 18:39:48 IST 2012
[    0.000000] Command line: root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend
[    0.000000] BIOS-provided physical RAM map:
[    0.000000]  BIOS-e820: 0000000000000000 - 000000000009dc00 (usable)
[    0.000000]  BIOS-e820: 000000000009dc00 - 00000000000a0000 (reserved)
[    0.000000]  BIOS-e820: 00000000000f0000 - 0000000000100000 (reserved)
[    0.000000]  BIOS-e820: 0000000000100000 - 000000001fffd000 (usable)
[    0.000000]  BIOS-e820: 000000001fffd000 - 0000000020000000 (reserved)
[    0.000000]  BIOS-e820: 00000000feffc000 - 00000000ff000000 (reserved)
[    0.000000]  BIOS-e820: 00000000fffc0000 - 0000000100000000 (reserved)
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] DMI 2.4 present.
[    0.000000] No AGP bridge found
[    0.000000] last_pfn = 0x1fffd max_arch_pfn = 0x400000000
[    0.000000] x86 PAT enabled: cpu 0, old 0x70406, new 0x7010600070106
[    0.000000] init_memory_mapping: 0000000000000000-000000001fffd000
[    0.000000] RAMDISK: 1fa2e000 - 1fff0000
[    0.000000] ACPI: RSDP 00000000000fd3f0 00014 (v00 BOCHS )
[    0.000000] ACPI: RSDT 000000001fffd660 00034 (v01 BOCHS  BXPCRSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: FACP 000000001fffff80 00074 (v01 BOCHS  BXPCFACP 00000001 BXPC 00000001)
[    0.000000] ACPI: DSDT 000000001fffd9b0 02589 (v01   BXPC   BXDSDT 00000001 INTL 20100528)
[    0.000000] ACPI: FACS 000000001fffff40 00040
[    0.000000] ACPI: SSDT 000000001fffd7e0 001C1 (v01 BOCHS  BXPCSSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: APIC 000000001fffd6e0 0008A (v01 BOCHS  BXPCAPIC 00000001 BXPC 00000001)
[    0.000000] ACPI: HPET 000000001fffd6a0 00038 (v01 BOCHS  BXPCHPET 00000001 BXPC 00000001)
[    0.000000] kvm-clock: Using msrs 4b564d01 and 4b564d00
[    0.000000] kvm-clock: cpu 0, msr 0:18509c1, boot clock
[    0.000000] Zone PFN ranges:
[    0.000000]   DMA      0x00000010 -> 0x00001000
[    0.000000]   DMA32    0x00001000 -> 0x00100000
[    0.000000]   Normal   empty
[    0.000000] Movable zone start PFN for each node
[    0.000000] Early memory PFN ranges
[    0.000000]     0: 0x00000010 -> 0x0000009d
[    0.000000]     0: 0x00000100 -> 0x0001fffd
[    0.000000] ACPI: PM-Timer IO Port: 0xb008
[    0.000000] ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x01] lapic_id[0x01] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x03] lapic_id[0x03] enabled)
[    0.000000] ACPI: IOAPIC (id[0x04] address[0xfec00000] gsi_base[0])
[    0.000000] IOAPIC[0]: apic_id 4, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 5 global_irq 5 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 10 global_irq 10 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 11 global_irq 11 high level)
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a201 base: 0xfed00000
[    0.000000] SMP: Allowing 4 CPUs, 0 hotplug CPUs
[    0.000000] PM: Registered nosave memory: 000000000009d000 - 000000000009e000
[    0.000000] PM: Registered nosave memory: 000000000009e000 - 00000000000a0000
[    0.000000] PM: Registered nosave memory: 00000000000a0000 - 00000000000f0000
[    0.000000] PM: Registered nosave memory: 00000000000f0000 - 0000000000100000
[    0.000000] Allocating PCI resources starting at 20000000 (gap: 20000000:deffc000)
[    0.000000] Booting paravirtualized kernel on KVM
[    0.000000] setup_percpu: NR_CPUS:4 nr_cpumask_bits:4 nr_cpu_ids:4 nr_node_ids:1
[    0.000000] PERCPU: Embedded 474 pages/cpu @ffff88001f000000 s1910784 r8192 d22528 u2097152
[    0.000000] Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 128901
[    0.000000] Kernel command line: root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend
[    0.000000] PID hash table entries: 2048 (order: 2, 16384 bytes)
[    0.000000] Dentry cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.000000] Inode-cache hash table entries: 32768 (order: 6, 262144 bytes)
[    0.000000] xsave/xrstor: enabled xstate_bv 0x7, cntxt size 0x340
[    0.000000] Checking aperture...
[    0.000000] No AGP bridge found
[    0.000000] Memory: 481644k/524276k available (3162k kernel code, 460k absent, 42172k reserved, 3483k data, 2420k init)
[    0.000000] SLUB: Genslabs=15, HWalign=64, Order=0-3, MinObjects=0, CPUs=4, Nodes=1
[    0.000000] Preemptible hierarchical RCU implementation.
[    0.000000] NR_IRQS:4352 nr_irqs:712 16
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [tty0] enabled
[    0.000000] console [ttyS0] enabled
[    0.000000] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.000000] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.000000] ... MAX_LOCK_DEPTH:          48
[    0.000000] ... MAX_LOCKDEP_KEYS:        8191
[    0.000000] ... CLASSHASH_SIZE:          4096
[    0.000000] ... MAX_LOCKDEP_ENTRIES:     16384
[    0.000000] ... MAX_LOCKDEP_CHAINS:      32768
[    0.000000] ... CHAINHASH_SIZE:          16384
[    0.000000]  memory used by lock dependency info: 6367 kB
[    0.000000]  per task-struct memory footprint: 2688 bytes
[    0.000000] Detected 2691.258 MHz processor.
[    0.008000] Calibrating delay loop (skipped) preset value.. 5382.51 BogoMIPS (lpj=10765032)
[    0.008000] pid_max: default: 32768 minimum: 301
[    0.010143] Mount-cache hash table entries: 256
[    0.020001] ACPI: Core revision 20120111
[    0.141619] ftrace: allocating 14018 entries in 55 pages
[    0.151144] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.196275] CPU0: Intel(R) Core(TM) i7-2620M CPU @ 2.70GHz stepping 07
[    0.200011] APIC calibration not consistent with PM-Timer: 91ms instead of 100ms
[    0.200011] APIC delta adjusted to PM-Timer: 6250032 (5749686)
[    0.200011] Performance Events: unsupported p6 CPU model 42 no PMU driver, software events only.
[    0.220527] lockdep: fixing up alternatives.
[    0.232533] Booting Node   0, Processors  #1
[    0.008000] kvm-clock: cpu 1, msr 0:1f3d19c1, secondary cpu clock
[    0.256113] lockdep: fixing up alternatives.
[    0.257316]  #2
[    0.008000] kvm-clock: cpu 2, msr 0:1f5d19c1, secondary cpu clock
[    0.280143] lockdep: fixing up alternatives.
[    0.281629]  #3 Ok.
[    0.008000] kvm-clock: cpu 3, msr 0:1f7d19c1, secondary cpu clock
[    0.296077] Brought up 4 CPUs
[    0.296992] Total of 4 processors activated (21530.06 BogoMIPS).
[    0.316988] NET: Registered protocol family 16
[    0.321184] ACPI: bus type pci registered
[    0.322202] PCI: Using configuration type 1 for base access
[    0.348634] bio: create slab <bio-0> at 0
[    0.350786] ACPI: Added _OSI(Module Device)
[    0.353256] ACPI: Added _OSI(Processor Device)
[    0.354509] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.356065] ACPI: Added _OSI(Processor Aggregator Device)
[    0.559295] ACPI: Interpreter enabled
[    0.560039] ACPI: (supports S0 S3 S4 S5)
[    0.562446] ACPI: Using IOAPIC for interrupt routing
[    0.857221] ACPI: No dock devices found.
[    0.858208] PCI: Ignoring host bridge windows from ACPI; if necessary, use "pci=use_crs" and report a bug
[    0.861656] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.864746] PCI host bridge to bus 0000:00
[    0.865675] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
[    0.866989] pci_bus 0000:00: root bus resource [mem 0x00000000-0xffffffffff]
[    0.871061] pci 0000:00:01.3: quirk: [io  0xb000-0xb03f] claimed by PIIX4 ACPI
[    0.872068] pci 0000:00:01.3: quirk: [io  0xb100-0xb10f] claimed by PIIX4 SMB
[    0.949295]  pci0000:00: Unable to request _OSC control (_OSC support mask: 0x18)
[    1.558297] ACPI: PCI Interrupt Link [LNKA] (IRQs 5 *10 11)
[    1.563546] ACPI: PCI Interrupt Link [LNKB] (IRQs 5 *10 11)
[    1.570707] ACPI: PCI Interrupt Link [LNKC] (IRQs 5 10 *11)
[    1.576655] ACPI: PCI Interrupt Link [LNKD] (IRQs 5 10 *11)
[    1.582843] ACPI: PCI Interrupt Link [LNKS] (IRQs 9) *0
[    1.586374] vgaarb: device added: PCI:0000:00:02.0,decodes=io+mem,owns=io+mem,locks=none
[    1.588884] vgaarb: loaded
[    1.589714] vgaarb: bridge control possible 0000:00:02.0
[    1.597144] SCSI subsystem initialized
[    1.600542] PCI: Using ACPI for IRQ routing
[    1.606292] HPET: 3 timers in total, 0 timers will be used for per-cpu timer
[    1.626553] Switching to clocksource kvm-clock
[    1.895262] pnp: PnP ACPI init
[    1.896376] ACPI: bus type pnp registered
[    1.926511] pnp: PnP ACPI: found 8 devices
[    1.927524] ACPI: ACPI bus type pnp unregistered
[    1.971474] NET: Registered protocol family 2
[    1.973160] IP route cache hash table entries: 4096 (order: 3, 32768 bytes)
[    1.975901] TCP established hash table entries: 16384 (order: 6, 262144 bytes)
[    1.978510] TCP bind hash table entries: 16384 (order: 8, 1310720 bytes)
[    1.981014] TCP: Hash tables configured (established 16384 bind 16384)
[    1.982280] TCP reno registered
[    1.983765] UDP hash table entries: 256 (order: 3, 49152 bytes)
[    1.985783] UDP-Lite hash table entries: 256 (order: 3, 49152 bytes)
[    1.989160] NET: Registered protocol family 1
[    1.990739] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    1.992981] pci 0000:00:01.0: PIIX3: Enabling Passive Release
[    1.994737] pci 0000:00:01.0: Activating ISA DMA hang workarounds
[    1.997711] Trying to unpack rootfs image as initramfs...
[    2.233326] debug: unmapping init memory ffff88001fa2e000..ffff88001fff0000
[    2.245530] audit: initializing netlink socket (disabled)
[    2.247191] type=2000 audit(1328879438.240:1): initialized
[    2.254089] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    2.354852] msgmni has been set to 940
[    2.358392] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 254)
[    2.361069] io scheduler noop registered (default)
[    2.362417] start plist test
[    2.364468] end plist test
[    2.366458] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[    2.367958] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[    2.371062] acpiphp: Slot [1] registered
[    2.373414] acpiphp: Slot [2] registered
[    2.375738] acpiphp: Slot [3] registered
[    2.378232] acpiphp: Slot [4] registered
[    2.380430] acpiphp: Slot [5] registered
[    2.382982] acpiphp: Slot [6] registered
[    2.385529] acpiphp: Slot [7] registered
[    2.394686] acpiphp: Slot [8] registered
[    2.396926] acpiphp: Slot [9] registered
[    2.399437] acpiphp: Slot [10] registered
[    2.402496] acpiphp: Slot [11] registered
[    2.405816] acpiphp: Slot [12] registered
[    2.408017] acpiphp: Slot [13] registered
[    2.410715] acpiphp: Slot [14] registered
[    2.412753] acpiphp: Slot [15] registered
[    2.414808] acpiphp: Slot [16] registered
[    2.416849] acpiphp: Slot [17] registered
[    2.418799] acpiphp: Slot [18] registered
[    2.420801] acpiphp: Slot [19] registered
[    2.422858] acpiphp: Slot [20] registered
[    2.425851] acpiphp: Slot [21] registered
[    2.427757] acpiphp: Slot [22] registered
[    2.429776] acpiphp: Slot [23] registered
[    2.431693] acpiphp: Slot [24] registered
[    2.434041] acpiphp: Slot [25] registered
[    2.436163] acpiphp: Slot [26] registered
[    2.438146] acpiphp: Slot [27] registered
[    2.441263] acpiphp: Slot [28] registered
[    2.443180] acpiphp: Slot [29] registered
[    2.445183] acpiphp: Slot [30] registered
[    2.447125] acpiphp: Slot [31] registered
[    2.638121] ACPI: PCI Interrupt Link [LNKC] enabled at IRQ 11
[    2.645567] ACPI: PCI Interrupt Link [LNKD] enabled at IRQ 10
[    2.651847] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
�[    2.932550] serial8250: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
[    2.960030] 00:06: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
[    2.964517] Non-volatile memory driver v1.3
[    2.965508] Linux agpgart interface v0.103
[    3.002429] brd: module loaded
[    3.023044] loop: module loaded
[    3.030536]  vda: vda1 vda2
[    3.046770] scsi0 : ata_piix
[    3.052222] scsi1 : ata_piix
[    3.054620] ata1: PATA max MWDMA2 cmd 0x1f0 ctl 0x3f6 bmdma 0xc060 irq 14
[    3.055938] ata2: PATA max MWDMA2 cmd 0x170 ctl 0x376 bmdma 0xc068 irq 15
[    3.065731] e1000: Intel(R) PRO/1000 Network Driver - version 7.3.21-k8-NAPI
[    3.067108] e1000: Copyright (c) 1999-2006 Intel Corporation.
[    3.069335] i8042: PNP: PS/2 Controller [PNP0303:KBD,PNP0f13:MOU] at 0x60,0x64 irq 1,12
[    3.072770] serio: i8042 KBD port at 0x60,0x64 irq 1
[    3.074612] serio: i8042 AUX port at 0x60,0x64 irq 12
[    3.077138] mousedev: PS/2 mouse device common for all mice
[    3.080696] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input0
[    3.086992] device-mapper: ioctl: 4.22.0-ioctl (2011-10-19) initialised: dm-devel@redhat.com
[    3.089016] cpuidle: using governor ladder
[    3.089979] cpuidle: using governor menu
[    3.091007] TCP cubic registered
[    3.091920] NET: Registered protocol family 17
[    3.095075] kmemleak: Kernel memory leak detector initialized
[    3.095122] kmemleak: Automatic memory scanning thread started
[    3.213173] ata2.00: ATAPI: QEMU DVD-ROM, 1.0.50, max UDMA/100
[    3.215130] ata2.00: configured for MWDMA2
[    3.218559] scsi 1:0:0:0: CD-ROM            QEMU     QEMU DVD-ROM     1.0. PQ: 0 ANSI: 5
[    3.223417] sr0: scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray
[    3.224743] cdrom: Uniform CD-ROM driver Revision: 3.20
[    3.232171] Refined TSC clocksource calibration: 2691.272 MHz.
[    3.233667] debug: unmapping init memory ffffffff8167f000..ffffffff818dc000
[    3.235507] Write protecting the kernel read-only data: 6144k
[    3.237158] debug: unmapping init memory ffff880001318000..ffff880001400000
[    3.238618] debug: unmapping init memory ffff8800015a0000..ffff880001600000
modprobe: FATAL: Could not load /lib/modules/3.3.0-rc2+/modules.dep: No such file or directory

[    3.372305] dracut: dracut-004-256.el6
[    3.463437] udev: starting version 147
[    3.465571] udevd (74): /proc/74/oom_adj is deprecated, please use /proc/74/oom_score_adj instead.
[    3.867773] dracut: Starting plymouth daemon
[    4.433636] EXT4-fs (vda1): INFO: recovery required on readonly filesystem
[    4.435071] EXT4-fs (vda1): write access will be enabled during recovery
[    4.514528] EXT4-fs (vda1): recovery complete
[    4.517143] EXT4-fs (vda1): mounted filesystem with ordered data mode. Opts: (null)
[    4.539644] dracut: Mounted root filesystem /dev/vda1
modprobe: FATAL: Could not load /lib/modules/3.3.0-rc2+/modules.dep: No such file or directory

[    4.706036] dracut: /sbin/load_policy: Can't load policy: No such device
[    4.902961] dracut: Switching root
     	       Welcome to Fedora 
Starting udev: [    5.302643] udev[224]: starting version 161
[  OK  ]
Setting hostname localhost.localdomain:  [  OK  ]
Checking filesystems
Checking all file systems.
[/sbin/fsck.ext4 (1) -- /] fsck.ext4 -a /dev/vda1 
/dev/vda1: clean, 31463/589824 files, 650384/2359040 blocks
[  OK  ]
Remounting root filesystem in read-write mode:  [    6.881187] EXT4-fs (vda1): re-mounted. Opts: (null)
[  OK  ]
Mounting local filesystems:  [  OK  ]
Enabling local filesystem quotas:  quotaon: Warning: No quota format detected in the kernel.
[  OK  ]
Enabling /etc/fstab swaps:  [    7.498226] Adding 1048572k swap on /dev/vda2.  Priority:-1 extents:1 across:1048572k 
[  OK  ]
Entering non-interactive startup
Starting auditd: [  OK  ]
Starting system logger: [  OK  ]
Starting irqbalance: [  OK  ]
Starting system message bus: [  OK  ]
Starting acpi daemon: [  OK  ]
Retrigger failed udev events[  OK  ]
Starting crond: [  OK  ]
[  OK  ] atd: [  OK  ]

Fedora release 14 (Laughlin)
Kernel 3.3.0-rc2+ on an x86_64 (/dev/ttyS0)

localhost.localdomain login: [   19.927622] PM: Syncing filesystems ... done.
[   19.949462] Freezing user space processes ... (elapsed 0.02 seconds) done.
[   19.973325] PM: Preallocating image memory... done (allocated 36953 pages)
[   20.027658] PM: Allocated 147812 kbytes in 0.05 seconds (2956.24 MB/s)
[   20.028688] Freezing remaining freezable tasks ... (elapsed 0.01 seconds) done.
[   20.051109] PM: freeze of devices complete after 5.348 msecs
[   20.054308] PM: late freeze of devices complete after 1.492 msecs
[   20.056204] ACPI: Preparing to enter system sleep state S4
[   20.058297] PM: Saving platform NVS memory
[   20.059239] Disabling non-boot CPUs ...
[   20.064208] CPU 1 is now offline
[   20.070176] CPU 2 is now offline
[   20.075103] CPU 3 is now offline
[   20.075929] lockdep: fixing up alternatives.
[   20.078259] PM: Creating hibernation image:
[   20.080938] PM: Need to copy 36297 pages
[   20.080938] PM: Hibernation image created (36297 pages copied)
[   20.080938] kvm-clock: cpu 0, msr 0:1f1d19c1, primary cpu clock, resume
[   20.086520] Enabling non-boot CPUs ...
[   20.098149] lockdep: fixing up alternatives.
[   20.098998] Booting Node 0 Processor 1 APIC 0x1
[   20.066584] kvm-clock: cpu 1, msr 0:1f3d19c1, secondary cpu clock
[   20.066584] Calibrating delay loop (skipped) already calibrated this CPU
[   20.140629] CPU1 is up
[   20.142184] lockdep: fixing up alternatives.
[   20.143416] Booting Node 0 Processor 2 APIC 0x2
[   20.072385] kvm-clock: cpu 2, msr 0:1f5d19c1, secondary cpu clock
[   20.072385] Calibrating delay loop (skipped) already calibrated this CPU
[   20.180614] CPU2 is up
[   20.182809] lockdep: fixing up alternatives.
[   20.183847] Booting Node 0 Processor 3 APIC 0x3
[   20.077703] kvm-clock: cpu 3, msr 0:1f7d19c1, secondary cpu clock
[   20.077703] Calibrating delay loop (skipped) already calibrated this CPU
[   20.224597] CPU3 is up
[   20.230960] ACPI: Waking up from system sleep state S4
[   20.233680] PM: early thaw of devices complete after 0.304 msecs
[   20.401644] ata2.00: configured for MWDMA2
[   20.403151] PM: thaw of devices complete after 168.149 msecs
[   20.408851] PM: Using 3 thread(s) for compression.
[   20.408853] PM: Compressing and saving image data (36368 pages) ... done
[   22.576196] PM: Wrote 145472 kbytes in 2.16 seconds (67.34 MB/s)
[   22.579450] PM: S|
[   22.636548] PM: hibernate of devices complete after 2.872 msecs
[   22.638290] PM: late hibernate of devices complete after 0.331 msecs
[   22.639740] ACPI: Preparing to enter system sleep state S4
[   22.641962] PM: Saving platform NVS memory
[   22.642935] Disabling non-boot CPUs ...
[   22.648517] CPU 1 is now offline
[   22.756125] CPU 2 is now offline
[   22.761957] CPU 3 is now offline
[   22.762818] lockdep: fixing up alternatives.


./x86_64-softmmu/qemu-system-x86_64 -kernel ~/src/linux/arch/x86/boot/bzImage  -append 'root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend' -initrd ~/tmp/initramfs-2.6.32-217.el6.x86_64.img -drive file=/guests/f14-suspend.qcow2,if=none,id=dr0 -device virtio-blk-pci,drive=dr0 -net nic,model=virtio -net user  -serial stdio  -enable-kvm -m 512  -cpu host -smp 4
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Linux version 3.3.0-rc2+ (amit@amit.redhat.com) (gcc version 4.6.2 20111027 (Red Hat 4.6.2-1) (GCC) ) #295 SMP PREEMPT Fri Feb 10 18:39:48 IST 2012
[    0.000000] Command line: root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend
[    0.000000] BIOS-provided physical RAM map:
[    0.000000]  BIOS-e820: 0000000000000000 - 000000000009dc00 (usable)
[    0.000000]  BIOS-e820: 000000000009dc00 - 00000000000a0000 (reserved)
[    0.000000]  BIOS-e820: 00000000000f0000 - 0000000000100000 (reserved)
[    0.000000]  BIOS-e820: 0000000000100000 - 000000001fffd000 (usable)
[    0.000000]  BIOS-e820: 000000001fffd000 - 0000000020000000 (reserved)
[    0.000000]  BIOS-e820: 00000000feffc000 - 00000000ff000000 (reserved)
[    0.000000]  BIOS-e820: 00000000fffc0000 - 0000000100000000 (reserved)
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] DMI 2.4 present.
[    0.000000] No AGP bridge found
[    0.000000] last_pfn = 0x1fffd max_arch_pfn = 0x400000000
[    0.000000] x86 PAT enabled: cpu 0, old 0x70406, new 0x7010600070106
[    0.000000] init_memory_mapping: 0000000000000000-000000001fffd000
[    0.000000] RAMDISK: 1fa2e000 - 1fff0000
[    0.000000] ACPI: RSDP 00000000000fd3f0 00014 (v00 BOCHS )
[    0.000000] ACPI: RSDT 000000001fffd660 00034 (v01 BOCHS  BXPCRSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: FACP 000000001fffff80 00074 (v01 BOCHS  BXPCFACP 00000001 BXPC 00000001)
[    0.000000] ACPI: DSDT 000000001fffd9b0 02589 (v01   BXPC   BXDSDT 00000001 INTL 20100528)
[    0.000000] ACPI: FACS 000000001fffff40 00040
[    0.000000] ACPI: SSDT 000000001fffd7e0 001C1 (v01 BOCHS  BXPCSSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: APIC 000000001fffd6e0 0008A (v01 BOCHS  BXPCAPIC 00000001 BXPC 00000001)
[    0.000000] ACPI: HPET 000000001fffd6a0 00038 (v01 BOCHS  BXPCHPET 00000001 BXPC 00000001)
[    0.000000] kvm-clock: Using msrs 4b564d01 and 4b564d00
[    0.000000] kvm-clock: cpu 0, msr 0:18509c1, boot clock
[    0.000000] Zone PFN ranges:
[    0.000000]   DMA      0x00000010 -> 0x00001000
[    0.000000]   DMA32    0x00001000 -> 0x00100000
[    0.000000]   Normal   empty
[    0.000000] Movable zone start PFN for each node
[    0.000000] Early memory PFN ranges
[    0.000000]     0: 0x00000010 -> 0x0000009d
[    0.000000]     0: 0x00000100 -> 0x0001fffd
[    0.000000] ACPI: PM-Timer IO Port: 0xb008
[    0.000000] ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x01] lapic_id[0x01] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x03] lapic_id[0x03] enabled)
[    0.000000] ACPI: IOAPIC (id[0x04] address[0xfec00000] gsi_base[0])
[    0.000000] IOAPIC[0]: apic_id 4, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 5 global_irq 5 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 10 global_irq 10 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 11 global_irq 11 high level)
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a201 base: 0xfed00000
[    0.000000] SMP: Allowing 4 CPUs, 0 hotplug CPUs
[    0.000000] PM: Registered nosave memory: 000000000009d000 - 000000000009e000
[    0.000000] PM: Registered nosave memory: 000000000009e000 - 00000000000a0000
[    0.000000] PM: Registered nosave memory: 00000000000a0000 - 00000000000f0000
[    0.000000] PM: Registered nosave memory: 00000000000f0000 - 0000000000100000
[    0.000000] Allocating PCI resources starting at 20000000 (gap: 20000000:deffc000)
[    0.000000] Booting paravirtualized kernel on KVM
[    0.000000] setup_percpu: NR_CPUS:4 nr_cpumask_bits:4 nr_cpu_ids:4 nr_node_ids:1
[    0.000000] PERCPU: Embedded 474 pages/cpu @ffff88001f000000 s1910784 r8192 d22528 u2097152
[    0.000000] Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 128901
[    0.000000] Kernel command line: root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend
[    0.000000] PID hash table entries: 2048 (order: 2, 16384 bytes)
[    0.000000] Dentry cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.000000] Inode-cache hash table entries: 32768 (order: 6, 262144 bytes)
[    0.000000] xsave/xrstor: enabled xstate_bv 0x7, cntxt size 0x340
[    0.000000] Checking aperture...
[    0.000000] No AGP bridge found
[    0.000000] Memory: 481644k/524276k available (3162k kernel code, 460k absent, 42172k reserved, 3483k data, 2420k init)
[    0.000000] SLUB: Genslabs=15, HWalign=64, Order=0-3, MinObjects=0, CPUs=4, Nodes=1
[    0.000000] Preemptible hierarchical RCU implementation.
[    0.000000] NR_IRQS:4352 nr_irqs:712 16
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [tty0] enabled
[    0.000000] console [ttyS0] enabled
[    0.000000] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.000000] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.000000] ... MAX_LOCK_DEPTH:          48
[    0.000000] ... MAX_LOCKDEP_KEYS:        8191
[    0.000000] ... CLASSHASH_SIZE:          4096
[    0.000000] ... MAX_LOCKDEP_ENTRIES:     16384
[    0.000000] ... MAX_LOCKDEP_CHAINS:      32768
[    0.000000] ... CHAINHASH_SIZE:          16384
[    0.000000]  memory used by lock dependency info: 6367 kB
[    0.000000]  per task-struct memory footprint: 2688 bytes
[    0.000000] Detected 2691.258 MHz processor.
[    0.008000] Calibrating delay loop (skipped) preset value.. 5382.51 BogoMIPS (lpj=10765032)
[    0.008000] pid_max: default: 32768 minimum: 301
[    0.012212] Mount-cache hash table entries: 256
[    0.018408] ACPI: Core revision 20120111
[    0.141561] ftrace: allocating 14018 entries in 55 pages
[    0.150614] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.195946] CPU0: Intel(R) Core(TM) i7-2620M CPU @ 2.70GHz stepping 07
[    0.200011] APIC calibration not consistent with PM-Timer: 91ms instead of 100ms
[    0.200011] APIC delta adjusted to PM-Timer: 6249972 (5729866)
[    0.200011] Performance Events: unsupported p6 CPU model 42 no PMU driver, software events only.
[    0.220474] lockdep: fixing up alternatives.
[    0.232381] Booting Node   0, Processors  #1
[    0.008000] kvm-clock: cpu 1, msr 0:1f3d19c1, secondary cpu clock
[    0.260131] lockdep: fixing up alternatives.
[    0.261537]  #2
[    0.008000] kvm-clock: cpu 2, msr 0:1f5d19c1, secondary cpu clock
[    0.284158] lockdep: fixing up alternatives.
[    0.285621]  #3 Ok.
[    0.008000] kvm-clock: cpu 3, msr 0:1f7d19c1, secondary cpu clock
[    0.300098] Brought up 4 CPUs
[    0.300948] Total of 4 processors activated (21530.06 BogoMIPS).
[    0.314752] NET: Registered protocol family 16
[    0.318516] ACPI: bus type pci registered
[    0.320059] PCI: Using configuration type 1 for base access
[    0.348760] bio: create slab <bio-0> at 0
[    0.351121] ACPI: Added _OSI(Module Device)
[    0.353139] ACPI: Added _OSI(Processor Device)
[    0.354521] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.356109] ACPI: Added _OSI(Processor Aggregator Device)
[    0.561419] ACPI: Interpreter enabled
[    0.562374] ACPI: (supports S0 S3 S4 S5)
[    0.564868] ACPI: Using IOAPIC for interrupt routing
[    0.859396] ACPI: No dock devices found.
[    0.860059] PCI: Ignoring host bridge windows from ACPI; if necessary, use "pci=use_crs" and report a bug
[    0.862495] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.866923] PCI host bridge to bus 0000:00
[    0.867948] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
[    0.868060] pci_bus 0000:00: root bus resource [mem 0x00000000-0xffffffffff]
[    0.873543] pci 0000:00:01.3: quirk: [io  0xb000-0xb03f] claimed by PIIX4 ACPI
[    0.875301] pci 0000:00:01.3: quirk: [io  0xb100-0xb10f] claimed by PIIX4 SMB
[    0.950413]  pci0000:00: Unable to request _OSC control (_OSC support mask: 0x18)
[    1.562832] ACPI: PCI Interrupt Link [LNKA] (IRQs 5 *10 11)
[    1.567293] ACPI: PCI Interrupt Link [LNKB] (IRQs 5 *10 11)
[    1.573939] ACPI: PCI Interrupt Link [LNKC] (IRQs 5 10 *11)
[    1.579086] ACPI: PCI Interrupt Link [LNKD] (IRQs 5 10 *11)
[    1.584993] ACPI: PCI Interrupt Link [LNKS] (IRQs 9) *0
[    1.591202] vgaarb: device added: PCI:0000:00:02.0,decodes=io+mem,owns=io+mem,locks=none
[    1.592213] vgaarb: loaded
[    1.593197] vgaarb: bridge control possible 0000:00:02.0
[    1.600683] SCSI subsystem initialized
[    1.602115] PCI: Using ACPI for IRQ routing
[    1.608286] HPET: 3 timers in total, 0 timers will be used for per-cpu timer
[    1.622720] Switching to clocksource kvm-clock
[    1.890639] pnp: PnP ACPI init
[    1.891666] ACPI: bus type pnp registered
[    1.922936] pnp: PnP ACPI: found 8 devices
[    1.923944] ACPI: ACPI bus type pnp unregistered
[    1.968395] NET: Registered protocol family 2
[    1.969976] IP route cache hash table entries: 4096 (order: 3, 32768 bytes)
[    1.973286] TCP established hash table entries: 16384 (order: 6, 262144 bytes)
[    1.976340] TCP bind hash table entries: 16384 (order: 8, 1310720 bytes)
[    1.980266] TCP: Hash tables configured (established 16384 bind 16384)
[    1.982352] TCP reno registered
[    1.983533] UDP hash table entries: 256 (order: 3, 49152 bytes)
[    1.985221] UDP-Lite hash table entries: 256 (order: 3, 49152 bytes)
[    1.987980] NET: Registered protocol family 1
[    1.989836] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    1.991366] pci 0000:00:01.0: PIIX3: Enabling Passive Release
[    1.993029] pci 0000:00:01.0: Activating ISA DMA hang workarounds
[    1.996522] Trying to unpack rootfs image as initramfs...
[    2.232249] debug: unmapping init memory ffff88001fa2e000..ffff88001fff0000
[    2.245977] audit: initializing netlink socket (disabled)
[    2.247348] type=2000 audit(1328879465.244:1): initialized
[    2.251524] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    2.351738] msgmni has been set to 940
[    2.355237] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 254)
[    2.357442] io scheduler noop registered (default)
[    2.358481] start plist test
[    2.359779] end plist test
[    2.361469] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[    2.362634] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[    2.366210] acpiphp: Slot [1] registered
[    2.368727] acpiphp: Slot [2] registered
[    2.370920] acpiphp: Slot [3] registered
[    2.373578] acpiphp: Slot [4] registered
[    2.375487] acpiphp: Slot [5] registered
[    2.377552] acpiphp: Slot [6] registered
[    2.379646] acpiphp: Slot [7] registered
[    2.387397] acpiphp: Slot [8] registered
[    2.390006] acpiphp: Slot [9] registered
[    2.391886] acpiphp: Slot [10] registered
[    2.394514] acpiphp: Slot [11] registered
[    2.396815] acpiphp: Slot [12] registered
[    2.398788] acpiphp: Slot [13] registered
[    2.400736] acpiphp: Slot [14] registered
[    2.402639] acpiphp: Slot [15] registered
[    2.405391] acpiphp: Slot [16] registered
[    2.407292] acpiphp: Slot [17] registered
[    2.409260] acpiphp: Slot [18] registered
[    2.411407] acpiphp: Slot [19] registered
[    2.413523] acpiphp: Slot [20] registered
[    2.415518] acpiphp: Slot [21] registered
[    2.417621] acpiphp: Slot [22] registered
[    2.419568] acpiphp: Slot [23] registered
[    2.422532] acpiphp: Slot [24] registered
[    2.424886] acpiphp: Slot [25] registered
[    2.426863] acpiphp: Slot [26] registered
[    2.428817] acpiphp: Slot [27] registered
[    2.430857] acpiphp: Slot [28] registered
[    2.432838] acpiphp: Slot [29] registered
[    2.434813] acpiphp: Slot [30] registered
[    2.437884] acpiphp: Slot [31] registered
[    2.625069] ACPI: PCI Interrupt Link [LNKC] enabled at IRQ 11
[    2.635091] ACPI: PCI Interrupt Link [LNKD] enabled at IRQ 10
[    2.638752] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
�[    2.920580] serial8250: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
[    2.947902] 00:06: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
[    2.952259] Non-volatile memory driver v1.3
[    2.953273] Linux agpgart interface v0.103
[    2.990450] brd: module loaded
[    3.010653] loop: module loaded
[    3.018104]  vda: vda1 vda2
[    3.035013] scsi0 : ata_piix
[    3.040861] scsi1 : ata_piix
[    3.043167] ata1: PATA max MWDMA2 cmd 0x1f0 ctl 0x3f6 bmdma 0xc060 irq 14
[    3.044520] ata2: PATA max MWDMA2 cmd 0x170 ctl 0x376 bmdma 0xc068 irq 15
[    3.053183] e1000: Intel(R) PRO/1000 Network Driver - version 7.3.21-k8-NAPI
[    3.054888] e1000: Copyright (c) 1999-2006 Intel Corporation.
[    3.058466] i8042: PNP: PS/2 Controller [PNP0303:KBD,PNP0f13:MOU] at 0x60,0x64 irq 1,12
[    3.063337] serio: i8042 KBD port at 0x60,0x64 irq 1
[    3.065592] serio: i8042 AUX port at 0x60,0x64 irq 12
[    3.068414] mousedev: PS/2 mouse device common for all mice
[    3.073460] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input0
[    3.077951] device-mapper: ioctl: 4.22.0-ioctl (2011-10-19) initialised: dm-devel@redhat.com
[    3.080248] cpuidle: using governor ladder
[    3.081293] cpuidle: using governor menu
[    3.082362] TCP cubic registered
[    3.083268] NET: Registered protocol family 17
[    3.087082] kmemleak: Kernel memory leak detector initialized
[    3.087108] kmemleak: Automatic memory scanning thread started
[    3.201317] ata2.00: ATAPI: QEMU DVD-ROM, 1.0.50, max UDMA/100
[    3.204954] ata2.00: configured for MWDMA2
[    3.208856] scsi 1:0:0:0: CD-ROM            QEMU     QEMU DVD-ROM     1.0. PQ: 0 ANSI: 5
[    3.213979] sr0: scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray
[    3.215259] cdrom: Uniform CD-ROM driver Revision: 3.20
[    3.224212] debug: unmapping init memory ffffffff8167f000..ffffffff818dc000
[    3.226117] Write protecting the kernel read-only data: 6144k
[    3.227745] debug: unmapping init memory ffff880001318000..ffff880001400000
[    3.229802] debug: unmapping init memory ffff8800015a0000..ffff880001600000
[    3.234772] Refined TSC clocksource calibration: 2691.274 MHz.
modprobe: FATAL: Could not load /lib/modules/3.3.0-rc2+/modules.dep: No such file or directory

[    3.376631] dracut: dracut-004-256.el6
[    3.465878] udev: starting version 147
[    3.468440] udevd (74): /proc/74/oom_adj is deprecated, please use /proc/74/oom_score_adj instead.
[    3.787549] dracut: Starting plymouth daemon
[    4.232996] PM: Starting manual resume from disk
[    4.276874] Freezing user space processes ... (elapsed 0.01 seconds) done.
[    4.304223] PM: Using 3 thread(s) for decompression.
[    4.304224] PM: Loading and decompressing image data (36368 pages) ... done
[    6.263529] PM: Read 145472 kbytes in 1.95 seconds (74.60 MB/s)
[    6.277105] PM: quiesce of devices complete after 5.331 msecs
[    6.279903] PM: late quiesce of devices complete after 1.229 msecs
[    6.281790] Disabling non-boot CPUs ...
[    6.286658] CPU 1 is now offline
[    6.392122] CPU 2 is now offline
[    6.397382] CPU 3 is now offline
[    6.397988] lockdep: fixing up alternatives.
[   20.080938] kvm-clock: cpu 0, msr 0:1f1d19c1, primary cpu clock, resume
[   20.080938] PM: Restoring platform NVS memory
[   20.086361] Enabling non-boot CPUs ...
[   20.097892] lockdep: fixing up alternatives.
[   20.099072] Booting Node 0 Processor 1 APIC 0x1
[   20.066584] kvm-clock: cpu 1, msr 0:1f3d19c1, secondary cpu clock
[   20.066584] Calibrating delay loop (skipped) already calibrated this CPU
[   20.140678] CPU1 is up
[   20.142345] lockdep: fixing up alternatives.
[   20.143531] Booting Node 0 Processor 2 APIC 0x2
[   20.072385] kvm-clock: cpu 2, msr 0:1f5d19c1, secondary cpu clock
[   20.072385] Calibrating delay loop (skipped) already calibrated this CPU
[   20.188725] CPU2 is up
[   20.191285] lockdep: fixing up alternatives.
[   20.192714] Booting Node 0 Processor 3 APIC 0x3
[   20.077703] kvm-clock: cpu 3, msr 0:1f7d19c1, secondary cpu clock
[   20.077703] Calibrating delay loop (skipped) already calibrated this CPU
[   20.232565] CPU3 is up
[   20.234208] ACPI: Waking up from system sleep state S4
[   20.236994] PM: early restore of devices complete after 1.341 msecs
[   20.449753] ata2.00: configured for MWDMA2
[   20.451505] PM: restore of devices complete after 164.925 msecs
[   20.454164] Restarting tasks ... done.
[   26.954508] PM: Syncing filesystems ... done.
[   26.962114] Freezing user space processes ... (elapsed 0.01 seconds) done.
[   26.981693] PM: Preallocating image memory... done (allocated 36779 pages)
[   27.031627] PM: Allocated 147116 kbytes in 0.04 seconds (3677.90 MB/s)
[   27.033069] Freezing remaining freezable tasks ... (elapsed 0.01 seconds) done.
[   27.053831] PM: freeze of devices complete after 3.511 msecs
[   27.057025] PM: late freeze of devices complete after 1.345 msecs
[   27.058970] ACPI: Preparing to enter system sleep state S4
[   27.061183] PM: Saving platform NVS memory
[   27.062449] Disabling non-boot CPUs ...
[   27.071546] Broke affinity for irq 1
[   27.172141] CPU 1 is now offline
[   27.280127] CPU 2 is now offline
[   27.285542] CPU 3 is now offline
[   27.286668] lockdep: fixing up alternatives.
[   27.289201] PM: Creating hibernation image:
[   27.292051] PM: Need to copy 36433 pages
[   27.292051] PM: Hibernation image created (36433 pages copied)
[   27.292051] kvm-clock: cpu 0, msr 0:1f1d19c1, primary cpu clock, resume
[   27.297473] Enabling non-boot CPUs ...
[   27.307663] lockdep: fixing up alternatives.
[   27.309283] Booting Node 0 Processor 1 APIC 0x1
[   27.074626] kvm-clock: cpu 1, msr 0:1f3d19c1, secondary cpu clock
[   27.074626] Calibrating delay loop (skipped) already calibrated this CPU
[   27.356732] CPU1 is up
[   27.359102] lockdep: fixing up alternatives.
[   27.360289] Booting Node 0 Processor 2 APIC 0x2
[   27.181025] kvm-clock: cpu 2, msr 0:1f5d19c1, secondary cpu clock
[   27.181025] Calibrating delay loop (skipped) already calibrated this CPU
[   27.404533] CPU2 is up
[   27.406678] lockdep: fixing up alternatives.
[   27.407643] Booting Node 0 Processor 3 APIC 0x3
[   27.288009] kvm-clock: cpu 3, msr 0:1f7d19c1, secondary cpu clock
[   27.288009] Calibrating delay loop (skipped) already calibrated this CPU
[   27.448610] CPU3 is up
[   27.453028] ACPI: Waking up from system sleep state S4
[   27.455146] PM: early thaw of devices complete after 0.398 msecs
[   27.621265] ata2.00: configured for MWDMA2
[   27.623344] PM: thaw of devices complete after 166.397 msecs
[   27.628990] PM: Using 3 thread(s) for compression.
[   27.628991] PM: Compressing and saving image data (36505 pages) ... done
[   29.748964] PM: Wrote 146020 kbytes in 2.11 seconds (69.20 MB/s)
[   29.752724] PM: S|
[   29.809222] PM: hibernate of devices complete after 2.545 msecs
[   29.810821] PM: late hibernate of devices complete after 0.281 msecs
[   29.812158] ACPI: Preparing to enter system sleep state S4
[   29.813841] PM: Saving platform NVS memory
[   29.814814] Disabling non-boot CPUs ...
[   29.920146] CPU 1 is now offline
[   30.028156] CPU 2 is now offline
[   30.033019] Broke affinity for irq 15
[   30.034989] CPU 3 is now offline
[   30.036411] lockdep: fixing up alternatives.



./x86_64-softmmu/qemu-system-x86_64 -kernel ~/src/linux/arch/x86/boot/bzImage  -append 'root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend' -initrd ~/tmp/initramfs-2.6.32-217.el6.x86_64.img -drive file=/guests/f14-suspend.qcow2,if=none,id=dr0 -device virtio-blk-pci,drive=dr0 -net nic,model=virtio -net user  -serial stdio  -enable-kvm -m 512  -cpu host -smp 4
[    0.000000] Initializing cgroup subsys cpu
[    0.000000] Linux version 3.3.0-rc2+ (amit@amit.redhat.com) (gcc version 4.6.2 20111027 (Red Hat 4.6.2-1) (GCC) ) #295 SMP PREEMPT Fri Feb 10 18:39:48 IST 2012
[    0.000000] Command line: root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend
[    0.000000] BIOS-provided physical RAM map:
[    0.000000]  BIOS-e820: 0000000000000000 - 000000000009dc00 (usable)
[    0.000000]  BIOS-e820: 000000000009dc00 - 00000000000a0000 (reserved)
[    0.000000]  BIOS-e820: 00000000000f0000 - 0000000000100000 (reserved)
[    0.000000]  BIOS-e820: 0000000000100000 - 000000001fffd000 (usable)
[    0.000000]  BIOS-e820: 000000001fffd000 - 0000000020000000 (reserved)
[    0.000000]  BIOS-e820: 00000000feffc000 - 00000000ff000000 (reserved)
[    0.000000]  BIOS-e820: 00000000fffc0000 - 0000000100000000 (reserved)
[    0.000000] NX (Execute Disable) protection: active
[    0.000000] DMI 2.4 present.
[    0.000000] No AGP bridge found
[    0.000000] last_pfn = 0x1fffd max_arch_pfn = 0x400000000
[    0.000000] x86 PAT enabled: cpu 0, old 0x70406, new 0x7010600070106
[    0.000000] init_memory_mapping: 0000000000000000-000000001fffd000
[    0.000000] RAMDISK: 1fa2e000 - 1fff0000
[    0.000000] ACPI: RSDP 00000000000fd3f0 00014 (v00 BOCHS )
[    0.000000] ACPI: RSDT 000000001fffd660 00034 (v01 BOCHS  BXPCRSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: FACP 000000001fffff80 00074 (v01 BOCHS  BXPCFACP 00000001 BXPC 00000001)
[    0.000000] ACPI: DSDT 000000001fffd9b0 02589 (v01   BXPC   BXDSDT 00000001 INTL 20100528)
[    0.000000] ACPI: FACS 000000001fffff40 00040
[    0.000000] ACPI: SSDT 000000001fffd7e0 001C1 (v01 BOCHS  BXPCSSDT 00000001 BXPC 00000001)
[    0.000000] ACPI: APIC 000000001fffd6e0 0008A (v01 BOCHS  BXPCAPIC 00000001 BXPC 00000001)
[    0.000000] ACPI: HPET 000000001fffd6a0 00038 (v01 BOCHS  BXPCHPET 00000001 BXPC 00000001)
[    0.000000] kvm-clock: Using msrs 4b564d01 and 4b564d00
[    0.000000] kvm-clock: cpu 0, msr 0:18509c1, boot clock
[    0.000000] Zone PFN ranges:
[    0.000000]   DMA      0x00000010 -> 0x00001000
[    0.000000]   DMA32    0x00001000 -> 0x00100000
[    0.000000]   Normal   empty
[    0.000000] Movable zone start PFN for each node
[    0.000000] Early memory PFN ranges
[    0.000000]     0: 0x00000010 -> 0x0000009d
[    0.000000]     0: 0x00000100 -> 0x0001fffd
[    0.000000] ACPI: PM-Timer IO Port: 0xb008
[    0.000000] ACPI: LAPIC (acpi_id[0x00] lapic_id[0x00] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x01] lapic_id[0x01] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x02] lapic_id[0x02] enabled)
[    0.000000] ACPI: LAPIC (acpi_id[0x03] lapic_id[0x03] enabled)
[    0.000000] ACPI: IOAPIC (id[0x04] address[0xfec00000] gsi_base[0])
[    0.000000] IOAPIC[0]: apic_id 4, version 17, address 0xfec00000, GSI 0-23
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 0 global_irq 2 dfl dfl)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 5 global_irq 5 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 9 global_irq 9 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 10 global_irq 10 high level)
[    0.000000] ACPI: INT_SRC_OVR (bus 0 bus_irq 11 global_irq 11 high level)
[    0.000000] Using ACPI (MADT) for SMP configuration information
[    0.000000] ACPI: HPET id: 0x8086a201 base: 0xfed00000
[    0.000000] SMP: Allowing 4 CPUs, 0 hotplug CPUs
[    0.000000] PM: Registered nosave memory: 000000000009d000 - 000000000009e000
[    0.000000] PM: Registered nosave memory: 000000000009e000 - 00000000000a0000
[    0.000000] PM: Registered nosave memory: 00000000000a0000 - 00000000000f0000
[    0.000000] PM: Registered nosave memory: 00000000000f0000 - 0000000000100000
[    0.000000] Allocating PCI resources starting at 20000000 (gap: 20000000:deffc000)
[    0.000000] Booting paravirtualized kernel on KVM
[    0.000000] setup_percpu: NR_CPUS:4 nr_cpumask_bits:4 nr_cpu_ids:4 nr_node_ids:1
[    0.000000] PERCPU: Embedded 474 pages/cpu @ffff88001f000000 s1910784 r8192 d22528 u2097152
[    0.000000] Built 1 zonelists in Zone order, mobility grouping on.  Total pages: 128901
[    0.000000] Kernel command line: root=/dev/vda1 console=tty0 console=ttyS0 no_console_suspend
[    0.000000] PID hash table entries: 2048 (order: 2, 16384 bytes)
[    0.000000] Dentry cache hash table entries: 65536 (order: 7, 524288 bytes)
[    0.000000] Inode-cache hash table entries: 32768 (order: 6, 262144 bytes)
[    0.000000] xsave/xrstor: enabled xstate_bv 0x7, cntxt size 0x340
[    0.000000] Checking aperture...
[    0.000000] No AGP bridge found
[    0.000000] Memory: 481644k/524276k available (3162k kernel code, 460k absent, 42172k reserved, 3483k data, 2420k init)
[    0.000000] SLUB: Genslabs=15, HWalign=64, Order=0-3, MinObjects=0, CPUs=4, Nodes=1
[    0.000000] Preemptible hierarchical RCU implementation.
[    0.000000] NR_IRQS:4352 nr_irqs:712 16
[    0.000000] Console: colour VGA+ 80x25
[    0.000000] console [tty0] enabled
[    0.000000] console [ttyS0] enabled
[    0.000000] Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar
[    0.000000] ... MAX_LOCKDEP_SUBCLASSES:  8
[    0.000000] ... MAX_LOCK_DEPTH:          48
[    0.000000] ... MAX_LOCKDEP_KEYS:        8191
[    0.000000] ... CLASSHASH_SIZE:          4096
[    0.000000] ... MAX_LOCKDEP_ENTRIES:     16384
[    0.000000] ... MAX_LOCKDEP_CHAINS:      32768
[    0.000000] ... CHAINHASH_SIZE:          16384
[    0.000000]  memory used by lock dependency info: 6367 kB
[    0.000000]  per task-struct memory footprint: 2688 bytes
[    0.000000] Detected 2691.258 MHz processor.
[    0.012000] Calibrating delay loop (skipped) preset value.. 5382.51 BogoMIPS (lpj=10765032)
[    0.012000] pid_max: default: 32768 minimum: 301
[    0.012000] Mount-cache hash table entries: 256
[    0.019120] ACPI: Core revision 20120111
[    0.144778] ftrace: allocating 14018 entries in 55 pages
[    0.153954] ..TIMER: vector=0x30 apic1=0 pin1=2 apic2=-1 pin2=-1
[    0.198922] CPU0: Intel(R) Core(TM) i7-2620M CPU @ 2.70GHz stepping 07
[    0.204012] APIC calibration not consistent with PM-Timer: 92ms instead of 100ms
[    0.204012] APIC delta adjusted to PM-Timer: 6249983 (5750078)
[    0.204012] Performance Events: unsupported p6 CPU model 42 no PMU driver, software events only.
[    0.224671] lockdep: fixing up alternatives.
[    0.236366] Booting Node   0, Processors  #1
[    0.012000] kvm-clock: cpu 1, msr 0:1f3d19c1, secondary cpu clock
[    0.264156] lockdep: fixing up alternatives.
[    0.265510]  #2
[    0.012000] kvm-clock: cpu 2, msr 0:1f5d19c1, secondary cpu clock
[    0.288199] lockdep: fixing up alternatives.
[    0.289706]  #3 Ok.
[    0.012000] kvm-clock: cpu 3, msr 0:1f7d19c1, secondary cpu clock
[    0.304090] Brought up 4 CPUs
[    0.305140] Total of 4 processors activated (21530.06 BogoMIPS).
[    0.320274] NET: Registered protocol family 16
[    0.324019] ACPI: bus type pci registered
[    0.324594] PCI: Using configuration type 1 for base access
[    0.353632] bio: create slab <bio-0> at 0
[    0.356647] ACPI: Added _OSI(Module Device)
[    0.358035] ACPI: Added _OSI(Processor Device)
[    0.359445] ACPI: Added _OSI(3.0 _SCP Extensions)
[    0.360842] ACPI: Added _OSI(Processor Aggregator Device)
[    0.565985] ACPI: Interpreter enabled
[    0.567277] ACPI: (supports S0 S3 S4 S5)
[    0.571106] ACPI: Using IOAPIC for interrupt routing
[    0.878052] ACPI: No dock devices found.
[    0.879152] PCI: Ignoring host bridge windows from ACPI; if necessary, use "pci=use_crs" and report a bug
[    0.880511] ACPI: PCI Root Bridge [PCI0] (domain 0000 [bus 00-ff])
[    0.885429] PCI host bridge to bus 0000:00
[    0.886466] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
[    0.887737] pci_bus 0000:00: root bus resource [mem 0x00000000-0xffffffffff]
[    0.892837] pci 0000:00:01.3: quirk: [io  0xb000-0xb03f] claimed by PIIX4 ACPI
[    0.894750] pci 0000:00:01.3: quirk: [io  0xb100-0xb10f] claimed by PIIX4 SMB
[    0.970335]  pci0000:00: Unable to request _OSC control (_OSC support mask: 0x18)
[    1.583715] ACPI: PCI Interrupt Link [LNKA] (IRQs 5 *10 11)
[    1.590390] ACPI: PCI Interrupt Link [LNKB] (IRQs 5 *10 11)
[    1.596519] ACPI: PCI Interrupt Link [LNKC] (IRQs 5 10 *11)
[    1.602490] ACPI: PCI Interrupt Link [LNKD] (IRQs 5 10 *11)
[    1.609401] ACPI: PCI Interrupt Link [LNKS] (IRQs 9) *0
[    1.614133] vgaarb: device added: PCI:0000:00:02.0,decodes=io+mem,owns=io+mem,locks=none
[    1.616905] vgaarb: loaded
[    1.617710] vgaarb: bridge control possible 0000:00:02.0
[    1.622992] SCSI subsystem initialized
[    1.625239] PCI: Using ACPI for IRQ routing
[    1.630700] HPET: 3 timers in total, 0 timers will be used for per-cpu timer
[    1.654453] Switching to clocksource kvm-clock
[    1.922667] pnp: PnP ACPI init
[    1.923703] ACPI: bus type pnp registered
[    1.954649] pnp: PnP ACPI: found 8 devices
[    1.955665] ACPI: ACPI bus type pnp unregistered
[    1.999911] NET: Registered protocol family 2
[    2.001578] IP route cache hash table entries: 4096 (order: 3, 32768 bytes)
[    2.004869] TCP established hash table entries: 16384 (order: 6, 262144 bytes)
[    2.006856] TCP bind hash table entries: 16384 (order: 8, 1310720 bytes)
[    2.009280] TCP: Hash tables configured (established 16384 bind 16384)
[    2.010583] TCP reno registered
[    2.011458] UDP hash table entries: 256 (order: 3, 49152 bytes)
[    2.013055] UDP-Lite hash table entries: 256 (order: 3, 49152 bytes)
[    2.015547] NET: Registered protocol family 1
[    2.016802] pci 0000:00:00.0: Limiting direct PCI/PCI transfers
[    2.018184] pci 0000:00:01.0: PIIX3: Enabling Passive Release
[    2.019566] pci 0000:00:01.0: Activating ISA DMA hang workarounds
[    2.022396] Trying to unpack rootfs image as initramfs...
[    2.258350] debug: unmapping init memory ffff88001fa2e000..ffff88001fff0000
[    2.271675] audit: initializing netlink socket (disabled)
[    2.273096] type=2000 audit(1328879486.272:1): initialized
[    2.279331] HugeTLB registered 2 MB page size, pre-allocated 0 pages
[    2.377735] msgmni has been set to 940
[    2.381564] Block layer SCSI generic (bsg) driver version 0.4 loaded (major 254)
[    2.383252] io scheduler noop registered (default)
[    2.384845] start plist test
[    2.386357] end plist test
[    2.387986] pci_hotplug: PCI Hot Plug PCI Core version: 0.5
[    2.389326] acpiphp: ACPI Hot Plug PCI Controller Driver version: 0.5
[    2.392128] acpiphp: Slot [1] registered
[    2.394065] acpiphp: Slot [2] registered
[    2.395957] acpiphp: Slot [3] registered
[    2.398352] acpiphp: Slot [4] registered
[    2.400313] acpiphp: Slot [5] registered
[    2.402295] acpiphp: Slot [6] registered
[    2.404291] acpiphp: Slot [7] registered
[    2.412976] acpiphp: Slot [8] registered
[    2.415126] acpiphp: Slot [9] registered
[    2.417209] acpiphp: Slot [10] registered
[    2.419484] acpiphp: Slot [11] registered
[    2.421641] acpiphp: Slot [12] registered
[    2.423644] acpiphp: Slot [13] registered
[    2.425771] acpiphp: Slot [14] registered
[    2.427717] acpiphp: Slot [15] registered
[    2.430512] acpiphp: Slot [16] registered
[    2.432502] acpiphp: Slot [17] registered
[    2.434420] acpiphp: Slot [18] registered
[    2.436616] acpiphp: Slot [19] registered
[    2.438625] acpiphp: Slot [20] registered
[    2.440626] acpiphp: Slot [21] registered
[    2.442586] acpiphp: Slot [22] registered
[    2.445915] acpiphp: Slot [23] registered
[    2.447828] acpiphp: Slot [24] registered
[    2.449849] acpiphp: Slot [25] registered
[    2.451754] acpiphp: Slot [26] registered
[    2.453822] acpiphp: Slot [27] registered
[    2.455815] acpiphp: Slot [28] registered
[    2.457862] acpiphp: Slot [29] registered
[    2.459817] acpiphp: Slot [30] registered
[    2.463363] acpiphp: Slot [31] registered
[    2.652004] ACPI: PCI Interrupt Link [LNKC] enabled at IRQ 11
[    2.662122] ACPI: PCI Interrupt Link [LNKD] enabled at IRQ 10
[    2.665594] Serial: 8250/16550 driver, 4 ports, IRQ sharing enabled
�[    2.944554] serial8250: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
[    2.972203] 00:06: ttyS0 at I/O 0x3f8 (irq = 4) is a 16550A
[    2.976490] Non-volatile memory driver v1.3
[    2.977506] Linux agpgart interface v0.103
[    3.014141] brd: module loaded
[    3.033693] loop: module loaded
[    3.041319]  vda: vda1 vda2
[    3.059399] scsi0 : ata_piix
[    3.065431] scsi1 : ata_piix
[    3.068000] ata1: PATA max MWDMA2 cmd 0x1f0 ctl 0x3f6 bmdma 0xc060 irq 14
[    3.070550] ata2: PATA max MWDMA2 cmd 0x170 ctl 0x376 bmdma 0xc068 irq 15
[    3.081620] e1000: Intel(R) PRO/1000 Network Driver - version 7.3.21-k8-NAPI
[    3.083229] e1000: Copyright (c) 1999-2006 Intel Corporation.
[    3.086006] i8042: PNP: PS/2 Controller [PNP0303:KBD,PNP0f13:MOU] at 0x60,0x64 irq 1,12
[    3.089426] serio: i8042 KBD port at 0x60,0x64 irq 1
[    3.091309] serio: i8042 AUX port at 0x60,0x64 irq 12
[    3.093860] mousedev: PS/2 mouse device common for all mice
[    3.097682] input: AT Translated Set 2 keyboard as /devices/platform/i8042/serio0/input/input0
[    3.107859] device-mapper: ioctl: 4.22.0-ioctl (2011-10-19) initialised: dm-devel@redhat.com
[    3.110152] cpuidle: using governor ladder
[    3.111213] cpuidle: using governor menu
[    3.112544] TCP cubic registered
[    3.113505] NET: Registered protocol family 17
[    3.116525] kmemleak: Kernel memory leak detector initialized
[    3.116550] kmemleak: Automatic memory scanning thread started
[    3.229160] ata2.00: ATAPI: QEMU DVD-ROM, 1.0.50, max UDMA/100
[    3.230765] ata2.00: configured for MWDMA2
[    3.234229] scsi 1:0:0:0: CD-ROM            QEMU     QEMU DVD-ROM     1.0. PQ: 0 ANSI: 5
[    3.240337] sr0: scsi3-mmc drive: 4x/4x cd/rw xa/form2 tray
[    3.241736] cdrom: Uniform CD-ROM driver Revision: 3.20
[    3.250342] debug: unmapping init memory ffffffff8167f000..ffffffff818dc000
[    3.252205] Write protecting the kernel read-only data: 6144k
[    3.254771] debug: unmapping init memory ffff880001318000..ffff880001400000
[    3.257081] debug: unmapping init memory ffff8800015a0000..ffff880001600000
[    3.260140] Refined TSC clocksource calibration: 2691.271 MHz.
modprobe: FATAL: Could not load /lib/modules/3.3.0-rc2+/modules.dep: No such file or directory

[    3.398866] dracut: dracut-004-256.el6
[    3.490652] udev: starting version 147
[    3.492541] udevd (74): /proc/74/oom_adj is deprecated, please use /proc/74/oom_score_adj instead.
[    3.810111] dracut: Starting plymouth daemon
[    4.288856] PM: Starting manual resume from disk
[    4.326388] Freezing user space processes ... (elapsed 0.02 seconds) done.
[    4.363688] PM: Using 3 thread(s) for decompression.
[    4.363689] PM: Loading and decompressing image data (36505 pages) ... done
[    6.317330] PM: Read 146020 kbytes in 1.94 seconds (75.26 MB/s)
[    6.331360] PM: quiesce of devices complete after 5.255 msecs
[    6.333865] PM: late quiesce of devices complete after 1.067 msecs
[    6.335089] Disabling non-boot CPUs ...
[    6.340486] CPU 1 is now offline
[    6.448137] CPU 2 is now offline
[    6.556109] CPU 3 is now offline
[    6.557140] lockdep: fixing up alternatives.
[   27.292051] kvm-clock: cpu 0, msr 0:1f1d19c1, primary cpu clock, resume
[   27.292051] PM: Restoring platform NVS memory
[   27.292051] Enabling non-boot CPUs ...
qemu: terminating on signal 2


		Amit

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 13:18         ` Amit Shah
@ 2012-02-10 20:58           ` Igor Mammedov
  2012-02-13 12:39             ` Amit Shah
  0 siblings, 1 reply; 24+ messages in thread
From: Igor Mammedov @ 2012-02-10 20:58 UTC (permalink / raw)
  To: Amit Shah
  Cc: Marcelo Tosatti, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi,
	Gleb Natapov

I've to revoke my ack and say NAK to this patch. Patch itself is in right
direction but clock restore happens too late.

With patch I've used to hunt down overflow for cpu hotplug
(maybe it will be better for it to be in kernel, which will help to detect
  overflow problem without spending a lot of time to debug it?):

diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c
index 42eb330..0081e10 100644
--- a/arch/x86/kernel/pvclock.c
+++ b/arch/x86/kernel/pvclock.c
@@ -43,7 +43,10 @@ void pvclock_set_flags(u8 flags)

  static u64 pvclock_get_nsec_offset(struct pvclock_shadow_time *shadow)
  {
-       u64 delta = native_read_tsc() - shadow->tsc_timestamp;
+       u64 tsc = native_read_tsc();
+       u64 shadow_timestamp = shadow->tsc_timestamp;
+       u64 delta = tsc - shadow_timestamp;
+       BUG_ON (tsc < shadow_timestamp);
         return pvclock_scale_delta(delta, shadow->tsc_to_nsec_mul,
                                    shadow->tsc_shift);
  }

I've got following back-trace:

#29 0xffffffff81004925 in oops_end (flags=70, regs=<optimized out>, signr=11) at arch/x86/kernel/dumpstack.c:246
#30 0xffffffff81004a77 in die (str=0xffffffff8152ca7f "invalid opcode", regs=0xffff8800149859e8, err=0)
     at arch/x86/kernel/dumpstack.c:305
#31 0xffffffff81001fed in do_trap (trapnr=6, signr=4, str=0xffffffff8152ca7f "invalid opcode", regs=0xffff8800149859e8,
     error_code=0, info=0xffff880014985948) at arch/x86/kernel/traps.c:168
#32 0xffffffff810021f7 in do_invalid_op (regs=0xffff8800149859e8, error_code=0) at arch/x86/kernel/traps.c:209
#33 <signal handler called>
#34 pvclock_get_nsec_offset (shadow=0xffff880014985a98) at arch/x86/kernel/pvclock.c:51
#35 pvclock_clocksource_read (src=0xffff88001f1d1ac0) at arch/x86/kernel/pvclock.c:107
#36 0xffffffff8101aa97 in kvm_clock_read () at arch/x86/kernel/kvmclock.c:79
#37 0xffffffff810087cc in paravirt_sched_clock () at /home/imammedov/fc15/linux-2.6/arch/x86/include/asm/paravirt.h:230
#38 sched_clock () at arch/x86/kernel/tsc.c:71
#39 0xffffffff8104f025 in sched_clock_local (scd=0xffff88001f1d2600) at kernel/sched/clock.c:147
#40 0xffffffff8104f1b7 in sched_clock_cpu (cpu=0) at kernel/sched/clock.c:232
#41 0xffffffff8104f1e5 in local_clock () at kernel/sched/clock.c:316
#42 0xffffffff810699d2 in lockstat_clock () at kernel/lockdep.c:158
#43 __lock_acquire (lock=0xffffffff816131b8, subclass=<optimized out>, trylock=0, read=0, check=2, hardirqs_off=<optimized out>,
     nest_lock=0x0, ip=18446744071578924098, references=0) at kernel/lockdep.c:3098
#44 0xffffffff8106a924 in lock_acquire (lock=0xffffffff816131b8, subclass=0, trylock=0, read=0, check=2, nest_lock=0x0,
     ip=18446744071578924098) at kernel/lockdep.c:3555
#45 0xffffffff8136b1c3 in __raw_spin_lock (lock=0xffffffff816131a0) at include/linux/spinlock_api_smp.h:143
#46 _raw_spin_lock (lock=0xffffffff816131a0) at kernel/spinlock.c:137
#47 0xffffffff81013442 in prepare_set () at arch/x86/kernel/cpu/mtrr/generic.c:676
#48 0xffffffff8101368c in generic_set_all () at arch/x86/kernel/cpu/mtrr/generic.c:723
#49 0xffffffff8101258b in mtrr_bp_restore () at arch/x86/kernel/cpu/mtrr/main.c:738
#50 0xffffffff812cabfa in __restore_processor_state (ctxt=<optimized out>) at arch/x86/power/cpu.c:227
#51 restore_processor_state () at arch/x86/power/cpu.c:233
---Type <return> to continue, or q <return> to quit---
#52 0xffffffff8105a1b6 in create_image (platform_mode=0) at kernel/power/hibernate.c:291
#53 hibernation_snapshot (platform_mode=0) at kernel/power/hibernate.c:363
#54 0xffffffff8105a825 in hibernate () at kernel/power/hibernate.c:629
#55 0xffffffff81058868 in state_store (kobj=<optimized out>, attr=<optimized out>,
     buf=0xffff88001366e000 <Address 0xffff88001366e000 out of bounds>, n=5) at kernel/power/main.c:284
#56 0xffffffff811e6cc3 in kobj_attr_store (kobj=<optimized out>, attr=<optimized out>, buf=<optimized out>,
     count=<optimized out>) at lib/kobject.c:699
#57 0xffffffff811372c2 in flush_write_buffer (count=<optimized out>, buffer=<optimized out>, dentry=<optimized out>)
     at fs/sysfs/file.c:202
#58 sysfs_write_file (file=<optimized out>, buf=0x7f9ce710d000 <Address 0x7f9ce710d000 out of bounds>, count=<optimized out>,
     ppos=0xffff880014985f58) at fs/sysfs/file.c:236
#59 0xffffffff810ebfbe in vfs_write (file=0xffff88001cb48000, buf=0x7f9ce710d000 <Address 0x7f9ce710d000 out of bounds>,
     count=<optimized out>, pos=0xffff880014985f58) at fs/read_write.c:435
#60 0xffffffff810ec175 in sys_write (fd=<optimized out>, buf=0x7f9ce710d000 <Address 0x7f9ce710d000 out of bounds>,
     count=<optimized out>) at fs/read_write.c:487

that shows an access to clock happens right before x86_platform.restore_sched_clock_state
is called.

Moving x86_platform.restore_sched_clock_state before mtrr_bp_restore solves issue.
It isn't bugged on me for 20 save/restore cycles with this change, without this change
it bugs on 2nd-3rd cycle.


BTW Amit,
your config doesn't have CONFIG_KVM_GUEST set, which causes primary cpu clock to be
uninitialized too in case of SMP kernel. Commit ca3f10172eea9b95b moved it from
kvmclock_init to kvm_guest_init but forgot to change Kconfig too and kvm_guest_init
is called only when KVM_GUEST is selected.  This commit created following implicit deps

if SMP && KVM_CLOCK then select KVM_GUEST

But I don't know if it's possible to express it in Kconfig. That rises a question:
Do we need KVM_CLOCK option? Maybe better to use KVM_GUEST instead and remove KVM_CLOCK option?
Or make KVM_CLOCK depend on KVM_GUEST...

^ permalink raw reply related	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 20:58           ` Igor Mammedov
@ 2012-02-13 12:39             ` Amit Shah
  0 siblings, 0 replies; 24+ messages in thread
From: Amit Shah @ 2012-02-13 12:39 UTC (permalink / raw)
  To: Igor Mammedov
  Cc: Marcelo Tosatti, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi,
	Gleb Natapov

On (Fri) 10 Feb 2012 [21:58:47], Igor Mammedov wrote:
> BTW Amit,
> your config doesn't have CONFIG_KVM_GUEST set, which causes primary cpu clock to be
> uninitialized too in case of SMP kernel.

Interesting.  I didn't notice that.  However, if I enable that option,
resume fails for me even the first time.

		Amit

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 12:43           ` Igor Mammedov
@ 2012-02-13 12:46             ` Amit Shah
  0 siblings, 0 replies; 24+ messages in thread
From: Amit Shah @ 2012-02-13 12:46 UTC (permalink / raw)
  To: Igor Mammedov
  Cc: Marcelo Tosatti, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On (Fri) 10 Feb 2012 [13:43:05], Igor Mammedov wrote:
> Another thing is to try smp guest without kvmclock and see if it helps.
> It might be just something else.

Nope, it's related to kvmclock.

		Amit

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-10 12:33         ` Marcelo Tosatti
  2012-02-10 12:43           ` Igor Mammedov
@ 2012-02-13 12:56           ` Amit Shah
  1 sibling, 0 replies; 24+ messages in thread
From: Amit Shah @ 2012-02-13 12:56 UTC (permalink / raw)
  To: Marcelo Tosatti
  Cc: Igor Mammedov, kvm, tglx, mingo, hpa, x86, johnstul, riel, avi

On (Fri) 10 Feb 2012 [10:33:37], Marcelo Tosatti wrote:
> On Fri, Feb 10, 2012 at 10:32:16AM -0200, Marcelo Tosatti wrote:
> > On Fri, Feb 10, 2012 at 03:32:11PM +0530, Amit Shah wrote:
> > > On (Thu) 09 Feb 2012 [16:13:29], Igor Mammedov wrote:
> > > 
> > > > Stalls are probably caused by uninitialized percpu hv_clock, with
> > > > following patch I don't see stalls. Although I might be just lucky.
> > > > http://git.kernel.org/?p=virt/kvm/kvm.git;a=commit;h=e2971ac7e1d186af059e088d305496c5cb47d487
> > > 
> > > Your commit does make things better, I don't see any stalls on the
> > > first resume.
> > > 
> > > However, a subsequent s4 causes the stall to re-appear on resume, and
> > > this time there are no stall messages; the kernel just sits there
> > > spinning on something.  I've not found the solution to this one yet (I
> > > had a commit similar to Marcelo's in the works, which got me to the
> > > previous works-but-stalls behaviour).
> > 
> > I cannot reproduce it here. Suspend/resume are operating normally after
> > several iterations. Igor do you see anything similar?
> > 
> > Amit, can you please enable CONFIG_PRINTK_TIME=y and post a full dmesg 
> > (both during suspend and also the new kernel during resume).
> 
> Also is it reproducible with UP guest?

Yes, it is.

		Amit

^ permalink raw reply	[flat|nested] 24+ messages in thread

* x86: kvmclock: abstract save/restore sched_clock_state (v2)
  2012-02-07 21:05 x86: kvmclock: abstract save/restore sched_clock_state Marcelo Tosatti
  2012-02-08 10:53 ` Igor Mammedov
  2012-02-09 12:27 ` Amit Shah
@ 2012-02-13 13:07 ` Marcelo Tosatti
  2012-02-13 15:20   ` Igor Mammedov
                     ` (2 more replies)
  2012-03-01  9:58 ` x86: kvmclock: abstract save/restore sched_clock_state Thomas Gleixner
  3 siblings, 3 replies; 24+ messages in thread
From: Marcelo Tosatti @ 2012-02-13 13:07 UTC (permalink / raw)
  To: kvm; +Cc: tglx, mingo, hpa, x86, johnstul, riel, avi, imammedo, amit.shah


Upon resume from hibernation, CPU 0's hvclock area contains the old
values for system_time and tsc_timestamp. It is necessary for the
hypervisor to update these values with uptodate ones before the CPU uses
them.

Abstract TSC's save/restore sched_clock_state functions and use
restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.

Also move restore_sched_clock_state before __restore_processor_state,
since the later calls CONFIG_LOCK_STAT's lockstat_clock (also for TSC).
Thanks to Igor Mammedov for tracking it down.

Fixes suspend-to-disk with kvmclock.

Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index 15d9915..c91e8b9 100644
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -61,7 +61,7 @@ extern void check_tsc_sync_source(int cpu);
 extern void check_tsc_sync_target(void);
 
 extern int notsc_setup(char *);
-extern void save_sched_clock_state(void);
-extern void restore_sched_clock_state(void);
+extern void tsc_save_sched_clock_state(void);
+extern void tsc_restore_sched_clock_state(void);
 
 #endif /* _ASM_X86_TSC_H */
diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index 5d0afac..baaca8d 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -162,6 +162,8 @@ struct x86_cpuinit_ops {
  * @is_untracked_pat_range	exclude from PAT logic
  * @nmi_init			enable NMI on cpus
  * @i8042_detect		pre-detect if i8042 controller exists
+ * @save_sched_clock_state:	save state for sched_clock() on suspend
+ * @restore_sched_clock_state:	restore state for sched_clock() on resume
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -173,6 +175,8 @@ struct x86_platform_ops {
 	void (*nmi_init)(void);
 	unsigned char (*get_nmi_reason)(void);
 	int (*i8042_detect)(void);
+	void (*save_sched_clock_state)(void);
+	void (*restore_sched_clock_state)(void);
 };
 
 struct pci_dev;
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index ca4e735..57e6b78 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -136,6 +136,15 @@ int kvm_register_clock(char *txt)
 	return ret;
 }
 
+void kvm_save_sched_clock_state(void)
+{
+}
+
+void kvm_restore_sched_clock_state(void)
+{
+	kvm_register_clock("primary cpu clock, resume");
+}
+
 #ifdef CONFIG_X86_LOCAL_APIC
 static void __cpuinit kvm_setup_secondary_clock(void)
 {
@@ -195,6 +204,8 @@ void __init kvmclock_init(void)
 	x86_cpuinit.early_percpu_clock_init =
 		kvm_setup_secondary_clock;
 #endif
+	x86_platform.save_sched_clock_state = kvm_save_sched_clock_state;
+	x86_platform.restore_sched_clock_state = kvm_restore_sched_clock_state;
 	machine_ops.shutdown  = kvm_shutdown;
 #ifdef CONFIG_KEXEC
 	machine_ops.crash_shutdown  = kvm_crash_shutdown;
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index a62c201..aed2aa1 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -629,7 +629,7 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
 
 static unsigned long long cyc2ns_suspend;
 
-void save_sched_clock_state(void)
+void tsc_save_sched_clock_state(void)
 {
 	if (!sched_clock_stable)
 		return;
@@ -645,7 +645,7 @@ void save_sched_clock_state(void)
  * that sched_clock() continues from the point where it was left off during
  * suspend.
  */
-void restore_sched_clock_state(void)
+void tsc_restore_sched_clock_state(void)
 {
 	unsigned long long offset;
 	unsigned long flags;
diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
index 6f2ec53..e9f265f 100644
--- a/arch/x86/kernel/x86_init.c
+++ b/arch/x86/kernel/x86_init.c
@@ -108,7 +108,9 @@ struct x86_platform_ops x86_platform = {
 	.is_untracked_pat_range		= is_ISA_range,
 	.nmi_init			= default_nmi_init,
 	.get_nmi_reason			= default_get_nmi_reason,
-	.i8042_detect			= default_i8042_detect
+	.i8042_detect			= default_i8042_detect,
+	.save_sched_clock_state 	= tsc_save_sched_clock_state,
+	.restore_sched_clock_state 	= tsc_restore_sched_clock_state,
 };
 
 EXPORT_SYMBOL_GPL(x86_platform);
diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
index f10c0af..0e76a28 100644
--- a/arch/x86/power/cpu.c
+++ b/arch/x86/power/cpu.c
@@ -114,7 +114,7 @@ static void __save_processor_state(struct saved_context *ctxt)
 void save_processor_state(void)
 {
 	__save_processor_state(&saved_context);
-	save_sched_clock_state();
+	x86_platform.save_sched_clock_state();
 }
 #ifdef CONFIG_X86_32
 EXPORT_SYMBOL(save_processor_state);
@@ -230,8 +230,8 @@ static void __restore_processor_state(struct saved_context *ctxt)
 /* Needed by apm.c */
 void restore_processor_state(void)
 {
+	x86_platform.restore_sched_clock_state();
 	__restore_processor_state(&saved_context);
-	restore_sched_clock_state();
 }
 #ifdef CONFIG_X86_32
 EXPORT_SYMBOL(restore_processor_state);


^ permalink raw reply related	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state (v2)
  2012-02-13 13:07 ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Marcelo Tosatti
@ 2012-02-13 15:20   ` Igor Mammedov
  2012-02-13 15:52     ` Marcelo Tosatti
  2012-02-13 15:45   ` [PATCH RFC] pvclock: Make pv_clock more robust and fixup it if overflow happens Igor Mammedov
  2012-02-13 16:26   ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Amit Shah
  2 siblings, 1 reply; 24+ messages in thread
From: Igor Mammedov @ 2012-02-13 15:20 UTC (permalink / raw)
  To: Marcelo Tosatti
  Cc: kvm, tglx, mingo, hpa, x86, johnstul, riel, avi, amit.shah

On 02/13/2012 02:07 PM, Marcelo Tosatti wrote:
>
> Upon resume from hibernation, CPU 0's hvclock area contains the old
> values for system_time and tsc_timestamp. It is necessary for the
> hypervisor to update these values with uptodate ones before the CPU uses
> them.
>
> Abstract TSC's save/restore sched_clock_state functions and use
> restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.
>
> Also move restore_sched_clock_state before __restore_processor_state,
> since the later calls CONFIG_LOCK_STAT's lockstat_clock (also for TSC).
> Thanks to Igor Mammedov for tracking it down.
>
> Fixes suspend-to-disk with kvmclock.
>
> Signed-off-by: Marcelo Tosatti<mtosatti@redhat.com>
>
> diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
> index 15d9915..c91e8b9 100644
> --- a/arch/x86/include/asm/tsc.h
> +++ b/arch/x86/include/asm/tsc.h
> @@ -61,7 +61,7 @@ extern void check_tsc_sync_source(int cpu);
>   extern void check_tsc_sync_target(void);
>
>   extern int notsc_setup(char *);
> -extern void save_sched_clock_state(void);
> -extern void restore_sched_clock_state(void);
> +extern void tsc_save_sched_clock_state(void);
> +extern void tsc_restore_sched_clock_state(void);
>
>   #endif /* _ASM_X86_TSC_H */
> diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
> index 5d0afac..baaca8d 100644
> --- a/arch/x86/include/asm/x86_init.h
> +++ b/arch/x86/include/asm/x86_init.h
> @@ -162,6 +162,8 @@ struct x86_cpuinit_ops {
>    * @is_untracked_pat_range	exclude from PAT logic
>    * @nmi_init			enable NMI on cpus
>    * @i8042_detect		pre-detect if i8042 controller exists
> + * @save_sched_clock_state:	save state for sched_clock() on suspend
> + * @restore_sched_clock_state:	restore state for sched_clock() on resume
>    */
>   struct x86_platform_ops {
>   	unsigned long (*calibrate_tsc)(void);
> @@ -173,6 +175,8 @@ struct x86_platform_ops {
>   	void (*nmi_init)(void);
>   	unsigned char (*get_nmi_reason)(void);
>   	int (*i8042_detect)(void);
> +	void (*save_sched_clock_state)(void);
> +	void (*restore_sched_clock_state)(void);
>   };
>
>   struct pci_dev;
> diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
> index ca4e735..57e6b78 100644
> --- a/arch/x86/kernel/kvmclock.c
> +++ b/arch/x86/kernel/kvmclock.c
> @@ -136,6 +136,15 @@ int kvm_register_clock(char *txt)
>   	return ret;
>   }
>
> +void kvm_save_sched_clock_state(void)
> +{
> +}
> +
> +void kvm_restore_sched_clock_state(void)
> +{
> +	kvm_register_clock("primary cpu clock, resume");
> +}
> +
>   #ifdef CONFIG_X86_LOCAL_APIC
>   static void __cpuinit kvm_setup_secondary_clock(void)
>   {
> @@ -195,6 +204,8 @@ void __init kvmclock_init(void)
>   	x86_cpuinit.early_percpu_clock_init =
>   		kvm_setup_secondary_clock;
>   #endif
> +	x86_platform.save_sched_clock_state = kvm_save_sched_clock_state;
> +	x86_platform.restore_sched_clock_state = kvm_restore_sched_clock_state;
>   	machine_ops.shutdown  = kvm_shutdown;
>   #ifdef CONFIG_KEXEC
>   	machine_ops.crash_shutdown  = kvm_crash_shutdown;
> diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
> index a62c201..aed2aa1 100644
> --- a/arch/x86/kernel/tsc.c
> +++ b/arch/x86/kernel/tsc.c
> @@ -629,7 +629,7 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
>
>   static unsigned long long cyc2ns_suspend;
>
> -void save_sched_clock_state(void)
> +void tsc_save_sched_clock_state(void)
>   {
>   	if (!sched_clock_stable)
>   		return;
> @@ -645,7 +645,7 @@ void save_sched_clock_state(void)
>    * that sched_clock() continues from the point where it was left off during
>    * suspend.
>    */
> -void restore_sched_clock_state(void)
> +void tsc_restore_sched_clock_state(void)
>   {
>   	unsigned long long offset;
>   	unsigned long flags;
> diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
> index 6f2ec53..e9f265f 100644
> --- a/arch/x86/kernel/x86_init.c
> +++ b/arch/x86/kernel/x86_init.c
> @@ -108,7 +108,9 @@ struct x86_platform_ops x86_platform = {
>   	.is_untracked_pat_range		= is_ISA_range,
>   	.nmi_init			= default_nmi_init,
>   	.get_nmi_reason			= default_get_nmi_reason,
> -	.i8042_detect			= default_i8042_detect
> +	.i8042_detect			= default_i8042_detect,
> +	.save_sched_clock_state 	= tsc_save_sched_clock_state,
> +	.restore_sched_clock_state 	= tsc_restore_sched_clock_state,
>   };
>
>   EXPORT_SYMBOL_GPL(x86_platform);
> diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
> index f10c0af..0e76a28 100644
> --- a/arch/x86/power/cpu.c
> +++ b/arch/x86/power/cpu.c
> @@ -114,7 +114,7 @@ static void __save_processor_state(struct saved_context *ctxt)
>   void save_processor_state(void)
>   {
>   	__save_processor_state(&saved_context);
> -	save_sched_clock_state();
> +	x86_platform.save_sched_clock_state();
>   }
>   #ifdef CONFIG_X86_32
>   EXPORT_SYMBOL(save_processor_state);
> @@ -230,8 +230,8 @@ static void __restore_processor_state(struct saved_context *ctxt)
>   /* Needed by apm.c */
>   void restore_processor_state(void)
>   {
> +	x86_platform.restore_sched_clock_state();
Isn't it too early? It is scarry to say hypervisor to write to some
memory location and than completely replace page-tables and half of
cpu state in __restore_processor_state. Wouldn't that have a potential
of writing into a place that is not restored hv_clock and restored
hv_clock might still be stale?

>   	__restore_processor_state(&saved_context);
> -	restore_sched_clock_state();
>   }
>   #ifdef CONFIG_X86_32
>   EXPORT_SYMBOL(restore_processor_state);
>
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

-- 
Thanks,
  Igor

^ permalink raw reply	[flat|nested] 24+ messages in thread

* [PATCH RFC] pvclock: Make pv_clock more robust and fixup it if overflow happens
  2012-02-13 13:07 ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Marcelo Tosatti
  2012-02-13 15:20   ` Igor Mammedov
@ 2012-02-13 15:45   ` Igor Mammedov
  2012-02-13 17:48     ` Marcelo Tosatti
  2012-02-13 16:26   ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Amit Shah
  2 siblings, 1 reply; 24+ messages in thread
From: Igor Mammedov @ 2012-02-13 15:45 UTC (permalink / raw)
  To: linux-kernel
  Cc: kvm, tglx, mingo, hpa, x86, jacob.jun.pan, alan, feng.tang,
	konrad.wilk, avi, glommer, johnstul, riel, tj, kosaki.motohiro,
	akpm, amit.shah

Instead of hunting misterious stalls/hungs all over the kernel when
overflow occurs at pvclock.c:pvclock_get_nsec_offset

    u64 delta = native_read_tsc() - shadow->tsc_timestamp;

and introducing hooks when places of unexpected access found, pv_clock
should be initialized for the calling cpu if overflow condition is detected.

Signed-off-by: Igor Mammedov <imammedo@redhat.com>
---
 arch/x86/kernel/pvclock.c |   18 +++++++++++++++---
 1 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c
index 42eb330..b486756 100644
--- a/arch/x86/kernel/pvclock.c
+++ b/arch/x86/kernel/pvclock.c
@@ -41,9 +41,14 @@ void pvclock_set_flags(u8 flags)
 	valid_flags = flags;
 }
 
-static u64 pvclock_get_nsec_offset(struct pvclock_shadow_time *shadow)
+static u64 pvclock_get_nsec_offset(struct pvclock_shadow_time *shadow,
+				   bool *overflow)
 {
-	u64 delta = native_read_tsc() - shadow->tsc_timestamp;
+	u64 delta;
+	u64 tsc = native_read_tsc();
+	u64 shadow_timestamp = shadow->tsc_timestamp;
+	*overflow = tsc < shadow_timestamp;
+	delta = tsc - shadow_timestamp;
 	return pvclock_scale_delta(delta, shadow->tsc_to_nsec_mul,
 				   shadow->tsc_shift);
 }
@@ -94,12 +99,19 @@ cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
 	unsigned version;
 	cycle_t ret, offset;
 	u64 last;
+	bool overflow;
 
 	do {
 		version = pvclock_get_time_values(&shadow, src);
 		barrier();
-		offset = pvclock_get_nsec_offset(&shadow);
+		offset = pvclock_get_nsec_offset(&shadow, &overflow);
 		ret = shadow.system_timestamp + offset;
+		if (unlikely(overflow)) {
+			memset(src, 0, sizeof(*src));
+			barrier();
+			x86_cpuinit.early_percpu_clock_init();
+			continue;
+		}
 		barrier();
 	} while (version != src->version);
 
-- 
1.7.7.6


^ permalink raw reply related	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state (v2)
  2012-02-13 15:20   ` Igor Mammedov
@ 2012-02-13 15:52     ` Marcelo Tosatti
  2012-02-15 10:28       ` Avi Kivity
  0 siblings, 1 reply; 24+ messages in thread
From: Marcelo Tosatti @ 2012-02-13 15:52 UTC (permalink / raw)
  To: Igor Mammedov; +Cc: kvm, tglx, mingo, hpa, x86, johnstul, riel, avi, amit.shah

On Mon, Feb 13, 2012 at 04:20:24PM +0100, Igor Mammedov wrote:
> On 02/13/2012 02:07 PM, Marcelo Tosatti wrote:
> >
> >Upon resume from hibernation, CPU 0's hvclock area contains the old
> >values for system_time and tsc_timestamp. It is necessary for the
> >hypervisor to update these values with uptodate ones before the CPU uses
> >them.
> >
> >Abstract TSC's save/restore sched_clock_state functions and use
> >restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.
> >
> >Also move restore_sched_clock_state before __restore_processor_state,
> >since the later calls CONFIG_LOCK_STAT's lockstat_clock (also for TSC).
> >Thanks to Igor Mammedov for tracking it down.
> >
> >Fixes suspend-to-disk with kvmclock.
> >
> >Signed-off-by: Marcelo Tosatti<mtosatti@redhat.com>
> >
> >diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
> >index 15d9915..c91e8b9 100644
> >--- a/arch/x86/include/asm/tsc.h
> >+++ b/arch/x86/include/asm/tsc.h
> >@@ -61,7 +61,7 @@ extern void check_tsc_sync_source(int cpu);
> >  extern void check_tsc_sync_target(void);
> >
> >  extern int notsc_setup(char *);
> >-extern void save_sched_clock_state(void);
> >-extern void restore_sched_clock_state(void);
> >+extern void tsc_save_sched_clock_state(void);
> >+extern void tsc_restore_sched_clock_state(void);
> >
> >  #endif /* _ASM_X86_TSC_H */
> >diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
> >index 5d0afac..baaca8d 100644
> >--- a/arch/x86/include/asm/x86_init.h
> >+++ b/arch/x86/include/asm/x86_init.h
> >@@ -162,6 +162,8 @@ struct x86_cpuinit_ops {
> >   * @is_untracked_pat_range	exclude from PAT logic
> >   * @nmi_init			enable NMI on cpus
> >   * @i8042_detect		pre-detect if i8042 controller exists
> >+ * @save_sched_clock_state:	save state for sched_clock() on suspend
> >+ * @restore_sched_clock_state:	restore state for sched_clock() on resume
> >   */
> >  struct x86_platform_ops {
> >  	unsigned long (*calibrate_tsc)(void);
> >@@ -173,6 +175,8 @@ struct x86_platform_ops {
> >  	void (*nmi_init)(void);
> >  	unsigned char (*get_nmi_reason)(void);
> >  	int (*i8042_detect)(void);
> >+	void (*save_sched_clock_state)(void);
> >+	void (*restore_sched_clock_state)(void);
> >  };
> >
> >  struct pci_dev;
> >diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
> >index ca4e735..57e6b78 100644
> >--- a/arch/x86/kernel/kvmclock.c
> >+++ b/arch/x86/kernel/kvmclock.c
> >@@ -136,6 +136,15 @@ int kvm_register_clock(char *txt)
> >  	return ret;
> >  }
> >
> >+void kvm_save_sched_clock_state(void)
> >+{
> >+}
> >+
> >+void kvm_restore_sched_clock_state(void)
> >+{
> >+	kvm_register_clock("primary cpu clock, resume");
> >+}
> >+
> >  #ifdef CONFIG_X86_LOCAL_APIC
> >  static void __cpuinit kvm_setup_secondary_clock(void)
> >  {
> >@@ -195,6 +204,8 @@ void __init kvmclock_init(void)
> >  	x86_cpuinit.early_percpu_clock_init =
> >  		kvm_setup_secondary_clock;
> >  #endif
> >+	x86_platform.save_sched_clock_state = kvm_save_sched_clock_state;
> >+	x86_platform.restore_sched_clock_state = kvm_restore_sched_clock_state;
> >  	machine_ops.shutdown  = kvm_shutdown;
> >  #ifdef CONFIG_KEXEC
> >  	machine_ops.crash_shutdown  = kvm_crash_shutdown;
> >diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
> >index a62c201..aed2aa1 100644
> >--- a/arch/x86/kernel/tsc.c
> >+++ b/arch/x86/kernel/tsc.c
> >@@ -629,7 +629,7 @@ static void set_cyc2ns_scale(unsigned long cpu_khz, int cpu)
> >
> >  static unsigned long long cyc2ns_suspend;
> >
> >-void save_sched_clock_state(void)
> >+void tsc_save_sched_clock_state(void)
> >  {
> >  	if (!sched_clock_stable)
> >  		return;
> >@@ -645,7 +645,7 @@ void save_sched_clock_state(void)
> >   * that sched_clock() continues from the point where it was left off during
> >   * suspend.
> >   */
> >-void restore_sched_clock_state(void)
> >+void tsc_restore_sched_clock_state(void)
> >  {
> >  	unsigned long long offset;
> >  	unsigned long flags;
> >diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
> >index 6f2ec53..e9f265f 100644
> >--- a/arch/x86/kernel/x86_init.c
> >+++ b/arch/x86/kernel/x86_init.c
> >@@ -108,7 +108,9 @@ struct x86_platform_ops x86_platform = {
> >  	.is_untracked_pat_range		= is_ISA_range,
> >  	.nmi_init			= default_nmi_init,
> >  	.get_nmi_reason			= default_get_nmi_reason,
> >-	.i8042_detect			= default_i8042_detect
> >+	.i8042_detect			= default_i8042_detect,
> >+	.save_sched_clock_state 	= tsc_save_sched_clock_state,
> >+	.restore_sched_clock_state 	= tsc_restore_sched_clock_state,
> >  };
> >
> >  EXPORT_SYMBOL_GPL(x86_platform);
> >diff --git a/arch/x86/power/cpu.c b/arch/x86/power/cpu.c
> >index f10c0af..0e76a28 100644
> >--- a/arch/x86/power/cpu.c
> >+++ b/arch/x86/power/cpu.c
> >@@ -114,7 +114,7 @@ static void __save_processor_state(struct saved_context *ctxt)
> >  void save_processor_state(void)
> >  {
> >  	__save_processor_state(&saved_context);
> >-	save_sched_clock_state();
> >+	x86_platform.save_sched_clock_state();
> >  }
> >  #ifdef CONFIG_X86_32
> >  EXPORT_SYMBOL(save_processor_state);
> >@@ -230,8 +230,8 @@ static void __restore_processor_state(struct saved_context *ctxt)
> >  /* Needed by apm.c */
> >  void restore_processor_state(void)
> >  {
> >+	x86_platform.restore_sched_clock_state();
> Isn't it too early? It is scarry to say hypervisor to write to some
> memory location and than completely replace page-tables and half of
> cpu state in __restore_processor_state. Wouldn't that have a potential
> of writing into a place that is not restored hv_clock and restored
> hv_clock might still be stale?

No, memory is copied in swsusp_arch_resume(), which happens
before restore_processor_state. restore_processor_state() is only
setting up registers and MTRR.


^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state (v2)
  2012-02-13 13:07 ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Marcelo Tosatti
  2012-02-13 15:20   ` Igor Mammedov
  2012-02-13 15:45   ` [PATCH RFC] pvclock: Make pv_clock more robust and fixup it if overflow happens Igor Mammedov
@ 2012-02-13 16:26   ` Amit Shah
  2 siblings, 0 replies; 24+ messages in thread
From: Amit Shah @ 2012-02-13 16:26 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: kvm, tglx, mingo, hpa, x86, johnstul, riel, avi, imammedo

On (Mon) 13 Feb 2012 [11:07:27], Marcelo Tosatti wrote:
> 
> Upon resume from hibernation, CPU 0's hvclock area contains the old
> values for system_time and tsc_timestamp. It is necessary for the
> hypervisor to update these values with uptodate ones before the CPU uses
> them.
> 
> Abstract TSC's save/restore sched_clock_state functions and use
> restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.
> 
> Also move restore_sched_clock_state before __restore_processor_state,
> since the later calls CONFIG_LOCK_STAT's lockstat_clock (also for TSC).
> Thanks to Igor Mammedov for tracking it down.
> 
> Fixes suspend-to-disk with kvmclock.
> 
> Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>

This works fine, thanks.

Tested-by: Amit Shah <amit.shah@redhat.com>

		Amit

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: [PATCH RFC] pvclock: Make pv_clock more robust and fixup it if overflow happens
  2012-02-13 15:45   ` [PATCH RFC] pvclock: Make pv_clock more robust and fixup it if overflow happens Igor Mammedov
@ 2012-02-13 17:48     ` Marcelo Tosatti
  2012-02-13 18:15       ` Igor Mammedov
  0 siblings, 1 reply; 24+ messages in thread
From: Marcelo Tosatti @ 2012-02-13 17:48 UTC (permalink / raw)
  To: Igor Mammedov
  Cc: linux-kernel, kvm, tglx, mingo, hpa, x86, jacob.jun.pan, alan,
	feng.tang, konrad.wilk, avi, glommer, johnstul, riel, tj,
	kosaki.motohiro, akpm, amit.shah

On Mon, Feb 13, 2012 at 04:45:59PM +0100, Igor Mammedov wrote:
> Instead of hunting misterious stalls/hungs all over the kernel when
> overflow occurs at pvclock.c:pvclock_get_nsec_offset
> 
>     u64 delta = native_read_tsc() - shadow->tsc_timestamp;
> 
> and introducing hooks when places of unexpected access found, pv_clock
> should be initialized for the calling cpu if overflow condition is detected.
> 
> Signed-off-by: Igor Mammedov <imammedo@redhat.com>

Igor,

I disagree. This is fixing the symptom not the root cause. Additionally,
Xen also uses pvclock_clocksource_read.

How about adding a BUG_ON to detect the overflow, this way hunting for
the problem is not necessary.

>  arch/x86/kernel/pvclock.c |   18 +++++++++++++++---
>  1 files changed, 15 insertions(+), 3 deletions(-)
> 
> diff --git a/arch/x86/kernel/pvclock.c b/arch/x86/kernel/pvclock.c
> index 42eb330..b486756 100644
> --- a/arch/x86/kernel/pvclock.c
> +++ b/arch/x86/kernel/pvclock.c
> @@ -41,9 +41,14 @@ void pvclock_set_flags(u8 flags)
>  	valid_flags = flags;
>  }
>  
> -static u64 pvclock_get_nsec_offset(struct pvclock_shadow_time *shadow)
> +static u64 pvclock_get_nsec_offset(struct pvclock_shadow_time *shadow,
> +				   bool *overflow)
>  {
> -	u64 delta = native_read_tsc() - shadow->tsc_timestamp;
> +	u64 delta;
> +	u64 tsc = native_read_tsc();
> +	u64 shadow_timestamp = shadow->tsc_timestamp;
> +	*overflow = tsc < shadow_timestamp;
> +	delta = tsc - shadow_timestamp;
>  	return pvclock_scale_delta(delta, shadow->tsc_to_nsec_mul,
>  				   shadow->tsc_shift);
>  }
> @@ -94,12 +99,19 @@ cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
>  	unsigned version;
>  	cycle_t ret, offset;
>  	u64 last;
> +	bool overflow;
>  
>  	do {
>  		version = pvclock_get_time_values(&shadow, src);
>  		barrier();
> -		offset = pvclock_get_nsec_offset(&shadow);
> +		offset = pvclock_get_nsec_offset(&shadow, &overflow);
>  		ret = shadow.system_timestamp + offset;
> +		if (unlikely(overflow)) {
> +			memset(src, 0, sizeof(*src));
> +			barrier();
> +			x86_cpuinit.early_percpu_clock_init();
> +			continue;
> +		}
>  		barrier();
>  	} while (version != src->version);
>  
> -- 
> 1.7.7.6
> 
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: [PATCH RFC] pvclock: Make pv_clock more robust and fixup it if overflow happens
  2012-02-13 17:48     ` Marcelo Tosatti
@ 2012-02-13 18:15       ` Igor Mammedov
  0 siblings, 0 replies; 24+ messages in thread
From: Igor Mammedov @ 2012-02-13 18:15 UTC (permalink / raw)
  To: Marcelo Tosatti
  Cc: linux-kernel, kvm, tglx, mingo, hpa, x86, jacob.jun.pan, alan,
	feng.tang, konrad.wilk, avi, glommer, johnstul, riel, tj,
	kosaki.motohiro, akpm, amit.shah

On 02/13/2012 06:48 PM, Marcelo Tosatti wrote:
> On Mon, Feb 13, 2012 at 04:45:59PM +0100, Igor Mammedov wrote:
>> Instead of hunting misterious stalls/hungs all over the kernel when
>> overflow occurs at pvclock.c:pvclock_get_nsec_offset
>>
>>      u64 delta = native_read_tsc() - shadow->tsc_timestamp;
>>
>> and introducing hooks when places of unexpected access found, pv_clock
>> should be initialized for the calling cpu if overflow condition is detected.
>>
>> Signed-off-by: Igor Mammedov<imammedo@redhat.com>
>
> Igor,
>
> I disagree. This is fixing the symptom not the root cause. Additionally,
> Xen also uses pvclock_clocksource_read.
>
> How about adding a BUG_ON to detect the overflow, this way hunting for
> the problem is not necessary.
>
Ok, I'll repost bug_on version.

^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state (v2)
  2012-02-13 15:52     ` Marcelo Tosatti
@ 2012-02-15 10:28       ` Avi Kivity
  0 siblings, 0 replies; 24+ messages in thread
From: Avi Kivity @ 2012-02-15 10:28 UTC (permalink / raw)
  To: Marcelo Tosatti
  Cc: Igor Mammedov, kvm, tglx, mingo, hpa, x86, johnstul, riel, amit.shah

On 02/13/2012 05:52 PM, Marcelo Tosatti wrote:
> > >  {
> > >+	x86_platform.restore_sched_clock_state();
> > Isn't it too early? It is scarry to say hypervisor to write to some
> > memory location and than completely replace page-tables and half of
> > cpu state in __restore_processor_state. Wouldn't that have a potential
> > of writing into a place that is not restored hv_clock and restored
> > hv_clock might still be stale?
>
> No, memory is copied in swsusp_arch_resume(), which happens
> before restore_processor_state. restore_processor_state() is only
> setting up registers and MTRR.
>

In addition, kvmclock uses physical addresses, so page table changes
don't matter.

Note we could have done this in
__save_processor_state()/__restore_processor_state() (it's just reading
and writing an MSR, like we do for MSR_IA32_MISC_ENABLE), but I think
your patch is the right way.  I'd like an ack from the x86 maintainers
though.

-- 
error compiling committee.c: too many arguments to function


^ permalink raw reply	[flat|nested] 24+ messages in thread

* Re: x86: kvmclock: abstract save/restore sched_clock_state
  2012-02-07 21:05 x86: kvmclock: abstract save/restore sched_clock_state Marcelo Tosatti
                   ` (2 preceding siblings ...)
  2012-02-13 13:07 ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Marcelo Tosatti
@ 2012-03-01  9:58 ` Thomas Gleixner
  3 siblings, 0 replies; 24+ messages in thread
From: Thomas Gleixner @ 2012-03-01  9:58 UTC (permalink / raw)
  To: Marcelo Tosatti; +Cc: kvm, mingo, hpa, x86, johnstul, riel, avi, imammedo

On Tue, 7 Feb 2012, Marcelo Tosatti wrote:
> 
> Upon resume from hibernation, CPU 0's hvclock area contains the old
> values for system_time and tsc_timestamp. It is necessary for the
> hypervisor to update these values with uptodate ones before the CPU uses
> them.
> 
> Abstract TSC's save/restore sched_clock_state functions and use
> restore_state to write to KVM_SYSTEM_TIME MSR, forcing an update.
> 
> Fixes suspend-to-disk with kvmclock.
> 
> Signed-off-by: Marcelo Tosatti <mtosatti@redhat.com>
> 
> diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
> index 15d9915..c91e8b9 100644
> --- a/arch/x86/include/asm/tsc.h
> +++ b/arch/x86/include/asm/tsc.h
> @@ -61,7 +61,7 @@ extern void check_tsc_sync_source(int cpu);
>  extern void check_tsc_sync_target(void);
>  
>  extern int notsc_setup(char *);
> -extern void save_sched_clock_state(void);
> -extern void restore_sched_clock_state(void);
> +extern void tsc_save_sched_clock_state(void);
> +extern void tsc_restore_sched_clock_state(void);
>  
>  #endif /* _ASM_X86_TSC_H */
> diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
> index 5d0afac..baaca8d 100644
> --- a/arch/x86/include/asm/x86_init.h
> +++ b/arch/x86/include/asm/x86_init.h
> @@ -162,6 +162,8 @@ struct x86_cpuinit_ops {
>   * @is_untracked_pat_range	exclude from PAT logic
>   * @nmi_init			enable NMI on cpus
>   * @i8042_detect		pre-detect if i8042 controller exists
> + * @save_sched_clock_state:	save state for sched_clock() on suspend
> + * @restore_sched_clock_state:	restore state for sched_clock() on resume
>   */
>  struct x86_platform_ops {
>  	unsigned long (*calibrate_tsc)(void);
> @@ -173,6 +175,8 @@ struct x86_platform_ops {
>  	void (*nmi_init)(void);
>  	unsigned char (*get_nmi_reason)(void);
>  	int (*i8042_detect)(void);
> +	void (*save_sched_clock_state)(void);
> +	void (*restore_sched_clock_state)(void);
>  };
>  
>  struct pci_dev;
> diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
> index ca4e735..57e6b78 100644
> --- a/arch/x86/kernel/kvmclock.c
> +++ b/arch/x86/kernel/kvmclock.c
> @@ -136,6 +136,15 @@ int kvm_register_clock(char *txt)
>  	return ret;
>  }
>  
> +void kvm_save_sched_clock_state(void)

static ?

> +{
> +}
> +
> +void kvm_restore_sched_clock_state(void)

Ditto

> +{
> +	kvm_register_clock("primary cpu clock, resume");
> +}
> +

Otherwise: Reviewed-by: Thomas Gleixner <tglx@linutronix.de>

^ permalink raw reply	[flat|nested] 24+ messages in thread

end of thread, other threads:[~2012-03-01  9:58 UTC | newest]

Thread overview: 24+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2012-02-07 21:05 x86: kvmclock: abstract save/restore sched_clock_state Marcelo Tosatti
2012-02-08 10:53 ` Igor Mammedov
2012-02-09 12:27 ` Amit Shah
2012-02-09 15:13   ` Igor Mammedov
2012-02-10 10:02     ` Amit Shah
2012-02-10 10:11       ` Igor Mammedov
2012-02-10 10:23         ` Amit Shah
2012-02-10 12:32       ` Marcelo Tosatti
2012-02-10 12:33         ` Marcelo Tosatti
2012-02-10 12:43           ` Igor Mammedov
2012-02-13 12:46             ` Amit Shah
2012-02-13 12:56           ` Amit Shah
2012-02-10 13:18         ` Amit Shah
2012-02-10 20:58           ` Igor Mammedov
2012-02-13 12:39             ` Amit Shah
2012-02-13 13:07 ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Marcelo Tosatti
2012-02-13 15:20   ` Igor Mammedov
2012-02-13 15:52     ` Marcelo Tosatti
2012-02-15 10:28       ` Avi Kivity
2012-02-13 15:45   ` [PATCH RFC] pvclock: Make pv_clock more robust and fixup it if overflow happens Igor Mammedov
2012-02-13 17:48     ` Marcelo Tosatti
2012-02-13 18:15       ` Igor Mammedov
2012-02-13 16:26   ` x86: kvmclock: abstract save/restore sched_clock_state (v2) Amit Shah
2012-03-01  9:58 ` x86: kvmclock: abstract save/restore sched_clock_state Thomas Gleixner

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.