[Qemu-devel] [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Qemu-devel] [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Liu, Jinsong]

Currently HVM S3 has a bug coming from the difference between
qemu-traditioanl and qemu-xen. For qemu-traditional, the way
to resume from hvm s3 is via 'xl trigger' command. However,
for qemu-xen, the way to resume from hvm s3 inherited from
standard qemu, i.e. via QMP, and it doesn't work under Xen.

The root cause is, for qemu-xen, 'xl trigger' command didn't reset
devices, while QMP didn't unpause hvm domain though they did qemu
system reset.

We have 2 patches to fix the HVM S3 bug: qemu-xen patch and xl patch.
This patch is the qemu-xen patch. It registers a wakeup later notify,
so that when 'xl trigger' command invokes QMP system_wakeup and after
qemu system reset, it hypercalls to hypervisor to unpause domain, then
hvm S3 resumes successfully.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 vl.c      |   13 +++++++++++++
 xen-all.c |    9 +++++++++
 2 files changed, 22 insertions(+), 0 deletions(-)

diff --git a/vl.c b/vl.c
index 5314f55..aeebd83 100644
--- a/vl.c
+++ b/vl.c
@@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
     NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
 static NotifierList wakeup_notifiers =
     NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
+static NotifierList wakeup_later_notifiers =
+    NOTIFIER_LIST_INITIALIZER(wakeup_later_notifiers);
 static uint32_t wakeup_reason_mask = ~0;
 static RunState vmstop_requested = RUN_STATE_MAX;
 
@@ -1625,6 +1627,11 @@ static void qemu_system_suspend(void)
     monitor_protocol_event(QEVENT_SUSPEND, NULL);
 }
 
+static void qemu_system_wakeup(void)
+{
+    notifier_list_notify(&wakeup_later_notifiers, NULL);
+}
+
 void qemu_system_suspend_request(void)
 {
     if (runstate_check(RUN_STATE_SUSPENDED)) {
@@ -1668,6 +1675,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
     notifier_list_add(&wakeup_notifiers, notifier);
 }
 
+void qemu_register_wakeup_later_notifier(Notifier *notifier)
+{
+    notifier_list_add(&wakeup_later_notifiers, notifier);
+}
+
 void qemu_system_killed(int signal, pid_t pid)
 {
     shutdown_signal = signal;
@@ -1744,6 +1756,7 @@ static bool main_loop_should_exit(void)
         cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_SILENT);
         resume_all_vcpus();
+        qemu_system_wakeup();
         monitor_protocol_event(QEVENT_WAKEUP, NULL);
     }
     if (qemu_powerdown_requested()) {
diff --git a/xen-all.c b/xen-all.c
index 15be8ed..3353f63 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -97,6 +97,7 @@ typedef struct XenIOState {
 
     Notifier exit;
     Notifier suspend;
+    Notifier wakeup_later;
 } XenIOState;
 
 /* Xen specific function for piix pci */
@@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
     xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
 }
 
+static void xen_wakeup_later_notifier(Notifier *notifier, void *data)
+{
+    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
+}
+
 /* Xen Interrupt Controller */
 
 static void xen_set_irq(void *opaque, int irq, int level)
@@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
     state->suspend.notify = xen_suspend_notifier;
     qemu_register_suspend_notifier(&state->suspend);
 
+    state->wakeup_later.notify = xen_wakeup_later_notifier;
+    qemu_register_wakeup_later_notifier(&state->wakeup_later);
+
     xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
     DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
     state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-- 
1.7.1

Re: [Qemu-devel] [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Anthony PERARD]

On 29/08/13 09:25, Liu, Jinsong wrote:
> Currently HVM S3 has a bug coming from the difference between
> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
> to resume from hvm s3 is via 'xl trigger' command. However,
> for qemu-xen, the way to resume from hvm s3 inherited from
> standard qemu, i.e. via QMP, and it doesn't work under Xen.
> 
> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
> devices, while QMP didn't unpause hvm domain though they did qemu
> system reset.
> 
> We have 2 patches to fix the HVM S3 bug: qemu-xen patch and xl patch.
> This patch is the qemu-xen patch. It registers a wakeup later notify,
> so that when 'xl trigger' command invokes QMP system_wakeup and after
> qemu system reset, it hypercalls to hypervisor to unpause domain, then
> hvm S3 resumes successfully.
> 
> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> ---
>  vl.c      |   13 +++++++++++++
>  xen-all.c |    9 +++++++++
>  2 files changed, 22 insertions(+), 0 deletions(-)
> 
> diff --git a/vl.c b/vl.c
> index 5314f55..aeebd83 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>  static NotifierList wakeup_notifiers =
>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
> +static NotifierList wakeup_later_notifiers =
> +    NOTIFIER_LIST_INITIALIZER(wakeup_later_notifiers);

Maybe late_wakeup_notifiers would be a better description for this list.

>  static uint32_t wakeup_reason_mask = ~0;
>  static RunState vmstop_requested = RUN_STATE_MAX;
>  
> @@ -1625,6 +1627,11 @@ static void qemu_system_suspend(void)
>      monitor_protocol_event(QEVENT_SUSPEND, NULL);
>  }
>  
> +static void qemu_system_wakeup(void)
> +{
> +    notifier_list_notify(&wakeup_later_notifiers, NULL);
> +}
> +

I don't think it's useful to have this function with only the
list_notify. You could call directly list_notify in the
main_loop_should_exit() function bellow.

>  void qemu_system_suspend_request(void)
>  {
>      if (runstate_check(RUN_STATE_SUSPENDED)) {
> @@ -1668,6 +1675,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
>      notifier_list_add(&wakeup_notifiers, notifier);
>  }
>  
> +void qemu_register_wakeup_later_notifier(Notifier *notifier)
> +{
> +    notifier_list_add(&wakeup_later_notifiers, notifier);
> +}
> +
>  void qemu_system_killed(int signal, pid_t pid)
>  {
>      shutdown_signal = signal;
> @@ -1744,6 +1756,7 @@ static bool main_loop_should_exit(void)
>          cpu_synchronize_all_states();
>          qemu_system_reset(VMRESET_SILENT);
>          resume_all_vcpus();
> +        qemu_system_wakeup();
>          monitor_protocol_event(QEVENT_WAKEUP, NULL);
>      }
>      if (qemu_powerdown_requested()) {
> diff --git a/xen-all.c b/xen-all.c
> index 15be8ed..3353f63 100644
> --- a/xen-all.c
> +++ b/xen-all.c
> @@ -97,6 +97,7 @@ typedef struct XenIOState {
>  
>      Notifier exit;
>      Notifier suspend;
> +    Notifier wakeup_later;
>  } XenIOState;
>  
>  /* Xen specific function for piix pci */
> @@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
>      xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
>  }
>  
> +static void xen_wakeup_later_notifier(Notifier *notifier, void *data)
> +{
> +    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
> +}
> +
>  /* Xen Interrupt Controller */
>  
>  static void xen_set_irq(void *opaque, int irq, int level)
> @@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
>      state->suspend.notify = xen_suspend_notifier;
>      qemu_register_suspend_notifier(&state->suspend);
>  
> +    state->wakeup_later.notify = xen_wakeup_later_notifier;
> +    qemu_register_wakeup_later_notifier(&state->wakeup_later);
> +
>      xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
>      DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
>      state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
> 

Could you break this path into two? One which add the notifier list and
a second one with the Xen specific part?

Thanks,

-- 
Anthony PERARD

[Qemu-devel] [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Liu, Jinsong]

[-- Attachment #1: Type: text/plain, Size: 2867 bytes --]

>From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00 2001
From: Liu Jinsong <jinsong.liu@intel.com>
Date: Sun, 1 Sep 2013 23:39:14 +0800
Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu wakeup

Currently HVM S3 has a bug coming from the difference between
qemu-traditioanl and qemu-xen. For qemu-traditional, the way
to resume from hvm s3 is via 'xl trigger' command. However,
for qemu-xen, the way to resume from hvm s3 inherited from
standard qemu, i.e. via QMP, and it doesn't work under Xen.

The root cause is, for qemu-xen, 'xl trigger' command didn't reset
devices, while QMP didn't unpause hvm domain though they did qemu
system reset.

We have two qemu-xen patches and one xl patch to fix the HVM S3 bug.
This patch is the qemu-xen patch 1. It provides a later wakeup notifier
and a register function, and notifies the later wakeup list when
qemu wakup by 'xl trigger' command.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 sysemu.h |    1 +
 vl.c     |    8 ++++++++
 2 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/sysemu.h b/sysemu.h
index b71f244..4dbcab7 100644
--- a/sysemu.h
+++ b/sysemu.h
@@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier *notifier);
 void qemu_system_wakeup_request(WakeupReason reason);
 void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
 void qemu_register_wakeup_notifier(Notifier *notifier);
+void qemu_register_later_wakeup_notifier(Notifier *notifier);
 void qemu_system_shutdown_request(void);
 void qemu_system_powerdown_request(void);
 void qemu_register_powerdown_notifier(Notifier *notifier);
diff --git a/vl.c b/vl.c
index 5314f55..1c4842d 100644
--- a/vl.c
+++ b/vl.c
@@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
     NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
 static NotifierList wakeup_notifiers =
     NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
+static NotifierList later_wakeup_notifiers =
+    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
 static uint32_t wakeup_reason_mask = ~0;
 static RunState vmstop_requested = RUN_STATE_MAX;
 
@@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
     notifier_list_add(&wakeup_notifiers, notifier);
 }
 
+void qemu_register_later_wakeup_notifier(Notifier *notifier)
+{
+    notifier_list_add(&later_wakeup_notifiers, notifier);
+}
+
 void qemu_system_killed(int signal, pid_t pid)
 {
     shutdown_signal = signal;
@@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
         cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_SILENT);
         resume_all_vcpus();
+        notifier_list_notify(&later_wakeup_notifiers, NULL);
         monitor_protocol_event(QEVENT_WAKEUP, NULL);
     }
     if (qemu_powerdown_requested()) {
-- 
1.7.1

[-- Attachment #2: 0001-Add-later-wakeup-logic-when-qemu-wakeup.patch --]
[-- Type: application/octet-stream, Size: 2752 bytes --]

From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00 2001
From: Liu Jinsong <jinsong.liu@intel.com>
Date: Sun, 1 Sep 2013 23:39:14 +0800
Subject: [PATCH 1/2] Add later wakeup logic when qemu wakeup

Currently HVM S3 has a bug coming from the difference between
qemu-traditioanl and qemu-xen. For qemu-traditional, the way
to resume from hvm s3 is via 'xl trigger' command. However,
for qemu-xen, the way to resume from hvm s3 inherited from
standard qemu, i.e. via QMP, and it doesn't work under Xen.

The root cause is, for qemu-xen, 'xl trigger' command didn't reset
devices, while QMP didn't unpause hvm domain though they did qemu
system reset.

We have two qemu patches and one xl patch to fix the HVM S3 bug.
This patch is the qemu patch 1. It provides a later wakeup notifier
and a register function, and notifies the later wakeup list when
qemu wakup.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 sysemu.h |    1 +
 vl.c     |    8 ++++++++
 2 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/sysemu.h b/sysemu.h
index b71f244..4dbcab7 100644
--- a/sysemu.h
+++ b/sysemu.h
@@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier *notifier);
 void qemu_system_wakeup_request(WakeupReason reason);
 void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
 void qemu_register_wakeup_notifier(Notifier *notifier);
+void qemu_register_later_wakeup_notifier(Notifier *notifier);
 void qemu_system_shutdown_request(void);
 void qemu_system_powerdown_request(void);
 void qemu_register_powerdown_notifier(Notifier *notifier);
diff --git a/vl.c b/vl.c
index 5314f55..1c4842d 100644
--- a/vl.c
+++ b/vl.c
@@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
     NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
 static NotifierList wakeup_notifiers =
     NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
+static NotifierList later_wakeup_notifiers =
+    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
 static uint32_t wakeup_reason_mask = ~0;
 static RunState vmstop_requested = RUN_STATE_MAX;
 
@@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
     notifier_list_add(&wakeup_notifiers, notifier);
 }
 
+void qemu_register_later_wakeup_notifier(Notifier *notifier)
+{
+    notifier_list_add(&later_wakeup_notifiers, notifier);
+}
+
 void qemu_system_killed(int signal, pid_t pid)
 {
     shutdown_signal = signal;
@@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
         cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_SILENT);
         resume_all_vcpus();
+        notifier_list_notify(&later_wakeup_notifiers, NULL);
         monitor_protocol_event(QEVENT_WAKEUP, NULL);
     }
     if (qemu_powerdown_requested()) {
-- 
1.7.1

Re: [Qemu-devel] [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Anthony PERARD]

On 01/09/13 10:51, Liu, Jinsong wrote:
> From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00 2001
> From: Liu Jinsong <jinsong.liu@intel.com>
> Date: Sun, 1 Sep 2013 23:39:14 +0800
> Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu wakeup
> 
> Currently HVM S3 has a bug coming from the difference between
> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
> to resume from hvm s3 is via 'xl trigger' command. However,
> for qemu-xen, the way to resume from hvm s3 inherited from
> standard qemu, i.e. via QMP, and it doesn't work under Xen.
> 
> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
> devices, while QMP didn't unpause hvm domain though they did qemu
> system reset.
> 
> We have two qemu-xen patches and one xl patch to fix the HVM S3 bug.
> This patch is the qemu-xen patch 1. It provides a later wakeup notifier
> and a register function, and notifies the later wakeup list when
> qemu wakup by 'xl trigger' command.
> 
> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> ---
>  sysemu.h |    1 +
>  vl.c     |    8 ++++++++
>  2 files changed, 9 insertions(+), 0 deletions(-)
> 
> diff --git a/sysemu.h b/sysemu.h
> index b71f244..4dbcab7 100644
> --- a/sysemu.h
> +++ b/sysemu.h
> @@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier *notifier);
>  void qemu_system_wakeup_request(WakeupReason reason);
>  void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
>  void qemu_register_wakeup_notifier(Notifier *notifier);
> +void qemu_register_later_wakeup_notifier(Notifier *notifier);
>  void qemu_system_shutdown_request(void);
>  void qemu_system_powerdown_request(void);
>  void qemu_register_powerdown_notifier(Notifier *notifier);
> diff --git a/vl.c b/vl.c
> index 5314f55..1c4842d 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>  static NotifierList wakeup_notifiers =
>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
> +static NotifierList later_wakeup_notifiers =
> +    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
>  static uint32_t wakeup_reason_mask = ~0;
>  static RunState vmstop_requested = RUN_STATE_MAX;
>  
> @@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
>      notifier_list_add(&wakeup_notifiers, notifier);
>  }
>  
> +void qemu_register_later_wakeup_notifier(Notifier *notifier)
> +{
> +    notifier_list_add(&later_wakeup_notifiers, notifier);
> +}
> +
>  void qemu_system_killed(int signal, pid_t pid)
>  {
>      shutdown_signal = signal;
> @@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
>          cpu_synchronize_all_states();
>          qemu_system_reset(VMRESET_SILENT);
>          resume_all_vcpus();
> +        notifier_list_notify(&later_wakeup_notifiers, NULL);
>          monitor_protocol_event(QEVENT_WAKEUP, NULL);
>      }
>      if (qemu_powerdown_requested()) {
> 

The patch those not apply properly to QEMU (upstream) but it just
because the file sysemu.h have been moved to include/sysemu/sysemu.h

Once this is fix:
Acked-by: Anthony PERARD <anthony.perard@citrix.com>

-- 
Anthony PERARD

Re: [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Liu, Jinsong]

Anthony PERARD wrote:
> On 01/09/13 10:51, Liu, Jinsong wrote:
>> From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00
>> 2001 
>> From: Liu Jinsong <jinsong.liu@intel.com>
>> Date: Sun, 1 Sep 2013 23:39:14 +0800
>> Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu
>> wakeup 
>> 
>> Currently HVM S3 has a bug coming from the difference between
>> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
>> to resume from hvm s3 is via 'xl trigger' command. However,
>> for qemu-xen, the way to resume from hvm s3 inherited from
>> standard qemu, i.e. via QMP, and it doesn't work under Xen.
>> 
>> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
>> devices, while QMP didn't unpause hvm domain though they did qemu
>> system reset.
>> 
>> We have two qemu-xen patches and one xl patch to fix the HVM S3 bug.
>> This patch is the qemu-xen patch 1. It provides a later wakeup
>> notifier 
>> and a register function, and notifies the later wakeup list when
>> qemu wakup by 'xl trigger' command.
>> 
>> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
>> ---
>>  sysemu.h |    1 +
>>  vl.c     |    8 ++++++++
>>  2 files changed, 9 insertions(+), 0 deletions(-)
>> 
>> diff --git a/sysemu.h b/sysemu.h
>> index b71f244..4dbcab7 100644
>> --- a/sysemu.h
>> +++ b/sysemu.h
>> @@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier
>>  *notifier); void qemu_system_wakeup_request(WakeupReason reason);
>>  void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
>>  void qemu_register_wakeup_notifier(Notifier *notifier);
>> +void qemu_register_later_wakeup_notifier(Notifier *notifier);
>>  void qemu_system_shutdown_request(void);
>>  void qemu_system_powerdown_request(void);
>>  void qemu_register_powerdown_notifier(Notifier *notifier);
>> diff --git a/vl.c b/vl.c
>> index 5314f55..1c4842d 100644
>> --- a/vl.c
>> +++ b/vl.c
>> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>>  static NotifierList wakeup_notifiers =
>>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
>> +static NotifierList later_wakeup_notifiers =
>> +    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
>>  static uint32_t wakeup_reason_mask = ~0;
>>  static RunState vmstop_requested = RUN_STATE_MAX;
>> 
>> @@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier
>>      *notifier) notifier_list_add(&wakeup_notifiers, notifier);  }
>> 
>> +void qemu_register_later_wakeup_notifier(Notifier *notifier) +{
>> +    notifier_list_add(&later_wakeup_notifiers, notifier); +}
>> +
>>  void qemu_system_killed(int signal, pid_t pid)
>>  {
>>      shutdown_signal = signal;
>> @@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
>>          cpu_synchronize_all_states();
>>          qemu_system_reset(VMRESET_SILENT);
>>          resume_all_vcpus();
>> +        notifier_list_notify(&later_wakeup_notifiers, NULL);
>>          monitor_protocol_event(QEVENT_WAKEUP, NULL);      }
>>      if (qemu_powerdown_requested()) {
>> 
> 
> The patch those not apply properly to QEMU (upstream) but it just
> because the file sysemu.h have been moved to include/sysemu/sysemu.h
> 
> Once this is fix:
> Acked-by: Anthony PERARD <anthony.perard@citrix.com>

Yes. The patches are for qemu-xen tree, to fix xen hvm s3 issue.

Where should the 2 patches be checked in? qemu upstream (then backport to qemu-xen tree), or, qemu-xen tree?

Thanks,
Jinsong

Re: [Qemu-devel] [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Liu, Jinsong]

Anthony PERARD wrote:
> On 01/09/13 10:51, Liu, Jinsong wrote:
>> From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00
>> 2001 
>> From: Liu Jinsong <jinsong.liu@intel.com>
>> Date: Sun, 1 Sep 2013 23:39:14 +0800
>> Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu
>> wakeup 
>> 
>> Currently HVM S3 has a bug coming from the difference between
>> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
>> to resume from hvm s3 is via 'xl trigger' command. However,
>> for qemu-xen, the way to resume from hvm s3 inherited from
>> standard qemu, i.e. via QMP, and it doesn't work under Xen.
>> 
>> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
>> devices, while QMP didn't unpause hvm domain though they did qemu
>> system reset.
>> 
>> We have two qemu-xen patches and one xl patch to fix the HVM S3 bug.
>> This patch is the qemu-xen patch 1. It provides a later wakeup
>> notifier 
>> and a register function, and notifies the later wakeup list when
>> qemu wakup by 'xl trigger' command.
>> 
>> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
>> ---
>>  sysemu.h |    1 +
>>  vl.c     |    8 ++++++++
>>  2 files changed, 9 insertions(+), 0 deletions(-)
>> 
>> diff --git a/sysemu.h b/sysemu.h
>> index b71f244..4dbcab7 100644
>> --- a/sysemu.h
>> +++ b/sysemu.h
>> @@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier
>>  *notifier); void qemu_system_wakeup_request(WakeupReason reason);
>>  void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
>>  void qemu_register_wakeup_notifier(Notifier *notifier);
>> +void qemu_register_later_wakeup_notifier(Notifier *notifier);
>>  void qemu_system_shutdown_request(void);
>>  void qemu_system_powerdown_request(void);
>>  void qemu_register_powerdown_notifier(Notifier *notifier);
>> diff --git a/vl.c b/vl.c
>> index 5314f55..1c4842d 100644
>> --- a/vl.c
>> +++ b/vl.c
>> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>>  static NotifierList wakeup_notifiers =
>>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
>> +static NotifierList later_wakeup_notifiers =
>> +    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
>>  static uint32_t wakeup_reason_mask = ~0;
>>  static RunState vmstop_requested = RUN_STATE_MAX;
>> 
>> @@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier
>>      *notifier) notifier_list_add(&wakeup_notifiers, notifier);  }
>> 
>> +void qemu_register_later_wakeup_notifier(Notifier *notifier) +{
>> +    notifier_list_add(&later_wakeup_notifiers, notifier); +}
>> +
>>  void qemu_system_killed(int signal, pid_t pid)
>>  {
>>      shutdown_signal = signal;
>> @@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
>>          cpu_synchronize_all_states();
>>          qemu_system_reset(VMRESET_SILENT);
>>          resume_all_vcpus();
>> +        notifier_list_notify(&later_wakeup_notifiers, NULL);
>>          monitor_protocol_event(QEVENT_WAKEUP, NULL);      }
>>      if (qemu_powerdown_requested()) {
>> 
> 
> The patch those not apply properly to QEMU (upstream) but it just
> because the file sysemu.h have been moved to include/sysemu/sysemu.h
> 
> Once this is fix:
> Acked-by: Anthony PERARD <anthony.perard@citrix.com>

Yes. The patches are for qemu-xen tree, to fix xen hvm s3 issue.

Where should the 2 patches be checked in? qemu upstream (then backport to qemu-xen tree), or, qemu-xen tree?

Thanks,
Jinsong

Re: [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Stefano Stabellini]

On Tue, 3 Sep 2013, Liu, Jinsong wrote:
> Anthony PERARD wrote:
> > On 01/09/13 10:51, Liu, Jinsong wrote:
> >> From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00
> >> 2001 
> >> From: Liu Jinsong <jinsong.liu@intel.com>
> >> Date: Sun, 1 Sep 2013 23:39:14 +0800
> >> Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu
> >> wakeup 
> >> 
> >> Currently HVM S3 has a bug coming from the difference between
> >> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
> >> to resume from hvm s3 is via 'xl trigger' command. However,
> >> for qemu-xen, the way to resume from hvm s3 inherited from
> >> standard qemu, i.e. via QMP, and it doesn't work under Xen.
> >> 
> >> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
> >> devices, while QMP didn't unpause hvm domain though they did qemu
> >> system reset.
> >> 
> >> We have two qemu-xen patches and one xl patch to fix the HVM S3 bug.
> >> This patch is the qemu-xen patch 1. It provides a later wakeup
> >> notifier 
> >> and a register function, and notifies the later wakeup list when
> >> qemu wakup by 'xl trigger' command.
> >> 
> >> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> >> ---
> >>  sysemu.h |    1 +
> >>  vl.c     |    8 ++++++++
> >>  2 files changed, 9 insertions(+), 0 deletions(-)
> >> 
> >> diff --git a/sysemu.h b/sysemu.h
> >> index b71f244..4dbcab7 100644
> >> --- a/sysemu.h
> >> +++ b/sysemu.h
> >> @@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier
> >>  *notifier); void qemu_system_wakeup_request(WakeupReason reason);
> >>  void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
> >>  void qemu_register_wakeup_notifier(Notifier *notifier);
> >> +void qemu_register_later_wakeup_notifier(Notifier *notifier);
> >>  void qemu_system_shutdown_request(void);
> >>  void qemu_system_powerdown_request(void);
> >>  void qemu_register_powerdown_notifier(Notifier *notifier);
> >> diff --git a/vl.c b/vl.c
> >> index 5314f55..1c4842d 100644
> >> --- a/vl.c
> >> +++ b/vl.c
> >> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
> >>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
> >>  static NotifierList wakeup_notifiers =
> >>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
> >> +static NotifierList later_wakeup_notifiers =
> >> +    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
> >>  static uint32_t wakeup_reason_mask = ~0;
> >>  static RunState vmstop_requested = RUN_STATE_MAX;
> >> 
> >> @@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier
> >>      *notifier) notifier_list_add(&wakeup_notifiers, notifier);  }
> >> 
> >> +void qemu_register_later_wakeup_notifier(Notifier *notifier) +{
> >> +    notifier_list_add(&later_wakeup_notifiers, notifier); +}
> >> +
> >>  void qemu_system_killed(int signal, pid_t pid)
> >>  {
> >>      shutdown_signal = signal;
> >> @@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
> >>          cpu_synchronize_all_states();
> >>          qemu_system_reset(VMRESET_SILENT);
> >>          resume_all_vcpus();
> >> +        notifier_list_notify(&later_wakeup_notifiers, NULL);
> >>          monitor_protocol_event(QEVENT_WAKEUP, NULL);      }
> >>      if (qemu_powerdown_requested()) {
> >> 
> > 
> > The patch those not apply properly to QEMU (upstream) but it just
> > because the file sysemu.h have been moved to include/sysemu/sysemu.h
> > 
> > Once this is fix:
> > Acked-by: Anthony PERARD <anthony.perard@citrix.com>
> 
> Yes. The patches are for qemu-xen tree, to fix xen hvm s3 issue.
> 
> Where should the 2 patches be checked in? qemu upstream (then backport to qemu-xen tree), or, qemu-xen tree?

In general patches should go to qemu upstream first and then
be backported to qemu-xen.

Re: [Qemu-devel] [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Stefano Stabellini]

On Tue, 3 Sep 2013, Liu, Jinsong wrote:
> Anthony PERARD wrote:
> > On 01/09/13 10:51, Liu, Jinsong wrote:
> >> From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00
> >> 2001 
> >> From: Liu Jinsong <jinsong.liu@intel.com>
> >> Date: Sun, 1 Sep 2013 23:39:14 +0800
> >> Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu
> >> wakeup 
> >> 
> >> Currently HVM S3 has a bug coming from the difference between
> >> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
> >> to resume from hvm s3 is via 'xl trigger' command. However,
> >> for qemu-xen, the way to resume from hvm s3 inherited from
> >> standard qemu, i.e. via QMP, and it doesn't work under Xen.
> >> 
> >> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
> >> devices, while QMP didn't unpause hvm domain though they did qemu
> >> system reset.
> >> 
> >> We have two qemu-xen patches and one xl patch to fix the HVM S3 bug.
> >> This patch is the qemu-xen patch 1. It provides a later wakeup
> >> notifier 
> >> and a register function, and notifies the later wakeup list when
> >> qemu wakup by 'xl trigger' command.
> >> 
> >> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> >> ---
> >>  sysemu.h |    1 +
> >>  vl.c     |    8 ++++++++
> >>  2 files changed, 9 insertions(+), 0 deletions(-)
> >> 
> >> diff --git a/sysemu.h b/sysemu.h
> >> index b71f244..4dbcab7 100644
> >> --- a/sysemu.h
> >> +++ b/sysemu.h
> >> @@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier
> >>  *notifier); void qemu_system_wakeup_request(WakeupReason reason);
> >>  void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
> >>  void qemu_register_wakeup_notifier(Notifier *notifier);
> >> +void qemu_register_later_wakeup_notifier(Notifier *notifier);
> >>  void qemu_system_shutdown_request(void);
> >>  void qemu_system_powerdown_request(void);
> >>  void qemu_register_powerdown_notifier(Notifier *notifier);
> >> diff --git a/vl.c b/vl.c
> >> index 5314f55..1c4842d 100644
> >> --- a/vl.c
> >> +++ b/vl.c
> >> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
> >>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
> >>  static NotifierList wakeup_notifiers =
> >>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
> >> +static NotifierList later_wakeup_notifiers =
> >> +    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
> >>  static uint32_t wakeup_reason_mask = ~0;
> >>  static RunState vmstop_requested = RUN_STATE_MAX;
> >> 
> >> @@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier
> >>      *notifier) notifier_list_add(&wakeup_notifiers, notifier);  }
> >> 
> >> +void qemu_register_later_wakeup_notifier(Notifier *notifier) +{
> >> +    notifier_list_add(&later_wakeup_notifiers, notifier); +}
> >> +
> >>  void qemu_system_killed(int signal, pid_t pid)
> >>  {
> >>      shutdown_signal = signal;
> >> @@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
> >>          cpu_synchronize_all_states();
> >>          qemu_system_reset(VMRESET_SILENT);
> >>          resume_all_vcpus();
> >> +        notifier_list_notify(&later_wakeup_notifiers, NULL);
> >>          monitor_protocol_event(QEVENT_WAKEUP, NULL);      }
> >>      if (qemu_powerdown_requested()) {
> >> 
> > 
> > The patch those not apply properly to QEMU (upstream) but it just
> > because the file sysemu.h have been moved to include/sysemu/sysemu.h
> > 
> > Once this is fix:
> > Acked-by: Anthony PERARD <anthony.perard@citrix.com>
> 
> Yes. The patches are for qemu-xen tree, to fix xen hvm s3 issue.
> 
> Where should the 2 patches be checked in? qemu upstream (then backport to qemu-xen tree), or, qemu-xen tree?

In general patches should go to qemu upstream first and then
be backported to qemu-xen.

Re: [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Liu, Jinsong]

Stefano Stabellini wrote:
> On Tue, 3 Sep 2013, Liu, Jinsong wrote:
>> Anthony PERARD wrote:
>>> On 01/09/13 10:51, Liu, Jinsong wrote:
>>>> From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00
>>>> 2001 From: Liu Jinsong <jinsong.liu@intel.com>
>>>> Date: Sun, 1 Sep 2013 23:39:14 +0800
>>>> Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu
>>>> wakeup 
>>>> 
>>>> Currently HVM S3 has a bug coming from the difference between
>>>> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
>>>> to resume from hvm s3 is via 'xl trigger' command. However,
>>>> for qemu-xen, the way to resume from hvm s3 inherited from
>>>> standard qemu, i.e. via QMP, and it doesn't work under Xen.
>>>> 
>>>> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
>>>> devices, while QMP didn't unpause hvm domain though they did qemu
>>>> system reset. 
>>>> 
>>>> We have two qemu-xen patches and one xl patch to fix the HVM S3
>>>> bug. This patch is the qemu-xen patch 1. It provides a later
>>>> wakeup notifier and a register function, and notifies the later
>>>> wakeup list when qemu wakup by 'xl trigger' command.
>>>> 
>>>> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
>>>> ---
>>>>  sysemu.h |    1 +
>>>>  vl.c     |    8 ++++++++
>>>>  2 files changed, 9 insertions(+), 0 deletions(-)
>>>> 
>>>> diff --git a/sysemu.h b/sysemu.h
>>>> index b71f244..4dbcab7 100644
>>>> --- a/sysemu.h
>>>> +++ b/sysemu.h
>>>> @@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier
>>>>  *notifier); void qemu_system_wakeup_request(WakeupReason reason);
>>>>  void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
>>>>  void qemu_register_wakeup_notifier(Notifier *notifier);
>>>> +void qemu_register_later_wakeup_notifier(Notifier *notifier);
>>>>  void qemu_system_shutdown_request(void);
>>>>  void qemu_system_powerdown_request(void);
>>>>  void qemu_register_powerdown_notifier(Notifier *notifier);
>>>> diff --git a/vl.c b/vl.c
>>>> index 5314f55..1c4842d 100644
>>>> --- a/vl.c
>>>> +++ b/vl.c
>>>> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>>>>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>>>>  static NotifierList wakeup_notifiers =
>>>>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
>>>> +static NotifierList later_wakeup_notifiers =
>>>> +    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
>>>>  static uint32_t wakeup_reason_mask = ~0;
>>>>  static RunState vmstop_requested = RUN_STATE_MAX;
>>>> 
>>>> @@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier
>>>>      *notifier) notifier_list_add(&wakeup_notifiers, notifier);  }
>>>> 
>>>> +void qemu_register_later_wakeup_notifier(Notifier *notifier) +{
>>>> +    notifier_list_add(&later_wakeup_notifiers, notifier); +} +
>>>>  void qemu_system_killed(int signal, pid_t pid)
>>>>  {
>>>>      shutdown_signal = signal;
>>>> @@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
>>>>          cpu_synchronize_all_states();
>>>>          qemu_system_reset(VMRESET_SILENT);
>>>>          resume_all_vcpus();
>>>> +        notifier_list_notify(&later_wakeup_notifiers, NULL);
>>>>          monitor_protocol_event(QEVENT_WAKEUP, NULL);      }
>>>>      if (qemu_powerdown_requested()) {
>>>> 
>>> 
>>> The patch those not apply properly to QEMU (upstream) but it just
>>> because the file sysemu.h have been moved to include/sysemu/sysemu.h
>>> 
>>> Once this is fix:
>>> Acked-by: Anthony PERARD <anthony.perard@citrix.com>
>> 
>> Yes. The patches are for qemu-xen tree, to fix xen hvm s3 issue.
>> 
>> Where should the 2 patches be checked in? qemu upstream (then
>> backport to qemu-xen tree), or, qemu-xen tree? 
> 
> In general patches should go to qemu upstream first and then
> be backported to qemu-xen.

OK, I will adjust system.h and then send to qemu upstream first.

Thanks,
Jinsong

Re: [Qemu-devel] [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Liu, Jinsong]

Stefano Stabellini wrote:
> On Tue, 3 Sep 2013, Liu, Jinsong wrote:
>> Anthony PERARD wrote:
>>> On 01/09/13 10:51, Liu, Jinsong wrote:
>>>> From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00
>>>> 2001 From: Liu Jinsong <jinsong.liu@intel.com>
>>>> Date: Sun, 1 Sep 2013 23:39:14 +0800
>>>> Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu
>>>> wakeup 
>>>> 
>>>> Currently HVM S3 has a bug coming from the difference between
>>>> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
>>>> to resume from hvm s3 is via 'xl trigger' command. However,
>>>> for qemu-xen, the way to resume from hvm s3 inherited from
>>>> standard qemu, i.e. via QMP, and it doesn't work under Xen.
>>>> 
>>>> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
>>>> devices, while QMP didn't unpause hvm domain though they did qemu
>>>> system reset. 
>>>> 
>>>> We have two qemu-xen patches and one xl patch to fix the HVM S3
>>>> bug. This patch is the qemu-xen patch 1. It provides a later
>>>> wakeup notifier and a register function, and notifies the later
>>>> wakeup list when qemu wakup by 'xl trigger' command.
>>>> 
>>>> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
>>>> ---
>>>>  sysemu.h |    1 +
>>>>  vl.c     |    8 ++++++++
>>>>  2 files changed, 9 insertions(+), 0 deletions(-)
>>>> 
>>>> diff --git a/sysemu.h b/sysemu.h
>>>> index b71f244..4dbcab7 100644
>>>> --- a/sysemu.h
>>>> +++ b/sysemu.h
>>>> @@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier
>>>>  *notifier); void qemu_system_wakeup_request(WakeupReason reason);
>>>>  void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
>>>>  void qemu_register_wakeup_notifier(Notifier *notifier);
>>>> +void qemu_register_later_wakeup_notifier(Notifier *notifier);
>>>>  void qemu_system_shutdown_request(void);
>>>>  void qemu_system_powerdown_request(void);
>>>>  void qemu_register_powerdown_notifier(Notifier *notifier);
>>>> diff --git a/vl.c b/vl.c
>>>> index 5314f55..1c4842d 100644
>>>> --- a/vl.c
>>>> +++ b/vl.c
>>>> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>>>>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>>>>  static NotifierList wakeup_notifiers =
>>>>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
>>>> +static NotifierList later_wakeup_notifiers =
>>>> +    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
>>>>  static uint32_t wakeup_reason_mask = ~0;
>>>>  static RunState vmstop_requested = RUN_STATE_MAX;
>>>> 
>>>> @@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier
>>>>      *notifier) notifier_list_add(&wakeup_notifiers, notifier);  }
>>>> 
>>>> +void qemu_register_later_wakeup_notifier(Notifier *notifier) +{
>>>> +    notifier_list_add(&later_wakeup_notifiers, notifier); +} +
>>>>  void qemu_system_killed(int signal, pid_t pid)
>>>>  {
>>>>      shutdown_signal = signal;
>>>> @@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
>>>>          cpu_synchronize_all_states();
>>>>          qemu_system_reset(VMRESET_SILENT);
>>>>          resume_all_vcpus();
>>>> +        notifier_list_notify(&later_wakeup_notifiers, NULL);
>>>>          monitor_protocol_event(QEVENT_WAKEUP, NULL);      }
>>>>      if (qemu_powerdown_requested()) {
>>>> 
>>> 
>>> The patch those not apply properly to QEMU (upstream) but it just
>>> because the file sysemu.h have been moved to include/sysemu/sysemu.h
>>> 
>>> Once this is fix:
>>> Acked-by: Anthony PERARD <anthony.perard@citrix.com>
>> 
>> Yes. The patches are for qemu-xen tree, to fix xen hvm s3 issue.
>> 
>> Where should the 2 patches be checked in? qemu upstream (then
>> backport to qemu-xen tree), or, qemu-xen tree? 
> 
> In general patches should go to qemu upstream first and then
> be backported to qemu-xen.

OK, I will adjust system.h and then send to qemu upstream first.

Thanks,
Jinsong

Re: [PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Anthony PERARD]

On 01/09/13 10:51, Liu, Jinsong wrote:
> From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00 2001
> From: Liu Jinsong <jinsong.liu@intel.com>
> Date: Sun, 1 Sep 2013 23:39:14 +0800
> Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu wakeup
> 
> Currently HVM S3 has a bug coming from the difference between
> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
> to resume from hvm s3 is via 'xl trigger' command. However,
> for qemu-xen, the way to resume from hvm s3 inherited from
> standard qemu, i.e. via QMP, and it doesn't work under Xen.
> 
> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
> devices, while QMP didn't unpause hvm domain though they did qemu
> system reset.
> 
> We have two qemu-xen patches and one xl patch to fix the HVM S3 bug.
> This patch is the qemu-xen patch 1. It provides a later wakeup notifier
> and a register function, and notifies the later wakeup list when
> qemu wakup by 'xl trigger' command.
> 
> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> ---
>  sysemu.h |    1 +
>  vl.c     |    8 ++++++++
>  2 files changed, 9 insertions(+), 0 deletions(-)
> 
> diff --git a/sysemu.h b/sysemu.h
> index b71f244..4dbcab7 100644
> --- a/sysemu.h
> +++ b/sysemu.h
> @@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier *notifier);
>  void qemu_system_wakeup_request(WakeupReason reason);
>  void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
>  void qemu_register_wakeup_notifier(Notifier *notifier);
> +void qemu_register_later_wakeup_notifier(Notifier *notifier);
>  void qemu_system_shutdown_request(void);
>  void qemu_system_powerdown_request(void);
>  void qemu_register_powerdown_notifier(Notifier *notifier);
> diff --git a/vl.c b/vl.c
> index 5314f55..1c4842d 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>  static NotifierList wakeup_notifiers =
>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
> +static NotifierList later_wakeup_notifiers =
> +    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
>  static uint32_t wakeup_reason_mask = ~0;
>  static RunState vmstop_requested = RUN_STATE_MAX;
>  
> @@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
>      notifier_list_add(&wakeup_notifiers, notifier);
>  }
>  
> +void qemu_register_later_wakeup_notifier(Notifier *notifier)
> +{
> +    notifier_list_add(&later_wakeup_notifiers, notifier);
> +}
> +
>  void qemu_system_killed(int signal, pid_t pid)
>  {
>      shutdown_signal = signal;
> @@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
>          cpu_synchronize_all_states();
>          qemu_system_reset(VMRESET_SILENT);
>          resume_all_vcpus();
> +        notifier_list_notify(&later_wakeup_notifiers, NULL);
>          monitor_protocol_event(QEVENT_WAKEUP, NULL);
>      }
>      if (qemu_powerdown_requested()) {
> 

The patch those not apply properly to QEMU (upstream) but it just
because the file sysemu.h have been moved to include/sysemu/sysemu.h

Once this is fix:
Acked-by: Anthony PERARD <anthony.perard@citrix.com>

-- 
Anthony PERARD

[PATCH 1/2] qem-xen: add later wakeup logic when qemu wakeup

[Liu, Jinsong]

[-- Attachment #1: Type: text/plain, Size: 2867 bytes --]

>From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00 2001
From: Liu Jinsong <jinsong.liu@intel.com>
Date: Sun, 1 Sep 2013 23:39:14 +0800
Subject: [PATCH 1/2] qemu-xen: add later wakeup logic when qemu wakeup

Currently HVM S3 has a bug coming from the difference between
qemu-traditioanl and qemu-xen. For qemu-traditional, the way
to resume from hvm s3 is via 'xl trigger' command. However,
for qemu-xen, the way to resume from hvm s3 inherited from
standard qemu, i.e. via QMP, and it doesn't work under Xen.

The root cause is, for qemu-xen, 'xl trigger' command didn't reset
devices, while QMP didn't unpause hvm domain though they did qemu
system reset.

We have two qemu-xen patches and one xl patch to fix the HVM S3 bug.
This patch is the qemu-xen patch 1. It provides a later wakeup notifier
and a register function, and notifies the later wakeup list when
qemu wakup by 'xl trigger' command.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 sysemu.h |    1 +
 vl.c     |    8 ++++++++
 2 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/sysemu.h b/sysemu.h
index b71f244..4dbcab7 100644
--- a/sysemu.h
+++ b/sysemu.h
@@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier *notifier);
 void qemu_system_wakeup_request(WakeupReason reason);
 void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
 void qemu_register_wakeup_notifier(Notifier *notifier);
+void qemu_register_later_wakeup_notifier(Notifier *notifier);
 void qemu_system_shutdown_request(void);
 void qemu_system_powerdown_request(void);
 void qemu_register_powerdown_notifier(Notifier *notifier);
diff --git a/vl.c b/vl.c
index 5314f55..1c4842d 100644
--- a/vl.c
+++ b/vl.c
@@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
     NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
 static NotifierList wakeup_notifiers =
     NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
+static NotifierList later_wakeup_notifiers =
+    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
 static uint32_t wakeup_reason_mask = ~0;
 static RunState vmstop_requested = RUN_STATE_MAX;
 
@@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
     notifier_list_add(&wakeup_notifiers, notifier);
 }
 
+void qemu_register_later_wakeup_notifier(Notifier *notifier)
+{
+    notifier_list_add(&later_wakeup_notifiers, notifier);
+}
+
 void qemu_system_killed(int signal, pid_t pid)
 {
     shutdown_signal = signal;
@@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
         cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_SILENT);
         resume_all_vcpus();
+        notifier_list_notify(&later_wakeup_notifiers, NULL);
         monitor_protocol_event(QEVENT_WAKEUP, NULL);
     }
     if (qemu_powerdown_requested()) {
-- 
1.7.1

[-- Attachment #2: 0001-Add-later-wakeup-logic-when-qemu-wakeup.patch --]
[-- Type: application/octet-stream, Size: 2752 bytes --]

From 86ad3bb83a984ad7bbc00b81d6a0bfc1abc543ca Mon Sep 17 00:00:00 2001
From: Liu Jinsong <jinsong.liu@intel.com>
Date: Sun, 1 Sep 2013 23:39:14 +0800
Subject: [PATCH 1/2] Add later wakeup logic when qemu wakeup

Currently HVM S3 has a bug coming from the difference between
qemu-traditioanl and qemu-xen. For qemu-traditional, the way
to resume from hvm s3 is via 'xl trigger' command. However,
for qemu-xen, the way to resume from hvm s3 inherited from
standard qemu, i.e. via QMP, and it doesn't work under Xen.

The root cause is, for qemu-xen, 'xl trigger' command didn't reset
devices, while QMP didn't unpause hvm domain though they did qemu
system reset.

We have two qemu patches and one xl patch to fix the HVM S3 bug.
This patch is the qemu patch 1. It provides a later wakeup notifier
and a register function, and notifies the later wakeup list when
qemu wakup.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 sysemu.h |    1 +
 vl.c     |    8 ++++++++
 2 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/sysemu.h b/sysemu.h
index b71f244..4dbcab7 100644
--- a/sysemu.h
+++ b/sysemu.h
@@ -49,6 +49,7 @@ void qemu_register_suspend_notifier(Notifier *notifier);
 void qemu_system_wakeup_request(WakeupReason reason);
 void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
 void qemu_register_wakeup_notifier(Notifier *notifier);
+void qemu_register_later_wakeup_notifier(Notifier *notifier);
 void qemu_system_shutdown_request(void);
 void qemu_system_powerdown_request(void);
 void qemu_register_powerdown_notifier(Notifier *notifier);
diff --git a/vl.c b/vl.c
index 5314f55..1c4842d 100644
--- a/vl.c
+++ b/vl.c
@@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
     NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
 static NotifierList wakeup_notifiers =
     NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
+static NotifierList later_wakeup_notifiers =
+    NOTIFIER_LIST_INITIALIZER(later_wakeup_notifiers);
 static uint32_t wakeup_reason_mask = ~0;
 static RunState vmstop_requested = RUN_STATE_MAX;
 
@@ -1668,6 +1670,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
     notifier_list_add(&wakeup_notifiers, notifier);
 }
 
+void qemu_register_later_wakeup_notifier(Notifier *notifier)
+{
+    notifier_list_add(&later_wakeup_notifiers, notifier);
+}
+
 void qemu_system_killed(int signal, pid_t pid)
 {
     shutdown_signal = signal;
@@ -1744,6 +1751,7 @@ static bool main_loop_should_exit(void)
         cpu_synchronize_all_states();
         qemu_system_reset(VMRESET_SILENT);
         resume_all_vcpus();
+        notifier_list_notify(&later_wakeup_notifiers, NULL);
         monitor_protocol_event(QEVENT_WAKEUP, NULL);
     }
     if (qemu_powerdown_requested()) {
-- 
1.7.1


[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[PATCH 2/2] qemu-xen: add qemu xen logic for HVM S3 resume

[Liu, Jinsong]

[-- Attachment #1: Type: text/plain, Size: 1747 bytes --]

>From e7d4bd70eae8da131dc3ff2cec70cb2c7b6575a9 Mon Sep 17 00:00:00 2001
From: Liu Jinsong <jinsong.liu@intel.com>
Date: Mon, 2 Sep 2013 00:39:20 +0800
Subject: [PATCH 2/2] qemu-xen: add qemu xen logic for HVM S3 resume

This patch is qemu-xen patch 2 to fix HVM S3 bug, adding qemu
xen logic. When qemu wakeup, qemu xen logic is notified and
hypercall to xen hypervisor to unpause domain.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 xen-all.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/xen-all.c b/xen-all.c
index 15be8ed..bef946b 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -97,6 +97,7 @@ typedef struct XenIOState {
 
     Notifier exit;
     Notifier suspend;
+    Notifier later_wakeup;
 } XenIOState;
 
 /* Xen specific function for piix pci */
@@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
     xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
 }
 
+static void xen_later_wakeup_notifier(Notifier *notifier, void *data)
+{
+    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
+}
+
 /* Xen Interrupt Controller */
 
 static void xen_set_irq(void *opaque, int irq, int level)
@@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
     state->suspend.notify = xen_suspend_notifier;
     qemu_register_suspend_notifier(&state->suspend);
 
+    state->later_wakeup.notify = xen_later_wakeup_notifier;
+    qemu_register_later_wakeup_notifier(&state->later_wakeup);
+
     xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
     DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
     state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-- 
1.7.1

[-- Attachment #2: 0002-Add-qemu-xen-logic-for-HVM-S3-resume.patch --]
[-- Type: application/octet-stream, Size: 1683 bytes --]

From e7d4bd70eae8da131dc3ff2cec70cb2c7b6575a9 Mon Sep 17 00:00:00 2001
From: Liu Jinsong <jinsong.liu@intel.com>
Date: Mon, 2 Sep 2013 00:39:20 +0800
Subject: [PATCH 2/2] Add qemu xen logic for HVM S3 resume

This patch is qemu patch 2 to fix HVM S3 bug, adding qemu
xen logic. When qemu wakeup, qemu xen logic is notified and
hypercall to xen hypervisor to unpause domain.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 xen-all.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/xen-all.c b/xen-all.c
index 15be8ed..bef946b 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -97,6 +97,7 @@ typedef struct XenIOState {
 
     Notifier exit;
     Notifier suspend;
+    Notifier later_wakeup;
 } XenIOState;
 
 /* Xen specific function for piix pci */
@@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
     xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
 }
 
+static void xen_later_wakeup_notifier(Notifier *notifier, void *data)
+{
+    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
+}
+
 /* Xen Interrupt Controller */
 
 static void xen_set_irq(void *opaque, int irq, int level)
@@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
     state->suspend.notify = xen_suspend_notifier;
     qemu_register_suspend_notifier(&state->suspend);
 
+    state->later_wakeup.notify = xen_later_wakeup_notifier;
+    qemu_register_later_wakeup_notifier(&state->later_wakeup);
+
     xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
     DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
     state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-- 
1.7.1


[-- Attachment #3: Type: text/plain, Size: 126 bytes --]

_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
http://lists.xen.org/xen-devel

[Qemu-devel] [PATCH 2/2] qemu-xen: add qemu xen logic for HVM S3 resume

[Liu, Jinsong]

[-- Attachment #1: Type: text/plain, Size: 1747 bytes --]

>From e7d4bd70eae8da131dc3ff2cec70cb2c7b6575a9 Mon Sep 17 00:00:00 2001
From: Liu Jinsong <jinsong.liu@intel.com>
Date: Mon, 2 Sep 2013 00:39:20 +0800
Subject: [PATCH 2/2] qemu-xen: add qemu xen logic for HVM S3 resume

This patch is qemu-xen patch 2 to fix HVM S3 bug, adding qemu
xen logic. When qemu wakeup, qemu xen logic is notified and
hypercall to xen hypervisor to unpause domain.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 xen-all.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/xen-all.c b/xen-all.c
index 15be8ed..bef946b 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -97,6 +97,7 @@ typedef struct XenIOState {
 
     Notifier exit;
     Notifier suspend;
+    Notifier later_wakeup;
 } XenIOState;
 
 /* Xen specific function for piix pci */
@@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
     xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
 }
 
+static void xen_later_wakeup_notifier(Notifier *notifier, void *data)
+{
+    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
+}
+
 /* Xen Interrupt Controller */
 
 static void xen_set_irq(void *opaque, int irq, int level)
@@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
     state->suspend.notify = xen_suspend_notifier;
     qemu_register_suspend_notifier(&state->suspend);
 
+    state->later_wakeup.notify = xen_later_wakeup_notifier;
+    qemu_register_later_wakeup_notifier(&state->later_wakeup);
+
     xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
     DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
     state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-- 
1.7.1

[-- Attachment #2: 0002-Add-qemu-xen-logic-for-HVM-S3-resume.patch --]
[-- Type: application/octet-stream, Size: 1683 bytes --]

From e7d4bd70eae8da131dc3ff2cec70cb2c7b6575a9 Mon Sep 17 00:00:00 2001
From: Liu Jinsong <jinsong.liu@intel.com>
Date: Mon, 2 Sep 2013 00:39:20 +0800
Subject: [PATCH 2/2] Add qemu xen logic for HVM S3 resume

This patch is qemu patch 2 to fix HVM S3 bug, adding qemu
xen logic. When qemu wakeup, qemu xen logic is notified and
hypercall to xen hypervisor to unpause domain.

Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
---
 xen-all.c |    9 +++++++++
 1 files changed, 9 insertions(+), 0 deletions(-)

diff --git a/xen-all.c b/xen-all.c
index 15be8ed..bef946b 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -97,6 +97,7 @@ typedef struct XenIOState {
 
     Notifier exit;
     Notifier suspend;
+    Notifier later_wakeup;
 } XenIOState;
 
 /* Xen specific function for piix pci */
@@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
     xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
 }
 
+static void xen_later_wakeup_notifier(Notifier *notifier, void *data)
+{
+    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
+}
+
 /* Xen Interrupt Controller */
 
 static void xen_set_irq(void *opaque, int irq, int level)
@@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
     state->suspend.notify = xen_suspend_notifier;
     qemu_register_suspend_notifier(&state->suspend);
 
+    state->later_wakeup.notify = xen_later_wakeup_notifier;
+    qemu_register_later_wakeup_notifier(&state->later_wakeup);
+
     xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
     DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
     state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
-- 
1.7.1

Re: [Qemu-devel] [PATCH 2/2] qemu-xen: add qemu xen logic for HVM S3 resume

[Anthony PERARD]

On 01/09/13 10:54, Liu, Jinsong wrote:
> From e7d4bd70eae8da131dc3ff2cec70cb2c7b6575a9 Mon Sep 17 00:00:00 2001
> From: Liu Jinsong <jinsong.liu@intel.com>
> Date: Mon, 2 Sep 2013 00:39:20 +0800
> Subject: [PATCH 2/2] qemu-xen: add qemu xen logic for HVM S3 resume
> 
> This patch is qemu-xen patch 2 to fix HVM S3 bug, adding qemu
> xen logic. When qemu wakeup, qemu xen logic is notified and
> hypercall to xen hypervisor to unpause domain.
> 
> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> ---
>  xen-all.c |    9 +++++++++
>  1 files changed, 9 insertions(+), 0 deletions(-)
> 
> diff --git a/xen-all.c b/xen-all.c
> index 15be8ed..bef946b 100644
> --- a/xen-all.c
> +++ b/xen-all.c
> @@ -97,6 +97,7 @@ typedef struct XenIOState {
>  
>      Notifier exit;
>      Notifier suspend;
> +    Notifier later_wakeup;
>  } XenIOState;
>  
>  /* Xen specific function for piix pci */
> @@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
>      xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
>  }
>  
> +static void xen_later_wakeup_notifier(Notifier *notifier, void *data)
> +{
> +    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
> +}
> +
>  /* Xen Interrupt Controller */
>  
>  static void xen_set_irq(void *opaque, int irq, int level)
> @@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
>      state->suspend.notify = xen_suspend_notifier;
>      qemu_register_suspend_notifier(&state->suspend);
>  
> +    state->later_wakeup.notify = xen_later_wakeup_notifier;
> +    qemu_register_later_wakeup_notifier(&state->later_wakeup);
> +
>      xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
>      DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
>      state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
> 

Acked-by: Anthony PERARD <anthony.perard@citrix.com>

-- 
Anthony PERARD

Re: [PATCH 2/2] qemu-xen: add qemu xen logic for HVM S3 resume

[Anthony PERARD]

On 01/09/13 10:54, Liu, Jinsong wrote:
> From e7d4bd70eae8da131dc3ff2cec70cb2c7b6575a9 Mon Sep 17 00:00:00 2001
> From: Liu Jinsong <jinsong.liu@intel.com>
> Date: Mon, 2 Sep 2013 00:39:20 +0800
> Subject: [PATCH 2/2] qemu-xen: add qemu xen logic for HVM S3 resume
> 
> This patch is qemu-xen patch 2 to fix HVM S3 bug, adding qemu
> xen logic. When qemu wakeup, qemu xen logic is notified and
> hypercall to xen hypervisor to unpause domain.
> 
> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> ---
>  xen-all.c |    9 +++++++++
>  1 files changed, 9 insertions(+), 0 deletions(-)
> 
> diff --git a/xen-all.c b/xen-all.c
> index 15be8ed..bef946b 100644
> --- a/xen-all.c
> +++ b/xen-all.c
> @@ -97,6 +97,7 @@ typedef struct XenIOState {
>  
>      Notifier exit;
>      Notifier suspend;
> +    Notifier later_wakeup;
>  } XenIOState;
>  
>  /* Xen specific function for piix pci */
> @@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
>      xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
>  }
>  
> +static void xen_later_wakeup_notifier(Notifier *notifier, void *data)
> +{
> +    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
> +}
> +
>  /* Xen Interrupt Controller */
>  
>  static void xen_set_irq(void *opaque, int irq, int level)
> @@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
>      state->suspend.notify = xen_suspend_notifier;
>      qemu_register_suspend_notifier(&state->suspend);
>  
> +    state->later_wakeup.notify = xen_later_wakeup_notifier;
> +    qemu_register_later_wakeup_notifier(&state->later_wakeup);
> +
>      xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
>      DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
>      state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
> 

Acked-by: Anthony PERARD <anthony.perard@citrix.com>

-- 
Anthony PERARD

Re: [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Liu, Jinsong]

> 
> Could you break this path into two? One which add the notifier list
> and a second one with the Xen specific part?
> 

Updated and sent out.

Thanks,
Jinsong

Re: [Qemu-devel] [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Liu, Jinsong]

> 
> Could you break this path into two? One which add the notifier list
> and a second one with the Xen specific part?
> 

Updated and sent out.

Thanks,
Jinsong

Re: [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Anthony PERARD]

On 29/08/13 09:25, Liu, Jinsong wrote:
> Currently HVM S3 has a bug coming from the difference between
> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
> to resume from hvm s3 is via 'xl trigger' command. However,
> for qemu-xen, the way to resume from hvm s3 inherited from
> standard qemu, i.e. via QMP, and it doesn't work under Xen.
> 
> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
> devices, while QMP didn't unpause hvm domain though they did qemu
> system reset.
> 
> We have 2 patches to fix the HVM S3 bug: qemu-xen patch and xl patch.
> This patch is the qemu-xen patch. It registers a wakeup later notify,
> so that when 'xl trigger' command invokes QMP system_wakeup and after
> qemu system reset, it hypercalls to hypervisor to unpause domain, then
> hvm S3 resumes successfully.
> 
> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> ---
>  vl.c      |   13 +++++++++++++
>  xen-all.c |    9 +++++++++
>  2 files changed, 22 insertions(+), 0 deletions(-)
> 
> diff --git a/vl.c b/vl.c
> index 5314f55..aeebd83 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>  static NotifierList wakeup_notifiers =
>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
> +static NotifierList wakeup_later_notifiers =
> +    NOTIFIER_LIST_INITIALIZER(wakeup_later_notifiers);

Maybe late_wakeup_notifiers would be a better description for this list.

>  static uint32_t wakeup_reason_mask = ~0;
>  static RunState vmstop_requested = RUN_STATE_MAX;
>  
> @@ -1625,6 +1627,11 @@ static void qemu_system_suspend(void)
>      monitor_protocol_event(QEVENT_SUSPEND, NULL);
>  }
>  
> +static void qemu_system_wakeup(void)
> +{
> +    notifier_list_notify(&wakeup_later_notifiers, NULL);
> +}
> +

I don't think it's useful to have this function with only the
list_notify. You could call directly list_notify in the
main_loop_should_exit() function bellow.

>  void qemu_system_suspend_request(void)
>  {
>      if (runstate_check(RUN_STATE_SUSPENDED)) {
> @@ -1668,6 +1675,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
>      notifier_list_add(&wakeup_notifiers, notifier);
>  }
>  
> +void qemu_register_wakeup_later_notifier(Notifier *notifier)
> +{
> +    notifier_list_add(&wakeup_later_notifiers, notifier);
> +}
> +
>  void qemu_system_killed(int signal, pid_t pid)
>  {
>      shutdown_signal = signal;
> @@ -1744,6 +1756,7 @@ static bool main_loop_should_exit(void)
>          cpu_synchronize_all_states();
>          qemu_system_reset(VMRESET_SILENT);
>          resume_all_vcpus();
> +        qemu_system_wakeup();
>          monitor_protocol_event(QEVENT_WAKEUP, NULL);
>      }
>      if (qemu_powerdown_requested()) {
> diff --git a/xen-all.c b/xen-all.c
> index 15be8ed..3353f63 100644
> --- a/xen-all.c
> +++ b/xen-all.c
> @@ -97,6 +97,7 @@ typedef struct XenIOState {
>  
>      Notifier exit;
>      Notifier suspend;
> +    Notifier wakeup_later;
>  } XenIOState;
>  
>  /* Xen specific function for piix pci */
> @@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
>      xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
>  }
>  
> +static void xen_wakeup_later_notifier(Notifier *notifier, void *data)
> +{
> +    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
> +}
> +
>  /* Xen Interrupt Controller */
>  
>  static void xen_set_irq(void *opaque, int irq, int level)
> @@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
>      state->suspend.notify = xen_suspend_notifier;
>      qemu_register_suspend_notifier(&state->suspend);
>  
> +    state->wakeup_later.notify = xen_wakeup_later_notifier;
> +    qemu_register_wakeup_later_notifier(&state->wakeup_later);
> +
>      xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
>      DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
>      state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
> 

Could you break this path into two? One which add the notifier list and
a second one with the Xen specific part?

Thanks,

-- 
Anthony PERARD

Re: [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Paolo Bonzini]

Il 29/08/2013 10:25, Liu, Jinsong ha scritto:
> Currently HVM S3 has a bug coming from the difference between
> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
> to resume from hvm s3 is via 'xl trigger' command. However,
> for qemu-xen, the way to resume from hvm s3 inherited from
> standard qemu, i.e. via QMP, and it doesn't work under Xen.
> 
> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
> devices, while QMP didn't unpause hvm domain though they did qemu
> system reset.
> 
> We have 2 patches to fix the HVM S3 bug: qemu-xen patch and xl patch.
> This patch is the qemu-xen patch. It registers a wakeup later notify,
> so that when 'xl trigger' command invokes QMP system_wakeup and after
> qemu system reset, it hypercalls to hypervisor to unpause domain, then
> hvm S3 resumes successfully.
> 
> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> ---
>  vl.c      |   13 +++++++++++++
>  xen-all.c |    9 +++++++++
>  2 files changed, 22 insertions(+), 0 deletions(-)
> 
> diff --git a/vl.c b/vl.c
> index 5314f55..aeebd83 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>  static NotifierList wakeup_notifiers =
>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
> +static NotifierList wakeup_later_notifiers =
> +    NOTIFIER_LIST_INITIALIZER(wakeup_later_notifiers);
>  static uint32_t wakeup_reason_mask = ~0;
>  static RunState vmstop_requested = RUN_STATE_MAX;
>  
> @@ -1625,6 +1627,11 @@ static void qemu_system_suspend(void)
>      monitor_protocol_event(QEVENT_SUSPEND, NULL);
>  }
>  
> +static void qemu_system_wakeup(void)
> +{
> +    notifier_list_notify(&wakeup_later_notifiers, NULL);
> +}
> +
>  void qemu_system_suspend_request(void)
>  {
>      if (runstate_check(RUN_STATE_SUSPENDED)) {
> @@ -1668,6 +1675,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
>      notifier_list_add(&wakeup_notifiers, notifier);
>  }
>  
> +void qemu_register_wakeup_later_notifier(Notifier *notifier)
> +{
> +    notifier_list_add(&wakeup_later_notifiers, notifier);
> +}
> +
>  void qemu_system_killed(int signal, pid_t pid)
>  {
>      shutdown_signal = signal;
> @@ -1744,6 +1756,7 @@ static bool main_loop_should_exit(void)
>          cpu_synchronize_all_states();
>          qemu_system_reset(VMRESET_SILENT);
>          resume_all_vcpus();
> +        qemu_system_wakeup();

Does Xen work if the hypercall is placed before resume_all_vcpus?  In
this case, you can just move the wakeup_notifiers invocation from
qemu_system_wakeup_request to here, and avoid introducing a separate list.

Paolo

>          monitor_protocol_event(QEVENT_WAKEUP, NULL);
>      }
>      if (qemu_powerdown_requested()) {
> diff --git a/xen-all.c b/xen-all.c
> index 15be8ed..3353f63 100644
> --- a/xen-all.c
> +++ b/xen-all.c
> @@ -97,6 +97,7 @@ typedef struct XenIOState {
>  
>      Notifier exit;
>      Notifier suspend;
> +    Notifier wakeup_later;
>  } XenIOState;
>  
>  /* Xen specific function for piix pci */
> @@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
>      xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
>  }
>  
> +static void xen_wakeup_later_notifier(Notifier *notifier, void *data)
> +{
> +    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
> +}
> +
>  /* Xen Interrupt Controller */
>  
>  static void xen_set_irq(void *opaque, int irq, int level)
> @@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
>      state->suspend.notify = xen_suspend_notifier;
>      qemu_register_suspend_notifier(&state->suspend);
>  
> +    state->wakeup_later.notify = xen_wakeup_later_notifier;
> +    qemu_register_wakeup_later_notifier(&state->wakeup_later);
> +
>      xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
>      DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
>      state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
> 

Re: [Qemu-devel] [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Paolo Bonzini]

Il 29/08/2013 10:25, Liu, Jinsong ha scritto:
> Currently HVM S3 has a bug coming from the difference between
> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
> to resume from hvm s3 is via 'xl trigger' command. However,
> for qemu-xen, the way to resume from hvm s3 inherited from
> standard qemu, i.e. via QMP, and it doesn't work under Xen.
> 
> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
> devices, while QMP didn't unpause hvm domain though they did qemu
> system reset.
> 
> We have 2 patches to fix the HVM S3 bug: qemu-xen patch and xl patch.
> This patch is the qemu-xen patch. It registers a wakeup later notify,
> so that when 'xl trigger' command invokes QMP system_wakeup and after
> qemu system reset, it hypercalls to hypervisor to unpause domain, then
> hvm S3 resumes successfully.
> 
> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
> ---
>  vl.c      |   13 +++++++++++++
>  xen-all.c |    9 +++++++++
>  2 files changed, 22 insertions(+), 0 deletions(-)
> 
> diff --git a/vl.c b/vl.c
> index 5314f55..aeebd83 100644
> --- a/vl.c
> +++ b/vl.c
> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>  static NotifierList wakeup_notifiers =
>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
> +static NotifierList wakeup_later_notifiers =
> +    NOTIFIER_LIST_INITIALIZER(wakeup_later_notifiers);
>  static uint32_t wakeup_reason_mask = ~0;
>  static RunState vmstop_requested = RUN_STATE_MAX;
>  
> @@ -1625,6 +1627,11 @@ static void qemu_system_suspend(void)
>      monitor_protocol_event(QEVENT_SUSPEND, NULL);
>  }
>  
> +static void qemu_system_wakeup(void)
> +{
> +    notifier_list_notify(&wakeup_later_notifiers, NULL);
> +}
> +
>  void qemu_system_suspend_request(void)
>  {
>      if (runstate_check(RUN_STATE_SUSPENDED)) {
> @@ -1668,6 +1675,11 @@ void qemu_register_wakeup_notifier(Notifier *notifier)
>      notifier_list_add(&wakeup_notifiers, notifier);
>  }
>  
> +void qemu_register_wakeup_later_notifier(Notifier *notifier)
> +{
> +    notifier_list_add(&wakeup_later_notifiers, notifier);
> +}
> +
>  void qemu_system_killed(int signal, pid_t pid)
>  {
>      shutdown_signal = signal;
> @@ -1744,6 +1756,7 @@ static bool main_loop_should_exit(void)
>          cpu_synchronize_all_states();
>          qemu_system_reset(VMRESET_SILENT);
>          resume_all_vcpus();
> +        qemu_system_wakeup();

Does Xen work if the hypercall is placed before resume_all_vcpus?  In
this case, you can just move the wakeup_notifiers invocation from
qemu_system_wakeup_request to here, and avoid introducing a separate list.

Paolo

>          monitor_protocol_event(QEVENT_WAKEUP, NULL);
>      }
>      if (qemu_powerdown_requested()) {
> diff --git a/xen-all.c b/xen-all.c
> index 15be8ed..3353f63 100644
> --- a/xen-all.c
> +++ b/xen-all.c
> @@ -97,6 +97,7 @@ typedef struct XenIOState {
>  
>      Notifier exit;
>      Notifier suspend;
> +    Notifier wakeup_later;
>  } XenIOState;
>  
>  /* Xen specific function for piix pci */
> @@ -139,6 +140,11 @@ static void xen_suspend_notifier(Notifier *notifier, void *data)
>      xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 3);
>  }
>  
> +static void xen_wakeup_later_notifier(Notifier *notifier, void *data)
> +{
> +    xc_set_hvm_param(xen_xc, xen_domid, HVM_PARAM_ACPI_S_STATE, 0);
> +}
> +
>  /* Xen Interrupt Controller */
>  
>  static void xen_set_irq(void *opaque, int irq, int level)
> @@ -1106,6 +1112,9 @@ int xen_hvm_init(void)
>      state->suspend.notify = xen_suspend_notifier;
>      qemu_register_suspend_notifier(&state->suspend);
>  
> +    state->wakeup_later.notify = xen_wakeup_later_notifier;
> +    qemu_register_wakeup_later_notifier(&state->wakeup_later);
> +
>      xc_get_hvm_param(xen_xc, xen_domid, HVM_PARAM_IOREQ_PFN, &ioreq_pfn);
>      DPRINTF("shared page at pfn %lx\n", ioreq_pfn);
>      state->shared_page = xc_map_foreign_range(xen_xc, xen_domid, XC_PAGE_SIZE,
> 

Re: [Qemu-devel] [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Liu, Jinsong]

Paolo Bonzini wrote:
> Il 29/08/2013 10:25, Liu, Jinsong ha scritto:
>> Currently HVM S3 has a bug coming from the difference between
>> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
>> to resume from hvm s3 is via 'xl trigger' command. However,
>> for qemu-xen, the way to resume from hvm s3 inherited from
>> standard qemu, i.e. via QMP, and it doesn't work under Xen.
>> 
>> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
>> devices, while QMP didn't unpause hvm domain though they did qemu
>> system reset.
>> 
>> We have 2 patches to fix the HVM S3 bug: qemu-xen patch and xl patch.
>> This patch is the qemu-xen patch. It registers a wakeup later notify,
>> so that when 'xl trigger' command invokes QMP system_wakeup and after
>> qemu system reset, it hypercalls to hypervisor to unpause domain,
>> then 
>> hvm S3 resumes successfully.
>> 
>> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
>> ---
>>  vl.c      |   13 +++++++++++++
>>  xen-all.c |    9 +++++++++
>>  2 files changed, 22 insertions(+), 0 deletions(-)
>> 
>> diff --git a/vl.c b/vl.c
>> index 5314f55..aeebd83 100644
>> --- a/vl.c
>> +++ b/vl.c
>> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>>  static NotifierList wakeup_notifiers =
>>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
>> +static NotifierList wakeup_later_notifiers =
>> +    NOTIFIER_LIST_INITIALIZER(wakeup_later_notifiers);
>>  static uint32_t wakeup_reason_mask = ~0;
>>  static RunState vmstop_requested = RUN_STATE_MAX;
>> 
>> @@ -1625,6 +1627,11 @@ static void qemu_system_suspend(void)
>>      monitor_protocol_event(QEVENT_SUSPEND, NULL);
>>  }
>> 
>> +static void qemu_system_wakeup(void)
>> +{
>> +    notifier_list_notify(&wakeup_later_notifiers, NULL); +}
>> +
>>  void qemu_system_suspend_request(void)
>>  {
>>      if (runstate_check(RUN_STATE_SUSPENDED)) {
>> @@ -1668,6 +1675,11 @@ void qemu_register_wakeup_notifier(Notifier
>>      *notifier) notifier_list_add(&wakeup_notifiers, notifier);  }
>> 
>> +void qemu_register_wakeup_later_notifier(Notifier *notifier) +{
>> +    notifier_list_add(&wakeup_later_notifiers, notifier); +}
>> +
>>  void qemu_system_killed(int signal, pid_t pid)
>>  {
>>      shutdown_signal = signal;
>> @@ -1744,6 +1756,7 @@ static bool main_loop_should_exit(void)
>>          cpu_synchronize_all_states();
>>          qemu_system_reset(VMRESET_SILENT);
>>          resume_all_vcpus();
>> +        qemu_system_wakeup();
> 
> Does Xen work if the hypercall is placed before resume_all_vcpus?  In
> this case, you can just move the wakeup_notifiers invocation from
> qemu_system_wakeup_request to here, and avoid introducing a separate
> list. 
> 
> Paolo
> 

No problem. Have updated and sent out.

Thanks,
Jinsong

Re: [PATCH V2] qemu-xen: HVM domain S3 bugfix

[Liu, Jinsong]

Paolo Bonzini wrote:
> Il 29/08/2013 10:25, Liu, Jinsong ha scritto:
>> Currently HVM S3 has a bug coming from the difference between
>> qemu-traditioanl and qemu-xen. For qemu-traditional, the way
>> to resume from hvm s3 is via 'xl trigger' command. However,
>> for qemu-xen, the way to resume from hvm s3 inherited from
>> standard qemu, i.e. via QMP, and it doesn't work under Xen.
>> 
>> The root cause is, for qemu-xen, 'xl trigger' command didn't reset
>> devices, while QMP didn't unpause hvm domain though they did qemu
>> system reset.
>> 
>> We have 2 patches to fix the HVM S3 bug: qemu-xen patch and xl patch.
>> This patch is the qemu-xen patch. It registers a wakeup later notify,
>> so that when 'xl trigger' command invokes QMP system_wakeup and after
>> qemu system reset, it hypercalls to hypervisor to unpause domain,
>> then 
>> hvm S3 resumes successfully.
>> 
>> Signed-off-by: Liu Jinsong <jinsong.liu@intel.com>
>> ---
>>  vl.c      |   13 +++++++++++++
>>  xen-all.c |    9 +++++++++
>>  2 files changed, 22 insertions(+), 0 deletions(-)
>> 
>> diff --git a/vl.c b/vl.c
>> index 5314f55..aeebd83 100644
>> --- a/vl.c
>> +++ b/vl.c
>> @@ -1478,6 +1478,8 @@ static NotifierList suspend_notifiers =
>>      NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
>>  static NotifierList wakeup_notifiers =
>>      NOTIFIER_LIST_INITIALIZER(wakeup_notifiers);
>> +static NotifierList wakeup_later_notifiers =
>> +    NOTIFIER_LIST_INITIALIZER(wakeup_later_notifiers);
>>  static uint32_t wakeup_reason_mask = ~0;
>>  static RunState vmstop_requested = RUN_STATE_MAX;
>> 
>> @@ -1625,6 +1627,11 @@ static void qemu_system_suspend(void)
>>      monitor_protocol_event(QEVENT_SUSPEND, NULL);
>>  }
>> 
>> +static void qemu_system_wakeup(void)
>> +{
>> +    notifier_list_notify(&wakeup_later_notifiers, NULL); +}
>> +
>>  void qemu_system_suspend_request(void)
>>  {
>>      if (runstate_check(RUN_STATE_SUSPENDED)) {
>> @@ -1668,6 +1675,11 @@ void qemu_register_wakeup_notifier(Notifier
>>      *notifier) notifier_list_add(&wakeup_notifiers, notifier);  }
>> 
>> +void qemu_register_wakeup_later_notifier(Notifier *notifier) +{
>> +    notifier_list_add(&wakeup_later_notifiers, notifier); +}
>> +
>>  void qemu_system_killed(int signal, pid_t pid)
>>  {
>>      shutdown_signal = signal;
>> @@ -1744,6 +1756,7 @@ static bool main_loop_should_exit(void)
>>          cpu_synchronize_all_states();
>>          qemu_system_reset(VMRESET_SILENT);
>>          resume_all_vcpus();
>> +        qemu_system_wakeup();
> 
> Does Xen work if the hypercall is placed before resume_all_vcpus?  In
> this case, you can just move the wakeup_notifiers invocation from
> qemu_system_wakeup_request to here, and avoid introducing a separate
> list. 
> 
> Paolo
> 

No problem. Have updated and sent out.

Thanks,
Jinsong