This is the user space tools support to offline a page. 

Changeset 19286:dd489125a2e7 implement the HV side page offline logic. This patch add support to use space tools.

When a page owned by a PV domain is marked "offline pending", user space tools will try to suspend the guest, scan the whole page table and update the page table entry one by one. If no more reference to the page, the page become "offlined".

xc_offline_page() is added will update all reference to the offline pending pages.
xc_map_m2p() is exported from xc_domain_save.c so that we don't need implement it again.

Signed-off-by: Jiang, Yunhong <yunhong.jiang@intel.com>

diff -r fa6b2732778f tools/libxc/Makefile
--- a/tools/libxc/Makefile	Wed Mar 18 00:19:25 2009 +0800
+++ b/tools/libxc/Makefile	Wed Mar 18 00:19:28 2009 +0800
@@ -31,6 +31,7 @@ GUEST_SRCS-y :=
 GUEST_SRCS-y :=
 GUEST_SRCS-y += xg_private.c
 GUEST_SRCS-$(CONFIG_MIGRATE) += xc_domain_restore.c xc_domain_save.c
+GUEST_SRCS-y += xc_offline_page.c
 GUEST_SRCS-$(CONFIG_HVM) += xc_hvm_build.c
 
 vpath %.c ../../xen/common/libelf
diff -r fa6b2732778f tools/libxc/xc_domain_save.c
--- a/tools/libxc/xc_domain_save.c	Wed Mar 18 00:19:25 2009 +0800
+++ b/tools/libxc/xc_domain_save.c	Wed Mar 18 00:19:28 2009 +0800
@@ -510,9 +510,10 @@ static int canonicalize_pagetable(unsign
     return race;
 }
 
-static xen_pfn_t *xc_map_m2p(int xc_handle,
-                                 unsigned long max_mfn,
-                                 int prot)
+xen_pfn_t *xc_map_m2p(int xc_handle,
+                      unsigned long max_mfn,
+                      int prot,
+                      unsigned long *mfn0)
 {
     struct xen_machphys_mfn_list xmml;
     privcmd_mmap_entry_t *entries;
@@ -561,7 +562,8 @@ static xen_pfn_t *xc_map_m2p(int xc_hand
         goto err2;
     }
 
-    m2p_mfn0 = entries[0].mfn;
+    if (mfn0)
+        *mfn0 = entries[0].mfn;
 
 err2:
     free(entries);
@@ -1058,7 +1060,7 @@ int xc_domain_save(int xc_handle, int io
     }
 
     /* Setup the mfn_to_pfn table mapping */
-    if ( !(live_m2p = xc_map_m2p(xc_handle, max_mfn, PROT_READ)) )
+    if ( !(live_m2p = xc_map_m2p(xc_handle, max_mfn, PROT_READ, &m2p_mfn0)) )
     {
         ERROR("Failed to map live M2P table");
         goto out;
diff -r fa6b2732778f tools/libxc/xc_offline_page.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/libxc/xc_offline_page.c	Wed Mar 18 01:42:25 2009 +0800
@@ -0,0 +1,388 @@
+/******************************************************************************
+ * xc_offline_page.c
+ *
+ * Helper functions to replace a offlining page
+ *
+ * Copyright (c) 2003, K A Fraser.
+ * Copyright (c) 2009, Intel Corporation.
+ */
+
+#include <inttypes.h>
+#include <time.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/time.h>
+
+#include "xc_private.h"
+#include "xc_dom.h"
+#include "xg_private.h"
+#include "xg_save_restore.h"
+
+#undef DEBUG
+#define DEBUG(_f, _a...) fprintf(stderr, _f , ## _a)
+
+int xc_mark_page_online(int xc, unsigned long start,
+                        unsigned long end, uint32_t *status)
+{
+    DECLARE_SYSCTL;
+    int ret = -1;
+
+    if ( !status || (end < start) )
+        return -EINVAL;
+
+    if (lock_pages(status, sizeof(uint32_t)*(end - start + 1)))
+    {
+        fprintf(stderr, "Could not lock memory for Xen hypercall");
+        return -EINVAL;
+    }
+
+    sysctl.cmd = XEN_SYSCTL_page_offline_op;
+    sysctl.u.page_offline.start = start;
+    sysctl.u.page_offline.cmd = sysctl_page_online;
+    sysctl.u.page_offline.end = end;
+    set_xen_guest_handle(sysctl.u.page_offline.status, status);
+    ret = xc_sysctl(xc, &sysctl);
+
+    unlock_pages(status, sizeof(uint32_t)*(end - start + 1));
+
+    return ret;
+}
+
+int xc_mark_page_offline(int xc, unsigned long start,
+                          unsigned long end, uint32_t *status)
+{
+    DECLARE_SYSCTL;
+    int ret = -1;
+
+    if ( !status || (end < start) )
+        return -EINVAL;
+
+    if (lock_pages(status, sizeof(uint32_t)*(end - start + 1)))
+    {
+        fprintf(stderr, "Could not lock memory for Xen hypercall");
+        return -EINVAL;
+    }
+
+    sysctl.cmd = XEN_SYSCTL_page_offline_op;
+    sysctl.u.page_offline.start = start;
+    sysctl.u.page_offline.cmd = sysctl_page_offline;
+    sysctl.u.page_offline.end = end;
+    set_xen_guest_handle(sysctl.u.page_offline.status, status);
+    ret = xc_sysctl(xc, &sysctl);
+
+    unlock_pages(status, sizeof(uint32_t)*(end - start + 1));
+
+    return ret;
+}
+
+int xc_query_page_offline_status(int xc, unsigned long start,
+                                 unsigned long end, uint32_t *status)
+{
+    DECLARE_SYSCTL;
+    int ret = -1;
+
+    if ( !status || (end < start) )
+        return -EINVAL;
+
+    if (lock_pages(status, sizeof(uint32_t)*(end - start + 1)))
+    {
+        fprintf(stderr, "Could not lock memory for Xen hypercall");
+        return -EINVAL;
+    }
+
+    sysctl.cmd = XEN_SYSCTL_page_offline_op;
+    sysctl.u.page_offline.start = start;
+    sysctl.u.page_offline.cmd = sysctl_query_page_offline;
+    sysctl.u.page_offline.end = end;
+    set_xen_guest_handle(sysctl.u.page_offline.status, status);
+    ret = xc_sysctl(xc, &sysctl);
+
+    unlock_pages(status, sizeof(uint32_t)*(end - start + 1));
+
+    return ret;
+}
+
+#define MAX_OFFLINE_BATCH 1024
+
+static xen_pfn_t pfn_to_mfn(xen_pfn_t pfn, xen_pfn_t *p2m, int guest_width)
+{
+  return ((xen_pfn_t) ((guest_width==8)?
+                       (((uint64_t *)p2m)[(pfn)]):
+                       ((((uint32_t *)p2m)[(pfn)]) == 0xffffffffU ?
+                            (-1UL) :
+                            (((uint32_t *)p2m)[(pfn)]))));
+}
+
+static int swap_page(int xc_handle, xen_pfn_t mfn, xen_pfn_t new_mfn,
+                    int domid, xen_pfn_t *p2m, unsigned long *pfn_type,
+                    int p2m_size, int pt_levels, int guest_width)
+{
+    int nr_swaped = 0, pte_num;
+    uint64_t i;
+    struct mmuext_op swap[MAX_OFFLINE_BATCH];
+    void *content = NULL;
+
+    pte_num = PAGE_SIZE / ((pt_levels == 2) ? 4 : 8);
+
+    for (i = 0; i < p2m_size; i++)
+    {
+        xen_pfn_t table_mfn = pfn_to_mfn(i, p2m, guest_width);
+        uint64_t pte;
+        int j;
+
+        if ( table_mfn == INVALID_P2M_ENTRY )
+            continue;
+
+        if ( pfn_type[i] & XEN_DOMCTL_PFINFO_LTABTYPE_MASK )
+        {
+            content = xc_map_foreign_range(xc_handle, domid, PAGE_SIZE,
+                                            PROT_READ, table_mfn);
+            if (!content)
+                goto out;
+
+            for (j = 0; j < pte_num; j++)
+            {
+                if ( pt_levels == 2 )
+                    pte = ((const uint32_t*)content)[j];
+                else
+                    pte = ((const uint64_t*)content)[j];
+
+                if (!(pte & _PAGE_PRESENT))
+                    continue;
+
+                /* Hit one entry */
+                if ( ((pte >> PAGE_SHIFT) & MFN_MASK_X86) == mfn )
+                {
+                    swap[nr_swaped].cmd = MMUEXT_SWAP_PAGE;
+                    swap[nr_swaped].arg1.mfn = table_mfn << PAGE_SHIFT;
+                    swap[nr_swaped].arg1.mfn |= j * ( (pt_levels == 2) ?
+                                        sizeof(uint32_t): sizeof(uint64_t) );
+                    swap[nr_swaped].arg2.new_mfn = new_mfn;
+                    nr_swaped ++;
+                }
+
+                if ( nr_swaped == MAX_OFFLINE_BATCH )
+                {
+                    if ( xc_mmuext_op(xc_handle, swap, nr_swaped, domid) < 0 )
+                    {
+                        ERROR("Failed to swap");
+                        goto out;
+                    }
+                    nr_swaped = 0;
+                }
+            }
+
+            if ( nr_swaped )
+            {
+                if ( xc_mmuext_op(xc_handle, swap, nr_swaped, domid) < 0 )
+                {
+                    ERROR("Failed to swap");
+                    goto out;
+                }
+                nr_swaped = 0;
+            }
+
+            munmap(content, PAGE_SIZE);
+            content = NULL;
+        }
+    }
+    return 0;
+out:
+    /* XXX Shall we take action if we have fail to swap? */
+    if (content)
+        munmap(content, PAGE_SIZE);
+
+    return -1;
+}
+
+int xc_replace_page(int xc_handle, xen_pfn_t pfn, int domid,
+                    xen_pfn_t *p2m, unsigned long *pfn_type,
+                    int p2m_size, int pt_levels, int guest_width)
+{
+    uint32_t status;
+    xc_dominfo_t info;
+    xen_pfn_t mfn, new_mfn;
+    struct mmuext_op unpin;
+    int rc, max_pages, broken = 0;
+    void *old_p, *tmp_p = NULL, *new_p = NULL;
+
+    if (!p2m || !pfn_type)
+        return -EINVAL;
+
+    mfn = pfn_to_mfn(pfn, p2m, guest_width);
+
+    /* This page has no mfn established?? */
+    if (mfn == INVALID_P2M_ENTRY)
+        return -EINVAL;
+
+    /* Target domain should be suspended already */
+    if ( (xc_domain_getinfo(xc_handle, domid, 1, &info) != 1) ||
+         !info.shutdown || (info.shutdown_reason != SHUTDOWN_suspend) )
+    {
+        ERROR("Domain not in suspended state");
+        return -1;
+    }
+
+    /* Check if pages are offline pending or not */
+    rc = xc_query_page_offline_status(xc_handle, mfn, mfn, &status);
+
+    if (rc)
+        return rc;
+
+    if ( !(status & PG_OFFLINE_STATUS_OFFLINE_PENDING) )
+    {
+        ERROR("Page %lx(mfn %lx) is not offline pending %x\n",
+               pfn, mfn, status);
+        return -EINVAL;
+    }
+
+    /* Unpin the page if it is pined */
+    if (pfn_type[pfn] & XEN_DOMCTL_PFINFO_LPINTAB)
+    {
+        unpin.cmd = MMUEXT_UNPIN_TABLE;
+        unpin.arg1.mfn = mfn;
+
+        if ( xc_mmuext_op(xc_handle, &unpin, 1, domid) < 0 )
+        {
+            ERROR("Failed to unpin page %lx", pfn);
+            return -EINVAL;
+        }
+    }
+
+    /* We increase the page limitation temp */
+    max_pages = xc_memory_op(xc_handle, XENMEM_maximum_reservation , &domid);
+    if (max_pages < 0)
+    {
+        ERROR("Failed to get max mfn\n");
+        goto undo_unpin;
+    }
+
+    max_pages ++;
+    xc_domain_setmaxmem(xc_handle, domid, max_pages << 2);
+
+    if (lock_pages(&new_mfn, sizeof(xen_pfn_t)))
+    {
+        ERROR("Could not lock new_mfn\n");
+        goto undo_maxmem;
+    }
+    rc = xc_domain_memory_increase_reservation(xc_handle, domid, 1, 0,
+                                     0x0, &new_mfn);
+
+    if (rc < 0)
+    {
+        ERROR("Failed to increase reservation \n");
+        goto undo_maxmem;
+    }
+
+    unlock_pages(&new_mfn, sizeof(xen_pfn_t));
+
+    /* Copy content from old page to new one */
+    old_p = xc_map_foreign_range(xc_handle, domid, PAGE_SIZE,
+                                PROT_READ, mfn);
+    tmp_p = malloc(PAGE_SIZE);
+
+    new_p = xc_map_foreign_range(xc_handle, domid, PAGE_SIZE,
+                                PROT_READ, new_mfn);
+
+    if (!old_p || !tmp_p || !new_p)
+        goto undo_increase;
+
+    memcpy(tmp_p, old_p, PAGE_SIZE);
+    munmap(old_p, PAGE_SIZE);
+
+    rc = swap_page(xc_handle, mfn, new_mfn, domid,
+                    p2m, pfn_type, p2m_size, pt_levels, guest_width);
+
+    if (rc)
+    {
+        ERROR("swap page failed\n");
+        /* No recover action now for swap fail */
+        broken = 1;
+        goto unmap_tmp_p;
+    }
+
+    /* Check if pages are offlined already */
+    rc = xc_query_page_offline_status(xc_handle,
+                            pfn_to_mfn(pfn, p2m, guest_width),
+                            pfn_to_mfn(pfn, p2m, guest_width),
+                            &status);
+
+    if (rc)
+    {
+        ERROR("Fail to query offline status\n");
+        goto unmap_tmp_p;
+    }
+
+    if ( !(status & PG_OFFLINE_STATUS_OFFLINED) )
+    {
+        ERROR("page is still not offlined, is it in granted to others? \n");
+        goto offline_error;
+    }
+    else
+    {
+        DEBUG("Now page is offlined %lx\n", pfn);
+        /* Update the p2m table */
+        p2m[pfn] = new_mfn;
+        memcpy(new_p, tmp_p, PAGE_SIZE);
+        munmap(new_p, PAGE_SIZE);
+        xc_domain_setmaxmem(xc_handle, domid, (max_pages - 1 ) << 2);
+    }
+
+    return 0;
+
+offline_error:
+    if (swap_page(xc_handle, new_mfn, mfn, domid,
+                  p2m, pfn_type, p2m_size, pt_levels, guest_width) < 0)
+        goto broken;
+
+unmap_tmp_p:
+    free(tmp_p);
+    munmap(new_p, PAGE_SIZE);
+
+
+undo_increase:
+    if (xc_domain_memory_decrease_reservation(xc_handle, domid, 1, 0,
+                                              &new_mfn))
+        goto broken;
+
+undo_maxmem:
+    xc_domain_setmaxmem(xc_handle, domid, (max_pages - 1 ) << 2);
+
+undo_unpin:
+    if (pfn_type[pfn] & XEN_DOMCTL_PFINFO_LPINTAB)
+    {
+        switch ( pfn_type[pfn] & XEN_DOMCTL_PFINFO_LTABTYPE_MASK )
+        {
+            case XEN_DOMCTL_PFINFO_L1TAB:
+                unpin.cmd = MMUEXT_PIN_L1_TABLE;
+                break;
+
+            case XEN_DOMCTL_PFINFO_L2TAB:
+                unpin.cmd = MMUEXT_PIN_L2_TABLE;
+                break;
+
+            case XEN_DOMCTL_PFINFO_L3TAB:
+                unpin.cmd = MMUEXT_PIN_L3_TABLE;
+                break;
+
+            case XEN_DOMCTL_PFINFO_L4TAB:
+                unpin.cmd = MMUEXT_PIN_L4_TABLE;
+                break;
+
+            default:
+                ERROR("Unpined for non pate table page\n");
+                break;
+        }
+        unpin.arg1.mfn = mfn;
+
+        if ( xc_mmuext_op(xc_handle, &unpin, 1, domid) < 0 )
+        {
+            ERROR("failed to pin the mfn again\n");
+            goto broken;
+        }
+    }
+    if (!broken)
+        return -1;
+broken:
+    return -2;
+}
diff -r fa6b2732778f tools/libxc/xenguest.h
--- a/tools/libxc/xenguest.h	Wed Mar 18 00:19:25 2009 +0800
+++ b/tools/libxc/xenguest.h	Wed Mar 18 00:19:28 2009 +0800
@@ -29,6 +29,31 @@ int xc_domain_save(int xc_handle, int io
                    void *(*init_qemu_maps)(int, unsigned),  /* HVM only */
                    void (*qemu_flip_buffer)(int, int));     /* HVM only */
 
+int xc_replace_page(int xc, xen_pfn_t pfn, int domid,
+                    xen_pfn_t *p2m, unsigned long *pfn_type,
+                    int p2m_size, int pt_levels, int guest_width);
+
+int xc_mark_page_online(int xc, unsigned long start,
+                        unsigned long end, uint32_t *status);
+
+int xc_mark_page_offline(int xc, unsigned long start,
+                          unsigned long end, uint32_t *status);
+
+int xc_query_page_offline_status(int xc, unsigned long start,
+                                 unsigned long end, uint32_t *status);
+
+/**
+ * This function map msp table
+ * @parm xc_handle a handle to an open hypervisor interface
+ * @parm max_mfn the max pfn
+ * @parm prot the flags to map, such as read/write etc
+ * @parm mfn0 return the first mfn, can be NULL
+ * @return mapped m2p table on success, NULL on failure
+ */
+xen_pfn_t *xc_map_m2p(int xc_handle,
+                      unsigned long max_mfn,
+                      int prot,
+                      unsigned long *mfn0);
 
 /**
  * This function will restore a saved domain.
diff -r fa6b2732778f tools/xcutils/Makefile
--- a/tools/xcutils/Makefile	Wed Mar 18 00:19:25 2009 +0800
+++ b/tools/xcutils/Makefile	Wed Mar 18 00:19:28 2009 +0800
@@ -14,7 +14,7 @@ CFLAGS += -Werror
 CFLAGS += -Werror
 CFLAGS += $(CFLAGS_libxenctrl) $(CFLAGS_libxenguest) $(CFLAGS_libxenstore)
 
-PROGRAMS = xc_restore xc_save readnotes lsevtchn
+PROGRAMS = xc_restore xc_save readnotes lsevtchn xc_offline
 
 LDLIBS   = $(LDFLAGS_libxenctrl) $(LDFLAGS_libxenguest) $(LDFLAGS_libxenstore)
 
diff -r fa6b2732778f tools/xcutils/xc_offline.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tools/xcutils/xc_offline.c	Wed Mar 18 01:42:44 2009 +0800
@@ -0,0 +1,327 @@
+/*
+ * This file is subject to the terms and conditions of the GNU General
+ * Public License.  See the file "COPYING" in the main directory of
+ * this archive for more details.
+ *
+ * Copyright (C) 2005 by Christian Limpach
+ *
+ */
+
+#include <err.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/ipc.h>
+#include <sys/shm.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <err.h>
+
+#include <xs.h>
+#include <xenctrl.h>
+#include <xenguest.h>
+#include <xc_private.h>
+#include <xc_core.h>
+
+#undef ERROR
+#undef DEBUG
+#define ERROR(fmr, args...) do { fprintf(stderr, "ERROR: " fmr , ##args); } while (0)
+#define DEBUG(fmr, args...) do { fprintf(stderr, "DEBUG: " fmr , ##args); } while (0)
+
+#define FPP             (PAGE_SIZE/(guest_width))
+
+/* Number of entries in the pfn_to_mfn_frame_list_list */
+#define P2M_FLL_ENTRIES (((p2m_size)+(FPP*FPP)-1)/(FPP*FPP))
+
+static xen_pfn_t pfn_to_mfn(xen_pfn_t pfn, xen_pfn_t *p2m, int guest_width)
+{
+  return ((xen_pfn_t) ((guest_width==8)?
+                       (((uint64_t *)p2m)[(pfn)]):
+                       ((((uint32_t *)p2m)[(pfn)]) == 0xffffffffU ?
+                            (-1UL) :
+                            (((uint32_t *)p2m)[(pfn)]))));
+}
+
+static int get_pt_level(int xc_handle, uint32_t domid,
+                        unsigned int *pt_level,
+                        unsigned int *guest_width)
+{
+    DECLARE_DOMCTL;
+    xen_capabilities_info_t xen_caps = "";
+
+    if (xc_version(xc_handle, XENVER_capabilities, &xen_caps) != 0)
+        return -1;
+
+    memset(&domctl, 0, sizeof(domctl));
+    domctl.domain = domid;
+    domctl.cmd = XEN_DOMCTL_get_address_size;
+
+    if ( do_domctl(xc_handle, &domctl) != 0 )
+        return -1;
+
+    *guest_width = domctl.u.address_size.size / 8;
+
+    if (strstr(xen_caps, "xen-3.0-x86_64"))
+        /* Depends on whether it's a compat 32-on-64 guest */
+        *pt_level = ( (*guest_width == 8) ? 4 : 3 );
+    else if (strstr(xen_caps, "xen-3.0-x86_32p"))
+        *pt_level = 3;
+    else if (strstr(xen_caps, "xen-3.0-x86_32"))
+        *pt_level = 2;
+    else
+        return -1;
+
+    return 0;
+}
+
+#define PG_OFFLINE_STATUS_HANDLED (1UL << 14)
+int
+main(int argc, char **argv)
+{
+    unsigned long start, end;
+    int xc_handle = 0, i, num, rc;
+    uint32_t *status = NULL;
+
+    if (argc != 3)
+        fprintf(stderr, "usage: %s start end", argv[0]);
+
+    start = strtoul(argv[1], NULL, 0);
+    end = strtoul(argv[2], NULL, 0);
+
+    xc_handle = xc_interface_open();
+
+    if (!xc_handle)
+        return -1;
+
+    num = end - start + 1;
+
+    rc = -ENOMEM;
+    status  = malloc(num * sizeof(uint32_t));
+    if (!status)
+        return -EINVAL;
+    memset(status, 0, sizeof(uint32_t)*num);
+
+    rc = xc_mark_page_offline(xc_handle, start, end, status);
+
+    if (rc)
+    {
+        ERROR("fail to mark pages offline %x\n", rc);
+        return -EINVAL;
+    }
+
+    for (i = 0; i < num; i++)
+    {
+        DEBUG("pfn %lx status %x\n", start + i, status[i]);
+
+        if (status[i] & PG_OFFLINE_STATUS_HANDLED)
+            continue;
+
+        switch (status[i] & PG_OFFLINE_STATUS_MASK)
+        {
+            case PG_OFFLINE_OFFLINED:
+                DEBUG("offlined page %lx\n", start + i);
+                status[i] |= PG_OFFLINE_STATUS_HANDLED;
+                break;
+            break;
+
+            case PG_OFFLINE_PENDING:
+            {
+                uint32_t domid, j;
+                int port, xce = -1, rc;
+                unsigned long p2m_size;
+                xen_pfn_t *p2m_table = NULL;
+                xen_pfn_t *m2p_table = NULL;
+                xc_dominfo_t info;
+                uint64_aligned_t shared_info_frame;
+                shared_info_any_t *live_shinfo = NULL;
+                uint32_t *pfn_type = NULL;
+                unsigned long *pfn_real = NULL, max_mfn;
+                int suspend_evtchn = -1, suspended = 0;
+                uint32_t pt_level = 0, guest_width = 0;
+
+                domid = status[i] >> PG_OFFLINE_OWNER_SHIFT;
+
+                if ( !domid || (domid > DOMID_FIRST_RESERVED) )
+                {
+                    DEBUG("Dom0's page can't be LM");
+                    goto failed;
+                }
+
+                if ( get_pt_level(xc_handle, domid, &pt_level, &guest_width) )
+                {
+                    ERROR("Unable to get PT level info.");
+                    goto failed;
+                }
+
+                if ( xc_domain_getinfo(xc_handle, domid, 1, &info) != 1 )
+                {
+                    ERROR("Could not get domain info");
+                    goto failed;
+                }
+
+                if (info.hvm)
+                {
+                    DEBUG("please Live migrate dom %x\n", domid);
+                    goto failed;
+                }
+
+                /* Map the p2m table and M2P table */
+                shared_info_frame = info.shared_info_frame;
+
+                live_shinfo = xc_map_foreign_range(xc_handle, domid,
+                  PAGE_SIZE, PROT_READ, shared_info_frame);
+                if ( !live_shinfo )
+                {
+                    ERROR("Couldn't map live_shinfo");
+                    goto failed;
+                }
+
+                if ( (rc = xc_core_arch_map_p2m(xc_handle, guest_width, &info,
+                                          live_shinfo, &p2m_table,  &p2m_size)) )
+                {
+                    ERROR("Couldn't map p2m table %x\n", rc);
+                    goto failed;
+                }
+
+                max_mfn = xc_memory_op(xc_handle, XENMEM_maximum_ram_page, NULL);
+                if ( !(m2p_table = xc_map_m2p(xc_handle, max_mfn, PROT_READ, NULL)) )
+                {
+                    ERROR("Failed to map live M2P table");
+                    goto failed;
+                }
+
+                /* Suspend the guest */
+                port = xs_suspend_evtchn_port(domid);
+                if (port < 0)
+                {
+                    ERROR("Dom %x: No suspsend port, try live migration\n",
+                            domid);
+                    goto failed;
+                }
+
+                xce = xc_evtchn_open();
+                if (xce < 0)
+                {
+                    ERROR("Dom %x: fail to open evtchn\n",
+                            domid);
+                            goto failed;
+                }
+
+                suspend_evtchn =
+                  xc_suspend_evtchn_init(xc_handle, xce, domid, port);
+                if (suspend_evtchn < 0)
+                {
+                    ERROR("suspend event channel initialization failed\n");
+                    goto failed;
+                }
+
+                rc = xc_evtchn_notify(xce, suspend_evtchn);
+                if (rc < 0) {
+                    ERROR("failed to notify suspend channel: %d", rc);
+                    goto failed;
+                }
+                if (xc_await_suspend(xce, suspend_evtchn) < 0) {
+                    ERROR("suspend failed");\
+                    goto failed;
+                }
+                suspended = 1;
+
+                /* Get pfn type */
+                pfn_type = malloc(sizeof(uint32_t) * p2m_size);
+                if (!pfn_type)
+                {
+                    ERROR("Failed to malloc pfn_type\n");
+                    goto failed;
+                }
+                memset(pfn_type, 0, sizeof(uint32_t) * p2m_size);
+
+                pfn_real = malloc(sizeof(unsigned long) * p2m_size);
+                if (!pfn_real)
+                {
+                    ERROR("Failed to malloc pfn_real\n");
+                    goto failed;
+                }
+                memset(pfn_real, 0, sizeof(unsigned long) * p2m_size);
+
+                for (j = 0; j < p2m_size; j++)
+                    pfn_type[j] = pfn_to_mfn(j, p2m_table, guest_width);
+
+                if ( lock_pages(pfn_type, p2m_size * sizeof(*pfn_type)) )
+                {
+                    ERROR("Unable to lock pfn_type array");
+                    goto failed;
+                }
+                if ( lock_pages(pfn_real, p2m_size * sizeof(*pfn_real)) )
+                {
+                    ERROR("Unable to lock pfn_real array");
+                    goto failed;
+                }
+
+                for (j = 0; j < p2m_size ; j+=1024)
+                {
+                    int count = ((p2m_size - j ) > 1024 ) ? 1024: (p2m_size - j);
+                    if ( ( rc = xc_get_pfn_type_batch(xc_handle, domid, count,
+                              pfn_type + j)) )
+                    {
+                        ERROR("Failed to get pfn_type %x\n", rc);
+                         goto failed;
+                    }
+                }
+
+                /* Now replace the page */
+                for (j = 0; j < p2m_size; j++)
+                    pfn_real[j] = pfn_type[j];
+
+                for (j = i ; j < num; j++)
+                {
+                    if ( ((status[j]& PG_OFFLINE_STATUS_MASK) == PG_OFFLINE_PENDING) &&
+                         ((status[j] >> PG_OFFLINE_OWNER_SHIFT) == domid) )
+                    {
+#define mfn_to_pfn(_mfn)  (m2p_table[(_mfn)])
+                             rc = xc_replace_page(xc_handle, mfn_to_pfn(start + j),
+                               domid, p2m_table,
+                               pfn_real, p2m_size, pt_level,
+                               guest_width);
+                             if (rc)
+                             {
+                                 ERROR("Failed to replace page %x\n", j);
+                                 goto failed;
+                             }
+
+                             status[j] |= PG_OFFLINE_STATUS_HANDLED;
+                    }
+                }
+
+failed:
+                status[i] |= PG_OFFLINE_STATUS_HANDLED;
+
+                if (p2m_table)
+                    munmap(p2m_table, P2M_FLL_ENTRIES * PAGE_SIZE);
+                if (pfn_type)
+                    free(pfn_type);
+                if (live_shinfo)
+                    munmap(live_shinfo, PAGE_SIZE);
+                if (suspend_evtchn > 0)
+                    xc_suspend_evtchn_release(xc_handle, suspend_evtchn);
+                if (suspended)
+                    xc_domain_resume(xc_handle, domid, 1);
+                if (xce > 0)
+                    xc_evtchn_close(xce);
+                break;
+            }
+            default:
+            {
+                ERROR("Error status result %x\n", status[i]);
+                break;
+            }
+        }
+
+    }
+    if (xc_handle)
+        xc_interface_close(xc_handle);
+    if (status)
+        free(status);
+    return 0;
+}
