[PATCH v2 0/4] support linear p2m list in migrate stream v2

[PATCH v2 0/4] support linear p2m list in migrate stream v2

[Juergen Gross]

Add support for the virtual mapped linear p2m list of pv-domains in the
v2 migrate stream. This will allow to migrate domains larger than 512
GB.

Tested with 32- and 64-bit pv-domains both with and without linear p2m
list and with a hvm domain.

Changes in V2:
- Added some sanity tests in patch 2 as suggested by Andrew Cooper
- Modified patch 3 according to Andrew Cooper's requests: rename of
  check_iteration to check_vm_state, call check_vm_state after each
  checkpoint, don't change check_vm_state hook but do the check decision
  internally
- Modified docs/features/migration.pandoc according to changes done in the
  series in patch 4 (requested by Andrew Cooper)

Juergen Gross (4):
  libxc: split mapping p2m leaves into a separate function
  libxc: support of linear p2m list for migration of pv-domains
  libxc: stop migration in case of p2m list structural changes
  libxc: set flag for support of linear p2m list in domain builder

 docs/features/migration.pandoc    |   7 +-
 tools/libxc/xc_dom_compat_linux.c |   2 +-
 tools/libxc/xc_dom_core.c         |   2 +
 tools/libxc/xc_sr_common.h        |  12 ++
 tools/libxc/xc_sr_save.c          |   7 +-
 tools/libxc/xc_sr_save_x86_hvm.c  |   7 +
 tools/libxc/xc_sr_save_x86_pv.c   | 303 +++++++++++++++++++++++++++++++++-----
 7 files changed, 300 insertions(+), 40 deletions(-)

-- 
2.6.2

[PATCH v2 1/4] libxc: split mapping p2m leaves into a separate function

[Juergen Gross]

In order to prepare using the virtual mapped linear p2m list for
migration split mapping of the p2m leaf pages into a separate function.

Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>
---
 tools/libxc/xc_sr_save_x86_pv.c | 77 ++++++++++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 32 deletions(-)

diff --git a/tools/libxc/xc_sr_save_x86_pv.c b/tools/libxc/xc_sr_save_x86_pv.c
index c8d6f0b..d7acd37 100644
--- a/tools/libxc/xc_sr_save_x86_pv.c
+++ b/tools/libxc/xc_sr_save_x86_pv.c
@@ -68,6 +68,50 @@ static int copy_mfns_from_guest(const struct xc_sr_context *ctx,
 }
 
 /*
+ * Map the p2m leave pages and build an array of their pfns.
+ */
+static int map_p2m_leaves(struct xc_sr_context *ctx, xen_pfn_t *mfns,
+                          size_t n_mfns)
+{
+    xc_interface *xch = ctx->xch;
+    unsigned x;
+
+    ctx->x86_pv.p2m = xc_map_foreign_pages(xch, ctx->domid, PROT_READ,
+                                           mfns, n_mfns);
+    if ( !ctx->x86_pv.p2m )
+    {
+        PERROR("Failed to map p2m frames");
+        return -1;
+    }
+
+    ctx->save.p2m_size = ctx->x86_pv.max_pfn + 1;
+    ctx->x86_pv.p2m_frames = n_mfns;
+    ctx->x86_pv.p2m_pfns = malloc(n_mfns * sizeof(*mfns));
+    if ( !ctx->x86_pv.p2m_pfns )
+    {
+        ERROR("Cannot allocate %zu bytes for p2m pfns list",
+              n_mfns * sizeof(*mfns));
+        return -1;
+    }
+
+    /* Convert leaf frames from mfns to pfns. */
+    for ( x = 0; x < n_mfns; ++x )
+    {
+        if ( !mfn_in_pseudophysmap(ctx, mfns[x]) )
+        {
+            ERROR("Bad mfn in p2m_frame_list[%u]", x);
+            dump_bad_pseudophysmap_entry(ctx, mfns[x]);
+            errno = ERANGE;
+            return -1;
+        }
+
+        ctx->x86_pv.p2m_pfns[x] = mfn_to_pfn(ctx, mfns[x]);
+    }
+
+    return 0;
+}
+
+/*
  * Walk the guests frame list list and frame list to identify and map the
  * frames making up the guests p2m table.  Construct a list of pfns making up
  * the table.
@@ -173,7 +217,6 @@ static int map_p2m(struct xc_sr_context *ctx)
     ctx->x86_pv.p2m_frames = (ctx->x86_pv.max_pfn + fpp) / fpp;
     DPRINTF("max_pfn %#lx, p2m_frames %d", ctx->x86_pv.max_pfn,
             ctx->x86_pv.p2m_frames);
-    ctx->save.p2m_size = ctx->x86_pv.max_pfn + 1;
     fl_entries  = (ctx->x86_pv.max_pfn / fpp) + 1;
 
     /* Map the guest mid p2m frames. */
@@ -211,38 +254,8 @@ static int map_p2m(struct xc_sr_context *ctx)
     }
 
     /* Map the p2m leaves themselves. */
-    ctx->x86_pv.p2m = xc_map_foreign_pages(xch, ctx->domid, PROT_READ,
-                                           local_fl, fl_entries);
-    if ( !ctx->x86_pv.p2m )
-    {
-        PERROR("Failed to map p2m frames");
-        goto err;
-    }
+    rc = map_p2m_leaves(ctx, local_fl, fl_entries);
 
-    ctx->x86_pv.p2m_frames = fl_entries;
-    ctx->x86_pv.p2m_pfns = malloc(local_fl_size);
-    if ( !ctx->x86_pv.p2m_pfns )
-    {
-        ERROR("Cannot allocate %zu bytes for p2m pfns list",
-              local_fl_size);
-        goto err;
-    }
-
-    /* Convert leaf frames from mfns to pfns. */
-    for ( x = 0; x < fl_entries; ++x )
-    {
-        if ( !mfn_in_pseudophysmap(ctx, local_fl[x]) )
-        {
-            ERROR("Bad mfn in p2m_frame_list[%u]", x);
-            dump_bad_pseudophysmap_entry(ctx, local_fl[x]);
-            errno = ERANGE;
-            goto err;
-        }
-
-        ctx->x86_pv.p2m_pfns[x] = mfn_to_pfn(ctx, local_fl[x]);
-    }
-
-    rc = 0;
 err:
 
     free(local_fl);
-- 
2.6.2

[PATCH v2 2/4] libxc: support of linear p2m list for migration of pv-domains

[Juergen Gross]

In order to be able to migrate pv-domains with more than 512 GB of RAM
the p2m information can be specified by the guest kernel via a virtual
mapped linear p2m list instead of a 3 level tree.

Add support for this new p2m format in libxc.

As the sanity checking of the virtual p2m address needs defines for the
xen regions use those defines when doing page table checks as well.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 tools/libxc/xc_sr_save_x86_pv.c | 193 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 184 insertions(+), 9 deletions(-)

diff --git a/tools/libxc/xc_sr_save_x86_pv.c b/tools/libxc/xc_sr_save_x86_pv.c
index d7acd37..98e9011 100644
--- a/tools/libxc/xc_sr_save_x86_pv.c
+++ b/tools/libxc/xc_sr_save_x86_pv.c
@@ -3,6 +3,18 @@
 
 #include "xc_sr_common_x86_pv.h"
 
+/* Check a 64 bit virtual address for being canonical. */
+static inline bool is_canonical_address(xen_vaddr_t vaddr)
+{
+    return ((int64_t)vaddr >> 47) == ((int64_t)vaddr >> 63);
+}
+
+#define HYPERVISOR_VIRT_START_X86_64 0xFFFF800000000000ULL
+#define HYPERVISOR_VIRT_END_X86_64   0xFFFF87FFFFFFFFFFULL
+
+#define HYPERVISOR_VIRT_START_X86_32 0x00000000F5800000ULL
+#define HYPERVISOR_VIRT_END_X86_32   0x00000000FFFFFFFFULL
+
 /*
  * Maps the guests shared info page.
  */
@@ -116,7 +128,7 @@ static int map_p2m_leaves(struct xc_sr_context *ctx, xen_pfn_t *mfns,
  * frames making up the guests p2m table.  Construct a list of pfns making up
  * the table.
  */
-static int map_p2m(struct xc_sr_context *ctx)
+static int map_p2m_tree(struct xc_sr_context *ctx)
 {
     /* Terminology:
      *
@@ -138,8 +150,6 @@ static int map_p2m(struct xc_sr_context *ctx)
     void *guest_fl = NULL;
     size_t local_fl_size;
 
-    ctx->x86_pv.max_pfn = GET_FIELD(ctx->x86_pv.shinfo, arch.max_pfn,
-                                    ctx->x86_pv.width) - 1;
     fpp = PAGE_SIZE / ctx->x86_pv.width;
     fll_entries = (ctx->x86_pv.max_pfn / (fpp * fpp)) + 1;
     if ( fll_entries > fpp )
@@ -270,6 +280,170 @@ err:
 }
 
 /*
+ * Map the guest p2m frames specified via a cr3 value, a virtual address, and
+ * the maximum pfn. PTE entries are 64 bits vor both, 32 and 64 bit guests as
+ * in 32 bit case we support PAE guests only.
+ */
+static int map_p2m_list(struct xc_sr_context *ctx, uint64_t p2m_cr3)
+{
+    xc_interface *xch = ctx->xch;
+    xen_vaddr_t p2m_vaddr, p2m_end, mask, off;
+    xen_pfn_t p2m_mfn, mfn, saved_mfn, max_pfn;
+    uint64_t *ptes;
+    xen_pfn_t *mfns;
+    unsigned fpp, n_pages, level, shift, idx_start, idx_end, idx, saved_idx;
+    int rc = -1;
+
+    p2m_mfn = cr3_to_mfn(ctx, p2m_cr3);
+    assert(p2m_mfn != 0);
+    if ( p2m_mfn > ctx->x86_pv.max_mfn )
+    {
+        ERROR("Bad p2m_cr3 value %#lx", p2m_cr3);
+        errno = ERANGE;
+        return -1;
+    }
+
+    p2m_vaddr = GET_FIELD(ctx->x86_pv.shinfo, arch.p2m_vaddr,
+                          ctx->x86_pv.width);
+    fpp = PAGE_SIZE / ctx->x86_pv.width;
+    ctx->x86_pv.p2m_frames = ctx->x86_pv.max_pfn / fpp + 1;
+    p2m_end = p2m_vaddr + ctx->x86_pv.p2m_frames * PAGE_SIZE - 1;
+
+    if ( ctx->x86_pv.width == 8 )
+    {
+        mask = 0x0000ffffffffffffULL;
+        if ( !is_canonical_address(p2m_vaddr) ||
+             !is_canonical_address(p2m_end) ||
+             p2m_end < p2m_vaddr ||
+             (p2m_vaddr <= HYPERVISOR_VIRT_END_X86_64 &&
+              p2m_end > HYPERVISOR_VIRT_START_X86_64) )
+        {
+            ERROR("Bad virtual p2m address range %#lx-%#lx",
+                  p2m_vaddr, p2m_end);
+            errno = ERANGE;
+            return -1;
+        }
+    }
+    else
+    {
+        mask = 0x00000000ffffffffULL;
+        if ( p2m_vaddr > mask || p2m_end > mask || p2m_end < p2m_vaddr ||
+             (p2m_vaddr <= HYPERVISOR_VIRT_END_X86_32 &&
+              p2m_end > HYPERVISOR_VIRT_START_X86_32) )
+        {
+            ERROR("Bad virtual p2m address range %#lx-%#lx",
+                  p2m_vaddr, p2m_end);
+            errno = ERANGE;
+            return -1;
+        }
+    }
+
+    DPRINTF("p2m list from %#lx to %#lx, root at %#lx", p2m_vaddr, p2m_end,
+            p2m_mfn);
+    DPRINTF("max_pfn %#lx, p2m_frames %d", ctx->x86_pv.max_pfn,
+            ctx->x86_pv.p2m_frames);
+
+    mfns = malloc(sizeof(*mfns));
+    if ( !mfns )
+    {
+        ERROR("Cannot allocate memory for array of %u mfns", 1);
+        goto err;
+    }
+    mfns[0] = p2m_mfn;
+    off = 0;
+    saved_mfn = 0;
+    idx_start = idx_end = saved_idx = 0;
+
+    for ( level = ctx->x86_pv.levels; level > 0; level-- )
+    {
+        n_pages = idx_end - idx_start + 1;
+        ptes = xc_map_foreign_pages(xch, ctx->domid, PROT_READ, mfns, n_pages);
+        if ( !ptes )
+        {
+            PERROR("Failed to map %u page table pages for p2m list", n_pages);
+            goto err;
+        }
+        free(mfns);
+
+        shift = level * 9 + 3;
+        idx_start = ((p2m_vaddr - off) & mask) >> shift;
+        idx_end = ((p2m_end - off) & mask) >> shift;
+        idx = idx_end - idx_start + 1;
+        mfns = malloc(sizeof(*mfns) * idx);
+        if ( !mfns )
+        {
+            ERROR("Cannot allocate memory for array of %u mfns", idx);
+            goto err;
+        }
+
+        for ( idx = idx_start; idx <= idx_end; idx++ )
+        {
+            mfn = pte_to_frame(ptes[idx]);
+            if ( mfn == 0 || mfn > ctx->x86_pv.max_mfn )
+            {
+                ERROR("Bad mfn %#lx during page table walk for vaddr %#lx at level %d of p2m list",
+                      mfn, off + ((xen_vaddr_t)idx << shift), level);
+                errno = ERANGE;
+                goto err;
+            }
+            mfns[idx - idx_start] = mfn;
+
+            /* Maximum pfn check at level 2. Same reasoning as for p2m tree. */
+            if ( level == 2 )
+            {
+                if ( mfn != saved_mfn )
+                {
+                    saved_mfn = mfn;
+                    saved_idx = idx - idx_start;
+                }
+            }
+        }
+
+        if ( level == 2 )
+        {
+            max_pfn = ((xen_pfn_t)saved_idx << 9) * fpp - 1;
+            if ( max_pfn < ctx->x86_pv.max_pfn )
+            {
+                ctx->x86_pv.max_pfn = max_pfn;
+                ctx->x86_pv.p2m_frames = (ctx->x86_pv.max_pfn + fpp) / fpp;
+                p2m_end = p2m_vaddr + ctx->x86_pv.p2m_frames * PAGE_SIZE - 1;
+                idx_end = idx_start + saved_idx;
+            }
+        }
+
+        munmap(ptes, n_pages * PAGE_SIZE);
+        ptes = NULL;
+        off = p2m_vaddr & ((mask >> shift) << shift);
+    }
+
+    /* Map the p2m leaves themselves. */
+    rc = map_p2m_leaves(ctx, mfns, idx_end - idx_start + 1);
+
+err:
+    free(mfns);
+    if ( ptes )
+        munmap(ptes, n_pages * PAGE_SIZE);
+
+    return rc;
+}
+
+/*
+ * Map the guest p2m frames.
+ * Depending on guest support this might either be a virtual mapped linear
+ * list (preferred format) or a 3 level tree linked via mfns.
+ */
+static int map_p2m(struct xc_sr_context *ctx)
+{
+    uint64_t p2m_cr3;
+
+    ctx->x86_pv.max_pfn = GET_FIELD(ctx->x86_pv.shinfo, arch.max_pfn,
+                                    ctx->x86_pv.width) - 1;
+    p2m_cr3 = GET_FIELD(ctx->x86_pv.shinfo, arch.p2m_cr3, ctx->x86_pv.width);
+
+    return p2m_cr3 ? map_p2m_list(ctx, p2m_cr3) : map_p2m_tree(ctx);
+}
+
+/*
  * Obtain a specific vcpus basic state and write an X86_PV_VCPU_BASIC record
  * into the stream.  Performs mfn->pfn conversion on architectural state.
  */
@@ -681,8 +855,8 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
         /* 64bit guests only have Xen mappings in their L4 tables. */
         if ( type == XEN_DOMCTL_PFINFO_L4TAB )
         {
-            xen_first = 256;
-            xen_last = 271;
+            xen_first = (HYPERVISOR_VIRT_START_X86_64 >> 39) & 511;
+            xen_last = (HYPERVISOR_VIRT_END_X86_64 >> 39) & 511;
         }
     }
     else
@@ -698,7 +872,7 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
             /* 32bit guests can only use the first 4 entries of their L3 tables.
              * All other are potentially used by Xen. */
             xen_first = 4;
-            xen_last = 512;
+            xen_last = 511;
             break;
 
         case XEN_DOMCTL_PFINFO_L2TAB:
@@ -709,10 +883,11 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
              *
              * ...which is conveniently unavailable to us in a 64bit build.
              */
-            if ( pte_to_frame(src[428]) == ctx->x86_pv.compat_m2p_mfn0 )
+            i = (HYPERVISOR_VIRT_START_X86_32 >> 21) & 511;
+            if ( pte_to_frame(src[i]) == ctx->x86_pv.compat_m2p_mfn0 )
             {
-                xen_first = 428;
-                xen_last = 512;
+                xen_first = i;
+                xen_last = (HYPERVISOR_VIRT_END_X86_32 >> 21) & 511;
             }
             break;
         }
-- 
2.6.2

[PATCH v2 3/4] libxc: stop migration in case of p2m list structural changes

[Juergen Gross]

With support of the virtual mapped linear p2m list for migration it is
now possible to detect structural changes of the p2m list which before
would either lead to a crashing or otherwise wrong behaving domU.

A guest supporting the linear p2m list will increment the
p2m_generation counter located in the shared info page before and after
each modification of a mapping related to the p2m list. A change of
that counter can be detected by the tools and reacted upon.

As such a change should occur only very rarely once the domU is up the
most simple reaction is to cancel migration in such an event.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 tools/libxc/xc_sr_common.h       | 12 +++++++++++
 tools/libxc/xc_sr_save.c         |  7 ++++++-
 tools/libxc/xc_sr_save_x86_hvm.c |  7 +++++++
 tools/libxc/xc_sr_save_x86_pv.c  | 45 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/tools/libxc/xc_sr_common.h b/tools/libxc/xc_sr_common.h
index 9aecde2..60b43e8 100644
--- a/tools/libxc/xc_sr_common.h
+++ b/tools/libxc/xc_sr_common.h
@@ -83,6 +83,15 @@ struct xc_sr_save_ops
     int (*end_of_checkpoint)(struct xc_sr_context *ctx);
 
     /**
+     * Check state of guest to decide whether it makes sense to continue
+     * migration.  This is called in each iteration or checkpoint to check
+     * whether all criteria for the migration are still met.  If that's not
+     * the case either migration is cancelled via a bad rc or the situation
+     * is handled, e.g. by sending appropriate records.
+     */
+    int (*check_vm_state)(struct xc_sr_context *ctx);
+
+    /**
      * Clean up the local environment.  Will be called exactly once, either
      * after a successful save, or upon encountering an error.
      */
@@ -280,6 +289,9 @@ struct xc_sr_context
             /* Read-only mapping of guests shared info page */
             shared_info_any_t *shinfo;
 
+            /* p2m generation count for verifying validity of local p2m. */
+            uint64_t p2m_generation;
+
             union
             {
                 struct
diff --git a/tools/libxc/xc_sr_save.c b/tools/libxc/xc_sr_save.c
index cefcef5..88d85ef 100644
--- a/tools/libxc/xc_sr_save.c
+++ b/tools/libxc/xc_sr_save.c
@@ -394,7 +394,8 @@ static int send_dirty_pages(struct xc_sr_context *ctx,
         DPRINTF("Bitmap contained more entries than expected...");
 
     xc_report_progress_step(xch, entries, entries);
-    return 0;
+
+    return ctx->save.ops.check_vm_state(ctx);
 }
 
 /*
@@ -751,6 +752,10 @@ static int save(struct xc_sr_context *ctx, uint16_t guest_type)
         if ( rc )
             goto err;
 
+        rc = ctx->save.ops.check_vm_state(ctx);
+        if ( rc )
+            goto err;
+
         if ( ctx->save.live )
             rc = send_domain_memory_live(ctx);
         else if ( ctx->save.checkpointed )
diff --git a/tools/libxc/xc_sr_save_x86_hvm.c b/tools/libxc/xc_sr_save_x86_hvm.c
index f3d6cee..e347b3b 100644
--- a/tools/libxc/xc_sr_save_x86_hvm.c
+++ b/tools/libxc/xc_sr_save_x86_hvm.c
@@ -175,6 +175,12 @@ static int x86_hvm_start_of_checkpoint(struct xc_sr_context *ctx)
     return 0;
 }
 
+static int x86_hvm_check_vm_state(struct xc_sr_context *ctx)
+{
+    /* no-op */
+    return 0;
+}
+
 static int x86_hvm_end_of_checkpoint(struct xc_sr_context *ctx)
 {
     int rc;
@@ -221,6 +227,7 @@ struct xc_sr_save_ops save_ops_x86_hvm =
     .start_of_stream     = x86_hvm_start_of_stream,
     .start_of_checkpoint = x86_hvm_start_of_checkpoint,
     .end_of_checkpoint   = x86_hvm_end_of_checkpoint,
+    .check_vm_state      = x86_hvm_check_vm_state,
     .cleanup             = x86_hvm_cleanup,
 };
 
diff --git a/tools/libxc/xc_sr_save_x86_pv.c b/tools/libxc/xc_sr_save_x86_pv.c
index 98e9011..63caf2e 100644
--- a/tools/libxc/xc_sr_save_x86_pv.c
+++ b/tools/libxc/xc_sr_save_x86_pv.c
@@ -280,6 +280,39 @@ err:
 }
 
 /*
+ * Get p2m_generation count.
+ * Returns an error if the generation count has changed since the last call.
+ */
+static int get_p2m_generation(struct xc_sr_context *ctx)
+{
+    uint64_t p2m_generation;
+    int rc;
+
+    p2m_generation = GET_FIELD(ctx->x86_pv.shinfo, arch.p2m_generation,
+                               ctx->x86_pv.width);
+
+    rc = (p2m_generation == ctx->x86_pv.p2m_generation) ? 0 : -1;
+    ctx->x86_pv.p2m_generation = p2m_generation;
+
+    return rc;
+}
+
+static int x86_pv_check_vm_state_p2m_list(struct xc_sr_context *ctx)
+{
+    xc_interface *xch = ctx->xch;
+    int rc;
+
+    if ( !ctx->save.live )
+        return 0;
+
+    rc = get_p2m_generation(ctx);
+    if ( rc )
+        ERROR("p2m generation count changed. Migration aborted.");
+
+    return rc;
+}
+
+/*
  * Map the guest p2m frames specified via a cr3 value, a virtual address, and
  * the maximum pfn. PTE entries are 64 bits vor both, 32 and 64 bit guests as
  * in 32 bit case we support PAE guests only.
@@ -303,6 +336,8 @@ static int map_p2m_list(struct xc_sr_context *ctx, uint64_t p2m_cr3)
         return -1;
     }
 
+    get_p2m_generation(ctx);
+
     p2m_vaddr = GET_FIELD(ctx->x86_pv.shinfo, arch.p2m_vaddr,
                           ctx->x86_pv.width);
     fpp = PAGE_SIZE / ctx->x86_pv.width;
@@ -436,6 +471,7 @@ static int map_p2m(struct xc_sr_context *ctx)
 {
     uint64_t p2m_cr3;
 
+    ctx->x86_pv.p2m_generation = ~0ULL;
     ctx->x86_pv.max_pfn = GET_FIELD(ctx->x86_pv.shinfo, arch.max_pfn,
                                     ctx->x86_pv.width) - 1;
     p2m_cr3 = GET_FIELD(ctx->x86_pv.shinfo, arch.p2m_cr3, ctx->x86_pv.width);
@@ -1076,6 +1112,14 @@ static int x86_pv_end_of_checkpoint(struct xc_sr_context *ctx)
     return 0;
 }
 
+static int x86_pv_check_vm_state(struct xc_sr_context *ctx)
+{
+    if ( ctx->x86_pv.p2m_generation == ~0ULL )
+        return 0;
+
+    return x86_pv_check_vm_state_p2m_list(ctx);
+}
+
 /*
  * save_ops function.  Cleanup.
  */
@@ -1103,6 +1147,7 @@ struct xc_sr_save_ops save_ops_x86_pv =
     .start_of_stream     = x86_pv_start_of_stream,
     .start_of_checkpoint = x86_pv_start_of_checkpoint,
     .end_of_checkpoint   = x86_pv_end_of_checkpoint,
+    .check_vm_state      = x86_pv_check_vm_state,
     .cleanup             = x86_pv_cleanup,
 };
 
-- 
2.6.2

[PATCH v2 4/4] libxc: set flag for support of linear p2m list in domain builder

[Juergen Gross]

Set the SIF_VIRT_P2M_4TOOLS flag for pv-domUs in the domain builder
to indicate the Xen tools have full support for the virtual mapped
linear p2m list.

This will enable pv-domUs to drop support of the 3 level p2m tree
and use the linear list only. Without setting this flag some kernels
might limit themselves to 512 GB memory size in order not to break
migration.

Signed-off-by: Juergen Gross <jgross@suse.com>
---
 docs/features/migration.pandoc    | 7 ++++---
 tools/libxc/xc_dom_compat_linux.c | 2 +-
 tools/libxc/xc_dom_core.c         | 2 ++
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/docs/features/migration.pandoc b/docs/features/migration.pandoc
index 9852a19..151c50d 100644
--- a/docs/features/migration.pandoc
+++ b/docs/features/migration.pandoc
@@ -1,5 +1,5 @@
 % Migration
-% Revision 1
+% Revision 2
 
 \clearpage
 
@@ -95,7 +95,6 @@ scenarios, which will involve starting with VMs from Xen 4.5
 # Areas for improvement
 
 * Arm support
-* Linear P2M support for x86 PV
 * Live looping parameters
 
 # Known issues
@@ -105,7 +104,8 @@ scenarios, which will involve starting with VMs from Xen 4.5
 * x86 HVM with nested-virt (no relevant information included in the
   stream)
 * x86 PV ballooning (P2M marked dirty, target frame not marked)
-* x86 PV P2M structure changes (not noticed, stale mappings used)
+* x86 PV P2M structure changes (not noticed, stale mappings used) for
+  guests not using the linear p2m layout
 
 # References
 
@@ -120,4 +120,5 @@ for Migration v2
 Date       Revision Version  Notes
 ---------- -------- -------- -------------------------------------------
 2015-10-24 1        Xen 4.6  Document written
+2015-12-11 2        Xen 4.7  Support of linear p2m list
 ---------- -------- -------- -------------------------------------------
diff --git a/tools/libxc/xc_dom_compat_linux.c b/tools/libxc/xc_dom_compat_linux.c
index abbc09f..c922c61 100644
--- a/tools/libxc/xc_dom_compat_linux.c
+++ b/tools/libxc/xc_dom_compat_linux.c
@@ -59,7 +59,7 @@ int xc_linux_build(xc_interface *xch, uint32_t domid,
          ((rc = xc_dom_ramdisk_file(dom, initrd_name)) != 0) )
         goto out;
 
-    dom->flags = flags;
+    dom->flags |= flags;
     dom->console_evtchn = console_evtchn;
     dom->xenstore_evtchn = store_evtchn;
 
diff --git a/tools/libxc/xc_dom_core.c b/tools/libxc/xc_dom_core.c
index 2061ba6..55c779d 100644
--- a/tools/libxc/xc_dom_core.c
+++ b/tools/libxc/xc_dom_core.c
@@ -777,6 +777,8 @@ struct xc_dom_image *xc_dom_allocate(xc_interface *xch,
     dom->parms.elf_paddr_offset = UNSET_ADDR;
     dom->parms.p2m_base = UNSET_ADDR;
 
+    dom->flags = SIF_VIRT_P2M_4TOOLS;
+
     dom->alloc_malloc += sizeof(*dom);
     return dom;
 
-- 
2.6.2

Re: [PATCH v2 2/4] libxc: support of linear p2m list for migration of pv-domains

[Andrew Cooper]

On 15/12/15 06:33, Juergen Gross wrote:
> In order to be able to migrate pv-domains with more than 512 GB of RAM
> the p2m information can be specified by the guest kernel via a virtual
> mapped linear p2m list instead of a 3 level tree.
>
> Add support for this new p2m format in libxc.
>
> As the sanity checking of the virtual p2m address needs defines for the
> xen regions use those defines when doing page table checks as well.
>
> Signed-off-by: Juergen Gross <jgross@suse.com>

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>, with a few
comments/suggestions inline.

> ---
>  tools/libxc/xc_sr_save_x86_pv.c | 193 ++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 184 insertions(+), 9 deletions(-)
>
> diff --git a/tools/libxc/xc_sr_save_x86_pv.c b/tools/libxc/xc_sr_save_x86_pv.c
> index d7acd37..98e9011 100644
> --- a/tools/libxc/xc_sr_save_x86_pv.c
> +++ b/tools/libxc/xc_sr_save_x86_pv.c
> @@ -3,6 +3,18 @@
>  
>  #include "xc_sr_common_x86_pv.h"
>  
> +/* Check a 64 bit virtual address for being canonical. */
> +static inline bool is_canonical_address(xen_vaddr_t vaddr)
> +{
> +    return ((int64_t)vaddr >> 47) == ((int64_t)vaddr >> 63);
> +}
> +
> +#define HYPERVISOR_VIRT_START_X86_64 0xFFFF800000000000ULL
> +#define HYPERVISOR_VIRT_END_X86_64   0xFFFF87FFFFFFFFFFULL
> +
> +#define HYPERVISOR_VIRT_START_X86_32 0x00000000F5800000ULL
> +#define HYPERVISOR_VIRT_END_X86_32   0x00000000FFFFFFFFULL

These might be better in the x86_pv header file, as they could plausibly
be useful on the restore side as well.

Ideally the virtual ranges would come from the Xen public header files,
but that would involve a non-trivial modifications to be able to get the
all the information.

> +
>  /*
>   * Maps the guests shared info page.
>   */
> @@ -116,7 +128,7 @@ static int map_p2m_leaves(struct xc_sr_context *ctx, xen_pfn_t *mfns,
>   * frames making up the guests p2m table.  Construct a list of pfns making up
>   * the table.
>   */
> -static int map_p2m(struct xc_sr_context *ctx)
> +static int map_p2m_tree(struct xc_sr_context *ctx)
>  {
>      /* Terminology:
>       *
> @@ -681,8 +855,8 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
>          /* 64bit guests only have Xen mappings in their L4 tables. */
>          if ( type == XEN_DOMCTL_PFINFO_L4TAB )
>          {
> -            xen_first = 256;
> -            xen_last = 271;
> +            xen_first = (HYPERVISOR_VIRT_START_X86_64 >> 39) & 511;
> +            xen_last = (HYPERVISOR_VIRT_END_X86_64 >> 39) & 511;

Would you mind using appropriate PAGETABLE_SHIFT defines from xg_private.h ?

>          }
>      }
>      else
> @@ -698,7 +872,7 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
>              /* 32bit guests can only use the first 4 entries of their L3 tables.
>               * All other are potentially used by Xen. */
>              xen_first = 4;
> -            xen_last = 512;
> +            xen_last = 511;
>              break;
>  
>          case XEN_DOMCTL_PFINFO_L2TAB:
> @@ -709,10 +883,11 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
>               *
>               * ...which is conveniently unavailable to us in a 64bit build.
>               */

The comment partially in context is now stale.

~Andrew

> -            if ( pte_to_frame(src[428]) == ctx->x86_pv.compat_m2p_mfn0 )
> +            i = (HYPERVISOR_VIRT_START_X86_32 >> 21) & 511;
> +            if ( pte_to_frame(src[i]) == ctx->x86_pv.compat_m2p_mfn0 )
>              {
> -                xen_first = 428;
> -                xen_last = 512;
> +                xen_first = i;
> +                xen_last = (HYPERVISOR_VIRT_END_X86_32 >> 21) & 511;
>              }
>              break;
>          }

Re: [PATCH v2 3/4] libxc: stop migration in case of p2m list structural changes

[Andrew Cooper]

On 15/12/15 06:33, Juergen Gross wrote:
> With support of the virtual mapped linear p2m list for migration it is
> now possible to detect structural changes of the p2m list which before
> would either lead to a crashing or otherwise wrong behaving domU.
>
> A guest supporting the linear p2m list will increment the
> p2m_generation counter located in the shared info page before and after
> each modification of a mapping related to the p2m list. A change of
> that counter can be detected by the tools and reacted upon.
>
> As such a change should occur only very rarely once the domU is up the
> most simple reaction is to cancel migration in such an event.
>
> Signed-off-by: Juergen Gross <jgross@suse.com>

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

Re: [PATCH v2 4/4] libxc: set flag for support of linear p2m list in domain builder

[Andrew Cooper]

On 15/12/15 06:33, Juergen Gross wrote:
> Set the SIF_VIRT_P2M_4TOOLS flag for pv-domUs in the domain builder
> to indicate the Xen tools have full support for the virtual mapped
> linear p2m list.
>
> This will enable pv-domUs to drop support of the 3 level p2m tree
> and use the linear list only. Without setting this flag some kernels
> might limit themselves to 512 GB memory size in order not to break
> migration.
>
> Signed-off-by: Juergen Gross <jgross@suse.com>

Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>

Re: [PATCH v2 2/4] libxc: support of linear p2m list for migration of pv-domains

[Juergen Gross]

On 15/12/15 12:52, Andrew Cooper wrote:
> On 15/12/15 06:33, Juergen Gross wrote:
>> In order to be able to migrate pv-domains with more than 512 GB of RAM
>> the p2m information can be specified by the guest kernel via a virtual
>> mapped linear p2m list instead of a 3 level tree.
>>
>> Add support for this new p2m format in libxc.
>>
>> As the sanity checking of the virtual p2m address needs defines for the
>> xen regions use those defines when doing page table checks as well.
>>
>> Signed-off-by: Juergen Gross <jgross@suse.com>
> 
> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>, with a few
> comments/suggestions inline.
> 
>> ---
>>  tools/libxc/xc_sr_save_x86_pv.c | 193 ++++++++++++++++++++++++++++++++++++++--
>>  1 file changed, 184 insertions(+), 9 deletions(-)
>>
>> diff --git a/tools/libxc/xc_sr_save_x86_pv.c b/tools/libxc/xc_sr_save_x86_pv.c
>> index d7acd37..98e9011 100644
>> --- a/tools/libxc/xc_sr_save_x86_pv.c
>> +++ b/tools/libxc/xc_sr_save_x86_pv.c
>> @@ -3,6 +3,18 @@
>>  
>>  #include "xc_sr_common_x86_pv.h"
>>  
>> +/* Check a 64 bit virtual address for being canonical. */
>> +static inline bool is_canonical_address(xen_vaddr_t vaddr)
>> +{
>> +    return ((int64_t)vaddr >> 47) == ((int64_t)vaddr >> 63);
>> +}
>> +
>> +#define HYPERVISOR_VIRT_START_X86_64 0xFFFF800000000000ULL
>> +#define HYPERVISOR_VIRT_END_X86_64   0xFFFF87FFFFFFFFFFULL
>> +
>> +#define HYPERVISOR_VIRT_START_X86_32 0x00000000F5800000ULL
>> +#define HYPERVISOR_VIRT_END_X86_32   0x00000000FFFFFFFFULL
> 
> These might be better in the x86_pv header file, as they could plausibly
> be useful on the restore side as well.
> 
> Ideally the virtual ranges would come from the Xen public header files,
> but that would involve a non-trivial modifications to be able to get the
> all the information.

Hmm, I've thought of that, too.

What would be easy is to have above defines in e.g.
xen/include/public/arch-x86/xen.h and use the appropriate ones in
xen/include/public/arch-x86/xen-x86_[32|64].h to define
HYPERVISOR_VIRT_START et al.

What do you think?

> 
>> +
>>  /*
>>   * Maps the guests shared info page.
>>   */
>> @@ -116,7 +128,7 @@ static int map_p2m_leaves(struct xc_sr_context *ctx, xen_pfn_t *mfns,
>>   * frames making up the guests p2m table.  Construct a list of pfns making up
>>   * the table.
>>   */
>> -static int map_p2m(struct xc_sr_context *ctx)
>> +static int map_p2m_tree(struct xc_sr_context *ctx)
>>  {
>>      /* Terminology:
>>       *
>> @@ -681,8 +855,8 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
>>          /* 64bit guests only have Xen mappings in their L4 tables. */
>>          if ( type == XEN_DOMCTL_PFINFO_L4TAB )
>>          {
>> -            xen_first = 256;
>> -            xen_last = 271;
>> +            xen_first = (HYPERVISOR_VIRT_START_X86_64 >> 39) & 511;
>> +            xen_last = (HYPERVISOR_VIRT_END_X86_64 >> 39) & 511;
> 
> Would you mind using appropriate PAGETABLE_SHIFT defines from xg_private.h ?

No, I don't mind.

> 
>>          }
>>      }
>>      else
>> @@ -698,7 +872,7 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
>>              /* 32bit guests can only use the first 4 entries of their L3 tables.
>>               * All other are potentially used by Xen. */
>>              xen_first = 4;
>> -            xen_last = 512;
>> +            xen_last = 511;
>>              break;
>>  
>>          case XEN_DOMCTL_PFINFO_L2TAB:
>> @@ -709,10 +883,11 @@ static int normalise_pagetable(struct xc_sr_context *ctx, const uint64_t *src,
>>               *
>>               * ...which is conveniently unavailable to us in a 64bit build.
>>               */
> 
> The comment partially in context is now stale.

Uuh, yes, of course. Will remove/change it.


Thanks for the quick review,

Juergen

Re: [PATCH v2 2/4] libxc: support of linear p2m list for migration of pv-domains

[Andrew Cooper]

On 15/12/15 12:14, Juergen Gross wrote:
> On 15/12/15 12:52, Andrew Cooper wrote:
>> On 15/12/15 06:33, Juergen Gross wrote:
>>> In order to be able to migrate pv-domains with more than 512 GB of RAM
>>> the p2m information can be specified by the guest kernel via a virtual
>>> mapped linear p2m list instead of a 3 level tree.
>>>
>>> Add support for this new p2m format in libxc.
>>>
>>> As the sanity checking of the virtual p2m address needs defines for the
>>> xen regions use those defines when doing page table checks as well.
>>>
>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>, with a few
>> comments/suggestions inline.
>>
>>> ---
>>>  tools/libxc/xc_sr_save_x86_pv.c | 193 ++++++++++++++++++++++++++++++++++++++--
>>>  1 file changed, 184 insertions(+), 9 deletions(-)
>>>
>>> diff --git a/tools/libxc/xc_sr_save_x86_pv.c b/tools/libxc/xc_sr_save_x86_pv.c
>>> index d7acd37..98e9011 100644
>>> --- a/tools/libxc/xc_sr_save_x86_pv.c
>>> +++ b/tools/libxc/xc_sr_save_x86_pv.c
>>> @@ -3,6 +3,18 @@
>>>  
>>>  #include "xc_sr_common_x86_pv.h"
>>>  
>>> +/* Check a 64 bit virtual address for being canonical. */
>>> +static inline bool is_canonical_address(xen_vaddr_t vaddr)
>>> +{
>>> +    return ((int64_t)vaddr >> 47) == ((int64_t)vaddr >> 63);
>>> +}
>>> +
>>> +#define HYPERVISOR_VIRT_START_X86_64 0xFFFF800000000000ULL
>>> +#define HYPERVISOR_VIRT_END_X86_64   0xFFFF87FFFFFFFFFFULL
>>> +
>>> +#define HYPERVISOR_VIRT_START_X86_32 0x00000000F5800000ULL
>>> +#define HYPERVISOR_VIRT_END_X86_32   0x00000000FFFFFFFFULL
>> These might be better in the x86_pv header file, as they could plausibly
>> be useful on the restore side as well.
>>
>> Ideally the virtual ranges would come from the Xen public header files,
>> but that would involve a non-trivial modifications to be able to get the
>> all the information.
> Hmm, I've thought of that, too.
>
> What would be easy is to have above defines in e.g.
> xen/include/public/arch-x86/xen.h and use the appropriate ones in
> xen/include/public/arch-x86/xen-x86_[32|64].h to define
> HYPERVISOR_VIRT_START et al.
>
> What do you think?

That was what I was thinking, but I wouldn't block this series on it.

It can certainly be improved at a later point.

~Andrew

Re: [PATCH v2 2/4] libxc: support of linear p2m list for migration of pv-domains

[Juergen Gross]

On 15/12/15 14:56, Andrew Cooper wrote:
> On 15/12/15 12:14, Juergen Gross wrote:
>> On 15/12/15 12:52, Andrew Cooper wrote:
>>> On 15/12/15 06:33, Juergen Gross wrote:
>>>> In order to be able to migrate pv-domains with more than 512 GB of RAM
>>>> the p2m information can be specified by the guest kernel via a virtual
>>>> mapped linear p2m list instead of a 3 level tree.
>>>>
>>>> Add support for this new p2m format in libxc.
>>>>
>>>> As the sanity checking of the virtual p2m address needs defines for the
>>>> xen regions use those defines when doing page table checks as well.
>>>>
>>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>>> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>, with a few
>>> comments/suggestions inline.
>>>
>>>> ---
>>>>  tools/libxc/xc_sr_save_x86_pv.c | 193 ++++++++++++++++++++++++++++++++++++++--
>>>>  1 file changed, 184 insertions(+), 9 deletions(-)
>>>>
>>>> diff --git a/tools/libxc/xc_sr_save_x86_pv.c b/tools/libxc/xc_sr_save_x86_pv.c
>>>> index d7acd37..98e9011 100644
>>>> --- a/tools/libxc/xc_sr_save_x86_pv.c
>>>> +++ b/tools/libxc/xc_sr_save_x86_pv.c
>>>> @@ -3,6 +3,18 @@
>>>>  
>>>>  #include "xc_sr_common_x86_pv.h"
>>>>  
>>>> +/* Check a 64 bit virtual address for being canonical. */
>>>> +static inline bool is_canonical_address(xen_vaddr_t vaddr)
>>>> +{
>>>> +    return ((int64_t)vaddr >> 47) == ((int64_t)vaddr >> 63);
>>>> +}
>>>> +
>>>> +#define HYPERVISOR_VIRT_START_X86_64 0xFFFF800000000000ULL
>>>> +#define HYPERVISOR_VIRT_END_X86_64   0xFFFF87FFFFFFFFFFULL
>>>> +
>>>> +#define HYPERVISOR_VIRT_START_X86_32 0x00000000F5800000ULL
>>>> +#define HYPERVISOR_VIRT_END_X86_32   0x00000000FFFFFFFFULL
>>> These might be better in the x86_pv header file, as they could plausibly
>>> be useful on the restore side as well.
>>>
>>> Ideally the virtual ranges would come from the Xen public header files,
>>> but that would involve a non-trivial modifications to be able to get the
>>> all the information.
>> Hmm, I've thought of that, too.
>>
>> What would be easy is to have above defines in e.g.
>> xen/include/public/arch-x86/xen.h and use the appropriate ones in
>> xen/include/public/arch-x86/xen-x86_[32|64].h to define
>> HYPERVISOR_VIRT_START et al.
>>
>> What do you think?
> 
> That was what I was thinking, but I wouldn't block this series on it.
> 
> It can certainly be improved at a later point.

Is this an Ack for the patch in the current form?


Juergen

Re: [PATCH v2 2/4] libxc: support of linear p2m list for migration of pv-domains

[Andrew Cooper]

On 15/12/15 15:55, Juergen Gross wrote:
> On 15/12/15 14:56, Andrew Cooper wrote:
>> On 15/12/15 12:14, Juergen Gross wrote:
>>> On 15/12/15 12:52, Andrew Cooper wrote:
>>>> On 15/12/15 06:33, Juergen Gross wrote:
>>>>> In order to be able to migrate pv-domains with more than 512 GB of RAM
>>>>> the p2m information can be specified by the guest kernel via a virtual
>>>>> mapped linear p2m list instead of a 3 level tree.
>>>>>
>>>>> Add support for this new p2m format in libxc.
>>>>>
>>>>> As the sanity checking of the virtual p2m address needs defines for the
>>>>> xen regions use those defines when doing page table checks as well.
>>>>>
>>>>> Signed-off-by: Juergen Gross <jgross@suse.com>
>>>> Reviewed-by: Andrew Cooper <andrew.cooper3@citrix.com>, with a few
>>>> comments/suggestions inline.
>>>>
>>>>> ---
>>>>>  tools/libxc/xc_sr_save_x86_pv.c | 193 ++++++++++++++++++++++++++++++++++++++--
>>>>>  1 file changed, 184 insertions(+), 9 deletions(-)
>>>>>
>>>>> diff --git a/tools/libxc/xc_sr_save_x86_pv.c b/tools/libxc/xc_sr_save_x86_pv.c
>>>>> index d7acd37..98e9011 100644
>>>>> --- a/tools/libxc/xc_sr_save_x86_pv.c
>>>>> +++ b/tools/libxc/xc_sr_save_x86_pv.c
>>>>> @@ -3,6 +3,18 @@
>>>>>  
>>>>>  #include "xc_sr_common_x86_pv.h"
>>>>>  
>>>>> +/* Check a 64 bit virtual address for being canonical. */
>>>>> +static inline bool is_canonical_address(xen_vaddr_t vaddr)
>>>>> +{
>>>>> +    return ((int64_t)vaddr >> 47) == ((int64_t)vaddr >> 63);
>>>>> +}
>>>>> +
>>>>> +#define HYPERVISOR_VIRT_START_X86_64 0xFFFF800000000000ULL
>>>>> +#define HYPERVISOR_VIRT_END_X86_64   0xFFFF87FFFFFFFFFFULL
>>>>> +
>>>>> +#define HYPERVISOR_VIRT_START_X86_32 0x00000000F5800000ULL
>>>>> +#define HYPERVISOR_VIRT_END_X86_32   0x00000000FFFFFFFFULL
>>>> These might be better in the x86_pv header file, as they could plausibly
>>>> be useful on the restore side as well.
>>>>
>>>> Ideally the virtual ranges would come from the Xen public header files,
>>>> but that would involve a non-trivial modifications to be able to get the
>>>> all the information.
>>> Hmm, I've thought of that, too.
>>>
>>> What would be easy is to have above defines in e.g.
>>> xen/include/public/arch-x86/xen.h and use the appropriate ones in
>>> xen/include/public/arch-x86/xen-x86_[32|64].h to define
>>> HYPERVISOR_VIRT_START et al.
>>>
>>> What do you think?
>> That was what I was thinking, but I wouldn't block this series on it.
>>
>> It can certainly be improved at a later point.
> Is this an Ack for the patch in the current form?

My R-b stands (although preferably with the other tweaks incorporated).

I do not suggest that you block this series on fixing the public header
files.

~Andrew