[PATCH RFC 20/20] tools: expose postcopy live migration support in libxl and xl

From: Joshua Otto <jtotto@uwaterloo.ca>
To: xen-devel@lists.xenproject.org
Cc: wei.liu2@citrix.com, andrew.cooper3@citrix.com,
	ian.jackson@eu.citrix.com, czylin@uwaterloo.ca,
	Joshua Otto <jtotto@uwaterloo.ca>,
	imhy.yang@gmail.com, hjarmstr@uwaterloo.ca
Subject: [PATCH RFC 20/20] tools: expose postcopy live migration support in libxl and xl
Date: Mon, 27 Mar 2017 05:06:32 -0400	[thread overview]
Message-ID: <1490605592-12189-21-git-send-email-jtotto@uwaterloo.ca> (raw)
In-Reply-To: <1490605592-12189-1-git-send-email-jtotto@uwaterloo.ca>

- Add a 'memory_strategy' parameter to libxl_domain_live_migrate(),
  which specifies how the remainder of the memory migration should be
  approached after the iterative precopy phase is completed.
- Plug this parameter into the libxl migration precopy policy
  implementation.
- Add --postcopy to xl migrate, and skip the xl-level handshaking at
  both sides when postcopy migration occurs.

Signed-off-by: Joshua Otto <jtotto@uwaterloo.ca>
---
 tools/libxl/libxl.h          |  6 ++++-
 tools/libxl/libxl_dom_save.c | 19 ++++++-------
 tools/libxl/libxl_domain.c   |  9 ++++---
 tools/libxl/libxl_internal.h |  1 +
 tools/xl/xl.h                |  7 ++++-
 tools/xl/xl_cmdtable.c       |  5 +++-
 tools/xl/xl_migrate.c        | 63 +++++++++++++++++++++++++++++++++++++++-----
 tools/xl/xl_vmcontrol.c      |  8 ++++--
 8 files changed, 94 insertions(+), 24 deletions(-)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 51e8760..3a2f7ea 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -1401,7 +1401,7 @@ int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t domid, int send_fd,
                               int flags, /* LIBXL_SUSPEND_* */
                               unsigned int precopy_iterations,
                               unsigned int precopy_dirty_threshold,
-                              int recv_fd,
+                              int recv_fd, int memory_strategy,
                               bool *postcopy_transitioned, /* OUT */
                               const libxl_asyncop_how *ao_how)
                               LIBXL_EXTERNAL_CALLERS_ONLY;
@@ -1409,6 +1409,10 @@ int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t domid, int send_fd,
 #define LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT 5
 #define LIBXL_LM_DIRTY_THRESHOLD_DEFAULT 50
 
+#define LIBXL_LM_MEMORY_STOP_AND_COPY 0
+#define LIBXL_LM_MEMORY_POSTCOPY 1
+#define LIBXL_LM_MEMORY_DEFAULT LIBXL_LM_MEMORY_STOP_AND_COPY
+
 /* @param suspend_cancel [from xenctrl.h:xc_domain_resume( @param fast )]
  *   If this parameter is true, use co-operative resume. The guest
  *   must support this.
diff --git a/tools/libxl/libxl_dom_save.c b/tools/libxl/libxl_dom_save.c
index 9e565ae..9d5d435 100644
--- a/tools/libxl/libxl_dom_save.c
+++ b/tools/libxl/libxl_dom_save.c
@@ -333,18 +333,19 @@ int libxl__save_emulator_xenstore_data(libxl__domain_save_state *dss,
  * the precopy phase of live migrations, and is responsible for deciding when
  * the precopy phase should terminate and what should be done next.
  */
-static int libxl__save_live_migration_simple_precopy_policy(
-    struct precopy_stats stats, void *user)
+static int libxl__save_live_migration_precopy_policy(struct precopy_stats stats,
+                                                     void *user)
 {
     libxl__save_helper_state *shs = user;
     libxl__domain_save_state *dss = shs->caller_state;
 
-    if (stats.dirty_count >= 0 &&
-        stats.dirty_count <= dss->precopy_dirty_threshold)
-        return XGS_POLICY_STOP_AND_COPY;
-
-    if (stats.iteration >= dss->precopy_iterations)
-        return XGS_POLICY_STOP_AND_COPY;
+    if ((stats.dirty_count >= 0 &&
+         stats.dirty_count <= dss->precopy_dirty_threshold) ||
+        (stats.iteration >= dss->precopy_iterations)) {
+        return (dss->memory_strategy == LIBXL_LM_MEMORY_POSTCOPY)
+            ? XGS_POLICY_POSTCOPY
+            : XGS_POLICY_STOP_AND_COPY;
+    }
 
     return XGS_POLICY_CONTINUE_PRECOPY;
 }
@@ -452,7 +453,7 @@ void libxl__domain_save(libxl__egc *egc, libxl__domain_save_state *dss)
             libxl__save_live_migration_postcopy_transition_callback;
     }
 
-    callbacks->precopy_policy = libxl__save_live_migration_simple_precopy_policy;
+    callbacks->precopy_policy = libxl__save_live_migration_precopy_policy;
     callbacks->switch_qemu_logdirty = libxl__domain_suspend_common_switch_qemu_logdirty;
 
     dss->sws.ao  = dss->ao;
diff --git a/tools/libxl/libxl_domain.c b/tools/libxl/libxl_domain.c
index ea778a6..feec293 100644
--- a/tools/libxl/libxl_domain.c
+++ b/tools/libxl/libxl_domain.c
@@ -489,6 +489,7 @@ static void domain_suspend_cb(libxl__egc *egc,
 static int do_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
                              unsigned int precopy_iterations,
                              unsigned int precopy_dirty_threshold, int recv_fd,
+                             int memory_strategy,
                              bool *postcopy_transitioned,
                              const libxl_asyncop_how *ao_how)
 {
@@ -510,7 +511,8 @@ static int do_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
     dss->domid = domid;
     dss->fd = fd;
     dss->recv_fd = recv_fd;
-    dss->postcopy_transitioned = postcopy_resumed_remotely;
+    dss->memory_strategy = memory_strategy;
+    dss->postcopy_transitioned = postcopy_transitioned;
     dss->type = type;
     dss->live = flags & LIBXL_SUSPEND_LIVE;
     dss->debug = flags & LIBXL_SUSPEND_DEBUG;
@@ -536,12 +538,13 @@ int libxl_domain_suspend(libxl_ctx *ctx, uint32_t domid, int fd, int flags,
     return do_domain_suspend(ctx, domid, fd, flags,
                              LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT,
                              LIBXL_LM_DIRTY_THRESHOLD_DEFAULT, -1,
-                             NULL, ao_how);
+                             LIBXL_LM_MEMORY_DEFAULT, NULL, ao_how);
 }
 
 int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t domid, int send_fd,
                               int flags, unsigned int precopy_iterations,
                               unsigned int precopy_dirty_threshold, int recv_fd,
+                              int memory_strategy,
                               bool *postcopy_transitioned,
                               const libxl_asyncop_how *ao_how)
 {
@@ -553,7 +556,7 @@ int libxl_domain_live_migrate(libxl_ctx *ctx, uint32_t domid, int send_fd,
     flags |= LIBXL_SUSPEND_LIVE;
 
     return do_domain_suspend(ctx, domid, send_fd, flags, precopy_iterations,
-                             precopy_dirty_threshold, recv_fd,
+                             precopy_dirty_threshold, recv_fd, memory_strategy,
                              postcopy_transitioned, ao_how);
 }
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 0a7c0d1..209cee5 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -3313,6 +3313,7 @@ struct libxl__domain_save_state {
     int fd;
     int fdfl; /* original flags on fd */
     int recv_fd;
+    int memory_strategy;
     bool *postcopy_transitioned;
     libxl_domain_type type;
     int live;
diff --git a/tools/xl/xl.h b/tools/xl/xl.h
index aa95b77..279c716 100644
--- a/tools/xl/xl.h
+++ b/tools/xl/xl.h
@@ -48,6 +48,7 @@ struct domain_create {
     bool userspace_colo_proxy;
     int migrate_fd; /* -1 means none */
     int send_back_fd; /* -1 means none */
+    bool *postcopy_resumed;
     char **migration_domname_r; /* from malloc */
 };
 
@@ -66,7 +67,6 @@ static const char migrate_permission_to_go[]=
     "domain is yours, you are cleared to unpause";
 static const char migrate_report[]=
     "my copy unpause results are as follows";
-#endif
 
   /* followed by one byte:
    *     0: everything went well, domain is running
@@ -76,6 +76,11 @@ static const char migrate_report[]=
    *            from target to source
    */
 
+static const char migrate_postcopy_sync[]=
+    "postcopy migration completed successfully";
+
+#endif
+
 #define XL_MANDATORY_FLAG_JSON (1U << 0) /* config data is in JSON format */
 #define XL_MANDATORY_FLAG_STREAMv2 (1U << 1) /* stream is v2 */
 #define XL_MANDATORY_FLAG_ALL  (XL_MANDATORY_FLAG_JSON |        \
diff --git a/tools/xl/xl_cmdtable.c b/tools/xl/xl_cmdtable.c
index 6df66fb..7bd2d1b 100644
--- a/tools/xl/xl_cmdtable.c
+++ b/tools/xl/xl_cmdtable.c
@@ -169,7 +169,10 @@ struct cmd_spec cmd_table[] = {
       "--precopy-iterations Perform at most this many iterations of the precopy\n"
       "                     memory migration loop before suspending the domain.\n"
       "--precopy-threshold  If fewer than this many pages are dirty at the end of a\n"
-      "                     copy round, exit the precopy loop and suspend the domain."
+      "                     copy round, exit the precopy loop and suspend the domain.\n"
+      "--postcopy           At the end of the iterative precopy phase, transition to a\n"
+      "                     postcopy memory migration rather than performing a stop-and-copy\n"
+      "                     migration of the outstanding dirty pages.\n"
     },
     { "restore",
       &main_restore, 0, 1,
diff --git a/tools/xl/xl_migrate.c b/tools/xl/xl_migrate.c
index 1ffc32b..43c7d8e 100644
--- a/tools/xl/xl_migrate.c
+++ b/tools/xl/xl_migrate.c
@@ -179,7 +179,8 @@ static void migrate_do_preamble(int send_fd, int recv_fd, pid_t child,
 static void migrate_domain(uint32_t domid, const char *rune, int debug,
                            const char *override_config_file,
                            unsigned int precopy_iterations,
-                           unsigned int precopy_dirty_threshold)
+                           unsigned int precopy_dirty_threshold,
+                           int memory_strategy)
 {
     pid_t child = -1;
     int rc;
@@ -210,18 +211,32 @@ static void migrate_domain(uint32_t domid, const char *rune, int debug,
         flags |= LIBXL_SUSPEND_DEBUG;
     rc = libxl_domain_live_migrate(ctx, domid, send_fd, flags,
                                    precopy_iterations, precopy_dirty_threshold,
-                                   recv_fd, &postcopy_transitioned, NULL);
-    assert(!postcopy_transitioned);
-
+                                   recv_fd, memory_strategy,
+                                   &postcopy_transitioned, NULL);
     if (rc) {
         fprintf(stderr, "migration sender: libxl_domain_suspend failed"
                 " (rc=%d)\n", rc);
-        if (rc == ERROR_GUEST_TIMEDOUT)
+        if (postcopy_transitioned)
+            goto failed_postcopy;
+        else if (rc == ERROR_GUEST_TIMEDOUT)
             goto failed_suspend;
         else
             goto failed_resume;
     }
 
+    /* No need for additional ceremony if we already resumed the guest as part
+     * of a postcopy live migration. */
+    if (postcopy_transitioned) {
+        /* It doesn't matter if something happens to the pipe after we get to
+         * this point - we only bother to synchronize here for tidiness. */
+        migrate_read_fixedmessage(recv_fd, migrate_postcopy_sync,
+                                  sizeof(migrate_postcopy_sync),
+                                  "postcopy sync", rune);
+        libxl_domain_destroy(ctx, domid, 0);
+        fprintf(stderr, "Migration successful.\n");
+        exit(EXIT_SUCCESS);
+    }
+
     //fprintf(stderr, "migration sender: Transfer complete.\n");
     // Should only be printed when debugging as it's a bit messy with
     // progress indication.
@@ -320,6 +335,21 @@ static void migrate_domain(uint32_t domid, const char *rune, int debug,
     close(send_fd);
     migration_child_report(recv_fd);
     exit(EXIT_FAILURE);
+
+ failed_postcopy:
+    if (common_domname) {
+        xasprintf(&away_domname, "%s--postcopy-inconsistent", common_domname);
+        libxl_domain_rename(ctx, domid, common_domname, away_domname);
+    }
+
+    fprintf(stderr,
+ "** Migration failed during memory postcopy **\n"
+ "It's possible that the guest has executed/is executing at the destination,\n"
+ " so resuming it here now may be unsafe.\n");
+
+    close(send_fd);
+    migration_child_report(recv_fd);
+    exit(EXIT_FAILURE);
 }
 
 static void migrate_receive(int debug, int daemonize, int monitor,
@@ -333,6 +363,7 @@ static void migrate_receive(int debug, int daemonize, int monitor,
     int rc, rc2;
     char rc_buf;
     char *migration_domname;
+    bool postcopy_resumed;
     struct domain_create dom_info;
 
     signal(SIGPIPE, SIG_IGN);
@@ -352,6 +383,7 @@ static void migrate_receive(int debug, int daemonize, int monitor,
     dom_info.paused = 1;
     dom_info.migrate_fd = recv_fd;
     dom_info.send_back_fd = send_fd;
+    dom_info.postcopy_resumed = &postcopy_resumed;
     dom_info.migration_domname_r = &migration_domname;
     dom_info.checkpointed_stream = checkpointed;
     dom_info.colo_proxy_script = colo_proxy_script;
@@ -414,6 +446,18 @@ static void migrate_receive(int debug, int daemonize, int monitor,
         break;
     }
 
+    /* No need for additional ceremony if we already resumed the guest as part
+     * of a postcopy live migration. */
+    if (postcopy_resumed) {
+        libxl_write_exactly(ctx, send_fd, migrate_postcopy_sync,
+                            sizeof(migrate_postcopy_sync),
+                            "migration ack stream", "postcopy sync");
+        fprintf(stderr, "migration target: Domain started successsfully.\n");
+        libxl_domain_rename(ctx, domid, migration_domname, common_domname);
+        exit(EXIT_SUCCESS);
+    }
+
+
     fprintf(stderr, "migration target: Transfer complete,"
             " requesting permission to start domain.\n");
 
@@ -545,12 +589,14 @@ int main_migrate(int argc, char **argv)
     char *host;
     int opt, daemonize = 1, monitor = 1, debug = 0, pause_after_migration = 0;
     int precopy_iterations = LIBXL_LM_PRECOPY_ITERATIONS_DEFAULT,
-        precopy_dirty_threshold = LIBXL_LM_DIRTY_THRESHOLD_DEFAULT;
+        precopy_dirty_threshold = LIBXL_LM_DIRTY_THRESHOLD_DEFAULT,
+        memory_strategy = LIBXL_LM_MEMORY_DEFAULT;
     static struct option opts[] = {
         {"debug", 0, 0, 0x100},
         {"live", 0, 0, 0x200},
         {"precopy-iterations", 1, 0, 'i'},
         {"precopy-threshold", 1, 0, 'd'},
+        {"postcopy", 0, 0, 0x400},
         COMMON_LONG_OPTS
     };
 
@@ -591,6 +637,9 @@ int main_migrate(int argc, char **argv)
     case 0x200: /* --live */
         /* ignored for compatibility with xm */
         break;
+    case 0x400: /* --postcopy */
+        memory_strategy = LIBXL_LM_MEMORY_POSTCOPY;
+        break;
     }
 
     domid = find_domain(argv[optind]);
@@ -622,7 +671,7 @@ int main_migrate(int argc, char **argv)
     }
 
     migrate_domain(domid, rune, debug, config_filename, precopy_iterations,
-                   precopy_dirty_threshold);
+                   precopy_dirty_threshold, memory_strategy);
     return EXIT_SUCCESS;
 }
 
diff --git a/tools/xl/xl_vmcontrol.c b/tools/xl/xl_vmcontrol.c
index 47ba9f3..62e09c1 100644
--- a/tools/xl/xl_vmcontrol.c
+++ b/tools/xl/xl_vmcontrol.c
@@ -655,6 +655,7 @@ int create_domain(struct domain_create *dom_info)
     const char *config_source = NULL;
     const char *restore_source = NULL;
     int migrate_fd = dom_info->migrate_fd;
+    bool *postcopy_resumed = dom_info->postcopy_resumed;
     bool config_in_json;
 
     int i;
@@ -675,6 +676,9 @@ int create_domain(struct domain_create *dom_info)
 
     int restoring = (restore_file || (migrate_fd >= 0));
 
+    if (postcopy_resumed)
+        *postcopy_resumed = false;
+
     libxl_domain_config_init(&d_config);
 
     if (restoring) {
@@ -882,8 +886,8 @@ start:
 
         ret = libxl_domain_create_restore(ctx, &d_config,
                                           &domid, restore_fd,
-                                          send_back_fd, NULL, &params,
-                                          0, autoconnect_console_how);
+                                          send_back_fd, postcopy_resumed,
+                                          &params, 0, autoconnect_console_how);
 
         libxl_domain_restore_params_dispose(&params);
 
-- 
2.7.4


_______________________________________________
Xen-devel mailing list
Xen-devel@lists.xen.org
https://lists.xen.org/xen-devel
