From mboxrd@z Thu Jan 1 00:00:00 1970 From: Wen Congyang Subject: [RFC Patch v3 08/22] primary vm suspend/get_dirty_pfn/resume/checkpoint code Date: Fri, 5 Sep 2014 17:25:43 +0800 Message-ID: <1409909158-19243-9-git-send-email-wency@cn.fujitsu.com> References: <1409909158-19243-1-git-send-email-wency@cn.fujitsu.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1409909158-19243-1-git-send-email-wency@cn.fujitsu.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: xen devel Cc: Ian Campbell , Wen Congyang , Ian Jackson , Jiang Yunhong , Dong Eddie , Yang Hongyang , Lai Jiangshan List-Id: xen-devel@lists.xenproject.org We will do the following things again and again: 1. Suspend primary vm a. Suspend primary vm b. do postsuspend c. Read LIBXL_COLO_SVM_SUSPENDED to master d. Read secondary vm's dirty page information to master(count + pfn list) 2. Get dirty pfn list a. Return secondary vm's dirty pfn list 3. Resume primary vm a. Read LIBXL_COLO_SVM_READY from slave b. Do presume c. Resume primary vm d. Read LIBXL_COLO_SVM_RESUMED from slave 4. Wait a new checkpoint a. Wait a new checkpoint(not implemented) b. Send LIBXL_COLO_NEW_CHECKPOINT to slave Signed-off-by: Wen Congyang --- tools/libxc/xenguest.h | 12 + tools/libxl/Makefile | 2 +- tools/libxl/libxl.c | 6 +- tools/libxl/libxl_colo.h | 10 + tools/libxl/libxl_colo_save.c | 608 +++++++++++++++++++++++++++++++++++++ tools/libxl/libxl_dom.c | 13 +- tools/libxl/libxl_internal.h | 32 +- tools/libxl/libxl_save_msgs_gen.pl | 1 + tools/libxl/libxl_types.idl | 1 + 9 files changed, 677 insertions(+), 8 deletions(-) create mode 100644 tools/libxl/libxl_colo_save.c diff --git a/tools/libxc/xenguest.h b/tools/libxc/xenguest.h index d3061c7..1aeaad2 100644 --- a/tools/libxc/xenguest.h +++ b/tools/libxc/xenguest.h @@ -72,6 +72,18 @@ struct save_callbacks { */ int (*toolstack_save)(uint32_t domid, uint8_t **buf, uint32_t *len, void *data); + /* Called after the guest is suspended. + * + * returns the list of dirty pfn: + * struct { + * uint64_t count; + * uint64_t pfn[]; + * }; + * + * Note: the caller must free the return value. + */ + uint8_t *(*get_dirty_pfn)(void *data); + /* to be provided as the last argument to each callback function */ void* data; }; diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index c026bdd..1c32ae2 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -57,7 +57,7 @@ LIBXL_OBJS-y += libxl_nonetbuffer.o endif LIBXL_OBJS-y += libxl_remus.o libxl_checkpoint_device.o libxl_remus_disk_drbd.o -LIBXL_OBJS-y += libxl_colo_restore.o +LIBXL_OBJS-y += libxl_colo_restore.o libxl_colo_save.o LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 8e7d87c..c86b988 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -18,6 +18,7 @@ #include "libxl_internal.h" #include "libxl_remus.h" +#include "libxl_colo.h" #define PAGE_TO_MEMKB(pages) ((pages) * 4) #define BACKEND_STRING_SIZE 5 @@ -831,7 +832,10 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, assert(info); /* Point of no return */ - libxl__remus_setup(egc, &dss->rs); + if (libxl_defbool_val(info->colo)) + libxl__colo_save_setup(egc, &dss->css); + else + libxl__remus_setup(egc, &dss->rs); return AO_INPROGRESS; out: diff --git a/tools/libxl/libxl_colo.h b/tools/libxl/libxl_colo.h index 91df275..26a2563 100644 --- a/tools/libxl/libxl_colo.h +++ b/tools/libxl/libxl_colo.h @@ -35,4 +35,14 @@ extern void libxl__colo_restore_teardown(libxl__egc *egc, libxl__colo_restore_state *crs, int rc); +extern void libxl__colo_save_domain_suspend_callback(void *data); +extern void libxl__colo_save_domain_resume_callback(void *data); +extern void libxl__colo_save_domain_checkpoint_callback(void *data); +extern void libxl__colo_save_get_dirty_pfn_callback(void *data); +extern void libxl__colo_save_setup(libxl__egc *egc, + libxl__colo_save_state *css); +extern void libxl__colo_save_teardown(libxl__egc *egc, + libxl__colo_save_state *css, + int rc); + #endif diff --git a/tools/libxl/libxl_colo_save.c b/tools/libxl/libxl_colo_save.c new file mode 100644 index 0000000..7b76d3f --- /dev/null +++ b/tools/libxl/libxl_colo_save.c @@ -0,0 +1,608 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author: Wen Congyang + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" +#include "libxl_colo.h" + +static const libxl__checkpoint_device_instance_ops *colo_ops[] = { + NULL, +}; + +/* ================= colo: setup save environment ================= */ +static void colo_save_setup_done(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); +static void colo_save_setup_failed(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); + +void libxl__colo_save_setup(libxl__egc *egc, libxl__colo_save_state *css) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + + /* Convenience aliases */ + libxl__checkpoint_devices_state *const cds = &css->cds; + + STATE_AO_GC(dss->ao); + + if (dss->type != LIBXL_DOMAIN_TYPE_HVM) { + LOG(ERROR, "COLO only supports hvm now"); + goto out; + } + + css->send_fd = dss->fd; + css->recv_fd = dss->recv_fd; + css->svm_running = false; + + /* TODO: disk/nic support */ + cds->device_kind_flags = 0; + cds->ops = colo_ops; + cds->callback = colo_save_setup_done; + cds->ao = ao; + cds->domid = dss->domid; + + libxl__checkpoint_devices_setup(egc, &css->cds); + + return; + +out: + libxl__ao_complete(egc, ao, ERROR_FAIL); +} + +static void colo_save_setup_done(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = CONTAINER_OF(cds, *css, cds); + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + STATE_AO_GC(cds->ao); + + if (!rc) { + libxl__domain_suspend(egc, dss); + return; + } + + LOG(ERROR, "COLO: failed to setup device for guest with domid %u", + dss->domid); + css->cds.callback = colo_save_setup_failed; + libxl__checkpoint_devices_teardown(egc, &css->cds); +} + +static void colo_save_setup_failed(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + STATE_AO_GC(cds->ao); + + if (rc) + LOG(ERROR, "COLO: failed to teardown device after setup failed" + " for guest with domid %u, rc %d", cds->domid, rc); + + libxl__ao_complete(egc, ao, rc); +} + + +/* ================= colo: teardown save environment ================= */ +static void colo_teardown_done(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); + +void libxl__colo_save_teardown(libxl__egc *egc, + libxl__colo_save_state *css, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + + STATE_AO_GC(css->cds.ao); + + LOG(WARN, "COLO: Domain suspend terminated with rc %d," + " teardown COLO devices...", rc); + dss->css.cds.callback = colo_teardown_done; + libxl__checkpoint_devices_teardown(egc, &dss->css.cds); + return; +} + +static void colo_teardown_done(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = CONTAINER_OF(cds, *css, cds); + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + dss->callback(egc, dss, rc); +} + +/* + * checkpoint callbacks are called in the following order: + * 1. suspend + * 2. resume + * 3. checkpoint + */ +static void colo_common_read_done(libxl__egc *egc, + libxl__datareader_state *drs, + ssize_t real_size, int errnoval); +/* ===================== colo: suspend primary vm ===================== */ +/* + * Do the following things when suspending primary vm: + * 1. suspend primary vm + * 2. do postsuspend + * 3. read LIBXL_COLO_SVM_SUSPENDED + * 4. read secondary vm's dirty pages + */ +static void colo_suspend_primary_vm_done(libxl__egc *egc, + libxl__domain_suspend_state2 *dss2, + int ok); +static void colo_postsuspend_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); +static void colo_read_pfn(libxl__egc *egc, libxl__colo_save_state *css); + +void libxl__colo_save_domain_suspend_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__egc *egc = shs->egc; + libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); + + /* Convenience aliases */ + libxl__domain_suspend_state2 *dss2 = &dss->dss2; + + dss2->callback_common_done = colo_suspend_primary_vm_done; + libxl__domain_suspend2(egc, dss2); +} + +static void colo_suspend_primary_vm_done(libxl__egc *egc, + libxl__domain_suspend_state2 *dss2, + int ok) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(dss2, *dss, dss2); + + STATE_AO_GC(dss2->ao); + + if (!ok) { + LOG(ERROR, "cannot suspend primary vm"); + goto out; + } + + /* Convenience aliases */ + libxl__checkpoint_devices_state *const cds = &dss->css.cds; + + cds->callback = colo_postsuspend_cb; + libxl__checkpoint_devices_postsuspend(egc, cds); + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + +static void colo_postsuspend_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + int ok = 0; + libxl__colo_save_state *css = CONTAINER_OF(cds, *css, cds); + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + + /* Convenience aliases */ + libxl__datareader_state *const drs = &css->drs; + + STATE_AO_GC(cds->ao); + + if (rc) { + LOG(ERROR, "postsuspend fails"); + goto out; + } + + if (!css->svm_running) { + ok = 1; + goto out; + } + + /* + * read LIBXL_COLO_SVM_SUSPENDED and the count of + * secondary vm's dirty pages. + */ + memset(drs, 0, sizeof(*drs)); + drs->ao = ao; + drs->readfd = css->recv_fd; + drs->readsize = sizeof(css->temp_buff); + drs->readwhat = "colo stream"; + drs->callback = colo_common_read_done; + drs->buf = css->temp_buff; + css->callback = colo_read_pfn; + + if (libxl__datareader_start(drs)) { + LOG(ERROR, "libxl__datareader_start() fails"); + goto out; + } + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + +static void colo_read_pfn(libxl__egc *egc, libxl__colo_save_state *css) +{ + int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + STATE_AO_GC(css->cds.ao); + + /* Convenience aliases */ + libxl__datareader_state *const drs = &css->drs; + + assert(!css->buff); + css->section = css->temp_buff[0]; + css->count = *(uint64_t *)(&css->temp_buff[1]); + + if (css->section != LIBXL_COLO_SVM_SUSPENDED) { + LOG(ERROR, "invalid section: %d, expected: %d", + css->section, LIBXL_COLO_SVM_SUSPENDED); + goto out; + } + + css->buff = libxl__zalloc(NOGC, sizeof(uint64_t) * (css->count + 1)); + css->buff[0] = css->count; + + if (css->count == 0) { + /* no dirty pages */ + ok = 1; + goto out; + } + + /* read the pfn of secondary vm's dirty pages */ + memset(drs, 0, sizeof(*drs)); + drs->ao = ao; + drs->readfd = css->recv_fd; + drs->readsize = css->count * sizeof(uint64_t); + drs->readwhat = "colo stream"; + drs->callback = colo_common_read_done; + drs->buf = css->buff + 1; + css->callback = NULL; + + if (libxl__datareader_start(drs)) { + LOG(ERROR, "libxl__datareader_start() fails"); + goto out; + } + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + + +/* ===================== colo: get dirty pfn ===================== */ +void libxl__colo_save_get_dirty_pfn_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__egc *egc = shs->egc; + libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); + uint64_t size; + + /* Convenience aliases */ + libxl__colo_save_state *const css = &dss->css; + + assert(css->buff); + size = sizeof(uint64_t) * (css->count + 1); + + libxl__xc_domain_saverestore_async_callback_done_with_data(egc, shs, + (uint8_t *)css->buff, + size); + free(css->buff); + css->buff = NULL; +} + + +/* ===================== colo: resume primary vm ===================== */ +/* + * Do the following things when resuming primary vm: + * 1. read LIBXL_COLO_SVM_READY + * 2. do preresume + * 3. resume primary vm + * 4. read LIBXL_COLO_SVM_RESUMED + */ +static void colo_preresume_dm_saved(libxl__egc *egc, + libxl__domain_suspend_state *dss, int rc); +static void colo_read_svm_ready_done(libxl__egc *egc, + libxl__colo_save_state *css); +static void colo_preresume_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); +static void colo_read_svm_resumed_done(libxl__egc *egc, + libxl__colo_save_state *css); + +void libxl__colo_save_domain_resume_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__egc *egc = shs->egc; + libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); + + /* This would go into tailbuf. */ + if (dss->hvm) { + libxl__domain_save_device_model(egc, dss, colo_preresume_dm_saved); + } else { + colo_preresume_dm_saved(egc, dss, 0); + } + + return; +} + +static void colo_preresume_dm_saved(libxl__egc *egc, + libxl__domain_suspend_state *dss, int rc) +{ + /* Convenience aliases */ + libxl__colo_save_state *const css = &dss->css; + libxl__datareader_state *const drs = &css->drs; + + STATE_AO_GC(css->cds.ao); + + if (rc) { + LOG(ERROR, "Failed to save device model. Terminating COLO.."); + goto out; + } + + /* read LIBXL_COLO_SVM_READY */ + memset(drs, 0, sizeof(*drs)); + drs->ao = ao; + drs->readfd = css->recv_fd; + drs->readsize = sizeof(css->section); + drs->readwhat = "colo stream"; + drs->callback = colo_common_read_done; + drs->buf = &css->section; + css->callback = colo_read_svm_ready_done; + + if (libxl__datareader_start(drs)) { + LOG(ERROR, "libxl__datareader_start() fails"); + goto out; + } + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void colo_read_svm_ready_done(libxl__egc *egc, + libxl__colo_save_state *css) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + + STATE_AO_GC(css->cds.ao); + + if (css->section != LIBXL_COLO_SVM_READY) { + LOG(ERROR, "invalid section: %d, expected: %d", + css->section, LIBXL_COLO_SVM_READY); + goto out; + } + + css->svm_running = true; + css->cds.callback = colo_preresume_cb; + libxl__checkpoint_devices_preresume(egc, &css->cds); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void colo_preresume_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = CONTAINER_OF(cds, *css, cds); + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + + /* Convenience aliases */ + libxl__datareader_state *const drs = &css->drs; + + STATE_AO_GC(cds->ao); + + if (rc) { + LOG(ERROR, "preresume fails"); + goto out; + } + + /* Resumes the domain and the device model */ + if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1, 0)) { + LOG(ERROR, "cannot resume primary vm"); + goto out; + } + + /* read LIBXL_COLO_SVM_RESUMED */ + memset(drs, 0, sizeof(*drs)); + drs->ao = ao; + drs->readfd = css->recv_fd; + drs->readsize = sizeof(css->section); + drs->readwhat = "colo stream"; + drs->callback = colo_common_read_done; + drs->buf = &css->section; + css->callback = colo_read_svm_resumed_done; + + if (libxl__datareader_start(drs)) { + LOG(ERROR, "libxl__datareader_start() fails"); + goto out; + } + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void colo_read_svm_resumed_done(libxl__egc *egc, + libxl__colo_save_state *css) +{ + int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + + STATE_AO_GC(css->cds.ao); + + if (css->section != LIBXL_COLO_SVM_RESUMED) { + LOG(ERROR, "invalid section: %d, expected: %d", + css->section, LIBXL_COLO_SVM_RESUMED); + goto out; + } + + ok = 1; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + + +/* ===================== colo: wait new checkpoint ===================== */ +/* + * Do the following things: + * 1. do commit + * 2. wait for a new checkpoint + * 3. write LIBXL_COLO_NEW_CHECKPOINT + */ +static void colo_device_commit_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); +static void colo_start_new_checkpoint(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc); +static void colo_send_data_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval); + +void libxl__colo_save_domain_checkpoint_callback(void *data) +{ + libxl__save_helper_state *shs = data; + libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); + libxl__egc *egc = dss->shs.egc; + + /* Convenience aliases */ + libxl__checkpoint_devices_state *const cds = &dss->css.cds; + + cds->callback = colo_device_commit_cb; + libxl__checkpoint_devices_commit(egc, cds); +} + +static void colo_device_commit_cb(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = CONTAINER_OF(cds, *css, cds); + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + + STATE_AO_GC(cds->ao); + + if (rc) { + LOG(ERROR, "commit fails"); + goto out; + } + + /* TODO: wait a new checkpoint */ + colo_start_new_checkpoint(egc, cds, 0); + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void colo_start_new_checkpoint(libxl__egc *egc, + libxl__checkpoint_devices_state *cds, + int rc) +{ + libxl__colo_save_state *css = CONTAINER_OF(cds, *css, cds); + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + uint8_t section = LIBXL_COLO_NEW_CHECKPOINT; + + /* Convenience aliases */ + libxl__datacopier_state *const dc = &css->dc; + + STATE_AO_GC(cds->ao); + + if (rc) + goto out; + + /* write LIBXL_COLO_NEW_CHECKPOINT */ + memset(dc, 0, sizeof(*dc)); + dc->ao = ao; + dc->readfd = -1; + dc->writefd = css->send_fd; + dc->maxsz = INT_MAX; + dc->copywhat = "new checkpoint is triggered"; + dc->writewhat = "colo stream"; + dc->callback = colo_send_data_done; + + rc = libxl__datacopier_start(dc); + if (rc) { + LOG(ERROR, "libxl__datacopier_start() fails"); + goto out; + } + + /* tell slave that a new checkpoint is triggered */ + libxl__datacopier_prefixdata(egc, dc, §ion, sizeof(section)); + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void colo_send_data_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + libxl__colo_save_state *css = CONTAINER_OF(dc, *css, dc); + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + int ok; + + STATE_AO_GC(dc->ao); + + if (onwrite == -1 || errnoval) { + LOG(ERROR, "cannot start a new checkpoint"); + ok = 0; + goto out; + } + + /* Everything is OK */ + ok = 1; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + + +/* ===================== colo: common callback ===================== */ +static void colo_common_read_done(libxl__egc *egc, + libxl__datareader_state *drs, + ssize_t real_size, int errnoval) +{ + int ok = 0; + libxl__colo_save_state *css = CONTAINER_OF(drs, *css, drs); + libxl__domain_suspend_state *dss = CONTAINER_OF(css, *dss, css); + STATE_AO_GC(drs->ao); + + if (real_size < drs->readsize) { + LOG(ERROR, "reading data fails: %lld", (long long)real_size); + goto out; + } + + if (!css->callback) { + /* Everything is OK */ + ok = 1; + goto out; + } + + css->callback(egc, css); + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c index e49f40f..ea08e41 100644 --- a/tools/libxl/libxl_dom.c +++ b/tools/libxl/libxl_dom.c @@ -20,6 +20,7 @@ #include "libxl_internal.h" #include "libxl_arch.h" #include "libxl_remus.h" +#include "libxl_colo.h" #include #include @@ -1624,7 +1625,12 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss) } memset(callbacks, 0, sizeof(*callbacks)); - if (r_info != NULL) { + if (r_info != NULL && libxl_defbool_val(r_info->colo)) { + callbacks->suspend = libxl__colo_save_domain_suspend_callback; + callbacks->postcopy = libxl__colo_save_domain_resume_callback; + callbacks->checkpoint = libxl__colo_save_domain_checkpoint_callback; + callbacks->get_dirty_pfn = libxl__colo_save_get_dirty_pfn_callback; + } else if (r_info != NULL) { callbacks->suspend = libxl__remus_domain_suspend_callback; callbacks->postcopy = libxl__remus_domain_resume_callback; callbacks->checkpoint = libxl__remus_domain_checkpoint_callback; @@ -1785,7 +1791,10 @@ static void domain_suspend_done(libxl__egc *egc, xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid, dss2->guest_evtchn.port, &dss2->guest_evtchn_lockfd); - if (dss->remus) { + if (dss->remus && libxl_defbool_val(dss->remus->colo)) { + libxl__colo_save_teardown(egc, &dss->css, rc); + return; + } else if (dss->remus) { libxl__remus_teardown(egc, &dss->rs, rc); return; } diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index d5b3197..1cc90fb 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -2526,7 +2526,7 @@ typedef struct libxl__save_helper_state { /* * The abstract checkpoint device layer exposes a common * set of API to [external] libxl for manipulating devices attached to - * a guest protected by Remus. The device layer also exposes a set of + * a guest protected by Remus/COLO. The device layer also exposes a set of * [internal] interfaces that every device type must implement. * * The following API are exposed to libxl: @@ -2544,7 +2544,7 @@ typedef struct libxl__save_helper_state { * +libxl__checkpoint_devices_commit * * Each device type needs to implement the interfaces specified in - * the libxl__checkpoint_device_instance_ops if it wishes to support Remus. + * the libxl__checkpoint_device_instance_ops if it wishes to support Remus/COLO. * * The high-level control flow through the checkpoint device layer is shown * below: @@ -2564,7 +2564,7 @@ typedef struct libxl__checkpoint_device_instance_ops libxl__checkpoint_device_in /* * Interfaces to be implemented by every device subkind that wishes to - * support Remus. Functions must be implemented unless otherwise + * support Remus/COLO. Functions must be implemented unless otherwise * stated. Many of these functions are asynchronous. They call * dev->aodev.callback when done. The actual implementations may be * synchronous and call dev->aodev.callback directly (as the last @@ -2716,6 +2716,25 @@ struct libxl__remus_state { _hidden int libxl__netbuffer_enabled(libxl__gc *gc); +/*----- colo related state structure -----*/ +typedef struct libxl__colo_save_state libxl__colo_save_state; +struct libxl__colo_save_state { + libxl__checkpoint_devices_state cds; + int send_fd; + int recv_fd; + + /* private */ + libxl__datacopier_state dc; + libxl__datareader_state drs; + uint8_t section; + uint64_t count; + uint64_t *buff; + /* read section and count, and then store it in temp_buff */ + uint8_t temp_buff[9]; + void (*callback)(libxl__egc *, libxl__colo_save_state *); + bool svm_running; +}; + /*----- Domain suspend (save) state structure -----*/ typedef struct libxl__domain_suspend_state libxl__domain_suspend_state; @@ -2779,7 +2798,12 @@ struct libxl__domain_suspend_state { libxl__domain_suspend_state2 dss2; int hvm; int xcflags; - libxl__remus_state rs; + union { + /* for Remus */ + libxl__remus_state rs; + /* for COLO */ + libxl__colo_save_state css; + }; libxl__save_helper_state shs; libxl__logdirty_switch logdirty; /* private for libxl__domain_save_device_model */ diff --git a/tools/libxl/libxl_save_msgs_gen.pl b/tools/libxl/libxl_save_msgs_gen.pl index 0239cac..fbb2d67 100755 --- a/tools/libxl/libxl_save_msgs_gen.pl +++ b/tools/libxl/libxl_save_msgs_gen.pl @@ -36,6 +36,7 @@ our @msgs = ( 'unsigned long', 'console_mfn'] ], [ 9, 'srW', "complete", [qw(int retval int errnoval)] ], + [ 10, 'scxAB', "get_dirty_pfn", [] ], ); #---------------------------------------- diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index 29b024a..a8be7ba 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -611,6 +611,7 @@ libxl_domain_remus_info = Struct("domain_remus_info",[ ("netbuf", libxl_defbool), ("netbufscript", string), ("diskbuf", libxl_defbool), + ("colo", libxl_defbool) ]) libxl_event_type = Enumeration("event_type", [ -- 1.9.3