From mboxrd@z Thu Jan 1 00:00:00 1970 From: Andrew Cooper Subject: [PATCH 20/27] tools/libxl: Infrastructure for writing a v2 stream Date: Mon, 15 Jun 2015 14:44:33 +0100 Message-ID: <1434375880-30914-21-git-send-email-andrew.cooper3@citrix.com> References: <1434375880-30914-1-git-send-email-andrew.cooper3@citrix.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1434375880-30914-1-git-send-email-andrew.cooper3@citrix.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: Xen-devel Cc: Wei Liu , Ian Campbell , Andrew Cooper , Ian Jackson , Ross Lagerwall , Yang Hongyang List-Id: xen-devel@lists.xenproject.org From: Ross Lagerwall Signed-off-by: Ross Lagerwall Signed-off-by: Andrew Cooper CC: Ian Campbell CC: Ian Jackson CC: Wei Liu --- tools/libxl/Makefile | 2 +- tools/libxl/libxl_internal.h | 33 +++ tools/libxl/libxl_stream_write.c | 536 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 570 insertions(+), 1 deletion(-) create mode 100644 tools/libxl/libxl_stream_write.c diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index ca0ae3e..63e32f7 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -94,7 +94,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \ libxl_dom.o libxl_exec.o libxl_xshelp.o libxl_device.o \ libxl_internal.o libxl_utils.o libxl_uuid.o \ libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o \ - libxl_stream_read.o \ + libxl_stream_read.o libxl_stream_write.o \ libxl_save_callout.o _libxl_save_msgs_callout.o \ libxl_convert_callout.o \ libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y) diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 5482950..82cd792 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -2868,6 +2868,38 @@ typedef void libxl__domain_suspend_cb(libxl__egc*, typedef void libxl__save_device_model_cb(libxl__egc*, libxl__domain_suspend_state*, int rc); +/* State for writing a libxl migration v2 stream */ +typedef struct libxl__stream_write_state libxl__stream_write_state; + +struct libxl__stream_write_state { + /* filled by the user */ + libxl__ao *ao; + int fd; + uint32_t domid; + void (*completion_callback)(libxl__egc *egc, + libxl__domain_suspend_state *dss, + int rc); + /* Private */ + int rc; + int joined_rc; + size_t padding; + bool running; + libxl__datacopier_state dc; +}; + +_hidden void libxl__stream_write_start(libxl__egc *egc, + libxl__stream_write_state *stream); + +_hidden void libxl__stream_write_abort(libxl__egc *egc, + libxl__stream_write_state *stream, + int rc); + +static inline bool libxl__stream_write_inuse( + const libxl__stream_write_state *stream) +{ + return stream->running; +} + typedef struct libxl__logdirty_switch { const char *cmd; const char *cmd_path; @@ -2907,6 +2939,7 @@ struct libxl__domain_suspend_state { /* private for libxl__domain_save_device_model */ libxl__save_device_model_cb *save_dm_callback; libxl__datacopier_state save_dm_datacopier; + libxl__stream_write_state sws; }; diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c new file mode 100644 index 0000000..856d72e --- /dev/null +++ b/tools/libxl/libxl_stream_write.c @@ -0,0 +1,536 @@ +/* + * Copyright (C) 2015 Citrix Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +/* + * Infrastructure for writing a domain to a libxl migration v2 stream. + * + * Entry points from outside: + * - libxl__stream_write_start() + * - Start writing a stream from the start. + * + * In normal operation, there are two tasks running at once; this stream + * processing, and the the libxl-save-helper. check_stream_finished() is used + * to join all the tasks in both success and error cases. + * + * Nomenclature for event callbacks: + * - $FOO_done(): Completion callback for $FOO + * - write_$FOO(): Set up writing a $FOO + * - $BAR_header(): A $BAR record header only + * - $BAR_record(): A complete $BAR record with header and content + * + * The main loop for a plain VM writes: + * - Stream header + * - Libxc record + * - Toolstack record + * - if (hvm), Qemu record + * - End record + */ + +static const uint8_t zero_padding[1U << REC_ALIGN_ORDER] = { 0 }; + +static void stream_success(libxl__egc *egc, + libxl__stream_write_state *stream); +static void stream_failed(libxl__egc *egc, + libxl__stream_write_state *stream, int ret); +static void stream_done(libxl__egc *egc, + libxl__stream_write_state *stream); + +static void check_stream_finished(libxl__egc *egc, + libxl__domain_suspend_state *dcs, + int rc, const char *what); + +/* Event callbacks for plain VM. */ +static void stream_header_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval); +static void libxc_header_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval); +/* libxl__xc_domain_save_done() lives here, event-order wise. */ +static void write_toolstack_record(libxl__egc *egc, + libxl__stream_write_state *stream); +static void toolstack_record_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval); +static void write_emulator_record(libxl__egc *egc, + libxl__stream_write_state *stream); +static void emulator_body_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval); +static void emulator_padding_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval); +static void write_end_record(libxl__egc *egc, + libxl__stream_write_state *stream); +static void end_record_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval); + +void libxl__stream_write_start(libxl__egc *egc, + libxl__stream_write_state *stream) +{ + libxl__datacopier_state *dc = &stream->dc; + STATE_AO_GC(stream->ao); + struct libxl_sr_hdr hdr = { 0 }; + int ret = 0; + + assert(!stream->running); + stream->running = true; + + memset(dc, 0, sizeof(*dc)); + dc->readwhat = ""; + dc->copywhat = "suspend header"; + dc->writewhat = "save/migration stream"; + dc->ao = ao; + dc->readfd = -1; + dc->writefd = stream->fd; + dc->maxsz = INT_MAX; + dc->bytes_to_read = INT_MAX; + dc->callback = stream_header_done; + + ret = libxl__datacopier_start(dc); + if (ret) + goto err; + + hdr.ident = htobe64(RESTORE_STREAM_IDENT); + hdr.version = htobe32(RESTORE_STREAM_VERSION); + hdr.options = htobe32(0); + + libxl__datacopier_prefixdata(egc, dc, &hdr, sizeof(hdr)); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +void libxl__stream_write_abort(libxl__egc *egc, + libxl__stream_write_state *stream, int rc) +{ + stream_failed(egc, stream, rc); +} + +static void stream_success(libxl__egc *egc, libxl__stream_write_state *stream) +{ + stream->rc = 0; + stream->running = false; + + stream_done(egc, stream); +} + +static void stream_failed(libxl__egc *egc, + libxl__stream_write_state *stream, int rc) +{ + assert(rc); + stream->rc = rc; + + if (stream->running) { + stream->running = false; + stream_done(egc, stream); + } +} + +static void stream_done(libxl__egc *egc, + libxl__stream_write_state *stream) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); + + assert(!stream->running); + + check_stream_finished(egc, dss, stream->rc, "stream"); +} + +static void check_stream_finished(libxl__egc *egc, + libxl__domain_suspend_state *dss, + int rc, const char *what) +{ + libxl__stream_write_state *stream = &dss->sws; + STATE_AO_GC(dss->ao); + + LOG(INFO, "Task '%s' joining (rc %d)", what, rc); + + if (rc && !stream->joined_rc) { + bool skip = false; + /* First reported failure from joining tasks. Tear everything down */ + stream->joined_rc = rc; + + if (libxl__stream_write_inuse(&dss->sws)) { + skip = true; + libxl__stream_write_abort(egc, &dss->sws, rc); + } + + if (libxl__save_helper_inuse(&dss->shs)) { + skip = true; + libxl__save_helper_abort(egc, &dss->shs); + } + + /* There is at least one more active task to join - wait for its + callback */ + if ( skip ) + return; + } + + if (libxl__stream_write_inuse(&dss->sws)) + LOG(DEBUG, "stream still in use"); + else if (libxl__save_helper_inuse(&dss->shs)) + LOG(DEBUG, "save/restore still in use"); + else { + LOG(INFO, "Join complete: result %d", stream->joined_rc); + stream->completion_callback(egc, dss, stream->joined_rc); + } +} + +static void stream_header_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); + STATE_AO_GC(stream->ao); + struct libxl_sr_rec_hdr rec = { REC_TYPE_LIBXC_CONTEXT, 0 }; + int ret = 0; + + if (onwrite || errnoval) { + ret = ERROR_FAIL; + goto err; + } + + dc->copywhat = "suspend footer"; + dc->writewhat = "save/migration stream"; + dc->callback = libxc_header_done; + + ret = libxl__datacopier_start(dc); + if (ret) + goto err; + + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +static void libxc_header_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); + STATE_AO_GC(stream->ao); + int ret = 0; + + if (onwrite || errnoval) { + ret = ERROR_FAIL; + goto err; + } + + libxl__xc_domain_save(egc, dss); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +static void __attribute__((used)) +will_be_libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void, + int rc, int retval, int errnoval) +{ + libxl__domain_suspend_state *dss = dss_void; + libxl__stream_write_state *stream = &dss->sws; + STATE_AO_GC(dss->ao); + + if (rc) + goto err; + + if (retval) { + LOGEV(ERROR, errnoval, "saving domain: %s", + dss->guest_responded ? + "domain responded to suspend request" : + "domain did not respond to suspend request"); + if ( !dss->guest_responded ) + rc = ERROR_GUEST_TIMEDOUT; + else + rc = ERROR_FAIL; + goto err; + } + + write_toolstack_record(egc, stream); + return; + + err: + assert(rc); + check_stream_finished(egc, dss, rc, "save/restore helper"); +} + +static void write_toolstack_record(libxl__egc *egc, + libxl__stream_write_state *stream) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); + libxl__datacopier_state *dc = &stream->dc; + STATE_AO_GC(stream->ao); + struct libxl_sr_rec_hdr rec = { REC_TYPE_XENSTORE_DATA, 0 }; + int ret = 0; + uint8_t *toolstack_buf = NULL; /* We must free this. */ + uint32_t toolstack_len, padding_len; + + ret = libxl__toolstack_save(dss->domid, &toolstack_buf, + &toolstack_len, dss); + if (ret) + goto err; + + dc->copywhat = "toolstack record"; + dc->writewhat = "save/migration stream"; + dc->callback = toolstack_record_done; + + ret = libxl__datacopier_start(dc); + if (ret) + goto err; + + rec.length = toolstack_len; + + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); + libxl__datacopier_prefixdata(egc, dc, toolstack_buf, toolstack_len); + + padding_len = ROUNDUP(rec.length, REC_ALIGN_ORDER) - rec.length; + if (padding_len) + libxl__datacopier_prefixdata(egc, dc, zero_padding, padding_len); + + free(toolstack_buf); + return; + + err: + assert(ret); + free(toolstack_buf); + stream_failed(egc, stream, ret); +} + +static void toolstack_record_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); + STATE_AO_GC(stream->ao); + int ret = 0; + + if (onwrite || errnoval) { + ret = ERROR_FAIL; + goto err; + } + + if (dss->type == LIBXL_DOMAIN_TYPE_HVM) + write_emulator_record(egc, stream); + else + write_end_record(egc, stream); + + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +static void write_emulator_record(libxl__egc *egc, + libxl__stream_write_state *stream) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); + libxl__datacopier_state *dc = &stream->dc; + STATE_AO_GC(stream->ao); + struct libxl_sr_rec_hdr rec = { REC_TYPE_EMULATOR_CONTEXT, 0 }; + struct libxl_sr_emulator_hdr ehdr = { 0 }; + struct stat st; + int ret = 0; + uint32_t qemu_state_len; + + assert(dss->type == LIBXL_DOMAIN_TYPE_HVM); + + /* Convenience aliases */ + const char *const filename = dss->dm_savefile; + const uint32_t domid = dss->domid; + + switch(libxl__device_model_version_running(gc, domid)) { + case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: + ehdr.id = EMULATOR_QEMU_TRADITIONAL; + break; + + case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN: + ehdr.id = EMULATOR_QEMU_UPSTREAM; + break; + + default: + ret = ERROR_FAIL; + goto err; + } + + ret = libxl__domain_suspend_device_model(gc, dss); + if (ret) + goto err; + + dc->readwhat = GCSPRINTF("qemu save file %s", filename); + dc->copywhat = "emulator record"; + dc->writewhat = "save/migration stream"; + dc->callback = emulator_body_done; + + dc->readfd = open(filename, O_RDONLY); + if (dc->readfd < 0) { + LOGE(ERROR, "unable to open %s", dc->readwhat); + goto err; + } + + if (fstat(dc->readfd, &st)) + { + LOGE(ERROR, "unable to fstat %s", dc->readwhat); + goto err; + } + + if (!S_ISREG(st.st_mode)) { + LOG(ERROR, "%s is not a plain file!", dc->readwhat); + goto err; + } + + qemu_state_len = st.st_size; + rec.length = qemu_state_len + sizeof(ehdr); + + ret = libxl__datacopier_start(dc); + if (ret) + goto err; + + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); + libxl__datacopier_prefixdata(egc, dc, &ehdr, sizeof(ehdr)); + + stream->padding = ROUNDUP(qemu_state_len, REC_ALIGN_ORDER) - qemu_state_len; + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +static void emulator_body_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); + STATE_AO_GC(stream->ao); + int ret = 0; + + if (onwrite || errnoval) { + ret = ERROR_FAIL; + goto err; + } + + dc->readwhat = ""; + dc->readfd = -1; + + if (stream->padding) { + assert(stream->padding < (1U << REC_ALIGN_ORDER)); + + dc->copywhat = "emulator padding"; + dc->writewhat = "save/migration stream"; + dc->callback = emulator_padding_done; + + ret = libxl__datacopier_start(dc); + if (ret) + goto err; + + libxl__datacopier_prefixdata(egc, dc, zero_padding, stream->padding); + return; + } + + emulator_padding_done(egc, dc, 0, 0); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +static void emulator_padding_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); + STATE_AO_GC(stream->ao); + int ret = 0; + + if (onwrite || errnoval) { + ret = ERROR_FAIL; + goto err; + } + + write_end_record(egc, stream); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +static void write_end_record(libxl__egc *egc, + libxl__stream_write_state *stream) +{ + libxl__datacopier_state *dc = &stream->dc; + STATE_AO_GC(stream->ao); + struct libxl_sr_rec_hdr rec = { REC_TYPE_END, 0 }; + int ret = 0; + + dc->copywhat = "suspend footer"; + dc->writewhat = "save/migration stream"; + dc->callback = end_record_done; + + ret = libxl__datacopier_start(dc); + if (ret) + goto err; + + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +static void end_record_done(libxl__egc *egc, + libxl__datacopier_state *dc, + int onwrite, int errnoval) +{ + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); + STATE_AO_GC(stream->ao); + int ret = 0; + + if (onwrite || errnoval) { + ret = ERROR_FAIL; + goto err; + } + + stream_success(egc, stream); + return; + + err: + assert(ret); + stream_failed(egc, stream, ret); +} + +/* + * Local variables: + * mode: C + * c-basic-offset: 4 + * indent-tabs-mode: nil + * End: + */ -- 1.7.10.4