From mboxrd@z Thu Jan 1 00:00:00 1970 From: Wen Congyang Subject: Re: [PATCH 20/27] tools/libxl: Infrastructure for writing a v2 stream Date: Wed, 17 Jun 2015 10:24:23 +0800 Message-ID: <5580DA57.9060300@cn.fujitsu.com> References: <1434375880-30914-1-git-send-email-andrew.cooper3@citrix.com> <1434375880-30914-21-git-send-email-andrew.cooper3@citrix.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1434375880-30914-21-git-send-email-andrew.cooper3@citrix.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: Andrew Cooper , Xen-devel Cc: Ian Jackson , Yang Hongyang , Wei Liu , Ian Campbell , Ross Lagerwall List-Id: xen-devel@lists.xenproject.org On 06/15/2015 09:44 PM, Andrew Cooper wrote: > From: Ross Lagerwall > > Signed-off-by: Ross Lagerwall > Signed-off-by: Andrew Cooper > CC: Ian Campbell > CC: Ian Jackson > CC: Wei Liu > --- > tools/libxl/Makefile | 2 +- > tools/libxl/libxl_internal.h | 33 +++ > tools/libxl/libxl_stream_write.c | 536 ++++++++++++++++++++++++++++++++++++++ > 3 files changed, 570 insertions(+), 1 deletion(-) > create mode 100644 tools/libxl/libxl_stream_write.c > > diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile > index ca0ae3e..63e32f7 100644 > --- a/tools/libxl/Makefile > +++ b/tools/libxl/Makefile > @@ -94,7 +94,7 @@ LIBXL_OBJS = flexarray.o libxl.o libxl_create.o libxl_dm.o libxl_pci.o \ > libxl_dom.o libxl_exec.o libxl_xshelp.o libxl_device.o \ > libxl_internal.o libxl_utils.o libxl_uuid.o \ > libxl_json.o libxl_aoutils.o libxl_numa.o libxl_vnuma.o \ > - libxl_stream_read.o \ > + libxl_stream_read.o libxl_stream_write.o \ > libxl_save_callout.o _libxl_save_msgs_callout.o \ > libxl_convert_callout.o \ > libxl_qmp.o libxl_event.o libxl_fork.o $(LIBXL_OBJS-y) > diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h > index 5482950..82cd792 100644 > --- a/tools/libxl/libxl_internal.h > +++ b/tools/libxl/libxl_internal.h > @@ -2868,6 +2868,38 @@ typedef void libxl__domain_suspend_cb(libxl__egc*, > typedef void libxl__save_device_model_cb(libxl__egc*, > libxl__domain_suspend_state*, int rc); > > +/* State for writing a libxl migration v2 stream */ > +typedef struct libxl__stream_write_state libxl__stream_write_state; > + > +struct libxl__stream_write_state { > + /* filled by the user */ > + libxl__ao *ao; > + int fd; > + uint32_t domid; > + void (*completion_callback)(libxl__egc *egc, > + libxl__domain_suspend_state *dss, > + int rc); > + /* Private */ > + int rc; > + int joined_rc; > + size_t padding; > + bool running; > + libxl__datacopier_state dc; > +}; > + > +_hidden void libxl__stream_write_start(libxl__egc *egc, > + libxl__stream_write_state *stream); > + > +_hidden void libxl__stream_write_abort(libxl__egc *egc, > + libxl__stream_write_state *stream, > + int rc); > + > +static inline bool libxl__stream_write_inuse( > + const libxl__stream_write_state *stream) > +{ > + return stream->running; > +} > + > typedef struct libxl__logdirty_switch { > const char *cmd; > const char *cmd_path; > @@ -2907,6 +2939,7 @@ struct libxl__domain_suspend_state { > /* private for libxl__domain_save_device_model */ > libxl__save_device_model_cb *save_dm_callback; > libxl__datacopier_state save_dm_datacopier; > + libxl__stream_write_state sws; > }; > > > diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c > new file mode 100644 > index 0000000..856d72e > --- /dev/null > +++ b/tools/libxl/libxl_stream_write.c > @@ -0,0 +1,536 @@ > +/* > + * Copyright (C) 2015 Citrix Ltd. > + * > + * This program is free software; you can redistribute it and/or modify > + * it under the terms of the GNU Lesser General Public License as published > + * by the Free Software Foundation; version 2.1 only. with the special > + * exception on linking described in file LICENSE. > + * > + * This program is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > + * GNU Lesser General Public License for more details. > + */ > + > +#include "libxl_osdeps.h" /* must come before any other headers */ > + > +#include "libxl_internal.h" > + > +/* > + * Infrastructure for writing a domain to a libxl migration v2 stream. > + * > + * Entry points from outside: > + * - libxl__stream_write_start() > + * - Start writing a stream from the start. > + * > + * In normal operation, there are two tasks running at once; this stream > + * processing, and the the libxl-save-helper. check_stream_finished() is used > + * to join all the tasks in both success and error cases. > + * > + * Nomenclature for event callbacks: > + * - $FOO_done(): Completion callback for $FOO > + * - write_$FOO(): Set up writing a $FOO > + * - $BAR_header(): A $BAR record header only > + * - $BAR_record(): A complete $BAR record with header and content > + * > + * The main loop for a plain VM writes: > + * - Stream header > + * - Libxc record > + * - Toolstack record > + * - if (hvm), Qemu record > + * - End record > + */ > + > +static const uint8_t zero_padding[1U << REC_ALIGN_ORDER] = { 0 }; > + > +static void stream_success(libxl__egc *egc, > + libxl__stream_write_state *stream); > +static void stream_failed(libxl__egc *egc, > + libxl__stream_write_state *stream, int ret); > +static void stream_done(libxl__egc *egc, > + libxl__stream_write_state *stream); > + > +static void check_stream_finished(libxl__egc *egc, > + libxl__domain_suspend_state *dcs, > + int rc, const char *what); > + > +/* Event callbacks for plain VM. */ > +static void stream_header_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval); > +static void libxc_header_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval); > +/* libxl__xc_domain_save_done() lives here, event-order wise. */ > +static void write_toolstack_record(libxl__egc *egc, > + libxl__stream_write_state *stream); > +static void toolstack_record_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval); > +static void write_emulator_record(libxl__egc *egc, > + libxl__stream_write_state *stream); > +static void emulator_body_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval); > +static void emulator_padding_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval); > +static void write_end_record(libxl__egc *egc, > + libxl__stream_write_state *stream); > +static void end_record_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval); > + > +void libxl__stream_write_start(libxl__egc *egc, > + libxl__stream_write_state *stream) > +{ > + libxl__datacopier_state *dc = &stream->dc; > + STATE_AO_GC(stream->ao); > + struct libxl_sr_hdr hdr = { 0 }; > + int ret = 0; > + > + assert(!stream->running); > + stream->running = true; > + > + memset(dc, 0, sizeof(*dc)); > + dc->readwhat = ""; > + dc->copywhat = "suspend header"; > + dc->writewhat = "save/migration stream"; > + dc->ao = ao; > + dc->readfd = -1; > + dc->writefd = stream->fd; > + dc->maxsz = INT_MAX; > + dc->bytes_to_read = INT_MAX; > + dc->callback = stream_header_done; > + > + ret = libxl__datacopier_start(dc); > + if (ret) > + goto err; > + > + hdr.ident = htobe64(RESTORE_STREAM_IDENT); > + hdr.version = htobe32(RESTORE_STREAM_VERSION); > + hdr.options = htobe32(0); > + > + libxl__datacopier_prefixdata(egc, dc, &hdr, sizeof(hdr)); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +void libxl__stream_write_abort(libxl__egc *egc, > + libxl__stream_write_state *stream, int rc) > +{ > + stream_failed(egc, stream, rc); > +} > + > +static void stream_success(libxl__egc *egc, libxl__stream_write_state *stream) > +{ > + stream->rc = 0; > + stream->running = false; > + > + stream_done(egc, stream); > +} > + > +static void stream_failed(libxl__egc *egc, > + libxl__stream_write_state *stream, int rc) > +{ > + assert(rc); > + stream->rc = rc; > + > + if (stream->running) { > + stream->running = false; > + stream_done(egc, stream); > + } > +} > + > +static void stream_done(libxl__egc *egc, > + libxl__stream_write_state *stream) > +{ > + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); > + > + assert(!stream->running); > + > + check_stream_finished(egc, dss, stream->rc, "stream"); > +} > + > +static void check_stream_finished(libxl__egc *egc, > + libxl__domain_suspend_state *dss, > + int rc, const char *what) > +{ > + libxl__stream_write_state *stream = &dss->sws; > + STATE_AO_GC(dss->ao); > + > + LOG(INFO, "Task '%s' joining (rc %d)", what, rc); > + > + if (rc && !stream->joined_rc) { > + bool skip = false; > + /* First reported failure from joining tasks. Tear everything down */ > + stream->joined_rc = rc; > + > + if (libxl__stream_write_inuse(&dss->sws)) { > + skip = true; > + libxl__stream_write_abort(egc, &dss->sws, rc); > + } > + > + if (libxl__save_helper_inuse(&dss->shs)) { > + skip = true; > + libxl__save_helper_abort(egc, &dss->shs); > + } > + > + /* There is at least one more active task to join - wait for its > + callback */ > + if ( skip ) > + return; > + } > + > + if (libxl__stream_write_inuse(&dss->sws)) > + LOG(DEBUG, "stream still in use"); > + else if (libxl__save_helper_inuse(&dss->shs)) > + LOG(DEBUG, "save/restore still in use"); > + else { > + LOG(INFO, "Join complete: result %d", stream->joined_rc); > + stream->completion_callback(egc, dss, stream->joined_rc); > + } > +} > + > +static void stream_header_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval) > +{ > + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); > + STATE_AO_GC(stream->ao); > + struct libxl_sr_rec_hdr rec = { REC_TYPE_LIBXC_CONTEXT, 0 }; > + int ret = 0; > + > + if (onwrite || errnoval) { > + ret = ERROR_FAIL; > + goto err; > + } > + > + dc->copywhat = "suspend footer"; > + dc->writewhat = "save/migration stream"; > + dc->callback = libxc_header_done; > + > + ret = libxl__datacopier_start(dc); > + if (ret) > + goto err; > + > + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +static void libxc_header_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval) > +{ > + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); > + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); > + STATE_AO_GC(stream->ao); > + int ret = 0; > + > + if (onwrite || errnoval) { > + ret = ERROR_FAIL; > + goto err; > + } > + > + libxl__xc_domain_save(egc, dss); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +static void __attribute__((used)) > +will_be_libxl__xc_domain_save_done(libxl__egc *egc, void *dss_void, > + int rc, int retval, int errnoval) > +{ > + libxl__domain_suspend_state *dss = dss_void; > + libxl__stream_write_state *stream = &dss->sws; > + STATE_AO_GC(dss->ao); > + > + if (rc) > + goto err; > + > + if (retval) { > + LOGEV(ERROR, errnoval, "saving domain: %s", > + dss->guest_responded ? > + "domain responded to suspend request" : > + "domain did not respond to suspend request"); > + if ( !dss->guest_responded ) > + rc = ERROR_GUEST_TIMEDOUT; > + else > + rc = ERROR_FAIL; > + goto err; > + } > + > + write_toolstack_record(egc, stream); > + return; > + > + err: > + assert(rc); > + check_stream_finished(egc, dss, rc, "save/restore helper"); > +} > + > +static void write_toolstack_record(libxl__egc *egc, > + libxl__stream_write_state *stream) > +{ > + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); > + libxl__datacopier_state *dc = &stream->dc; > + STATE_AO_GC(stream->ao); > + struct libxl_sr_rec_hdr rec = { REC_TYPE_XENSTORE_DATA, 0 }; > + int ret = 0; > + uint8_t *toolstack_buf = NULL; /* We must free this. */ > + uint32_t toolstack_len, padding_len; > + > + ret = libxl__toolstack_save(dss->domid, &toolstack_buf, > + &toolstack_len, dss); > + if (ret) > + goto err; > + > + dc->copywhat = "toolstack record"; > + dc->writewhat = "save/migration stream"; > + dc->callback = toolstack_record_done; > + > + ret = libxl__datacopier_start(dc); > + if (ret) > + goto err; > + > + rec.length = toolstack_len; > + > + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); > + libxl__datacopier_prefixdata(egc, dc, toolstack_buf, toolstack_len); > + > + padding_len = ROUNDUP(rec.length, REC_ALIGN_ORDER) - rec.length; > + if (padding_len) > + libxl__datacopier_prefixdata(egc, dc, zero_padding, padding_len); > + > + free(toolstack_buf); > + return; > + > + err: > + assert(ret); > + free(toolstack_buf); > + stream_failed(egc, stream, ret); > +} > + > +static void toolstack_record_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval) > +{ > + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); > + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); > + STATE_AO_GC(stream->ao); > + int ret = 0; > + > + if (onwrite || errnoval) { > + ret = ERROR_FAIL; > + goto err; > + } > + > + if (dss->type == LIBXL_DOMAIN_TYPE_HVM) > + write_emulator_record(egc, stream); > + else > + write_end_record(egc, stream); > + > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +static void write_emulator_record(libxl__egc *egc, > + libxl__stream_write_state *stream) > +{ > + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); > + libxl__datacopier_state *dc = &stream->dc; > + STATE_AO_GC(stream->ao); > + struct libxl_sr_rec_hdr rec = { REC_TYPE_EMULATOR_CONTEXT, 0 }; > + struct libxl_sr_emulator_hdr ehdr = { 0 }; > + struct stat st; > + int ret = 0; > + uint32_t qemu_state_len; > + > + assert(dss->type == LIBXL_DOMAIN_TYPE_HVM); > + > + /* Convenience aliases */ > + const char *const filename = dss->dm_savefile; > + const uint32_t domid = dss->domid; > + > + switch(libxl__device_model_version_running(gc, domid)) { > + case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN_TRADITIONAL: > + ehdr.id = EMULATOR_QEMU_TRADITIONAL; > + break; > + > + case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN: > + ehdr.id = EMULATOR_QEMU_UPSTREAM; > + break; > + > + default: > + ret = ERROR_FAIL; > + goto err; > + } > + > + ret = libxl__domain_suspend_device_model(gc, dss); > + if (ret) > + goto err; > + > + dc->readwhat = GCSPRINTF("qemu save file %s", filename); > + dc->copywhat = "emulator record"; > + dc->writewhat = "save/migration stream"; > + dc->callback = emulator_body_done; > + > + dc->readfd = open(filename, O_RDONLY); > + if (dc->readfd < 0) { > + LOGE(ERROR, "unable to open %s", dc->readwhat); > + goto err; > + } > + > + if (fstat(dc->readfd, &st)) > + { > + LOGE(ERROR, "unable to fstat %s", dc->readwhat); > + goto err; > + } > + > + if (!S_ISREG(st.st_mode)) { > + LOG(ERROR, "%s is not a plain file!", dc->readwhat); > + goto err; > + } > + > + qemu_state_len = st.st_size; > + rec.length = qemu_state_len + sizeof(ehdr); > + > + ret = libxl__datacopier_start(dc); > + if (ret) > + goto err; > + > + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); > + libxl__datacopier_prefixdata(egc, dc, &ehdr, sizeof(ehdr)); > + > + stream->padding = ROUNDUP(qemu_state_len, REC_ALIGN_ORDER) - qemu_state_len; > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +static void emulator_body_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval) > +{ > + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); > + STATE_AO_GC(stream->ao); > + int ret = 0; I think you forget to close dc->readfd here. Thanks Wen Congyang > + > + if (onwrite || errnoval) { > + ret = ERROR_FAIL; > + goto err; > + } > + > + dc->readwhat = ""; > + dc->readfd = -1; > + > + if (stream->padding) { > + assert(stream->padding < (1U << REC_ALIGN_ORDER)); > + > + dc->copywhat = "emulator padding"; > + dc->writewhat = "save/migration stream"; > + dc->callback = emulator_padding_done; > + > + ret = libxl__datacopier_start(dc); > + if (ret) > + goto err; > + > + libxl__datacopier_prefixdata(egc, dc, zero_padding, stream->padding); > + return; > + } > + > + emulator_padding_done(egc, dc, 0, 0); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +static void emulator_padding_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval) > +{ > + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); > + STATE_AO_GC(stream->ao); > + int ret = 0; > + > + if (onwrite || errnoval) { > + ret = ERROR_FAIL; > + goto err; > + } > + > + write_end_record(egc, stream); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +static void write_end_record(libxl__egc *egc, > + libxl__stream_write_state *stream) > +{ > + libxl__datacopier_state *dc = &stream->dc; > + STATE_AO_GC(stream->ao); > + struct libxl_sr_rec_hdr rec = { REC_TYPE_END, 0 }; > + int ret = 0; > + > + dc->copywhat = "suspend footer"; > + dc->writewhat = "save/migration stream"; > + dc->callback = end_record_done; > + > + ret = libxl__datacopier_start(dc); > + if (ret) > + goto err; > + > + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +static void end_record_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval) > +{ > + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); > + STATE_AO_GC(stream->ao); > + int ret = 0; > + > + if (onwrite || errnoval) { > + ret = ERROR_FAIL; > + goto err; > + } > + > + stream_success(egc, stream); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +/* > + * Local variables: > + * mode: C > + * c-basic-offset: 4 > + * indent-tabs-mode: nil > + * End: > + */ >