From mboxrd@z Thu Jan 1 00:00:00 1970 From: Vitaly Kuznetsov Subject: [PATCH v9 11/11] (lib)xl: soft reset support Date: Thu, 16 Jul 2015 18:27:26 +0200 Message-ID: <1437064046-12174-12-git-send-email-vkuznets@redhat.com> References: <1437064046-12174-1-git-send-email-vkuznets@redhat.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: Received: from mail6.bemta5.messagelabs.com ([195.245.231.135]) by lists.xen.org with esmtp (Exim 4.72) (envelope-from ) id 1ZFm17-0004sI-O5 for xen-devel@lists.xenproject.org; Thu, 16 Jul 2015 16:28:14 +0000 In-Reply-To: <1437064046-12174-1-git-send-email-vkuznets@redhat.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: xen-devel@lists.xenproject.org Cc: Andrew Jones , Julien Grall , Keir Fraser , Ian Campbell , Stefano Stabellini , Andrew Cooper , Ian Jackson , Olaf Hering , Tim Deegan , David Vrabel , Jan Beulich , Wei Liu , Daniel De Graaf List-Id: xen-devel@lists.xenproject.org Use existing create/restore path to perform 'soft reset' for HVM domains. Tear everything down, e.g. destroy domain's device model, remove the domain from xenstore, save toolstack record and start over. Signed-off-by: Vitaly Kuznetsov --- Changes since v8: - Pause/unpase the domain when doing xc_domain_soft_reset [Konrad Rzeszutek Wilk] - Redistribution of hunks between this patch and PATCH 3 [Ian Jackson] - Shorten lines to 70-75 chars [Ian Jackson] - Rephrase 'soft-reset' action in xl.cfg.pod.5 [Ian Jackson] Changes since v7 - 'Reset everything approach': XEN_DOMCTL_soft_reset doesn't destroy the original domain any more. - libxl__domain_soft_reset_state introduced [Ian Campbell] - Separate do_domain_soft_reset to not intertwine with do_domain_create [Ian Campbell] Signed-off-by: Vitaly Kuznetsov --- docs/man/xl.cfg.pod.5 | 7 +- tools/libxl/libxl.c | 23 +++++- tools/libxl/libxl.h | 15 ++++ tools/libxl/libxl_create.c | 193 ++++++++++++++++++++++++++++++++++++++----- tools/libxl/libxl_internal.h | 4 + tools/libxl/libxl_types.idl | 2 + tools/libxl/xl.h | 1 + tools/libxl/xl_cmdimpl.c | 25 +++++- 8 files changed, 240 insertions(+), 30 deletions(-) diff --git a/docs/man/xl.cfg.pod.5 b/docs/man/xl.cfg.pod.5 index 3489b27..caed758 100644 --- a/docs/man/xl.cfg.pod.5 +++ b/docs/man/xl.cfg.pod.5 @@ -346,6 +346,11 @@ destroy the domain. write a "coredump" of the domain to F and then restart the domain. +=item B + +reset all Xen specific interfaces for the Xen-aware HVM domain allowing +it to reastablish these interfaces and continue executing the domain. + =back The default for C is C. @@ -367,7 +372,7 @@ Action to take if the domain crashes. Default is C. =item B Action to take if the domain performs 'soft reset' (e.g. does kexec). -Default is C. +Default is C. =back diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 38aff8d..2aa66a3 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -1495,6 +1495,7 @@ void libxl__domain_destroy(libxl__egc *egc, libxl__domain_destroy_state *dds) dds->stubdom.ao = ao; dds->stubdom.domid = stubdomid; dds->stubdom.callback = stubdom_destroy_callback; + dds->stubdom.soft_reset = false; libxl__destroy_domid(egc, &dds->stubdom); } else { dds->stubdom_finished = 1; @@ -1503,6 +1504,7 @@ void libxl__domain_destroy(libxl__egc *egc, libxl__domain_destroy_state *dds) dds->domain.ao = ao; dds->domain.domid = dds->domid; dds->domain.callback = domain_destroy_callback; + dds->domain.soft_reset = dds->soft_reset; libxl__destroy_domid(egc, &dds->domain); } @@ -1683,10 +1685,14 @@ static void devices_destroy_cb(libxl__egc *egc, /* Clean up qemu-save and qemu-resume files. They are * intermediate files created by libxc. Unfortunately they - * don't fit in existing userdata scheme very well. + * don't fit in existing userdata scheme very well. In soft reset + * case we need to keep the file. */ - rc = libxl__remove_file(gc, libxl__device_model_savefile(gc, domid)); - if (rc < 0) goto out; + if (!dis->soft_reset) { + rc = libxl__remove_file(gc, + libxl__device_model_savefile(gc, domid)); + if (rc < 0) goto out; + } rc = libxl__remove_file(gc, GCSPRINTF(XC_DEVICE_MODEL_RESTORE_FILE".%u", domid)); if (rc < 0) goto out; @@ -1697,7 +1703,16 @@ static void devices_destroy_cb(libxl__egc *egc, ctx->xch = xc_interface_open(ctx->lg,0,0); if (!ctx->xch) goto badchild; - rc = xc_domain_destroy(ctx->xch, domid); + if (!dis->soft_reset) { + rc = xc_domain_destroy(ctx->xch, domid); + } + else { + rc = xc_domain_pause(ctx->xch, domid); + if (rc < 0) goto badchild; + rc = xc_domain_soft_reset(ctx->xch, domid); + if (rc < 0) goto badchild; + rc = xc_domain_unpause(ctx->xch, domid); + } if (rc < 0) goto badchild; _exit(0); diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h index e9d63c9..a579383 100644 --- a/tools/libxl/libxl.h +++ b/tools/libxl/libxl.h @@ -205,6 +205,13 @@ #define LIBXL_HAVE_BUILDINFO_ARM_GIC_VERSION 1 /* + * LIBXL_HAVE_SOFT_RESET indicates that libxl supports performing + * 'soft reset' for domains and there is 'soft_reset' shutdown reason + * in enum libxl_shutdown_reason. + */ +#define LIBXL_HAVE_SOFT_RESET 1 + +/* * libxl ABI compatibility * * The only guarantee which libxl makes regarding ABI compatibility @@ -1101,6 +1108,14 @@ int static inline libxl_domain_create_restore_0x040200( #endif +int libxl_domain_soft_reset(libxl_ctx *ctx, + libxl_domain_config *d_config, + uint32_t domid, + const libxl_asyncop_how *ao_how, + const libxl_asyncprogress_how + *aop_console_how) + LIBXL_EXTERNAL_CALLERS_ONLY; + /* A progress report will be made via ao_console_how, of type * domain_create_console_available, when the domain's primary * console is available and can be connected to. diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c index b785ddd..2e66e00 100644 --- a/tools/libxl/libxl_create.c +++ b/tools/libxl/libxl_create.c @@ -484,8 +484,6 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_config *d_config, /* convenience aliases */ libxl_domain_create_info *info = &d_config->c_info; - assert(!libxl_domid_valid_guest(*domid)); - uuid_string = libxl__uuid2string(gc, info->uuid); if (!uuid_string) { rc = ERROR_NOMEM; @@ -506,7 +504,6 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_config *d_config, } flags |= XEN_DOMCTL_CDF_hap; } - *domid = -1; /* Ultimately, handle is an array of 16 uint8_t, same as uuid */ libxl_uuid_copy(ctx, (libxl_uuid *)handle, &info->uuid); @@ -518,13 +515,17 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_config *d_config, goto out; } - ret = xc_domain_create_config(ctx->xch, info->ssidref, - handle, flags, domid, - xc_config); - if (ret < 0) { - LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "domain creation fail"); - rc = ERROR_FAIL; - goto out; + /* Valid domid here means we're soft resetting */ + if (!libxl_domid_valid_guest(*domid)) { + ret = xc_domain_create_config(ctx->xch, info->ssidref, + handle, flags, domid, + xc_config); + if (ret < 0) { + LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, + "domain creation fail"); + rc = ERROR_FAIL; + goto out; + } } rc = libxl__arch_domain_save_config(gc, d_config, xc_config); @@ -734,9 +735,8 @@ static void initiate_domain_create(libxl__egc *egc, libxl_domain_config *const d_config = dcs->guest_config; libxl__domain_build_state *const state = &dcs->build_state; const int restore_fd = dcs->restore_fd; - memset(&dcs->build_state, 0, sizeof(dcs->build_state)); - domid = 0; + domid = dcs->domid_soft_reset; if (d_config->c_info.ssid_label) { char *s = d_config->c_info.ssid_label; @@ -884,7 +884,7 @@ static void initiate_domain_create(libxl__egc *egc, d_config->nics[i].devid = ++last_devid; } - if (restore_fd >= 0) { + if (restore_fd >= 0 || dcs->domid_soft_reset != INVALID_DOMID) { LOG(DEBUG, "restoring, not running bootloader"); domcreate_bootloader_done(egc, &dcs->bl, 0); } else { @@ -958,7 +958,7 @@ static void domcreate_bootloader_done(libxl__egc *egc, dcs->dmss.dm.callback = domcreate_devmodel_started; dcs->dmss.callback = domcreate_devmodel_started; - if ( restore_fd < 0 ) { + if ( restore_fd < 0 && dcs->domid_soft_reset == INVALID_DOMID) { rc = libxl__domain_build(gc, d_config, domid, state); domcreate_rebuild_done(egc, dcs, rc); return; @@ -988,9 +988,11 @@ static void domcreate_bootloader_done(libxl__egc *egc, rc = ERROR_INVAL; goto out; } - libxl__xc_domain_restore(egc, dcs, - hvm, pae, superpages); - return; + if ( restore_fd >= 0 ) { + libxl__xc_domain_restore(egc, dcs, + hvm, pae, superpages); + return; + } out: libxl__xc_domain_restore_done(egc, dcs, rc, 0, 0); @@ -1074,8 +1076,12 @@ void libxl__xc_domain_restore_done(libxl__egc *egc, void *dcs_void, goto out; if (info->type == LIBXL_DOMAIN_TYPE_HVM) { - state->saved_state = GCSPRINTF( - XC_DEVICE_MODEL_RESTORE_FILE".%d", domid); + if (fd != -1) + state->saved_state = GCSPRINTF( + XC_DEVICE_MODEL_RESTORE_FILE".%d", domid); + else + state->saved_state = GCSPRINTF( + XC_DEVICE_MODEL_SAVE_FILE".%d", domid); } out: @@ -1084,9 +1090,12 @@ out: libxl__file_reference_unmap(&state->pv_ramdisk); } - esave = errno; - libxl_fd_set_nonblock(ctx, fd, 0); - errno = esave; + /* fd == -1 here means we're doing soft reset */ + if (fd != -1) { + esave = errno; + libxl_fd_set_nonblock(ctx, fd, 0); + errno = esave; + } domcreate_rebuild_done(egc, dcs, ret); } @@ -1507,6 +1516,14 @@ typedef struct { uint32_t *domid_out; } libxl__app_domain_create_state; +typedef struct { + libxl__app_domain_create_state cdcs; + libxl__domain_destroy_state dds; + libxl__domain_suspend_state dss; + uint8_t *toolstack_buf; + uint32_t toolstack_len; +} libxl__domain_soft_reset_state; + static void domain_create_cb(libxl__egc *egc, libxl__domain_create_state *dcs, int rc, uint32_t domid); @@ -1528,6 +1545,7 @@ static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config, cdcs->dcs.restore_fd = restore_fd; cdcs->dcs.callback = domain_create_cb; cdcs->dcs.checkpointed_stream = checkpointed_stream; + cdcs->dcs.domid_soft_reset = INVALID_DOMID; libxl__ao_progress_gethow(&cdcs->dcs.aop_console_how, aop_console_how); cdcs->domid_out = domid; @@ -1536,6 +1554,122 @@ static int do_domain_create(libxl_ctx *ctx, libxl_domain_config *d_config, return AO_INPROGRESS; } +static void domain_soft_reset_cb(libxl__egc *egc, + libxl__domain_destroy_state *dds, + int rc) +{ + STATE_AO_GC(dds->ao); + libxl__domain_soft_reset_state *srs = CONTAINER_OF(dds, *srs, dds); + libxl__app_domain_create_state *cdcs = &srs->cdcs; + + if (rc) { + LOG(ERROR, "destruction of domain %u failed", dds->domid); + goto error; + } + + rc = libxl__toolstack_restore(cdcs->dcs.domid_soft_reset, + srs->toolstack_buf, + srs->toolstack_len, + &cdcs->dcs.shs); + if (rc) { + LOG(ERROR, "failed to restore toolstack record"); + goto error; + } + + initiate_domain_create(egc, &cdcs->dcs); + return; + +error: + domcreate_complete(egc, &cdcs->dcs, rc); +} + +static int do_domain_soft_reset(libxl_ctx *ctx, + libxl_domain_config *d_config, + uint32_t domid_soft_reset, + const libxl_asyncop_how *ao_how, + const libxl_asyncprogress_how + *aop_console_how) +{ + AO_CREATE(ctx, 0, ao_how); + libxl__domain_soft_reset_state *srs; + libxl__app_domain_create_state *cdcs; + libxl__domain_create_state *dcs; + libxl__domain_build_state *state; + libxl__domain_suspend_state *dss; + char *dom_path, *xs_store_mfn, *xs_console_mfn; + uint32_t domid_out; + int rc; + + GCNEW(srs); + cdcs = &srs->cdcs; + dcs = &cdcs->dcs; + state = &dcs->build_state; + dss = &srs->dss; + + srs->cdcs.dcs.ao = ao; + srs->cdcs.dcs.guest_config = d_config; + libxl_domain_config_init(&srs->cdcs.dcs.guest_config_saved); + libxl_domain_config_copy(ctx, &srs->cdcs.dcs.guest_config_saved, + d_config); + cdcs->dcs.restore_fd = -1; + cdcs->dcs.domid_soft_reset = domid_soft_reset; + cdcs->dcs.callback = domain_create_cb; + libxl__ao_progress_gethow(&srs->cdcs.dcs.aop_console_how, + aop_console_how); + cdcs->domid_out = &domid_out; + + dom_path = libxl__xs_get_dompath(gc, domid_soft_reset); + if (!dom_path) { + LOG(ERROR, "failed to read domain path"); + return AO_CREATE_FAIL(ERROR_FAIL); + } + + xs_store_mfn = xs_read(ctx->xsh, XBT_NULL, + GCSPRINTF("%s/store/ring-ref", dom_path), + NULL); + state->store_mfn = xs_store_mfn ? atol(xs_store_mfn): 0; + free(xs_store_mfn); + + xs_console_mfn = xs_read(ctx->xsh, XBT_NULL, + GCSPRINTF("%s/console/ring-ref", dom_path), + NULL); + state->console_mfn = xs_console_mfn ? atol(xs_console_mfn): 0; + free(xs_console_mfn); + + dss->ao = ao; + dss->domid = domid_soft_reset; + dss->dm_savefile = GCSPRINTF(XC_DEVICE_MODEL_SAVE_FILE".%d", + domid_soft_reset); + + rc = libxl__toolstack_save(domid_soft_reset, &srs->toolstack_buf, + &srs->toolstack_len, dss); + if (rc) { + LOG(ERROR, "failed to save toolstack record"); + return AO_CREATE_FAIL(ERROR_FAIL); + } + + rc = libxl__domain_suspend_device_model(gc, dss); + if (rc) { + LOG(ERROR, "failed to suspend device model"); + return AO_CREATE_FAIL(ERROR_FAIL); + } + + /* + * On the domain creation path it will be introduced to xenstore + * with (probably) different store/console channels so we need to + * release it here. + */ + xs_release_domain(ctx->xsh, cdcs->dcs.domid_soft_reset); + + srs->dds.ao = ao; + srs->dds.domid = domid_soft_reset; + srs->dds.callback = domain_soft_reset_cb; + srs->dds.soft_reset = true; + libxl__domain_destroy(egc, &srs->dds); + + return AO_INPROGRESS; +} + static void domain_create_cb(libxl__egc *egc, libxl__domain_create_state *dcs, int rc, uint32_t domid) @@ -1567,6 +1701,21 @@ int libxl_domain_create_restore(libxl_ctx *ctx, libxl_domain_config *d_config, params->checkpointed_stream, ao_how, aop_console_how); } +int libxl_domain_soft_reset(libxl_ctx *ctx, + libxl_domain_config *d_config, + uint32_t domid, + const libxl_asyncop_how *ao_how, + const libxl_asyncprogress_how + *aop_console_how) +{ + libxl_domain_build_info *const info = &d_config->b_info; + + if (info->type != LIBXL_DOMAIN_TYPE_HVM) return ERROR_INVAL; + + return do_domain_soft_reset(ctx, d_config, domid, ao_how, + aop_console_how); +} + /* * Local variables: * mode: C diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 5235d25..a648734 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -107,6 +107,7 @@ #define STUBDOM_SPECIAL_CONSOLES 3 #define TAP_DEVICE_SUFFIX "-emu" #define DOMID_XS_PATH "domid" +#define INVALID_DOMID ~0 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(a[0])) @@ -3098,6 +3099,7 @@ struct libxl__destroy_domid_state { /* private to implementation */ libxl__devices_remove_state drs; libxl__ev_child destroyer; + bool soft_reset; }; struct libxl__domain_destroy_state { @@ -3112,6 +3114,7 @@ struct libxl__domain_destroy_state { int stubdom_finished; libxl__destroy_domid_state domain; int domain_finished; + bool soft_reset; }; /* @@ -3212,6 +3215,7 @@ struct libxl__domain_create_state { libxl_domain_config *guest_config; libxl_domain_config guest_config_saved; /* vanilla config */ int restore_fd; + uint32_t domid_soft_reset; libxl__domain_create_cb *callback; libxl_asyncprogress_how aop_console_how; /* private to domain_create */ diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index 337257a..f887b28 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -127,6 +127,8 @@ libxl_action_on_shutdown = Enumeration("action_on_shutdown", [ (5, "COREDUMP_DESTROY"), (6, "COREDUMP_RESTART"), + + (7, "SOFT_RESET"), ], init_val = "LIBXL_ACTION_ON_SHUTDOWN_DESTROY") libxl_trigger = Enumeration("trigger", [ diff --git a/tools/libxl/xl.h b/tools/libxl/xl.h index 6c19c0d..0021112 100644 --- a/tools/libxl/xl.h +++ b/tools/libxl/xl.h @@ -194,6 +194,7 @@ typedef enum { DOMAIN_RESTART_NONE = 0, /* No domain restart */ DOMAIN_RESTART_NORMAL, /* Domain should be restarted */ DOMAIN_RESTART_RENAME, /* Domain should be renamed and restarted */ + DOMAIN_RESTART_SOFT_RESET, /* Soft reset should be performed */ } domain_restart_type; extern void printf_info_sexp(int domid, libxl_domain_config *d_config, FILE *fh); diff --git a/tools/libxl/xl_cmdimpl.c b/tools/libxl/xl_cmdimpl.c index f1604b1..97a3aac 100644 --- a/tools/libxl/xl_cmdimpl.c +++ b/tools/libxl/xl_cmdimpl.c @@ -130,6 +130,8 @@ static const char *action_on_shutdown_names[] = { [LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_DESTROY] = "coredump-destroy", [LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_RESTART] = "coredump-restart", + + [LIBXL_ACTION_ON_SHUTDOWN_SOFT_RESET] = "soft-reset", }; /* Optional data, in order: @@ -1338,7 +1340,7 @@ static void parse_config_data(const char *config_source, } if (xlu_cfg_get_string (config, "on_soft_reset", &buf, 0)) - buf = "restart"; + buf = "soft-reset"; if (!parse_action_on_shutdown(buf, &d_config->on_soft_reset)) { fprintf(stderr, "Unknown on_soft_reset action \"%s\" specified\n", buf); exit(1); @@ -2391,6 +2393,11 @@ static domain_restart_type handle_domain_death(uint32_t *r_domid, *r_domid = INVALID_DOMID; break; + case LIBXL_ACTION_ON_SHUTDOWN_SOFT_RESET: + reload_domain_config(*r_domid, d_config); + restart = DOMAIN_RESTART_SOFT_RESET; + break; + case LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_DESTROY: case LIBXL_ACTION_ON_SHUTDOWN_COREDUMP_RESTART: /* Already handled these above. */ @@ -2566,6 +2573,7 @@ static uint32_t create_domain(struct domain_create *dom_info) int restore_fd = -1; const libxl_asyncprogress_how *autoconnect_console_how; struct save_file_header hdr; + uint32_t domid_soft_reset = INVALID_DOMID; int restoring = (restore_file || (migrate_fd >= 0)); @@ -2775,7 +2783,13 @@ start: * restore/migrate-receive it again. */ restoring = 0; - }else{ + } else if ( domid_soft_reset != INVALID_DOMID ) { + /* Do soft reset */ + ret = libxl_domain_soft_reset(ctx, &d_config, domid_soft_reset, + 0, autoconnect_console_how); + domid = domid_soft_reset; + domid_soft_reset = INVALID_DOMID; + } else { ret = libxl_domain_create_new(ctx, &d_config, &domid, 0, autoconnect_console_how); } @@ -2839,8 +2853,13 @@ start: event->u.domain_shutdown.shutdown_reason, event->u.domain_shutdown.shutdown_reason); switch (handle_domain_death(&domid, event, &d_config)) { + case DOMAIN_RESTART_SOFT_RESET: + domid_soft_reset = domid; + domid = INVALID_DOMID; + /* fall through */ case DOMAIN_RESTART_RENAME: - if (!preserve_domain(&domid, event, &d_config)) { + if (domid_soft_reset == INVALID_DOMID && + !preserve_domain(&domid, event, &d_config)) { /* If we fail then exit leaving the old domain in place. */ ret = -1; goto out; -- 2.4.3