From mboxrd@z Thu Jan 1 00:00:00 1970 From: Yang Hongyang Subject: [PATCH for-4.5 v21 06/14] libxl/remus: introduce an abstract Remus device layer Date: Fri, 26 Sep 2014 14:13:11 +0800 Message-ID: <1411711999-3183-7-git-send-email-yanghy@cn.fujitsu.com> References: <1411711999-3183-1-git-send-email-yanghy@cn.fujitsu.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1411711999-3183-1-git-send-email-yanghy@cn.fujitsu.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: xen-devel@lists.xen.org Cc: ian.campbell@citrix.com, wency@cn.fujitsu.com, ian.jackson@eu.citrix.com, yunhong.jiang@intel.com, eddie.dong@intel.com, rshriram@cs.ubc.ca, laijs@cn.fujitsu.com List-Id: xen-devel@lists.xenproject.org Introduce an abstract device layer that allows the Remus logic in libxl to control a guest's devices in a device-agnostic manner. The device layer also exposes a set of internal interfaces that a device type must implement, if it wishes to support Remus. The following API are exposed to libxl: One-time configuration operations: *libxl__remus_devices_setup > Enable output buffering for NICs, setup disk replication, etc. *libxl__remus_devices_teardown > Disable network output buffering and disk replication; teardown any associated external setups like qdiscs for NICs. Operations executed every checkpoint (in order of invocation): *libxl__remus_devices_postsuspend *libxl__remus_devices_preresume *libxl__remus_devices_commit Each device type needs to implement the interfaces specified in the libxl__remus_device_instance_ops if it wishes to support Remus. The high-level control flow through the Remus device layer is shown below: xl remus |-> libxl_domain_remus_start |-> libxl__remus_devices_setup |-> Per-checkpoint libxl__remus_devices_[postsuspend,preresume,commit] ... |-> On backup failure/network error/other errors libxl__remus_devices_teardown callback processing * Only call the per-device libxl__multidev_one_callback when the iteration has succeded or failed. * The final callback (called by multidev) is a trivial shim to shuffle the pointers and notify our own caller. Signed-off-by: Yang Hongyang Signed-off-by: Ian Jackson Signed-off-by: Wen Congyang Signed-off-by: Lai Jiangshan For comments: Signed-off-by: Shriram Rajagopalan --- tools/libxl/Makefile | 2 + tools/libxl/libxl.c | 46 +++++- tools/libxl/libxl_dom.c | 168 ++++++++++++++++++++-- tools/libxl/libxl_internal.h | 162 +++++++++++++++++++++ tools/libxl/libxl_remus_device.c | 304 +++++++++++++++++++++++++++++++++++++++ tools/libxl/libxl_types.idl | 2 + 6 files changed, 667 insertions(+), 17 deletions(-) create mode 100644 tools/libxl/libxl_remus_device.c diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile index 58f9975..da3cddb 100644 --- a/tools/libxl/Makefile +++ b/tools/libxl/Makefile @@ -56,6 +56,8 @@ else LIBXL_OBJS-y += libxl_nonetbuffer.o endif +LIBXL_OBJS-y += libxl_remus_device.o + LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c index 3735f55..e108e40 100644 --- a/tools/libxl/libxl.c +++ b/tools/libxl/libxl.c @@ -782,6 +782,10 @@ out: return ptr; } +static void libxl__remus_setup_done(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc); +static void libxl__remus_setup_failed(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc); static void remus_failover_cb(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); @@ -813,16 +817,50 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info, assert(info); - /* TBD: Remus setup - i.e. attach qdisc, enable disk buffering, etc */ + /* Convenience aliases */ + libxl__remus_devices_state *const rds = &dss->rds; + rds->ao = ao; + rds->domid = domid; + rds->callback = libxl__remus_setup_done; /* Point of no return */ - libxl__domain_suspend(egc, dss); + libxl__remus_devices_setup(egc, rds); return AO_INPROGRESS; out: return AO_ABORT(rc); } +static void libxl__remus_setup_done(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (!rc) { + libxl__domain_suspend(egc, dss); + return; + } + + LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d", + dss->domid, rc); + rds->callback = libxl__remus_setup_failed; + libxl__remus_devices_teardown(egc, rds); +} + +static void libxl__remus_setup_failed(libxl__egc *egc, + libxl__remus_devices_state *rds, int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + LOG(ERROR, "Remus: failed to teardown device after setup failed" + " for guest with domid %u, rc %d", dss->domid, rc); + + dss->callback(egc, dss, rc); +} + static void remus_failover_cb(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { @@ -832,10 +870,6 @@ static void remus_failover_cb(libxl__egc *egc, * backup died or some network error occurred preventing us * from sending checkpoints. */ - - /* TBD: Remus cleanup - i.e. detach qdisc, release other - * resources. - */ libxl__ao_complete(egc, ao, rc); } diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c index bd21841..e9d29b5 100644 --- a/tools/libxl/libxl_dom.c +++ b/tools/libxl/libxl_dom.c @@ -920,8 +920,6 @@ static void domain_suspend_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); static void domain_suspend_callback_common_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int ok); -static void remus_domain_suspend_callback_common_done(libxl__egc *egc, - libxl__domain_suspend_state *dss, int ok); /*----- complicated callback, called by xc_domain_save -----*/ @@ -1583,6 +1581,14 @@ static void domain_suspend_callback_common_done(libxl__egc *egc, } /*----- remus callbacks -----*/ +static void remus_domain_suspend_callback_common_done(libxl__egc *egc, + libxl__domain_suspend_state *dss, int ok); +static void remus_devices_postsuspend_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); +static void remus_devices_preresume_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); static void libxl__remus_domain_suspend_callback(void *data) { @@ -1597,32 +1603,77 @@ static void libxl__remus_domain_suspend_callback(void *data) static void remus_domain_suspend_callback_common_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int ok) { - /* REMUS TODO: Issue disk and network checkpoint reqs. */ + if (!ok) + goto out; + + libxl__remus_devices_state *const rds = &dss->rds; + rds->callback = remus_devices_postsuspend_cb; + libxl__remus_devices_postsuspend(egc, rds); + return; + +out: libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); } -static void libxl__remus_domain_resume_callback(void *data) +static void remus_devices_postsuspend_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) { int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + if (rc) + goto out; + + ok = 1; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); +} + +static void libxl__remus_domain_resume_callback(void *data) +{ libxl__save_helper_state *shs = data; libxl__egc *egc = shs->egc; libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs); STATE_AO_GC(dss->ao); + libxl__remus_devices_state *const rds = &dss->rds; + rds->callback = remus_devices_preresume_cb; + libxl__remus_devices_preresume(egc, rds); +} + +static void remus_devices_preresume_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + int ok = 0; + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + goto out; + /* Resumes the domain and the device model */ - if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1)) + rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1); + if (rc) goto out; - /* REMUS TODO: Deal with disk. Start a new network output buffer */ ok = 1; + out: - libxl__xc_domain_saverestore_async_callback_done(egc, shs, ok); + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok); } /*----- remus asynchronous checkpoint callback -----*/ static void remus_checkpoint_dm_saved(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); +static void remus_devices_commit_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, + const struct timeval *requested_abs); static void libxl__remus_domain_checkpoint_callback(void *data) { @@ -1642,10 +1693,73 @@ static void libxl__remus_domain_checkpoint_callback(void *data) static void remus_checkpoint_dm_saved(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { - /* REMUS TODO: Wait for disk and memory ack, release network buffer */ - /* REMUS TODO: make this asynchronous */ - assert(!rc); /* REMUS TODO handle this error properly */ - usleep(dss->interval * 1000); + /* Convenience aliases */ + libxl__remus_devices_state *const rds = &dss->rds; + + STATE_AO_GC(dss->ao); + + if (rc) { + LOG(ERROR, "Failed to save device model. Terminating Remus.."); + goto out; + } + + rds->callback = remus_devices_commit_cb; + libxl__remus_devices_commit(egc, rds); + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void remus_devices_commit_cb(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + + STATE_AO_GC(dss->ao); + + if (rc) { + LOG(ERROR, "Failed to do device commit op." + " Terminating Remus.."); + goto out; + } + + /* + * At this point, we have successfully checkpointed the guest and + * committed it at the backup. We'll come back after the checkpoint + * interval to checkpoint the guest again. Until then, let the guest + * continue execution. + */ + + /* Set checkpoint interval timeout */ + rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout, + remus_next_checkpoint, + dss->interval); + + if (rc) + goto out; + + return; + +out: + libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0); +} + +static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev, + const struct timeval *requested_abs) +{ + libxl__domain_suspend_state *dss = + CONTAINER_OF(ev, *dss, checkpoint_timeout); + + STATE_AO_GC(dss->ao); + + /* + * Time to checkpoint the guest again. We return 1 to libxc + * (xc_domain_save.c). in order to continue executing the infinite loop + * (suspend, checkpoint, resume) in xc_domain_save(). + */ libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1); } @@ -1860,6 +1974,10 @@ static void save_device_model_datacopier_done(libxl__egc *egc, dss->save_dm_callback(egc, dss, our_rc); } +static void remus_teardown_done(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc); + static void domain_suspend_done(libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) { @@ -1874,6 +1992,34 @@ static void domain_suspend_done(libxl__egc *egc, xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid, dss->guest_evtchn.port, &dss->guest_evtchn_lockfd); + if (!dss->remus) { + remus_teardown_done(egc, &dss->rds, rc); + return; + } + + /* + * With Remus, if we reach this point, it means either + * backup died or some network error occurred preventing us + * from sending checkpoints. Teardown the network buffers and + * release netlink resources. This is an async op. + */ + LOG(WARN, "Remus: Domain suspend terminated with rc %d," + " teardown Remus devices...", rc); + dss->rds.callback = remus_teardown_done; + libxl__remus_devices_teardown(egc, &dss->rds); +} + +static void remus_teardown_done(libxl__egc *egc, + libxl__remus_devices_state *rds, + int rc) +{ + libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds); + STATE_AO_GC(dss->ao); + + if (rc) + LOG(ERROR, "Remus: failed to teardown device for guest with domid %u," + " rc %d", dss->domid, rc); + dss->callback(egc, dss, rc); } diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h index 09742d9..35fbdcd 100644 --- a/tools/libxl/libxl_internal.h +++ b/tools/libxl/libxl_internal.h @@ -2586,6 +2586,166 @@ typedef struct libxl__save_helper_state { * marshalling and xc callback functions */ } libxl__save_helper_state; +/*----- remus device related state structure -----*/ +/* + * The abstract Remus device layer exposes a common + * set of API to [external] libxl for manipulating devices attached to + * a guest protected by Remus. The device layer also exposes a set of + * [internal] interfaces that every device type must implement. + * + * The following API are exposed to libxl: + * + * One-time configuration operations: + * +libxl__remus_devices_setup + * > Enable output buffering for NICs, setup disk replication, etc. + * +libxl__remus_devices_teardown + * > Disable output buffering and disk replication; teardown any + * associated external setups like qdiscs for NICs. + * + * Operations executed every checkpoint (in order of invocation): + * +libxl__remus_devices_postsuspend + * +libxl__remus_devices_preresume + * +libxl__remus_devices_commit + * + * Each device type needs to implement the interfaces specified in + * the libxl__remus_device_instance_ops if it wishes to support Remus. + * + * The high-level control flow through the Remus device layer is shown below: + * + * xl remus + * |-> libxl_domain_remus_start + * |-> libxl__remus_devices_setup + * |-> Per-checkpoint libxl__remus_devices_[postsuspend,preresume,commit] + * ... + * |-> On backup failure, network error or other internal errors: + * libxl__remus_devices_teardown + */ + +typedef struct libxl__remus_device libxl__remus_device; +typedef struct libxl__remus_devices_state libxl__remus_devices_state; +typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops; + +/* + * Interfaces to be implemented by every device subkind that wishes to + * support Remus. Functions must be implemented unless otherwise + * stated. Many of these functions are asynchronous. They call + * dev->aodev.callback when done. The actual implementations may be + * synchronous and call dev->aodev.callback directly (as the last + * thing they do). + */ +struct libxl__remus_device_instance_ops { + /* the device kind this ops belongs to... */ + libxl__device_kind kind; + + /* + * Checkpoint operations. May be NULL, meaning the op is not + * implemented and the caller should treat them as a no-op (and do + * nothing when checkpointing). + * Asynchronous. + */ + + void (*postsuspend)(libxl__egc *egc, libxl__remus_device *dev); + void (*preresume)(libxl__egc *egc, libxl__remus_device *dev); + void (*commit)(libxl__egc *egc, libxl__remus_device *dev); + + /* + * setup() and teardown() are refer to the actual remus device. + * Asynchronous. + * teardown is called even if setup fails. + */ + /* + * setup() should first determines whether the subkind matches the specific + * device. If matched, the device will then be managed with this set of + * subkind operations. + * Yields 0 if the device successfully set up. + * REMUS_DEVOPS_DOES_NOT_MATCH if the ops does not match the device. + * any other rc indicates failure. + */ + void (*setup)(libxl__egc *egc, libxl__remus_device *dev); + void (*teardown)(libxl__egc *egc, libxl__remus_device *dev); +}; + +typedef void libxl__remus_callback(libxl__egc *, + libxl__remus_devices_state *, int rc); + +/* + * State associated with a remus invocation, including parameters + * passed to the remus abstract device layer by the remus + * save/restore machinery. + */ +struct libxl__remus_devices_state { + /*---- must be set by caller of libxl__remus_device_(setup|teardown) ----*/ + + libxl__ao *ao; + uint32_t domid; + libxl__remus_callback *callback; + int device_kind_flags; + + /*----- private for abstract layer only -----*/ + + int num_devices; + /* + * this array is allocated before setup the remus devices by the + * remus abstract layer. + * devs may be NULL, means there's no remus devices that has been set up. + * the size of this array is 'num_devices', which is the total number + * of libxl nic devices and disk devices(num_nics + num_disks). + */ + libxl__remus_device **devs; + + libxl_device_nic *nics; + int num_nics; + libxl_device_disk *disks; + int num_disks; + + libxl__multidev multidev; +}; + +/* + * Information about a single device being handled by remus. + * Allocated by the remus abstract layer. + */ +struct libxl__remus_device { + /*----- shared between abstract and concrete layers -----*/ + /* + * if this is true, that means the subkind ops match the device + */ + bool matched; + + /*----- set by remus device abstruct layer -----*/ + /* libxl__device_* which this remus device related to */ + const void *backend_dev; + libxl__device_kind kind; + libxl__remus_devices_state *rds; + libxl__ao_device aodev; + + /*----- private for abstract layer only -----*/ + + /* + * Control and state variables for the asynchronous callback + * based loops which iterate over device subkinds, and over + * individual devices. + */ + int ops_index; + const libxl__remus_device_instance_ops *ops; + + /*----- private for concrete (device-specific) layer -----*/ + + /* concrete device's private data */ + void *concrete_data; +}; + +/* the following 5 APIs are async ops, call rds->callback when done */ +_hidden void libxl__remus_devices_setup(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden void libxl__remus_devices_teardown(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden void libxl__remus_devices_postsuspend(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden void libxl__remus_devices_preresume(libxl__egc *egc, + libxl__remus_devices_state *rds); +_hidden void libxl__remus_devices_commit(libxl__egc *egc, + libxl__remus_devices_state *rds); _hidden int libxl__netbuffer_enabled(libxl__gc *gc); /*----- Domain suspend (save) state structure -----*/ @@ -2626,6 +2786,8 @@ struct libxl__domain_suspend_state { libxl__ev_xswatch guest_watch; libxl__ev_time guest_timeout; const char *dm_savefile; + libxl__remus_devices_state rds; + libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */ int interval; /* checkpoint interval (for Remus) */ libxl__save_helper_state shs; libxl__logdirty_switch logdirty; diff --git a/tools/libxl/libxl_remus_device.c b/tools/libxl/libxl_remus_device.c new file mode 100644 index 0000000..4e77587 --- /dev/null +++ b/tools/libxl/libxl_remus_device.c @@ -0,0 +1,304 @@ +/* + * Copyright (C) 2014 FUJITSU LIMITED + * Author: Yang Hongyang + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation; version 2.1 only. with the special + * exception on linking described in file LICENSE. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + */ + +#include "libxl_osdeps.h" /* must come before any other headers */ + +#include "libxl_internal.h" + +static const libxl__remus_device_instance_ops *remus_ops[] = { + NULL, +}; + +/*----- helper functions -----*/ + +static int init_device_subkind(libxl__remus_devices_state *rds) +{ + /* init device subkind-specific state in the libxl ctx */ + return 0; +} + +static void cleanup_device_subkind(libxl__remus_devices_state *rds) +{ + /* cleanup device subkind-specific state in the libxl ctx */ +} + +/*----- setup() and teardown() -----*/ + +/* callbacks */ + +static void all_devices_setup_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc); +static void device_setup_iterate(libxl__egc *egc, + libxl__ao_device *aodev); +static void devices_teardown_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc); + +/* remus device setup and teardown */ + +static libxl__remus_device* remus_device_init(libxl__egc *egc, + libxl__remus_devices_state *rds, + libxl__device_kind kind, + void *libxl_dev) +{ + libxl__remus_device *dev = NULL; + + STATE_AO_GC(rds->ao); + GCNEW(dev); + dev->backend_dev = libxl_dev; + dev->kind = kind; + dev->rds = rds; + + return dev; +} + +static void remus_devices_setup(libxl__egc *egc, + libxl__remus_devices_state *rds); + +void libxl__remus_devices_setup(libxl__egc *egc, libxl__remus_devices_state *rds) +{ + int i, rc; + + STATE_AO_GC(rds->ao); + + rc = init_device_subkind(rds); + if (rc) + goto out; + + rds->num_devices = 0; + rds->num_nics = 0; + rds->num_disks = 0; + + if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VIF)) + rds->nics = libxl_device_nic_list(CTX, rds->domid, &rds->num_nics); + + if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_VBD)) + rds->disks = libxl_device_disk_list(CTX, rds->domid, &rds->num_disks); + + if (rds->num_nics == 0 && rds->num_disks == 0) + goto out; + + GCNEW_ARRAY(rds->devs, rds->num_nics + rds->num_disks); + + for (i = 0; i < rds->num_nics; i++) { + rds->devs[rds->num_devices++] = remus_device_init(egc, rds, + LIBXL__DEVICE_KIND_VIF, + &rds->nics[i]); + } + + for (i = 0; i < rds->num_disks; i++) { + rds->devs[rds->num_devices++] = remus_device_init(egc, rds, + LIBXL__DEVICE_KIND_VBD, + &rds->disks[i]); + } + + remus_devices_setup(egc, rds); + + return; + +out: + rds->callback(egc, rds, rc); +} + +static void remus_devices_setup(libxl__egc *egc, + libxl__remus_devices_state *rds) +{ + int i, rc; + + STATE_AO_GC(rds->ao); + + libxl__multidev_begin(ao, &rds->multidev); + rds->multidev.callback = all_devices_setup_cb; + for (i = 0; i < rds->num_devices; i++) { + libxl__remus_device *dev = rds->devs[i]; + dev->ops_index = -1; + libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev); + + dev->aodev.rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED; + dev->aodev.callback = device_setup_iterate; + device_setup_iterate(egc,&dev->aodev); + } + + rc = 0; + libxl__multidev_prepared(egc, &rds->multidev, rc); +} + + +static void device_setup_iterate(libxl__egc *egc, libxl__ao_device *aodev) +{ + libxl__remus_device *dev = CONTAINER_OF(aodev, *dev, aodev); + EGC_GC; + + if (aodev->rc != ERROR_REMUS_DEVICE_NOT_SUPPORTED && + aodev->rc != ERROR_REMUS_DEVOPS_DOES_NOT_MATCH) + /* might be success or disaster */ + goto out; + + do { + dev->ops = remus_ops[++dev->ops_index]; + if (!dev->ops) { + libxl_device_nic * nic = NULL; + libxl_device_disk * disk = NULL; + uint32_t domid; + int devid; + if (dev->kind == LIBXL__DEVICE_KIND_VIF) { + nic = (libxl_device_nic *)dev->backend_dev; + domid = nic->backend_domid; + devid = nic->devid; + } else if (dev->kind == LIBXL__DEVICE_KIND_VBD) { + disk = (libxl_device_disk *)dev->backend_dev; + domid = disk->backend_domid; + devid = libxl__device_disk_dev_number(disk->vdev, NULL, NULL); + } else { + LOG(ERROR,"device kind not handled by remus: %s", + libxl__device_kind_to_string(dev->kind)); + aodev->rc = ERROR_FAIL; + goto out; + } + LOG(ERROR,"device not handled by remus" + " (device=%s:%"PRId32"/%"PRId32")", + libxl__device_kind_to_string(dev->kind), + domid, devid); + aodev->rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED; + goto out; + } + } while (dev->ops->kind != dev->kind); + + /* found the next ops_index to try */ + assert(dev->aodev.callback == device_setup_iterate); + dev->ops->setup(egc,dev); + return; + + out: + libxl__multidev_one_callback(egc,aodev); +} + +static void all_devices_setup_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc) +{ + STATE_AO_GC(multidev->ao); + + /* Convenience aliases */ + libxl__remus_devices_state *const rds = + CONTAINER_OF(multidev, *rds, multidev); + + rds->callback(egc, rds, rc); +} + +void libxl__remus_devices_teardown(libxl__egc *egc, + libxl__remus_devices_state *rds) +{ + int i; + libxl__remus_device *dev; + + STATE_AO_GC(rds->ao); + + libxl__multidev_begin(ao, &rds->multidev); + rds->multidev.callback = devices_teardown_cb; + for (i = 0; i < rds->num_devices; i++) { + dev = rds->devs[i]; + if (!dev->ops || !dev->matched) + continue; + + libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev); + dev->ops->teardown(egc,dev); + } + + libxl__multidev_prepared(egc, &rds->multidev, 0); +} + +static void devices_teardown_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc) +{ + int i; + + STATE_AO_GC(multidev->ao); + + /* Convenience aliases */ + libxl__remus_devices_state *const rds = + CONTAINER_OF(multidev, *rds, multidev); + + /* clean nic */ + for (i = 0; i < rds->num_nics; i++) + libxl_device_nic_dispose(&rds->nics[i]); + free(rds->nics); + rds->nics = NULL; + rds->num_nics = 0; + + /* clean disk */ + for (i = 0; i < rds->num_disks; i++) + libxl_device_disk_dispose(&rds->disks[i]); + free(rds->disks); + rds->disks = NULL; + rds->num_disks = 0; + + cleanup_device_subkind(rds); + + rds->callback(egc, rds, rc); +} + +/*----- checkpointing APIs -----*/ + +/* callbacks */ + +static void devices_checkpoint_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc); + +/* API implementations */ + +#define define_remus_checkpoint_api(api) \ +void libxl__remus_devices_##api(libxl__egc *egc, \ + libxl__remus_devices_state *rds) \ +{ \ + int i; \ + libxl__remus_device *dev; \ + \ + STATE_AO_GC(rds->ao); \ + \ + libxl__multidev_begin(ao, &rds->multidev); \ + rds->multidev.callback = devices_checkpoint_cb; \ + for (i = 0; i < rds->num_devices; i++) { \ + dev = rds->devs[i]; \ + if (!dev->matched || !dev->ops->api) \ + continue; \ + libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);\ + dev->ops->api(egc,dev); \ + } \ + \ + libxl__multidev_prepared(egc, &rds->multidev, 0); \ +} + +define_remus_checkpoint_api(postsuspend); + +define_remus_checkpoint_api(preresume); + +define_remus_checkpoint_api(commit); + +static void devices_checkpoint_cb(libxl__egc *egc, + libxl__multidev *multidev, + int rc) +{ + STATE_AO_GC(multidev->ao); + + /* Convenience aliases */ + libxl__remus_devices_state *const rds = + CONTAINER_OF(multidev, *rds, multidev); + + rds->callback(egc, rds, rc); +} diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl index 7f9e7c7..da4c52d 100644 --- a/tools/libxl/libxl_types.idl +++ b/tools/libxl/libxl_types.idl @@ -61,6 +61,8 @@ libxl_error = Enumeration("error", [ (-15, "LOCK_FAIL"), (-16, "JSON_CONFIG_EMPTY"), (-17, "DEVICE_EXISTS"), + (-18, "REMUS_DEVOPS_DOES_NOT_MATCH"), + (-19, "REMUS_DEVICE_NOT_SUPPORTED"), ], value_namespace = "") libxl_domain_type = Enumeration("domain_type", [ -- 1.9.1