[PATCH for-4.5 v20 04/12] libxl/remus: introduce an abstract Remus device layer

[Yang Hongyang <yanghy@cn.fujitsu.com>]

Introduce an abstract device layer that allows the Remus
logic in libxl to control a guest's devices in a device-agnostic
manner. The device layer also exposes a set of internal interfaces
that a device type must implement, if it wishes to support Remus.

The following API are exposed to libxl:

One-time configuration operations:
  *libxl__remus_devices_setup
    > Enable output buffering for NICs, setup disk replication, etc.
  *libxl__remus_devices_teardown
    > Disable network output buffering and disk replication;
      teardown any associated external setups like qdiscs for NICs.

Operations executed every checkpoint (in order of invocation):
  *libxl__remus_devices_postsuspend
  *libxl__remus_devices_preresume
  *libxl__remus_devices_commit

Each device type needs to implement the interfaces specified in
the libxl__remus_device_instance_ops if it wishes to support Remus.

The high-level control flow through the Remus device layer is shown below:

xl remus
  |->  libxl_domain_remus_start
    |-> libxl__remus_devices_setup
      |-> Per-checkpoint libxl__remus_devices_[postsuspend,preresume,commit]
        ...
        |-> On backup failure/network error/other errors
            libxl__remus_devices_teardown

Signed-off-by: Yang Hongyang <yanghy@cn.fujitsu.com>
Signed-off-by: Wen Congyang <wency@cn.fujitsu.com>
Signed-off-by: Lai Jiangshan <laijs@cn.fujitsu.com>

For comments:
Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
Signed-off-by: Shriram Rajagopalan <rshriram@cs.ubc.ca>
---
 tools/libxl/Makefile                 |   2 +
 tools/libxl/libxl.c                  |  47 +++++-
 tools/libxl/libxl_dom.c              | 168 +++++++++++++++++++--
 tools/libxl/libxl_internal.h         | 163 +++++++++++++++++++++
 tools/libxl/libxl_remus_device.c     | 273 +++++++++++++++++++++++++++++++++++
 tools/libxl/libxl_types.idl          |   2 +
 tools/libxl/libxl_types_internal.idl |   2 +
 7 files changed, 640 insertions(+), 17 deletions(-)
 create mode 100644 tools/libxl/libxl_remus_device.c

diff --git a/tools/libxl/Makefile b/tools/libxl/Makefile
index 4605124..28b8616 100644
--- a/tools/libxl/Makefile
+++ b/tools/libxl/Makefile
@@ -56,6 +56,8 @@ else
 LIBXL_OBJS-y += libxl_nonetbuffer.o
 endif
 
+LIBXL_OBJS-y += libxl_remus_device.o
+
 LIBXL_OBJS-$(CONFIG_X86) += libxl_cpuid.o libxl_x86.o
 LIBXL_OBJS-$(CONFIG_ARM) += libxl_nocpuid.o libxl_arm.o
 
diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 3735f55..728fdec 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -782,6 +782,10 @@ out:
     return ptr;
 }
 
+static void libxl__remus_setup_done(libxl__egc *egc,
+                                    libxl__remus_devices_state *rds, int rc);
+static void libxl__remus_setup_failed(libxl__egc *egc,
+                                      libxl__remus_devices_state *rds, int rc);
 static void remus_failover_cb(libxl__egc *egc,
                               libxl__domain_suspend_state *dss, int rc);
 
@@ -813,16 +817,51 @@ int libxl_domain_remus_start(libxl_ctx *ctx, libxl_domain_remus_info *info,
 
     assert(info);
 
-    /* TBD: Remus setup - i.e. attach qdisc, enable disk buffering, etc */
+    /* Convenience aliases */
+    libxl__remus_devices_state *const rds = &dss->rds;
+    rds->ao = ao;
+    rds->egc = egc;
+    rds->domid = domid;
+    rds->callback = libxl__remus_setup_done;
 
     /* Point of no return */
-    libxl__domain_suspend(egc, dss);
+    libxl__remus_devices_setup(egc, rds);
     return AO_INPROGRESS;
 
  out:
     return AO_ABORT(rc);
 }
 
+static void libxl__remus_setup_done(libxl__egc *egc,
+                                    libxl__remus_devices_state *rds, int rc)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    STATE_AO_GC(dss->ao);
+
+    if (!rc) {
+        libxl__domain_suspend(egc, dss);
+        return;
+    }
+
+    LOG(ERROR, "Remus: failed to setup device for guest with domid %u, rc %d",
+        dss->domid, rc);
+    rds->callback = libxl__remus_setup_failed;
+    libxl__remus_devices_teardown(egc, rds);
+}
+
+static void libxl__remus_setup_failed(libxl__egc *egc,
+                                      libxl__remus_devices_state *rds, int rc)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    STATE_AO_GC(dss->ao);
+
+    if (rc)
+        LOG(ERROR, "Remus: failed to teardown device after setup failed"
+            " for guest with domid %u, rc %d", dss->domid, rc);
+
+    dss->callback(egc, dss, rc);
+}
+
 static void remus_failover_cb(libxl__egc *egc,
                               libxl__domain_suspend_state *dss, int rc)
 {
@@ -832,10 +871,6 @@ static void remus_failover_cb(libxl__egc *egc,
      * backup died or some network error occurred preventing us
      * from sending checkpoints.
      */
-
-    /* TBD: Remus cleanup - i.e. detach qdisc, release other
-     * resources.
-     */
     libxl__ao_complete(egc, ao, rc);
 }
 
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index bd21841..e9d29b5 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -920,8 +920,6 @@ static void domain_suspend_done(libxl__egc *egc,
                         libxl__domain_suspend_state *dss, int rc);
 static void domain_suspend_callback_common_done(libxl__egc *egc,
                                 libxl__domain_suspend_state *dss, int ok);
-static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
-                                libxl__domain_suspend_state *dss, int ok);
 
 /*----- complicated callback, called by xc_domain_save -----*/
 
@@ -1583,6 +1581,14 @@ static void domain_suspend_callback_common_done(libxl__egc *egc,
 }
 
 /*----- remus callbacks -----*/
+static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
+                                libxl__domain_suspend_state *dss, int ok);
+static void remus_devices_postsuspend_cb(libxl__egc *egc,
+                                         libxl__remus_devices_state *rds,
+                                         int rc);
+static void remus_devices_preresume_cb(libxl__egc *egc,
+                                       libxl__remus_devices_state *rds,
+                                       int rc);
 
 static void libxl__remus_domain_suspend_callback(void *data)
 {
@@ -1597,32 +1603,77 @@ static void libxl__remus_domain_suspend_callback(void *data)
 static void remus_domain_suspend_callback_common_done(libxl__egc *egc,
                                 libxl__domain_suspend_state *dss, int ok)
 {
-    /* REMUS TODO: Issue disk and network checkpoint reqs. */
+    if (!ok)
+        goto out;
+
+    libxl__remus_devices_state *const rds = &dss->rds;
+    rds->callback = remus_devices_postsuspend_cb;
+    libxl__remus_devices_postsuspend(egc, rds);
+    return;
+
+out:
     libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
 }
 
-static void libxl__remus_domain_resume_callback(void *data)
+static void remus_devices_postsuspend_cb(libxl__egc *egc,
+                                         libxl__remus_devices_state *rds,
+                                         int rc)
 {
     int ok = 0;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+
+    if (rc)
+        goto out;
+
+    ok = 1;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
+}
+
+static void libxl__remus_domain_resume_callback(void *data)
+{
     libxl__save_helper_state *shs = data;
     libxl__egc *egc = shs->egc;
     libxl__domain_suspend_state *dss = CONTAINER_OF(shs, *dss, shs);
     STATE_AO_GC(dss->ao);
 
+    libxl__remus_devices_state *const rds = &dss->rds;
+    rds->callback = remus_devices_preresume_cb;
+    libxl__remus_devices_preresume(egc, rds);
+}
+
+static void remus_devices_preresume_cb(libxl__egc *egc,
+                                       libxl__remus_devices_state *rds,
+                                       int rc)
+{
+    int ok = 0;
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    STATE_AO_GC(dss->ao);
+
+    if (rc)
+        goto out;
+
     /* Resumes the domain and the device model */
-    if (libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1))
+    rc = libxl__domain_resume(gc, dss->domid, /* Fast Suspend */1);
+    if (rc)
         goto out;
 
-    /* REMUS TODO: Deal with disk. Start a new network output buffer */
     ok = 1;
+
 out:
-    libxl__xc_domain_saverestore_async_callback_done(egc, shs, ok);
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, ok);
 }
 
 /*----- remus asynchronous checkpoint callback -----*/
 
 static void remus_checkpoint_dm_saved(libxl__egc *egc,
                                       libxl__domain_suspend_state *dss, int rc);
+static void remus_devices_commit_cb(libxl__egc *egc,
+                                    libxl__remus_devices_state *rds,
+                                    int rc);
+static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
+                                  const struct timeval *requested_abs);
 
 static void libxl__remus_domain_checkpoint_callback(void *data)
 {
@@ -1642,10 +1693,73 @@ static void libxl__remus_domain_checkpoint_callback(void *data)
 static void remus_checkpoint_dm_saved(libxl__egc *egc,
                                       libxl__domain_suspend_state *dss, int rc)
 {
-    /* REMUS TODO: Wait for disk and memory ack, release network buffer */
-    /* REMUS TODO: make this asynchronous */
-    assert(!rc); /* REMUS TODO handle this error properly */
-    usleep(dss->interval * 1000);
+    /* Convenience aliases */
+    libxl__remus_devices_state *const rds = &dss->rds;
+
+    STATE_AO_GC(dss->ao);
+
+    if (rc) {
+        LOG(ERROR, "Failed to save device model. Terminating Remus..");
+        goto out;
+    }
+
+    rds->callback = remus_devices_commit_cb;
+    libxl__remus_devices_commit(egc, rds);
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
+}
+
+static void remus_devices_commit_cb(libxl__egc *egc,
+                                    libxl__remus_devices_state *rds,
+                                    int rc)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+
+    STATE_AO_GC(dss->ao);
+
+    if (rc) {
+        LOG(ERROR, "Failed to do device commit op."
+            " Terminating Remus..");
+        goto out;
+    }
+
+    /*
+     * At this point, we have successfully checkpointed the guest and
+     * committed it at the backup. We'll come back after the checkpoint
+     * interval to checkpoint the guest again. Until then, let the guest
+     * continue execution.
+     */
+
+    /* Set checkpoint interval timeout */
+    rc = libxl__ev_time_register_rel(gc, &dss->checkpoint_timeout,
+                                     remus_next_checkpoint,
+                                     dss->interval);
+
+    if (rc)
+        goto out;
+
+    return;
+
+out:
+    libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 0);
+}
+
+static void remus_next_checkpoint(libxl__egc *egc, libxl__ev_time *ev,
+                                  const struct timeval *requested_abs)
+{
+    libxl__domain_suspend_state *dss =
+                            CONTAINER_OF(ev, *dss, checkpoint_timeout);
+
+    STATE_AO_GC(dss->ao);
+
+    /*
+     * Time to checkpoint the guest again. We return 1 to libxc
+     * (xc_domain_save.c). in order to continue executing the infinite loop
+     * (suspend, checkpoint, resume) in xc_domain_save().
+     */
     libxl__xc_domain_saverestore_async_callback_done(egc, &dss->shs, 1);
 }
 
@@ -1860,6 +1974,10 @@ static void save_device_model_datacopier_done(libxl__egc *egc,
     dss->save_dm_callback(egc, dss, our_rc);
 }
 
+static void remus_teardown_done(libxl__egc *egc,
+                                       libxl__remus_devices_state *rds,
+                                       int rc);
+
 static void domain_suspend_done(libxl__egc *egc,
                         libxl__domain_suspend_state *dss, int rc)
 {
@@ -1874,6 +1992,34 @@ static void domain_suspend_done(libxl__egc *egc,
         xc_suspend_evtchn_release(CTX->xch, CTX->xce, domid,
                            dss->guest_evtchn.port, &dss->guest_evtchn_lockfd);
 
+    if (!dss->remus) {
+        remus_teardown_done(egc, &dss->rds, rc);
+        return;
+    }
+
+    /*
+     * With Remus, if we reach this point, it means either
+     * backup died or some network error occurred preventing us
+     * from sending checkpoints. Teardown the network buffers and
+     * release netlink resources.  This is an async op.
+     */
+    LOG(WARN, "Remus: Domain suspend terminated with rc %d,"
+        " teardown Remus devices...", rc);
+    dss->rds.callback = remus_teardown_done;
+    libxl__remus_devices_teardown(egc, &dss->rds);
+}
+
+static void remus_teardown_done(libxl__egc *egc,
+                                       libxl__remus_devices_state *rds,
+                                       int rc)
+{
+    libxl__domain_suspend_state *dss = CONTAINER_OF(rds, *dss, rds);
+    STATE_AO_GC(dss->ao);
+
+    if (rc)
+        LOG(ERROR, "Remus: failed to teardown device for guest with domid %u,"
+            " rc %d", dss->domid, rc);
+
     dss->callback(egc, dss, rc);
 }
 
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 229f63b..7be373f 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -2573,6 +2573,167 @@ typedef struct libxl__save_helper_state {
                       * marshalling and xc callback functions */
 } libxl__save_helper_state;
 
+/*----- remus device related state structure -----*/
+/*
+ * The abstract Remus device layer exposes a common
+ * set of API to [external] libxl for manipulating devices attached to
+ * a guest protected by Remus. The device layer also exposes a set of
+ * [internal] interfaces that every device type must implement.
+ *
+ * The following API are exposed to libxl:
+ *
+ * One-time configuration operations:
+ *  +libxl__remus_devices_setup
+ *    > Enable output buffering for NICs, setup disk replication, etc.
+ *  +libxl__remus_devices_teardown
+ *    > Disable output buffering and disk replication; teardown any
+ *       associated external setups like qdiscs for NICs.
+ *
+ * Operations executed every checkpoint (in order of invocation):
+ *  +libxl__remus_devices_postsuspend
+ *  +libxl__remus_devices_preresume
+ *  +libxl__remus_devices_commit
+ *
+ * Each device type needs to implement the interfaces specified in
+ * the libxl__remus_device_instance_ops if it wishes to support Remus.
+ *
+ * The high-level control flow through the Remus device layer is shown below:
+ *
+ * xl remus
+ *  |->  libxl_domain_remus_start
+ *    |-> libxl__remus_devices_setup
+ *      |-> Per-checkpoint libxl__remus_devices_[postsuspend,preresume,commit]
+ *        ...
+ *        |-> On backup failure, network error or other internal errors:
+ *            libxl__remus_devices_teardown
+ */
+
+typedef struct libxl__remus_device libxl__remus_device;
+typedef struct libxl__remus_devices_state libxl__remus_devices_state;
+typedef struct libxl__remus_device_instance_ops libxl__remus_device_instance_ops;
+
+/*
+ * Interfaces to be implemented by every device subkind that wishes to
+ * support Remus. Functions must be implemented unless otherwise
+ * stated. Many of these functions are asynchronous. They call
+ * dev->aodev.callback when done.  The actual implementations may be
+ * synchronous and call dev->aodev.callback directly (as the last
+ * thing they do).
+ */
+struct libxl__remus_device_instance_ops {
+    /* the device kind this ops belongs to... */
+    libxl__device_kind kind;
+
+    /*
+     * Checkpoint operations. May be NULL, meaning the op is not
+     * implemented and the caller should treat them as a no-op (and do
+     * nothing when checkpointing).
+     * Asynchronous.
+     */
+
+    void (*postsuspend)(libxl__remus_device *dev);
+    void (*preresume)(libxl__remus_device *dev);
+    void (*commit)(libxl__remus_device *dev);
+
+    /*
+     * setup() and teardown() are refer to the actual remus device.
+     * Asynchronous.
+     * teardown is called even if setup fails.
+     */
+    /*
+     * setup() should first determines whether the subkind matches the specific
+     * device. If matched, the device will then be managed with this set of
+     * subkind operations.
+     * Yields 0 if the device successfully set up.
+     * REMUS_DEVOPS_DOES_NOT_MATCH if the ops does not match the device.
+     * any other rc indicates failure.
+     */
+    void (*setup)(libxl__remus_device *dev);
+    void (*teardown)(libxl__remus_device *dev);
+};
+
+typedef void libxl__remus_callback(libxl__egc *,
+                                   libxl__remus_devices_state *, int rc);
+
+/*
+ * State associated with a remus invocation, including parameters
+ * passed to the remus abstract device layer by the remus
+ * save/restore machinery.
+ */
+struct libxl__remus_devices_state {
+    /*---- must be set by caller of libxl__remus_device_(setup|teardown) ----*/
+
+    libxl__ao *ao;
+    libxl__egc *egc;
+    uint32_t domid;
+    libxl__remus_callback *callback;
+    int device_kind_flags;
+
+    /*----- private for abstract layer only -----*/
+
+    int num_devices;
+    /*
+     * this array is allocated before setup the remus devices by the
+     * remus abstract layer.
+     * devs may be NULL, means there's no remus devices that has been set up.
+     * the size of this array is 'num_devices', which is the total number
+     * of libxl nic devices and disk devices(num_nics + num_disks).
+     */
+    libxl__remus_device **devs;
+
+    libxl_device_nic *nics;
+    int num_nics;
+    libxl_device_disk *disks;
+    int num_disks;
+
+    libxl__multidev multidev;
+};
+
+/*
+ * Information about a single device being handled by remus.
+ * Allocated by the remus abstract layer.
+ */
+struct libxl__remus_device {
+    /*----- shared between abstract and concrete layers -----*/
+    /*
+     * if this is true, that means the subkind ops match the device
+     */
+    bool matched;
+
+    /*----- set by remus device abstruct layer -----*/
+    /* libxl__device_* which this remus device related to */
+    const void *backend_dev;
+    libxl__device_kind kind;
+    libxl__remus_devices_state *rds;
+    libxl__ao_device aodev;
+
+    /*----- private for abstract layer only -----*/
+
+    /*
+     * Control and state variables for the asynchronous callback
+     * based loops which iterate over device subkinds, and over
+     * individual devices.
+     */
+    int ops_index;
+    const libxl__remus_device_instance_ops *ops;
+
+    /*----- private for concrete (device-specific) layer -----*/
+
+    /* concrete device's private data */
+    void *concrete_data;
+};
+
+/* the following 5 APIs are async ops, call rds->callback when done */
+_hidden void libxl__remus_devices_setup(libxl__egc *egc,
+                                        libxl__remus_devices_state *rds);
+_hidden void libxl__remus_devices_teardown(libxl__egc *egc,
+                                           libxl__remus_devices_state *rds);
+_hidden void libxl__remus_devices_postsuspend(libxl__egc *egc,
+                                              libxl__remus_devices_state *rds);
+_hidden void libxl__remus_devices_preresume(libxl__egc *egc,
+                                            libxl__remus_devices_state *rds);
+_hidden void libxl__remus_devices_commit(libxl__egc *egc,
+                                         libxl__remus_devices_state *rds);
 _hidden int libxl__netbuffer_enabled(libxl__gc *gc);
 
 /*----- Domain suspend (save) state structure -----*/
@@ -2613,6 +2774,8 @@ struct libxl__domain_suspend_state {
     libxl__ev_xswatch guest_watch;
     libxl__ev_time guest_timeout;
     const char *dm_savefile;
+    libxl__remus_devices_state rds;
+    libxl__ev_time checkpoint_timeout; /* used for Remus checkpoint */
     int interval; /* checkpoint interval (for Remus) */
     libxl__save_helper_state shs;
     libxl__logdirty_switch logdirty;
diff --git a/tools/libxl/libxl_remus_device.c b/tools/libxl/libxl_remus_device.c
new file mode 100644
index 0000000..be87f1e
--- /dev/null
+++ b/tools/libxl/libxl_remus_device.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright (C) 2014 FUJITSU LIMITED
+ * Author: Yang Hongyang <yanghy@cn.fujitsu.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ */
+
+#include "libxl_osdeps.h" /* must come before any other headers */
+
+#include "libxl_internal.h"
+
+static const libxl__remus_device_instance_ops *remus_ops[] = {
+    NULL,
+};
+
+/*----- helper functions -----*/
+
+static int init_device_subkind(libxl__remus_devices_state *rds)
+{
+    /* init device subkind-specific state in the libxl ctx */
+    return 0;
+}
+
+static void cleanup_device_subkind(libxl__remus_devices_state *rds)
+{
+    /* cleanup device subkind-specific state in the libxl ctx */
+}
+
+/*----- setup() and teardown() -----*/
+
+/* callbacks */
+
+static void devices_setup_cb(libxl__egc *egc,
+                             libxl__multidev *multidev,
+                             int rc);
+static void devices_teardown_cb(libxl__egc *egc,
+                                libxl__multidev *multidev,
+                                int rc);
+
+/* remus device setup and teardown */
+
+static libxl__remus_device* remus_device_init(libxl__egc *egc,
+                                              libxl__remus_devices_state *rds,
+                                              libxl__device_kind kind,
+                                              void *libxl_dev)
+{
+    libxl__remus_device *dev = NULL;
+
+    STATE_AO_GC(rds->ao);
+    GCNEW(dev);
+    dev->backend_dev = libxl_dev;
+    dev->kind = kind;
+    dev->rds = rds;
+    dev->ops_index = -1;
+
+    return dev;
+}
+
+static void remus_devices_setup(libxl__egc *egc,
+                                libxl__remus_devices_state *rds);
+
+void libxl__remus_devices_setup(libxl__egc *egc, libxl__remus_devices_state *rds)
+{
+    int i, rc;
+
+    STATE_AO_GC(rds->ao);
+
+    rc = init_device_subkind(rds);
+    if (rc)
+        goto out;
+
+    rds->num_devices = 0;
+    rds->num_nics = 0;
+    rds->num_disks = 0;
+
+    if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_REMUS_NIC))
+        rds->nics = libxl_device_nic_list(CTX, rds->domid, &rds->num_nics);
+
+    if (rds->device_kind_flags & (1 << LIBXL__DEVICE_KIND_REMUS_DISK))
+        rds->disks = libxl_device_disk_list(CTX, rds->domid, &rds->num_disks);
+
+    if (rds->num_nics == 0 && rds->num_disks == 0)
+        goto out;
+
+    GCNEW_ARRAY(rds->devs, rds->num_nics + rds->num_disks);
+
+    for (i = 0; i < rds->num_nics; i++) {
+        rds->devs[rds->num_devices++] = remus_device_init(egc, rds,
+                                                LIBXL__DEVICE_KIND_REMUS_NIC,
+                                                &rds->nics[i]);
+    }
+
+    for (i = 0; i < rds->num_disks; i++) {
+        rds->devs[rds->num_devices++] = remus_device_init(egc, rds,
+                                                LIBXL__DEVICE_KIND_REMUS_DISK,
+                                                &rds->disks[i]);
+    }
+
+    remus_devices_setup(egc, rds);
+
+    return;
+
+out:
+    rds->callback(egc, rds, rc);
+}
+
+static void remus_devices_setup(libxl__egc *egc,
+                                libxl__remus_devices_state *rds)
+{
+    int i, rc;
+    libxl__remus_device *dev;
+
+    STATE_AO_GC(rds->ao);
+
+    libxl__multidev_begin(ao, &rds->multidev);
+    rds->multidev.callback = devices_setup_cb;
+    for (i = 0; i < rds->num_devices; i++) {
+        dev = rds->devs[i];
+        if (dev->matched)
+            continue;
+
+        /* find avaliable ops */
+        do {
+            dev->ops = remus_ops[++dev->ops_index];
+            if (!dev->ops) {
+                rc = ERROR_REMUS_DEVICE_NOT_SUPPORTED;
+                goto out;
+            }
+        } while (dev->ops->kind != dev->kind);
+
+        libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);
+        dev->ops->setup(dev);
+    }
+
+    rc = 0;
+out:
+    libxl__multidev_prepared(egc, &rds->multidev, rc);
+}
+
+static void devices_setup_cb(libxl__egc *egc,
+                             libxl__multidev *multidev,
+                             int rc)
+{
+    STATE_AO_GC(multidev->ao);
+
+    /* Convenience aliases */
+    libxl__remus_devices_state *const rds =
+                            CONTAINER_OF(multidev, *rds, multidev);
+
+    /*
+     * if the error is ERROR_REMUS_DEVOPS_DOES_NOT_MATCH, begin next iter
+     * if there are devices that can't be set up, the rc will become
+     * ERROR_FAIL or ERROR_REMUS_DEVICE_NOT_SUPPORTED at last anyway.
+     */
+    if (rc == ERROR_REMUS_DEVOPS_DOES_NOT_MATCH) {
+        remus_devices_setup(egc, rds);
+        return;
+    }
+
+    rds->callback(egc, rds, rc);
+}
+
+void libxl__remus_devices_teardown(libxl__egc *egc,
+                                   libxl__remus_devices_state *rds)
+{
+    int i;
+    libxl__remus_device *dev;
+
+    STATE_AO_GC(rds->ao);
+
+    libxl__multidev_begin(ao, &rds->multidev);
+    rds->multidev.callback = devices_teardown_cb;
+    for (i = 0; i < rds->num_devices; i++) {
+        dev = rds->devs[i];
+        if (!dev->ops || !dev->matched)
+            continue;
+
+        libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);
+        dev->ops->teardown(dev);
+    }
+
+    libxl__multidev_prepared(egc, &rds->multidev, 0);
+}
+
+static void devices_teardown_cb(libxl__egc *egc,
+                                libxl__multidev *multidev,
+                                int rc)
+{
+    int i;
+
+    STATE_AO_GC(multidev->ao);
+
+    /* Convenience aliases */
+    libxl__remus_devices_state *const rds =
+                            CONTAINER_OF(multidev, *rds, multidev);
+
+    /* clean nic */
+    for (i = 0; i < rds->num_nics; i++)
+        libxl_device_nic_dispose(&rds->nics[i]);
+    free(rds->nics);
+    rds->nics = NULL;
+    rds->num_nics = 0;
+
+    /* clean disk */
+    for (i = 0; i < rds->num_disks; i++)
+        libxl_device_disk_dispose(&rds->disks[i]);
+    free(rds->disks);
+    rds->disks = NULL;
+    rds->num_disks = 0;
+
+    cleanup_device_subkind(rds);
+
+    rds->callback(egc, rds, rc);
+}
+
+/*----- checkpointing APIs -----*/
+
+/* callbacks */
+
+static void devices_checkpoint_cb(libxl__egc *egc,
+                                  libxl__multidev *multidev,
+                                  int rc);
+
+/* API implementations */
+
+#define define_remus_checkpoint_api(api)                                \
+void libxl__remus_devices_##api(libxl__egc *egc,                        \
+                                libxl__remus_devices_state *rds)        \
+{                                                                       \
+    int i;                                                              \
+    libxl__remus_device *dev;                                           \
+                                                                        \
+    STATE_AO_GC(rds->ao);                                               \
+                                                                        \
+    libxl__multidev_begin(ao, &rds->multidev);                          \
+    rds->multidev.callback = devices_checkpoint_cb;                     \
+    for (i = 0; i < rds->num_devices; i++) {                            \
+        dev = rds->devs[i];                                             \
+        if (!dev->matched || !dev->ops->api)                            \
+            continue;                                                   \
+        libxl__multidev_prepare_with_aodev(&rds->multidev, &dev->aodev);\
+        dev->ops->api(dev);                                             \
+    }                                                                   \
+                                                                        \
+    libxl__multidev_prepared(egc, &rds->multidev, 0);                   \
+}
+
+define_remus_checkpoint_api(postsuspend);
+
+define_remus_checkpoint_api(preresume);
+
+define_remus_checkpoint_api(commit);
+
+static void devices_checkpoint_cb(libxl__egc *egc,
+                                  libxl__multidev *multidev,
+                                  int rc)
+{
+    STATE_AO_GC(multidev->ao);
+
+    /* Convenience aliases */
+    libxl__remus_devices_state *const rds =
+                            CONTAINER_OF(multidev, *rds, multidev);
+
+    rds->callback(egc, rds, rc);
+}
diff --git a/tools/libxl/libxl_types.idl b/tools/libxl/libxl_types.idl
index 7f9e7c7..da4c52d 100644
--- a/tools/libxl/libxl_types.idl
+++ b/tools/libxl/libxl_types.idl
@@ -61,6 +61,8 @@ libxl_error = Enumeration("error", [
     (-15, "LOCK_FAIL"),
     (-16, "JSON_CONFIG_EMPTY"),
     (-17, "DEVICE_EXISTS"),
+    (-18, "REMUS_DEVOPS_DOES_NOT_MATCH"),
+    (-19, "REMUS_DEVICE_NOT_SUPPORTED"),
     ], value_namespace = "")
 
 libxl_domain_type = Enumeration("domain_type", [
diff --git a/tools/libxl/libxl_types_internal.idl b/tools/libxl/libxl_types_internal.idl
index 800361b..720232e 100644
--- a/tools/libxl/libxl_types_internal.idl
+++ b/tools/libxl/libxl_types_internal.idl
@@ -22,6 +22,8 @@ libxl__device_kind = Enumeration("device_kind", [
     (6, "VKBD"),
     (7, "CONSOLE"),
     (8, "VTPM"),
+    (9, "REMUS_NIC"),
+    (10, "REMUS_DISK"),
     ])
 
 libxl__console_backend = Enumeration("console_backend", [
-- 
1.9.1