[Qemu-devel] [XEN][RFC PATCH 15/15] xl: Launch and destroy all device models

[Julien Grall <julien.grall@citrix.com>]

This patch permits to launch and destroy all device models.
For the moment it's a fork of libxl__build_device*

Signed-off-by: Julien Grall <julien.grall@citrix.com>
---
 tools/libxl/libxl.c          |    8 +-
 tools/libxl/libxl_create.c   |   30 +++++-
 tools/libxl/libxl_dm.c       |  225 +++++++++++++++++++++++++++++++++++++++---
 tools/libxl/libxl_dom.c      |    6 +-
 tools/libxl/libxl_internal.h |    9 ++
 5 files changed, 256 insertions(+), 22 deletions(-)

diff --git a/tools/libxl/libxl.c b/tools/libxl/libxl.c
index 5344366..b578ada 100644
--- a/tools/libxl/libxl.c
+++ b/tools/libxl/libxl.c
@@ -1047,7 +1047,8 @@ int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid)
 
     switch (libxl__domain_type(gc, domid)) {
     case LIBXL_DOMAIN_TYPE_HVM:
-        dm_present = 1;
+        pid = libxl__xs_read(gc, XBT_NULL, libxl__sprintf(gc, "/local/domain/%d/image/device-model-pid", domid));
+        dm_present = (pid != NULL);
         break;
     case LIBXL_DOMAIN_TYPE_PV:
         pid = libxl__xs_read(gc, XBT_NULL, libxl__sprintf(gc, "/local/domain/%d/image/device-model-pid", domid));
@@ -1073,8 +1074,11 @@ int libxl_domain_destroy(libxl_ctx *ctx, uint32_t domid)
         if (libxl__destroy_device_model(gc, domid) < 0)
             LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "libxl__destroy_device_model failed for %d", domid);
 
-        libxl__qmp_cleanup(gc, domid);
+        libxl__qmp_cleanup(gc, domid, 0);
     }
+
+    libxl__destroy_dms(gc, domid);
+
     if (libxl__devices_destroy(gc, domid) < 0)
         LIBXL__LOG(ctx, LIBXL__LOG_ERROR, 
                    "libxl__devices_destroy failed for %d", domid);
diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
index 8417661..43a97f5 100644
--- a/tools/libxl/libxl_create.c
+++ b/tools/libxl/libxl_create.c
@@ -26,6 +26,10 @@ void libxl_domain_config_dispose(libxl_domain_config *d_config)
 {
     int i;
 
+    for (i=0; i<d_config->num_dms; i++)
+        libxl_dm_dispose(&d_config->dms[i]);
+    free(d_config->dms);
+
     for (i=0; i<d_config->num_disks; i++)
         libxl_device_disk_dispose(&d_config->disks[i]);
     free(d_config->disks);
@@ -80,6 +84,7 @@ int libxl__domain_build_info_setdefault(libxl__gc *gc,
             switch (b_info->device_model_version) {
             case 1: b_info->u.hvm.bios = LIBXL_BIOS_TYPE_ROMBIOS; break;
             case 2: b_info->u.hvm.bios = LIBXL_BIOS_TYPE_SEABIOS; break;
+            case 3: b_info->u.hvm.bios = LIBXL_BIOS_TYPE_SEABIOS; break;
             default:return ERROR_INVAL;
             }
 
@@ -90,6 +95,7 @@ int libxl__domain_build_info_setdefault(libxl__gc *gc,
                 return ERROR_INVAL;
             break;
         case 2:
+        case 3:
             if (b_info->u.hvm.bios == LIBXL_BIOS_TYPE_ROMBIOS)
                 return ERROR_INVAL;
             break;
@@ -621,12 +627,24 @@ static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
         libxl_device_vkb_add(ctx, domid, &vkb);
         libxl_device_vkb_dispose(&vkb);
 
-        ret = libxl__create_device_model(gc, domid, d_config,
+        if (d_config->b_info.device_model_version
+            == LIBXL_DEVICE_MODEL_VERSION_MULTIPLE_QEMU_XEN) {
+            ret = libxl__launch_dms(gc, domid, &state, d_config);
+            if (ret <  0) {
+                LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
+                           "failed to launch device models: %d\n", ret);
+                goto error_out;
+            }
+        }
+        else
+        {
+            ret = libxl__create_device_model(gc, domid, ~0, d_config,
                                          &state, &dm_starting);
-        if (ret < 0) {
-            LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
-                       "failed to create device model: %d", ret);
-            goto error_out;
+            if (ret < 0) {
+                LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
+                           "failed to create device model: %d", ret);
+                goto error_out;
+            }
         }
         break;
     }
@@ -667,7 +685,7 @@ static int do_domain_create(libxl__gc *gc, libxl_domain_config *d_config,
     if (dm_starting) {
         if (d_config->b_info.device_model_version
             == LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN) {
-            libxl__qmp_initializations(gc, domid, d_config);
+            libxl__qmp_initializations(gc, domid, d_config, 0);
         }
         ret = libxl__confirm_device_model_startup(gc, &state, dm_starting);
         if (ret < 0) {
diff --git a/tools/libxl/libxl_dm.c b/tools/libxl/libxl_dm.c
index 1261499..150b03a 100644
--- a/tools/libxl/libxl_dm.c
+++ b/tools/libxl/libxl_dm.c
@@ -308,6 +308,7 @@ static char *dm_spice_options(libxl__gc *gc,
 
 static char ** libxl__build_device_model_args_new(libxl__gc *gc,
                                         const char *dm, int guest_domid,
+                                        uint32_t dmid,
                                         const libxl_domain_config *guest_config,
                                         const libxl__domain_build_state *state)
 {
@@ -324,6 +325,11 @@ static char ** libxl__build_device_model_args_new(libxl__gc *gc,
     flexarray_t *dm_args;
     int i;
 
+    if (dmid == ~0)
+        dmid = 0;
+    else
+        dmid = guest_config->dms[dmid].id;
+
     dm_args = flexarray_make(16, 1);
     if (!dm_args)
         return NULL;
@@ -332,11 +338,15 @@ static char ** libxl__build_device_model_args_new(libxl__gc *gc,
                       "-xen-domid",
                       libxl__sprintf(gc, "%d", guest_domid), NULL);
 
+    flexarray_append(dm_args, "-xen-dmid");
+    flexarray_append(dm_args,
+                     libxl__sprintf(gc, "%u", dmid));
+
     flexarray_append(dm_args, "-chardev");
     flexarray_append(dm_args,
                      libxl__sprintf(gc, "socket,id=libxl-cmd,"
-                                    "path=%s/qmp-libxl-%d,server,nowait",
-                                    libxl_run_dir_path(), guest_domid));
+                                    "path=%s/qmp-libxl-%u-%u,server,nowait",
+                                    libxl_run_dir_path(), guest_domid, dmid));
 
     flexarray_append(dm_args, "-mon");
     flexarray_append(dm_args, "chardev=libxl-cmd,mode=control");
@@ -455,6 +465,7 @@ static char ** libxl__build_device_model_args_new(libxl__gc *gc,
                 } else {
                     ifname = vifs[i].ifname;
                 }
+                ifname = libxl__sprintf(gc, "%s.%u", ifname, dmid);
                 flexarray_append(dm_args, "-device");
                 flexarray_append(dm_args,
                    libxl__sprintf(gc, "%s,id=nic%d,netdev=net%d,mac=%s",
@@ -573,6 +584,7 @@ static char ** libxl__build_device_model_args_new(libxl__gc *gc,
 
 static char ** libxl__build_device_model_args(libxl__gc *gc,
                                         const char *dm, int guest_domid,
+                                        uint32_t dmid,
                                         const libxl_domain_config *guest_config,
                                         const libxl__domain_build_state *state)
 {
@@ -584,9 +596,10 @@ static char ** libxl__build_device_model_args(libxl__gc *gc,
                                                   guest_domid, guest_config,
                                                   state);
     case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
+    case LIBXL_DEVICE_MODEL_VERSION_MULTIPLE_QEMU_XEN:
         return libxl__build_device_model_args_new(gc, dm,
-                                                  guest_domid, guest_config,
-                                                  state);
+                                                  guest_domid, dmid,
+                                                  guest_config, state);
     default:
         LIBXL__LOG_ERRNO(ctx, LIBXL__LOG_ERROR, "unknown device model version %d",
                          guest_config->b_info.device_model_version);
@@ -748,7 +761,7 @@ static int libxl__create_stubdom(libxl__gc *gc,
     if (ret)
         goto out;
 
-    args = libxl__build_device_model_args(gc, "stubdom-dm", guest_domid,
+    args = libxl__build_device_model_args(gc, "stubdom-dm", guest_domid, 0,
                                           guest_config, d_state);
     if (!args) {
         ret = ERROR_FAIL;
@@ -882,6 +895,7 @@ out:
 
 int libxl__create_device_model(libxl__gc *gc,
                               int domid,
+                              uint32_t dmid,
                               libxl_domain_config *guest_config,
                               libxl__domain_build_state *state,
                               libxl__spawner_starting **starting_r)
@@ -899,13 +913,18 @@ int libxl__create_device_model(libxl__gc *gc,
     char *vm_path;
     char **pass_stuff;
     const char *dm;
+    int i = 0;
 
     if (libxl_defbool_val(b_info->device_model_stubdomain)) {
         rc = libxl__create_stubdom(gc, domid, guest_config, state, starting_r);
         goto out;
     }
 
-    dm = libxl__domain_device_model(gc, b_info);
+    if (dmid == ~0)
+        dm = libxl__domain_device_model(gc, b_info);
+    else
+        dm = guest_config->dms[dmid].path;
+
     if (!dm) {
         rc = ERROR_FAIL;
         goto out;
@@ -916,7 +935,7 @@ int libxl__create_device_model(libxl__gc *gc,
         rc = ERROR_FAIL;
         goto out;
     }
-    args = libxl__build_device_model_args(gc, dm, domid, guest_config, state);
+    args = libxl__build_device_model_args(gc, dm, domid, dmid, guest_config, state);
     if (!args) {
         rc = ERROR_FAIL;
         goto out;
@@ -930,7 +949,11 @@ int libxl__create_device_model(libxl__gc *gc,
         free(path);
     }
 
-    path = libxl__sprintf(gc, "/local/domain/0/device-model/%d", domid);
+    if (dmid == ~0)
+        path = libxl__sprintf(gc, "/local/domain/0/device-model/%d", domid);
+    else
+        path = libxl__sprintf(gc, "/local/domain/0/dms/%u/%u", domid,
+                              guest_config->dms[dmid].id);
     xs_mkdir(ctx->xsh, XBT_NULL, path);
 
     if (b_info->type == LIBXL_DOMAIN_TYPE_HVM &&
@@ -939,8 +962,57 @@ int libxl__create_device_model(libxl__gc *gc,
         libxl__xs_write(gc, XBT_NULL, libxl__sprintf(gc, "%s/disable_pf", path),
                     "%d", !libxl_defbool_val(b_info->u.hvm.xen_platform_pci));
 
+    if (dmid != ~0) {
+        path = libxl__sprintf(gc, "%s/image/dms/%u/pci",
+                              libxl__xs_get_dompath(gc, domid),
+                              guest_config->dms[dmid].id);
+        xs_mkdir(ctx->xsh, XBT_NULL, path);
+
+        if (guest_config->dms[dmid].pcis) {
+            for (i = 0; guest_config->dms[dmid].pcis[i]; i++) {
+                path = xs_get_domain_path(ctx->xsh, domid);
+                path = libxl__sprintf(gc, "%s/image/dms/%u/pci/%u",
+                                      path, guest_config->dms[dmid].id, i);
+                libxl__xs_write(gc, XBT_NULL, path,
+                                "%s", guest_config->dms[dmid].pcis[i]);
+            }
+        }
+
+        path = libxl__sprintf(gc, "%s/image/dms/%u/mmio",
+                              libxl__xs_get_dompath(gc, domid),
+                              guest_config->dms[dmid].id);
+        xs_mkdir(ctx->xsh, XBT_NULL, path);
+
+        if (guest_config->dms[dmid].mmios) {
+            for (i = 0; guest_config->dms[dmid].mmios[i]; i++) {
+                path = xs_get_domain_path(ctx->xsh, domid);
+                path = libxl__sprintf(gc, "%s/image/dms/%u/mmio/%u",
+                                      path, guest_config->dms[dmid].id, i);
+                libxl__xs_write(gc, XBT_NULL, path,
+                                "%s", guest_config->dms[dmid].mmios[i]);
+            }
+        }
+
+        path = libxl__sprintf(gc, "%s/image/dms/%u/pio",
+                              libxl__xs_get_dompath(gc, domid),
+                              guest_config->dms[dmid].id);
+        xs_mkdir(ctx->xsh, XBT_NULL, path);
+
+        if (guest_config->dms[dmid].pios) {
+            for (i = 0; guest_config->dms[dmid].pios[i]; i++) {
+                path = xs_get_domain_path(ctx->xsh, domid);
+                path = libxl__sprintf(gc, "%s/image/dms/%u/pio/%u",
+                                      path, guest_config->dms[dmid].id, i);
+                libxl__xs_write(gc, XBT_NULL, path,
+                                "%s", guest_config->dms[dmid].pios[i]);
+            }
+        }
+    }
+
     libxl_create_logfile(ctx,
-                         libxl__sprintf(gc, "qemu-dm-%s", c_info->name),
+                         libxl__sprintf(gc, "qemu-%s-%s",
+                                        (dmid == ~0) ? "dm" : guest_config->dms[dmid].name,
+                                        c_info->name),
                          &logfile);
     logfile_w = open(logfile, O_WRONLY|O_CREAT|O_APPEND, 0644);
     free(logfile);
@@ -960,7 +1032,15 @@ int libxl__create_device_model(libxl__gc *gc,
 
     p->domid = domid;
     p->dom_path = libxl__xs_get_dompath(gc, domid);
-    p->pid_path = "image/device-model-pid";
+    if (dmid == ~0) {
+        p->pid_path = "image/device-model-pid";
+        p->dmid = 0;
+    }
+    else {
+        p->pid_path = libxl__sprintf(gc, "image/dms/%u-pid", guest_config->dms[dmid].id);
+        p->dmid = guest_config->dms[dmid].id;
+    }
+
     if (!p->dom_path) {
         rc = ERROR_FAIL;
         goto out_close;
@@ -985,7 +1065,8 @@ retry_transaction:
         }
     }
 
-    rc = libxl__spawn_spawn(gc, p->for_spawn, "device model",
+    path = (dmid == ~0) ? "device model" : guest_config->dms[dmid].name;
+    rc = libxl__spawn_spawn(gc, p->for_spawn, path,
                             libxl_spawner_record_pid, p);
     if (rc < 0)
         goto out_close;
@@ -1011,8 +1092,14 @@ int libxl__confirm_device_model_startup(libxl__gc *gc,
 {
     char *path;
     int domid = starting->domid;
+    uint32_t dmid = starting->dmid;
     int ret, ret2;
-    path = libxl__sprintf(gc, "/local/domain/0/device-model/%d/state", domid);
+
+    if (!dmid)
+        path = libxl__sprintf(gc, "/local/domain/0/device-model/%d/state", domid);
+    else
+        path = libxl__sprintf(gc, "/local/domain/0/dms/%u/%u/state", domid, dmid);
+
     ret = libxl__spawn_confirm_offspring_startup(gc,
                                      LIBXL_DEVICE_MODEL_START_TIMEOUT,
                                      "Device Model", path, "running", starting);
@@ -1125,10 +1212,122 @@ int libxl__create_xenpv_qemu(libxl__gc *gc, uint32_t domid,
                              libxl__domain_build_state *state,
                              libxl__spawner_starting **starting_r)
 {
-    libxl__create_device_model(gc, domid, guest_config, state, starting_r);
+    libxl__create_device_model(gc, domid, ~0, guest_config, state, starting_r);
     return 0;
 }
 
+int libxl__launch_dms(libxl__gc *gc,
+                      libxl_domid domid,
+                      libxl__domain_build_state *state,
+                      libxl_domain_config *guest_config)
+{
+    char *path;
+    libxl_ctx *ctx = libxl__gc_owner(gc);
+    libxl_dm *dm = NULL;
+    int i;
+    libxl__spawner_starting *dm_starting = 0;
+    int ret = 0;
+
+    path = libxl__sprintf(gc, "/local/domain/0/dms/%u", domid);
+    xs_mkdir(ctx->xsh, XBT_NULL, path);
+    path = xs_get_domain_path(ctx->xsh, domid);
+    xs_mkdir(ctx->xsh, XBT_NULL, libxl__sprintf(gc, "%s/image/dms", path));
+    free(path);
+
+    for (i = 0; i < guest_config->num_dms; i++)
+    {
+        dm = &guest_config->dms[i];
+        ret = libxl__create_device_model(gc, domid, i, guest_config,
+                                         state, &dm_starting);
+        if (ret < 0)
+            fprintf(stderr, "Can't launch dm %s\n",
+                    guest_config->dms[i].name);
+        if (dm_starting) {
+            libxl__qmp_initializations(gc, domid, guest_config, dm->id);
+            ret = libxl__confirm_device_model_startup(gc, state, dm_starting);
+            if (ret < 0) {
+                LIBXL__LOG(ctx, LIBXL__LOG_ERROR,
+                           "dm %s did not start: %d",
+                           guest_config->dms[i].name,
+                           ret);
+                break;
+            }
+        }
+    }
+
+    return ret;
+}
+
+static int libxl_destroy_dm(libxl__gc *gc,
+                            libxl_domid domid,
+                            char *dmid)
+{
+    libxl_ctx *ctx = libxl__gc_owner(gc);
+    char *path;
+    char *pid;
+    int ret = 0;
+
+    path = libxl__sprintf(gc, "/local/domain/%u/image/dms/%s-pid",
+                          domid, dmid);
+
+
+    pid = libxl__xs_read(gc, XBT_NULL, path);
+    if (!pid)
+        return ERROR_FAIL;
+
+    ret = kill(atoi(pid), SIGHUP);
+
+    if (ret < 0 && errno == ESRCH) {
+        LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "Daemon %s already exited", dmid);
+        ret = 0;
+    } else if (ret == 0) {
+        LIBXL__LOG(ctx, LIBXL__LOG_DEBUG, "Daemon %s signaled", dmid);
+        ret = 0;
+    } else {
+        LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "failed to kill Daemon %s [%d]",
+                   dmid, atoi(pid));
+        ret = ERROR_FAIL;
+    }
+
+    if (!ret)
+    {
+        path = libxl__sprintf(gc, "/local/domain/0/dms/%u/%s",
+                             domid, dmid);
+
+        xs_rm(ctx->xsh, XBT_NULL, path);
+    }
+
+    return 0;
+}
+
+int libxl__destroy_dms(libxl__gc *gc,
+                       libxl_domid domid)
+{
+    libxl_ctx *ctx = libxl__gc_owner(gc);
+    int ret = 0;
+    char **dir = NULL;
+    unsigned int n;
+    char *path;
+    unsigned int i = 0;
+
+    path = libxl__sprintf(gc, "/local/domain/0/dms/%u", domid);
+
+    dir = libxl__xs_directory(gc, XBT_NULL, path, &n);
+    if (dir)
+    {
+        for (i = 0; i < n; i++)
+        {
+            if (libxl_destroy_dm(gc, domid, dir[i]))
+                ret = ERROR_FAIL;
+        }
+    }
+
+    if (!ret)
+        xs_rm(ctx->xsh, XBT_NULL, libxl__sprintf(gc, "/local/domain/0/dms/%u", domid));
+
+    return ret;
+}
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c
index 9b33267..a08b445 100644
--- a/tools/libxl/libxl_dom.c
+++ b/tools/libxl/libxl_dom.c
@@ -339,6 +339,9 @@ static const char *libxl__domain_firmware(libxl__gc *gc,
         case LIBXL_DEVICE_MODEL_VERSION_QEMU_XEN:
             firmware = "hvmloader";
             break;
+        case LIBXL_DEVICE_MODEL_VERSION_MULTIPLE_QEMU_XEN:
+            firmware = "hvmloader";
+            break;
         default:
             LIBXL__LOG(ctx, LIBXL__LOG_ERROR, "invalid device model version %d",
                        info->device_model_version);
@@ -364,7 +367,8 @@ int libxl__build_hvm(libxl__gc *gc, uint32_t domid,
         domid,
         (info->max_memkb - info->video_memkb) / 1024,
         (info->target_memkb - info->video_memkb) / 1024,
-        firmware);
+        firmware,
+        info->u.hvm.max_servers * 2 + 1);
     if (ret) {
         LIBXL__LOG_ERRNOVAL(ctx, LIBXL__LOG_ERROR, ret, "hvm building failed");
         goto out;
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 52a2429..d1dd083 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -893,11 +893,20 @@ _hidden int libxl__domain_build(libxl__gc *gc,
                                 uint32_t domid,
                                 libxl__domain_build_state *state);
 
+/* for deamon create */
+_hidden int libxl__launch_dms(libxl__gc *gc,
+                              libxl_domid domid,
+                              libxl__domain_build_state *state,
+                              libxl_domain_config *guest_config);
+_hidden int libxl__destroy_dms(libxl__gc *gc,
+                               libxl_domid domid);
+
 /* for device model creation */
 _hidden const char *libxl__domain_device_model(libxl__gc *gc,
                                         const libxl_domain_build_info *info);
 _hidden int libxl__create_device_model(libxl__gc *gc,
                               int domid,
+                              uint32_t dmid,
                               libxl_domain_config *guest_config,
                               libxl__domain_build_state *state,
                               libxl__spawner_starting **starting_r);
-- 
Julien Grall