[PATCH 18/20] libxl: provide progress reporting for long-running operations

[Ian Jackson <ian.jackson@eu.citrix.com>]

This will be used for reporting, during domain creation, that the
console is ready.

Signed-off-by: Ian Jackson <ian.jackson@eu.citrix.com>
---
 tools/libxl/libxl.h          |   45 +++++++++++++++
 tools/libxl/libxl_event.c    |  122 ++++++++++++++++++++++++++++++++----------
 tools/libxl/libxl_internal.h |   28 ++++++++++
 3 files changed, 167 insertions(+), 28 deletions(-)

diff --git a/tools/libxl/libxl.h b/tools/libxl/libxl.h
index 6f59364..1bfe684 100644
--- a/tools/libxl/libxl.h
+++ b/tools/libxl/libxl.h
@@ -435,6 +435,51 @@ typedef struct {
     } u;
 } libxl_asyncop_how;
 
+/*
+ * Some more complex asynchronous operations can report intermediate
+ * progress.  How this is to be reported is controlled, for each
+ * function, by a parameter
+ *    libxl_asyncprogress_how *aop_FOO_how;
+ * for each kind of progress FOO supported by that function.  Each
+ * such kind of progress is associated with an event type.
+ *
+ * The function description will document whether, when, and how
+ * many times, the intermediate progress will be reported, and
+ * what the corresponding event type(s) are.
+ *
+ * If aop_FOO_how==NULL, intermediate progress reports are discarded.
+ *
+ * If aop_FOO_how->callback==NULL, intermediate progress reports
+ * generate libxl events which can be obtained from libxl_event_wait
+ * or libxl_event_check.
+ *
+ * If aop_FOO_how->callback!=NULL, libxl will report intermediate
+ * progress by calling callback(ctx, &event, for_callback).
+ *
+ * The rules for these events are otherwise the same as those for
+ * ordinary events.  The reentrancy and threading rules for the
+ * callback are the same as those for ao completion callbacks.
+ *
+ * Note that the callback, if provided, is responsible for freeing
+ * the event.
+ *
+ * If callbacks are requested, they will be made, and returned, before
+ * the long-running libxl operation is considered finished (so if the
+ * long-running libxl operation was invoked with ao_how==NULL then any
+ * callbacks will occur strictly before the long-running operation
+ * returns).  However, the callbacks may occur on any thread.
+ *
+ * In general, otherwise, no promises are made about the relative
+ * order of callbacks in a multithreaded program.  In particular
+ * different callbacks relating to the same long-running operation may
+ * be delivered out of order.
+ */
+
+typedef struct {
+    void (*callback)(libxl_ctx *ctx, libxl_event*, void *for_callback);
+    libxl_ev_user for_event; /* always used */
+    void *for_callback; /* passed to callback */
+} libxl_asyncprogress_how;
 
 #define LIBXL_VERSION 0
 
diff --git a/tools/libxl/libxl_event.c b/tools/libxl/libxl_event.c
index 1b7495a..f355e3d 100644
--- a/tools/libxl/libxl_event.c
+++ b/tools/libxl/libxl_event.c
@@ -899,12 +899,25 @@ static void egc_run_callbacks(libxl__egc *egc)
 {
     EGC_GC;
     libxl_event *ev, *ev_tmp;
+    libxl__aop_occurred *aop, *aop_tmp;
 
     LIBXL_TAILQ_FOREACH_SAFE(ev, &egc->occurred_for_callback, link, ev_tmp) {
         LIBXL_TAILQ_REMOVE(&egc->occurred_for_callback, ev, link);
         CTX->event_hooks->event_occurs(CTX->event_hooks_user, ev);
     }
 
+    LIBXL_TAILQ_FOREACH_SAFE(aop, &egc->aops_for_callback, entry, aop_tmp) {
+        LIBXL_TAILQ_REMOVE(&egc->aops_for_callback, aop, entry);
+        aop->how->callback(CTX, aop->ev, aop->how->for_callback);
+
+        CTX_LOCK;
+        aop->ao->progress_reports_outstanding--;
+        libxl__ao_complete_check_progress_reports(egc, aop->ao);
+        CTX_UNLOCK;
+
+        free(aop);
+    }
+
     libxl__ao *ao, *ao_tmp;
     LIBXL_TAILQ_FOREACH_SAFE(ao, &egc->aos_for_callback,
                              entry_for_callback, ao_tmp) {
@@ -1285,6 +1298,7 @@ void libxl__ao_abort(libxl__ao *ao)
     assert(ao->magic == LIBXL__AO_MAGIC);
     assert(ao->in_initiator);
     assert(!ao->complete);
+    assert(!ao->progress_reports_outstanding);
     libxl__ao__destroy(CTX, ao);
 }
 
@@ -1295,6 +1309,23 @@ void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, int rc)
     ao->complete = 1;
     ao->rc = rc;
 
+    libxl__ao_complete_check_progress_reports(egc, ao);
+}
+
+void libxl__ao_complete_check_progress_reports(libxl__egc *egc, libxl__ao *ao)
+{
+    /* We don't consider an ao complete if it has any outstanding
+     * callbacks.  These callbacks might be outstanding on other
+     * threads, queued up in the other threads' egc's.  Those threads
+     * will, after making the callback, take out the lock again,
+     * decrememt progress_reports_outstanding, and call us again.
+     */
+
+    assert(ao->progress_reports_outstanding >= 0);
+
+    if (!ao->complete || ao->progress_reports_outstanding)
+        return;
+
     if (ao->poller) {
         assert(ao->in_initiator);
         if (!ao->constructing)
@@ -1316,34 +1347,6 @@ void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, int rc)
         libxl__ao__destroy(libxl__gc_owner(&egc->gc), ao);
 }
 
-libxl__ao *libxl__ao_create(libxl_ctx *ctx, uint32_t domid,
-                            const libxl_asyncop_how *how)
-{
-    libxl__ao *ao;
-
-    ao = calloc(1, sizeof(*ao));
-    if (!ao) goto out;
-
-    ao->magic = LIBXL__AO_MAGIC;
-    ao->constructing = 1;
-    ao->in_initiator = 1;
-    ao->poller = 0;
-    ao->domid = domid;
-    LIBXL_INIT_GC(ao->gc, ctx);
-
-    if (how) {
-        ao->how = *how;
-    } else {
-        ao->poller = libxl__poller_get(ctx);
-        if (!ao->poller) goto out;
-    }
-    return ao;
-
- out:
-    if (ao) libxl__ao__destroy(ctx, ao);
-    return NULL;
-}
-
 int libxl__ao_inprogress(libxl__ao *ao)
 {
     AO_GC;
@@ -1401,6 +1404,69 @@ int libxl__ao_inprogress(libxl__ao *ao)
 }
 
 
+/* progress reporting */
+
+/* The application indicates a desire to ignore events by passing NULL
+ * for how.  But we want to copy *how.  So we have this dummy function
+ * whose address is stored in callback if the app passed how==NULL. */
+static void dummy_asyncprogress_callback_ignore
+  (libxl_ctx *ctx, libxl_event *ev, void *for_callback) { }
+
+void libxl__ao_progress_gethow(libxl_asyncprogress_how *in_state,
+                               const libxl_asyncprogress_how *from_app) {
+    if (from_app)
+        *in_state = *from_app;
+    else
+        in_state->callback = dummy_asyncprogress_callback_ignore;
+}
+
+void libxl__ao_progress_report(libxl__egc *egc, libxl__ao *ao,
+        const libxl_asyncprogress_how *how, libxl_event *ev)
+{
+    ev->for_user = how->for_event;
+    if (how->callback == dummy_asyncprogress_callback_ignore) {
+        /* ignore */
+    } else if (how->callback) {
+        libxl__aop_occurred *aop = libxl__zalloc(0, sizeof(*aop));
+        ao->progress_reports_outstanding++;
+        aop->ao = ao;
+        aop->ev = ev;
+        aop->how = how;
+    } else {
+        libxl__event_occurred(egc, ev);
+    }
+}
+
+
+libxl__ao *libxl__ao_create(libxl_ctx *ctx, uint32_t domid,
+                            const libxl_asyncop_how *how)
+{
+    libxl__ao *ao;
+
+    ao = calloc(1, sizeof(*ao));
+    if (!ao) goto out;
+
+    ao->magic = LIBXL__AO_MAGIC;
+    ao->constructing = 1;
+    ao->in_initiator = 1;
+    ao->poller = 0;
+    ao->domid = domid;
+    LIBXL_INIT_GC(ao->gc, ctx);
+
+    if (how) {
+        ao->how = *how;
+    } else {
+        ao->poller = libxl__poller_get(ctx);
+        if (!ao->poller) goto out;
+    }
+    return ao;
+
+ out:
+    if (ao) libxl__ao__destroy(ctx, ao);
+    return NULL;
+}
+
+
 /*
  * Local variables:
  * mode: C
diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
index 56c3eec..4e97f5e 100644
--- a/tools/libxl/libxl_internal.h
+++ b/tools/libxl/libxl_internal.h
@@ -118,6 +118,7 @@ _hidden void libxl__log(libxl_ctx *ctx, xentoollog_level msglevel, int errnoval,
 typedef struct libxl__gc libxl__gc;
 typedef struct libxl__egc libxl__egc;
 typedef struct libxl__ao libxl__ao;
+typedef struct libxl__aop_occurred libxl__aop_occurred;
 
 void libxl__alloc_failed(libxl_ctx *, const char *func,
                          size_t nmemb, size_t size) __attribute__((noreturn));
@@ -372,6 +373,14 @@ struct libxl__egc {
     struct libxl__gc gc;
     struct libxl__event_list occurred_for_callback;
     LIBXL_TAILQ_HEAD(, libxl__ao) aos_for_callback;
+    LIBXL_TAILQ_HEAD(, libxl__aop_occurred) aops_for_callback;
+};
+
+struct libxl__aop_occurred {
+    LIBXL_TAILQ_ENTRY(libxl__aop_occurred) entry;
+    libxl__ao *ao;
+    libxl_event *ev;
+    const libxl_asyncprogress_how *how;
 };
 
 #define LIBXL__AO_MAGIC              0xA0FACE00ul
@@ -380,6 +389,7 @@ struct libxl__egc {
 struct libxl__ao {
     uint32_t magic;
     unsigned constructing:1, in_initiator:1, complete:1, notified:1;
+    int progress_reports_outstanding;
     int rc;
     libxl__gc gc;
     libxl_asyncop_how how;
@@ -1369,6 +1379,7 @@ libxl__device_model_version_running(libxl__gc *gc, uint32_t domid);
         LIBXL_INIT_GC((egc).gc,ctx);                    \
         LIBXL_TAILQ_INIT(&(egc).occurred_for_callback); \
         LIBXL_TAILQ_INIT(&(egc).aos_for_callback);      \
+        LIBXL_TAILQ_INIT(&(egc).aops_for_callback);     \
     } while(0)
 
 _hidden void libxl__egc_cleanup(libxl__egc *egc);
@@ -1415,6 +1426,9 @@ _hidden void libxl__egc_cleanup(libxl__egc *egc);
  *        pointer to the internal event generation request routines
  *        libxl__evgen_FOO, so that at some point a CALLBACK will be
  *        made when the operation is complete.
+ *      - if the operation provides progress reports, the aop_how(s)
+ *        must be copied into the per-operation structure using
+ *        libxl__ao_progress_gethow.
  *
  * - If initiation is successful, the initiating function needs
  *   to run libxl__ao_inprogress right before unlocking and
@@ -1424,6 +1438,10 @@ _hidden void libxl__egc_cleanup(libxl__egc *egc);
  *   call libxl__ao_abort before unlocking and returning whatever
  *   error code is appropriate (AO_ABORT macro).
  *
+ * - If the operation supports progress reports, it may generate
+ *   suitable events with NEW_EVENT and report them with
+ *   libxl__ao_progress_report (with the ctx locked).
+ *
  * - Later, some callback function, whose callback has been requested
  *   directly or indirectly, should call libxl__ao_complete (with the
  *   ctx locked, as it will generally already be in any event callback
@@ -1479,8 +1497,18 @@ _hidden int libxl__ao_inprogress(libxl__ao *ao); /* temporarily unlocks */
 _hidden void libxl__ao_abort(libxl__ao *ao);
 _hidden void libxl__ao_complete(libxl__egc *egc, libxl__ao *ao, int rc);
 
+/* Can be called at any time.  Use is essential for any aop user. */
+_hidden void libxl__ao_progress_gethow(libxl_asyncprogress_how *in_state,
+                                       const libxl_asyncprogress_how *from_app);
+
+/* Must be called with the ctx locked.  Will fill in ev->for_user,
+ * so caller need not do that. */
+_hidden void libxl__ao_progress_report(libxl__egc *egc, libxl__ao *ao,
+   const libxl_asyncprogress_how *how, libxl_event *ev /* consumed */);
+
 /* For use by ao machinery ONLY */
 _hidden void libxl__ao__destroy(libxl_ctx*, libxl__ao *ao);
+_hidden void libxl__ao_complete_check_progress_reports(libxl__egc*, libxl__ao*);
 
 
 /*
-- 
1.7.2.5