Re: [PATCH 25/27] tools/libxl: [RFC] Handle checkpoint records in a libxl migration v2 stream

[Wen Congyang <wency@cn.fujitsu.com>]

On 06/15/2015 09:44 PM, Andrew Cooper wrote:
> This is the final bit of untangling for Remus.
> 
> Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
> CC: Ian Campbell <Ian.Campbell@citrix.com>
> CC: Ian Jackson <Ian.Jackson@eu.citrix.com>
> CC: Wei Liu <wei.liu2@citrix.com>
> ---
>  tools/libxl/libxl_create.c      |   25 ++++++++++++++++
>  tools/libxl/libxl_internal.h    |    6 ++++
>  tools/libxl/libxl_stream_read.c |   62 +++++++++++++++++++++++++++++++++++++++
>  3 files changed, 93 insertions(+)
> 
> diff --git a/tools/libxl/libxl_create.c b/tools/libxl/libxl_create.c
> index 7dd7130..ac918bd 100644
> --- a/tools/libxl/libxl_create.c
> +++ b/tools/libxl/libxl_create.c
> @@ -747,6 +747,27 @@ static int store_libxl_entry(libxl__gc *gc, uint32_t domid,
>          libxl_device_model_version_to_string(b_info->device_model_version));
>  }
>  
> +/*----- remus asynchronous checkpoint callback -----*/
> +
> +static void remus_checkpoint_stream_done(
> +    libxl__egc *egc, libxl__domain_create_state *dcs, int rc);
> +
> +static void libxl__remus_domain_checkpoint_callback(void *data)
> +{
> +    libxl__save_helper_state *shs = data;
> +    libxl__domain_create_state *dcs = CONTAINER_OF(shs, *dcs, shs);
> +    libxl__egc *egc = dcs->shs.egc;
> +    STATE_AO_GC(dcs->ao);
> +
> +    libxl__stream_read_start_checkpoint(egc, &dcs->srs);
> +}
> +
> +static void remus_checkpoint_stream_done(
> +    libxl__egc *egc, libxl__domain_create_state *dcs, int rc)
> +{
> +    libxl__xc_domain_saverestore_async_callback_done(egc, &dcs->shs, rc);
> +}
> +
>  /*----- main domain creation -----*/
>  
>  /* We have a linear control flow; only one event callback is
> @@ -1008,6 +1029,8 @@ static void domcreate_bootloader_done(libxl__egc *egc,
>      libxl_domain_config *const d_config = dcs->guest_config;
>      const int restore_fd = dcs->restore_fd;
>      libxl__domain_build_state *const state = &dcs->build_state;
> +    libxl__srm_restore_autogen_callbacks *const callbacks =
> +        &dcs->shs.callbacks.restore.a;
>  
>      if (rc) {
>          domcreate_rebuild_done(egc, dcs, rc);
> @@ -1035,6 +1058,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
>      }
>  
>      /* Restore */
> +    callbacks->checkpoint = libxl__remus_domain_checkpoint_callback;
>  
>      rc = libxl__build_pre(gc, domid, d_config, state);
>      if (rc)
> @@ -1044,6 +1068,7 @@ static void domcreate_bootloader_done(libxl__egc *egc,
>      dcs->srs.fd = restore_fd;
>      dcs->srs.legacy = (dcs->restore_params.stream_version == 1);
>      dcs->srs.completion_callback = domcreate_stream_done;
> +    dcs->srs.checkpoint_callback = remus_checkpoint_stream_done;
>  
>      libxl__stream_read_start(egc, &dcs->srs);
>      return;
> diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h
> index bf1c377..e271a0b 100644
> --- a/tools/libxl/libxl_internal.h
> +++ b/tools/libxl/libxl_internal.h
> @@ -3205,11 +3205,15 @@ struct libxl__stream_read_state {
>      void (*completion_callback)(libxl__egc *egc,
>                                  libxl__domain_create_state *dcs,
>                                  int rc);
> +    void (*checkpoint_callback)(libxl__egc *egc,
> +                                libxl__domain_create_state *dcs,
> +                                int rc);
>      /* Private */
>      libxl__carefd *v2_carefd;
>      int rc;
>      int joined_rc;
>      bool running;
> +    bool in_checkpoint;
>      libxl__datacopier_state dc;
>      size_t expected_len;
>      libxl_sr_hdr hdr;
> @@ -3222,6 +3226,8 @@ _hidden void libxl__stream_read_start(libxl__egc *egc,
>  
>  _hidden void libxl__stream_read_continue(libxl__egc *egc,
>                                           libxl__stream_read_state *stream);
> +_hidden void libxl__stream_read_start_checkpoint(
> +    libxl__egc *egc, libxl__stream_read_state *stream);
>  
>  _hidden void libxl__stream_read_abort(libxl__egc *egc,
>                                        libxl__stream_read_state *stream, int rc);
> diff --git a/tools/libxl/libxl_stream_read.c b/tools/libxl/libxl_stream_read.c
> index a8cd2c3..09ef0aa 100644
> --- a/tools/libxl/libxl_stream_read.c
> +++ b/tools/libxl/libxl_stream_read.c
> @@ -80,6 +80,10 @@ static void emulator_padding_done(libxl__egc *egc,
>                                    libxl__datacopier_state *dc,
>                                    int onwrite, int errnoval);
>  
> +/* Error handling for checkpoint mini-loop. */
> +static void checkpoint_done(libxl__egc *egc,
> +                            libxl__stream_read_state *stream, int rc);
> +
>  void libxl__stream_read_start(libxl__egc *egc,
>                                libxl__stream_read_state *stream)
>  {
> @@ -162,6 +166,35 @@ void libxl__stream_read_continue(libxl__egc *egc,
>      stream_failed(egc, stream, ret);
>  }
>  
> +void libxl__stream_read_start_checkpoint(libxl__egc *egc,
> +                                         libxl__stream_read_state *stream)
> +{
> +    libxl__datacopier_state *dc = &stream->dc;
> +    int ret = 0;
> +
> +    assert(stream->running);
> +    assert(!stream->in_checkpoint);
> +    stream->in_checkpoint = true;
> +

I think you can call libxl__stream_read_continue() here.

Thanks
Wen Congyang

> +    /* Read a record header. */
> +    dc->readwhat = "record header";
> +    dc->readbuf = &stream->rec_hdr;
> +    stream->expected_len = dc->bytes_to_read = sizeof(stream->rec_hdr);
> +    dc->used = 0;
> +    dc->callback = record_header_done;
> +
> +    ret = libxl__datacopier_start(dc);
> +    if (ret)
> +        goto err;
> +
> +    assert(!ret);
> +    return;
> +
> + err:
> +    assert(ret);
> +    stream_failed(egc, stream, ret);
> +}
> +
>  void libxl__stream_read_abort(libxl__egc *egc,
>                                libxl__stream_read_state *stream, int rc)
>  {
> @@ -182,6 +215,15 @@ static void stream_failed(libxl__egc *egc,
>      assert(rc);
>      stream->rc = rc;
>  
> +    /*
> +     *If we are in a checkpoint, pass the failure to libxc, which will come
> +     * back around to us via libxl__xc_domain_restore_done().
> +     */
> +    if (stream->in_checkpoint) {
> +        checkpoint_done(egc, stream, rc);
> +        return;
> +    }
> +
>      if (stream->running) {
>          stream->running = false;
>          stream_done(egc, stream);
> @@ -194,6 +236,7 @@ static void stream_done(libxl__egc *egc,
>      libxl__domain_create_state *dcs = CONTAINER_OF(stream, *dcs, srs);
>  
>      assert(!stream->running);
> +    assert(!stream->in_checkpoint);
>  
>      if (stream->v2_carefd)
>          libxl__carefd_close(stream->v2_carefd);
> @@ -452,6 +495,15 @@ static void process_record(libxl__egc *egc,
>          read_emulator_body(egc, stream);
>          break;
>  
> +    case REC_TYPE_CHECKPOINT_END:
> +        if (!stream->in_checkpoint) {
> +            LOG(ERROR, "Unexpected CHECKPOINT_END record in stream");
> +            ret = ERROR_FAIL;
> +            goto err;
> +        }
> +        checkpoint_done(egc, stream, 0);
> +        break;
> +
>      default:
>          LOG(ERROR, "Unrecognised record 0x%08x", rec_hdr->type);
>          ret = ERROR_FAIL;
> @@ -592,6 +644,16 @@ static void emulator_padding_done(libxl__egc *egc,
>      stream_failed(egc, stream, ret);
>  }
>  
> +static void checkpoint_done(libxl__egc *egc,
> +                            libxl__stream_read_state *stream, int rc)
> +{
> +    libxl__domain_create_state *dcs = CONTAINER_OF(stream, *dcs, srs);
> +
> +    assert(stream->in_checkpoint);
> +    stream->in_checkpoint = false;
> +    stream->checkpoint_callback(egc, dcs, rc);
> +}
> +
>  /*
>   * Local variables:
>   * mode: C
>