From mboxrd@z Thu Jan 1 00:00:00 1970 From: Wen Congyang Subject: Re: [PATCH 23/27] tools/libxl: [RFC] Write checkpoint records into the stream Date: Thu, 18 Jun 2015 11:13:52 +0800 Message-ID: <55823770.9040801@cn.fujitsu.com> References: <1434375880-30914-1-git-send-email-andrew.cooper3@citrix.com> <1434375880-30914-24-git-send-email-andrew.cooper3@citrix.com> Mime-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit Return-path: In-Reply-To: <1434375880-30914-24-git-send-email-andrew.cooper3@citrix.com> List-Unsubscribe: , List-Post: List-Help: List-Subscribe: , Sender: xen-devel-bounces@lists.xen.org Errors-To: xen-devel-bounces@lists.xen.org To: Andrew Cooper , Xen-devel Cc: Ian Jackson , Yang Hongyang , Wei Liu , Ian Campbell List-Id: xen-devel@lists.xenproject.org On 06/15/2015 09:44 PM, Andrew Cooper wrote: > when signalled to do so by libxl__remus_domain_checkpoint_callback() > > Signed-off-by: Andrew Cooper > CC: Ian Campbell > CC: Ian Jackson > CC: Wei Liu > --- > tools/libxl/libxl_dom.c | 16 +++--- > tools/libxl/libxl_internal.h | 7 +++ > tools/libxl/libxl_stream_write.c | 111 ++++++++++++++++++++++++++++++++++++-- > 3 files changed, 121 insertions(+), 13 deletions(-) > > diff --git a/tools/libxl/libxl_dom.c b/tools/libxl/libxl_dom.c > index 06bfaab..3597a91 100644 > --- a/tools/libxl/libxl_dom.c > +++ b/tools/libxl/libxl_dom.c > @@ -1867,8 +1867,8 @@ static void remus_devices_preresume_cb(libxl__egc *egc, > > /*----- remus asynchronous checkpoint callback -----*/ > > -static void remus_checkpoint_dm_saved(libxl__egc *egc, > - libxl__domain_suspend_state *dss, int rc); > +static void remus_checkpoint_stream_written( > + libxl__egc *egc, libxl__domain_suspend_state *dss, int rc); > static void remus_devices_commit_cb(libxl__egc *egc, > libxl__remus_devices_state *rds, > int rc); > @@ -1882,16 +1882,11 @@ static void libxl__remus_domain_checkpoint_callback(void *data) > libxl__egc *egc = dss->shs.egc; > STATE_AO_GC(dss->ao); > > - /* This would go into tailbuf. */ > - if (dss->hvm) { > - libxl__domain_save_device_model(egc, dss, remus_checkpoint_dm_saved); > - } else { > - remus_checkpoint_dm_saved(egc, dss, 0); > - } > + libxl__stream_write_start_checkpoint(egc, &dss->sws); > } > > -static void remus_checkpoint_dm_saved(libxl__egc *egc, > - libxl__domain_suspend_state *dss, int rc) > +static void remus_checkpoint_stream_written( > + libxl__egc *egc, libxl__domain_suspend_state *dss, int rc) > { > /* Convenience aliases */ > libxl__remus_devices_state *const rds = &dss->rds; > @@ -2036,6 +2031,7 @@ void libxl__domain_suspend(libxl__egc *egc, libxl__domain_suspend_state *dss) > callbacks->suspend = libxl__remus_domain_suspend_callback; > callbacks->postcopy = libxl__remus_domain_resume_callback; > callbacks->checkpoint = libxl__remus_domain_checkpoint_callback; > + dss->sws.checkpoint_callback = remus_checkpoint_stream_written; > } else > callbacks->suspend = libxl__domain_suspend_callback; > > diff --git a/tools/libxl/libxl_internal.h b/tools/libxl/libxl_internal.h > index 82cd792..bf1c377 100644 > --- a/tools/libxl/libxl_internal.h > +++ b/tools/libxl/libxl_internal.h > @@ -2879,17 +2879,24 @@ struct libxl__stream_write_state { > void (*completion_callback)(libxl__egc *egc, > libxl__domain_suspend_state *dss, > int rc); > + void (*checkpoint_callback)(libxl__egc *egc, > + libxl__domain_suspend_state *dss, > + int rc); > /* Private */ > int rc; > int joined_rc; > size_t padding; > bool running; > + bool in_checkpoint; > libxl__datacopier_state dc; > }; > > _hidden void libxl__stream_write_start(libxl__egc *egc, > libxl__stream_write_state *stream); > > +_hidden void libxl__stream_write_start_checkpoint( > + libxl__egc *egc, libxl__stream_write_state *stream); > + > _hidden void libxl__stream_write_abort(libxl__egc *egc, > libxl__stream_write_state *stream, > int rc); > diff --git a/tools/libxl/libxl_stream_write.c b/tools/libxl/libxl_stream_write.c > index d28a8a5..40f2cb7 100644 > --- a/tools/libxl/libxl_stream_write.c > +++ b/tools/libxl/libxl_stream_write.c > @@ -23,6 +23,9 @@ > * - libxl__stream_write_start() > * - Start writing a stream from the start. > * > + * - libxl__stream_write_start() > + * - Write the records which form a checkpoint into a stream. > + * > * In normal operation, there are two tasks running at once; this stream > * processing, and the the libxl-save-helper. check_stream_finished() is used > * to join all the tasks in both success and error cases. > @@ -39,6 +42,12 @@ > * - Toolstack record > * - if (hvm), Qemu record > * - End record > + * > + * For checkpointed stream, there is a second loop which is triggered by a > + * save-helper checkpoint callback. It writes: > + * - Toolstack record > + * - if (hvm), Qemu record > + * - Checkpoint end record > */ > > static const uint8_t zero_padding[1U << REC_ALIGN_ORDER] = { 0 }; > @@ -81,6 +90,16 @@ static void end_record_done(libxl__egc *egc, > libxl__datacopier_state *dc, > int onwrite, int errnoval); > > +/* Event callbacks unique to checkpointed streams. */ > +static void checkpoint_done(libxl__egc *egc, > + libxl__stream_write_state *stream, > + int rc); > +static void write_checkpoint_end_record(libxl__egc *egc, > + libxl__stream_write_state *stream); > +static void checkpoint_end_record_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval); > + > void libxl__stream_write_start(libxl__egc *egc, > libxl__stream_write_state *stream) > { > @@ -119,6 +138,16 @@ void libxl__stream_write_start(libxl__egc *egc, > stream_failed(egc, stream, ret); > } > > +void libxl__stream_write_start_checkpoint(libxl__egc *egc, > + libxl__stream_write_state *stream) > +{ > + assert(stream->running); > + assert(!stream->in_checkpoint); > + stream->in_checkpoint = true; > + > + write_toolstack_record(egc, stream); > +} > + > void libxl__stream_write_abort(libxl__egc *egc, > libxl__stream_write_state *stream, int rc) > { > @@ -130,6 +159,7 @@ static void stream_success(libxl__egc *egc, libxl__stream_write_state *stream) > stream->rc = 0; > stream->running = false; > > + assert(!stream->in_checkpoint); > stream_done(egc, stream); > } > > @@ -139,6 +169,15 @@ static void stream_failed(libxl__egc *egc, > assert(rc); > stream->rc = rc; > > + /* > + *If we are in a checkpoint, pass the failure to libxc, which will come > + * back around to us via libxl__xc_domain_save_done(). > + */ > + if (stream->in_checkpoint) { I think we should set running to false here too. Thanks Wen Congyang > + checkpoint_done(egc, stream, rc); > + return; > + } > + > if (stream->running) { > stream->running = false; > stream_done(egc, stream); > @@ -151,6 +190,7 @@ static void stream_done(libxl__egc *egc, > libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); > > assert(!stream->running); > + assert(!stream->in_checkpoint); > > check_stream_finished(egc, dss, stream->rc, "stream"); > } > @@ -335,8 +375,12 @@ static void toolstack_record_done(libxl__egc *egc, > > if (dss->type == LIBXL_DOMAIN_TYPE_HVM) > write_emulator_record(egc, stream); > - else > - write_end_record(egc, stream); > + else { > + if (stream->in_checkpoint) > + write_checkpoint_end_record(egc, stream); > + else > + write_end_record(egc, stream); > + } > > return; > > @@ -473,7 +517,10 @@ static void emulator_padding_done(libxl__egc *egc, > goto err; > } > > - write_end_record(egc, stream); > + if (stream->in_checkpoint) > + write_checkpoint_end_record(egc, stream); > + else > + write_end_record(egc, stream); > return; > > err: > @@ -526,6 +573,64 @@ static void end_record_done(libxl__egc *egc, > stream_failed(egc, stream, ret); > } > > +static void checkpoint_done(libxl__egc *egc, > + libxl__stream_write_state *stream, > + int rc) > +{ > + libxl__domain_suspend_state *dss = CONTAINER_OF(stream, *dss, sws); > + > + assert(stream->in_checkpoint); > + stream->in_checkpoint = false; > + stream->checkpoint_callback(egc, dss, rc); > +} > + > +static void write_checkpoint_end_record(libxl__egc *egc, > + libxl__stream_write_state *stream) > +{ > + libxl__datacopier_state *dc = &stream->dc; > + STATE_AO_GC(stream->ao); > + struct libxl_sr_rec_hdr rec = { REC_TYPE_CHECKPOINT_END, 0 }; > + int ret = 0; > + > + assert(stream->in_checkpoint); > + > + dc->copywhat = "checkpoint record"; > + dc->writewhat = "save/migration stream"; > + dc->callback = checkpoint_end_record_done; > + > + ret = libxl__datacopier_start(dc); > + if (ret) > + goto err; > + > + libxl__datacopier_prefixdata(egc, dc, &rec, sizeof(rec)); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > +static void checkpoint_end_record_done(libxl__egc *egc, > + libxl__datacopier_state *dc, > + int onwrite, int errnoval) > +{ > + libxl__stream_write_state *stream = CONTAINER_OF(dc, *stream, dc); > + STATE_AO_GC(stream->ao); > + int ret = 0; > + > + if (onwrite || errnoval) { > + ret = ERROR_FAIL; > + goto err; > + } > + > + checkpoint_done(egc, stream, 0); > + return; > + > + err: > + assert(ret); > + stream_failed(egc, stream, ret); > +} > + > /* > * Local variables: > * mode: C >