[Qemu-devel] [PATCH 3/3] replay: introduce block devices record/replay

[Pavel Dovgalyuk <Pavel.Dovgaluk@ispras.ru>]

This patch introduces a set of functions that implement recording
and replaying of block devices' operations. These functions form a thin
layer between blk_aio_ functions and replay subsystem.
All asynchronous block requests are added to the queue to be processed
at deterministically invoked record/replay checkpoints.
Queue is flushed at checkpoints and information about processed requests
is recorded to the log. In replay phase the queue is matched with
events read from the log. Therefore block devices requests are processed
deterministically.

Signed-off-by: Pavel Dovgalyuk <pavel.dovgaluk@ispras.ru>
---
 block/block-backend.c          |   71 +++++++++++---
 include/sysemu/block-backend.h |   16 +++
 include/sysemu/replay.h        |   27 +++++
 replay/Makefile.objs           |    1 
 replay/replay-block.c          |  202 ++++++++++++++++++++++++++++++++++++++++
 replay/replay-events.c         |   51 ++++++++++
 replay/replay-internal.h       |   24 +++++
 stubs/replay.c                 |   46 +++++++++
 8 files changed, 423 insertions(+), 15 deletions(-)
 create mode 100755 replay/replay-block.c

diff --git a/block/block-backend.c b/block/block-backend.c
index f41d326..d8820b9 100644
--- a/block/block-backend.c
+++ b/block/block-backend.c
@@ -16,6 +16,7 @@
 #include "block/throttle-groups.h"
 #include "sysemu/blockdev.h"
 #include "sysemu/sysemu.h"
+#include "sysemu/replay.h"
 #include "qapi-event.h"
 
 /* Number of coroutines to reserve per attached device model */
@@ -655,14 +656,14 @@ BlockAIOCB *blk_abort_aio_request(BlockBackend *blk,
 
     bh = aio_bh_new(blk_get_aio_context(blk), error_callback_bh, acb);
     acb->bh = bh;
-    qemu_bh_schedule(bh);
+    replay_bh_schedule_event(bh);
 
     return &acb->common;
 }
 
-BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
-                                 int nb_sectors, BdrvRequestFlags flags,
-                                 BlockCompletionFunc *cb, void *opaque)
+BlockAIOCB *blk_aio_write_zeroes_impl(BlockBackend *blk, int64_t sector_num,
+                                      int nb_sectors, BdrvRequestFlags flags,
+                                      BlockCompletionFunc *cb, void *opaque)
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
@@ -673,6 +674,13 @@ BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
                                  cb, opaque);
 }
 
+BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
+                                 int nb_sectors, BdrvRequestFlags flags,
+                                 BlockCompletionFunc *cb, void *opaque)
+{
+    return replay_aio_write_zeroes(blk, sector_num, nb_sectors, flags, cb, opaque);
+}
+
 int blk_pread(BlockBackend *blk, int64_t offset, void *buf, int count)
 {
     int ret = blk_check_byte_request(blk, offset, count);
@@ -720,7 +728,7 @@ int64_t blk_nb_sectors(BlockBackend *blk)
     return bdrv_nb_sectors(blk->bs);
 }
 
-BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
+BlockAIOCB *blk_aio_readv_impl(BlockBackend *blk, int64_t sector_num,
                           QEMUIOVector *iov, int nb_sectors,
                           BlockCompletionFunc *cb, void *opaque)
 {
@@ -732,9 +740,16 @@ BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
     return bdrv_aio_readv(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
 }
 
-BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
-                           QEMUIOVector *iov, int nb_sectors,
-                           BlockCompletionFunc *cb, void *opaque)
+BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
+                          QEMUIOVector *iov, int nb_sectors,
+                          BlockCompletionFunc *cb, void *opaque)
+{
+    return replay_aio_readv(blk, sector_num, iov, nb_sectors, cb, opaque);
+}
+
+BlockAIOCB *blk_aio_writev_impl(BlockBackend *blk, int64_t sector_num,
+                                QEMUIOVector *iov, int nb_sectors,
+                                BlockCompletionFunc *cb, void *opaque)
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
@@ -744,8 +759,15 @@ BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
     return bdrv_aio_writev(blk->bs, sector_num, iov, nb_sectors, cb, opaque);
 }
 
-BlockAIOCB *blk_aio_flush(BlockBackend *blk,
-                          BlockCompletionFunc *cb, void *opaque)
+BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
+                           QEMUIOVector *iov, int nb_sectors,
+                           BlockCompletionFunc *cb, void *opaque)
+{
+    return replay_aio_writev(blk, sector_num, iov, nb_sectors, cb, opaque);
+}
+
+BlockAIOCB *blk_aio_flush_impl(BlockBackend *blk,
+                               BlockCompletionFunc *cb, void *opaque)
 {
     if (!blk_is_available(blk)) {
         return blk_abort_aio_request(blk, cb, opaque, -ENOMEDIUM);
@@ -754,9 +776,15 @@ BlockAIOCB *blk_aio_flush(BlockBackend *blk,
     return bdrv_aio_flush(blk->bs, cb, opaque);
 }
 
-BlockAIOCB *blk_aio_discard(BlockBackend *blk,
-                            int64_t sector_num, int nb_sectors,
-                            BlockCompletionFunc *cb, void *opaque)
+BlockAIOCB *blk_aio_flush(BlockBackend *blk,
+                          BlockCompletionFunc *cb, void *opaque)
+{
+    return replay_aio_flush(blk, cb, opaque);
+}
+                          
+BlockAIOCB *blk_aio_discard_impl(BlockBackend *blk,
+                                 int64_t sector_num, int nb_sectors,
+                                 BlockCompletionFunc *cb, void *opaque)
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
     if (ret < 0) {
@@ -766,6 +794,13 @@ BlockAIOCB *blk_aio_discard(BlockBackend *blk,
     return bdrv_aio_discard(blk->bs, sector_num, nb_sectors, cb, opaque);
 }
 
+BlockAIOCB *blk_aio_discard(BlockBackend *blk,
+                            int64_t sector_num, int nb_sectors,
+                            BlockCompletionFunc *cb, void *opaque)
+{
+    return replay_aio_discard(blk, sector_num, nb_sectors, cb, opaque);
+}
+
 void blk_aio_cancel(BlockAIOCB *acb)
 {
     bdrv_aio_cancel(acb);
@@ -799,8 +834,8 @@ int blk_ioctl(BlockBackend *blk, unsigned long int req, void *buf)
     return bdrv_ioctl(blk->bs, req, buf);
 }
 
-BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
-                          BlockCompletionFunc *cb, void *opaque)
+BlockAIOCB *blk_aio_ioctl_impl(BlockBackend *blk, unsigned long int req, void *buf,
+                               BlockCompletionFunc *cb, void *opaque)
 {
     if (!blk_is_available(blk)) {
         return blk_abort_aio_request(blk, cb, opaque, -ENOMEDIUM);
@@ -809,6 +844,12 @@ BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
     return bdrv_aio_ioctl(blk->bs, req, buf, cb, opaque);
 }
 
+BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
+                          BlockCompletionFunc *cb, void *opaque)
+{
+    return replay_aio_ioctl(blk, req, buf, cb, opaque);
+}
+
 int blk_co_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors)
 {
     int ret = blk_check_request(blk, sector_num, nb_sectors);
diff --git a/include/sysemu/block-backend.h b/include/sysemu/block-backend.h
index dc24476..cf05d56 100644
--- a/include/sysemu/block-backend.h
+++ b/include/sysemu/block-backend.h
@@ -100,6 +100,9 @@ int blk_write_zeroes(BlockBackend *blk, int64_t sector_num,
 BlockAIOCB *blk_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
                                  int nb_sectors, BdrvRequestFlags flags,
                                  BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_write_zeroes_impl(BlockBackend *blk, int64_t sector_num,
+                                      int nb_sectors, BdrvRequestFlags flags,
+                                      BlockCompletionFunc *cb, void *opaque);
 int blk_pread(BlockBackend *blk, int64_t offset, void *buf, int count);
 int blk_pwrite(BlockBackend *blk, int64_t offset, const void *buf, int count);
 int64_t blk_getlength(BlockBackend *blk);
@@ -108,20 +111,33 @@ int64_t blk_nb_sectors(BlockBackend *blk);
 BlockAIOCB *blk_aio_readv(BlockBackend *blk, int64_t sector_num,
                           QEMUIOVector *iov, int nb_sectors,
                           BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_readv_impl(BlockBackend *blk, int64_t sector_num,
+                          QEMUIOVector *iov, int nb_sectors,
+                          BlockCompletionFunc *cb, void *opaque);
 BlockAIOCB *blk_aio_writev(BlockBackend *blk, int64_t sector_num,
                            QEMUIOVector *iov, int nb_sectors,
                            BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_writev_impl(BlockBackend *blk, int64_t sector_num,
+                                QEMUIOVector *iov, int nb_sectors,
+                                BlockCompletionFunc *cb, void *opaque);
 BlockAIOCB *blk_aio_flush(BlockBackend *blk,
                           BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_flush_impl(BlockBackend *blk,
+                               BlockCompletionFunc *cb, void *opaque);
 BlockAIOCB *blk_aio_discard(BlockBackend *blk,
                             int64_t sector_num, int nb_sectors,
                             BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_discard_impl(BlockBackend *blk,
+                                 int64_t sector_num, int nb_sectors,
+                                 BlockCompletionFunc *cb, void *opaque);
 void blk_aio_cancel(BlockAIOCB *acb);
 void blk_aio_cancel_async(BlockAIOCB *acb);
 int blk_aio_multiwrite(BlockBackend *blk, BlockRequest *reqs, int num_reqs);
 int blk_ioctl(BlockBackend *blk, unsigned long int req, void *buf);
 BlockAIOCB *blk_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
                           BlockCompletionFunc *cb, void *opaque);
+BlockAIOCB *blk_aio_ioctl_impl(BlockBackend *blk, unsigned long int req, void *buf,
+                               BlockCompletionFunc *cb, void *opaque);
 int blk_co_discard(BlockBackend *blk, int64_t sector_num, int nb_sectors);
 int blk_co_flush(BlockBackend *blk);
 int blk_flush(BlockBackend *blk);
diff --git a/include/sysemu/replay.h b/include/sysemu/replay.h
index c879231..c06e355 100644
--- a/include/sysemu/replay.h
+++ b/include/sysemu/replay.h
@@ -17,6 +17,8 @@
 #include "qapi-types.h"
 #include "qapi/error.h"
 #include "qemu/typedefs.h"
+#include "block/aio.h"
+#include "block/block.h"
 
 /* replay clock kinds */
 enum ReplayClockKind {
@@ -125,6 +127,31 @@ void replay_register_char_driver(struct CharDriverState *chr);
 /*! Saves write to char device event to the log */
 void replay_chr_be_write(struct CharDriverState *s, uint8_t *buf, int len);
 
+/* Block devices */
+
+/*! Asynchronous read function. Adds read request to the queue. */
+BlockAIOCB *replay_aio_readv(BlockBackend *blk, int64_t sector_num,
+                             QEMUIOVector *iov, int nb_sectors,
+                             BlockCompletionFunc *cb, void *opaque);
+/*! Asynchronous write function. Adds write request to the queue. */
+BlockAIOCB *replay_aio_writev(BlockBackend *blk, int64_t sector_num,
+                              QEMUIOVector *iov, int nb_sectors,
+                              BlockCompletionFunc *cb, void *opaque);
+/*! Asynchronous write zeroes function. Adds this request to the queue. */
+BlockAIOCB *replay_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
+                                    int nb_sectors, BdrvRequestFlags flags,
+                                    BlockCompletionFunc *cb, void *opaque);
+/*! Asynchronous flush function. Adds flush request to the queue. */
+BlockAIOCB *replay_aio_flush(BlockBackend *blk,
+                             BlockCompletionFunc *cb, void *opaque);
+/*! Asynchronous discard function. Adds discard request to the queue. */
+BlockAIOCB *replay_aio_discard(BlockBackend *blk, int64_t sector_num,
+                               int nb_sectors, BlockCompletionFunc *cb,
+                               void *opaque);
+/*! Asynchronous ioctl function. Adds ioctl request to the queue. */
+BlockAIOCB *replay_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
+                             BlockCompletionFunc *cb, void *opaque);
+
 /* Other data */
 
 /*! Writes or reads integer value to/from replay log. */
diff --git a/replay/Makefile.objs b/replay/Makefile.objs
index 70e5572..050c0ea 100644
--- a/replay/Makefile.objs
+++ b/replay/Makefile.objs
@@ -4,3 +4,4 @@ common-obj-y += replay-events.o
 common-obj-y += replay-time.o
 common-obj-y += replay-input.o
 common-obj-y += replay-char.o
+common-obj-y += replay-block.o
diff --git a/replay/replay-block.c b/replay/replay-block.c
new file mode 100755
index 0000000..7fc68cf
--- /dev/null
+++ b/replay/replay-block.c
@@ -0,0 +1,202 @@
+/*
+ * replay-block.c
+ *
+ * Copyright (c) 2010-2016 Institute for System Programming
+ *                         of the Russian Academy of Sciences.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+#include "sysemu/replay.h"
+#include "sysemu/block-backend.h"
+#include "replay-internal.h"
+
+static void replay_aio_cancel(BlockAIOCB *acb);
+
+typedef struct ReplayAIOCB {
+    BlockAIOCB common;
+    BlockRequest req;
+    BlockBackend *blk;
+    ReplayAsyncEventKind kind;
+} ReplayAIOCB;
+
+static const AIOCBInfo replay_aiocb_info = {
+    .aiocb_size   = sizeof(ReplayAIOCB),
+    .cancel_async = replay_aio_cancel
+};
+
+static ReplayAIOCB *replay_aio_create(ReplayAsyncEventKind kind,
+                                      BlockBackend *blk,
+                                      BlockCompletionFunc *cb, void *opaque)
+{
+    ReplayAIOCB *acb = g_malloc0(sizeof(*acb));
+    acb->kind = kind;
+    /* Make correct AIOCB to return to the caller */
+    acb->common.aiocb_info = &replay_aiocb_info;
+    acb->common.bs = blk_bs(blk);
+    acb->common.cb = cb;
+    acb->common.opaque = opaque;
+    acb->common.refcnt = 1;
+    /* Data to restore and pass request to the bdrv */
+    acb->blk = blk;
+    acb->req.error = -EINPROGRESS;
+    return acb;
+}
+
+BlockAIOCB *replay_aio_readv(BlockBackend *blk, int64_t sector_num,
+                             QEMUIOVector *iov, int nb_sectors,
+                             BlockCompletionFunc *cb, void *opaque)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return blk_aio_readv_impl(blk, sector_num, iov, nb_sectors, cb, opaque);
+    } else {
+        ReplayAIOCB *acb = replay_aio_create(REPLAY_ASYNC_EVENT_BLOCK_READV,
+                                             blk, cb, opaque);
+        acb->req.sector = sector_num;
+        acb->req.nb_sectors = nb_sectors;
+        acb->req.qiov = iov;
+        replay_add_event(REPLAY_ASYNC_EVENT_BLOCK_READV, acb, NULL, 0);
+
+        return &acb->common;
+    }
+}
+
+void replay_event_block_readv_run(void *opaque)
+{
+    ReplayAIOCB *acb = opaque;
+    blk_aio_readv_impl(acb->blk, acb->req.sector, acb->req.qiov,
+                       acb->req.nb_sectors, acb->common.cb,
+                       acb->common.opaque);
+    blk_drain_all();
+}
+
+BlockAIOCB *replay_aio_writev(BlockBackend *blk, int64_t sector_num,
+                              QEMUIOVector *iov, int nb_sectors,
+                              BlockCompletionFunc *cb, void *opaque)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return blk_aio_writev_impl(blk, sector_num, iov, nb_sectors, cb, opaque);
+    } else {
+        ReplayAIOCB *acb = replay_aio_create(REPLAY_ASYNC_EVENT_BLOCK_WRITEV,
+                                             blk, cb, opaque);
+        acb->req.sector = sector_num;
+        acb->req.nb_sectors = nb_sectors;
+        acb->req.qiov = iov;
+        replay_add_event(REPLAY_ASYNC_EVENT_BLOCK_WRITEV, acb, NULL, 0);
+
+        return &acb->common;
+    }
+}
+
+void replay_event_block_writev_run(void *opaque)
+{
+    ReplayAIOCB *acb = opaque;
+    blk_aio_writev_impl(acb->blk, acb->req.sector, acb->req.qiov,
+                        acb->req.nb_sectors, acb->common.cb,
+                        acb->common.opaque);
+    blk_drain_all();
+}
+
+BlockAIOCB *replay_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
+                                    int nb_sectors, BdrvRequestFlags flags,
+                                    BlockCompletionFunc *cb, void *opaque)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return blk_aio_write_zeroes_impl(blk, sector_num, nb_sectors, flags, cb, opaque);
+    } else {
+        ReplayAIOCB *acb = replay_aio_create(REPLAY_ASYNC_EVENT_BLOCK_WRITE_ZEROES,
+                                             blk, cb, opaque);
+        acb->req.sector = sector_num;
+        acb->req.nb_sectors = nb_sectors;
+        acb->req.flags = flags;
+        replay_add_event(REPLAY_ASYNC_EVENT_BLOCK_WRITE_ZEROES, acb, NULL, 0);
+
+        return &acb->common;
+    }
+}
+
+void replay_event_block_write_zeroes_run(void *opaque)
+{
+    ReplayAIOCB *acb = opaque;
+    blk_aio_write_zeroes_impl(acb->blk, acb->req.sector, acb->req.nb_sectors,
+                              acb->req.flags, acb->common.cb,
+                              acb->common.opaque);
+    blk_drain_all();
+}
+
+BlockAIOCB *replay_aio_flush(BlockBackend *blk,
+                             BlockCompletionFunc *cb, void *opaque)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return blk_aio_flush_impl(blk, cb, opaque);
+    } else {
+        ReplayAIOCB *acb = replay_aio_create(REPLAY_ASYNC_EVENT_BLOCK_FLUSH,
+                                             blk, cb, opaque);
+        replay_add_event(REPLAY_ASYNC_EVENT_BLOCK_FLUSH, acb, NULL, 0);
+
+        return &acb->common;
+    }
+}
+
+void replay_event_block_flush_run(void *opaque)
+{
+    ReplayAIOCB *acb = opaque;
+    blk_aio_flush_impl(acb->blk, acb->common.cb, acb->common.opaque);
+    blk_drain_all();
+}
+
+static void replay_aio_cancel(BlockAIOCB *acb)
+{
+    ReplayAIOCB *racb = container_of(acb, ReplayAIOCB, common);;
+    replay_remove_event(racb->kind, acb, NULL, 0);
+}
+
+BlockAIOCB *replay_aio_discard(BlockBackend *blk, int64_t sector_num,
+                               int nb_sectors, BlockCompletionFunc *cb,
+                               void *opaque)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return blk_aio_discard_impl(blk, sector_num, nb_sectors, cb, opaque);
+    } else {
+        ReplayAIOCB *acb = replay_aio_create(REPLAY_ASYNC_EVENT_BLOCK_DISCARD,
+                                             blk, cb, opaque);
+        acb->req.sector = sector_num;
+        acb->req.nb_sectors = nb_sectors;
+        replay_add_event(REPLAY_ASYNC_EVENT_BLOCK_DISCARD, acb, NULL, 0);
+
+        return &acb->common;
+    }
+}
+
+void replay_event_block_discard_run(void *opaque)
+{
+    ReplayAIOCB *acb = opaque;
+    blk_aio_discard_impl(acb->blk, acb->req.sector, acb->req.nb_sectors,
+                         acb->common.cb, acb->common.opaque);
+    blk_drain_all();
+}
+
+BlockAIOCB *replay_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
+                             BlockCompletionFunc *cb, void *opaque)
+{
+    if (replay_mode == REPLAY_MODE_NONE) {
+        return blk_aio_ioctl_impl(blk, req, buf, cb, opaque);
+    } else {
+        ReplayAIOCB *acb = replay_aio_create(REPLAY_ASYNC_EVENT_BLOCK_IOCTL,
+                                             blk, cb, opaque);
+        acb->req.req = req;
+        acb->req.buf = buf;
+        replay_add_event(REPLAY_ASYNC_EVENT_BLOCK_IOCTL, acb, NULL, 0);
+
+        return &acb->common;
+    }
+}
+
+void replay_event_block_ioctl_run(void *opaque)
+{
+    ReplayAIOCB *acb = opaque;
+    blk_aio_ioctl_impl(acb->blk, acb->req.req, acb->req.buf,
+                       acb->common.cb, acb->common.opaque);
+    blk_drain_all();
+}
diff --git a/replay/replay-events.c b/replay/replay-events.c
index 7fc7ed0..1551257 100644
--- a/replay/replay-events.c
+++ b/replay/replay-events.c
@@ -50,6 +50,24 @@ static void replay_run_event(Event *event)
     case REPLAY_ASYNC_EVENT_CHAR:
         replay_event_char_run(event->opaque);
         break;
+    case REPLAY_ASYNC_EVENT_BLOCK_READV:
+        replay_event_block_readv_run(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_BLOCK_WRITEV:
+        replay_event_block_writev_run(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_BLOCK_WRITE_ZEROES:
+        replay_event_block_write_zeroes_run(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_BLOCK_FLUSH:
+        replay_event_block_flush_run(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_BLOCK_DISCARD:
+        replay_event_block_discard_run(event->opaque);
+        break;
+    case REPLAY_ASYNC_EVENT_BLOCK_IOCTL:
+        replay_event_block_ioctl_run(event->opaque);
+        break;
     default:
         error_report("Replay: invalid async event ID (%d) in the queue",
                     event->event_kind);
@@ -132,6 +150,24 @@ void replay_add_event(ReplayAsyncEventKind event_kind,
     replay_mutex_unlock();
 }
 
+void replay_remove_event(ReplayAsyncEventKind event_kind,
+                         void *opaque,
+                         void *opaque2, uint64_t id)
+{
+    Event *event;
+    replay_mutex_lock();
+    QTAILQ_FOREACH(event, &events_list, events) {
+        if (event->event_kind == event_kind
+            && event->opaque == opaque
+            && event->opaque2 == opaque2
+            && event->id == id) {
+            QTAILQ_REMOVE(&events_list, event, events);
+            break;
+        }
+    }
+    replay_mutex_unlock();
+}
+
 void replay_bh_schedule_event(QEMUBH *bh)
 {
     if (replay_mode != REPLAY_MODE_NONE) {
@@ -173,6 +209,13 @@ static void replay_save_event(Event *event, int checkpoint)
         case REPLAY_ASYNC_EVENT_CHAR:
             replay_event_char_save(event->opaque);
             break;
+        case REPLAY_ASYNC_EVENT_BLOCK_READV:
+        case REPLAY_ASYNC_EVENT_BLOCK_WRITEV:
+        case REPLAY_ASYNC_EVENT_BLOCK_WRITE_ZEROES:
+        case REPLAY_ASYNC_EVENT_BLOCK_FLUSH:
+        case REPLAY_ASYNC_EVENT_BLOCK_DISCARD:
+        case REPLAY_ASYNC_EVENT_BLOCK_IOCTL:
+            break;
         default:
             error_report("Unknown ID %d of replay event", read_event_kind);
             exit(1);
@@ -231,6 +274,14 @@ static Event *replay_read_event(int checkpoint)
         event->event_kind = read_event_kind;
         event->opaque = replay_event_char_read();
         return event;
+    case REPLAY_ASYNC_EVENT_BLOCK_READV:
+    case REPLAY_ASYNC_EVENT_BLOCK_WRITEV:
+    case REPLAY_ASYNC_EVENT_BLOCK_WRITE_ZEROES:
+    case REPLAY_ASYNC_EVENT_BLOCK_FLUSH:
+    case REPLAY_ASYNC_EVENT_BLOCK_DISCARD:
+    case REPLAY_ASYNC_EVENT_BLOCK_IOCTL:
+        /* search in the queue */
+        break;
     default:
         error_report("Unknown ID %d of replay event", read_event_kind);
         exit(1);
diff --git a/replay/replay-internal.h b/replay/replay-internal.h
index 34eee5b..8f705d1 100644
--- a/replay/replay-internal.h
+++ b/replay/replay-internal.h
@@ -47,6 +47,12 @@ enum ReplayAsyncEventKind {
     REPLAY_ASYNC_EVENT_INPUT,
     REPLAY_ASYNC_EVENT_INPUT_SYNC,
     REPLAY_ASYNC_EVENT_CHAR,
+    REPLAY_ASYNC_EVENT_BLOCK_READV,
+    REPLAY_ASYNC_EVENT_BLOCK_WRITEV,
+    REPLAY_ASYNC_EVENT_BLOCK_WRITE_ZEROES,
+    REPLAY_ASYNC_EVENT_BLOCK_FLUSH,
+    REPLAY_ASYNC_EVENT_BLOCK_DISCARD,
+    REPLAY_ASYNC_EVENT_BLOCK_IOCTL,
     REPLAY_ASYNC_COUNT
 };
 
@@ -131,6 +137,9 @@ void replay_read_events(int checkpoint);
 /*! Adds specified async event to the queue */
 void replay_add_event(ReplayAsyncEventKind event_kind, void *opaque,
                       void *opaque2, uint64_t id);
+/*! Removes specified async event from the queue */
+void replay_remove_event(ReplayAsyncEventKind event_kind,
+                         void *opaque, void *opaque2, uint64_t id);
 
 /* Input events */
 
@@ -152,4 +161,19 @@ void replay_event_char_save(void *opaque);
 /*! Reads char event from the file. */
 void *replay_event_char_read(void);
 
+/* Block devices */
+
+/*! Called to run block device readv event. */
+void replay_event_block_readv_run(void *opaque);
+/*! Called to run block device writev event. */
+void replay_event_block_writev_run(void *opaque);
+/*! Called to run block device write zeroes event. */
+void replay_event_block_write_zeroes_run(void *opaque);
+/*! Called to run block device flush event. */
+void replay_event_block_flush_run(void *opaque);
+/*! Called to run block device discard event. */
+void replay_event_block_discard_run(void *opaque);
+/*! Called to run block device ioctl event. */
+void replay_event_block_ioctl_run(void *opaque);
+
 #endif
diff --git a/stubs/replay.c b/stubs/replay.c
index 8f98790..54a5f61 100644
--- a/stubs/replay.c
+++ b/stubs/replay.c
@@ -1,6 +1,7 @@
 #include "sysemu/replay.h"
 #include <stdlib.h>
 #include "sysemu/sysemu.h"
+#include "sysemu/block-backend.h"
 
 ReplayMode replay_mode;
 
@@ -29,3 +30,48 @@ bool replay_events_enabled(void)
 void replay_finish(void)
 {
 }
+
+BlockAIOCB *replay_aio_readv(BlockBackend *blk, int64_t sector_num,
+                             QEMUIOVector *iov, int nb_sectors,
+                             BlockCompletionFunc *cb, void *opaque)
+{
+    return blk_aio_readv_impl(blk, sector_num, iov, nb_sectors, cb, opaque);
+}
+
+BlockAIOCB *replay_aio_writev(BlockBackend *blk, int64_t sector_num,
+                              QEMUIOVector *iov, int nb_sectors,
+                              BlockCompletionFunc *cb, void *opaque)
+{
+    return blk_aio_writev_impl(blk, sector_num, iov, nb_sectors, cb, opaque);
+}
+
+BlockAIOCB *replay_aio_flush(BlockBackend *blk,
+                             BlockCompletionFunc *cb, void *opaque)
+{
+    return blk_aio_flush_impl(blk, cb, opaque);
+}
+
+BlockAIOCB *replay_aio_discard(BlockBackend *blk,
+                               int64_t sector_num, int nb_sectors,
+                               BlockCompletionFunc *cb, void *opaque)
+{
+    return blk_aio_discard_impl(blk, sector_num, nb_sectors, cb, opaque);
+}
+
+BlockAIOCB *replay_aio_ioctl(BlockBackend *blk, unsigned long int req, void *buf,
+                             BlockCompletionFunc *cb, void *opaque)
+{
+    return blk_aio_ioctl_impl(blk, req, buf, cb, opaque);
+}
+
+BlockAIOCB *replay_aio_write_zeroes(BlockBackend *blk, int64_t sector_num,
+                                    int nb_sectors, BdrvRequestFlags flags,
+                                    BlockCompletionFunc *cb, void *opaque)
+{
+    return blk_aio_write_zeroes_impl(blk, sector_num, nb_sectors, flags, cb, opaque);
+}
+
+void replay_bh_schedule_event(QEMUBH *bh)
+{
+    qemu_bh_schedule(bh);
+}