[PATCH v1 0/4] migration/debug: Add migration ram consistency check

[PATCH v1 0/4] migration/debug: Add migration ram consistency check

[Chuan Zheng]

Sometimes we want to debug whether the ramblock we migrate is same between Src and Dst.
For example, we could want to check ram when develop something related to dirty log sync.
Consistency check is implemented in this series, it will sha256sum all migratable
ramblock and print both in Src and Dst.

Check results are shown as follow:
Src:
CheckPoint: migrate_fd_cleanup, Ramblock: mach-virt.ram, CheckValue: 4422e2e8f26835f32ee3a9f13e1df2772d48f973a58381f6a549ebbcfe485b72
CheckPoint: migrate_fd_cleanup, Ramblock: virt.flash0, CheckValue: d5584b740ffcf81df8123ebc833793a71a03d47c1bb5a97170d05d18665c8b2e
CheckPoint: migrate_fd_cleanup, Ramblock: virt.flash1, CheckValue: 1d6c818dfa81a88ca5b7b1da231a9ba57f4f87677c6ba8e76196195b5aa05f0c
CheckPoint: migrate_fd_cleanup, Ramblock: /rom@etc/acpi/tables, CheckValue: db4c25623cb0192a70b56b5700e304e87c46f3016bc4b43b458a831b93d1cd54
CheckPoint: migrate_fd_cleanup, Ramblock: /rom@etc/table-loader, CheckValue: b3e0b1026cd4df920884f7d090b90cfb64b4a3ab407feeb632300aabd9fb28fe
CheckPoint: migrate_fd_cleanup, Ramblock: /rom@etc/acpi/rsdp, CheckValue: 7af8a2bc8c5f78db788a47ed60b30bffee50f28783529ee55224f9e3613cc28a

Dst:
CheckPoint: qemu_loadvm_state, Ramblock: mach-virt.ram, CheckValue: 4422e2e8f26835f32ee3a9f13e1df2772d48f973a58381f6a549ebbcfe485b72
CheckPoint: qemu_loadvm_state, Ramblock: virt.flash0, CheckValue: d5584b740ffcf81df8123ebc833793a71a03d47c1bb5a97170d05d18665c8b2e
CheckPoint: qemu_loadvm_state, Ramblock: virt.flash1, CheckValue: 1d6c818dfa81a88ca5b7b1da231a9ba57f4f87677c6ba8e76196195b5aa05f0c
CheckPoint: qemu_loadvm_state, Ramblock: /rom@etc/acpi/tables, CheckValue: db4c25623cb0192a70b56b5700e304e87c46f3016bc4b43b458a831b93d1cd54
CheckPoint: qemu_loadvm_state, Ramblock: /rom@etc/table-loader, CheckValue: b3e0b1026cd4df920884f7d090b90cfb64b4a3ab407feeb632300aabd9fb28fe
CheckPoint: qemu_loadvm_state, Ramblock: /rom@etc/acpi/rsdp, CheckValue: 7af8a2bc8c5f78db788a47ed60b30bffee50f28783529ee55224f9e3613cc28a

Not for sure if it is valuable, any discussion and comment is welcome.

Chuan Zheng (4):
  migration/debug: Introduce foreach_migratable_block()
  migration/debug: Implement migration memory consistency check
  migration/debug: add checkpoint for migration consistency check
  migration/debug: add DEBUG_MIGRATION_CONSISTENCY_CHECK macros

 migration/migration.c |  3 ++
 migration/ram.c       | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++
 migration/ram.h       |  7 +++++
 migration/savevm.c    |  9 ++++++
 4 files changed, 105 insertions(+)

-- 
1.8.3.1

[PATCH v1 1/4] migration/debug: Introduce foreach_migratable_block()

[Chuan Zheng]

Signed-off-by: Chuan Zheng <zhengchuan@huawei.com>
---
 migration/ram.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/migration/ram.c b/migration/ram.c
index 433489d..aa39908 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -166,6 +166,22 @@ bool ramblock_is_ignored(RAMBlock *block)
 
 #undef RAMBLOCK_FOREACH
 
+static int foreach_migratable_block(RAMBlockIterFunc func, void *opaque)
+{
+    RAMBlock *block;
+    int ret = 0;
+
+    rcu_read_lock();
+    RAMBLOCK_FOREACH_MIGRATABLE(block) {
+        ret = func(block, opaque);
+        if (ret) {
+            break;
+        }
+    }
+    rcu_read_unlock();
+    return ret;
+}
+
 int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
 {
     RAMBlock *block;
-- 
1.8.3.1

[PATCH v1 2/4] migration/debug: Implement migration memory consistency check

[Chuan Zheng]

Traverse all migratable ramblocks, calculate sha256 for memory consistency check

Signed-off-by: Chuan Zheng <zhengchuan@huawei.com>
---
 migration/ram.c | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 migration/ram.h |  7 ++++++
 2 files changed, 80 insertions(+)

diff --git a/migration/ram.c b/migration/ram.c
index aa39908..f04594e 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -55,6 +55,8 @@
 #include "sysemu/cpu-throttle.h"
 #include "savevm.h"
 #include "qemu/iov.h"
+#include "crypto/hash.h"
+#include "qemu/typedefs.h"
 #include "multifd.h"
 
 /***********************************************************/
@@ -198,6 +200,77 @@ int foreach_not_ignored_block(RAMBlockIterFunc func, void *opaque)
     return ret;
 }
 
+#define SHA256_DIGEST_LENGTH 32
+#define SHA256_CHUNK_SIZE 0x80000000
+
+static void ram_debug_dump_sha256(uint8_t *md, const char *idstr,
+                                  const char *prefix)
+{
+    int i;
+    char buf[2 * SHA256_DIGEST_LENGTH + 1] = { 0 };
+
+    for (i = 0; i < SHA256_DIGEST_LENGTH; i++) {
+        sprintf(&buf[2 * i], "%02x", md[i]);
+    }
+
+    fprintf(stderr, "CheckPoint: %s, Ramblock: %s, CheckValue: %s\n",
+            prefix, idstr, buf);
+}
+
+static void ram_debug_calc_sha256(RAMBlock *block, const char *idstr,
+                                  const char *prefix)
+{
+    uint8_t *md = NULL;
+    size_t sha256_len;
+    size_t i, niov;
+    void *addr = NULL;
+    ram_addr_t remaining = 0;
+    size_t resultlen = 0;
+    struct iovec *iov_array = NULL;
+
+    sha256_len = qcrypto_hash_digest_len(QCRYPTO_HASH_ALG_SHA256);
+    assert(sha256_len == SHA256_DIGEST_LENGTH);
+
+    niov = DIV_ROUND_UP(qemu_ram_get_used_length(block), SHA256_CHUNK_SIZE);
+    iov_array = g_malloc0_n(niov, sizeof(struct iovec));
+
+    addr = qemu_ram_get_host_addr(block);
+    remaining = qemu_ram_get_used_length(block);
+    for (i = 0; i < niov; i++) {
+        iov_array[i].iov_base = addr;
+        iov_array[i].iov_len = MIN(SHA256_CHUNK_SIZE, remaining);
+        addr += SHA256_CHUNK_SIZE;
+        remaining -= SHA256_CHUNK_SIZE;
+    }
+
+    if (qcrypto_hash_bytesv(QCRYPTO_HASH_ALG_SHA256,
+                            iov_array, niov,
+                            &md, &resultlen, NULL) || !md) {
+        fprintf(stderr, "Consistency check(%s) calc failed.\n", prefix);
+        goto out;
+    }
+
+    ram_debug_dump_sha256(md, idstr, prefix);
+
+out:
+    g_free(iov_array);
+}
+
+static int ram_debug_consistency(RAMBlock *block, void *opaque)
+{
+    const char *prefix = opaque;
+    const char *idstr = qemu_ram_get_idstr(block);
+
+    ram_debug_calc_sha256(block, idstr, prefix);
+
+    return 0;
+}
+
+void migration_debug_ram_consistency(const char *prefix)
+{
+    foreach_migratable_block(ram_debug_consistency, (void *)prefix);
+}
+
 static void ramblock_recv_map_init(void)
 {
     RAMBlock *rb;
diff --git a/migration/ram.h b/migration/ram.h
index 011e854..d73de6e 100644
--- a/migration/ram.h
+++ b/migration/ram.h
@@ -47,6 +47,13 @@ bool ramblock_is_ignored(RAMBlock *block);
     INTERNAL_RAMBLOCK_FOREACH(block)                   \
         if (!qemu_ram_is_migratable(block)) {} else
 
+void migration_debug_ram_consistency(const char *prefix);
+
+#define MIGRATION_RAM_CONSISTENCY_CHECK()              \
+do {                                              \
+        migration_debug_ram_consistency(__func__);      \
+} while (0)
+
 int xbzrle_cache_resize(int64_t new_size, Error **errp);
 uint64_t ram_bytes_remaining(void);
 uint64_t ram_bytes_total(void);
-- 
1.8.3.1

[PATCH v1 3/4] migration/debug: add checkpoint for migration consistency check

[Chuan Zheng]

Signed-off-by: Chuan Zheng <zhengchuan@huawei.com>
---
 migration/migration.c | 1 +
 migration/savevm.c    | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/migration/migration.c b/migration/migration.c
index 0575ecb..f4434c4 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1678,6 +1678,7 @@ static void migrate_fd_cleanup(MigrationState *s)
         tmp = s->to_dst_file;
         s->to_dst_file = NULL;
         qemu_mutex_unlock(&s->qemu_file_lock);
+        MIGRATION_RAM_CONSISTENCY_CHECK();
         /*
          * Close the file handle without the lock to make sure the
          * critical section won't block for long.
diff --git a/migration/savevm.c b/migration/savevm.c
index ff33e21..5f989a8 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -2636,6 +2636,13 @@ int qemu_loadvm_state(QEMUFile *f)
     }
 
     qemu_loadvm_state_cleanup();
+    /*
+     * cpu_synchronize_all_post_init->kvm_put_msrs will update the
+     * kvmclock share-mem.
+     * So this's the latest point at which the ram is the same as source
+     */
+    MIGRATION_RAM_CONSISTENCY_CHECK();
+
     cpu_synchronize_all_post_init();
 
     return ret;
-- 
1.8.3.1

[PATCH v1 4/4] migration/debug: add DEBUG_MIGRATION_CONSISTENCY_CHECK macros

[Chuan Zheng]

Signed-off-by: Chuan Zheng <zhengchuan@huawei.com>
---
 migration/migration.c | 2 ++
 migration/savevm.c    | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/migration/migration.c b/migration/migration.c
index f4434c4..fe2bb3a 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -1678,7 +1678,9 @@ static void migrate_fd_cleanup(MigrationState *s)
         tmp = s->to_dst_file;
         s->to_dst_file = NULL;
         qemu_mutex_unlock(&s->qemu_file_lock);
+#ifdef DEBUG_MIGRATION_CONSISTENCY_CHECK
         MIGRATION_RAM_CONSISTENCY_CHECK();
+#endif
         /*
          * Close the file handle without the lock to make sure the
          * critical section won't block for long.
diff --git a/migration/savevm.c b/migration/savevm.c
index 5f989a8..102cd0e 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -2636,12 +2636,14 @@ int qemu_loadvm_state(QEMUFile *f)
     }
 
     qemu_loadvm_state_cleanup();
+#ifdef DEBUG_MIGRATION_CONSISTENCY_CHECK
     /*
      * cpu_synchronize_all_post_init->kvm_put_msrs will update the
      * kvmclock share-mem.
      * So this's the latest point at which the ram is the same as source
      */
     MIGRATION_RAM_CONSISTENCY_CHECK();
+#endif
 
     cpu_synchronize_all_post_init();
 
-- 
1.8.3.1