[PATCH 0/4] Add function to dump block layer for debugging

[PATCH 0/4] Add function to dump block layer for debugging

[Vladimir Sementsov-Ogievskiy]

Hi all!

Here is a new function dbg_dump_block_layer() to help with block layer
code debugging.

Usage:

1. Add dbg_dump_block_layer("/path/to/dump.json") call in some place
were you are trying to understand the relations in block graph and run
your test. Or just call dbg_dump_block_layer("/path/to/dump.json") from
gdb session if attached to running Qemu.

2. Convert json to png image:

 .scripts/render_block_graph.py --json /path/to/dump.json /path/to/out

And get your /path/to/out dot file and /path/to/out.png image.

Vladimir Sementsov-Ogievskiy (4):
  block: bdrv_get_xdbg_block_graph() drop unused errp argument
  blockjob: add block_jobs_info_list()
  block: add dbg_dump_block_layer()
  scripts/render_block_graph.py: add ability to parse json files

 qapi/block-core.json          | 13 +++++++++
 include/block/block.h         |  5 +++-
 include/block/blockjob.h      |  7 +++++
 block.c                       | 53 +++++++++++++++++++++++++++++++++--
 blockdev.c                    | 28 ++----------------
 blockjob.c                    | 29 +++++++++++++++++++
 MAINTAINERS                   |  5 ++++
 scripts/render_block_graph.py | 53 ++++++++++++++++++++++-------------
 8 files changed, 145 insertions(+), 48 deletions(-)

-- 
2.21.3

[PATCH 1/4] block: bdrv_get_xdbg_block_graph() drop unused errp argument

[Vladimir Sementsov-Ogievskiy]

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
 include/block/block.h | 2 +-
 block.c               | 2 +-
 blockdev.c            | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index c9d7c58765..d17151abd7 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -524,7 +524,7 @@ void bdrv_eject(BlockDriverState *bs, bool eject_flag);
 const char *bdrv_get_format_name(BlockDriverState *bs);
 BlockDriverState *bdrv_find_node(const char *node_name);
 BlockDeviceInfoList *bdrv_named_nodes_list(bool flat, Error **errp);
-XDbgBlockGraph *bdrv_get_xdbg_block_graph(Error **errp);
+XDbgBlockGraph *bdrv_get_xdbg_block_graph(void);
 BlockDriverState *bdrv_lookup_bs(const char *device,
                                  const char *node_name,
                                  Error **errp);
diff --git a/block.c b/block.c
index f1cedac362..eca16a5e29 100644
--- a/block.c
+++ b/block.c
@@ -5338,7 +5338,7 @@ static void xdbg_graph_add_edge(XDbgBlockGraphConstructor *gr, void *parent,
 }
 
 
-XDbgBlockGraph *bdrv_get_xdbg_block_graph(Error **errp)
+XDbgBlockGraph *bdrv_get_xdbg_block_graph(void)
 {
     BlockBackend *blk;
     BlockJob *job;
diff --git a/blockdev.c b/blockdev.c
index fe6fb5dc1d..6f103e5ce7 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -2911,7 +2911,7 @@ BlockDeviceInfoList *qmp_query_named_block_nodes(bool has_flat,
 
 XDbgBlockGraph *qmp_x_debug_query_block_graph(Error **errp)
 {
-    return bdrv_get_xdbg_block_graph(errp);
+    return bdrv_get_xdbg_block_graph();
 }
 
 void qmp_blockdev_backup(BlockdevBackup *backup, Error **errp)
-- 
2.21.3

[PATCH 2/4] blockjob: add block_jobs_info_list()

[Vladimir Sementsov-Ogievskiy]

Move qmp_query_block_jobs() functionality to blockjob.c. We'll need it
in block.c which is not always compiled together with blockdev.c

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
 include/block/blockjob.h |  7 +++++++
 blockdev.c               | 26 +-------------------------
 blockjob.c               | 29 +++++++++++++++++++++++++++++
 3 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index 35faa3aa26..2e66d529e2 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -173,4 +173,11 @@ bool block_job_is_internal(BlockJob *job);
  */
 const BlockJobDriver *block_job_driver(BlockJob *job);
 
+/**
+ * block_jobs_info_list:
+ *
+ * Returns the list of jobs info.
+ */
+BlockJobInfoList *block_jobs_info_list(Error **errp);
+
 #endif
diff --git a/blockdev.c b/blockdev.c
index 6f103e5ce7..b5f11c524b 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -3670,31 +3670,7 @@ void qmp_x_blockdev_change(const char *parent, bool has_child,
 
 BlockJobInfoList *qmp_query_block_jobs(Error **errp)
 {
-    BlockJobInfoList *head = NULL, **p_next = &head;
-    BlockJob *job;
-
-    for (job = block_job_next(NULL); job; job = block_job_next(job)) {
-        BlockJobInfoList *elem;
-        AioContext *aio_context;
-
-        if (block_job_is_internal(job)) {
-            continue;
-        }
-        elem = g_new0(BlockJobInfoList, 1);
-        aio_context = blk_get_aio_context(job->blk);
-        aio_context_acquire(aio_context);
-        elem->value = block_job_query(job, errp);
-        aio_context_release(aio_context);
-        if (!elem->value) {
-            g_free(elem);
-            qapi_free_BlockJobInfoList(head);
-            return NULL;
-        }
-        *p_next = elem;
-        p_next = &elem->next;
-    }
-
-    return head;
+    return block_jobs_info_list(errp);
 }
 
 void qmp_x_blockdev_set_iothread(const char *node_name, StrOrNull *iothread,
diff --git a/blockjob.c b/blockjob.c
index 98ac8af982..9d0bed01c2 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -515,3 +515,32 @@ BlockErrorAction block_job_error_action(BlockJob *job, BlockdevOnError on_err,
     }
     return action;
 }
+
+BlockJobInfoList *block_jobs_info_list(Error **errp)
+{
+    BlockJobInfoList *head = NULL, **p_next = &head;
+    BlockJob *job;
+
+    for (job = block_job_next(NULL); job; job = block_job_next(job)) {
+        BlockJobInfoList *elem;
+        AioContext *aio_context;
+
+        if (block_job_is_internal(job)) {
+            continue;
+        }
+        elem = g_new0(BlockJobInfoList, 1);
+        aio_context = blk_get_aio_context(job->blk);
+        aio_context_acquire(aio_context);
+        elem->value = block_job_query(job, errp);
+        aio_context_release(aio_context);
+        if (!elem->value) {
+            g_free(elem);
+            qapi_free_BlockJobInfoList(head);
+            return NULL;
+        }
+        *p_next = elem;
+        p_next = &elem->next;
+    }
+
+    return head;
+}
-- 
2.21.3

[PATCH 3/4] block: add dbg_dump_block_layer()

[Vladimir Sementsov-Ogievskiy]

Add function for debugging: we already have x-debug-query-block-graph
qmp command and scripts/render_block_graph.py which can dump
block-layer graph for running vm using qmp command. But when debug
block layer code, its often needed to dump some intermediate state
during some operation, so separate qmp command doesn't help.

So, let's introduce a function which can dump needed information into
json file.

In next commit we'll update scripts/render_block_graph.py so that it
will be able to parse json files.

For new function to not crash if we have mirror_top filter node not yet
bdrv_append()ed (to debug why it can't be appended), make
bdrv_get_allocated_file_size() and bdrv_refresh_filename() not crash on
filters without a child.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
 qapi/block-core.json  | 13 +++++++++++
 include/block/block.h |  3 +++
 block.c               | 51 ++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 66 insertions(+), 1 deletion(-)

diff --git a/qapi/block-core.json b/qapi/block-core.json
index 04ad80bc1e..3f8e5a3822 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -1908,6 +1908,19 @@
 ##
 { 'command': 'x-debug-query-block-graph', 'returns': 'XDbgBlockGraph' }
 
+##
+# @XDbgBlockLayerDump:
+#
+# Unite query-named-block-nodes, query-block-jobs and
+# x-debug-query-block-graph results into one structure for block_layer_dump()
+# function.
+#
+# Since: 5.3
+##
+{ 'struct': 'XDbgBlockLayerDump',
+  'data': { 'nodes': ['BlockDeviceInfo'], 'jobs': ['BlockJobInfo'],
+            'graph': 'XDbgBlockGraph' } }
+
 ##
 # @drive-mirror:
 #
diff --git a/include/block/block.h b/include/block/block.h
index d17151abd7..6a75bbc84c 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -836,4 +836,7 @@ int coroutine_fn bdrv_co_copy_range(BdrvChild *src, uint64_t src_offset,
                                     BdrvChild *dst, uint64_t dst_offset,
                                     uint64_t bytes, BdrvRequestFlags read_flags,
                                     BdrvRequestFlags write_flags);
+
+int dbg_dump_block_layer(const char *filename, Error **errp);
+
 #endif
diff --git a/block.c b/block.c
index eca16a5e29..82f847a88e 100644
--- a/block.c
+++ b/block.c
@@ -5035,7 +5035,8 @@ int64_t bdrv_get_allocated_file_size(BlockDriverState *bs)
         return -ENOTSUP;
     } else if (drv->is_filter) {
         /* Filter drivers default to the size of their filtered child */
-        return bdrv_get_allocated_file_size(bdrv_filter_bs(bs));
+        return bdrv_filter_child(bs) ?
+            bdrv_get_allocated_file_size(bdrv_filter_bs(bs)) : -ENOMEDIUM;
     } else {
         /* Other drivers default to summing their children's sizes */
         return bdrv_sum_allocated_file_size(bs);
@@ -5382,6 +5383,51 @@ XDbgBlockGraph *bdrv_get_xdbg_block_graph(void)
     return xdbg_graph_finalize(gr);
 }
 
+int dbg_dump_block_layer(const char *filename, Error **errp)
+{
+    ERRP_GUARD();
+    Visitor *v;
+    g_autoptr(XDbgBlockLayerDump) dump = g_new0(XDbgBlockLayerDump, 1);
+    QObject *obj = NULL;
+    QString *json;
+    FILE *f;
+
+    dump->nodes = bdrv_named_nodes_list(false, errp);
+    if (*errp) {
+        return -EINVAL;
+    }
+    dump->jobs = block_jobs_info_list(errp);
+    if (*errp) {
+        return -EINVAL;
+    }
+    dump->graph = bdrv_get_xdbg_block_graph();
+
+    v = qobject_output_visitor_new(&obj);
+    if (visit_type_XDbgBlockLayerDump(v, "unused", &dump, errp)) {
+        visit_complete(v, &obj);
+    }
+    visit_free(v);
+    if (*errp) {
+        return -EINVAL;
+    }
+
+    json = qobject_to_json_pretty(obj);
+    qobject_unref(obj);
+
+    f = fopen(filename, "w");
+    if (!f) {
+        error_setg_errno(errp, errno, "Can't open file '%s'", filename);
+        qobject_unref(json);
+        return -EINVAL;
+    }
+
+    fputs(qstring_get_str(json), f);
+    fclose(f);
+    qobject_unref(json);
+
+    return 0;
+}
+
 BlockDriverState *bdrv_lookup_bs(const char *device,
                                  const char *node_name,
                                  Error **errp)
@@ -6885,6 +6931,9 @@ void bdrv_refresh_filename(BlockDriverState *bs)
     if (bs->implicit) {
         /* For implicit nodes, just copy everything from the single child */
         child = QLIST_FIRST(&bs->children);
+        if (!child) {
+            return;
+        }
         assert(QLIST_NEXT(child, next) == NULL);
 
         pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
-- 
2.21.3

[PATCH 4/4] scripts/render_block_graph.py: add ability to parse json files

[Vladimir Sementsov-Ogievskiy]

Add an option to use json file (generated by dbg_dump_block_layer())
as input.

Also, add myself as maintainer of the script.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
 MAINTAINERS                   |  5 ++++
 scripts/render_block_graph.py | 53 ++++++++++++++++++++++-------------
 2 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/MAINTAINERS b/MAINTAINERS
index 2e018a0c1d..e780c89e0e 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -2408,6 +2408,11 @@ M: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
 S: Maintained
 F: scripts/simplebench/
 
+Block graph visualization
+M: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
+S: Maintained
+F: scripts/render_block_graph.py
+
 QAPI
 M: Markus Armbruster <armbru@redhat.com>
 M: Michael Roth <mdroth@linux.vnet.ibm.com>
diff --git a/scripts/render_block_graph.py b/scripts/render_block_graph.py
index da6acf050d..4a11aba697 100755
--- a/scripts/render_block_graph.py
+++ b/scripts/render_block_graph.py
@@ -2,7 +2,7 @@
 #
 # Render Qemu Block Graph
 #
-# Copyright (c) 2018 Virtuozzo International GmbH. All rights reserved.
+# Copyright (c) 2018-2020 Virtuozzo International GmbH.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -40,19 +40,14 @@ def perm(arr):
     return s
 
 
-def render_block_graph(qmp, filename, format='png'):
+def do_render_block_graph(nodes, jobs, block_graph, filename, format='png'):
     '''
     Render graph in text (dot) representation into "@filename" and
     representation in @format into "@filename.@format"
     '''
+    bds_nodes = {n['node-name']: n for n in nodes}
 
-    bds_nodes = qmp.command('query-named-block-nodes')
-    bds_nodes = {n['node-name']: n for n in bds_nodes}
-
-    job_nodes = qmp.command('query-block-jobs')
-    job_nodes = {n['device']: n for n in job_nodes}
-
-    block_graph = qmp.command('x-debug-query-block-graph')
+    job_nodes = {n['device']: n for n in jobs}
 
     graph = Digraph(comment='Block Nodes Graph')
     graph.format = format
@@ -93,6 +88,19 @@ def render_block_graph(qmp, filename, format='png'):
     graph.render(filename)
 
 
+def render_block_graph(qmp, filename, format='png'):
+    '''
+    Render graph in text (dot) representation into "@filename" and
+    representation in @format into "@filename.@format"
+    '''
+
+    nodes = qmp.command('query-named-block-nodes')
+    jobs = qmp.command('query-block-jobs')
+    graph = qmp.command('x-debug-query-block-graph')
+
+    do_render_block_graph(nodes, jobs, graph, filename, format)
+
+
 class LibvirtGuest():
     def __init__(self, name):
         self.name = name
@@ -111,15 +119,22 @@ class LibvirtGuest():
 
 
 if __name__ == '__main__':
-    obj = sys.argv[1]
-    out = sys.argv[2]
-
-    if os.path.exists(obj):
-        # assume unix socket
-        qmp = QEMUMonitorProtocol(obj)
-        qmp.connect()
+    if sys.argv[1] == '--json':
+        json_file = sys.argv[2]
+        out = sys.argv[3]
+        with open(json_file) as f:
+            dump = json.load(f)
+        do_render_block_graph(dump['nodes'], dump['jobs'], dump['graph'], out)
     else:
-        # assume libvirt guest name
-        qmp = LibvirtGuest(obj)
+        obj = sys.argv[1]
+        out = sys.argv[2]
+
+        if os.path.exists(obj):
+            # assume unix socket
+            qmp = QEMUMonitorProtocol(obj)
+            qmp.connect()
+        else:
+            # assume libvirt guest name
+            qmp = LibvirtGuest(obj)
 
-    render_block_graph(qmp, out)
+        render_block_graph(qmp, out)
-- 
2.21.3

Re: [PATCH 3/4] block: add dbg_dump_block_layer()

[Eric Blake]

On 11/19/20 7:16 AM, Vladimir Sementsov-Ogievskiy wrote:
> Add function for debugging: we already have x-debug-query-block-graph
> qmp command and scripts/render_block_graph.py which can dump
> block-layer graph for running vm using qmp command. But when debug
> block layer code, its often needed to dump some intermediate state
> during some operation, so separate qmp command doesn't help.
> 
> So, let's introduce a function which can dump needed information into
> json file.
> 
> In next commit we'll update scripts/render_block_graph.py so that it
> will be able to parse json files.
> 
> For new function to not crash if we have mirror_top filter node not yet
> bdrv_append()ed (to debug why it can't be appended), make
> bdrv_get_allocated_file_size() and bdrv_refresh_filename() not crash on
> filters without a child.
> 
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> ---

> +++ b/qapi/block-core.json
> @@ -1908,6 +1908,19 @@
>  ##
>  { 'command': 'x-debug-query-block-graph', 'returns': 'XDbgBlockGraph' }
>  
> +##
> +# @XDbgBlockLayerDump:
> +#
> +# Unite query-named-block-nodes, query-block-jobs and
> +# x-debug-query-block-graph results into one structure for block_layer_dump()
> +# function.
> +#
> +# Since: 5.3

The next release will be numbered 6.0, not 5.3.

-- 
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org