* [PATCH v5 01/13] qcow2.py: python style fixes
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:17 ` Vladimir Sementsov-Ogievskiy
2020-06-08 20:52 ` Eric Blake
2020-06-06 8:17 ` [PATCH v5 02/13] qcow2.py: add licensing blurb Vladimir Sementsov-Ogievskiy
` (11 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:17 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
Fix flake8 complaints. Leave the only chunk of lines over 79 characters:
initialization of cmds variable. Leave it for another day, when it
should be refactored to utilize argparse instead of hand-written
parsing.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
tests/qemu-iotests/qcow2.py | 95 ++++++++++++++++++++++---------------
1 file changed, 56 insertions(+), 39 deletions(-)
diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index 94a07b2f6f..d99f4ee3e8 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -4,6 +4,7 @@ import sys
import struct
import string
+
class QcowHeaderExtension:
def __init__(self, magic, length, data):
@@ -11,14 +12,15 @@ class QcowHeaderExtension:
padding = 8 - (length % 8)
data += b"\0" * padding
- self.magic = magic
+ self.magic = magic
self.length = length
- self.data = data
+ self.data = data
@classmethod
def create(cls, magic, data):
return QcowHeaderExtension(magic, len(data), data)
+
class QcowHeader:
uint32_t = 'I'
@@ -26,27 +28,27 @@ class QcowHeader:
fields = [
# Version 2 header fields
- [ uint32_t, '%#x', 'magic' ],
- [ uint32_t, '%d', 'version' ],
- [ uint64_t, '%#x', 'backing_file_offset' ],
- [ uint32_t, '%#x', 'backing_file_size' ],
- [ uint32_t, '%d', 'cluster_bits' ],
- [ uint64_t, '%d', 'size' ],
- [ uint32_t, '%d', 'crypt_method' ],
- [ uint32_t, '%d', 'l1_size' ],
- [ uint64_t, '%#x', 'l1_table_offset' ],
- [ uint64_t, '%#x', 'refcount_table_offset' ],
- [ uint32_t, '%d', 'refcount_table_clusters' ],
- [ uint32_t, '%d', 'nb_snapshots' ],
- [ uint64_t, '%#x', 'snapshot_offset' ],
+ [uint32_t, '%#x', 'magic'],
+ [uint32_t, '%d', 'version'],
+ [uint64_t, '%#x', 'backing_file_offset'],
+ [uint32_t, '%#x', 'backing_file_size'],
+ [uint32_t, '%d', 'cluster_bits'],
+ [uint64_t, '%d', 'size'],
+ [uint32_t, '%d', 'crypt_method'],
+ [uint32_t, '%d', 'l1_size'],
+ [uint64_t, '%#x', 'l1_table_offset'],
+ [uint64_t, '%#x', 'refcount_table_offset'],
+ [uint32_t, '%d', 'refcount_table_clusters'],
+ [uint32_t, '%d', 'nb_snapshots'],
+ [uint64_t, '%#x', 'snapshot_offset'],
# Version 3 header fields
- [ uint64_t, 'mask', 'incompatible_features' ],
- [ uint64_t, 'mask', 'compatible_features' ],
- [ uint64_t, 'mask', 'autoclear_features' ],
- [ uint32_t, '%d', 'refcount_order' ],
- [ uint32_t, '%d', 'header_length' ],
- ];
+ [uint64_t, 'mask', 'incompatible_features'],
+ [uint64_t, 'mask', 'compatible_features'],
+ [uint64_t, 'mask', 'autoclear_features'],
+ [uint32_t, '%d', 'refcount_order'],
+ [uint32_t, '%d', 'header_length'],
+ ]
fmt = '>' + ''.join(field[0] for field in fields)
@@ -59,7 +61,7 @@ class QcowHeader:
header = struct.unpack(QcowHeader.fmt, buf)
self.__dict__ = dict((field[2], header[i])
- for i, field in enumerate(QcowHeader.fields))
+ for i, field in enumerate(QcowHeader.fields))
self.set_defaults()
self.cluster_size = 1 << self.cluster_bits
@@ -96,7 +98,8 @@ class QcowHeader:
else:
padded = (length + 7) & ~7
data = fd.read(padded)
- self.extensions.append(QcowHeaderExtension(magic, length, data))
+ self.extensions.append(QcowHeaderExtension(magic, length,
+ data))
def update_extensions(self, fd):
@@ -108,14 +111,13 @@ class QcowHeader:
fd.write(buf)
fd.write(ex.data)
- if self.backing_file != None:
+ if self.backing_file is not None:
self.backing_file_offset = fd.tell()
fd.write(self.backing_file)
if fd.tell() > self.cluster_size:
raise Exception("I think I just broke the image...")
-
def update(self, fd):
header_bytes = self.header_length
@@ -163,19 +165,21 @@ def cmd_dump_header(fd):
h.dump()
h.dump_extensions()
+
def cmd_dump_header_exts(fd):
h = QcowHeader(fd)
h.dump_extensions()
+
def cmd_set_header(fd, name, value):
try:
value = int(value, 0)
- except:
+ except ValueError:
print("'%s' is not a valid number" % value)
sys.exit(1)
fields = (field[2] for field in QcowHeader.fields)
- if not name in fields:
+ if name not in fields:
print("'%s' is not a known header field" % name)
sys.exit(1)
@@ -183,25 +187,29 @@ def cmd_set_header(fd, name, value):
h.__dict__[name] = value
h.update(fd)
+
def cmd_add_header_ext(fd, magic, data):
try:
magic = int(magic, 0)
- except:
+ except ValueError:
print("'%s' is not a valid magic number" % magic)
sys.exit(1)
h = QcowHeader(fd)
- h.extensions.append(QcowHeaderExtension.create(magic, data.encode('ascii')))
+ h.extensions.append(QcowHeaderExtension.create(magic,
+ data.encode('ascii')))
h.update(fd)
+
def cmd_add_header_ext_stdio(fd, magic):
data = sys.stdin.read()
cmd_add_header_ext(fd, magic, data)
+
def cmd_del_header_ext(fd, magic):
try:
magic = int(magic, 0)
- except:
+ except ValueError:
print("'%s' is not a valid magic number" % magic)
sys.exit(1)
@@ -219,12 +227,13 @@ def cmd_del_header_ext(fd, magic):
h.update(fd)
+
def cmd_set_feature_bit(fd, group, bit):
try:
bit = int(bit, 0)
if bit < 0 or bit >= 64:
raise ValueError
- except:
+ except ValueError:
print("'%s' is not a valid bit number in range [0, 64)" % bit)
sys.exit(1)
@@ -236,21 +245,27 @@ def cmd_set_feature_bit(fd, group, bit):
elif group == 'autoclear':
h.autoclear_features |= 1 << bit
else:
- print("'%s' is not a valid group, try 'incompatible', 'compatible', or 'autoclear'" % group)
+ print("'%s' is not a valid group, try "
+ "'incompatible', 'compatible', or 'autoclear'" % group)
sys.exit(1)
h.update(fd)
+
cmds = [
- [ 'dump-header', cmd_dump_header, 0, 'Dump image header and header extensions' ],
- [ 'dump-header-exts', cmd_dump_header_exts, 0, 'Dump image header extensions' ],
- [ 'set-header', cmd_set_header, 2, 'Set a field in the header'],
- [ 'add-header-ext', cmd_add_header_ext, 2, 'Add a header extension' ],
- [ 'add-header-ext-stdio', cmd_add_header_ext_stdio, 1, 'Add a header extension, data from stdin' ],
- [ 'del-header-ext', cmd_del_header_ext, 1, 'Delete a header extension' ],
- [ 'set-feature-bit', cmd_set_feature_bit, 2, 'Set a feature bit'],
+ ['dump-header', cmd_dump_header, 0,
+ 'Dump image header and header extensions'],
+ ['dump-header-exts', cmd_dump_header_exts, 0,
+ 'Dump image header extensions'],
+ ['set-header', cmd_set_header, 2, 'Set a field in the header'],
+ ['add-header-ext', cmd_add_header_ext, 2, 'Add a header extension'],
+ ['add-header-ext-stdio', cmd_add_header_ext_stdio, 1,
+ 'Add a header extension, data from stdin'],
+ ['del-header-ext', cmd_del_header_ext, 1, 'Delete a header extension'],
+ ['set-feature-bit', cmd_set_feature_bit, 2, 'Set a feature bit'],
]
+
def main(filename, cmd, args):
fd = open(filename, "r+b")
try:
@@ -267,6 +282,7 @@ def main(filename, cmd, args):
finally:
fd.close()
+
def usage():
print("Usage: %s <file> <cmd> [<arg>, ...]" % sys.argv[0])
print("")
@@ -274,6 +290,7 @@ def usage():
for name, handler, num_args, desc in cmds:
print(" %-20s - %s" % (name, desc))
+
if __name__ == '__main__':
if len(sys.argv) < 3:
usage()
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH v5 01/13] qcow2.py: python style fixes
2020-06-06 8:17 ` [PATCH v5 01/13] qcow2.py: python style fixes Vladimir Sementsov-Ogievskiy
@ 2020-06-08 20:52 ` Eric Blake
0 siblings, 0 replies; 23+ messages in thread
From: Eric Blake @ 2020-06-08 20:52 UTC (permalink / raw)
To: Vladimir Sementsov-Ogievskiy, qemu-block
Cc: kwolf, den, andrey.shinkevich, qemu-devel, mreitz
On 6/6/20 3:17 AM, Vladimir Sementsov-Ogievskiy wrote:
> Fix flake8 complaints. Leave the only chunk of lines over 79 characters:
Stale sentence since you fixed that; on my Fedora 32 machine, with
flake8 3.7.7, this patch silences all warnings.
> initialization of cmds variable. Leave it for another day, when it
> should be refactored to utilize argparse instead of hand-written
> parsing.
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> ---
> tests/qemu-iotests/qcow2.py | 95 ++++++++++++++++++++++---------------
> 1 file changed, 56 insertions(+), 39 deletions(-)
Reviewed-by: Eric Blake <eblake@redhat.com>
Tested-by: Eric Blake <eblake@redhat.com>
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH v5 02/13] qcow2.py: add licensing blurb
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
2020-06-06 8:17 ` [PATCH v5 01/13] qcow2.py: python style fixes Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:17 ` Vladimir Sementsov-Ogievskiy
2020-06-08 21:01 ` Eric Blake
2020-06-06 8:17 ` [PATCH v5 03/13] qcow2.py: move qcow2 format classes to separate module Vladimir Sementsov-Ogievskiy
` (10 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:17 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
Add classic heading, which is missing here. Keep copyright place empty,
for anyone who have added (or will add) some intellectual property
here.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
tests/qemu-iotests/qcow2.py | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index d99f4ee3e8..2da434a013 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -1,4 +1,20 @@
#!/usr/bin/env python3
+#
+# Manipulations with qcow2 image
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
import sys
import struct
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH v5 02/13] qcow2.py: add licensing blurb
2020-06-06 8:17 ` [PATCH v5 02/13] qcow2.py: add licensing blurb Vladimir Sementsov-Ogievskiy
@ 2020-06-08 21:01 ` Eric Blake
0 siblings, 0 replies; 23+ messages in thread
From: Eric Blake @ 2020-06-08 21:01 UTC (permalink / raw)
To: Vladimir Sementsov-Ogievskiy, qemu-block
Cc: kwolf, Eduardo Habkost, qemu-devel, mreitz, Paolo Bonzini,
stefanha, andrey.shinkevich, den, Philippe Mathieu-Daudé
On 6/6/20 3:17 AM, Vladimir Sementsov-Ogievskiy wrote:
> Add classic heading, which is missing here. Keep copyright place empty,
> for anyone who have added (or will add) some intellectual property
> here.
It's not so much intellectual property (since that term is at odds with
open source), but authorship rights.
Looking at git history, the file has been touched by:
Kevin Wolf
Stefan Hajnoczi (while at IBM)
Eduardo Habkost
Max Reitz
Philippe Mathieu-Daudé
Paolo Bonzini
where Stefan was the only contributor without a redhat.com address at
the time. So if anything, a Red Hat copyright is most likely; but you
are also correct that it is incorrect to add a copyright line on someone
else's behalf without their permission.
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> ---
> tests/qemu-iotests/qcow2.py | 16 ++++++++++++++++
> 1 file changed, 16 insertions(+)
>
> diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
> index d99f4ee3e8..2da434a013 100755
> --- a/tests/qemu-iotests/qcow2.py
> +++ b/tests/qemu-iotests/qcow2.py
> @@ -1,4 +1,20 @@
> #!/usr/bin/env python3
> +#
> +# Manipulations with qcow2 image
> +#
I've cc'd all prior authors; if Kevin agrees, and unless anyone speaks
up to the contrary, I'm willing to add:
# Copyright (C) 2012 Red Hat, Inc.
for Kevin's initial contribution, without worrying about subsequent
contributions.
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program. If not, see <http://www.gnu.org/licenses/>.
> +#
>
> import sys
> import struct
>
Adding a copyright line could be a followup patch, so in the meantime,
making what was previously an implicit license now explicit is fine even
if it is odd to assert GPL without also asserting Copyright.
Reviewed-by: Eric Blake <eblake@redhat.com>
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH v5 03/13] qcow2.py: move qcow2 format classes to separate module
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
2020-06-06 8:17 ` [PATCH v5 01/13] qcow2.py: python style fixes Vladimir Sementsov-Ogievskiy
2020-06-06 8:17 ` [PATCH v5 02/13] qcow2.py: add licensing blurb Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:17 ` Vladimir Sementsov-Ogievskiy
2020-06-08 21:05 ` Eric Blake
2020-06-06 8:17 ` [PATCH v5 04/13] qcow2_format.py: drop new line printing at end of dump() Vladimir Sementsov-Ogievskiy
` (9 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:17 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
We are going to enhance qcow2 format parsing by adding more structure
classes. Let's split format parsing from utility code.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
tests/qemu-iotests/qcow2.py | 160 +-------------------------
tests/qemu-iotests/qcow2_format.py | 173 +++++++++++++++++++++++++++++
2 files changed, 177 insertions(+), 156 deletions(-)
create mode 100644 tests/qemu-iotests/qcow2_format.py
diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index 2da434a013..e968869ea6 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -17,163 +17,11 @@
#
import sys
-import struct
-import string
-
-
-class QcowHeaderExtension:
-
- def __init__(self, magic, length, data):
- if length % 8 != 0:
- padding = 8 - (length % 8)
- data += b"\0" * padding
-
- self.magic = magic
- self.length = length
- self.data = data
-
- @classmethod
- def create(cls, magic, data):
- return QcowHeaderExtension(magic, len(data), data)
-
-
-class QcowHeader:
-
- uint32_t = 'I'
- uint64_t = 'Q'
-
- fields = [
- # Version 2 header fields
- [uint32_t, '%#x', 'magic'],
- [uint32_t, '%d', 'version'],
- [uint64_t, '%#x', 'backing_file_offset'],
- [uint32_t, '%#x', 'backing_file_size'],
- [uint32_t, '%d', 'cluster_bits'],
- [uint64_t, '%d', 'size'],
- [uint32_t, '%d', 'crypt_method'],
- [uint32_t, '%d', 'l1_size'],
- [uint64_t, '%#x', 'l1_table_offset'],
- [uint64_t, '%#x', 'refcount_table_offset'],
- [uint32_t, '%d', 'refcount_table_clusters'],
- [uint32_t, '%d', 'nb_snapshots'],
- [uint64_t, '%#x', 'snapshot_offset'],
-
- # Version 3 header fields
- [uint64_t, 'mask', 'incompatible_features'],
- [uint64_t, 'mask', 'compatible_features'],
- [uint64_t, 'mask', 'autoclear_features'],
- [uint32_t, '%d', 'refcount_order'],
- [uint32_t, '%d', 'header_length'],
- ]
-
- fmt = '>' + ''.join(field[0] for field in fields)
-
- def __init__(self, fd):
-
- buf_size = struct.calcsize(QcowHeader.fmt)
-
- fd.seek(0)
- buf = fd.read(buf_size)
-
- header = struct.unpack(QcowHeader.fmt, buf)
- self.__dict__ = dict((field[2], header[i])
- for i, field in enumerate(QcowHeader.fields))
-
- self.set_defaults()
- self.cluster_size = 1 << self.cluster_bits
-
- fd.seek(self.header_length)
- self.load_extensions(fd)
-
- if self.backing_file_offset:
- fd.seek(self.backing_file_offset)
- self.backing_file = fd.read(self.backing_file_size)
- else:
- self.backing_file = None
-
- def set_defaults(self):
- if self.version == 2:
- self.incompatible_features = 0
- self.compatible_features = 0
- self.autoclear_features = 0
- self.refcount_order = 4
- self.header_length = 72
-
- def load_extensions(self, fd):
- self.extensions = []
-
- if self.backing_file_offset != 0:
- end = min(self.cluster_size, self.backing_file_offset)
- else:
- end = self.cluster_size
-
- while fd.tell() < end:
- (magic, length) = struct.unpack('>II', fd.read(8))
- if magic == 0:
- break
- else:
- padded = (length + 7) & ~7
- data = fd.read(padded)
- self.extensions.append(QcowHeaderExtension(magic, length,
- data))
-
- def update_extensions(self, fd):
-
- fd.seek(self.header_length)
- extensions = self.extensions
- extensions.append(QcowHeaderExtension(0, 0, b""))
- for ex in extensions:
- buf = struct.pack('>II', ex.magic, ex.length)
- fd.write(buf)
- fd.write(ex.data)
-
- if self.backing_file is not None:
- self.backing_file_offset = fd.tell()
- fd.write(self.backing_file)
-
- if fd.tell() > self.cluster_size:
- raise Exception("I think I just broke the image...")
-
- def update(self, fd):
- header_bytes = self.header_length
-
- self.update_extensions(fd)
-
- fd.seek(0)
- header = tuple(self.__dict__[f] for t, p, f in QcowHeader.fields)
- buf = struct.pack(QcowHeader.fmt, *header)
- buf = buf[0:header_bytes-1]
- fd.write(buf)
-
- def dump(self):
- for f in QcowHeader.fields:
- value = self.__dict__[f[2]]
- if f[1] == 'mask':
- bits = []
- for bit in range(64):
- if value & (1 << bit):
- bits.append(bit)
- value_str = str(bits)
- else:
- value_str = f[1] % value
-
- print("%-25s" % f[2], value_str)
- print("")
-
- def dump_extensions(self):
- for ex in self.extensions:
-
- data = ex.data[:ex.length]
- if all(c in string.printable.encode('ascii') for c in data):
- data = "'%s'" % data.decode('ascii')
- else:
- data = "<binary>"
- print("Header extension:")
- print("%-25s %#x" % ("magic", ex.magic))
- print("%-25s %d" % ("length", ex.length))
- print("%-25s %s" % ("data", data))
- print("")
+from qcow2_format import (
+ QcowHeader,
+ QcowHeaderExtension
+)
def cmd_dump_header(fd):
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
new file mode 100644
index 0000000000..0e517c1bbc
--- /dev/null
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -0,0 +1,173 @@
+# Library for manipulations with qcow2 image
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import struct
+import string
+
+
+class QcowHeaderExtension:
+
+ def __init__(self, magic, length, data):
+ if length % 8 != 0:
+ padding = 8 - (length % 8)
+ data += b"\0" * padding
+
+ self.magic = magic
+ self.length = length
+ self.data = data
+
+ @classmethod
+ def create(cls, magic, data):
+ return QcowHeaderExtension(magic, len(data), data)
+
+
+class QcowHeader:
+
+ uint32_t = 'I'
+ uint64_t = 'Q'
+
+ fields = [
+ # Version 2 header fields
+ [uint32_t, '%#x', 'magic'],
+ [uint32_t, '%d', 'version'],
+ [uint64_t, '%#x', 'backing_file_offset'],
+ [uint32_t, '%#x', 'backing_file_size'],
+ [uint32_t, '%d', 'cluster_bits'],
+ [uint64_t, '%d', 'size'],
+ [uint32_t, '%d', 'crypt_method'],
+ [uint32_t, '%d', 'l1_size'],
+ [uint64_t, '%#x', 'l1_table_offset'],
+ [uint64_t, '%#x', 'refcount_table_offset'],
+ [uint32_t, '%d', 'refcount_table_clusters'],
+ [uint32_t, '%d', 'nb_snapshots'],
+ [uint64_t, '%#x', 'snapshot_offset'],
+
+ # Version 3 header fields
+ [uint64_t, 'mask', 'incompatible_features'],
+ [uint64_t, 'mask', 'compatible_features'],
+ [uint64_t, 'mask', 'autoclear_features'],
+ [uint32_t, '%d', 'refcount_order'],
+ [uint32_t, '%d', 'header_length'],
+ ]
+
+ fmt = '>' + ''.join(field[0] for field in fields)
+
+ def __init__(self, fd):
+
+ buf_size = struct.calcsize(QcowHeader.fmt)
+
+ fd.seek(0)
+ buf = fd.read(buf_size)
+
+ header = struct.unpack(QcowHeader.fmt, buf)
+ self.__dict__ = dict((field[2], header[i])
+ for i, field in enumerate(QcowHeader.fields))
+
+ self.set_defaults()
+ self.cluster_size = 1 << self.cluster_bits
+
+ fd.seek(self.header_length)
+ self.load_extensions(fd)
+
+ if self.backing_file_offset:
+ fd.seek(self.backing_file_offset)
+ self.backing_file = fd.read(self.backing_file_size)
+ else:
+ self.backing_file = None
+
+ def set_defaults(self):
+ if self.version == 2:
+ self.incompatible_features = 0
+ self.compatible_features = 0
+ self.autoclear_features = 0
+ self.refcount_order = 4
+ self.header_length = 72
+
+ def load_extensions(self, fd):
+ self.extensions = []
+
+ if self.backing_file_offset != 0:
+ end = min(self.cluster_size, self.backing_file_offset)
+ else:
+ end = self.cluster_size
+
+ while fd.tell() < end:
+ (magic, length) = struct.unpack('>II', fd.read(8))
+ if magic == 0:
+ break
+ else:
+ padded = (length + 7) & ~7
+ data = fd.read(padded)
+ self.extensions.append(QcowHeaderExtension(magic, length,
+ data))
+
+ def update_extensions(self, fd):
+
+ fd.seek(self.header_length)
+ extensions = self.extensions
+ extensions.append(QcowHeaderExtension(0, 0, b""))
+ for ex in extensions:
+ buf = struct.pack('>II', ex.magic, ex.length)
+ fd.write(buf)
+ fd.write(ex.data)
+
+ if self.backing_file is not None:
+ self.backing_file_offset = fd.tell()
+ fd.write(self.backing_file)
+
+ if fd.tell() > self.cluster_size:
+ raise Exception("I think I just broke the image...")
+
+ def update(self, fd):
+ header_bytes = self.header_length
+
+ self.update_extensions(fd)
+
+ fd.seek(0)
+ header = tuple(self.__dict__[f] for t, p, f in QcowHeader.fields)
+ buf = struct.pack(QcowHeader.fmt, *header)
+ buf = buf[0:header_bytes-1]
+ fd.write(buf)
+
+ def dump(self):
+ for f in QcowHeader.fields:
+ value = self.__dict__[f[2]]
+ if f[1] == 'mask':
+ bits = []
+ for bit in range(64):
+ if value & (1 << bit):
+ bits.append(bit)
+ value_str = str(bits)
+ else:
+ value_str = f[1] % value
+
+ print("%-25s" % f[2], value_str)
+ print("")
+
+ def dump_extensions(self):
+ for ex in self.extensions:
+
+ data = ex.data[:ex.length]
+ if all(c in string.printable.encode('ascii') for c in data):
+ data = "'%s'" % data.decode('ascii')
+ else:
+ data = "<binary>"
+
+ print("Header extension:")
+ print("%-25s %#x" % ("magic", ex.magic))
+ print("%-25s %d" % ("length", ex.length))
+ print("%-25s %s" % ("data", data))
+ print("")
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH v5 03/13] qcow2.py: move qcow2 format classes to separate module
2020-06-06 8:17 ` [PATCH v5 03/13] qcow2.py: move qcow2 format classes to separate module Vladimir Sementsov-Ogievskiy
@ 2020-06-08 21:05 ` Eric Blake
0 siblings, 0 replies; 23+ messages in thread
From: Eric Blake @ 2020-06-08 21:05 UTC (permalink / raw)
To: Vladimir Sementsov-Ogievskiy, qemu-block
Cc: kwolf, den, andrey.shinkevich, qemu-devel, mreitz
On 6/6/20 3:17 AM, Vladimir Sementsov-Ogievskiy wrote:
> We are going to enhance qcow2 format parsing by adding more structure
> classes. Let's split format parsing from utility code.
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> ---
> tests/qemu-iotests/qcow2.py | 160 +-------------------------
> tests/qemu-iotests/qcow2_format.py | 173 +++++++++++++++++++++++++++++
> 2 files changed, 177 insertions(+), 156 deletions(-)
> create mode 100644 tests/qemu-iotests/qcow2_format.py
>
> +++ b/tests/qemu-iotests/qcow2_format.py
> @@ -0,0 +1,173 @@
> +# Library for manipulations with qcow2 image
> +#
Any copyright line added in 2/13 should also be added here. Again, that
can be a followup for authorship reasons if we decide what copyright
line is best.
Otherwise, this is a clean code motion patch where the difference in
line length is the boilerplate header and the change to imports.
Reviewed-by: Eric Blake <eblake@redhat.com>
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH v5 04/13] qcow2_format.py: drop new line printing at end of dump()
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (2 preceding siblings ...)
2020-06-06 8:17 ` [PATCH v5 03/13] qcow2.py: move qcow2 format classes to separate module Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:17 ` Vladimir Sementsov-Ogievskiy
2020-06-06 8:17 ` [PATCH v5 05/13] qcow2_format.py: use tuples instead of lists for fields Vladimir Sementsov-Ogievskiy
` (8 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:17 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
This will simplify further conversion. To compensate, print this empty
line directly in cmd_dump_header().
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/qcow2.py | 1 +
tests/qemu-iotests/qcow2_format.py | 1 -
2 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index e968869ea6..8c187e9a72 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -27,6 +27,7 @@ from qcow2_format import (
def cmd_dump_header(fd):
h = QcowHeader(fd)
h.dump()
+ print()
h.dump_extensions()
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 0e517c1bbc..2b6c9612ae 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -155,7 +155,6 @@ class QcowHeader:
value_str = f[1] % value
print("%-25s" % f[2], value_str)
- print("")
def dump_extensions(self):
for ex in self.extensions:
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH v5 05/13] qcow2_format.py: use tuples instead of lists for fields
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (3 preceding siblings ...)
2020-06-06 8:17 ` [PATCH v5 04/13] qcow2_format.py: drop new line printing at end of dump() Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:17 ` Vladimir Sementsov-Ogievskiy
2020-06-06 8:17 ` [PATCH v5 06/13] qcow2_format.py: use modern string formatting Vladimir Sementsov-Ogievskiy
` (7 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:17 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
No need in lists: it's a constant variable.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/qcow2_format.py | 40 +++++++++++++++---------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 2b6c9612ae..e2f08ed691 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -39,29 +39,29 @@ class QcowHeader:
uint32_t = 'I'
uint64_t = 'Q'
- fields = [
+ fields = (
# Version 2 header fields
- [uint32_t, '%#x', 'magic'],
- [uint32_t, '%d', 'version'],
- [uint64_t, '%#x', 'backing_file_offset'],
- [uint32_t, '%#x', 'backing_file_size'],
- [uint32_t, '%d', 'cluster_bits'],
- [uint64_t, '%d', 'size'],
- [uint32_t, '%d', 'crypt_method'],
- [uint32_t, '%d', 'l1_size'],
- [uint64_t, '%#x', 'l1_table_offset'],
- [uint64_t, '%#x', 'refcount_table_offset'],
- [uint32_t, '%d', 'refcount_table_clusters'],
- [uint32_t, '%d', 'nb_snapshots'],
- [uint64_t, '%#x', 'snapshot_offset'],
+ (uint32_t, '%#x', 'magic'),
+ (uint32_t, '%d', 'version'),
+ (uint64_t, '%#x', 'backing_file_offset'),
+ (uint32_t, '%#x', 'backing_file_size'),
+ (uint32_t, '%d', 'cluster_bits'),
+ (uint64_t, '%d', 'size'),
+ (uint32_t, '%d', 'crypt_method'),
+ (uint32_t, '%d', 'l1_size'),
+ (uint64_t, '%#x', 'l1_table_offset'),
+ (uint64_t, '%#x', 'refcount_table_offset'),
+ (uint32_t, '%d', 'refcount_table_clusters'),
+ (uint32_t, '%d', 'nb_snapshots'),
+ (uint64_t, '%#x', 'snapshot_offset'),
# Version 3 header fields
- [uint64_t, 'mask', 'incompatible_features'],
- [uint64_t, 'mask', 'compatible_features'],
- [uint64_t, 'mask', 'autoclear_features'],
- [uint32_t, '%d', 'refcount_order'],
- [uint32_t, '%d', 'header_length'],
- ]
+ (uint64_t, 'mask', 'incompatible_features'),
+ (uint64_t, 'mask', 'compatible_features'),
+ (uint64_t, 'mask', 'autoclear_features'),
+ (uint32_t, '%d', 'refcount_order'),
+ (uint32_t, '%d', 'header_length'),
+ )
fmt = '>' + ''.join(field[0] for field in fields)
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH v5 06/13] qcow2_format.py: use modern string formatting
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (4 preceding siblings ...)
2020-06-06 8:17 ` [PATCH v5 05/13] qcow2_format.py: use tuples instead of lists for fields Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:17 ` Vladimir Sementsov-Ogievskiy
2020-06-06 8:18 ` [PATCH v5 07/13] qcow2_format.py: use strings to specify c-type of struct fields Vladimir Sementsov-Ogievskiy
` (6 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:17 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
Use .format and f-strings instead of old %style. Also, the file uses
both '' and "" quotes, for consistency let's use '', except for cases
when we need '' inside the string (use "" to avoid extra escaping).
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/qcow2_format.py | 54 +++++++++++++++---------------
1 file changed, 27 insertions(+), 27 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index e2f08ed691..da66df3408 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -23,7 +23,7 @@ class QcowHeaderExtension:
def __init__(self, magic, length, data):
if length % 8 != 0:
padding = 8 - (length % 8)
- data += b"\0" * padding
+ data += b'\0' * padding
self.magic = magic
self.length = length
@@ -41,26 +41,26 @@ class QcowHeader:
fields = (
# Version 2 header fields
- (uint32_t, '%#x', 'magic'),
- (uint32_t, '%d', 'version'),
- (uint64_t, '%#x', 'backing_file_offset'),
- (uint32_t, '%#x', 'backing_file_size'),
- (uint32_t, '%d', 'cluster_bits'),
- (uint64_t, '%d', 'size'),
- (uint32_t, '%d', 'crypt_method'),
- (uint32_t, '%d', 'l1_size'),
- (uint64_t, '%#x', 'l1_table_offset'),
- (uint64_t, '%#x', 'refcount_table_offset'),
- (uint32_t, '%d', 'refcount_table_clusters'),
- (uint32_t, '%d', 'nb_snapshots'),
- (uint64_t, '%#x', 'snapshot_offset'),
+ (uint32_t, '{:#x}', 'magic'),
+ (uint32_t, '{}', 'version'),
+ (uint64_t, '{:#x}', 'backing_file_offset'),
+ (uint32_t, '{:#x}', 'backing_file_size'),
+ (uint32_t, '{}', 'cluster_bits'),
+ (uint64_t, '{}', 'size'),
+ (uint32_t, '{}', 'crypt_method'),
+ (uint32_t, '{}', 'l1_size'),
+ (uint64_t, '{:#x}', 'l1_table_offset'),
+ (uint64_t, '{:#x}', 'refcount_table_offset'),
+ (uint32_t, '{}', 'refcount_table_clusters'),
+ (uint32_t, '{}', 'nb_snapshots'),
+ (uint64_t, '{:#x}', 'snapshot_offset'),
# Version 3 header fields
(uint64_t, 'mask', 'incompatible_features'),
(uint64_t, 'mask', 'compatible_features'),
(uint64_t, 'mask', 'autoclear_features'),
- (uint32_t, '%d', 'refcount_order'),
- (uint32_t, '%d', 'header_length'),
+ (uint32_t, '{}', 'refcount_order'),
+ (uint32_t, '{}', 'header_length'),
)
fmt = '>' + ''.join(field[0] for field in fields)
@@ -118,7 +118,7 @@ class QcowHeader:
fd.seek(self.header_length)
extensions = self.extensions
- extensions.append(QcowHeaderExtension(0, 0, b""))
+ extensions.append(QcowHeaderExtension(0, 0, b''))
for ex in extensions:
buf = struct.pack('>II', ex.magic, ex.length)
fd.write(buf)
@@ -129,7 +129,7 @@ class QcowHeader:
fd.write(self.backing_file)
if fd.tell() > self.cluster_size:
- raise Exception("I think I just broke the image...")
+ raise Exception('I think I just broke the image...')
def update(self, fd):
header_bytes = self.header_length
@@ -152,21 +152,21 @@ class QcowHeader:
bits.append(bit)
value_str = str(bits)
else:
- value_str = f[1] % value
+ value_str = f[1].format(value)
- print("%-25s" % f[2], value_str)
+ print(f'{f[2]:<25} {value_str}')
def dump_extensions(self):
for ex in self.extensions:
data = ex.data[:ex.length]
if all(c in string.printable.encode('ascii') for c in data):
- data = "'%s'" % data.decode('ascii')
+ data = f"'{ data.decode('ascii') }'"
else:
- data = "<binary>"
+ data = '<binary>'
- print("Header extension:")
- print("%-25s %#x" % ("magic", ex.magic))
- print("%-25s %d" % ("length", ex.length))
- print("%-25s %s" % ("data", data))
- print("")
+ print('Header extension:')
+ print(f'{"magic":<25} {ex.magic:#x}')
+ print(f'{"length":<25} {ex.length}')
+ print(f'{"data":<25} {data}')
+ print()
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH v5 07/13] qcow2_format.py: use strings to specify c-type of struct fields
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (5 preceding siblings ...)
2020-06-06 8:17 ` [PATCH v5 06/13] qcow2_format.py: use modern string formatting Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:18 ` Vladimir Sementsov-Ogievskiy
2020-06-06 8:18 ` [PATCH v5 08/13] qcow2_format.py: separate generic functionality of structure classes Vladimir Sementsov-Ogievskiy
` (5 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:18 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
We are going to move field-parsing to super-class, this will be simpler
with simple string specifiers instead of variables.
For some reason, python doesn't allow the definition of ctypes variable
in the class alongside fields: it would not be available then for use
by the 'for' operator. Don't worry: ctypes will be moved to metaclass
soon.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/qcow2_format.py | 50 +++++++++++++++++-------------
1 file changed, 28 insertions(+), 22 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index da66df3408..28f2bfa63b 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -34,36 +34,42 @@ class QcowHeaderExtension:
return QcowHeaderExtension(magic, len(data), data)
-class QcowHeader:
+# Mapping from c types to python struct format
+ctypes = {
+ 'u8': 'B',
+ 'u16': 'H',
+ 'u32': 'I',
+ 'u64': 'Q'
+}
+
- uint32_t = 'I'
- uint64_t = 'Q'
+class QcowHeader:
fields = (
# Version 2 header fields
- (uint32_t, '{:#x}', 'magic'),
- (uint32_t, '{}', 'version'),
- (uint64_t, '{:#x}', 'backing_file_offset'),
- (uint32_t, '{:#x}', 'backing_file_size'),
- (uint32_t, '{}', 'cluster_bits'),
- (uint64_t, '{}', 'size'),
- (uint32_t, '{}', 'crypt_method'),
- (uint32_t, '{}', 'l1_size'),
- (uint64_t, '{:#x}', 'l1_table_offset'),
- (uint64_t, '{:#x}', 'refcount_table_offset'),
- (uint32_t, '{}', 'refcount_table_clusters'),
- (uint32_t, '{}', 'nb_snapshots'),
- (uint64_t, '{:#x}', 'snapshot_offset'),
+ ('u32', '{:#x}', 'magic'),
+ ('u32', '{}', 'version'),
+ ('u64', '{:#x}', 'backing_file_offset'),
+ ('u32', '{:#x}', 'backing_file_size'),
+ ('u32', '{}', 'cluster_bits'),
+ ('u64', '{}', 'size'),
+ ('u32', '{}', 'crypt_method'),
+ ('u32', '{}', 'l1_size'),
+ ('u64', '{:#x}', 'l1_table_offset'),
+ ('u64', '{:#x}', 'refcount_table_offset'),
+ ('u32', '{}', 'refcount_table_clusters'),
+ ('u32', '{}', 'nb_snapshots'),
+ ('u64', '{:#x}', 'snapshot_offset'),
# Version 3 header fields
- (uint64_t, 'mask', 'incompatible_features'),
- (uint64_t, 'mask', 'compatible_features'),
- (uint64_t, 'mask', 'autoclear_features'),
- (uint32_t, '{}', 'refcount_order'),
- (uint32_t, '{}', 'header_length'),
+ ('u64', 'mask', 'incompatible_features'),
+ ('u64', 'mask', 'compatible_features'),
+ ('u64', 'mask', 'autoclear_features'),
+ ('u32', '{}', 'refcount_order'),
+ ('u32', '{}', 'header_length'),
)
- fmt = '>' + ''.join(field[0] for field in fields)
+ fmt = '>' + ''.join(ctypes[f[0]] for f in fields)
def __init__(self, fd):
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH v5 08/13] qcow2_format.py: separate generic functionality of structure classes
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (6 preceding siblings ...)
2020-06-06 8:18 ` [PATCH v5 07/13] qcow2_format.py: use strings to specify c-type of struct fields Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:18 ` Vladimir Sementsov-Ogievskiy
2020-06-08 21:28 ` Eric Blake
2020-06-06 8:18 ` [PATCH v5 09/13] qcow2_format.py: add field-formatting class Vladimir Sementsov-Ogievskiy
` (4 subsequent siblings)
12 siblings, 1 reply; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:18 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
We are going to introduce more Qcow2 structure types, defined like
QcowHeader. Move generic functionality into base class to be reused for
further structure classes.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/qcow2_format.py | 101 +++++++++++++++++++----------
1 file changed, 66 insertions(+), 35 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 28f2bfa63b..898d388b8a 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -1,5 +1,7 @@
# Library for manipulations with qcow2 image
#
+# Copyright (c) 2020 Virtuozzo International GmbH.
+#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
@@ -18,6 +20,68 @@ import struct
import string
+class Qcow2StructMeta(type):
+
+ # Mapping from c types to python struct format
+ ctypes = {
+ 'u8': 'B',
+ 'u16': 'H',
+ 'u32': 'I',
+ 'u64': 'Q'
+ }
+
+ def __init__(self, name, bases, attrs):
+ if 'fields' in attrs:
+ self.fmt = '>' + ''.join(self.ctypes[f[0]] for f in self.fields)
+
+
+class Qcow2Struct(metaclass=Qcow2StructMeta):
+
+ """Qcow2Struct: base class for qcow2 data structures
+
+ Successors should define fields class variable, which is: list of tuples,
+ each of three elements:
+ - c-type (one of 'u8', 'u16', 'u32', 'u64')
+ - format (format_spec to use with .format() when dump or 'mask' to dump
+ bitmasks)
+ - field name
+ """
+
+ def __init__(self, fd=None, offset=None, data=None):
+ """
+ Two variants:
+ 1. Specify data. fd and offset must be None.
+ 2. Specify fd and offset, data must be None. offset may be omitted
+ in this case, than current position of fd is used.
+ """
+ if data is None:
+ assert fd is not None
+ buf_size = struct.calcsize(self.fmt)
+ if offset is not None:
+ fd.seek(offset)
+ data = fd.read(buf_size)
+ else:
+ assert fd is None and offset is None
+
+ values = struct.unpack(self.fmt, data)
+ self.__dict__ = dict((field[2], values[i])
+ for i, field in enumerate(self.fields))
+
+ def dump(self):
+ for f in self.fields:
+ value = self.__dict__[f[2]]
+ if f[1] == 'mask':
+ bits = []
+ for bit in range(64):
+ if value & (1 << bit):
+ bits.append(bit)
+ value_str = str(bits)
+ else:
+ value_str = f[1].format(value)
+
+ print('{:<25} {}'.format(f[2], value_str))
+
+
class QcowHeaderExtension:
def __init__(self, magic, length, data):
@@ -34,16 +98,7 @@ class QcowHeaderExtension:
return QcowHeaderExtension(magic, len(data), data)
-# Mapping from c types to python struct format
-ctypes = {
- 'u8': 'B',
- 'u16': 'H',
- 'u32': 'I',
- 'u64': 'Q'
-}
-
-
-class QcowHeader:
+class QcowHeader(Qcow2Struct):
fields = (
# Version 2 header fields
@@ -69,18 +124,8 @@ class QcowHeader:
('u32', '{}', 'header_length'),
)
- fmt = '>' + ''.join(ctypes[f[0]] for f in fields)
-
def __init__(self, fd):
-
- buf_size = struct.calcsize(QcowHeader.fmt)
-
- fd.seek(0)
- buf = fd.read(buf_size)
-
- header = struct.unpack(QcowHeader.fmt, buf)
- self.__dict__ = dict((field[2], header[i])
- for i, field in enumerate(QcowHeader.fields))
+ super().__init__(fd=fd, offset=0)
self.set_defaults()
self.cluster_size = 1 << self.cluster_bits
@@ -148,20 +193,6 @@ class QcowHeader:
buf = buf[0:header_bytes-1]
fd.write(buf)
- def dump(self):
- for f in QcowHeader.fields:
- value = self.__dict__[f[2]]
- if f[1] == 'mask':
- bits = []
- for bit in range(64):
- if value & (1 << bit):
- bits.append(bit)
- value_str = str(bits)
- else:
- value_str = f[1].format(value)
-
- print(f'{f[2]:<25} {value_str}')
-
def dump_extensions(self):
for ex in self.extensions:
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH v5 08/13] qcow2_format.py: separate generic functionality of structure classes
2020-06-06 8:18 ` [PATCH v5 08/13] qcow2_format.py: separate generic functionality of structure classes Vladimir Sementsov-Ogievskiy
@ 2020-06-08 21:28 ` Eric Blake
0 siblings, 0 replies; 23+ messages in thread
From: Eric Blake @ 2020-06-08 21:28 UTC (permalink / raw)
To: Vladimir Sementsov-Ogievskiy, qemu-block
Cc: kwolf, den, andrey.shinkevich, qemu-devel, mreitz
On 6/6/20 3:18 AM, Vladimir Sementsov-Ogievskiy wrote:
> We are going to introduce more Qcow2 structure types, defined like
> QcowHeader. Move generic functionality into base class to be reused for
> further structure classes.
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
> ---
> tests/qemu-iotests/qcow2_format.py | 101 +++++++++++++++++++----------
> 1 file changed, 66 insertions(+), 35 deletions(-)
>
> diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
> index 28f2bfa63b..898d388b8a 100644
> --- a/tests/qemu-iotests/qcow2_format.py
> +++ b/tests/qemu-iotests/qcow2_format.py
> @@ -1,5 +1,7 @@
> # Library for manipulations with qcow2 image
> #
> +# Copyright (c) 2020 Virtuozzo International GmbH.
Deferring the addition of this line until this point in the series is
fine. Depending on the answer I get to adding any other copyright line
in 2/ and 3/13, or if that is deferred to later, we can figure out the
merge conflict by keeping both.
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH v5 09/13] qcow2_format.py: add field-formatting class
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (7 preceding siblings ...)
2020-06-06 8:18 ` [PATCH v5 08/13] qcow2_format.py: separate generic functionality of structure classes Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:18 ` Vladimir Sementsov-Ogievskiy
2020-06-06 8:18 ` [PATCH v5 10/13] qcow2_format.py: QcowHeaderExtension: add dump method Vladimir Sementsov-Ogievskiy
` (3 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:18 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
Allow formatter class in structure definition instead of hacking with
'mask'. This will simplify further introduction of new formatters.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/qcow2_format.py | 35 +++++++++++++++++++++---------
1 file changed, 25 insertions(+), 10 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 898d388b8a..74a82f9263 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -20,6 +20,25 @@ import struct
import string
+class Qcow2Field:
+
+ def __init__(self, value):
+ self.value = value
+
+ def __str__(self):
+ return str(self.value)
+
+
+class Flags64(Qcow2Field):
+
+ def __str__(self):
+ bits = []
+ for bit in range(64):
+ if self.value & (1 << bit):
+ bits.append(bit)
+ return str(bits)
+
+
class Qcow2StructMeta(type):
# Mapping from c types to python struct format
@@ -70,14 +89,10 @@ class Qcow2Struct(metaclass=Qcow2StructMeta):
def dump(self):
for f in self.fields:
value = self.__dict__[f[2]]
- if f[1] == 'mask':
- bits = []
- for bit in range(64):
- if value & (1 << bit):
- bits.append(bit)
- value_str = str(bits)
- else:
+ if isinstance(f[1], str):
value_str = f[1].format(value)
+ else:
+ value_str = str(f[1](value))
print('{:<25} {}'.format(f[2], value_str))
@@ -117,9 +132,9 @@ class QcowHeader(Qcow2Struct):
('u64', '{:#x}', 'snapshot_offset'),
# Version 3 header fields
- ('u64', 'mask', 'incompatible_features'),
- ('u64', 'mask', 'compatible_features'),
- ('u64', 'mask', 'autoclear_features'),
+ ('u64', Flags64, 'incompatible_features'),
+ ('u64', Flags64, 'compatible_features'),
+ ('u64', Flags64, 'autoclear_features'),
('u32', '{}', 'refcount_order'),
('u32', '{}', 'header_length'),
)
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH v5 10/13] qcow2_format.py: QcowHeaderExtension: add dump method
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (8 preceding siblings ...)
2020-06-06 8:18 ` [PATCH v5 09/13] qcow2_format.py: add field-formatting class Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:18 ` Vladimir Sementsov-Ogievskiy
2020-06-06 8:18 ` [PATCH v5 11/13] qcow2_format: refactor QcowHeaderExtension as a subclass of Qcow2Struct Vladimir Sementsov-Ogievskiy
` (2 subsequent siblings)
12 siblings, 0 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:18 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
Obviously, for-loop body in dump_extensions should be the dump method
of extension.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/qcow2_format.py | 22 ++++++++++++----------
1 file changed, 12 insertions(+), 10 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 74a82f9263..d4ad5431b2 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -108,6 +108,17 @@ class QcowHeaderExtension:
self.length = length
self.data = data
+ def dump(self):
+ data = self.data[:self.length]
+ if all(c in string.printable.encode('ascii') for c in data):
+ data = f"'{ data.decode('ascii') }'"
+ else:
+ data = '<binary>'
+
+ print(f'{"magic":<25} {self.magic:#x}')
+ print(f'{"length":<25} {self.length}')
+ print(f'{"data":<25} {data}')
+
@classmethod
def create(cls, magic, data):
return QcowHeaderExtension(magic, len(data), data)
@@ -210,15 +221,6 @@ class QcowHeader(Qcow2Struct):
def dump_extensions(self):
for ex in self.extensions:
-
- data = ex.data[:ex.length]
- if all(c in string.printable.encode('ascii') for c in data):
- data = f"'{ data.decode('ascii') }'"
- else:
- data = '<binary>'
-
print('Header extension:')
- print(f'{"magic":<25} {ex.magic:#x}')
- print(f'{"length":<25} {ex.length}')
- print(f'{"data":<25} {data}')
+ ex.dump()
print()
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH v5 11/13] qcow2_format: refactor QcowHeaderExtension as a subclass of Qcow2Struct
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (9 preceding siblings ...)
2020-06-06 8:18 ` [PATCH v5 10/13] qcow2_format.py: QcowHeaderExtension: add dump method Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:18 ` Vladimir Sementsov-Ogievskiy
2020-06-06 8:18 ` [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics Vladimir Sementsov-Ogievskiy
2020-06-06 8:18 ` [PATCH v5 13/13] qcow2_format.py: dump bitmaps header extension Vladimir Sementsov-Ogievskiy
12 siblings, 0 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:18 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
Only two fields we can parse by generic code, but that is better than
nothing. Keep further refactoring of variable-length fields for another
day.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/qcow2_format.py | 53 +++++++++++++++++++++---------
1 file changed, 37 insertions(+), 16 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index d4ad5431b2..32371e42da 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -97,16 +97,41 @@ class Qcow2Struct(metaclass=Qcow2StructMeta):
print('{:<25} {}'.format(f[2], value_str))
-class QcowHeaderExtension:
+class QcowHeaderExtension(Qcow2Struct):
- def __init__(self, magic, length, data):
- if length % 8 != 0:
- padding = 8 - (length % 8)
- data += b'\0' * padding
+ fields = (
+ ('u32', '{:#x}', 'magic'),
+ ('u32', '{}', 'length')
+ # length bytes of data follows
+ # then padding to next multiply of 8
+ )
- self.magic = magic
- self.length = length
- self.data = data
+ def __init__(self, magic=None, length=None, data=None, fd=None):
+ """
+ Support both loading from fd and creation from user data.
+ For fd-based creation current position in a file will be used to read
+ the data.
+
+ This should be somehow refactored and functionality should be moved to
+ superclass (to allow creation of any qcow2 struct), but then, fields
+ of variable length (data here) should be supported in base class
+ somehow. So, it's a TODO. We'll see how to properly refactor this when
+ we have more qcow2 structures.
+ """
+ if fd is None:
+ assert all(v is not None for v in (magic, length, data))
+ self.magic = magic
+ self.length = length
+ if length % 8 != 0:
+ padding = 8 - (length % 8)
+ data += b'\0' * padding
+ self.data = data
+ else:
+ assert all(v is None for v in (magic, length, data))
+ super().__init__(fd=fd)
+ padded = (self.length + 7) & ~7
+ self.data = fd.read(padded)
+ assert self.data is not None
def dump(self):
data = self.data[:self.length]
@@ -115,8 +140,7 @@ class QcowHeaderExtension:
else:
data = '<binary>'
- print(f'{"magic":<25} {self.magic:#x}')
- print(f'{"length":<25} {self.length}')
+ super().dump()
print(f'{"data":<25} {data}')
@classmethod
@@ -182,14 +206,11 @@ class QcowHeader(Qcow2Struct):
end = self.cluster_size
while fd.tell() < end:
- (magic, length) = struct.unpack('>II', fd.read(8))
- if magic == 0:
+ ext = QcowHeaderExtension(fd=fd)
+ if ext.magic == 0:
break
else:
- padded = (length + 7) & ~7
- data = fd.read(padded)
- self.extensions.append(QcowHeaderExtension(magic, length,
- data))
+ self.extensions.append(ext)
def update_extensions(self, fd):
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (10 preceding siblings ...)
2020-06-06 8:18 ` [PATCH v5 11/13] qcow2_format: refactor QcowHeaderExtension as a subclass of Qcow2Struct Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:18 ` Vladimir Sementsov-Ogievskiy
2020-06-08 5:48 ` Andrey Shinkevich
2020-06-09 19:08 ` Eric Blake
2020-06-06 8:18 ` [PATCH v5 13/13] qcow2_format.py: dump bitmaps header extension Vladimir Sementsov-Ogievskiy
12 siblings, 2 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:18 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
Suggested-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
tests/qemu-iotests/031.out | 22 +++++++++++-----------
tests/qemu-iotests/036.out | 4 ++--
tests/qemu-iotests/061.out | 14 +++++++-------
tests/qemu-iotests/qcow2_format.py | 17 ++++++++++++++++-
4 files changed, 36 insertions(+), 21 deletions(-)
diff --git a/tests/qemu-iotests/031.out b/tests/qemu-iotests/031.out
index 5a4beda6a2..4b21d6a9ba 100644
--- a/tests/qemu-iotests/031.out
+++ b/tests/qemu-iotests/031.out
@@ -25,7 +25,7 @@ refcount_order 4
header_length 72
Header extension:
-magic 0x12345678
+magic 0x12345678 (<unknown>)
length 31
data 'This is a test header extension'
@@ -53,7 +53,7 @@ refcount_order 4
header_length 72
Header extension:
-magic 0x12345678
+magic 0x12345678 (<unknown>)
length 31
data 'This is a test header extension'
@@ -81,12 +81,12 @@ refcount_order 4
header_length 72
Header extension:
-magic 0xe2792aca
+magic 0xe2792aca (Backing format)
length 11
data 'host_device'
Header extension:
-magic 0x12345678
+magic 0x12345678 (<unknown>)
length 31
data 'This is a test header extension'
@@ -116,12 +116,12 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
Header extension:
-magic 0x12345678
+magic 0x12345678 (<unknown>)
length 31
data 'This is a test header extension'
@@ -149,12 +149,12 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
Header extension:
-magic 0x12345678
+magic 0x12345678 (<unknown>)
length 31
data 'This is a test header extension'
@@ -182,17 +182,17 @@ refcount_order 4
header_length 112
Header extension:
-magic 0xe2792aca
+magic 0xe2792aca (Backing format)
length 11
data 'host_device'
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
Header extension:
-magic 0x12345678
+magic 0x12345678 (<unknown>)
length 31
data 'This is a test header extension'
diff --git a/tests/qemu-iotests/036.out b/tests/qemu-iotests/036.out
index e409acf60e..a9bed828e5 100644
--- a/tests/qemu-iotests/036.out
+++ b/tests/qemu-iotests/036.out
@@ -25,7 +25,7 @@ incompatible_features []
compatible_features []
autoclear_features [63]
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
@@ -37,7 +37,7 @@ incompatible_features []
compatible_features []
autoclear_features []
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
diff --git a/tests/qemu-iotests/061.out b/tests/qemu-iotests/061.out
index a51ad1b5ba..2f03cf045c 100644
--- a/tests/qemu-iotests/061.out
+++ b/tests/qemu-iotests/061.out
@@ -25,7 +25,7 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
@@ -83,7 +83,7 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
@@ -139,7 +139,7 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
@@ -194,7 +194,7 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
@@ -263,7 +263,7 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
@@ -325,7 +325,7 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
@@ -354,7 +354,7 @@ refcount_order 4
header_length 112
Header extension:
-magic 0x6803f857
+magic 0x6803f857 (Feature table)
length 336
data <binary>
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 32371e42da..40b5bf467b 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -39,6 +39,12 @@ class Flags64(Qcow2Field):
return str(bits)
+class Enum(Qcow2Field):
+
+ def __str__(self):
+ return f'{self.value:#x} ({self.mapping.get(self.value, "<unknown>")})'
+
+
class Qcow2StructMeta(type):
# Mapping from c types to python struct format
@@ -99,8 +105,17 @@ class Qcow2Struct(metaclass=Qcow2StructMeta):
class QcowHeaderExtension(Qcow2Struct):
+ class Magic(Enum):
+ mapping = {
+ 0xe2792aca: 'Backing format',
+ 0x6803f857: 'Feature table',
+ 0x0537be77: 'Crypto header',
+ 0x23852875: 'Bitmaps',
+ 0x44415441: 'Data file'
+ }
+
fields = (
- ('u32', '{:#x}', 'magic'),
+ ('u32', Magic, 'magic'),
('u32', '{}', 'length')
# length bytes of data follows
# then padding to next multiply of 8
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics
2020-06-06 8:18 ` [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics Vladimir Sementsov-Ogievskiy
@ 2020-06-08 5:48 ` Andrey Shinkevich
2020-06-08 9:52 ` Andrey Shinkevich
2020-06-09 19:08 ` Eric Blake
1 sibling, 1 reply; 23+ messages in thread
From: Andrey Shinkevich @ 2020-06-08 5:48 UTC (permalink / raw)
To: Vladimir Sementsov-Ogievskiy, qemu-block
Cc: kwolf, Denis Lunev, qemu-devel, mreitz
[-- Attachment #1: Type: text/plain, Size: 1503 bytes --]
________________________________
From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Sent: Saturday, June 6, 2020 11:18 AM
To: qemu-block@nongnu.org <qemu-block@nongnu.org>
Cc: qemu-devel@nongnu.org <qemu-devel@nongnu.org>; mreitz@redhat.com <mreitz@redhat.com>; kwolf@redhat.com <kwolf@redhat.com>; eblake@redhat.com <eblake@redhat.com>; Denis Lunev <den@virtuozzo.com>; Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>; Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Subject: [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics
Suggested-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
tests/qemu-iotests/031.out | 22 +++++++++++-----------
tests/qemu-iotests/036.out | 4 ++--
tests/qemu-iotests/061.out | 14 +++++++-------
tests/qemu-iotests/qcow2_format.py | 17 ++++++++++++++++-
4 files changed, 36 insertions(+), 21 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 32371e42da..40b5bf467b 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -39,6 +39,12 @@ class Flags64(Qcow2Field):
return str(bits)
+class Enum(Qcow2Field):
+
+ def __str__(self):
+ return f'{self.value:#x} ({self.mapping.get(self.value, "<unknown>")})'
I've got the error E0100: invalid syntax [pylama]
Andrey
[-- Attachment #2: Type: text/html, Size: 2746 bytes --]
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics
2020-06-08 5:48 ` Andrey Shinkevich
@ 2020-06-08 9:52 ` Andrey Shinkevich
0 siblings, 0 replies; 23+ messages in thread
From: Andrey Shinkevich @ 2020-06-08 9:52 UTC (permalink / raw)
To: Vladimir Sementsov-Ogievskiy, qemu-block
Cc: kwolf, Denis Lunev, qemu-devel, mreitz
[-- Attachment #1: Type: text/plain, Size: 2122 bytes --]
________________________________
From: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Sent: Monday, June 8, 2020 8:48 AM
To: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>; qemu-block@nongnu.org <qemu-block@nongnu.org>
Cc: qemu-devel@nongnu.org <qemu-devel@nongnu.org>; mreitz@redhat.com <mreitz@redhat.com>; kwolf@redhat.com <kwolf@redhat.com>; eblake@redhat.com <eblake@redhat.com>; Denis Lunev <den@virtuozzo.com>
Subject: Re: [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics
________________________________
From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Sent: Saturday, June 6, 2020 11:18 AM
To: qemu-block@nongnu.org <qemu-block@nongnu.org>
Cc: qemu-devel@nongnu.org <qemu-devel@nongnu.org>; mreitz@redhat.com <mreitz@redhat.com>; kwolf@redhat.com <kwolf@redhat.com>; eblake@redhat.com <eblake@redhat.com>; Denis Lunev <den@virtuozzo.com>; Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>; Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Subject: [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics
Suggested-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
---
tests/qemu-iotests/031.out | 22 +++++++++++-----------
tests/qemu-iotests/036.out | 4 ++--
tests/qemu-iotests/061.out | 14 +++++++-------
tests/qemu-iotests/qcow2_format.py | 17 ++++++++++++++++-
4 files changed, 36 insertions(+), 21 deletions(-)
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 32371e42da..40b5bf467b 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -39,6 +39,12 @@ class Flags64(Qcow2Field):
return str(bits)
+class Enum(Qcow2Field):
+
+ def __str__(self):
+ return f'{self.value:#x} ({self.mapping.get(self.value, "<unknown>")})'
I've got the error E0100: invalid syntax [pylama]
Andrey
The script itself works. Probably, my python syntax checker is obsolete.
Andrey
[-- Attachment #2: Type: text/html, Size: 3832 bytes --]
^ permalink raw reply related [flat|nested] 23+ messages in thread
* Re: [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics
2020-06-06 8:18 ` [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics Vladimir Sementsov-Ogievskiy
2020-06-08 5:48 ` Andrey Shinkevich
@ 2020-06-09 19:08 ` Eric Blake
1 sibling, 0 replies; 23+ messages in thread
From: Eric Blake @ 2020-06-09 19:08 UTC (permalink / raw)
To: Vladimir Sementsov-Ogievskiy, qemu-block
Cc: kwolf, den, andrey.shinkevich, qemu-devel, mreitz
On 6/6/20 3:18 AM, Vladimir Sementsov-Ogievskiy wrote:
> Suggested-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> ---
> tests/qemu-iotests/031.out | 22 +++++++++++-----------
> tests/qemu-iotests/036.out | 4 ++--
> tests/qemu-iotests/061.out | 14 +++++++-------
> tests/qemu-iotests/qcow2_format.py | 17 ++++++++++++++++-
> 4 files changed, 36 insertions(+), 21 deletions(-)
>
Reviewed-by: Eric Blake <eblake@redhat.com>
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 23+ messages in thread
* [PATCH v5 13/13] qcow2_format.py: dump bitmaps header extension
2020-06-06 8:17 [PATCH v5 00/13] iotests: Dump QCOW2 dirty bitmaps metadata Vladimir Sementsov-Ogievskiy
` (11 preceding siblings ...)
2020-06-06 8:18 ` [PATCH v5 12/13] qcow2: QcowHeaderExtension print names for extension magics Vladimir Sementsov-Ogievskiy
@ 2020-06-06 8:18 ` Vladimir Sementsov-Ogievskiy
2020-06-08 22:00 ` Eric Blake
12 siblings, 1 reply; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-06 8:18 UTC (permalink / raw)
To: qemu-block; +Cc: kwolf, vsementsov, qemu-devel, mreitz, andrey.shinkevich, den
Add class for bitmap extension and dump its fields. Further work is to
dump bitmap directory.
Test new functionality inside 291 iotest.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
---
tests/qemu-iotests/291 | 4 +++
tests/qemu-iotests/291.out | 33 +++++++++++++++++++++++
tests/qemu-iotests/qcow2_format.py | 42 +++++++++++++++++++++++-------
3 files changed, 70 insertions(+), 9 deletions(-)
diff --git a/tests/qemu-iotests/291 b/tests/qemu-iotests/291
index 3ca83b9cd1..e0cffc7cb1 100755
--- a/tests/qemu-iotests/291
+++ b/tests/qemu-iotests/291
@@ -62,6 +62,8 @@ $QEMU_IO -c 'w 1M 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io
$QEMU_IMG bitmap --disable -f $IMGFMT "$TEST_IMG" b1
$QEMU_IMG bitmap --enable -f $IMGFMT "$TEST_IMG" b2
$QEMU_IO -c 'w 2M 1M' -f $IMGFMT "$TEST_IMG" | _filter_qemu_io
+echo "Check resulting qcow2 header extensions:"
+$PYTHON qcow2.py "$TEST_IMG" dump-header-exts
echo
echo "=== Bitmap preservation not possible to non-qcow2 ==="
@@ -88,6 +90,8 @@ $QEMU_IMG bitmap --merge tmp -f $IMGFMT "$TEST_IMG" b0
$QEMU_IMG bitmap --remove --image-opts \
driver=$IMGFMT,file.driver=file,file.filename="$TEST_IMG" tmp
$QEMU_IMG info "$TEST_IMG" | _filter_img_info --format-specific
+echo "Check resulting qcow2 header extensions:"
+$PYTHON qcow2.py "$TEST_IMG" dump-header-exts
echo
echo "=== Check bitmap contents ==="
diff --git a/tests/qemu-iotests/291.out b/tests/qemu-iotests/291.out
index 8c62017567..1d4f9cd96d 100644
--- a/tests/qemu-iotests/291.out
+++ b/tests/qemu-iotests/291.out
@@ -14,6 +14,25 @@ wrote 1048576/1048576 bytes at offset 1048576
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1048576/1048576 bytes at offset 2097152
1 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Check resulting qcow2 header extensions:
+Header extension:
+magic 3799591626 (Backing format)
+length 5
+data 'qcow2'
+
+Header extension:
+magic 1745090647 (Feature table)
+length 336
+data <binary>
+
+Header extension:
+magic 595929205 (Bitmaps)
+length 24
+nb_bitmaps 2
+reserved32 0
+bitmap_directory_size 0x40
+bitmap_directory_offset 0x510000
+
=== Bitmap preservation not possible to non-qcow2 ===
@@ -65,6 +84,20 @@ Format specific information:
granularity: 65536
refcount bits: 16
corrupt: false
+Check resulting qcow2 header extensions:
+Header extension:
+magic 1745090647 (Feature table)
+length 336
+data <binary>
+
+Header extension:
+magic 595929205 (Bitmaps)
+length 24
+nb_bitmaps 3
+reserved32 0
+bitmap_directory_size 0x60
+bitmap_directory_offset 0x520000
+
=== Check bitmap contents ===
diff --git a/tests/qemu-iotests/qcow2_format.py b/tests/qemu-iotests/qcow2_format.py
index 40b5bf467b..0f65fd161d 100644
--- a/tests/qemu-iotests/qcow2_format.py
+++ b/tests/qemu-iotests/qcow2_format.py
@@ -103,6 +103,19 @@ class Qcow2Struct(metaclass=Qcow2StructMeta):
print('{:<25} {}'.format(f[2], value_str))
+class Qcow2BitmapExt(Qcow2Struct):
+
+ fields = (
+ ('u32', '{}', 'nb_bitmaps'),
+ ('u32', '{}', 'reserved32'),
+ ('u64', '{:#x}', 'bitmap_directory_size'),
+ ('u64', '{:#x}', 'bitmap_directory_offset')
+ )
+
+
+QCOW2_EXT_MAGIC_BITMAPS = 0x23852875
+
+
class QcowHeaderExtension(Qcow2Struct):
class Magic(Enum):
@@ -110,7 +123,7 @@ class QcowHeaderExtension(Qcow2Struct):
0xe2792aca: 'Backing format',
0x6803f857: 'Feature table',
0x0537be77: 'Crypto header',
- 0x23852875: 'Bitmaps',
+ QCOW2_EXT_MAGIC_BITMAPS: 'Bitmaps',
0x44415441: 'Data file'
}
@@ -130,8 +143,11 @@ class QcowHeaderExtension(Qcow2Struct):
This should be somehow refactored and functionality should be moved to
superclass (to allow creation of any qcow2 struct), but then, fields
of variable length (data here) should be supported in base class
- somehow. So, it's a TODO. We'll see how to properly refactor this when
- we have more qcow2 structures.
+ somehow. Note also, that we probably want to parse different
+ extensions. Should they be subclasses of this class, or how to do it
+ better? Should it be something like QAPI union with discriminator field
+ (magic here). So, it's a TODO. We'll see how to properly refactor this
+ when we have more qcow2 structures.
"""
if fd is None:
assert all(v is not None for v in (magic, length, data))
@@ -148,15 +164,23 @@ class QcowHeaderExtension(Qcow2Struct):
self.data = fd.read(padded)
assert self.data is not None
- def dump(self):
- data = self.data[:self.length]
- if all(c in string.printable.encode('ascii') for c in data):
- data = f"'{ data.decode('ascii') }'"
+ if self.magic == QCOW2_EXT_MAGIC_BITMAPS:
+ self.obj = Qcow2BitmapExt(data=self.data)
else:
- data = '<binary>'
+ self.obj = None
+ def dump(self):
super().dump()
- print(f'{"data":<25} {data}')
+
+ if self.obj is None:
+ data = self.data[:self.length]
+ if all(c in string.printable.encode('ascii') for c in data):
+ data = f"'{ data.decode('ascii') }'"
+ else:
+ data = '<binary>'
+ print(f'{"data":<25} {data}')
+ else:
+ self.obj.dump()
@classmethod
def create(cls, magic, data):
--
2.21.0
^ permalink raw reply related [flat|nested] 23+ messages in thread* Re: [PATCH v5 13/13] qcow2_format.py: dump bitmaps header extension
2020-06-06 8:18 ` [PATCH v5 13/13] qcow2_format.py: dump bitmaps header extension Vladimir Sementsov-Ogievskiy
@ 2020-06-08 22:00 ` Eric Blake
2020-06-09 7:11 ` Vladimir Sementsov-Ogievskiy
0 siblings, 1 reply; 23+ messages in thread
From: Eric Blake @ 2020-06-08 22:00 UTC (permalink / raw)
To: Vladimir Sementsov-Ogievskiy, qemu-block
Cc: kwolf, den, andrey.shinkevich, qemu-devel, mreitz
On 6/6/20 3:18 AM, Vladimir Sementsov-Ogievskiy wrote:
> Add class for bitmap extension and dump its fields. Further work is to
> dump bitmap directory.
>
> Test new functionality inside 291 iotest.
>
> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
> Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
> ---
> @@ -88,6 +90,8 @@ $QEMU_IMG bitmap --merge tmp -f $IMGFMT "$TEST_IMG" b0
> $QEMU_IMG bitmap --remove --image-opts \
> driver=$IMGFMT,file.driver=file,file.filename="$TEST_IMG" tmp
> $QEMU_IMG info "$TEST_IMG" | _filter_img_info --format-specific
> +echo "Check resulting qcow2 header extensions:"
> +$PYTHON qcow2.py "$TEST_IMG" dump-header-exts
Context conflict with my pending patch to resolve an issue reported by
Kevin:
https://lists.gnu.org/archive/html/qemu-devel/2020-06/msg02057.html
but that is easy enough to resolve.
I'll be queuing this series through my bitmaps tree, hopefully with a
pull request on Tuesday.
--
Eric Blake, Principal Software Engineer
Red Hat, Inc. +1-919-301-3226
Virtualization: qemu.org | libvirt.org
^ permalink raw reply [flat|nested] 23+ messages in thread
* Re: [PATCH v5 13/13] qcow2_format.py: dump bitmaps header extension
2020-06-08 22:00 ` Eric Blake
@ 2020-06-09 7:11 ` Vladimir Sementsov-Ogievskiy
0 siblings, 0 replies; 23+ messages in thread
From: Vladimir Sementsov-Ogievskiy @ 2020-06-09 7:11 UTC (permalink / raw)
To: Eric Blake, qemu-block; +Cc: kwolf, den, andrey.shinkevich, qemu-devel, mreitz
09.06.2020 01:00, Eric Blake wrote:
> On 6/6/20 3:18 AM, Vladimir Sementsov-Ogievskiy wrote:
>> Add class for bitmap extension and dump its fields. Further work is to
>> dump bitmap directory.
>>
>> Test new functionality inside 291 iotest.
>>
>> Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
>> Reviewed-by: Andrey Shinkevich <andrey.shinkevich@virtuozzo.com>
>> ---
>
>> @@ -88,6 +90,8 @@ $QEMU_IMG bitmap --merge tmp -f $IMGFMT "$TEST_IMG" b0
>> $QEMU_IMG bitmap --remove --image-opts \
>> driver=$IMGFMT,file.driver=file,file.filename="$TEST_IMG" tmp
>> $QEMU_IMG info "$TEST_IMG" | _filter_img_info --format-specific
>> +echo "Check resulting qcow2 header extensions:"
>> +$PYTHON qcow2.py "$TEST_IMG" dump-header-exts
>
> Context conflict with my pending patch to resolve an issue reported by Kevin:
>
> https://lists.gnu.org/archive/html/qemu-devel/2020-06/msg02057.html
>
> but that is easy enough to resolve.
>
> I'll be queuing this series through my bitmaps tree, hopefully with a pull request on Tuesday.
>
Great! Thanks!
--
Best regards,
Vladimir
^ permalink raw reply [flat|nested] 23+ messages in thread