From: John Snow <jsnow@redhat.com>
To: qemu-devel@nongnu.org
Cc: peter.maydell@linaro.org, jsnow@redhat.com,
qemu-stable@nongnu.org, qemu-block@nongnu.org,
Max Reitz <mreitz@redhat.com>
Subject: [Qemu-devel] [PULL 23/36] iotests/257: add EmulatedBitmap class
Date: Fri, 16 Aug 2019 19:13:05 -0400 [thread overview]
Message-ID: <20190816231318.8650-24-jsnow@redhat.com> (raw)
In-Reply-To: <20190816231318.8650-1-jsnow@redhat.com>
Represent a bitmap with an object that we can mark and clear bits in.
This makes it easier to manage partial writes when we don't write a
full group's worth of patterns before an error.
Signed-off-by: John Snow <jsnow@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-id: 20190716000117.25219-3-jsnow@redhat.com
Signed-off-by: John Snow <jsnow@redhat.com>
---
tests/qemu-iotests/257 | 124 +++++++++++++++++++++++++----------------
1 file changed, 75 insertions(+), 49 deletions(-)
diff --git a/tests/qemu-iotests/257 b/tests/qemu-iotests/257
index 02f9ae06490..bc66ea03b24 100755
--- a/tests/qemu-iotests/257
+++ b/tests/qemu-iotests/257
@@ -85,6 +85,59 @@ GROUPS = [
Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
]
+
+class EmulatedBitmap:
+ def __init__(self, granularity=GRANULARITY):
+ self._bits = set()
+ self.granularity = granularity
+
+ def dirty_bits(self, bits):
+ self._bits |= set(bits)
+
+ def dirty_group(self, n):
+ self.dirty_bits(GROUPS[n].bits(self.granularity))
+
+ def clear(self):
+ self._bits = set()
+
+ def clear_bits(self, bits):
+ self._bits -= set(bits)
+
+ def clear_bit(self, bit):
+ self.clear_bits({bit})
+
+ def clear_group(self, n):
+ self.clear_bits(GROUPS[n].bits(self.granularity))
+
+ @property
+ def first_bit(self):
+ return sorted(self.bits)[0]
+
+ @property
+ def bits(self):
+ return self._bits
+
+ @property
+ def count(self):
+ return len(self.bits)
+
+ def compare(self, qmp_bitmap):
+ """
+ Print a nice human-readable message checking that a bitmap as reported
+ by the QMP interface has as many bits set as we expect it to.
+ """
+
+ name = qmp_bitmap.get('name', '(anonymous)')
+ log("= Checking Bitmap {:s} =".format(name))
+
+ want = self.count
+ have = qmp_bitmap['count'] // qmp_bitmap['granularity']
+
+ log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
+ want, have, "OK!" if want == have else "ERROR!"))
+ log('')
+
+
class Drive:
"""Represents, vaguely, a drive attached to a VM.
Includes format, graph, and device information."""
@@ -195,27 +248,6 @@ def perform_writes(drive, n):
log('')
return bitmaps
-def calculate_bits(groups=None):
- """Calculate how many bits we expect to see dirtied."""
- if groups:
- bits = set.union(*(GROUPS[group].bits(GRANULARITY) for group in groups))
- return len(bits)
- return 0
-
-def bitmap_comparison(bitmap, groups=None, want=0):
- """
- Print a nice human-readable message checking that this bitmap has as
- many bits set as we expect it to.
- """
- log("= Checking Bitmap {:s} =".format(bitmap.get('name', '(anonymous)')))
-
- if groups:
- want = calculate_bits(groups)
- have = bitmap['count'] // bitmap['granularity']
-
- log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
- want, have, "OK!" if want == have else "ERROR!"))
- log('')
def compare_images(image, reference, baseimg=None, expected_match=True):
"""
@@ -321,12 +353,13 @@ def test_bitmap_sync(bsync_mode, failure=None):
vm.qmp_log("block-dirty-bitmap-add", node=drive0.name,
name="bitmap0", granularity=GRANULARITY)
log('')
+ ebitmap = EmulatedBitmap()
# 1 - Writes and Reference Backup
bitmaps = perform_writes(drive0, 1)
- dirty_groups = {1}
+ ebitmap.dirty_group(1)
bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
- bitmap_comparison(bitmap, groups=dirty_groups)
+ ebitmap.compare(bitmap)
reference_backup(drive0, 1, fbackup1)
# 1 - Bitmap Backup (Optional induced failure)
@@ -342,54 +375,47 @@ def test_bitmap_sync(bsync_mode, failure=None):
log('')
bitmaps = perform_writes(drive0, 2)
# Named bitmap (static, should be unchanged)
- bitmap_comparison(get_bitmap(bitmaps, drive0.device, 'bitmap0'),
- groups=dirty_groups)
+ ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
# Anonymous bitmap (dynamic, shows new writes)
- bitmap_comparison(get_bitmap(bitmaps, drive0.device, '',
- recording=True), groups={2})
- dirty_groups.add(2)
+ anonymous = EmulatedBitmap()
+ anonymous.dirty_group(2)
+ anonymous.compare(get_bitmap(bitmaps, drive0.device, '',
+ recording=True))
+
+ # Simulate the order in which this will happen:
+ # group 1 gets cleared first, then group two gets written.
+ if ((bsync_mode == 'on-success' and not failure) or
+ (bsync_mode == 'always')):
+ ebitmap.clear_group(1)
+ ebitmap.dirty_group(2)
vm.run_job(job, auto_dismiss=True, auto_finalize=False,
pre_finalize=_callback,
cancel=(failure == 'simulated'))
bitmaps = query_bitmaps(vm)
- bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
log(bitmaps, indent=2)
log('')
- if ((bsync_mode == 'on-success' and not failure) or
- (bsync_mode == 'always' and failure != 'intermediate')):
- dirty_groups.remove(1)
-
if bsync_mode == 'always' and failure == 'intermediate':
# We manage to copy one sector (one bit) before the error.
- bitmap_comparison(bitmap,
- want=calculate_bits(groups=dirty_groups) - 1)
- else:
- bitmap_comparison(bitmap, groups=dirty_groups)
+ ebitmap.clear_bit(ebitmap.first_bit)
+ ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
# 2 - Writes and Reference Backup
bitmaps = perform_writes(drive0, 3)
- dirty_groups.add(3)
- bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
- if bsync_mode == 'always' and failure == 'intermediate':
- # We're one bit short, still.
- bitmap_comparison(bitmap,
- want=calculate_bits(groups=dirty_groups) - 1)
- else:
- bitmap_comparison(bitmap, groups=dirty_groups)
+ ebitmap.dirty_group(3)
+ ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
reference_backup(drive0, 2, fbackup2)
# 2 - Bitmap Backup (In failure modes, this is a recovery.)
job = bitmap_backup(drive0, 2, bsync2, "bitmap0", bsync_mode)
vm.run_job(job, auto_dismiss=True, auto_finalize=False)
bitmaps = query_bitmaps(vm)
- bitmap = get_bitmap(bitmaps, drive0.device, 'bitmap0')
log(bitmaps, indent=2)
log('')
- bitmap_comparison(bitmap, groups={}
- if bsync_mode != 'never'
- else dirty_groups)
+ if bsync_mode != 'never':
+ ebitmap.clear()
+ ebitmap.compare(get_bitmap(bitmaps, drive0.device, 'bitmap0'))
log('--- Cleanup ---\n')
vm.qmp_log("block-dirty-bitmap-remove",
--
2.21.0
next prev parent reply other threads:[~2019-08-16 23:36 UTC|newest]
Thread overview: 38+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-08-16 23:12 [Qemu-devel] [PULL 00/36] Bitmaps patches John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 01/36] qapi/block-core: Introduce BackupCommon John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 02/36] drive-backup: create do_backup_common John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 03/36] blockdev-backup: utilize do_backup_common John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 04/36] qapi: add BitmapSyncMode enum John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 05/36] block/backup: Add mirror sync mode 'bitmap' John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 06/36] block/backup: add 'never' policy to bitmap sync mode John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 07/36] hbitmap: Fix merge when b is empty, and result is not an alias of a John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 08/36] hbitmap: enable merging across granularities John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 09/36] block/dirty-bitmap: add bdrv_dirty_bitmap_merge_internal John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 10/36] block/dirty-bitmap: add bdrv_dirty_bitmap_get John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 11/36] block/backup: upgrade copy_bitmap to BdrvDirtyBitmap John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 12/36] block/backup: add 'always' bitmap sync policy John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 13/36] iotests: add testing shim for script-style python tests John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 14/36] iotests: teach run_job to cancel pending jobs John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 15/36] iotests: teach FilePath to produce multiple paths John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 16/36] iotests: Add virtio-scsi device helper John Snow
2019-08-16 23:12 ` [Qemu-devel] [PULL 17/36] iotests: add test 257 for bitmap-mode backups John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 18/36] block/backup: loosen restriction on readonly bitmaps John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 19/36] blockdev: reduce aio_context locked sections in bitmap add/remove John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 20/36] qapi: implement block-dirty-bitmap-remove transaction action John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 21/36] iotests: test bitmap moving inside 254 John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 22/36] iotests/257: add Pattern class John Snow
2019-08-16 23:13 ` John Snow [this message]
2019-08-16 23:13 ` [Qemu-devel] [PULL 24/36] iotests/257: Refactor backup helpers John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 25/36] block/backup: hoist bitmap check into QMP interface John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 26/36] iotests/257: test API failures John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 27/36] block/backup: improve sync=bitmap work estimates John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 28/36] block/backup: centralize copy_bitmap initialization John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 29/36] block/backup: add backup_is_cluster_allocated John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 30/36] block/backup: teach TOP to never copy unallocated regions John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 31/36] block/backup: support bitmap sync modes for non-bitmap backups John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 32/36] iotests/257: test traditional sync modes John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 33/36] qapi: add dirty-bitmaps to query-named-block-nodes result John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 34/36] block/backup: deal with zero detection John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 35/36] block/backup: refactor write_flags John Snow
2019-08-16 23:13 ` [Qemu-devel] [PULL 36/36] tests/test-hbitmap: test next_zero and _next_dirty_area after truncate John Snow
2019-08-19 11:32 ` [Qemu-devel] [PULL 00/36] Bitmaps patches Peter Maydell
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190816231318.8650-24-jsnow@redhat.com \
--to=jsnow@redhat.com \
--cc=mreitz@redhat.com \
--cc=peter.maydell@linaro.org \
--cc=qemu-block@nongnu.org \
--cc=qemu-devel@nongnu.org \
--cc=qemu-stable@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).