[PULL 0/2] bitmaps patches for -rc3, 2020-08-03

[PULL 0/2] bitmaps patches for -rc3, 2020-08-03

[Eric Blake]

The following changes since commit 6c5dfc9ccb643a0d50fdec9f10806b14960571d1:

  Merge remote-tracking branch 'remotes/armbru/tags/pull-qapi-2020-08-03' into staging (2020-08-03 12:21:57 +0100)

are available in the Git repository at:

  https://repo.or.cz/qemu/ericb.git tags/pull-bitmaps-2020-08-03

for you to fetch changes up to edadc99a2ee90daeaaf4fba21d623ec8efe7c8e6:

  iotests/169: Test source cont with backing bmap (2020-08-03 08:59:37 -0500)

----------------------------------------------------------------
bitmaps patches for 2020-08-03

- fix bitmap migration involving read-only bitmap from backing chain

----------------------------------------------------------------
Max Reitz (2):
      qcow2: Release read-only bitmaps when inactivated
      iotests/169: Test source cont with backing bmap

 block/qcow2-bitmap.c       | 23 ++++++++++++++---
 tests/qemu-iotests/169     | 64 +++++++++++++++++++++++++++++++++++++++++++++-
 tests/qemu-iotests/169.out |  4 +--
 3 files changed, 84 insertions(+), 7 deletions(-)

-- 
2.28.0

[PULL 1/2] qcow2: Release read-only bitmaps when inactivated

[Eric Blake]

From: Max Reitz <mreitz@redhat.com>

During migration, we release all bitmaps after storing them on disk, as
long as they are (1) stored on disk, (2) not read-only, and (3)
consistent.

(2) seems arbitrary, though.  The reason we do not release them is
because we do not write them, as there is no need to; and then we just
forget about all bitmaps that we have not written to the file.  However,
read-only persistent bitmaps are still in the file and in sync with
their in-memory representation, so we may as well release them just like
any R/W bitmap that we have updated.

It leads to actual problems, too: After migration, letting the source
continue may result in an error if there were any bitmaps on read-only
nodes (such as backing images), because those have not been released by
bdrv_inactive_all(), but bdrv_invalidate_cache_all() attempts to reload
them (which fails, because they are still present in memory).

Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200730120234.49288-2-mreitz@redhat.com>
Tested-by: Peter Krempa <pkrempa@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Eric Blake <eblake@redhat.com>
---
 block/qcow2-bitmap.c | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index 1f38806ca6ea..8c34b2aef7cd 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -1562,11 +1562,22 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
         Qcow2Bitmap *bm;

         if (!bdrv_dirty_bitmap_get_persistence(bitmap) ||
-            bdrv_dirty_bitmap_readonly(bitmap) ||
             bdrv_dirty_bitmap_inconsistent(bitmap)) {
             continue;
         }

+        if (bdrv_dirty_bitmap_readonly(bitmap)) {
+            /*
+             * Store the bitmap in the associated Qcow2Bitmap so it
+             * can be released later
+             */
+            bm = find_bitmap_by_name(bm_list, name);
+            if (bm) {
+                bm->dirty_bitmap = bitmap;
+            }
+            continue;
+        }
+
         need_write = true;

         if (check_constraints_on_bitmap(bs, name, granularity, errp) < 0) {
@@ -1618,7 +1629,9 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,

     /* allocate clusters and store bitmaps */
     QSIMPLEQ_FOREACH(bm, bm_list, entry) {
-        if (bm->dirty_bitmap == NULL) {
+        BdrvDirtyBitmap *bitmap = bm->dirty_bitmap;
+
+        if (bitmap == NULL || bdrv_dirty_bitmap_readonly(bitmap)) {
             continue;
         }

@@ -1641,6 +1654,7 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
         g_free(tb);
     }

+success:
     if (release_stored) {
         QSIMPLEQ_FOREACH(bm, bm_list, entry) {
             if (bm->dirty_bitmap == NULL) {
@@ -1651,13 +1665,14 @@ void qcow2_store_persistent_dirty_bitmaps(BlockDriverState *bs,
         }
     }

-success:
     bitmap_list_free(bm_list);
     return;

 fail:
     QSIMPLEQ_FOREACH(bm, bm_list, entry) {
-        if (bm->dirty_bitmap == NULL || bm->table.offset == 0) {
+        if (bm->dirty_bitmap == NULL || bm->table.offset == 0 ||
+            bdrv_dirty_bitmap_readonly(bm->dirty_bitmap))
+        {
             continue;
         }

-- 
2.28.0

[PULL 2/2] iotests/169: Test source cont with backing bmap

[Eric Blake]

From: Max Reitz <mreitz@redhat.com>

Test migrating from a VM with a persistent bitmap in the backing chain,
and then continuing that VM after the migration

Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-Id: <20200730120234.49288-3-mreitz@redhat.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Signed-off-by: Eric Blake <eblake@redhat.com>
---
 tests/qemu-iotests/169     | 64 +++++++++++++++++++++++++++++++++++++-
 tests/qemu-iotests/169.out |  4 +--
 2 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/tests/qemu-iotests/169 b/tests/qemu-iotests/169
index 2c5a132aa315..40afb1529986 100755
--- a/tests/qemu-iotests/169
+++ b/tests/qemu-iotests/169
@@ -24,11 +24,12 @@ import time
 import itertools
 import operator
 import re
-from iotests import qemu_img
+from iotests import qemu_img, qemu_img_create, Timeout


 disk_a = os.path.join(iotests.test_dir, 'disk_a')
 disk_b = os.path.join(iotests.test_dir, 'disk_b')
+base_a = os.path.join(iotests.test_dir, 'base_a')
 size = '1M'
 mig_file = os.path.join(iotests.test_dir, 'mig_file')
 mig_cmd = 'exec: cat > ' + mig_file
@@ -234,6 +235,67 @@ for cmb in list(itertools.product((True, False), repeat=2)):
     inject_test_case(TestDirtyBitmapMigration, name,
                      'do_test_migration_resume_source', *list(cmb))

+
+class TestDirtyBitmapBackingMigration(iotests.QMPTestCase):
+    def setUp(self):
+        qemu_img_create('-f', iotests.imgfmt, base_a, size)
+        qemu_img_create('-f', iotests.imgfmt, '-F', iotests.imgfmt,
+                        '-b', base_a, disk_a, size)
+
+        for f in (disk_a, base_a):
+            qemu_img('bitmap', '--add', f, 'bmap0')
+
+        blockdev = {
+            'node-name': 'node0',
+            'driver': iotests.imgfmt,
+            'file': {
+                'driver': 'file',
+                'filename': disk_a
+            },
+            'backing': {
+                'node-name': 'node0-base',
+                'driver': iotests.imgfmt,
+                'file': {
+                    'driver': 'file',
+                    'filename': base_a
+                }
+            }
+        }
+
+        self.vm = iotests.VM()
+        self.vm.launch()
+
+        result = self.vm.qmp('blockdev-add', **blockdev)
+        self.assert_qmp(result, 'return', {})
+
+        # Check that the bitmaps are there
+        for node in self.vm.qmp('query-named-block-nodes', flat=True)['return']:
+            if 'node0' in node['node-name']:
+                self.assert_qmp(node, 'dirty-bitmaps[0]/name', 'bmap0')
+
+        caps = [{'capability': 'events', 'state': True}]
+        result = self.vm.qmp('migrate-set-capabilities', capabilities=caps)
+        self.assert_qmp(result, 'return', {})
+
+    def tearDown(self):
+        self.vm.shutdown()
+        for f in (disk_a, base_a):
+            os.remove(f)
+
+    def test_cont_on_source(self):
+        """
+        Continue the source after migration.
+        """
+        result = self.vm.qmp('migrate', uri=f'exec: cat > /dev/null')
+        self.assert_qmp(result, 'return', {})
+
+        with Timeout(10, 'Migration timeout'):
+            self.vm.wait_migration('postmigrate')
+
+        result = self.vm.qmp('cont')
+        self.assert_qmp(result, 'return', {})
+
+
 if __name__ == '__main__':
     iotests.main(supported_fmts=['qcow2'],
                  supported_protocols=['file'])
diff --git a/tests/qemu-iotests/169.out b/tests/qemu-iotests/169.out
index 5c26d15c0d06..cafb8161f7b1 100644
--- a/tests/qemu-iotests/169.out
+++ b/tests/qemu-iotests/169.out
@@ -1,5 +1,5 @@
-....................................
+.....................................
 ----------------------------------------------------------------------
-Ran 36 tests
+Ran 37 tests

 OK
-- 
2.28.0

Re: [PULL 0/2] bitmaps patches for -rc3, 2020-08-03

[Peter Maydell]

On Mon, 3 Aug 2020 at 15:13, Eric Blake <eblake@redhat.com> wrote:
>
> The following changes since commit 6c5dfc9ccb643a0d50fdec9f10806b14960571d1:
>
>   Merge remote-tracking branch 'remotes/armbru/tags/pull-qapi-2020-08-03' into staging (2020-08-03 12:21:57 +0100)
>
> are available in the Git repository at:
>
>   https://repo.or.cz/qemu/ericb.git tags/pull-bitmaps-2020-08-03
>
> for you to fetch changes up to edadc99a2ee90daeaaf4fba21d623ec8efe7c8e6:
>
>   iotests/169: Test source cont with backing bmap (2020-08-03 08:59:37 -0500)
>
> ----------------------------------------------------------------
> bitmaps patches for 2020-08-03
>
> - fix bitmap migration involving read-only bitmap from backing chain
>


Applied, thanks.

Please update the changelog at https://wiki.qemu.org/ChangeLog/5.1
for any user-visible changes.

-- PMM