[Qemu-devel] [PULL 0/2] NBD patches through 2018-02-09

[Qemu-devel] [PULL 0/2] NBD patches through 2018-02-09

[Eric Blake]

The following changes since commit c7b02d7d032d6022060e4b393827c963c93ce63f:

  Merge remote-tracking branch 'remotes/stsquad/tags/pull-travis-speedup-090218-1' into staging (2018-02-09 16:12:34 +0000)

are available in the Git repository at:

  git://repo.or.cz/qemu/ericb.git tags/pull-nbd-2018-02-09

for you to fetch changes up to e24d813b29d3a478a9309078487efc8ce8599f22:

  block: Simplify bdrv_can_write_zeroes_with_unmap() (2018-02-09 12:32:44 -0600)

----------------------------------------------------------------
nbd patches for 2018-02-09

- Vladimir Sementsov-Ogievskiy: iotests: 205: support luks format
- Eric Blake: block: Simplify bdrv_can_write_zeroes_with_unmap()

----------------------------------------------------------------
Eric Blake (1):
      block: Simplify bdrv_can_write_zeroes_with_unmap()

Vladimir Sementsov-Ogievskiy (1):
      iotests: 205: support luks format

 include/block/block.h         | 11 -----------
 block.c                       |  8 +-------
 block/crypto.c                |  1 -
 block/file-posix.c            |  3 +--
 block/iscsi.c                 |  6 ++++--
 block/nbd.c                   | 11 -----------
 block/qcow2.c                 |  3 +--
 block/qed.c                   |  1 -
 tests/qemu-iotests/205        |  4 ++--
 tests/qemu-iotests/iotests.py | 31 +++++++++++++++++++++++++++++++
 10 files changed, 40 insertions(+), 39 deletions(-)

-- 
2.14.3

[Qemu-devel] [PULL 1/2] iotests: 205: support luks format

[Eric Blake]

From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>

Support default luks options in VM.add_drive and in new library
function qemu_img_create. Use it in 205 iotests.

Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Message-Id: <20180206182507.21753-1-vsementsov@virtuozzo.com>
Reviewed-by: Daniel P. Berrange <berrange@redhat.com>
Signed-off-by: Eric Blake <eblake@redhat.com>
---
 tests/qemu-iotests/205        |  4 ++--
 tests/qemu-iotests/iotests.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/tests/qemu-iotests/205 b/tests/qemu-iotests/205
index 10388920dc3..e7b2eae51d5 100644
--- a/tests/qemu-iotests/205
+++ b/tests/qemu-iotests/205
@@ -22,7 +22,7 @@ import os
 import sys
 import iotests
 import time
-from iotests import qemu_img, qemu_io, filter_qemu_io, QemuIoInteractive
+from iotests import qemu_img_create, qemu_io, filter_qemu_io, QemuIoInteractive

 nbd_sock = 'nbd_sock'
 nbd_uri = 'nbd+unix:///exp?socket=' + nbd_sock
@@ -31,7 +31,7 @@ disk = os.path.join(iotests.test_dir, 'disk')

 class TestNbdServerRemove(iotests.QMPTestCase):
     def setUp(self):
-        qemu_img('create', '-f', iotests.imgfmt, disk, '1M')
+        qemu_img_create('-f', iotests.imgfmt, disk, '1M')

         self.vm = iotests.VM().add_drive(disk)
         self.vm.launch()
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index 5a10b2d5345..1bcc9ca57dc 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -58,6 +58,11 @@ qemu_default_machine = os.environ.get('QEMU_DEFAULT_MACHINE')
 socket_scm_helper = os.environ.get('SOCKET_SCM_HELPER', 'socket_scm_helper')
 debug = False

+luks_default_secret_object = 'secret,id=keysec0,data=' + \
+                             os.environ['IMGKEYSECRET']
+luks_default_key_secret_opt = 'key-secret=keysec0'
+
+
 def qemu_img(*args):
     '''Run qemu-img and return the exit code'''
     devnull = open('/dev/null', 'r+')
@@ -66,6 +71,25 @@ def qemu_img(*args):
         sys.stderr.write('qemu-img received signal %i: %s\n' % (-exitcode, ' '.join(qemu_img_args + list(args))))
     return exitcode

+def qemu_img_create(*args):
+    args = list(args)
+
+    # default luks support
+    if '-f' in args and args[args.index('-f') + 1] == 'luks':
+        if '-o' in args:
+            i = args.index('-o')
+            if 'key-secret' not in args[i + 1]:
+                args[i + 1].append(luks_default_key_secret_opt)
+                args.insert(i + 2, '--object')
+                args.insert(i + 3, luks_default_secret_object)
+        else:
+            args = ['-o', luks_default_key_secret_opt,
+                    '--object', luks_default_secret_object] + args
+
+    args.insert(0, 'create')
+
+    return qemu_img(*args)
+
 def qemu_img_verbose(*args):
     '''Run qemu-img without suppressing its output and return the exit code'''
     exitcode = subprocess.call(qemu_img_args + list(args))
@@ -263,6 +287,13 @@ class VM(qtest.QEMUQtestMachine):
         if opts:
             options.append(opts)

+        if format == 'luks' and 'key-secret' not in opts:
+            # default luks support
+            if luks_default_secret_object not in self._args:
+                self.add_object(luks_default_secret_object)
+
+            options.append(luks_default_key_secret_opt)
+
         self._args.append('-drive')
         self._args.append(','.join(options))
         self._num_drives += 1
-- 
2.14.3

[Qemu-devel] [PULL 2/2] block: Simplify bdrv_can_write_zeroes_with_unmap()

[Eric Blake]

We don't need the can_write_zeroes_with_unmap field in
BlockDriverInfo, because it is redundant information with
supported_zero_flags & BDRV_REQ_MAY_UNMAP.  Note that
BlockDriverInfo and supported_zero_flags are both per-device
settings, rather than global state about the driver as a
whole, which means one or both of these bits of information
can already be conditional.  Let's audit how they were set:

crypto: always setting can_write_ to false is pointless (the
struct starts life zero-initialized), no use of supported_

nbd: just recently fixed to set can_write_ if supported_
includes MAY_UNMAP (thus this commit effectively reverts
bca80059e and solves the problem mentioned there in a more
global way)

file-posix, iscsi, qcow2: can_write_ is conditional, while
supported_ was unconditional; but passing MAY_UNMAP would
fail with ENOTSUP if the condition wasn't met

qed: can_write_ is unconditional, but pwrite_zeroes lacks
support for MAY_UNMAP and supported_ is not set. Perhaps
support can be added later (since it would be similar to
qcow2), but for now claiming false is no real loss

all other drivers: can_write_ is not set, and supported_ is
either unset or a passthrough

Simplify the code by moving the conditional into
supported_zero_flags for all drivers, then dropping the
now-unused BDI field.  For callers that relied on
bdrv_can_write_zeroes_with_unmap(), we return the same
per-device settings for drivers that had conditions (no
observable change in behavior there); and can now return
true (instead of false) for drivers that support passthrough
(for example, the commit driver) which gives those drivers
the same fix as nbd just got in bca80059e.  For callers that
relied on supported_zero_flags, we now have a few more places
that can avoid a wasted call to pwrite_zeroes() that will
just fail with ENOTSUP.

Suggested-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Eric Blake <eblake@redhat.com>
Message-Id: <20180126193439.20219-1-eblake@redhat.com>
Reviewed-by: Stefan Hajnoczi <stefanha@redhat.com>
---
 include/block/block.h | 11 -----------
 block.c               |  8 +-------
 block/crypto.c        |  1 -
 block/file-posix.c    |  3 +--
 block/iscsi.c         |  6 ++++--
 block/nbd.c           | 11 -----------
 block/qcow2.c         |  3 +--
 block/qed.c           |  1 -
 8 files changed, 7 insertions(+), 37 deletions(-)

diff --git a/include/block/block.h b/include/block/block.h
index 24ef816960e..19b3ab9cb5e 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -26,17 +26,6 @@ typedef struct BlockDriverInfo {
      * to the LBPRZ flag in the SCSI logical block provisioning page.
      */
     bool unallocated_blocks_are_zero;
-    /*
-     * True if the driver can optimize writing zeroes by unmapping
-     * sectors. This is equivalent to the BLKDISCARDZEROES ioctl in Linux
-     * with the difference that in qemu a discard is allowed to silently
-     * fail. Therefore we have to use bdrv_pwrite_zeroes with the
-     * BDRV_REQ_MAY_UNMAP flag for an optimized zero write with unmapping.
-     * After this call the driver has to guarantee that the contents read
-     * back as zero. It is additionally required that the block device is
-     * opened with BDRV_O_UNMAP flag for this to work.
-     */
-    bool can_write_zeroes_with_unmap;
     /*
      * True if this block driver only supports compressed writes
      */
diff --git a/block.c b/block.c
index f94585b2304..814e5a02da6 100644
--- a/block.c
+++ b/block.c
@@ -4010,17 +4010,11 @@ bool bdrv_unallocated_blocks_are_zero(BlockDriverState *bs)

 bool bdrv_can_write_zeroes_with_unmap(BlockDriverState *bs)
 {
-    BlockDriverInfo bdi;
-
     if (!(bs->open_flags & BDRV_O_UNMAP)) {
         return false;
     }

-    if (bdrv_get_info(bs, &bdi) == 0) {
-        return bdi.can_write_zeroes_with_unmap;
-    }
-
-    return false;
+    return bs->supported_zero_flags & BDRV_REQ_MAY_UNMAP;
 }

 const char *bdrv_get_encrypted_filename(BlockDriverState *bs)
diff --git a/block/crypto.c b/block/crypto.c
index 70e3691cd84..3df66947c5a 100644
--- a/block/crypto.c
+++ b/block/crypto.c
@@ -576,7 +576,6 @@ static int block_crypto_get_info_luks(BlockDriverState *bs,
     }

     bdi->unallocated_blocks_are_zero = false;
-    bdi->can_write_zeroes_with_unmap = false;
     bdi->cluster_size = subbdi.cluster_size;

     return 0;
diff --git a/block/file-posix.c b/block/file-posix.c
index dd8d7cbbd21..ca49c1a98ae 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -549,7 +549,6 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,

     s->has_discard = true;
     s->has_write_zeroes = true;
-    bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP;
     if ((bs->open_flags & BDRV_O_NOCACHE) != 0) {
         s->needs_alignment = true;
     }
@@ -599,6 +598,7 @@ static int raw_open_common(BlockDriverState *bs, QDict *options,
     }
 #endif

+    bs->supported_zero_flags = s->discard_zeroes ? BDRV_REQ_MAY_UNMAP : 0;
     ret = 0;
 fail:
     if (filename && (bdrv_flags & BDRV_O_TEMPORARY)) {
@@ -2223,7 +2223,6 @@ static int raw_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     BDRVRawState *s = bs->opaque;

     bdi->unallocated_blocks_are_zero = s->discard_zeroes;
-    bdi->can_write_zeroes_with_unmap = s->discard_zeroes;
     return 0;
 }

diff --git a/block/iscsi.c b/block/iscsi.c
index 9f99ae5e07b..421983dd6ff 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -1877,7 +1877,6 @@ static int iscsi_open(BlockDriverState *bs, QDict *options, int flags,
     if (iscsilun->dpofua) {
         bs->supported_write_flags = BDRV_REQ_FUA;
     }
-    bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP;

     /* Check the write protect flag of the LUN if we want to write */
     if (iscsilun->type == TYPE_DISK && (flags & BDRV_O_RDWR) &&
@@ -1961,6 +1960,10 @@ static int iscsi_open(BlockDriverState *bs, QDict *options, int flags,
         }
     }

+    if (iscsilun->lbprz && iscsilun->lbp.lbpws) {
+        bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP;
+    }
+
 out:
     qemu_opts_del(opts);
     g_free(initiator_name);
@@ -2160,7 +2163,6 @@ static int iscsi_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
     IscsiLun *iscsilun = bs->opaque;
     bdi->unallocated_blocks_are_zero = iscsilun->lbprz;
-    bdi->can_write_zeroes_with_unmap = iscsilun->lbprz && iscsilun->lbp.lbpws;
     bdi->cluster_size = iscsilun->cluster_sectors * BDRV_SECTOR_SIZE;
     return 0;
 }
diff --git a/block/nbd.c b/block/nbd.c
index 411eeb42a7d..ef81a9f53ba 100644
--- a/block/nbd.c
+++ b/block/nbd.c
@@ -566,14 +566,6 @@ static void nbd_refresh_filename(BlockDriverState *bs, QDict *options)
     bs->full_open_options = opts;
 }

-static int nbd_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
-{
-    if (bs->supported_zero_flags & BDRV_REQ_MAY_UNMAP) {
-        bdi->can_write_zeroes_with_unmap = true;
-    }
-    return 0;
-}
-
 static BlockDriver bdrv_nbd = {
     .format_name                = "nbd",
     .protocol_name              = "nbd",
@@ -591,7 +583,6 @@ static BlockDriver bdrv_nbd = {
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
     .bdrv_refresh_filename      = nbd_refresh_filename,
-    .bdrv_get_info              = nbd_get_info,
 };

 static BlockDriver bdrv_nbd_tcp = {
@@ -611,7 +602,6 @@ static BlockDriver bdrv_nbd_tcp = {
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
     .bdrv_refresh_filename      = nbd_refresh_filename,
-    .bdrv_get_info              = nbd_get_info,
 };

 static BlockDriver bdrv_nbd_unix = {
@@ -631,7 +621,6 @@ static BlockDriver bdrv_nbd_unix = {
     .bdrv_detach_aio_context    = nbd_detach_aio_context,
     .bdrv_attach_aio_context    = nbd_attach_aio_context,
     .bdrv_refresh_filename      = nbd_refresh_filename,
-    .bdrv_get_info              = nbd_get_info,
 };

 static void bdrv_nbd_init(void)
diff --git a/block/qcow2.c b/block/qcow2.c
index a64a572785d..801e29fc56e 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1479,7 +1479,7 @@ static int qcow2_do_open(BlockDriverState *bs, QDict *options, int flags,

     /* Initialise locks */
     qemu_co_mutex_init(&s->lock);
-    bs->supported_zero_flags = BDRV_REQ_MAY_UNMAP;
+    bs->supported_zero_flags = header.version >= 3 ? BDRV_REQ_MAY_UNMAP : 0;

     /* Repair image if dirty */
     if (!(flags & (BDRV_O_CHECK | BDRV_O_INACTIVE)) && !bs->read_only &&
@@ -3771,7 +3771,6 @@ static int qcow2_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
 {
     BDRVQcow2State *s = bs->opaque;
     bdi->unallocated_blocks_are_zero = true;
-    bdi->can_write_zeroes_with_unmap = (s->qcow_version >= 3);
     bdi->cluster_size = s->cluster_size;
     bdi->vm_state_offset = qcow2_vm_state_offset(s);
     return 0;
diff --git a/block/qed.c b/block/qed.c
index 205dbf16e35..c6ff3ab015d 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -1438,7 +1438,6 @@ static int bdrv_qed_get_info(BlockDriverState *bs, BlockDriverInfo *bdi)
     bdi->cluster_size = s->header.cluster_size;
     bdi->is_dirty = s->header.features & QED_F_NEED_CHECK;
     bdi->unallocated_blocks_are_zero = true;
-    bdi->can_write_zeroes_with_unmap = true;
     return 0;
 }

-- 
2.14.3

Re: [Qemu-devel] [PULL 0/2] NBD patches through 2018-02-09

[Peter Maydell]

On 9 February 2018 at 19:08, Eric Blake <eblake@redhat.com> wrote:
> The following changes since commit c7b02d7d032d6022060e4b393827c963c93ce63f:
>
>   Merge remote-tracking branch 'remotes/stsquad/tags/pull-travis-speedup-090218-1' into staging (2018-02-09 16:12:34 +0000)
>
> are available in the Git repository at:
>
>   git://repo.or.cz/qemu/ericb.git tags/pull-nbd-2018-02-09
>
> for you to fetch changes up to e24d813b29d3a478a9309078487efc8ce8599f22:
>
>   block: Simplify bdrv_can_write_zeroes_with_unmap() (2018-02-09 12:32:44 -0600)
>
> ----------------------------------------------------------------
> nbd patches for 2018-02-09
>
> - Vladimir Sementsov-Ogievskiy: iotests: 205: support luks format
> - Eric Blake: block: Simplify bdrv_can_write_zeroes_with_unmap()
>

Applied, thanks.

-- PMM