* [Qemu-devel] [PULL 01/15] qemu-io: add pattern file for write command
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
@ 2019-08-27 18:22 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 02/15] block: fix permission update in bdrv_replace_node Max Reitz
` (14 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:22 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
From: Denis Plotnikov <dplotnikov@virtuozzo.com>
The patch allows to provide a pattern file for write
command. There was no similar ability before.
Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com>
Message-id: 20190820164616.4072-1-dplotnikov@virtuozzo.com
Reviewed-by: Eric Blake <eblake@redhat.com>
[mreitz: Keep optstring in alphabetical order]
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
qemu-io-cmds.c | 99 +++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 93 insertions(+), 6 deletions(-)
diff --git a/qemu-io-cmds.c b/qemu-io-cmds.c
index 8904733961..d46fa166d3 100644
--- a/qemu-io-cmds.c
+++ b/qemu-io-cmds.c
@@ -350,6 +350,79 @@ static void qemu_io_free(void *p)
qemu_vfree(p);
}
+/*
+ * qemu_io_alloc_from_file()
+ *
+ * Allocates the buffer and populates it with the content of the given file
+ * up to @len bytes. If the file length is less than @len, then the buffer
+ * is populated with the file content cyclically.
+ *
+ * @blk - the block backend where the buffer content is going to be written to
+ * @len - the buffer length
+ * @file_name - the file to read the content from
+ *
+ * Returns: the buffer pointer on success
+ * NULL on error
+ */
+static void *qemu_io_alloc_from_file(BlockBackend *blk, size_t len,
+ const char *file_name)
+{
+ char *buf, *buf_origin;
+ FILE *f = fopen(file_name, "r");
+ int pattern_len;
+
+ if (!f) {
+ perror(file_name);
+ return NULL;
+ }
+
+ if (qemuio_misalign) {
+ len += MISALIGN_OFFSET;
+ }
+
+ buf_origin = buf = blk_blockalign(blk, len);
+
+ if (qemuio_misalign) {
+ buf_origin += MISALIGN_OFFSET;
+ buf += MISALIGN_OFFSET;
+ len -= MISALIGN_OFFSET;
+ }
+
+ pattern_len = fread(buf_origin, 1, len, f);
+
+ if (ferror(f)) {
+ perror(file_name);
+ goto error;
+ }
+
+ if (pattern_len == 0) {
+ fprintf(stderr, "%s: file is empty\n", file_name);
+ goto error;
+ }
+
+ fclose(f);
+
+ if (len > pattern_len) {
+ len -= pattern_len;
+ buf += pattern_len;
+
+ while (len > 0) {
+ size_t len_to_copy = MIN(pattern_len, len);
+
+ memcpy(buf, buf_origin, len_to_copy);
+
+ len -= len_to_copy;
+ buf += len_to_copy;
+ }
+ }
+
+ return buf_origin;
+
+error:
+ qemu_io_free(buf_origin);
+ return NULL;
+}
+
static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
{
uint64_t i;
@@ -948,6 +1021,7 @@ static void write_help(void)
" -n, -- with -z, don't allow slow fallback\n"
" -p, -- ignored for backwards compatibility\n"
" -P, -- use different pattern to fill file\n"
+" -s, -- use a pattern file to fill the write buffer\n"
" -C, -- report statistics in a machine parsable format\n"
" -q, -- quiet mode, do not show I/O statistics\n"
" -u, -- with -z, allow unmapping\n"
@@ -964,7 +1038,7 @@ static const cmdinfo_t write_cmd = {
.perm = BLK_PERM_WRITE,
.argmin = 2,
.argmax = -1,
- .args = "[-bcCfnquz] [-P pattern] off len",
+ .args = "[-bcCfnquz] [-P pattern | -s source_file] off len",
.oneline = "writes a number of bytes at a specified offset",
.help = write_help,
};
@@ -973,7 +1047,7 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
{
struct timespec t1, t2;
bool Cflag = false, qflag = false, bflag = false;
- bool Pflag = false, zflag = false, cflag = false;
+ bool Pflag = false, zflag = false, cflag = false, sflag = false;
int flags = 0;
int c, cnt, ret;
char *buf = NULL;
@@ -982,8 +1056,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
/* Some compilers get confused and warn if this is not initialized. */
int64_t total = 0;
int pattern = 0xcd;
+ const char *file_name = NULL;
- while ((c = getopt(argc, argv, "bcCfnpP:quz")) != -1) {
+ while ((c = getopt(argc, argv, "bcCfnpP:qs:uz")) != -1) {
switch (c) {
case 'b':
bflag = true;
@@ -1013,6 +1088,10 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
case 'q':
qflag = true;
break;
+ case 's':
+ sflag = true;
+ file_name = optarg;
+ break;
case 'u':
flags |= BDRV_REQ_MAY_UNMAP;
break;
@@ -1050,8 +1129,9 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
return -EINVAL;
}
- if (zflag && Pflag) {
- printf("-z and -P cannot be specified at the same time\n");
+ if (zflag + Pflag + sflag > 1) {
+ printf("Only one of -z, -P, and -s "
+ "can be specified at the same time\n");
return -EINVAL;
}
@@ -1087,7 +1167,14 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
}
if (!zflag) {
- buf = qemu_io_alloc(blk, count, pattern);
+ if (sflag) {
+ buf = qemu_io_alloc_from_file(blk, count, file_name);
+ if (!buf) {
+ return -EINVAL;
+ }
+ } else {
+ buf = qemu_io_alloc(blk, count, pattern);
+ }
}
clock_gettime(CLOCK_MONOTONIC, &t1);
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 02/15] block: fix permission update in bdrv_replace_node
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
2019-08-27 18:22 ` [Qemu-devel] [PULL 01/15] qemu-io: add pattern file for write command Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 03/15] block: posix: Always allocate the first block Max Reitz
` (13 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
It's wrong to OR shared permissions. It may lead to crash on further
permission updates.
Also, no needs to consider previously calculated permissions, as at
this point we already bind all new parents and bdrv_get_cumulative_perm
result is enough. So fix the bug by just set permissions by
bdrv_get_cumulative_perm result.
Bug was introduced in long ago 234ac1a9025, in 2.9.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Message-id: 20190824100740.61635-1-vsementsov@virtuozzo.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
block.c | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/block.c b/block.c
index 874a29a983..5944124845 100644
--- a/block.c
+++ b/block.c
@@ -4165,7 +4165,6 @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
{
BdrvChild *c, *next;
GSList *list = NULL, *p;
- uint64_t old_perm, old_shared;
uint64_t perm = 0, shared = BLK_PERM_ALL;
int ret;
@@ -4211,8 +4210,8 @@ void bdrv_replace_node(BlockDriverState *from, BlockDriverState *to,
bdrv_unref(from);
}
- bdrv_get_cumulative_perm(to, &old_perm, &old_shared);
- bdrv_set_perm(to, old_perm | perm, old_shared | shared);
+ bdrv_get_cumulative_perm(to, &perm, &shared);
+ bdrv_set_perm(to, perm, shared);
out:
g_slist_free(list);
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 03/15] block: posix: Always allocate the first block
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
2019-08-27 18:22 ` [Qemu-devel] [PULL 01/15] qemu-io: add pattern file for write command Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 02/15] block: fix permission update in bdrv_replace_node Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 04/15] iotests: Test allocate_first_block() with O_DIRECT Max Reitz
` (12 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
From: Nir Soffer <nirsof@gmail.com>
When creating an image with preallocation "off" or "falloc", the first
block of the image is typically not allocated. When using Gluster
storage backed by XFS filesystem, reading this block using direct I/O
succeeds regardless of request length, fooling alignment detection.
In this case we fallback to a safe value (4096) instead of the optimal
value (512), which may lead to unneeded data copying when aligning
requests. Allocating the first block avoids the fallback.
Since we allocate the first block even with preallocation=off, we no
longer create images with zero disk size:
$ ./qemu-img create -f raw test.raw 1g
Formatting 'test.raw', fmt=raw size=1073741824
$ ls -lhs test.raw
4.0K -rw-r--r--. 1 nsoffer nsoffer 1.0G Aug 16 23:48 test.raw
And converting the image requires additional cluster:
$ ./qemu-img measure -f raw -O qcow2 test.raw
required size: 458752
fully allocated size: 1074135040
When using format like vmdk with multiple files per image, we allocate
one block per file:
$ ./qemu-img create -f vmdk -o subformat=twoGbMaxExtentFlat test.vmdk 4g
Formatting 'test.vmdk', fmt=vmdk size=4294967296 compat6=off hwversion=undefined subformat=twoGbMaxExtentFlat
$ ls -lhs test*.vmdk
4.0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f001.vmdk
4.0K -rw-r--r--. 1 nsoffer nsoffer 2.0G Aug 27 03:23 test-f002.vmdk
4.0K -rw-r--r--. 1 nsoffer nsoffer 353 Aug 27 03:23 test.vmdk
I did quick performance test for copying disks with qemu-img convert to
new raw target image to Gluster storage with sector size of 512 bytes:
for i in $(seq 10); do
rm -f dst.raw
sleep 10
time ./qemu-img convert -f raw -O raw -t none -T none src.raw dst.raw
done
Here is a table comparing the total time spent:
Type Before(s) After(s) Diff(%)
---------------------------------------
real 530.028 469.123 -11.4
user 17.204 10.768 -37.4
sys 17.881 7.011 -60.7
We can see very clear improvement in CPU usage.
Signed-off-by: Nir Soffer <nsoffer@redhat.com>
Message-id: 20190827010528.8818-2-nsoffer@redhat.com
Reviewed-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
block/file-posix.c | 51 +++++++++++++++++++
tests/qemu-iotests/059.out | 2 +-
tests/qemu-iotests/{150.out => 150.out.qcow2} | 0
tests/qemu-iotests/150.out.raw | 12 +++++
tests/qemu-iotests/175 | 19 ++++---
tests/qemu-iotests/175.out | 8 +--
tests/qemu-iotests/178.out.qcow2 | 4 +-
tests/qemu-iotests/221.out | 12 +++--
tests/qemu-iotests/253.out | 12 +++--
9 files changed, 99 insertions(+), 21 deletions(-)
rename tests/qemu-iotests/{150.out => 150.out.qcow2} (100%)
create mode 100644 tests/qemu-iotests/150.out.raw
diff --git a/block/file-posix.c b/block/file-posix.c
index fbeb0068db..447f937aa1 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -1749,6 +1749,43 @@ static int handle_aiocb_discard(void *opaque)
return ret;
}
+/*
+ * Help alignment probing by allocating the first block.
+ *
+ * When reading with direct I/O from unallocated area on Gluster backed by XFS,
+ * reading succeeds regardless of request length. In this case we fallback to
+ * safe alignment which is not optimal. Allocating the first block avoids this
+ * fallback.
+ *
+ * fd may be opened with O_DIRECT, but we don't know the buffer alignment or
+ * request alignment, so we use safe values.
+ *
+ * Returns: 0 on success, -errno on failure. Since this is an optimization,
+ * caller may ignore failures.
+ */
+static int allocate_first_block(int fd, size_t max_size)
+{
+ size_t write_size = (max_size < MAX_BLOCKSIZE)
+ ? BDRV_SECTOR_SIZE
+ : MAX_BLOCKSIZE;
+ size_t max_align = MAX(MAX_BLOCKSIZE, getpagesize());
+ void *buf;
+ ssize_t n;
+ int ret;
+
+ buf = qemu_memalign(max_align, write_size);
+ memset(buf, 0, write_size);
+
+ do {
+ n = pwrite(fd, buf, write_size, 0);
+ } while (n == -1 && errno == EINTR);
+
+ ret = (n == -1) ? -errno : 0;
+
+ qemu_vfree(buf);
+ return ret;
+}
+
static int handle_aiocb_truncate(void *opaque)
{
RawPosixAIOData *aiocb = opaque;
@@ -1788,6 +1825,17 @@ static int handle_aiocb_truncate(void *opaque)
/* posix_fallocate() doesn't set errno. */
error_setg_errno(errp, -result,
"Could not preallocate new data");
+ } else if (current_length == 0) {
+ /*
+ * posix_fallocate() uses fallocate() if the filesystem
+ * supports it, or fallback to manually writing zeroes. If
+ * fallocate() was used, unaligned reads from the fallocated
+ * area in raw_probe_alignment() will succeed, hence we need to
+ * allocate the first block.
+ *
+ * Optimize future alignment probing; ignore failures.
+ */
+ allocate_first_block(fd, offset);
}
} else {
result = 0;
@@ -1849,6 +1897,9 @@ static int handle_aiocb_truncate(void *opaque)
if (ftruncate(fd, offset) != 0) {
result = -errno;
error_setg_errno(errp, -result, "Could not resize file");
+ } else if (current_length == 0 && offset > current_length) {
+ /* Optimize future alignment probing; ignore failures. */
+ allocate_first_block(fd, offset);
}
return result;
default:
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index 4fab42a28c..fe3f861f3c 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -27,7 +27,7 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMax
image: TEST_DIR/t.vmdk
file format: vmdk
virtual size: 0.977 TiB (1073741824000 bytes)
-disk size: 16 KiB
+disk size: 1.97 MiB
Format specific information:
cid: XXXXXXXX
parent cid: XXXXXXXX
diff --git a/tests/qemu-iotests/150.out b/tests/qemu-iotests/150.out.qcow2
similarity index 100%
rename from tests/qemu-iotests/150.out
rename to tests/qemu-iotests/150.out.qcow2
diff --git a/tests/qemu-iotests/150.out.raw b/tests/qemu-iotests/150.out.raw
new file mode 100644
index 0000000000..3cdc7727a5
--- /dev/null
+++ b/tests/qemu-iotests/150.out.raw
@@ -0,0 +1,12 @@
+QA output created by 150
+
+=== Mapping sparse conversion ===
+
+Offset Length File
+0 0x1000 TEST_DIR/t.IMGFMT
+
+=== Mapping non-sparse conversion ===
+
+Offset Length File
+0 0x100000 TEST_DIR/t.IMGFMT
+*** done
diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175
index 51e62c8276..7ba28b3c1b 100755
--- a/tests/qemu-iotests/175
+++ b/tests/qemu-iotests/175
@@ -37,14 +37,16 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
# the file size. This function hides the resulting difference in the
# stat -c '%b' output.
# Parameter 1: Number of blocks an empty file occupies
-# Parameter 2: Image size in bytes
+# Parameter 2: Minimal number of blocks in an image
+# Parameter 3: Image size in bytes
_filter_blocks()
{
extra_blocks=$1
- img_size=$2
+ min_blocks=$2
+ img_size=$3
- sed -e "s/blocks=$extra_blocks\\(\$\\|[^0-9]\\)/nothing allocated/" \
- -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/everything allocated/"
+ sed -e "s/blocks=$min_blocks\\(\$\\|[^0-9]\\)/min allocation/" \
+ -e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/max allocation/"
}
# get standard environment, filters and checks
@@ -60,16 +62,21 @@ size=$((1 * 1024 * 1024))
touch "$TEST_DIR/empty"
extra_blocks=$(stat -c '%b' "$TEST_DIR/empty")
+# We always write the first byte; check how many blocks this filesystem
+# allocates to match empty image alloation.
+printf "\0" > "$TEST_DIR/empty"
+min_blocks=$(stat -c '%b' "$TEST_DIR/empty")
+
echo
echo "== creating image with default preallocation =="
_make_test_img $size | _filter_imgfmt
-stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size
+stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
for mode in off full falloc; do
echo
echo "== creating image with preallocation $mode =="
IMGOPTS=preallocation=$mode _make_test_img $size | _filter_imgfmt
- stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $size
+ stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
done
# success, all done
diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out
index 6d9a5ed84e..263e521262 100644
--- a/tests/qemu-iotests/175.out
+++ b/tests/qemu-iotests/175.out
@@ -2,17 +2,17 @@ QA output created by 175
== creating image with default preallocation ==
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576
-size=1048576, nothing allocated
+size=1048576, min allocation
== creating image with preallocation off ==
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=off
-size=1048576, nothing allocated
+size=1048576, min allocation
== creating image with preallocation full ==
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=full
-size=1048576, everything allocated
+size=1048576, max allocation
== creating image with preallocation falloc ==
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc
-size=1048576, everything allocated
+size=1048576, max allocation
*** done
diff --git a/tests/qemu-iotests/178.out.qcow2 b/tests/qemu-iotests/178.out.qcow2
index 55a8dc926f..9e7d8c44df 100644
--- a/tests/qemu-iotests/178.out.qcow2
+++ b/tests/qemu-iotests/178.out.qcow2
@@ -101,7 +101,7 @@ converted image file size in bytes: 196608
== raw input image with data (human) ==
Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824
-required size: 393216
+required size: 458752
fully allocated size: 1074135040
wrote 512/512 bytes at offset 512
512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
@@ -257,7 +257,7 @@ converted image file size in bytes: 196608
Formatting 'TEST_DIR/t.qcow2', fmt=IMGFMT size=1073741824
{
- "required": 393216,
+ "required": 458752,
"fully-allocated": 1074135040
}
wrote 512/512 bytes at offset 512
diff --git a/tests/qemu-iotests/221.out b/tests/qemu-iotests/221.out
index 9f9dd52bb0..dca024a0c3 100644
--- a/tests/qemu-iotests/221.out
+++ b/tests/qemu-iotests/221.out
@@ -3,14 +3,18 @@ QA output created by 221
=== Check mapping of unaligned raw image ===
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=65537
-[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
-[{ "start": 0, "length": 66048, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 61952, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
wrote 1/1 bytes at offset 65536
1 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
{ "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
{ "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
-[{ "start": 0, "length": 65536, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 61440, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
{ "start": 65536, "length": 1, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
{ "start": 65537, "length": 511, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
*** done
diff --git a/tests/qemu-iotests/253.out b/tests/qemu-iotests/253.out
index 607c0baa0b..3d08b305d7 100644
--- a/tests/qemu-iotests/253.out
+++ b/tests/qemu-iotests/253.out
@@ -3,12 +3,16 @@ QA output created by 253
=== Check mapping of unaligned raw image ===
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048575
-[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
-[{ "start": 0, "length": 1048576, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 1044480, "depth": 0, "zero": true, "data": false, "offset": OFFSET}]
wrote 65535/65535 bytes at offset 983040
63.999 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
-[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
{ "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
-[{ "start": 0, "length": 983040, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
+[{ "start": 0, "length": 4096, "depth": 0, "zero": false, "data": true, "offset": OFFSET},
+{ "start": 4096, "length": 978944, "depth": 0, "zero": true, "data": false, "offset": OFFSET},
{ "start": 983040, "length": 65536, "depth": 0, "zero": false, "data": true, "offset": OFFSET}]
*** done
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 04/15] iotests: Test allocate_first_block() with O_DIRECT
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (2 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 03/15] block: posix: Always allocate the first block Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 05/15] iotests: Fix _filter_img_create() Max Reitz
` (11 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
From: Nir Soffer <nirsof@gmail.com>
Using block_resize we can test allocate_first_block() with file
descriptor opened with O_DIRECT, ensuring that it works for any size
larger than 4096 bytes.
Testing smaller sizes is tricky as the result depends on the filesystem
used for testing. For example on NFS any size will work since O_DIRECT
does not require any alignment.
Signed-off-by: Nir Soffer <nsoffer@redhat.com>
Reviewed-by: Max Reitz <mreitz@redhat.com>
Message-id: 20190827010528.8818-3-nsoffer@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/qemu-iotests/175 | 28 ++++++++++++++++++++++++++++
tests/qemu-iotests/175.out | 8 ++++++++
2 files changed, 36 insertions(+)
diff --git a/tests/qemu-iotests/175 b/tests/qemu-iotests/175
index 7ba28b3c1b..55db2803ed 100755
--- a/tests/qemu-iotests/175
+++ b/tests/qemu-iotests/175
@@ -49,6 +49,23 @@ _filter_blocks()
-e "s/blocks=$((extra_blocks + img_size / 512))\\(\$\\|[^0-9]\\)/max allocation/"
}
+# Resize image using block_resize.
+# Parameter 1: image path
+# Parameter 2: new size
+_block_resize()
+{
+ local path=$1
+ local size=$2
+
+ $QEMU -qmp stdio -nographic -nodefaults \
+ -blockdev file,node-name=file,filename=$path,cache.direct=on \
+ <<EOF
+{'execute': 'qmp_capabilities'}
+{'execute': 'block_resize', 'arguments': {'node-name': 'file', 'size': $size}}
+{'execute': 'quit'}
+EOF
+}
+
# get standard environment, filters and checks
. ./common.rc
. ./common.filter
@@ -57,6 +74,9 @@ _supported_fmt raw
_supported_proto file
_supported_os Linux
+_default_cache_mode none
+_supported_cache_modes none directsync
+
size=$((1 * 1024 * 1024))
touch "$TEST_DIR/empty"
@@ -79,6 +99,14 @@ for mode in off full falloc; do
stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $size
done
+for new_size in 4096 1048576; do
+ echo
+ echo "== resize empty image with block_resize =="
+ _make_test_img 0 | _filter_imgfmt
+ _block_resize $TEST_IMG $new_size >/dev/null
+ stat -c "size=%s, blocks=%b" $TEST_IMG | _filter_blocks $extra_blocks $min_blocks $new_size
+done
+
# success, all done
echo "*** done"
rm -f $seq.full
diff --git a/tests/qemu-iotests/175.out b/tests/qemu-iotests/175.out
index 263e521262..39c2ee0f62 100644
--- a/tests/qemu-iotests/175.out
+++ b/tests/qemu-iotests/175.out
@@ -15,4 +15,12 @@ size=1048576, max allocation
== creating image with preallocation falloc ==
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1048576 preallocation=falloc
size=1048576, max allocation
+
+== resize empty image with block_resize ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
+size=4096, min allocation
+
+== resize empty image with block_resize ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0
+size=1048576, min allocation
*** done
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 05/15] iotests: Fix _filter_img_create()
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (3 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 04/15] iotests: Test allocate_first_block() with O_DIRECT Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 06/15] vmdk: Use bdrv_dirname() for relative extent paths Max Reitz
` (10 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
fe646693acc changed qemu-img create's output so that it no longer prints
single quotes around parameter values. The subformat and adapter_type
filters in _filter_img_create() have never been adapted to that change.
Fixes: fe646693acc13ac48b98435d14149ab04dc597bc
Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Message-id: 20190815153638.4600-2-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/qemu-iotests/059.out | 16 ++++++++--------
tests/qemu-iotests/common.filter | 4 ++--
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index fe3f861f3c..b2e718d29f 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -13,17 +13,17 @@ Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
qemu-io: can't open device TEST_DIR/t.vmdk: L1 size too big
=== Testing monolithicFlat creation and opening ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
image: TEST_DIR/t.IMGFMT
file format: IMGFMT
virtual size: 2 GiB (2147483648 bytes)
=== Testing monolithicFlat with zeroed_grain ===
qemu-img: TEST_DIR/t.IMGFMT: Flat image can't enable zeroed grain
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=2147483648
=== Testing big twoGbMaxExtentFlat ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000 subformat=twoGbMaxExtentFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824000
image: TEST_DIR/t.vmdk
file format: vmdk
virtual size: 0.977 TiB (1073741824000 bytes)
@@ -2038,7 +2038,7 @@ Format specific information:
qemu-img: Could not open 'TEST_DIR/t.IMGFMT': Invalid extent line: RW 12582912 VMFS "dummy.IMGFMT" 1
=== Testing truncated sparse ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400 subformat=monolithicSparse
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=107374182400
qemu-img: Could not open 'TEST_DIR/t.IMGFMT': File truncated, expecting at least 13172736 bytes
=== Converting to streamOptimized from image with small cluster size===
@@ -2049,7 +2049,7 @@ wrote 512/512 bytes at offset 10240
512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
=== Testing monolithicFlat with internally generated JSON file name ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
=== Testing version 3 ===
@@ -2259,7 +2259,7 @@ read 512/512 bytes at offset 64931328
512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
=== Testing 4TB monolithicFlat creation and IO ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104 subformat=monolithicFlat
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=4398046511104
image: TEST_DIR/t.IMGFMT
file format: IMGFMT
virtual size: 4 TiB (4398046511104 bytes)
@@ -2333,7 +2333,7 @@ read 1024/1024 bytes at offset 966367641600
1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
=== Testing qemu-img map on extents ===
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=monolithicSparse
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544
wrote 1024/1024 bytes at offset 65024
1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1024/1024 bytes at offset 2147483136
@@ -2344,7 +2344,7 @@ Offset Length Mapped to File
0 0x20000 0x3f0000 TEST_DIR/t.vmdk
0x7fff0000 0x20000 0x410000 TEST_DIR/t.vmdk
0x140000000 0x10000 0x430000 TEST_DIR/t.vmdk
-Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544 subformat=twoGbMaxExtentSparse
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=33285996544
wrote 1024/1024 bytes at offset 65024
1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
wrote 1024/1024 bytes at offset 2147483136
diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
index 8e9235d6fe..445a1c23e0 100644
--- a/tests/qemu-iotests/common.filter
+++ b/tests/qemu-iotests/common.filter
@@ -130,8 +130,8 @@ _filter_img_create()
-e "s# compat6=\\(on\\|off\\)##g" \
-e "s# static=\\(on\\|off\\)##g" \
-e "s# zeroed_grain=\\(on\\|off\\)##g" \
- -e "s# subformat='[^']*'##g" \
- -e "s# adapter_type='[^']*'##g" \
+ -e "s# subformat=[^ ]*##g" \
+ -e "s# adapter_type=[^ ]*##g" \
-e "s# hwversion=[^ ]*##g" \
-e "s# lazy_refcounts=\\(on\\|off\\)##g" \
-e "s# block_size=[0-9]\\+##g" \
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 06/15] vmdk: Use bdrv_dirname() for relative extent paths
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (4 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 05/15] iotests: Fix _filter_img_create() Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 07/15] iotests: Keep testing broken " Max Reitz
` (9 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
This makes iotest 033 pass with e.g. subformat=monolithicFlat. It also
turns a former error in 059 into success.
Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-id: 20190815153638.4600-3-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
block/vmdk.c | 54 ++++++++++++++++++++++++--------------
tests/qemu-iotests/059 | 7 +++--
tests/qemu-iotests/059.out | 4 ++-
3 files changed, 42 insertions(+), 23 deletions(-)
diff --git a/block/vmdk.c b/block/vmdk.c
index fd78fd0ccf..a7f82e665e 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -1076,8 +1076,7 @@ static const char *next_line(const char *s)
}
static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
- const char *desc_file_path, QDict *options,
- Error **errp)
+ QDict *options, Error **errp)
{
int ret;
int matches;
@@ -1087,6 +1086,7 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
const char *p, *np;
int64_t sectors = 0;
int64_t flat_offset;
+ char *desc_file_dir = NULL;
char *extent_path;
BdrvChild *extent_file;
BDRVVmdkState *s = bs->opaque;
@@ -1130,16 +1130,23 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
continue;
}
- if (!path_is_absolute(fname) && !path_has_protocol(fname) &&
- !desc_file_path[0])
- {
- bdrv_refresh_filename(bs->file->bs);
- error_setg(errp, "Cannot use relative extent paths with VMDK "
- "descriptor file '%s'", bs->file->bs->filename);
- return -EINVAL;
- }
+ if (path_is_absolute(fname)) {
+ extent_path = g_strdup(fname);
+ } else {
+ if (!desc_file_dir) {
+ desc_file_dir = bdrv_dirname(bs->file->bs, errp);
+ if (!desc_file_dir) {
+ bdrv_refresh_filename(bs->file->bs);
+ error_prepend(errp, "Cannot use relative paths with VMDK "
+ "descriptor file '%s': ",
+ bs->file->bs->filename);
+ ret = -EINVAL;
+ goto out;
+ }
+ }
- extent_path = path_combine(desc_file_path, fname);
+ extent_path = g_strconcat(desc_file_dir, fname, NULL);
+ }
ret = snprintf(extent_opt_prefix, 32, "extents.%d", s->num_extents);
assert(ret < 32);
@@ -1149,7 +1156,8 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
g_free(extent_path);
if (local_err) {
error_propagate(errp, local_err);
- return -EINVAL;
+ ret = -EINVAL;
+ goto out;
}
/* save to extents array */
@@ -1160,7 +1168,7 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
0, 0, 0, 0, 0, &extent, errp);
if (ret < 0) {
bdrv_unref_child(bs, extent_file);
- return ret;
+ goto out;
}
extent->flat_start_offset = flat_offset << 9;
} else if (!strcmp(type, "SPARSE") || !strcmp(type, "VMFSSPARSE")) {
@@ -1175,24 +1183,27 @@ static int vmdk_parse_extents(const char *desc, BlockDriverState *bs,
g_free(buf);
if (ret) {
bdrv_unref_child(bs, extent_file);
- return ret;
+ goto out;
}
extent = &s->extents[s->num_extents - 1];
} else if (!strcmp(type, "SESPARSE")) {
ret = vmdk_open_se_sparse(bs, extent_file, bs->open_flags, errp);
if (ret) {
bdrv_unref_child(bs, extent_file);
- return ret;
+ goto out;
}
extent = &s->extents[s->num_extents - 1];
} else {
error_setg(errp, "Unsupported extent type '%s'", type);
bdrv_unref_child(bs, extent_file);
- return -ENOTSUP;
+ ret = -ENOTSUP;
+ goto out;
}
extent->type = g_strdup(type);
}
- return 0;
+
+ ret = 0;
+ goto out;
invalid:
np = next_line(p);
@@ -1201,7 +1212,11 @@ invalid:
np--;
}
error_setg(errp, "Invalid extent line: %.*s", (int)(np - p), p);
- return -EINVAL;
+ ret = -EINVAL;
+
+out:
+ g_free(desc_file_dir);
+ return ret;
}
static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
@@ -1228,8 +1243,7 @@ static int vmdk_open_desc_file(BlockDriverState *bs, int flags, char *buf,
}
s->create_type = g_strdup(ct);
s->desc_offset = 0;
- ret = vmdk_parse_extents(buf, bs, bs->file->bs->exact_filename, options,
- errp);
+ ret = vmdk_parse_extents(buf, bs, options, errp);
exit:
return ret;
}
diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
index 279aee6815..fbed5f9483 100755
--- a/tests/qemu-iotests/059
+++ b/tests/qemu-iotests/059
@@ -114,9 +114,12 @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2
echo
echo "=== Testing monolithicFlat with internally generated JSON file name ==="
+# Should work, because bdrv_dirname() works fine with blkdebug
IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
-$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" 2>&1 \
- | _filter_testdir | _filter_imgfmt
+$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \
+ -c info \
+ 2>&1 \
+ | _filter_testdir | _filter_imgfmt | _filter_img_info
_cleanup_test_img
echo
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index b2e718d29f..a51b571d27 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -2050,7 +2050,9 @@ wrote 512/512 bytes at offset 10240
=== Testing monolithicFlat with internally generated JSON file name ===
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
-qemu-io: can't open: Cannot use relative extent paths with VMDK descriptor file 'json:{"image": {"driver": "file", "filename": "TEST_DIR/t.IMGFMT"}, "driver": "blkdebug", "inject-error.0.event": "read_aio"}'
+format name: IMGFMT
+cluster size: 0 bytes
+vm state offset: 0 bytes
=== Testing version 3 ===
image: TEST_DIR/iotest-version3.IMGFMT
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 07/15] iotests: Keep testing broken relative extent paths
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (5 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 06/15] vmdk: Use bdrv_dirname() for relative extent paths Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 08/15] vmdk: Reject invalid compressed writes Max Reitz
` (8 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
We had a test for a case where relative extent paths did not work, but
unfortunately we just fixed the underlying problem, so it works now.
This patch adds a new test case that still fails.
Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Message-id: 20190815153638.4600-4-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/qemu-iotests/059 | 27 +++++++++++++++++++++++++++
tests/qemu-iotests/059.out | 4 ++++
2 files changed, 31 insertions(+)
diff --git a/tests/qemu-iotests/059 b/tests/qemu-iotests/059
index fbed5f9483..10bfbaecec 100755
--- a/tests/qemu-iotests/059
+++ b/tests/qemu-iotests/059
@@ -114,6 +114,8 @@ $QEMU_IMG convert -f qcow2 -O vmdk -o subformat=streamOptimized "$TEST_IMG.qcow2
echo
echo "=== Testing monolithicFlat with internally generated JSON file name ==="
+
+echo '--- blkdebug ---'
# Should work, because bdrv_dirname() works fine with blkdebug
IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
$QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TEST_IMG,file.inject-error.0.event=read_aio" \
@@ -122,6 +124,31 @@ $QEMU_IO -c "open -o driver=$IMGFMT,file.driver=blkdebug,file.image.filename=$TE
| _filter_testdir | _filter_imgfmt | _filter_img_info
_cleanup_test_img
+echo '--- quorum ---'
+# Should not work, because bdrv_dirname() does not work with quorum
+IMGOPTS="subformat=monolithicFlat" _make_test_img 64M
+cp "$TEST_IMG" "$TEST_IMG.orig"
+
+filename="json:{
+ \"driver\": \"$IMGFMT\",
+ \"file\": {
+ \"driver\": \"quorum\",
+ \"children\": [ {
+ \"driver\": \"file\",
+ \"filename\": \"$TEST_IMG\"
+ }, {
+ \"driver\": \"file\",
+ \"filename\": \"$TEST_IMG.orig\"
+ } ],
+ \"vote-threshold\": 1
+ } }"
+
+filename=$(echo "$filename" | tr '\n' ' ' | sed -e 's/\s\+/ /g')
+$QEMU_IMG info "$filename" 2>&1 \
+ | sed -e "s/'json:[^']*'/\$QUORUM_FILE/g" \
+ | _filter_testdir | _filter_imgfmt | _filter_img_info
+
+
echo
echo "=== Testing version 3 ==="
_use_sample_img iotest-version3.vmdk.bz2
diff --git a/tests/qemu-iotests/059.out b/tests/qemu-iotests/059.out
index a51b571d27..39bf7e211d 100644
--- a/tests/qemu-iotests/059.out
+++ b/tests/qemu-iotests/059.out
@@ -2049,10 +2049,14 @@ wrote 512/512 bytes at offset 10240
512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
=== Testing monolithicFlat with internally generated JSON file name ===
+--- blkdebug ---
Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
format name: IMGFMT
cluster size: 0 bytes
vm state offset: 0 bytes
+--- quorum ---
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864
+qemu-img: Could not open $QUORUM_FILE: Cannot use relative paths with VMDK descriptor file $QUORUM_FILE: Cannot generate a base directory for quorum nodes
=== Testing version 3 ===
image: TEST_DIR/iotest-version3.IMGFMT
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 08/15] vmdk: Reject invalid compressed writes
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (6 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 07/15] iotests: Keep testing broken " Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 09/15] iotests: Disable broken streamOptimized tests Max Reitz
` (7 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
Compressed writes generally have to write full clusters, not just in
theory but also in practice when it comes to vmdk's streamOptimized
subformat. It currently is just silently broken for writes with
non-zero in-cluster offsets:
$ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M
$ qemu-io -c 'write 4k 4k' -c 'read 4k 4k' foo.vmdk
wrote 4096/4096 bytes at offset 4096
4 KiB, 1 ops; 00.01 sec (443.724 KiB/sec and 110.9309 ops/sec)
read failed: Invalid argument
(The technical reason is that vmdk_write_extent() just writes the
incomplete compressed data actually to offset 4k. When reading the
data, vmdk_read_extent() looks at offset 0 and finds the compressed data
size to be 0, because that is what it reads from there. This yields an
error.)
For incomplete writes with zero in-cluster offsets, the error path when
reading the rest of the cluster is a bit different, but the result is
the same:
$ qemu-img create -f vmdk -o subformat=streamOptimized foo.vmdk 1M
$ qemu-io -c 'write 0k 4k' -c 'read 4k 4k' foo.vmdk
wrote 4096/4096 bytes at offset 0
4 KiB, 1 ops; 00.01 sec (362.641 KiB/sec and 90.6603 ops/sec)
read failed: Invalid argument
(Here, vmdk_read_extent() finds the data and then sees that the
uncompressed data is short.)
It is better to reject invalid writes than to make the user believe they
might have succeeded and then fail when trying to read it back.
Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Message-id: 20190815153638.4600-5-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
block/vmdk.c | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/block/vmdk.c b/block/vmdk.c
index a7f82e665e..fed3b50c8a 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -1734,6 +1734,16 @@ static int vmdk_write_extent(VmdkExtent *extent, int64_t cluster_offset,
if (extent->compressed) {
void *compressed_data;
+ /* Only whole clusters */
+ if (offset_in_cluster ||
+ n_bytes > (extent->cluster_sectors * SECTOR_SIZE) ||
+ (n_bytes < (extent->cluster_sectors * SECTOR_SIZE) &&
+ offset + n_bytes != extent->end_sector * SECTOR_SIZE))
+ {
+ ret = -EINVAL;
+ goto out;
+ }
+
if (!extent->has_marker) {
ret = -EINVAL;
goto out;
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 09/15] iotests: Disable broken streamOptimized tests
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (7 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 08/15] vmdk: Reject invalid compressed writes Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 10/15] iotests: Disable 110 for vmdk.twoGbMaxExtentSparse Max Reitz
` (6 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
streamOptimized does not support writes that do not span exactly one
cluster. Furthermore, it cannot rewrite already allocated clusters.
As such, many iotests do not work with it. Disable them.
Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-id: 20190815153638.4600-6-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/qemu-iotests/002 | 1 +
tests/qemu-iotests/003 | 1 +
tests/qemu-iotests/005 | 3 ++-
tests/qemu-iotests/009 | 1 +
tests/qemu-iotests/010 | 1 +
tests/qemu-iotests/011 | 1 +
tests/qemu-iotests/017 | 3 ++-
tests/qemu-iotests/018 | 3 ++-
tests/qemu-iotests/019 | 3 ++-
tests/qemu-iotests/020 | 3 ++-
tests/qemu-iotests/027 | 1 +
tests/qemu-iotests/032 | 1 +
tests/qemu-iotests/033 | 1 +
tests/qemu-iotests/034 | 3 ++-
tests/qemu-iotests/037 | 3 ++-
tests/qemu-iotests/063 | 3 ++-
tests/qemu-iotests/072 | 1 +
tests/qemu-iotests/105 | 3 ++-
tests/qemu-iotests/197 | 1 +
tests/qemu-iotests/215 | 1 +
tests/qemu-iotests/251 | 1 +
21 files changed, 30 insertions(+), 9 deletions(-)
diff --git a/tests/qemu-iotests/002 b/tests/qemu-iotests/002
index fd413bce48..1a0d411df5 100755
--- a/tests/qemu-iotests/002
+++ b/tests/qemu-iotests/002
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt generic
_supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
size=128M
diff --git a/tests/qemu-iotests/003 b/tests/qemu-iotests/003
index ccd3a39dfb..33eeade0de 100755
--- a/tests/qemu-iotests/003
+++ b/tests/qemu-iotests/003
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt generic
_supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
size=128M
offset=67M
diff --git a/tests/qemu-iotests/005 b/tests/qemu-iotests/005
index 9c7681c19b..58442762fe 100755
--- a/tests/qemu-iotests/005
+++ b/tests/qemu-iotests/005
@@ -43,7 +43,8 @@ _supported_fmt generic
_supported_proto generic
_supported_os Linux
_unsupported_imgopts "subformat=twoGbMaxExtentFlat" \
- "subformat=twoGbMaxExtentSparse"
+ "subformat=twoGbMaxExtentSparse" \
+ "subformat=streamOptimized"
# vpc is limited to 127GB, so we can't test it here
if [ "$IMGFMT" = "vpc" ]; then
diff --git a/tests/qemu-iotests/009 b/tests/qemu-iotests/009
index 51b200db1d..4dc7d210f9 100755
--- a/tests/qemu-iotests/009
+++ b/tests/qemu-iotests/009
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt generic
_supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
size=6G
diff --git a/tests/qemu-iotests/010 b/tests/qemu-iotests/010
index 48c533f632..df809b3088 100755
--- a/tests/qemu-iotests/010
+++ b/tests/qemu-iotests/010
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt generic
_supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
size=6G
diff --git a/tests/qemu-iotests/011 b/tests/qemu-iotests/011
index 56f704b5b9..57b99ae4a9 100755
--- a/tests/qemu-iotests/011
+++ b/tests/qemu-iotests/011
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt generic
_supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
size=6G
diff --git a/tests/qemu-iotests/017 b/tests/qemu-iotests/017
index 79875de454..0a4b854e65 100755
--- a/tests/qemu-iotests/017
+++ b/tests/qemu-iotests/017
@@ -41,7 +41,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow qcow2 vmdk qed
_supported_proto generic
_unsupported_proto vxhs
-_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
+ "subformat=streamOptimized"
TEST_OFFSETS="0 4294967296"
diff --git a/tests/qemu-iotests/018 b/tests/qemu-iotests/018
index 78169838ba..c69ce09209 100755
--- a/tests/qemu-iotests/018
+++ b/tests/qemu-iotests/018
@@ -41,7 +41,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow qcow2 vmdk qed
_supported_proto file
_supported_os Linux
-_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
+ "streamOptimized"
TEST_OFFSETS="0 4294967296"
diff --git a/tests/qemu-iotests/019 b/tests/qemu-iotests/019
index a56dd30bed..b4f5234609 100755
--- a/tests/qemu-iotests/019
+++ b/tests/qemu-iotests/019
@@ -47,7 +47,8 @@ _supported_proto file
_supported_os Linux
_unsupported_imgopts "subformat=monolithicFlat" \
"subformat=twoGbMaxExtentFlat" \
- "subformat=twoGbMaxExtentSparse"
+ "subformat=twoGbMaxExtentSparse" \
+ "subformat=streamOptimized"
TEST_OFFSETS="0 4294967296"
CLUSTER_SIZE=65536
diff --git a/tests/qemu-iotests/020 b/tests/qemu-iotests/020
index 6b0ebb37d2..f41b92f35f 100755
--- a/tests/qemu-iotests/020
+++ b/tests/qemu-iotests/020
@@ -44,7 +44,8 @@ _supported_fmt qcow qcow2 vmdk qed
_supported_proto file
_unsupported_imgopts "subformat=monolithicFlat" \
"subformat=twoGbMaxExtentFlat" \
- "subformat=twoGbMaxExtentSparse"
+ "subformat=twoGbMaxExtentSparse" \
+ "subformat=streamOptimized"
TEST_OFFSETS="0 4294967296"
diff --git a/tests/qemu-iotests/027 b/tests/qemu-iotests/027
index 4cb638022a..494be0921f 100755
--- a/tests/qemu-iotests/027
+++ b/tests/qemu-iotests/027
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt vmdk qcow qcow2 qed
_supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
size=128M
diff --git a/tests/qemu-iotests/032 b/tests/qemu-iotests/032
index 988a8c5d8f..8337a4d825 100755
--- a/tests/qemu-iotests/032
+++ b/tests/qemu-iotests/032
@@ -42,6 +42,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
# This works for any image format (though unlikely to segfault for raw)
_supported_fmt generic
_supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
echo
echo === Prepare image ===
diff --git a/tests/qemu-iotests/033 b/tests/qemu-iotests/033
index 362a48c0a0..8b40991d55 100755
--- a/tests/qemu-iotests/033
+++ b/tests/qemu-iotests/033
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt generic
_supported_proto generic
+_unsupported_imgopts "subformat=streamOptimized"
size=128M
diff --git a/tests/qemu-iotests/034 b/tests/qemu-iotests/034
index 324bed28c6..da4cea1571 100755
--- a/tests/qemu-iotests/034
+++ b/tests/qemu-iotests/034
@@ -41,7 +41,8 @@ _supported_proto file
_supported_os Linux
_unsupported_imgopts "subformat=monolithicFlat" \
"subformat=twoGbMaxExtentFlat" \
- "subformat=twoGbMaxExtentSparse"
+ "subformat=twoGbMaxExtentSparse" \
+ "subformat=streamOptimized"
CLUSTER_SIZE=4k
size=128M
diff --git a/tests/qemu-iotests/037 b/tests/qemu-iotests/037
index 4946b9be92..e6517acbd4 100755
--- a/tests/qemu-iotests/037
+++ b/tests/qemu-iotests/037
@@ -40,7 +40,8 @@ _supported_fmt qcow qcow2 vmdk qed
_supported_proto file
_unsupported_imgopts "subformat=monolithicFlat" \
"subformat=twoGbMaxExtentFlat" \
- "subformat=twoGbMaxExtentSparse"
+ "subformat=twoGbMaxExtentSparse" \
+ "subformat=streamOptimized"
CLUSTER_SIZE=4k
size=128M
diff --git a/tests/qemu-iotests/063 b/tests/qemu-iotests/063
index fe4892e467..7cf0427af4 100755
--- a/tests/qemu-iotests/063
+++ b/tests/qemu-iotests/063
@@ -43,7 +43,8 @@ _supported_fmt qcow qcow2 vmdk qed raw
_supported_proto file
_unsupported_imgopts "subformat=monolithicFlat" \
"subformat=twoGbMaxExtentFlat" \
- "subformat=twoGbMaxExtentSparse"
+ "subformat=twoGbMaxExtentSparse" \
+ "subformat=streamOptimized"
_make_test_img 4M
diff --git a/tests/qemu-iotests/072 b/tests/qemu-iotests/072
index 661b36da2d..f0b73e7e65 100755
--- a/tests/qemu-iotests/072
+++ b/tests/qemu-iotests/072
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt vpc vmdk vhdx vdi qed qcow2 qcow
_supported_proto file
+_unsupported_imgopts "subformat=streamOptimized"
IMG_SIZE=64M
diff --git a/tests/qemu-iotests/105 b/tests/qemu-iotests/105
index 3346e8cb25..4d55a2d3ef 100755
--- a/tests/qemu-iotests/105
+++ b/tests/qemu-iotests/105
@@ -39,7 +39,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow2 vmdk vhdx qed
_supported_proto generic
_unsupported_imgopts "subformat=twoGbMaxExtentFlat" \
- "subformat=twoGbMaxExtentSparse"
+ "subformat=twoGbMaxExtentSparse" \
+ "subformat=streamOptimized"
echo
echo "creating large image"
diff --git a/tests/qemu-iotests/197 b/tests/qemu-iotests/197
index 383d7d7f61..1d4f6786db 100755
--- a/tests/qemu-iotests/197
+++ b/tests/qemu-iotests/197
@@ -53,6 +53,7 @@ _supported_fmt generic
_supported_proto generic
# LUKS support may be possible, but it complicates things.
_unsupported_fmt luks
+_unsupported_imgopts "subformat=streamOptimized"
echo
echo '=== Copy-on-read ==='
diff --git a/tests/qemu-iotests/215 b/tests/qemu-iotests/215
index 958c14f5a0..2eb377d682 100755
--- a/tests/qemu-iotests/215
+++ b/tests/qemu-iotests/215
@@ -50,6 +50,7 @@ _supported_fmt generic
_supported_proto generic
# LUKS support may be possible, but it complicates things.
_unsupported_fmt luks
+_unsupported_imgopts "subformat=streamOptimized"
echo
echo '=== Copy-on-read ==='
diff --git a/tests/qemu-iotests/251 b/tests/qemu-iotests/251
index 13f85de9cd..7918ba3559 100755
--- a/tests/qemu-iotests/251
+++ b/tests/qemu-iotests/251
@@ -40,6 +40,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt generic
_supported_proto file
_supported_os Linux
+_unsupported_imgopts "subformat=streamOptimized"
if [ "$IMGOPTSSYNTAX" = "true" ]; then
# We use json:{} filenames here, so we cannot work with additional options.
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 10/15] iotests: Disable 110 for vmdk.twoGbMaxExtentSparse
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (8 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 09/15] iotests: Disable broken streamOptimized tests Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 11/15] iotests: Disable 126 for flat vmdk subformats Max Reitz
` (5 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
The error message for the test case where we have a quorum node for
which no directory name can be generated is different: For
twoGbMaxExtentSparse, it complains that it cannot open the extent file.
For other (sub)formats, it just notes that it cannot determine the
backing file path. Both are fine, but just disable twoGbMaxExtentSparse
for simplicity's sake.
Signed-off-by: Max Reitz <mreitz@redhat.com>
Reviewed-by: John Snow <jsnow@redhat.com>
Message-id: 20190815153638.4600-7-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/qemu-iotests/110 | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tests/qemu-iotests/110 b/tests/qemu-iotests/110
index 2cdc7c8a72..2ef516baf1 100755
--- a/tests/qemu-iotests/110
+++ b/tests/qemu-iotests/110
@@ -40,7 +40,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
# Any format supporting backing files
_supported_fmt qed qcow qcow2 vmdk
_supported_proto file
-_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat"
+_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
+ "subformat=twoGbMaxExtentSparse"
TEST_IMG_REL=$(basename "$TEST_IMG")
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 11/15] iotests: Disable 126 for flat vmdk subformats
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (9 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 10/15] iotests: Disable 110 for vmdk.twoGbMaxExtentSparse Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 12/15] file-posix: fix request_alignment typo Max Reitz
` (4 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
iotest 126 requires backing file support, which flat vmdks cannot offer.
Skip this test for such subformats.
Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-id: 20190815153638.4600-8-mreitz@redhat.com
Reviewed-by: John Snow <jsnow@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/qemu-iotests/126 | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tests/qemu-iotests/126 b/tests/qemu-iotests/126
index 9b0dcf9255..b7fce1e59d 100755
--- a/tests/qemu-iotests/126
+++ b/tests/qemu-iotests/126
@@ -33,6 +33,8 @@ status=1 # failure is the default!
# Needs backing file support
_supported_fmt qcow qcow2 qed vmdk
+_unsupported_imgopts "subformat=monolithicFlat" \
+ "subformat=twoGbMaxExtentFlat"
# This is the default protocol (and we want to test the difference between
# colons which separate a protocol prefix from the rest and colons which are
# just part of the filename, so we cannot test protocols which require a prefix)
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 12/15] file-posix: fix request_alignment typo
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (10 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 11/15] iotests: Disable 126 for flat vmdk subformats Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 13/15] iotests: Check for enabled drivers before testing them Max Reitz
` (3 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
From: Stefan Hajnoczi <stefanha@redhat.com>
Fixes: a6b257a08e3d72219f03e461a52152672fec0612
("file-posix: Handle undetectable alignment")
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Message-id: 20190827101328.4062-1-stefanha@redhat.com
Reviewed-by: Thomas Huth <thuth@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
block/file-posix.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/block/file-posix.c b/block/file-posix.c
index 447f937aa1..71f168ee2f 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -380,7 +380,7 @@ static void raw_probe_alignment(BlockDriverState *bs, int fd, Error **errp)
for (i = 0; i < ARRAY_SIZE(alignments); i++) {
align = alignments[i];
if (raw_is_io_aligned(fd, buf + align, max_align)) {
- /* Fallback to request_aligment. */
+ /* Fallback to request_alignment. */
s->buf_align = (align != 1) ? align : bs->bl.request_alignment;
break;
}
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 13/15] iotests: Check for enabled drivers before testing them
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (11 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 12/15] file-posix: fix request_alignment typo Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 14/15] tests/check-block: Skip iotests when sanitizers are enabled Max Reitz
` (2 subsequent siblings)
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
From: Thomas Huth <thuth@redhat.com>
It is possible to enable only a subset of the block drivers with the
"--block-drv-rw-whitelist" option of the "configure" script. All other
drivers are marked as unusable (or only included as read-only with the
"--block-drv-ro-whitelist" option). If an iotest is now using such a
disabled block driver, it is failing - which is bad, since at least the
tests in the "auto" group should be able to deal with this situation.
Thus let's introduce a "_require_drivers" function that can be used by
the shell tests to check for the availability of certain drivers first,
and marks the test as "not run" if one of the drivers is missing.
This patch mainly targets the test in the "auto" group which should
never fail in such a case, but also improves some of the other tests
along the way. Note that we also assume that the "qcow2" and "file"
drivers are always available - otherwise it does not make sense to
run "make check-block" at all (which only tests with qcow2 by default).
Signed-off-by: Thomas Huth <thuth@redhat.com>
Message-id: 20190823133552.11680-1-thuth@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/qemu-iotests/071 | 1 +
tests/qemu-iotests/081 | 4 +---
tests/qemu-iotests/099 | 1 +
tests/qemu-iotests/120 | 1 +
tests/qemu-iotests/162 | 4 +---
tests/qemu-iotests/184 | 1 +
tests/qemu-iotests/186 | 1 +
tests/qemu-iotests/common.rc | 14 ++++++++++++++
8 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/tests/qemu-iotests/071 b/tests/qemu-iotests/071
index 1cca9233d0..fab526666b 100755
--- a/tests/qemu-iotests/071
+++ b/tests/qemu-iotests/071
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow2
_supported_proto file
+_require_drivers blkdebug blkverify
do_run_qemu()
{
diff --git a/tests/qemu-iotests/081 b/tests/qemu-iotests/081
index c418bab093..85acdf76d4 100755
--- a/tests/qemu-iotests/081
+++ b/tests/qemu-iotests/081
@@ -41,6 +41,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt raw
_supported_proto file
_supported_os Linux
+_require_drivers quorum
do_run_qemu()
{
@@ -55,9 +56,6 @@ run_qemu()
| _filter_qemu_io | _filter_generated_node_ids
}
-test_quorum=$($QEMU_IMG --help|grep quorum)
-[ "$test_quorum" = "" ] && _supported_fmt quorum
-
quorum="driver=raw,file.driver=quorum,file.vote-threshold=2"
quorum="$quorum,file.children.0.file.filename=$TEST_DIR/1.raw"
quorum="$quorum,file.children.1.file.filename=$TEST_DIR/2.raw"
diff --git a/tests/qemu-iotests/099 b/tests/qemu-iotests/099
index ae02f27afe..c3cf66798a 100755
--- a/tests/qemu-iotests/099
+++ b/tests/qemu-iotests/099
@@ -42,6 +42,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow qcow2 qed vdi vhdx vmdk vpc
_supported_proto file
_supported_os Linux
+_require_drivers blkdebug blkverify
_unsupported_imgopts "subformat=monolithicFlat" "subformat=twoGbMaxExtentFlat" \
"subformat=twoGbMaxExtentSparse"
diff --git a/tests/qemu-iotests/120 b/tests/qemu-iotests/120
index e9b4fbb009..2931a7550f 100755
--- a/tests/qemu-iotests/120
+++ b/tests/qemu-iotests/120
@@ -40,6 +40,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt generic
_supported_proto file
_unsupported_fmt luks
+_require_drivers raw
_make_test_img 64M
diff --git a/tests/qemu-iotests/162 b/tests/qemu-iotests/162
index 4e5ed74fd5..2d719afbed 100755
--- a/tests/qemu-iotests/162
+++ b/tests/qemu-iotests/162
@@ -39,9 +39,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
. ./common.filter
_supported_fmt generic
-
-test_ssh=$($QEMU_IMG --help | grep '^Supported formats:.* ssh\( \|$\)')
-[ "$test_ssh" = "" ] && _notrun "ssh support required"
+_require_drivers ssh
echo
echo '=== NBD ==='
diff --git a/tests/qemu-iotests/184 b/tests/qemu-iotests/184
index cb0c181228..33dd8d2a4f 100755
--- a/tests/qemu-iotests/184
+++ b/tests/qemu-iotests/184
@@ -33,6 +33,7 @@ trap "exit \$status" 0 1 2 3 15
. ./common.filter
_supported_os Linux
+_require_drivers throttle
do_run_qemu()
{
diff --git a/tests/qemu-iotests/186 b/tests/qemu-iotests/186
index 5f6b18c150..3ea0442d44 100755
--- a/tests/qemu-iotests/186
+++ b/tests/qemu-iotests/186
@@ -38,6 +38,7 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow2
_supported_proto file
+_require_drivers null-co
if [ "$QEMU_DEFAULT_MACHINE" != "pc" ]; then
_notrun "Requires a PC machine"
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index 5502c3da2f..ee20be8920 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -520,5 +520,19 @@ _require_command()
[ -x "$c" ] || _notrun "$1 utility required, skipped this test"
}
+# Check that a set of drivers has been whitelisted in the QEMU binary
+#
+_require_drivers()
+{
+ available=$($QEMU -drive format=help | \
+ sed -e '/Supported formats:/!d' -e 's/Supported formats://')
+ for driver
+ do
+ if ! echo "$available" | grep -q " $driver\( \|$\)"; then
+ _notrun "$driver not available"
+ fi
+ done
+}
+
# make sure this script returns success
true
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 14/15] tests/check-block: Skip iotests when sanitizers are enabled
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (12 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 13/15] iotests: Check for enabled drivers before testing them Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-08-27 18:23 ` [Qemu-devel] [PULL 15/15] iotests: Unify cache mode quoting Max Reitz
2019-09-03 8:39 ` [Qemu-devel] [PULL 00/15] Block patches Peter Maydell
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
From: Thomas Huth <thuth@redhat.com>
The sanitizers (especially the address sanitizer from Clang) are
sometimes printing out warnings or false positives - this spoils
the output of the iotests, causing some of the tests to fail.
Thus let's skip the automatic iotests during "make check" when the
user configured QEMU with --enable-sanitizers.
Signed-off-by: Thomas Huth <thuth@redhat.com>
Message-id: 20190823084203.29734-1-thuth@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/check-block.sh | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/tests/check-block.sh b/tests/check-block.sh
index c8b6cec3f6..679aedec50 100755
--- a/tests/check-block.sh
+++ b/tests/check-block.sh
@@ -21,6 +21,11 @@ if grep -q "TARGET_GPROF=y" *-softmmu/config-target.mak 2>/dev/null ; then
exit 0
fi
+if grep -q "CFLAGS.*-fsanitize" config-host.mak 2>/dev/null ; then
+ echo "Sanitizers are enabled ==> Not running the qemu-iotests."
+ exit 0
+fi
+
if [ -z "$(find . -name 'qemu-system-*' -print)" ]; then
echo "No qemu-system binary available ==> Not running the qemu-iotests."
exit 0
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* [Qemu-devel] [PULL 15/15] iotests: Unify cache mode quoting
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (13 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 14/15] tests/check-block: Skip iotests when sanitizers are enabled Max Reitz
@ 2019-08-27 18:23 ` Max Reitz
2019-09-03 8:39 ` [Qemu-devel] [PULL 00/15] Block patches Peter Maydell
15 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-08-27 18:23 UTC (permalink / raw)
To: qemu-block; +Cc: Peter Maydell, qemu-devel, Max Reitz
From: Nir Soffer <nirsof@gmail.com>
Quoting cache mode is not needed, and most tests use unquoted values.
Unify all test to use the same style.
Message-id: 20190827173432.7656-1-nsoffer@redhat.com
Signed-off-by: Nir Soffer <nsoffer@redhat.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
---
tests/qemu-iotests/026 | 4 ++--
tests/qemu-iotests/039 | 4 ++--
tests/qemu-iotests/052 | 2 +-
tests/qemu-iotests/091 | 4 ++--
4 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/tests/qemu-iotests/026 b/tests/qemu-iotests/026
index e30243608b..ffb18ab6b5 100755
--- a/tests/qemu-iotests/026
+++ b/tests/qemu-iotests/026
@@ -41,8 +41,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
# Currently only qcow2 supports rebasing
_supported_fmt qcow2
_supported_proto file
-_default_cache_mode "writethrough"
-_supported_cache_modes "writethrough" "none"
+_default_cache_mode writethrough
+_supported_cache_modes writethrough none
# The refcount table tests expect a certain minimum width for refcount entries
# (so that the refcount table actually needs to grow); that minimum is 16 bits,
# being the default refcount entry width.
diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039
index 0d4e963bd4..7c730d94a7 100755
--- a/tests/qemu-iotests/039
+++ b/tests/qemu-iotests/039
@@ -42,8 +42,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow2
_supported_proto file
_supported_os Linux
-_default_cache_mode "writethrough"
-_supported_cache_modes "writethrough"
+_default_cache_mode writethrough
+_supported_cache_modes writethrough
size=128M
diff --git a/tests/qemu-iotests/052 b/tests/qemu-iotests/052
index 6e2ecbfe21..45a140910d 100755
--- a/tests/qemu-iotests/052
+++ b/tests/qemu-iotests/052
@@ -40,7 +40,7 @@ _supported_fmt generic
_supported_proto file
# Don't do O_DIRECT on tmpfs
-_supported_cache_modes "writeback" "writethrough" "unsafe"
+_supported_cache_modes writeback writethrough unsafe
size=128M
_make_test_img $size
diff --git a/tests/qemu-iotests/091 b/tests/qemu-iotests/091
index d62ef18a02..f4b44659ae 100755
--- a/tests/qemu-iotests/091
+++ b/tests/qemu-iotests/091
@@ -46,8 +46,8 @@ trap "_cleanup; exit \$status" 0 1 2 3 15
_supported_fmt qcow2
_supported_proto file
_supported_os Linux
-_default_cache_mode "none"
-_supported_cache_modes "writethrough" "none" "writeback"
+_default_cache_mode none
+_supported_cache_modes writethrough none writeback
size=1G
--
2.21.0
^ permalink raw reply related [flat|nested] 18+ messages in thread* Re: [Qemu-devel] [PULL 00/15] Block patches
2019-08-27 18:22 [Qemu-devel] [PULL 00/15] Block patches Max Reitz
` (14 preceding siblings ...)
2019-08-27 18:23 ` [Qemu-devel] [PULL 15/15] iotests: Unify cache mode quoting Max Reitz
@ 2019-09-03 8:39 ` Peter Maydell
2019-09-03 12:50 ` Max Reitz
15 siblings, 1 reply; 18+ messages in thread
From: Peter Maydell @ 2019-09-03 8:39 UTC (permalink / raw)
To: Max Reitz; +Cc: QEMU Developers, Qemu-block
On Tue, 27 Aug 2019 at 19:23, Max Reitz <mreitz@redhat.com> wrote:
>
> The following changes since commit 23919ddfd56135cad3cb468a8f54d5a595f024f4:
>
> Merge remote-tracking branch 'remotes/aperard/tags/pull-xen-20190827' into staging (2019-08-27 15:52:36 +0100)
>
> are available in the Git repository at:
>
> https://github.com/XanClic/qemu.git tags/pull-block-2019-08-27
>
> for you to fetch changes up to bb043c056cffcc2f3ce88bfdaf2e76e455c09e2c:
>
> iotests: Unify cache mode quoting (2019-08-27 19:48:44 +0200)
>
> ----------------------------------------------------------------
> Block patches:
> - qemu-io now accepts a file to read a write pattern from
> - Ensure that raw files have their first block allocated so we can probe
> the O_DIRECT alignment if necessary
> - Various fixes
Fails make check running the iotests (on some platforms,
including x86-64 Linux):
Not run: 220
Failures: 071 099 120 184 186
Failed 5 of 105 tests
/home/petmay01/linaro/qemu-for-merges/tests/Makefile.include:1100:
recipe for target 'check-tests/check-block.sh' failed
The printed diff output for the failures generally looks like:
--- /home/petmay01/linaro/qemu-for-merges/tests/qemu-iotests/071.out
2018-12-19 15:31:00.523062228 +0000
+++ /home/petmay01/linaro/qemu-for-merges/build/all/tests/qemu-iotests/071.out.bad
2019-09-03 09:01:43.665180692 +0100
@@ -1,4 +1,5 @@
QA output created by 071
+Unable to init server: Could not connect: Connection refused
thanks
-- PMM
^ permalink raw reply [flat|nested] 18+ messages in thread* Re: [Qemu-devel] [PULL 00/15] Block patches
2019-09-03 8:39 ` [Qemu-devel] [PULL 00/15] Block patches Peter Maydell
@ 2019-09-03 12:50 ` Max Reitz
0 siblings, 0 replies; 18+ messages in thread
From: Max Reitz @ 2019-09-03 12:50 UTC (permalink / raw)
To: Peter Maydell; +Cc: QEMU Developers, Qemu-block
[-- Attachment #1.1: Type: text/plain, Size: 1867 bytes --]
On 03.09.19 10:39, Peter Maydell wrote:
> On Tue, 27 Aug 2019 at 19:23, Max Reitz <mreitz@redhat.com> wrote:
>>
>> The following changes since commit 23919ddfd56135cad3cb468a8f54d5a595f024f4:
>>
>> Merge remote-tracking branch 'remotes/aperard/tags/pull-xen-20190827' into staging (2019-08-27 15:52:36 +0100)
>>
>> are available in the Git repository at:
>>
>> https://github.com/XanClic/qemu.git tags/pull-block-2019-08-27
>>
>> for you to fetch changes up to bb043c056cffcc2f3ce88bfdaf2e76e455c09e2c:
>>
>> iotests: Unify cache mode quoting (2019-08-27 19:48:44 +0200)
>>
>> ----------------------------------------------------------------
>> Block patches:
>> - qemu-io now accepts a file to read a write pattern from
>> - Ensure that raw files have their first block allocated so we can probe
>> the O_DIRECT alignment if necessary
>> - Various fixes
>
> Fails make check running the iotests (on some platforms,
> including x86-64 Linux):
>
> Not run: 220
> Failures: 071 099 120 184 186
> Failed 5 of 105 tests
> /home/petmay01/linaro/qemu-for-merges/tests/Makefile.include:1100:
> recipe for target 'check-tests/check-block.sh' failed
>
> The printed diff output for the failures generally looks like:
> --- /home/petmay01/linaro/qemu-for-merges/tests/qemu-iotests/071.out
> 2018-12-19 15:31:00.523062228 +0000
> +++ /home/petmay01/linaro/qemu-for-merges/build/all/tests/qemu-iotests/071.out.bad
> 2019-09-03 09:01:43.665180692 +0100
> @@ -1,4 +1,5 @@
> QA output created by 071
> +Unable to init server: Could not connect: Connection refused
OK, I think I know which patch is to blame. (The problem is probably
that you don’t have a $DISPLAY on your test machine. Neither had I
until a couple of weeks ago.,,)
(Well, I personally blame adding the iotests to make check, but, well.)
Max
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 488 bytes --]
^ permalink raw reply [flat|nested] 18+ messages in thread