From: Denis Plotnikov <dplotnikov@virtuozzo.com>
To: kwolf@redhat.com, mreitz@redhat.com, eblake@redhat.com,
armbru@redhat.com
Cc: vsementsov@virtuozzo.com, den@virtuozzo.com,
qemu-block@nongnu.org, qemu-devel@nongnu.org
Subject: [Qemu-devel] [PATCH v2 3/3] qcow2: add zstd cluster compression
Date: Thu, 4 Jul 2019 16:09:49 +0300 [thread overview]
Message-ID: <20190704130949.14017-4-dplotnikov@virtuozzo.com> (raw)
In-Reply-To: <20190704130949.14017-1-dplotnikov@virtuozzo.com>
zstd significantly reduces cluster compression time.
It provides better compression performance maintaining
the same level of compression ratio in comparison with
zlib, which, by the moment, has been the only compression
method available.
The performance test results:
Test compresses and decompresses qemu qcow2 image with just
installed rhel-7.6 guest.
Image cluster size: 64K. Image on disk size: 2.2G
The test was conducted with brd disk to reduce the influence
of disk subsystem to the test results.
The results is given in seconds.
compress cmd:
time ./qemu-img convert -O qcow2 -c -o compression_type=[zlib|zstd]
src.img [zlib|zstd]_compressed.img
decompress cmd
time ./qemu-img convert -O qcow2
[zlib|zstd]_compressed.img uncompressed.img
compression decompression
zlib zstd zlib zstd
------------------------------------------------------------
real 65.5 16.3 (-75 %) 1.9 1.6 (-16 %)
user 65.0 15.8 5.3 2.5
sys 3.3 0.2 2.0 2.0
Both ZLIB and ZSTD gave the same compression ratio: 1.57
compressed image size in both cases: 1.4G
Signed-off-by: Denis Plotnikov <dplotnikov@virtuozzo.com>
---
block/qcow2.c | 99 ++++++++++++++++++++++++++++++++++++++++++
configure | 32 ++++++++++++++
docs/interop/qcow2.txt | 19 ++++++++
qapi/block-core.json | 3 +-
4 files changed, 152 insertions(+), 1 deletion(-)
diff --git a/block/qcow2.c b/block/qcow2.c
index a107f76e98..252eba636f 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -27,6 +27,11 @@
#define ZLIB_CONST
#include <zlib.h>
+#ifdef CONFIG_ZSTD
+#include <zstd.h>
+#include <zstd_errors.h>
+#endif
+
#include "block/block_int.h"
#include "block/qdict.h"
#include "sysemu/block-backend.h"
@@ -1206,6 +1211,9 @@ static int check_compression_type(BDRVQcow2State *s, Error **errp)
{
switch (s->compression_type) {
case QCOW2_COMPRESSION_TYPE_ZLIB:
+#ifdef CONFIG_ZSTD
+ case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif
break;
default:
@@ -3200,6 +3208,9 @@ qcow2_co_create(BlockdevCreateOptions *create_options, Error **errp)
switch(qcow2_opts->compression_type) {
case QCOW2_COMPRESSION_TYPE_ZLIB:
+#ifdef CONFIG_ZSTD
+ case QCOW2_COMPRESSION_TYPE_ZSTD:
+#endif
break;
default:
@@ -4092,6 +4103,84 @@ static ssize_t qcow2_zlib_decompress(void *dest, size_t dest_size,
return ret;
}
+#ifdef CONFIG_ZSTD
+/*
+ * qcow2_zstd_compress()
+ *
+ * Compress @src_size bytes of data using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: compressed size on success
+ * a negative error code on fail
+ */
+
+static ssize_t qcow2_zstd_compress(void *dest, size_t dest_size,
+ const void *src, size_t src_size)
+{
+ ssize_t ret;
+ uint32_t *c_size = dest;
+ /* steal some bytes to store compressed chunk size */
+ char *d_buf = ((char *) dest) + sizeof(*c_size);
+
+ if (dest_size < sizeof(*c_size)) {
+ return -ENOMEM;
+ }
+
+ dest_size -= sizeof(*c_size);
+
+ ret = ZSTD_compress(d_buf, dest_size, src, src_size, 5);
+
+ if (ZSTD_isError(ret)) {
+ if (ret == ZSTD_error_dstSize_tooSmall) {
+ return -ENOMEM;
+ } else {
+ return -EIO;
+ }
+ }
+
+ /* store the compressed chunk size in the very beginning of the buffer */
+ *c_size = ret;
+
+ return ret + sizeof(ret);
+}
+
+/*
+ * qcow2_zstd_decompress()
+ *
+ * Decompress some data (not more than @src_size bytes) to produce exactly
+ * @dest_size bytes using zstd compression method
+ *
+ * @dest - destination buffer, @dest_size bytes
+ * @src - source buffer, @src_size bytes
+ *
+ * Returns: 0 on success
+ * -EIO on fail
+ */
+
+static ssize_t qcow2_zstd_decompress(void *dest, size_t dest_size,
+ const void *src, size_t src_size)
+{
+ ssize_t ret;
+ /*
+ * zstd decompress wants to know the exact lenght of the data
+ * for that purpose, on the compression the length is stored in
+ * the very beginning of the compressed buffer
+ */
+ const uint32_t *s_size = src;
+ const char *s_buf = ((char *) src) + sizeof(*s_size);
+
+ ret = ZSTD_decompress(dest, dest_size, s_buf, *s_size);
+
+ if (ZSTD_isError(ret)) {
+ return -EIO;
+ }
+
+ return 0;
+}
+#endif
+
#define MAX_COMPRESS_THREADS 4
typedef ssize_t (*Qcow2CompressFunc)(void *dest, size_t dest_size,
@@ -4180,6 +4269,11 @@ qcow2_co_compress(BlockDriverState *bs, void *dest, size_t dest_size,
fn = qcow2_zlib_compress;
break;
+#ifdef CONFIG_ZSTD
+ case QCOW2_COMPRESSION_TYPE_ZSTD:
+ fn = qcow2_zstd_compress;
+ break;
+#endif
default:
return -ENOTSUP;
}
@@ -4212,6 +4306,11 @@ qcow2_co_decompress(BlockDriverState *bs, void *dest, size_t dest_size,
fn = qcow2_zlib_decompress;
break;
+#ifdef CONFIG_ZSTD
+ case QCOW2_COMPRESSION_TYPE_ZSTD:
+ fn = qcow2_zstd_decompress;
+ break;
+#endif
default:
return -ENOTSUP;
}
diff --git a/configure b/configure
index 1c563a7027..57a80e38e7 100755
--- a/configure
+++ b/configure
@@ -433,6 +433,7 @@ opengl_dmabuf="no"
cpuid_h="no"
avx2_opt=""
zlib="yes"
+zstd=""
capstone=""
lzo=""
snappy=""
@@ -1333,6 +1334,10 @@ for opt do
;;
--disable-lzfse) lzfse="no"
;;
+ --enable-zstd) zstd="yes"
+ ;;
+ --disable-zstd) zstd="no"
+ ;;
--enable-guest-agent) guest_agent="yes"
;;
--disable-guest-agent) guest_agent="no"
@@ -1788,6 +1793,7 @@ disabled with --disable-FEATURE, default is enabled if available:
(for reading bzip2-compressed dmg images)
lzfse support of lzfse compression library
(for reading lzfse-compressed dmg images)
+ zstd support of zstd compression library
seccomp seccomp support
coroutine-pool coroutine freelist (better performance)
glusterfs GlusterFS backend
@@ -2374,6 +2380,29 @@ EOF
fi
fi
+#########################################
+# zstd check
+
+if test "$zstd" != "no" ; then
+ if $pkg_config --exists libzstd; then
+ zstd_cflags=$($pkg_config --cflags libzstd)
+ zstd_libs=$($pkg_config --libs libzstd)
+ QEMU_CFLAGS="$zstd_cflags $QEMU_CFLAGS"
+ LIBS="$zstd_libs $LIBS"
+ else
+ cat > $TMPC << EOF
+#include <zstd.h>
+int main(void) { ZSTD_versionNumber(); return 0; }
+EOF
+ if compile_prog "" "-lzstd" ; then
+ LIBS="$LIBS -lzstd"
+ else
+ error_exit "zstd check failed" \
+ "Make sure to have the zstd libs and headers installed."
+ fi
+ fi
+fi
+
##########################################
# libseccomp check
@@ -7253,6 +7282,9 @@ fi
if test "$sheepdog" = "yes" ; then
echo "CONFIG_SHEEPDOG=y" >> $config_host_mak
fi
+if test "$zstd" = "yes" ; then
+ echo "CONFIG_ZSTD=y" >> $config_host_mak
+fi
if test "$tcg_interpreter" = "yes"; then
QEMU_INCLUDES="-iquote \$(SRC_PATH)/tcg/tci $QEMU_INCLUDES"
diff --git a/docs/interop/qcow2.txt b/docs/interop/qcow2.txt
index 7cf068f814..4344e858cb 100644
--- a/docs/interop/qcow2.txt
+++ b/docs/interop/qcow2.txt
@@ -538,6 +538,9 @@ Compressed Clusters Descriptor (x = 62 - (cluster_bits - 8)):
Another compressed cluster may map to the tail of the final
sector used by this compressed cluster.
+ The layout of the compressed data depends on the compression
+ type used for the image (see compressed cluster layout).
+
If a cluster is unallocated, read requests shall read the data from the backing
file (except if bit 0 in the Standard Cluster Descriptor is set). If there is
no backing file or the backing file is smaller than the image, they shall read
@@ -790,3 +793,19 @@ In the image file the 'enabled' state is reflected by the 'auto' flag. If this
flag is set, the software must consider the bitmap as 'enabled' and start
tracking virtual disk changes to this bitmap from the first write to the
virtual disk. If this flag is not set then the bitmap is disabled.
+
+=== Compressed cluster layout ===
+
+The compressed cluster data may have a different layout depending on the
+compression type used for the image, and store specific data for the particular
+compression type.
+
+Compressed data layout for the available compression types:
+(x = data_space_length - 1)
+
+ zlib:
+ Byte 0 - x: the compressed data content
+ all the space provided used for compressed data
+ zstd:
+ Byte 0 - 3: the length of compressed data
+ 4 - x: the compressed data content
diff --git a/qapi/block-core.json b/qapi/block-core.json
index 835dd3c37f..2021e03a84 100644
--- a/qapi/block-core.json
+++ b/qapi/block-core.json
@@ -4215,11 +4215,12 @@
# Compression type used in qcow2 image file
#
# @zlib: zlib compression, see <http://zlib.net/>
+# @zstd: zstd compression, see <http://github.com/facebook/zstd>
#
# Since: 4.1
##
{ 'enum': 'Qcow2CompressionType',
- 'data': [ 'zlib' ] }
+ 'data': [ 'zlib', { 'name': 'zstd', 'if': 'defined(CONFIG_ZSTD)' } ] }
##
# @BlockdevCreateOptionsQcow2:
--
2.17.0
next prev parent reply other threads:[~2019-07-04 13:12 UTC|newest]
Thread overview: 19+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-07-04 13:09 [Qemu-devel] [PATCH v2 0/3] add zstd cluster compression Denis Plotnikov
2019-07-04 13:09 ` [Qemu-devel] [PATCH v2 1/3] qcow2: introduce compression type feature Denis Plotnikov
2019-07-09 6:15 ` Markus Armbruster
2019-08-07 23:12 ` Max Reitz
2019-08-08 0:09 ` Eric Blake
2019-08-08 12:48 ` Max Reitz
2019-08-08 0:18 ` Eric Blake
2019-08-08 12:50 ` Max Reitz
2019-08-08 14:07 ` Kevin Wolf
2019-07-04 13:09 ` [Qemu-devel] [PATCH v2 2/3] qcow2: rework the cluster compression routine Denis Plotnikov
2019-08-08 13:30 ` Max Reitz
2019-08-08 14:39 ` Max Reitz
2019-07-04 13:09 ` Denis Plotnikov [this message]
2019-07-09 6:18 ` [Qemu-devel] [PATCH v2 3/3] qcow2: add zstd cluster compression Markus Armbruster
2019-07-30 14:42 ` Denis Plotnikov
2019-08-08 0:24 ` Eric Blake
2019-08-08 14:44 ` Max Reitz
2019-07-30 14:45 ` [Qemu-devel] [PATCH v2 0/3] " Denis Plotnikov
2019-08-07 7:12 ` Denis Plotnikov
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190704130949.14017-4-dplotnikov@virtuozzo.com \
--to=dplotnikov@virtuozzo.com \
--cc=armbru@redhat.com \
--cc=den@virtuozzo.com \
--cc=eblake@redhat.com \
--cc=kwolf@redhat.com \
--cc=mreitz@redhat.com \
--cc=qemu-block@nongnu.org \
--cc=qemu-devel@nongnu.org \
--cc=vsementsov@virtuozzo.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).