QEMU-Devel Archive on lore.kernel.org
 help / color / Atom feed
* [PATCH] qemu-img: add support for rate limit in qemu-img convert
@ 2020-10-18  5:36 Zhengui li
  2020-10-18  5:42 ` no-reply
  0 siblings, 1 reply; 5+ messages in thread
From: Zhengui li @ 2020-10-18  5:36 UTC (permalink / raw)
  To: pbonzini, stefanha, mreitz, kwolf
  Cc: xieyingtai, lizhengui, qemu-devel, qemu-block

From: Zhengui <lizhengui@huawei.com>

Currently, there is no rate limit for qemu-img convert. This may
cause the task of qemu-img convert to consume all the bandwidth
of the storage. This will affect the IO performance of other processes
and virtual machines under shared storage. So we add support for
offline rate limit in qemu-img convert to get better quality of sevice.

Signed-off-by: Zhengui <lizhengui@huawei.com>
---
 qemu-img-cmds.hx |  4 ++--
 qemu-img.c       | 35 ++++++++++++++++++++++++++++++++++-
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
index ed55b76..70c0bf7 100644
--- a/qemu-img-cmds.hx
+++ b/qemu-img-cmds.hx
@@ -46,9 +46,9 @@ SRST
 ERST
 
 DEF("convert", img_convert,
-    "convert [--object objectdef] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")
+    "convert [--object objectdef] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-r rate_limit] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")
 SRST
-.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-m NUM_COROUTINES] [-W] [--salvage] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
+.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-r RATE_LIMIT] [-m NUM_COROUTINES] [-W] [--salvage] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
 ERST
 
 DEF("create", img_create,
diff --git a/qemu-img.c b/qemu-img.c
index 67c83b47..d27d8a6 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -50,6 +50,8 @@
 #include "block/qapi.h"
 #include "crypto/init.h"
 #include "trace/control.h"
+#include "qemu/throttle.h"
+#include "block/throttle-groups.h"
 
 #define QEMU_IMG_VERSION "qemu-img version " QEMU_FULL_VERSION \
                           "\n" QEMU_COPYRIGHT "\n"
@@ -1672,6 +1674,7 @@ enum ImgConvertBlockStatus {
 };
 
 #define MAX_COROUTINES 16
+#define CONVERT_THROTTLE_GROUP "img_convert"
 
 typedef struct ImgConvertState {
     BlockBackend **src;
@@ -2187,6 +2190,17 @@ static int convert_copy_bitmaps(BlockDriverState *src, BlockDriverState *dst)
 
 #define MAX_BUF_SECTORS 32768
 
+static void set_rate_limit(BlockBackend *blk, int64_t rate_limit)
+{
+    ThrottleConfig cfg;
+
+    throttle_config_init(&cfg);
+    cfg.buckets[THROTTLE_BPS_WRITE].avg = rate_limit;
+
+    blk_io_limits_enable(blk, CONVERT_THROTTLE_GROUP);
+    blk_set_io_limits(blk, &cfg);
+}
+
 static int img_convert(int argc, char **argv)
 {
     int c, bs_i, flags, src_flags = 0;
@@ -2207,6 +2221,7 @@ static int img_convert(int argc, char **argv)
     bool force_share = false;
     bool explict_min_sparse = false;
     bool bitmaps = false;
+    int64_t rate_limit = 0;
 
     ImgConvertState s = (ImgConvertState) {
         /* Need at least 4k of zeros for sparse detection */
@@ -2229,7 +2244,7 @@ static int img_convert(int argc, char **argv)
             {"bitmaps", no_argument, 0, OPTION_BITMAPS},
             {0, 0, 0, 0}
         };
-        c = getopt_long(argc, argv, ":hf:O:B:Cco:l:S:pt:T:qnm:WU",
+        c = getopt_long(argc, argv, ":hf:O:B:Cco:l:S:pt:T:qnm:WUr:",
                         long_options, NULL);
         if (c == -1) {
             break;
@@ -2326,6 +2341,16 @@ static int img_convert(int argc, char **argv)
         case 'U':
             force_share = true;
             break;
+        case 'r': {
+            char *end;
+            unsigned long long sval = strtoull(optarg, &end, 10);
+            if (*end != '\0' || sval > UINT_MAX) {
+                error_report("rate limit parse failed");
+                ret = -1;
+                goto fail_getopt;
+            }
+            rate_limit = (int64_t)sval * 1024 * 1024;
+        }   break;
         case OPTION_OBJECT: {
             QemuOpts *object_opts;
             object_opts = qemu_opts_parse_noisily(&qemu_object_opts,
@@ -2715,6 +2740,10 @@ static int img_convert(int argc, char **argv)
         s.cluster_sectors = bdi.cluster_size / BDRV_SECTOR_SIZE;
     }
 
+    if (rate_limit) {
+        set_rate_limit(s.target, rate_limit);
+    }
+
     ret = convert_do_copy(&s);
 
     /* Now copy the bitmaps */
@@ -2730,6 +2759,10 @@ out:
     qemu_opts_del(opts);
     qemu_opts_free(create_opts);
     qemu_opts_del(sn_opts);
+    if (s.target && rate_limit &&
+        blk_get_public(s.target)->throttle_group_member.throttle_state) {
+        blk_io_limits_disable(s.target);
+    }
     qobject_unref(open_opts);
     blk_unref(s.target);
     if (s.src) {
-- 
1.8.3.1



^ permalink raw reply	[flat|nested] 5+ messages in thread

* Re: [PATCH] qemu-img: add support for rate limit in qemu-img convert
  2020-10-18  5:36 [PATCH] qemu-img: add support for rate limit in qemu-img convert Zhengui li
@ 2020-10-18  5:42 ` no-reply
  0 siblings, 0 replies; 5+ messages in thread
From: no-reply @ 2020-10-18  5:42 UTC (permalink / raw)
  To: lizhengui
  Cc: kwolf, xieyingtai, lizhengui, qemu-block, qemu-devel, mreitz,
	stefanha, pbonzini

Patchew URL: https://patchew.org/QEMU/1602999390-21324-1-git-send-email-lizhengui@huawei.com/



Hi,

This series seems to have some coding style problems. See output below for
more information:

Type: series
Message-id: 1602999390-21324-1-git-send-email-lizhengui@huawei.com
Subject: [PATCH] qemu-img: add support for rate limit in qemu-img convert

=== TEST SCRIPT BEGIN ===
#!/bin/bash
git rev-parse base > /dev/null || exit 0
git config --local diff.renamelimit 0
git config --local diff.renames True
git config --local diff.algorithm histogram
./scripts/checkpatch.pl --mailback base..
=== TEST SCRIPT END ===

Updating 3c8cf5a9c21ff8782164d1def7f44bd888713384
From https://github.com/patchew-project/qemu
 * [new tag]         patchew/1602999390-21324-1-git-send-email-lizhengui@huawei.com -> patchew/1602999390-21324-1-git-send-email-lizhengui@huawei.com
Switched to a new branch 'test'
c39672c qemu-img: add support for rate limit in qemu-img convert

=== OUTPUT BEGIN ===
ERROR: consider using qemu_strtoull in preference to strtoull
#94: FILE: qemu-img.c:2336:
+            unsigned long long sval = strtoull(optarg, &end, 10);

total: 1 errors, 0 warnings, 94 lines checked

Commit c39672c23e5d (qemu-img: add support for rate limit in qemu-img convert) has style problems, please review.  If any of these errors
are false positives report them to the maintainer, see
CHECKPATCH in MAINTAINERS.
=== OUTPUT END ===

Test command exited with code: 1


The full log is available at
http://patchew.org/logs/1602999390-21324-1-git-send-email-lizhengui@huawei.com/testing.checkpatch/?type=message.
---
Email generated automatically by Patchew [https://patchew.org/].
Please send your feedback to patchew-devel@redhat.com

^ permalink raw reply	[flat|nested] 5+ messages in thread

* Re: [PATCH] qemu-img: add support for rate limit in qemu-img convert
  2020-10-18  6:34 Zhengui li
  2020-10-19 11:45 ` Alberto Garcia
@ 2020-10-19 20:13 ` Eric Blake
  1 sibling, 0 replies; 5+ messages in thread
From: Eric Blake @ 2020-10-19 20:13 UTC (permalink / raw)
  To: Zhengui li, pbonzini, stefanha, mreitz, kwolf
  Cc: xieyingtai, qemu-devel, qemu-block

On 10/18/20 1:34 AM, Zhengui li wrote:
> From: Zhengui <lizhengui@huawei.com>
> 
> Currently, there is no rate limit for qemu-img convert. This may
> cause the task of qemu-img convert to consume all the bandwidth
> of the storage. This will affect the IO performance of other processes
> and virtual machines under shared storage. So we add support for
> offline rate limit in qemu-img convert to get better quality of sevice.
> 

service

Also, I'd suggest bundling your related patches into a 0/N series (it 
took me a while to notice that you had two separate emails, one for 
commit and one for convert, sent at nearly the same time, because the 
subject line was long enough to truncate the important part in my view-pane)

> Signed-off-by: Zhengui <lizhengui@huawei.com>
> ---
>   qemu-img-cmds.hx |  4 ++--
>   qemu-img.c       | 34 +++++++++++++++++++++++++++++++++-
>   2 files changed, 35 insertions(+), 3 deletions(-)
> 

-- 
Eric Blake, Principal Software Engineer
Red Hat, Inc.           +1-919-301-3226
Virtualization:  qemu.org | libvirt.org



^ permalink raw reply	[flat|nested] 5+ messages in thread

* Re: [PATCH] qemu-img: add support for rate limit in qemu-img convert
  2020-10-18  6:34 Zhengui li
@ 2020-10-19 11:45 ` Alberto Garcia
  2020-10-19 20:13 ` Eric Blake
  1 sibling, 0 replies; 5+ messages in thread
From: Alberto Garcia @ 2020-10-19 11:45 UTC (permalink / raw)
  To: Zhengui li, pbonzini, stefanha, mreitz, kwolf
  Cc: xieyingtai, lizhengui, qemu-devel, qemu-block

On Sun 18 Oct 2020 08:34:39 AM CEST, Zhengui li wrote:
> @@ -2729,6 +2757,10 @@ out:
>      qemu_opts_del(opts);
>      qemu_opts_free(create_opts);
>      qemu_opts_del(sn_opts);
> +    if (s.target && rate_limit &&
> +        blk_get_public(s.target)->throttle_group_member.throttle_state) {
> +        blk_io_limits_disable(s.target);
> +    }
>      qobject_unref(open_opts);
>      blk_unref(s.target);
>      if (s.src) {

Apart from the comments that I wrote to the other patch, which also
apply to this one, blk_delete() already calls blk_io_limits_disable() so
I don't think you need to do it manually here.

Berto


^ permalink raw reply	[flat|nested] 5+ messages in thread

* [PATCH] qemu-img: add support for rate limit in qemu-img convert
@ 2020-10-18  6:34 Zhengui li
  2020-10-19 11:45 ` Alberto Garcia
  2020-10-19 20:13 ` Eric Blake
  0 siblings, 2 replies; 5+ messages in thread
From: Zhengui li @ 2020-10-18  6:34 UTC (permalink / raw)
  To: pbonzini, stefanha, mreitz, kwolf
  Cc: xieyingtai, lizhengui, qemu-devel, qemu-block

From: Zhengui <lizhengui@huawei.com>

Currently, there is no rate limit for qemu-img convert. This may
cause the task of qemu-img convert to consume all the bandwidth
of the storage. This will affect the IO performance of other processes
and virtual machines under shared storage. So we add support for
offline rate limit in qemu-img convert to get better quality of sevice.

Signed-off-by: Zhengui <lizhengui@huawei.com>
---
 qemu-img-cmds.hx |  4 ++--
 qemu-img.c       | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
index ed55b76..70c0bf7 100644
--- a/qemu-img-cmds.hx
+++ b/qemu-img-cmds.hx
@@ -46,9 +46,9 @@ SRST
 ERST
 
 DEF("convert", img_convert,
-    "convert [--object objectdef] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")
+    "convert [--object objectdef] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f fmt] [-t cache] [-T src_cache] [-O output_fmt] [-B backing_file] [-o options] [-l snapshot_param] [-S sparse_size] [-r rate_limit] [-m num_coroutines] [-W] [--salvage] filename [filename2 [...]] output_filename")
 SRST
-.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-m NUM_COROUTINES] [-W] [--salvage] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
+.. option:: convert [--object OBJECTDEF] [--image-opts] [--target-image-opts] [--target-is-zero] [--bitmaps] [-U] [-C] [-c] [-p] [-q] [-n] [-f FMT] [-t CACHE] [-T SRC_CACHE] [-O OUTPUT_FMT] [-B BACKING_FILE] [-o OPTIONS] [-l SNAPSHOT_PARAM] [-S SPARSE_SIZE] [-r RATE_LIMIT] [-m NUM_COROUTINES] [-W] [--salvage] FILENAME [FILENAME2 [...]] OUTPUT_FILENAME
 ERST
 
 DEF("create", img_create,
diff --git a/qemu-img.c b/qemu-img.c
index 74e4d64..ffda3c7 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -50,6 +50,8 @@
 #include "block/qapi.h"
 #include "crypto/init.h"
 #include "trace/control.h"
+#include "qemu/throttle.h"
+#include "block/throttle-groups.h"
 
 #define QEMU_IMG_VERSION "qemu-img version " QEMU_FULL_VERSION \
                           "\n" QEMU_COPYRIGHT "\n"
@@ -1671,6 +1673,7 @@ enum ImgConvertBlockStatus {
 };
 
 #define MAX_COROUTINES 16
+#define CONVERT_THROTTLE_GROUP "img_convert"
 
 typedef struct ImgConvertState {
     BlockBackend **src;
@@ -2186,6 +2189,17 @@ static int convert_copy_bitmaps(BlockDriverState *src, BlockDriverState *dst)
 
 #define MAX_BUF_SECTORS 32768
 
+static void set_rate_limit(BlockBackend *blk, int64_t rate_limit)
+{
+    ThrottleConfig cfg;
+
+    throttle_config_init(&cfg);
+    cfg.buckets[THROTTLE_BPS_WRITE].avg = rate_limit;
+
+    blk_io_limits_enable(blk, CONVERT_THROTTLE_GROUP);
+    blk_set_io_limits(blk, &cfg);
+}
+
 static int img_convert(int argc, char **argv)
 {
     int c, bs_i, flags, src_flags = 0;
@@ -2206,6 +2220,7 @@ static int img_convert(int argc, char **argv)
     bool force_share = false;
     bool explict_min_sparse = false;
     bool bitmaps = false;
+    int64_t rate_limit = 0;
 
     ImgConvertState s = (ImgConvertState) {
         /* Need at least 4k of zeros for sparse detection */
@@ -2228,7 +2243,7 @@ static int img_convert(int argc, char **argv)
             {"bitmaps", no_argument, 0, OPTION_BITMAPS},
             {0, 0, 0, 0}
         };
-        c = getopt_long(argc, argv, ":hf:O:B:Cco:l:S:pt:T:qnm:WU",
+        c = getopt_long(argc, argv, ":hf:O:B:Cco:l:S:pt:T:qnm:WUr:",
                         long_options, NULL);
         if (c == -1) {
             break;
@@ -2325,6 +2340,15 @@ static int img_convert(int argc, char **argv)
         case 'U':
             force_share = true;
             break;
+        case 'r': {
+            unsigned long long sval;
+            if (qemu_strtou64(optarg, NULL, 10, &sval)) {
+                error_report("rate limit parse failed");
+                ret = -1;
+                goto fail_getopt;
+            }
+            rate_limit = (int64_t)sval * 1024 * 1024;
+        }   break;
         case OPTION_OBJECT: {
             QemuOpts *object_opts;
             object_opts = qemu_opts_parse_noisily(&qemu_object_opts,
@@ -2714,6 +2738,10 @@ static int img_convert(int argc, char **argv)
         s.cluster_sectors = bdi.cluster_size / BDRV_SECTOR_SIZE;
     }
 
+    if (rate_limit) {
+        set_rate_limit(s.target, rate_limit);
+    }
+
     ret = convert_do_copy(&s);
 
     /* Now copy the bitmaps */
@@ -2729,6 +2757,10 @@ out:
     qemu_opts_del(opts);
     qemu_opts_free(create_opts);
     qemu_opts_del(sn_opts);
+    if (s.target && rate_limit &&
+        blk_get_public(s.target)->throttle_group_member.throttle_state) {
+        blk_io_limits_disable(s.target);
+    }
     qobject_unref(open_opts);
     blk_unref(s.target);
     if (s.src) {
-- 
1.8.3.1



^ permalink raw reply	[flat|nested] 5+ messages in thread

end of thread, back to index

Thread overview: 5+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-10-18  5:36 [PATCH] qemu-img: add support for rate limit in qemu-img convert Zhengui li
2020-10-18  5:42 ` no-reply
2020-10-18  6:34 Zhengui li
2020-10-19 11:45 ` Alberto Garcia
2020-10-19 20:13 ` Eric Blake

QEMU-Devel Archive on lore.kernel.org

Archives are clonable:
	git clone --mirror https://lore.kernel.org/qemu-devel/0 qemu-devel/git/0.git
	git clone --mirror https://lore.kernel.org/qemu-devel/1 qemu-devel/git/1.git

	# If you have public-inbox 1.1+ installed, you may
	# initialize and index your mirror using the following commands:
	public-inbox-init -V2 qemu-devel qemu-devel/ https://lore.kernel.org/qemu-devel \
		qemu-devel@nongnu.org
	public-inbox-index qemu-devel

Example config snippet for mirrors

Newsgroup available over NNTP:
	nntp://nntp.lore.kernel.org/org.nongnu.qemu-devel


AGPL code for this site: git clone https://public-inbox.org/public-inbox.git