[Qemu-devel] [PATCH v2 0/3] migration: add speed limit for multifd migration

[Qemu-devel] [PATCH v2 0/3] migration: add speed limit for multifd migration

[Ivan Ren]

Currently multifd migration has not been limited and it will consume
the whole bandwidth of Nic. These two patches add speed limitation to
it.

This is the v2 patches, differences with v1:
1. change qemu_file_update_rate_transfer interface name
   to qemu_file_update_transfer
2. add a new patch to update ram_counters for multifd sync packet

Ivan Ren (3):
  migration: add qemu_file_update_transfer interface
  migration: add speed limit for multifd migration
  migration: update ram_counters for multifd sync packet

 migration/qemu-file.c |  5 +++++
 migration/qemu-file.h |  1 +
 migration/ram.c       | 24 ++++++++++++++----------
 3 files changed, 20 insertions(+), 10 deletions(-)

-- 
2.17.2 (Apple Git-113)

[Qemu-devel] [PATCH v2 1/3] migration: add qemu_file_update_transfer interface

[Ivan Ren]

Add qemu_file_update_transfer for just update bytes_xfer for speed
limitation. This will be used for further migration feature such as
multifd migration.

Signed-off-by: Ivan Ren <ivanren@tencent.com>
---
 migration/qemu-file.c | 5 +++++
 migration/qemu-file.h | 1 +
 2 files changed, 6 insertions(+)

diff --git a/migration/qemu-file.c b/migration/qemu-file.c
index 0431585502..18f480529a 100644
--- a/migration/qemu-file.c
+++ b/migration/qemu-file.c
@@ -615,6 +615,11 @@ void qemu_file_reset_rate_limit(QEMUFile *f)
     f->bytes_xfer = 0;
 }
 
+void qemu_file_update_transfer(QEMUFile *f, int64_t len)
+{
+    f->bytes_xfer += len;
+}
+
 void qemu_put_be16(QEMUFile *f, unsigned int v)
 {
     qemu_put_byte(f, v >> 8);
diff --git a/migration/qemu-file.h b/migration/qemu-file.h
index 13baf896bd..5de9fa2e96 100644
--- a/migration/qemu-file.h
+++ b/migration/qemu-file.h
@@ -147,6 +147,7 @@ int qemu_peek_byte(QEMUFile *f, int offset);
 void qemu_file_skip(QEMUFile *f, int size);
 void qemu_update_position(QEMUFile *f, size_t size);
 void qemu_file_reset_rate_limit(QEMUFile *f);
+void qemu_file_update_transfer(QEMUFile *f, int64_t len);
 void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate);
 int64_t qemu_file_get_rate_limit(QEMUFile *f);
 void qemu_file_set_error(QEMUFile *f, int ret);
-- 
2.17.2 (Apple Git-113)

[Qemu-devel] [PATCH v2 2/3] migration: add speed limit for multifd migration

[Ivan Ren]

Limit the speed of multifd migration through common speed limitation
qemu file.

Signed-off-by: Ivan Ren <ivanren@tencent.com>
---
 migration/ram.c | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/migration/ram.c b/migration/ram.c
index 889148dd84..88ddd2bbe2 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -922,7 +922,7 @@ struct {
  * false.
  */
 
-static int multifd_send_pages(void)
+static int multifd_send_pages(RAMState *rs)
 {
     int i;
     static int next_channel;
@@ -954,6 +954,7 @@ static int multifd_send_pages(void)
     multifd_send_state->pages = p->pages;
     p->pages = pages;
     transferred = ((uint64_t) pages->used) * TARGET_PAGE_SIZE + p->packet_len;
+    qemu_file_update_transfer(rs->f, transferred);
     ram_counters.multifd_bytes += transferred;
     ram_counters.transferred += transferred;;
     qemu_mutex_unlock(&p->mutex);
@@ -962,7 +963,7 @@ static int multifd_send_pages(void)
     return 1;
 }
 
-static int multifd_queue_page(RAMBlock *block, ram_addr_t offset)
+static int multifd_queue_page(RAMState *rs, RAMBlock *block, ram_addr_t offset)
 {
     MultiFDPages_t *pages = multifd_send_state->pages;
 
@@ -981,12 +982,12 @@ static int multifd_queue_page(RAMBlock *block, ram_addr_t offset)
         }
     }
 
-    if (multifd_send_pages() < 0) {
+    if (multifd_send_pages(rs) < 0) {
         return -1;
     }
 
     if (pages->block != block) {
-        return  multifd_queue_page(block, offset);
+        return  multifd_queue_page(rs, block, offset);
     }
 
     return 1;
@@ -1054,7 +1055,7 @@ void multifd_save_cleanup(void)
     multifd_send_state = NULL;
 }
 
-static void multifd_send_sync_main(void)
+static void multifd_send_sync_main(RAMState *rs)
 {
     int i;
 
@@ -1062,7 +1063,7 @@ static void multifd_send_sync_main(void)
         return;
     }
     if (multifd_send_state->pages->used) {
-        if (multifd_send_pages() < 0) {
+        if (multifd_send_pages(rs) < 0) {
             error_report("%s: multifd_send_pages fail", __func__);
             return;
         }
@@ -1083,6 +1084,7 @@ static void multifd_send_sync_main(void)
         p->packet_num = multifd_send_state->packet_num++;
         p->flags |= MULTIFD_FLAG_SYNC;
         p->pending_job++;
+        qemu_file_update_transfer(rs->f, p->packet_len);
         qemu_mutex_unlock(&p->mutex);
         qemu_sem_post(&p->sem);
     }
@@ -2079,7 +2081,7 @@ static int ram_save_page(RAMState *rs, PageSearchStatus *pss, bool last_stage)
 static int ram_save_multifd_page(RAMState *rs, RAMBlock *block,
                                  ram_addr_t offset)
 {
-    if (multifd_queue_page(block, offset) < 0) {
+    if (multifd_queue_page(rs, block, offset) < 0) {
         return -1;
     }
     ram_counters.normal++;
@@ -3482,7 +3484,7 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
     ram_control_before_iterate(f, RAM_CONTROL_SETUP);
     ram_control_after_iterate(f, RAM_CONTROL_SETUP);
 
-    multifd_send_sync_main();
+    multifd_send_sync_main(*rsp);
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
     qemu_fflush(f);
 
@@ -3570,7 +3572,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
     ram_control_after_iterate(f, RAM_CONTROL_ROUND);
 
 out:
-    multifd_send_sync_main();
+    multifd_send_sync_main(rs);
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
     qemu_fflush(f);
     ram_counters.transferred += 8;
@@ -3629,7 +3631,7 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
 
     rcu_read_unlock();
 
-    multifd_send_sync_main();
+    multifd_send_sync_main(rs);
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
     qemu_fflush(f);
 
-- 
2.17.2 (Apple Git-113)

[Qemu-devel] [PATCH v2 3/3] migration: update ram_counters for multifd sync packet

[Ivan Ren]

Multifd sync will send MULTIFD_FLAG_SYNC flag info to destination, add
these bytes to ram_counters record.

Signed-off-by: Ivan Ren <ivanren@tencent.com>
---
 migration/ram.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/migration/ram.c b/migration/ram.c
index 88ddd2bbe2..20b6eebb7c 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -1085,6 +1085,8 @@ static void multifd_send_sync_main(RAMState *rs)
         p->flags |= MULTIFD_FLAG_SYNC;
         p->pending_job++;
         qemu_file_update_transfer(rs->f, p->packet_len);
+        ram_counters.multifd_bytes += p->packet_len;
+        ram_counters.transferred += p->packet_len;
         qemu_mutex_unlock(&p->mutex);
         qemu_sem_post(&p->sem);
     }
-- 
2.17.2 (Apple Git-113)

Re: [Qemu-devel] [PATCH v2 1/3] migration: add qemu_file_update_transfer interface

[Wei Yang]

On Mon, Jul 29, 2019 at 04:01:19PM +0800, Ivan Ren wrote:
>Add qemu_file_update_transfer for just update bytes_xfer for speed
>limitation. This will be used for further migration feature such as
>multifd migration.
>
>Signed-off-by: Ivan Ren <ivanren@tencent.com>

Well I have reviewed this patch, suppose you need add my Reviewed-by.

>---
> migration/qemu-file.c | 5 +++++
> migration/qemu-file.h | 1 +
> 2 files changed, 6 insertions(+)
>
>diff --git a/migration/qemu-file.c b/migration/qemu-file.c
>index 0431585502..18f480529a 100644
>--- a/migration/qemu-file.c
>+++ b/migration/qemu-file.c
>@@ -615,6 +615,11 @@ void qemu_file_reset_rate_limit(QEMUFile *f)
>     f->bytes_xfer = 0;
> }
> 
>+void qemu_file_update_transfer(QEMUFile *f, int64_t len)
>+{
>+    f->bytes_xfer += len;
>+}
>+
> void qemu_put_be16(QEMUFile *f, unsigned int v)
> {
>     qemu_put_byte(f, v >> 8);
>diff --git a/migration/qemu-file.h b/migration/qemu-file.h
>index 13baf896bd..5de9fa2e96 100644
>--- a/migration/qemu-file.h
>+++ b/migration/qemu-file.h
>@@ -147,6 +147,7 @@ int qemu_peek_byte(QEMUFile *f, int offset);
> void qemu_file_skip(QEMUFile *f, int size);
> void qemu_update_position(QEMUFile *f, size_t size);
> void qemu_file_reset_rate_limit(QEMUFile *f);
>+void qemu_file_update_transfer(QEMUFile *f, int64_t len);
> void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate);
> int64_t qemu_file_get_rate_limit(QEMUFile *f);
> void qemu_file_set_error(QEMUFile *f, int ret);
>-- 
>2.17.2 (Apple Git-113)
>

-- 
Wei Yang
Help you, Help me

Re: [Qemu-devel] [PATCH v2 3/3] migration: update ram_counters for multifd sync packet

[Wei Yang]

On Mon, Jul 29, 2019 at 04:01:21PM +0800, Ivan Ren wrote:
>Multifd sync will send MULTIFD_FLAG_SYNC flag info to destination, add
>these bytes to ram_counters record.
>
>Signed-off-by: Ivan Ren <ivanren@tencent.com>

Also this is suggested by me, it would be better to add Suggested-by.

>---
> migration/ram.c | 2 ++
> 1 file changed, 2 insertions(+)
>
>diff --git a/migration/ram.c b/migration/ram.c
>index 88ddd2bbe2..20b6eebb7c 100644
>--- a/migration/ram.c
>+++ b/migration/ram.c
>@@ -1085,6 +1085,8 @@ static void multifd_send_sync_main(RAMState *rs)
>         p->flags |= MULTIFD_FLAG_SYNC;
>         p->pending_job++;
>         qemu_file_update_transfer(rs->f, p->packet_len);
>+        ram_counters.multifd_bytes += p->packet_len;
>+        ram_counters.transferred += p->packet_len;
>         qemu_mutex_unlock(&p->mutex);
>         qemu_sem_post(&p->sem);
>     }
>-- 
>2.17.2 (Apple Git-113)
>

-- 
Wei Yang
Help you, Help me

Re: [Qemu-devel] [PATCH v2 0/3] migration: add speed limit for multifd migration

[Wei Yang]

On Mon, Jul 29, 2019 at 04:01:18PM +0800, Ivan Ren wrote:
>Currently multifd migration has not been limited and it will consume
>the whole bandwidth of Nic. These two patches add speed limitation to
>it.
>
>This is the v2 patches, differences with v1:
>1. change qemu_file_update_rate_transfer interface name
>   to qemu_file_update_transfer
>2. add a new patch to update ram_counters for multifd sync packet

Usually we cc those who gave us comment.

>
>Ivan Ren (3):
>  migration: add qemu_file_update_transfer interface
>  migration: add speed limit for multifd migration
>  migration: update ram_counters for multifd sync packet
>
> migration/qemu-file.c |  5 +++++
> migration/qemu-file.h |  1 +
> migration/ram.c       | 24 ++++++++++++++----------
> 3 files changed, 20 insertions(+), 10 deletions(-)
>
>-- 
>2.17.2 (Apple Git-113)
>

-- 
Wei Yang
Help you, Help me

Re: [Qemu-devel] [PATCH v2 3/3] migration: update ram_counters for multifd sync packet

[Ivan Ren]

Thanks
I'll send a new version with these suggest info and review info.

On Tue, Jul 30, 2019 at 8:42 AM Wei Yang <richardw.yang@linux.intel.com>
wrote:

> On Mon, Jul 29, 2019 at 04:01:21PM +0800, Ivan Ren wrote:
> >Multifd sync will send MULTIFD_FLAG_SYNC flag info to destination, add
> >these bytes to ram_counters record.
> >
> >Signed-off-by: Ivan Ren <ivanren@tencent.com>
>
> Also this is suggested by me, it would be better to add Suggested-by.
>
> >---
> > migration/ram.c | 2 ++
> > 1 file changed, 2 insertions(+)
> >
> >diff --git a/migration/ram.c b/migration/ram.c
> >index 88ddd2bbe2..20b6eebb7c 100644
> >--- a/migration/ram.c
> >+++ b/migration/ram.c
> >@@ -1085,6 +1085,8 @@ static void multifd_send_sync_main(RAMState *rs)
> >         p->flags |= MULTIFD_FLAG_SYNC;
> >         p->pending_job++;
> >         qemu_file_update_transfer(rs->f, p->packet_len);
> >+        ram_counters.multifd_bytes += p->packet_len;
> >+        ram_counters.transferred += p->packet_len;
> >         qemu_mutex_unlock(&p->mutex);
> >         qemu_sem_post(&p->sem);
> >     }
> >--
> >2.17.2 (Apple Git-113)
> >
>
> --
> Wei Yang
> Help you, Help me
>