[Qemu-devel] [PATCH v4 0/3] High downtime with 95+ throttle pct

[Qemu-devel] [PATCH v4 0/3] High downtime with 95+ throttle pct

[Yury Kotov]

Hi,

V4:
* The test was simplified to prevent false fails.

V3:
* Rebase fixes (migrate_set_parameter -> migrate_set_parameter_int)

V2:
* Added a test
* Fixed qemu_cond_timedwait for qsp

I wrote a test for migration auto converge and found out a strange thing:
1. Enable auto converge
2. Set max-bandwidth 1Gb/s
3. Set downtime-limit 1ms
4. Run standard test (just writes a byte per page)
5. Wait for converge
6. It's converged with 99% throttle percentage
7. The result downtime was about 300-600ms   <<<<

It's much higher than expected 1ms. I figured out that cpu_throttle_thread()
function sleeps for 100ms+ for high throttle percentage (>=95%) in VCPU thread.
And it sleeps even after a cpu kick.

Fixed it by using timedwait for ms part of sleep.
E.g timedwait(halt_cond, 1ms) + usleep(500).

Regards,
Yury

Yury Kotov (3):
  qemu-thread: Add qemu_cond_timedwait
  cpus: Fix throttling during vm_stop
  tests/migration: Add a test for auto converge

 cpus.c                   |  27 +++++++---
 include/qemu/thread.h    |  18 +++++++
 tests/migration-test.c   | 103 ++++++++++++++++++++++++++++++++++-----
 util/qemu-thread-posix.c |  40 ++++++++++-----
 util/qemu-thread-win32.c |  16 ++++++
 util/qsp.c               |  18 +++++++
 6 files changed, 191 insertions(+), 31 deletions(-)

-- 
2.22.0

[Qemu-devel] [PATCH v4 1/3] qemu-thread: Add qemu_cond_timedwait

[Yury Kotov]

Signed-off-by: Yury Kotov <yury-kotov@yandex-team.ru>
---
 include/qemu/thread.h    | 18 ++++++++++++++++++
 util/qemu-thread-posix.c | 40 ++++++++++++++++++++++++++++------------
 util/qemu-thread-win32.c | 16 ++++++++++++++++
 util/qsp.c               | 18 ++++++++++++++++++
 4 files changed, 80 insertions(+), 12 deletions(-)

diff --git a/include/qemu/thread.h b/include/qemu/thread.h
index 55d83a907c..d0cd7b9ae0 100644
--- a/include/qemu/thread.h
+++ b/include/qemu/thread.h
@@ -34,6 +34,8 @@ typedef void (*QemuRecMutexLockFunc)(QemuRecMutex *m, const char *f, int l);
 typedef int (*QemuRecMutexTrylockFunc)(QemuRecMutex *m, const char *f, int l);
 typedef void (*QemuCondWaitFunc)(QemuCond *c, QemuMutex *m, const char *f,
                                  int l);
+typedef void (*QemuCondTimedWaitFunc)(QemuCond *c, QemuMutex *m, int ms,
+                                      const char *f, int l);
 
 extern QemuMutexLockFunc qemu_bql_mutex_lock_func;
 extern QemuMutexLockFunc qemu_mutex_lock_func;
@@ -41,6 +43,7 @@ extern QemuMutexTrylockFunc qemu_mutex_trylock_func;
 extern QemuRecMutexLockFunc qemu_rec_mutex_lock_func;
 extern QemuRecMutexTrylockFunc qemu_rec_mutex_trylock_func;
 extern QemuCondWaitFunc qemu_cond_wait_func;
+extern QemuCondTimedWaitFunc qemu_cond_timedwait_func;
 
 /* convenience macros to bypass the profiler */
 #define qemu_mutex_lock__raw(m)                         \
@@ -63,6 +66,8 @@ extern QemuCondWaitFunc qemu_cond_wait_func;
             qemu_rec_mutex_trylock_impl(m, __FILE__, __LINE__);
 #define qemu_cond_wait(c, m)                                            \
             qemu_cond_wait_impl(c, m, __FILE__, __LINE__);
+#define qemu_cond_timedwait(c, m, ms)                                   \
+            qemu_cond_wait_impl(c, m, ms, __FILE__, __LINE__);
 #else
 #define qemu_mutex_lock(m) ({                                           \
             QemuMutexLockFunc _f = atomic_read(&qemu_mutex_lock_func);  \
@@ -89,6 +94,11 @@ extern QemuCondWaitFunc qemu_cond_wait_func;
             QemuCondWaitFunc _f = atomic_read(&qemu_cond_wait_func);    \
             _f(c, m, __FILE__, __LINE__);                               \
         })
+
+#define qemu_cond_timedwait(c, m, ms) ({                                       \
+            QemuCondTimedWaitFunc _f = atomic_read(&qemu_cond_timedwait_func); \
+            _f(c, m, ms, __FILE__, __LINE__);                                  \
+        })
 #endif
 
 #define qemu_mutex_unlock(mutex) \
@@ -134,12 +144,20 @@ void qemu_cond_signal(QemuCond *cond);
 void qemu_cond_broadcast(QemuCond *cond);
 void qemu_cond_wait_impl(QemuCond *cond, QemuMutex *mutex,
                          const char *file, const int line);
+void qemu_cond_timedwait_impl(QemuCond *cond, QemuMutex *mutex, int ms,
+                              const char *file, const int line);
 
 static inline void (qemu_cond_wait)(QemuCond *cond, QemuMutex *mutex)
 {
     qemu_cond_wait(cond, mutex);
 }
 
+static inline void (qemu_cond_timedwait)(QemuCond *cond, QemuMutex *mutex,
+                                         int ms)
+{
+    qemu_cond_timedwait(cond, mutex, ms);
+}
+
 void qemu_sem_init(QemuSemaphore *sem, int init);
 void qemu_sem_post(QemuSemaphore *sem);
 void qemu_sem_wait(QemuSemaphore *sem);
diff --git a/util/qemu-thread-posix.c b/util/qemu-thread-posix.c
index 1bf5e65dea..eed777d9ec 100644
--- a/util/qemu-thread-posix.c
+++ b/util/qemu-thread-posix.c
@@ -36,6 +36,18 @@ static void error_exit(int err, const char *msg)
     abort();
 }
 
+static void compute_abs_deadline(struct timespec *ts, int ms)
+{
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    ts->tv_nsec = tv.tv_usec * 1000 + (ms % 1000) * 1000000;
+    ts->tv_sec = tv.tv_sec + ms / 1000;
+    if (ts->tv_nsec >= 1000000000) {
+        ts->tv_sec++;
+        ts->tv_nsec -= 1000000000;
+    }
+}
+
 void qemu_mutex_init(QemuMutex *mutex)
 {
     int err;
@@ -164,6 +176,22 @@ void qemu_cond_wait_impl(QemuCond *cond, QemuMutex *mutex, const char *file, con
         error_exit(err, __func__);
 }
 
+void qemu_cond_timedwait_impl(QemuCond *cond, QemuMutex *mutex, int ms,
+                              const char *file, const int line)
+{
+    int err;
+    struct timespec ts;
+
+    assert(cond->initialized);
+    trace_qemu_mutex_unlock(mutex, file, line);
+    compute_abs_deadline(&ts, ms);
+    err = pthread_cond_timedwait(&cond->cond, &mutex->lock, &ts);
+    trace_qemu_mutex_locked(mutex, file, line);
+    if (err && err != ETIMEDOUT) {
+        error_exit(err, __func__);
+    }
+}
+
 void qemu_sem_init(QemuSemaphore *sem, int init)
 {
     int rc;
@@ -238,18 +266,6 @@ void qemu_sem_post(QemuSemaphore *sem)
 #endif
 }
 
-static void compute_abs_deadline(struct timespec *ts, int ms)
-{
-    struct timeval tv;
-    gettimeofday(&tv, NULL);
-    ts->tv_nsec = tv.tv_usec * 1000 + (ms % 1000) * 1000000;
-    ts->tv_sec = tv.tv_sec + ms / 1000;
-    if (ts->tv_nsec >= 1000000000) {
-        ts->tv_sec++;
-        ts->tv_nsec -= 1000000000;
-    }
-}
-
 int qemu_sem_timedwait(QemuSemaphore *sem, int ms)
 {
     int rc;
diff --git a/util/qemu-thread-win32.c b/util/qemu-thread-win32.c
index 572f88535d..5faa01cb61 100644
--- a/util/qemu-thread-win32.c
+++ b/util/qemu-thread-win32.c
@@ -145,6 +145,22 @@ void qemu_cond_wait_impl(QemuCond *cond, QemuMutex *mutex, const char *file, con
     qemu_mutex_post_lock(mutex, file, line);
 }
 
+void qemu_cond_timedwait_impl(QemuCond *cond, QemuMutex *mutex, int ms,
+                              const char *file, const int line)
+{
+    int rc = 0;
+
+    assert(cond->initialized);
+    trace_qemu_mutex_unlock(mutex, file, line);
+    if (!SleepConditionVariableSRW(&cond->var, &mutex->lock, ms, 0)) {
+        rc = GetLastError();
+    }
+    trace_qemu_mutex_locked(mutex, file, line);
+    if (rc && rc != ERROR_TIMEOUT) {
+        error_exit(rc, __func__);
+    }
+}
+
 void qemu_sem_init(QemuSemaphore *sem, int init)
 {
     /* Manual reset.  */
diff --git a/util/qsp.c b/util/qsp.c
index 5264c97342..904dcb7436 100644
--- a/util/qsp.c
+++ b/util/qsp.c
@@ -131,6 +131,7 @@ QemuRecMutexLockFunc qemu_rec_mutex_lock_func = qemu_rec_mutex_lock_impl;
 QemuRecMutexTrylockFunc qemu_rec_mutex_trylock_func =
     qemu_rec_mutex_trylock_impl;
 QemuCondWaitFunc qemu_cond_wait_func = qemu_cond_wait_impl;
+QemuCondTimedWaitFunc qemu_cond_timedwait_func = qemu_cond_timedwait_impl;
 
 /*
  * It pays off to _not_ hash callsite->file; hashing a string is slow, and
@@ -412,6 +413,21 @@ qsp_cond_wait(QemuCond *cond, QemuMutex *mutex, const char *file, int line)
     qsp_entry_record(e, t1 - t0);
 }
 
+static void
+qsp_cond_timedwait(QemuCond *cond, QemuMutex *mutex, int ms,
+                   const char *file, int line)
+{
+    QSPEntry *e;
+    int64_t t0, t1;
+
+    t0 = get_clock();
+    qemu_cond_timedwait_impl(cond, mutex, ms, file, line);
+    t1 = get_clock();
+
+    e = qsp_entry_get(cond, file, line, QSP_CONDVAR);
+    qsp_entry_record(e, t1 - t0);
+}
+
 bool qsp_is_enabled(void)
 {
     return atomic_read(&qemu_mutex_lock_func) == qsp_mutex_lock;
@@ -425,6 +441,7 @@ void qsp_enable(void)
     atomic_set(&qemu_rec_mutex_lock_func, qsp_rec_mutex_lock);
     atomic_set(&qemu_rec_mutex_trylock_func, qsp_rec_mutex_trylock);
     atomic_set(&qemu_cond_wait_func, qsp_cond_wait);
+    atomic_set(&qemu_cond_timedwait_func, qsp_cond_timedwait);
 }
 
 void qsp_disable(void)
@@ -435,6 +452,7 @@ void qsp_disable(void)
     atomic_set(&qemu_rec_mutex_lock_func, qemu_rec_mutex_lock_impl);
     atomic_set(&qemu_rec_mutex_trylock_func, qemu_rec_mutex_trylock_impl);
     atomic_set(&qemu_cond_wait_func, qemu_cond_wait_impl);
+    atomic_set(&qemu_cond_timedwait_func, qemu_cond_timedwait_impl);
 }
 
 static gint qsp_tree_cmp(gconstpointer ap, gconstpointer bp, gpointer up)
-- 
2.22.0

[Qemu-devel] [PATCH v4 2/3] cpus: Fix throttling during vm_stop

[Yury Kotov]

Throttling thread sleeps in VCPU thread. For high throttle percentage
this sleep is more than 10ms. E.g. for 60% - 15ms, for 99% - 990ms.
vm_stop() kicks all VCPUs and waits for them. It's called at the end of
migration and because of the long sleep the migration downtime might be
more than 100ms even for downtime-limit 1ms.
Use qemu_cond_timedwait for high percentage to wake up during vm_stop.

Signed-off-by: Yury Kotov <yury-kotov@yandex-team.ru>
---
 cpus.c | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/cpus.c b/cpus.c
index 927a00aa90..3baedd554c 100644
--- a/cpus.c
+++ b/cpus.c
@@ -74,6 +74,8 @@
 
 #endif /* CONFIG_LINUX */
 
+static QemuMutex qemu_global_mutex;
+
 int64_t max_delay;
 int64_t max_advance;
 
@@ -777,7 +779,7 @@ static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
 {
     double pct;
     double throttle_ratio;
-    long sleeptime_ns;
+    int64_t sleeptime_ns;
 
     if (!cpu_throttle_get_percentage()) {
         return;
@@ -785,11 +787,22 @@ static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
 
     pct = (double)cpu_throttle_get_percentage()/100;
     throttle_ratio = pct / (1 - pct);
-    sleeptime_ns = (long)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS);
-
-    qemu_mutex_unlock_iothread();
-    g_usleep(sleeptime_ns / 1000); /* Convert ns to us for usleep call */
-    qemu_mutex_lock_iothread();
+    /* Add 1ns to fix double's rounding error (like 0.9999999...) */
+    sleeptime_ns = (int64_t)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS + 1);
+
+    while (sleeptime_ns >= SCALE_MS && !cpu->stop) {
+        int64_t start, end;
+        start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+        qemu_cond_timedwait(cpu->halt_cond, &qemu_global_mutex,
+                            sleeptime_ns / SCALE_MS);
+        end = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
+        sleeptime_ns -= end - start;
+    }
+    if (sleeptime_ns >= SCALE_US && !cpu->stop) {
+        qemu_mutex_unlock_iothread();
+        g_usleep(sleeptime_ns / SCALE_US);
+        qemu_mutex_lock_iothread();
+    }
     atomic_set(&cpu->throttle_thread_scheduled, 0);
 }
 
@@ -1167,8 +1180,6 @@ static void qemu_init_sigbus(void)
 }
 #endif /* !CONFIG_LINUX */
 
-static QemuMutex qemu_global_mutex;
-
 static QemuThread io_thread;
 
 /* cpu creation */
-- 
2.22.0

[Qemu-devel] [PATCH v4 3/3] tests/migration: Add a test for auto converge

[Yury Kotov]

Signed-off-by: Yury Kotov <yury-kotov@yandex-team.ru>
---
 tests/migration-test.c | 103 ++++++++++++++++++++++++++++++++++++-----
 1 file changed, 92 insertions(+), 11 deletions(-)

diff --git a/tests/migration-test.c b/tests/migration-test.c
index a4feb9545d..b783ae47b3 100644
--- a/tests/migration-test.c
+++ b/tests/migration-test.c
@@ -241,6 +241,17 @@ static int64_t read_ram_property_int(QTestState *who, const char *property)
     return result;
 }
 
+static int64_t read_migrate_property_int(QTestState *who, const char *property)
+{
+    QDict *rsp_return;
+    int64_t result;
+
+    rsp_return = migrate_query(who);
+    result = qdict_get_try_int(rsp_return, property, 0);
+    qobject_unref(rsp_return);
+    return result;
+}
+
 static uint64_t get_migration_pass(QTestState *who)
 {
     return read_ram_property_int(who, "dirty-sync-count");
@@ -255,20 +266,22 @@ static void read_blocktime(QTestState *who)
     qobject_unref(rsp_return);
 }
 
+static bool check_migration_status(QTestState *who, const char *status)
+{
+    bool completed;
+    char *current_status;
+
+    current_status = migrate_query_status(who);
+    completed = strcmp(current_status, status) == 0;
+    g_assert_cmpstr(current_status, !=, "failed");
+    g_free(current_status);
+    return completed;
+}
+
 static void wait_for_migration_status(QTestState *who,
                                       const char *goal)
 {
-    while (true) {
-        bool completed;
-        char *status;
-
-        status = migrate_query_status(who);
-        completed = strcmp(status, goal) == 0;
-        g_assert_cmpstr(status, !=,  "failed");
-        g_free(status);
-        if (completed) {
-            return;
-        }
+    while (!check_migration_status(who, goal)) {
         usleep(1000);
     }
 }
@@ -1121,6 +1134,73 @@ static void test_migrate_fd_proto(void)
     test_migrate_end(from, to, true);
 }
 
+static void test_migrate_auto_converge(void)
+{
+    char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
+    QTestState *from, *to;
+    int64_t remaining, percentage;
+
+    /*
+     * We want the test to be fast enough, but stable.
+     * Throttle percentages are chosen to cover all cases (init, increment, max)
+     */
+    static const int64_t expected_pcts[] = { 0, 1, 51, 98 };
+    const int64_t max_bandwidth = 200000000; /* ~200Mb/s */
+    const int64_t downtime_limit = 50; /* 50ms */
+    /*
+     * We migrate through unix-socket (> 500Mb/s).
+     * Thus, expected migration speed ~= bandwidth limit (< 500Mb/s).
+     * So, we can predict expected_threshold
+     */
+    const int64_t expected_threshold = max_bandwidth * downtime_limit / 1000;
+
+    if (test_migrate_start(&from, &to, uri, false, false)) {
+        return;
+    }
+
+    migrate_set_capability(from, "auto-converge", true);
+    migrate_set_parameter_int(from, "cpu-throttle-initial", expected_pcts[1]);
+    migrate_set_parameter_int(from, "cpu-throttle-increment",
+                              expected_pcts[2] - expected_pcts[1]);
+    migrate_set_parameter_int(from, "max-cpu-throttle", expected_pcts[3]);
+
+    migrate_set_parameter_int(from, "max-bandwidth", max_bandwidth);
+    migrate_set_parameter_int(from, "downtime-limit", downtime_limit);
+
+    /* To check remaining size after precopy */
+    migrate_set_capability(from, "pause-before-switchover", true);
+
+    /* Wait for the first serial output from the source */
+    wait_for_serial("src_serial");
+
+    migrate(from, uri, "{}");
+
+    /*
+     * Wait for pre-switchover status to check last throttle percentage
+     * and remaining. These values will be zeroed later
+     */
+    wait_for_migration_status(from, "pre-switchover");
+
+    /* We expect that migration can't converge without throttling */
+    percentage = read_migrate_property_int(from, "cpu-throttle-percentage");
+    g_assert_cmpint(percentage, >, 0);
+
+    remaining = read_ram_property_int(from, "remaining");
+    g_assert_cmpint(remaining, <, expected_threshold);
+
+    wait_command(from, "{ 'execute': 'migrate-continue' , 'arguments':"
+                       "  { 'state': 'pre-switchover' } }");
+
+    qtest_qmp_eventwait(to, "RESUME");
+
+    wait_for_serial("dest_serial");
+    wait_for_migration_complete(from);
+
+    g_free(uri);
+
+    test_migrate_end(from, to, true);
+}
+
 int main(int argc, char **argv)
 {
     char template[] = "/tmp/migration-test-XXXXXX";
@@ -1176,6 +1256,7 @@ int main(int argc, char **argv)
     /* qtest_add_func("/migration/ignore_shared", test_ignore_shared); */
     qtest_add_func("/migration/xbzrle/unix", test_xbzrle_unix);
     qtest_add_func("/migration/fd_proto", test_migrate_fd_proto);
+    qtest_add_func("/migration/auto_converge", test_migrate_auto_converge);
 
     ret = g_test_run();
 
-- 
2.22.0

Re: [Qemu-devel] [PATCH v4 3/3] tests/migration: Add a test for auto converge

[Dr. David Alan Gilbert]

* Yury Kotov (yury-kotov@yandex-team.ru) wrote:
> Signed-off-by: Yury Kotov <yury-kotov@yandex-team.ru>

OK, I think that's worth a go; lets see how it does in heavy CI systems/

Dave

> ---
>  tests/migration-test.c | 103 ++++++++++++++++++++++++++++++++++++-----
>  1 file changed, 92 insertions(+), 11 deletions(-)
> 
> diff --git a/tests/migration-test.c b/tests/migration-test.c
> index a4feb9545d..b783ae47b3 100644
> --- a/tests/migration-test.c
> +++ b/tests/migration-test.c
> @@ -241,6 +241,17 @@ static int64_t read_ram_property_int(QTestState *who, const char *property)
>      return result;
>  }
>  
> +static int64_t read_migrate_property_int(QTestState *who, const char *property)
> +{
> +    QDict *rsp_return;
> +    int64_t result;
> +
> +    rsp_return = migrate_query(who);
> +    result = qdict_get_try_int(rsp_return, property, 0);
> +    qobject_unref(rsp_return);
> +    return result;
> +}
> +
>  static uint64_t get_migration_pass(QTestState *who)
>  {
>      return read_ram_property_int(who, "dirty-sync-count");
> @@ -255,20 +266,22 @@ static void read_blocktime(QTestState *who)
>      qobject_unref(rsp_return);
>  }
>  
> +static bool check_migration_status(QTestState *who, const char *status)
> +{
> +    bool completed;
> +    char *current_status;
> +
> +    current_status = migrate_query_status(who);
> +    completed = strcmp(current_status, status) == 0;
> +    g_assert_cmpstr(current_status, !=, "failed");
> +    g_free(current_status);
> +    return completed;
> +}
> +
>  static void wait_for_migration_status(QTestState *who,
>                                        const char *goal)
>  {
> -    while (true) {
> -        bool completed;
> -        char *status;
> -
> -        status = migrate_query_status(who);
> -        completed = strcmp(status, goal) == 0;
> -        g_assert_cmpstr(status, !=,  "failed");
> -        g_free(status);
> -        if (completed) {
> -            return;
> -        }
> +    while (!check_migration_status(who, goal)) {
>          usleep(1000);
>      }
>  }
> @@ -1121,6 +1134,73 @@ static void test_migrate_fd_proto(void)
>      test_migrate_end(from, to, true);
>  }
>  
> +static void test_migrate_auto_converge(void)
> +{
> +    char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
> +    QTestState *from, *to;
> +    int64_t remaining, percentage;
> +
> +    /*
> +     * We want the test to be fast enough, but stable.
> +     * Throttle percentages are chosen to cover all cases (init, increment, max)
> +     */
> +    static const int64_t expected_pcts[] = { 0, 1, 51, 98 };
> +    const int64_t max_bandwidth = 200000000; /* ~200Mb/s */
> +    const int64_t downtime_limit = 50; /* 50ms */
> +    /*
> +     * We migrate through unix-socket (> 500Mb/s).
> +     * Thus, expected migration speed ~= bandwidth limit (< 500Mb/s).
> +     * So, we can predict expected_threshold
> +     */
> +    const int64_t expected_threshold = max_bandwidth * downtime_limit / 1000;
> +
> +    if (test_migrate_start(&from, &to, uri, false, false)) {
> +        return;
> +    }
> +
> +    migrate_set_capability(from, "auto-converge", true);
> +    migrate_set_parameter_int(from, "cpu-throttle-initial", expected_pcts[1]);
> +    migrate_set_parameter_int(from, "cpu-throttle-increment",
> +                              expected_pcts[2] - expected_pcts[1]);
> +    migrate_set_parameter_int(from, "max-cpu-throttle", expected_pcts[3]);
> +
> +    migrate_set_parameter_int(from, "max-bandwidth", max_bandwidth);
> +    migrate_set_parameter_int(from, "downtime-limit", downtime_limit);
> +
> +    /* To check remaining size after precopy */
> +    migrate_set_capability(from, "pause-before-switchover", true);
> +
> +    /* Wait for the first serial output from the source */
> +    wait_for_serial("src_serial");
> +
> +    migrate(from, uri, "{}");
> +
> +    /*
> +     * Wait for pre-switchover status to check last throttle percentage
> +     * and remaining. These values will be zeroed later
> +     */
> +    wait_for_migration_status(from, "pre-switchover");
> +
> +    /* We expect that migration can't converge without throttling */
> +    percentage = read_migrate_property_int(from, "cpu-throttle-percentage");
> +    g_assert_cmpint(percentage, >, 0);
> +
> +    remaining = read_ram_property_int(from, "remaining");
> +    g_assert_cmpint(remaining, <, expected_threshold);
> +
> +    wait_command(from, "{ 'execute': 'migrate-continue' , 'arguments':"
> +                       "  { 'state': 'pre-switchover' } }");
> +
> +    qtest_qmp_eventwait(to, "RESUME");
> +
> +    wait_for_serial("dest_serial");
> +    wait_for_migration_complete(from);
> +
> +    g_free(uri);
> +
> +    test_migrate_end(from, to, true);
> +}
> +
>  int main(int argc, char **argv)
>  {
>      char template[] = "/tmp/migration-test-XXXXXX";
> @@ -1176,6 +1256,7 @@ int main(int argc, char **argv)
>      /* qtest_add_func("/migration/ignore_shared", test_ignore_shared); */
>      qtest_add_func("/migration/xbzrle/unix", test_xbzrle_unix);
>      qtest_add_func("/migration/fd_proto", test_migrate_fd_proto);
> +    qtest_add_func("/migration/auto_converge", test_migrate_auto_converge);
>  
>      ret = g_test_run();
>  
> -- 
> 2.22.0
> 
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK

Re: [Qemu-devel] [PATCH v4 3/3] tests/migration: Add a test for auto converge

[Dr. David Alan Gilbert]

* Dr. David Alan Gilbert (dgilbert@redhat.com) wrote:
> * Yury Kotov (yury-kotov@yandex-team.ru) wrote:
> > Signed-off-by: Yury Kotov <yury-kotov@yandex-team.ru>
> 
> OK, I think that's worth a go; lets see how it does in heavy CI systems/
> 
> Dave

and I meant:


Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>

> > ---
> >  tests/migration-test.c | 103 ++++++++++++++++++++++++++++++++++++-----
> >  1 file changed, 92 insertions(+), 11 deletions(-)
> > 
> > diff --git a/tests/migration-test.c b/tests/migration-test.c
> > index a4feb9545d..b783ae47b3 100644
> > --- a/tests/migration-test.c
> > +++ b/tests/migration-test.c
> > @@ -241,6 +241,17 @@ static int64_t read_ram_property_int(QTestState *who, const char *property)
> >      return result;
> >  }
> >  
> > +static int64_t read_migrate_property_int(QTestState *who, const char *property)
> > +{
> > +    QDict *rsp_return;
> > +    int64_t result;
> > +
> > +    rsp_return = migrate_query(who);
> > +    result = qdict_get_try_int(rsp_return, property, 0);
> > +    qobject_unref(rsp_return);
> > +    return result;
> > +}
> > +
> >  static uint64_t get_migration_pass(QTestState *who)
> >  {
> >      return read_ram_property_int(who, "dirty-sync-count");
> > @@ -255,20 +266,22 @@ static void read_blocktime(QTestState *who)
> >      qobject_unref(rsp_return);
> >  }
> >  
> > +static bool check_migration_status(QTestState *who, const char *status)
> > +{
> > +    bool completed;
> > +    char *current_status;
> > +
> > +    current_status = migrate_query_status(who);
> > +    completed = strcmp(current_status, status) == 0;
> > +    g_assert_cmpstr(current_status, !=, "failed");
> > +    g_free(current_status);
> > +    return completed;
> > +}
> > +
> >  static void wait_for_migration_status(QTestState *who,
> >                                        const char *goal)
> >  {
> > -    while (true) {
> > -        bool completed;
> > -        char *status;
> > -
> > -        status = migrate_query_status(who);
> > -        completed = strcmp(status, goal) == 0;
> > -        g_assert_cmpstr(status, !=,  "failed");
> > -        g_free(status);
> > -        if (completed) {
> > -            return;
> > -        }
> > +    while (!check_migration_status(who, goal)) {
> >          usleep(1000);
> >      }
> >  }
> > @@ -1121,6 +1134,73 @@ static void test_migrate_fd_proto(void)
> >      test_migrate_end(from, to, true);
> >  }
> >  
> > +static void test_migrate_auto_converge(void)
> > +{
> > +    char *uri = g_strdup_printf("unix:%s/migsocket", tmpfs);
> > +    QTestState *from, *to;
> > +    int64_t remaining, percentage;
> > +
> > +    /*
> > +     * We want the test to be fast enough, but stable.
> > +     * Throttle percentages are chosen to cover all cases (init, increment, max)
> > +     */
> > +    static const int64_t expected_pcts[] = { 0, 1, 51, 98 };
> > +    const int64_t max_bandwidth = 200000000; /* ~200Mb/s */
> > +    const int64_t downtime_limit = 50; /* 50ms */
> > +    /*
> > +     * We migrate through unix-socket (> 500Mb/s).
> > +     * Thus, expected migration speed ~= bandwidth limit (< 500Mb/s).
> > +     * So, we can predict expected_threshold
> > +     */
> > +    const int64_t expected_threshold = max_bandwidth * downtime_limit / 1000;
> > +
> > +    if (test_migrate_start(&from, &to, uri, false, false)) {
> > +        return;
> > +    }
> > +
> > +    migrate_set_capability(from, "auto-converge", true);
> > +    migrate_set_parameter_int(from, "cpu-throttle-initial", expected_pcts[1]);
> > +    migrate_set_parameter_int(from, "cpu-throttle-increment",
> > +                              expected_pcts[2] - expected_pcts[1]);
> > +    migrate_set_parameter_int(from, "max-cpu-throttle", expected_pcts[3]);
> > +
> > +    migrate_set_parameter_int(from, "max-bandwidth", max_bandwidth);
> > +    migrate_set_parameter_int(from, "downtime-limit", downtime_limit);
> > +
> > +    /* To check remaining size after precopy */
> > +    migrate_set_capability(from, "pause-before-switchover", true);
> > +
> > +    /* Wait for the first serial output from the source */
> > +    wait_for_serial("src_serial");
> > +
> > +    migrate(from, uri, "{}");
> > +
> > +    /*
> > +     * Wait for pre-switchover status to check last throttle percentage
> > +     * and remaining. These values will be zeroed later
> > +     */
> > +    wait_for_migration_status(from, "pre-switchover");
> > +
> > +    /* We expect that migration can't converge without throttling */
> > +    percentage = read_migrate_property_int(from, "cpu-throttle-percentage");
> > +    g_assert_cmpint(percentage, >, 0);
> > +
> > +    remaining = read_ram_property_int(from, "remaining");
> > +    g_assert_cmpint(remaining, <, expected_threshold);
> > +
> > +    wait_command(from, "{ 'execute': 'migrate-continue' , 'arguments':"
> > +                       "  { 'state': 'pre-switchover' } }");
> > +
> > +    qtest_qmp_eventwait(to, "RESUME");
> > +
> > +    wait_for_serial("dest_serial");
> > +    wait_for_migration_complete(from);
> > +
> > +    g_free(uri);
> > +
> > +    test_migrate_end(from, to, true);
> > +}
> > +
> >  int main(int argc, char **argv)
> >  {
> >      char template[] = "/tmp/migration-test-XXXXXX";
> > @@ -1176,6 +1256,7 @@ int main(int argc, char **argv)
> >      /* qtest_add_func("/migration/ignore_shared", test_ignore_shared); */
> >      qtest_add_func("/migration/xbzrle/unix", test_xbzrle_unix);
> >      qtest_add_func("/migration/fd_proto", test_migrate_fd_proto);
> > +    qtest_add_func("/migration/auto_converge", test_migrate_auto_converge);
> >  
> >      ret = g_test_run();
> >  
> > -- 
> > 2.22.0
> > 
> --
> Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK

Re: [Qemu-devel] [PATCH v4 0/3] High downtime with 95+ throttle pct

[Yury Kotov]

Ping

23.07.2019, 16:42, "Yury Kotov" <yury-kotov@yandex-team.ru>:
> Hi,
>
> V4:
> * The test was simplified to prevent false fails.
>
> V3:
> * Rebase fixes (migrate_set_parameter -> migrate_set_parameter_int)
>
> V2:
> * Added a test
> * Fixed qemu_cond_timedwait for qsp
>
> I wrote a test for migration auto converge and found out a strange thing:
> 1. Enable auto converge
> 2. Set max-bandwidth 1Gb/s
> 3. Set downtime-limit 1ms
> 4. Run standard test (just writes a byte per page)
> 5. Wait for converge
> 6. It's converged with 99% throttle percentage
> 7. The result downtime was about 300-600ms <<<<
>
> It's much higher than expected 1ms. I figured out that cpu_throttle_thread()
> function sleeps for 100ms+ for high throttle percentage (>=95%) in VCPU thread.
> And it sleeps even after a cpu kick.
>
> Fixed it by using timedwait for ms part of sleep.
> E.g timedwait(halt_cond, 1ms) + usleep(500).
>
> Regards,
> Yury
>
> Yury Kotov (3):
>   qemu-thread: Add qemu_cond_timedwait
>   cpus: Fix throttling during vm_stop
>   tests/migration: Add a test for auto converge
>
>  cpus.c | 27 +++++++---
>  include/qemu/thread.h | 18 +++++++
>  tests/migration-test.c | 103 ++++++++++++++++++++++++++++++++++-----
>  util/qemu-thread-posix.c | 40 ++++++++++-----
>  util/qemu-thread-win32.c | 16 ++++++
>  util/qsp.c | 18 +++++++
>  6 files changed, 191 insertions(+), 31 deletions(-)
>
> --
> 2.22.0

Re: [Qemu-devel] [PATCH v4 0/3] High downtime with 95+ throttle pct

[Yury Kotov]

Ping ping

07.08.2019, 10:42, "Yury Kotov" <yury-kotov@yandex-team.ru>:
> Ping
>
> 23.07.2019, 16:42, "Yury Kotov" <yury-kotov@yandex-team.ru>:
>>  Hi,
>>
>>  V4:
>>  * The test was simplified to prevent false fails.
>>
>>  V3:
>>  * Rebase fixes (migrate_set_parameter -> migrate_set_parameter_int)
>>
>>  V2:
>>  * Added a test
>>  * Fixed qemu_cond_timedwait for qsp
>>
>>  I wrote a test for migration auto converge and found out a strange thing:
>>  1. Enable auto converge
>>  2. Set max-bandwidth 1Gb/s
>>  3. Set downtime-limit 1ms
>>  4. Run standard test (just writes a byte per page)
>>  5. Wait for converge
>>  6. It's converged with 99% throttle percentage
>>  7. The result downtime was about 300-600ms <<<<
>>
>>  It's much higher than expected 1ms. I figured out that cpu_throttle_thread()
>>  function sleeps for 100ms+ for high throttle percentage (>=95%) in VCPU thread.
>>  And it sleeps even after a cpu kick.
>>
>>  Fixed it by using timedwait for ms part of sleep.
>>  E.g timedwait(halt_cond, 1ms) + usleep(500).
>>
>>  Regards,
>>  Yury
>>
>>  Yury Kotov (3):
>>    qemu-thread: Add qemu_cond_timedwait
>>    cpus: Fix throttling during vm_stop
>>    tests/migration: Add a test for auto converge
>>
>>   cpus.c | 27 +++++++---
>>   include/qemu/thread.h | 18 +++++++
>>   tests/migration-test.c | 103 ++++++++++++++++++++++++++++++++++-----
>>   util/qemu-thread-posix.c | 40 ++++++++++-----
>>   util/qemu-thread-win32.c | 16 ++++++
>>   util/qsp.c | 18 +++++++
>>   6 files changed, 191 insertions(+), 31 deletions(-)
>>
>>  --
>>  2.22.0

Re: [Qemu-devel] [PATCH v4 0/3] High downtime with 95+ throttle pct

[Paolo Bonzini]

On 15/08/19 11:13, Yury Kotov wrote:
> Ping ping

Hi,

sorry for the delay, I was waiting for the 4.1 release.

I would like to make a small change so that preemption of QEMU does not
result in overly long sleeps.  The following patch on top of yours computes
the throttle-end time just once.  Of course you can still be unlucky if
you are preempted at the wrong time, but the window becomes much smaller.

diff --git a/cpus.c b/cpus.c
index d091dbd..d7e2145 100644
--- a/cpus.c
+++ b/cpus.c
@@ -781,7 +781,7 @@ static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
 {
     double pct;
     double throttle_ratio;
-    int64_t sleeptime_ns;
+    int64_t sleeptime_ns, end;
 
     if (!cpu_throttle_get_percentage()) {
         return;
@@ -792,18 +792,17 @@ static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
     /* Add 1ns to fix double's rounding error (like 0.9999999...) */
     sleeptime_ns = (int64_t)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS + 1);
 
-    while (sleeptime_ns >= SCALE_MS && !cpu->stop) {
-        int64_t start, end;
-        start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-        qemu_cond_timedwait(cpu->halt_cond, &qemu_global_mutex,
-                            sleeptime_ns / SCALE_MS);
-        end = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
-        sleeptime_ns -= end - start;
-    }
-    if (sleeptime_ns >= SCALE_US && !cpu->stop) {
-        qemu_mutex_unlock_iothread();
-        g_usleep(sleeptime_ns / SCALE_US);
-        qemu_mutex_lock_iothread();
+    end = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + sleeptime_ns;
+    while (sleeptime_ns > 0 && !cpu->stop) {
+        if (sleeptime_ns > SCALE_MS) {
+            qemu_cond_timedwait(cpu->halt_cond, &qemu_global_mutex,
+                                sleeptime_ns / SCALE_MS);
+        } else {
+            qemu_mutex_unlock_iothread();
+            g_usleep(sleeptime_ns / SCALE_US);
+            qemu_mutex_lock_iothread();
+        }
+        sleeptime_ns = end - qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
     }
     atomic_set(&cpu->throttle_thread_scheduled, 0);
 }

Re: [Qemu-devel] [PATCH v4 0/3] High downtime with 95+ throttle pct

[Paolo Bonzini]

On 19/08/19 18:39, Paolo Bonzini wrote:
> On 15/08/19 11:13, Yury Kotov wrote:
>> Ping ping
> 
> Hi,
> 
> sorry for the delay, I was waiting for the 4.1 release.
> 
> I would like to make a small change so that preemption of QEMU does not
> result in overly long sleeps.  The following patch on top of yours computes
> the throttle-end time just once.  Of course you can still be unlucky if
> you are preempted at the wrong time, but the window becomes much smaller.

The new unit test is hanging for me on aarch64-softmmu.

Paolo

> diff --git a/cpus.c b/cpus.c
> index d091dbd..d7e2145 100644
> --- a/cpus.c
> +++ b/cpus.c
> @@ -781,7 +781,7 @@ static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
>  {
>      double pct;
>      double throttle_ratio;
> -    int64_t sleeptime_ns;
> +    int64_t sleeptime_ns, end;
>  
>      if (!cpu_throttle_get_percentage()) {
>          return;
> @@ -792,18 +792,17 @@ static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
>      /* Add 1ns to fix double's rounding error (like 0.9999999...) */
>      sleeptime_ns = (int64_t)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS + 1);
>  
> -    while (sleeptime_ns >= SCALE_MS && !cpu->stop) {
> -        int64_t start, end;
> -        start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
> -        qemu_cond_timedwait(cpu->halt_cond, &qemu_global_mutex,
> -                            sleeptime_ns / SCALE_MS);
> -        end = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
> -        sleeptime_ns -= end - start;
> -    }
> -    if (sleeptime_ns >= SCALE_US && !cpu->stop) {
> -        qemu_mutex_unlock_iothread();
> -        g_usleep(sleeptime_ns / SCALE_US);
> -        qemu_mutex_lock_iothread();
> +    end = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + sleeptime_ns;
> +    while (sleeptime_ns > 0 && !cpu->stop) {
> +        if (sleeptime_ns > SCALE_MS) {
> +            qemu_cond_timedwait(cpu->halt_cond, &qemu_global_mutex,
> +                                sleeptime_ns / SCALE_MS);
> +        } else {
> +            qemu_mutex_unlock_iothread();
> +            g_usleep(sleeptime_ns / SCALE_US);
> +            qemu_mutex_lock_iothread();
> +        }
> +        sleeptime_ns = end - qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
>      }
>      atomic_set(&cpu->throttle_thread_scheduled, 0);
>  }
> 

Re: [Qemu-devel] [PATCH v4 0/3] High downtime with 95+ throttle pct

[Yury Kotov]

Hi,

19.08.2019, 20:11, "Paolo Bonzini" <pbonzini@redhat.com>:
> On 19/08/19 18:39, Paolo Bonzini wrote:
>>  On 15/08/19 11:13, Yury Kotov wrote:
>>>  Ping ping
>>
>>  Hi,
>>
>>  sorry for the delay, I was waiting for the 4.1 release.
>>
>>  I would like to make a small change so that preemption of QEMU does not
>>  result in overly long sleeps. The following patch on top of yours computes
>>  the throttle-end time just once. Of course you can still be unlucky if
>>  you are preempted at the wrong time, but the window becomes much smaller.

Thanks for your suggestion! I'll use it in the v5.

>
> The new unit test is hanging for me on aarch64-softmmu.
>

Ok, I'll try to fix it in v5.

> Paolo
>
>>  diff --git a/cpus.c b/cpus.c
>>  index d091dbd..d7e2145 100644
>>  --- a/cpus.c
>>  +++ b/cpus.c
>>  @@ -781,7 +781,7 @@ static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
>>   {
>>       double pct;
>>       double throttle_ratio;
>>  - int64_t sleeptime_ns;
>>  + int64_t sleeptime_ns, end;
>>
>>       if (!cpu_throttle_get_percentage()) {
>>           return;
>>  @@ -792,18 +792,17 @@ static void cpu_throttle_thread(CPUState *cpu, run_on_cpu_data opaque)
>>       /* Add 1ns to fix double's rounding error (like 0.9999999...) */
>>       sleeptime_ns = (int64_t)(throttle_ratio * CPU_THROTTLE_TIMESLICE_NS + 1);
>>
>>  - while (sleeptime_ns >= SCALE_MS && !cpu->stop) {
>>  - int64_t start, end;
>>  - start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
>>  - qemu_cond_timedwait(cpu->halt_cond, &qemu_global_mutex,
>>  - sleeptime_ns / SCALE_MS);
>>  - end = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
>>  - sleeptime_ns -= end - start;
>>  - }
>>  - if (sleeptime_ns >= SCALE_US && !cpu->stop) {
>>  - qemu_mutex_unlock_iothread();
>>  - g_usleep(sleeptime_ns / SCALE_US);
>>  - qemu_mutex_lock_iothread();
>>  + end = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + sleeptime_ns;
>>  + while (sleeptime_ns > 0 && !cpu->stop) {
>>  + if (sleeptime_ns > SCALE_MS) {
>>  + qemu_cond_timedwait(cpu->halt_cond, &qemu_global_mutex,
>>  + sleeptime_ns / SCALE_MS);
>>  + } else {
>>  + qemu_mutex_unlock_iothread();
>>  + g_usleep(sleeptime_ns / SCALE_US);
>>  + qemu_mutex_lock_iothread();
>>  + }
>>  + sleeptime_ns = end - qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
>>       }
>>       atomic_set(&cpu->throttle_thread_scheduled, 0);
>>   }

Regards,
Yury