[Qemu-devel] [PATCH 1/9] throttle: Extract timers from ThrottleState into a separate ThrottleTimers structure

[Alberto Garcia <berto@igalia.com>]

From: Benoît Canet <benoit.canet@nodalink.com>

Group throttling will share ThrottleState between multiple bs.
As a consequence the ThrottleState will be accessed by multiple aio context.

Timers are tied to their aio context so they must go out of the ThrottleState structure.

This commit paves the way for each bs of a common ThrottleState to have its own
timer.

Signed-off-by: Benoit Canet <benoit.canet@nodalink.com>
Signed-off-by: Alberto Garcia <berto@igalia.com>
---
 block.c                   | 35 ++++++++++++--------
 include/block/block_int.h |  1 +
 include/qemu/throttle.h   | 38 ++++++++++++++--------
 tests/test-throttle.c     | 82 ++++++++++++++++++++++++++---------------------
 util/throttle.c           | 73 ++++++++++++++++++++++++-----------------
 5 files changed, 135 insertions(+), 94 deletions(-)

diff --git a/block.c b/block.c
index 210fd5f..b7469a7 100644
--- a/block.c
+++ b/block.c
@@ -130,7 +130,7 @@ void bdrv_set_io_limits(BlockDriverState *bs,
 {
     int i;
 
-    throttle_config(&bs->throttle_state, cfg);
+    throttle_config(&bs->throttle_state, &bs->throttle_timers, cfg);
 
     for (i = 0; i < 2; i++) {
         qemu_co_enter_next(&bs->throttled_reqs[i]);
@@ -163,7 +163,7 @@ void bdrv_io_limits_disable(BlockDriverState *bs)
 
     bdrv_start_throttled_reqs(bs);
 
-    throttle_destroy(&bs->throttle_state);
+    throttle_timers_destroy(&bs->throttle_timers);
 }
 
 static void bdrv_throttle_read_timer_cb(void *opaque)
@@ -182,12 +182,13 @@ static void bdrv_throttle_write_timer_cb(void *opaque)
 void bdrv_io_limits_enable(BlockDriverState *bs)
 {
     assert(!bs->io_limits_enabled);
-    throttle_init(&bs->throttle_state,
-                  bdrv_get_aio_context(bs),
-                  QEMU_CLOCK_VIRTUAL,
-                  bdrv_throttle_read_timer_cb,
-                  bdrv_throttle_write_timer_cb,
-                  bs);
+    throttle_init(&bs->throttle_state);
+    throttle_timers_init(&bs->throttle_timers,
+                         bdrv_get_aio_context(bs),
+                         QEMU_CLOCK_VIRTUAL,
+                         bdrv_throttle_read_timer_cb,
+                         bdrv_throttle_write_timer_cb,
+                         bs);
     bs->io_limits_enabled = true;
 }
 
@@ -201,7 +202,9 @@ static void bdrv_io_limits_intercept(BlockDriverState *bs,
                                      bool is_write)
 {
     /* does this io must wait */
-    bool must_wait = throttle_schedule_timer(&bs->throttle_state, is_write);
+    bool must_wait = throttle_schedule_timer(&bs->throttle_state,
+                                             &bs->throttle_timers,
+                                             is_write);
 
     /* if must wait or any request of this type throttled queue the IO */
     if (must_wait ||
@@ -214,7 +217,8 @@ static void bdrv_io_limits_intercept(BlockDriverState *bs,
 
 
     /* if the next request must wait -> do nothing */
-    if (throttle_schedule_timer(&bs->throttle_state, is_write)) {
+    if (throttle_schedule_timer(&bs->throttle_state, &bs->throttle_timers,
+                                is_write)) {
         return;
     }
 
@@ -2050,6 +2054,9 @@ static void bdrv_move_feature_fields(BlockDriverState *bs_dest,
     memcpy(&bs_dest->throttle_state,
            &bs_src->throttle_state,
            sizeof(ThrottleState));
+    memcpy(&bs_dest->throttle_timers,
+           &bs_src->throttle_timers,
+           sizeof(ThrottleTimers));
     bs_dest->throttled_reqs[0]  = bs_src->throttled_reqs[0];
     bs_dest->throttled_reqs[1]  = bs_src->throttled_reqs[1];
     bs_dest->io_limits_enabled  = bs_src->io_limits_enabled;
@@ -2111,7 +2118,7 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
     assert(QLIST_EMPTY(&bs_new->dirty_bitmaps));
     assert(bs_new->job == NULL);
     assert(bs_new->io_limits_enabled == false);
-    assert(!throttle_have_timer(&bs_new->throttle_state));
+    assert(!throttle_timers_are_initialized(&bs_new->throttle_timers));
 
     tmp = *bs_new;
     *bs_new = *bs_old;
@@ -2128,7 +2135,7 @@ void bdrv_swap(BlockDriverState *bs_new, BlockDriverState *bs_old)
     /* Check a few fields that should remain attached to the device */
     assert(bs_new->job == NULL);
     assert(bs_new->io_limits_enabled == false);
-    assert(!throttle_have_timer(&bs_new->throttle_state));
+    assert(!throttle_timers_are_initialized(&bs_new->throttle_timers));
 
     /* insert the nodes back into the graph node list if needed */
     if (bs_new->node_name[0] != '\0') {
@@ -5785,7 +5792,7 @@ void bdrv_detach_aio_context(BlockDriverState *bs)
     }
 
     if (bs->io_limits_enabled) {
-        throttle_detach_aio_context(&bs->throttle_state);
+        throttle_timers_detach_aio_context(&bs->throttle_timers);
     }
     if (bs->drv->bdrv_detach_aio_context) {
         bs->drv->bdrv_detach_aio_context(bs);
@@ -5821,7 +5828,7 @@ void bdrv_attach_aio_context(BlockDriverState *bs,
         bs->drv->bdrv_attach_aio_context(bs, new_context);
     }
     if (bs->io_limits_enabled) {
-        throttle_attach_aio_context(&bs->throttle_state, new_context);
+        throttle_timers_attach_aio_context(&bs->throttle_timers, new_context);
     }
 
     QLIST_FOREACH(ban, &bs->aio_notifiers, list) {
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 7ad1950..c98acb6 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -360,6 +360,7 @@ struct BlockDriverState {
 
     /* I/O throttling */
     ThrottleState throttle_state;
+    ThrottleTimers throttle_timers;
     CoQueue      throttled_reqs[2];
     bool         io_limits_enabled;
 
diff --git a/include/qemu/throttle.h b/include/qemu/throttle.h
index b890613..2c560db 100644
--- a/include/qemu/throttle.h
+++ b/include/qemu/throttle.h
@@ -65,14 +65,17 @@ typedef struct ThrottleConfig {
 typedef struct ThrottleState {
     ThrottleConfig cfg;       /* configuration */
     int64_t previous_leak;    /* timestamp of the last leak done */
-    QEMUTimer * timers[2];    /* timers used to do the throttling */
+} ThrottleState;
+
+typedef struct ThrottleTimers {
+    QEMUTimer *timers[2];     /* timers used to do the throttling */
     QEMUClockType clock_type; /* the clock used */
 
     /* Callbacks */
     QEMUTimerCB *read_timer_cb;
     QEMUTimerCB *write_timer_cb;
     void *timer_opaque;
-} ThrottleState;
+} ThrottleTimers;
 
 /* operations on single leaky buckets */
 void throttle_leak_bucket(LeakyBucket *bkt, int64_t delta);
@@ -86,20 +89,23 @@ bool throttle_compute_timer(ThrottleState *ts,
                             int64_t *next_timestamp);
 
 /* init/destroy cycle */
-void throttle_init(ThrottleState *ts,
-                   AioContext *aio_context,
-                   QEMUClockType clock_type,
-                   void (read_timer)(void *),
-                   void (write_timer)(void *),
-                   void *timer_opaque);
+void throttle_init(ThrottleState *ts);
+
+void throttle_timers_init(ThrottleTimers *tt,
+                          AioContext *aio_context,
+                          QEMUClockType clock_type,
+                          QEMUTimerCB *read_timer_cb,
+                          QEMUTimerCB *write_timer_cb,
+                          void *timer_opaque);
 
-void throttle_destroy(ThrottleState *ts);
+void throttle_timers_destroy(ThrottleTimers *tt);
 
-void throttle_detach_aio_context(ThrottleState *ts);
+void throttle_timers_detach_aio_context(ThrottleTimers *tt);
 
-void throttle_attach_aio_context(ThrottleState *ts, AioContext *new_context);
+void throttle_timers_attach_aio_context(ThrottleTimers *tt,
+                                        AioContext *new_context);
 
-bool throttle_have_timer(ThrottleState *ts);
+bool throttle_timers_are_initialized(ThrottleTimers *tt);
 
 /* configuration */
 bool throttle_enabled(ThrottleConfig *cfg);
@@ -108,12 +114,16 @@ bool throttle_conflicting(ThrottleConfig *cfg);
 
 bool throttle_is_valid(ThrottleConfig *cfg);
 
-void throttle_config(ThrottleState *ts, ThrottleConfig *cfg);
+void throttle_config(ThrottleState *ts,
+                     ThrottleTimers *tt,
+                     ThrottleConfig *cfg);
 
 void throttle_get_config(ThrottleState *ts, ThrottleConfig *cfg);
 
 /* usage */
-bool throttle_schedule_timer(ThrottleState *ts, bool is_write);
+bool throttle_schedule_timer(ThrottleState *ts,
+                             ThrottleTimers *tt,
+                             bool is_write);
 
 void throttle_account(ThrottleState *ts, bool is_write, uint64_t size);
 
diff --git a/tests/test-throttle.c b/tests/test-throttle.c
index d8ba415..458f577 100644
--- a/tests/test-throttle.c
+++ b/tests/test-throttle.c
@@ -20,6 +20,7 @@ static AioContext     *ctx;
 static LeakyBucket    bkt;
 static ThrottleConfig cfg;
 static ThrottleState  ts;
+static ThrottleTimers tt;
 
 /* useful function */
 static bool double_cmp(double x, double y)
@@ -103,17 +104,19 @@ static void test_init(void)
 {
     int i;
 
-    /* fill the structure with crap */
+    /* fill the structures with crap */
     memset(&ts, 1, sizeof(ts));
+    memset(&tt, 1, sizeof(tt));
 
-    /* init the structure */
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
+    /* init structures */
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
 
     /* check initialized fields */
-    g_assert(ts.clock_type == QEMU_CLOCK_VIRTUAL);
-    g_assert(ts.timers[0]);
-    g_assert(ts.timers[1]);
+    g_assert(tt.clock_type == QEMU_CLOCK_VIRTUAL);
+    g_assert(tt.timers[0]);
+    g_assert(tt.timers[1]);
 
     /* check other fields where cleared */
     g_assert(!ts.previous_leak);
@@ -124,17 +127,18 @@ static void test_init(void)
         g_assert(!ts.cfg.buckets[i].level);
     }
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 }
 
 static void test_destroy(void)
 {
     int i;
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
-    throttle_destroy(&ts);
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
+    throttle_timers_destroy(&tt);
     for (i = 0; i < 2; i++) {
-        g_assert(!ts.timers[i]);
+        g_assert(!tt.timers[i]);
     }
 }
 
@@ -170,11 +174,12 @@ static void test_config_functions(void)
 
     orig_cfg.op_size = 1;
 
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
     /* structure reset by throttle_init previous_leak should be null */
     g_assert(!ts.previous_leak);
-    throttle_config(&ts, &orig_cfg);
+    throttle_config(&ts, &tt, &orig_cfg);
 
     /* has previous leak been initialized by throttle_config ? */
     g_assert(ts.previous_leak);
@@ -182,7 +187,7 @@ static void test_config_functions(void)
     /* get back the fixed configuration */
     throttle_get_config(&ts, &final_cfg);
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 
     g_assert(final_cfg.buckets[THROTTLE_BPS_TOTAL].avg == 153);
     g_assert(final_cfg.buckets[THROTTLE_BPS_READ].avg  == 56);
@@ -323,43 +328,47 @@ static void test_is_valid(void)
 
 static void test_have_timer(void)
 {
-    /* zero the structure */
+    /* zero structures */
     memset(&ts, 0, sizeof(ts));
+    memset(&tt, 0, sizeof(tt));
 
     /* no timer set should return false */
-    g_assert(!throttle_have_timer(&ts));
+    g_assert(!throttle_timers_are_initialized(&tt));
 
-    /* init the structure */
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
+    /* init structures */
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
 
     /* timer set by init should return true */
-    g_assert(throttle_have_timer(&ts));
+    g_assert(throttle_timers_are_initialized(&tt));
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 }
 
 static void test_detach_attach(void)
 {
-    /* zero the structure */
+    /* zero structures */
     memset(&ts, 0, sizeof(ts));
+    memset(&tt, 0, sizeof(tt));
 
     /* init the structure */
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
 
     /* timer set by init should return true */
-    g_assert(throttle_have_timer(&ts));
+    g_assert(throttle_timers_are_initialized(&tt));
 
     /* timer should no longer exist after detaching */
-    throttle_detach_aio_context(&ts);
-    g_assert(!throttle_have_timer(&ts));
+    throttle_timers_detach_aio_context(&tt);
+    g_assert(!throttle_timers_are_initialized(&tt));
 
     /* timer should exist again after attaching */
-    throttle_attach_aio_context(&ts, ctx);
-    g_assert(throttle_have_timer(&ts));
+    throttle_timers_attach_aio_context(&tt, ctx);
+    g_assert(throttle_timers_are_initialized(&tt));
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 }
 
 static bool do_test_accounting(bool is_ops, /* are we testing bps or ops */
@@ -387,9 +396,10 @@ static bool do_test_accounting(bool is_ops, /* are we testing bps or ops */
 
     cfg.op_size = op_size;
 
-    throttle_init(&ts, ctx, QEMU_CLOCK_VIRTUAL,
-                  read_timer_cb, write_timer_cb, &ts);
-    throttle_config(&ts, &cfg);
+    throttle_init(&ts);
+    throttle_timers_init(&tt, ctx, QEMU_CLOCK_VIRTUAL,
+                         read_timer_cb, write_timer_cb, &ts);
+    throttle_config(&ts, &tt, &cfg);
 
     /* account a read */
     throttle_account(&ts, false, size);
@@ -414,7 +424,7 @@ static bool do_test_accounting(bool is_ops, /* are we testing bps or ops */
         return false;
     }
 
-    throttle_destroy(&ts);
+    throttle_timers_destroy(&tt);
 
     return true;
 }
diff --git a/util/throttle.c b/util/throttle.c
index f976ac7..d76a48e 100644
--- a/util/throttle.c
+++ b/util/throttle.c
@@ -159,29 +159,36 @@ bool throttle_compute_timer(ThrottleState *ts,
 }
 
 /* Add timers to event loop */
-void throttle_attach_aio_context(ThrottleState *ts, AioContext *new_context)
+void throttle_timers_attach_aio_context(ThrottleTimers *tt,
+                                        AioContext *new_context)
 {
-    ts->timers[0] = aio_timer_new(new_context, ts->clock_type, SCALE_NS,
-                                  ts->read_timer_cb, ts->timer_opaque);
-    ts->timers[1] = aio_timer_new(new_context, ts->clock_type, SCALE_NS,
-                                  ts->write_timer_cb, ts->timer_opaque);
+    tt->timers[0] = aio_timer_new(new_context, tt->clock_type, SCALE_NS,
+                                  tt->read_timer_cb, tt->timer_opaque);
+    tt->timers[1] = aio_timer_new(new_context, tt->clock_type, SCALE_NS,
+                                  tt->write_timer_cb, tt->timer_opaque);
 }
 
 /* To be called first on the ThrottleState */
-void throttle_init(ThrottleState *ts,
-                   AioContext *aio_context,
-                   QEMUClockType clock_type,
-                   QEMUTimerCB *read_timer_cb,
-                   QEMUTimerCB *write_timer_cb,
-                   void *timer_opaque)
+void throttle_init(ThrottleState *ts)
 {
     memset(ts, 0, sizeof(ThrottleState));
+}
+
+/* To be called first on the ThrottleTimers */
+void throttle_timers_init(ThrottleTimers *tt,
+                          AioContext *aio_context,
+                          QEMUClockType clock_type,
+                          QEMUTimerCB *read_timer_cb,
+                          QEMUTimerCB *write_timer_cb,
+                          void *timer_opaque)
+{
+    memset(tt, 0, sizeof(ThrottleTimers));
 
-    ts->clock_type = clock_type;
-    ts->read_timer_cb = read_timer_cb;
-    ts->write_timer_cb = write_timer_cb;
-    ts->timer_opaque = timer_opaque;
-    throttle_attach_aio_context(ts, aio_context);
+    tt->clock_type = clock_type;
+    tt->read_timer_cb = read_timer_cb;
+    tt->write_timer_cb = write_timer_cb;
+    tt->timer_opaque = timer_opaque;
+    throttle_timers_attach_aio_context(tt, aio_context);
 }
 
 /* destroy a timer */
@@ -195,25 +202,25 @@ static void throttle_timer_destroy(QEMUTimer **timer)
 }
 
 /* Remove timers from event loop */
-void throttle_detach_aio_context(ThrottleState *ts)
+void throttle_timers_detach_aio_context(ThrottleTimers *tt)
 {
     int i;
 
     for (i = 0; i < 2; i++) {
-        throttle_timer_destroy(&ts->timers[i]);
+        throttle_timer_destroy(&tt->timers[i]);
     }
 }
 
-/* To be called last on the ThrottleState */
-void throttle_destroy(ThrottleState *ts)
+/* To be called last on the ThrottleTimers */
+void throttle_timers_destroy(ThrottleTimers *tt)
 {
-    throttle_detach_aio_context(ts);
+    throttle_timers_detach_aio_context(tt);
 }
 
 /* is any throttling timer configured */
-bool throttle_have_timer(ThrottleState *ts)
+bool throttle_timers_are_initialized(ThrottleTimers *tt)
 {
-    if (ts->timers[0]) {
+    if (tt->timers[0]) {
         return true;
     }
 
@@ -324,9 +331,12 @@ static void throttle_cancel_timer(QEMUTimer *timer)
 /* Used to configure the throttle
  *
  * @ts: the throttle state we are working on
+ * @tt: the throttle timers we use in this aio context
  * @cfg: the config to set
  */
-void throttle_config(ThrottleState *ts, ThrottleConfig *cfg)
+void throttle_config(ThrottleState *ts,
+                     ThrottleTimers *tt,
+                     ThrottleConfig *cfg)
 {
     int i;
 
@@ -336,10 +346,10 @@ void throttle_config(ThrottleState *ts, ThrottleConfig *cfg)
         throttle_fix_bucket(&ts->cfg.buckets[i]);
     }
 
-    ts->previous_leak = qemu_clock_get_ns(ts->clock_type);
+    ts->previous_leak = qemu_clock_get_ns(tt->clock_type);
 
     for (i = 0; i < 2; i++) {
-        throttle_cancel_timer(ts->timers[i]);
+        throttle_cancel_timer(tt->timers[i]);
     }
 }
 
@@ -358,12 +368,15 @@ void throttle_get_config(ThrottleState *ts, ThrottleConfig *cfg)
  *
  * NOTE: this function is not unit tested due to it's usage of timer_mod
  *
+ * @tt:       the timers structure
  * @is_write: the type of operation (read/write)
  * @ret:      true if the timer has been scheduled else false
  */
-bool throttle_schedule_timer(ThrottleState *ts, bool is_write)
+bool throttle_schedule_timer(ThrottleState *ts,
+                             ThrottleTimers *tt,
+                             bool is_write)
 {
-    int64_t now = qemu_clock_get_ns(ts->clock_type);
+    int64_t now = qemu_clock_get_ns(tt->clock_type);
     int64_t next_timestamp;
     bool must_wait;
 
@@ -378,12 +391,12 @@ bool throttle_schedule_timer(ThrottleState *ts, bool is_write)
     }
 
     /* request throttled and timer pending -> do nothing */
-    if (timer_pending(ts->timers[is_write])) {
+    if (timer_pending(tt->timers[is_write])) {
         return true;
     }
 
     /* request throttled and timer not pending -> arm timer */
-    timer_mod(ts->timers[is_write], next_timestamp);
+    timer_mod(tt->timers[is_write], next_timestamp);
     return true;
 }
 
-- 
2.1.4