[PATCH v2 3/4] kvm: Introduce a dirty rate calculation method based on dirty ring

[wucy11@chinatelecom.cn]

From: Chongyun Wu <wucy11@chinatelecom.cn>

A new structure KVMDirtyRingDirtyCounter is introduced in
KVMDirtyRingReaper to record the number of dirty pages
within a period of time.

When kvm_dirty_ring_mark_page collects dirty pages, if it
finds that the current dirty pages are not duplicates, it
increases the dirty_pages_period count.

Divide the dirty_pages_period count by the interval to get
the dirty page rate for this period.

And use dirty_pages_period_peak_rate to count the highest
dirty page rate, to solve the problem that the dirty page
collection rate may change greatly during a period of time,
resulting in a large change in the dirty page rate.

Through sufficient testing, it is found that the dirty rate
calculated after kvm_dirty_ring_flush usually matches the actual
pressure, and the dirty rate counted per second may change in the
subsequent seconds, so record the peak dirty rate as the real
dirty pages rate.

This dirty pages rate is mainly used as the subsequent autoconverge
calculation speed limit throttle.

Signed-off-by: Chongyun Wu <wucy11@chinatelecom.cn>
---
 accel/kvm/kvm-all.c  | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 include/sysemu/kvm.h |  2 ++
 2 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c
index 5e02700..a158b1c 100644
--- a/accel/kvm/kvm-all.c
+++ b/accel/kvm/kvm-all.c
@@ -114,6 +114,13 @@ enum KVMDirtyRingReaperSpeedControl {
     KVM_DIRTY_RING_REAPER_SPEED_CONTROL_DOWN
 };
 
+struct KVMDirtyRingDirtyCounter {
+    int64_t time_last_count;
+    uint64_t dirty_pages_period;
+    int64_t dirty_pages_rate;
+    int64_t dirty_pages_period_peak_rate;
+};
+
 /*
  * KVM reaper instance, responsible for collecting the KVM dirty bits
  * via the dirty ring.
@@ -128,6 +135,7 @@ struct KVMDirtyRingReaper {
     uint64_t ring_full_cnt;
     float ratio_adjust_threshold;
     int stable_count_threshold;
+    struct KVMDirtyRingDirtyCounter counter; /* Calculate dirty pages rate */
 };
 
 struct KVMState
@@ -739,7 +747,9 @@ static void kvm_dirty_ring_mark_page(KVMState *s, uint32_t as_id,
         return;
     }
 
-    set_bit(offset, mem->dirty_bmap);
+    if (!test_and_set_bit(offset, mem->dirty_bmap)) {
+        s->reaper.counter.dirty_pages_period++;
+    }
 }
 
 static bool dirty_gfn_is_dirtied(struct kvm_dirty_gfn *gfn)
@@ -783,6 +793,56 @@ static uint32_t kvm_dirty_ring_reap_one(KVMState *s, CPUState *cpu)
     return count;
 }
 
+int64_t kvm_dirty_ring_get_rate(void)
+{
+    return kvm_state->reaper.counter.dirty_pages_rate;
+}
+
+int64_t kvm_dirty_ring_get_peak_rate(void)
+{
+    return kvm_state->reaper.counter.dirty_pages_period_peak_rate;
+}
+
+static void kvm_dirty_ring_reap_count(KVMState *s)
+{
+    int64_t spend_time = 0;
+    int64_t end_time;
+
+    if (!s->reaper.counter.time_last_count) {
+        s->reaper.counter.time_last_count =
+            qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+    }
+
+    end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME);
+    spend_time = end_time - s->reaper.counter.time_last_count;
+
+    if (!s->reaper.counter.dirty_pages_period ||
+        !spend_time) {
+        return;
+    }
+
+    /*
+     * More than 1 second = 1000 millisecons,
+     * or trigger by kvm_log_sync_global which spend time
+     * more than 300 milliscons.
+     */
+    if (spend_time > 1000) {
+        /* Count the dirty page rate during this period */
+        s->reaper.counter.dirty_pages_rate =
+            s->reaper.counter.dirty_pages_period * 1000 / spend_time;
+        /* Update the peak dirty page rate at this period */
+        if (s->reaper.counter.dirty_pages_rate >
+            s->reaper.counter.dirty_pages_period_peak_rate) {
+            s->reaper.counter.dirty_pages_period_peak_rate =
+                s->reaper.counter.dirty_pages_rate;
+        }
+
+        /* Reset counters */
+        s->reaper.counter.dirty_pages_period = 0;
+        s->reaper.counter.time_last_count = 0;
+    }
+}
+
 /* Must be with slots_lock held */
 static uint64_t kvm_dirty_ring_reap_locked(KVMState *s)
 {
@@ -793,6 +853,8 @@ static uint64_t kvm_dirty_ring_reap_locked(KVMState *s)
 
     stamp = get_clock();
 
+    kvm_dirty_ring_reap_count(s);
+
     CPU_FOREACH(cpu) {
         total += kvm_dirty_ring_reap_one(s, cpu);
     }
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index a783c78..05846f9 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -582,4 +582,6 @@ bool kvm_cpu_check_are_resettable(void);
 bool kvm_arch_cpu_check_are_resettable(void);
 
 bool kvm_dirty_ring_enabled(void);
+int64_t kvm_dirty_ring_get_rate(void);
+int64_t kvm_dirty_ring_get_peak_rate(void);
 #endif
-- 
1.8.3.1