From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-16.8 required=3.0 tests=BAYES_00, HEADER_FROM_DIFFERENT_DOMAINS,INCLUDES_CR_TRAILER,INCLUDES_PATCH, MAILING_LIST_MULTI,SPF_HELO_NONE,SPF_PASS,URIBL_BLOCKED,USER_AGENT_GIT autolearn=unavailable autolearn_force=no version=3.4.0 Received: from mail.kernel.org (mail.kernel.org [198.145.29.99]) by smtp.lore.kernel.org (Postfix) with ESMTP id 113B9C47080 for ; Mon, 31 May 2021 17:16:53 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id DF7DC610A8 for ; Mon, 31 May 2021 17:16:52 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S231971AbhEaRSb (ORCPT ); Mon, 31 May 2021 13:18:31 -0400 Received: from prt-mail.chinatelecom.cn ([42.123.76.219]:58424 "EHLO chinatelecom.cn" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S233794AbhEaRRY (ORCPT ); Mon, 31 May 2021 13:17:24 -0400 HMM_SOURCE_IP: 172.18.0.218:42706.1466219582 HMM_ATTACHE_NUM: 0000 HMM_SOURCE_TYPE: SMTP Received: from clientip-202.80.192.39?logid-8790cfef0ba147a299cbd8acf210f1a0 (unknown [172.18.0.218]) by chinatelecom.cn (HERMES) with SMTP id 5A2E42800B2; Tue, 1 Jun 2021 01:06:29 +0800 (CST) X-189-SAVE-TO-SEND: +huangy81@chinatelecom.cn Received: from ([172.18.0.218]) by app0025 with ESMTP id 8790cfef0ba147a299cbd8acf210f1a0 for qemu-devel@nongnu.org; Tue Jun 1 01:06:27 2021 X-Transaction-ID: 8790cfef0ba147a299cbd8acf210f1a0 X-filter-score: filter<0> X-Real-From: huangy81@chinatelecom.cn X-Receive-IP: 172.18.0.218 X-MEDUSA-Status: 0 Sender: huangy81@chinatelecom.cn From: huangy81@chinatelecom.cn To: Cc: Paolo Bonzini , , Juan Quintela , "Dr. David Alan Gilbert" , Peter Xu , Hyman Subject: [PATCH v1 6/6] migration/dirtyrate: implement dirty-ring dirtyrate calculation Date: Tue, 1 Jun 2021 01:06:29 +0800 Message-Id: <27607e12038273706273203b2146c5d4a40ac487.1622479162.git.huangy81@chinatelecom.cn> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: References: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org From: Hyman Huang(黄勇) use dirty ring feature to implement dirtyrate calculation. to enable it, set vcpu option as true in qmp calc-dirty-rate. Signed-off-by: Hyman Huang(黄勇) --- migration/dirtyrate.c | 146 ++++++++++++++++++++++++++++++++++++++--- migration/trace-events | 1 + 2 files changed, 139 insertions(+), 8 deletions(-) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index da6500c8ec..028c11d117 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -16,14 +16,22 @@ #include "cpu.h" #include "exec/ramblock.h" #include "qemu/rcu_queue.h" +#include "qemu/main-loop.h" #include "sysemu/kvm.h" #include "qapi/qapi-commands-migration.h" #include "ram.h" #include "trace.h" #include "dirtyrate.h" +typedef enum { + CALC_NONE = 0, + CALC_DIRTY_RING, + CALC_SAMPLE_PAGES, +} CalcMethod; + static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; static struct DirtyRateStat DirtyStat; +static CalcMethod last_method = CALC_NONE; static int64_t set_sample_page_period(int64_t msec, int64_t initial_time) { @@ -64,6 +72,7 @@ static struct DirtyRateInfo *query_dirty_rate_info(void) { int64_t dirty_rate = DirtyStat.dirty_rate; struct DirtyRateInfo *info = g_malloc0(sizeof(DirtyRateInfo)); + DirtyRateVcpuList *head = NULL, **tail = &head; if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) { info->has_dirty_rate = true; @@ -73,6 +82,22 @@ static struct DirtyRateInfo *query_dirty_rate_info(void) info->status = CalculatingState; info->start_time = DirtyStat.start_time; info->calc_time = DirtyStat.calc_time; + info->has_vcpu = true; + + if (last_method == CALC_DIRTY_RING) { + int i = 0; + info->vcpu = true; + info->has_vcpu_dirty_rate = true; + for (i = 0; i < DirtyStat.method.vcpu.nvcpu; i++) { + DirtyRateVcpu *rate = g_malloc0(sizeof(DirtyRateVcpu)); + rate->id = DirtyStat.method.vcpu.rates[i].id; + rate->dirty_rate = DirtyStat.method.vcpu.rates[i].dirty_rate; + QAPI_LIST_APPEND(tail, rate); + } + info->vcpu_dirty_rate = head; + } else { + info->vcpu = false; + } trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState)); @@ -87,13 +112,29 @@ static void init_dirtyrate_stat(int64_t start_time, DirtyStat.start_time = start_time; DirtyStat.calc_time = calc_time; - if (config.vcpu) { - DirtyStat.method.vcpu.nvcpu = -1; - DirtyStat.method.vcpu.rates = NULL; - } else { - DirtyStat.method.vm.total_dirty_samples = 0; - DirtyStat.method.vm.total_sample_count = 0; - DirtyStat.method.vm.total_block_mem_MB = 0; + switch (last_method) { + case CALC_NONE: + case CALC_SAMPLE_PAGES: + if (config.vcpu) { + DirtyStat.method.vcpu.nvcpu = -1; + DirtyStat.method.vcpu.rates = NULL; + } else { + DirtyStat.method.vm.total_dirty_samples = 0; + DirtyStat.method.vm.total_sample_count = 0; + DirtyStat.method.vm.total_block_mem_MB = 0; + } + break; + case CALC_DIRTY_RING: + if (!config.vcpu) { + g_free(DirtyStat.method.vcpu.rates); + DirtyStat.method.vcpu.rates = NULL; + DirtyStat.method.vm.total_dirty_samples = 0; + DirtyStat.method.vm.total_sample_count = 0; + DirtyStat.method.vm.total_block_mem_MB = 0; + } + break; + default: + break; } } @@ -331,7 +372,84 @@ static bool compare_page_hash_info(struct RamblockDirtyInfo *info, return true; } -static void calculate_dirtyrate(struct DirtyRateConfig config) +static void stat_dirtypages(CPUState *cpu, bool start) +{ + cpu->stat_dirty_pages = start; +} + +static void start_kvm_dirty_log(void) +{ + qemu_mutex_lock_iothread(); + memory_global_dirty_log_start(); + qemu_mutex_unlock_iothread(); +} + +static void stop_kvm_dirty_log(void) +{ + qemu_mutex_lock_iothread(); + memory_global_dirty_log_stop(); + qemu_mutex_unlock_iothread(); +} + +static int64_t do_calculate_dirtyrate_vcpu(CPUState *cpu) +{ + uint64_t memory_size_MB; + int64_t time_s; + + memory_size_MB = (cpu->dirty_pages * TARGET_PAGE_SIZE) >> 20; + time_s = DirtyStat.calc_time; + + return memory_size_MB / time_s; +} + +static void calculate_dirtyrate_vcpu(struct DirtyRateConfig config) +{ + CPUState *cpu; + int64_t msec = 0; + int64_t start_time; + uint64_t dirtyrate = 0; + uint64_t dirtyrate_sum = 0; + int nvcpu, i = 0; + + CPU_FOREACH(cpu) { + stat_dirtypages(cpu, true); + nvcpu++; + } + + DirtyStat.method.vcpu.nvcpu = nvcpu; + + if (last_method != CALC_DIRTY_RING) { + DirtyStat.method.vcpu.rates = + g_malloc0(sizeof(DirtyRateVcpu) * nvcpu); + } + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + DirtyStat.start_time = start_time / 1000; + + start_kvm_dirty_log(); + + msec = config.sample_period_seconds * 1000; + msec = set_sample_page_period(msec, start_time); + DirtyStat.calc_time = msec / 1000; + + CPU_FOREACH(cpu) { + stat_dirtypages(cpu, false); + } + + stop_kvm_dirty_log(); + + CPU_FOREACH(cpu) { + dirtyrate = do_calculate_dirtyrate_vcpu(cpu); + DirtyStat.method.vcpu.rates[i].id = cpu->cpu_index; + DirtyStat.method.vcpu.rates[i].dirty_rate = dirtyrate; + dirtyrate_sum += dirtyrate; + i++; + } + + DirtyStat.dirty_rate = dirtyrate_sum / nvcpu; +} + +static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config) { struct RamblockDirtyInfo *block_dinfo = NULL; int block_count = 0; @@ -364,6 +482,18 @@ out: rcu_unregister_thread(); } +static void calculate_dirtyrate(struct DirtyRateConfig config) +{ + if (config.vcpu) { + calculate_dirtyrate_vcpu(config); + last_method = CALC_DIRTY_RING; + } else { + calculate_dirtyrate_sample_vm(config); + last_method = CALC_SAMPLE_PAGES; + } + trace_calculate_dirtyrate(DirtyStat.dirty_rate); +} + void *get_dirtyrate_thread(void *arg) { struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg; diff --git a/migration/trace-events b/migration/trace-events index 668c562fed..5a80b39a62 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -330,6 +330,7 @@ get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t crc) "ramblock n calc_page_dirty_rate(const char *idstr, uint32_t new_crc, uint32_t old_crc) "ramblock name: %s, new crc: %" PRIu32 ", old crc: %" PRIu32 skip_sample_ramblock(const char *idstr, uint64_t ramblock_size) "ramblock name: %s, ramblock size: %" PRIu64 find_page_matched(const char *idstr) "ramblock %s addr or size changed" +calculate_dirtyrate(int64_t dirtyrate) "dirty rate: %" PRIi64 # block.c migration_block_init_shared(const char *blk_device_name) "Start migration for %s with shared base image" -- 2.24.3 From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-16.8 required=3.0 tests=BAYES_00, HEADER_FROM_DIFFERENT_DOMAINS,INCLUDES_CR_TRAILER,INCLUDES_PATCH, MAILING_LIST_MULTI,SPF_HELO_NONE,SPF_PASS,URIBL_BLOCKED,USER_AGENT_GIT autolearn=ham autolearn_force=no version=3.4.0 Received: from mail.kernel.org (mail.kernel.org [198.145.29.99]) by smtp.lore.kernel.org (Postfix) with ESMTP id 4A18DC47082 for ; Mon, 31 May 2021 17:14:51 +0000 (UTC) Received: from lists.gnu.org (lists.gnu.org [209.51.188.17]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 05BD9610A8 for ; Mon, 31 May 2021 17:14:50 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 05BD9610A8 Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=chinatelecom.cn Authentication-Results: mail.kernel.org; spf=pass smtp.mailfrom=qemu-devel-bounces+qemu-devel=archiver.kernel.org@nongnu.org Received: from localhost ([::1]:51226 helo=lists1p.gnu.org) by lists.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1lnlV4-0003lY-3W for qemu-devel@archiver.kernel.org; Mon, 31 May 2021 13:14:50 -0400 Received: from eggs.gnu.org ([2001:470:142:3::10]:55900) by lists.gnu.org with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.90_1) (envelope-from ) id 1lnlN5-0001bp-Js for qemu-devel@nongnu.org; Mon, 31 May 2021 13:06:36 -0400 Received: from prt-mail.chinatelecom.cn ([42.123.76.219]:51924 helo=chinatelecom.cn) by eggs.gnu.org with esmtp (Exim 4.90_1) (envelope-from ) id 1lnlN0-0005rU-22 for qemu-devel@nongnu.org; Mon, 31 May 2021 13:06:35 -0400 HMM_SOURCE_IP: 172.18.0.218:42706.1466219582 HMM_ATTACHE_NUM: 0000 HMM_SOURCE_TYPE: SMTP Received: from clientip-202.80.192.39?logid-8790cfef0ba147a299cbd8acf210f1a0 (unknown [172.18.0.218]) by chinatelecom.cn (HERMES) with SMTP id 5A2E42800B2; Tue, 1 Jun 2021 01:06:29 +0800 (CST) X-189-SAVE-TO-SEND: +huangy81@chinatelecom.cn Received: from ([172.18.0.218]) by app0025 with ESMTP id 8790cfef0ba147a299cbd8acf210f1a0 for qemu-devel@nongnu.org; Tue Jun 1 01:06:27 2021 X-Transaction-ID: 8790cfef0ba147a299cbd8acf210f1a0 X-filter-score: filter<0> X-Real-From: huangy81@chinatelecom.cn X-Receive-IP: 172.18.0.218 X-MEDUSA-Status: 0 From: huangy81@chinatelecom.cn To: Subject: [PATCH v1 6/6] migration/dirtyrate: implement dirty-ring dirtyrate calculation Date: Tue, 1 Jun 2021 01:06:29 +0800 Message-Id: <27607e12038273706273203b2146c5d4a40ac487.1622479162.git.huangy81@chinatelecom.cn> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: References: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Received-SPF: pass client-ip=42.123.76.219; envelope-from=huangy81@chinatelecom.cn; helo=chinatelecom.cn X-Spam_score_int: -18 X-Spam_score: -1.9 X-Spam_bar: - X-Spam_report: (-1.9 / 5.0 requ) BAYES_00=-1.9, SPF_HELO_PASS=-0.001, SPF_PASS=-0.001 autolearn=ham autolearn_force=no X-Spam_action: no action X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: kvm@vger.kernel.org, Juan Quintela , Hyman , "Dr. David Alan Gilbert" , Peter Xu , Paolo Bonzini Errors-To: qemu-devel-bounces+qemu-devel=archiver.kernel.org@nongnu.org Sender: "Qemu-devel" From: Hyman Huang(黄勇) use dirty ring feature to implement dirtyrate calculation. to enable it, set vcpu option as true in qmp calc-dirty-rate. Signed-off-by: Hyman Huang(黄勇) --- migration/dirtyrate.c | 146 ++++++++++++++++++++++++++++++++++++++--- migration/trace-events | 1 + 2 files changed, 139 insertions(+), 8 deletions(-) diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index da6500c8ec..028c11d117 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -16,14 +16,22 @@ #include "cpu.h" #include "exec/ramblock.h" #include "qemu/rcu_queue.h" +#include "qemu/main-loop.h" #include "sysemu/kvm.h" #include "qapi/qapi-commands-migration.h" #include "ram.h" #include "trace.h" #include "dirtyrate.h" +typedef enum { + CALC_NONE = 0, + CALC_DIRTY_RING, + CALC_SAMPLE_PAGES, +} CalcMethod; + static int CalculatingState = DIRTY_RATE_STATUS_UNSTARTED; static struct DirtyRateStat DirtyStat; +static CalcMethod last_method = CALC_NONE; static int64_t set_sample_page_period(int64_t msec, int64_t initial_time) { @@ -64,6 +72,7 @@ static struct DirtyRateInfo *query_dirty_rate_info(void) { int64_t dirty_rate = DirtyStat.dirty_rate; struct DirtyRateInfo *info = g_malloc0(sizeof(DirtyRateInfo)); + DirtyRateVcpuList *head = NULL, **tail = &head; if (qatomic_read(&CalculatingState) == DIRTY_RATE_STATUS_MEASURED) { info->has_dirty_rate = true; @@ -73,6 +82,22 @@ static struct DirtyRateInfo *query_dirty_rate_info(void) info->status = CalculatingState; info->start_time = DirtyStat.start_time; info->calc_time = DirtyStat.calc_time; + info->has_vcpu = true; + + if (last_method == CALC_DIRTY_RING) { + int i = 0; + info->vcpu = true; + info->has_vcpu_dirty_rate = true; + for (i = 0; i < DirtyStat.method.vcpu.nvcpu; i++) { + DirtyRateVcpu *rate = g_malloc0(sizeof(DirtyRateVcpu)); + rate->id = DirtyStat.method.vcpu.rates[i].id; + rate->dirty_rate = DirtyStat.method.vcpu.rates[i].dirty_rate; + QAPI_LIST_APPEND(tail, rate); + } + info->vcpu_dirty_rate = head; + } else { + info->vcpu = false; + } trace_query_dirty_rate_info(DirtyRateStatus_str(CalculatingState)); @@ -87,13 +112,29 @@ static void init_dirtyrate_stat(int64_t start_time, DirtyStat.start_time = start_time; DirtyStat.calc_time = calc_time; - if (config.vcpu) { - DirtyStat.method.vcpu.nvcpu = -1; - DirtyStat.method.vcpu.rates = NULL; - } else { - DirtyStat.method.vm.total_dirty_samples = 0; - DirtyStat.method.vm.total_sample_count = 0; - DirtyStat.method.vm.total_block_mem_MB = 0; + switch (last_method) { + case CALC_NONE: + case CALC_SAMPLE_PAGES: + if (config.vcpu) { + DirtyStat.method.vcpu.nvcpu = -1; + DirtyStat.method.vcpu.rates = NULL; + } else { + DirtyStat.method.vm.total_dirty_samples = 0; + DirtyStat.method.vm.total_sample_count = 0; + DirtyStat.method.vm.total_block_mem_MB = 0; + } + break; + case CALC_DIRTY_RING: + if (!config.vcpu) { + g_free(DirtyStat.method.vcpu.rates); + DirtyStat.method.vcpu.rates = NULL; + DirtyStat.method.vm.total_dirty_samples = 0; + DirtyStat.method.vm.total_sample_count = 0; + DirtyStat.method.vm.total_block_mem_MB = 0; + } + break; + default: + break; } } @@ -331,7 +372,84 @@ static bool compare_page_hash_info(struct RamblockDirtyInfo *info, return true; } -static void calculate_dirtyrate(struct DirtyRateConfig config) +static void stat_dirtypages(CPUState *cpu, bool start) +{ + cpu->stat_dirty_pages = start; +} + +static void start_kvm_dirty_log(void) +{ + qemu_mutex_lock_iothread(); + memory_global_dirty_log_start(); + qemu_mutex_unlock_iothread(); +} + +static void stop_kvm_dirty_log(void) +{ + qemu_mutex_lock_iothread(); + memory_global_dirty_log_stop(); + qemu_mutex_unlock_iothread(); +} + +static int64_t do_calculate_dirtyrate_vcpu(CPUState *cpu) +{ + uint64_t memory_size_MB; + int64_t time_s; + + memory_size_MB = (cpu->dirty_pages * TARGET_PAGE_SIZE) >> 20; + time_s = DirtyStat.calc_time; + + return memory_size_MB / time_s; +} + +static void calculate_dirtyrate_vcpu(struct DirtyRateConfig config) +{ + CPUState *cpu; + int64_t msec = 0; + int64_t start_time; + uint64_t dirtyrate = 0; + uint64_t dirtyrate_sum = 0; + int nvcpu, i = 0; + + CPU_FOREACH(cpu) { + stat_dirtypages(cpu, true); + nvcpu++; + } + + DirtyStat.method.vcpu.nvcpu = nvcpu; + + if (last_method != CALC_DIRTY_RING) { + DirtyStat.method.vcpu.rates = + g_malloc0(sizeof(DirtyRateVcpu) * nvcpu); + } + + start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + DirtyStat.start_time = start_time / 1000; + + start_kvm_dirty_log(); + + msec = config.sample_period_seconds * 1000; + msec = set_sample_page_period(msec, start_time); + DirtyStat.calc_time = msec / 1000; + + CPU_FOREACH(cpu) { + stat_dirtypages(cpu, false); + } + + stop_kvm_dirty_log(); + + CPU_FOREACH(cpu) { + dirtyrate = do_calculate_dirtyrate_vcpu(cpu); + DirtyStat.method.vcpu.rates[i].id = cpu->cpu_index; + DirtyStat.method.vcpu.rates[i].dirty_rate = dirtyrate; + dirtyrate_sum += dirtyrate; + i++; + } + + DirtyStat.dirty_rate = dirtyrate_sum / nvcpu; +} + +static void calculate_dirtyrate_sample_vm(struct DirtyRateConfig config) { struct RamblockDirtyInfo *block_dinfo = NULL; int block_count = 0; @@ -364,6 +482,18 @@ out: rcu_unregister_thread(); } +static void calculate_dirtyrate(struct DirtyRateConfig config) +{ + if (config.vcpu) { + calculate_dirtyrate_vcpu(config); + last_method = CALC_DIRTY_RING; + } else { + calculate_dirtyrate_sample_vm(config); + last_method = CALC_SAMPLE_PAGES; + } + trace_calculate_dirtyrate(DirtyStat.dirty_rate); +} + void *get_dirtyrate_thread(void *arg) { struct DirtyRateConfig config = *(struct DirtyRateConfig *)arg; diff --git a/migration/trace-events b/migration/trace-events index 668c562fed..5a80b39a62 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -330,6 +330,7 @@ get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t crc) "ramblock n calc_page_dirty_rate(const char *idstr, uint32_t new_crc, uint32_t old_crc) "ramblock name: %s, new crc: %" PRIu32 ", old crc: %" PRIu32 skip_sample_ramblock(const char *idstr, uint64_t ramblock_size) "ramblock name: %s, ramblock size: %" PRIu64 find_page_matched(const char *idstr) "ramblock %s addr or size changed" +calculate_dirtyrate(int64_t dirtyrate) "dirty rate: %" PRIi64 # block.c migration_block_init_shared(const char *blk_device_name) "Start migration for %s with shared base image" -- 2.24.3