[PATCH 0/5] Support RISC-V migration

[PATCH 0/5] Support RISC-V migration

[Yifei Jiang]

This patches supported RISC-V migration based on tcg accel. And we have
verified related migration features such as snapshot and live migration.

A few weeks ago, we submitted RFC patches about supporting RISC-V migration
based on kvm accel: https://www.spinics.net/lists/kvm/msg223605.html.
And we found that tcg accelerated migration can be supported with a few
changes. Most of the devices have already implemented the migration
interface, so, to achieve the tcg accelerated migration, we just need to
add vmstate of both cpu and sifive_plic.

Yifei Jiang (5):
  target/riscv: Add basic vmstate description of CPU
  target/riscv: Add PMP state description
  target/riscv: Add H extention state description
  target/riscv: Add V extention state description
  target/riscv: Add sifive_plic vmstate

 hw/intc/sifive_plic.c    |  26 +++++-
 hw/intc/sifive_plic.h    |   1 +
 target/riscv/cpu.c       |   7 --
 target/riscv/cpu.h       |   4 +
 target/riscv/machine.c   | 184 +++++++++++++++++++++++++++++++++++++++
 target/riscv/meson.build |   3 +-
 6 files changed, 214 insertions(+), 9 deletions(-)
 create mode 100644 target/riscv/machine.c

-- 
2.19.1

[PATCH 0/5] Support RISC-V migration

[Yifei Jiang]

This patches supported RISC-V migration based on tcg accel. And we have
verified related migration features such as snapshot and live migration.

A few weeks ago, we submitted RFC patches about supporting RISC-V migration
based on kvm accel: https://www.spinics.net/lists/kvm/msg223605.html.
And we found that tcg accelerated migration can be supported with a few
changes. Most of the devices have already implemented the migration
interface, so, to achieve the tcg accelerated migration, we just need to
add vmstate of both cpu and sifive_plic.

Yifei Jiang (5):
  target/riscv: Add basic vmstate description of CPU
  target/riscv: Add PMP state description
  target/riscv: Add H extention state description
  target/riscv: Add V extention state description
  target/riscv: Add sifive_plic vmstate

 hw/intc/sifive_plic.c    |  26 +++++-
 hw/intc/sifive_plic.h    |   1 +
 target/riscv/cpu.c       |   7 --
 target/riscv/cpu.h       |   4 +
 target/riscv/machine.c   | 184 +++++++++++++++++++++++++++++++++++++++
 target/riscv/meson.build |   3 +-
 6 files changed, 214 insertions(+), 9 deletions(-)
 create mode 100644 target/riscv/machine.c

-- 
2.19.1

[PATCH 1/5] target/riscv: Add basic vmstate description of CPU

[Yifei Jiang]

Add basic CPU state description to the newly created machine.c

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 target/riscv/cpu.c       |  7 -----
 target/riscv/cpu.h       |  4 +++
 target/riscv/machine.c   | 59 ++++++++++++++++++++++++++++++++++++++++
 target/riscv/meson.build |  3 +-
 4 files changed, 65 insertions(+), 8 deletions(-)
 create mode 100644 target/riscv/machine.c

diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c
index 0bbfd7f457..bf396e2916 100644
--- a/target/riscv/cpu.c
+++ b/target/riscv/cpu.c
@@ -496,13 +496,6 @@ static void riscv_cpu_init(Object *obj)
     cpu_set_cpustate_pointers(cpu);
 }
 
-#ifndef CONFIG_USER_ONLY
-static const VMStateDescription vmstate_riscv_cpu = {
-    .name = "cpu",
-    .unmigratable = 1,
-};
-#endif
-
 static Property riscv_cpu_properties[] = {
     DEFINE_PROP_BOOL("i", RISCVCPU, cfg.ext_i, true),
     DEFINE_PROP_BOOL("e", RISCVCPU, cfg.ext_e, false),
diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h
index de275782e6..8440ea0793 100644
--- a/target/riscv/cpu.h
+++ b/target/riscv/cpu.h
@@ -311,6 +311,10 @@ extern const char * const riscv_fpr_regnames[];
 extern const char * const riscv_excp_names[];
 extern const char * const riscv_intr_names[];
 
+#ifndef CONFIG_USER_ONLY
+extern const VMStateDescription vmstate_riscv_cpu;
+#endif
+
 const char *riscv_cpu_get_trap_name(target_ulong cause, bool async);
 void riscv_cpu_do_interrupt(CPUState *cpu);
 int riscv_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg);
diff --git a/target/riscv/machine.c b/target/riscv/machine.c
new file mode 100644
index 0000000000..3451b888b1
--- /dev/null
+++ b/target/riscv/machine.c
@@ -0,0 +1,59 @@
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "migration/cpu.h"
+
+const VMStateDescription vmstate_riscv_cpu = {
+    .name = "cpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL_ARRAY(env.gpr, RISCVCPU, 32),
+        VMSTATE_UINT64_ARRAY(env.fpr, RISCVCPU, 32),
+        VMSTATE_UINTTL(env.pc, RISCVCPU),
+        VMSTATE_UINTTL(env.load_res, RISCVCPU),
+        VMSTATE_UINTTL(env.load_val, RISCVCPU),
+        VMSTATE_UINTTL(env.frm, RISCVCPU),
+        VMSTATE_UINTTL(env.badaddr, RISCVCPU),
+        VMSTATE_UINTTL(env.guest_phys_fault_addr, RISCVCPU),
+        VMSTATE_UINTTL(env.priv_ver, RISCVCPU),
+        VMSTATE_UINTTL(env.vext_ver, RISCVCPU),
+        VMSTATE_UINTTL(env.misa, RISCVCPU),
+        VMSTATE_UINTTL(env.misa_mask, RISCVCPU),
+        VMSTATE_UINT32(env.features, RISCVCPU),
+        VMSTATE_UINTTL(env.priv, RISCVCPU),
+        VMSTATE_UINTTL(env.virt, RISCVCPU),
+        VMSTATE_UINTTL(env.resetvec, RISCVCPU),
+        VMSTATE_UINTTL(env.mhartid, RISCVCPU),
+        VMSTATE_UINTTL(env.mstatus, RISCVCPU),
+        VMSTATE_UINTTL(env.mip, RISCVCPU),
+        VMSTATE_UINT32(env.miclaim, RISCVCPU),
+        VMSTATE_UINTTL(env.mie, RISCVCPU),
+        VMSTATE_UINTTL(env.mideleg, RISCVCPU),
+        VMSTATE_UINTTL(env.sptbr, RISCVCPU),
+        VMSTATE_UINTTL(env.satp, RISCVCPU),
+        VMSTATE_UINTTL(env.sbadaddr, RISCVCPU),
+        VMSTATE_UINTTL(env.mbadaddr, RISCVCPU),
+        VMSTATE_UINTTL(env.medeleg, RISCVCPU),
+        VMSTATE_UINTTL(env.stvec, RISCVCPU),
+        VMSTATE_UINTTL(env.sepc, RISCVCPU),
+        VMSTATE_UINTTL(env.scause, RISCVCPU),
+        VMSTATE_UINTTL(env.mtvec, RISCVCPU),
+        VMSTATE_UINTTL(env.mepc, RISCVCPU),
+        VMSTATE_UINTTL(env.mcause, RISCVCPU),
+        VMSTATE_UINTTL(env.mtval, RISCVCPU),
+        VMSTATE_UINTTL(env.scounteren, RISCVCPU),
+        VMSTATE_UINTTL(env.mcounteren, RISCVCPU),
+        VMSTATE_UINTTL(env.sscratch, RISCVCPU),
+        VMSTATE_UINTTL(env.mscratch, RISCVCPU),
+        VMSTATE_UINT64(env.mfromhost, RISCVCPU),
+        VMSTATE_UINT64(env.mtohost, RISCVCPU),
+        VMSTATE_UINT64(env.timecmp, RISCVCPU),
+
+#ifdef TARGET_RISCV32
+        VMSTATE_UINTTL(env.mstatush, RISCVCPU),
+#endif
+        VMSTATE_END_OF_LIST()
+    }
+};
diff --git a/target/riscv/meson.build b/target/riscv/meson.build
index abd647fea1..14a5c62dac 100644
--- a/target/riscv/meson.build
+++ b/target/riscv/meson.build
@@ -27,7 +27,8 @@ riscv_ss.add(files(
 riscv_softmmu_ss = ss.source_set()
 riscv_softmmu_ss.add(files(
   'pmp.c',
-  'monitor.c'
+  'monitor.c',
+  'machine.c'
 ))
 
 target_arch += {'riscv': riscv_ss}
-- 
2.19.1

[PATCH 1/5] target/riscv: Add basic vmstate description of CPU

[Yifei Jiang]

Add basic CPU state description to the newly created machine.c

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 target/riscv/cpu.c       |  7 -----
 target/riscv/cpu.h       |  4 +++
 target/riscv/machine.c   | 59 ++++++++++++++++++++++++++++++++++++++++
 target/riscv/meson.build |  3 +-
 4 files changed, 65 insertions(+), 8 deletions(-)
 create mode 100644 target/riscv/machine.c

diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c
index 0bbfd7f457..bf396e2916 100644
--- a/target/riscv/cpu.c
+++ b/target/riscv/cpu.c
@@ -496,13 +496,6 @@ static void riscv_cpu_init(Object *obj)
     cpu_set_cpustate_pointers(cpu);
 }
 
-#ifndef CONFIG_USER_ONLY
-static const VMStateDescription vmstate_riscv_cpu = {
-    .name = "cpu",
-    .unmigratable = 1,
-};
-#endif
-
 static Property riscv_cpu_properties[] = {
     DEFINE_PROP_BOOL("i", RISCVCPU, cfg.ext_i, true),
     DEFINE_PROP_BOOL("e", RISCVCPU, cfg.ext_e, false),
diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h
index de275782e6..8440ea0793 100644
--- a/target/riscv/cpu.h
+++ b/target/riscv/cpu.h
@@ -311,6 +311,10 @@ extern const char * const riscv_fpr_regnames[];
 extern const char * const riscv_excp_names[];
 extern const char * const riscv_intr_names[];
 
+#ifndef CONFIG_USER_ONLY
+extern const VMStateDescription vmstate_riscv_cpu;
+#endif
+
 const char *riscv_cpu_get_trap_name(target_ulong cause, bool async);
 void riscv_cpu_do_interrupt(CPUState *cpu);
 int riscv_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg);
diff --git a/target/riscv/machine.c b/target/riscv/machine.c
new file mode 100644
index 0000000000..3451b888b1
--- /dev/null
+++ b/target/riscv/machine.c
@@ -0,0 +1,59 @@
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "qemu/error-report.h"
+#include "sysemu/kvm.h"
+#include "migration/cpu.h"
+
+const VMStateDescription vmstate_riscv_cpu = {
+    .name = "cpu",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL_ARRAY(env.gpr, RISCVCPU, 32),
+        VMSTATE_UINT64_ARRAY(env.fpr, RISCVCPU, 32),
+        VMSTATE_UINTTL(env.pc, RISCVCPU),
+        VMSTATE_UINTTL(env.load_res, RISCVCPU),
+        VMSTATE_UINTTL(env.load_val, RISCVCPU),
+        VMSTATE_UINTTL(env.frm, RISCVCPU),
+        VMSTATE_UINTTL(env.badaddr, RISCVCPU),
+        VMSTATE_UINTTL(env.guest_phys_fault_addr, RISCVCPU),
+        VMSTATE_UINTTL(env.priv_ver, RISCVCPU),
+        VMSTATE_UINTTL(env.vext_ver, RISCVCPU),
+        VMSTATE_UINTTL(env.misa, RISCVCPU),
+        VMSTATE_UINTTL(env.misa_mask, RISCVCPU),
+        VMSTATE_UINT32(env.features, RISCVCPU),
+        VMSTATE_UINTTL(env.priv, RISCVCPU),
+        VMSTATE_UINTTL(env.virt, RISCVCPU),
+        VMSTATE_UINTTL(env.resetvec, RISCVCPU),
+        VMSTATE_UINTTL(env.mhartid, RISCVCPU),
+        VMSTATE_UINTTL(env.mstatus, RISCVCPU),
+        VMSTATE_UINTTL(env.mip, RISCVCPU),
+        VMSTATE_UINT32(env.miclaim, RISCVCPU),
+        VMSTATE_UINTTL(env.mie, RISCVCPU),
+        VMSTATE_UINTTL(env.mideleg, RISCVCPU),
+        VMSTATE_UINTTL(env.sptbr, RISCVCPU),
+        VMSTATE_UINTTL(env.satp, RISCVCPU),
+        VMSTATE_UINTTL(env.sbadaddr, RISCVCPU),
+        VMSTATE_UINTTL(env.mbadaddr, RISCVCPU),
+        VMSTATE_UINTTL(env.medeleg, RISCVCPU),
+        VMSTATE_UINTTL(env.stvec, RISCVCPU),
+        VMSTATE_UINTTL(env.sepc, RISCVCPU),
+        VMSTATE_UINTTL(env.scause, RISCVCPU),
+        VMSTATE_UINTTL(env.mtvec, RISCVCPU),
+        VMSTATE_UINTTL(env.mepc, RISCVCPU),
+        VMSTATE_UINTTL(env.mcause, RISCVCPU),
+        VMSTATE_UINTTL(env.mtval, RISCVCPU),
+        VMSTATE_UINTTL(env.scounteren, RISCVCPU),
+        VMSTATE_UINTTL(env.mcounteren, RISCVCPU),
+        VMSTATE_UINTTL(env.sscratch, RISCVCPU),
+        VMSTATE_UINTTL(env.mscratch, RISCVCPU),
+        VMSTATE_UINT64(env.mfromhost, RISCVCPU),
+        VMSTATE_UINT64(env.mtohost, RISCVCPU),
+        VMSTATE_UINT64(env.timecmp, RISCVCPU),
+
+#ifdef TARGET_RISCV32
+        VMSTATE_UINTTL(env.mstatush, RISCVCPU),
+#endif
+        VMSTATE_END_OF_LIST()
+    }
+};
diff --git a/target/riscv/meson.build b/target/riscv/meson.build
index abd647fea1..14a5c62dac 100644
--- a/target/riscv/meson.build
+++ b/target/riscv/meson.build
@@ -27,7 +27,8 @@ riscv_ss.add(files(
 riscv_softmmu_ss = ss.source_set()
 riscv_softmmu_ss.add(files(
   'pmp.c',
-  'monitor.c'
+  'monitor.c',
+  'machine.c'
 ))
 
 target_arch += {'riscv': riscv_ss}
-- 
2.19.1

[PATCH 2/5] target/riscv: Add PMP state description

[Yifei Jiang]

In the case of supporting PMP feature, add PMP state description
to vmstate_riscv_cpu.

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 target/riscv/machine.c | 49 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/target/riscv/machine.c b/target/riscv/machine.c
index 3451b888b1..b1fc839b43 100644
--- a/target/riscv/machine.c
+++ b/target/riscv/machine.c
@@ -4,6 +4,51 @@
 #include "sysemu/kvm.h"
 #include "migration/cpu.h"
 
+static bool pmp_needed(void *opaque)
+{
+    RISCVCPU *cpu = opaque;
+    CPURISCVState *env = &cpu->env;
+
+    return riscv_feature(env, RISCV_FEATURE_PMP);
+}
+
+static const VMStateDescription vmstate_pmp_entry = {
+    .name = "cpu/pmp/entry",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL(addr_reg, pmp_entry_t),
+        VMSTATE_UINT8(cfg_reg, pmp_entry_t),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_pmp_addr = {
+    .name = "cpu/pmp/addr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL(sa, pmp_addr_t),
+        VMSTATE_UINTTL(ea, pmp_addr_t),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_pmp = {
+    .name = "cpu/pmp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pmp_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_ARRAY(env.pmp_state.pmp, RISCVCPU, MAX_RISCV_PMPS,
+                             0, vmstate_pmp_entry, pmp_entry_t),
+        VMSTATE_STRUCT_ARRAY(env.pmp_state.addr, RISCVCPU, MAX_RISCV_PMPS,
+                             0, vmstate_pmp_addr, pmp_addr_t),
+        VMSTATE_UINT32(env.pmp_state.num_rules, RISCVCPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_riscv_cpu = {
     .name = "cpu",
     .version_id = 1,
@@ -55,5 +100,9 @@ const VMStateDescription vmstate_riscv_cpu = {
         VMSTATE_UINTTL(env.mstatush, RISCVCPU),
 #endif
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_pmp,
+        NULL
     }
 };
-- 
2.19.1

[PATCH 2/5] target/riscv: Add PMP state description

[Yifei Jiang]

In the case of supporting PMP feature, add PMP state description
to vmstate_riscv_cpu.

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 target/riscv/machine.c | 49 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/target/riscv/machine.c b/target/riscv/machine.c
index 3451b888b1..b1fc839b43 100644
--- a/target/riscv/machine.c
+++ b/target/riscv/machine.c
@@ -4,6 +4,51 @@
 #include "sysemu/kvm.h"
 #include "migration/cpu.h"
 
+static bool pmp_needed(void *opaque)
+{
+    RISCVCPU *cpu = opaque;
+    CPURISCVState *env = &cpu->env;
+
+    return riscv_feature(env, RISCV_FEATURE_PMP);
+}
+
+static const VMStateDescription vmstate_pmp_entry = {
+    .name = "cpu/pmp/entry",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL(addr_reg, pmp_entry_t),
+        VMSTATE_UINT8(cfg_reg, pmp_entry_t),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_pmp_addr = {
+    .name = "cpu/pmp/addr",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL(sa, pmp_addr_t),
+        VMSTATE_UINTTL(ea, pmp_addr_t),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_pmp = {
+    .name = "cpu/pmp",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = pmp_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT_ARRAY(env.pmp_state.pmp, RISCVCPU, MAX_RISCV_PMPS,
+                             0, vmstate_pmp_entry, pmp_entry_t),
+        VMSTATE_STRUCT_ARRAY(env.pmp_state.addr, RISCVCPU, MAX_RISCV_PMPS,
+                             0, vmstate_pmp_addr, pmp_addr_t),
+        VMSTATE_UINT32(env.pmp_state.num_rules, RISCVCPU),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_riscv_cpu = {
     .name = "cpu",
     .version_id = 1,
@@ -55,5 +100,9 @@ const VMStateDescription vmstate_riscv_cpu = {
         VMSTATE_UINTTL(env.mstatush, RISCVCPU),
 #endif
         VMSTATE_END_OF_LIST()
+    },
+    .subsections = (const VMStateDescription * []) {
+        &vmstate_pmp,
+        NULL
     }
 };
-- 
2.19.1

[PATCH 3/5] target/riscv: Add H extention state description

[Yifei Jiang]

In the case of supporting H extention, add H extention description
to vmstate_riscv_cpu.

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 target/riscv/machine.c | 51 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/target/riscv/machine.c b/target/riscv/machine.c
index b1fc839b43..6a528bc1a5 100644
--- a/target/riscv/machine.c
+++ b/target/riscv/machine.c
@@ -49,6 +49,56 @@ static const VMStateDescription vmstate_pmp = {
     }
 };
 
+static bool hyper_needed(void *opaque)
+{
+    RISCVCPU *cpu = opaque;
+    CPURISCVState *env = &cpu->env;
+
+    return riscv_has_ext(env, RVH);
+}
+
+static const VMStateDescription vmstate_hyper = {
+    .name = "cpu/hyper",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyper_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL(env.hstatus, RISCVCPU),
+        VMSTATE_UINTTL(env.hedeleg, RISCVCPU),
+        VMSTATE_UINTTL(env.hideleg, RISCVCPU),
+        VMSTATE_UINTTL(env.hcounteren, RISCVCPU),
+        VMSTATE_UINTTL(env.htval, RISCVCPU),
+        VMSTATE_UINTTL(env.htinst, RISCVCPU),
+        VMSTATE_UINTTL(env.hgatp, RISCVCPU),
+        VMSTATE_UINT64(env.htimedelta, RISCVCPU),
+
+        VMSTATE_UINTTL(env.vsstatus, RISCVCPU),
+        VMSTATE_UINTTL(env.vstvec, RISCVCPU),
+        VMSTATE_UINTTL(env.vsscratch, RISCVCPU),
+        VMSTATE_UINTTL(env.vsepc, RISCVCPU),
+        VMSTATE_UINTTL(env.vscause, RISCVCPU),
+        VMSTATE_UINTTL(env.vstval, RISCVCPU),
+        VMSTATE_UINTTL(env.vsatp, RISCVCPU),
+
+        VMSTATE_UINTTL(env.mtval2, RISCVCPU),
+        VMSTATE_UINTTL(env.mtinst, RISCVCPU),
+
+        VMSTATE_UINTTL(env.stvec_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.sscratch_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.sepc_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.scause_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.stval_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.satp_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.mstatus_hs, RISCVCPU),
+
+#ifdef TARGET_RISCV32
+        VMSTATE_UINTTL(env.vsstatush, RISCVCPU),
+        VMSTATE_UINTTL(env.mstatush_hs, RISCVCPU),
+#endif
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_riscv_cpu = {
     .name = "cpu",
     .version_id = 1,
@@ -103,6 +153,7 @@ const VMStateDescription vmstate_riscv_cpu = {
     },
     .subsections = (const VMStateDescription * []) {
         &vmstate_pmp,
+        &vmstate_hyper,
         NULL
     }
 };
-- 
2.19.1

[PATCH 3/5] target/riscv: Add H extention state description

[Yifei Jiang]

In the case of supporting H extention, add H extention description
to vmstate_riscv_cpu.

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 target/riscv/machine.c | 51 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/target/riscv/machine.c b/target/riscv/machine.c
index b1fc839b43..6a528bc1a5 100644
--- a/target/riscv/machine.c
+++ b/target/riscv/machine.c
@@ -49,6 +49,56 @@ static const VMStateDescription vmstate_pmp = {
     }
 };
 
+static bool hyper_needed(void *opaque)
+{
+    RISCVCPU *cpu = opaque;
+    CPURISCVState *env = &cpu->env;
+
+    return riscv_has_ext(env, RVH);
+}
+
+static const VMStateDescription vmstate_hyper = {
+    .name = "cpu/hyper",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = hyper_needed,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINTTL(env.hstatus, RISCVCPU),
+        VMSTATE_UINTTL(env.hedeleg, RISCVCPU),
+        VMSTATE_UINTTL(env.hideleg, RISCVCPU),
+        VMSTATE_UINTTL(env.hcounteren, RISCVCPU),
+        VMSTATE_UINTTL(env.htval, RISCVCPU),
+        VMSTATE_UINTTL(env.htinst, RISCVCPU),
+        VMSTATE_UINTTL(env.hgatp, RISCVCPU),
+        VMSTATE_UINT64(env.htimedelta, RISCVCPU),
+
+        VMSTATE_UINTTL(env.vsstatus, RISCVCPU),
+        VMSTATE_UINTTL(env.vstvec, RISCVCPU),
+        VMSTATE_UINTTL(env.vsscratch, RISCVCPU),
+        VMSTATE_UINTTL(env.vsepc, RISCVCPU),
+        VMSTATE_UINTTL(env.vscause, RISCVCPU),
+        VMSTATE_UINTTL(env.vstval, RISCVCPU),
+        VMSTATE_UINTTL(env.vsatp, RISCVCPU),
+
+        VMSTATE_UINTTL(env.mtval2, RISCVCPU),
+        VMSTATE_UINTTL(env.mtinst, RISCVCPU),
+
+        VMSTATE_UINTTL(env.stvec_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.sscratch_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.sepc_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.scause_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.stval_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.satp_hs, RISCVCPU),
+        VMSTATE_UINTTL(env.mstatus_hs, RISCVCPU),
+
+#ifdef TARGET_RISCV32
+        VMSTATE_UINTTL(env.vsstatush, RISCVCPU),
+        VMSTATE_UINTTL(env.mstatush_hs, RISCVCPU),
+#endif
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 const VMStateDescription vmstate_riscv_cpu = {
     .name = "cpu",
     .version_id = 1,
@@ -103,6 +153,7 @@ const VMStateDescription vmstate_riscv_cpu = {
     },
     .subsections = (const VMStateDescription * []) {
         &vmstate_pmp,
+        &vmstate_hyper,
         NULL
     }
 };
-- 
2.19.1

[PATCH 4/5] target/riscv: Add V extention state description

[Yifei Jiang]

In the case of supporting V extention, add V extention description
to vmstate_riscv_cpu.

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 target/riscv/machine.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/target/riscv/machine.c b/target/riscv/machine.c
index 6a528bc1a5..8912a77b71 100644
--- a/target/riscv/machine.c
+++ b/target/riscv/machine.c
@@ -57,6 +57,30 @@ static bool hyper_needed(void *opaque)
     return riscv_has_ext(env, RVH);
 }
 
+static bool vector_needed(void *opaque)
+{
+    RISCVCPU *cpu = opaque;
+    CPURISCVState *env = &cpu->env;
+
+    return riscv_has_ext(env, RVV);
+}
+
+static const VMStateDescription vmstate_vector = {
+    .name = "cpu/vector",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = vector_needed,
+    .fields = (VMStateField[]) {
+            VMSTATE_UINT64_ARRAY(env.vreg, RISCVCPU, 32 * RV_VLEN_MAX / 64),
+            VMSTATE_UINTTL(env.vxrm, RISCVCPU),
+            VMSTATE_UINTTL(env.vxsat, RISCVCPU),
+            VMSTATE_UINTTL(env.vl, RISCVCPU),
+            VMSTATE_UINTTL(env.vstart, RISCVCPU),
+            VMSTATE_UINTTL(env.vtype, RISCVCPU),
+            VMSTATE_END_OF_LIST()
+        }
+};
+
 static const VMStateDescription vmstate_hyper = {
     .name = "cpu/hyper",
     .version_id = 1,
@@ -154,6 +178,7 @@ const VMStateDescription vmstate_riscv_cpu = {
     .subsections = (const VMStateDescription * []) {
         &vmstate_pmp,
         &vmstate_hyper,
+        &vmstate_vector,
         NULL
     }
 };
-- 
2.19.1

[PATCH 4/5] target/riscv: Add V extention state description

[Yifei Jiang]

In the case of supporting V extention, add V extention description
to vmstate_riscv_cpu.

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 target/riscv/machine.c | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/target/riscv/machine.c b/target/riscv/machine.c
index 6a528bc1a5..8912a77b71 100644
--- a/target/riscv/machine.c
+++ b/target/riscv/machine.c
@@ -57,6 +57,30 @@ static bool hyper_needed(void *opaque)
     return riscv_has_ext(env, RVH);
 }
 
+static bool vector_needed(void *opaque)
+{
+    RISCVCPU *cpu = opaque;
+    CPURISCVState *env = &cpu->env;
+
+    return riscv_has_ext(env, RVV);
+}
+
+static const VMStateDescription vmstate_vector = {
+    .name = "cpu/vector",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .needed = vector_needed,
+    .fields = (VMStateField[]) {
+            VMSTATE_UINT64_ARRAY(env.vreg, RISCVCPU, 32 * RV_VLEN_MAX / 64),
+            VMSTATE_UINTTL(env.vxrm, RISCVCPU),
+            VMSTATE_UINTTL(env.vxsat, RISCVCPU),
+            VMSTATE_UINTTL(env.vl, RISCVCPU),
+            VMSTATE_UINTTL(env.vstart, RISCVCPU),
+            VMSTATE_UINTTL(env.vtype, RISCVCPU),
+            VMSTATE_END_OF_LIST()
+        }
+};
+
 static const VMStateDescription vmstate_hyper = {
     .name = "cpu/hyper",
     .version_id = 1,
@@ -154,6 +178,7 @@ const VMStateDescription vmstate_riscv_cpu = {
     .subsections = (const VMStateDescription * []) {
         &vmstate_pmp,
         &vmstate_hyper,
+        &vmstate_vector,
         NULL
     }
 };
-- 
2.19.1

[PATCH 5/5] target/riscv: Add sifive_plic vmstate

[Yifei Jiang]

Add sifive_plic vmstate for supporting sifive_plic migration.
Current vmstate framework only supports one structure parameter
as num field to describe variable length arrays, so introduce
num_enables.

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 hw/intc/sifive_plic.c | 26 +++++++++++++++++++++++++-
 hw/intc/sifive_plic.h |  1 +
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/hw/intc/sifive_plic.c b/hw/intc/sifive_plic.c
index f42fd695d8..97a1a27a9a 100644
--- a/hw/intc/sifive_plic.c
+++ b/hw/intc/sifive_plic.c
@@ -30,6 +30,7 @@
 #include "hw/intc/sifive_plic.h"
 #include "target/riscv/cpu.h"
 #include "sysemu/sysemu.h"
+#include "migration/vmstate.h"
 
 #define RISCV_DEBUG_PLIC 0
 
@@ -448,11 +449,12 @@ static void sifive_plic_realize(DeviceState *dev, Error **errp)
                           TYPE_SIFIVE_PLIC, plic->aperture_size);
     parse_hart_config(plic);
     plic->bitfield_words = (plic->num_sources + 31) >> 5;
+    plic->num_enables = plic->bitfield_words * plic->num_addrs;
     plic->source_priority = g_new0(uint32_t, plic->num_sources);
     plic->target_priority = g_new(uint32_t, plic->num_addrs);
     plic->pending = g_new0(uint32_t, plic->bitfield_words);
     plic->claimed = g_new0(uint32_t, plic->bitfield_words);
-    plic->enable = g_new0(uint32_t, plic->bitfield_words * plic->num_addrs);
+    plic->enable = g_new0(uint32_t, plic->num_enables);
     sysbus_init_mmio(SYS_BUS_DEVICE(dev), &plic->mmio);
     qdev_init_gpio_in(dev, sifive_plic_irq_request, plic->num_sources);
 
@@ -472,12 +474,34 @@ static void sifive_plic_realize(DeviceState *dev, Error **errp)
     msi_nonbroken = true;
 }
 
+static const VMStateDescription vmstate_sifive_plic = {
+    .name = "riscv_sifive_plic",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+            VMSTATE_VARRAY_UINT32(source_priority, SiFivePLICState,
+                                  num_sources, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_VARRAY_UINT32(target_priority, SiFivePLICState,
+                                  num_addrs, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_VARRAY_UINT32(pending, SiFivePLICState, bitfield_words, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_VARRAY_UINT32(claimed, SiFivePLICState, bitfield_words, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_VARRAY_UINT32(enable, SiFivePLICState, num_enables, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_END_OF_LIST()
+        }
+};
+
 static void sifive_plic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
 
     device_class_set_props(dc, sifive_plic_properties);
     dc->realize = sifive_plic_realize;
+    dc->vmsd = &vmstate_sifive_plic;
 }
 
 static const TypeInfo sifive_plic_info = {
diff --git a/hw/intc/sifive_plic.h b/hw/intc/sifive_plic.h
index b75b1f145d..1e451a270c 100644
--- a/hw/intc/sifive_plic.h
+++ b/hw/intc/sifive_plic.h
@@ -52,6 +52,7 @@ struct SiFivePLICState {
     uint32_t num_addrs;
     uint32_t num_harts;
     uint32_t bitfield_words;
+    uint32_t num_enables;
     PLICAddr *addr_config;
     uint32_t *source_priority;
     uint32_t *target_priority;
-- 
2.19.1

[PATCH 5/5] target/riscv: Add sifive_plic vmstate

[Yifei Jiang]

Add sifive_plic vmstate for supporting sifive_plic migration.
Current vmstate framework only supports one structure parameter
as num field to describe variable length arrays, so introduce
num_enables.

Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
---
 hw/intc/sifive_plic.c | 26 +++++++++++++++++++++++++-
 hw/intc/sifive_plic.h |  1 +
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/hw/intc/sifive_plic.c b/hw/intc/sifive_plic.c
index f42fd695d8..97a1a27a9a 100644
--- a/hw/intc/sifive_plic.c
+++ b/hw/intc/sifive_plic.c
@@ -30,6 +30,7 @@
 #include "hw/intc/sifive_plic.h"
 #include "target/riscv/cpu.h"
 #include "sysemu/sysemu.h"
+#include "migration/vmstate.h"
 
 #define RISCV_DEBUG_PLIC 0
 
@@ -448,11 +449,12 @@ static void sifive_plic_realize(DeviceState *dev, Error **errp)
                           TYPE_SIFIVE_PLIC, plic->aperture_size);
     parse_hart_config(plic);
     plic->bitfield_words = (plic->num_sources + 31) >> 5;
+    plic->num_enables = plic->bitfield_words * plic->num_addrs;
     plic->source_priority = g_new0(uint32_t, plic->num_sources);
     plic->target_priority = g_new(uint32_t, plic->num_addrs);
     plic->pending = g_new0(uint32_t, plic->bitfield_words);
     plic->claimed = g_new0(uint32_t, plic->bitfield_words);
-    plic->enable = g_new0(uint32_t, plic->bitfield_words * plic->num_addrs);
+    plic->enable = g_new0(uint32_t, plic->num_enables);
     sysbus_init_mmio(SYS_BUS_DEVICE(dev), &plic->mmio);
     qdev_init_gpio_in(dev, sifive_plic_irq_request, plic->num_sources);
 
@@ -472,12 +474,34 @@ static void sifive_plic_realize(DeviceState *dev, Error **errp)
     msi_nonbroken = true;
 }
 
+static const VMStateDescription vmstate_sifive_plic = {
+    .name = "riscv_sifive_plic",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+            VMSTATE_VARRAY_UINT32(source_priority, SiFivePLICState,
+                                  num_sources, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_VARRAY_UINT32(target_priority, SiFivePLICState,
+                                  num_addrs, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_VARRAY_UINT32(pending, SiFivePLICState, bitfield_words, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_VARRAY_UINT32(claimed, SiFivePLICState, bitfield_words, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_VARRAY_UINT32(enable, SiFivePLICState, num_enables, 0,
+                                  vmstate_info_uint32, uint32_t),
+            VMSTATE_END_OF_LIST()
+        }
+};
+
 static void sifive_plic_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
 
     device_class_set_props(dc, sifive_plic_properties);
     dc->realize = sifive_plic_realize;
+    dc->vmsd = &vmstate_sifive_plic;
 }
 
 static const TypeInfo sifive_plic_info = {
diff --git a/hw/intc/sifive_plic.h b/hw/intc/sifive_plic.h
index b75b1f145d..1e451a270c 100644
--- a/hw/intc/sifive_plic.h
+++ b/hw/intc/sifive_plic.h
@@ -52,6 +52,7 @@ struct SiFivePLICState {
     uint32_t num_addrs;
     uint32_t num_harts;
     uint32_t bitfield_words;
+    uint32_t num_enables;
     PLICAddr *addr_config;
     uint32_t *source_priority;
     uint32_t *target_priority;
-- 
2.19.1

Re: [PATCH 1/5] target/riscv: Add basic vmstate description of CPU

[Richard Henderson]

On 9/28/20 9:03 PM, Yifei Jiang wrote:
> --- /dev/null
> +++ b/target/riscv/machine.c
> @@ -0,0 +1,59 @@
> +#include "qemu/osdep.h"
> +#include "cpu.h"

All new files must contain license boilerplate.

Otherwise, considering the followups, this seems ok.


r~

Re: [PATCH 3/5] target/riscv: Add H extention state description

[Richard Henderson]

On 9/28/20 9:03 PM, Yifei Jiang wrote:
> +        VMSTATE_UINTTL(env.vsstatus, RISCVCPU),
> +        VMSTATE_UINTTL(env.vstvec, RISCVCPU),
> +        VMSTATE_UINTTL(env.vsscratch, RISCVCPU),
> +        VMSTATE_UINTTL(env.vsepc, RISCVCPU),
> +        VMSTATE_UINTTL(env.vscause, RISCVCPU),
> +        VMSTATE_UINTTL(env.vstval, RISCVCPU),
> +        VMSTATE_UINTTL(env.vsatp, RISCVCPU),

So... if I understand things correctly, this is synthetic state, internal to
QEMU.  It is generally better to only serialize architectural state, so that if
qemu internals are rearranged, it is easy to decide on the correct sequence of
operations.

It seems like this should be re-generated with a post_load hook, calling some
of the code currently in riscv_cpu_swap_hypervisor_regs().  Note that some
minor rearrangement would be needed to call that code from this new context.


r~

Re: [PATCH 4/5] target/riscv: Add V extention state description

[Richard Henderson]

On 9/28/20 9:03 PM, Yifei Jiang wrote:
> In the case of supporting V extention, add V extention description
> to vmstate_riscv_cpu.
> 
> Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
> Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
> ---
>  target/riscv/machine.c | 25 +++++++++++++++++++++++++
>  1 file changed, 25 insertions(+)

Reviewed-by: Richard Henderson <richard.henderson@linaro.org>

Though of course this is racing with the v1.0 patch set, which changes the set
of vector csrs.


r~

Re: [PATCH 3/5] target/riscv: Add H extention state description

[Alistair Francis]

On Thu, Oct 1, 2020 at 10:56 AM Richard Henderson
<richard.henderson@linaro.org> wrote:
>
> On 9/28/20 9:03 PM, Yifei Jiang wrote:
> > +        VMSTATE_UINTTL(env.vsstatus, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vstvec, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsscratch, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsepc, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vscause, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vstval, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsatp, RISCVCPU),
>
> So... if I understand things correctly, this is synthetic state, internal to
> QEMU.  It is generally better to only serialize architectural state, so that if
> qemu internals are rearranged, it is easy to decide on the correct sequence of
> operations.

I don't think the virtual registers are synthetic, they contain what
the guest Hypervisor/Hypervisor guest wrote to those CSRs. I don't
think we could re-generate them from anything else.

There are some other registers in this series that I think can be
re-generated. The PMP is a good example of that, where the PMP config
data could be re-generated from the CSRs.

Alistair

>
> It seems like this should be re-generated with a post_load hook, calling some
> of the code currently in riscv_cpu_swap_hypervisor_regs().  Note that some
> minor rearrangement would be needed to call that code from this new context.
>
>
> r~
>

Re: [PATCH 3/5] target/riscv: Add H extention state description

[Alistair Francis]

On Thu, Oct 1, 2020 at 10:56 AM Richard Henderson
<richard.henderson@linaro.org> wrote:
>
> On 9/28/20 9:03 PM, Yifei Jiang wrote:
> > +        VMSTATE_UINTTL(env.vsstatus, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vstvec, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsscratch, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsepc, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vscause, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vstval, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsatp, RISCVCPU),
>
> So... if I understand things correctly, this is synthetic state, internal to
> QEMU.  It is generally better to only serialize architectural state, so that if
> qemu internals are rearranged, it is easy to decide on the correct sequence of
> operations.

I don't think the virtual registers are synthetic, they contain what
the guest Hypervisor/Hypervisor guest wrote to those CSRs. I don't
think we could re-generate them from anything else.

There are some other registers in this series that I think can be
re-generated. The PMP is a good example of that, where the PMP config
data could be re-generated from the CSRs.

Alistair

>
> It seems like this should be re-generated with a post_load hook, calling some
> of the code currently in riscv_cpu_swap_hypervisor_regs().  Note that some
> minor rearrangement would be needed to call that code from this new context.
>
>
> r~
>

Re: [PATCH 2/5] target/riscv: Add PMP state description

[Alistair Francis]

On Mon, Sep 28, 2020 at 7:05 PM Yifei Jiang <jiangyifei@huawei.com> wrote:
>
> In the case of supporting PMP feature, add PMP state description
> to vmstate_riscv_cpu.
>
> Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
> Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
> ---
>  target/riscv/machine.c | 49 ++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 49 insertions(+)
>
> diff --git a/target/riscv/machine.c b/target/riscv/machine.c
> index 3451b888b1..b1fc839b43 100644
> --- a/target/riscv/machine.c
> +++ b/target/riscv/machine.c
> @@ -4,6 +4,51 @@
>  #include "sysemu/kvm.h"
>  #include "migration/cpu.h"
>
> +static bool pmp_needed(void *opaque)
> +{
> +    RISCVCPU *cpu = opaque;
> +    CPURISCVState *env = &cpu->env;
> +
> +    return riscv_feature(env, RISCV_FEATURE_PMP);
> +}
> +
> +static const VMStateDescription vmstate_pmp_entry = {
> +    .name = "cpu/pmp/entry",
> +    .version_id = 1,
> +    .minimum_version_id = 1,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_UINTTL(addr_reg, pmp_entry_t),
> +        VMSTATE_UINT8(cfg_reg, pmp_entry_t),
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
> +static const VMStateDescription vmstate_pmp_addr = {
> +    .name = "cpu/pmp/addr",
> +    .version_id = 1,
> +    .minimum_version_id = 1,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_UINTTL(sa, pmp_addr_t),
> +        VMSTATE_UINTTL(ea, pmp_addr_t),
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
> +static const VMStateDescription vmstate_pmp = {
> +    .name = "cpu/pmp",
> +    .version_id = 1,
> +    .minimum_version_id = 1,
> +    .needed = pmp_needed,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_STRUCT_ARRAY(env.pmp_state.pmp, RISCVCPU, MAX_RISCV_PMPS,
> +                             0, vmstate_pmp_entry, pmp_entry_t),
> +        VMSTATE_STRUCT_ARRAY(env.pmp_state.addr, RISCVCPU, MAX_RISCV_PMPS,
> +                             0, vmstate_pmp_addr, pmp_addr_t),
> +        VMSTATE_UINT32(env.pmp_state.num_rules, RISCVCPU),

I would prefer to re-generate these from the PMP CSRs post load.

Alistair

> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
>  const VMStateDescription vmstate_riscv_cpu = {
>      .name = "cpu",
>      .version_id = 1,
> @@ -55,5 +100,9 @@ const VMStateDescription vmstate_riscv_cpu = {
>          VMSTATE_UINTTL(env.mstatush, RISCVCPU),
>  #endif
>          VMSTATE_END_OF_LIST()
> +    },
> +    .subsections = (const VMStateDescription * []) {
> +        &vmstate_pmp,
> +        NULL
>      }
>  };
> --
> 2.19.1
>
>

Re: [PATCH 2/5] target/riscv: Add PMP state description

[Alistair Francis]

On Mon, Sep 28, 2020 at 7:05 PM Yifei Jiang <jiangyifei@huawei.com> wrote:
>
> In the case of supporting PMP feature, add PMP state description
> to vmstate_riscv_cpu.
>
> Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
> Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
> ---
>  target/riscv/machine.c | 49 ++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 49 insertions(+)
>
> diff --git a/target/riscv/machine.c b/target/riscv/machine.c
> index 3451b888b1..b1fc839b43 100644
> --- a/target/riscv/machine.c
> +++ b/target/riscv/machine.c
> @@ -4,6 +4,51 @@
>  #include "sysemu/kvm.h"
>  #include "migration/cpu.h"
>
> +static bool pmp_needed(void *opaque)
> +{
> +    RISCVCPU *cpu = opaque;
> +    CPURISCVState *env = &cpu->env;
> +
> +    return riscv_feature(env, RISCV_FEATURE_PMP);
> +}
> +
> +static const VMStateDescription vmstate_pmp_entry = {
> +    .name = "cpu/pmp/entry",
> +    .version_id = 1,
> +    .minimum_version_id = 1,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_UINTTL(addr_reg, pmp_entry_t),
> +        VMSTATE_UINT8(cfg_reg, pmp_entry_t),
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
> +static const VMStateDescription vmstate_pmp_addr = {
> +    .name = "cpu/pmp/addr",
> +    .version_id = 1,
> +    .minimum_version_id = 1,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_UINTTL(sa, pmp_addr_t),
> +        VMSTATE_UINTTL(ea, pmp_addr_t),
> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
> +static const VMStateDescription vmstate_pmp = {
> +    .name = "cpu/pmp",
> +    .version_id = 1,
> +    .minimum_version_id = 1,
> +    .needed = pmp_needed,
> +    .fields = (VMStateField[]) {
> +        VMSTATE_STRUCT_ARRAY(env.pmp_state.pmp, RISCVCPU, MAX_RISCV_PMPS,
> +                             0, vmstate_pmp_entry, pmp_entry_t),
> +        VMSTATE_STRUCT_ARRAY(env.pmp_state.addr, RISCVCPU, MAX_RISCV_PMPS,
> +                             0, vmstate_pmp_addr, pmp_addr_t),
> +        VMSTATE_UINT32(env.pmp_state.num_rules, RISCVCPU),

I would prefer to re-generate these from the PMP CSRs post load.

Alistair

> +        VMSTATE_END_OF_LIST()
> +    }
> +};
> +
>  const VMStateDescription vmstate_riscv_cpu = {
>      .name = "cpu",
>      .version_id = 1,
> @@ -55,5 +100,9 @@ const VMStateDescription vmstate_riscv_cpu = {
>          VMSTATE_UINTTL(env.mstatush, RISCVCPU),
>  #endif
>          VMSTATE_END_OF_LIST()
> +    },
> +    .subsections = (const VMStateDescription * []) {
> +        &vmstate_pmp,
> +        NULL
>      }
>  };
> --
> 2.19.1
>
>

RE: [PATCH 1/5] target/riscv: Add basic vmstate description of CPU

[Jiangyifei]

> -----Original Message-----
> From: Richard Henderson [mailto:richard.henderson@linaro.org]
> Sent: Friday, October 2, 2020 1:23 AM
> To: Jiangyifei <jiangyifei@huawei.com>; qemu-devel@nongnu.org;
> qemu-riscv@nongnu.org
> Cc: Zhanghailiang <zhang.zhanghailiang@huawei.com>;
> sagark@eecs.berkeley.edu; kbastian@mail.uni-paderborn.de; Zhangxiaofeng
> (F) <victor.zhangxiaofeng@huawei.com>; Alistair.Francis@wdc.com; yinyipeng
> <yinyipeng1@huawei.com>; palmer@dabbelt.com; Wubin (H)
> <wu.wubin@huawei.com>; dengkai (A) <dengkai1@huawei.com>
> Subject: Re: [PATCH 1/5] target/riscv: Add basic vmstate description of CPU
> 
> On 9/28/20 9:03 PM, Yifei Jiang wrote:
> > --- /dev/null
> > +++ b/target/riscv/machine.c
> > @@ -0,0 +1,59 @@
> > +#include "qemu/osdep.h"
> > +#include "cpu.h"
> 
> All new files must contain license boilerplate.
> 
> Otherwise, considering the followups, this seems ok.
> 

Thanks, I'll add license boilerplate in the next series.

Yifei

> 
> r~

RE: [PATCH 1/5] target/riscv: Add basic vmstate description of CPU

[Jiangyifei]

> -----Original Message-----
> From: Richard Henderson [mailto:richard.henderson@linaro.org]
> Sent: Friday, October 2, 2020 1:23 AM
> To: Jiangyifei <jiangyifei@huawei.com>; qemu-devel@nongnu.org;
> qemu-riscv@nongnu.org
> Cc: Zhanghailiang <zhang.zhanghailiang@huawei.com>;
> sagark@eecs.berkeley.edu; kbastian@mail.uni-paderborn.de; Zhangxiaofeng
> (F) <victor.zhangxiaofeng@huawei.com>; Alistair.Francis@wdc.com; yinyipeng
> <yinyipeng1@huawei.com>; palmer@dabbelt.com; Wubin (H)
> <wu.wubin@huawei.com>; dengkai (A) <dengkai1@huawei.com>
> Subject: Re: [PATCH 1/5] target/riscv: Add basic vmstate description of CPU
> 
> On 9/28/20 9:03 PM, Yifei Jiang wrote:
> > --- /dev/null
> > +++ b/target/riscv/machine.c
> > @@ -0,0 +1,59 @@
> > +#include "qemu/osdep.h"
> > +#include "cpu.h"
> 
> All new files must contain license boilerplate.
> 
> Otherwise, considering the followups, this seems ok.
> 

Thanks, I'll add license boilerplate in the next series.

Yifei

> 
> r~

RE: [PATCH 3/5] target/riscv: Add H extention state description

[Jiangyifei]

> -----Original Message-----
> From: Richard Henderson [mailto:richard.henderson@linaro.org]
> Sent: Friday, October 2, 2020 1:28 AM
> To: Jiangyifei <jiangyifei@huawei.com>; qemu-devel@nongnu.org;
> qemu-riscv@nongnu.org
> Cc: Zhanghailiang <zhang.zhanghailiang@huawei.com>;
> sagark@eecs.berkeley.edu; kbastian@mail.uni-paderborn.de; Zhangxiaofeng
> (F) <victor.zhangxiaofeng@huawei.com>; Alistair.Francis@wdc.com; yinyipeng
> <yinyipeng1@huawei.com>; palmer@dabbelt.com; Wubin (H)
> <wu.wubin@huawei.com>; dengkai (A) <dengkai1@huawei.com>
> Subject: Re: [PATCH 3/5] target/riscv: Add H extention state description
> 
> On 9/28/20 9:03 PM, Yifei Jiang wrote:
> > +        VMSTATE_UINTTL(env.vsstatus, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vstvec, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsscratch, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsepc, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vscause, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vstval, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsatp, RISCVCPU),
> 
> So... if I understand things correctly, this is synthetic state, internal to QEMU.
> It is generally better to only serialize architectural state, so that if qemu
> internals are rearranged, it is easy to decide on the correct sequence of
> operations.
> 
> It seems like this should be re-generated with a post_load hook, calling some of
> the code currently in riscv_cpu_swap_hypervisor_regs().  Note that some
> minor rearrangement would be needed to call that code from this new context.
> 
> 
> r~

Thank you for your comments.

As Alistair said, Those vs* are Virtual Supervisor CSRs. Hypervisor may write those.
Actually，except *_hs, all states are real CSRs and cannot be regenerated.
*_hs are backup of Supervisor CSRs when V=1，so, I don't think could re-generate them.

In conclusion, I think all H extension states in this patch need to be described in vmstate.

Yifei

RE: [PATCH 3/5] target/riscv: Add H extention state description

[Jiangyifei]

> -----Original Message-----
> From: Richard Henderson [mailto:richard.henderson@linaro.org]
> Sent: Friday, October 2, 2020 1:28 AM
> To: Jiangyifei <jiangyifei@huawei.com>; qemu-devel@nongnu.org;
> qemu-riscv@nongnu.org
> Cc: Zhanghailiang <zhang.zhanghailiang@huawei.com>;
> sagark@eecs.berkeley.edu; kbastian@mail.uni-paderborn.de; Zhangxiaofeng
> (F) <victor.zhangxiaofeng@huawei.com>; Alistair.Francis@wdc.com; yinyipeng
> <yinyipeng1@huawei.com>; palmer@dabbelt.com; Wubin (H)
> <wu.wubin@huawei.com>; dengkai (A) <dengkai1@huawei.com>
> Subject: Re: [PATCH 3/5] target/riscv: Add H extention state description
> 
> On 9/28/20 9:03 PM, Yifei Jiang wrote:
> > +        VMSTATE_UINTTL(env.vsstatus, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vstvec, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsscratch, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsepc, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vscause, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vstval, RISCVCPU),
> > +        VMSTATE_UINTTL(env.vsatp, RISCVCPU),
> 
> So... if I understand things correctly, this is synthetic state, internal to QEMU.
> It is generally better to only serialize architectural state, so that if qemu
> internals are rearranged, it is easy to decide on the correct sequence of
> operations.
> 
> It seems like this should be re-generated with a post_load hook, calling some of
> the code currently in riscv_cpu_swap_hypervisor_regs().  Note that some
> minor rearrangement would be needed to call that code from this new context.
> 
> 
> r~

Thank you for your comments.

As Alistair said, Those vs* are Virtual Supervisor CSRs. Hypervisor may write those.
Actually，except *_hs, all states are real CSRs and cannot be regenerated.
*_hs are backup of Supervisor CSRs when V=1，so, I don't think could re-generate them.

In conclusion, I think all H extension states in this patch need to be described in vmstate.

Yifei

RE: [PATCH 2/5] target/riscv: Add PMP state description

[Jiangyifei]

> -----Original Message-----
> From: Alistair Francis [mailto:alistair23@gmail.com]
> Sent: Tuesday, October 6, 2020 6:11 AM
> To: Jiangyifei <jiangyifei@huawei.com>
> Cc: qemu-devel@nongnu.org Developers <qemu-devel@nongnu.org>; open
> list:RISC-V <qemu-riscv@nongnu.org>; Zhanghailiang
> <zhang.zhanghailiang@huawei.com>; Sagar Karandikar
> <sagark@eecs.berkeley.edu>; Bastian Koppelmann
> <kbastian@mail.uni-paderborn.de>; Zhangxiaofeng (F)
> <victor.zhangxiaofeng@huawei.com>; Alistair Francis
> <Alistair.Francis@wdc.com>; yinyipeng <yinyipeng1@huawei.com>; Palmer
> Dabbelt <palmer@dabbelt.com>; Wubin (H) <wu.wubin@huawei.com>;
> dengkai (A) <dengkai1@huawei.com>
> Subject: Re: [PATCH 2/5] target/riscv: Add PMP state description
> 
> On Mon, Sep 28, 2020 at 7:05 PM Yifei Jiang <jiangyifei@huawei.com> wrote:
> >
> > In the case of supporting PMP feature, add PMP state description to
> > vmstate_riscv_cpu.
> >
> > Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
> > Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
> > ---
> >  target/riscv/machine.c | 49
> > ++++++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 49 insertions(+)
> >
> > diff --git a/target/riscv/machine.c b/target/riscv/machine.c index
> > 3451b888b1..b1fc839b43 100644
> > --- a/target/riscv/machine.c
> > +++ b/target/riscv/machine.c
> > @@ -4,6 +4,51 @@
> >  #include "sysemu/kvm.h"
> >  #include "migration/cpu.h"
> >
> > +static bool pmp_needed(void *opaque)
> > +{
> > +    RISCVCPU *cpu = opaque;
> > +    CPURISCVState *env = &cpu->env;
> > +
> > +    return riscv_feature(env, RISCV_FEATURE_PMP); }
> > +
> > +static const VMStateDescription vmstate_pmp_entry = {
> > +    .name = "cpu/pmp/entry",
> > +    .version_id = 1,
> > +    .minimum_version_id = 1,
> > +    .fields = (VMStateField[]) {
> > +        VMSTATE_UINTTL(addr_reg, pmp_entry_t),
> > +        VMSTATE_UINT8(cfg_reg, pmp_entry_t),
> > +        VMSTATE_END_OF_LIST()
> > +    }
> > +};
> > +
> > +static const VMStateDescription vmstate_pmp_addr = {
> > +    .name = "cpu/pmp/addr",
> > +    .version_id = 1,
> > +    .minimum_version_id = 1,
> > +    .fields = (VMStateField[]) {
> > +        VMSTATE_UINTTL(sa, pmp_addr_t),
> > +        VMSTATE_UINTTL(ea, pmp_addr_t),
> > +        VMSTATE_END_OF_LIST()
> > +    }
> > +};
> > +
> > +static const VMStateDescription vmstate_pmp = {
> > +    .name = "cpu/pmp",
> > +    .version_id = 1,
> > +    .minimum_version_id = 1,
> > +    .needed = pmp_needed,
> > +    .fields = (VMStateField[]) {
> > +        VMSTATE_STRUCT_ARRAY(env.pmp_state.pmp, RISCVCPU,
> MAX_RISCV_PMPS,
> > +                             0, vmstate_pmp_entry, pmp_entry_t),
> > +        VMSTATE_STRUCT_ARRAY(env.pmp_state.addr, RISCVCPU,
> MAX_RISCV_PMPS,
> > +                             0, vmstate_pmp_addr, pmp_addr_t),
> > +        VMSTATE_UINT32(env.pmp_state.num_rules, RISCVCPU),
> 
> I would prefer to re-generate these from the PMP CSRs post load.
> 
> Alistair
> 

Thanks for your advice. Yes, you are right.
vmstate_pmp_entry cannot be regenerated, but vmstate_pmp_addr and num_rules could be regenerated.

I'll add post_load hook for regenerating vmstate_pmp_addr and num_rules in the next series.

Yifei

> > +        VMSTATE_END_OF_LIST()
> > +    }
> > +};
> > +
> >  const VMStateDescription vmstate_riscv_cpu = {
> >      .name = "cpu",
> >      .version_id = 1,
> > @@ -55,5 +100,9 @@ const VMStateDescription vmstate_riscv_cpu = {
> >          VMSTATE_UINTTL(env.mstatush, RISCVCPU),  #endif
> >          VMSTATE_END_OF_LIST()
> > +    },
> > +    .subsections = (const VMStateDescription * []) {
> > +        &vmstate_pmp,
> > +        NULL
> >      }
> >  };
> > --
> > 2.19.1
> >
> >

RE: [PATCH 2/5] target/riscv: Add PMP state description

[Jiangyifei]

> -----Original Message-----
> From: Alistair Francis [mailto:alistair23@gmail.com]
> Sent: Tuesday, October 6, 2020 6:11 AM
> To: Jiangyifei <jiangyifei@huawei.com>
> Cc: qemu-devel@nongnu.org Developers <qemu-devel@nongnu.org>; open
> list:RISC-V <qemu-riscv@nongnu.org>; Zhanghailiang
> <zhang.zhanghailiang@huawei.com>; Sagar Karandikar
> <sagark@eecs.berkeley.edu>; Bastian Koppelmann
> <kbastian@mail.uni-paderborn.de>; Zhangxiaofeng (F)
> <victor.zhangxiaofeng@huawei.com>; Alistair Francis
> <Alistair.Francis@wdc.com>; yinyipeng <yinyipeng1@huawei.com>; Palmer
> Dabbelt <palmer@dabbelt.com>; Wubin (H) <wu.wubin@huawei.com>;
> dengkai (A) <dengkai1@huawei.com>
> Subject: Re: [PATCH 2/5] target/riscv: Add PMP state description
> 
> On Mon, Sep 28, 2020 at 7:05 PM Yifei Jiang <jiangyifei@huawei.com> wrote:
> >
> > In the case of supporting PMP feature, add PMP state description to
> > vmstate_riscv_cpu.
> >
> > Signed-off-by: Yifei Jiang <jiangyifei@huawei.com>
> > Signed-off-by: Yipeng Yin <yinyipeng1@huawei.com>
> > ---
> >  target/riscv/machine.c | 49
> > ++++++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 49 insertions(+)
> >
> > diff --git a/target/riscv/machine.c b/target/riscv/machine.c index
> > 3451b888b1..b1fc839b43 100644
> > --- a/target/riscv/machine.c
> > +++ b/target/riscv/machine.c
> > @@ -4,6 +4,51 @@
> >  #include "sysemu/kvm.h"
> >  #include "migration/cpu.h"
> >
> > +static bool pmp_needed(void *opaque)
> > +{
> > +    RISCVCPU *cpu = opaque;
> > +    CPURISCVState *env = &cpu->env;
> > +
> > +    return riscv_feature(env, RISCV_FEATURE_PMP); }
> > +
> > +static const VMStateDescription vmstate_pmp_entry = {
> > +    .name = "cpu/pmp/entry",
> > +    .version_id = 1,
> > +    .minimum_version_id = 1,
> > +    .fields = (VMStateField[]) {
> > +        VMSTATE_UINTTL(addr_reg, pmp_entry_t),
> > +        VMSTATE_UINT8(cfg_reg, pmp_entry_t),
> > +        VMSTATE_END_OF_LIST()
> > +    }
> > +};
> > +
> > +static const VMStateDescription vmstate_pmp_addr = {
> > +    .name = "cpu/pmp/addr",
> > +    .version_id = 1,
> > +    .minimum_version_id = 1,
> > +    .fields = (VMStateField[]) {
> > +        VMSTATE_UINTTL(sa, pmp_addr_t),
> > +        VMSTATE_UINTTL(ea, pmp_addr_t),
> > +        VMSTATE_END_OF_LIST()
> > +    }
> > +};
> > +
> > +static const VMStateDescription vmstate_pmp = {
> > +    .name = "cpu/pmp",
> > +    .version_id = 1,
> > +    .minimum_version_id = 1,
> > +    .needed = pmp_needed,
> > +    .fields = (VMStateField[]) {
> > +        VMSTATE_STRUCT_ARRAY(env.pmp_state.pmp, RISCVCPU,
> MAX_RISCV_PMPS,
> > +                             0, vmstate_pmp_entry, pmp_entry_t),
> > +        VMSTATE_STRUCT_ARRAY(env.pmp_state.addr, RISCVCPU,
> MAX_RISCV_PMPS,
> > +                             0, vmstate_pmp_addr, pmp_addr_t),
> > +        VMSTATE_UINT32(env.pmp_state.num_rules, RISCVCPU),
> 
> I would prefer to re-generate these from the PMP CSRs post load.
> 
> Alistair
> 

Thanks for your advice. Yes, you are right.
vmstate_pmp_entry cannot be regenerated, but vmstate_pmp_addr and num_rules could be regenerated.

I'll add post_load hook for regenerating vmstate_pmp_addr and num_rules in the next series.

Yifei

> > +        VMSTATE_END_OF_LIST()
> > +    }
> > +};
> > +
> >  const VMStateDescription vmstate_riscv_cpu = {
> >      .name = "cpu",
> >      .version_id = 1,
> > @@ -55,5 +100,9 @@ const VMStateDescription vmstate_riscv_cpu = {
> >          VMSTATE_UINTTL(env.mstatush, RISCVCPU),  #endif
> >          VMSTATE_END_OF_LIST()
> > +    },
> > +    .subsections = (const VMStateDescription * []) {
> > +        &vmstate_pmp,
> > +        NULL
> >      }
> >  };
> > --
> > 2.19.1
> >
> >