[kvmtool RFC PATCH 0/8] KVMTOOL RISC-V support

[kvmtool RFC PATCH 0/8] KVMTOOL RISC-V support

[Anup Patel]

This series adds RISC-V support for KVMTOOL and it is based on
the v10 of KVM RISC-V series. The KVM RISC-V patches are not yet
merged in Linux kernel but it will be good to get early review
for KVMTOOL RISC-V support.

The KVMTOOL RISC-V patches can be found in riscv_master branch at:
https//github.com/kvm-riscv/kvmtool.git

The KVM RISC-V patches can be found in riscv_kvm_master branch at:
https//github.com/kvm-riscv/linux.git

The QEMU RISC-V hypervisor emulation is done by Alistair and is
available in mainline/alistair/riscv-hyp-ext-v0.5.1 branch at:
https://github.com/kvm-riscv/qemu.git

Anup Patel (8):
  update_headers: Sync-up ABI headers with Linux-5.5-rc3
  riscv: Initial skeletal support
  riscv: Implement Guest/VM arch functions
  riscv: Implement Guest/VM VCPU arch functions
  riscv: Add PLIC device emulation
  riscv: Generate FDT at runtime for Guest/VM
  riscv: Handle SBI calls forwarded to user space
  riscv: Generate PCI host DT node

 INSTALL                             |   7 +-
 Makefile                            |  18 +-
 arm/aarch32/include/asm/kvm.h       |   7 +-
 arm/aarch64/include/asm/kvm.h       |   9 +-
 include/linux/kvm.h                 |  25 ++
 powerpc/include/asm/kvm.h           |   3 +
 riscv/fdt.c                         | 195 ++++++++++
 riscv/include/asm/kvm.h             | 127 +++++++
 riscv/include/kvm/barrier.h         |  14 +
 riscv/include/kvm/fdt-arch.h        |   8 +
 riscv/include/kvm/kvm-arch.h        |  79 ++++
 riscv/include/kvm/kvm-config-arch.h |  15 +
 riscv/include/kvm/kvm-cpu-arch.h    |  53 +++
 riscv/include/kvm/sbi.h             |  48 +++
 riscv/ioport.c                      |  11 +
 riscv/irq.c                         |  13 +
 riscv/kvm-cpu.c                     | 404 ++++++++++++++++++++
 riscv/kvm.c                         | 175 +++++++++
 riscv/pci.c                         | 109 ++++++
 riscv/plic.c                        | 558 ++++++++++++++++++++++++++++
 util/update_headers.sh              |   2 +-
 21 files changed, 1871 insertions(+), 9 deletions(-)
 create mode 100644 riscv/fdt.c
 create mode 100644 riscv/include/asm/kvm.h
 create mode 100644 riscv/include/kvm/barrier.h
 create mode 100644 riscv/include/kvm/fdt-arch.h
 create mode 100644 riscv/include/kvm/kvm-arch.h
 create mode 100644 riscv/include/kvm/kvm-config-arch.h
 create mode 100644 riscv/include/kvm/kvm-cpu-arch.h
 create mode 100644 riscv/include/kvm/sbi.h
 create mode 100644 riscv/ioport.c
 create mode 100644 riscv/irq.c
 create mode 100644 riscv/kvm-cpu.c
 create mode 100644 riscv/kvm.c
 create mode 100644 riscv/pci.c
 create mode 100644 riscv/plic.c

-- 
2.17.1

[kvmtool RFC PATCH 1/8] update_headers: Sync-up ABI headers with Linux-5.5-rc3

[Anup Patel]

We sync-up all ABI headers with Linux-5.5-rc3. This will allow
us to use ONE_REG interface for KVMTOOL RISC-V port.

Signed-off-by: Anup Patel <anup.patel@wdc.com>
---
 arm/aarch32/include/asm/kvm.h |  7 +++++--
 arm/aarch64/include/asm/kvm.h |  9 +++++++--
 include/linux/kvm.h           | 25 +++++++++++++++++++++++++
 powerpc/include/asm/kvm.h     |  3 +++
 4 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/arm/aarch32/include/asm/kvm.h b/arm/aarch32/include/asm/kvm.h
index a4217c1..03cd7c1 100644
--- a/arm/aarch32/include/asm/kvm.h
+++ b/arm/aarch32/include/asm/kvm.h
@@ -131,8 +131,9 @@ struct kvm_vcpu_events {
 	struct {
 		__u8 serror_pending;
 		__u8 serror_has_esr;
+		__u8 ext_dabt_pending;
 		/* Align it to 8 bytes */
-		__u8 pad[6];
+		__u8 pad[5];
 		__u64 serror_esr;
 	} exception;
 	__u32 reserved[12];
@@ -266,8 +267,10 @@ struct kvm_vcpu_events {
 #define   KVM_DEV_ARM_ITS_CTRL_RESET		4
 
 /* KVM_IRQ_LINE irq field index values */
+#define KVM_ARM_IRQ_VCPU2_SHIFT		28
+#define KVM_ARM_IRQ_VCPU2_MASK		0xf
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
-#define KVM_ARM_IRQ_TYPE_MASK		0xff
+#define KVM_ARM_IRQ_TYPE_MASK		0xf
 #define KVM_ARM_IRQ_VCPU_SHIFT		16
 #define KVM_ARM_IRQ_VCPU_MASK		0xff
 #define KVM_ARM_IRQ_NUM_SHIFT		0
diff --git a/arm/aarch64/include/asm/kvm.h b/arm/aarch64/include/asm/kvm.h
index 9a50771..820e575 100644
--- a/arm/aarch64/include/asm/kvm.h
+++ b/arm/aarch64/include/asm/kvm.h
@@ -164,8 +164,9 @@ struct kvm_vcpu_events {
 	struct {
 		__u8 serror_pending;
 		__u8 serror_has_esr;
+		__u8 ext_dabt_pending;
 		/* Align it to 8 bytes */
-		__u8 pad[6];
+		__u8 pad[5];
 		__u64 serror_esr;
 	} exception;
 	__u32 reserved[12];
@@ -323,10 +324,14 @@ struct kvm_vcpu_events {
 #define KVM_ARM_VCPU_TIMER_CTRL		1
 #define   KVM_ARM_VCPU_TIMER_IRQ_VTIMER		0
 #define   KVM_ARM_VCPU_TIMER_IRQ_PTIMER		1
+#define KVM_ARM_VCPU_PVTIME_CTRL	2
+#define   KVM_ARM_VCPU_PVTIME_IPA	0
 
 /* KVM_IRQ_LINE irq field index values */
+#define KVM_ARM_IRQ_VCPU2_SHIFT		28
+#define KVM_ARM_IRQ_VCPU2_MASK		0xf
 #define KVM_ARM_IRQ_TYPE_SHIFT		24
-#define KVM_ARM_IRQ_TYPE_MASK		0xff
+#define KVM_ARM_IRQ_TYPE_MASK		0xf
 #define KVM_ARM_IRQ_VCPU_SHIFT		16
 #define KVM_ARM_IRQ_VCPU_MASK		0xff
 #define KVM_ARM_IRQ_NUM_SHIFT		0
diff --git a/include/linux/kvm.h b/include/linux/kvm.h
index 5e3f12d..b6a90dd 100644
--- a/include/linux/kvm.h
+++ b/include/linux/kvm.h
@@ -235,6 +235,8 @@ struct kvm_hyperv_exit {
 #define KVM_EXIT_S390_STSI        25
 #define KVM_EXIT_IOAPIC_EOI       26
 #define KVM_EXIT_HYPERV           27
+#define KVM_EXIT_ARM_NISV         28
+#define KVM_EXIT_RISCV_SBI        28
 
 /* For KVM_EXIT_INTERNAL_ERROR */
 /* Emulate instruction failed. */
@@ -243,6 +245,8 @@ struct kvm_hyperv_exit {
 #define KVM_INTERNAL_ERROR_SIMUL_EX	2
 /* Encounter unexpected vm-exit due to delivery event. */
 #define KVM_INTERNAL_ERROR_DELIVERY_EV	3
+/* Encounter unexpected vm-exit reason */
+#define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON	4
 
 /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */
 struct kvm_run {
@@ -392,6 +396,18 @@ struct kvm_run {
 		} eoi;
 		/* KVM_EXIT_HYPERV */
 		struct kvm_hyperv_exit hyperv;
+		/* KVM_EXIT_ARM_NISV */
+		struct {
+			__u64 esr_iss;
+			__u64 fault_ipa;
+		} arm_nisv;
+		/* KVM_EXIT_RISCV_SBI */
+		struct {
+			unsigned long extension_id;
+			unsigned long function_id;
+			unsigned long args[6];
+			unsigned long ret[2];
+		} riscv_sbi;
 		/* Fix the size of the union. */
 		char padding[256];
 	};
@@ -996,6 +1012,11 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_ARM_PTRAUTH_ADDRESS 171
 #define KVM_CAP_ARM_PTRAUTH_GENERIC 172
 #define KVM_CAP_PMU_EVENT_FILTER 173
+#define KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 174
+#define KVM_CAP_HYPERV_DIRECT_TLBFLUSH 175
+#define KVM_CAP_PPC_GUEST_DEBUG_SSTEP 176
+#define KVM_CAP_ARM_NISV_TO_USER 177
+#define KVM_CAP_ARM_INJECT_EXT_DABT 178
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -1142,6 +1163,7 @@ struct kvm_dirty_tlb {
 #define KVM_REG_S390		0x5000000000000000ULL
 #define KVM_REG_ARM64		0x6000000000000000ULL
 #define KVM_REG_MIPS		0x7000000000000000ULL
+#define KVM_REG_RISCV		0x8000000000000000ULL
 
 #define KVM_REG_SIZE_SHIFT	52
 #define KVM_REG_SIZE_MASK	0x00f0000000000000ULL
@@ -1222,6 +1244,8 @@ enum kvm_device_type {
 #define KVM_DEV_TYPE_ARM_VGIC_ITS	KVM_DEV_TYPE_ARM_VGIC_ITS
 	KVM_DEV_TYPE_XIVE,
 #define KVM_DEV_TYPE_XIVE		KVM_DEV_TYPE_XIVE
+	KVM_DEV_TYPE_ARM_PV_TIME,
+#define KVM_DEV_TYPE_ARM_PV_TIME	KVM_DEV_TYPE_ARM_PV_TIME
 	KVM_DEV_TYPE_MAX,
 };
 
@@ -1332,6 +1356,7 @@ struct kvm_s390_ucas_mapping {
 #define KVM_PPC_GET_CPU_CHAR	  _IOR(KVMIO,  0xb1, struct kvm_ppc_cpu_char)
 /* Available with KVM_CAP_PMU_EVENT_FILTER */
 #define KVM_SET_PMU_EVENT_FILTER  _IOW(KVMIO,  0xb2, struct kvm_pmu_event_filter)
+#define KVM_PPC_SVM_OFF		  _IO(KVMIO,  0xb3)
 
 /* ioctl for vm fd */
 #define KVM_CREATE_DEVICE	  _IOWR(KVMIO,  0xe0, struct kvm_create_device)
diff --git a/powerpc/include/asm/kvm.h b/powerpc/include/asm/kvm.h
index b0f72de..264e266 100644
--- a/powerpc/include/asm/kvm.h
+++ b/powerpc/include/asm/kvm.h
@@ -667,6 +667,8 @@ struct kvm_ppc_cpu_char {
 
 /* PPC64 eXternal Interrupt Controller Specification */
 #define KVM_DEV_XICS_GRP_SOURCES	1	/* 64-bit source attributes */
+#define KVM_DEV_XICS_GRP_CTRL		2
+#define   KVM_DEV_XICS_NR_SERVERS	1
 
 /* Layout of 64-bit source attribute values */
 #define  KVM_XICS_DESTINATION_SHIFT	0
@@ -683,6 +685,7 @@ struct kvm_ppc_cpu_char {
 #define KVM_DEV_XIVE_GRP_CTRL		1
 #define   KVM_DEV_XIVE_RESET		1
 #define   KVM_DEV_XIVE_EQ_SYNC		2
+#define   KVM_DEV_XIVE_NR_SERVERS	3
 #define KVM_DEV_XIVE_GRP_SOURCE		2	/* 64-bit source identifier */
 #define KVM_DEV_XIVE_GRP_SOURCE_CONFIG	3	/* 64-bit source identifier */
 #define KVM_DEV_XIVE_GRP_EQ_CONFIG	4	/* 64-bit EQ identifier */
-- 
2.17.1

[kvmtool RFC PATCH 2/8] riscv: Initial skeletal support

[Anup Patel]

This patch adds initial skeletal KVMTOOL RISC-V support which
just compiles for RV32 and RV64 host.

Signed-off-by: Anup Patel <anup.patel@wdc.com>
---
 INSTALL                             |   7 +-
 Makefile                            |  15 +++-
 riscv/include/asm/kvm.h             | 127 ++++++++++++++++++++++++++++
 riscv/include/kvm/barrier.h         |  14 +++
 riscv/include/kvm/fdt-arch.h        |   4 +
 riscv/include/kvm/kvm-arch.h        |  58 +++++++++++++
 riscv/include/kvm/kvm-config-arch.h |   9 ++
 riscv/include/kvm/kvm-cpu-arch.h    |  49 +++++++++++
 riscv/ioport.c                      |  11 +++
 riscv/irq.c                         |  13 +++
 riscv/kvm-cpu.c                     |  64 ++++++++++++++
 riscv/kvm.c                         |  61 +++++++++++++
 util/update_headers.sh              |   2 +-
 13 files changed, 429 insertions(+), 5 deletions(-)
 create mode 100644 riscv/include/asm/kvm.h
 create mode 100644 riscv/include/kvm/barrier.h
 create mode 100644 riscv/include/kvm/fdt-arch.h
 create mode 100644 riscv/include/kvm/kvm-arch.h
 create mode 100644 riscv/include/kvm/kvm-config-arch.h
 create mode 100644 riscv/include/kvm/kvm-cpu-arch.h
 create mode 100644 riscv/ioport.c
 create mode 100644 riscv/irq.c
 create mode 100644 riscv/kvm-cpu.c
 create mode 100644 riscv/kvm.c

diff --git a/INSTALL b/INSTALL
index ca8e022..951b123 100644
--- a/INSTALL
+++ b/INSTALL
@@ -26,8 +26,8 @@ For Fedora based systems:
 For OpenSUSE based systems:
 	# zypper install glibc-devel-static
 
-Architectures which require device tree (PowerPC, ARM, ARM64) also require
-libfdt.
+Architectures which require device tree (PowerPC, ARM, ARM64, RISC-V) also
+require libfdt.
 	deb: $ sudo apt-get install libfdt-dev
 	Fedora: # yum install libfdt-devel
 	OpenSUSE: # zypper install libfdt1-devel
@@ -64,6 +64,7 @@ to the Linux name of the architecture. Architectures supported:
 - arm
 - arm64
 - mips
+- riscv
 If ARCH is not provided, the target architecture will be automatically
 determined by running "uname -m" on your host, resulting in a native build.
 
@@ -81,7 +82,7 @@ On multiarch system you should be able to install those be appending
 the architecture name after the package (example for ARM64):
 $ sudo apt-get install libfdt-dev:arm64
 
-PowerPC and ARM/ARM64 require libfdt to be installed. If you cannot use
+PowerPC, ARM/ARM64 and RISC-V require libfdt to be installed. If you cannot use
 precompiled mulitarch packages, you could either copy the required header and
 library files from an installed target system into the SYSROOT (you will need
 /usr/include/*fdt*.h and /usr/lib64/libfdt-v.v.v.so and its symlinks), or you
diff --git a/Makefile b/Makefile
index 3862112..972fa63 100644
--- a/Makefile
+++ b/Makefile
@@ -106,7 +106,8 @@ OBJS	+= hw/i8042.o
 
 # Translate uname -m into ARCH string
 ARCH ?= $(shell uname -m | sed -e s/i.86/i386/ -e s/ppc.*/powerpc/ \
-	  -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/)
+	  -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/ \
+	  -e s/riscv64/riscv/ -e s/riscv32/riscv/)
 
 ifeq ($(ARCH),i386)
 	ARCH         := x86
@@ -190,6 +191,18 @@ ifeq ($(ARCH),mips)
 	OBJS		+= mips/kvm.o
 	OBJS		+= mips/kvm-cpu.o
 endif
+
+# RISC-V (RV32 and RV64)
+ifeq ($(ARCH),riscv)
+	DEFINES		+= -DCONFIG_RISCV
+	ARCH_INCLUDE	:= riscv/include
+	OBJS		+= riscv/ioport.o
+	OBJS		+= riscv/irq.o
+	OBJS		+= riscv/kvm.o
+	OBJS		+= riscv/kvm-cpu.o
+
+	ARCH_WANT_LIBFDT := y
+endif
 ###
 
 ifeq (,$(ARCH_INCLUDE))
diff --git a/riscv/include/asm/kvm.h b/riscv/include/asm/kvm.h
new file mode 100644
index 0000000..f4274c2
--- /dev/null
+++ b/riscv/include/asm/kvm.h
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2019 Western Digital Corporation or its affiliates.
+ *
+ * Authors:
+ *     Anup Patel <anup.patel@wdc.com>
+ */
+
+#ifndef __LINUX_KVM_RISCV_H
+#define __LINUX_KVM_RISCV_H
+
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+#include <asm/ptrace.h>
+
+#define __KVM_HAVE_READONLY_MEM
+
+#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
+
+#define KVM_INTERRUPT_SET	-1U
+#define KVM_INTERRUPT_UNSET	-2U
+
+/* for KVM_GET_REGS and KVM_SET_REGS */
+struct kvm_regs {
+};
+
+/* for KVM_GET_FPU and KVM_SET_FPU */
+struct kvm_fpu {
+};
+
+/* KVM Debug exit structure */
+struct kvm_debug_exit_arch {
+};
+
+/* for KVM_SET_GUEST_DEBUG */
+struct kvm_guest_debug_arch {
+};
+
+/* definition of registers in kvm_run */
+struct kvm_sync_regs {
+};
+
+/* for KVM_GET_SREGS and KVM_SET_SREGS */
+struct kvm_sregs {
+};
+
+/* CONFIG registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
+struct kvm_riscv_config {
+	unsigned long isa;
+};
+
+/* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
+struct kvm_riscv_core {
+	struct user_regs_struct regs;
+	unsigned long mode;
+};
+
+/* Possible privilege modes for kvm_riscv_core */
+#define KVM_RISCV_MODE_S	1
+#define KVM_RISCV_MODE_U	0
+
+/* CSR registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
+struct kvm_riscv_csr {
+	unsigned long sstatus;
+	unsigned long sie;
+	unsigned long stvec;
+	unsigned long sscratch;
+	unsigned long sepc;
+	unsigned long scause;
+	unsigned long stval;
+	unsigned long sip;
+	unsigned long satp;
+};
+
+/* TIMER registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
+struct kvm_riscv_timer {
+	u64 frequency;
+	u64 time;
+	u64 compare;
+	u64 state;
+};
+
+/* Possible states for kvm_riscv_timer */
+#define KVM_RISCV_TIMER_STATE_OFF	0
+#define KVM_RISCV_TIMER_STATE_ON	1
+
+#define KVM_REG_SIZE(id)		\
+	(1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
+
+/* If you need to interpret the index values, here is the key: */
+#define KVM_REG_RISCV_TYPE_MASK		0x00000000FF000000
+#define KVM_REG_RISCV_TYPE_SHIFT	24
+
+/* Config registers are mapped as type 1 */
+#define KVM_REG_RISCV_CONFIG		(0x01 << KVM_REG_RISCV_TYPE_SHIFT)
+#define KVM_REG_RISCV_CONFIG_REG(name)	\
+	(offsetof(struct kvm_riscv_config, name) / sizeof(unsigned long))
+
+/* Core registers are mapped as type 2 */
+#define KVM_REG_RISCV_CORE		(0x02 << KVM_REG_RISCV_TYPE_SHIFT)
+#define KVM_REG_RISCV_CORE_REG(name)	\
+		(offsetof(struct kvm_riscv_core, name) / sizeof(unsigned long))
+
+/* Control and status registers are mapped as type 3 */
+#define KVM_REG_RISCV_CSR		(0x03 << KVM_REG_RISCV_TYPE_SHIFT)
+#define KVM_REG_RISCV_CSR_REG(name)	\
+		(offsetof(struct kvm_riscv_csr, name) / sizeof(unsigned long))
+
+/* Timer registers are mapped as type 4 */
+#define KVM_REG_RISCV_TIMER		(0x04 << KVM_REG_RISCV_TYPE_SHIFT)
+#define KVM_REG_RISCV_TIMER_REG(name)	\
+		(offsetof(struct kvm_riscv_timer, name) / sizeof(u64))
+
+/* F extension registers are mapped as type 5 */
+#define KVM_REG_RISCV_FP_F		(0x05 << KVM_REG_RISCV_TYPE_SHIFT)
+#define KVM_REG_RISCV_FP_F_REG(name)	\
+		(offsetof(struct __riscv_f_ext_state, name) / sizeof(u32))
+
+/* D extension registers are mapped as type 6 */
+#define KVM_REG_RISCV_FP_D		(0x06 << KVM_REG_RISCV_TYPE_SHIFT)
+#define KVM_REG_RISCV_FP_D_REG(name)	\
+		(offsetof(struct __riscv_d_ext_state, name) / sizeof(u64))
+
+#endif
+
+#endif /* __LINUX_KVM_RISCV_H */
diff --git a/riscv/include/kvm/barrier.h b/riscv/include/kvm/barrier.h
new file mode 100644
index 0000000..235f610
--- /dev/null
+++ b/riscv/include/kvm/barrier.h
@@ -0,0 +1,14 @@
+#ifndef KVM__KVM_BARRIER_H
+#define KVM__KVM_BARRIER_H
+
+#define nop()		__asm__ __volatile__ ("nop")
+
+#define RISCV_FENCE(p, s) \
+	__asm__ __volatile__ ("fence " #p "," #s : : : "memory")
+
+/* These barriers need to enforce ordering on both devices or memory. */
+#define mb()		RISCV_FENCE(iorw,iorw)
+#define rmb()		RISCV_FENCE(ir,ir)
+#define wmb()		RISCV_FENCE(ow,ow)
+
+#endif /* KVM__KVM_BARRIER_H */
diff --git a/riscv/include/kvm/fdt-arch.h b/riscv/include/kvm/fdt-arch.h
new file mode 100644
index 0000000..9450fc5
--- /dev/null
+++ b/riscv/include/kvm/fdt-arch.h
@@ -0,0 +1,4 @@
+#ifndef KVM__KVM_FDT_H
+#define KVM__KVM_FDT_H
+
+#endif /* KVM__KVM_FDT_H */
diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
new file mode 100644
index 0000000..7e9c578
--- /dev/null
+++ b/riscv/include/kvm/kvm-arch.h
@@ -0,0 +1,58 @@
+#ifndef KVM__KVM_ARCH_H
+#define KVM__KVM_ARCH_H
+
+#include <stdbool.h>
+#include <linux/const.h>
+#include <linux/types.h>
+
+#define RISCV_IOPORT		0x00000000ULL
+#define RISCV_IOPORT_SIZE	0x00010000ULL
+#define RISCV_PLIC		0x0c000000ULL
+#define RISCV_PLIC_SIZE		0x04000000ULL
+#define RISCV_MMIO		0x10000000ULL
+#define RISCV_MMIO_SIZE		0x20000000ULL
+#define RISCV_PCI		0x30000000ULL
+#define RISCV_PCI_CFG_SIZE	0x10000000ULL
+#define RISCV_PCI_SIZE		0x50000000ULL
+#define RISCV_PCI_MMIO_SIZE	(RISCV_PCI_SIZE - RISCV_PCI_CFG_SIZE)
+
+#define RISCV_RAM		0x80000000ULL
+
+#define RISCV_LOMAP_MAX_MEMORY	((1ULL << 32) - RISCV_RAM)
+#define RISCV_HIMAP_MAX_MEMORY	((1ULL << 40) - RISCV_RAM)
+
+#if __riscv_xlen == 64
+#define RISCV_MAX_MEMORY(kvm)	RISCV_HIMAP_MAX_MEMORY
+#elif __riscv_xlen == 32
+#define RISCV_MAX_MEMORY(kvm)	RISCV_LOMAP_MAX_MEMORY
+#endif
+
+#define KVM_IOPORT_AREA		RISCV_IOPORT
+#define KVM_PCI_CFG_AREA	RISCV_PCI
+#define KVM_PCI_MMIO_AREA	(KVM_PCI_CFG_AREA + RISCV_PCI_CFG_SIZE)
+#define KVM_VIRTIO_MMIO_AREA	RISCV_MMIO
+
+#define KVM_IOEVENTFD_HAS_PIO	0
+
+#define KVM_IRQ_OFFSET		0
+
+#define KVM_VM_TYPE		0
+
+#define VIRTIO_DEFAULT_TRANS(kvm)	VIRTIO_MMIO
+
+#define VIRTIO_RING_ENDIAN	VIRTIO_ENDIAN_LE
+
+struct kvm;
+
+struct kvm_arch {
+};
+
+static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
+{
+	u64 limit = KVM_IOPORT_AREA + RISCV_IOPORT_SIZE;
+	return phys_addr >= KVM_IOPORT_AREA && phys_addr < limit;
+}
+
+enum irq_type;
+
+#endif /* KVM__KVM_ARCH_H */
diff --git a/riscv/include/kvm/kvm-config-arch.h b/riscv/include/kvm/kvm-config-arch.h
new file mode 100644
index 0000000..60c7333
--- /dev/null
+++ b/riscv/include/kvm/kvm-config-arch.h
@@ -0,0 +1,9 @@
+#ifndef KVM__KVM_CONFIG_ARCH_H
+#define KVM__KVM_CONFIG_ARCH_H
+
+#include "kvm/parse-options.h"
+
+struct kvm_config_arch {
+};
+
+#endif /* KVM__KVM_CONFIG_ARCH_H */
diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
new file mode 100644
index 0000000..09a50e8
--- /dev/null
+++ b/riscv/include/kvm/kvm-cpu-arch.h
@@ -0,0 +1,49 @@
+#ifndef KVM__KVM_CPU_ARCH_H
+#define KVM__KVM_CPU_ARCH_H
+
+#include <linux/kvm.h>
+#include <pthread.h>
+#include <stdbool.h>
+
+#include "kvm/kvm.h"
+
+struct kvm;
+
+struct kvm_cpu {
+	pthread_t	thread;
+
+	unsigned long   cpu_id;
+
+	struct kvm	*kvm;
+	int		vcpu_fd;
+	struct kvm_run	*kvm_run;
+	struct kvm_cpu_task	*task;
+
+	u8		is_running;
+	u8		paused;
+	u8		needs_nmi;
+
+	struct kvm_coalesced_mmio_ring	*ring;
+};
+
+static inline bool kvm_cpu__emulate_io(struct kvm_cpu *vcpu, u16 port,
+				       void *data, int direction,
+				       int size, u32 count)
+{
+	return false;
+}
+
+static inline bool kvm_cpu__emulate_mmio(struct kvm_cpu *vcpu, u64 phys_addr,
+					 u8 *data, u32 len, u8 is_write)
+{
+	if (riscv_addr_in_ioport_region(phys_addr)) {
+		int direction = is_write ? KVM_EXIT_IO_OUT : KVM_EXIT_IO_IN;
+		u16 port = (phys_addr - KVM_IOPORT_AREA) & USHRT_MAX;
+
+		return kvm__emulate_io(vcpu, port, data, direction, len, 1);
+	}
+
+	return kvm__emulate_mmio(vcpu, phys_addr, data, len, is_write);
+}
+
+#endif /* KVM__KVM_CPU_ARCH_H */
diff --git a/riscv/ioport.c b/riscv/ioport.c
new file mode 100644
index 0000000..bdd30b6
--- /dev/null
+++ b/riscv/ioport.c
@@ -0,0 +1,11 @@
+#include "kvm/ioport.h"
+#include "kvm/irq.h"
+
+void ioport__setup_arch(struct kvm *kvm)
+{
+}
+
+void ioport__map_irq(u8 *irq)
+{
+	*irq = irq__alloc_line();
+}
diff --git a/riscv/irq.c b/riscv/irq.c
new file mode 100644
index 0000000..8e605ef
--- /dev/null
+++ b/riscv/irq.c
@@ -0,0 +1,13 @@
+#include "kvm/kvm.h"
+#include "kvm/kvm-cpu.h"
+#include "kvm/irq.h"
+
+void kvm__irq_line(struct kvm *kvm, int irq, int level)
+{
+	/* TODO: */
+}
+
+void kvm__irq_trigger(struct kvm *kvm, int irq)
+{
+	/* TODO: */
+}
diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
new file mode 100644
index 0000000..e4b8fa5
--- /dev/null
+++ b/riscv/kvm-cpu.c
@@ -0,0 +1,64 @@
+#include "kvm/kvm-cpu.h"
+#include "kvm/kvm.h"
+#include "kvm/virtio.h"
+#include "kvm/term.h"
+
+#include <asm/ptrace.h>
+
+static int debug_fd;
+
+void kvm_cpu__set_debug_fd(int fd)
+{
+	debug_fd = fd;
+}
+
+int kvm_cpu__get_debug_fd(void)
+{
+	return debug_fd;
+}
+
+struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
+{
+	/* TODO: */
+	return NULL;
+}
+
+void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
+{
+}
+
+void kvm_cpu__delete(struct kvm_cpu *vcpu)
+{
+	/* TODO: */
+}
+
+bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
+{
+	/* TODO: */
+	return false;
+}
+
+void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
+{
+	/* TODO: */
+}
+
+void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
+{
+	/* TODO: */
+}
+
+int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
+{
+	return VIRTIO_ENDIAN_LE;
+}
+
+void kvm_cpu__show_code(struct kvm_cpu *vcpu)
+{
+	/* TODO: */
+}
+
+void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
+{
+	/* TODO: */
+}
diff --git a/riscv/kvm.c b/riscv/kvm.c
new file mode 100644
index 0000000..e816ef5
--- /dev/null
+++ b/riscv/kvm.c
@@ -0,0 +1,61 @@
+#include "kvm/kvm.h"
+#include "kvm/util.h"
+#include "kvm/fdt.h"
+
+#include <linux/kernel.h>
+#include <linux/kvm.h>
+#include <linux/sizes.h>
+
+struct kvm_ext kvm_req_ext[] = {
+	{ DEFINE_KVM_EXT(KVM_CAP_ONE_REG) },
+	{ 0, 0 },
+};
+
+bool kvm__arch_cpu_supports_vm(void)
+{
+	/* The KVM capability check is enough. */
+	return true;
+}
+
+void kvm__init_ram(struct kvm *kvm)
+{
+	/* TODO: */
+}
+
+void kvm__arch_delete_ram(struct kvm *kvm)
+{
+	/* TODO: */
+}
+
+void kvm__arch_read_term(struct kvm *kvm)
+{
+	/* TODO: */
+}
+
+void kvm__arch_set_cmdline(char *cmdline, bool video)
+{
+	/* TODO: */
+}
+
+void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
+{
+	/* TODO: */
+}
+
+bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
+				 const char *kernel_cmdline)
+{
+	/* TODO: */
+	return true;
+}
+
+bool kvm__load_firmware(struct kvm *kvm, const char *firmware_filename)
+{
+	/* TODO: Firmware loading to be supported later. */
+	return false;
+}
+
+int kvm__arch_setup_firmware(struct kvm *kvm)
+{
+	return 0;
+}
diff --git a/util/update_headers.sh b/util/update_headers.sh
index bf87ef6..78eba1f 100755
--- a/util/update_headers.sh
+++ b/util/update_headers.sh
@@ -36,7 +36,7 @@ copy_optional_arch () {
 	fi
 }
 
-for arch in arm arm64 mips powerpc x86
+for arch in arm arm64 mips powerpc riscv x86
 do
 	case "$arch" in
 		arm) KVMTOOL_PATH=arm/aarch32 ;;
-- 
2.17.1

[kvmtool RFC PATCH 3/8] riscv: Implement Guest/VM arch functions

[Anup Patel]

This patch implements all kvm__arch_<xyz> Guest/VM arch functions.

These functions mostly deal with:
1. Guest/VM RAM initialization
2. Updating terminals on character read
3. Loading kernel and initrd images

Firmware loading is not implemented currently because initially we
will be booting kernel directly without any bootloader. In future,
we will certainly support firmware loading.

Signed-off-by: Anup Patel <anup.patel@wdc.com>
---
 riscv/include/kvm/kvm-arch.h |  15 +++++
 riscv/kvm.c                  | 126 +++++++++++++++++++++++++++++++++--
 2 files changed, 135 insertions(+), 6 deletions(-)

diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
index 7e9c578..b3ec2d6 100644
--- a/riscv/include/kvm/kvm-arch.h
+++ b/riscv/include/kvm/kvm-arch.h
@@ -45,6 +45,21 @@
 struct kvm;
 
 struct kvm_arch {
+	/*
+	 * We may have to align the guest memory for virtio, so keep the
+	 * original pointers here for munmap.
+	 */
+	void	*ram_alloc_start;
+	u64	ram_alloc_size;
+
+	/*
+	 * Guest addresses for memory layout.
+	 */
+	u64	memory_guest_start;
+	u64	kern_guest_start;
+	u64	initrd_guest_start;
+	u64	initrd_size;
+	u64	dtb_guest_start;
 };
 
 static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
diff --git a/riscv/kvm.c b/riscv/kvm.c
index e816ef5..c0d3639 100644
--- a/riscv/kvm.c
+++ b/riscv/kvm.c
@@ -1,5 +1,7 @@
 #include "kvm/kvm.h"
 #include "kvm/util.h"
+#include "kvm/8250-serial.h"
+#include "kvm/virtio-console.h"
 #include "kvm/fdt.h"
 
 #include <linux/kernel.h>
@@ -19,33 +21,145 @@ bool kvm__arch_cpu_supports_vm(void)
 
 void kvm__init_ram(struct kvm *kvm)
 {
-	/* TODO: */
+	int err;
+	u64 phys_start, phys_size;
+	void *host_mem;
+
+	phys_start	= RISCV_RAM;
+	phys_size	= kvm->ram_size;
+	host_mem	= kvm->ram_start;
+
+	err = kvm__register_ram(kvm, phys_start, phys_size, host_mem);
+	if (err)
+		die("Failed to register %lld bytes of memory at physical "
+		    "address 0x%llx [err %d]", phys_size, phys_start, err);
+
+	kvm->arch.memory_guest_start = phys_start;
 }
 
 void kvm__arch_delete_ram(struct kvm *kvm)
 {
-	/* TODO: */
+	munmap(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size);
 }
 
 void kvm__arch_read_term(struct kvm *kvm)
 {
-	/* TODO: */
+	serial8250__update_consoles(kvm);
+	virtio_console__inject_interrupt(kvm);
 }
 
 void kvm__arch_set_cmdline(char *cmdline, bool video)
 {
-	/* TODO: */
 }
 
 void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
 {
-	/* TODO: */
+	/*
+	 * Allocate guest memory. We must align our buffer to 64K to
+	 * correlate with the maximum guest page size for virtio-mmio.
+	 * If using THP, then our minimal alignment becomes 2M.
+	 * 2M trumps 64K, so let's go with that.
+	 */
+	kvm->ram_size = min(ram_size, (u64)RISCV_MAX_MEMORY(kvm));
+	kvm->arch.ram_alloc_size = kvm->ram_size + SZ_2M;
+	kvm->arch.ram_alloc_start = mmap_anon_or_hugetlbfs(kvm, hugetlbfs_path,
+						kvm->arch.ram_alloc_size);
+
+	if (kvm->arch.ram_alloc_start == MAP_FAILED)
+		die("Failed to map %lld bytes for guest memory (%d)",
+		    kvm->arch.ram_alloc_size, errno);
+
+	kvm->ram_start = (void *)ALIGN((unsigned long)kvm->arch.ram_alloc_start,
+					SZ_2M);
+
+	madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
+		MADV_MERGEABLE);
+
+	madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
+		MADV_HUGEPAGE);
 }
 
+#define FDT_ALIGN	SZ_4M
+#define INITRD_ALIGN	4
 bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
 				 const char *kernel_cmdline)
 {
-	/* TODO: */
+	void *pos, *kernel_end, *limit;
+	unsigned long guest_addr, kernel_offset;
+	ssize_t file_size;
+
+	/*
+	 * Linux requires the initrd and dtb to be mapped inside lowmem,
+	 * so we can't just place them at the top of memory.
+	 */
+	limit = kvm->ram_start + min(kvm->ram_size, (u64)SZ_256M) - 1;
+
+#if __riscv_xlen == 64
+	/* Linux expects to be booted at 2M boundary for RV64 */
+	kernel_offset = 0x200000;
+#else
+	/* Linux expects to be booted at 4M boundary for RV32 */
+	kernel_offset = 0x400000;
+#endif
+
+	pos = kvm->ram_start + kernel_offset;
+	kvm->arch.kern_guest_start = host_to_guest_flat(kvm, pos);
+	file_size = read_file(fd_kernel, pos, limit - pos);
+	if (file_size < 0) {
+		if (errno == ENOMEM)
+			die("kernel image too big to fit in guest memory.");
+
+		die_perror("kernel read");
+	}
+	kernel_end = pos + file_size;
+	pr_debug("Loaded kernel to 0x%llx (%zd bytes)",
+		 kvm->arch.kern_guest_start, file_size);
+
+	/* Place FDT just after kernel at FDT_ALIGN address */
+	pos = kernel_end + FDT_ALIGN;
+	guest_addr = ALIGN(host_to_guest_flat(kvm, pos), FDT_ALIGN);
+	pos = guest_flat_to_host(kvm, guest_addr);
+	if (pos < kernel_end)
+		die("fdt overlaps with kernel image.");
+
+	kvm->arch.dtb_guest_start = guest_addr;
+	pr_debug("Placing fdt at 0x%llx - 0x%llx",
+		 kvm->arch.dtb_guest_start,
+		 host_to_guest_flat(kvm, limit));
+	limit = pos;
+
+	/* ... and finally the initrd, if we have one. */
+	if (fd_initrd != -1) {
+		struct stat sb;
+		unsigned long initrd_start;
+
+		if (fstat(fd_initrd, &sb))
+			die_perror("fstat");
+
+		pos -= (sb.st_size + INITRD_ALIGN);
+		guest_addr = ALIGN(host_to_guest_flat(kvm, pos), INITRD_ALIGN);
+		pos = guest_flat_to_host(kvm, guest_addr);
+		if (pos < kernel_end)
+			die("initrd overlaps with kernel image.");
+
+		initrd_start = guest_addr;
+		file_size = read_file(fd_initrd, pos, limit - pos);
+		if (file_size == -1) {
+			if (errno == ENOMEM)
+				die("initrd too big to fit in guest memory.");
+
+			die_perror("initrd read");
+		}
+
+		kvm->arch.initrd_guest_start = initrd_start;
+		kvm->arch.initrd_size = file_size;
+		pr_debug("Loaded initrd to 0x%llx (%llu bytes)",
+			 kvm->arch.initrd_guest_start,
+			 kvm->arch.initrd_size);
+	} else {
+		kvm->arch.initrd_size = 0;
+	}
+
 	return true;
 }
 
-- 
2.17.1

[kvmtool RFC PATCH 4/8] riscv: Implement Guest/VM VCPU arch functions

[Anup Patel]

This patch implements kvm_cpu__<xyz> Guest/VM VCPU arch functions.

These functions mostly deal with:
1. VCPU allocation and initialization
2. VCPU reset
3. VCPU show/dump code
4. VCPU show/dump registers

We also save RISC-V ISA, XLEN, and TIMEBASE frequency for each VCPU
so that it can be later used for generating Guest/VM FDT.

Signed-off-by: Anup Patel <anup.patel@wdc.com>
---
 riscv/include/kvm/kvm-cpu-arch.h |   4 +
 riscv/kvm-cpu.c                  | 307 ++++++++++++++++++++++++++++++-
 2 files changed, 304 insertions(+), 7 deletions(-)

diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
index 09a50e8..035965e 100644
--- a/riscv/include/kvm/kvm-cpu-arch.h
+++ b/riscv/include/kvm/kvm-cpu-arch.h
@@ -14,6 +14,10 @@ struct kvm_cpu {
 
 	unsigned long   cpu_id;
 
+	unsigned long	riscv_xlen;
+	unsigned long	riscv_isa;
+	unsigned long	riscv_timebase;
+
 	struct kvm	*kvm;
 	int		vcpu_fd;
 	struct kvm_run	*kvm_run;
diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
index e4b8fa5..1565275 100644
--- a/riscv/kvm-cpu.c
+++ b/riscv/kvm-cpu.c
@@ -17,10 +17,84 @@ int kvm_cpu__get_debug_fd(void)
 	return debug_fd;
 }
 
+static __u64 __kvm_reg_id(__u64 type, __u64 idx)
+{
+	__u64 id = KVM_REG_RISCV | type | idx;
+
+	if (sizeof(unsigned long) == 8)
+		id |= KVM_REG_SIZE_U64;
+	else
+		id |= KVM_REG_SIZE_U32;
+
+	return id;
+}
+
+#define RISCV_CONFIG_REG(name)	__kvm_reg_id(KVM_REG_RISCV_CONFIG, \
+					     KVM_REG_RISCV_CONFIG_REG(name))
+
+#define RISCV_CORE_REG(name)	__kvm_reg_id(KVM_REG_RISCV_CORE, \
+					     KVM_REG_RISCV_CORE_REG(name))
+
+#define RISCV_CSR_REG(name)	__kvm_reg_id(KVM_REG_RISCV_CSR, \
+					     KVM_REG_RISCV_CSR_REG(name))
+
+#define RISCV_TIMER_REG(name)	__kvm_reg_id(KVM_REG_RISCV_TIMER, \
+					     KVM_REG_RISCV_TIMER_REG(name))
+
 struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
 {
-	/* TODO: */
-	return NULL;
+	struct kvm_cpu *vcpu;
+	unsigned long timebase = 0, isa = 0;
+	int coalesced_offset, mmap_size;
+	struct kvm_one_reg reg;
+
+	vcpu = calloc(1, sizeof(struct kvm_cpu));
+	if (!vcpu)
+		return NULL;
+
+	vcpu->vcpu_fd = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, cpu_id);
+	if (vcpu->vcpu_fd < 0)
+		die_perror("KVM_CREATE_VCPU ioctl");
+
+	reg.id = RISCV_CONFIG_REG(isa);
+	reg.addr = (unsigned long)&isa;
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (config.isa)");
+
+	reg.id = RISCV_TIMER_REG(frequency);
+	reg.addr = (unsigned long)&timebase;
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (config.timebase)");
+
+	mmap_size = ioctl(kvm->sys_fd, KVM_GET_VCPU_MMAP_SIZE, 0);
+	if (mmap_size < 0)
+		die_perror("KVM_GET_VCPU_MMAP_SIZE ioctl");
+
+	vcpu->kvm_run = mmap(NULL, mmap_size, PROT_RW, MAP_SHARED,
+			     vcpu->vcpu_fd, 0);
+	if (vcpu->kvm_run == MAP_FAILED)
+		die("unable to mmap vcpu fd");
+
+	coalesced_offset = ioctl(kvm->sys_fd, KVM_CHECK_EXTENSION,
+				 KVM_CAP_COALESCED_MMIO);
+	if (coalesced_offset)
+		vcpu->ring = (void *)vcpu->kvm_run +
+			     (coalesced_offset * PAGE_SIZE);
+
+	reg.id = RISCV_CONFIG_REG(isa);
+	reg.addr = (unsigned long)&isa;
+	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
+		die("KVM_SET_ONE_REG failed (config.isa)");
+
+	/* Populate the vcpu structure. */
+	vcpu->kvm		= kvm;
+	vcpu->cpu_id		= cpu_id;
+	vcpu->riscv_isa		= isa;
+	vcpu->riscv_xlen	= __riscv_xlen;
+	vcpu->riscv_timebase	= timebase;
+	vcpu->is_running	= true;
+
+	return vcpu;
 }
 
 void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
@@ -29,7 +103,7 @@ void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
 
 void kvm_cpu__delete(struct kvm_cpu *vcpu)
 {
-	/* TODO: */
+	free(vcpu);
 }
 
 bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
@@ -40,12 +114,43 @@ bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
 
 void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
 {
-	/* TODO: */
 }
 
 void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
 {
-	/* TODO: */
+	struct kvm *kvm = vcpu->kvm;
+	struct kvm_mp_state mp_state;
+	struct kvm_one_reg reg;
+	unsigned long data;
+
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_MP_STATE, &mp_state) < 0)
+		die_perror("KVM_GET_MP_STATE failed");
+
+	/*
+	 * If MP state is stopped then it means Linux KVM RISC-V emulates
+	 * SBI v0.2 (or higher) with HART power managment and give VCPU
+	 * will power-up at boot-time by boot VCPU. For such VCPU, we
+	 * don't update PC, A0 and A1 here.
+	 */
+	if (mp_state.mp_state == KVM_MP_STATE_STOPPED)
+		return;
+
+	reg.addr = (unsigned long)&data;
+
+	data	= kvm->arch.kern_guest_start;
+	reg.id	= RISCV_CORE_REG(regs.pc);
+	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
+		die_perror("KVM_SET_ONE_REG failed (pc)");
+
+	data	= vcpu->cpu_id;
+	reg.id	= RISCV_CORE_REG(regs.a0);
+	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
+		die_perror("KVM_SET_ONE_REG failed (a0)");
+
+	data	= kvm->arch.dtb_guest_start;
+	reg.id	= RISCV_CORE_REG(regs.a1);
+	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
+		die_perror("KVM_SET_ONE_REG failed (a1)");
 }
 
 int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
@@ -55,10 +160,198 @@ int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
 
 void kvm_cpu__show_code(struct kvm_cpu *vcpu)
 {
-	/* TODO: */
+	struct kvm_one_reg reg;
+	unsigned long data;
+	int debug_fd = kvm_cpu__get_debug_fd();
+
+	reg.addr = (unsigned long)&data;
+
+	dprintf(debug_fd, "\n*PC:\n");
+	reg.id = RISCV_CORE_REG(regs.pc);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (show_code @ PC)");
+
+	kvm__dump_mem(vcpu->kvm, data, 32, debug_fd);
+
+	dprintf(debug_fd, "\n*RA:\n");
+	reg.id = RISCV_CORE_REG(regs.ra);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (show_code @ RA)");
+
+	kvm__dump_mem(vcpu->kvm, data, 32, debug_fd);
 }
 
 void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
 {
-	/* TODO: */
+	struct kvm_one_reg reg;
+	unsigned long data;
+	int debug_fd = kvm_cpu__get_debug_fd();
+
+	reg.addr = (unsigned long)&data;
+	dprintf(debug_fd, "\n Registers:\n");
+
+	reg.id		= RISCV_CORE_REG(mode);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (mode)");
+	dprintf(debug_fd, " MODE:  0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.pc);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (pc)");
+	dprintf(debug_fd, " PC:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.ra);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (ra)");
+	dprintf(debug_fd, " RA:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.sp);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (sp)");
+	dprintf(debug_fd, " SP:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.gp);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (gp)");
+	dprintf(debug_fd, " GP:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.tp);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (tp)");
+	dprintf(debug_fd, " TP:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.t0);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (t0)");
+	dprintf(debug_fd, " T0:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.t1);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (t1)");
+	dprintf(debug_fd, " T1:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.t2);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (t2)");
+	dprintf(debug_fd, " T2:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s0);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s0)");
+	dprintf(debug_fd, " S0:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s1);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s1)");
+	dprintf(debug_fd, " S1:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.a0);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (a0)");
+	dprintf(debug_fd, " A0:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.a1);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (a1)");
+	dprintf(debug_fd, " A1:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.a2);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (a2)");
+	dprintf(debug_fd, " A2:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.a3);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (a3)");
+	dprintf(debug_fd, " A3:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.a4);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (a4)");
+	dprintf(debug_fd, " A4:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.a5);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (a5)");
+	dprintf(debug_fd, " A5:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.a6);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (a6)");
+	dprintf(debug_fd, " A6:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.a7);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (a7)");
+	dprintf(debug_fd, " A7:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s2);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s2)");
+	dprintf(debug_fd, " S2:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s3);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s3)");
+	dprintf(debug_fd, " S3:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s4);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s4)");
+	dprintf(debug_fd, " S4:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s5);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s5)");
+	dprintf(debug_fd, " S5:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s6);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s6)");
+	dprintf(debug_fd, " S6:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s7);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s7)");
+	dprintf(debug_fd, " S7:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s8);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s8)");
+	dprintf(debug_fd, " S8:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s9);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s9)");
+	dprintf(debug_fd, " S9:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s10);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s10)");
+	dprintf(debug_fd, " S10:   0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.s11);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (s11)");
+	dprintf(debug_fd, " S11:   0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.t3);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (t3)");
+	dprintf(debug_fd, " T3:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.t4);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (t4)");
+	dprintf(debug_fd, " T4:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.t5);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (t5)");
+	dprintf(debug_fd, " T5:    0x%lx\n", data);
+
+	reg.id		= RISCV_CORE_REG(regs.t6);
+	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
+		die("KVM_GET_ONE_REG failed (t6)");
+	dprintf(debug_fd, " T6:    0x%lx\n", data);
 }
-- 
2.17.1

[kvmtool RFC PATCH 5/8] riscv: Add PLIC device emulation

[Anup Patel]

The PLIC (platform level interrupt controller) manages peripheral
interrupts in RISC-V world. The per-CPU interrupts are managed
using CPU CSRs hence virtualized in-kernel by KVM RISC-V.

This patch adds PLIC device emulation for KVMTOOL RISC-V.

Signed-off-by: Anup Patel <anup.patel@wdc.com>
---
 Makefile                     |   1 +
 riscv/include/kvm/kvm-arch.h |   2 +
 riscv/irq.c                  |   4 +-
 riscv/plic.c                 | 508 +++++++++++++++++++++++++++++++++++
 4 files changed, 513 insertions(+), 2 deletions(-)
 create mode 100644 riscv/plic.c

diff --git a/Makefile b/Makefile
index 972fa63..3220ad3 100644
--- a/Makefile
+++ b/Makefile
@@ -200,6 +200,7 @@ ifeq ($(ARCH),riscv)
 	OBJS		+= riscv/irq.o
 	OBJS		+= riscv/kvm.o
 	OBJS		+= riscv/kvm-cpu.o
+	OBJS		+= riscv/plic.o
 
 	ARCH_WANT_LIBFDT := y
 endif
diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
index b3ec2d6..b1af70f 100644
--- a/riscv/include/kvm/kvm-arch.h
+++ b/riscv/include/kvm/kvm-arch.h
@@ -70,4 +70,6 @@ static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
 
 enum irq_type;
 
+void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge);
+
 #endif /* KVM__KVM_ARCH_H */
diff --git a/riscv/irq.c b/riscv/irq.c
index 8e605ef..78a582d 100644
--- a/riscv/irq.c
+++ b/riscv/irq.c
@@ -4,10 +4,10 @@
 
 void kvm__irq_line(struct kvm *kvm, int irq, int level)
 {
-	/* TODO: */
+	plic__irq_trig(kvm, irq, level, false);
 }
 
 void kvm__irq_trigger(struct kvm *kvm, int irq)
 {
-	/* TODO: */
+	plic__irq_trig(kvm, irq, 1, true);
 }
diff --git a/riscv/plic.c b/riscv/plic.c
new file mode 100644
index 0000000..93bfbc5
--- /dev/null
+++ b/riscv/plic.c
@@ -0,0 +1,508 @@
+
+#include "kvm/devices.h"
+#include "kvm/ioeventfd.h"
+#include "kvm/ioport.h"
+#include "kvm/kvm.h"
+#include "kvm/kvm-cpu.h"
+#include "kvm/irq.h"
+#include "kvm/mutex.h"
+
+#include <linux/byteorder.h>
+#include <linux/kernel.h>
+#include <linux/kvm.h>
+#include <linux/sizes.h>
+
+/*
+ * From the RISC-V Privlidged Spec v1.10:
+ *
+ * Global interrupt sources are assigned small unsigned integer identifiers,
+ * beginning at the value 1.  An interrupt ID of 0 is reserved to mean no
+ * interrupt.  Interrupt identifiers are also used to break ties when two or
+ * more interrupt sources have the same assigned priority. Smaller values of
+ * interrupt ID take precedence over larger values of interrupt ID.
+ *
+ * While the RISC-V supervisor spec doesn't define the maximum number of
+ * devices supported by the PLIC, the largest number supported by devices
+ * marked as 'riscv,plic0' (which is the only device type this driver supports,
+ * and is the only extant PLIC as of now) is 1024.  As mentioned above, device
+ * 0 is defined to be non-existant so this device really only supports 1023
+ * devices.
+ */
+
+#define MAX_DEVICES	1024
+#define MAX_CONTEXTS	15872
+
+/*
+ * The PLIC consists of memory-mapped control registers, with a memory map as
+ * follows:
+ *
+ * base + 0x000000: Reserved (interrupt source 0 does not exist)
+ * base + 0x000004: Interrupt source 1 priority
+ * base + 0x000008: Interrupt source 2 priority
+ * ...
+ * base + 0x000FFC: Interrupt source 1023 priority
+ * base + 0x001000: Pending 0
+ * base + 0x001FFF: Pending
+ * base + 0x002000: Enable bits for sources 0-31 on context 0
+ * base + 0x002004: Enable bits for sources 32-63 on context 0
+ * ...
+ * base + 0x0020FC: Enable bits for sources 992-1023 on context 0
+ * base + 0x002080: Enable bits for sources 0-31 on context 1
+ * ...
+ * base + 0x002100: Enable bits for sources 0-31 on context 2
+ * ...
+ * base + 0x1F1F80: Enable bits for sources 992-1023 on context 15871
+ * base + 0x1F1F84: Reserved
+ * ...		    (higher context IDs would fit here, but wouldn't fit
+ *		     inside the per-context priority vector)
+ * base + 0x1FFFFC: Reserved
+ * base + 0x200000: Priority threshold for context 0
+ * base + 0x200004: Claim/complete for context 0
+ * base + 0x200008: Reserved
+ * ...
+ * base + 0x200FFC: Reserved
+ * base + 0x201000: Priority threshold for context 1
+ * base + 0x201004: Claim/complete for context 1
+ * ...
+ * base + 0xFFE000: Priority threshold for context 15871
+ * base + 0xFFE004: Claim/complete for context 15871
+ * base + 0xFFE008: Reserved
+ * ...
+ * base + 0xFFFFFC: Reserved
+ */
+
+/* Each interrupt source has a priority register associated with it. */
+#define PRIORITY_BASE		0
+#define PRIORITY_PER_ID		4
+
+/*
+ * Each hart context has a vector of interupt enable bits associated with it.
+ * There's one bit for each interrupt source.
+ */
+#define ENABLE_BASE		0x2000
+#define ENABLE_PER_HART		0x80
+
+/*
+ * Each hart context has a set of control registers associated with it.  Right
+ * now there's only two: a source priority threshold over which the hart will
+ * take an interrupt, and a register to claim interrupts.
+ */
+#define CONTEXT_BASE		0x200000
+#define CONTEXT_PER_HART	0x1000
+#define CONTEXT_THRESHOLD	0
+#define CONTEXT_CLAIM		4
+
+#define REG_SIZE		0x1000000
+
+struct plic_state;
+
+struct plic_context {
+	/* State to which this belongs */
+	struct plic_state *s;
+
+	/* Static Configuration */
+	u32 num;
+	struct kvm_cpu *vcpu;
+
+	/* Local IRQ state */
+	struct mutex irq_lock;
+	u8 irq_priority_threshold;
+	u32 irq_enable[MAX_DEVICES/32];
+	u32 irq_pending[MAX_DEVICES/32];
+	u8 irq_pending_priority[MAX_DEVICES];
+	u32 irq_claimed[MAX_DEVICES/32];
+	u32 irq_autoclear[MAX_DEVICES/32];
+};
+
+struct plic_state {
+	bool ready;
+	struct kvm *kvm;
+	struct device_header dev_hdr;
+
+	/* Static Configuration */
+	u32 num_irq;
+	u32 num_irq_word;
+	u32 max_prio;
+
+	/* Context Array */
+	u32 num_context;
+	struct plic_context *contexts;
+
+	/* Global IRQ state */
+	struct mutex irq_lock;
+	u8 irq_priority[MAX_DEVICES];
+	u32 irq_level[MAX_DEVICES/32];
+};
+
+static struct plic_state plic;
+
+/* Note: Must be called with c->irq_lock held */
+static u32 __plic_context_best_pending_irq(struct plic_state *s,
+					   struct plic_context *c)
+{
+	u8 best_irq_prio = 0;
+	u32 i, j, irq, best_irq = 0;
+
+	for (i = 0; i < s->num_irq_word; i++) {
+		if (!c->irq_pending[i])
+			continue;
+
+		for (j = 0; j < 32; j++) {
+			irq = i * 32 + j;
+			if ((s->num_irq <= irq) ||
+			    !(c->irq_pending[i] & (1 << j)) ||
+			    (c->irq_claimed[i] & (1 << j)))
+				continue;
+
+			if (!best_irq ||
+			    (best_irq_prio < c->irq_pending_priority[irq])) {
+				best_irq = irq;
+				best_irq_prio = c->irq_pending_priority[irq];
+			}
+		}
+	}
+
+	return best_irq;
+}
+
+/* Note: Must be called with c->irq_lock held */
+static void __plic_context_irq_update(struct plic_state *s,
+				      struct plic_context *c)
+{
+	u32 best_irq = __plic_context_best_pending_irq(s, c);
+	u32 virq = (best_irq) ? KVM_INTERRUPT_SET : KVM_INTERRUPT_UNSET;
+
+	if (ioctl(c->vcpu->vcpu_fd, KVM_INTERRUPT, &virq) < 0)
+		pr_warning("KVM_INTERRUPT failed");
+}
+
+/* Note: Must be called with c->irq_lock held */
+static u32 __plic_context_irq_claim(struct plic_state *s,
+				    struct plic_context *c)
+{
+	u32 virq = KVM_INTERRUPT_UNSET;
+	u32 best_irq = __plic_context_best_pending_irq(s, c);
+	u32 best_irq_word = best_irq / 32;
+	u32 best_irq_mask = (1 << (best_irq % 32));
+
+	if (ioctl(c->vcpu->vcpu_fd, KVM_INTERRUPT, &virq) < 0)
+		pr_warning("KVM_INTERRUPT failed");
+
+	if (best_irq) {
+		if (c->irq_autoclear[best_irq_word] & best_irq_mask) {
+			c->irq_pending[best_irq_word] &= ~best_irq_mask;
+			c->irq_pending_priority[best_irq] = 0;
+			c->irq_claimed[best_irq_word] &= ~best_irq_mask;
+			c->irq_autoclear[best_irq_word] &= ~best_irq_mask;
+		} else
+			c->irq_claimed[best_irq_word] |= best_irq_mask;
+	}
+
+	__plic_context_irq_update(s, c);
+
+	return best_irq;
+}
+
+void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge)
+{
+	bool irq_marked = false;
+	u8 i, irq_prio, irq_word;
+	u32 irq_mask;
+	struct plic_context *c = NULL;
+	struct plic_state *s = &plic;
+
+	if (!s->ready)
+		return;
+
+	if (irq <= 0 || s->num_irq <= (u32)irq)
+		goto done;
+
+	mutex_lock(&s->irq_lock);
+
+	irq_prio = s->irq_priority[irq];
+	irq_word = irq / 32;
+	irq_mask = 1 << (irq % 32);
+
+	if (level)
+		s->irq_level[irq_word] |= irq_mask;
+	else
+		s->irq_level[irq_word] &= ~irq_mask;
+
+	/*
+	 * Note: PLIC interrupts are level-triggered. As of now,
+	 * there is no notion of edge-triggered interrupts. To
+	 * handle this we auto-clear edge-triggered interrupts
+	 * when PLIC context CLAIM register is read.
+	 */
+	for (i = 0; i < s->num_context; i++) {
+		c = &s->contexts[i];
+
+		mutex_lock(&c->irq_lock);
+		if (c->irq_enable[irq_word] & irq_mask) {
+			if (level) {
+				c->irq_pending[irq_word] |= irq_mask;
+				c->irq_pending_priority[irq] = irq_prio;
+				if (edge)
+					c->irq_autoclear[irq_word] |= irq_mask;
+			} else {
+				c->irq_pending[irq_word] &= ~irq_mask;
+				c->irq_pending_priority[irq] = 0;
+				c->irq_claimed[irq_word] &= ~irq_mask;
+				c->irq_autoclear[irq_word] &= ~irq_mask;
+			}
+			__plic_context_irq_update(s, c);
+			irq_marked = true;
+		}
+		mutex_unlock(&c->irq_lock);
+
+		if (irq_marked)
+			break;
+	}
+
+done:
+	mutex_unlock(&s->irq_lock);
+}
+
+static void plic__priority_read(struct plic_state *s,
+				u64 offset, void *data)
+{
+	u32 irq = (offset >> 2);
+
+	if (irq == 0 || irq >= s->num_irq)
+		return;
+
+	mutex_lock(&s->irq_lock);
+	ioport__write32(data, s->irq_priority[irq]);
+	mutex_unlock(&s->irq_lock);
+}
+
+static void plic__priority_write(struct plic_state *s,
+				 u64 offset, void *data)
+{
+	u32 val, irq = (offset >> 2);
+
+	if (irq == 0 || irq >= s->num_irq)
+		return;
+
+	mutex_lock(&s->irq_lock);
+	val = ioport__read32(data);
+	val &= ((1 << PRIORITY_PER_ID) - 1);
+	s->irq_priority[irq] = val;
+	mutex_unlock(&s->irq_lock);
+}
+
+static void plic__context_enable_read(struct plic_state *s,
+				      struct plic_context *c,
+				      u64 offset, void *data)
+{
+	u32 irq_word = offset >> 2;
+
+	if (s->num_irq_word < irq_word)
+		return;
+
+	mutex_lock(&c->irq_lock);
+	ioport__write32(data, c->irq_enable[irq_word]);
+	mutex_unlock(&c->irq_lock);
+}
+
+static void plic__context_enable_write(struct plic_state *s,
+				       struct plic_context *c,
+				       u64 offset, void *data)
+{
+	u8 irq_prio;
+	u32 i, irq, irq_mask;
+	u32 irq_word = offset >> 2;
+	u32 old_val, new_val, xor_val;
+
+	if (s->num_irq_word < irq_word)
+		return;
+
+	mutex_lock(&s->irq_lock);
+
+	mutex_lock(&c->irq_lock);
+
+	old_val = c->irq_enable[irq_word];
+	new_val = ioport__read32(data);
+
+	if (irq_word == 0)
+		new_val &= ~0x1;
+
+	c->irq_enable[irq_word] = new_val;
+
+	xor_val = old_val ^ new_val;
+	for (i = 0; i < 32; i++) {
+		irq = irq_word * 32 + i;
+		irq_mask = 1 << i;
+		irq_prio = s->irq_priority[irq];
+		if (!(xor_val & irq_mask))
+			continue;
+		if ((new_val & irq_mask) &&
+		    (s->irq_level[irq_word] & irq_mask)) {
+			c->irq_pending[irq_word] |= irq_mask;
+			c->irq_pending_priority[irq] = irq_prio;
+		} else if (!(new_val & irq_mask)) {
+			c->irq_pending[irq_word] &= ~irq_mask;
+			c->irq_pending_priority[irq] = 0;
+			c->irq_claimed[irq_word] &= ~irq_mask;
+		}
+	}
+
+	__plic_context_irq_update(s, c);
+
+	mutex_unlock(&c->irq_lock);
+
+	mutex_unlock(&s->irq_lock);
+}
+
+static void plic__context_read(struct plic_state *s,
+			       struct plic_context *c,
+			       u64 offset, void *data)
+{
+	mutex_lock(&c->irq_lock);
+
+	switch (offset) {
+	case CONTEXT_THRESHOLD:
+		ioport__write32(data, c->irq_priority_threshold);
+		break;
+	case CONTEXT_CLAIM:
+		ioport__write32(data, __plic_context_irq_claim(s, c));
+		break;
+	default:
+		break;
+	};
+
+	mutex_unlock(&c->irq_lock);
+}
+
+static void plic__context_write(struct plic_state *s,
+				struct plic_context *c,
+				u64 offset, void *data)
+{
+	u32 val;
+	bool irq_update = false;
+
+	mutex_lock(&c->irq_lock);
+
+	switch (offset) {
+	case CONTEXT_THRESHOLD:
+		val = ioport__read32(data);
+		val &= ((1 << PRIORITY_PER_ID) - 1);
+		if (val <= s->max_prio)
+			c->irq_priority_threshold = val;
+		else
+			irq_update = true;
+		break;
+	case CONTEXT_CLAIM:
+		break;
+	default:
+		irq_update = true;
+		break;
+	};
+
+	if (irq_update)
+		__plic_context_irq_update(s, c);
+
+	mutex_unlock(&c->irq_lock);
+}
+
+static void plic__mmio_callback(struct kvm_cpu *vcpu,
+				u64 addr, u8 *data, u32 len,
+				u8 is_write, void *ptr)
+{
+	u32 cntx;
+	struct plic_state *s = ptr;
+
+	if (len != 4)
+		die("plic: invalid len=%d", len);
+
+	addr &= ~0x3;
+	addr -= RISCV_PLIC;
+
+	if (is_write) {
+		if (PRIORITY_BASE <= addr && addr < ENABLE_BASE) {
+			plic__priority_write(s, addr, data);
+		} else if (ENABLE_BASE <= addr && addr < CONTEXT_BASE) {
+			cntx = (addr - ENABLE_BASE) / ENABLE_PER_HART;
+			addr -= cntx * ENABLE_PER_HART + ENABLE_BASE;
+			if (cntx < s->num_context)
+				plic__context_enable_write(s,
+							   &s->contexts[cntx],
+							   addr, data);
+		} else if (CONTEXT_BASE <= addr && addr < REG_SIZE) {
+			cntx = (addr - CONTEXT_BASE) / CONTEXT_PER_HART;
+			addr -= cntx * CONTEXT_PER_HART + CONTEXT_BASE;
+			if (cntx < s->num_context)
+				plic__context_write(s, &s->contexts[cntx],
+						    addr, data);
+		}
+	} else {
+		if (PRIORITY_BASE <= addr && addr < ENABLE_BASE) {
+			plic__priority_read(s, addr, data);
+		} else if (ENABLE_BASE <= addr && addr < CONTEXT_BASE) {
+			cntx = (addr - ENABLE_BASE) / ENABLE_PER_HART;
+			addr -= cntx * ENABLE_PER_HART + ENABLE_BASE;
+			if (cntx < s->num_context)
+				plic__context_enable_read(s,
+							  &s->contexts[cntx],
+							  addr, data);
+		} else if (CONTEXT_BASE <= addr && addr < REG_SIZE) {
+			cntx = (addr - CONTEXT_BASE) / CONTEXT_PER_HART;
+			addr -= cntx * CONTEXT_PER_HART + CONTEXT_BASE;
+			if (cntx < s->num_context)
+				plic__context_read(s, &s->contexts[cntx],
+						   addr, data);
+		}
+	}
+}
+
+static int plic__init(struct kvm *kvm)
+{
+	u32 i;
+	struct plic_context *c;
+
+	plic.kvm = kvm;
+	plic.dev_hdr = (struct device_header) {
+		.bus_type	= DEVICE_BUS_MMIO,
+	};
+
+	plic.num_irq = MAX_DEVICES;
+	plic.num_irq_word = plic.num_irq / 32;
+	if ((plic.num_irq_word * 32) < plic.num_irq)
+		plic.num_irq_word++;
+	plic.max_prio = (1UL << PRIORITY_PER_ID) - 1;
+
+	plic.num_context = kvm->nrcpus * 2;
+	plic.contexts = calloc(plic.num_context, sizeof(struct plic_context));
+	if (!plic.contexts)
+		return -ENOMEM;
+	for (i = 0; i < plic.num_context; i++) {
+		c = &plic.contexts[i];
+		c->s = &plic;
+		c->num = i;
+		c->vcpu = kvm->cpus[i / 2];
+		mutex_init(&c->irq_lock);
+	}
+
+	mutex_init(&plic.irq_lock);
+
+	kvm__register_mmio(kvm, RISCV_PLIC, RISCV_PLIC_SIZE,
+			   false, plic__mmio_callback, &plic);
+
+	device__register(&plic.dev_hdr);
+
+	plic.ready = true;
+
+	return 0;
+
+}
+dev_init(plic__init);
+
+static int plic__exit(struct kvm *kvm)
+{
+	plic.ready = false;
+	kvm__deregister_mmio(kvm, RISCV_PLIC);
+	free(plic.contexts);
+
+	return 0;
+}
+dev_exit(plic__exit);
-- 
2.17.1

[kvmtool RFC PATCH 6/8] riscv: Generate FDT at runtime for Guest/VM

[Anup Patel]

We generate FDT at runtime for RISC-V Guest/VM so that KVMTOOL users
don't have to pass FDT separately via command-line parameters.

Also, we provide "--dump-dtb <filename>" command-line option to dump
generated FDT into a file for debugging purpose.

Signed-off-by: Atish Patra <atish.patra@wdc.com>
Signed-off-by: Anup Patel <anup.patel@wdc.com>
---
 Makefile                            |   1 +
 riscv/fdt.c                         | 192 ++++++++++++++++++++++++++++
 riscv/include/kvm/fdt-arch.h        |   4 +
 riscv/include/kvm/kvm-arch.h        |   2 +
 riscv/include/kvm/kvm-config-arch.h |   6 +
 riscv/plic.c                        |  50 ++++++++
 6 files changed, 255 insertions(+)
 create mode 100644 riscv/fdt.c

diff --git a/Makefile b/Makefile
index 3220ad3..fb78fa2 100644
--- a/Makefile
+++ b/Makefile
@@ -196,6 +196,7 @@ endif
 ifeq ($(ARCH),riscv)
 	DEFINES		+= -DCONFIG_RISCV
 	ARCH_INCLUDE	:= riscv/include
+	OBJS		+= riscv/fdt.o
 	OBJS		+= riscv/ioport.o
 	OBJS		+= riscv/irq.o
 	OBJS		+= riscv/kvm.o
diff --git a/riscv/fdt.c b/riscv/fdt.c
new file mode 100644
index 0000000..3b56e30
--- /dev/null
+++ b/riscv/fdt.c
@@ -0,0 +1,192 @@
+#include "kvm/devices.h"
+#include "kvm/fdt.h"
+#include "kvm/kvm.h"
+#include "kvm/kvm-cpu.h"
+
+#include <stdbool.h>
+
+#include <linux/byteorder.h>
+#include <linux/kernel.h>
+#include <linux/sizes.h>
+
+static void dump_fdt(const char *dtb_file, void *fdt)
+{
+	int count, fd;
+
+	fd = open(dtb_file, O_CREAT | O_TRUNC | O_RDWR, 0666);
+	if (fd < 0)
+		die("Failed to write dtb to %s", dtb_file);
+
+	count = write(fd, fdt, FDT_MAX_SIZE);
+	if (count < 0)
+		die_perror("Failed to dump dtb");
+
+	pr_debug("Wrote %d bytes to dtb %s", count, dtb_file);
+	close(fd);
+}
+
+#define CPU_NAME_MAX_LEN 15
+#define CPU_ISA_MAX_LEN 128
+static void generate_cpu_nodes(void *fdt, struct kvm *kvm)
+{
+	int cpu, pos, i, index, valid_isa_len;
+	const char *valid_isa_order = "IEMAFDQCLBJTPVNSUHKORWXYZG";
+
+	_FDT(fdt_begin_node(fdt, "cpus"));
+	_FDT(fdt_property_cell(fdt, "#address-cells", 0x1));
+	_FDT(fdt_property_cell(fdt, "#size-cells", 0x0));
+	_FDT(fdt_property_cell(fdt, "timebase-frequency",
+				kvm->cpus[0]->riscv_timebase));
+
+	for (cpu = 0; cpu < kvm->nrcpus; ++cpu) {
+		char cpu_name[CPU_NAME_MAX_LEN];
+		char cpu_isa[CPU_ISA_MAX_LEN];
+		struct kvm_cpu *vcpu = kvm->cpus[cpu];
+
+		snprintf(cpu_name, CPU_NAME_MAX_LEN, "cpu@%x", cpu);
+
+		snprintf(cpu_isa, CPU_ISA_MAX_LEN, "rv%ld", vcpu->riscv_xlen);
+		pos = strlen(cpu_isa);
+		valid_isa_len = strlen(valid_isa_order);
+		for (i = 0; i < valid_isa_len; i++) {
+			index = valid_isa_order[i] - 'A';
+			if (vcpu->riscv_isa & (1 << (index)))
+				cpu_isa[pos++] = 'a' + index;
+		}
+		cpu_isa[pos] = '\0';
+
+		_FDT(fdt_begin_node(fdt, cpu_name));
+		_FDT(fdt_property_string(fdt, "device_type", "cpu"));
+		_FDT(fdt_property_string(fdt, "compatible", "riscv"));
+		if (vcpu->riscv_xlen == 64)
+			_FDT(fdt_property_string(fdt, "mmu-type",
+						 "riscv,sv48"));
+		else
+			_FDT(fdt_property_string(fdt, "mmu-type",
+						 "riscv,sv32"));
+		_FDT(fdt_property_string(fdt, "riscv,isa", cpu_isa));
+		_FDT(fdt_property_cell(fdt, "reg", cpu));
+		_FDT(fdt_property_string(fdt, "status", "okay"));
+
+		_FDT(fdt_begin_node(fdt, "interrupt-controller"));
+		_FDT(fdt_property_string(fdt, "compatible", "riscv,cpu-intc"));
+		_FDT(fdt_property_cell(fdt, "#interrupt-cells", 1));
+		_FDT(fdt_property(fdt, "interrupt-controller", NULL, 0));
+		_FDT(fdt_property_cell(fdt, "phandle",
+					PHANDLE_CPU_INTC_BASE + cpu));
+		_FDT(fdt_end_node(fdt));
+
+		_FDT(fdt_end_node(fdt));
+	}
+
+	_FDT(fdt_end_node(fdt));
+}
+
+static int setup_fdt(struct kvm *kvm)
+{
+	struct device_header *dev_hdr;
+	u8 staging_fdt[FDT_MAX_SIZE];
+	u64 mem_reg_prop[]	= {
+		cpu_to_fdt64(kvm->arch.memory_guest_start),
+		cpu_to_fdt64(kvm->ram_size),
+	};
+	void *fdt		= staging_fdt;
+	void *fdt_dest		= guest_flat_to_host(kvm,
+						     kvm->arch.dtb_guest_start);
+	void (*generate_mmio_fdt_nodes)(void *, struct device_header *,
+					void (*)(void *, u8, enum irq_type));
+
+	/* Create new tree without a reserve map */
+	_FDT(fdt_create(fdt, FDT_MAX_SIZE));
+	_FDT(fdt_finish_reservemap(fdt));
+
+	/* Header */
+	_FDT(fdt_begin_node(fdt, ""));
+	_FDT(fdt_property_cell(fdt, "interrupt-parent", PHANDLE_PLIC));
+	_FDT(fdt_property_string(fdt, "compatible", "linux,dummy-virt"));
+	_FDT(fdt_property_cell(fdt, "#address-cells", 0x2));
+	_FDT(fdt_property_cell(fdt, "#size-cells", 0x2));
+
+	/* /chosen */
+	_FDT(fdt_begin_node(fdt, "chosen"));
+
+	/* Pass on our amended command line to a Linux kernel only. */
+	if (kvm->cfg.firmware_filename) {
+		if (kvm->cfg.kernel_cmdline)
+			_FDT(fdt_property_string(fdt, "bootargs",
+						 kvm->cfg.kernel_cmdline));
+	} else
+		_FDT(fdt_property_string(fdt, "bootargs",
+					 kvm->cfg.real_cmdline));
+
+	_FDT(fdt_property_string(fdt, "stdout-path", "serial0"));
+
+	/* Initrd */
+	if (kvm->arch.initrd_size != 0) {
+		u64 ird_st_prop = cpu_to_fdt64(kvm->arch.initrd_guest_start);
+		u64 ird_end_prop = cpu_to_fdt64(kvm->arch.initrd_guest_start +
+					       kvm->arch.initrd_size);
+
+		_FDT(fdt_property(fdt, "linux,initrd-start",
+				   &ird_st_prop, sizeof(ird_st_prop)));
+		_FDT(fdt_property(fdt, "linux,initrd-end",
+				   &ird_end_prop, sizeof(ird_end_prop)));
+	}
+
+	_FDT(fdt_end_node(fdt));
+
+	/* Memory */
+	_FDT(fdt_begin_node(fdt, "memory"));
+	_FDT(fdt_property_string(fdt, "device_type", "memory"));
+	_FDT(fdt_property(fdt, "reg", mem_reg_prop, sizeof(mem_reg_prop)));
+	_FDT(fdt_end_node(fdt));
+
+	/* CPUs */
+	generate_cpu_nodes(fdt, kvm);
+
+	/* Simple Bus */
+	_FDT(fdt_begin_node(fdt, "smb"));
+	_FDT(fdt_property_string(fdt, "compatible", "simple-bus"));
+	_FDT(fdt_property_cell(fdt, "#address-cells", 0x2));
+	_FDT(fdt_property_cell(fdt, "#size-cells", 0x2));
+	_FDT(fdt_property(fdt, "ranges", NULL, 0));
+
+	/* Virtio MMIO devices */
+	dev_hdr = device__first_dev(DEVICE_BUS_MMIO);
+	while (dev_hdr) {
+		generate_mmio_fdt_nodes = dev_hdr->data;
+		generate_mmio_fdt_nodes(fdt, dev_hdr, plic__generate_irq_prop);
+		dev_hdr = device__next_dev(dev_hdr);
+	}
+
+	/* IOPORT devices */
+	dev_hdr = device__first_dev(DEVICE_BUS_IOPORT);
+	while (dev_hdr) {
+		generate_mmio_fdt_nodes = dev_hdr->data;
+		generate_mmio_fdt_nodes(fdt, dev_hdr, plic__generate_irq_prop);
+		dev_hdr = device__next_dev(dev_hdr);
+	}
+
+	_FDT(fdt_end_node(fdt));
+
+	if (fdt_stdout_path) {
+		_FDT(fdt_begin_node(fdt, "aliases"));
+		_FDT(fdt_property_string(fdt, "serial0", fdt_stdout_path));
+		_FDT(fdt_end_node(fdt));
+
+		free(fdt_stdout_path);
+		fdt_stdout_path = NULL;
+	}
+
+	/* Finalise. */
+	_FDT(fdt_end_node(fdt));
+	_FDT(fdt_finish(fdt));
+
+	_FDT(fdt_open_into(fdt, fdt_dest, FDT_MAX_SIZE));
+	_FDT(fdt_pack(fdt_dest));
+
+	if (kvm->cfg.arch.dump_dtb_filename)
+		dump_fdt(kvm->cfg.arch.dump_dtb_filename, fdt_dest);
+	return 0;
+}
+late_init(setup_fdt);
diff --git a/riscv/include/kvm/fdt-arch.h b/riscv/include/kvm/fdt-arch.h
index 9450fc5..f7548e8 100644
--- a/riscv/include/kvm/fdt-arch.h
+++ b/riscv/include/kvm/fdt-arch.h
@@ -1,4 +1,8 @@
 #ifndef KVM__KVM_FDT_H
 #define KVM__KVM_FDT_H
 
+enum phandles {PHANDLE_RESERVED = 0, PHANDLE_PLIC, PHANDLES_MAX};
+
+#define PHANDLE_CPU_INTC_BASE	PHANDLES_MAX
+
 #endif /* KVM__KVM_FDT_H */
diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
index b1af70f..7d3fd73 100644
--- a/riscv/include/kvm/kvm-arch.h
+++ b/riscv/include/kvm/kvm-arch.h
@@ -70,6 +70,8 @@ static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
 
 enum irq_type;
 
+void plic__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
+
 void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge);
 
 #endif /* KVM__KVM_ARCH_H */
diff --git a/riscv/include/kvm/kvm-config-arch.h b/riscv/include/kvm/kvm-config-arch.h
index 60c7333..526fca2 100644
--- a/riscv/include/kvm/kvm-config-arch.h
+++ b/riscv/include/kvm/kvm-config-arch.h
@@ -4,6 +4,12 @@
 #include "kvm/parse-options.h"
 
 struct kvm_config_arch {
+	const char	*dump_dtb_filename;
 };
 
+#define OPT_ARCH_RUN(pfx, cfg)						\
+	pfx,								\
+	OPT_STRING('\0', "dump-dtb", &(cfg)->dump_dtb_filename,		\
+		   ".dtb file", "Dump generated .dtb to specified file"),
+
 #endif /* KVM__KVM_CONFIG_ARCH_H */
diff --git a/riscv/plic.c b/riscv/plic.c
index 93bfbc5..1112d16 100644
--- a/riscv/plic.c
+++ b/riscv/plic.c
@@ -1,5 +1,6 @@
 
 #include "kvm/devices.h"
+#include "kvm/fdt.h"
 #include "kvm/ioeventfd.h"
 #include "kvm/ioport.h"
 #include "kvm/kvm.h"
@@ -455,6 +456,54 @@ static void plic__mmio_callback(struct kvm_cpu *vcpu,
 	}
 }
 
+void plic__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type)
+{
+	u32 irq_prop[] = {
+		cpu_to_fdt32(irq)
+	};
+
+	_FDT(fdt_property(fdt, "interrupts", irq_prop, sizeof(irq_prop)));
+}
+
+static void plic__generate_fdt_node(void *fdt,
+				    struct device_header *dev_hdr,
+				    void (*generate_irq_prop)(void *fdt,
+							      u8 irq,
+							      enum irq_type))
+{
+	u32 i;
+	u32 reg_cells[4], *irq_cells;
+
+	reg_cells[0] = 0;
+	reg_cells[1] = cpu_to_fdt32(RISCV_PLIC);
+	reg_cells[2] = 0;
+	reg_cells[3] = cpu_to_fdt32(RISCV_PLIC_SIZE);
+
+	irq_cells = calloc(plic.num_context * 2, sizeof(u32));
+	if (!irq_cells)
+		die("Failed to alloc irq_cells");
+
+	_FDT(fdt_begin_node(fdt, "interrupt-controller@0c000000"));
+	_FDT(fdt_property_string(fdt, "compatible", "riscv,plic0"));
+	_FDT(fdt_property(fdt, "reg", reg_cells, sizeof(reg_cells)));
+	_FDT(fdt_property_cell(fdt, "#interrupt-cells", 1));
+	_FDT(fdt_property(fdt, "interrupt-controller", NULL, 0));
+	_FDT(fdt_property_cell(fdt, "riscv,max-priority", plic.max_prio));
+	_FDT(fdt_property_cell(fdt, "riscv,ndev", MAX_DEVICES));
+	_FDT(fdt_property_cell(fdt, "phandle", PHANDLE_PLIC));
+	for (i = 0; i < (plic.num_context / 2); i++) {
+		irq_cells[4*i + 0] = cpu_to_fdt32(PHANDLE_CPU_INTC_BASE + i);
+		irq_cells[4*i + 1] = cpu_to_fdt32(0xffffffff);
+		irq_cells[4*i + 2] = cpu_to_fdt32(PHANDLE_CPU_INTC_BASE + i);
+		irq_cells[4*i + 3] = cpu_to_fdt32(9);
+	}
+	_FDT(fdt_property(fdt, "interrupts-extended", irq_cells,
+			  sizeof(u32) * plic.num_context * 2));
+	_FDT(fdt_end_node(fdt));
+
+	free(irq_cells);
+}
+
 static int plic__init(struct kvm *kvm)
 {
 	u32 i;
@@ -463,6 +512,7 @@ static int plic__init(struct kvm *kvm)
 	plic.kvm = kvm;
 	plic.dev_hdr = (struct device_header) {
 		.bus_type	= DEVICE_BUS_MMIO,
+		.data		= plic__generate_fdt_node,
 	};
 
 	plic.num_irq = MAX_DEVICES;
-- 
2.17.1

[kvmtool RFC PATCH 7/8] riscv: Handle SBI calls forwarded to user space

[Anup Patel]

The kernel KVM RISC-V module will forward certain SBI calls
to user space. These forwared SBI calls will usually be the
SBI calls which cannot be emulated in kernel space such as
PUTCHAR and GETCHAR calls.

This patch extends kvm_cpu__handle_exit() to handle SBI calls
forwarded to user space.

Signed-off-by: Atish Patra <atish.patra@wdc.com>
Signed-off-by: Anup Patel <anup.patel@wdc.com>
---
 riscv/include/kvm/sbi.h | 48 ++++++++++++++++++++++++++++++++++++++++
 riscv/kvm-cpu.c         | 49 ++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 96 insertions(+), 1 deletion(-)
 create mode 100644 riscv/include/kvm/sbi.h

diff --git a/riscv/include/kvm/sbi.h b/riscv/include/kvm/sbi.h
new file mode 100644
index 0000000..f4b4182
--- /dev/null
+++ b/riscv/include/kvm/sbi.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Common SBI related defines and macros to be used by RISC-V kernel,
+ * RISC-V KVM and userspace.
+ *
+ * Copyright (c) 2019 Western Digital Corporation or its affiliates.
+ */
+
+#ifndef __RISCV_SBI_H__
+#define __RISCV_SBI_H__
+
+enum sbi_ext_id {
+	SBI_EXT_0_1_SET_TIMER = 0x0,
+	SBI_EXT_0_1_CONSOLE_PUTCHAR = 0x1,
+	SBI_EXT_0_1_CONSOLE_GETCHAR = 0x2,
+	SBI_EXT_0_1_CLEAR_IPI = 0x3,
+	SBI_EXT_0_1_SEND_IPI = 0x4,
+	SBI_EXT_0_1_REMOTE_FENCE_I = 0x5,
+	SBI_EXT_0_1_REMOTE_SFENCE_VMA = 0x6,
+	SBI_EXT_0_1_REMOTE_SFENCE_VMA_ASID = 0x7,
+	SBI_EXT_0_1_SHUTDOWN = 0x8,
+	SBI_EXT_BASE = 0x10,
+};
+
+enum sbi_ext_base_fid {
+	SBI_BASE_GET_SPEC_VERSION = 0,
+	SBI_BASE_GET_IMP_ID,
+	SBI_BASE_GET_IMP_VERSION,
+	SBI_BASE_PROBE_EXT,
+	SBI_BASE_GET_MVENDORID,
+	SBI_BASE_GET_MARCHID,
+	SBI_BASE_GET_MIMPID,
+};
+
+#define SBI_SPEC_VERSION_DEFAULT	0x1
+#define SBI_SPEC_VERSION_MAJOR_OFFSET	24
+#define SBI_SPEC_VERSION_MAJOR_MASK	0x7f
+#define SBI_SPEC_VERSION_MINOR_MASK	0xffffff
+
+/* SBI return error codes */
+#define SBI_SUCCESS		0
+#define SBI_ERR_FAILURE		-1
+#define SBI_ERR_NOT_SUPPORTED	-2
+#define SBI_ERR_INVALID_PARAM   -3
+#define SBI_ERR_DENIED		-4
+#define SBI_ERR_INVALID_ADDRESS -5
+
+#endif
diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
index 1565275..4f52b92 100644
--- a/riscv/kvm-cpu.c
+++ b/riscv/kvm-cpu.c
@@ -1,6 +1,7 @@
 #include "kvm/kvm-cpu.h"
 #include "kvm/kvm.h"
 #include "kvm/virtio.h"
+#include "kvm/sbi.h"
 #include "kvm/term.h"
 
 #include <asm/ptrace.h>
@@ -106,9 +107,55 @@ void kvm_cpu__delete(struct kvm_cpu *vcpu)
 	free(vcpu);
 }
 
+static bool kvm_cpu_riscv_sbi(struct kvm_cpu *vcpu)
+{
+	char ch;
+	bool ret = true;
+	int dfd = kvm_cpu__get_debug_fd();
+
+	switch (vcpu->kvm_run->riscv_sbi.extension_id) {
+	case SBI_EXT_0_1_CONSOLE_PUTCHAR:
+		ch = vcpu->kvm_run->riscv_sbi.args[0];
+		term_putc(&ch, 1, 0);
+		vcpu->kvm_run->riscv_sbi.ret[0] = 0;
+		break;
+	case SBI_EXT_0_1_CONSOLE_GETCHAR:
+		if (term_readable(0))
+			vcpu->kvm_run->riscv_sbi.ret[0] =
+					term_getc(vcpu->kvm, 0);
+		else
+			vcpu->kvm_run->riscv_sbi.ret[0] = SBI_ERR_FAILURE;
+		break;
+	default:
+		dprintf(dfd, "Unhandled SBI call\n");
+		dprintf(dfd, "extension_id=0x%lx function_id=0x%lx\n",
+			vcpu->kvm_run->riscv_sbi.extension_id,
+			vcpu->kvm_run->riscv_sbi.function_id);
+		dprintf(dfd, "args[0]=0x%lx args[1]=0x%lx\n",
+			vcpu->kvm_run->riscv_sbi.args[0],
+			vcpu->kvm_run->riscv_sbi.args[1]);
+		dprintf(dfd, "args[2]=0x%lx args[3]=0x%lx\n",
+			vcpu->kvm_run->riscv_sbi.args[2],
+			vcpu->kvm_run->riscv_sbi.args[3]);
+		dprintf(dfd, "args[4]=0x%lx args[5]=0x%lx\n",
+			vcpu->kvm_run->riscv_sbi.args[4],
+			vcpu->kvm_run->riscv_sbi.args[5]);
+		ret = false;
+		break;
+	};
+
+	return ret;
+}
+
 bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
 {
-	/* TODO: */
+	switch (vcpu->kvm_run->exit_reason) {
+	case KVM_EXIT_RISCV_SBI:
+		return kvm_cpu_riscv_sbi(vcpu);
+	default:
+		break;
+	};
+
 	return false;
 }
 
-- 
2.17.1

[kvmtool RFC PATCH 8/8] riscv: Generate PCI host DT node

[Anup Patel]

This patch extends FDT generation to generate PCI host DT node.

Of course, PCI host for Guest/VM is not useful at the moment
because it's mostly for PCI pass-through and we don't have
IOMMU and interrupt routing available for KVM RISC-V. In future,
we might be able to use PCI host for VirtIO PCI transport or
other software emulated PCI devices.

Signed-off-by: Anup Patel <anup.patel@wdc.com>
---
 Makefile                     |   1 +
 riscv/fdt.c                  |   3 +
 riscv/include/kvm/kvm-arch.h |   2 +
 riscv/pci.c                  | 109 +++++++++++++++++++++++++++++++++++
 4 files changed, 115 insertions(+)
 create mode 100644 riscv/pci.c

diff --git a/Makefile b/Makefile
index fb78fa2..fa5e4f7 100644
--- a/Makefile
+++ b/Makefile
@@ -201,6 +201,7 @@ ifeq ($(ARCH),riscv)
 	OBJS		+= riscv/irq.o
 	OBJS		+= riscv/kvm.o
 	OBJS		+= riscv/kvm-cpu.o
+	OBJS		+= riscv/pci.o
 	OBJS		+= riscv/plic.o
 
 	ARCH_WANT_LIBFDT := y
diff --git a/riscv/fdt.c b/riscv/fdt.c
index 3b56e30..fc8aa88 100644
--- a/riscv/fdt.c
+++ b/riscv/fdt.c
@@ -167,6 +167,9 @@ static int setup_fdt(struct kvm *kvm)
 		dev_hdr = device__next_dev(dev_hdr);
 	}
 
+	/* PCI host controller */
+	pci__generate_fdt_nodes(fdt);
+
 	_FDT(fdt_end_node(fdt));
 
 	if (fdt_stdout_path) {
diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
index 7d3fd73..54a8c69 100644
--- a/riscv/include/kvm/kvm-arch.h
+++ b/riscv/include/kvm/kvm-arch.h
@@ -74,4 +74,6 @@ void plic__generate_irq_prop(void *fdt, u8 irq, enum irq_type irq_type);
 
 void plic__irq_trig(struct kvm *kvm, int irq, int level, bool edge);
 
+void pci__generate_fdt_nodes(void *fdt);
+
 #endif /* KVM__KVM_ARCH_H */
diff --git a/riscv/pci.c b/riscv/pci.c
new file mode 100644
index 0000000..666a452
--- /dev/null
+++ b/riscv/pci.c
@@ -0,0 +1,109 @@
+#include "kvm/devices.h"
+#include "kvm/fdt.h"
+#include "kvm/kvm.h"
+#include "kvm/of_pci.h"
+#include "kvm/pci.h"
+#include "kvm/util.h"
+
+/*
+ * An entry in the interrupt-map table looks like:
+ * <pci unit address> <pci interrupt pin> <plic phandle> <plic interrupt>
+ */
+
+struct of_interrupt_map_entry {
+	struct of_pci_irq_mask		pci_irq_mask;
+	u32				plic_phandle;
+	u32				plic_irq;
+} __attribute__((packed));
+
+void pci__generate_fdt_nodes(void *fdt)
+{
+	struct device_header *dev_hdr;
+	struct of_interrupt_map_entry irq_map[OF_PCI_IRQ_MAP_MAX];
+	unsigned nentries = 0;
+	/* Bus range */
+	u32 bus_range[] = { cpu_to_fdt32(0), cpu_to_fdt32(1), };
+	/* Configuration Space */
+	u64 cfg_reg_prop[] = { cpu_to_fdt64(KVM_PCI_CFG_AREA),
+			       cpu_to_fdt64(RISCV_PCI_CFG_SIZE), };
+	/* Describe the memory ranges */
+	struct of_pci_ranges_entry ranges[] = {
+		{
+			.pci_addr = {
+				.hi	= cpu_to_fdt32(of_pci_b_ss(OF_PCI_SS_IO)),
+				.mid	= 0,
+				.lo	= 0,
+			},
+			.cpu_addr	= cpu_to_fdt64(KVM_IOPORT_AREA),
+			.length		= cpu_to_fdt64(RISCV_IOPORT_SIZE),
+		},
+		{
+			.pci_addr = {
+				.hi	= cpu_to_fdt32(of_pci_b_ss(OF_PCI_SS_M32)),
+				.mid	= cpu_to_fdt32(KVM_PCI_MMIO_AREA >> 32),
+				.lo	= cpu_to_fdt32(KVM_PCI_MMIO_AREA),
+			},
+			.cpu_addr	= cpu_to_fdt64(KVM_PCI_MMIO_AREA),
+			.length		= cpu_to_fdt64(RISCV_PCI_MMIO_SIZE),
+		},
+	};
+
+	/* Boilerplate PCI properties */
+	_FDT(fdt_begin_node(fdt, "pci"));
+	_FDT(fdt_property_string(fdt, "device_type", "pci"));
+	_FDT(fdt_property_cell(fdt, "#address-cells", 0x3));
+	_FDT(fdt_property_cell(fdt, "#size-cells", 0x2));
+	_FDT(fdt_property_cell(fdt, "#interrupt-cells", 0x1));
+	_FDT(fdt_property_string(fdt, "compatible", "pci-host-cam-generic"));
+	_FDT(fdt_property(fdt, "dma-coherent", NULL, 0));
+
+	_FDT(fdt_property(fdt, "bus-range", bus_range, sizeof(bus_range)));
+	_FDT(fdt_property(fdt, "reg", &cfg_reg_prop, sizeof(cfg_reg_prop)));
+	_FDT(fdt_property(fdt, "ranges", ranges, sizeof(ranges)));
+
+	/* Generate the interrupt map ... */
+	dev_hdr = device__first_dev(DEVICE_BUS_PCI);
+	while (dev_hdr && nentries < ARRAY_SIZE(irq_map)) {
+		struct of_interrupt_map_entry *entry = &irq_map[nentries];
+		struct pci_device_header *pci_hdr = dev_hdr->data;
+		u8 dev_num = dev_hdr->dev_num;
+		u8 pin = pci_hdr->irq_pin;
+		u8 irq = pci_hdr->irq_line;
+
+		*entry = (struct of_interrupt_map_entry) {
+			.pci_irq_mask = {
+				.pci_addr = {
+					.hi	= cpu_to_fdt32(of_pci_b_ddddd(dev_num)),
+					.mid	= 0,
+					.lo	= 0,
+				},
+				.pci_pin	= cpu_to_fdt32(pin),
+			},
+			.plic_phandle	= cpu_to_fdt32(PHANDLE_PLIC),
+			.plic_irq	= cpu_to_fdt32(irq),
+		};
+
+		nentries++;
+		dev_hdr = device__next_dev(dev_hdr);
+	}
+
+	_FDT(fdt_property(fdt, "interrupt-map", irq_map,
+			  sizeof(struct of_interrupt_map_entry) * nentries));
+
+	/* ... and the corresponding mask. */
+	if (nentries) {
+		struct of_pci_irq_mask irq_mask = {
+			.pci_addr = {
+				.hi	= cpu_to_fdt32(of_pci_b_ddddd(-1)),
+				.mid	= 0,
+				.lo	= 0,
+			},
+			.pci_pin	= cpu_to_fdt32(7),
+		};
+
+		_FDT(fdt_property(fdt, "interrupt-map-mask", &irq_mask,
+				  sizeof(irq_mask)));
+	}
+
+	_FDT(fdt_end_node(fdt));
+}
-- 
2.17.1

Re: [kvmtool RFC PATCH 2/8] riscv: Initial skeletal support

[Alexandru Elisei]

Hi,

I don't know much about the RISC-V architecture, so I'm only going to comment on
the more generic code.

On 12/25/19 3:00 AM, Anup Patel wrote:
> This patch adds initial skeletal KVMTOOL RISC-V support which
> just compiles for RV32 and RV64 host.
>
> Signed-off-by: Anup Patel <anup.patel@wdc.com>
> ---
>  INSTALL                             |   7 +-
>  Makefile                            |  15 +++-
>  riscv/include/asm/kvm.h             | 127 ++++++++++++++++++++++++++++
>  riscv/include/kvm/barrier.h         |  14 +++
>  riscv/include/kvm/fdt-arch.h        |   4 +
>  riscv/include/kvm/kvm-arch.h        |  58 +++++++++++++
>  riscv/include/kvm/kvm-config-arch.h |   9 ++
>  riscv/include/kvm/kvm-cpu-arch.h    |  49 +++++++++++
>  riscv/ioport.c                      |  11 +++
>  riscv/irq.c                         |  13 +++
>  riscv/kvm-cpu.c                     |  64 ++++++++++++++
>  riscv/kvm.c                         |  61 +++++++++++++
>  util/update_headers.sh              |   2 +-
>  13 files changed, 429 insertions(+), 5 deletions(-)
>  create mode 100644 riscv/include/asm/kvm.h
>  create mode 100644 riscv/include/kvm/barrier.h
>  create mode 100644 riscv/include/kvm/fdt-arch.h
>  create mode 100644 riscv/include/kvm/kvm-arch.h
>  create mode 100644 riscv/include/kvm/kvm-config-arch.h
>  create mode 100644 riscv/include/kvm/kvm-cpu-arch.h
>  create mode 100644 riscv/ioport.c
>  create mode 100644 riscv/irq.c
>  create mode 100644 riscv/kvm-cpu.c
>  create mode 100644 riscv/kvm.c
>
> diff --git a/INSTALL b/INSTALL
> index ca8e022..951b123 100644
> --- a/INSTALL
> +++ b/INSTALL
> @@ -26,8 +26,8 @@ For Fedora based systems:
>  For OpenSUSE based systems:
>  	# zypper install glibc-devel-static
>  
> -Architectures which require device tree (PowerPC, ARM, ARM64) also require
> -libfdt.
> +Architectures which require device tree (PowerPC, ARM, ARM64, RISC-V) also
> +require libfdt.
>  	deb: $ sudo apt-get install libfdt-dev
>  	Fedora: # yum install libfdt-devel
>  	OpenSUSE: # zypper install libfdt1-devel
> @@ -64,6 +64,7 @@ to the Linux name of the architecture. Architectures supported:
>  - arm
>  - arm64
>  - mips
> +- riscv
>  If ARCH is not provided, the target architecture will be automatically
>  determined by running "uname -m" on your host, resulting in a native build.
>  
> @@ -81,7 +82,7 @@ On multiarch system you should be able to install those be appending
>  the architecture name after the package (example for ARM64):
>  $ sudo apt-get install libfdt-dev:arm64
>  
> -PowerPC and ARM/ARM64 require libfdt to be installed. If you cannot use
> +PowerPC, ARM/ARM64 and RISC-V require libfdt to be installed. If you cannot use
>  precompiled mulitarch packages, you could either copy the required header and
>  library files from an installed target system into the SYSROOT (you will need
>  /usr/include/*fdt*.h and /usr/lib64/libfdt-v.v.v.so and its symlinks), or you
> diff --git a/Makefile b/Makefile
> index 3862112..972fa63 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -106,7 +106,8 @@ OBJS	+= hw/i8042.o
>  
>  # Translate uname -m into ARCH string
>  ARCH ?= $(shell uname -m | sed -e s/i.86/i386/ -e s/ppc.*/powerpc/ \
> -	  -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/)
> +	  -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/ \
> +	  -e s/riscv64/riscv/ -e s/riscv32/riscv/)
>  
>  ifeq ($(ARCH),i386)
>  	ARCH         := x86
> @@ -190,6 +191,18 @@ ifeq ($(ARCH),mips)
>  	OBJS		+= mips/kvm.o
>  	OBJS		+= mips/kvm-cpu.o
>  endif
> +
> +# RISC-V (RV32 and RV64)
> +ifeq ($(ARCH),riscv)
> +	DEFINES		+= -DCONFIG_RISCV
> +	ARCH_INCLUDE	:= riscv/include
> +	OBJS		+= riscv/ioport.o
> +	OBJS		+= riscv/irq.o
> +	OBJS		+= riscv/kvm.o
> +	OBJS		+= riscv/kvm-cpu.o
> +
> +	ARCH_WANT_LIBFDT := y
> +endif
>  ###
>  
>  ifeq (,$(ARCH_INCLUDE))
> diff --git a/riscv/include/asm/kvm.h b/riscv/include/asm/kvm.h
> new file mode 100644
> index 0000000..f4274c2
> --- /dev/null
> +++ b/riscv/include/asm/kvm.h
> @@ -0,0 +1,127 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright (C) 2019 Western Digital Corporation or its affiliates.
> + *
> + * Authors:
> + *     Anup Patel <anup.patel@wdc.com>
> + */
> +
> +#ifndef __LINUX_KVM_RISCV_H
> +#define __LINUX_KVM_RISCV_H
> +
> +#ifndef __ASSEMBLY__
> +
> +#include <linux/types.h>
> +#include <asm/ptrace.h>
> +
> +#define __KVM_HAVE_READONLY_MEM
> +
> +#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
> +
> +#define KVM_INTERRUPT_SET	-1U
> +#define KVM_INTERRUPT_UNSET	-2U
> +
> +/* for KVM_GET_REGS and KVM_SET_REGS */
> +struct kvm_regs {
> +};
> +
> +/* for KVM_GET_FPU and KVM_SET_FPU */
> +struct kvm_fpu {
> +};
> +
> +/* KVM Debug exit structure */
> +struct kvm_debug_exit_arch {
> +};
> +
> +/* for KVM_SET_GUEST_DEBUG */
> +struct kvm_guest_debug_arch {
> +};
> +
> +/* definition of registers in kvm_run */
> +struct kvm_sync_regs {
> +};
> +
> +/* for KVM_GET_SREGS and KVM_SET_SREGS */
> +struct kvm_sregs {
> +};
> +
> +/* CONFIG registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> +struct kvm_riscv_config {
> +	unsigned long isa;
> +};
> +
> +/* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> +struct kvm_riscv_core {
> +	struct user_regs_struct regs;
> +	unsigned long mode;
> +};
> +
> +/* Possible privilege modes for kvm_riscv_core */
> +#define KVM_RISCV_MODE_S	1
> +#define KVM_RISCV_MODE_U	0
> +
> +/* CSR registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> +struct kvm_riscv_csr {
> +	unsigned long sstatus;
> +	unsigned long sie;
> +	unsigned long stvec;
> +	unsigned long sscratch;
> +	unsigned long sepc;
> +	unsigned long scause;
> +	unsigned long stval;
> +	unsigned long sip;
> +	unsigned long satp;
> +};
> +
> +/* TIMER registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> +struct kvm_riscv_timer {
> +	u64 frequency;
> +	u64 time;
> +	u64 compare;
> +	u64 state;
> +};
> +
> +/* Possible states for kvm_riscv_timer */
> +#define KVM_RISCV_TIMER_STATE_OFF	0
> +#define KVM_RISCV_TIMER_STATE_ON	1
> +
> +#define KVM_REG_SIZE(id)		\
> +	(1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
> +
> +/* If you need to interpret the index values, here is the key: */
> +#define KVM_REG_RISCV_TYPE_MASK		0x00000000FF000000
> +#define KVM_REG_RISCV_TYPE_SHIFT	24
> +
> +/* Config registers are mapped as type 1 */
> +#define KVM_REG_RISCV_CONFIG		(0x01 << KVM_REG_RISCV_TYPE_SHIFT)
> +#define KVM_REG_RISCV_CONFIG_REG(name)	\
> +	(offsetof(struct kvm_riscv_config, name) / sizeof(unsigned long))
> +
> +/* Core registers are mapped as type 2 */
> +#define KVM_REG_RISCV_CORE		(0x02 << KVM_REG_RISCV_TYPE_SHIFT)
> +#define KVM_REG_RISCV_CORE_REG(name)	\
> +		(offsetof(struct kvm_riscv_core, name) / sizeof(unsigned long))
> +
> +/* Control and status registers are mapped as type 3 */
> +#define KVM_REG_RISCV_CSR		(0x03 << KVM_REG_RISCV_TYPE_SHIFT)
> +#define KVM_REG_RISCV_CSR_REG(name)	\
> +		(offsetof(struct kvm_riscv_csr, name) / sizeof(unsigned long))
> +
> +/* Timer registers are mapped as type 4 */
> +#define KVM_REG_RISCV_TIMER		(0x04 << KVM_REG_RISCV_TYPE_SHIFT)
> +#define KVM_REG_RISCV_TIMER_REG(name)	\
> +		(offsetof(struct kvm_riscv_timer, name) / sizeof(u64))
> +
> +/* F extension registers are mapped as type 5 */
> +#define KVM_REG_RISCV_FP_F		(0x05 << KVM_REG_RISCV_TYPE_SHIFT)
> +#define KVM_REG_RISCV_FP_F_REG(name)	\
> +		(offsetof(struct __riscv_f_ext_state, name) / sizeof(u32))
> +
> +/* D extension registers are mapped as type 6 */
> +#define KVM_REG_RISCV_FP_D		(0x06 << KVM_REG_RISCV_TYPE_SHIFT)
> +#define KVM_REG_RISCV_FP_D_REG(name)	\
> +		(offsetof(struct __riscv_d_ext_state, name) / sizeof(u64))
> +
> +#endif
> +
> +#endif /* __LINUX_KVM_RISCV_H */
> diff --git a/riscv/include/kvm/barrier.h b/riscv/include/kvm/barrier.h
> new file mode 100644
> index 0000000..235f610
> --- /dev/null
> +++ b/riscv/include/kvm/barrier.h
> @@ -0,0 +1,14 @@
> +#ifndef KVM__KVM_BARRIER_H
> +#define KVM__KVM_BARRIER_H
> +
> +#define nop()		__asm__ __volatile__ ("nop")
> +
> +#define RISCV_FENCE(p, s) \
> +	__asm__ __volatile__ ("fence " #p "," #s : : : "memory")
> +
> +/* These barriers need to enforce ordering on both devices or memory. */
> +#define mb()		RISCV_FENCE(iorw,iorw)
> +#define rmb()		RISCV_FENCE(ir,ir)
> +#define wmb()		RISCV_FENCE(ow,ow)
> +
> +#endif /* KVM__KVM_BARRIER_H */
> diff --git a/riscv/include/kvm/fdt-arch.h b/riscv/include/kvm/fdt-arch.h
> new file mode 100644
> index 0000000..9450fc5
> --- /dev/null
> +++ b/riscv/include/kvm/fdt-arch.h
> @@ -0,0 +1,4 @@
> +#ifndef KVM__KVM_FDT_H
> +#define KVM__KVM_FDT_H
> +
> +#endif /* KVM__KVM_FDT_H */
> diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
> new file mode 100644
> index 0000000..7e9c578
> --- /dev/null
> +++ b/riscv/include/kvm/kvm-arch.h
> @@ -0,0 +1,58 @@
> +#ifndef KVM__KVM_ARCH_H
> +#define KVM__KVM_ARCH_H
> +
> +#include <stdbool.h>
> +#include <linux/const.h>
> +#include <linux/types.h>
> +
> +#define RISCV_IOPORT		0x00000000ULL
> +#define RISCV_IOPORT_SIZE	0x00010000ULL
> +#define RISCV_PLIC		0x0c000000ULL
> +#define RISCV_PLIC_SIZE		0x04000000ULL
> +#define RISCV_MMIO		0x10000000ULL
> +#define RISCV_MMIO_SIZE		0x20000000ULL
> +#define RISCV_PCI		0x30000000ULL
> +#define RISCV_PCI_CFG_SIZE	0x10000000ULL

In the DTB you're advertising the PCI node as CAM compatible, which is the right
thing to do. Legacy PCI configuration space is 16MB, not 256MB (PCI Express is 256MB).

> +#define RISCV_PCI_SIZE		0x50000000ULL
> +#define RISCV_PCI_MMIO_SIZE	(RISCV_PCI_SIZE - RISCV_PCI_CFG_SIZE)
> +
> +#define RISCV_RAM		0x80000000ULL

I'm not sure about the reasons for choosing RAM to start at 2GB. For arm we do the
same, but qemu starts memory at 1GB and this mismatch has caused some issues in
the past. For example, 32 bit kvm-unit-tests currently do not run under kvmtool
because the text address is hardcoded to the qemu default value.

As a more general observation, I know that other architectures (like arm) declare
the memory layout in hexadecimal numbers, but it might be a better idea to use the
sizes from include/linux/sizes.h, since it makes the memory layout a lot more
readable and mistakes are easier to spot.

> +
> +#define RISCV_LOMAP_MAX_MEMORY	((1ULL << 32) - RISCV_RAM)
> +#define RISCV_HIMAP_MAX_MEMORY	((1ULL << 40) - RISCV_RAM)
> +
> +#if __riscv_xlen == 64
> +#define RISCV_MAX_MEMORY(kvm)	RISCV_HIMAP_MAX_MEMORY
> +#elif __riscv_xlen == 32
> +#define RISCV_MAX_MEMORY(kvm)	RISCV_LOMAP_MAX_MEMORY
> +#endif
> +
> +#define KVM_IOPORT_AREA		RISCV_IOPORT
> +#define KVM_PCI_CFG_AREA	RISCV_PCI
> +#define KVM_PCI_MMIO_AREA	(KVM_PCI_CFG_AREA + RISCV_PCI_CFG_SIZE)
> +#define KVM_VIRTIO_MMIO_AREA	RISCV_MMIO
> +
> +#define KVM_IOEVENTFD_HAS_PIO	0
> +
> +#define KVM_IRQ_OFFSET		0
> +
> +#define KVM_VM_TYPE		0
> +
> +#define VIRTIO_DEFAULT_TRANS(kvm)	VIRTIO_MMIO
> +
> +#define VIRTIO_RING_ENDIAN	VIRTIO_ENDIAN_LE
> +
> +struct kvm;
> +
> +struct kvm_arch {
> +};
> +
> +static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
> +{
> +	u64 limit = KVM_IOPORT_AREA + RISCV_IOPORT_SIZE;
> +	return phys_addr >= KVM_IOPORT_AREA && phys_addr < limit;
> +}
> +
> +enum irq_type;
> +
> +#endif /* KVM__KVM_ARCH_H */
> diff --git a/riscv/include/kvm/kvm-config-arch.h b/riscv/include/kvm/kvm-config-arch.h
> new file mode 100644
> index 0000000..60c7333
> --- /dev/null
> +++ b/riscv/include/kvm/kvm-config-arch.h
> @@ -0,0 +1,9 @@
> +#ifndef KVM__KVM_CONFIG_ARCH_H
> +#define KVM__KVM_CONFIG_ARCH_H
> +
> +#include "kvm/parse-options.h"
> +
> +struct kvm_config_arch {
> +};
> +
> +#endif /* KVM__KVM_CONFIG_ARCH_H */
> diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
> new file mode 100644
> index 0000000..09a50e8
> --- /dev/null
> +++ b/riscv/include/kvm/kvm-cpu-arch.h
> @@ -0,0 +1,49 @@
> +#ifndef KVM__KVM_CPU_ARCH_H
> +#define KVM__KVM_CPU_ARCH_H
> +
> +#include <linux/kvm.h>
> +#include <pthread.h>
> +#include <stdbool.h>
> +
> +#include "kvm/kvm.h"
> +
> +struct kvm;

Shouldn't kvm.h already have a definition for struct kvm? Also, the arm
corresponding header doesn't have the include here, I don't think it's needed
(unless I'm missing something).

Thanks,
Alex
> +
> +struct kvm_cpu {
> +	pthread_t	thread;
> +
> +	unsigned long   cpu_id;
> +
> +	struct kvm	*kvm;
> +	int		vcpu_fd;
> +	struct kvm_run	*kvm_run;
> +	struct kvm_cpu_task	*task;
> +
> +	u8		is_running;
> +	u8		paused;
> +	u8		needs_nmi;
> +
> +	struct kvm_coalesced_mmio_ring	*ring;
> +};
> +
> +static inline bool kvm_cpu__emulate_io(struct kvm_cpu *vcpu, u16 port,
> +				       void *data, int direction,
> +				       int size, u32 count)
> +{
> +	return false;
> +}
> +
> +static inline bool kvm_cpu__emulate_mmio(struct kvm_cpu *vcpu, u64 phys_addr,
> +					 u8 *data, u32 len, u8 is_write)
> +{
> +	if (riscv_addr_in_ioport_region(phys_addr)) {
> +		int direction = is_write ? KVM_EXIT_IO_OUT : KVM_EXIT_IO_IN;
> +		u16 port = (phys_addr - KVM_IOPORT_AREA) & USHRT_MAX;
> +
> +		return kvm__emulate_io(vcpu, port, data, direction, len, 1);
> +	}
> +
> +	return kvm__emulate_mmio(vcpu, phys_addr, data, len, is_write);
> +}
> +
> +#endif /* KVM__KVM_CPU_ARCH_H */
> diff --git a/riscv/ioport.c b/riscv/ioport.c
> new file mode 100644
> index 0000000..bdd30b6
> --- /dev/null
> +++ b/riscv/ioport.c
> @@ -0,0 +1,11 @@
> +#include "kvm/ioport.h"
> +#include "kvm/irq.h"
> +
> +void ioport__setup_arch(struct kvm *kvm)
> +{
> +}
> +
> +void ioport__map_irq(u8 *irq)
> +{
> +	*irq = irq__alloc_line();
> +}
> diff --git a/riscv/irq.c b/riscv/irq.c
> new file mode 100644
> index 0000000..8e605ef
> --- /dev/null
> +++ b/riscv/irq.c
> @@ -0,0 +1,13 @@
> +#include "kvm/kvm.h"
> +#include "kvm/kvm-cpu.h"
> +#include "kvm/irq.h"
> +
> +void kvm__irq_line(struct kvm *kvm, int irq, int level)
> +{
> +	/* TODO: */
> +}
> +
> +void kvm__irq_trigger(struct kvm *kvm, int irq)
> +{
> +	/* TODO: */
> +}
> diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
> new file mode 100644
> index 0000000..e4b8fa5
> --- /dev/null
> +++ b/riscv/kvm-cpu.c
> @@ -0,0 +1,64 @@
> +#include "kvm/kvm-cpu.h"
> +#include "kvm/kvm.h"
> +#include "kvm/virtio.h"
> +#include "kvm/term.h"
> +
> +#include <asm/ptrace.h>
> +
> +static int debug_fd;
> +
> +void kvm_cpu__set_debug_fd(int fd)
> +{
> +	debug_fd = fd;
> +}
> +
> +int kvm_cpu__get_debug_fd(void)
> +{
> +	return debug_fd;
> +}
> +
> +struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
> +{
> +	/* TODO: */
> +	return NULL;
> +}
> +
> +void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
> +{
> +}
> +
> +void kvm_cpu__delete(struct kvm_cpu *vcpu)
> +{
> +	/* TODO: */
> +}
> +
> +bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
> +{
> +	/* TODO: */
> +	return false;
> +}
> +
> +void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
> +{
> +	/* TODO: */
> +}
> +
> +void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
> +{
> +	/* TODO: */
> +}
> +
> +int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
> +{
> +	return VIRTIO_ENDIAN_LE;
> +}
> +
> +void kvm_cpu__show_code(struct kvm_cpu *vcpu)
> +{
> +	/* TODO: */
> +}
> +
> +void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
> +{
> +	/* TODO: */
> +}
> diff --git a/riscv/kvm.c b/riscv/kvm.c
> new file mode 100644
> index 0000000..e816ef5
> --- /dev/null
> +++ b/riscv/kvm.c
> @@ -0,0 +1,61 @@
> +#include "kvm/kvm.h"
> +#include "kvm/util.h"
> +#include "kvm/fdt.h"
> +
> +#include <linux/kernel.h>
> +#include <linux/kvm.h>
> +#include <linux/sizes.h>
> +
> +struct kvm_ext kvm_req_ext[] = {
> +	{ DEFINE_KVM_EXT(KVM_CAP_ONE_REG) },
> +	{ 0, 0 },
> +};
> +
> +bool kvm__arch_cpu_supports_vm(void)
> +{
> +	/* The KVM capability check is enough. */
> +	return true;
> +}
> +
> +void kvm__init_ram(struct kvm *kvm)
> +{
> +	/* TODO: */
> +}
> +
> +void kvm__arch_delete_ram(struct kvm *kvm)
> +{
> +	/* TODO: */
> +}
> +
> +void kvm__arch_read_term(struct kvm *kvm)
> +{
> +	/* TODO: */
> +}
> +
> +void kvm__arch_set_cmdline(char *cmdline, bool video)
> +{
> +	/* TODO: */
> +}
> +
> +void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
> +{
> +	/* TODO: */
> +}
> +
> +bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
> +				 const char *kernel_cmdline)
> +{
> +	/* TODO: */
> +	return true;
> +}
> +
> +bool kvm__load_firmware(struct kvm *kvm, const char *firmware_filename)
> +{
> +	/* TODO: Firmware loading to be supported later. */
> +	return false;
> +}
> +
> +int kvm__arch_setup_firmware(struct kvm *kvm)
> +{
> +	return 0;
> +}
> diff --git a/util/update_headers.sh b/util/update_headers.sh
> index bf87ef6..78eba1f 100755
> --- a/util/update_headers.sh
> +++ b/util/update_headers.sh
> @@ -36,7 +36,7 @@ copy_optional_arch () {
>  	fi
>  }
>  
> -for arch in arm arm64 mips powerpc x86
> +for arch in arm arm64 mips powerpc riscv x86
>  do
>  	case "$arch" in
>  		arm) KVMTOOL_PATH=arm/aarch32 ;;

Re: [kvmtool RFC PATCH 3/8] riscv: Implement Guest/VM arch functions

[Alexandru Elisei]

Hello,

On 12/25/19 3:00 AM, Anup Patel wrote:
> This patch implements all kvm__arch_<xyz> Guest/VM arch functions.
>
> These functions mostly deal with:
> 1. Guest/VM RAM initialization
> 2. Updating terminals on character read
> 3. Loading kernel and initrd images
>
> Firmware loading is not implemented currently because initially we
> will be booting kernel directly without any bootloader. In future,
> we will certainly support firmware loading.
>
> Signed-off-by: Anup Patel <anup.patel@wdc.com>
> ---
>  riscv/include/kvm/kvm-arch.h |  15 +++++
>  riscv/kvm.c                  | 126 +++++++++++++++++++++++++++++++++--
>  2 files changed, 135 insertions(+), 6 deletions(-)
>
> diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
> index 7e9c578..b3ec2d6 100644
> --- a/riscv/include/kvm/kvm-arch.h
> +++ b/riscv/include/kvm/kvm-arch.h
> @@ -45,6 +45,21 @@
>  struct kvm;
>  
>  struct kvm_arch {
> +	/*
> +	 * We may have to align the guest memory for virtio, so keep the
> +	 * original pointers here for munmap.
> +	 */
> +	void	*ram_alloc_start;
> +	u64	ram_alloc_size;
> +
> +	/*
> +	 * Guest addresses for memory layout.
> +	 */
> +	u64	memory_guest_start;
> +	u64	kern_guest_start;
> +	u64	initrd_guest_start;
> +	u64	initrd_size;
> +	u64	dtb_guest_start;
>  };
>  
>  static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
> diff --git a/riscv/kvm.c b/riscv/kvm.c
> index e816ef5..c0d3639 100644
> --- a/riscv/kvm.c
> +++ b/riscv/kvm.c
> @@ -1,5 +1,7 @@
>  #include "kvm/kvm.h"
>  #include "kvm/util.h"
> +#include "kvm/8250-serial.h"
> +#include "kvm/virtio-console.h"
>  #include "kvm/fdt.h"
>  
>  #include <linux/kernel.h>
> @@ -19,33 +21,145 @@ bool kvm__arch_cpu_supports_vm(void)
>  
>  void kvm__init_ram(struct kvm *kvm)
>  {
> -	/* TODO: */
> +	int err;
> +	u64 phys_start, phys_size;
> +	void *host_mem;
> +
> +	phys_start	= RISCV_RAM;
> +	phys_size	= kvm->ram_size;
> +	host_mem	= kvm->ram_start;
> +
> +	err = kvm__register_ram(kvm, phys_start, phys_size, host_mem);
> +	if (err)
> +		die("Failed to register %lld bytes of memory at physical "
> +		    "address 0x%llx [err %d]", phys_size, phys_start, err);
> +
> +	kvm->arch.memory_guest_start = phys_start;
>  }
>  
>  void kvm__arch_delete_ram(struct kvm *kvm)
>  {
> -	/* TODO: */
> +	munmap(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size);
>  }
>  
>  void kvm__arch_read_term(struct kvm *kvm)
>  {
> -	/* TODO: */
> +	serial8250__update_consoles(kvm);
> +	virtio_console__inject_interrupt(kvm);
>  }
>  
>  void kvm__arch_set_cmdline(char *cmdline, bool video)
>  {
> -	/* TODO: */
>  }
>  
>  void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
>  {
> -	/* TODO: */
> +	/*
> +	 * Allocate guest memory. We must align our buffer to 64K to
> +	 * correlate with the maximum guest page size for virtio-mmio.
> +	 * If using THP, then our minimal alignment becomes 2M.
> +	 * 2M trumps 64K, so let's go with that.
> +	 */
> +	kvm->ram_size = min(ram_size, (u64)RISCV_MAX_MEMORY(kvm));
> +	kvm->arch.ram_alloc_size = kvm->ram_size + SZ_2M;
> +	kvm->arch.ram_alloc_start = mmap_anon_or_hugetlbfs(kvm, hugetlbfs_path,
> +						kvm->arch.ram_alloc_size);
> +
> +	if (kvm->arch.ram_alloc_start == MAP_FAILED)
> +		die("Failed to map %lld bytes for guest memory (%d)",
> +		    kvm->arch.ram_alloc_size, errno);
> +
> +	kvm->ram_start = (void *)ALIGN((unsigned long)kvm->arch.ram_alloc_start,
> +					SZ_2M);
> +
> +	madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
> +		MADV_MERGEABLE);
> +
> +	madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
> +		MADV_HUGEPAGE);
>  }
>  
> +#define FDT_ALIGN	SZ_4M
> +#define INITRD_ALIGN	4
>  bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
>  				 const char *kernel_cmdline)
>  {
> -	/* TODO: */
> +	void *pos, *kernel_end, *limit;
> +	unsigned long guest_addr, kernel_offset;
> +	ssize_t file_size;
> +
> +	/*
> +	 * Linux requires the initrd and dtb to be mapped inside lowmem,
> +	 * so we can't just place them at the top of memory.
> +	 */
> +	limit = kvm->ram_start + min(kvm->ram_size, (u64)SZ_256M) - 1;
> +
> +#if __riscv_xlen == 64
> +	/* Linux expects to be booted at 2M boundary for RV64 */
> +	kernel_offset = 0x200000;
> +#else
> +	/* Linux expects to be booted at 4M boundary for RV32 */
> +	kernel_offset = 0x400000;
> +#endif
> +
> +	pos = kvm->ram_start + kernel_offset;
> +	kvm->arch.kern_guest_start = host_to_guest_flat(kvm, pos);
> +	file_size = read_file(fd_kernel, pos, limit - pos);
> +	if (file_size < 0) {
> +		if (errno == ENOMEM)
> +			die("kernel image too big to fit in guest memory.");
> +
> +		die_perror("kernel read");
> +	}
> +	kernel_end = pos + file_size;
> +	pr_debug("Loaded kernel to 0x%llx (%zd bytes)",
> +		 kvm->arch.kern_guest_start, file_size);
> +
> +	/* Place FDT just after kernel at FDT_ALIGN address */
> +	pos = kernel_end + FDT_ALIGN;
> +	guest_addr = ALIGN(host_to_guest_flat(kvm, pos), FDT_ALIGN);
> +	pos = guest_flat_to_host(kvm, guest_addr);
> +	if (pos < kernel_end)
> +		die("fdt overlaps with kernel image.");
> +
> +	kvm->arch.dtb_guest_start = guest_addr;
> +	pr_debug("Placing fdt at 0x%llx - 0x%llx",
> +		 kvm->arch.dtb_guest_start,
> +		 host_to_guest_flat(kvm, limit));
> +	limit = pos;

This doesn't look right. pos points to the start of the DTB, not to the top of
free memory. You probably want to delete the line.

> +
> +	/* ... and finally the initrd, if we have one. */
> +	if (fd_initrd != -1) {
> +		struct stat sb;
> +		unsigned long initrd_start;
> +
> +		if (fstat(fd_initrd, &sb))
> +			die_perror("fstat");
> +
> +		pos -= (sb.st_size + INITRD_ALIGN);

This too doesn't look right. You're overwriting the DTB and most likely the kernel
with the initrd.

Thanks,
Alex
> +		guest_addr = ALIGN(host_to_guest_flat(kvm, pos), INITRD_ALIGN);
> +		pos = guest_flat_to_host(kvm, guest_addr);
> +		if (pos < kernel_end)
> +			die("initrd overlaps with kernel image.");
> +
> +		initrd_start = guest_addr;
> +		file_size = read_file(fd_initrd, pos, limit - pos);
> +		if (file_size == -1) {
> +			if (errno == ENOMEM)
> +				die("initrd too big to fit in guest memory.");
> +
> +			die_perror("initrd read");
> +		}
> +
> +		kvm->arch.initrd_guest_start = initrd_start;
> +		kvm->arch.initrd_size = file_size;
> +		pr_debug("Loaded initrd to 0x%llx (%llu bytes)",
> +			 kvm->arch.initrd_guest_start,
> +			 kvm->arch.initrd_size);
> +	} else {
> +		kvm->arch.initrd_size = 0;
> +	}
> +
>  	return true;
>  }
>  

Re: [kvmtool RFC PATCH 4/8] riscv: Implement Guest/VM VCPU arch functions

[Alexandru Elisei]

Hi,

On 12/25/19 3:00 AM, Anup Patel wrote:
> This patch implements kvm_cpu__<xyz> Guest/VM VCPU arch functions.
>
> These functions mostly deal with:
> 1. VCPU allocation and initialization
> 2. VCPU reset
> 3. VCPU show/dump code
> 4. VCPU show/dump registers
>
> We also save RISC-V ISA, XLEN, and TIMEBASE frequency for each VCPU
> so that it can be later used for generating Guest/VM FDT.
>
> Signed-off-by: Anup Patel <anup.patel@wdc.com>
> ---
>  riscv/include/kvm/kvm-cpu-arch.h |   4 +
>  riscv/kvm-cpu.c                  | 307 ++++++++++++++++++++++++++++++-
>  2 files changed, 304 insertions(+), 7 deletions(-)
>
> diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
> index 09a50e8..035965e 100644
> --- a/riscv/include/kvm/kvm-cpu-arch.h
> +++ b/riscv/include/kvm/kvm-cpu-arch.h
> @@ -14,6 +14,10 @@ struct kvm_cpu {
>  
>  	unsigned long   cpu_id;
>  
> +	unsigned long	riscv_xlen;
> +	unsigned long	riscv_isa;
> +	unsigned long	riscv_timebase;
> +
>  	struct kvm	*kvm;
>  	int		vcpu_fd;
>  	struct kvm_run	*kvm_run;
> diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
> index e4b8fa5..1565275 100644
> --- a/riscv/kvm-cpu.c
> +++ b/riscv/kvm-cpu.c
> @@ -17,10 +17,84 @@ int kvm_cpu__get_debug_fd(void)
>  	return debug_fd;
>  }
>  
> +static __u64 __kvm_reg_id(__u64 type, __u64 idx)
> +{
> +	__u64 id = KVM_REG_RISCV | type | idx;
> +
> +	if (sizeof(unsigned long) == 8)

This looks fragile. As far as I know, according to C99 the minimum width of
unsigned long is 32 bits. Why not use __riscv_xlen instead?

Thanks,
Alex
> +		id |= KVM_REG_SIZE_U64;
> +	else
> +		id |= KVM_REG_SIZE_U32;
> +
> +	return id;
> +}
> +
> +#define RISCV_CONFIG_REG(name)	__kvm_reg_id(KVM_REG_RISCV_CONFIG, \
> +					     KVM_REG_RISCV_CONFIG_REG(name))
> +
> +#define RISCV_CORE_REG(name)	__kvm_reg_id(KVM_REG_RISCV_CORE, \
> +					     KVM_REG_RISCV_CORE_REG(name))
> +
> +#define RISCV_CSR_REG(name)	__kvm_reg_id(KVM_REG_RISCV_CSR, \
> +					     KVM_REG_RISCV_CSR_REG(name))
> +
> +#define RISCV_TIMER_REG(name)	__kvm_reg_id(KVM_REG_RISCV_TIMER, \
> +					     KVM_REG_RISCV_TIMER_REG(name))
> +
>  struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
>  {
> -	/* TODO: */
> -	return NULL;
> +	struct kvm_cpu *vcpu;
> +	unsigned long timebase = 0, isa = 0;
> +	int coalesced_offset, mmap_size;
> +	struct kvm_one_reg reg;
> +
> +	vcpu = calloc(1, sizeof(struct kvm_cpu));
> +	if (!vcpu)
> +		return NULL;
> +
> +	vcpu->vcpu_fd = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, cpu_id);
> +	if (vcpu->vcpu_fd < 0)
> +		die_perror("KVM_CREATE_VCPU ioctl");
> +
> +	reg.id = RISCV_CONFIG_REG(isa);
> +	reg.addr = (unsigned long)&isa;
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (config.isa)");
> +
> +	reg.id = RISCV_TIMER_REG(frequency);
> +	reg.addr = (unsigned long)&timebase;
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (config.timebase)");
> +
> +	mmap_size = ioctl(kvm->sys_fd, KVM_GET_VCPU_MMAP_SIZE, 0);
> +	if (mmap_size < 0)
> +		die_perror("KVM_GET_VCPU_MMAP_SIZE ioctl");
> +
> +	vcpu->kvm_run = mmap(NULL, mmap_size, PROT_RW, MAP_SHARED,
> +			     vcpu->vcpu_fd, 0);
> +	if (vcpu->kvm_run == MAP_FAILED)
> +		die("unable to mmap vcpu fd");
> +
> +	coalesced_offset = ioctl(kvm->sys_fd, KVM_CHECK_EXTENSION,
> +				 KVM_CAP_COALESCED_MMIO);
> +	if (coalesced_offset)
> +		vcpu->ring = (void *)vcpu->kvm_run +
> +			     (coalesced_offset * PAGE_SIZE);
> +
> +	reg.id = RISCV_CONFIG_REG(isa);
> +	reg.addr = (unsigned long)&isa;
> +	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
> +		die("KVM_SET_ONE_REG failed (config.isa)");
> +
> +	/* Populate the vcpu structure. */
> +	vcpu->kvm		= kvm;
> +	vcpu->cpu_id		= cpu_id;
> +	vcpu->riscv_isa		= isa;
> +	vcpu->riscv_xlen	= __riscv_xlen;
> +	vcpu->riscv_timebase	= timebase;
> +	vcpu->is_running	= true;
> +
> +	return vcpu;
>  }
>  
>  void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
> @@ -29,7 +103,7 @@ void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
>  
>  void kvm_cpu__delete(struct kvm_cpu *vcpu)
>  {
> -	/* TODO: */
> +	free(vcpu);
>  }
>  
>  bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
> @@ -40,12 +114,43 @@ bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
>  
>  void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
>  {
> -	/* TODO: */
>  }
>  
>  void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
>  {
> -	/* TODO: */
> +	struct kvm *kvm = vcpu->kvm;
> +	struct kvm_mp_state mp_state;
> +	struct kvm_one_reg reg;
> +	unsigned long data;
> +
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_MP_STATE, &mp_state) < 0)
> +		die_perror("KVM_GET_MP_STATE failed");
> +
> +	/*
> +	 * If MP state is stopped then it means Linux KVM RISC-V emulates
> +	 * SBI v0.2 (or higher) with HART power managment and give VCPU
> +	 * will power-up at boot-time by boot VCPU. For such VCPU, we
> +	 * don't update PC, A0 and A1 here.
> +	 */
> +	if (mp_state.mp_state == KVM_MP_STATE_STOPPED)
> +		return;
> +
> +	reg.addr = (unsigned long)&data;
> +
> +	data	= kvm->arch.kern_guest_start;
> +	reg.id	= RISCV_CORE_REG(regs.pc);
> +	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
> +		die_perror("KVM_SET_ONE_REG failed (pc)");
> +
> +	data	= vcpu->cpu_id;
> +	reg.id	= RISCV_CORE_REG(regs.a0);
> +	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
> +		die_perror("KVM_SET_ONE_REG failed (a0)");
> +
> +	data	= kvm->arch.dtb_guest_start;
> +	reg.id	= RISCV_CORE_REG(regs.a1);
> +	if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
> +		die_perror("KVM_SET_ONE_REG failed (a1)");
>  }
>  
>  int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
> @@ -55,10 +160,198 @@ int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
>  
>  void kvm_cpu__show_code(struct kvm_cpu *vcpu)
>  {
> -	/* TODO: */
> +	struct kvm_one_reg reg;
> +	unsigned long data;
> +	int debug_fd = kvm_cpu__get_debug_fd();
> +
> +	reg.addr = (unsigned long)&data;
> +
> +	dprintf(debug_fd, "\n*PC:\n");
> +	reg.id = RISCV_CORE_REG(regs.pc);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (show_code @ PC)");
> +
> +	kvm__dump_mem(vcpu->kvm, data, 32, debug_fd);
> +
> +	dprintf(debug_fd, "\n*RA:\n");
> +	reg.id = RISCV_CORE_REG(regs.ra);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (show_code @ RA)");
> +
> +	kvm__dump_mem(vcpu->kvm, data, 32, debug_fd);
>  }
>  
>  void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
>  {
> -	/* TODO: */
> +	struct kvm_one_reg reg;
> +	unsigned long data;
> +	int debug_fd = kvm_cpu__get_debug_fd();
> +
> +	reg.addr = (unsigned long)&data;
> +	dprintf(debug_fd, "\n Registers:\n");
> +
> +	reg.id		= RISCV_CORE_REG(mode);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (mode)");
> +	dprintf(debug_fd, " MODE:  0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.pc);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (pc)");
> +	dprintf(debug_fd, " PC:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.ra);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (ra)");
> +	dprintf(debug_fd, " RA:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.sp);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (sp)");
> +	dprintf(debug_fd, " SP:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.gp);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (gp)");
> +	dprintf(debug_fd, " GP:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.tp);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (tp)");
> +	dprintf(debug_fd, " TP:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.t0);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (t0)");
> +	dprintf(debug_fd, " T0:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.t1);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (t1)");
> +	dprintf(debug_fd, " T1:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.t2);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (t2)");
> +	dprintf(debug_fd, " T2:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s0);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s0)");
> +	dprintf(debug_fd, " S0:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s1);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s1)");
> +	dprintf(debug_fd, " S1:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.a0);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (a0)");
> +	dprintf(debug_fd, " A0:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.a1);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (a1)");
> +	dprintf(debug_fd, " A1:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.a2);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (a2)");
> +	dprintf(debug_fd, " A2:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.a3);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (a3)");
> +	dprintf(debug_fd, " A3:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.a4);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (a4)");
> +	dprintf(debug_fd, " A4:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.a5);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (a5)");
> +	dprintf(debug_fd, " A5:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.a6);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (a6)");
> +	dprintf(debug_fd, " A6:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.a7);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (a7)");
> +	dprintf(debug_fd, " A7:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s2);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s2)");
> +	dprintf(debug_fd, " S2:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s3);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s3)");
> +	dprintf(debug_fd, " S3:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s4);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s4)");
> +	dprintf(debug_fd, " S4:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s5);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s5)");
> +	dprintf(debug_fd, " S5:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s6);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s6)");
> +	dprintf(debug_fd, " S6:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s7);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s7)");
> +	dprintf(debug_fd, " S7:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s8);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s8)");
> +	dprintf(debug_fd, " S8:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s9);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s9)");
> +	dprintf(debug_fd, " S9:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s10);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s10)");
> +	dprintf(debug_fd, " S10:   0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.s11);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (s11)");
> +	dprintf(debug_fd, " S11:   0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.t3);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (t3)");
> +	dprintf(debug_fd, " T3:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.t4);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (t4)");
> +	dprintf(debug_fd, " T4:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.t5);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (t5)");
> +	dprintf(debug_fd, " T5:    0x%lx\n", data);
> +
> +	reg.id		= RISCV_CORE_REG(regs.t6);
> +	if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> +		die("KVM_GET_ONE_REG failed (t6)");
> +	dprintf(debug_fd, " T6:    0x%lx\n", data);
>  }

Re: [kvmtool RFC PATCH 4/8] riscv: Implement Guest/VM VCPU arch functions

[Anup Patel]

On Wed, Jan 8, 2020 at 6:52 PM Alexandru Elisei
<alexandru.elisei@arm.com> wrote:
>
> Hi,
>
> On 12/25/19 3:00 AM, Anup Patel wrote:
> > This patch implements kvm_cpu__<xyz> Guest/VM VCPU arch functions.
> >
> > These functions mostly deal with:
> > 1. VCPU allocation and initialization
> > 2. VCPU reset
> > 3. VCPU show/dump code
> > 4. VCPU show/dump registers
> >
> > We also save RISC-V ISA, XLEN, and TIMEBASE frequency for each VCPU
> > so that it can be later used for generating Guest/VM FDT.
> >
> > Signed-off-by: Anup Patel <anup.patel@wdc.com>
> > ---
> >  riscv/include/kvm/kvm-cpu-arch.h |   4 +
> >  riscv/kvm-cpu.c                  | 307 ++++++++++++++++++++++++++++++-
> >  2 files changed, 304 insertions(+), 7 deletions(-)
> >
> > diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
> > index 09a50e8..035965e 100644
> > --- a/riscv/include/kvm/kvm-cpu-arch.h
> > +++ b/riscv/include/kvm/kvm-cpu-arch.h
> > @@ -14,6 +14,10 @@ struct kvm_cpu {
> >
> >       unsigned long   cpu_id;
> >
> > +     unsigned long   riscv_xlen;
> > +     unsigned long   riscv_isa;
> > +     unsigned long   riscv_timebase;
> > +
> >       struct kvm      *kvm;
> >       int             vcpu_fd;
> >       struct kvm_run  *kvm_run;
> > diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
> > index e4b8fa5..1565275 100644
> > --- a/riscv/kvm-cpu.c
> > +++ b/riscv/kvm-cpu.c
> > @@ -17,10 +17,84 @@ int kvm_cpu__get_debug_fd(void)
> >       return debug_fd;
> >  }
> >
> > +static __u64 __kvm_reg_id(__u64 type, __u64 idx)
> > +{
> > +     __u64 id = KVM_REG_RISCV | type | idx;
> > +
> > +     if (sizeof(unsigned long) == 8)
>
> This looks fragile. As far as I know, according to C99 the minimum width of
> unsigned long is 32 bits. Why not use __riscv_xlen instead?

Good suggestion. I will use __riscv_xlen here.

>
> Thanks,
> Alex
> > +             id |= KVM_REG_SIZE_U64;
> > +     else
> > +             id |= KVM_REG_SIZE_U32;
> > +
> > +     return id;
> > +}
> > +
> > +#define RISCV_CONFIG_REG(name)       __kvm_reg_id(KVM_REG_RISCV_CONFIG, \
> > +                                          KVM_REG_RISCV_CONFIG_REG(name))
> > +
> > +#define RISCV_CORE_REG(name) __kvm_reg_id(KVM_REG_RISCV_CORE, \
> > +                                          KVM_REG_RISCV_CORE_REG(name))
> > +
> > +#define RISCV_CSR_REG(name)  __kvm_reg_id(KVM_REG_RISCV_CSR, \
> > +                                          KVM_REG_RISCV_CSR_REG(name))
> > +
> > +#define RISCV_TIMER_REG(name)        __kvm_reg_id(KVM_REG_RISCV_TIMER, \
> > +                                          KVM_REG_RISCV_TIMER_REG(name))
> > +
> >  struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
> >  {
> > -     /* TODO: */
> > -     return NULL;
> > +     struct kvm_cpu *vcpu;
> > +     unsigned long timebase = 0, isa = 0;
> > +     int coalesced_offset, mmap_size;
> > +     struct kvm_one_reg reg;
> > +
> > +     vcpu = calloc(1, sizeof(struct kvm_cpu));
> > +     if (!vcpu)
> > +             return NULL;
> > +
> > +     vcpu->vcpu_fd = ioctl(kvm->vm_fd, KVM_CREATE_VCPU, cpu_id);
> > +     if (vcpu->vcpu_fd < 0)
> > +             die_perror("KVM_CREATE_VCPU ioctl");
> > +
> > +     reg.id = RISCV_CONFIG_REG(isa);
> > +     reg.addr = (unsigned long)&isa;
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (config.isa)");
> > +
> > +     reg.id = RISCV_TIMER_REG(frequency);
> > +     reg.addr = (unsigned long)&timebase;
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (config.timebase)");
> > +
> > +     mmap_size = ioctl(kvm->sys_fd, KVM_GET_VCPU_MMAP_SIZE, 0);
> > +     if (mmap_size < 0)
> > +             die_perror("KVM_GET_VCPU_MMAP_SIZE ioctl");
> > +
> > +     vcpu->kvm_run = mmap(NULL, mmap_size, PROT_RW, MAP_SHARED,
> > +                          vcpu->vcpu_fd, 0);
> > +     if (vcpu->kvm_run == MAP_FAILED)
> > +             die("unable to mmap vcpu fd");
> > +
> > +     coalesced_offset = ioctl(kvm->sys_fd, KVM_CHECK_EXTENSION,
> > +                              KVM_CAP_COALESCED_MMIO);
> > +     if (coalesced_offset)
> > +             vcpu->ring = (void *)vcpu->kvm_run +
> > +                          (coalesced_offset * PAGE_SIZE);
> > +
> > +     reg.id = RISCV_CONFIG_REG(isa);
> > +     reg.addr = (unsigned long)&isa;
> > +     if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
> > +             die("KVM_SET_ONE_REG failed (config.isa)");
> > +
> > +     /* Populate the vcpu structure. */
> > +     vcpu->kvm               = kvm;
> > +     vcpu->cpu_id            = cpu_id;
> > +     vcpu->riscv_isa         = isa;
> > +     vcpu->riscv_xlen        = __riscv_xlen;
> > +     vcpu->riscv_timebase    = timebase;
> > +     vcpu->is_running        = true;
> > +
> > +     return vcpu;
> >  }
> >
> >  void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
> > @@ -29,7 +103,7 @@ void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
> >
> >  void kvm_cpu__delete(struct kvm_cpu *vcpu)
> >  {
> > -     /* TODO: */
> > +     free(vcpu);
> >  }
> >
> >  bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
> > @@ -40,12 +114,43 @@ bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
> >
> >  void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
> >  {
> > -     /* TODO: */
> >  }
> >
> >  void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
> >  {
> > -     /* TODO: */
> > +     struct kvm *kvm = vcpu->kvm;
> > +     struct kvm_mp_state mp_state;
> > +     struct kvm_one_reg reg;
> > +     unsigned long data;
> > +
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_MP_STATE, &mp_state) < 0)
> > +             die_perror("KVM_GET_MP_STATE failed");
> > +
> > +     /*
> > +      * If MP state is stopped then it means Linux KVM RISC-V emulates
> > +      * SBI v0.2 (or higher) with HART power managment and give VCPU
> > +      * will power-up at boot-time by boot VCPU. For such VCPU, we
> > +      * don't update PC, A0 and A1 here.
> > +      */
> > +     if (mp_state.mp_state == KVM_MP_STATE_STOPPED)
> > +             return;
> > +
> > +     reg.addr = (unsigned long)&data;
> > +
> > +     data    = kvm->arch.kern_guest_start;
> > +     reg.id  = RISCV_CORE_REG(regs.pc);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
> > +             die_perror("KVM_SET_ONE_REG failed (pc)");
> > +
> > +     data    = vcpu->cpu_id;
> > +     reg.id  = RISCV_CORE_REG(regs.a0);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
> > +             die_perror("KVM_SET_ONE_REG failed (a0)");
> > +
> > +     data    = kvm->arch.dtb_guest_start;
> > +     reg.id  = RISCV_CORE_REG(regs.a1);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_SET_ONE_REG, &reg) < 0)
> > +             die_perror("KVM_SET_ONE_REG failed (a1)");
> >  }
> >
> >  int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
> > @@ -55,10 +160,198 @@ int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
> >
> >  void kvm_cpu__show_code(struct kvm_cpu *vcpu)
> >  {
> > -     /* TODO: */
> > +     struct kvm_one_reg reg;
> > +     unsigned long data;
> > +     int debug_fd = kvm_cpu__get_debug_fd();
> > +
> > +     reg.addr = (unsigned long)&data;
> > +
> > +     dprintf(debug_fd, "\n*PC:\n");
> > +     reg.id = RISCV_CORE_REG(regs.pc);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (show_code @ PC)");
> > +
> > +     kvm__dump_mem(vcpu->kvm, data, 32, debug_fd);
> > +
> > +     dprintf(debug_fd, "\n*RA:\n");
> > +     reg.id = RISCV_CORE_REG(regs.ra);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (show_code @ RA)");
> > +
> > +     kvm__dump_mem(vcpu->kvm, data, 32, debug_fd);
> >  }
> >
> >  void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
> >  {
> > -     /* TODO: */
> > +     struct kvm_one_reg reg;
> > +     unsigned long data;
> > +     int debug_fd = kvm_cpu__get_debug_fd();
> > +
> > +     reg.addr = (unsigned long)&data;
> > +     dprintf(debug_fd, "\n Registers:\n");
> > +
> > +     reg.id          = RISCV_CORE_REG(mode);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (mode)");
> > +     dprintf(debug_fd, " MODE:  0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.pc);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (pc)");
> > +     dprintf(debug_fd, " PC:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.ra);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (ra)");
> > +     dprintf(debug_fd, " RA:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.sp);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (sp)");
> > +     dprintf(debug_fd, " SP:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.gp);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (gp)");
> > +     dprintf(debug_fd, " GP:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.tp);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (tp)");
> > +     dprintf(debug_fd, " TP:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.t0);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (t0)");
> > +     dprintf(debug_fd, " T0:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.t1);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (t1)");
> > +     dprintf(debug_fd, " T1:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.t2);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (t2)");
> > +     dprintf(debug_fd, " T2:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s0);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s0)");
> > +     dprintf(debug_fd, " S0:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s1);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s1)");
> > +     dprintf(debug_fd, " S1:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.a0);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (a0)");
> > +     dprintf(debug_fd, " A0:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.a1);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (a1)");
> > +     dprintf(debug_fd, " A1:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.a2);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (a2)");
> > +     dprintf(debug_fd, " A2:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.a3);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (a3)");
> > +     dprintf(debug_fd, " A3:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.a4);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (a4)");
> > +     dprintf(debug_fd, " A4:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.a5);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (a5)");
> > +     dprintf(debug_fd, " A5:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.a6);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (a6)");
> > +     dprintf(debug_fd, " A6:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.a7);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (a7)");
> > +     dprintf(debug_fd, " A7:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s2);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s2)");
> > +     dprintf(debug_fd, " S2:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s3);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s3)");
> > +     dprintf(debug_fd, " S3:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s4);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s4)");
> > +     dprintf(debug_fd, " S4:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s5);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s5)");
> > +     dprintf(debug_fd, " S5:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s6);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s6)");
> > +     dprintf(debug_fd, " S6:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s7);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s7)");
> > +     dprintf(debug_fd, " S7:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s8);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s8)");
> > +     dprintf(debug_fd, " S8:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s9);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s9)");
> > +     dprintf(debug_fd, " S9:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s10);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s10)");
> > +     dprintf(debug_fd, " S10:   0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.s11);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (s11)");
> > +     dprintf(debug_fd, " S11:   0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.t3);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (t3)");
> > +     dprintf(debug_fd, " T3:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.t4);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (t4)");
> > +     dprintf(debug_fd, " T4:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.t5);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (t5)");
> > +     dprintf(debug_fd, " T5:    0x%lx\n", data);
> > +
> > +     reg.id          = RISCV_CORE_REG(regs.t6);
> > +     if (ioctl(vcpu->vcpu_fd, KVM_GET_ONE_REG, &reg) < 0)
> > +             die("KVM_GET_ONE_REG failed (t6)");
> > +     dprintf(debug_fd, " T6:    0x%lx\n", data);
> >  }

Regards,
Anup

Re: [kvmtool RFC PATCH 2/8] riscv: Initial skeletal support

[Anup Patel]

On Wed, Jan 8, 2020 at 6:52 PM Alexandru Elisei
<alexandru.elisei@arm.com> wrote:
>
> Hi,
>
> I don't know much about the RISC-V architecture, so I'm only going to comment on
> the more generic code.

Sure, no problem.

>
> On 12/25/19 3:00 AM, Anup Patel wrote:
> > This patch adds initial skeletal KVMTOOL RISC-V support which
> > just compiles for RV32 and RV64 host.
> >
> > Signed-off-by: Anup Patel <anup.patel@wdc.com>
> > ---
> >  INSTALL                             |   7 +-
> >  Makefile                            |  15 +++-
> >  riscv/include/asm/kvm.h             | 127 ++++++++++++++++++++++++++++
> >  riscv/include/kvm/barrier.h         |  14 +++
> >  riscv/include/kvm/fdt-arch.h        |   4 +
> >  riscv/include/kvm/kvm-arch.h        |  58 +++++++++++++
> >  riscv/include/kvm/kvm-config-arch.h |   9 ++
> >  riscv/include/kvm/kvm-cpu-arch.h    |  49 +++++++++++
> >  riscv/ioport.c                      |  11 +++
> >  riscv/irq.c                         |  13 +++
> >  riscv/kvm-cpu.c                     |  64 ++++++++++++++
> >  riscv/kvm.c                         |  61 +++++++++++++
> >  util/update_headers.sh              |   2 +-
> >  13 files changed, 429 insertions(+), 5 deletions(-)
> >  create mode 100644 riscv/include/asm/kvm.h
> >  create mode 100644 riscv/include/kvm/barrier.h
> >  create mode 100644 riscv/include/kvm/fdt-arch.h
> >  create mode 100644 riscv/include/kvm/kvm-arch.h
> >  create mode 100644 riscv/include/kvm/kvm-config-arch.h
> >  create mode 100644 riscv/include/kvm/kvm-cpu-arch.h
> >  create mode 100644 riscv/ioport.c
> >  create mode 100644 riscv/irq.c
> >  create mode 100644 riscv/kvm-cpu.c
> >  create mode 100644 riscv/kvm.c
> >
> > diff --git a/INSTALL b/INSTALL
> > index ca8e022..951b123 100644
> > --- a/INSTALL
> > +++ b/INSTALL
> > @@ -26,8 +26,8 @@ For Fedora based systems:
> >  For OpenSUSE based systems:
> >       # zypper install glibc-devel-static
> >
> > -Architectures which require device tree (PowerPC, ARM, ARM64) also require
> > -libfdt.
> > +Architectures which require device tree (PowerPC, ARM, ARM64, RISC-V) also
> > +require libfdt.
> >       deb: $ sudo apt-get install libfdt-dev
> >       Fedora: # yum install libfdt-devel
> >       OpenSUSE: # zypper install libfdt1-devel
> > @@ -64,6 +64,7 @@ to the Linux name of the architecture. Architectures supported:
> >  - arm
> >  - arm64
> >  - mips
> > +- riscv
> >  If ARCH is not provided, the target architecture will be automatically
> >  determined by running "uname -m" on your host, resulting in a native build.
> >
> > @@ -81,7 +82,7 @@ On multiarch system you should be able to install those be appending
> >  the architecture name after the package (example for ARM64):
> >  $ sudo apt-get install libfdt-dev:arm64
> >
> > -PowerPC and ARM/ARM64 require libfdt to be installed. If you cannot use
> > +PowerPC, ARM/ARM64 and RISC-V require libfdt to be installed. If you cannot use
> >  precompiled mulitarch packages, you could either copy the required header and
> >  library files from an installed target system into the SYSROOT (you will need
> >  /usr/include/*fdt*.h and /usr/lib64/libfdt-v.v.v.so and its symlinks), or you
> > diff --git a/Makefile b/Makefile
> > index 3862112..972fa63 100644
> > --- a/Makefile
> > +++ b/Makefile
> > @@ -106,7 +106,8 @@ OBJS      += hw/i8042.o
> >
> >  # Translate uname -m into ARCH string
> >  ARCH ?= $(shell uname -m | sed -e s/i.86/i386/ -e s/ppc.*/powerpc/ \
> > -       -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/)
> > +       -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/ \
> > +       -e s/riscv64/riscv/ -e s/riscv32/riscv/)
> >
> >  ifeq ($(ARCH),i386)
> >       ARCH         := x86
> > @@ -190,6 +191,18 @@ ifeq ($(ARCH),mips)
> >       OBJS            += mips/kvm.o
> >       OBJS            += mips/kvm-cpu.o
> >  endif
> > +
> > +# RISC-V (RV32 and RV64)
> > +ifeq ($(ARCH),riscv)
> > +     DEFINES         += -DCONFIG_RISCV
> > +     ARCH_INCLUDE    := riscv/include
> > +     OBJS            += riscv/ioport.o
> > +     OBJS            += riscv/irq.o
> > +     OBJS            += riscv/kvm.o
> > +     OBJS            += riscv/kvm-cpu.o
> > +
> > +     ARCH_WANT_LIBFDT := y
> > +endif
> >  ###
> >
> >  ifeq (,$(ARCH_INCLUDE))
> > diff --git a/riscv/include/asm/kvm.h b/riscv/include/asm/kvm.h
> > new file mode 100644
> > index 0000000..f4274c2
> > --- /dev/null
> > +++ b/riscv/include/asm/kvm.h
> > @@ -0,0 +1,127 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +/*
> > + * Copyright (C) 2019 Western Digital Corporation or its affiliates.
> > + *
> > + * Authors:
> > + *     Anup Patel <anup.patel@wdc.com>
> > + */
> > +
> > +#ifndef __LINUX_KVM_RISCV_H
> > +#define __LINUX_KVM_RISCV_H
> > +
> > +#ifndef __ASSEMBLY__
> > +
> > +#include <linux/types.h>
> > +#include <asm/ptrace.h>
> > +
> > +#define __KVM_HAVE_READONLY_MEM
> > +
> > +#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
> > +
> > +#define KVM_INTERRUPT_SET    -1U
> > +#define KVM_INTERRUPT_UNSET  -2U
> > +
> > +/* for KVM_GET_REGS and KVM_SET_REGS */
> > +struct kvm_regs {
> > +};
> > +
> > +/* for KVM_GET_FPU and KVM_SET_FPU */
> > +struct kvm_fpu {
> > +};
> > +
> > +/* KVM Debug exit structure */
> > +struct kvm_debug_exit_arch {
> > +};
> > +
> > +/* for KVM_SET_GUEST_DEBUG */
> > +struct kvm_guest_debug_arch {
> > +};
> > +
> > +/* definition of registers in kvm_run */
> > +struct kvm_sync_regs {
> > +};
> > +
> > +/* for KVM_GET_SREGS and KVM_SET_SREGS */
> > +struct kvm_sregs {
> > +};
> > +
> > +/* CONFIG registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> > +struct kvm_riscv_config {
> > +     unsigned long isa;
> > +};
> > +
> > +/* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> > +struct kvm_riscv_core {
> > +     struct user_regs_struct regs;
> > +     unsigned long mode;
> > +};
> > +
> > +/* Possible privilege modes for kvm_riscv_core */
> > +#define KVM_RISCV_MODE_S     1
> > +#define KVM_RISCV_MODE_U     0
> > +
> > +/* CSR registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> > +struct kvm_riscv_csr {
> > +     unsigned long sstatus;
> > +     unsigned long sie;
> > +     unsigned long stvec;
> > +     unsigned long sscratch;
> > +     unsigned long sepc;
> > +     unsigned long scause;
> > +     unsigned long stval;
> > +     unsigned long sip;
> > +     unsigned long satp;
> > +};
> > +
> > +/* TIMER registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> > +struct kvm_riscv_timer {
> > +     u64 frequency;
> > +     u64 time;
> > +     u64 compare;
> > +     u64 state;
> > +};
> > +
> > +/* Possible states for kvm_riscv_timer */
> > +#define KVM_RISCV_TIMER_STATE_OFF    0
> > +#define KVM_RISCV_TIMER_STATE_ON     1
> > +
> > +#define KVM_REG_SIZE(id)             \
> > +     (1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
> > +
> > +/* If you need to interpret the index values, here is the key: */
> > +#define KVM_REG_RISCV_TYPE_MASK              0x00000000FF000000
> > +#define KVM_REG_RISCV_TYPE_SHIFT     24
> > +
> > +/* Config registers are mapped as type 1 */
> > +#define KVM_REG_RISCV_CONFIG         (0x01 << KVM_REG_RISCV_TYPE_SHIFT)
> > +#define KVM_REG_RISCV_CONFIG_REG(name)       \
> > +     (offsetof(struct kvm_riscv_config, name) / sizeof(unsigned long))
> > +
> > +/* Core registers are mapped as type 2 */
> > +#define KVM_REG_RISCV_CORE           (0x02 << KVM_REG_RISCV_TYPE_SHIFT)
> > +#define KVM_REG_RISCV_CORE_REG(name) \
> > +             (offsetof(struct kvm_riscv_core, name) / sizeof(unsigned long))
> > +
> > +/* Control and status registers are mapped as type 3 */
> > +#define KVM_REG_RISCV_CSR            (0x03 << KVM_REG_RISCV_TYPE_SHIFT)
> > +#define KVM_REG_RISCV_CSR_REG(name)  \
> > +             (offsetof(struct kvm_riscv_csr, name) / sizeof(unsigned long))
> > +
> > +/* Timer registers are mapped as type 4 */
> > +#define KVM_REG_RISCV_TIMER          (0x04 << KVM_REG_RISCV_TYPE_SHIFT)
> > +#define KVM_REG_RISCV_TIMER_REG(name)        \
> > +             (offsetof(struct kvm_riscv_timer, name) / sizeof(u64))
> > +
> > +/* F extension registers are mapped as type 5 */
> > +#define KVM_REG_RISCV_FP_F           (0x05 << KVM_REG_RISCV_TYPE_SHIFT)
> > +#define KVM_REG_RISCV_FP_F_REG(name) \
> > +             (offsetof(struct __riscv_f_ext_state, name) / sizeof(u32))
> > +
> > +/* D extension registers are mapped as type 6 */
> > +#define KVM_REG_RISCV_FP_D           (0x06 << KVM_REG_RISCV_TYPE_SHIFT)
> > +#define KVM_REG_RISCV_FP_D_REG(name) \
> > +             (offsetof(struct __riscv_d_ext_state, name) / sizeof(u64))
> > +
> > +#endif
> > +
> > +#endif /* __LINUX_KVM_RISCV_H */
> > diff --git a/riscv/include/kvm/barrier.h b/riscv/include/kvm/barrier.h
> > new file mode 100644
> > index 0000000..235f610
> > --- /dev/null
> > +++ b/riscv/include/kvm/barrier.h
> > @@ -0,0 +1,14 @@
> > +#ifndef KVM__KVM_BARRIER_H
> > +#define KVM__KVM_BARRIER_H
> > +
> > +#define nop()                __asm__ __volatile__ ("nop")
> > +
> > +#define RISCV_FENCE(p, s) \
> > +     __asm__ __volatile__ ("fence " #p "," #s : : : "memory")
> > +
> > +/* These barriers need to enforce ordering on both devices or memory. */
> > +#define mb()         RISCV_FENCE(iorw,iorw)
> > +#define rmb()                RISCV_FENCE(ir,ir)
> > +#define wmb()                RISCV_FENCE(ow,ow)
> > +
> > +#endif /* KVM__KVM_BARRIER_H */
> > diff --git a/riscv/include/kvm/fdt-arch.h b/riscv/include/kvm/fdt-arch.h
> > new file mode 100644
> > index 0000000..9450fc5
> > --- /dev/null
> > +++ b/riscv/include/kvm/fdt-arch.h
> > @@ -0,0 +1,4 @@
> > +#ifndef KVM__KVM_FDT_H
> > +#define KVM__KVM_FDT_H
> > +
> > +#endif /* KVM__KVM_FDT_H */
> > diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
> > new file mode 100644
> > index 0000000..7e9c578
> > --- /dev/null
> > +++ b/riscv/include/kvm/kvm-arch.h
> > @@ -0,0 +1,58 @@
> > +#ifndef KVM__KVM_ARCH_H
> > +#define KVM__KVM_ARCH_H
> > +
> > +#include <stdbool.h>
> > +#include <linux/const.h>
> > +#include <linux/types.h>
> > +
> > +#define RISCV_IOPORT         0x00000000ULL
> > +#define RISCV_IOPORT_SIZE    0x00010000ULL
> > +#define RISCV_PLIC           0x0c000000ULL
> > +#define RISCV_PLIC_SIZE              0x04000000ULL
> > +#define RISCV_MMIO           0x10000000ULL
> > +#define RISCV_MMIO_SIZE              0x20000000ULL
> > +#define RISCV_PCI            0x30000000ULL
> > +#define RISCV_PCI_CFG_SIZE   0x10000000ULL
>
> In the DTB you're advertising the PCI node as CAM compatible, which is the right
> thing to do. Legacy PCI configuration space is 16MB, not 256MB (PCI Express is 256MB).

I was confused here so I did what was done for ARM. I will check with other
architectures and update like you suggested.

>
> > +#define RISCV_PCI_SIZE               0x50000000ULL
> > +#define RISCV_PCI_MMIO_SIZE  (RISCV_PCI_SIZE - RISCV_PCI_CFG_SIZE)
> > +
> > +#define RISCV_RAM            0x80000000ULL
>
> I'm not sure about the reasons for choosing RAM to start at 2GB. For arm we do the
> same, but qemu starts memory at 1GB and this mismatch has caused some issues in
> the past. For example, 32 bit kvm-unit-tests currently do not run under kvmtool
> because the text address is hardcoded to the qemu default value.

Actually in RISC-V world, it is kind of a un-documented standard that
RAM starts at
2GB (0x80000000) for both RV32 and RV64. This is true for all existing HW,
QEMU RISC-V virt machine and here. This will be soon explicitly documented in
RISC-V Unix platform spec.

>
> As a more general observation, I know that other architectures (like arm) declare
> the memory layout in hexadecimal numbers, but it might be a better idea to use the
> sizes from include/linux/sizes.h, since it makes the memory layout a lot more
> readable and mistakes are easier to spot.

Sure, I will try to use linux/sizes.h in next patch version.

>
> > +
> > +#define RISCV_LOMAP_MAX_MEMORY       ((1ULL << 32) - RISCV_RAM)
> > +#define RISCV_HIMAP_MAX_MEMORY       ((1ULL << 40) - RISCV_RAM)
> > +
> > +#if __riscv_xlen == 64
> > +#define RISCV_MAX_MEMORY(kvm)        RISCV_HIMAP_MAX_MEMORY
> > +#elif __riscv_xlen == 32
> > +#define RISCV_MAX_MEMORY(kvm)        RISCV_LOMAP_MAX_MEMORY
> > +#endif
> > +
> > +#define KVM_IOPORT_AREA              RISCV_IOPORT
> > +#define KVM_PCI_CFG_AREA     RISCV_PCI
> > +#define KVM_PCI_MMIO_AREA    (KVM_PCI_CFG_AREA + RISCV_PCI_CFG_SIZE)
> > +#define KVM_VIRTIO_MMIO_AREA RISCV_MMIO
> > +
> > +#define KVM_IOEVENTFD_HAS_PIO        0
> > +
> > +#define KVM_IRQ_OFFSET               0
> > +
> > +#define KVM_VM_TYPE          0
> > +
> > +#define VIRTIO_DEFAULT_TRANS(kvm)    VIRTIO_MMIO
> > +
> > +#define VIRTIO_RING_ENDIAN   VIRTIO_ENDIAN_LE
> > +
> > +struct kvm;
> > +
> > +struct kvm_arch {
> > +};
> > +
> > +static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
> > +{
> > +     u64 limit = KVM_IOPORT_AREA + RISCV_IOPORT_SIZE;
> > +     return phys_addr >= KVM_IOPORT_AREA && phys_addr < limit;
> > +}
> > +
> > +enum irq_type;
> > +
> > +#endif /* KVM__KVM_ARCH_H */
> > diff --git a/riscv/include/kvm/kvm-config-arch.h b/riscv/include/kvm/kvm-config-arch.h
> > new file mode 100644
> > index 0000000..60c7333
> > --- /dev/null
> > +++ b/riscv/include/kvm/kvm-config-arch.h
> > @@ -0,0 +1,9 @@
> > +#ifndef KVM__KVM_CONFIG_ARCH_H
> > +#define KVM__KVM_CONFIG_ARCH_H
> > +
> > +#include "kvm/parse-options.h"
> > +
> > +struct kvm_config_arch {
> > +};
> > +
> > +#endif /* KVM__KVM_CONFIG_ARCH_H */
> > diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
> > new file mode 100644
> > index 0000000..09a50e8
> > --- /dev/null
> > +++ b/riscv/include/kvm/kvm-cpu-arch.h
> > @@ -0,0 +1,49 @@
> > +#ifndef KVM__KVM_CPU_ARCH_H
> > +#define KVM__KVM_CPU_ARCH_H
> > +
> > +#include <linux/kvm.h>
> > +#include <pthread.h>
> > +#include <stdbool.h>
> > +
> > +#include "kvm/kvm.h"
> > +
> > +struct kvm;
>
> Shouldn't kvm.h already have a definition for struct kvm? Also, the arm
> corresponding header doesn't have the include here, I don't think it's needed
> (unless I'm missing something).

Sure, I will drop this forward declaration here.

Regards,
Anup

>
> Thanks,
> Alex
> > +
> > +struct kvm_cpu {
> > +     pthread_t       thread;
> > +
> > +     unsigned long   cpu_id;
> > +
> > +     struct kvm      *kvm;
> > +     int             vcpu_fd;
> > +     struct kvm_run  *kvm_run;
> > +     struct kvm_cpu_task     *task;
> > +
> > +     u8              is_running;
> > +     u8              paused;
> > +     u8              needs_nmi;
> > +
> > +     struct kvm_coalesced_mmio_ring  *ring;
> > +};
> > +
> > +static inline bool kvm_cpu__emulate_io(struct kvm_cpu *vcpu, u16 port,
> > +                                    void *data, int direction,
> > +                                    int size, u32 count)
> > +{
> > +     return false;
> > +}
> > +
> > +static inline bool kvm_cpu__emulate_mmio(struct kvm_cpu *vcpu, u64 phys_addr,
> > +                                      u8 *data, u32 len, u8 is_write)
> > +{
> > +     if (riscv_addr_in_ioport_region(phys_addr)) {
> > +             int direction = is_write ? KVM_EXIT_IO_OUT : KVM_EXIT_IO_IN;
> > +             u16 port = (phys_addr - KVM_IOPORT_AREA) & USHRT_MAX;
> > +
> > +             return kvm__emulate_io(vcpu, port, data, direction, len, 1);
> > +     }
> > +
> > +     return kvm__emulate_mmio(vcpu, phys_addr, data, len, is_write);
> > +}
> > +
> > +#endif /* KVM__KVM_CPU_ARCH_H */
> > diff --git a/riscv/ioport.c b/riscv/ioport.c
> > new file mode 100644
> > index 0000000..bdd30b6
> > --- /dev/null
> > +++ b/riscv/ioport.c
> > @@ -0,0 +1,11 @@
> > +#include "kvm/ioport.h"
> > +#include "kvm/irq.h"
> > +
> > +void ioport__setup_arch(struct kvm *kvm)
> > +{
> > +}
> > +
> > +void ioport__map_irq(u8 *irq)
> > +{
> > +     *irq = irq__alloc_line();
> > +}
> > diff --git a/riscv/irq.c b/riscv/irq.c
> > new file mode 100644
> > index 0000000..8e605ef
> > --- /dev/null
> > +++ b/riscv/irq.c
> > @@ -0,0 +1,13 @@
> > +#include "kvm/kvm.h"
> > +#include "kvm/kvm-cpu.h"
> > +#include "kvm/irq.h"
> > +
> > +void kvm__irq_line(struct kvm *kvm, int irq, int level)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +void kvm__irq_trigger(struct kvm *kvm, int irq)
> > +{
> > +     /* TODO: */
> > +}
> > diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
> > new file mode 100644
> > index 0000000..e4b8fa5
> > --- /dev/null
> > +++ b/riscv/kvm-cpu.c
> > @@ -0,0 +1,64 @@
> > +#include "kvm/kvm-cpu.h"
> > +#include "kvm/kvm.h"
> > +#include "kvm/virtio.h"
> > +#include "kvm/term.h"
> > +
> > +#include <asm/ptrace.h>
> > +
> > +static int debug_fd;
> > +
> > +void kvm_cpu__set_debug_fd(int fd)
> > +{
> > +     debug_fd = fd;
> > +}
> > +
> > +int kvm_cpu__get_debug_fd(void)
> > +{
> > +     return debug_fd;
> > +}
> > +
> > +struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
> > +{
> > +     /* TODO: */
> > +     return NULL;
> > +}
> > +
> > +void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
> > +{
> > +}
> > +
> > +void kvm_cpu__delete(struct kvm_cpu *vcpu)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
> > +{
> > +     /* TODO: */
> > +     return false;
> > +}
> > +
> > +void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
> > +{
> > +     return VIRTIO_ENDIAN_LE;
> > +}
> > +
> > +void kvm_cpu__show_code(struct kvm_cpu *vcpu)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
> > +{
> > +     /* TODO: */
> > +}
> > diff --git a/riscv/kvm.c b/riscv/kvm.c
> > new file mode 100644
> > index 0000000..e816ef5
> > --- /dev/null
> > +++ b/riscv/kvm.c
> > @@ -0,0 +1,61 @@
> > +#include "kvm/kvm.h"
> > +#include "kvm/util.h"
> > +#include "kvm/fdt.h"
> > +
> > +#include <linux/kernel.h>
> > +#include <linux/kvm.h>
> > +#include <linux/sizes.h>
> > +
> > +struct kvm_ext kvm_req_ext[] = {
> > +     { DEFINE_KVM_EXT(KVM_CAP_ONE_REG) },
> > +     { 0, 0 },
> > +};
> > +
> > +bool kvm__arch_cpu_supports_vm(void)
> > +{
> > +     /* The KVM capability check is enough. */
> > +     return true;
> > +}
> > +
> > +void kvm__init_ram(struct kvm *kvm)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +void kvm__arch_delete_ram(struct kvm *kvm)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +void kvm__arch_read_term(struct kvm *kvm)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +void kvm__arch_set_cmdline(char *cmdline, bool video)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
> > +{
> > +     /* TODO: */
> > +}
> > +
> > +bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
> > +                              const char *kernel_cmdline)
> > +{
> > +     /* TODO: */
> > +     return true;
> > +}
> > +
> > +bool kvm__load_firmware(struct kvm *kvm, const char *firmware_filename)
> > +{
> > +     /* TODO: Firmware loading to be supported later. */
> > +     return false;
> > +}
> > +
> > +int kvm__arch_setup_firmware(struct kvm *kvm)
> > +{
> > +     return 0;
> > +}
> > diff --git a/util/update_headers.sh b/util/update_headers.sh
> > index bf87ef6..78eba1f 100755
> > --- a/util/update_headers.sh
> > +++ b/util/update_headers.sh
> > @@ -36,7 +36,7 @@ copy_optional_arch () {
> >       fi
> >  }
> >
> > -for arch in arm arm64 mips powerpc x86
> > +for arch in arm arm64 mips powerpc riscv x86
> >  do
> >       case "$arch" in
> >               arm) KVMTOOL_PATH=arm/aarch32 ;;

Re: [kvmtool RFC PATCH 2/8] riscv: Initial skeletal support

[Alexandru Elisei]

Hi,

On 1/10/20 3:30 AM, Anup Patel wrote:
> On Wed, Jan 8, 2020 at 6:52 PM Alexandru Elisei
> <alexandru.elisei@arm.com> wrote:
>> Hi,
>>
>> I don't know much about the RISC-V architecture, so I'm only going to comment on
>> the more generic code.
> Sure, no problem.
>
>> On 12/25/19 3:00 AM, Anup Patel wrote:
>>> This patch adds initial skeletal KVMTOOL RISC-V support which
>>> just compiles for RV32 and RV64 host.
>>>
>>> Signed-off-by: Anup Patel <anup.patel@wdc.com>
>>> ---
>>>  INSTALL                             |   7 +-
>>>  Makefile                            |  15 +++-
>>>  riscv/include/asm/kvm.h             | 127 ++++++++++++++++++++++++++++
>>>  riscv/include/kvm/barrier.h         |  14 +++
>>>  riscv/include/kvm/fdt-arch.h        |   4 +
>>>  riscv/include/kvm/kvm-arch.h        |  58 +++++++++++++
>>>  riscv/include/kvm/kvm-config-arch.h |   9 ++
>>>  riscv/include/kvm/kvm-cpu-arch.h    |  49 +++++++++++
>>>  riscv/ioport.c                      |  11 +++
>>>  riscv/irq.c                         |  13 +++
>>>  riscv/kvm-cpu.c                     |  64 ++++++++++++++
>>>  riscv/kvm.c                         |  61 +++++++++++++
>>>  util/update_headers.sh              |   2 +-
>>>  13 files changed, 429 insertions(+), 5 deletions(-)
>>>  create mode 100644 riscv/include/asm/kvm.h
>>>  create mode 100644 riscv/include/kvm/barrier.h
>>>  create mode 100644 riscv/include/kvm/fdt-arch.h
>>>  create mode 100644 riscv/include/kvm/kvm-arch.h
>>>  create mode 100644 riscv/include/kvm/kvm-config-arch.h
>>>  create mode 100644 riscv/include/kvm/kvm-cpu-arch.h
>>>  create mode 100644 riscv/ioport.c
>>>  create mode 100644 riscv/irq.c
>>>  create mode 100644 riscv/kvm-cpu.c
>>>  create mode 100644 riscv/kvm.c
>>>
>>> diff --git a/INSTALL b/INSTALL
>>> index ca8e022..951b123 100644
>>> --- a/INSTALL
>>> +++ b/INSTALL
>>> @@ -26,8 +26,8 @@ For Fedora based systems:
>>>  For OpenSUSE based systems:
>>>       # zypper install glibc-devel-static
>>>
>>> -Architectures which require device tree (PowerPC, ARM, ARM64) also require
>>> -libfdt.
>>> +Architectures which require device tree (PowerPC, ARM, ARM64, RISC-V) also
>>> +require libfdt.
>>>       deb: $ sudo apt-get install libfdt-dev
>>>       Fedora: # yum install libfdt-devel
>>>       OpenSUSE: # zypper install libfdt1-devel
>>> @@ -64,6 +64,7 @@ to the Linux name of the architecture. Architectures supported:
>>>  - arm
>>>  - arm64
>>>  - mips
>>> +- riscv
>>>  If ARCH is not provided, the target architecture will be automatically
>>>  determined by running "uname -m" on your host, resulting in a native build.
>>>
>>> @@ -81,7 +82,7 @@ On multiarch system you should be able to install those be appending
>>>  the architecture name after the package (example for ARM64):
>>>  $ sudo apt-get install libfdt-dev:arm64
>>>
>>> -PowerPC and ARM/ARM64 require libfdt to be installed. If you cannot use
>>> +PowerPC, ARM/ARM64 and RISC-V require libfdt to be installed. If you cannot use
>>>  precompiled mulitarch packages, you could either copy the required header and
>>>  library files from an installed target system into the SYSROOT (you will need
>>>  /usr/include/*fdt*.h and /usr/lib64/libfdt-v.v.v.so and its symlinks), or you
>>> diff --git a/Makefile b/Makefile
>>> index 3862112..972fa63 100644
>>> --- a/Makefile
>>> +++ b/Makefile
>>> @@ -106,7 +106,8 @@ OBJS      += hw/i8042.o
>>>
>>>  # Translate uname -m into ARCH string
>>>  ARCH ?= $(shell uname -m | sed -e s/i.86/i386/ -e s/ppc.*/powerpc/ \
>>> -       -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/)
>>> +       -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/ \
>>> +       -e s/riscv64/riscv/ -e s/riscv32/riscv/)
>>>
>>>  ifeq ($(ARCH),i386)
>>>       ARCH         := x86
>>> @@ -190,6 +191,18 @@ ifeq ($(ARCH),mips)
>>>       OBJS            += mips/kvm.o
>>>       OBJS            += mips/kvm-cpu.o
>>>  endif
>>> +
>>> +# RISC-V (RV32 and RV64)
>>> +ifeq ($(ARCH),riscv)
>>> +     DEFINES         += -DCONFIG_RISCV
>>> +     ARCH_INCLUDE    := riscv/include
>>> +     OBJS            += riscv/ioport.o
>>> +     OBJS            += riscv/irq.o
>>> +     OBJS            += riscv/kvm.o
>>> +     OBJS            += riscv/kvm-cpu.o
>>> +
>>> +     ARCH_WANT_LIBFDT := y
>>> +endif
>>>  ###
>>>
>>>  ifeq (,$(ARCH_INCLUDE))
>>> diff --git a/riscv/include/asm/kvm.h b/riscv/include/asm/kvm.h
>>> new file mode 100644
>>> index 0000000..f4274c2
>>> --- /dev/null
>>> +++ b/riscv/include/asm/kvm.h
>>> @@ -0,0 +1,127 @@
>>> +/* SPDX-License-Identifier: GPL-2.0 */
>>> +/*
>>> + * Copyright (C) 2019 Western Digital Corporation or its affiliates.
>>> + *
>>> + * Authors:
>>> + *     Anup Patel <anup.patel@wdc.com>
>>> + */
>>> +
>>> +#ifndef __LINUX_KVM_RISCV_H
>>> +#define __LINUX_KVM_RISCV_H
>>> +
>>> +#ifndef __ASSEMBLY__
>>> +
>>> +#include <linux/types.h>
>>> +#include <asm/ptrace.h>
>>> +
>>> +#define __KVM_HAVE_READONLY_MEM
>>> +
>>> +#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
>>> +
>>> +#define KVM_INTERRUPT_SET    -1U
>>> +#define KVM_INTERRUPT_UNSET  -2U
>>> +
>>> +/* for KVM_GET_REGS and KVM_SET_REGS */
>>> +struct kvm_regs {
>>> +};
>>> +
>>> +/* for KVM_GET_FPU and KVM_SET_FPU */
>>> +struct kvm_fpu {
>>> +};
>>> +
>>> +/* KVM Debug exit structure */
>>> +struct kvm_debug_exit_arch {
>>> +};
>>> +
>>> +/* for KVM_SET_GUEST_DEBUG */
>>> +struct kvm_guest_debug_arch {
>>> +};
>>> +
>>> +/* definition of registers in kvm_run */
>>> +struct kvm_sync_regs {
>>> +};
>>> +
>>> +/* for KVM_GET_SREGS and KVM_SET_SREGS */
>>> +struct kvm_sregs {
>>> +};
>>> +
>>> +/* CONFIG registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
>>> +struct kvm_riscv_config {
>>> +     unsigned long isa;
>>> +};
>>> +
>>> +/* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
>>> +struct kvm_riscv_core {
>>> +     struct user_regs_struct regs;
>>> +     unsigned long mode;
>>> +};
>>> +
>>> +/* Possible privilege modes for kvm_riscv_core */
>>> +#define KVM_RISCV_MODE_S     1
>>> +#define KVM_RISCV_MODE_U     0
>>> +
>>> +/* CSR registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
>>> +struct kvm_riscv_csr {
>>> +     unsigned long sstatus;
>>> +     unsigned long sie;
>>> +     unsigned long stvec;
>>> +     unsigned long sscratch;
>>> +     unsigned long sepc;
>>> +     unsigned long scause;
>>> +     unsigned long stval;
>>> +     unsigned long sip;
>>> +     unsigned long satp;
>>> +};
>>> +
>>> +/* TIMER registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
>>> +struct kvm_riscv_timer {
>>> +     u64 frequency;
>>> +     u64 time;
>>> +     u64 compare;
>>> +     u64 state;
>>> +};
>>> +
>>> +/* Possible states for kvm_riscv_timer */
>>> +#define KVM_RISCV_TIMER_STATE_OFF    0
>>> +#define KVM_RISCV_TIMER_STATE_ON     1
>>> +
>>> +#define KVM_REG_SIZE(id)             \
>>> +     (1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
>>> +
>>> +/* If you need to interpret the index values, here is the key: */
>>> +#define KVM_REG_RISCV_TYPE_MASK              0x00000000FF000000
>>> +#define KVM_REG_RISCV_TYPE_SHIFT     24
>>> +
>>> +/* Config registers are mapped as type 1 */
>>> +#define KVM_REG_RISCV_CONFIG         (0x01 << KVM_REG_RISCV_TYPE_SHIFT)
>>> +#define KVM_REG_RISCV_CONFIG_REG(name)       \
>>> +     (offsetof(struct kvm_riscv_config, name) / sizeof(unsigned long))
>>> +
>>> +/* Core registers are mapped as type 2 */
>>> +#define KVM_REG_RISCV_CORE           (0x02 << KVM_REG_RISCV_TYPE_SHIFT)
>>> +#define KVM_REG_RISCV_CORE_REG(name) \
>>> +             (offsetof(struct kvm_riscv_core, name) / sizeof(unsigned long))
>>> +
>>> +/* Control and status registers are mapped as type 3 */
>>> +#define KVM_REG_RISCV_CSR            (0x03 << KVM_REG_RISCV_TYPE_SHIFT)
>>> +#define KVM_REG_RISCV_CSR_REG(name)  \
>>> +             (offsetof(struct kvm_riscv_csr, name) / sizeof(unsigned long))
>>> +
>>> +/* Timer registers are mapped as type 4 */
>>> +#define KVM_REG_RISCV_TIMER          (0x04 << KVM_REG_RISCV_TYPE_SHIFT)
>>> +#define KVM_REG_RISCV_TIMER_REG(name)        \
>>> +             (offsetof(struct kvm_riscv_timer, name) / sizeof(u64))
>>> +
>>> +/* F extension registers are mapped as type 5 */
>>> +#define KVM_REG_RISCV_FP_F           (0x05 << KVM_REG_RISCV_TYPE_SHIFT)
>>> +#define KVM_REG_RISCV_FP_F_REG(name) \
>>> +             (offsetof(struct __riscv_f_ext_state, name) / sizeof(u32))
>>> +
>>> +/* D extension registers are mapped as type 6 */
>>> +#define KVM_REG_RISCV_FP_D           (0x06 << KVM_REG_RISCV_TYPE_SHIFT)
>>> +#define KVM_REG_RISCV_FP_D_REG(name) \
>>> +             (offsetof(struct __riscv_d_ext_state, name) / sizeof(u64))
>>> +
>>> +#endif
>>> +
>>> +#endif /* __LINUX_KVM_RISCV_H */
>>> diff --git a/riscv/include/kvm/barrier.h b/riscv/include/kvm/barrier.h
>>> new file mode 100644
>>> index 0000000..235f610
>>> --- /dev/null
>>> +++ b/riscv/include/kvm/barrier.h
>>> @@ -0,0 +1,14 @@
>>> +#ifndef KVM__KVM_BARRIER_H
>>> +#define KVM__KVM_BARRIER_H
>>> +
>>> +#define nop()                __asm__ __volatile__ ("nop")
>>> +
>>> +#define RISCV_FENCE(p, s) \
>>> +     __asm__ __volatile__ ("fence " #p "," #s : : : "memory")
>>> +
>>> +/* These barriers need to enforce ordering on both devices or memory. */
>>> +#define mb()         RISCV_FENCE(iorw,iorw)
>>> +#define rmb()                RISCV_FENCE(ir,ir)
>>> +#define wmb()                RISCV_FENCE(ow,ow)
>>> +
>>> +#endif /* KVM__KVM_BARRIER_H */
>>> diff --git a/riscv/include/kvm/fdt-arch.h b/riscv/include/kvm/fdt-arch.h
>>> new file mode 100644
>>> index 0000000..9450fc5
>>> --- /dev/null
>>> +++ b/riscv/include/kvm/fdt-arch.h
>>> @@ -0,0 +1,4 @@
>>> +#ifndef KVM__KVM_FDT_H
>>> +#define KVM__KVM_FDT_H
>>> +
>>> +#endif /* KVM__KVM_FDT_H */
>>> diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
>>> new file mode 100644
>>> index 0000000..7e9c578
>>> --- /dev/null
>>> +++ b/riscv/include/kvm/kvm-arch.h
>>> @@ -0,0 +1,58 @@
>>> +#ifndef KVM__KVM_ARCH_H
>>> +#define KVM__KVM_ARCH_H
>>> +
>>> +#include <stdbool.h>
>>> +#include <linux/const.h>
>>> +#include <linux/types.h>
>>> +
>>> +#define RISCV_IOPORT         0x00000000ULL
>>> +#define RISCV_IOPORT_SIZE    0x00010000ULL
>>> +#define RISCV_PLIC           0x0c000000ULL
>>> +#define RISCV_PLIC_SIZE              0x04000000ULL
>>> +#define RISCV_MMIO           0x10000000ULL
>>> +#define RISCV_MMIO_SIZE              0x20000000ULL
>>> +#define RISCV_PCI            0x30000000ULL
>>> +#define RISCV_PCI_CFG_SIZE   0x10000000ULL
>> In the DTB you're advertising the PCI node as CAM compatible, which is the right
>> thing to do. Legacy PCI configuration space is 16MB, not 256MB (PCI Express is 256MB).
> I was confused here so I did what was done for ARM. I will check with other
> architectures and update like you suggested.

For ARM, it's 16 MB (taken from arm/include/arm-common/kvm-arch.h)

#define ARM_PCI_CFG_SIZE    (1ULL << 24)

which was duplicated from include/kvm/pci.h:

#define PCI_CFG_SIZE        (1ULL << 24)**

It's not a question of what other architectures do, it's about kvmtool emulating
the legacy PCI 3.0 protocol which uses 24 bits for device addressing. You can
declare the PCI configuration space to be 256 MB (i.e 28 bit addresses), but
kvmtool will only use the bottom 16MB.

Thanks,
Alex
>
>>> +#define RISCV_PCI_SIZE               0x50000000ULL
>>> +#define RISCV_PCI_MMIO_SIZE  (RISCV_PCI_SIZE - RISCV_PCI_CFG_SIZE)
>>> +
>>> +#define RISCV_RAM            0x80000000ULL
>> I'm not sure about the reasons for choosing RAM to start at 2GB. For arm we do the
>> same, but qemu starts memory at 1GB and this mismatch has caused some issues in
>> the past. For example, 32 bit kvm-unit-tests currently do not run under kvmtool
>> because the text address is hardcoded to the qemu default value.
> Actually in RISC-V world, it is kind of a un-documented standard that
> RAM starts at
> 2GB (0x80000000) for both RV32 and RV64. This is true for all existing HW,
> QEMU RISC-V virt machine and here. This will be soon explicitly documented in
> RISC-V Unix platform spec.
>
>> As a more general observation, I know that other architectures (like arm) declare
>> the memory layout in hexadecimal numbers, but it might be a better idea to use the
>> sizes from include/linux/sizes.h, since it makes the memory layout a lot more
>> readable and mistakes are easier to spot.
> Sure, I will try to use linux/sizes.h in next patch version.
>
>>> +
>>> +#define RISCV_LOMAP_MAX_MEMORY       ((1ULL << 32) - RISCV_RAM)
>>> +#define RISCV_HIMAP_MAX_MEMORY       ((1ULL << 40) - RISCV_RAM)
>>> +
>>> +#if __riscv_xlen == 64
>>> +#define RISCV_MAX_MEMORY(kvm)        RISCV_HIMAP_MAX_MEMORY
>>> +#elif __riscv_xlen == 32
>>> +#define RISCV_MAX_MEMORY(kvm)        RISCV_LOMAP_MAX_MEMORY
>>> +#endif
>>> +
>>> +#define KVM_IOPORT_AREA              RISCV_IOPORT
>>> +#define KVM_PCI_CFG_AREA     RISCV_PCI
>>> +#define KVM_PCI_MMIO_AREA    (KVM_PCI_CFG_AREA + RISCV_PCI_CFG_SIZE)
>>> +#define KVM_VIRTIO_MMIO_AREA RISCV_MMIO
>>> +
>>> +#define KVM_IOEVENTFD_HAS_PIO        0
>>> +
>>> +#define KVM_IRQ_OFFSET               0
>>> +
>>> +#define KVM_VM_TYPE          0
>>> +
>>> +#define VIRTIO_DEFAULT_TRANS(kvm)    VIRTIO_MMIO
>>> +
>>> +#define VIRTIO_RING_ENDIAN   VIRTIO_ENDIAN_LE
>>> +
>>> +struct kvm;
>>> +
>>> +struct kvm_arch {
>>> +};
>>> +
>>> +static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
>>> +{
>>> +     u64 limit = KVM_IOPORT_AREA + RISCV_IOPORT_SIZE;
>>> +     return phys_addr >= KVM_IOPORT_AREA && phys_addr < limit;
>>> +}
>>> +
>>> +enum irq_type;
>>> +
>>> +#endif /* KVM__KVM_ARCH_H */
>>> diff --git a/riscv/include/kvm/kvm-config-arch.h b/riscv/include/kvm/kvm-config-arch.h
>>> new file mode 100644
>>> index 0000000..60c7333
>>> --- /dev/null
>>> +++ b/riscv/include/kvm/kvm-config-arch.h
>>> @@ -0,0 +1,9 @@
>>> +#ifndef KVM__KVM_CONFIG_ARCH_H
>>> +#define KVM__KVM_CONFIG_ARCH_H
>>> +
>>> +#include "kvm/parse-options.h"
>>> +
>>> +struct kvm_config_arch {
>>> +};
>>> +
>>> +#endif /* KVM__KVM_CONFIG_ARCH_H */
>>> diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
>>> new file mode 100644
>>> index 0000000..09a50e8
>>> --- /dev/null
>>> +++ b/riscv/include/kvm/kvm-cpu-arch.h
>>> @@ -0,0 +1,49 @@
>>> +#ifndef KVM__KVM_CPU_ARCH_H
>>> +#define KVM__KVM_CPU_ARCH_H
>>> +
>>> +#include <linux/kvm.h>
>>> +#include <pthread.h>
>>> +#include <stdbool.h>
>>> +
>>> +#include "kvm/kvm.h"
>>> +
>>> +struct kvm;
>> Shouldn't kvm.h already have a definition for struct kvm? Also, the arm
>> corresponding header doesn't have the include here, I don't think it's needed
>> (unless I'm missing something).
> Sure, I will drop this forward declaration here.
>
> Regards,
> Anup
>
>> Thanks,
>> Alex
>>> +
>>> +struct kvm_cpu {
>>> +     pthread_t       thread;
>>> +
>>> +     unsigned long   cpu_id;
>>> +
>>> +     struct kvm      *kvm;
>>> +     int             vcpu_fd;
>>> +     struct kvm_run  *kvm_run;
>>> +     struct kvm_cpu_task     *task;
>>> +
>>> +     u8              is_running;
>>> +     u8              paused;
>>> +     u8              needs_nmi;
>>> +
>>> +     struct kvm_coalesced_mmio_ring  *ring;
>>> +};
>>> +
>>> +static inline bool kvm_cpu__emulate_io(struct kvm_cpu *vcpu, u16 port,
>>> +                                    void *data, int direction,
>>> +                                    int size, u32 count)
>>> +{
>>> +     return false;
>>> +}
>>> +
>>> +static inline bool kvm_cpu__emulate_mmio(struct kvm_cpu *vcpu, u64 phys_addr,
>>> +                                      u8 *data, u32 len, u8 is_write)
>>> +{
>>> +     if (riscv_addr_in_ioport_region(phys_addr)) {
>>> +             int direction = is_write ? KVM_EXIT_IO_OUT : KVM_EXIT_IO_IN;
>>> +             u16 port = (phys_addr - KVM_IOPORT_AREA) & USHRT_MAX;
>>> +
>>> +             return kvm__emulate_io(vcpu, port, data, direction, len, 1);
>>> +     }
>>> +
>>> +     return kvm__emulate_mmio(vcpu, phys_addr, data, len, is_write);
>>> +}
>>> +
>>> +#endif /* KVM__KVM_CPU_ARCH_H */
>>> diff --git a/riscv/ioport.c b/riscv/ioport.c
>>> new file mode 100644
>>> index 0000000..bdd30b6
>>> --- /dev/null
>>> +++ b/riscv/ioport.c
>>> @@ -0,0 +1,11 @@
>>> +#include "kvm/ioport.h"
>>> +#include "kvm/irq.h"
>>> +
>>> +void ioport__setup_arch(struct kvm *kvm)
>>> +{
>>> +}
>>> +
>>> +void ioport__map_irq(u8 *irq)
>>> +{
>>> +     *irq = irq__alloc_line();
>>> +}
>>> diff --git a/riscv/irq.c b/riscv/irq.c
>>> new file mode 100644
>>> index 0000000..8e605ef
>>> --- /dev/null
>>> +++ b/riscv/irq.c
>>> @@ -0,0 +1,13 @@
>>> +#include "kvm/kvm.h"
>>> +#include "kvm/kvm-cpu.h"
>>> +#include "kvm/irq.h"
>>> +
>>> +void kvm__irq_line(struct kvm *kvm, int irq, int level)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +void kvm__irq_trigger(struct kvm *kvm, int irq)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
>>> new file mode 100644
>>> index 0000000..e4b8fa5
>>> --- /dev/null
>>> +++ b/riscv/kvm-cpu.c
>>> @@ -0,0 +1,64 @@
>>> +#include "kvm/kvm-cpu.h"
>>> +#include "kvm/kvm.h"
>>> +#include "kvm/virtio.h"
>>> +#include "kvm/term.h"
>>> +
>>> +#include <asm/ptrace.h>
>>> +
>>> +static int debug_fd;
>>> +
>>> +void kvm_cpu__set_debug_fd(int fd)
>>> +{
>>> +     debug_fd = fd;
>>> +}
>>> +
>>> +int kvm_cpu__get_debug_fd(void)
>>> +{
>>> +     return debug_fd;
>>> +}
>>> +
>>> +struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
>>> +{
>>> +     /* TODO: */
>>> +     return NULL;
>>> +}
>>> +
>>> +void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
>>> +{
>>> +}
>>> +
>>> +void kvm_cpu__delete(struct kvm_cpu *vcpu)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
>>> +{
>>> +     /* TODO: */
>>> +     return false;
>>> +}
>>> +
>>> +void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
>>> +{
>>> +     return VIRTIO_ENDIAN_LE;
>>> +}
>>> +
>>> +void kvm_cpu__show_code(struct kvm_cpu *vcpu)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> diff --git a/riscv/kvm.c b/riscv/kvm.c
>>> new file mode 100644
>>> index 0000000..e816ef5
>>> --- /dev/null
>>> +++ b/riscv/kvm.c
>>> @@ -0,0 +1,61 @@
>>> +#include "kvm/kvm.h"
>>> +#include "kvm/util.h"
>>> +#include "kvm/fdt.h"
>>> +
>>> +#include <linux/kernel.h>
>>> +#include <linux/kvm.h>
>>> +#include <linux/sizes.h>
>>> +
>>> +struct kvm_ext kvm_req_ext[] = {
>>> +     { DEFINE_KVM_EXT(KVM_CAP_ONE_REG) },
>>> +     { 0, 0 },
>>> +};
>>> +
>>> +bool kvm__arch_cpu_supports_vm(void)
>>> +{
>>> +     /* The KVM capability check is enough. */
>>> +     return true;
>>> +}
>>> +
>>> +void kvm__init_ram(struct kvm *kvm)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +void kvm__arch_delete_ram(struct kvm *kvm)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +void kvm__arch_read_term(struct kvm *kvm)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +void kvm__arch_set_cmdline(char *cmdline, bool video)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
>>> +{
>>> +     /* TODO: */
>>> +}
>>> +
>>> +bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
>>> +                              const char *kernel_cmdline)
>>> +{
>>> +     /* TODO: */
>>> +     return true;
>>> +}
>>> +
>>> +bool kvm__load_firmware(struct kvm *kvm, const char *firmware_filename)
>>> +{
>>> +     /* TODO: Firmware loading to be supported later. */
>>> +     return false;
>>> +}
>>> +
>>> +int kvm__arch_setup_firmware(struct kvm *kvm)
>>> +{
>>> +     return 0;
>>> +}
>>> diff --git a/util/update_headers.sh b/util/update_headers.sh
>>> index bf87ef6..78eba1f 100755
>>> --- a/util/update_headers.sh
>>> +++ b/util/update_headers.sh
>>> @@ -36,7 +36,7 @@ copy_optional_arch () {
>>>       fi
>>>  }
>>>
>>> -for arch in arm arm64 mips powerpc x86
>>> +for arch in arm arm64 mips powerpc riscv x86
>>>  do
>>>       case "$arch" in
>>>               arm) KVMTOOL_PATH=arm/aarch32 ;;

Re: [kvmtool RFC PATCH 2/8] riscv: Initial skeletal support

[Anup Patel]

On Fri, Jan 10, 2020 at 3:17 PM Alexandru Elisei
<alexandru.elisei@arm.com> wrote:
>
> Hi,
>
> On 1/10/20 3:30 AM, Anup Patel wrote:
> > On Wed, Jan 8, 2020 at 6:52 PM Alexandru Elisei
> > <alexandru.elisei@arm.com> wrote:
> >> Hi,
> >>
> >> I don't know much about the RISC-V architecture, so I'm only going to comment on
> >> the more generic code.
> > Sure, no problem.
> >
> >> On 12/25/19 3:00 AM, Anup Patel wrote:
> >>> This patch adds initial skeletal KVMTOOL RISC-V support which
> >>> just compiles for RV32 and RV64 host.
> >>>
> >>> Signed-off-by: Anup Patel <anup.patel@wdc.com>
> >>> ---
> >>>  INSTALL                             |   7 +-
> >>>  Makefile                            |  15 +++-
> >>>  riscv/include/asm/kvm.h             | 127 ++++++++++++++++++++++++++++
> >>>  riscv/include/kvm/barrier.h         |  14 +++
> >>>  riscv/include/kvm/fdt-arch.h        |   4 +
> >>>  riscv/include/kvm/kvm-arch.h        |  58 +++++++++++++
> >>>  riscv/include/kvm/kvm-config-arch.h |   9 ++
> >>>  riscv/include/kvm/kvm-cpu-arch.h    |  49 +++++++++++
> >>>  riscv/ioport.c                      |  11 +++
> >>>  riscv/irq.c                         |  13 +++
> >>>  riscv/kvm-cpu.c                     |  64 ++++++++++++++
> >>>  riscv/kvm.c                         |  61 +++++++++++++
> >>>  util/update_headers.sh              |   2 +-
> >>>  13 files changed, 429 insertions(+), 5 deletions(-)
> >>>  create mode 100644 riscv/include/asm/kvm.h
> >>>  create mode 100644 riscv/include/kvm/barrier.h
> >>>  create mode 100644 riscv/include/kvm/fdt-arch.h
> >>>  create mode 100644 riscv/include/kvm/kvm-arch.h
> >>>  create mode 100644 riscv/include/kvm/kvm-config-arch.h
> >>>  create mode 100644 riscv/include/kvm/kvm-cpu-arch.h
> >>>  create mode 100644 riscv/ioport.c
> >>>  create mode 100644 riscv/irq.c
> >>>  create mode 100644 riscv/kvm-cpu.c
> >>>  create mode 100644 riscv/kvm.c
> >>>
> >>> diff --git a/INSTALL b/INSTALL
> >>> index ca8e022..951b123 100644
> >>> --- a/INSTALL
> >>> +++ b/INSTALL
> >>> @@ -26,8 +26,8 @@ For Fedora based systems:
> >>>  For OpenSUSE based systems:
> >>>       # zypper install glibc-devel-static
> >>>
> >>> -Architectures which require device tree (PowerPC, ARM, ARM64) also require
> >>> -libfdt.
> >>> +Architectures which require device tree (PowerPC, ARM, ARM64, RISC-V) also
> >>> +require libfdt.
> >>>       deb: $ sudo apt-get install libfdt-dev
> >>>       Fedora: # yum install libfdt-devel
> >>>       OpenSUSE: # zypper install libfdt1-devel
> >>> @@ -64,6 +64,7 @@ to the Linux name of the architecture. Architectures supported:
> >>>  - arm
> >>>  - arm64
> >>>  - mips
> >>> +- riscv
> >>>  If ARCH is not provided, the target architecture will be automatically
> >>>  determined by running "uname -m" on your host, resulting in a native build.
> >>>
> >>> @@ -81,7 +82,7 @@ On multiarch system you should be able to install those be appending
> >>>  the architecture name after the package (example for ARM64):
> >>>  $ sudo apt-get install libfdt-dev:arm64
> >>>
> >>> -PowerPC and ARM/ARM64 require libfdt to be installed. If you cannot use
> >>> +PowerPC, ARM/ARM64 and RISC-V require libfdt to be installed. If you cannot use
> >>>  precompiled mulitarch packages, you could either copy the required header and
> >>>  library files from an installed target system into the SYSROOT (you will need
> >>>  /usr/include/*fdt*.h and /usr/lib64/libfdt-v.v.v.so and its symlinks), or you
> >>> diff --git a/Makefile b/Makefile
> >>> index 3862112..972fa63 100644
> >>> --- a/Makefile
> >>> +++ b/Makefile
> >>> @@ -106,7 +106,8 @@ OBJS      += hw/i8042.o
> >>>
> >>>  # Translate uname -m into ARCH string
> >>>  ARCH ?= $(shell uname -m | sed -e s/i.86/i386/ -e s/ppc.*/powerpc/ \
> >>> -       -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/)
> >>> +       -e s/armv.*/arm/ -e s/aarch64.*/arm64/ -e s/mips64/mips/ \
> >>> +       -e s/riscv64/riscv/ -e s/riscv32/riscv/)
> >>>
> >>>  ifeq ($(ARCH),i386)
> >>>       ARCH         := x86
> >>> @@ -190,6 +191,18 @@ ifeq ($(ARCH),mips)
> >>>       OBJS            += mips/kvm.o
> >>>       OBJS            += mips/kvm-cpu.o
> >>>  endif
> >>> +
> >>> +# RISC-V (RV32 and RV64)
> >>> +ifeq ($(ARCH),riscv)
> >>> +     DEFINES         += -DCONFIG_RISCV
> >>> +     ARCH_INCLUDE    := riscv/include
> >>> +     OBJS            += riscv/ioport.o
> >>> +     OBJS            += riscv/irq.o
> >>> +     OBJS            += riscv/kvm.o
> >>> +     OBJS            += riscv/kvm-cpu.o
> >>> +
> >>> +     ARCH_WANT_LIBFDT := y
> >>> +endif
> >>>  ###
> >>>
> >>>  ifeq (,$(ARCH_INCLUDE))
> >>> diff --git a/riscv/include/asm/kvm.h b/riscv/include/asm/kvm.h
> >>> new file mode 100644
> >>> index 0000000..f4274c2
> >>> --- /dev/null
> >>> +++ b/riscv/include/asm/kvm.h
> >>> @@ -0,0 +1,127 @@
> >>> +/* SPDX-License-Identifier: GPL-2.0 */
> >>> +/*
> >>> + * Copyright (C) 2019 Western Digital Corporation or its affiliates.
> >>> + *
> >>> + * Authors:
> >>> + *     Anup Patel <anup.patel@wdc.com>
> >>> + */
> >>> +
> >>> +#ifndef __LINUX_KVM_RISCV_H
> >>> +#define __LINUX_KVM_RISCV_H
> >>> +
> >>> +#ifndef __ASSEMBLY__
> >>> +
> >>> +#include <linux/types.h>
> >>> +#include <asm/ptrace.h>
> >>> +
> >>> +#define __KVM_HAVE_READONLY_MEM
> >>> +
> >>> +#define KVM_COALESCED_MMIO_PAGE_OFFSET 1
> >>> +
> >>> +#define KVM_INTERRUPT_SET    -1U
> >>> +#define KVM_INTERRUPT_UNSET  -2U
> >>> +
> >>> +/* for KVM_GET_REGS and KVM_SET_REGS */
> >>> +struct kvm_regs {
> >>> +};
> >>> +
> >>> +/* for KVM_GET_FPU and KVM_SET_FPU */
> >>> +struct kvm_fpu {
> >>> +};
> >>> +
> >>> +/* KVM Debug exit structure */
> >>> +struct kvm_debug_exit_arch {
> >>> +};
> >>> +
> >>> +/* for KVM_SET_GUEST_DEBUG */
> >>> +struct kvm_guest_debug_arch {
> >>> +};
> >>> +
> >>> +/* definition of registers in kvm_run */
> >>> +struct kvm_sync_regs {
> >>> +};
> >>> +
> >>> +/* for KVM_GET_SREGS and KVM_SET_SREGS */
> >>> +struct kvm_sregs {
> >>> +};
> >>> +
> >>> +/* CONFIG registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> >>> +struct kvm_riscv_config {
> >>> +     unsigned long isa;
> >>> +};
> >>> +
> >>> +/* CORE registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> >>> +struct kvm_riscv_core {
> >>> +     struct user_regs_struct regs;
> >>> +     unsigned long mode;
> >>> +};
> >>> +
> >>> +/* Possible privilege modes for kvm_riscv_core */
> >>> +#define KVM_RISCV_MODE_S     1
> >>> +#define KVM_RISCV_MODE_U     0
> >>> +
> >>> +/* CSR registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> >>> +struct kvm_riscv_csr {
> >>> +     unsigned long sstatus;
> >>> +     unsigned long sie;
> >>> +     unsigned long stvec;
> >>> +     unsigned long sscratch;
> >>> +     unsigned long sepc;
> >>> +     unsigned long scause;
> >>> +     unsigned long stval;
> >>> +     unsigned long sip;
> >>> +     unsigned long satp;
> >>> +};
> >>> +
> >>> +/* TIMER registers for KVM_GET_ONE_REG and KVM_SET_ONE_REG */
> >>> +struct kvm_riscv_timer {
> >>> +     u64 frequency;
> >>> +     u64 time;
> >>> +     u64 compare;
> >>> +     u64 state;
> >>> +};
> >>> +
> >>> +/* Possible states for kvm_riscv_timer */
> >>> +#define KVM_RISCV_TIMER_STATE_OFF    0
> >>> +#define KVM_RISCV_TIMER_STATE_ON     1
> >>> +
> >>> +#define KVM_REG_SIZE(id)             \
> >>> +     (1U << (((id) & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT))
> >>> +
> >>> +/* If you need to interpret the index values, here is the key: */
> >>> +#define KVM_REG_RISCV_TYPE_MASK              0x00000000FF000000
> >>> +#define KVM_REG_RISCV_TYPE_SHIFT     24
> >>> +
> >>> +/* Config registers are mapped as type 1 */
> >>> +#define KVM_REG_RISCV_CONFIG         (0x01 << KVM_REG_RISCV_TYPE_SHIFT)
> >>> +#define KVM_REG_RISCV_CONFIG_REG(name)       \
> >>> +     (offsetof(struct kvm_riscv_config, name) / sizeof(unsigned long))
> >>> +
> >>> +/* Core registers are mapped as type 2 */
> >>> +#define KVM_REG_RISCV_CORE           (0x02 << KVM_REG_RISCV_TYPE_SHIFT)
> >>> +#define KVM_REG_RISCV_CORE_REG(name) \
> >>> +             (offsetof(struct kvm_riscv_core, name) / sizeof(unsigned long))
> >>> +
> >>> +/* Control and status registers are mapped as type 3 */
> >>> +#define KVM_REG_RISCV_CSR            (0x03 << KVM_REG_RISCV_TYPE_SHIFT)
> >>> +#define KVM_REG_RISCV_CSR_REG(name)  \
> >>> +             (offsetof(struct kvm_riscv_csr, name) / sizeof(unsigned long))
> >>> +
> >>> +/* Timer registers are mapped as type 4 */
> >>> +#define KVM_REG_RISCV_TIMER          (0x04 << KVM_REG_RISCV_TYPE_SHIFT)
> >>> +#define KVM_REG_RISCV_TIMER_REG(name)        \
> >>> +             (offsetof(struct kvm_riscv_timer, name) / sizeof(u64))
> >>> +
> >>> +/* F extension registers are mapped as type 5 */
> >>> +#define KVM_REG_RISCV_FP_F           (0x05 << KVM_REG_RISCV_TYPE_SHIFT)
> >>> +#define KVM_REG_RISCV_FP_F_REG(name) \
> >>> +             (offsetof(struct __riscv_f_ext_state, name) / sizeof(u32))
> >>> +
> >>> +/* D extension registers are mapped as type 6 */
> >>> +#define KVM_REG_RISCV_FP_D           (0x06 << KVM_REG_RISCV_TYPE_SHIFT)
> >>> +#define KVM_REG_RISCV_FP_D_REG(name) \
> >>> +             (offsetof(struct __riscv_d_ext_state, name) / sizeof(u64))
> >>> +
> >>> +#endif
> >>> +
> >>> +#endif /* __LINUX_KVM_RISCV_H */
> >>> diff --git a/riscv/include/kvm/barrier.h b/riscv/include/kvm/barrier.h
> >>> new file mode 100644
> >>> index 0000000..235f610
> >>> --- /dev/null
> >>> +++ b/riscv/include/kvm/barrier.h
> >>> @@ -0,0 +1,14 @@
> >>> +#ifndef KVM__KVM_BARRIER_H
> >>> +#define KVM__KVM_BARRIER_H
> >>> +
> >>> +#define nop()                __asm__ __volatile__ ("nop")
> >>> +
> >>> +#define RISCV_FENCE(p, s) \
> >>> +     __asm__ __volatile__ ("fence " #p "," #s : : : "memory")
> >>> +
> >>> +/* These barriers need to enforce ordering on both devices or memory. */
> >>> +#define mb()         RISCV_FENCE(iorw,iorw)
> >>> +#define rmb()                RISCV_FENCE(ir,ir)
> >>> +#define wmb()                RISCV_FENCE(ow,ow)
> >>> +
> >>> +#endif /* KVM__KVM_BARRIER_H */
> >>> diff --git a/riscv/include/kvm/fdt-arch.h b/riscv/include/kvm/fdt-arch.h
> >>> new file mode 100644
> >>> index 0000000..9450fc5
> >>> --- /dev/null
> >>> +++ b/riscv/include/kvm/fdt-arch.h
> >>> @@ -0,0 +1,4 @@
> >>> +#ifndef KVM__KVM_FDT_H
> >>> +#define KVM__KVM_FDT_H
> >>> +
> >>> +#endif /* KVM__KVM_FDT_H */
> >>> diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
> >>> new file mode 100644
> >>> index 0000000..7e9c578
> >>> --- /dev/null
> >>> +++ b/riscv/include/kvm/kvm-arch.h
> >>> @@ -0,0 +1,58 @@
> >>> +#ifndef KVM__KVM_ARCH_H
> >>> +#define KVM__KVM_ARCH_H
> >>> +
> >>> +#include <stdbool.h>
> >>> +#include <linux/const.h>
> >>> +#include <linux/types.h>
> >>> +
> >>> +#define RISCV_IOPORT         0x00000000ULL
> >>> +#define RISCV_IOPORT_SIZE    0x00010000ULL
> >>> +#define RISCV_PLIC           0x0c000000ULL
> >>> +#define RISCV_PLIC_SIZE              0x04000000ULL
> >>> +#define RISCV_MMIO           0x10000000ULL
> >>> +#define RISCV_MMIO_SIZE              0x20000000ULL
> >>> +#define RISCV_PCI            0x30000000ULL
> >>> +#define RISCV_PCI_CFG_SIZE   0x10000000ULL
> >> In the DTB you're advertising the PCI node as CAM compatible, which is the right
> >> thing to do. Legacy PCI configuration space is 16MB, not 256MB (PCI Express is 256MB).
> > I was confused here so I did what was done for ARM. I will check with other
> > architectures and update like you suggested.
>
> For ARM, it's 16 MB (taken from arm/include/arm-common/kvm-arch.h)
>
> #define ARM_PCI_CFG_SIZE    (1ULL << 24)
>
> which was duplicated from include/kvm/pci.h:
>
> #define PCI_CFG_SIZE        (1ULL << 24)**
>
> It's not a question of what other architectures do, it's about kvmtool emulating
> the legacy PCI 3.0 protocol which uses 24 bits for device addressing. You can
> declare the PCI configuration space to be 256 MB (i.e 28 bit addresses), but
> kvmtool will only use the bottom 16MB.

Thanks for the PCI related explanation.

I will update PCI config size to 16M.

Regards,
Anup

>
> Thanks,
> Alex
> >
> >>> +#define RISCV_PCI_SIZE               0x50000000ULL
> >>> +#define RISCV_PCI_MMIO_SIZE  (RISCV_PCI_SIZE - RISCV_PCI_CFG_SIZE)
> >>> +
> >>> +#define RISCV_RAM            0x80000000ULL
> >> I'm not sure about the reasons for choosing RAM to start at 2GB. For arm we do the
> >> same, but qemu starts memory at 1GB and this mismatch has caused some issues in
> >> the past. For example, 32 bit kvm-unit-tests currently do not run under kvmtool
> >> because the text address is hardcoded to the qemu default value.
> > Actually in RISC-V world, it is kind of a un-documented standard that
> > RAM starts at
> > 2GB (0x80000000) for both RV32 and RV64. This is true for all existing HW,
> > QEMU RISC-V virt machine and here. This will be soon explicitly documented in
> > RISC-V Unix platform spec.
> >
> >> As a more general observation, I know that other architectures (like arm) declare
> >> the memory layout in hexadecimal numbers, but it might be a better idea to use the
> >> sizes from include/linux/sizes.h, since it makes the memory layout a lot more
> >> readable and mistakes are easier to spot.
> > Sure, I will try to use linux/sizes.h in next patch version.
> >
> >>> +
> >>> +#define RISCV_LOMAP_MAX_MEMORY       ((1ULL << 32) - RISCV_RAM)
> >>> +#define RISCV_HIMAP_MAX_MEMORY       ((1ULL << 40) - RISCV_RAM)
> >>> +
> >>> +#if __riscv_xlen == 64
> >>> +#define RISCV_MAX_MEMORY(kvm)        RISCV_HIMAP_MAX_MEMORY
> >>> +#elif __riscv_xlen == 32
> >>> +#define RISCV_MAX_MEMORY(kvm)        RISCV_LOMAP_MAX_MEMORY
> >>> +#endif
> >>> +
> >>> +#define KVM_IOPORT_AREA              RISCV_IOPORT
> >>> +#define KVM_PCI_CFG_AREA     RISCV_PCI
> >>> +#define KVM_PCI_MMIO_AREA    (KVM_PCI_CFG_AREA + RISCV_PCI_CFG_SIZE)
> >>> +#define KVM_VIRTIO_MMIO_AREA RISCV_MMIO
> >>> +
> >>> +#define KVM_IOEVENTFD_HAS_PIO        0
> >>> +
> >>> +#define KVM_IRQ_OFFSET               0
> >>> +
> >>> +#define KVM_VM_TYPE          0
> >>> +
> >>> +#define VIRTIO_DEFAULT_TRANS(kvm)    VIRTIO_MMIO
> >>> +
> >>> +#define VIRTIO_RING_ENDIAN   VIRTIO_ENDIAN_LE
> >>> +
> >>> +struct kvm;
> >>> +
> >>> +struct kvm_arch {
> >>> +};
> >>> +
> >>> +static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
> >>> +{
> >>> +     u64 limit = KVM_IOPORT_AREA + RISCV_IOPORT_SIZE;
> >>> +     return phys_addr >= KVM_IOPORT_AREA && phys_addr < limit;
> >>> +}
> >>> +
> >>> +enum irq_type;
> >>> +
> >>> +#endif /* KVM__KVM_ARCH_H */
> >>> diff --git a/riscv/include/kvm/kvm-config-arch.h b/riscv/include/kvm/kvm-config-arch.h
> >>> new file mode 100644
> >>> index 0000000..60c7333
> >>> --- /dev/null
> >>> +++ b/riscv/include/kvm/kvm-config-arch.h
> >>> @@ -0,0 +1,9 @@
> >>> +#ifndef KVM__KVM_CONFIG_ARCH_H
> >>> +#define KVM__KVM_CONFIG_ARCH_H
> >>> +
> >>> +#include "kvm/parse-options.h"
> >>> +
> >>> +struct kvm_config_arch {
> >>> +};
> >>> +
> >>> +#endif /* KVM__KVM_CONFIG_ARCH_H */
> >>> diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
> >>> new file mode 100644
> >>> index 0000000..09a50e8
> >>> --- /dev/null
> >>> +++ b/riscv/include/kvm/kvm-cpu-arch.h
> >>> @@ -0,0 +1,49 @@
> >>> +#ifndef KVM__KVM_CPU_ARCH_H
> >>> +#define KVM__KVM_CPU_ARCH_H
> >>> +
> >>> +#include <linux/kvm.h>
> >>> +#include <pthread.h>
> >>> +#include <stdbool.h>
> >>> +
> >>> +#include "kvm/kvm.h"
> >>> +
> >>> +struct kvm;
> >> Shouldn't kvm.h already have a definition for struct kvm? Also, the arm
> >> corresponding header doesn't have the include here, I don't think it's needed
> >> (unless I'm missing something).
> > Sure, I will drop this forward declaration here.
> >
> > Regards,
> > Anup
> >
> >> Thanks,
> >> Alex
> >>> +
> >>> +struct kvm_cpu {
> >>> +     pthread_t       thread;
> >>> +
> >>> +     unsigned long   cpu_id;
> >>> +
> >>> +     struct kvm      *kvm;
> >>> +     int             vcpu_fd;
> >>> +     struct kvm_run  *kvm_run;
> >>> +     struct kvm_cpu_task     *task;
> >>> +
> >>> +     u8              is_running;
> >>> +     u8              paused;
> >>> +     u8              needs_nmi;
> >>> +
> >>> +     struct kvm_coalesced_mmio_ring  *ring;
> >>> +};
> >>> +
> >>> +static inline bool kvm_cpu__emulate_io(struct kvm_cpu *vcpu, u16 port,
> >>> +                                    void *data, int direction,
> >>> +                                    int size, u32 count)
> >>> +{
> >>> +     return false;
> >>> +}
> >>> +
> >>> +static inline bool kvm_cpu__emulate_mmio(struct kvm_cpu *vcpu, u64 phys_addr,
> >>> +                                      u8 *data, u32 len, u8 is_write)
> >>> +{
> >>> +     if (riscv_addr_in_ioport_region(phys_addr)) {
> >>> +             int direction = is_write ? KVM_EXIT_IO_OUT : KVM_EXIT_IO_IN;
> >>> +             u16 port = (phys_addr - KVM_IOPORT_AREA) & USHRT_MAX;
> >>> +
> >>> +             return kvm__emulate_io(vcpu, port, data, direction, len, 1);
> >>> +     }
> >>> +
> >>> +     return kvm__emulate_mmio(vcpu, phys_addr, data, len, is_write);
> >>> +}
> >>> +
> >>> +#endif /* KVM__KVM_CPU_ARCH_H */
> >>> diff --git a/riscv/ioport.c b/riscv/ioport.c
> >>> new file mode 100644
> >>> index 0000000..bdd30b6
> >>> --- /dev/null
> >>> +++ b/riscv/ioport.c
> >>> @@ -0,0 +1,11 @@
> >>> +#include "kvm/ioport.h"
> >>> +#include "kvm/irq.h"
> >>> +
> >>> +void ioport__setup_arch(struct kvm *kvm)
> >>> +{
> >>> +}
> >>> +
> >>> +void ioport__map_irq(u8 *irq)
> >>> +{
> >>> +     *irq = irq__alloc_line();
> >>> +}
> >>> diff --git a/riscv/irq.c b/riscv/irq.c
> >>> new file mode 100644
> >>> index 0000000..8e605ef
> >>> --- /dev/null
> >>> +++ b/riscv/irq.c
> >>> @@ -0,0 +1,13 @@
> >>> +#include "kvm/kvm.h"
> >>> +#include "kvm/kvm-cpu.h"
> >>> +#include "kvm/irq.h"
> >>> +
> >>> +void kvm__irq_line(struct kvm *kvm, int irq, int level)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +void kvm__irq_trigger(struct kvm *kvm, int irq)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
> >>> new file mode 100644
> >>> index 0000000..e4b8fa5
> >>> --- /dev/null
> >>> +++ b/riscv/kvm-cpu.c
> >>> @@ -0,0 +1,64 @@
> >>> +#include "kvm/kvm-cpu.h"
> >>> +#include "kvm/kvm.h"
> >>> +#include "kvm/virtio.h"
> >>> +#include "kvm/term.h"
> >>> +
> >>> +#include <asm/ptrace.h>
> >>> +
> >>> +static int debug_fd;
> >>> +
> >>> +void kvm_cpu__set_debug_fd(int fd)
> >>> +{
> >>> +     debug_fd = fd;
> >>> +}
> >>> +
> >>> +int kvm_cpu__get_debug_fd(void)
> >>> +{
> >>> +     return debug_fd;
> >>> +}
> >>> +
> >>> +struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
> >>> +{
> >>> +     /* TODO: */
> >>> +     return NULL;
> >>> +}
> >>> +
> >>> +void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
> >>> +{
> >>> +}
> >>> +
> >>> +void kvm_cpu__delete(struct kvm_cpu *vcpu)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
> >>> +{
> >>> +     /* TODO: */
> >>> +     return false;
> >>> +}
> >>> +
> >>> +void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
> >>> +{
> >>> +     return VIRTIO_ENDIAN_LE;
> >>> +}
> >>> +
> >>> +void kvm_cpu__show_code(struct kvm_cpu *vcpu)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> diff --git a/riscv/kvm.c b/riscv/kvm.c
> >>> new file mode 100644
> >>> index 0000000..e816ef5
> >>> --- /dev/null
> >>> +++ b/riscv/kvm.c
> >>> @@ -0,0 +1,61 @@
> >>> +#include "kvm/kvm.h"
> >>> +#include "kvm/util.h"
> >>> +#include "kvm/fdt.h"
> >>> +
> >>> +#include <linux/kernel.h>
> >>> +#include <linux/kvm.h>
> >>> +#include <linux/sizes.h>
> >>> +
> >>> +struct kvm_ext kvm_req_ext[] = {
> >>> +     { DEFINE_KVM_EXT(KVM_CAP_ONE_REG) },
> >>> +     { 0, 0 },
> >>> +};
> >>> +
> >>> +bool kvm__arch_cpu_supports_vm(void)
> >>> +{
> >>> +     /* The KVM capability check is enough. */
> >>> +     return true;
> >>> +}
> >>> +
> >>> +void kvm__init_ram(struct kvm *kvm)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +void kvm__arch_delete_ram(struct kvm *kvm)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +void kvm__arch_read_term(struct kvm *kvm)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +void kvm__arch_set_cmdline(char *cmdline, bool video)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
> >>> +{
> >>> +     /* TODO: */
> >>> +}
> >>> +
> >>> +bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
> >>> +                              const char *kernel_cmdline)
> >>> +{
> >>> +     /* TODO: */
> >>> +     return true;
> >>> +}
> >>> +
> >>> +bool kvm__load_firmware(struct kvm *kvm, const char *firmware_filename)
> >>> +{
> >>> +     /* TODO: Firmware loading to be supported later. */
> >>> +     return false;
> >>> +}
> >>> +
> >>> +int kvm__arch_setup_firmware(struct kvm *kvm)
> >>> +{
> >>> +     return 0;
> >>> +}
> >>> diff --git a/util/update_headers.sh b/util/update_headers.sh
> >>> index bf87ef6..78eba1f 100755
> >>> --- a/util/update_headers.sh
> >>> +++ b/util/update_headers.sh
> >>> @@ -36,7 +36,7 @@ copy_optional_arch () {
> >>>       fi
> >>>  }
> >>>
> >>> -for arch in arm arm64 mips powerpc x86
> >>> +for arch in arm arm64 mips powerpc riscv x86
> >>>  do
> >>>       case "$arch" in
> >>>               arm) KVMTOOL_PATH=arm/aarch32 ;;

Re: [kvmtool RFC PATCH 2/8] riscv: Initial skeletal support

[Andre Przywara]

On Fri, 10 Jan 2020 17:05:28 +0530
Anup Patel <anup@brainfault.org> wrote:

Hi,

[ ... ]

> > >>> +
> > >>> +#define RISCV_IOPORT         0x00000000ULL
> > >>> +#define RISCV_IOPORT_SIZE    0x00010000ULL
> > >>> +#define RISCV_PLIC           0x0c000000ULL
> > >>> +#define RISCV_PLIC_SIZE              0x04000000ULL
> > >>> +#define RISCV_MMIO           0x10000000ULL
> > >>> +#define RISCV_MMIO_SIZE              0x20000000ULL
> > >>> +#define RISCV_PCI            0x30000000ULL
> > >>> +#define RISCV_PCI_CFG_SIZE   0x10000000ULL  
> > >> In the DTB you're advertising the PCI node as CAM compatible, which is the right
> > >> thing to do. Legacy PCI configuration space is 16MB, not 256MB (PCI Express is 256MB).  
> > > I was confused here so I did what was done for ARM. I will check with other
> > > architectures and update like you suggested.  
> >
> > For ARM, it's 16 MB (taken from arm/include/arm-common/kvm-arch.h)
> >
> > #define ARM_PCI_CFG_SIZE    (1ULL << 24)
> >
> > which was duplicated from include/kvm/pci.h:
> >
> > #define PCI_CFG_SIZE        (1ULL << 24)**
> >
> > It's not a question of what other architectures do, it's about kvmtool emulating
> > the legacy PCI 3.0 protocol which uses 24 bits for device addressing. You can
> > declare the PCI configuration space to be 256 MB (i.e 28 bit addresses), but
> > kvmtool will only use the bottom 16MB.  
> 
> Thanks for the PCI related explanation.
> 
> I will update PCI config size to 16M.

Actually, looking forward, you should at least reserve 256 MB, since we plan to upgrade kvmtool to PCIe soonish. You might as well keep your current 256MB, but have a comment about it currently using only the first 16MB. This comment could then be deleted after the upgrade.

Cheers,
Andre.

> >
> > Thanks,
> > Alex  
> > >  
> > >>> +#define RISCV_PCI_SIZE               0x50000000ULL
> > >>> +#define RISCV_PCI_MMIO_SIZE  (RISCV_PCI_SIZE - RISCV_PCI_CFG_SIZE)
> > >>> +
> > >>> +#define RISCV_RAM            0x80000000ULL  
> > >> I'm not sure about the reasons for choosing RAM to start at 2GB. For arm we do the
> > >> same, but qemu starts memory at 1GB and this mismatch has caused some issues in
> > >> the past. For example, 32 bit kvm-unit-tests currently do not run under kvmtool
> > >> because the text address is hardcoded to the qemu default value.  
> > > Actually in RISC-V world, it is kind of a un-documented standard that
> > > RAM starts at
> > > 2GB (0x80000000) for both RV32 and RV64. This is true for all existing HW,
> > > QEMU RISC-V virt machine and here. This will be soon explicitly documented in
> > > RISC-V Unix platform spec.
> > >  
> > >> As a more general observation, I know that other architectures (like arm) declare
> > >> the memory layout in hexadecimal numbers, but it might be a better idea to use the
> > >> sizes from include/linux/sizes.h, since it makes the memory layout a lot more
> > >> readable and mistakes are easier to spot.  
> > > Sure, I will try to use linux/sizes.h in next patch version.
> > >  
> > >>> +
> > >>> +#define RISCV_LOMAP_MAX_MEMORY       ((1ULL << 32) - RISCV_RAM)
> > >>> +#define RISCV_HIMAP_MAX_MEMORY       ((1ULL << 40) - RISCV_RAM)
> > >>> +
> > >>> +#if __riscv_xlen == 64
> > >>> +#define RISCV_MAX_MEMORY(kvm)        RISCV_HIMAP_MAX_MEMORY
> > >>> +#elif __riscv_xlen == 32
> > >>> +#define RISCV_MAX_MEMORY(kvm)        RISCV_LOMAP_MAX_MEMORY
> > >>> +#endif
> > >>> +
> > >>> +#define KVM_IOPORT_AREA              RISCV_IOPORT
> > >>> +#define KVM_PCI_CFG_AREA     RISCV_PCI
> > >>> +#define KVM_PCI_MMIO_AREA    (KVM_PCI_CFG_AREA + RISCV_PCI_CFG_SIZE)
> > >>> +#define KVM_VIRTIO_MMIO_AREA RISCV_MMIO
> > >>> +
> > >>> +#define KVM_IOEVENTFD_HAS_PIO        0
> > >>> +
> > >>> +#define KVM_IRQ_OFFSET               0
> > >>> +
> > >>> +#define KVM_VM_TYPE          0
> > >>> +
> > >>> +#define VIRTIO_DEFAULT_TRANS(kvm)    VIRTIO_MMIO
> > >>> +
> > >>> +#define VIRTIO_RING_ENDIAN   VIRTIO_ENDIAN_LE
> > >>> +
> > >>> +struct kvm;
> > >>> +
> > >>> +struct kvm_arch {
> > >>> +};
> > >>> +
> > >>> +static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
> > >>> +{
> > >>> +     u64 limit = KVM_IOPORT_AREA + RISCV_IOPORT_SIZE;
> > >>> +     return phys_addr >= KVM_IOPORT_AREA && phys_addr < limit;
> > >>> +}
> > >>> +
> > >>> +enum irq_type;
> > >>> +
> > >>> +#endif /* KVM__KVM_ARCH_H */
> > >>> diff --git a/riscv/include/kvm/kvm-config-arch.h b/riscv/include/kvm/kvm-config-arch.h
> > >>> new file mode 100644
> > >>> index 0000000..60c7333
> > >>> --- /dev/null
> > >>> +++ b/riscv/include/kvm/kvm-config-arch.h
> > >>> @@ -0,0 +1,9 @@
> > >>> +#ifndef KVM__KVM_CONFIG_ARCH_H
> > >>> +#define KVM__KVM_CONFIG_ARCH_H
> > >>> +
> > >>> +#include "kvm/parse-options.h"
> > >>> +
> > >>> +struct kvm_config_arch {
> > >>> +};
> > >>> +
> > >>> +#endif /* KVM__KVM_CONFIG_ARCH_H */
> > >>> diff --git a/riscv/include/kvm/kvm-cpu-arch.h b/riscv/include/kvm/kvm-cpu-arch.h
> > >>> new file mode 100644
> > >>> index 0000000..09a50e8
> > >>> --- /dev/null
> > >>> +++ b/riscv/include/kvm/kvm-cpu-arch.h
> > >>> @@ -0,0 +1,49 @@
> > >>> +#ifndef KVM__KVM_CPU_ARCH_H
> > >>> +#define KVM__KVM_CPU_ARCH_H
> > >>> +
> > >>> +#include <linux/kvm.h>
> > >>> +#include <pthread.h>
> > >>> +#include <stdbool.h>
> > >>> +
> > >>> +#include "kvm/kvm.h"
> > >>> +
> > >>> +struct kvm;  
> > >> Shouldn't kvm.h already have a definition for struct kvm? Also, the arm
> > >> corresponding header doesn't have the include here, I don't think it's needed
> > >> (unless I'm missing something).  
> > > Sure, I will drop this forward declaration here.
> > >
> > > Regards,
> > > Anup
> > >  
> > >> Thanks,
> > >> Alex  
> > >>> +
> > >>> +struct kvm_cpu {
> > >>> +     pthread_t       thread;
> > >>> +
> > >>> +     unsigned long   cpu_id;
> > >>> +
> > >>> +     struct kvm      *kvm;
> > >>> +     int             vcpu_fd;
> > >>> +     struct kvm_run  *kvm_run;
> > >>> +     struct kvm_cpu_task     *task;
> > >>> +
> > >>> +     u8              is_running;
> > >>> +     u8              paused;
> > >>> +     u8              needs_nmi;
> > >>> +
> > >>> +     struct kvm_coalesced_mmio_ring  *ring;
> > >>> +};
> > >>> +
> > >>> +static inline bool kvm_cpu__emulate_io(struct kvm_cpu *vcpu, u16 port,
> > >>> +                                    void *data, int direction,
> > >>> +                                    int size, u32 count)
> > >>> +{
> > >>> +     return false;
> > >>> +}
> > >>> +
> > >>> +static inline bool kvm_cpu__emulate_mmio(struct kvm_cpu *vcpu, u64 phys_addr,
> > >>> +                                      u8 *data, u32 len, u8 is_write)
> > >>> +{
> > >>> +     if (riscv_addr_in_ioport_region(phys_addr)) {
> > >>> +             int direction = is_write ? KVM_EXIT_IO_OUT : KVM_EXIT_IO_IN;
> > >>> +             u16 port = (phys_addr - KVM_IOPORT_AREA) & USHRT_MAX;
> > >>> +
> > >>> +             return kvm__emulate_io(vcpu, port, data, direction, len, 1);
> > >>> +     }
> > >>> +
> > >>> +     return kvm__emulate_mmio(vcpu, phys_addr, data, len, is_write);
> > >>> +}
> > >>> +
> > >>> +#endif /* KVM__KVM_CPU_ARCH_H */
> > >>> diff --git a/riscv/ioport.c b/riscv/ioport.c
> > >>> new file mode 100644
> > >>> index 0000000..bdd30b6
> > >>> --- /dev/null
> > >>> +++ b/riscv/ioport.c
> > >>> @@ -0,0 +1,11 @@
> > >>> +#include "kvm/ioport.h"
> > >>> +#include "kvm/irq.h"
> > >>> +
> > >>> +void ioport__setup_arch(struct kvm *kvm)
> > >>> +{
> > >>> +}
> > >>> +
> > >>> +void ioport__map_irq(u8 *irq)
> > >>> +{
> > >>> +     *irq = irq__alloc_line();
> > >>> +}
> > >>> diff --git a/riscv/irq.c b/riscv/irq.c
> > >>> new file mode 100644
> > >>> index 0000000..8e605ef
> > >>> --- /dev/null
> > >>> +++ b/riscv/irq.c
> > >>> @@ -0,0 +1,13 @@
> > >>> +#include "kvm/kvm.h"
> > >>> +#include "kvm/kvm-cpu.h"
> > >>> +#include "kvm/irq.h"
> > >>> +
> > >>> +void kvm__irq_line(struct kvm *kvm, int irq, int level)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +void kvm__irq_trigger(struct kvm *kvm, int irq)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> diff --git a/riscv/kvm-cpu.c b/riscv/kvm-cpu.c
> > >>> new file mode 100644
> > >>> index 0000000..e4b8fa5
> > >>> --- /dev/null
> > >>> +++ b/riscv/kvm-cpu.c
> > >>> @@ -0,0 +1,64 @@
> > >>> +#include "kvm/kvm-cpu.h"
> > >>> +#include "kvm/kvm.h"
> > >>> +#include "kvm/virtio.h"
> > >>> +#include "kvm/term.h"
> > >>> +
> > >>> +#include <asm/ptrace.h>
> > >>> +
> > >>> +static int debug_fd;
> > >>> +
> > >>> +void kvm_cpu__set_debug_fd(int fd)
> > >>> +{
> > >>> +     debug_fd = fd;
> > >>> +}
> > >>> +
> > >>> +int kvm_cpu__get_debug_fd(void)
> > >>> +{
> > >>> +     return debug_fd;
> > >>> +}
> > >>> +
> > >>> +struct kvm_cpu *kvm_cpu__arch_init(struct kvm *kvm, unsigned long cpu_id)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +     return NULL;
> > >>> +}
> > >>> +
> > >>> +void kvm_cpu__arch_nmi(struct kvm_cpu *cpu)
> > >>> +{
> > >>> +}
> > >>> +
> > >>> +void kvm_cpu__delete(struct kvm_cpu *vcpu)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +bool kvm_cpu__handle_exit(struct kvm_cpu *vcpu)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +     return false;
> > >>> +}
> > >>> +
> > >>> +void kvm_cpu__show_page_tables(struct kvm_cpu *vcpu)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +void kvm_cpu__reset_vcpu(struct kvm_cpu *vcpu)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +int kvm_cpu__get_endianness(struct kvm_cpu *vcpu)
> > >>> +{
> > >>> +     return VIRTIO_ENDIAN_LE;
> > >>> +}
> > >>> +
> > >>> +void kvm_cpu__show_code(struct kvm_cpu *vcpu)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +void kvm_cpu__show_registers(struct kvm_cpu *vcpu)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> diff --git a/riscv/kvm.c b/riscv/kvm.c
> > >>> new file mode 100644
> > >>> index 0000000..e816ef5
> > >>> --- /dev/null
> > >>> +++ b/riscv/kvm.c
> > >>> @@ -0,0 +1,61 @@
> > >>> +#include "kvm/kvm.h"
> > >>> +#include "kvm/util.h"
> > >>> +#include "kvm/fdt.h"
> > >>> +
> > >>> +#include <linux/kernel.h>
> > >>> +#include <linux/kvm.h>
> > >>> +#include <linux/sizes.h>
> > >>> +
> > >>> +struct kvm_ext kvm_req_ext[] = {
> > >>> +     { DEFINE_KVM_EXT(KVM_CAP_ONE_REG) },
> > >>> +     { 0, 0 },
> > >>> +};
> > >>> +
> > >>> +bool kvm__arch_cpu_supports_vm(void)
> > >>> +{
> > >>> +     /* The KVM capability check is enough. */
> > >>> +     return true;
> > >>> +}
> > >>> +
> > >>> +void kvm__init_ram(struct kvm *kvm)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +void kvm__arch_delete_ram(struct kvm *kvm)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +void kvm__arch_read_term(struct kvm *kvm)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +void kvm__arch_set_cmdline(char *cmdline, bool video)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +}
> > >>> +
> > >>> +bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
> > >>> +                              const char *kernel_cmdline)
> > >>> +{
> > >>> +     /* TODO: */
> > >>> +     return true;
> > >>> +}
> > >>> +
> > >>> +bool kvm__load_firmware(struct kvm *kvm, const char *firmware_filename)
> > >>> +{
> > >>> +     /* TODO: Firmware loading to be supported later. */
> > >>> +     return false;
> > >>> +}
> > >>> +
> > >>> +int kvm__arch_setup_firmware(struct kvm *kvm)
> > >>> +{
> > >>> +     return 0;
> > >>> +}
> > >>> diff --git a/util/update_headers.sh b/util/update_headers.sh
> > >>> index bf87ef6..78eba1f 100755
> > >>> --- a/util/update_headers.sh
> > >>> +++ b/util/update_headers.sh
> > >>> @@ -36,7 +36,7 @@ copy_optional_arch () {
> > >>>       fi
> > >>>  }
> > >>>
> > >>> -for arch in arm arm64 mips powerpc x86
> > >>> +for arch in arm arm64 mips powerpc riscv x86
> > >>>  do
> > >>>       case "$arch" in
> > >>>               arm) KVMTOOL_PATH=arm/aarch32 ;;  

Re: [kvmtool RFC PATCH 3/8] riscv: Implement Guest/VM arch functions

[Anup Patel]

On Wed, Jan 8, 2020 at 6:52 PM Alexandru Elisei
<alexandru.elisei@arm.com> wrote:
>
> Hello,
>
> On 12/25/19 3:00 AM, Anup Patel wrote:
> > This patch implements all kvm__arch_<xyz> Guest/VM arch functions.
> >
> > These functions mostly deal with:
> > 1. Guest/VM RAM initialization
> > 2. Updating terminals on character read
> > 3. Loading kernel and initrd images
> >
> > Firmware loading is not implemented currently because initially we
> > will be booting kernel directly without any bootloader. In future,
> > we will certainly support firmware loading.
> >
> > Signed-off-by: Anup Patel <anup.patel@wdc.com>
> > ---
> >  riscv/include/kvm/kvm-arch.h |  15 +++++
> >  riscv/kvm.c                  | 126 +++++++++++++++++++++++++++++++++--
> >  2 files changed, 135 insertions(+), 6 deletions(-)
> >
> > diff --git a/riscv/include/kvm/kvm-arch.h b/riscv/include/kvm/kvm-arch.h
> > index 7e9c578..b3ec2d6 100644
> > --- a/riscv/include/kvm/kvm-arch.h
> > +++ b/riscv/include/kvm/kvm-arch.h
> > @@ -45,6 +45,21 @@
> >  struct kvm;
> >
> >  struct kvm_arch {
> > +     /*
> > +      * We may have to align the guest memory for virtio, so keep the
> > +      * original pointers here for munmap.
> > +      */
> > +     void    *ram_alloc_start;
> > +     u64     ram_alloc_size;
> > +
> > +     /*
> > +      * Guest addresses for memory layout.
> > +      */
> > +     u64     memory_guest_start;
> > +     u64     kern_guest_start;
> > +     u64     initrd_guest_start;
> > +     u64     initrd_size;
> > +     u64     dtb_guest_start;
> >  };
> >
> >  static inline bool riscv_addr_in_ioport_region(u64 phys_addr)
> > diff --git a/riscv/kvm.c b/riscv/kvm.c
> > index e816ef5..c0d3639 100644
> > --- a/riscv/kvm.c
> > +++ b/riscv/kvm.c
> > @@ -1,5 +1,7 @@
> >  #include "kvm/kvm.h"
> >  #include "kvm/util.h"
> > +#include "kvm/8250-serial.h"
> > +#include "kvm/virtio-console.h"
> >  #include "kvm/fdt.h"
> >
> >  #include <linux/kernel.h>
> > @@ -19,33 +21,145 @@ bool kvm__arch_cpu_supports_vm(void)
> >
> >  void kvm__init_ram(struct kvm *kvm)
> >  {
> > -     /* TODO: */
> > +     int err;
> > +     u64 phys_start, phys_size;
> > +     void *host_mem;
> > +
> > +     phys_start      = RISCV_RAM;
> > +     phys_size       = kvm->ram_size;
> > +     host_mem        = kvm->ram_start;
> > +
> > +     err = kvm__register_ram(kvm, phys_start, phys_size, host_mem);
> > +     if (err)
> > +             die("Failed to register %lld bytes of memory at physical "
> > +                 "address 0x%llx [err %d]", phys_size, phys_start, err);
> > +
> > +     kvm->arch.memory_guest_start = phys_start;
> >  }
> >
> >  void kvm__arch_delete_ram(struct kvm *kvm)
> >  {
> > -     /* TODO: */
> > +     munmap(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size);
> >  }
> >
> >  void kvm__arch_read_term(struct kvm *kvm)
> >  {
> > -     /* TODO: */
> > +     serial8250__update_consoles(kvm);
> > +     virtio_console__inject_interrupt(kvm);
> >  }
> >
> >  void kvm__arch_set_cmdline(char *cmdline, bool video)
> >  {
> > -     /* TODO: */
> >  }
> >
> >  void kvm__arch_init(struct kvm *kvm, const char *hugetlbfs_path, u64 ram_size)
> >  {
> > -     /* TODO: */
> > +     /*
> > +      * Allocate guest memory. We must align our buffer to 64K to
> > +      * correlate with the maximum guest page size for virtio-mmio.
> > +      * If using THP, then our minimal alignment becomes 2M.
> > +      * 2M trumps 64K, so let's go with that.
> > +      */
> > +     kvm->ram_size = min(ram_size, (u64)RISCV_MAX_MEMORY(kvm));
> > +     kvm->arch.ram_alloc_size = kvm->ram_size + SZ_2M;
> > +     kvm->arch.ram_alloc_start = mmap_anon_or_hugetlbfs(kvm, hugetlbfs_path,
> > +                                             kvm->arch.ram_alloc_size);
> > +
> > +     if (kvm->arch.ram_alloc_start == MAP_FAILED)
> > +             die("Failed to map %lld bytes for guest memory (%d)",
> > +                 kvm->arch.ram_alloc_size, errno);
> > +
> > +     kvm->ram_start = (void *)ALIGN((unsigned long)kvm->arch.ram_alloc_start,
> > +                                     SZ_2M);
> > +
> > +     madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
> > +             MADV_MERGEABLE);
> > +
> > +     madvise(kvm->arch.ram_alloc_start, kvm->arch.ram_alloc_size,
> > +             MADV_HUGEPAGE);
> >  }
> >
> > +#define FDT_ALIGN    SZ_4M
> > +#define INITRD_ALIGN 4
> >  bool kvm__arch_load_kernel_image(struct kvm *kvm, int fd_kernel, int fd_initrd,
> >                                const char *kernel_cmdline)
> >  {
> > -     /* TODO: */
> > +     void *pos, *kernel_end, *limit;
> > +     unsigned long guest_addr, kernel_offset;
> > +     ssize_t file_size;
> > +
> > +     /*
> > +      * Linux requires the initrd and dtb to be mapped inside lowmem,
> > +      * so we can't just place them at the top of memory.
> > +      */
> > +     limit = kvm->ram_start + min(kvm->ram_size, (u64)SZ_256M) - 1;
> > +
> > +#if __riscv_xlen == 64
> > +     /* Linux expects to be booted at 2M boundary for RV64 */
> > +     kernel_offset = 0x200000;
> > +#else
> > +     /* Linux expects to be booted at 4M boundary for RV32 */
> > +     kernel_offset = 0x400000;
> > +#endif
> > +
> > +     pos = kvm->ram_start + kernel_offset;
> > +     kvm->arch.kern_guest_start = host_to_guest_flat(kvm, pos);
> > +     file_size = read_file(fd_kernel, pos, limit - pos);
> > +     if (file_size < 0) {
> > +             if (errno == ENOMEM)
> > +                     die("kernel image too big to fit in guest memory.");
> > +
> > +             die_perror("kernel read");
> > +     }
> > +     kernel_end = pos + file_size;
> > +     pr_debug("Loaded kernel to 0x%llx (%zd bytes)",
> > +              kvm->arch.kern_guest_start, file_size);
> > +
> > +     /* Place FDT just after kernel at FDT_ALIGN address */
> > +     pos = kernel_end + FDT_ALIGN;
> > +     guest_addr = ALIGN(host_to_guest_flat(kvm, pos), FDT_ALIGN);
> > +     pos = guest_flat_to_host(kvm, guest_addr);
> > +     if (pos < kernel_end)
> > +             die("fdt overlaps with kernel image.");
> > +
> > +     kvm->arch.dtb_guest_start = guest_addr;
> > +     pr_debug("Placing fdt at 0x%llx - 0x%llx",
> > +              kvm->arch.dtb_guest_start,
> > +              host_to_guest_flat(kvm, limit));
> > +     limit = pos;
>
> This doesn't look right. pos points to the start of the DTB, not to the top of
> free memory. You probably want to delete the line.

Actually, we are trying to place INITRD between kernel and DTB.

Having looked at it again, I think this is not right approach we should
place INITRD after DTB otherwise a bigger INITRD can easily get
corrupted by DTB creation.

>
> > +
> > +     /* ... and finally the initrd, if we have one. */
> > +     if (fd_initrd != -1) {
> > +             struct stat sb;
> > +             unsigned long initrd_start;
> > +
> > +             if (fstat(fd_initrd, &sb))
> > +                     die_perror("fstat");
> > +
> > +             pos -= (sb.st_size + INITRD_ALIGN);
>
> This too doesn't look right. You're overwriting the DTB and most likely the kernel
> with the initrd.

Sure, I will fix the placement of DTB and INITRD.

Regards,
Anup

>
> Thanks,
> Alex
> > +             guest_addr = ALIGN(host_to_guest_flat(kvm, pos), INITRD_ALIGN);
> > +             pos = guest_flat_to_host(kvm, guest_addr);
> > +             if (pos < kernel_end)
> > +                     die("initrd overlaps with kernel image.");
> > +
> > +             initrd_start = guest_addr;
> > +             file_size = read_file(fd_initrd, pos, limit - pos);
> > +             if (file_size == -1) {
> > +                     if (errno == ENOMEM)
> > +                             die("initrd too big to fit in guest memory.");
> > +
> > +                     die_perror("initrd read");
> > +             }
> > +
> > +             kvm->arch.initrd_guest_start = initrd_start;
> > +             kvm->arch.initrd_size = file_size;
> > +             pr_debug("Loaded initrd to 0x%llx (%llu bytes)",
> > +                      kvm->arch.initrd_guest_start,
> > +                      kvm->arch.initrd_size);
> > +     } else {
> > +             kvm->arch.initrd_size = 0;
> > +     }
> > +
> >       return true;
> >  }
> >