[PATCH v2 01/10] KVM: add commentary for kvm_debug_exit_arch struct

[PATCH v2 01/10] KVM: add commentary for kvm_debug_exit_arch struct

[Alex Bennée]

Bring into line with the commentary for the other structures and their
KVM_EXIT_* cases.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

---

v2
  - add comments for other exit types

diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 8055706..5eedf84 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -226,6 +226,7 @@ struct kvm_run {
 			__u32 count;
 			__u64 data_offset; /* relative to kvm_run start */
 		} io;
+		/* KVM_EXIT_DEBUG */
 		struct {
 			struct kvm_debug_exit_arch arch;
 		} debug;
@@ -274,6 +275,7 @@ struct kvm_run {
 			__u32 data;
 			__u8  is_write;
 		} dcr;
+		/* KVM_EXIT_INTERNAL_ERROR */
 		struct {
 			__u32 suberror;
 			/* Available with KVM_CAP_INTERNAL_ERROR_DATA: */
@@ -284,6 +286,7 @@ struct kvm_run {
 		struct {
 			__u64 gprs[32];
 		} osi;
+		/* KVM_EXIT_PAPR_HCALL */
 		struct {
 			__u64 nr;
 			__u64 ret;
-- 
2.3.4

[PATCH v2 02/10] KVM: define common __KVM_GUESTDBG_USE_SW/HW_BP values

[Alex Bennée]

Currently x86, powerpc and soon arm64 use the same two architecture
specific bits for guest debug support for software and hardware
breakpoints. This makes the shared values explicit while leaving the
gate open for another architecture to use some other value if they
really really want to.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
index ab4d473..1731569 100644
--- a/arch/powerpc/include/uapi/asm/kvm.h
+++ b/arch/powerpc/include/uapi/asm/kvm.h
@@ -310,8 +310,8 @@ struct kvm_guest_debug_arch {
  * and upper 16 bits are architecture specific. Architecture specific defines
  * that ioctl is for setting hardware breakpoint or software breakpoint.
  */
-#define KVM_GUESTDBG_USE_SW_BP		0x00010000
-#define KVM_GUESTDBG_USE_HW_BP		0x00020000
+#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
+#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
 
 /* definition of registers in kvm_run */
 struct kvm_sync_regs {
diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
index d7dcef5..1438202 100644
--- a/arch/x86/include/uapi/asm/kvm.h
+++ b/arch/x86/include/uapi/asm/kvm.h
@@ -250,8 +250,8 @@ struct kvm_debug_exit_arch {
 	__u64 dr7;
 };
 
-#define KVM_GUESTDBG_USE_SW_BP		0x00010000
-#define KVM_GUESTDBG_USE_HW_BP		0x00020000
+#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
+#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
 #define KVM_GUESTDBG_INJECT_DB		0x00040000
 #define KVM_GUESTDBG_INJECT_BP		0x00080000
 
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 5eedf84..ce2db14 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -525,8 +525,16 @@ struct kvm_s390_irq {
 
 /* for KVM_SET_GUEST_DEBUG */
 
-#define KVM_GUESTDBG_ENABLE		0x00000001
-#define KVM_GUESTDBG_SINGLESTEP		0x00000002
+#define KVM_GUESTDBG_ENABLE		(1 << 0)
+#define KVM_GUESTDBG_SINGLESTEP	(1 << 1)
+
+/*
+ * Architecture specific stuff uses the top 16 bits of the field,
+ * however there is some shared commonality for the common cases
+ */
+#define __KVM_GUESTDBG_USE_SW_BP	(1 << 16)
+#define __KVM_GUESTDBG_USE_HW_BP	(1 << 17)
+
 
 struct kvm_guest_debug {
 	__u32 control;
-- 
2.3.4

[PATCH v2 03/10] KVM: arm: guest debug, define API headers

[Alex Bennée]

This commit defines the API headers for guest debugging. There are two
architecture specific debug structures:

  - kvm_guest_debug_arch, allows us to pass in HW debug registers
  - kvm_debug_exit_arch, signals the exact debug exit and pc

The type of debugging being used is control by the architecture specific
control bits of the kvm_guest_debug->control flags in the ioctl
structure.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

---
v2
   - expose hsr and pc directly to user-space

diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 3ef77a4..6ee70a0 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -100,10 +100,24 @@ struct kvm_sregs {
 struct kvm_fpu {
 };
 
+/*
+ * See ARM ARM D7.3: Debug Registers
+ *
+ * The control registers are architecturally defined as 32 bits but are
+ * stored as 64 bit values along side the value registers and aligned
+ * with the rest 64 bit registers in the normal CPU context.
+ */
+#define KVM_ARM_NDBG_REGS 16
 struct kvm_guest_debug_arch {
+	__u64 dbg_bcr[KVM_ARM_NDBG_REGS];
+	__u64 dbg_bvr[KVM_ARM_NDBG_REGS];
+	__u64 dbg_wcr[KVM_ARM_NDBG_REGS];
+	__u64 dbg_wvr[KVM_ARM_NDBG_REGS];
 };
 
 struct kvm_debug_exit_arch {
+	__u64 pc;
+	__u32 hsr;
 };
 
 struct kvm_sync_regs {
@@ -207,4 +221,11 @@ struct kvm_arch_memory_slot {
 
 #endif
 
+/*
+ * Architecture related debug defines - upper 16 bits of
+ * kvm_guest_debug->control
+ */
+#define KVM_GUESTDBG_USE_SW_BP	        __KVM_GUESTDBG_USE_SW_BP
+#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
+
 #endif /* __ARM_KVM_H__ */
-- 
2.3.4

[PATCH v2 04/10] KVM: arm: guest debug, add stub KVM_SET_GUEST_DEBUG ioctl

[Alex Bennée]

This commit adds a stub function to support the KVM_SET_GUEST_DEBUG
ioctl. Currently any operation flag will return EINVAL. Actual
functionality will be added with further patches.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>.

---
v2
  - simplified form of the ioctl (stuff will go into setup_debug)

diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index b112efc..06c5064 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -2604,7 +2604,7 @@ handled.
 4.87 KVM_SET_GUEST_DEBUG
 
 Capability: KVM_CAP_SET_GUEST_DEBUG
-Architectures: x86, s390, ppc
+Architectures: x86, s390, ppc, arm64
 Type: vcpu ioctl
 Parameters: struct kvm_guest_debug (in)
 Returns: 0 on success; -1 on error
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 5560f74..445933d 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -183,6 +183,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_ARM_PSCI:
 	case KVM_CAP_ARM_PSCI_0_2:
 	case KVM_CAP_READONLY_MEM:
+	case KVM_CAP_SET_GUEST_DEBUG:
 		r = 1;
 		break;
 	case KVM_CAP_COALESCED_MMIO:
@@ -303,10 +304,21 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 	kvm_arm_set_running_vcpu(NULL);
 }
 
+#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)
+
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
 {
-	return -EINVAL;
+	if (dbg->control & KVM_GUESTDBG_ENABLE) {
+		if (dbg->control & ~KVM_GUESTDBG_VALID)
+			return -EINVAL;
+
+		vcpu->guest_debug = dbg->control;
+	} else {
+		/* If not enabled clear all flags */
+		vcpu->guest_debug = 0;
+	}
+	return 0;
 }
 
 
-- 
2.3.4

[PATCH v2 05/10] KVM: arm: introduce kvm_arch_setup/clear_debug()

[Alex Bennée]

This is a precursor for later patches which will need to do more to
setup debug state before entering the hyp.S switch code. The existing
functionality for setting mdcr_el2 has been moved out of hyp.S and now
uses the value kept in vcpu->arch.mdcr_el2.

This also moves the conditional setting of the TDA bit from the hyp code
into the C code.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

 create mode 100644 arch/arm64/kvm/debug.c

diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index 41008cd..8c01c97 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -242,5 +242,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
 static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
+static inline void kvm_arch_setup_debug(struct kvm_vcpu *vcpu) {}
+static inline void kvm_arch_clear_debug(struct kvm_vcpu *vcpu) {}
 
 #endif /* __ARM_KVM_HOST_H__ */
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 445933d..7ea8b0e 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -523,6 +523,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 
 		kvm_vgic_flush_hwstate(vcpu);
 		kvm_timer_flush_hwstate(vcpu);
+		kvm_arch_setup_debug(vcpu);
 
 		local_irq_disable();
 
@@ -569,6 +570,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
 		 * Back from guest
 		 *************************************************************/
 
+		kvm_arch_clear_debug(vcpu);
 		kvm_timer_sync_hwstate(vcpu);
 		kvm_vgic_sync_hwstate(vcpu);
 
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 8ac3c70..0631840 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -101,6 +101,7 @@ struct kvm_vcpu_arch {
 
 	/* HYP configuration */
 	u64 hcr_el2;
+	u32 mdcr_el2;
 
 	/* Exception Information */
 	struct kvm_vcpu_fault_info fault;
@@ -257,4 +258,7 @@ static inline void kvm_arch_sync_events(struct kvm *kvm) {}
 static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
 static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
 
+void kvm_arch_setup_debug(struct kvm_vcpu *vcpu);
+void kvm_arch_clear_debug(struct kvm_vcpu *vcpu);
+
 #endif /* __ARM64_KVM_HOST_H__ */
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index f7fa65d..cd06209 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -122,6 +122,7 @@ int main(void)
   DEFINE(VCPU_HPFAR_EL2,	offsetof(struct kvm_vcpu, arch.fault.hpfar_el2));
   DEFINE(VCPU_DEBUG_FLAGS,	offsetof(struct kvm_vcpu, arch.debug_flags));
   DEFINE(VCPU_HCR_EL2,		offsetof(struct kvm_vcpu, arch.hcr_el2));
+  DEFINE(VCPU_MDCR_EL2,	offsetof(struct kvm_vcpu, arch.mdcr_el2));
   DEFINE(VCPU_IRQ_LINES,	offsetof(struct kvm_vcpu, arch.irq_lines));
   DEFINE(VCPU_HOST_CONTEXT,	offsetof(struct kvm_vcpu, arch.host_cpu_context));
   DEFINE(VCPU_TIMER_CNTV_CTL,	offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
index 4e6e09e..6796d4a 100644
--- a/arch/arm64/kvm/Makefile
+++ b/arch/arm64/kvm/Makefile
@@ -17,7 +17,7 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o
 
 kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o
 kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
-kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
+kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o
 
 kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
 kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
new file mode 100644
index 0000000..8a29d0b
--- /dev/null
+++ b/arch/arm64/kvm/debug.c
@@ -0,0 +1,58 @@
+/*
+ * Debug and Guest Debug support
+ *
+ * Copyright (C) 2015 - Linaro Ltd
+ * Author: Alex Bennée <alex.bennee@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/kvm_host.h>
+
+#include <asm/kvm_arm.h>
+#include <asm/kvm_host.h>
+
+/**
+ * kvm_arch_setup_debug - set-up debug related stuff
+ *
+ * @vcpu:	the vcpu pointer
+ *
+ * This is called before each entry in to the hypervisor to setup any
+ * debug related registers. Currently this just ensures we will trap
+ * access to:
+ *  - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR)
+ *  - Debug ROM Address (MDCR_EL2_TDRA)
+ *  - Power down debug registers (MDCR_EL2_TDOSA)
+ *
+ * Additionally the hypervisor lazily saves/restores the debug
+ * register state. If it is not currently doing so (arch.debug_flags)
+ * then we also need to ensure we trap if the guest messes with them
+ * so we know we need to save them.
+ */
+
+void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
+{
+	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
+	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
+
+	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
+		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
+	else
+		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
+
+}
+
+void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
+{
+	/* Nothing to do yet */
+}
diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
index 5befd01..be92bfe1 100644
--- a/arch/arm64/kvm/hyp.S
+++ b/arch/arm64/kvm/hyp.S
@@ -768,17 +768,8 @@
 	mov	x2, #(1 << 15)	// Trap CP15 Cr=15
 	msr	hstr_el2, x2
 
-	mrs	x2, mdcr_el2
-	and	x2, x2, #MDCR_EL2_HPMN_MASK
-	orr	x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
-	orr	x2, x2, #(MDCR_EL2_TDRA | MDCR_EL2_TDOSA)
-
-	// Check for KVM_ARM64_DEBUG_DIRTY, and set debug to trap
-	// if not dirty.
-	ldr	x3, [x0, #VCPU_DEBUG_FLAGS]
-	tbnz	x3, #KVM_ARM64_DEBUG_DIRTY_SHIFT, 1f
-	orr	x2, x2,  #MDCR_EL2_TDA
-1:
+	// Monitor Debug Config - see kvm_arch_setup_debug()
+	ldr	x2, [x0, #VCPU_MDCR_EL2]
 	msr	mdcr_el2, x2
 .endm
 
-- 
2.3.4

[PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Alex Bennée]

This adds support for SW breakpoints inserted by userspace.

We do this by trapping all BKPT exceptions in the
hypervisor (MDCR_EL2_TDE). The kvm_debug_exit_arch carries the address
of the exception. If user-space doesn't know of the breakpoint then we
have a guest inserted breakpoint and the hypervisor needs to start again
and deliver the exception to guest.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

---
v2
  - update to use new exit struct
  - tweak for C setup
  - do our setup in debug_setup/clear code
  - fixed up comments

diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index 06c5064..17d4f9c 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -2626,7 +2626,7 @@ when running. Common control bits are:
 The top 16 bits of the control field are architecture specific control
 flags which can include the following:
 
-  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86]
+  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
   - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
   - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
   - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 7ea8b0e..d3bc8dc 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -304,7 +304,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 	kvm_arm_set_running_vcpu(NULL);
 }
 
-#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)
+#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)
 
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
index 8a29d0b..cff0475 100644
--- a/arch/arm64/kvm/debug.c
+++ b/arch/arm64/kvm/debug.c
@@ -45,11 +45,18 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
 	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
 
+	/* Trap debug register access? */
 	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
 		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
 	else
 		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
 
+	/* Trap breakpoints? */
+	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
+		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
+	else
+		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
+
 }
 
 void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 524fa25..ed1bbb4 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -82,6 +82,37 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	return 1;
 }
 
+/**
+ * kvm_handle_debug_exception - handle a debug exception instruction
+ *
+ * @vcpu:	the vcpu pointer
+ * @run:	access to the kvm_run structure for results
+ *
+ * We route all debug exceptions through the same handler as we
+ * just need to report the PC and the HSR values to userspace.
+ * Userspace may decide to re-inject the exception and deliver it to
+ * the guest if it wasn't for the host to deal with.
+ */
+static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
+{
+	u32 hsr = kvm_vcpu_get_hsr(vcpu);
+
+	run->exit_reason = KVM_EXIT_DEBUG;
+	run->debug.arch.hsr = hsr;
+
+	switch (hsr >> ESR_ELx_EC_SHIFT) {
+	case ESR_ELx_EC_BKPT32:
+	case ESR_ELx_EC_BRK64:
+		run->debug.arch.pc = *vcpu_pc(vcpu);
+		break;
+	default:
+		kvm_err("%s: un-handled case hsr: %#08x\n",
+			__func__, (unsigned int) hsr);
+		break;
+	}
+	return 0;
+}
+
 static exit_handle_fn arm_exit_handlers[] = {
 	[ESR_ELx_EC_WFx]	= kvm_handle_wfx,
 	[ESR_ELx_EC_CP15_32]	= kvm_handle_cp15_32,
@@ -96,6 +127,8 @@ static exit_handle_fn arm_exit_handlers[] = {
 	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
 	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
 	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
+	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
+	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
 };
 
 static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
-- 
2.3.4

[PATCH v2 07/10] KVM: arm64: guest debug, add support for single-step

[Alex Bennée]

This adds support for single-stepping the guest. As userspace can and
will manipulate guest registers before restarting any tweaking of the
registers has to occur just before control is passed back to the guest.
Furthermore while guest debugging is in effect we need to squash the
ability of the guest to single-step itself as we have no easy way of
re-entering the guest after the exception has been delivered to the
hypervisor.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

---
v2
  - Move pstate/mdscr manipulation into C
  - don't export guest_debug to assembly
  - add accessor for saved_debug regs
  - tweak save/restore of mdscr_el1

diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index d3bc8dc..c1ed8cb 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -304,7 +304,21 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 	kvm_arm_set_running_vcpu(NULL);
 }
 
-#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)
+#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE |    \
+			    KVM_GUESTDBG_USE_SW_BP | \
+			    KVM_GUESTDBG_SINGLESTEP)
+
+/**
+ * kvm_arch_vcpu_ioctl_set_guest_debug - Setup guest debugging
+ * @kvm:	pointer to the KVM struct
+ * @kvm_guest_debug: the ioctl data buffer
+ *
+ * This sets up the VM for guest debugging. Care has to be taken when
+ * manipulating guest registers as these will be set/cleared by the
+ * hyper-visor controller, typically before each kvm_run event. As a
+ * result modification of the guest registers needs to take place
+ * after they have been restored in the hyp.S trampoline code.
+ */
 
 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 					struct kvm_guest_debug *dbg)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 0631840..6a33647 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -121,6 +121,13 @@ struct kvm_vcpu_arch {
 	 * here.
 	 */
 
+	/* Registers pre any guest debug manipulations */
+	struct {
+		u32	pstate_ss_bit;
+		u32	mdscr_el1_bits;
+
+	} debug_saved_regs;
+
 	/* Don't run the guest */
 	bool pause;
 
@@ -143,6 +150,7 @@ struct kvm_vcpu_arch {
 
 #define vcpu_gp_regs(v)		(&(v)->arch.ctxt.gp_regs)
 #define vcpu_sys_reg(v,r)	((v)->arch.ctxt.sys_regs[(r)])
+#define vcpu_debug_saved_reg(v, r) ((v)->arch.debug_saved_regs.r)
 /*
  * CP14 and CP15 live in the same array, as they are backed by the
  * same system registers.
diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
index cff0475..b32362c 100644
--- a/arch/arm64/kvm/debug.c
+++ b/arch/arm64/kvm/debug.c
@@ -19,8 +19,16 @@
 
 #include <linux/kvm_host.h>
 
+#include <asm/debug-monitors.h>
+#include <asm/kvm_asm.h>
 #include <asm/kvm_arm.h>
 #include <asm/kvm_host.h>
+#include <asm/kvm_emulate.h>
+
+/* These are the bits of MDSCR_EL1 we may mess with */
+#define MDSCR_EL1_DEBUG_BITS	(DBG_MDSCR_SS | \
+				DBG_MDSCR_KDE | \
+				DBG_MDSCR_MDE)
 
 /**
  * kvm_arch_setup_debug - set-up debug related stuff
@@ -51,15 +59,46 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 	else
 		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
 
-	/* Trap breakpoints? */
-	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
+	/* Is Guest debugging in effect? */
+	if (vcpu->guest_debug) {
 		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
-	else
-		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
 
+		/* Save pstate/mdscr */
+		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
+			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
+		vcpu_debug_saved_reg(vcpu, mdscr_el1_bits) =
+			vcpu_sys_reg(vcpu, MDSCR_EL1) & MDSCR_EL1_DEBUG_BITS;
+		/*
+		 * Single Step (ARM ARM D2.12.3 The software step state
+		 * machine)
+		 *
+		 * If we are doing Single Step we need to manipulate
+		 * MDSCR_EL1.SS and PSTATE.SS. If not we need to
+		 * suppress the guest from messing with it.
+		 */
+		if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
+			*vcpu_cpsr(vcpu) |=  DBG_SPSR_SS;
+			vcpu_sys_reg(vcpu, MDSCR_EL1) |= DBG_MDSCR_SS;
+		} else {
+			*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
+			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
+		}
+
+	} else {
+		/* Debug operations can go straight to the guest */
+		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
+	}
 }
 
 void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
 {
-	/* Nothing to do yet */
+	if (vcpu->guest_debug) {
+		/* Restore pstate/mdscr bits we may have messed with */
+		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
+		*vcpu_cpsr(vcpu) |= vcpu_debug_saved_reg(vcpu, pstate_ss_bit);
+
+		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
+		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
+			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
+	}
 }
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index ed1bbb4..16accae 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -101,6 +101,7 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	run->debug.arch.hsr = hsr;
 
 	switch (hsr >> ESR_ELx_EC_SHIFT) {
+	case ESR_ELx_EC_SOFTSTP_LOW:
 	case ESR_ELx_EC_BKPT32:
 	case ESR_ELx_EC_BRK64:
 		run->debug.arch.pc = *vcpu_pc(vcpu);
@@ -127,6 +128,7 @@ static exit_handle_fn arm_exit_handlers[] = {
 	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
 	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
 	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
+	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
 	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
 	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
 };
-- 
2.3.4

[PATCH v2 08/10] KVM: arm64: guest debug, HW assisted debug support

[Alex Bennée]

This adds support for userspace to control the HW debug registers for
guest debug. We'll only copy the $ARCH defined number across as that is
all that hyp.S will use anyway. I've moved some helper functions into
the hw_breakpoint.h header for re-use.

As with single step we need to tweak the guest registers to enable the
exceptions so we need to save and restore those bits.

Two new capabilities have been added to the KVM_EXTENSION ioctl to allow
userspace to query the number of hardware break and watch points
available on the host hardware.

As QEMU tests for watchpoints based on the address and not the PC we
also need to export the value of far_el2 to userspace.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

---
v2
   - switched to C setup
   - replace host debug registers directly into context
   - minor tweak to api docs
   - setup right register for debug
   - add FAR_EL2 to debug exit structure
   - add support fro trapping debug register access

diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
index 17d4f9c..ac34093 100644
--- a/Documentation/virtual/kvm/api.txt
+++ b/Documentation/virtual/kvm/api.txt
@@ -2627,7 +2627,7 @@ The top 16 bits of the control field are architecture specific control
 flags which can include the following:
 
   - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
-  - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
+  - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390, arm64]
   - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
   - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
   - KVM_GUESTDBG_EXIT_PENDING:  trigger an immediate guest exit [s390]
@@ -2642,6 +2642,10 @@ updated to the correct (supplied) values.
 The second part of the structure is architecture specific and
 typically contains a set of debug registers.
 
+For arm64 the number of debug registers is implementation defined and
+can be determined by querying the KVM_CAP_GUEST_DEBUG_HW_BPS and
+KVM_CAP_GUEST_DEBUG_HW_WPS capabilities.
+
 When debug events exit the main run loop with the reason
 KVM_EXIT_DEBUG with the kvm_debug_exit_arch part of the kvm_run
 structure containing architecture specific debug information.
diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index c1ed8cb..a286026 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -306,6 +306,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 
 #define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE |    \
 			    KVM_GUESTDBG_USE_SW_BP | \
+			    KVM_GUESTDBG_USE_HW_BP | \
 			    KVM_GUESTDBG_SINGLESTEP)
 
 /**
@@ -328,6 +329,26 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
 			return -EINVAL;
 
 		vcpu->guest_debug = dbg->control;
+
+		/* Hardware assisted Break and Watch points */
+		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
+			int nb = get_num_brps();
+			int nw = get_num_wrps();
+
+			/* Copy across up to IMPDEF debug registers to our
+			 * shadow copy in the vcpu structure. The debug code
+			 * will then set them up before we re-enter the guest.
+			 */
+			memcpy(vcpu->arch.guest_debug_regs.dbg_bcr,
+				dbg->arch.dbg_bcr, sizeof(__u64)*nb);
+			memcpy(vcpu->arch.guest_debug_regs.dbg_bvr,
+				dbg->arch.dbg_bvr, sizeof(__u64)*nb);
+			memcpy(vcpu->arch.guest_debug_regs.dbg_wcr,
+				dbg->arch.dbg_wcr, sizeof(__u64)*nw);
+			memcpy(vcpu->arch.guest_debug_regs.dbg_wvr,
+				dbg->arch.dbg_wvr, sizeof(__u64)*nw);
+		}
+
 	} else {
 		/* If not enabled clear all flags */
 		vcpu->guest_debug = 0;
diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
index 52b484b..c450552 100644
--- a/arch/arm64/include/asm/hw_breakpoint.h
+++ b/arch/arm64/include/asm/hw_breakpoint.h
@@ -130,6 +130,18 @@ static inline void ptrace_hw_copy_thread(struct task_struct *task)
 }
 #endif
 
+/* Determine number of BRP registers available. */
+static inline int get_num_brps(void)
+{
+	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
+}
+
+/* Determine number of WRP registers available. */
+static inline int get_num_wrps(void)
+{
+	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
+}
+
 extern struct pmu perf_ops_bp;
 
 #endif	/* __KERNEL__ */
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 6a33647..2c359c9 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -106,8 +106,9 @@ struct kvm_vcpu_arch {
 	/* Exception Information */
 	struct kvm_vcpu_fault_info fault;
 
-	/* Debug state */
+	/* Guest debug state */
 	u64 debug_flags;
+	struct kvm_guest_debug_arch guest_debug_regs;
 
 	/* Pointer to host CPU context */
 	kvm_cpu_context_t *host_cpu_context;
@@ -126,6 +127,7 @@ struct kvm_vcpu_arch {
 		u32	pstate_ss_bit;
 		u32	mdscr_el1_bits;
 
+		struct kvm_guest_debug_arch debug_regs;
 	} debug_saved_regs;
 
 	/* Don't run the guest */
diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
index 6ee70a0..73f21e4 100644
--- a/arch/arm64/include/uapi/asm/kvm.h
+++ b/arch/arm64/include/uapi/asm/kvm.h
@@ -118,6 +118,7 @@ struct kvm_guest_debug_arch {
 struct kvm_debug_exit_arch {
 	__u64 pc;
 	__u32 hsr;
+	__u64 far;	/* used for watchpoints */
 };
 
 struct kvm_sync_regs {
diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
index 98bbe06..923be44 100644
--- a/arch/arm64/kernel/hw_breakpoint.c
+++ b/arch/arm64/kernel/hw_breakpoint.c
@@ -49,18 +49,6 @@ static DEFINE_PER_CPU(int, stepping_kernel_bp);
 static int core_num_brps;
 static int core_num_wrps;
 
-/* Determine number of BRP registers available. */
-static int get_num_brps(void)
-{
-	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
-}
-
-/* Determine number of WRP registers available. */
-static int get_num_wrps(void)
-{
-	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
-}
-
 int hw_breakpoint_slots(int type)
 {
 	/*
diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
index b32362c..3b368f3 100644
--- a/arch/arm64/kvm/debug.c
+++ b/arch/arm64/kvm/debug.c
@@ -50,14 +50,19 @@
 
 void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 {
+	bool trap_debug = false;
+
 	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
 	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
 
-	/* Trap debug register access? */
+	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
+
+	/*
+	 * If we are not treating debug registers are dirty we need
+	 * to trap if the guest starts accessing them.
+	 */
 	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
-		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
-	else
-		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
+		trap_debug = true;
 
 	/* Is Guest debugging in effect? */
 	if (vcpu->guest_debug) {
@@ -84,10 +89,69 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
 		}
 
+		/*
+		 * HW Break/Watch points
+		 */
+		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
+			int nb = get_num_brps();
+			int nw = get_num_wrps();
+			struct kvm_guest_debug_arch *saved, *host;
+
+			vcpu_sys_reg(vcpu, MDSCR_EL1) |=
+				(DBG_MDSCR_KDE|DBG_MDSCR_MDE);
+
+			/*
+			 * First we need to make a copy of the guest
+			 * debug registers before we wipe them out
+			 * with the ones we want to use.
+			 */
+			saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
+			host = &vcpu->arch.guest_debug_regs;
+
+			/* Save guest values */
+			memcpy(&saved->dbg_bcr,
+			       &vcpu_sys_reg(vcpu, DBGBCR0_EL1),
+			       sizeof(__u64)*nb);
+			memcpy(&saved->dbg_bvr,
+			       &vcpu_sys_reg(vcpu, DBGBVR0_EL1),
+			       sizeof(__u64)*nb);
+			memcpy(&saved->dbg_wcr,
+			       &vcpu_sys_reg(vcpu, DBGWCR0_EL1),
+			       sizeof(__u64)*nw);
+			memcpy(&saved->dbg_wvr,
+			       &vcpu_sys_reg(vcpu, DBGWVR0_EL1),
+			       sizeof(__u64)*nw);
+
+			/* Copy in host values */
+			memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1),
+			       &host->dbg_bcr,
+			       sizeof(__u64)*nb);
+			memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1),
+			       &host->dbg_bvr,
+			       sizeof(__u64)*nb);
+			memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1),
+			       &host->dbg_wcr,
+			       sizeof(__u64)*nw);
+			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
+			       &host->dbg_wvr,
+			       sizeof(__u64)*nw);
+
+			/* Make sure hyp.S copies them in/out */
+			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+			/* Also track guest changes */
+			trap_debug = true;
+		}
+
 	} else {
 		/* Debug operations can go straight to the guest */
 		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
 	}
+
+	/* Trap debug register access? */
+	if (trap_debug)
+		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
+	else
+		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
 }
 
 void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
@@ -100,5 +164,31 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
 		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
 		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
 			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
+
+		/*
+		 * If we were using HW debug we need to restore the
+		 * values the guest had set them up with
+		 */
+		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
+			struct kvm_guest_debug_arch *regs;
+			int nb = get_num_brps();
+			int nw = get_num_wrps();
+
+			regs = &vcpu_debug_saved_reg(vcpu, debug_regs);
+
+			/* Restore the saved debug register values */
+			memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1),
+			       &regs->dbg_bcr,
+			       sizeof(__u64)*nb);
+			memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1),
+			       &regs->dbg_bvr,
+			       sizeof(__u64)*nb);
+			memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1),
+			       &regs->dbg_wcr,
+			       sizeof(__u64)*nw);
+			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
+			       &regs->dbg_wvr,
+			       sizeof(__u64)*nw);
+		}
 	}
 }
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 16accae..460a1aa 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -101,7 +101,11 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
 	run->debug.arch.hsr = hsr;
 
 	switch (hsr >> ESR_ELx_EC_SHIFT) {
+	case ESR_ELx_EC_WATCHPT_LOW:
+		run->debug.arch.far = vcpu->arch.fault.far_el2;
+		/* fall through */
 	case ESR_ELx_EC_SOFTSTP_LOW:
+	case ESR_ELx_EC_BREAKPT_LOW:
 	case ESR_ELx_EC_BKPT32:
 	case ESR_ELx_EC_BRK64:
 		run->debug.arch.pc = *vcpu_pc(vcpu);
@@ -129,6 +133,8 @@ static exit_handle_fn arm_exit_handlers[] = {
 	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
 	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
 	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
+	[ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug,
+	[ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
 	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
 	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
 };
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index 0b43265..c2732c7 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -64,6 +64,12 @@ int kvm_arch_dev_ioctl_check_extension(long ext)
 	case KVM_CAP_ARM_EL1_32BIT:
 		r = cpu_has_32bit_el1();
 		break;
+	case KVM_CAP_GUEST_DEBUG_HW_BPS:
+		r = get_num_brps();
+		break;
+	case KVM_CAP_GUEST_DEBUG_HW_WPS:
+		r  = get_num_wrps();
+		break;
 	default:
 		r = 0;
 	}
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index c370b40..be9b188 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -196,16 +196,49 @@ static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
  * - If the dirty bit is set, save guest registers, restore host
  *   registers and clear the dirty bit. This ensure that the host can
  *   now use the debug registers.
+ *
+ * We also use this mechanism to set-up the debug registers for guest
+ * debugging. If this is the case we want to ensure the guest sees
+ * the right versions of the registers - even if they are not going to
+ * be effective while guest debug is using HW debug.
+ *
  */
+
 static bool trap_debug_regs(struct kvm_vcpu *vcpu,
 			    const struct sys_reg_params *p,
 			    const struct sys_reg_desc *r)
 {
-	if (p->is_write) {
-		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
-		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
+		struct kvm_guest_debug_arch *saved;
+		__u64 *val;
+
+		saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
+
+		if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
+			val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
+		else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
+			val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
+		else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
+			val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
+		else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
+			val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
+		else {
+			kvm_err("Bad register index %d\n", r->reg);
+			return false;
+		}
+
+		if (p->is_write)
+			*val = *vcpu_reg(vcpu, p->Rt);
+		else
+			*vcpu_reg(vcpu, p->Rt) = *val;
+
 	} else {
-		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
+		if (p->is_write) {
+			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
+			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+		} else {
+			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
+		}
 	}
 
 	return true;
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index ce2db14..0e48c0d 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -771,6 +771,8 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_PPC_ENABLE_HCALL 104
 #define KVM_CAP_CHECK_EXTENSION_VM 105
 #define KVM_CAP_S390_USER_SIGP 106
+#define KVM_CAP_GUEST_DEBUG_HW_BPS 107
+#define KVM_CAP_GUEST_DEBUG_HW_WPS 108
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
-- 
2.3.4

[PATCH v2 09/10] KVM: arm64: trap nested debug register access

[Alex Bennée]

When we are using the hardware registers for guest debug we need to deal
with the guests access to them. There is already a mechanism for dealing
with these accesses so we build on top of that.

  - mdscr_el1_bits is renamed as we save the whole register
  - any access to mdscr_el1 is now stored in the mirror location
  - if we are using HW assisted debug we do the same with DBG[WB][CV]R

There is one register (MDCCINT_EL1) which guest debug doesn't care about
so this behaves as before.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 2c359c9..3d32d45 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -122,10 +122,13 @@ struct kvm_vcpu_arch {
 	 * here.
 	 */
 
-	/* Registers pre any guest debug manipulations */
+	/* Registers before any guest debug manipulations. These
+	 * shadow registers are updated by the kvm_handle_sys_reg
+	 * trap handler if the guest accesses or updates them
+	 */
 	struct {
 		u32	pstate_ss_bit;
-		u32	mdscr_el1_bits;
+		u32	mdscr_el1;
 
 		struct kvm_guest_debug_arch debug_regs;
 	} debug_saved_regs;
diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
index 3b368f3..638c111 100644
--- a/arch/arm64/kvm/debug.c
+++ b/arch/arm64/kvm/debug.c
@@ -55,8 +55,6 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
 	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
 
-	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
-
 	/*
 	 * If we are not treating debug registers are dirty we need
 	 * to trap if the guest starts accessing them.
@@ -71,8 +69,10 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 		/* Save pstate/mdscr */
 		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
 			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
-		vcpu_debug_saved_reg(vcpu, mdscr_el1_bits) =
-			vcpu_sys_reg(vcpu, MDSCR_EL1) & MDSCR_EL1_DEBUG_BITS;
+
+		vcpu_debug_saved_reg(vcpu, mdscr_el1) =
+			vcpu_sys_reg(vcpu, MDSCR_EL1);
+
 		/*
 		 * Single Step (ARM ARM D2.12.3 The software step state
 		 * machine)
@@ -161,9 +161,8 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
 		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
 		*vcpu_cpsr(vcpu) |= vcpu_debug_saved_reg(vcpu, pstate_ss_bit);
 
-		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
-		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
-			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
+		vcpu_sys_reg(vcpu, MDSCR_EL1) =
+			vcpu_debug_saved_reg(vcpu, mdscr_el1);
 
 		/*
 		 * If we were using HW debug we need to restore the
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index be9b188..d43d7d1 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -208,39 +208,61 @@ static bool trap_debug_regs(struct kvm_vcpu *vcpu,
 			    const struct sys_reg_params *p,
 			    const struct sys_reg_desc *r)
 {
-	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
-		struct kvm_guest_debug_arch *saved;
-		__u64 *val;
-
-		saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
-
-		if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
-			val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
-		else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
-			val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
-		else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
-			val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
-		else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
-			val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
-		else {
-			kvm_err("Bad register index %d\n", r->reg);
-			return false;
+	if (vcpu->guest_debug) {
+
+		/* MDSCR_EL1 */
+		if (r->reg == MDSCR_EL1) {
+			if (p->is_write)
+				vcpu_debug_saved_reg(vcpu, mdscr_el1) =
+					*vcpu_reg(vcpu, p->Rt);
+			else
+				*vcpu_reg(vcpu, p->Rt) =
+					vcpu_debug_saved_reg(vcpu, mdscr_el1);
+
+			return true;
 		}
 
-		if (p->is_write)
-			*val = *vcpu_reg(vcpu, p->Rt);
-		else
-			*vcpu_reg(vcpu, p->Rt) = *val;
+		/* MDCCINT_EL1 */
+		if (r->reg == MDCCINT_EL1)
+			goto old;
+
+		/* We only shadow DBG* if guest being debugged */
+		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
+			struct kvm_guest_debug_arch *saved;
+			__u64 *val;
+
+			saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
+
+			if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
+				val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
+			else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
+				val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
+			else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
+				val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
+			else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
+				val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
+			else {
+				kvm_err("Bad register index %d\n", r->reg);
+				return false;
+			}
 
-	} else {
-		if (p->is_write) {
-			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
-			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
-		} else {
-			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
+			if (p->is_write)
+				*val = *vcpu_reg(vcpu, p->Rt);
+			else
+				*vcpu_reg(vcpu, p->Rt) = *val;
+
+			return true;
 		}
 	}
 
+old:
+	if (p->is_write) {
+		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
+		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
+	} else {
+		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
+	}
+
 	return true;
 }
 
-- 
2.3.4

[PATCH v2 10/10] KVM: arm64: add trace points for guest_debug debug

[Alex Bennée]

This includes trace points for:
  kvm_arch_setup_guest_debug
  kvm_arch_clear_guest_debug
  kvm_handle_guest_debug

I've also added some generic register setting trace events so I can
watch the register values being built up over time. The local
dump_dbg_regs() function dumps all the HW BKPT and WPT registers.

I've also added a #define trace_dreg to shorten some lines.

Signed-off-by: Alex Bennée <alex.bennee@linaro.org>

diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
index 638c111..7c96288 100644
--- a/arch/arm64/kvm/debug.c
+++ b/arch/arm64/kvm/debug.c
@@ -25,12 +25,37 @@
 #include <asm/kvm_host.h>
 #include <asm/kvm_emulate.h>
 
+#include "trace.h"
+
+#define trace_dreg(name, value) trace_kvm_arch_setup_debug_reg32(name, value)
+
 /* These are the bits of MDSCR_EL1 we may mess with */
 #define MDSCR_EL1_DEBUG_BITS	(DBG_MDSCR_SS | \
 				DBG_MDSCR_KDE | \
 				DBG_MDSCR_MDE)
 
 /**
+ * dump_dbg_regs - simple debug helper
+ *
+ * This provides a simple helper to dump the HW debug registers
+ */
+static void dump_dbg_regs(struct kvm_vcpu *vcpu, int nb, int nw)
+{
+	int i;
+
+	for (i = 0; i < nb; i++) {
+		trace_printk("bkpt%d: 0x%08x:0x%llx\n", i,
+			(u32) vcpu_sys_reg(vcpu, DBGBCR0_EL1 + i),
+			vcpu_sys_reg(vcpu, DBGBVR0_EL1 + i));
+	}
+	for (i = 0; i < nb; i++) {
+		trace_printk("wtpt%d: 0x%08x:0x%llx\n", i,
+			(u32) vcpu_sys_reg(vcpu, DBGWCR0_EL1 + i),
+			vcpu_sys_reg(vcpu, DBGWVR0_EL1 + i));
+	}
+}
+
+/**
  * kvm_arch_setup_debug - set-up debug related stuff
  *
  * @vcpu:	the vcpu pointer
@@ -52,9 +77,13 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 {
 	bool trap_debug = false;
 
+	trace_kvm_arch_setup_debug(vcpu->guest_debug);
+
 	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
 	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
 
+	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
+
 	/*
 	 * If we are not treating debug registers are dirty we need
 	 * to trap if the guest starts accessing them.
@@ -66,6 +95,8 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 	if (vcpu->guest_debug) {
 		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
 
+		trace_dreg("MDCR_EL2", vcpu->arch.mdcr_el2);
+
 		/* Save pstate/mdscr */
 		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
 			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
@@ -73,6 +104,11 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 		vcpu_debug_saved_reg(vcpu, mdscr_el1) =
 			vcpu_sys_reg(vcpu, MDSCR_EL1);
 
+		trace_dreg("Save: PSTATE.SS",
+			vcpu_debug_saved_reg(vcpu, pstate_ss_bit));
+		trace_dreg("Save: MDSCR",
+			vcpu_debug_saved_reg(vcpu, mdscr_el1));
+
 		/*
 		 * Single Step (ARM ARM D2.12.3 The software step state
 		 * machine)
@@ -88,6 +124,8 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 			*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
 			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
 		}
+		trace_dreg("SPSR_EL2", *vcpu_cpsr(vcpu));
+		trace_dreg("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
 
 		/*
 		 * HW Break/Watch points
@@ -136,6 +174,9 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 			       &host->dbg_wvr,
 			       sizeof(__u64)*nw);
 
+			if (trace_kvm_arch_setup_debug_reg32_enabled())
+				dump_dbg_regs(vcpu, nb, nw);
+
 			/* Make sure hyp.S copies them in/out */
 			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
 			/* Also track guest changes */
@@ -147,15 +188,24 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
 		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
 	}
 
+	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
+	trace_kvm_arch_setup_debug_reg32("MDSCR_EL1",
+					vcpu_sys_reg(vcpu, MDSCR_EL1));
+
+
 	/* Trap debug register access? */
 	if (trap_debug)
 		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
 	else
 		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
+
+	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
 }
 
 void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
 {
+	trace_kvm_arch_clear_debug(vcpu->guest_debug);
+
 	if (vcpu->guest_debug) {
 		/* Restore pstate/mdscr bits we may have messed with */
 		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
@@ -164,6 +214,8 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
 		vcpu_sys_reg(vcpu, MDSCR_EL1) =
 			vcpu_debug_saved_reg(vcpu, mdscr_el1);
 
+		trace_dreg("SPSR_EL2", *vcpu_cpsr(vcpu));
+		trace_dreg("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
 		/*
 		 * If we were using HW debug we need to restore the
 		 * values the guest had set them up with
@@ -188,6 +240,10 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
 			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
 			       &regs->dbg_wvr,
 			       sizeof(__u64)*nw);
+
+			if (trace_kvm_arch_setup_debug_reg32_enabled())
+				dump_dbg_regs(vcpu, nb, nw);
+
 		}
 	}
 }
diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
index 460a1aa..c6cc69a 100644
--- a/arch/arm64/kvm/handle_exit.c
+++ b/arch/arm64/kvm/handle_exit.c
@@ -97,6 +97,8 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
 {
 	u32 hsr = kvm_vcpu_get_hsr(vcpu);
 
+	trace_kvm_handle_guest_debug(*vcpu_pc(vcpu), hsr);
+
 	run->exit_reason = KVM_EXIT_DEBUG;
 	run->debug.arch.hsr = hsr;
 
diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h
index 157416e9..dffdb49 100644
--- a/arch/arm64/kvm/trace.h
+++ b/arch/arm64/kvm/trace.h
@@ -44,6 +44,72 @@ TRACE_EVENT(kvm_hvc_arm64,
 		  __entry->vcpu_pc, __entry->r0, __entry->imm)
 );
 
+TRACE_EVENT(kvm_handle_guest_debug,
+	TP_PROTO(unsigned long vcpu_pc, u32 hsr),
+	TP_ARGS(vcpu_pc, hsr),
+
+	TP_STRUCT__entry(
+		__field(unsigned long,	vcpu_pc)
+		__field(u32,		hsr)
+	),
+
+	TP_fast_assign(
+		__entry->vcpu_pc = vcpu_pc;
+		__entry->hsr = hsr;
+	),
+
+	TP_printk("debug exception at 0x%08lx (HSR: 0x%08x)",
+		__entry->vcpu_pc, __entry->hsr)
+);
+
+
+TRACE_EVENT(kvm_arch_setup_debug,
+	TP_PROTO(__u32 guest_debug),
+	TP_ARGS(guest_debug),
+
+	TP_STRUCT__entry(
+		__field(__u32, guest_debug)
+	),
+
+	TP_fast_assign(
+		__entry->guest_debug = guest_debug;
+	),
+
+	TP_printk("flags: 0x%08x", __entry->guest_debug)
+);
+
+TRACE_EVENT(kvm_arch_clear_debug,
+	TP_PROTO(__u32 guest_debug),
+	TP_ARGS(guest_debug),
+
+	TP_STRUCT__entry(
+		__field(__u32, guest_debug)
+	),
+
+	TP_fast_assign(
+		__entry->guest_debug = guest_debug;
+	),
+
+	TP_printk("flags: 0x%08x", __entry->guest_debug)
+);
+
+TRACE_EVENT(kvm_arch_setup_debug_reg32,
+	TP_PROTO(const char *name, __u32 value),
+	TP_ARGS(name, value),
+
+	TP_STRUCT__entry(
+		__field(const char *, name)
+		__field(__u32, value)
+	),
+
+	TP_fast_assign(
+		__entry->name = name;
+		__entry->value = value;
+	),
+
+	TP_printk("%s now 0x%08x", __entry->name, __entry->value)
+);
+
 #endif /* _TRACE_ARM64_KVM_H */
 
 #undef TRACE_INCLUDE_PATH
-- 
2.3.4

Re: [PATCH v2 01/10] KVM: add commentary for kvm_debug_exit_arch struct

[David Hildenbrand]

> Bring into line with the commentary for the other structures and their
> KVM_EXIT_* cases.

s/commentary/comments/ in the subject and description. Unless you want to add a
lengthy discussion :)

> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> 
> v2
>   - add comments for other exit types
> 
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index 8055706..5eedf84 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -226,6 +226,7 @@ struct kvm_run {
>  			__u32 count;
>  			__u64 data_offset; /* relative to kvm_run start */
>  		} io;
> +		/* KVM_EXIT_DEBUG */
>  		struct {
>  			struct kvm_debug_exit_arch arch;
>  		} debug;
> @@ -274,6 +275,7 @@ struct kvm_run {
>  			__u32 data;
>  			__u8  is_write;
>  		} dcr;
> +		/* KVM_EXIT_INTERNAL_ERROR */
>  		struct {
>  			__u32 suberror;
>  			/* Available with KVM_CAP_INTERNAL_ERROR_DATA: */
> @@ -284,6 +286,7 @@ struct kvm_run {
>  		struct {
>  			__u64 gprs[32];
>  		} osi;
> +		/* KVM_EXIT_PAPR_HCALL */
>  		struct {
>  			__u64 nr;
>  			__u64 ret;

Looks good to me.

David

Re: [PATCH v2 03/10] KVM: arm: guest debug, define API headers

[David Hildenbrand]

> This commit defines the API headers for guest debugging. There are two
> architecture specific debug structures:
> 
>   - kvm_guest_debug_arch, allows us to pass in HW debug registers
>   - kvm_debug_exit_arch, signals the exact debug exit and pc
> 
> The type of debugging being used is control by the architecture specific

s/control/controlled/

> control bits of the kvm_guest_debug->control flags in the ioctl
> structure.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>    - expose hsr and pc directly to user-space
> 
> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
> index 3ef77a4..6ee70a0 100644
> --- a/arch/arm64/include/uapi/asm/kvm.h
> +++ b/arch/arm64/include/uapi/asm/kvm.h
> @@ -100,10 +100,24 @@ struct kvm_sregs {
>  struct kvm_fpu {
>  };
> 
> +/*
> + * See ARM ARM D7.3: Debug Registers

Maybe drop one ARM

> + *
> + * The control registers are architecturally defined as 32 bits but are
> + * stored as 64 bit values along side the value registers and aligned
> + * with the rest 64 bit registers in the normal CPU context.
> + */
> +#define KVM_ARM_NDBG_REGS 16
>  struct kvm_guest_debug_arch {
> +	__u64 dbg_bcr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_bvr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_wcr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_wvr[KVM_ARM_NDBG_REGS];
>  };
> 
>  struct kvm_debug_exit_arch {
> +	__u64 pc;
> +	__u32 hsr;
>  };
> 
>  struct kvm_sync_regs {
> @@ -207,4 +221,11 @@ struct kvm_arch_memory_slot {
> 
>  #endif
> 
> +/*
> + * Architecture related debug defines - upper 16 bits of
> + * kvm_guest_debug->control
> + */
> +#define KVM_GUESTDBG_USE_SW_BP	        __KVM_GUESTDBG_USE_SW_BP
> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
> +
>  #endif /* __ARM_KVM_H__ */

Looks good to me!

David

Re: [PATCH v2 04/10] KVM: arm: guest debug, add stub KVM_SET_GUEST_DEBUG ioctl

[David Hildenbrand]

> This commit adds a stub function to support the KVM_SET_GUEST_DEBUG
> ioctl. Currently any operation flag will return EINVAL. Actual

Well it won't return -EINVAL if you push in KVM_GUESTDBG_ENABLE or 0.

"Any unsupported flag will return -EINVAL. For now, only KVM_GUESTDBG_ENABLE is
supported, although it won't have any effects."

> functionality will be added with further patches.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>.
> 
> ---
> v2
>   - simplified form of the ioctl (stuff will go into setup_debug)
> 
> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> index b112efc..06c5064 100644
> --- a/Documentation/virtual/kvm/api.txt
> +++ b/Documentation/virtual/kvm/api.txt
> @@ -2604,7 +2604,7 @@ handled.
>  4.87 KVM_SET_GUEST_DEBUG
> 
>  Capability: KVM_CAP_SET_GUEST_DEBUG
> -Architectures: x86, s390, ppc
> +Architectures: x86, s390, ppc, arm64
>  Type: vcpu ioctl
>  Parameters: struct kvm_guest_debug (in)
>  Returns: 0 on success; -1 on error
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 5560f74..445933d 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -183,6 +183,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>  	case KVM_CAP_ARM_PSCI:
>  	case KVM_CAP_ARM_PSCI_0_2:
>  	case KVM_CAP_READONLY_MEM:
> +	case KVM_CAP_SET_GUEST_DEBUG:
>  		r = 1;
>  		break;
>  	case KVM_CAP_COALESCED_MMIO:
> @@ -303,10 +304,21 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  	kvm_arm_set_running_vcpu(NULL);
>  }
> 
> +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)

That makes me rather think that it is another flag.

We(s390x) use VALID_GUESTDBG_FLAGS, what about that or KVM_GUESTDBG_VALID_MASK?

> +
>  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>  					struct kvm_guest_debug *dbg)
>  {
> -	return -EINVAL;
> +	if (dbg->control & KVM_GUESTDBG_ENABLE) {
> +		if (dbg->control & ~KVM_GUESTDBG_VALID)
> +			return -EINVAL;

I'd move that check directly to the start of the function and bail out on any
unsupported flag.

> +
> +		vcpu->guest_debug = dbg->control;
> +	} else {
> +		/* If not enabled clear all flags */
> +		vcpu->guest_debug = 0;
> +	}
> +	return 0;
>  }
> 
> 

David

Re: [PATCH v2 03/10] KVM: arm: guest debug, define API headers

[Alex Bennée]

David Hildenbrand <dahi@linux.vnet.ibm.com> writes:

>> This commit defines the API headers for guest debugging. There are two
>> architecture specific debug structures:
>> 
>>   - kvm_guest_debug_arch, allows us to pass in HW debug registers
>>   - kvm_debug_exit_arch, signals the exact debug exit and pc
>> 
>> The type of debugging being used is control by the architecture specific
>
> s/control/controlled/

good catch

>
>> control bits of the kvm_guest_debug->control flags in the ioctl
>> structure.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>> ---
>> v2
>>    - expose hsr and pc directly to user-space
>> 
>> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
>> index 3ef77a4..6ee70a0 100644
>> --- a/arch/arm64/include/uapi/asm/kvm.h
>> +++ b/arch/arm64/include/uapi/asm/kvm.h
>> @@ -100,10 +100,24 @@ struct kvm_sregs {
>>  struct kvm_fpu {
>>  };
>> 
>> +/*
>> + * See ARM ARM D7.3: Debug Registers
>
> Maybe drop one ARM

Well technically it is the ARM Architecture Reference Manual but that's
quite long winded.

>
>> + *
>> + * The control registers are architecturally defined as 32 bits but are
>> + * stored as 64 bit values along side the value registers and aligned
>> + * with the rest 64 bit registers in the normal CPU context.
>> + */
>> +#define KVM_ARM_NDBG_REGS 16
>>  struct kvm_guest_debug_arch {
>> +	__u64 dbg_bcr[KVM_ARM_NDBG_REGS];
>> +	__u64 dbg_bvr[KVM_ARM_NDBG_REGS];
>> +	__u64 dbg_wcr[KVM_ARM_NDBG_REGS];
>> +	__u64 dbg_wvr[KVM_ARM_NDBG_REGS];
>>  };
>> 
>>  struct kvm_debug_exit_arch {
>> +	__u64 pc;
>> +	__u32 hsr;
>>  };
>> 
>>  struct kvm_sync_regs {
>> @@ -207,4 +221,11 @@ struct kvm_arch_memory_slot {
>> 
>>  #endif
>> 
>> +/*
>> + * Architecture related debug defines - upper 16 bits of
>> + * kvm_guest_debug->control
>> + */
>> +#define KVM_GUESTDBG_USE_SW_BP	        __KVM_GUESTDBG_USE_SW_BP
>> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
>> +
>>  #endif /* __ARM_KVM_H__ */
>
> Looks good to me!

Is that a Reviewed-by?

>
> David

-- 
Alex Bennée

Re: [PATCH v2 03/10] KVM: arm: guest debug, define API headers

[David Hildenbrand]

> > Looks good to me!
> 
> Is that a Reviewed-by?

Now it is :)

Reviewed-by: David Hildenbrand <dahi@linux.vnet.ibm.com>

David

Re: [PATCH v2 03/10] KVM: arm: guest debug, define API headers

[Peter Maydell]

On 1 April 2015 at 17:01, Alex Bennée <alex.bennee@linaro.org> wrote:
>
> David Hildenbrand <dahi@linux.vnet.ibm.com> writes:

>>> +/*
>>> + * See ARM ARM D7.3: Debug Registers
>>
>> Maybe drop one ARM
>
> Well technically it is the ARM Architecture Reference Manual but that's
> quite long winded.

Dropping one "ARM" would be actively wrong, so don't do that.
You probably want to say 'v8 ARM ARM", assuming you mean that.

-- PMM

Re: [PATCH v2 05/10] KVM: arm: introduce kvm_arch_setup/clear_debug()

[David Hildenbrand]

> This is a precursor for later patches which will need to do more to
> setup debug state before entering the hyp.S switch code. The existing
> functionality for setting mdcr_el2 has been moved out of hyp.S and now
> uses the value kept in vcpu->arch.mdcr_el2.
> 
> This also moves the conditional setting of the TDA bit from the hyp code
> into the C code.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
>  create mode 100644 arch/arm64/kvm/debug.c
> 
> diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
> index 41008cd..8c01c97 100644
> --- a/arch/arm/include/asm/kvm_host.h
> +++ b/arch/arm/include/asm/kvm_host.h
> @@ -242,5 +242,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
>  static inline void kvm_arch_sync_events(struct kvm *kvm) {}
>  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
>  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
> +static inline void kvm_arch_setup_debug(struct kvm_vcpu *vcpu) {}
> +static inline void kvm_arch_clear_debug(struct kvm_vcpu *vcpu) {}

Do you really want to call these functions "kvm_arch.." although they are not
defined for other arch and not triggered by common code?

kvm_setup ... or kvm_arm_setup...

> 
>  #endif /* __ARM_KVM_HOST_H__ */
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 445933d..7ea8b0e 100644
> --- a/arch/arm/kvm/arm.c
[...]
> +#include <linux/kvm_host.h>
> +
> +#include <asm/kvm_arm.h>
> +#include <asm/kvm_host.h>
> +
> +/**
> + * kvm_arch_setup_debug - set-up debug related stuff
> + *
> + * @vcpu:	the vcpu pointer
> + *
> + * This is called before each entry in to the hypervisor to setup any
> + * debug related registers. Currently this just ensures we will trap
> + * access to:
> + *  - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR)
> + *  - Debug ROM Address (MDCR_EL2_TDRA)
> + *  - Power down debug registers (MDCR_EL2_TDOSA)
> + *
> + * Additionally the hypervisor lazily saves/restores the debug
> + * register state. If it is not currently doing so (arch.debug_flags)
> + * then we also need to ensure we trap if the guest messes with them
> + * so we know we need to save them.
> + */
> +
> +void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
> +{
> +	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
> +	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);

I'd put that into a single assignment.

> +
> +	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
> +	else
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
> +
> +}
> +
> +void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> +{
> +	/* Nothing to do yet */

Wonder if that would be the right place to clear MDCR_EL2_TDA unconditionally?
But see my comment below.

> +}
> diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
> index 5befd01..be92bfe1 100644
> --- a/arch/arm64/kvm/hyp.S
> +++ b/arch/arm64/kvm/hyp.S
> @@ -768,17 +768,8 @@
>  	mov	x2, #(1 << 15)	// Trap CP15 Cr=15
>  	msr	hstr_el2, x2
> 
> -	mrs	x2, mdcr_el2
> -	and	x2, x2, #MDCR_EL2_HPMN_MASK
> -	orr	x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
> -	orr	x2, x2, #(MDCR_EL2_TDRA | MDCR_EL2_TDOSA)
> -
> -	// Check for KVM_ARM64_DEBUG_DIRTY, and set debug to trap
> -	// if not dirty.
> -	ldr	x3, [x0, #VCPU_DEBUG_FLAGS]
> -	tbnz	x3, #KVM_ARM64_DEBUG_DIRTY_SHIFT, 1f
> -	orr	x2, x2,  #MDCR_EL2_TDA
> -1:

*neither an assembler nor arm expert*
The existing code always cleared all bits except MDCR_EL2_HPMN_MASK. So they
remained set.

We would now always overwrite these bits with the values from 
vcpu->arch.mdcr_el2, is that okay?

> +	// Monitor Debug Config - see kvm_arch_setup_debug()
> +	ldr	x2, [x0, #VCPU_MDCR_EL2]
>  	msr	mdcr_el2, x2
>  .endm
> 



David

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[David Hildenbrand]

> This adds support for SW breakpoints inserted by userspace.
> 
> We do this by trapping all BKPT exceptions in the
> hypervisor (MDCR_EL2_TDE). The kvm_debug_exit_arch carries the address
> of the exception. If user-space doesn't know of the breakpoint then we
> have a guest inserted breakpoint and the hypervisor needs to start again
> and deliver the exception to guest.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>   - update to use new exit struct
>   - tweak for C setup
>   - do our setup in debug_setup/clear code
>   - fixed up comments
> 
> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> index 06c5064..17d4f9c 100644
> --- a/Documentation/virtual/kvm/api.txt
> +++ b/Documentation/virtual/kvm/api.txt
> @@ -2626,7 +2626,7 @@ when running. Common control bits are:
>  The top 16 bits of the control field are architecture specific control
>  flags which can include the following:
> 
> -  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86]
> +  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
>    - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
>    - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
>    - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 7ea8b0e..d3bc8dc 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -304,7 +304,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  	kvm_arm_set_running_vcpu(NULL);
>  }
> 
> -#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)
> +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)
> 
>  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>  					struct kvm_guest_debug *dbg)
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index 8a29d0b..cff0475 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -45,11 +45,18 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
> 
> +	/* Trap debug register access? */

This should probably go to the earlier patch.

>  	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>  	else
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
> 
> +	/* Trap breakpoints? */
> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
> +	else
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;

Again, a candidate for clear_debug?

> +
>  }
> 
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index 524fa25..ed1bbb4 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -82,6 +82,37 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  	return 1;
>  }
> 
> +/**
> + * kvm_handle_debug_exception - handle a debug exception instruction

"kvm_handle_guest_debug"

> + *
> + * @vcpu:	the vcpu pointer
> + * @run:	access to the kvm_run structure for results
> + *
> + * We route all debug exceptions through the same handler as we
> + * just need to report the PC and the HSR values to userspace.
> + * Userspace may decide to re-inject the exception and deliver it to
> + * the guest if it wasn't for the host to deal with.
> + */
> +static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
> +{
> +	u32 hsr = kvm_vcpu_get_hsr(vcpu);
> +
> +	run->exit_reason = KVM_EXIT_DEBUG;
> +	run->debug.arch.hsr = hsr;
> +
> +	switch (hsr >> ESR_ELx_EC_SHIFT) {
> +	case ESR_ELx_EC_BKPT32:
> +	case ESR_ELx_EC_BRK64:
> +		run->debug.arch.pc = *vcpu_pc(vcpu);
> +		break;
> +	default:
> +		kvm_err("%s: un-handled case hsr: %#08x\n",
> +			__func__, (unsigned int) hsr);

Don't you want to fail hard in this case? This might result in many messages.
returning 0 feels wrong.

> +		break;
> +	}
> +	return 0;
> +}
> +
>  static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_WFx]	= kvm_handle_wfx,
>  	[ESR_ELx_EC_CP15_32]	= kvm_handle_cp15_32,
> @@ -96,6 +127,8 @@ static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
> +	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
> +	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>  };
> 
>  static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)


David

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Alex Bennée]

David Hildenbrand <dahi@linux.vnet.ibm.com> writes:

>> This adds support for SW breakpoints inserted by userspace.
>> 
>> We do this by trapping all BKPT exceptions in the
>> hypervisor (MDCR_EL2_TDE). The kvm_debug_exit_arch carries the address
>> of the exception. If user-space doesn't know of the breakpoint then we
>> have a guest inserted breakpoint and the hypervisor needs to start again
>> and deliver the exception to guest.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>> ---
>> v2
>>   - update to use new exit struct
>>   - tweak for C setup
>>   - do our setup in debug_setup/clear code
>>   - fixed up comments
>> 
>> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
>> index 06c5064..17d4f9c 100644
>> --- a/Documentation/virtual/kvm/api.txt
>> +++ b/Documentation/virtual/kvm/api.txt
>> @@ -2626,7 +2626,7 @@ when running. Common control bits are:
>>  The top 16 bits of the control field are architecture specific control
>>  flags which can include the following:
>> 
>> -  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86]
>> +  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
>>    - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
>>    - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
>>    - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
>> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
>> index 7ea8b0e..d3bc8dc 100644
>> --- a/arch/arm/kvm/arm.c
>> +++ b/arch/arm/kvm/arm.c
>> @@ -304,7 +304,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>>  	kvm_arm_set_running_vcpu(NULL);
>>  }
>> 
>> -#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)
>> +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)
>> 
>>  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>>  					struct kvm_guest_debug *dbg)
>> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
>> index 8a29d0b..cff0475 100644
>> --- a/arch/arm64/kvm/debug.c
>> +++ b/arch/arm64/kvm/debug.c
>> @@ -45,11 +45,18 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>> 
>> +	/* Trap debug register access? */
>
> This should probably go to the earlier patch.

Agreed.

>
>>  	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
>>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>>  	else
>>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>> 
>> +	/* Trap breakpoints? */
>> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
>> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
>> +	else
>> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>
> Again, a candidate for clear_debug?

I don't follow. Changes to mdcr_el2 will only get applied as we jump in
through the hyp.S code. We need to ensure the guest can use SW BKPTs if
we are not.

>
>> +
>>  }
>> 
>>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
>> index 524fa25..ed1bbb4 100644
>> --- a/arch/arm64/kvm/handle_exit.c
>> +++ b/arch/arm64/kvm/handle_exit.c
>> @@ -82,6 +82,37 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
>>  	return 1;
>>  }
>> 
>> +/**
>> + * kvm_handle_debug_exception - handle a debug exception instruction
>
> "kvm_handle_guest_debug"

Sure.

>
>> + *
>> + * @vcpu:	the vcpu pointer
>> + * @run:	access to the kvm_run structure for results
>> + *
>> + * We route all debug exceptions through the same handler as we
>> + * just need to report the PC and the HSR values to userspace.
>> + * Userspace may decide to re-inject the exception and deliver it to
>> + * the guest if it wasn't for the host to deal with.
>> + */
>> +static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>> +{
>> +	u32 hsr = kvm_vcpu_get_hsr(vcpu);
>> +
>> +	run->exit_reason = KVM_EXIT_DEBUG;
>> +	run->debug.arch.hsr = hsr;
>> +
>> +	switch (hsr >> ESR_ELx_EC_SHIFT) {
>> +	case ESR_ELx_EC_BKPT32:
>> +	case ESR_ELx_EC_BRK64:
>> +		run->debug.arch.pc = *vcpu_pc(vcpu);
>> +		break;
>> +	default:
>> +		kvm_err("%s: un-handled case hsr: %#08x\n",
>> +			__func__, (unsigned int) hsr);
>
> Don't you want to fail hard in this case? This might result in many messages.
> returning 0 feels wrong.

You mean a BUG_ON()? Although it would be a cock up on the hosts part to
have an un-handled exception enabled allowing the guest to trigger a
host panic seems excessive.

>
>> +		break;
>> +	}
>> +	return 0;
>> +}
>> +
>>  static exit_handle_fn arm_exit_handlers[] = {
>>  	[ESR_ELx_EC_WFx]	= kvm_handle_wfx,
>>  	[ESR_ELx_EC_CP15_32]	= kvm_handle_cp15_32,
>> @@ -96,6 +127,8 @@ static exit_handle_fn arm_exit_handlers[] = {
>>  	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
>>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
>> +	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
>> +	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>>  };
>> 
>>  static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
>
>
> David

-- 
Alex Bennée

Re: [PATCH v2 04/10] KVM: arm: guest debug, add stub KVM_SET_GUEST_DEBUG ioctl

[Andrew Jones]

On Wed, Apr 01, 2015 at 05:55:29PM +0200, David Hildenbrand wrote:
> > This commit adds a stub function to support the KVM_SET_GUEST_DEBUG
> > ioctl. Currently any operation flag will return EINVAL. Actual
> 
> Well it won't return -EINVAL if you push in KVM_GUESTDBG_ENABLE or 0.
> 
> "Any unsupported flag will return -EINVAL. For now, only KVM_GUESTDBG_ENABLE is
> supported, although it won't have any effects."
> 
> > functionality will be added with further patches.
> > 
> > Signed-off-by: Alex Bennée <alex.bennee@linaro.org>.
> > 
> > ---
> > v2
> >   - simplified form of the ioctl (stuff will go into setup_debug)
> > 
> > diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> > index b112efc..06c5064 100644
> > --- a/Documentation/virtual/kvm/api.txt
> > +++ b/Documentation/virtual/kvm/api.txt
> > @@ -2604,7 +2604,7 @@ handled.
> >  4.87 KVM_SET_GUEST_DEBUG
> > 
> >  Capability: KVM_CAP_SET_GUEST_DEBUG
> > -Architectures: x86, s390, ppc
> > +Architectures: x86, s390, ppc, arm64
> >  Type: vcpu ioctl
> >  Parameters: struct kvm_guest_debug (in)
> >  Returns: 0 on success; -1 on error
> > diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> > index 5560f74..445933d 100644
> > --- a/arch/arm/kvm/arm.c
> > +++ b/arch/arm/kvm/arm.c
> > @@ -183,6 +183,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
> >  	case KVM_CAP_ARM_PSCI:
> >  	case KVM_CAP_ARM_PSCI_0_2:
> >  	case KVM_CAP_READONLY_MEM:
> > +	case KVM_CAP_SET_GUEST_DEBUG:
> >  		r = 1;
> >  		break;
> >  	case KVM_CAP_COALESCED_MMIO:
> > @@ -303,10 +304,21 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
> >  	kvm_arm_set_running_vcpu(NULL);
> >  }
> > 
> > +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)
> 
> That makes me rather think that it is another flag.
> 
> We(s390x) use VALID_GUESTDBG_FLAGS, what about that or KVM_GUESTDBG_VALID_MASK?
> 
> > +
> >  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
> >  					struct kvm_guest_debug *dbg)
> >  {
> > -	return -EINVAL;
> > +	if (dbg->control & KVM_GUESTDBG_ENABLE) {
> > +		if (dbg->control & ~KVM_GUESTDBG_VALID)
> > +			return -EINVAL;
> 
> I'd move that check directly to the start of the function and bail out on any
> unsupported flag.
> 
> > +
> > +		vcpu->guest_debug = dbg->control;
> > +	} else {
> > +		/* If not enabled clear all flags */
> > +		vcpu->guest_debug = 0;
> > +	}
> > +	return 0;
> >  }
> > 
> > 
> 
> David
>

I don't see any follow-up from Alex on this, so I feel the need to
"+1" all David's comments here.

drew

Re: [PATCH v2 05/10] KVM: arm: introduce kvm_arch_setup/clear_debug()

[Andrew Jones]

On Tue, Mar 31, 2015 at 04:08:03PM +0100, Alex Bennée wrote:
> This is a precursor for later patches which will need to do more to
> setup debug state before entering the hyp.S switch code. The existing
> functionality for setting mdcr_el2 has been moved out of hyp.S and now
> uses the value kept in vcpu->arch.mdcr_el2.
> 
> This also moves the conditional setting of the TDA bit from the hyp code
> into the C code.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
>  create mode 100644 arch/arm64/kvm/debug.c
> 
> diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
> index 41008cd..8c01c97 100644
> --- a/arch/arm/include/asm/kvm_host.h
> +++ b/arch/arm/include/asm/kvm_host.h
> @@ -242,5 +242,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
>  static inline void kvm_arch_sync_events(struct kvm *kvm) {}
>  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
>  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
> +static inline void kvm_arch_setup_debug(struct kvm_vcpu *vcpu) {}
> +static inline void kvm_arch_clear_debug(struct kvm_vcpu *vcpu) {}
>  
>  #endif /* __ARM_KVM_HOST_H__ */
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 445933d..7ea8b0e 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -523,6 +523,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  
>  		kvm_vgic_flush_hwstate(vcpu);
>  		kvm_timer_flush_hwstate(vcpu);
> +		kvm_arch_setup_debug(vcpu);
>  
>  		local_irq_disable();
>  
> @@ -569,6 +570,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  		 * Back from guest
>  		 *************************************************************/
>  
> +		kvm_arch_clear_debug(vcpu);
>  		kvm_timer_sync_hwstate(vcpu);
>  		kvm_vgic_sync_hwstate(vcpu);
>  
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 8ac3c70..0631840 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -101,6 +101,7 @@ struct kvm_vcpu_arch {
>  
>  	/* HYP configuration */
>  	u64 hcr_el2;
> +	u32 mdcr_el2;
>  
>  	/* Exception Information */
>  	struct kvm_vcpu_fault_info fault;
> @@ -257,4 +258,7 @@ static inline void kvm_arch_sync_events(struct kvm *kvm) {}
>  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
>  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
>  
> +void kvm_arch_setup_debug(struct kvm_vcpu *vcpu);
> +void kvm_arch_clear_debug(struct kvm_vcpu *vcpu);
> +
>  #endif /* __ARM64_KVM_HOST_H__ */
> diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
> index f7fa65d..cd06209 100644
> --- a/arch/arm64/kernel/asm-offsets.c
> +++ b/arch/arm64/kernel/asm-offsets.c
> @@ -122,6 +122,7 @@ int main(void)
>    DEFINE(VCPU_HPFAR_EL2,	offsetof(struct kvm_vcpu, arch.fault.hpfar_el2));
>    DEFINE(VCPU_DEBUG_FLAGS,	offsetof(struct kvm_vcpu, arch.debug_flags));
>    DEFINE(VCPU_HCR_EL2,		offsetof(struct kvm_vcpu, arch.hcr_el2));
> +  DEFINE(VCPU_MDCR_EL2,	offsetof(struct kvm_vcpu, arch.mdcr_el2));
>    DEFINE(VCPU_IRQ_LINES,	offsetof(struct kvm_vcpu, arch.irq_lines));
>    DEFINE(VCPU_HOST_CONTEXT,	offsetof(struct kvm_vcpu, arch.host_cpu_context));
>    DEFINE(VCPU_TIMER_CNTV_CTL,	offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
> index 4e6e09e..6796d4a 100644
> --- a/arch/arm64/kvm/Makefile
> +++ b/arch/arm64/kvm/Makefile
> @@ -17,7 +17,7 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o
>  
>  kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o
>  kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
> -kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
> +kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o
>  
>  kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
>  kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> new file mode 100644
> index 0000000..8a29d0b
> --- /dev/null
> +++ b/arch/arm64/kvm/debug.c
> @@ -0,0 +1,58 @@
> +/*
> + * Debug and Guest Debug support
> + *
> + * Copyright (C) 2015 - Linaro Ltd
> + * Author: Alex Bennée <alex.bennee@linaro.org>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include <linux/kvm_host.h>
> +
> +#include <asm/kvm_arm.h>
> +#include <asm/kvm_host.h>

linux/kvm_host.h added above includes this asm/kvm_host already

> +
> +/**
> + * kvm_arch_setup_debug - set-up debug related stuff
> + *
> + * @vcpu:	the vcpu pointer
> + *
> + * This is called before each entry in to the hypervisor to setup any

into

> + * debug related registers. Currently this just ensures we will trap
> + * access to:
> + *  - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR)
> + *  - Debug ROM Address (MDCR_EL2_TDRA)
> + *  - Power down debug registers (MDCR_EL2_TDOSA)
> + *
> + * Additionally the hypervisor lazily saves/restores the debug
> + * register state. If it is not currently doing so (arch.debug_flags)
> + * then we also need to ensure we trap if the guest messes with them
> + * so we know we need to save them.
> + */
> +
> +void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
> +{
> +	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
> +	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
> +
> +	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)

missing () in the if (!(x & y)) condition

> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
> +	else
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
> +
> +}
> +
> +void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> +{
> +	/* Nothing to do yet */
> +}
> diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
> index 5befd01..be92bfe1 100644
> --- a/arch/arm64/kvm/hyp.S
> +++ b/arch/arm64/kvm/hyp.S
> @@ -768,17 +768,8 @@
>  	mov	x2, #(1 << 15)	// Trap CP15 Cr=15
>  	msr	hstr_el2, x2
>  
> -	mrs	x2, mdcr_el2
> -	and	x2, x2, #MDCR_EL2_HPMN_MASK
> -	orr	x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
> -	orr	x2, x2, #(MDCR_EL2_TDRA | MDCR_EL2_TDOSA)
> -
> -	// Check for KVM_ARM64_DEBUG_DIRTY, and set debug to trap
> -	// if not dirty.
> -	ldr	x3, [x0, #VCPU_DEBUG_FLAGS]
> -	tbnz	x3, #KVM_ARM64_DEBUG_DIRTY_SHIFT, 1f
> -	orr	x2, x2,  #MDCR_EL2_TDA
> -1:
> +	// Monitor Debug Config - see kvm_arch_setup_debug()
> +	ldr	x2, [x0, #VCPU_MDCR_EL2]
>  	msr	mdcr_el2, x2
>  .endm

We lose the preservation of HPMN, as I see David points out as well.
Based on the ARM ARM setting it to zero is most likely wrong, as
that results in constrained unpredictable behavior.

drew

Re: [PATCH v2 05/10] KVM: arm: introduce kvm_arch_setup/clear_debug()

[Andrew Jones]

On Wed, Apr 01, 2015 at 06:28:55PM +0200, David Hildenbrand wrote:
> > This is a precursor for later patches which will need to do more to
> > setup debug state before entering the hyp.S switch code. The existing
> > functionality for setting mdcr_el2 has been moved out of hyp.S and now
> > uses the value kept in vcpu->arch.mdcr_el2.
> > 
> > This also moves the conditional setting of the TDA bit from the hyp code
> > into the C code.
> > 
> > Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> > 
> >  create mode 100644 arch/arm64/kvm/debug.c
> > 
> > diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
> > index 41008cd..8c01c97 100644
> > --- a/arch/arm/include/asm/kvm_host.h
> > +++ b/arch/arm/include/asm/kvm_host.h
> > @@ -242,5 +242,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
> >  static inline void kvm_arch_sync_events(struct kvm *kvm) {}
> >  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
> >  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
> > +static inline void kvm_arch_setup_debug(struct kvm_vcpu *vcpu) {}
> > +static inline void kvm_arch_clear_debug(struct kvm_vcpu *vcpu) {}
> 
> Do you really want to call these functions "kvm_arch.." although they are not
> defined for other arch and not triggered by common code?
>

Agreed. If other arches want something similar we can rename/refactor
later. If these are arm-only functions, then we don't need the generic
prefix, which is actually even a bit confusing.

drew 

Re: [PATCH v2 07/10] KVM: arm64: guest debug, add support for single-step

[Andrew Jones]

On Tue, Mar 31, 2015 at 04:08:05PM +0100, Alex Bennée wrote:
> This adds support for single-stepping the guest. As userspace can and
> will manipulate guest registers before restarting any tweaking of the
> registers has to occur just before control is passed back to the guest.
> Furthermore while guest debugging is in effect we need to squash the
> ability of the guest to single-step itself as we have no easy way of
> re-entering the guest after the exception has been delivered to the
> hypervisor.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>   - Move pstate/mdscr manipulation into C
>   - don't export guest_debug to assembly
>   - add accessor for saved_debug regs
>   - tweak save/restore of mdscr_el1
> 
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index d3bc8dc..c1ed8cb 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -304,7 +304,21 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  	kvm_arm_set_running_vcpu(NULL);
>  }
>  
> -#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)
> +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE |    \
> +			    KVM_GUESTDBG_USE_SW_BP | \
> +			    KVM_GUESTDBG_SINGLESTEP)
> +
> +/**
> + * kvm_arch_vcpu_ioctl_set_guest_debug - Setup guest debugging
> + * @kvm:	pointer to the KVM struct
> + * @kvm_guest_debug: the ioctl data buffer
> + *
> + * This sets up the VM for guest debugging. Care has to be taken when
> + * manipulating guest registers as these will be set/cleared by the
> + * hyper-visor controller, typically before each kvm_run event. As a
> + * result modification of the guest registers needs to take place
> + * after they have been restored in the hyp.S trampoline code.
> + */
>  
>  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>  					struct kvm_guest_debug *dbg)
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 0631840..6a33647 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -121,6 +121,13 @@ struct kvm_vcpu_arch {
>  	 * here.
>  	 */
>  
> +	/* Registers pre any guest debug manipulations */
> +	struct {
> +		u32	pstate_ss_bit;
> +		u32	mdscr_el1_bits;
> +
> +	} debug_saved_regs;

Hmm, you have a struct called "regs", but then each member is
suffixed with _bit(s). This looks awkward.


> +
>  	/* Don't run the guest */
>  	bool pause;
>  
> @@ -143,6 +150,7 @@ struct kvm_vcpu_arch {
>  
>  #define vcpu_gp_regs(v)		(&(v)->arch.ctxt.gp_regs)
>  #define vcpu_sys_reg(v,r)	((v)->arch.ctxt.sys_regs[(r)])
> +#define vcpu_debug_saved_reg(v, r) ((v)->arch.debug_saved_regs.r)
>  /*
>   * CP14 and CP15 live in the same array, as they are backed by the
>   * same system registers.
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index cff0475..b32362c 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -19,8 +19,16 @@
>  
>  #include <linux/kvm_host.h>
>  
> +#include <asm/debug-monitors.h>
> +#include <asm/kvm_asm.h>
>  #include <asm/kvm_arm.h>
>  #include <asm/kvm_host.h>
> +#include <asm/kvm_emulate.h>
> +
> +/* These are the bits of MDSCR_EL1 we may mess with */
> +#define MDSCR_EL1_DEBUG_BITS	(DBG_MDSCR_SS | \
> +				DBG_MDSCR_KDE | \
> +				DBG_MDSCR_MDE)

_MASK instead of _BITS ?

>  
>  /**
>   * kvm_arch_setup_debug - set-up debug related stuff
> @@ -51,15 +59,46 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	else
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>  
> -	/* Trap breakpoints? */
> -	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
> +	/* Is Guest debugging in effect? */
> +	if (vcpu->guest_debug) {
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
> -	else
> -		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>  
> +		/* Save pstate/mdscr */
> +		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
> +			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
> +		vcpu_debug_saved_reg(vcpu, mdscr_el1_bits) =
> +			vcpu_sys_reg(vcpu, MDSCR_EL1) & MDSCR_EL1_DEBUG_BITS;

I think it would be clearer if we embed the masks into helper
functions, and, assuming we drop the _bits concept too, then

#define SPSR_DEBUG_MASK DBG_SPSR_SS

vcpu_debug_save_regs(vcpu)
{
  vcpu->arch.debug_saved_regs.pstate = *vcpu_cpsr(vcpu);
  vcpu->arch.debug_saved_regs.mdscr_el1 = vcpu_sys_reg(vcpu, MDSCR_EL1);
}

vcpu_debug_restore_regs(vcpu)
{
  *vcpu_cpsr(vcpu) |=
       (vcpu->arch.debug_saved_regs.pstate & SPSR_DEBUG_MASK);
  vcpu_sys_reg(vcpu, MDSCR_EL1) |=
       (vcpu->arch.debug_saved_regs.mdscr_el1 & MDSCR_EL1_DEBUG_MASK)
}

> +		/*
> +		 * Single Step (ARM ARM D2.12.3 The software step state
> +		 * machine)
> +		 *
> +		 * If we are doing Single Step we need to manipulate
> +		 * MDSCR_EL1.SS and PSTATE.SS. If not we need to
> +		 * suppress the guest from messing with it.
> +		 */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
> +			*vcpu_cpsr(vcpu) |=  DBG_SPSR_SS;
> +			vcpu_sys_reg(vcpu, MDSCR_EL1) |= DBG_MDSCR_SS;
> +		} else {
> +			*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
> +			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
> +		}
> +
> +	} else {
> +		/* Debug operations can go straight to the guest */
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
> +	}
>  }
>  
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  {
> -	/* Nothing to do yet */

This would now just be

  if (vcpu->guest_debug)
     vcpu_debug_restore_regs(vcpu);

> +	if (vcpu->guest_debug) {
> +		/* Restore pstate/mdscr bits we may have messed with */
> +		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
> +		*vcpu_cpsr(vcpu) |= vcpu_debug_saved_reg(vcpu, pstate_ss_bit);
> +
> +		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
> +		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
> +			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
> +	}
>  }
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index ed1bbb4..16accae 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -101,6 +101,7 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  	run->debug.arch.hsr = hsr;
>  
>  	switch (hsr >> ESR_ELx_EC_SHIFT) {
> +	case ESR_ELx_EC_SOFTSTP_LOW:
>  	case ESR_ELx_EC_BKPT32:
>  	case ESR_ELx_EC_BRK64:
>  		run->debug.arch.pc = *vcpu_pc(vcpu);
> @@ -127,6 +128,7 @@ static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
> +	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
>  	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
>  	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>  };
> -- 
> 2.3.4
> 

Re: [PATCH v2 07/10] KVM: arm64: guest debug, add support for single-step

[Alex Bennée]

Andrew Jones <drjones@redhat.com> writes:

> On Tue, Mar 31, 2015 at 04:08:05PM +0100, Alex Bennée wrote:
>> This adds support for single-stepping the guest. As userspace can and
>> will manipulate guest registers before restarting any tweaking of the
>> registers has to occur just before control is passed back to the guest.
>> Furthermore while guest debugging is in effect we need to squash the
>> ability of the guest to single-step itself as we have no easy way of
>> re-entering the guest after the exception has been delivered to the
>> hypervisor.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>> ---
>> v2
>>   - Move pstate/mdscr manipulation into C
>>   - don't export guest_debug to assembly
>>   - add accessor for saved_debug regs
>>   - tweak save/restore of mdscr_el1
>> 
>> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
>> index d3bc8dc..c1ed8cb 100644
>> --- a/arch/arm/kvm/arm.c
>> +++ b/arch/arm/kvm/arm.c
>> @@ -304,7 +304,21 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>>  	kvm_arm_set_running_vcpu(NULL);
>>  }
>>  
>> -#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)
>> +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE |    \
>> +			    KVM_GUESTDBG_USE_SW_BP | \
>> +			    KVM_GUESTDBG_SINGLESTEP)
>> +
>> +/**
>> + * kvm_arch_vcpu_ioctl_set_guest_debug - Setup guest debugging
>> + * @kvm:	pointer to the KVM struct
>> + * @kvm_guest_debug: the ioctl data buffer
>> + *
>> + * This sets up the VM for guest debugging. Care has to be taken when
>> + * manipulating guest registers as these will be set/cleared by the
>> + * hyper-visor controller, typically before each kvm_run event. As a
>> + * result modification of the guest registers needs to take place
>> + * after they have been restored in the hyp.S trampoline code.
>> + */
>>  
>>  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>>  					struct kvm_guest_debug *dbg)
>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>> index 0631840..6a33647 100644
>> --- a/arch/arm64/include/asm/kvm_host.h
>> +++ b/arch/arm64/include/asm/kvm_host.h
>> @@ -121,6 +121,13 @@ struct kvm_vcpu_arch {
>>  	 * here.
>>  	 */
>>  
>> +	/* Registers pre any guest debug manipulations */
>> +	struct {
>> +		u32	pstate_ss_bit;
>> +		u32	mdscr_el1_bits;
>> +
>> +	} debug_saved_regs;
>
> Hmm, you have a struct called "regs", but then each member is
> suffixed with _bit(s). This looks awkward.

Later on mdscr gets expanded and properly shadowed but your right the
pstate_ss_bit is a bit of a fiddle. I'll see if there is a neater way.

>
>
>> +
>>  	/* Don't run the guest */
>>  	bool pause;
>>  
>> @@ -143,6 +150,7 @@ struct kvm_vcpu_arch {
>>  
>>  #define vcpu_gp_regs(v)		(&(v)->arch.ctxt.gp_regs)
>>  #define vcpu_sys_reg(v,r)	((v)->arch.ctxt.sys_regs[(r)])
>> +#define vcpu_debug_saved_reg(v, r) ((v)->arch.debug_saved_regs.r)
>>  /*
>>   * CP14 and CP15 live in the same array, as they are backed by the
>>   * same system registers.
>> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
>> index cff0475..b32362c 100644
>> --- a/arch/arm64/kvm/debug.c
>> +++ b/arch/arm64/kvm/debug.c
>> @@ -19,8 +19,16 @@
>>  
>>  #include <linux/kvm_host.h>
>>  
>> +#include <asm/debug-monitors.h>
>> +#include <asm/kvm_asm.h>
>>  #include <asm/kvm_arm.h>
>>  #include <asm/kvm_host.h>
>> +#include <asm/kvm_emulate.h>
>> +
>> +/* These are the bits of MDSCR_EL1 we may mess with */
>> +#define MDSCR_EL1_DEBUG_BITS	(DBG_MDSCR_SS | \
>> +				DBG_MDSCR_KDE | \
>> +				DBG_MDSCR_MDE)
>
> _MASK instead of _BITS ?
>
>>  
>>  /**
>>   * kvm_arch_setup_debug - set-up debug related stuff
>> @@ -51,15 +59,46 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  	else
>>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>>  
>> -	/* Trap breakpoints? */
>> -	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
>> +	/* Is Guest debugging in effect? */
>> +	if (vcpu->guest_debug) {
>>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
>> -	else
>> -		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>>  
>> +		/* Save pstate/mdscr */
>> +		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
>> +			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
>> +		vcpu_debug_saved_reg(vcpu, mdscr_el1_bits) =
>> +			vcpu_sys_reg(vcpu, MDSCR_EL1) & MDSCR_EL1_DEBUG_BITS;
>
> I think it would be clearer if we embed the masks into helper
> functions, and, assuming we drop the _bits concept too, then
>
> #define SPSR_DEBUG_MASK DBG_SPSR_SS
>
> vcpu_debug_save_regs(vcpu)
> {
>   vcpu->arch.debug_saved_regs.pstate = *vcpu_cpsr(vcpu);
>   vcpu->arch.debug_saved_regs.mdscr_el1 = vcpu_sys_reg(vcpu, MDSCR_EL1);
> }
>
> vcpu_debug_restore_regs(vcpu)
> {
>   *vcpu_cpsr(vcpu) |=
>        (vcpu->arch.debug_saved_regs.pstate & SPSR_DEBUG_MASK);
>   vcpu_sys_reg(vcpu, MDSCR_EL1) |=
>        (vcpu->arch.debug_saved_regs.mdscr_el1 & MDSCR_EL1_DEBUG_MASK)
> }

Makes sense

>
>> +		/*
>> +		 * Single Step (ARM ARM D2.12.3 The software step state
>> +		 * machine)
>> +		 *
>> +		 * If we are doing Single Step we need to manipulate
>> +		 * MDSCR_EL1.SS and PSTATE.SS. If not we need to
>> +		 * suppress the guest from messing with it.
>> +		 */
>> +		if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
>> +			*vcpu_cpsr(vcpu) |=  DBG_SPSR_SS;
>> +			vcpu_sys_reg(vcpu, MDSCR_EL1) |= DBG_MDSCR_SS;
>> +		} else {
>> +			*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>> +			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
>> +		}
>> +
>> +	} else {
>> +		/* Debug operations can go straight to the guest */
>> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>> +	}
>>  }
>>  
>>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>>  {
>> -	/* Nothing to do yet */
>
> This would now just be
>
>   if (vcpu->guest_debug)
>      vcpu_debug_restore_regs(vcpu);
>
>> +	if (vcpu->guest_debug) {
>> +		/* Restore pstate/mdscr bits we may have messed with */
>> +		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>> +		*vcpu_cpsr(vcpu) |= vcpu_debug_saved_reg(vcpu, pstate_ss_bit);
>> +
>> +		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
>> +		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
>> +			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
>> +	}
>>  }
>> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
>> index ed1bbb4..16accae 100644
>> --- a/arch/arm64/kvm/handle_exit.c
>> +++ b/arch/arm64/kvm/handle_exit.c
>> @@ -101,6 +101,7 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>>  	run->debug.arch.hsr = hsr;
>>  
>>  	switch (hsr >> ESR_ELx_EC_SHIFT) {
>> +	case ESR_ELx_EC_SOFTSTP_LOW:
>>  	case ESR_ELx_EC_BKPT32:
>>  	case ESR_ELx_EC_BRK64:
>>  		run->debug.arch.pc = *vcpu_pc(vcpu);
>> @@ -127,6 +128,7 @@ static exit_handle_fn arm_exit_handlers[] = {
>>  	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
>>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
>> +	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
>>  	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
>>  	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>>  };
>> -- 
>> 2.3.4
>> 

-- 
Alex Bennée

Re: [PATCH v2 05/10] KVM: arm: introduce kvm_arch_setup/clear_debug()

[Alex Bennée]

Andrew Jones <drjones@redhat.com> writes:

> On Wed, Apr 01, 2015 at 06:28:55PM +0200, David Hildenbrand wrote:
>> > This is a precursor for later patches which will need to do more to
>> > setup debug state before entering the hyp.S switch code. The existing
>> > functionality for setting mdcr_el2 has been moved out of hyp.S and now
>> > uses the value kept in vcpu->arch.mdcr_el2.
>> > 
>> > This also moves the conditional setting of the TDA bit from the hyp code
>> > into the C code.
>> > 
>> > Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> > 
>> >  create mode 100644 arch/arm64/kvm/debug.c
>> > 
>> > diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
>> > index 41008cd..8c01c97 100644
>> > --- a/arch/arm/include/asm/kvm_host.h
>> > +++ b/arch/arm/include/asm/kvm_host.h
>> > @@ -242,5 +242,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
>> >  static inline void kvm_arch_sync_events(struct kvm *kvm) {}
>> >  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
>> >  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
>> > +static inline void kvm_arch_setup_debug(struct kvm_vcpu *vcpu) {}
>> > +static inline void kvm_arch_clear_debug(struct kvm_vcpu *vcpu) {}
>> 
>> Do you really want to call these functions "kvm_arch.." although they are not
>> defined for other arch and not triggered by common code?
>>
>
> Agreed. If other arches want something similar we can rename/refactor
> later. If these are arm-only functions, then we don't need the generic
> prefix, which is actually even a bit confusing.

They are arm only but I was keeping in mind the need to add guest debug
to ARMv7 at some point.

>
> drew 

-- 
Alex Bennée

Re: [PATCH v2 04/10] KVM: arm: guest debug, add stub KVM_SET_GUEST_DEBUG ioctl

[Alex Bennée]

Andrew Jones <drjones@redhat.com> writes:

> On Wed, Apr 01, 2015 at 05:55:29PM +0200, David Hildenbrand wrote:
>> > This commit adds a stub function to support the KVM_SET_GUEST_DEBUG
>> > ioctl. Currently any operation flag will return EINVAL. Actual
>> 
>> Well it won't return -EINVAL if you push in KVM_GUESTDBG_ENABLE or 0.
>> 
>> "Any unsupported flag will return -EINVAL. For now, only KVM_GUESTDBG_ENABLE is
>> supported, although it won't have any effects."
>> 
>> > functionality will be added with further patches.
>> > 
>> > Signed-off-by: Alex Bennée <alex.bennee@linaro.org>.
>> > 
>> > ---
>> > v2
>> >   - simplified form of the ioctl (stuff will go into setup_debug)
>> > 
>> > diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
>> > index b112efc..06c5064 100644
>> > --- a/Documentation/virtual/kvm/api.txt
>> > +++ b/Documentation/virtual/kvm/api.txt
>> > @@ -2604,7 +2604,7 @@ handled.
>> >  4.87 KVM_SET_GUEST_DEBUG
>> > 
>> >  Capability: KVM_CAP_SET_GUEST_DEBUG
>> > -Architectures: x86, s390, ppc
>> > +Architectures: x86, s390, ppc, arm64
>> >  Type: vcpu ioctl
>> >  Parameters: struct kvm_guest_debug (in)
>> >  Returns: 0 on success; -1 on error
>> > diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
>> > index 5560f74..445933d 100644
>> > --- a/arch/arm/kvm/arm.c
>> > +++ b/arch/arm/kvm/arm.c
>> > @@ -183,6 +183,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>> >  	case KVM_CAP_ARM_PSCI:
>> >  	case KVM_CAP_ARM_PSCI_0_2:
>> >  	case KVM_CAP_READONLY_MEM:
>> > +	case KVM_CAP_SET_GUEST_DEBUG:
>> >  		r = 1;
>> >  		break;
>> >  	case KVM_CAP_COALESCED_MMIO:
>> > @@ -303,10 +304,21 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>> >  	kvm_arm_set_running_vcpu(NULL);
>> >  }
>> > 
>> > +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)
>> 
>> That makes me rather think that it is another flag.
>> 
>> We(s390x) use VALID_GUESTDBG_FLAGS, what about that or KVM_GUESTDBG_VALID_MASK?
>> 
>> > +
>> >  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>> >  					struct kvm_guest_debug *dbg)
>> >  {
>> > -	return -EINVAL;
>> > +	if (dbg->control & KVM_GUESTDBG_ENABLE) {
>> > +		if (dbg->control & ~KVM_GUESTDBG_VALID)
>> > +			return -EINVAL;
>> 
>> I'd move that check directly to the start of the function and bail out on any
>> unsupported flag.
>> 
>> > +
>> > +		vcpu->guest_debug = dbg->control;
>> > +	} else {
>> > +		/* If not enabled clear all flags */
>> > +		vcpu->guest_debug = 0;
>> > +	}
>> > +	return 0;
>> >  }
>> > 
>> > 
>> 
>> David
>>
>
> I don't see any follow-up from Alex on this, so I feel the need to
> "+1" all David's comments here.

Yeah they make sense. Will do.

-- 
Alex Bennée

Re: [PATCH v2 08/10] KVM: arm64: guest debug, HW assisted debug support

[Andrew Jones]

On Tue, Mar 31, 2015 at 04:08:06PM +0100, Alex Bennée wrote:
> This adds support for userspace to control the HW debug registers for
> guest debug. We'll only copy the $ARCH defined number across as that is
> all that hyp.S will use anyway. I've moved some helper functions into
> the hw_breakpoint.h header for re-use.
> 
> As with single step we need to tweak the guest registers to enable the
> exceptions so we need to save and restore those bits.
> 
> Two new capabilities have been added to the KVM_EXTENSION ioctl to allow
> userspace to query the number of hardware break and watch points
> available on the host hardware.
> 
> As QEMU tests for watchpoints based on the address and not the PC we
> also need to export the value of far_el2 to userspace.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>    - switched to C setup
>    - replace host debug registers directly into context
>    - minor tweak to api docs
>    - setup right register for debug
>    - add FAR_EL2 to debug exit structure
>    - add support fro trapping debug register access
> 
> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> index 17d4f9c..ac34093 100644
> --- a/Documentation/virtual/kvm/api.txt
> +++ b/Documentation/virtual/kvm/api.txt
> @@ -2627,7 +2627,7 @@ The top 16 bits of the control field are architecture specific control
>  flags which can include the following:
>  
>    - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
> -  - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
> +  - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390, arm64]
>    - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
>    - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
>    - KVM_GUESTDBG_EXIT_PENDING:  trigger an immediate guest exit [s390]
> @@ -2642,6 +2642,10 @@ updated to the correct (supplied) values.
>  The second part of the structure is architecture specific and
>  typically contains a set of debug registers.
>  
> +For arm64 the number of debug registers is implementation defined and
> +can be determined by querying the KVM_CAP_GUEST_DEBUG_HW_BPS and
> +KVM_CAP_GUEST_DEBUG_HW_WPS capabilities.
> +
>  When debug events exit the main run loop with the reason
>  KVM_EXIT_DEBUG with the kvm_debug_exit_arch part of the kvm_run
>  structure containing architecture specific debug information.
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index c1ed8cb..a286026 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -306,6 +306,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  
>  #define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE |    \
>  			    KVM_GUESTDBG_USE_SW_BP | \
> +			    KVM_GUESTDBG_USE_HW_BP | \
>  			    KVM_GUESTDBG_SINGLESTEP)
>  
>  /**
> @@ -328,6 +329,26 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>  			return -EINVAL;
>  
>  		vcpu->guest_debug = dbg->control;
> +
> +		/* Hardware assisted Break and Watch points */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +			int nb = get_num_brps();
> +			int nw = get_num_wrps();
> +
> +			/* Copy across up to IMPDEF debug registers to our
> +			 * shadow copy in the vcpu structure. The debug code
> +			 * will then set them up before we re-enter the guest.
> +			 */

Is it necessary to limit the number copied here by anything other than
what the struct supports? Userspace gave us a struct kvm_guest_debug_arch,
so let's save the whole thing. How about just

vcpu->arch.guest_debug_regs = *dbg;

> +			memcpy(vcpu->arch.guest_debug_regs.dbg_bcr,
> +				dbg->arch.dbg_bcr, sizeof(__u64)*nb);
> +			memcpy(vcpu->arch.guest_debug_regs.dbg_bvr,
> +				dbg->arch.dbg_bvr, sizeof(__u64)*nb);
> +			memcpy(vcpu->arch.guest_debug_regs.dbg_wcr,
> +				dbg->arch.dbg_wcr, sizeof(__u64)*nw);
> +			memcpy(vcpu->arch.guest_debug_regs.dbg_wvr,
> +				dbg->arch.dbg_wvr, sizeof(__u64)*nw);
> +		}
> +
>  	} else {
>  		/* If not enabled clear all flags */
>  		vcpu->guest_debug = 0;
> diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
> index 52b484b..c450552 100644
> --- a/arch/arm64/include/asm/hw_breakpoint.h
> +++ b/arch/arm64/include/asm/hw_breakpoint.h
> @@ -130,6 +130,18 @@ static inline void ptrace_hw_copy_thread(struct task_struct *task)
>  }
>  #endif
>  
> +/* Determine number of BRP registers available. */
> +static inline int get_num_brps(void)
> +{
> +	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
> +}
> +
> +/* Determine number of WRP registers available. */
> +static inline int get_num_wrps(void)
> +{
> +	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
> +}
> +
>  extern struct pmu perf_ops_bp;
>  
>  #endif	/* __KERNEL__ */
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 6a33647..2c359c9 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -106,8 +106,9 @@ struct kvm_vcpu_arch {
>  	/* Exception Information */
>  	struct kvm_vcpu_fault_info fault;
>  
> -	/* Debug state */
> +	/* Guest debug state */
>  	u64 debug_flags;
> +	struct kvm_guest_debug_arch guest_debug_regs;
>  
>  	/* Pointer to host CPU context */
>  	kvm_cpu_context_t *host_cpu_context;
> @@ -126,6 +127,7 @@ struct kvm_vcpu_arch {
>  		u32	pstate_ss_bit;
>  		u32	mdscr_el1_bits;
>  
> +		struct kvm_guest_debug_arch debug_regs;
>  	} debug_saved_regs;
>  
>  	/* Don't run the guest */
> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
> index 6ee70a0..73f21e4 100644
> --- a/arch/arm64/include/uapi/asm/kvm.h
> +++ b/arch/arm64/include/uapi/asm/kvm.h
> @@ -118,6 +118,7 @@ struct kvm_guest_debug_arch {
>  struct kvm_debug_exit_arch {
>  	__u64 pc;
>  	__u32 hsr;
> +	__u64 far;	/* used for watchpoints */
>  };
>  
>  struct kvm_sync_regs {
> diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
> index 98bbe06..923be44 100644
> --- a/arch/arm64/kernel/hw_breakpoint.c
> +++ b/arch/arm64/kernel/hw_breakpoint.c
> @@ -49,18 +49,6 @@ static DEFINE_PER_CPU(int, stepping_kernel_bp);
>  static int core_num_brps;
>  static int core_num_wrps;
>  
> -/* Determine number of BRP registers available. */
> -static int get_num_brps(void)
> -{
> -	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
> -}
> -
> -/* Determine number of WRP registers available. */
> -static int get_num_wrps(void)
> -{
> -	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
> -}
> -
>  int hw_breakpoint_slots(int type)
>  {
>  	/*
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index b32362c..3b368f3 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -50,14 +50,19 @@
>  
>  void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  {
> +	bool trap_debug = false;
> +
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>  
> -	/* Trap debug register access? */
> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);

This is 2 patches too early. You don't introduce this tracepoint until
later.

> +
> +	/*
> +	 * If we are not treating debug registers are dirty we need
                                                  ^^ as
> +	 * to trap if the guest starts accessing them.
> +	 */
>  	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
> -		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
> -	else
> -		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
> +		trap_debug = true;

How about just

bool trap_debug = !(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY);

at the top instead?

>  
>  	/* Is Guest debugging in effect? */
>  	if (vcpu->guest_debug) {
> @@ -84,10 +89,69 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
>  		}
>  
> +		/*
> +		 * HW Break/Watch points
> +		 */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +			int nb = get_num_brps();
> +			int nw = get_num_wrps();
> +			struct kvm_guest_debug_arch *saved, *host;
> +
> +			vcpu_sys_reg(vcpu, MDSCR_EL1) |=
> +				(DBG_MDSCR_KDE|DBG_MDSCR_MDE);
> +
> +			/*
> +			 * First we need to make a copy of the guest
> +			 * debug registers before we wipe them out
> +			 * with the ones we want to use.
> +			 */
> +			saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
> +			host = &vcpu->arch.guest_debug_regs;

Again, I don't think we need to be so precise with our copy. We can
always copy extra without problems. It's just what we use that matters.
And how about a couple helpers?

debug_regs_save_to(struct kvm_vcpu *vcpu, struct kvm_guest_debug_arch *dst)
{
  memcpy(dst->dbg_bcr, &vcpu_sys_reg(vcpu, DBGBCR0_EL1), sizeof(u64)*KVM_ARM_NDBG_REGS);
  memcpy(dst->dbg_bvr, &vcpu_sys_reg(vcpu, DBGBVR0_EL1), sizeof(u64)*KVM_ARM_NDBG_REGS);
  memcpy(dst->dbg_wcr, &vcpu_sys_reg(vcpu, DBGWCR0_EL1), sizeof(u64)*KVM_ARM_NDBG_REGS);
  memcpy(dst->dbg_wvr, &vcpu_sys_reg(vcpu, DBGWVR0_EL1), sizeof(u64)*KVM_ARM_NDBG_REGS);
}
debug_regs_restore_from(struct kvm_vcpu *vcpu, struct kvm_guest_debug_arch *src)
{
  memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1), src->dbg_bcr, sizeof(u64)*KVM_ARM_NDBG_REGS);
  memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1), src->dbg_bvr, sizeof(u64)*KVM_ARM_NDBG_REGS);
  memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1), src->dbg_wcr, sizeof(u64)*KVM_ARM_NDBG_REGS);
  memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1), src->dbg_wvr, sizeof(u64)*KVM_ARM_NDBG_REGS);
}

> +
> +			/* Save guest values */
> +			memcpy(&saved->dbg_bcr,
> +			       &vcpu_sys_reg(vcpu, DBGBCR0_EL1),
> +			       sizeof(__u64)*nb);
> +			memcpy(&saved->dbg_bvr,
> +			       &vcpu_sys_reg(vcpu, DBGBVR0_EL1),
> +			       sizeof(__u64)*nb);
> +			memcpy(&saved->dbg_wcr,
> +			       &vcpu_sys_reg(vcpu, DBGWCR0_EL1),
> +			       sizeof(__u64)*nw);
> +			memcpy(&saved->dbg_wvr,
> +			       &vcpu_sys_reg(vcpu, DBGWVR0_EL1),
> +			       sizeof(__u64)*nw);
> +
> +			/* Copy in host values */
> +			memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1),
> +			       &host->dbg_bcr,
> +			       sizeof(__u64)*nb);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1),
> +			       &host->dbg_bvr,
> +			       sizeof(__u64)*nb);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1),
> +			       &host->dbg_wcr,
> +			       sizeof(__u64)*nw);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
> +			       &host->dbg_wvr,
> +			       sizeof(__u64)*nw);
> +
> +			/* Make sure hyp.S copies them in/out */
> +			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> +			/* Also track guest changes */
> +			trap_debug = true;
> +		}
> +
>  	} else {
>  		/* Debug operations can go straight to the guest */
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>  	}
> +
> +	/* Trap debug register access? */
> +	if (trap_debug)
> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
> +	else
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>  }
>  
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> @@ -100,5 +164,31 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
>  		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
>  			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
> +
> +		/*
> +		 * If we were using HW debug we need to restore the
> +		 * values the guest had set them up with
> +		 */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +			struct kvm_guest_debug_arch *regs;
> +			int nb = get_num_brps();
> +			int nw = get_num_wrps();
> +
> +			regs = &vcpu_debug_saved_reg(vcpu, debug_regs);
debug_regs_restore_from(vcpu, regs);
> +
> +			/* Restore the saved debug register values */
> +			memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1),
> +			       &regs->dbg_bcr,
> +			       sizeof(__u64)*nb);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1),
> +			       &regs->dbg_bvr,
> +			       sizeof(__u64)*nb);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1),
> +			       &regs->dbg_wcr,
> +			       sizeof(__u64)*nw);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
> +			       &regs->dbg_wvr,
> +			       sizeof(__u64)*nw);
> +		}
>  	}
>  }
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index 16accae..460a1aa 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -101,7 +101,11 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  	run->debug.arch.hsr = hsr;
>  
>  	switch (hsr >> ESR_ELx_EC_SHIFT) {
> +	case ESR_ELx_EC_WATCHPT_LOW:
> +		run->debug.arch.far = vcpu->arch.fault.far_el2;
> +		/* fall through */
>  	case ESR_ELx_EC_SOFTSTP_LOW:
> +	case ESR_ELx_EC_BREAKPT_LOW:
>  	case ESR_ELx_EC_BKPT32:
>  	case ESR_ELx_EC_BRK64:
>  		run->debug.arch.pc = *vcpu_pc(vcpu);
> @@ -129,6 +133,8 @@ static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
> +	[ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug,
> +	[ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
>  	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
>  	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>  };
> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
> index 0b43265..c2732c7 100644
> --- a/arch/arm64/kvm/reset.c
> +++ b/arch/arm64/kvm/reset.c
> @@ -64,6 +64,12 @@ int kvm_arch_dev_ioctl_check_extension(long ext)
>  	case KVM_CAP_ARM_EL1_32BIT:
>  		r = cpu_has_32bit_el1();
>  		break;
> +	case KVM_CAP_GUEST_DEBUG_HW_BPS:
> +		r = get_num_brps();
> +		break;
> +	case KVM_CAP_GUEST_DEBUG_HW_WPS:
> +		r  = get_num_wrps();
> +		break;
>  	default:
>  		r = 0;
>  	}
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index c370b40..be9b188 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -196,16 +196,49 @@ static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
>   * - If the dirty bit is set, save guest registers, restore host
>   *   registers and clear the dirty bit. This ensure that the host can
>   *   now use the debug registers.
> + *
> + * We also use this mechanism to set-up the debug registers for guest
setup
> + * debugging. If this is the case we want to ensure the guest sees
> + * the right versions of the registers - even if they are not going to
> + * be effective while guest debug is using HW debug.
> + *
no need for this blank comment line
>   */
> +
>  static bool trap_debug_regs(struct kvm_vcpu *vcpu,
>  			    const struct sys_reg_params *p,
>  			    const struct sys_reg_desc *r)
>  {
> -	if (p->is_write) {
> -		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
> -		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +		struct kvm_guest_debug_arch *saved;
> +		__u64 *val;
> +
> +		saved = &vcpu_debug_saved_reg(vcpu, debug_regs);

Here we don't bother enforcing num_brps/wrps, which is fine by me.

> +
> +		if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
> +			val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
> +		else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
> +			val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
> +		else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
> +			val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
> +		else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
> +			val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
> +		else {
> +			kvm_err("Bad register index %d\n", r->reg);
> +			return false;
> +		}
> +
> +		if (p->is_write)
> +			*val = *vcpu_reg(vcpu, p->Rt);
> +		else
> +			*vcpu_reg(vcpu, p->Rt) = *val;
> +
>  	} else {
> -		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +		if (p->is_write) {
> +			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
> +			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> +		} else {
> +			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +		}
>  	}
>  
>  	return true;
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index ce2db14..0e48c0d 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -771,6 +771,8 @@ struct kvm_ppc_smmu_info {
>  #define KVM_CAP_PPC_ENABLE_HCALL 104
>  #define KVM_CAP_CHECK_EXTENSION_VM 105
>  #define KVM_CAP_S390_USER_SIGP 106
> +#define KVM_CAP_GUEST_DEBUG_HW_BPS 107
> +#define KVM_CAP_GUEST_DEBUG_HW_WPS 108
>  
>  #ifdef KVM_CAP_IRQ_ROUTING
>  
> -- 
> 2.3.4
> 

Re: [PATCH v2 09/10] KVM: arm64: trap nested debug register access

[Andrew Jones]

On Tue, Mar 31, 2015 at 04:08:07PM +0100, Alex Bennée wrote:
> When we are using the hardware registers for guest debug we need to deal
> with the guests access to them. There is already a mechanism for dealing
> with these accesses so we build on top of that.
> 
>   - mdscr_el1_bits is renamed as we save the whole register
>   - any access to mdscr_el1 is now stored in the mirror location
>   - if we are using HW assisted debug we do the same with DBG[WB][CV]R
> 
> There is one register (MDCCINT_EL1) which guest debug doesn't care about
> so this behaves as before.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 2c359c9..3d32d45 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -122,10 +122,13 @@ struct kvm_vcpu_arch {
>  	 * here.
>  	 */
>  
> -	/* Registers pre any guest debug manipulations */
> +	/* Registers before any guest debug manipulations. These

starting comment /* on own line

> +	 * shadow registers are updated by the kvm_handle_sys_reg
> +	 * trap handler if the guest accesses or updates them
> +	 */
>  	struct {
>  		u32	pstate_ss_bit;
> -		u32	mdscr_el1_bits;
> +		u32	mdscr_el1;
>  
>  		struct kvm_guest_debug_arch debug_regs;
>  	} debug_saved_regs;
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index 3b368f3..638c111 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -55,8 +55,6 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>  
> -	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
> -

I guess I'll see this come back in the next patch. You must be playing
'now you see me, now you don't'

>  	/*
>  	 * If we are not treating debug registers are dirty we need
>  	 * to trap if the guest starts accessing them.
> @@ -71,8 +69,10 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  		/* Save pstate/mdscr */
>  		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
>  			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
> -		vcpu_debug_saved_reg(vcpu, mdscr_el1_bits) =
> -			vcpu_sys_reg(vcpu, MDSCR_EL1) & MDSCR_EL1_DEBUG_BITS;
> +
> +		vcpu_debug_saved_reg(vcpu, mdscr_el1) =
> +			vcpu_sys_reg(vcpu, MDSCR_EL1);
> +
>  		/*
>  		 * Single Step (ARM ARM D2.12.3 The software step state
>  		 * machine)
> @@ -161,9 +161,8 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>  		*vcpu_cpsr(vcpu) |= vcpu_debug_saved_reg(vcpu, pstate_ss_bit);
>  
> -		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
> -		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
> -			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
> +		vcpu_sys_reg(vcpu, MDSCR_EL1) =
> +			vcpu_debug_saved_reg(vcpu, mdscr_el1);
>  
>  		/*
>  		 * If we were using HW debug we need to restore the
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index be9b188..d43d7d1 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -208,39 +208,61 @@ static bool trap_debug_regs(struct kvm_vcpu *vcpu,
>  			    const struct sys_reg_params *p,
>  			    const struct sys_reg_desc *r)
>  {
> -	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> -		struct kvm_guest_debug_arch *saved;
> -		__u64 *val;
> -
> -		saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
> -
> -		if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
> -			val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
> -		else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
> -			val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
> -		else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
> -			val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
> -		else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
> -			val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
> -		else {
> -			kvm_err("Bad register index %d\n", r->reg);
> -			return false;
> +	if (vcpu->guest_debug) {
> +
> +		/* MDSCR_EL1 */
> +		if (r->reg == MDSCR_EL1) {
> +			if (p->is_write)
> +				vcpu_debug_saved_reg(vcpu, mdscr_el1) =
> +					*vcpu_reg(vcpu, p->Rt);
> +			else
> +				*vcpu_reg(vcpu, p->Rt) =
> +					vcpu_debug_saved_reg(vcpu, mdscr_el1);

With this lines wrapping, {}'s might be nice.

> +
> +			return true;
>  		}
>  
> -		if (p->is_write)
> -			*val = *vcpu_reg(vcpu, p->Rt);
> -		else
> -			*vcpu_reg(vcpu, p->Rt) = *val;
> +		/* MDCCINT_EL1 */
> +		if (r->reg == MDCCINT_EL1)
> +			goto old;

"old"? As in the old way this worked? Someday (soon) all this code will
be "old". How about just 'out'? Or use some other way to get the flow
such that we avoid code duplication, but doesn't require a goto?

> +
> +		/* We only shadow DBG* if guest being debugged */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +			struct kvm_guest_debug_arch *saved;
> +			__u64 *val;
> +
> +			saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
> +
> +			if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
> +				val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
> +			else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
> +				val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
> +			else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
> +				val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
> +			else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
> +				val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
> +			else {
> +				kvm_err("Bad register index %d\n", r->reg);
> +				return false;
> +			}
>  
> -	} else {
> -		if (p->is_write) {
> -			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
> -			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> -		} else {
> -			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +			if (p->is_write)
> +				*val = *vcpu_reg(vcpu, p->Rt);
> +			else
> +				*vcpu_reg(vcpu, p->Rt) = *val;
> +
> +			return true;
>  		}
>  	}
>  
> +old:
> +	if (p->is_write) {
> +		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
> +		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> +	} else {
> +		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +	}
> +
>  	return true;
>  }
>  
> -- 
> 2.3.4
> 

Re: [PATCH v2 10/10] KVM: arm64: add trace points for guest_debug debug

[Andrew Jones]

$SUBJECT | sed s/guest_debug debug/guest debug/ ?

On Tue, Mar 31, 2015 at 04:08:08PM +0100, Alex Bennée wrote:
> This includes trace points for:
>   kvm_arch_setup_guest_debug
>   kvm_arch_clear_guest_debug
>   kvm_handle_guest_debug
> 
> I've also added some generic register setting trace events so I can
> watch the register values being built up over time. The local
> dump_dbg_regs() function dumps all the HW BKPT and WPT registers.
> 
> I've also added a #define trace_dreg to shorten some lines.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index 638c111..7c96288 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -25,12 +25,37 @@
>  #include <asm/kvm_host.h>
>  #include <asm/kvm_emulate.h>
>  
> +#include "trace.h"
> +
> +#define trace_dreg(name, value) trace_kvm_arch_setup_debug_reg32(name, value)

I don't think this name shortener should be necessary

> +
>  /* These are the bits of MDSCR_EL1 we may mess with */
>  #define MDSCR_EL1_DEBUG_BITS	(DBG_MDSCR_SS | \
>  				DBG_MDSCR_KDE | \
>  				DBG_MDSCR_MDE)
>  
>  /**
> + * dump_dbg_regs - simple debug helper
> + *
> + * This provides a simple helper to dump the HW debug registers
> + */
> +static void dump_dbg_regs(struct kvm_vcpu *vcpu, int nb, int nw)
> +{
> +	int i;
> +
> +	for (i = 0; i < nb; i++) {
> +		trace_printk("bkpt%d: 0x%08x:0x%llx\n", i,
> +			(u32) vcpu_sys_reg(vcpu, DBGBCR0_EL1 + i),
> +			vcpu_sys_reg(vcpu, DBGBVR0_EL1 + i));
> +	}
> +	for (i = 0; i < nb; i++) {
> +		trace_printk("wtpt%d: 0x%08x:0x%llx\n", i,
> +			(u32) vcpu_sys_reg(vcpu, DBGWCR0_EL1 + i),
> +			vcpu_sys_reg(vcpu, DBGWVR0_EL1 + i));
> +	}
> +}
> +
> +/**
>   * kvm_arch_setup_debug - set-up debug related stuff
>   *
>   * @vcpu:	the vcpu pointer
> @@ -52,9 +77,13 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  {
>  	bool trap_debug = false;
>  
> +	trace_kvm_arch_setup_debug(vcpu->guest_debug);
> +
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>  
> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);

Not using your trace_dreg here? I guess you didn't think it was
necessary either.

> +
>  	/*
>  	 * If we are not treating debug registers are dirty we need
>  	 * to trap if the guest starts accessing them.
> @@ -66,6 +95,8 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	if (vcpu->guest_debug) {
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
>  
> +		trace_dreg("MDCR_EL2", vcpu->arch.mdcr_el2);
> +
>  		/* Save pstate/mdscr */
>  		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
>  			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
> @@ -73,6 +104,11 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  		vcpu_debug_saved_reg(vcpu, mdscr_el1) =
>  			vcpu_sys_reg(vcpu, MDSCR_EL1);
>  
> +		trace_dreg("Save: PSTATE.SS",
> +			vcpu_debug_saved_reg(vcpu, pstate_ss_bit));
> +		trace_dreg("Save: MDSCR",
> +			vcpu_debug_saved_reg(vcpu, mdscr_el1));
> +
>  		/*
>  		 * Single Step (ARM ARM D2.12.3 The software step state
>  		 * machine)
> @@ -88,6 +124,8 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  			*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>  			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
>  		}
> +		trace_dreg("SPSR_EL2", *vcpu_cpsr(vcpu));
> +		trace_dreg("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
>  
>  		/*
>  		 * HW Break/Watch points
> @@ -136,6 +174,9 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  			       &host->dbg_wvr,
>  			       sizeof(__u64)*nw);
>  
> +			if (trace_kvm_arch_setup_debug_reg32_enabled())
> +				dump_dbg_regs(vcpu, nb, nw);
> +
>  			/* Make sure hyp.S copies them in/out */
>  			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
>  			/* Also track guest changes */
> @@ -147,15 +188,24 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>  	}
>  
> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
> +	trace_kvm_arch_setup_debug_reg32("MDSCR_EL1",
> +					vcpu_sys_reg(vcpu, MDSCR_EL1));
> +
> +
>  	/* Trap debug register access? */
>  	if (trap_debug)
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>  	else
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
> +
> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
>  }
>  
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  {
> +	trace_kvm_arch_clear_debug(vcpu->guest_debug);
> +
>  	if (vcpu->guest_debug) {
>  		/* Restore pstate/mdscr bits we may have messed with */
>  		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
> @@ -164,6 +214,8 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  		vcpu_sys_reg(vcpu, MDSCR_EL1) =
>  			vcpu_debug_saved_reg(vcpu, mdscr_el1);
>  
> +		trace_dreg("SPSR_EL2", *vcpu_cpsr(vcpu));
> +		trace_dreg("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
>  		/*
>  		 * If we were using HW debug we need to restore the
>  		 * values the guest had set them up with
> @@ -188,6 +240,10 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
>  			       &regs->dbg_wvr,
>  			       sizeof(__u64)*nw);
> +
> +			if (trace_kvm_arch_setup_debug_reg32_enabled())
> +				dump_dbg_regs(vcpu, nb, nw);
> +
>  		}
>  	}
>  }
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index 460a1aa..c6cc69a 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -97,6 +97,8 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  {
>  	u32 hsr = kvm_vcpu_get_hsr(vcpu);
>  
> +	trace_kvm_handle_guest_debug(*vcpu_pc(vcpu), hsr);
> +
>  	run->exit_reason = KVM_EXIT_DEBUG;
>  	run->debug.arch.hsr = hsr;
>  
> diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h
> index 157416e9..dffdb49 100644
> --- a/arch/arm64/kvm/trace.h
> +++ b/arch/arm64/kvm/trace.h
> @@ -44,6 +44,72 @@ TRACE_EVENT(kvm_hvc_arm64,
>  		  __entry->vcpu_pc, __entry->r0, __entry->imm)
>  );
>  
> +TRACE_EVENT(kvm_handle_guest_debug,
> +	TP_PROTO(unsigned long vcpu_pc, u32 hsr),
> +	TP_ARGS(vcpu_pc, hsr),
> +
> +	TP_STRUCT__entry(
> +		__field(unsigned long,	vcpu_pc)
> +		__field(u32,		hsr)
> +	),
> +
> +	TP_fast_assign(
> +		__entry->vcpu_pc = vcpu_pc;
> +		__entry->hsr = hsr;
> +	),
> +
> +	TP_printk("debug exception at 0x%08lx (HSR: 0x%08x)",
> +		__entry->vcpu_pc, __entry->hsr)
> +);
> +
> +
> +TRACE_EVENT(kvm_arch_setup_debug,
> +	TP_PROTO(__u32 guest_debug),
> +	TP_ARGS(guest_debug),
> +
> +	TP_STRUCT__entry(
> +		__field(__u32, guest_debug)
> +	),
> +
> +	TP_fast_assign(
> +		__entry->guest_debug = guest_debug;
> +	),
> +
> +	TP_printk("flags: 0x%08x", __entry->guest_debug)
> +);
> +
> +TRACE_EVENT(kvm_arch_clear_debug,
> +	TP_PROTO(__u32 guest_debug),
> +	TP_ARGS(guest_debug),
> +
> +	TP_STRUCT__entry(
> +		__field(__u32, guest_debug)
> +	),
> +
> +	TP_fast_assign(
> +		__entry->guest_debug = guest_debug;
> +	),
> +
> +	TP_printk("flags: 0x%08x", __entry->guest_debug)
> +);
> +
> +TRACE_EVENT(kvm_arch_setup_debug_reg32,
> +	TP_PROTO(const char *name, __u32 value),
> +	TP_ARGS(name, value),
> +
> +	TP_STRUCT__entry(
> +		__field(const char *, name)
> +		__field(__u32, value)
> +	),
> +
> +	TP_fast_assign(
> +		__entry->name = name;
> +		__entry->value = value;
> +	),
> +
> +	TP_printk("%s now 0x%08x", __entry->name, __entry->value)
> +);
> +
>  #endif /* _TRACE_ARM64_KVM_H */
>  
>  #undef TRACE_INCLUDE_PATH
> -- 
> 2.3.4
> 

This looks like it would be super noisy. I'm sure that's nice for
developing/debugging, but can't we reduce the, now debugged, version
down to just one trace point per unique, and interesting, path? If you
have several variables you want to see, then maybe they can all be
on the same line.

Also, I'm not sure about the necessity of the dump_dbg_regs function,
but I think you should be able to embed it in the trace event code
using __print_array

drew

Re: [PATCH v2 01/10] KVM: add commentary for kvm_debug_exit_arch struct

[Andrew Jones]

On Tue, Mar 31, 2015 at 04:07:59PM +0100, Alex Bennée wrote:
> Bring into line with the commentary for the other structures and their
> KVM_EXIT_* cases.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> 
> v2
>   - add comments for other exit types
> 
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index 8055706..5eedf84 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -226,6 +226,7 @@ struct kvm_run {
>  			__u32 count;
>  			__u64 data_offset; /* relative to kvm_run start */
>  		} io;
> +		/* KVM_EXIT_DEBUG */
>  		struct {
>  			struct kvm_debug_exit_arch arch;
>  		} debug;
> @@ -274,6 +275,7 @@ struct kvm_run {
>  			__u32 data;
>  			__u8  is_write;
>  		} dcr;
> +		/* KVM_EXIT_INTERNAL_ERROR */
>  		struct {
>  			__u32 suberror;
>  			/* Available with KVM_CAP_INTERNAL_ERROR_DATA: */
> @@ -284,6 +286,7 @@ struct kvm_run {
>  		struct {
>  			__u64 gprs[32];
>  		} osi;
> +		/* KVM_EXIT_PAPR_HCALL */
>  		struct {
>  			__u64 nr;
>  			__u64 ret;
> -- 
> 2.3.4
> 

I echo David's commit message tweak, otherwise

Reviewed-by: Andrew Jones <drjones@redhat.com>

Re: [PATCH v2 02/10] KVM: define common __KVM_GUESTDBG_USE_SW/HW_BP values

[Andrew Jones]

On Tue, Mar 31, 2015 at 04:08:00PM +0100, Alex Bennée wrote:
> Currently x86, powerpc and soon arm64 use the same two architecture
> specific bits for guest debug support for software and hardware
> breakpoints. This makes the shared values explicit while leaving the
> gate open for another architecture to use some other value if they
> really really want to.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
> index ab4d473..1731569 100644
> --- a/arch/powerpc/include/uapi/asm/kvm.h
> +++ b/arch/powerpc/include/uapi/asm/kvm.h
> @@ -310,8 +310,8 @@ struct kvm_guest_debug_arch {
>   * and upper 16 bits are architecture specific. Architecture specific defines
>   * that ioctl is for setting hardware breakpoint or software breakpoint.
>   */
> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
>  
>  /* definition of registers in kvm_run */
>  struct kvm_sync_regs {
> diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
> index d7dcef5..1438202 100644
> --- a/arch/x86/include/uapi/asm/kvm.h
> +++ b/arch/x86/include/uapi/asm/kvm.h
> @@ -250,8 +250,8 @@ struct kvm_debug_exit_arch {
>  	__u64 dr7;
>  };
>  
> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
>  #define KVM_GUESTDBG_INJECT_DB		0x00040000
>  #define KVM_GUESTDBG_INJECT_BP		0x00080000
>  
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index 5eedf84..ce2db14 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -525,8 +525,16 @@ struct kvm_s390_irq {
>  
>  /* for KVM_SET_GUEST_DEBUG */
>  
> -#define KVM_GUESTDBG_ENABLE		0x00000001
> -#define KVM_GUESTDBG_SINGLESTEP		0x00000002
> +#define KVM_GUESTDBG_ENABLE		(1 << 0)
> +#define KVM_GUESTDBG_SINGLESTEP	(1 << 1)
> +
> +/*
> + * Architecture specific stuff uses the top 16 bits of the field,
> + * however there is some shared commonality for the common cases
> + */
> +#define __KVM_GUESTDBG_USE_SW_BP	(1 << 16)
> +#define __KVM_GUESTDBG_USE_HW_BP	(1 << 17)
> +
>  
>  struct kvm_guest_debug {
>  	__u32 control;
> -- 
> 2.3.4
>

Reviewed-by: Andrew Jones <drjones@redhat.com> 

Re: [PATCH v2 03/10] KVM: arm: guest debug, define API headers

[Andrew Jones]

On Tue, Mar 31, 2015 at 04:08:01PM +0100, Alex Bennée wrote:
> This commit defines the API headers for guest debugging. There are two
> architecture specific debug structures:
> 
>   - kvm_guest_debug_arch, allows us to pass in HW debug registers
>   - kvm_debug_exit_arch, signals the exact debug exit and pc
> 
> The type of debugging being used is control by the architecture specific
controlled
> control bits of the kvm_guest_debug->control flags in the ioctl
> structure.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>    - expose hsr and pc directly to user-space
> 
> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
> index 3ef77a4..6ee70a0 100644
> --- a/arch/arm64/include/uapi/asm/kvm.h
> +++ b/arch/arm64/include/uapi/asm/kvm.h
> @@ -100,10 +100,24 @@ struct kvm_sregs {
>  struct kvm_fpu {
>  };
>  
> +/*
> + * See ARM ARM D7.3: Debug Registers
> + *
> + * The control registers are architecturally defined as 32 bits but are
> + * stored as 64 bit values along side the value registers and aligned
> + * with the rest 64 bit registers in the normal CPU context.
                   ^ of the
> + */
> +#define KVM_ARM_NDBG_REGS 16
>  struct kvm_guest_debug_arch {
> +	__u64 dbg_bcr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_bvr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_wcr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_wvr[KVM_ARM_NDBG_REGS];
>  };
>  
>  struct kvm_debug_exit_arch {
> +	__u64 pc;
> +	__u32 hsr;
>  };
>  
>  struct kvm_sync_regs {
> @@ -207,4 +221,11 @@ struct kvm_arch_memory_slot {
>  
>  #endif
>  
> +/*
> + * Architecture related debug defines - upper 16 bits of
> + * kvm_guest_debug->control
> + */
> +#define KVM_GUESTDBG_USE_SW_BP	        __KVM_GUESTDBG_USE_SW_BP
> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
> +
>  #endif /* __ARM_KVM_H__ */
> -- 
> 2.3.4
>

couple typo nits, otherwise

Reviewed-by: Andrew Jones <drjones@redhat.com>

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Andrew Jones]

On Tue, Mar 31, 2015 at 04:08:04PM +0100, Alex Bennée wrote:
> This adds support for SW breakpoints inserted by userspace.
> 
> We do this by trapping all BKPT exceptions in the
> hypervisor (MDCR_EL2_TDE). The kvm_debug_exit_arch carries the address
> of the exception. If user-space doesn't know of the breakpoint then we
> have a guest inserted breakpoint and the hypervisor needs to start again
> and deliver the exception to guest.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>   - update to use new exit struct
>   - tweak for C setup
>   - do our setup in debug_setup/clear code
>   - fixed up comments
> 
> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> index 06c5064..17d4f9c 100644
> --- a/Documentation/virtual/kvm/api.txt
> +++ b/Documentation/virtual/kvm/api.txt
> @@ -2626,7 +2626,7 @@ when running. Common control bits are:
>  The top 16 bits of the control field are architecture specific control
>  flags which can include the following:
>  
> -  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86]
> +  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
>    - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
>    - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
>    - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 7ea8b0e..d3bc8dc 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -304,7 +304,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  	kvm_arm_set_running_vcpu(NULL);
>  }
>  
> -#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)
> +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)
>  
>  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>  					struct kvm_guest_debug *dbg)
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index 8a29d0b..cff0475 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -45,11 +45,18 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>  
> +	/* Trap debug register access? */
>  	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>  	else
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>  
> +	/* Trap breakpoints? */
> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
> +	else
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
> +
>  }
>  
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index 524fa25..ed1bbb4 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -82,6 +82,37 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  	return 1;
>  }
>  
> +/**
> + * kvm_handle_debug_exception - handle a debug exception instruction
> + *
> + * @vcpu:	the vcpu pointer
> + * @run:	access to the kvm_run structure for results
> + *
> + * We route all debug exceptions through the same handler as we
> + * just need to report the PC and the HSR values to userspace.
> + * Userspace may decide to re-inject the exception and deliver it to
> + * the guest if it wasn't for the host to deal with.
> + */
> +static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
> +{
> +	u32 hsr = kvm_vcpu_get_hsr(vcpu);
> +
> +	run->exit_reason = KVM_EXIT_DEBUG;
> +	run->debug.arch.hsr = hsr;
> +
> +	switch (hsr >> ESR_ELx_EC_SHIFT) {
> +	case ESR_ELx_EC_BKPT32:
> +	case ESR_ELx_EC_BRK64:
> +		run->debug.arch.pc = *vcpu_pc(vcpu);
> +		break;
> +	default:
> +		kvm_err("%s: un-handled case hsr: %#08x\n",
> +			__func__, (unsigned int) hsr);
> +		break;
> +	}
> +	return 0;
> +}
> +
>  static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_WFx]	= kvm_handle_wfx,
>  	[ESR_ELx_EC_CP15_32]	= kvm_handle_cp15_32,
> @@ -96,6 +127,8 @@ static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
> +	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
> +	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>  };
>  
>  static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
> -- 
> 2.3.4
>

I agree with David's name change suggestion, "kvm_handle_guest_debug",
otherwise

Reviewed-by: Andrew Jones <drjones@redhat.com>

Re: [PATCH v2 10/10] KVM: arm64: add trace points for guest_debug debug

[Alex Bennée]

Andrew Jones <drjones@redhat.com> writes:

> $SUBJECT | sed s/guest_debug debug/guest debug/ ?
>
> On Tue, Mar 31, 2015 at 04:08:08PM +0100, Alex Bennée wrote:
>> This includes trace points for:
>>   kvm_arch_setup_guest_debug
>>   kvm_arch_clear_guest_debug
>>   kvm_handle_guest_debug
>> 
>> I've also added some generic register setting trace events so I can
>> watch the register values being built up over time. The local
>> dump_dbg_regs() function dumps all the HW BKPT and WPT registers.
>> 
>> I've also added a #define trace_dreg to shorten some lines.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
>> index 638c111..7c96288 100644
>> --- a/arch/arm64/kvm/debug.c
>> +++ b/arch/arm64/kvm/debug.c
>> @@ -25,12 +25,37 @@
>>  #include <asm/kvm_host.h>
>>  #include <asm/kvm_emulate.h>
>>  
>> +#include "trace.h"
>> +
>> +#define trace_dreg(name, value) trace_kvm_arch_setup_debug_reg32(name, value)
>
> I don't think this name shortener should be necessary

Yeah it was just a quick hack as lower down the trace_ statements kept
wrapping and it looked ugly. I guess it would be less of an issue if I
shorten the setup/clear names.

>
>> +
>>  /* These are the bits of MDSCR_EL1 we may mess with */
>>  #define MDSCR_EL1_DEBUG_BITS	(DBG_MDSCR_SS | \
>>  				DBG_MDSCR_KDE | \
>>  				DBG_MDSCR_MDE)
>>  
>>  /**
>> + * dump_dbg_regs - simple debug helper
>> + *
>> + * This provides a simple helper to dump the HW debug registers
>> + */
>> +static void dump_dbg_regs(struct kvm_vcpu *vcpu, int nb, int nw)
>> +{
>> +	int i;
>> +
>> +	for (i = 0; i < nb; i++) {
>> +		trace_printk("bkpt%d: 0x%08x:0x%llx\n", i,
>> +			(u32) vcpu_sys_reg(vcpu, DBGBCR0_EL1 + i),
>> +			vcpu_sys_reg(vcpu, DBGBVR0_EL1 + i));
>> +	}
>> +	for (i = 0; i < nb; i++) {
>> +		trace_printk("wtpt%d: 0x%08x:0x%llx\n", i,
>> +			(u32) vcpu_sys_reg(vcpu, DBGWCR0_EL1 + i),
>> +			vcpu_sys_reg(vcpu, DBGWVR0_EL1 + i));
>> +	}
>> +}
>> +
>> +/**
>>   * kvm_arch_setup_debug - set-up debug related stuff
>>   *
>>   * @vcpu:	the vcpu pointer
>> @@ -52,9 +77,13 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  {
>>  	bool trap_debug = false;
>>  
>> +	trace_kvm_arch_setup_debug(vcpu->guest_debug);
>> +
>>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>>  
>> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
>
> Not using your trace_dreg here? I guess you didn't think it was
> necessary either.
>
>> +
>>  	/*
>>  	 * If we are not treating debug registers are dirty we need
>>  	 * to trap if the guest starts accessing them.
>> @@ -66,6 +95,8 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  	if (vcpu->guest_debug) {
>>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
>>  
>> +		trace_dreg("MDCR_EL2", vcpu->arch.mdcr_el2);
>> +
>>  		/* Save pstate/mdscr */
>>  		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
>>  			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
>> @@ -73,6 +104,11 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  		vcpu_debug_saved_reg(vcpu, mdscr_el1) =
>>  			vcpu_sys_reg(vcpu, MDSCR_EL1);
>>  
>> +		trace_dreg("Save: PSTATE.SS",
>> +			vcpu_debug_saved_reg(vcpu, pstate_ss_bit));
>> +		trace_dreg("Save: MDSCR",
>> +			vcpu_debug_saved_reg(vcpu, mdscr_el1));
>> +
>>  		/*
>>  		 * Single Step (ARM ARM D2.12.3 The software step state
>>  		 * machine)
>> @@ -88,6 +124,8 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  			*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>>  			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
>>  		}
>> +		trace_dreg("SPSR_EL2", *vcpu_cpsr(vcpu));
>> +		trace_dreg("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
>>  
>>  		/*
>>  		 * HW Break/Watch points
>> @@ -136,6 +174,9 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  			       &host->dbg_wvr,
>>  			       sizeof(__u64)*nw);
>>  
>> +			if (trace_kvm_arch_setup_debug_reg32_enabled())
>> +				dump_dbg_regs(vcpu, nb, nw);
>> +
>>  			/* Make sure hyp.S copies them in/out */
>>  			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
>>  			/* Also track guest changes */
>> @@ -147,15 +188,24 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>>  	}
>>  
>> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
>> +	trace_kvm_arch_setup_debug_reg32("MDSCR_EL1",
>> +					vcpu_sys_reg(vcpu, MDSCR_EL1));
>> +
>> +
>>  	/* Trap debug register access? */
>>  	if (trap_debug)
>>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>>  	else
>>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>> +
>> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
>>  }
>>  
>>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>>  {
>> +	trace_kvm_arch_clear_debug(vcpu->guest_debug);
>> +
>>  	if (vcpu->guest_debug) {
>>  		/* Restore pstate/mdscr bits we may have messed with */
>>  		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>> @@ -164,6 +214,8 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>>  		vcpu_sys_reg(vcpu, MDSCR_EL1) =
>>  			vcpu_debug_saved_reg(vcpu, mdscr_el1);
>>  
>> +		trace_dreg("SPSR_EL2", *vcpu_cpsr(vcpu));
>> +		trace_dreg("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
>>  		/*
>>  		 * If we were using HW debug we need to restore the
>>  		 * values the guest had set them up with
>> @@ -188,6 +240,10 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>>  			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
>>  			       &regs->dbg_wvr,
>>  			       sizeof(__u64)*nw);
>> +
>> +			if (trace_kvm_arch_setup_debug_reg32_enabled())
>> +				dump_dbg_regs(vcpu, nb, nw);
>> +
>>  		}
>>  	}
>>  }
>> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
>> index 460a1aa..c6cc69a 100644
>> --- a/arch/arm64/kvm/handle_exit.c
>> +++ b/arch/arm64/kvm/handle_exit.c
>> @@ -97,6 +97,8 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>>  {
>>  	u32 hsr = kvm_vcpu_get_hsr(vcpu);
>>  
>> +	trace_kvm_handle_guest_debug(*vcpu_pc(vcpu), hsr);
>> +
>>  	run->exit_reason = KVM_EXIT_DEBUG;
>>  	run->debug.arch.hsr = hsr;
>>  
>> diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h
>> index 157416e9..dffdb49 100644
>> --- a/arch/arm64/kvm/trace.h
>> +++ b/arch/arm64/kvm/trace.h
>> @@ -44,6 +44,72 @@ TRACE_EVENT(kvm_hvc_arm64,
>>  		  __entry->vcpu_pc, __entry->r0, __entry->imm)
>>  );
>>  
>> +TRACE_EVENT(kvm_handle_guest_debug,
>> +	TP_PROTO(unsigned long vcpu_pc, u32 hsr),
>> +	TP_ARGS(vcpu_pc, hsr),
>> +
>> +	TP_STRUCT__entry(
>> +		__field(unsigned long,	vcpu_pc)
>> +		__field(u32,		hsr)
>> +	),
>> +
>> +	TP_fast_assign(
>> +		__entry->vcpu_pc = vcpu_pc;
>> +		__entry->hsr = hsr;
>> +	),
>> +
>> +	TP_printk("debug exception at 0x%08lx (HSR: 0x%08x)",
>> +		__entry->vcpu_pc, __entry->hsr)
>> +);
>> +
>> +
>> +TRACE_EVENT(kvm_arch_setup_debug,
>> +	TP_PROTO(__u32 guest_debug),
>> +	TP_ARGS(guest_debug),
>> +
>> +	TP_STRUCT__entry(
>> +		__field(__u32, guest_debug)
>> +	),
>> +
>> +	TP_fast_assign(
>> +		__entry->guest_debug = guest_debug;
>> +	),
>> +
>> +	TP_printk("flags: 0x%08x", __entry->guest_debug)
>> +);
>> +
>> +TRACE_EVENT(kvm_arch_clear_debug,
>> +	TP_PROTO(__u32 guest_debug),
>> +	TP_ARGS(guest_debug),
>> +
>> +	TP_STRUCT__entry(
>> +		__field(__u32, guest_debug)
>> +	),
>> +
>> +	TP_fast_assign(
>> +		__entry->guest_debug = guest_debug;
>> +	),
>> +
>> +	TP_printk("flags: 0x%08x", __entry->guest_debug)
>> +);
>> +
>> +TRACE_EVENT(kvm_arch_setup_debug_reg32,
>> +	TP_PROTO(const char *name, __u32 value),
>> +	TP_ARGS(name, value),
>> +
>> +	TP_STRUCT__entry(
>> +		__field(const char *, name)
>> +		__field(__u32, value)
>> +	),
>> +
>> +	TP_fast_assign(
>> +		__entry->name = name;
>> +		__entry->value = value;
>> +	),
>> +
>> +	TP_printk("%s now 0x%08x", __entry->name, __entry->value)
>> +);
>> +
>>  #endif /* _TRACE_ARM64_KVM_H */
>>  
>>  #undef TRACE_INCLUDE_PATH
>> -- 
>> 2.3.4
>> 
>
> This looks like it would be super noisy. I'm sure that's nice for
> developing/debugging, but can't we reduce the, now debugged, version
> down to just one trace point per unique, and interesting, path? If you
> have several variables you want to see, then maybe they can all be
> on the same line.

Sure - I kept the multiple register references for debugging but your
right it can be compressed down now it's working.

>
> Also, I'm not sure about the necessity of the dump_dbg_regs function,
> but I think you should be able to embed it in the trace event code
> using __print_array

Thanks, I'll look at that.

>
> drew

-- 
Alex Bennée

Re: [PATCH v2 09/10] KVM: arm64: trap nested debug register access

[Alex Bennée]

Andrew Jones <drjones@redhat.com> writes:

> On Tue, Mar 31, 2015 at 04:08:07PM +0100, Alex Bennée wrote:
>> When we are using the hardware registers for guest debug we need to deal
>> with the guests access to them. There is already a mechanism for dealing
>> with these accesses so we build on top of that.
>> 
>>   - mdscr_el1_bits is renamed as we save the whole register
>>   - any access to mdscr_el1 is now stored in the mirror location
>>   - if we are using HW assisted debug we do the same with DBG[WB][CV]R
>> 
>> There is one register (MDCCINT_EL1) which guest debug doesn't care about
>> so this behaves as before.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>> index 2c359c9..3d32d45 100644
>> --- a/arch/arm64/include/asm/kvm_host.h
>> +++ b/arch/arm64/include/asm/kvm_host.h
>> @@ -122,10 +122,13 @@ struct kvm_vcpu_arch {
>>  	 * here.
>>  	 */
>>  
>> -	/* Registers pre any guest debug manipulations */
>> +	/* Registers before any guest debug manipulations. These
>
> starting comment /* on own line
>
>> +	 * shadow registers are updated by the kvm_handle_sys_reg
>> +	 * trap handler if the guest accesses or updates them
>> +	 */
>>  	struct {
>>  		u32	pstate_ss_bit;
>> -		u32	mdscr_el1_bits;
>> +		u32	mdscr_el1;
>>  
>>  		struct kvm_guest_debug_arch debug_regs;
>>  	} debug_saved_regs;
>> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
>> index 3b368f3..638c111 100644
>> --- a/arch/arm64/kvm/debug.c
>> +++ b/arch/arm64/kvm/debug.c
>> @@ -55,8 +55,6 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>>  
>> -	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
>> -
>
> I guess I'll see this come back in the next patch. You must be playing
> 'now you see me, now you don't'

Oops, missed that on the rebase.

>
>>  	/*
>>  	 * If we are not treating debug registers are dirty we need
>>  	 * to trap if the guest starts accessing them.
>> @@ -71,8 +69,10 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  		/* Save pstate/mdscr */
>>  		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
>>  			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
>> -		vcpu_debug_saved_reg(vcpu, mdscr_el1_bits) =
>> -			vcpu_sys_reg(vcpu, MDSCR_EL1) & MDSCR_EL1_DEBUG_BITS;
>> +
>> +		vcpu_debug_saved_reg(vcpu, mdscr_el1) =
>> +			vcpu_sys_reg(vcpu, MDSCR_EL1);
>> +
>>  		/*
>>  		 * Single Step (ARM ARM D2.12.3 The software step state
>>  		 * machine)
>> @@ -161,9 +161,8 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>>  		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>>  		*vcpu_cpsr(vcpu) |= vcpu_debug_saved_reg(vcpu, pstate_ss_bit);
>>  
>> -		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
>> -		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
>> -			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
>> +		vcpu_sys_reg(vcpu, MDSCR_EL1) =
>> +			vcpu_debug_saved_reg(vcpu, mdscr_el1);
>>  
>>  		/*
>>  		 * If we were using HW debug we need to restore the
>> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
>> index be9b188..d43d7d1 100644
>> --- a/arch/arm64/kvm/sys_regs.c
>> +++ b/arch/arm64/kvm/sys_regs.c
>> @@ -208,39 +208,61 @@ static bool trap_debug_regs(struct kvm_vcpu *vcpu,
>>  			    const struct sys_reg_params *p,
>>  			    const struct sys_reg_desc *r)
>>  {
>> -	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
>> -		struct kvm_guest_debug_arch *saved;
>> -		__u64 *val;
>> -
>> -		saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
>> -
>> -		if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
>> -			val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
>> -		else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
>> -			val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
>> -		else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
>> -			val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
>> -		else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
>> -			val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
>> -		else {
>> -			kvm_err("Bad register index %d\n", r->reg);
>> -			return false;
>> +	if (vcpu->guest_debug) {
>> +
>> +		/* MDSCR_EL1 */
>> +		if (r->reg == MDSCR_EL1) {
>> +			if (p->is_write)
>> +				vcpu_debug_saved_reg(vcpu, mdscr_el1) =
>> +					*vcpu_reg(vcpu, p->Rt);
>> +			else
>> +				*vcpu_reg(vcpu, p->Rt) =
>> +					vcpu_debug_saved_reg(vcpu, mdscr_el1);
>
> With this lines wrapping, {}'s might be nice.

My natural inclination is to wrap in {}'s but I know the kernel is a fan
of the single-statement if forms.

>
>> +
>> +			return true;
>>  		}
>>  
>> -		if (p->is_write)
>> -			*val = *vcpu_reg(vcpu, p->Rt);
>> -		else
>> -			*vcpu_reg(vcpu, p->Rt) = *val;
>> +		/* MDCCINT_EL1 */
>> +		if (r->reg == MDCCINT_EL1)
>> +			goto old;
>
> "old"? As in the old way this worked? Someday (soon) all this code will
> be "old". How about just 'out'? Or use some other way to get the flow
> such that we avoid code duplication, but doesn't require a goto?

I'll see if I can structure this better.

>
>> +
>> +		/* We only shadow DBG* if guest being debugged */
>> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
>> +			struct kvm_guest_debug_arch *saved;
>> +			__u64 *val;
>> +
>> +			saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
>> +
>> +			if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
>> +				val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
>> +			else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
>> +				val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
>> +			else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
>> +				val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
>> +			else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
>> +				val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
>> +			else {
>> +				kvm_err("Bad register index %d\n", r->reg);
>> +				return false;
>> +			}
>>  
>> -	} else {
>> -		if (p->is_write) {
>> -			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
>> -			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
>> -		} else {
>> -			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
>> +			if (p->is_write)
>> +				*val = *vcpu_reg(vcpu, p->Rt);
>> +			else
>> +				*vcpu_reg(vcpu, p->Rt) = *val;
>> +
>> +			return true;
>>  		}
>>  	}
>>  
>> +old:
>> +	if (p->is_write) {
>> +		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
>> +		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
>> +	} else {
>> +		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
>> +	}
>> +
>>  	return true;
>>  }
>>  
>> -- 
>> 2.3.4
>> 

-- 
Alex Bennée

Re: [PATCH v2 08/10] KVM: arm64: guest debug, HW assisted debug support

[Alex Bennée]

Andrew Jones <drjones@redhat.com> writes:

> On Tue, Mar 31, 2015 at 04:08:06PM +0100, Alex Bennée wrote:
>> This adds support for userspace to control the HW debug registers for
>> guest debug. We'll only copy the $ARCH defined number across as that is
>> all that hyp.S will use anyway. I've moved some helper functions into
>> the hw_breakpoint.h header for re-use.
>> 
>> As with single step we need to tweak the guest registers to enable the
>> exceptions so we need to save and restore those bits.
>> 
>> Two new capabilities have been added to the KVM_EXTENSION ioctl to allow
>> userspace to query the number of hardware break and watch points
>> available on the host hardware.
>> 
>> As QEMU tests for watchpoints based on the address and not the PC we
>> also need to export the value of far_el2 to userspace.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>> ---
>> v2
>>    - switched to C setup
>>    - replace host debug registers directly into context
>>    - minor tweak to api docs
>>    - setup right register for debug
>>    - add FAR_EL2 to debug exit structure
>>    - add support fro trapping debug register access
>> 
>> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
>> index 17d4f9c..ac34093 100644
>> --- a/Documentation/virtual/kvm/api.txt
>> +++ b/Documentation/virtual/kvm/api.txt
>> @@ -2627,7 +2627,7 @@ The top 16 bits of the control field are architecture specific control
>>  flags which can include the following:
>>  
>>    - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
>> -  - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
>> +  - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390, arm64]
>>    - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
>>    - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
>>    - KVM_GUESTDBG_EXIT_PENDING:  trigger an immediate guest exit [s390]
>> @@ -2642,6 +2642,10 @@ updated to the correct (supplied) values.
>>  The second part of the structure is architecture specific and
>>  typically contains a set of debug registers.
>>  
>> +For arm64 the number of debug registers is implementation defined and
>> +can be determined by querying the KVM_CAP_GUEST_DEBUG_HW_BPS and
>> +KVM_CAP_GUEST_DEBUG_HW_WPS capabilities.
>> +
>>  When debug events exit the main run loop with the reason
>>  KVM_EXIT_DEBUG with the kvm_debug_exit_arch part of the kvm_run
>>  structure containing architecture specific debug information.
>> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
>> index c1ed8cb..a286026 100644
>> --- a/arch/arm/kvm/arm.c
>> +++ b/arch/arm/kvm/arm.c
>> @@ -306,6 +306,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>>  
>>  #define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE |    \
>>  			    KVM_GUESTDBG_USE_SW_BP | \
>> +			    KVM_GUESTDBG_USE_HW_BP | \
>>  			    KVM_GUESTDBG_SINGLESTEP)
>>  
>>  /**
>> @@ -328,6 +329,26 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>>  			return -EINVAL;
>>  
>>  		vcpu->guest_debug = dbg->control;
>> +
>> +		/* Hardware assisted Break and Watch points */
>> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
>> +			int nb = get_num_brps();
>> +			int nw = get_num_wrps();
>> +
>> +			/* Copy across up to IMPDEF debug registers to our
>> +			 * shadow copy in the vcpu structure. The debug code
>> +			 * will then set them up before we re-enter the guest.
>> +			 */
>
> Is it necessary to limit the number copied here by anything other than
> what the struct supports? Userspace gave us a struct kvm_guest_debug_arch,
> so let's save the whole thing. How about just
>
> vcpu->arch.guest_debug_regs = *dbg;

Makes sense for the ioctl. I'll still limit it in the setup/clear function.

>
>> +			memcpy(vcpu->arch.guest_debug_regs.dbg_bcr,
>> +				dbg->arch.dbg_bcr, sizeof(__u64)*nb);
>> +			memcpy(vcpu->arch.guest_debug_regs.dbg_bvr,
>> +				dbg->arch.dbg_bvr, sizeof(__u64)*nb);
>> +			memcpy(vcpu->arch.guest_debug_regs.dbg_wcr,
>> +				dbg->arch.dbg_wcr, sizeof(__u64)*nw);
>> +			memcpy(vcpu->arch.guest_debug_regs.dbg_wvr,
>> +				dbg->arch.dbg_wvr, sizeof(__u64)*nw);
>> +		}
>> +
>>  	} else {
>>  		/* If not enabled clear all flags */
>>  		vcpu->guest_debug = 0;
>> diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
>> index 52b484b..c450552 100644
>> --- a/arch/arm64/include/asm/hw_breakpoint.h
>> +++ b/arch/arm64/include/asm/hw_breakpoint.h
>> @@ -130,6 +130,18 @@ static inline void ptrace_hw_copy_thread(struct task_struct *task)
>>  }
>>  #endif
>>  
>> +/* Determine number of BRP registers available. */
>> +static inline int get_num_brps(void)
>> +{
>> +	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
>> +}
>> +
>> +/* Determine number of WRP registers available. */
>> +static inline int get_num_wrps(void)
>> +{
>> +	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
>> +}
>> +
>>  extern struct pmu perf_ops_bp;
>>  
>>  #endif	/* __KERNEL__ */
>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>> index 6a33647..2c359c9 100644
>> --- a/arch/arm64/include/asm/kvm_host.h
>> +++ b/arch/arm64/include/asm/kvm_host.h
>> @@ -106,8 +106,9 @@ struct kvm_vcpu_arch {
>>  	/* Exception Information */
>>  	struct kvm_vcpu_fault_info fault;
>>  
>> -	/* Debug state */
>> +	/* Guest debug state */
>>  	u64 debug_flags;
>> +	struct kvm_guest_debug_arch guest_debug_regs;
>>  
>>  	/* Pointer to host CPU context */
>>  	kvm_cpu_context_t *host_cpu_context;
>> @@ -126,6 +127,7 @@ struct kvm_vcpu_arch {
>>  		u32	pstate_ss_bit;
>>  		u32	mdscr_el1_bits;
>>  
>> +		struct kvm_guest_debug_arch debug_regs;
>>  	} debug_saved_regs;
>>  
>>  	/* Don't run the guest */
>> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
>> index 6ee70a0..73f21e4 100644
>> --- a/arch/arm64/include/uapi/asm/kvm.h
>> +++ b/arch/arm64/include/uapi/asm/kvm.h
>> @@ -118,6 +118,7 @@ struct kvm_guest_debug_arch {
>>  struct kvm_debug_exit_arch {
>>  	__u64 pc;
>>  	__u32 hsr;
>> +	__u64 far;	/* used for watchpoints */
>>  };
>>  
>>  struct kvm_sync_regs {
>> diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
>> index 98bbe06..923be44 100644
>> --- a/arch/arm64/kernel/hw_breakpoint.c
>> +++ b/arch/arm64/kernel/hw_breakpoint.c
>> @@ -49,18 +49,6 @@ static DEFINE_PER_CPU(int, stepping_kernel_bp);
>>  static int core_num_brps;
>>  static int core_num_wrps;
>>  
>> -/* Determine number of BRP registers available. */
>> -static int get_num_brps(void)
>> -{
>> -	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
>> -}
>> -
>> -/* Determine number of WRP registers available. */
>> -static int get_num_wrps(void)
>> -{
>> -	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
>> -}
>> -
>>  int hw_breakpoint_slots(int type)
>>  {
>>  	/*
>> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
>> index b32362c..3b368f3 100644
>> --- a/arch/arm64/kvm/debug.c
>> +++ b/arch/arm64/kvm/debug.c
>> @@ -50,14 +50,19 @@
>>  
>>  void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  {
>> +	bool trap_debug = false;
>> +
>>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>>  
>> -	/* Trap debug register access? */
>> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
>
> This is 2 patches too early. You don't introduce this tracepoint until
> later.
>
>> +
>> +	/*
>> +	 * If we are not treating debug registers are dirty we need
>                                                   ^^ as
>> +	 * to trap if the guest starts accessing them.
>> +	 */
>>  	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
>> -		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>> -	else
>> -		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>> +		trap_debug = true;
>
> How about just
>
> bool trap_debug = !(vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY);
>
> at the top instead?
>
>>  
>>  	/* Is Guest debugging in effect? */
>>  	if (vcpu->guest_debug) {
>> @@ -84,10 +89,69 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>>  			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
>>  		}
>>  
>> +		/*
>> +		 * HW Break/Watch points
>> +		 */
>> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
>> +			int nb = get_num_brps();
>> +			int nw = get_num_wrps();
>> +			struct kvm_guest_debug_arch *saved, *host;
>> +
>> +			vcpu_sys_reg(vcpu, MDSCR_EL1) |=
>> +				(DBG_MDSCR_KDE|DBG_MDSCR_MDE);
>> +
>> +			/*
>> +			 * First we need to make a copy of the guest
>> +			 * debug registers before we wipe them out
>> +			 * with the ones we want to use.
>> +			 */
>> +			saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
>> +			host = &vcpu->arch.guest_debug_regs;
>
> Again, I don't think we need to be so precise with our copy. We can
> always copy extra without problems. It's just what we use that matters.
> And how about a couple helpers?
>
> debug_regs_save_to(struct kvm_vcpu *vcpu, struct kvm_guest_debug_arch *dst)
> {
>   memcpy(dst->dbg_bcr, &vcpu_sys_reg(vcpu, DBGBCR0_EL1), sizeof(u64)*KVM_ARM_NDBG_REGS);
>   memcpy(dst->dbg_bvr, &vcpu_sys_reg(vcpu, DBGBVR0_EL1), sizeof(u64)*KVM_ARM_NDBG_REGS);
>   memcpy(dst->dbg_wcr, &vcpu_sys_reg(vcpu, DBGWCR0_EL1), sizeof(u64)*KVM_ARM_NDBG_REGS);
>   memcpy(dst->dbg_wvr, &vcpu_sys_reg(vcpu, DBGWVR0_EL1), sizeof(u64)*KVM_ARM_NDBG_REGS);
> }
> debug_regs_restore_from(struct kvm_vcpu *vcpu, struct kvm_guest_debug_arch *src)
> {
>   memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1), src->dbg_bcr, sizeof(u64)*KVM_ARM_NDBG_REGS);
>   memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1), src->dbg_bvr, sizeof(u64)*KVM_ARM_NDBG_REGS);
>   memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1), src->dbg_wcr, sizeof(u64)*KVM_ARM_NDBG_REGS);
>   memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1), src->dbg_wvr, sizeof(u64)*KVM_ARM_NDBG_REGS);
> }
>
>> +
>> +			/* Save guest values */
>> +			memcpy(&saved->dbg_bcr,
>> +			       &vcpu_sys_reg(vcpu, DBGBCR0_EL1),
>> +			       sizeof(__u64)*nb);
>> +			memcpy(&saved->dbg_bvr,
>> +			       &vcpu_sys_reg(vcpu, DBGBVR0_EL1),
>> +			       sizeof(__u64)*nb);
>> +			memcpy(&saved->dbg_wcr,
>> +			       &vcpu_sys_reg(vcpu, DBGWCR0_EL1),
>> +			       sizeof(__u64)*nw);
>> +			memcpy(&saved->dbg_wvr,
>> +			       &vcpu_sys_reg(vcpu, DBGWVR0_EL1),
>> +			       sizeof(__u64)*nw);
>> +
>> +			/* Copy in host values */
>> +			memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1),
>> +			       &host->dbg_bcr,
>> +			       sizeof(__u64)*nb);
>> +			memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1),
>> +			       &host->dbg_bvr,
>> +			       sizeof(__u64)*nb);
>> +			memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1),
>> +			       &host->dbg_wcr,
>> +			       sizeof(__u64)*nw);
>> +			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
>> +			       &host->dbg_wvr,
>> +			       sizeof(__u64)*nw);
>> +
>> +			/* Make sure hyp.S copies them in/out */
>> +			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
>> +			/* Also track guest changes */
>> +			trap_debug = true;
>> +		}
>> +
>>  	} else {
>>  		/* Debug operations can go straight to the guest */
>>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>>  	}
>> +
>> +	/* Trap debug register access? */
>> +	if (trap_debug)
>> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>> +	else
>> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>>  }
>>  
>>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>> @@ -100,5 +164,31 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>>  		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
>>  		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
>>  			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
>> +
>> +		/*
>> +		 * If we were using HW debug we need to restore the
>> +		 * values the guest had set them up with
>> +		 */
>> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
>> +			struct kvm_guest_debug_arch *regs;
>> +			int nb = get_num_brps();
>> +			int nw = get_num_wrps();
>> +
>> +			regs = &vcpu_debug_saved_reg(vcpu, debug_regs);
> debug_regs_restore_from(vcpu, regs);
>> +
>> +			/* Restore the saved debug register values */
>> +			memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1),
>> +			       &regs->dbg_bcr,
>> +			       sizeof(__u64)*nb);
>> +			memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1),
>> +			       &regs->dbg_bvr,
>> +			       sizeof(__u64)*nb);
>> +			memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1),
>> +			       &regs->dbg_wcr,
>> +			       sizeof(__u64)*nw);
>> +			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
>> +			       &regs->dbg_wvr,
>> +			       sizeof(__u64)*nw);
>> +		}
>>  	}
>>  }
>> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
>> index 16accae..460a1aa 100644
>> --- a/arch/arm64/kvm/handle_exit.c
>> +++ b/arch/arm64/kvm/handle_exit.c
>> @@ -101,7 +101,11 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>>  	run->debug.arch.hsr = hsr;
>>  
>>  	switch (hsr >> ESR_ELx_EC_SHIFT) {
>> +	case ESR_ELx_EC_WATCHPT_LOW:
>> +		run->debug.arch.far = vcpu->arch.fault.far_el2;
>> +		/* fall through */
>>  	case ESR_ELx_EC_SOFTSTP_LOW:
>> +	case ESR_ELx_EC_BREAKPT_LOW:
>>  	case ESR_ELx_EC_BKPT32:
>>  	case ESR_ELx_EC_BRK64:
>>  		run->debug.arch.pc = *vcpu_pc(vcpu);
>> @@ -129,6 +133,8 @@ static exit_handle_fn arm_exit_handlers[] = {
>>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
>>  	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
>> +	[ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug,
>> +	[ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
>>  	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
>>  	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>>  };
>> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
>> index 0b43265..c2732c7 100644
>> --- a/arch/arm64/kvm/reset.c
>> +++ b/arch/arm64/kvm/reset.c
>> @@ -64,6 +64,12 @@ int kvm_arch_dev_ioctl_check_extension(long ext)
>>  	case KVM_CAP_ARM_EL1_32BIT:
>>  		r = cpu_has_32bit_el1();
>>  		break;
>> +	case KVM_CAP_GUEST_DEBUG_HW_BPS:
>> +		r = get_num_brps();
>> +		break;
>> +	case KVM_CAP_GUEST_DEBUG_HW_WPS:
>> +		r  = get_num_wrps();
>> +		break;
>>  	default:
>>  		r = 0;
>>  	}
>> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
>> index c370b40..be9b188 100644
>> --- a/arch/arm64/kvm/sys_regs.c
>> +++ b/arch/arm64/kvm/sys_regs.c
>> @@ -196,16 +196,49 @@ static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
>>   * - If the dirty bit is set, save guest registers, restore host
>>   *   registers and clear the dirty bit. This ensure that the host can
>>   *   now use the debug registers.
>> + *
>> + * We also use this mechanism to set-up the debug registers for guest
> setup
>> + * debugging. If this is the case we want to ensure the guest sees
>> + * the right versions of the registers - even if they are not going to
>> + * be effective while guest debug is using HW debug.
>> + *
> no need for this blank comment line
>>   */
>> +
>>  static bool trap_debug_regs(struct kvm_vcpu *vcpu,
>>  			    const struct sys_reg_params *p,
>>  			    const struct sys_reg_desc *r)
>>  {
>> -	if (p->is_write) {
>> -		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
>> -		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
>> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
>> +		struct kvm_guest_debug_arch *saved;
>> +		__u64 *val;
>> +
>> +		saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
>
> Here we don't bother enforcing num_brps/wrps, which is fine by me.
>
>> +
>> +		if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
>> +			val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
>> +		else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
>> +			val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
>> +		else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
>> +			val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
>> +		else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
>> +			val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
>> +		else {
>> +			kvm_err("Bad register index %d\n", r->reg);
>> +			return false;
>> +		}
>> +
>> +		if (p->is_write)
>> +			*val = *vcpu_reg(vcpu, p->Rt);
>> +		else
>> +			*vcpu_reg(vcpu, p->Rt) = *val;
>> +
>>  	} else {
>> -		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
>> +		if (p->is_write) {
>> +			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
>> +			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
>> +		} else {
>> +			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
>> +		}
>>  	}
>>  
>>  	return true;
>> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
>> index ce2db14..0e48c0d 100644
>> --- a/include/uapi/linux/kvm.h
>> +++ b/include/uapi/linux/kvm.h
>> @@ -771,6 +771,8 @@ struct kvm_ppc_smmu_info {
>>  #define KVM_CAP_PPC_ENABLE_HCALL 104
>>  #define KVM_CAP_CHECK_EXTENSION_VM 105
>>  #define KVM_CAP_S390_USER_SIGP 106
>> +#define KVM_CAP_GUEST_DEBUG_HW_BPS 107
>> +#define KVM_CAP_GUEST_DEBUG_HW_WPS 108
>>  
>>  #ifdef KVM_CAP_IRQ_ROUTING
>>  
>> -- 
>> 2.3.4
>> 

-- 
Alex Bennée

Re: [PATCH v2 01/10] KVM: add commentary for kvm_debug_exit_arch struct

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:07:59PM +0100, Alex Bennée wrote:
> Bring into line with the commentary for the other structures and their
> KVM_EXIT_* cases.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> 
> v2
>   - add comments for other exit types
> 
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index 8055706..5eedf84 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -226,6 +226,7 @@ struct kvm_run {
>  			__u32 count;
>  			__u64 data_offset; /* relative to kvm_run start */
>  		} io;
> +		/* KVM_EXIT_DEBUG */
>  		struct {
>  			struct kvm_debug_exit_arch arch;
>  		} debug;
> @@ -274,6 +275,7 @@ struct kvm_run {
>  			__u32 data;
>  			__u8  is_write;
>  		} dcr;
> +		/* KVM_EXIT_INTERNAL_ERROR */
>  		struct {
>  			__u32 suberror;
>  			/* Available with KVM_CAP_INTERNAL_ERROR_DATA: */
> @@ -284,6 +286,7 @@ struct kvm_run {
>  		struct {
>  			__u64 gprs[32];
>  		} osi;
> +		/* KVM_EXIT_PAPR_HCALL */
>  		struct {
>  			__u64 nr;
>  			__u64 ret;
> -- 
> 2.3.4
> 

I'm fine with this change as it is, but I think it should update the
documenation of the kvm_run structure along with it, and in there it
seems the debug struct is listed as 'unused'; this should be addressed
in this patch set somewhere if we start using this struct?

-Christoffer

Re: [PATCH v2 02/10] KVM: define common __KVM_GUESTDBG_USE_SW/HW_BP values

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:08:00PM +0100, Alex Bennée wrote:
> Currently x86, powerpc and soon arm64 use the same two architecture
> specific bits for guest debug support for software and hardware
> breakpoints. This makes the shared values explicit while leaving the
> gate open for another architecture to use some other value if they
> really really want to.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
> index ab4d473..1731569 100644
> --- a/arch/powerpc/include/uapi/asm/kvm.h
> +++ b/arch/powerpc/include/uapi/asm/kvm.h
> @@ -310,8 +310,8 @@ struct kvm_guest_debug_arch {
>   * and upper 16 bits are architecture specific. Architecture specific defines
>   * that ioctl is for setting hardware breakpoint or software breakpoint.
>   */
> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
>  
>  /* definition of registers in kvm_run */
>  struct kvm_sync_regs {
> diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
> index d7dcef5..1438202 100644
> --- a/arch/x86/include/uapi/asm/kvm.h
> +++ b/arch/x86/include/uapi/asm/kvm.h
> @@ -250,8 +250,8 @@ struct kvm_debug_exit_arch {
>  	__u64 dr7;
>  };
>  
> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
>  #define KVM_GUESTDBG_INJECT_DB		0x00040000
>  #define KVM_GUESTDBG_INJECT_BP		0x00080000
>  
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index 5eedf84..ce2db14 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -525,8 +525,16 @@ struct kvm_s390_irq {
>  
>  /* for KVM_SET_GUEST_DEBUG */
>  
> -#define KVM_GUESTDBG_ENABLE		0x00000001
> -#define KVM_GUESTDBG_SINGLESTEP		0x00000002
> +#define KVM_GUESTDBG_ENABLE		(1 << 0)
> +#define KVM_GUESTDBG_SINGLESTEP	(1 << 1)
> +
> +/*
> + * Architecture specific stuff uses the top 16 bits of the field,

can you be more specific than 'stuff' here?  features?

> + * however there is some shared commonality for the common cases

I don't like this sentence; shared commonality is a pleonasm and the use
of however makes it sounds like there's some caveat here.

If the top 16 bits are indeed arhictecture specific, then I think they
should just be defined in their architecture specific headers.  Unless
the idea here is that there's a fixed set of of flags that architectures
can choose to support, in which case it should simply be defined in the
common header.


> + */
> +#define __KVM_GUESTDBG_USE_SW_BP	(1 << 16)
> +#define __KVM_GUESTDBG_USE_HW_BP	(1 << 17)
> +
>  
>  struct kvm_guest_debug {
>  	__u32 control;
> -- 
> 2.3.4
> 

Re: [PATCH v2 03/10] KVM: arm: guest debug, define API headers

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:08:01PM +0100, Alex Bennée wrote:
> This commit defines the API headers for guest debugging. There are two
> architecture specific debug structures:
> 
>   - kvm_guest_debug_arch, allows us to pass in HW debug registers
>   - kvm_debug_exit_arch, signals the exact debug exit and pc
> 
> The type of debugging being used is control by the architecture specific
> control bits of the kvm_guest_debug->control flags in the ioctl
> structure.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>    - expose hsr and pc directly to user-space
> 
> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
> index 3ef77a4..6ee70a0 100644
> --- a/arch/arm64/include/uapi/asm/kvm.h
> +++ b/arch/arm64/include/uapi/asm/kvm.h
> @@ -100,10 +100,24 @@ struct kvm_sregs {
>  struct kvm_fpu {
>  };
>  
> +/*
> + * See ARM ARM D7.3: Debug Registers

see the ARM ARM for ??

> + *
> + * The control registers are architecturally defined as 32 bits but are
> + * stored as 64 bit values along side the value registers and aligned

do you mean alongside?

> + * with the rest 64 bit registers in the normal CPU context.

rest of the 64 bit

> + */

why do we store them as 64 bit values?  There's nothing prevented us
from defining them as __u32 is there?  Is this to make the ONE_REG
interface accessers more convenient?

> +#define KVM_ARM_NDBG_REGS 16

nit: is NDBG short for something I don't know about or is it
the number of debug registers we are noting here, in which case I think
KVM_ARM_NUM_DBG_REGS is more clear.

>  struct kvm_guest_debug_arch {
> +	__u64 dbg_bcr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_bvr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_wcr[KVM_ARM_NDBG_REGS];
> +	__u64 dbg_wvr[KVM_ARM_NDBG_REGS];
>  };
>  
>  struct kvm_debug_exit_arch {
> +	__u64 pc;
> +	__u32 hsr;
>  };
>  
>  struct kvm_sync_regs {
> @@ -207,4 +221,11 @@ struct kvm_arch_memory_slot {
>  
>  #endif
>  
> +/*
> + * Architecture related debug defines - upper 16 bits of
> + * kvm_guest_debug->control
> + */
> +#define KVM_GUESTDBG_USE_SW_BP	        __KVM_GUESTDBG_USE_SW_BP
> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
> +
>  #endif /* __ARM_KVM_H__ */
> -- 
> 2.3.4
> 

Thanks,
-Christoffer

Re: [PATCH v2 04/10] KVM: arm: guest debug, add stub KVM_SET_GUEST_DEBUG ioctl

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:08:02PM +0100, Alex Bennée wrote:
> This commit adds a stub function to support the KVM_SET_GUEST_DEBUG
> ioctl. Currently any operation flag will return EINVAL. Actual
> functionality will be added with further patches.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>.
> 
> ---
> v2
>   - simplified form of the ioctl (stuff will go into setup_debug)
> 
> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> index b112efc..06c5064 100644
> --- a/Documentation/virtual/kvm/api.txt
> +++ b/Documentation/virtual/kvm/api.txt
> @@ -2604,7 +2604,7 @@ handled.
>  4.87 KVM_SET_GUEST_DEBUG
>  
>  Capability: KVM_CAP_SET_GUEST_DEBUG
> -Architectures: x86, s390, ppc
> +Architectures: x86, s390, ppc, arm64
>  Type: vcpu ioctl
>  Parameters: struct kvm_guest_debug (in)
>  Returns: 0 on success; -1 on error
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 5560f74..445933d 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -183,6 +183,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>  	case KVM_CAP_ARM_PSCI:
>  	case KVM_CAP_ARM_PSCI_0_2:
>  	case KVM_CAP_READONLY_MEM:
> +	case KVM_CAP_SET_GUEST_DEBUG:
>  		r = 1;
>  		break;

shouldn't you wait with advertising this capability until you've
implemented support for it?


Thanks,
-Christoffer

Re: [PATCH v2 05/10] KVM: arm: introduce kvm_arch_setup/clear_debug()

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:08:03PM +0100, Alex Bennée wrote:
> This is a precursor for later patches which will need to do more to
> setup debug state before entering the hyp.S switch code. The existing
> functionality for setting mdcr_el2 has been moved out of hyp.S and now
> uses the value kept in vcpu->arch.mdcr_el2.
> 
> This also moves the conditional setting of the TDA bit from the hyp code
> into the C code.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
>  create mode 100644 arch/arm64/kvm/debug.c
> 
> diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
> index 41008cd..8c01c97 100644
> --- a/arch/arm/include/asm/kvm_host.h
> +++ b/arch/arm/include/asm/kvm_host.h
> @@ -242,5 +242,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
>  static inline void kvm_arch_sync_events(struct kvm *kvm) {}
>  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
>  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
> +static inline void kvm_arch_setup_debug(struct kvm_vcpu *vcpu) {}
> +static inline void kvm_arch_clear_debug(struct kvm_vcpu *vcpu) {}
>  
>  #endif /* __ARM_KVM_HOST_H__ */
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 445933d..7ea8b0e 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -523,6 +523,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  
>  		kvm_vgic_flush_hwstate(vcpu);
>  		kvm_timer_flush_hwstate(vcpu);
> +		kvm_arch_setup_debug(vcpu);
>  
>  		local_irq_disable();
>  
> @@ -569,6 +570,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  		 * Back from guest
>  		 *************************************************************/
>  
> +		kvm_arch_clear_debug(vcpu);
>  		kvm_timer_sync_hwstate(vcpu);
>  		kvm_vgic_sync_hwstate(vcpu);
>  
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 8ac3c70..0631840 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -101,6 +101,7 @@ struct kvm_vcpu_arch {
>  
>  	/* HYP configuration */
>  	u64 hcr_el2;
> +	u32 mdcr_el2;
>  
>  	/* Exception Information */
>  	struct kvm_vcpu_fault_info fault;
> @@ -257,4 +258,7 @@ static inline void kvm_arch_sync_events(struct kvm *kvm) {}
>  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
>  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
>  
> +void kvm_arch_setup_debug(struct kvm_vcpu *vcpu);
> +void kvm_arch_clear_debug(struct kvm_vcpu *vcpu);
> +
>  #endif /* __ARM64_KVM_HOST_H__ */
> diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
> index f7fa65d..cd06209 100644
> --- a/arch/arm64/kernel/asm-offsets.c
> +++ b/arch/arm64/kernel/asm-offsets.c
> @@ -122,6 +122,7 @@ int main(void)
>    DEFINE(VCPU_HPFAR_EL2,	offsetof(struct kvm_vcpu, arch.fault.hpfar_el2));
>    DEFINE(VCPU_DEBUG_FLAGS,	offsetof(struct kvm_vcpu, arch.debug_flags));
>    DEFINE(VCPU_HCR_EL2,		offsetof(struct kvm_vcpu, arch.hcr_el2));
> +  DEFINE(VCPU_MDCR_EL2,	offsetof(struct kvm_vcpu, arch.mdcr_el2));
>    DEFINE(VCPU_IRQ_LINES,	offsetof(struct kvm_vcpu, arch.irq_lines));
>    DEFINE(VCPU_HOST_CONTEXT,	offsetof(struct kvm_vcpu, arch.host_cpu_context));
>    DEFINE(VCPU_TIMER_CNTV_CTL,	offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
> index 4e6e09e..6796d4a 100644
> --- a/arch/arm64/kvm/Makefile
> +++ b/arch/arm64/kvm/Makefile
> @@ -17,7 +17,7 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o
>  
>  kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o
>  kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
> -kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
> +kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o
>  
>  kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
>  kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> new file mode 100644
> index 0000000..8a29d0b
> --- /dev/null
> +++ b/arch/arm64/kvm/debug.c
> @@ -0,0 +1,58 @@
> +/*
> + * Debug and Guest Debug support
> + *
> + * Copyright (C) 2015 - Linaro Ltd
> + * Author: Alex Bennée <alex.bennee@linaro.org>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License version 2 as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License
> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
> + */
> +
> +#include <linux/kvm_host.h>
> +
> +#include <asm/kvm_arm.h>
> +#include <asm/kvm_host.h>
> +
> +/**
> + * kvm_arch_setup_debug - set-up debug related stuff

nit: I think you want "set up" when it's a verb.

> + *
> + * @vcpu:	the vcpu pointer
> + *
> + * This is called before each entry in to the hypervisor to setup any

s/in to/into/
s/setup/set up/

> + * debug related registers. Currently this just ensures we will trap
> + * access to:

guest accesses to:

> + *  - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR)
> + *  - Debug ROM Address (MDCR_EL2_TDRA)
> + *  - Power down debug registers (MDCR_EL2_TDOSA)
> + *
> + * Additionally the hypervisor lazily saves/restores the debug
> + * register state. If it is not currently doing so (arch.debug_flags)
> + * then we also need to ensure we trap if the guest messes with them
> + * so we know we need to save them.

This paragraph is a little hard to make sense of.  If I understand it
correctly, the point is that when debugging the guest we need to make
sure guest accesses to the debug registers traps?  If so, I would
suggest something like:

  Additionally, KVM only traps guest accesses to the debug registers if
  the guest is not actively using them (see the KVM_ARM64_DEBUG_DIRTY
  flag on vcpu->arch.debug_flags).  Since the guest must not interfere
  with the hardware state when debugging the guest, we must ensure that
  trapping is enabled whenever we are debugging the guest.

> + */
> +
> +void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
> +{
> +	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
> +	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
> +
> +	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
> +	else
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
> +
> +}
> +
> +void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> +{
> +	/* Nothing to do yet */
> +}
> diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
> index 5befd01..be92bfe1 100644
> --- a/arch/arm64/kvm/hyp.S
> +++ b/arch/arm64/kvm/hyp.S
> @@ -768,17 +768,8 @@
>  	mov	x2, #(1 << 15)	// Trap CP15 Cr=15
>  	msr	hstr_el2, x2
>  
> -	mrs	x2, mdcr_el2
> -	and	x2, x2, #MDCR_EL2_HPMN_MASK
> -	orr	x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
> -	orr	x2, x2, #(MDCR_EL2_TDRA | MDCR_EL2_TDOSA)
> -
> -	// Check for KVM_ARM64_DEBUG_DIRTY, and set debug to trap
> -	// if not dirty.
> -	ldr	x3, [x0, #VCPU_DEBUG_FLAGS]
> -	tbnz	x3, #KVM_ARM64_DEBUG_DIRTY_SHIFT, 1f
> -	orr	x2, x2,  #MDCR_EL2_TDA
> -1:
> +	// Monitor Debug Config - see kvm_arch_setup_debug()
> +	ldr	x2, [x0, #VCPU_MDCR_EL2]
>  	msr	mdcr_el2, x2
>  .endm
>  

As the other reviewers noted, this is now setting the number of PMU
counters accessible to 0.  I'm fine with always setting mdcr_el2 from
memory, but I think we need to add code in hyp-init.S to read PMCR_EL0.N
and store that in a per-cpu (not vcpu) variable and configure mdcr_el2
using that value in kvm_arch_set_debug(), but then you need to call
kvm_arch_set_debug() from a non-preemptible section, which is probably a
good idea anyway.  Note that I couldn't see that we are initializing
MDCR_EL2 anywhere, so that should probably be a separate fix.

Thanks,
-Christoffer

Re: [PATCH v2 05/10] KVM: arm: introduce kvm_arch_setup/clear_debug()

[Christoffer Dall]

On Mon, Apr 13, 2015 at 04:36:23PM +0200, Christoffer Dall wrote:

[...]

> > +
> > +/**
> > + * kvm_arch_setup_debug - set-up debug related stuff
> 
> nit: I think you want "set up" when it's a verb.
> 
> > + *
> > + * @vcpu:	the vcpu pointer
> > + *
> > + * This is called before each entry in to the hypervisor to setup any
> 
> s/in to/into/
> s/setup/set up/
> 
> > + * debug related registers. Currently this just ensures we will trap
> > + * access to:
> 
> guest accesses to:
> 
> > + *  - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR)
> > + *  - Debug ROM Address (MDCR_EL2_TDRA)
> > + *  - Power down debug registers (MDCR_EL2_TDOSA)
> > + *
> > + * Additionally the hypervisor lazily saves/restores the debug
> > + * register state. If it is not currently doing so (arch.debug_flags)
> > + * then we also need to ensure we trap if the guest messes with them
> > + * so we know we need to save them.
> 
> This paragraph is a little hard to make sense of.  If I understand it
> correctly, the point is that when debugging the guest we need to make
> sure guest accesses to the debug registers traps?  If so, I would
> suggest something like:
> 
>   Additionally, KVM only traps guest accesses to the debug registers if
>   the guest is not actively using them (see the KVM_ARM64_DEBUG_DIRTY
>   flag on vcpu->arch.debug_flags).  Since the guest must not interfere
>   with the hardware state when debugging the guest, we must ensure that
>   trapping is enabled whenever we are debugging the guest.
> 

thinking about this, I don't think we're enforcing this yet, but maybe
that will come in the later patches or I misread the original paragraph.

-Christoffer

Re: [PATCH v2 02/10] KVM: define common __KVM_GUESTDBG_USE_SW/HW_BP values

[Alex Bennée]

Christoffer Dall <christoffer.dall@linaro.org> writes:

> On Tue, Mar 31, 2015 at 04:08:00PM +0100, Alex Bennée wrote:
>> Currently x86, powerpc and soon arm64 use the same two architecture
>> specific bits for guest debug support for software and hardware
>> breakpoints. This makes the shared values explicit while leaving the
>> gate open for another architecture to use some other value if they
>> really really want to.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>> diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
>> index ab4d473..1731569 100644
>> --- a/arch/powerpc/include/uapi/asm/kvm.h
>> +++ b/arch/powerpc/include/uapi/asm/kvm.h
>> @@ -310,8 +310,8 @@ struct kvm_guest_debug_arch {
>>   * and upper 16 bits are architecture specific. Architecture specific defines
>>   * that ioctl is for setting hardware breakpoint or software breakpoint.
>>   */
>> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
>> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
>> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
>> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
>>  
>>  /* definition of registers in kvm_run */
>>  struct kvm_sync_regs {
>> diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
>> index d7dcef5..1438202 100644
>> --- a/arch/x86/include/uapi/asm/kvm.h
>> +++ b/arch/x86/include/uapi/asm/kvm.h
>> @@ -250,8 +250,8 @@ struct kvm_debug_exit_arch {
>>  	__u64 dr7;
>>  };
>>  
>> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
>> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
>> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
>> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
>>  #define KVM_GUESTDBG_INJECT_DB		0x00040000
>>  #define KVM_GUESTDBG_INJECT_BP		0x00080000
>>  
>> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
>> index 5eedf84..ce2db14 100644
>> --- a/include/uapi/linux/kvm.h
>> +++ b/include/uapi/linux/kvm.h
>> @@ -525,8 +525,16 @@ struct kvm_s390_irq {
>>  
>>  /* for KVM_SET_GUEST_DEBUG */
>>  
>> -#define KVM_GUESTDBG_ENABLE		0x00000001
>> -#define KVM_GUESTDBG_SINGLESTEP		0x00000002
>> +#define KVM_GUESTDBG_ENABLE		(1 << 0)
>> +#define KVM_GUESTDBG_SINGLESTEP	(1 << 1)
>> +
>> +/*
>> + * Architecture specific stuff uses the top 16 bits of the field,
>
> can you be more specific than 'stuff' here?  features?
>
>> + * however there is some shared commonality for the common cases
>
> I don't like this sentence; shared commonality is a pleonasm and the use
> of however makes it sounds like there's some caveat here.

OK I can see that - after I looked it up ;-)

> If the top 16 bits are indeed arhictecture specific, then I think they
> should just be defined in their architecture specific headers.  Unless
> the idea here is that there's a fixed set of of flags that architectures
> can choose to support, in which case it should simply be defined in the
> common header.

Well an architecture might not support some features and want to use
those bits for something else? I didn't want to force the bottom two
of the architecture specific bits to wasted if the features don't exist.

>
>
>> + */
>> +#define __KVM_GUESTDBG_USE_SW_BP	(1 << 16)
>> +#define __KVM_GUESTDBG_USE_HW_BP	(1 << 17)
>> +
>>  
>>  struct kvm_guest_debug {
>>  	__u32 control;
>> -- 
>> 2.3.4
>> 

-- 
Alex Bennée

Re: [PATCH v2 02/10] KVM: define common __KVM_GUESTDBG_USE_SW/HW_BP values

[Andrew Jones]

On Mon, Apr 13, 2015 at 03:51:33PM +0100, Alex Bennée wrote:
> 
> Christoffer Dall <christoffer.dall@linaro.org> writes:
> 
> > On Tue, Mar 31, 2015 at 04:08:00PM +0100, Alex Bennée wrote:
> >> Currently x86, powerpc and soon arm64 use the same two architecture
> >> specific bits for guest debug support for software and hardware
> >> breakpoints. This makes the shared values explicit while leaving the
> >> gate open for another architecture to use some other value if they
> >> really really want to.
> >> 
> >> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> >> 
> >> diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
> >> index ab4d473..1731569 100644
> >> --- a/arch/powerpc/include/uapi/asm/kvm.h
> >> +++ b/arch/powerpc/include/uapi/asm/kvm.h
> >> @@ -310,8 +310,8 @@ struct kvm_guest_debug_arch {
> >>   * and upper 16 bits are architecture specific. Architecture specific defines
> >>   * that ioctl is for setting hardware breakpoint or software breakpoint.
> >>   */
> >> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
> >> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
> >> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
> >> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
> >>  
> >>  /* definition of registers in kvm_run */
> >>  struct kvm_sync_regs {
> >> diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
> >> index d7dcef5..1438202 100644
> >> --- a/arch/x86/include/uapi/asm/kvm.h
> >> +++ b/arch/x86/include/uapi/asm/kvm.h
> >> @@ -250,8 +250,8 @@ struct kvm_debug_exit_arch {
> >>  	__u64 dr7;
> >>  };
> >>  
> >> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
> >> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
> >> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
> >> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
> >>  #define KVM_GUESTDBG_INJECT_DB		0x00040000
> >>  #define KVM_GUESTDBG_INJECT_BP		0x00080000
> >>  
> >> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> >> index 5eedf84..ce2db14 100644
> >> --- a/include/uapi/linux/kvm.h
> >> +++ b/include/uapi/linux/kvm.h
> >> @@ -525,8 +525,16 @@ struct kvm_s390_irq {
> >>  
> >>  /* for KVM_SET_GUEST_DEBUG */
> >>  
> >> -#define KVM_GUESTDBG_ENABLE		0x00000001
> >> -#define KVM_GUESTDBG_SINGLESTEP		0x00000002
> >> +#define KVM_GUESTDBG_ENABLE		(1 << 0)
> >> +#define KVM_GUESTDBG_SINGLESTEP	(1 << 1)
> >> +
> >> +/*
> >> + * Architecture specific stuff uses the top 16 bits of the field,
> >
> > can you be more specific than 'stuff' here?  features?
> >
> >> + * however there is some shared commonality for the common cases
> >
> > I don't like this sentence; shared commonality is a pleonasm and the use
> > of however makes it sounds like there's some caveat here.
> 
> OK I can see that - after I looked it up ;-)
> 
> > If the top 16 bits are indeed arhictecture specific, then I think they
> > should just be defined in their architecture specific headers.  Unless
> > the idea here is that there's a fixed set of of flags that architectures
> > can choose to support, in which case it should simply be defined in the
> > common header.
> 
> Well an architecture might not support some features and want to use
> those bits for something else? I didn't want to force the bottom two
> of the architecture specific bits to wasted if the features don't exist.

This change comes from a discussion we had on v1 of this series
http://www.spinics.net/lists/kvm-arm/msg12409.html

> 
> >
> >
> >> + */
> >> +#define __KVM_GUESTDBG_USE_SW_BP	(1 << 16)
> >> +#define __KVM_GUESTDBG_USE_HW_BP	(1 << 17)
> >> +
> >>  
> >>  struct kvm_guest_debug {
> >>  	__u32 control;
> >> -- 
> >> 2.3.4
> >> 
> 
> -- 
> Alex Bennée

Re: [PATCH v2 05/10] KVM: arm: introduce kvm_arch_setup/clear_debug()

[Alex Bennée]

Christoffer Dall <christoffer.dall@linaro.org> writes:

> On Tue, Mar 31, 2015 at 04:08:03PM +0100, Alex Bennée wrote:
>> This is a precursor for later patches which will need to do more to
>> setup debug state before entering the hyp.S switch code. The existing
>> functionality for setting mdcr_el2 has been moved out of hyp.S and now
>> uses the value kept in vcpu->arch.mdcr_el2.
>> 
>> This also moves the conditional setting of the TDA bit from the hyp code
>> into the C code.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>>  create mode 100644 arch/arm64/kvm/debug.c
>> 
>> diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
>> index 41008cd..8c01c97 100644
>> --- a/arch/arm/include/asm/kvm_host.h
>> +++ b/arch/arm/include/asm/kvm_host.h
>> @@ -242,5 +242,7 @@ static inline void kvm_arch_hardware_unsetup(void) {}
>>  static inline void kvm_arch_sync_events(struct kvm *kvm) {}
>>  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
>>  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
>> +static inline void kvm_arch_setup_debug(struct kvm_vcpu *vcpu) {}
>> +static inline void kvm_arch_clear_debug(struct kvm_vcpu *vcpu) {}
>>  
>>  #endif /* __ARM_KVM_HOST_H__ */
>> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
>> index 445933d..7ea8b0e 100644
>> --- a/arch/arm/kvm/arm.c
>> +++ b/arch/arm/kvm/arm.c
>> @@ -523,6 +523,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
>>  
>>  		kvm_vgic_flush_hwstate(vcpu);
>>  		kvm_timer_flush_hwstate(vcpu);
>> +		kvm_arch_setup_debug(vcpu);
>>  
>>  		local_irq_disable();
>>  
>> @@ -569,6 +570,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
>>  		 * Back from guest
>>  		 *************************************************************/
>>  
>> +		kvm_arch_clear_debug(vcpu);
>>  		kvm_timer_sync_hwstate(vcpu);
>>  		kvm_vgic_sync_hwstate(vcpu);
>>  
>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>> index 8ac3c70..0631840 100644
>> --- a/arch/arm64/include/asm/kvm_host.h
>> +++ b/arch/arm64/include/asm/kvm_host.h
>> @@ -101,6 +101,7 @@ struct kvm_vcpu_arch {
>>  
>>  	/* HYP configuration */
>>  	u64 hcr_el2;
>> +	u32 mdcr_el2;
>>  
>>  	/* Exception Information */
>>  	struct kvm_vcpu_fault_info fault;
>> @@ -257,4 +258,7 @@ static inline void kvm_arch_sync_events(struct kvm *kvm) {}
>>  static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
>>  static inline void kvm_arch_sched_in(struct kvm_vcpu *vcpu, int cpu) {}
>>  
>> +void kvm_arch_setup_debug(struct kvm_vcpu *vcpu);
>> +void kvm_arch_clear_debug(struct kvm_vcpu *vcpu);
>> +
>>  #endif /* __ARM64_KVM_HOST_H__ */
>> diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
>> index f7fa65d..cd06209 100644
>> --- a/arch/arm64/kernel/asm-offsets.c
>> +++ b/arch/arm64/kernel/asm-offsets.c
>> @@ -122,6 +122,7 @@ int main(void)
>>    DEFINE(VCPU_HPFAR_EL2,	offsetof(struct kvm_vcpu, arch.fault.hpfar_el2));
>>    DEFINE(VCPU_DEBUG_FLAGS,	offsetof(struct kvm_vcpu, arch.debug_flags));
>>    DEFINE(VCPU_HCR_EL2,		offsetof(struct kvm_vcpu, arch.hcr_el2));
>> +  DEFINE(VCPU_MDCR_EL2,	offsetof(struct kvm_vcpu, arch.mdcr_el2));
>>    DEFINE(VCPU_IRQ_LINES,	offsetof(struct kvm_vcpu, arch.irq_lines));
>>    DEFINE(VCPU_HOST_CONTEXT,	offsetof(struct kvm_vcpu, arch.host_cpu_context));
>>    DEFINE(VCPU_TIMER_CNTV_CTL,	offsetof(struct kvm_vcpu, arch.timer_cpu.cntv_ctl));
>> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
>> index 4e6e09e..6796d4a 100644
>> --- a/arch/arm64/kvm/Makefile
>> +++ b/arch/arm64/kvm/Makefile
>> @@ -17,7 +17,7 @@ kvm-$(CONFIG_KVM_ARM_HOST) += $(ARM)/psci.o $(ARM)/perf.o
>>  
>>  kvm-$(CONFIG_KVM_ARM_HOST) += emulate.o inject_fault.o regmap.o
>>  kvm-$(CONFIG_KVM_ARM_HOST) += hyp.o hyp-init.o handle_exit.o
>> -kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
>> +kvm-$(CONFIG_KVM_ARM_HOST) += guest.o debug.o reset.o sys_regs.o sys_regs_generic_v8.o
>>  
>>  kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
>>  kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
>> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
>> new file mode 100644
>> index 0000000..8a29d0b
>> --- /dev/null
>> +++ b/arch/arm64/kvm/debug.c
>> @@ -0,0 +1,58 @@
>> +/*
>> + * Debug and Guest Debug support
>> + *
>> + * Copyright (C) 2015 - Linaro Ltd
>> + * Author: Alex Bennée <alex.bennee@linaro.org>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License version 2 as
>> + * published by the Free Software Foundation.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU General Public License
>> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
>> + */
>> +
>> +#include <linux/kvm_host.h>
>> +
>> +#include <asm/kvm_arm.h>
>> +#include <asm/kvm_host.h>
>> +
>> +/**
>> + * kvm_arch_setup_debug - set-up debug related stuff
>
> nit: I think you want "set up" when it's a verb.
>
>> + *
>> + * @vcpu:	the vcpu pointer
>> + *
>> + * This is called before each entry in to the hypervisor to setup any
>
> s/in to/into/
> s/setup/set up/
>
>> + * debug related registers. Currently this just ensures we will trap
>> + * access to:
>
> guest accesses to:
>
>> + *  - Performance monitors (MDCR_EL2_TPM/MDCR_EL2_TPMCR)
>> + *  - Debug ROM Address (MDCR_EL2_TDRA)
>> + *  - Power down debug registers (MDCR_EL2_TDOSA)
>> + *
>> + * Additionally the hypervisor lazily saves/restores the debug
>> + * register state. If it is not currently doing so (arch.debug_flags)
>> + * then we also need to ensure we trap if the guest messes with them
>> + * so we know we need to save them.
>
> This paragraph is a little hard to make sense of.  If I understand it
> correctly, the point is that when debugging the guest we need to make
> sure guest accesses to the debug registers traps?  If so, I would
> suggest something like:
>
>   Additionally, KVM only traps guest accesses to the debug registers if
>   the guest is not actively using them (see the KVM_ARM64_DEBUG_DIRTY
>   flag on vcpu->arch.debug_flags).  Since the guest must not interfere
>   with the hardware state when debugging the guest, we must ensure that
>   trapping is enabled whenever we are debugging the guest.
>
>> + */
>> +
>> +void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>> +{
>> +	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>> +	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>> +
>> +	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
>> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>> +	else
>> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>> +
>> +}
>> +
>> +void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>> +{
>> +	/* Nothing to do yet */
>> +}
>> diff --git a/arch/arm64/kvm/hyp.S b/arch/arm64/kvm/hyp.S
>> index 5befd01..be92bfe1 100644
>> --- a/arch/arm64/kvm/hyp.S
>> +++ b/arch/arm64/kvm/hyp.S
>> @@ -768,17 +768,8 @@
>>  	mov	x2, #(1 << 15)	// Trap CP15 Cr=15
>>  	msr	hstr_el2, x2
>>  
>> -	mrs	x2, mdcr_el2
>> -	and	x2, x2, #MDCR_EL2_HPMN_MASK
>> -	orr	x2, x2, #(MDCR_EL2_TPM | MDCR_EL2_TPMCR)
>> -	orr	x2, x2, #(MDCR_EL2_TDRA | MDCR_EL2_TDOSA)
>> -
>> -	// Check for KVM_ARM64_DEBUG_DIRTY, and set debug to trap
>> -	// if not dirty.
>> -	ldr	x3, [x0, #VCPU_DEBUG_FLAGS]
>> -	tbnz	x3, #KVM_ARM64_DEBUG_DIRTY_SHIFT, 1f
>> -	orr	x2, x2,  #MDCR_EL2_TDA
>> -1:
>> +	// Monitor Debug Config - see kvm_arch_setup_debug()
>> +	ldr	x2, [x0, #VCPU_MDCR_EL2]
>>  	msr	mdcr_el2, x2
>>  .endm
>>  
>
> As the other reviewers noted, this is now setting the number of PMU
> counters accessible to 0.  I'm fine with always setting mdcr_el2 from
> memory, but I think we need to add code in hyp-init.S to read PMCR_EL0.N
> and store that in a per-cpu (not vcpu) variable and configure mdcr_el2
> using that value in kvm_arch_set_debug(), but then you need to call
> kvm_arch_set_debug() from a non-preemptible section, which is probably a
> good idea anyway.  Note that I couldn't see that we are initializing
> MDCR_EL2 anywhere, so that should probably be a separate fix.

Yeah because I couldn't see it being set I figured it was OK to blat it
with zeros. But you are right we should set it up properly if we are
going to mess with it.

>
> Thanks,
> -Christoffer

-- 
Alex Bennée

Re: [PATCH v2 04/10] KVM: arm: guest debug, add stub KVM_SET_GUEST_DEBUG ioctl

[David Hildenbrand]

> On Tue, Mar 31, 2015 at 04:08:02PM +0100, Alex Bennée wrote:
> > This commit adds a stub function to support the KVM_SET_GUEST_DEBUG
> > ioctl. Currently any operation flag will return EINVAL. Actual
> > functionality will be added with further patches.
> > 
> > Signed-off-by: Alex Bennée <alex.bennee@linaro.org>.
> > 
> > ---
> > v2
> >   - simplified form of the ioctl (stuff will go into setup_debug)
> > 
> > diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> > index b112efc..06c5064 100644
> > --- a/Documentation/virtual/kvm/api.txt
> > +++ b/Documentation/virtual/kvm/api.txt
> > @@ -2604,7 +2604,7 @@ handled.
> >  4.87 KVM_SET_GUEST_DEBUG
> >  
> >  Capability: KVM_CAP_SET_GUEST_DEBUG
> > -Architectures: x86, s390, ppc
> > +Architectures: x86, s390, ppc, arm64
> >  Type: vcpu ioctl
> >  Parameters: struct kvm_guest_debug (in)
> >  Returns: 0 on success; -1 on error
> > diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> > index 5560f74..445933d 100644
> > --- a/arch/arm/kvm/arm.c
> > +++ b/arch/arm/kvm/arm.c
> > @@ -183,6 +183,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
> >  	case KVM_CAP_ARM_PSCI:
> >  	case KVM_CAP_ARM_PSCI_0_2:
> >  	case KVM_CAP_READONLY_MEM:
> > +	case KVM_CAP_SET_GUEST_DEBUG:
> >  		r = 1;
> >  		break;
> 
> shouldn't you wait with advertising this capability until you've
> implemented support for it?
> 

I think this would work for now, however it's not very practical
- in the end one has to sense which debug flags are actually supported.

Question is if he wants to add initial support and extend functionality and
flags with each patch or enable the whole set of features in one shot at the
end.

Doing the latter seems more practicable to me (especially as the debug features
are added in the same patch series).

David

Re: [PATCH v2 04/10] KVM: arm: guest debug, add stub KVM_SET_GUEST_DEBUG ioctl

[Alex Bennée]

David Hildenbrand <dahi@linux.vnet.ibm.com> writes:

>> On Tue, Mar 31, 2015 at 04:08:02PM +0100, Alex Bennée wrote:
>> > This commit adds a stub function to support the KVM_SET_GUEST_DEBUG
>> > ioctl. Currently any operation flag will return EINVAL. Actual
>> > functionality will be added with further patches.
>> > 
>> > Signed-off-by: Alex Bennée <alex.bennee@linaro.org>.
>> > 
>> > ---
>> > v2
>> >   - simplified form of the ioctl (stuff will go into setup_debug)
>> > 
>> > diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
>> > index b112efc..06c5064 100644
>> > --- a/Documentation/virtual/kvm/api.txt
>> > +++ b/Documentation/virtual/kvm/api.txt
>> > @@ -2604,7 +2604,7 @@ handled.
>> >  4.87 KVM_SET_GUEST_DEBUG
>> >  
>> >  Capability: KVM_CAP_SET_GUEST_DEBUG
>> > -Architectures: x86, s390, ppc
>> > +Architectures: x86, s390, ppc, arm64
>> >  Type: vcpu ioctl
>> >  Parameters: struct kvm_guest_debug (in)
>> >  Returns: 0 on success; -1 on error
>> > diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
>> > index 5560f74..445933d 100644
>> > --- a/arch/arm/kvm/arm.c
>> > +++ b/arch/arm/kvm/arm.c
>> > @@ -183,6 +183,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>> >  	case KVM_CAP_ARM_PSCI:
>> >  	case KVM_CAP_ARM_PSCI_0_2:
>> >  	case KVM_CAP_READONLY_MEM:
>> > +	case KVM_CAP_SET_GUEST_DEBUG:
>> >  		r = 1;
>> >  		break;
>> 
>> shouldn't you wait with advertising this capability until you've
>> implemented support for it?
>> 
>
> I think this would work for now, however it's not very practical
> - in the end one has to sense which debug flags are actually supported.
>
> Question is if he wants to add initial support and extend functionality and
> flags with each patch or enable the whole set of features in one shot at the
> end.

This is what in effect I was doing. Testing each feature in turn and
ensuring it failed gracefully when kernel features where not present
(both missing the capability or returning EINVAL).

> Doing the latter seems more practicable to me (especially as the debug features
> are added in the same patch series).

Well in practice the whole series will go in together so this is only
really relevant to what happens when you bisect the series.

>
> David

-- 
Alex Bennée

Re: [PATCH v2 02/10] KVM: define common __KVM_GUESTDBG_USE_SW/HW_BP values

[Christoffer Dall]

On Mon, Apr 13, 2015 at 03:51:33PM +0100, Alex Bennée wrote:
> 
> Christoffer Dall <christoffer.dall@linaro.org> writes:
> 
> > On Tue, Mar 31, 2015 at 04:08:00PM +0100, Alex Bennée wrote:
> >> Currently x86, powerpc and soon arm64 use the same two architecture
> >> specific bits for guest debug support for software and hardware
> >> breakpoints. This makes the shared values explicit while leaving the
> >> gate open for another architecture to use some other value if they
> >> really really want to.
> >> 
> >> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> >> 
> >> diff --git a/arch/powerpc/include/uapi/asm/kvm.h b/arch/powerpc/include/uapi/asm/kvm.h
> >> index ab4d473..1731569 100644
> >> --- a/arch/powerpc/include/uapi/asm/kvm.h
> >> +++ b/arch/powerpc/include/uapi/asm/kvm.h
> >> @@ -310,8 +310,8 @@ struct kvm_guest_debug_arch {
> >>   * and upper 16 bits are architecture specific. Architecture specific defines
> >>   * that ioctl is for setting hardware breakpoint or software breakpoint.
> >>   */
> >> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
> >> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
> >> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
> >> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
> >>  
> >>  /* definition of registers in kvm_run */
> >>  struct kvm_sync_regs {
> >> diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
> >> index d7dcef5..1438202 100644
> >> --- a/arch/x86/include/uapi/asm/kvm.h
> >> +++ b/arch/x86/include/uapi/asm/kvm.h
> >> @@ -250,8 +250,8 @@ struct kvm_debug_exit_arch {
> >>  	__u64 dr7;
> >>  };
> >>  
> >> -#define KVM_GUESTDBG_USE_SW_BP		0x00010000
> >> -#define KVM_GUESTDBG_USE_HW_BP		0x00020000
> >> +#define KVM_GUESTDBG_USE_SW_BP		__KVM_GUESTDBG_USE_SW_BP
> >> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
> >>  #define KVM_GUESTDBG_INJECT_DB		0x00040000
> >>  #define KVM_GUESTDBG_INJECT_BP		0x00080000
> >>  
> >> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> >> index 5eedf84..ce2db14 100644
> >> --- a/include/uapi/linux/kvm.h
> >> +++ b/include/uapi/linux/kvm.h
> >> @@ -525,8 +525,16 @@ struct kvm_s390_irq {
> >>  
> >>  /* for KVM_SET_GUEST_DEBUG */
> >>  
> >> -#define KVM_GUESTDBG_ENABLE		0x00000001
> >> -#define KVM_GUESTDBG_SINGLESTEP		0x00000002
> >> +#define KVM_GUESTDBG_ENABLE		(1 << 0)
> >> +#define KVM_GUESTDBG_SINGLESTEP	(1 << 1)
> >> +
> >> +/*
> >> + * Architecture specific stuff uses the top 16 bits of the field,
> >
> > can you be more specific than 'stuff' here?  features?
> >
> >> + * however there is some shared commonality for the common cases
> >
> > I don't like this sentence; shared commonality is a pleonasm and the use
> > of however makes it sounds like there's some caveat here.
> 
> OK I can see that - after I looked it up ;-)
> 
> > If the top 16 bits are indeed arhictecture specific, then I think they
> > should just be defined in their architecture specific headers.  Unless
> > the idea here is that there's a fixed set of of flags that architectures
> > can choose to support, in which case it should simply be defined in the
> > common header.
> 
> Well an architecture might not support some features and want to use
> those bits for something else? I didn't want to force the bottom two
> of the architecture specific bits to wasted if the features don't exist.
> 
In that case I think the definition is local to each architecture and
should indeed just be duplicated.  The __ definitions complicate more
than they help as they are exported to userspace etc.  The KVM
maintainers may have a different view on this though.

-Christoffer

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:08:04PM +0100, Alex Bennée wrote:
> This adds support for SW breakpoints inserted by userspace.
> 
> We do this by trapping all BKPT exceptions in the
> hypervisor (MDCR_EL2_TDE).

you mean trapping all exceptions in the guest to the hypervisor?

> The kvm_debug_exit_arch carries the address
> of the exception.

why?  can userspace not simply read out the PC using GET_ONE_REG?

> If user-space doesn't know of the breakpoint then we
> have a guest inserted breakpoint and the hypervisor needs to start again
> and deliver the exception to guest.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>   - update to use new exit struct
>   - tweak for C setup
>   - do our setup in debug_setup/clear code
>   - fixed up comments
> 
> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> index 06c5064..17d4f9c 100644
> --- a/Documentation/virtual/kvm/api.txt
> +++ b/Documentation/virtual/kvm/api.txt
> @@ -2626,7 +2626,7 @@ when running. Common control bits are:
>  The top 16 bits of the control field are architecture specific control
>  flags which can include the following:
>  
> -  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86]
> +  - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
>    - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
>    - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
>    - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index 7ea8b0e..d3bc8dc 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -304,7 +304,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  	kvm_arm_set_running_vcpu(NULL);
>  }
>  
> -#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE)
> +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)

nit: spaces around the operator

>  
>  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>  					struct kvm_guest_debug *dbg)
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index 8a29d0b..cff0475 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -45,11 +45,18 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>  
> +	/* Trap debug register access? */

other patch

>  	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>  	else
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>  
> +	/* Trap breakpoints? */
> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
> +	else
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;

so now you're trapping all debug exceptions, right?

what happens if the guest is using the hardware to debug debug stuff and
generates other kinds of debug exceptions, like a hardware breakpoint,
will we not see an unhandled exception and the guest being forcefully
killed?

> +
>  }
>  
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index 524fa25..ed1bbb4 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -82,6 +82,37 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  	return 1;
>  }
>  
> +/**
> + * kvm_handle_debug_exception - handle a debug exception instruction

handle a software breadkpoint exception

> + *
> + * @vcpu:	the vcpu pointer
> + * @run:	access to the kvm_run structure for results
> + *
> + * We route all debug exceptions through the same handler as we

all debug exceptions?  software breakpoints and all?  then why the above
shot text?

> + * just need to report the PC and the HSR values to userspace.
> + * Userspace may decide to re-inject the exception and deliver it to
> + * the guest if it wasn't for the host to deal with.

now I'm confused - does userspace setup the guest to receive an
exception or does it tell KVM to emulate an exception for the guest or
do we execute the breakpoint without trapping the debug exception?

> + */
> +static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
> +{
> +	u32 hsr = kvm_vcpu_get_hsr(vcpu);
> +
> +	run->exit_reason = KVM_EXIT_DEBUG;
> +	run->debug.arch.hsr = hsr;
> +
> +	switch (hsr >> ESR_ELx_EC_SHIFT) {
> +	case ESR_ELx_EC_BKPT32:
> +	case ESR_ELx_EC_BRK64:
> +		run->debug.arch.pc = *vcpu_pc(vcpu);
> +		break;
> +	default:
> +		kvm_err("%s: un-handled case hsr: %#08x\n",
> +			__func__, (unsigned int) hsr);

this should never happen right?

> +		break;
> +	}
> +	return 0;
> +}
> +
>  static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_WFx]	= kvm_handle_wfx,
>  	[ESR_ELx_EC_CP15_32]	= kvm_handle_cp15_32,
> @@ -96,6 +127,8 @@ static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
> +	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
> +	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>  };
>  
>  static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
> -- 
> 2.3.4
> 

Re: [PATCH v2 07/10] KVM: arm64: guest debug, add support for single-step

[Christoffer Dall]

Hi Alex,

On Tue, Mar 31, 2015 at 04:08:05PM +0100, Alex Bennée wrote:
> This adds support for single-stepping the guest. As userspace can and
> will manipulate guest registers before restarting any tweaking of the
> registers has to occur just before control is passed back to the guest.

this sentence is hard to read.  Do you mean:

(a) As userspace can and will manipulate guest register, we must ensure
that any tweaking of the registers before restarting the guest happens
immediately before...

or

(b) As userspace manipulates guest registers before restarting the
guest, we must ensure that any tweaking of the register happens
immediately before...

> Furthermore while guest debugging is in effect we need to squash the

Furthermore, while guest debugging is in effect,
(commas)

> ability of the guest to single-step itself as we have no easy way of
> re-entering the guest after the exception has been delivered to the
> hypervisor.

I'm not sure I understand this last part of the sentence.  Is the point
that if we trap on a guest single-step exception we cannot easily inject
such an exception back into the guest and therefore we trap the guest if
it tries to set itself up for single-stepping?

What is our recourse then?  To just ignore the single-step setting of
the guest and execute it as normal (while single-stepping the guest from
the outside)?

> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>   - Move pstate/mdscr manipulation into C
>   - don't export guest_debug to assembly
>   - add accessor for saved_debug regs
>   - tweak save/restore of mdscr_el1
> 
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index d3bc8dc..c1ed8cb 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -304,7 +304,21 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  	kvm_arm_set_running_vcpu(NULL);
>  }
>  
> -#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE|KVM_GUESTDBG_USE_SW_BP)
> +#define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE |    \
> +			    KVM_GUESTDBG_USE_SW_BP | \
> +			    KVM_GUESTDBG_SINGLESTEP)
> +
> +/**
> + * kvm_arch_vcpu_ioctl_set_guest_debug - Setup guest debugging
> + * @kvm:	pointer to the KVM struct
> + * @kvm_guest_debug: the ioctl data buffer
> + *
> + * This sets up the VM for guest debugging. Care has to be taken when
> + * manipulating guest registers as these will be set/cleared by the
> + * hyper-visor controller, typically before each kvm_run event. As a

which guest registers are set/cleared by userspace exactly?

s/by the hyper-visor controller/by userspace/

> + * result modification of the guest registers needs to take place

As a result, (comma)

s/needs to/must/

> + * after they have been restored in the hyp.S trampoline code.

trampoline code?  The trampoline code we are referring to is in
hyp-init.S.  Do you mean in EL2?  Then just sya in hyp.S or say in EL2
or in hyp mode.

> + */
>  
>  int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>  					struct kvm_guest_debug *dbg)
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 0631840..6a33647 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -121,6 +121,13 @@ struct kvm_vcpu_arch {
>  	 * here.
>  	 */
>  
> +	/* Registers pre any guest debug manipulations */

I couldn't find 'pre' as an independent word in any English
dictionaries.  I'm also not entirely sure what you mean?  Who modifies
this when, and why do we need to store this?

> +	struct {
> +		u32	pstate_ss_bit;
> +		u32	mdscr_el1_bits;
> +
> +	} debug_saved_regs;

If I understood this state correctly (see below), then guest_debug_state
is probably a better name for this struct.

> +
>  	/* Don't run the guest */
>  	bool pause;
>  
> @@ -143,6 +150,7 @@ struct kvm_vcpu_arch {
>  
>  #define vcpu_gp_regs(v)		(&(v)->arch.ctxt.gp_regs)
>  #define vcpu_sys_reg(v,r)	((v)->arch.ctxt.sys_regs[(r)])
> +#define vcpu_debug_saved_reg(v, r) ((v)->arch.debug_saved_regs.r)

hmm, not sure this is warranted if the 'saved_regs' is not the current
state of the VM, which is sort of what the vcpu_gp_regs() and friends
hint at.  Maybe I'm just not getting exactly what piece of state it is.

>  /*
>   * CP14 and CP15 live in the same array, as they are backed by the
>   * same system registers.
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index cff0475..b32362c 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -19,8 +19,16 @@
>  
>  #include <linux/kvm_host.h>
>  
> +#include <asm/debug-monitors.h>
> +#include <asm/kvm_asm.h>
>  #include <asm/kvm_arm.h>
>  #include <asm/kvm_host.h>
> +#include <asm/kvm_emulate.h>
> +
> +/* These are the bits of MDSCR_EL1 we may mess with */

we may mess with?  Can you be more specific?

> +#define MDSCR_EL1_DEBUG_BITS	(DBG_MDSCR_SS | \
> +				DBG_MDSCR_KDE | \
> +				DBG_MDSCR_MDE)
>  
>  /**
>   * kvm_arch_setup_debug - set-up debug related stuff
> @@ -51,15 +59,46 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	else
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>  
> -	/* Trap breakpoints? */
> -	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
> +	/* Is Guest debugging in effect? */
> +	if (vcpu->guest_debug) {
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
> -	else
> -		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>  
> +		/* Save pstate/mdscr */
> +		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
> +			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
> +		vcpu_debug_saved_reg(vcpu, mdscr_el1_bits) =
> +			vcpu_sys_reg(vcpu, MDSCR_EL1) & MDSCR_EL1_DEBUG_BITS;

nit: add blank line

> +		/*
> +		 * Single Step (ARM ARM D2.12.3 The software step state
> +		 * machine)
> +		 *
> +		 * If we are doing Single Step we need to manipulate
> +		 * MDSCR_EL1.SS and PSTATE.SS. If not we need to
> +		 * suppress the guest from messing with it.

If not, we must prevent the guest from modifying them.
     ^^^    ^^^    ^^^                            ^^^

Why must we prevent the guest from modifying them if we're not
single-stepping the guest?


> +		 */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) {
> +			*vcpu_cpsr(vcpu) |=  DBG_SPSR_SS;
> +			vcpu_sys_reg(vcpu, MDSCR_EL1) |= DBG_MDSCR_SS;
> +		} else {
> +			*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
> +			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
> +		}
> +
> +	} else {
> +		/* Debug operations can go straight to the guest */
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
> +	}
>  }
>  
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  {
> -	/* Nothing to do yet */
> +	if (vcpu->guest_debug) {
> +		/* Restore pstate/mdscr bits we may have messed with */

Can you be more specific than "messed with"?  "... bits that may have
been modified if we were debugging the guest" ?

Drew's suggestion may make this easier to read...


> +		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
> +		*vcpu_cpsr(vcpu) |= vcpu_debug_saved_reg(vcpu, pstate_ss_bit);
> +
> +		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
> +		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
> +			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);

So the debug_saved_reg is simply the guest's view of the debug
registers?  In that case I think vcpu_guest_dbg_reg() is more clear.



> +	}
>  }
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index ed1bbb4..16accae 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -101,6 +101,7 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  	run->debug.arch.hsr = hsr;
>  
>  	switch (hsr >> ESR_ELx_EC_SHIFT) {
> +	case ESR_ELx_EC_SOFTSTP_LOW:
>  	case ESR_ELx_EC_BKPT32:
>  	case ESR_ELx_EC_BRK64:
>  		run->debug.arch.pc = *vcpu_pc(vcpu);
> @@ -127,6 +128,7 @@ static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_SYS64]	= kvm_handle_sys_reg,
>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
> +	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
>  	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
>  	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>  };
> -- 
> 2.3.4
> 

Re: [PATCH v2 08/10] KVM: arm64: guest debug, HW assisted debug support

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:08:06PM +0100, Alex Bennée wrote:
> This adds support for userspace to control the HW debug registers for
> guest debug. We'll only copy the $ARCH defined number across as that is
> all that hyp.S will use anyway. 

I don't really understand what this sentence means?

> I've moved some helper functions into
> the hw_breakpoint.h header for re-use.
> 
> As with single step we need to tweak the guest registers to enable the
> exceptions so we need to save and restore those bits.
> 
> Two new capabilities have been added to the KVM_EXTENSION ioctl to allow
> userspace to query the number of hardware break and watch points
> available on the host hardware.
> 
> As QEMU tests for watchpoints based on the address and not the PC we
> also need to export the value of far_el2 to userspace.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> ---
> v2
>    - switched to C setup
>    - replace host debug registers directly into context
>    - minor tweak to api docs
>    - setup right register for debug
>    - add FAR_EL2 to debug exit structure
>    - add support fro trapping debug register access
> 
> diff --git a/Documentation/virtual/kvm/api.txt b/Documentation/virtual/kvm/api.txt
> index 17d4f9c..ac34093 100644
> --- a/Documentation/virtual/kvm/api.txt
> +++ b/Documentation/virtual/kvm/api.txt
> @@ -2627,7 +2627,7 @@ The top 16 bits of the control field are architecture specific control
>  flags which can include the following:
>  
>    - KVM_GUESTDBG_USE_SW_BP:     using software breakpoints [x86, arm64]
> -  - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390]
> +  - KVM_GUESTDBG_USE_HW_BP:     using hardware breakpoints [x86, s390, arm64]
>    - KVM_GUESTDBG_INJECT_DB:     inject DB type exception [x86]
>    - KVM_GUESTDBG_INJECT_BP:     inject BP type exception [x86]
>    - KVM_GUESTDBG_EXIT_PENDING:  trigger an immediate guest exit [s390]
> @@ -2642,6 +2642,10 @@ updated to the correct (supplied) values.
>  The second part of the structure is architecture specific and
>  typically contains a set of debug registers.
>  
> +For arm64 the number of debug registers is implementation defined and
> +can be determined by querying the KVM_CAP_GUEST_DEBUG_HW_BPS and
> +KVM_CAP_GUEST_DEBUG_HW_WPS capabilities.
> +

can you document their behavior more specifically?  I assume they both
return 0 if HW assisted debugging is not supported and return the number
of implemented hardware registers otherwise?

How does this work on big.LITTLE systems where cores may have a different
number of implemented registers?

>  When debug events exit the main run loop with the reason
>  KVM_EXIT_DEBUG with the kvm_debug_exit_arch part of the kvm_run
>  structure containing architecture specific debug information.
> diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
> index c1ed8cb..a286026 100644
> --- a/arch/arm/kvm/arm.c
> +++ b/arch/arm/kvm/arm.c
> @@ -306,6 +306,7 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
>  
>  #define KVM_GUESTDBG_VALID (KVM_GUESTDBG_ENABLE |    \
>  			    KVM_GUESTDBG_USE_SW_BP | \
> +			    KVM_GUESTDBG_USE_HW_BP | \
>  			    KVM_GUESTDBG_SINGLESTEP)
>  
>  /**
> @@ -328,6 +329,26 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
>  			return -EINVAL;
>  
>  		vcpu->guest_debug = dbg->control;
> +
> +		/* Hardware assisted Break and Watch points */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +			int nb = get_num_brps();
> +			int nw = get_num_wrps();
> +
> +			/* Copy across up to IMPDEF debug registers to our
> +			 * shadow copy in the vcpu structure. The debug code
> +			 * will then set them up before we re-enter the guest.
> +			 */
> +			memcpy(vcpu->arch.guest_debug_regs.dbg_bcr,
> +				dbg->arch.dbg_bcr, sizeof(__u64)*nb);
> +			memcpy(vcpu->arch.guest_debug_regs.dbg_bvr,
> +				dbg->arch.dbg_bvr, sizeof(__u64)*nb);
> +			memcpy(vcpu->arch.guest_debug_regs.dbg_wcr,
> +				dbg->arch.dbg_wcr, sizeof(__u64)*nw);
> +			memcpy(vcpu->arch.guest_debug_regs.dbg_wvr,
> +				dbg->arch.dbg_wvr, sizeof(__u64)*nw);
> +		}
> +
>  	} else {
>  		/* If not enabled clear all flags */
>  		vcpu->guest_debug = 0;
> diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
> index 52b484b..c450552 100644
> --- a/arch/arm64/include/asm/hw_breakpoint.h
> +++ b/arch/arm64/include/asm/hw_breakpoint.h
> @@ -130,6 +130,18 @@ static inline void ptrace_hw_copy_thread(struct task_struct *task)
>  }
>  #endif
>  
> +/* Determine number of BRP registers available. */
> +static inline int get_num_brps(void)
> +{
> +	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
> +}
> +
> +/* Determine number of WRP registers available. */
> +static inline int get_num_wrps(void)
> +{
> +	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
> +}
> +
>  extern struct pmu perf_ops_bp;
>  
>  #endif	/* __KERNEL__ */
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 6a33647..2c359c9 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -106,8 +106,9 @@ struct kvm_vcpu_arch {
>  	/* Exception Information */
>  	struct kvm_vcpu_fault_info fault;
>  
> -	/* Debug state */
> +	/* Guest debug state */
>  	u64 debug_flags;
> +	struct kvm_guest_debug_arch guest_debug_regs;
>  
>  	/* Pointer to host CPU context */
>  	kvm_cpu_context_t *host_cpu_context;
> @@ -126,6 +127,7 @@ struct kvm_vcpu_arch {
>  		u32	pstate_ss_bit;
>  		u32	mdscr_el1_bits;
>  
> +		struct kvm_guest_debug_arch debug_regs;
>  	} debug_saved_regs;
>  
>  	/* Don't run the guest */
> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
> index 6ee70a0..73f21e4 100644
> --- a/arch/arm64/include/uapi/asm/kvm.h
> +++ b/arch/arm64/include/uapi/asm/kvm.h
> @@ -118,6 +118,7 @@ struct kvm_guest_debug_arch {
>  struct kvm_debug_exit_arch {
>  	__u64 pc;
>  	__u32 hsr;
> +	__u64 far;	/* used for watchpoints */
>  };
>  
>  struct kvm_sync_regs {
> diff --git a/arch/arm64/kernel/hw_breakpoint.c b/arch/arm64/kernel/hw_breakpoint.c
> index 98bbe06..923be44 100644
> --- a/arch/arm64/kernel/hw_breakpoint.c
> +++ b/arch/arm64/kernel/hw_breakpoint.c
> @@ -49,18 +49,6 @@ static DEFINE_PER_CPU(int, stepping_kernel_bp);
>  static int core_num_brps;
>  static int core_num_wrps;
>  
> -/* Determine number of BRP registers available. */
> -static int get_num_brps(void)
> -{
> -	return ((read_cpuid(ID_AA64DFR0_EL1) >> 12) & 0xf) + 1;
> -}
> -
> -/* Determine number of WRP registers available. */
> -static int get_num_wrps(void)
> -{
> -	return ((read_cpuid(ID_AA64DFR0_EL1) >> 20) & 0xf) + 1;
> -}
> -
>  int hw_breakpoint_slots(int type)
>  {
>  	/*
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index b32362c..3b368f3 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -50,14 +50,19 @@
>  
>  void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  {
> +	bool trap_debug = false;
> +
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>  
> -	/* Trap debug register access? */
> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
> +
> +	/*
> +	 * If we are not treating debug registers are dirty we need
> +	 * to trap if the guest starts accessing them.
> +	 */

Hmmm, this comment could have been added when introducing the original
logic instead.  Also:

s/are/as/

I'm not sure the 'treating them as dirty' part of the comment is
helpful.  Perhaps you want something like:

  If the guest debug register state is dirty (the guest is actively
  accesing them), then we context-switch the registers in EL2.
  Otherwise, we trap-and-emulate all guest accesses to them.

>  	if (!vcpu->arch.debug_flags & KVM_ARM64_DEBUG_DIRTY)
> -		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
> -	else
> -		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
> +		trap_debug = true;
>  
>  	/* Is Guest debugging in effect? */
>  	if (vcpu->guest_debug) {
> @@ -84,10 +89,69 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
>  		}
>  
> +		/*
> +		 * HW Break/Watch points
> +		 */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +			int nb = get_num_brps();
> +			int nw = get_num_wrps();

again, don't you have to be careful to be in a non-preemptible section
here?

> +			struct kvm_guest_debug_arch *saved, *host;
> +
> +			vcpu_sys_reg(vcpu, MDSCR_EL1) |=
> +				(DBG_MDSCR_KDE|DBG_MDSCR_MDE);

Can you always set the KDE bit?  What if you only want debug exceptions
from guest user mode?  Is that supported through the KVM interface
and/or GDB?

style: spaces around the '|' operator.

> +
> +			/*
> +			 * First we need to make a copy of the guest
> +			 * debug registers before we wipe them out
> +			 * with the ones we want to use.
> +			 */

ok, I may be over-doing this, but you've sort of gotten me on the track
here; First typically implies that there's a Second, or subsequently, or
something like that here.

Also, regarding 'need to' vs. 'must', I found this a nice read:
http://www.kirkmahoney.com/blog/2008/02/must-vs-needs-to/

I think this comment would be more clear if it was written something
like:

  We are about to set the hw debug registers with values used to debug
  the VM.  Make a copy of the guest debug register state to preserve
  the guest's view of the debug register state.

> +			saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
> +			host = &vcpu->arch.guest_debug_regs;
> +
> +			/* Save guest values */
> +			memcpy(&saved->dbg_bcr,
> +			       &vcpu_sys_reg(vcpu, DBGBCR0_EL1),
> +			       sizeof(__u64)*nb);
> +			memcpy(&saved->dbg_bvr,
> +			       &vcpu_sys_reg(vcpu, DBGBVR0_EL1),
> +			       sizeof(__u64)*nb);
> +			memcpy(&saved->dbg_wcr,
> +			       &vcpu_sys_reg(vcpu, DBGWCR0_EL1),
> +			       sizeof(__u64)*nw);
> +			memcpy(&saved->dbg_wvr,
> +			       &vcpu_sys_reg(vcpu, DBGWVR0_EL1),
> +			       sizeof(__u64)*nw);
> +
> +			/* Copy in host values */
> +			memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1),
> +			       &host->dbg_bcr,
> +			       sizeof(__u64)*nb);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1),
> +			       &host->dbg_bvr,
> +			       sizeof(__u64)*nb);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1),
> +			       &host->dbg_wcr,
> +			       sizeof(__u64)*nw);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
> +			       &host->dbg_wvr,
> +			       sizeof(__u64)*nw);

you need spaces around all the '*' operators.


Instead of copying back and forth, could we just have a pointer on the
vcpu->arch struct to the debug regs struct that the EL2 code will
context-switch (if trapping is disabled) and then just shift the pointer
in C code instead of doing all this?

> +
> +			/* Make sure hyp.S copies them in/out */
> +			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> +			/* Also track guest changes */
> +			trap_debug = true;
> +		}
> +
>  	} else {
>  		/* Debug operations can go straight to the guest */
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>  	}
> +
> +	/* Trap debug register access? */
> +	if (trap_debug)
> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
> +	else
> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
>  }
>  
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> @@ -100,5 +164,31 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
>  		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
>  			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
> +
> +		/*
> +		 * If we were using HW debug we need to restore the
> +		 * values the guest had set them up with
> +		 */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +			struct kvm_guest_debug_arch *regs;
> +			int nb = get_num_brps();
> +			int nw = get_num_wrps();
> +
> +			regs = &vcpu_debug_saved_reg(vcpu, debug_regs);
> +
> +			/* Restore the saved debug register values */
> +			memcpy(&vcpu_sys_reg(vcpu, DBGBCR0_EL1),
> +			       &regs->dbg_bcr,
> +			       sizeof(__u64)*nb);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGBVR0_EL1),
> +			       &regs->dbg_bvr,
> +			       sizeof(__u64)*nb);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGWCR0_EL1),
> +			       &regs->dbg_wcr,
> +			       sizeof(__u64)*nw);
> +			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
> +			       &regs->dbg_wvr,
> +			       sizeof(__u64)*nw);

same style comment as above

> +		}

this is called on iteration of the run-loop, right?

it really seems excessive to do all this copying on every single
iteration.  I think it's cleaner if you have on piece of state on the
vcpu struct for the guest state, and one piece of state for guest
debugging, and for all emulation and userspace sync purposes you access
the guest state directly and you switch a pointer to the struct that
EL2 uses to save/restore the underlying hardware state.  Would that
work?


>  	}
>  }
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index 16accae..460a1aa 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -101,7 +101,11 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  	run->debug.arch.hsr = hsr;
>  
>  	switch (hsr >> ESR_ELx_EC_SHIFT) {
> +	case ESR_ELx_EC_WATCHPT_LOW:
> +		run->debug.arch.far = vcpu->arch.fault.far_el2;
> +		/* fall through */
>  	case ESR_ELx_EC_SOFTSTP_LOW:
> +	case ESR_ELx_EC_BREAKPT_LOW:
>  	case ESR_ELx_EC_BKPT32:
>  	case ESR_ELx_EC_BRK64:
>  		run->debug.arch.pc = *vcpu_pc(vcpu);
> @@ -129,6 +133,8 @@ static exit_handle_fn arm_exit_handlers[] = {
>  	[ESR_ELx_EC_IABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_DABT_LOW]	= kvm_handle_guest_abort,
>  	[ESR_ELx_EC_SOFTSTP_LOW]= kvm_handle_guest_debug,
> +	[ESR_ELx_EC_WATCHPT_LOW]= kvm_handle_guest_debug,
> +	[ESR_ELx_EC_BREAKPT_LOW]= kvm_handle_guest_debug,
>  	[ESR_ELx_EC_BKPT32]	= kvm_handle_guest_debug,
>  	[ESR_ELx_EC_BRK64]	= kvm_handle_guest_debug,
>  };
> diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
> index 0b43265..c2732c7 100644
> --- a/arch/arm64/kvm/reset.c
> +++ b/arch/arm64/kvm/reset.c
> @@ -64,6 +64,12 @@ int kvm_arch_dev_ioctl_check_extension(long ext)
>  	case KVM_CAP_ARM_EL1_32BIT:
>  		r = cpu_has_32bit_el1();
>  		break;
> +	case KVM_CAP_GUEST_DEBUG_HW_BPS:
> +		r = get_num_brps();
> +		break;
> +	case KVM_CAP_GUEST_DEBUG_HW_WPS:
> +		r  = get_num_wrps();
> +		break;
>  	default:
>  		r = 0;
>  	}
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index c370b40..be9b188 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -196,16 +196,49 @@ static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
>   * - If the dirty bit is set, save guest registers, restore host
>   *   registers and clear the dirty bit. This ensure that the host can
>   *   now use the debug registers.
> + *
> + * We also use this mechanism to set-up the debug registers for guest
                                    set up
> + * debugging. If this is the case we want to ensure the guest sees
> + * the right versions of the registers - even if they are not going to
> + * be effective while guest debug is using HW debug.
> + *
>   */
> +
>  static bool trap_debug_regs(struct kvm_vcpu *vcpu,
>  			    const struct sys_reg_params *p,
>  			    const struct sys_reg_desc *r)
>  {
> -	if (p->is_write) {
> -		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
> -		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +		struct kvm_guest_debug_arch *saved;
> +		__u64 *val;
> +
> +		saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
> +
> +		if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
> +			val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
> +		else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
> +			val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
> +		else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
> +			val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
> +		else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
> +			val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
> +		else {
> +			kvm_err("Bad register index %d\n", r->reg);
> +			return false;
> +		}

you should be able to greatly simplify this as well if you follow my
suggestion above.

> +
> +		if (p->is_write)
> +			*val = *vcpu_reg(vcpu, p->Rt);
> +		else
> +			*vcpu_reg(vcpu, p->Rt) = *val;
> +
>  	} else {
> -		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +		if (p->is_write) {
> +			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
> +			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> +		} else {
> +			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +		}
>  	}
>  
>  	return true;
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index ce2db14..0e48c0d 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -771,6 +771,8 @@ struct kvm_ppc_smmu_info {
>  #define KVM_CAP_PPC_ENABLE_HCALL 104
>  #define KVM_CAP_CHECK_EXTENSION_VM 105
>  #define KVM_CAP_S390_USER_SIGP 106
> +#define KVM_CAP_GUEST_DEBUG_HW_BPS 107
> +#define KVM_CAP_GUEST_DEBUG_HW_WPS 108
>  
>  #ifdef KVM_CAP_IRQ_ROUTING
>  
> -- 
> 2.3.4
> 

Re: [PATCH v2 08/10] KVM: arm64: guest debug, HW assisted debug support

[Christoffer Dall]

On Fri, Apr 10, 2015 at 02:25:21PM +0200, Andrew Jones wrote:

[...]

> > --- a/arch/arm64/kvm/sys_regs.c
> > +++ b/arch/arm64/kvm/sys_regs.c
> > @@ -196,16 +196,49 @@ static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
> >   * - If the dirty bit is set, save guest registers, restore host
> >   *   registers and clear the dirty bit. This ensure that the host can
> >   *   now use the debug registers.
> > + *
> > + * We also use this mechanism to set-up the debug registers for guest
> setup

since I'm in this mood:

setup: noun or adjective
set-up: noun derived from the phrasal verb, example "Run! It's a set-up."
set up: verb

-Christoffer

Re: [PATCH v2 09/10] KVM: arm64: trap nested debug register access

[Christoffer Dall]

On Mon, Apr 13, 2015 at 08:59:21AM +0100, Alex Bennée wrote:

[...]

> >> +		/* MDSCR_EL1 */
> >> +		if (r->reg == MDSCR_EL1) {
> >> +			if (p->is_write)
> >> +				vcpu_debug_saved_reg(vcpu, mdscr_el1) =
> >> +					*vcpu_reg(vcpu, p->Rt);
> >> +			else
> >> +				*vcpu_reg(vcpu, p->Rt) =
> >> +					vcpu_debug_saved_reg(vcpu, mdscr_el1);
> >
> > With this lines wrapping, {}'s might be nice.
> 
> My natural inclination is to wrap in {}'s but I know the kernel is a fan
> of the single-statement if forms.
> 
It's accepted to use braces for multi-line single statements - and I
prefer it too :)

-Christoffer

Re: [PATCH v2 09/10] KVM: arm64: trap nested debug register access

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:08:07PM +0100, Alex Bennée wrote:
> When we are using the hardware registers for guest debug we need to deal
> with the guests access to them. There is already a mechanism for dealing
> with these accesses so we build on top of that.
> 
>   - mdscr_el1_bits is renamed as we save the whole register
>   - any access to mdscr_el1 is now stored in the mirror location
>   - if we are using HW assisted debug we do the same with DBG[WB][CV]R
> 
> There is one register (MDCCINT_EL1) which guest debug doesn't care about
> so this behaves as before.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
> index 2c359c9..3d32d45 100644
> --- a/arch/arm64/include/asm/kvm_host.h
> +++ b/arch/arm64/include/asm/kvm_host.h
> @@ -122,10 +122,13 @@ struct kvm_vcpu_arch {
>  	 * here.
>  	 */
>  
> -	/* Registers pre any guest debug manipulations */
> +	/* Registers before any guest debug manipulations. These
> +	 * shadow registers are updated by the kvm_handle_sys_reg
> +	 * trap handler if the guest accesses or updates them
> +	 */
>  	struct {
>  		u32	pstate_ss_bit;
> -		u32	mdscr_el1_bits;
> +		u32	mdscr_el1;
>  
>  		struct kvm_guest_debug_arch debug_regs;
>  	} debug_saved_regs;
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index 3b368f3..638c111 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -55,8 +55,6 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>  
> -	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
> -
>  	/*
>  	 * If we are not treating debug registers are dirty we need
>  	 * to trap if the guest starts accessing them.
> @@ -71,8 +69,10 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  		/* Save pstate/mdscr */
>  		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
>  			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
> -		vcpu_debug_saved_reg(vcpu, mdscr_el1_bits) =
> -			vcpu_sys_reg(vcpu, MDSCR_EL1) & MDSCR_EL1_DEBUG_BITS;
> +
> +		vcpu_debug_saved_reg(vcpu, mdscr_el1) =
> +			vcpu_sys_reg(vcpu, MDSCR_EL1);
> +

you can avoid this churn in the patches by following Drew's advice to a
previous patch.

>  		/*
>  		 * Single Step (ARM ARM D2.12.3 The software step state
>  		 * machine)
> @@ -161,9 +161,8 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>  		*vcpu_cpsr(vcpu) |= vcpu_debug_saved_reg(vcpu, pstate_ss_bit);
>  
> -		vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~MDSCR_EL1_DEBUG_BITS;
> -		vcpu_sys_reg(vcpu, MDSCR_EL1) |=
> -			vcpu_debug_saved_reg(vcpu, mdscr_el1_bits);
> +		vcpu_sys_reg(vcpu, MDSCR_EL1) =
> +			vcpu_debug_saved_reg(vcpu, mdscr_el1);
>  
>  		/*
>  		 * If we were using HW debug we need to restore the
> diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
> index be9b188..d43d7d1 100644
> --- a/arch/arm64/kvm/sys_regs.c
> +++ b/arch/arm64/kvm/sys_regs.c
> @@ -208,39 +208,61 @@ static bool trap_debug_regs(struct kvm_vcpu *vcpu,
>  			    const struct sys_reg_params *p,
>  			    const struct sys_reg_desc *r)
>  {
> -	if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> -		struct kvm_guest_debug_arch *saved;
> -		__u64 *val;
> -
> -		saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
> -
> -		if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
> -			val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
> -		else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
> -			val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
> -		else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
> -			val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
> -		else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
> -			val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
> -		else {
> -			kvm_err("Bad register index %d\n", r->reg);
> -			return false;
> +	if (vcpu->guest_debug) {
> +
> +		/* MDSCR_EL1 */
> +		if (r->reg == MDSCR_EL1) {
> +			if (p->is_write)
> +				vcpu_debug_saved_reg(vcpu, mdscr_el1) =
> +					*vcpu_reg(vcpu, p->Rt);
> +			else
> +				*vcpu_reg(vcpu, p->Rt) =
> +					vcpu_debug_saved_reg(vcpu, mdscr_el1);
> +
> +			return true;
>  		}
>  
> -		if (p->is_write)
> -			*val = *vcpu_reg(vcpu, p->Rt);
> -		else
> -			*vcpu_reg(vcpu, p->Rt) = *val;
> +		/* MDCCINT_EL1 */
> +		if (r->reg == MDCCINT_EL1)
> +			goto old;
> +
> +		/* We only shadow DBG* if guest being debugged */
> +		if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
> +			struct kvm_guest_debug_arch *saved;
> +			__u64 *val;
> +
> +			saved = &vcpu_debug_saved_reg(vcpu, debug_regs);
> +
> +			if (r->reg >= DBGBCR0_EL1 && r->reg <= DBGBCR15_EL1)
> +				val = &saved->dbg_bcr[r->reg - DBGBCR0_EL1];
> +			else if (r->reg >= DBGBVR0_EL1 && r->reg <= DBGBVR15_EL1)
> +				val = &saved->dbg_bvr[r->reg - DBGBVR0_EL1];
> +			else if (r->reg >= DBGWCR0_EL1 && r->reg <= DBGWCR15_EL1)
> +				val = &saved->dbg_wcr[r->reg - DBGWCR0_EL1];
> +			else if (r->reg >= DBGWVR0_EL1 && r->reg <= DBGWVR15_EL1)
> +				val = &saved->dbg_wvr[r->reg - DBGWVR0_EL1];
> +			else {
> +				kvm_err("Bad register index %d\n", r->reg);
> +				return false;
> +			}
>  
> -	} else {
> -		if (p->is_write) {
> -			vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
> -			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> -		} else {
> -			*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +			if (p->is_write)
> +				*val = *vcpu_reg(vcpu, p->Rt);
> +			else
> +				*vcpu_reg(vcpu, p->Rt) = *val;
> +
> +			return true;
>  		}
>  	}
>  
> +old:
> +	if (p->is_write) {
> +		vcpu_sys_reg(vcpu, r->reg) = *vcpu_reg(vcpu, p->Rt);
> +		vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
> +	} else {
> +		*vcpu_reg(vcpu, p->Rt) = vcpu_sys_reg(vcpu, r->reg);
> +	}
> +

I really think this points to a problem with the design; the emulate
function should just emulate writes/reads to some state without this
complexity.  If there's some reason not to do this, you should put that
in the commit text.

>  	return true;
>  }
>  
> -- 
> 2.3.4
> 

Thanks,
-Christoffer

Re: [PATCH v2 10/10] KVM: arm64: add trace points for guest_debug debug

[Christoffer Dall]

On Tue, Mar 31, 2015 at 04:08:08PM +0100, Alex Bennée wrote:
> This includes trace points for:
>   kvm_arch_setup_guest_debug
>   kvm_arch_clear_guest_debug
>   kvm_handle_guest_debug
> 
> I've also added some generic register setting trace events so I can
> watch the register values being built up over time. The local
> dump_dbg_regs() function dumps all the HW BKPT and WPT registers.
> 
> I've also added a #define trace_dreg to shorten some lines.
> 
> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
> 
> diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
> index 638c111..7c96288 100644
> --- a/arch/arm64/kvm/debug.c
> +++ b/arch/arm64/kvm/debug.c
> @@ -25,12 +25,37 @@
>  #include <asm/kvm_host.h>
>  #include <asm/kvm_emulate.h>
>  
> +#include "trace.h"
> +
> +#define trace_dreg(name, value) trace_kvm_arch_setup_debug_reg32(name, value)
> +
>  /* These are the bits of MDSCR_EL1 we may mess with */
>  #define MDSCR_EL1_DEBUG_BITS	(DBG_MDSCR_SS | \
>  				DBG_MDSCR_KDE | \
>  				DBG_MDSCR_MDE)
>  
>  /**
> + * dump_dbg_regs - simple debug helper
> + *
> + * This provides a simple helper to dump the HW debug registers
> + */
> +static void dump_dbg_regs(struct kvm_vcpu *vcpu, int nb, int nw)
> +{
> +	int i;
> +
> +	for (i = 0; i < nb; i++) {
> +		trace_printk("bkpt%d: 0x%08x:0x%llx\n", i,
> +			(u32) vcpu_sys_reg(vcpu, DBGBCR0_EL1 + i),
> +			vcpu_sys_reg(vcpu, DBGBVR0_EL1 + i));
> +	}
> +	for (i = 0; i < nb; i++) {
> +		trace_printk("wtpt%d: 0x%08x:0x%llx\n", i,
> +			(u32) vcpu_sys_reg(vcpu, DBGWCR0_EL1 + i),
> +			vcpu_sys_reg(vcpu, DBGWVR0_EL1 + i));
> +	}
> +}
> +
> +/**
>   * kvm_arch_setup_debug - set-up debug related stuff
>   *
>   * @vcpu:	the vcpu pointer
> @@ -52,9 +77,13 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  {
>  	bool trap_debug = false;
>  
> +	trace_kvm_arch_setup_debug(vcpu->guest_debug);
> +
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TPM | MDCR_EL2_TPMCR);
>  	vcpu->arch.mdcr_el2 |= (MDCR_EL2_TDRA | MDCR_EL2_TDOSA);
>  
> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
> +
>  	/*
>  	 * If we are not treating debug registers are dirty we need
>  	 * to trap if the guest starts accessing them.
> @@ -66,6 +95,8 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  	if (vcpu->guest_debug) {
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
>  
> +		trace_dreg("MDCR_EL2", vcpu->arch.mdcr_el2);
> +
>  		/* Save pstate/mdscr */
>  		vcpu_debug_saved_reg(vcpu, pstate_ss_bit) =
>  			*vcpu_cpsr(vcpu) & DBG_SPSR_SS;
> @@ -73,6 +104,11 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  		vcpu_debug_saved_reg(vcpu, mdscr_el1) =
>  			vcpu_sys_reg(vcpu, MDSCR_EL1);
>  
> +		trace_dreg("Save: PSTATE.SS",
> +			vcpu_debug_saved_reg(vcpu, pstate_ss_bit));
> +		trace_dreg("Save: MDSCR",
> +			vcpu_debug_saved_reg(vcpu, mdscr_el1));
> +
>  		/*
>  		 * Single Step (ARM ARM D2.12.3 The software step state
>  		 * machine)
> @@ -88,6 +124,8 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  			*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
>  			vcpu_sys_reg(vcpu, MDSCR_EL1) &= ~DBG_MDSCR_SS;
>  		}
> +		trace_dreg("SPSR_EL2", *vcpu_cpsr(vcpu));
> +		trace_dreg("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
>  
>  		/*
>  		 * HW Break/Watch points
> @@ -136,6 +174,9 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  			       &host->dbg_wvr,
>  			       sizeof(__u64)*nw);
>  
> +			if (trace_kvm_arch_setup_debug_reg32_enabled())
> +				dump_dbg_regs(vcpu, nb, nw);
> +
>  			/* Make sure hyp.S copies them in/out */
>  			vcpu->arch.debug_flags |= KVM_ARM64_DEBUG_DIRTY;
>  			/* Also track guest changes */
> @@ -147,15 +188,24 @@ void kvm_arch_setup_debug(struct kvm_vcpu *vcpu)
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>  	}
>  
> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
> +	trace_kvm_arch_setup_debug_reg32("MDSCR_EL1",
> +					vcpu_sys_reg(vcpu, MDSCR_EL1));
> +
> +
>  	/* Trap debug register access? */
>  	if (trap_debug)
>  		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDA;
>  	else
>  		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDA;
> +
> +	trace_kvm_arch_setup_debug_reg32("MDCR_EL2", vcpu->arch.mdcr_el2);
>  }
>  
>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  {
> +	trace_kvm_arch_clear_debug(vcpu->guest_debug);
> +
>  	if (vcpu->guest_debug) {
>  		/* Restore pstate/mdscr bits we may have messed with */
>  		*vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
> @@ -164,6 +214,8 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  		vcpu_sys_reg(vcpu, MDSCR_EL1) =
>  			vcpu_debug_saved_reg(vcpu, mdscr_el1);
>  
> +		trace_dreg("SPSR_EL2", *vcpu_cpsr(vcpu));
> +		trace_dreg("MDSCR_EL1", vcpu_sys_reg(vcpu, MDSCR_EL1));
>  		/*
>  		 * If we were using HW debug we need to restore the
>  		 * values the guest had set them up with
> @@ -188,6 +240,10 @@ void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>  			memcpy(&vcpu_sys_reg(vcpu, DBGWVR0_EL1),
>  			       &regs->dbg_wvr,
>  			       sizeof(__u64)*nw);
> +
> +			if (trace_kvm_arch_setup_debug_reg32_enabled())
> +				dump_dbg_regs(vcpu, nb, nw);
> +
>  		}
>  	}
>  }
> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> index 460a1aa..c6cc69a 100644
> --- a/arch/arm64/kvm/handle_exit.c
> +++ b/arch/arm64/kvm/handle_exit.c
> @@ -97,6 +97,8 @@ static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>  {
>  	u32 hsr = kvm_vcpu_get_hsr(vcpu);
>  
> +	trace_kvm_handle_guest_debug(*vcpu_pc(vcpu), hsr);
> +
>  	run->exit_reason = KVM_EXIT_DEBUG;
>  	run->debug.arch.hsr = hsr;
>  
> diff --git a/arch/arm64/kvm/trace.h b/arch/arm64/kvm/trace.h
> index 157416e9..dffdb49 100644
> --- a/arch/arm64/kvm/trace.h
> +++ b/arch/arm64/kvm/trace.h
> @@ -44,6 +44,72 @@ TRACE_EVENT(kvm_hvc_arm64,
>  		  __entry->vcpu_pc, __entry->r0, __entry->imm)
>  );
>  
> +TRACE_EVENT(kvm_handle_guest_debug,
> +	TP_PROTO(unsigned long vcpu_pc, u32 hsr),
> +	TP_ARGS(vcpu_pc, hsr),
> +
> +	TP_STRUCT__entry(
> +		__field(unsigned long,	vcpu_pc)
> +		__field(u32,		hsr)
> +	),
> +
> +	TP_fast_assign(
> +		__entry->vcpu_pc = vcpu_pc;
> +		__entry->hsr = hsr;
> +	),
> +
> +	TP_printk("debug exception at 0x%08lx (HSR: 0x%08x)",
> +		__entry->vcpu_pc, __entry->hsr)
> +);
> +
> +
> +TRACE_EVENT(kvm_arch_setup_debug,
> +	TP_PROTO(__u32 guest_debug),
> +	TP_ARGS(guest_debug),
> +
> +	TP_STRUCT__entry(
> +		__field(__u32, guest_debug)
> +	),
> +
> +	TP_fast_assign(
> +		__entry->guest_debug = guest_debug;
> +	),
> +
> +	TP_printk("flags: 0x%08x", __entry->guest_debug)
> +);
> +
> +TRACE_EVENT(kvm_arch_clear_debug,
> +	TP_PROTO(__u32 guest_debug),
> +	TP_ARGS(guest_debug),
> +
> +	TP_STRUCT__entry(
> +		__field(__u32, guest_debug)
> +	),
> +
> +	TP_fast_assign(
> +		__entry->guest_debug = guest_debug;
> +	),
> +
> +	TP_printk("flags: 0x%08x", __entry->guest_debug)
> +);
> +
> +TRACE_EVENT(kvm_arch_setup_debug_reg32,
> +	TP_PROTO(const char *name, __u32 value),
> +	TP_ARGS(name, value),
> +
> +	TP_STRUCT__entry(
> +		__field(const char *, name)
> +		__field(__u32, value)
> +	),
> +
> +	TP_fast_assign(
> +		__entry->name = name;
> +		__entry->value = value;
> +	),
> +
> +	TP_printk("%s now 0x%08x", __entry->name, __entry->value)
> +);
> +
>  #endif /* _TRACE_ARM64_KVM_H */
>  
>  #undef TRACE_INCLUDE_PATH
> -- 
> 2.3.4
> 

I would guess that as far as merging any of this upstream keeping the
kvm_arch_set_debug() and kvm_handle_guest_debug() are worth keeping, but
only if the kvm_arch_set_debug() is only called when the guest is
actually being debugged from userspace.

Thanks,
-Christoffer

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Alex Bennée]

Zhichao Huang <zhichao.huang@linaro.org> writes:

> On Tue, Mar 31, 2015 at 04:08:04PM +0100, Alex Bennée wrote:
>> This adds support for SW breakpoints inserted by userspace.
>>
>> We do this by trapping all BKPT exceptions in the
>> hypervisor (MDCR_EL2_TDE).
>
> why should we trap all debug exceptions?
>
> The trap for cp14 register r/w seems enough to record relevant
> informations to context switch the dbg register while neccessary.

Lets think about this case when the SW breakpoint exception occurs:

If KVM doesn't trap it and pass it back to userspace to handle it would
have to deliver it to the guest. The guest not having inserted the
breakpoint in the first place would get very confused.

So what we actually do is re-route the exception to the hypervisor and
stop the VM and return to userspace with the debug information. Once in
QEMU we check to see if the SW breakpoint was one of the ones we
inserted at which point control is passed back to the host GDB (attached
via the GDB stub in QEMU). If it is not a breakpoint which was set-up by
the host then it must be one for the guest at which point we need to
ensure the exception is delivered to the guest for it to process.

-- 
Alex Bennée

Re: [PATCH v2 03/10] KVM: arm: guest debug, define API headers

[Alex Bennée]

Christoffer Dall <christoffer.dall@linaro.org> writes:

> On Tue, Mar 31, 2015 at 04:08:01PM +0100, Alex Bennée wrote:
>> This commit defines the API headers for guest debugging. There are two
>> architecture specific debug structures:
>> 
>>   - kvm_guest_debug_arch, allows us to pass in HW debug registers
>>   - kvm_debug_exit_arch, signals the exact debug exit and pc
>> 
>> The type of debugging being used is control by the architecture specific
>> control bits of the kvm_guest_debug->control flags in the ioctl
>> structure.
>> 
>> Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
>> 
>> ---
>> v2
>>    - expose hsr and pc directly to user-space
>> 
>> diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h
>> index 3ef77a4..6ee70a0 100644
>> --- a/arch/arm64/include/uapi/asm/kvm.h
>> +++ b/arch/arm64/include/uapi/asm/kvm.h
>> @@ -100,10 +100,24 @@ struct kvm_sregs {
>>  struct kvm_fpu {
>>  };
>>  
>> +/*
>> + * See ARM ARM D7.3: Debug Registers
>
> see the ARM ARM for ??
>
>> + *
>> + * The control registers are architecturally defined as 32 bits but are
>> + * stored as 64 bit values along side the value registers and aligned
>
> do you mean alongside?

sure.

>
>> + * with the rest 64 bit registers in the normal CPU context.
>
> rest of the 64 bit
>
>> + */
>
> why do we store them as 64 bit values?  There's nothing prevented us
> from defining them as __u32 is there?  Is this to make the ONE_REG
> interface accessers more convenient?

No but it will involve more fiddling when we copy them into the CPU
context which keeps everything as 64 bit aligned. Of course if we want
to remove the debug registers from the context and put a pointer in
place then this is fairly moot as we will need to change the hyp.S code
that copies the registers during the world switch. I was trying to
minimise the amount of change to the assembler in this series.

>
>> +#define KVM_ARM_NDBG_REGS 16
>
> nit: is NDBG short for something I don't know about or is it
> the number of debug registers we are noting here, in which case I think
> KVM_ARM_NUM_DBG_REGS is more clear.

OK.

>
>>  struct kvm_guest_debug_arch {
>> +	__u64 dbg_bcr[KVM_ARM_NDBG_REGS];
>> +	__u64 dbg_bvr[KVM_ARM_NDBG_REGS];
>> +	__u64 dbg_wcr[KVM_ARM_NDBG_REGS];
>> +	__u64 dbg_wvr[KVM_ARM_NDBG_REGS];
>>  };
>>  
>>  struct kvm_debug_exit_arch {
>> +	__u64 pc;
>> +	__u32 hsr;
>>  };
>>  
>>  struct kvm_sync_regs {
>> @@ -207,4 +221,11 @@ struct kvm_arch_memory_slot {
>>  
>>  #endif
>>  
>> +/*
>> + * Architecture related debug defines - upper 16 bits of
>> + * kvm_guest_debug->control
>> + */
>> +#define KVM_GUESTDBG_USE_SW_BP	        __KVM_GUESTDBG_USE_SW_BP
>> +#define KVM_GUESTDBG_USE_HW_BP		__KVM_GUESTDBG_USE_HW_BP
>> +
>>  #endif /* __ARM_KVM_H__ */
>> -- 
>> 2.3.4
>> 
>
> Thanks,
> -Christoffer

-- 
Alex Bennée

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Alex Bennée]

Christoffer Dall <christoffer.dall@linaro.org> writes:

> On Tue, Mar 31, 2015 at 04:08:04PM +0100, Alex Bennée wrote:
>> This adds support for SW breakpoints inserted by userspace.
>> 
>> We do this by trapping all BKPT exceptions in the
>> hypervisor (MDCR_EL2_TDE).
>
> you mean trapping all exceptions in the guest to the hypervisor?
>
>> The kvm_debug_exit_arch carries the address
>> of the exception.
>
> why?  can userspace not simply read out the PC using GET_ONE_REG?

Yes, I have re-worded and removed PC from the debug information.

<snip>
>>  
>> +	/* Trap breakpoints? */
>> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
>> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
>> +	else
>> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
>
> so now you're trapping all debug exceptions, right?
>
> what happens if the guest is using the hardware to debug debug stuff and
> generates other kinds of debug exceptions, like a hardware breakpoint,
> will we not see an unhandled exception and the guest being forcefully
> killed?

Yes until the later patches which stop the guest using HW debug
registers while we are using them.

>
>> +
>>  }
>>  
>>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
>> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
>> index 524fa25..ed1bbb4 100644
>> --- a/arch/arm64/kvm/handle_exit.c
>> +++ b/arch/arm64/kvm/handle_exit.c
>> @@ -82,6 +82,37 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
>>  	return 1;
>>  }
>>  
>> +/**
>> + * kvm_handle_debug_exception - handle a debug exception instruction
>
> handle a software breadkpoint exception
>
>> + *
>> + * @vcpu:	the vcpu pointer
>> + * @run:	access to the kvm_run structure for results
>> + *
>> + * We route all debug exceptions through the same handler as we
>
> all debug exceptions?  software breakpoints and all?  then why the above
> shot text?
>
>> + * just need to report the PC and the HSR values to userspace.
>> + * Userspace may decide to re-inject the exception and deliver it to
>> + * the guest if it wasn't for the host to deal with.
>
> now I'm confused - does userspace setup the guest to receive an
> exception or does it tell KVM to emulate an exception for the guest or
> do we execute the breakpoint without trapping the debug exception?

I've made it all go through userspace as we may have to translate the
hypervisor visible exception code to what the guest was expecting to see.

>
>> + */
>> +static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
>> +{
>> +	u32 hsr = kvm_vcpu_get_hsr(vcpu);
>> +
>> +	run->exit_reason = KVM_EXIT_DEBUG;
>> +	run->debug.arch.hsr = hsr;
>> +
>> +	switch (hsr >> ESR_ELx_EC_SHIFT) {
>> +	case ESR_ELx_EC_BKPT32:
>> +	case ESR_ELx_EC_BRK64:
>> +		run->debug.arch.pc = *vcpu_pc(vcpu);
>> +		break;
>> +	default:
>> +		kvm_err("%s: un-handled case hsr: %#08x\n",
>> +			__func__, (unsigned int) hsr);
>
> this should never happen right?

At the moment it could, at the end of the patch series we should cover
all the cases so it would indicate a bug. I've made it return an error
code so it fails hard as suggested by David.

-- 
Alex Bennée

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Christoffer Dall]

On Thu, Apr 23, 2015 at 03:26:53PM +0100, Alex Bennée wrote:
> 
> Christoffer Dall <christoffer.dall@linaro.org> writes:
> 
> > On Tue, Mar 31, 2015 at 04:08:04PM +0100, Alex Bennée wrote:
> >> This adds support for SW breakpoints inserted by userspace.
> >> 
> >> We do this by trapping all BKPT exceptions in the
> >> hypervisor (MDCR_EL2_TDE).
> >
> > you mean trapping all exceptions in the guest to the hypervisor?
> >
> >> The kvm_debug_exit_arch carries the address
> >> of the exception.
> >
> > why?  can userspace not simply read out the PC using GET_ONE_REG?
> 
> Yes, I have re-worded and removed PC from the debug information.
> 
> <snip>
> >>  
> >> +	/* Trap breakpoints? */
> >> +	if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
> >> +		vcpu->arch.mdcr_el2 |= MDCR_EL2_TDE;
> >> +	else
> >> +		vcpu->arch.mdcr_el2 &= ~MDCR_EL2_TDE;
> >
> > so now you're trapping all debug exceptions, right?
> >
> > what happens if the guest is using the hardware to debug debug stuff and
> > generates other kinds of debug exceptions, like a hardware breakpoint,
> > will we not see an unhandled exception and the guest being forcefully
> > killed?
> 
> Yes until the later patches which stop the guest using HW debug
> registers while we are using them.
> 
> >
> >> +
> >>  }
> >>  
> >>  void kvm_arch_clear_debug(struct kvm_vcpu *vcpu)
> >> diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c
> >> index 524fa25..ed1bbb4 100644
> >> --- a/arch/arm64/kvm/handle_exit.c
> >> +++ b/arch/arm64/kvm/handle_exit.c
> >> @@ -82,6 +82,37 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
> >>  	return 1;
> >>  }
> >>  
> >> +/**
> >> + * kvm_handle_debug_exception - handle a debug exception instruction
> >
> > handle a software breadkpoint exception
> >
> >> + *
> >> + * @vcpu:	the vcpu pointer
> >> + * @run:	access to the kvm_run structure for results
> >> + *
> >> + * We route all debug exceptions through the same handler as we
> >
> > all debug exceptions?  software breakpoints and all?  then why the above
> > shot text?
> >

I think the issue here was "debug exception instruction" making me think
this is just for software breakpoints...

Not sure what I meant by 'shot text' - probably 'short text'

> >> + * just need to report the PC and the HSR values to userspace.
> >> + * Userspace may decide to re-inject the exception and deliver it to
> >> + * the guest if it wasn't for the host to deal with.
> >
> > now I'm confused - does userspace setup the guest to receive an
> > exception or does it tell KVM to emulate an exception for the guest or
> > do we execute the breakpoint without trapping the debug exception?
> 
> I've made it all go through userspace as we may have to translate the
> hypervisor visible exception code to what the guest was expecting to see.
> 

ok, so I think you should re-phrase something like:

"Userspace may decide that this exception is caused by the guest using
debugging itself, and may in that case emulate the guest debug exception
in userspace before resuming KVM."

But does that really work?  Given that we don't support KVM-TCG
migration, this sounds a little strange.  Did we test it?

> >
> >> + */
> >> +static int kvm_handle_guest_debug(struct kvm_vcpu *vcpu, struct kvm_run *run)
> >> +{
> >> +	u32 hsr = kvm_vcpu_get_hsr(vcpu);
> >> +
> >> +	run->exit_reason = KVM_EXIT_DEBUG;
> >> +	run->debug.arch.hsr = hsr;
> >> +
> >> +	switch (hsr >> ESR_ELx_EC_SHIFT) {
> >> +	case ESR_ELx_EC_BKPT32:
> >> +	case ESR_ELx_EC_BRK64:
> >> +		run->debug.arch.pc = *vcpu_pc(vcpu);
> >> +		break;
> >> +	default:
> >> +		kvm_err("%s: un-handled case hsr: %#08x\n",
> >> +			__func__, (unsigned int) hsr);
> >
> > this should never happen right?
> 
> At the moment it could, at the end of the patch series we should cover
> all the cases so it would indicate a bug. I've made it return an error
> code so it fails hard as suggested by David.
> 
hmm, ok, so I'm not so worried about that kind of bisectability
(although it would be nice to keep that working too), but reading
patches that way is a bit annoying for reviewers, so I recommend you
deal with the patch ordering in some way that makes it more obvious what
happens as reviewers read the patches, one at a time.

Thanks,
-Christoffer

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Peter Maydell]

On 27 April 2015 at 21:04, Christoffer Dall <christoffer.dall@linaro.org> wrote:
> On Thu, Apr 23, 2015 at 03:26:53PM +0100, Alex Bennée wrote:
>>
>> Christoffer Dall <christoffer.dall@linaro.org> writes:
>>
>> > On Tue, Mar 31, 2015 at 04:08:04PM +0100, Alex Bennée wrote:
>> >> + * just need to report the PC and the HSR values to userspace.
>> >> + * Userspace may decide to re-inject the exception and deliver it to
>> >> + * the guest if it wasn't for the host to deal with.
>> >
>> > now I'm confused - does userspace setup the guest to receive an
>> > exception or does it tell KVM to emulate an exception for the guest or
>> > do we execute the breakpoint without trapping the debug exception?
>>
>> I've made it all go through userspace as we may have to translate the
>> hypervisor visible exception code to what the guest was expecting to see.
>>
>
> ok, so I think you should re-phrase something like:
>
> "Userspace may decide that this exception is caused by the guest using
> debugging itself, and may in that case emulate the guest debug exception
> in userspace before resuming KVM."
>
> But does that really work?  Given that we don't support KVM-TCG
> migration, this sounds a little strange.  Did we test it?

The QEMU patches have a TODO note at the point where you'd want
to do this... Design-wise you can do the reinjection in the
kernel or in userspace (ppc QEMU does this in userspace, for
instance) because it's pretty much just setting registers to fake
up the exception-entry into EL1. Code-wise QEMU's ARM code isn't
set up to do it right now, but it shouldn't be too difficult to
persuade the TCG exception-entry code to work for this case too.

Does the kernel already have a conveniently implemented "inject
exception into guest" lump of code? If so it might be less effort
to do it that way round, maybe.

-- PMM

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Alex Bennée]

Peter Maydell <peter.maydell@linaro.org> writes:

> On 27 April 2015 at 21:04, Christoffer Dall <christoffer.dall@linaro.org> wrote:
>> On Thu, Apr 23, 2015 at 03:26:53PM +0100, Alex Bennée wrote:
>>>
>>> Christoffer Dall <christoffer.dall@linaro.org> writes:
>>>
>>> > On Tue, Mar 31, 2015 at 04:08:04PM +0100, Alex Bennée wrote:
>>> >> + * just need to report the PC and the HSR values to userspace.
>>> >> + * Userspace may decide to re-inject the exception and deliver it to
>>> >> + * the guest if it wasn't for the host to deal with.
>>> >
>>> > now I'm confused - does userspace setup the guest to receive an
>>> > exception or does it tell KVM to emulate an exception for the guest or
>>> > do we execute the breakpoint without trapping the debug exception?
>>>
>>> I've made it all go through userspace as we may have to translate the
>>> hypervisor visible exception code to what the guest was expecting to see.
>>>
>>
>> ok, so I think you should re-phrase something like:
>>
>> "Userspace may decide that this exception is caused by the guest using
>> debugging itself, and may in that case emulate the guest debug exception
>> in userspace before resuming KVM."
>>
>> But does that really work?  Given that we don't support KVM-TCG
>> migration, this sounds a little strange.  Did we test it?
>
> The QEMU patches have a TODO note at the point where you'd want
> to do this... Design-wise you can do the reinjection in the
> kernel or in userspace (ppc QEMU does this in userspace, for
> instance) because it's pretty much just setting registers to fake
> up the exception-entry into EL1. Code-wise QEMU's ARM code isn't
> set up to do it right now, but it shouldn't be too difficult to
> persuade the TCG exception-entry code to work for this case too.
>
> Does the kernel already have a conveniently implemented "inject
> exception into guest" lump of code? If so it might be less effort
> to do it that way round, maybe.

So you pointed out we can't just re-inject the exceptions we get as we
need to map from things like ESR_ELx_EC_WATCHPT_LOW to
ESR_ELx_EC_WATCHPT_CUR before re-injection.

Of course if it is as simple as modifying the ESR_EL1 register and
returning +ve in the handle_exit path then I can do that but I assumed
if any other wrangling needs doing it should be done in userspace.

>
> -- PMM

-- 
Alex Bennée

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Peter Maydell]

On 28 April 2015 at 09:42, Alex Bennée <alex.bennee@linaro.org> wrote:
> Peter Maydell <peter.maydell@linaro.org> writes:
>> Does the kernel already have a conveniently implemented "inject
>> exception into guest" lump of code? If so it might be less effort
>> to do it that way round, maybe.
>
> So you pointed out we can't just re-inject the exceptions we get as we
> need to map from things like ESR_ELx_EC_WATCHPT_LOW to
> ESR_ELx_EC_WATCHPT_CUR before re-injection.
>
> Of course if it is as simple as modifying the ESR_EL1 register and
> returning +ve in the handle_exit path then I can do that but I assumed
> if any other wrangling needs doing it should be done in userspace.

Well, somebody's got to do it, and it's the same amount of work
either way (fiddling with ESR, making sure we direct the guest
to the right exception vector entry point, maybe a few other
things).

-- PMM

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Christoffer Dall]

On Tue, Apr 28, 2015 at 10:34:12AM +0100, Peter Maydell wrote:
> On 28 April 2015 at 09:42, Alex Bennée <alex.bennee@linaro.org> wrote:
> > Peter Maydell <peter.maydell@linaro.org> writes:
> >> Does the kernel already have a conveniently implemented "inject
> >> exception into guest" lump of code? If so it might be less effort
> >> to do it that way round, maybe.
> >
> > So you pointed out we can't just re-inject the exceptions we get as we
> > need to map from things like ESR_ELx_EC_WATCHPT_LOW to
> > ESR_ELx_EC_WATCHPT_CUR before re-injection.
> >
> > Of course if it is as simple as modifying the ESR_EL1 register and
> > returning +ve in the handle_exit path then I can do that but I assumed
> > if any other wrangling needs doing it should be done in userspace.
> 
> Well, somebody's got to do it, and it's the same amount of work
> either way (fiddling with ESR, making sure we direct the guest
> to the right exception vector entry point, maybe a few other
> things).
> 
We already have code in the kernel to inject data/instruction aborts,
but not sure how much benefit there is in re-using that.  It's up to you
really, but I think the kernel code should be clear about what the
intention is so that we don't end up in a situation where: (1) The
intended behavior is unclear/vague, and (2) it doesn't actually work in
practice so nobody can follow the code.

Thanks,
-Christoffer

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Alex Bennée]

Christoffer Dall <christoffer.dall@linaro.org> writes:

> On Tue, Apr 28, 2015 at 10:34:12AM +0100, Peter Maydell wrote:
>> On 28 April 2015 at 09:42, Alex Bennée <alex.bennee@linaro.org> wrote:
>> > Peter Maydell <peter.maydell@linaro.org> writes:
>> >> Does the kernel already have a conveniently implemented "inject
>> >> exception into guest" lump of code? If so it might be less effort
>> >> to do it that way round, maybe.
>> >
>> > So you pointed out we can't just re-inject the exceptions we get as we
>> > need to map from things like ESR_ELx_EC_WATCHPT_LOW to
>> > ESR_ELx_EC_WATCHPT_CUR before re-injection.
>> >
>> > Of course if it is as simple as modifying the ESR_EL1 register and
>> > returning +ve in the handle_exit path then I can do that but I assumed
>> > if any other wrangling needs doing it should be done in userspace.
>> 
>> Well, somebody's got to do it, and it's the same amount of work
>> either way (fiddling with ESR, making sure we direct the guest
>> to the right exception vector entry point, maybe a few other
>> things).
>> 
> We already have code in the kernel to inject data/instruction aborts,
> but not sure how much benefit there is in re-using that.  It's up to you
> really, but I think the kernel code should be clear about what the
> intention is so that we don't end up in a situation where: (1) The
> intended behavior is unclear/vague, and (2) it doesn't actually work in
> practice so nobody can follow the code.

Certainly there are some cases where the kernel doesn't have all the
information. For example it doesn't know if the soft break was inserted
by the guest or the host. That to me favours the "let userspace deal
with the ugly" approach.

-- 
Alex Bennée

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Christoffer Dall]

On Tue, Apr 28, 2015 at 03:37:01PM +0100, Alex Bennée wrote:
> 
> Christoffer Dall <christoffer.dall@linaro.org> writes:
> 
> > On Tue, Apr 28, 2015 at 10:34:12AM +0100, Peter Maydell wrote:
> >> On 28 April 2015 at 09:42, Alex Bennée <alex.bennee@linaro.org> wrote:
> >> > Peter Maydell <peter.maydell@linaro.org> writes:
> >> >> Does the kernel already have a conveniently implemented "inject
> >> >> exception into guest" lump of code? If so it might be less effort
> >> >> to do it that way round, maybe.
> >> >
> >> > So you pointed out we can't just re-inject the exceptions we get as we
> >> > need to map from things like ESR_ELx_EC_WATCHPT_LOW to
> >> > ESR_ELx_EC_WATCHPT_CUR before re-injection.
> >> >
> >> > Of course if it is as simple as modifying the ESR_EL1 register and
> >> > returning +ve in the handle_exit path then I can do that but I assumed
> >> > if any other wrangling needs doing it should be done in userspace.
> >> 
> >> Well, somebody's got to do it, and it's the same amount of work
> >> either way (fiddling with ESR, making sure we direct the guest
> >> to the right exception vector entry point, maybe a few other
> >> things).
> >> 
> > We already have code in the kernel to inject data/instruction aborts,
> > but not sure how much benefit there is in re-using that.  It's up to you
> > really, but I think the kernel code should be clear about what the
> > intention is so that we don't end up in a situation where: (1) The
> > intended behavior is unclear/vague, and (2) it doesn't actually work in
> > practice so nobody can follow the code.
> 
> Certainly there are some cases where the kernel doesn't have all the
> information. For example it doesn't know if the soft break was inserted
> by the guest or the host. That to me favours the "let userspace deal
> with the ugly" approach.
> 
Not sure I follow.

If it's an exception for the guest, then that must be because the guest
put in the breakpoint instruction, right?

However, that's a separate discussion from that of *how* userspace or
the kernel then injects an exception to the guest.

By using some QEMU TCG functionality or by QEMU calling back into KVM
and asking it to inject an exception for it.

What am I missing?

-Christoffer

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Alex Bennée]

Christoffer Dall <christoffer.dall@linaro.org> writes:

> On Tue, Apr 28, 2015 at 03:37:01PM +0100, Alex Bennée wrote:
>> 
>> Christoffer Dall <christoffer.dall@linaro.org> writes:
>> 
>> > On Tue, Apr 28, 2015 at 10:34:12AM +0100, Peter Maydell wrote:
>> >> On 28 April 2015 at 09:42, Alex Bennée <alex.bennee@linaro.org> wrote:
>> >> > Peter Maydell <peter.maydell@linaro.org> writes:
>> >> >> Does the kernel already have a conveniently implemented "inject
>> >> >> exception into guest" lump of code? If so it might be less effort
>> >> >> to do it that way round, maybe.
>> >> >
>> >> > So you pointed out we can't just re-inject the exceptions we get as we
>> >> > need to map from things like ESR_ELx_EC_WATCHPT_LOW to
>> >> > ESR_ELx_EC_WATCHPT_CUR before re-injection.
>> >> >
>> >> > Of course if it is as simple as modifying the ESR_EL1 register and
>> >> > returning +ve in the handle_exit path then I can do that but I assumed
>> >> > if any other wrangling needs doing it should be done in userspace.
>> >> 
>> >> Well, somebody's got to do it, and it's the same amount of work
>> >> either way (fiddling with ESR, making sure we direct the guest
>> >> to the right exception vector entry point, maybe a few other
>> >> things).
>> >> 
>> > We already have code in the kernel to inject data/instruction aborts,
>> > but not sure how much benefit there is in re-using that.  It's up to you
>> > really, but I think the kernel code should be clear about what the
>> > intention is so that we don't end up in a situation where: (1) The
>> > intended behavior is unclear/vague, and (2) it doesn't actually work in
>> > practice so nobody can follow the code.
>> 
>> Certainly there are some cases where the kernel doesn't have all the
>> information. For example it doesn't know if the soft break was inserted
>> by the guest or the host. That to me favours the "let userspace deal
>> with the ugly" approach.
>> 
> Not sure I follow.
>
> If it's an exception for the guest, then that must be because the guest
> put in the breakpoint instruction, right?

No the host can add breakpoint instructions as well. They both generate
the same (redirected) exception to the hypervisor which then has to
figure out who planted the breakpoint and where the eventual exception
will be handled.

> However, that's a separate discussion from that of *how* userspace or
> the kernel then injects an exception to the guest.
>
> By using some QEMU TCG functionality or by QEMU calling back into KVM
> and asking it to inject an exception for it.

I don't know if there is explicit TCG functionality to use but QEMU can
set the registers and PC up for exception entry and re-enter KVM.

>
> What am I missing?
>
> -Christoffer

-- 
Alex Bennée

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Christoffer Dall]

On Wed, Apr 29, 2015 at 10:18:18AM +0100, Alex Bennée wrote:
> 
> Christoffer Dall <christoffer.dall@linaro.org> writes:
> 
> > On Tue, Apr 28, 2015 at 03:37:01PM +0100, Alex Bennée wrote:
> >> 
> >> Christoffer Dall <christoffer.dall@linaro.org> writes:
> >> 
> >> > On Tue, Apr 28, 2015 at 10:34:12AM +0100, Peter Maydell wrote:
> >> >> On 28 April 2015 at 09:42, Alex Bennée <alex.bennee@linaro.org> wrote:
> >> >> > Peter Maydell <peter.maydell@linaro.org> writes:
> >> >> >> Does the kernel already have a conveniently implemented "inject
> >> >> >> exception into guest" lump of code? If so it might be less effort
> >> >> >> to do it that way round, maybe.
> >> >> >
> >> >> > So you pointed out we can't just re-inject the exceptions we get as we
> >> >> > need to map from things like ESR_ELx_EC_WATCHPT_LOW to
> >> >> > ESR_ELx_EC_WATCHPT_CUR before re-injection.
> >> >> >
> >> >> > Of course if it is as simple as modifying the ESR_EL1 register and
> >> >> > returning +ve in the handle_exit path then I can do that but I assumed
> >> >> > if any other wrangling needs doing it should be done in userspace.
> >> >> 
> >> >> Well, somebody's got to do it, and it's the same amount of work
> >> >> either way (fiddling with ESR, making sure we direct the guest
> >> >> to the right exception vector entry point, maybe a few other
> >> >> things).
> >> >> 
> >> > We already have code in the kernel to inject data/instruction aborts,
> >> > but not sure how much benefit there is in re-using that.  It's up to you
> >> > really, but I think the kernel code should be clear about what the
> >> > intention is so that we don't end up in a situation where: (1) The
> >> > intended behavior is unclear/vague, and (2) it doesn't actually work in
> >> > practice so nobody can follow the code.
> >> 
> >> Certainly there are some cases where the kernel doesn't have all the
> >> information. For example it doesn't know if the soft break was inserted
> >> by the guest or the host. That to me favours the "let userspace deal
> >> with the ugly" approach.
> >> 
> > Not sure I follow.
> >
> > If it's an exception for the guest, then that must be because the guest
> > put in the breakpoint instruction, right?
> 
> No the host can add breakpoint instructions as well. They both generate
> the same (redirected) exception to the hypervisor which then has to
> figure out who planted the breakpoint and where the eventual exception
> will be handled.

I understand this; let's just rewind here.

If you've concluded that the exception is for the guest, then the guest
must have placed the breakpoint instruction there, correct?  Otherwise,
the exception is for the hypervisor and the discussion about how to
inject an exception for the guest is invalid.

Or are you talking about the corner case where the host uses a soft
breakpoint to get a breakpoint on an instruction which is also a
breakpoint in the guest?

> 
> > However, that's a separate discussion from that of *how* userspace or
> > the kernel then injects an exception to the guest.
> >
> > By using some QEMU TCG functionality or by QEMU calling back into KVM
> > and asking it to inject an exception for it.
> 
> I don't know if there is explicit TCG functionality to use but QEMU can
> set the registers and PC up for exception entry and re-enter KVM.
> 

I also understand this.  I think Peter's point was exactly that if we
have existing code somewhere which we can reuse, then we should consider
reusing it.

Again, I don't care particularly which way, I just want the expected
working behavior to be clearly defined.

-Christoffer

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Alex Bennée]

Christoffer Dall <christoffer.dall@linaro.org> writes:

> On Wed, Apr 29, 2015 at 10:18:18AM +0100, Alex Bennée wrote:
>> 
>> Christoffer Dall <christoffer.dall@linaro.org> writes:
>> 
>> > On Tue, Apr 28, 2015 at 03:37:01PM +0100, Alex Bennée wrote:
>> >> 
>> >> Christoffer Dall <christoffer.dall@linaro.org> writes:
>> >> 
>> >> > On Tue, Apr 28, 2015 at 10:34:12AM +0100, Peter Maydell wrote:
>> >> >> On 28 April 2015 at 09:42, Alex Bennée <alex.bennee@linaro.org> wrote:
>> >> >> > Peter Maydell <peter.maydell@linaro.org> writes:
>> >> >> >> Does the kernel already have a conveniently implemented "inject
>> >> >> >> exception into guest" lump of code? If so it might be less effort
>> >> >> >> to do it that way round, maybe.
>> >> >> >
<snip>
>> >> 
>> >> Certainly there are some cases where the kernel doesn't have all the
>> >> information. For example it doesn't know if the soft break was inserted
>> >> by the guest or the host. That to me favours the "let userspace deal
>> >> with the ugly" approach.
>> >> 
>> > Not sure I follow.
>> >
>> > If it's an exception for the guest, then that must be because the guest
>> > put in the breakpoint instruction, right?
>> 
>> No the host can add breakpoint instructions as well. They both generate
>> the same (redirected) exception to the hypervisor which then has to
>> figure out who planted the breakpoint and where the eventual exception
>> will be handled.
>
> I understand this; let's just rewind here.
>
> If you've concluded that the exception is for the guest, then the guest
> must have placed the breakpoint instruction there, correct?  Otherwise,
> the exception is for the hypervisor and the discussion about how to
> inject an exception for the guest is invalid.

But only userspace has enough information to make that conclusion (after
searching the list of breakpoints it added to the code). So from
userspace we can:

  - re-enter KVM telling it to re-route the exception it just delivered
    to userspace somehow

  or

  - make the changes to deliver the exception in userspace and re-enter
    KVM as normal.

It seems to me if we have already exited into userspace it may as well
clean up if it has all the information it needs?

> Or are you talking about the corner case where the host uses a soft
> breakpoint to get a breakpoint on an instruction which is also a
> breakpoint in the guest?

I think in this case host debugging just wins.

>
>> 
>> > However, that's a separate discussion from that of *how* userspace or
>> > the kernel then injects an exception to the guest.
>> >
>> > By using some QEMU TCG functionality or by QEMU calling back into KVM
>> > and asking it to inject an exception for it.
>> 
>> I don't know if there is explicit TCG functionality to use but QEMU can
>> set the registers and PC up for exception entry and re-enter KVM.
>> 
>
> I also understand this.  I think Peter's point was exactly that if we
> have existing code somewhere which we can reuse, then we should consider
> reusing it.

I'm not sure such code exists. The only injection code I know of in KVMs
handle_exit code where a +ve return value signals KVM to deliver the
exception to the guest. This is used by the hvc and svc handlers after
calling kvm_inject_undefined() and the wfx handler which advances the PC
first.

> Again, I don't care particularly which way, I just want the expected
> working behavior to be clearly defined.

I think it makes sense to do it in userspace. I have the kernels
inject_fault code for reference for what needs setting up but I'll see
if I can find anything in QEMU that already handles this for some other
thing (although I don't think it does at first glance).

-- 
Alex Bennée

Re: [PATCH v2 06/10] KVM: arm64: guest debug, add SW break point support

[Christoffer Dall]

On Wed, Apr 29, 2015 at 5:08 PM, Alex Bennée <alex.bennee@linaro.org> wrote:
>
> Christoffer Dall <christoffer.dall@linaro.org> writes:
>
>> On Wed, Apr 29, 2015 at 10:18:18AM +0100, Alex Bennée wrote:
>>>
>>> Christoffer Dall <christoffer.dall@linaro.org> writes:
>>>
>>> > On Tue, Apr 28, 2015 at 03:37:01PM +0100, Alex Bennée wrote:
>>> >>
>>> >> Christoffer Dall <christoffer.dall@linaro.org> writes:
>>> >>
>>> >> > On Tue, Apr 28, 2015 at 10:34:12AM +0100, Peter Maydell wrote:
>>> >> >> On 28 April 2015 at 09:42, Alex Bennée <alex.bennee@linaro.org> wrote:
>>> >> >> > Peter Maydell <peter.maydell@linaro.org> writes:
>>> >> >> >> Does the kernel already have a conveniently implemented "inject
>>> >> >> >> exception into guest" lump of code? If so it might be less effort
>>> >> >> >> to do it that way round, maybe.
>>> >> >> >
> <snip>
>>> >>
>>> >> Certainly there are some cases where the kernel doesn't have all the
>>> >> information. For example it doesn't know if the soft break was inserted
>>> >> by the guest or the host. That to me favours the "let userspace deal
>>> >> with the ugly" approach.
>>> >>
>>> > Not sure I follow.
>>> >
>>> > If it's an exception for the guest, then that must be because the guest
>>> > put in the breakpoint instruction, right?
>>>
>>> No the host can add breakpoint instructions as well. They both generate
>>> the same (redirected) exception to the hypervisor which then has to
>>> figure out who planted the breakpoint and where the eventual exception
>>> will be handled.
>>
>> I understand this; let's just rewind here.
>>
>> If you've concluded that the exception is for the guest, then the guest
>> must have placed the breakpoint instruction there, correct?  Otherwise,
>> the exception is for the hypervisor and the discussion about how to
>> inject an exception for the guest is invalid.
>
> But only userspace has enough information to make that conclusion (after
> searching the list of breakpoints it added to the code). So from
> userspace we can:
>
>   - re-enter KVM telling it to re-route the exception it just delivered
>     to userspace somehow
>
>   or
>
>   - make the changes to deliver the exception in userspace and re-enter
>     KVM as normal.
>

ok, we agree and are talking about the same thing.  good.

> It seems to me if we have already exited into userspace it may as well
> clean up if it has all the information it needs?
>

depends on the complexity and size of the code really, imho.

>> Or are you talking about the corner case where the host uses a soft
>> breakpoint to get a breakpoint on an instruction which is also a
>> breakpoint in the guest?
>
> I think in this case host debugging just wins.
>
ok