[PATCH v2 0/6] kvm: x86: export TSC information to user-space

[PATCH v2 0/6] kvm: x86: export TSC information to user-space

[Luiz Capitulino]

[Introduction will follow]

Changelog
---------

v2

 - add tsc_offset field to struct kvm_vcpu_arch
 - drop read_tsc_offset()
 - add per-vcpu dir entries in debugfs
 - export TSC scaling info (besides TSC offset)
 - export the TSC offset as a signed number
 - drop patch that wrongly tried to improve error
   handling in kvm_create_vm_debugfs()

Intro
-----

This series exports a VM's TSC offset and TSC scaling
information to user-space via a new per-vcpu directory
in debugfs. For example:

  /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-offset
  /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio
  /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio-frac-bits

The TSC offset in particular is needed in user-space
in order for tracing tools, such as trace-cmd, to be
able to merge the host and guest traces using the
host TSC. This is explained in detail in this thread:

  [Qemu-devel] [RFC] host and guest kernel trace merging
  https://lists.nongnu.org/archive/html/qemu-devel/2016-03/msg00887.html

Luiz Capitulino (6):
  kvm: x86: add tsc_offset field to struct kvm_vcpu_arch
  kvm: x86: drop read_tsc_offset()
  kvm: kvm_destroy_vm_debugfs(): check debugfs_stat_data pointer
  kvm: add stubs for arch specific debugfs support
  kvm: create per-vcpu dirs in debugfs
  kvm: x86: export TSC information to user-space

 arch/arm/kvm/arm.c              | 10 ++++++
 arch/mips/kvm/mips.c            | 10 ++++++
 arch/powerpc/kvm/powerpc.c      | 10 ++++++
 arch/s390/kvm/kvm-s390.c        | 10 ++++++
 arch/x86/include/asm/kvm_host.h |  2 +-
 arch/x86/kvm/Makefile           |  2 +-
 arch/x86/kvm/debugfs.c          | 69 +++++++++++++++++++++++++++++++++++++++++
 arch/x86/kvm/svm.c              |  8 -----
 arch/x86/kvm/vmx.c              |  6 ----
 arch/x86/kvm/x86.c              | 12 +++++--
 include/linux/kvm_host.h        |  4 +++
 virt/kvm/kvm_main.c             | 40 ++++++++++++++++++++++--
 12 files changed, 161 insertions(+), 22 deletions(-)
 create mode 100644 arch/x86/kvm/debugfs.c

-- 
2.5.5

[PATCH 1/6] kvm: x86: add tsc_offset field to struct kvm_vcpu_arch

[Luiz Capitulino]

A future commit will want to easily read a vCPU's TSC offset,
so we store it in struct kvm_arch_vcpu_arch for easy access.

Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
---
 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/x86.c              | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 4c738c2..9b36a31 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -568,6 +568,7 @@ struct kvm_vcpu_arch {
 		struct kvm_steal_time steal;
 	} st;
 
+	u64 tsc_offset;
 	u64 last_guest_tsc;
 	u64 last_host_tsc;
 	u64 tsc_offset_adjustment;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 19f9f9e..cda4ca5 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1413,6 +1413,12 @@ u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
 }
 EXPORT_SYMBOL_GPL(kvm_read_l1_tsc);
 
+static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
+{
+	kvm_x86_ops->write_tsc_offset(vcpu, offset);
+	vcpu->arch.tsc_offset = offset;
+}
+
 void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
 {
 	struct kvm *kvm = vcpu->kvm;
@@ -1522,7 +1528,7 @@ void kvm_write_tsc(struct kvm_vcpu *vcpu, struct msr_data *msr)
 
 	if (guest_cpuid_has_tsc_adjust(vcpu) && !msr->host_initiated)
 		update_ia32_tsc_adjust_msr(vcpu, offset);
-	kvm_x86_ops->write_tsc_offset(vcpu, offset);
+	kvm_vcpu_write_tsc_offset(vcpu, offset);
 	raw_spin_unlock_irqrestore(&kvm->arch.tsc_write_lock, flags);
 
 	spin_lock(&kvm->arch.pvclock_gtod_sync_lock);
@@ -2750,7 +2756,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 		if (check_tsc_unstable()) {
 			u64 offset = kvm_compute_tsc_offset(vcpu,
 						vcpu->arch.last_guest_tsc);
-			kvm_x86_ops->write_tsc_offset(vcpu, offset);
+			kvm_vcpu_write_tsc_offset(vcpu, offset);
 			vcpu->arch.tsc_catchup = 1;
 		}
 		/*
-- 
2.5.5

[PATCH 2/6] kvm: x86: drop read_tsc_offset()

[Luiz Capitulino]

The TSC offset can now be read directly from struct kvm_arch_vcpu.

Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
---
 arch/x86/include/asm/kvm_host.h | 1 -
 arch/x86/kvm/svm.c              | 8 --------
 arch/x86/kvm/vmx.c              | 6 ------
 arch/x86/kvm/x86.c              | 2 +-
 4 files changed, 1 insertion(+), 16 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 9b36a31..c4c5ac5 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -954,7 +954,6 @@ struct kvm_x86_ops {
 
 	bool (*has_wbinvd_exit)(void);
 
-	u64 (*read_tsc_offset)(struct kvm_vcpu *vcpu);
 	void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
 
 	u64 (*read_l1_tsc)(struct kvm_vcpu *vcpu, u64 host_tsc);
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index db77c1c..8023d53 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -1119,13 +1119,6 @@ static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
 	seg->base = 0;
 }
 
-static u64 svm_read_tsc_offset(struct kvm_vcpu *vcpu)
-{
-	struct vcpu_svm *svm = to_svm(vcpu);
-
-	return svm->vmcb->control.tsc_offset;
-}
-
 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
@@ -5427,7 +5420,6 @@ static struct kvm_x86_ops svm_x86_ops = {
 
 	.has_wbinvd_exit = svm_has_wbinvd_exit,
 
-	.read_tsc_offset = svm_read_tsc_offset,
 	.write_tsc_offset = svm_write_tsc_offset,
 	.adjust_tsc_offset_guest = svm_adjust_tsc_offset_guest,
 	.read_l1_tsc = svm_read_l1_tsc,
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 5cede40..29cbd4b 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -2603,11 +2603,6 @@ static u64 vmx_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
 	return host_tsc + tsc_offset;
 }
 
-static u64 vmx_read_tsc_offset(struct kvm_vcpu *vcpu)
-{
-	return vmcs_read64(TSC_OFFSET);
-}
-
 /*
  * writes 'offset' into guest's timestamp counter offset register
  */
@@ -11274,7 +11269,6 @@ static struct kvm_x86_ops vmx_x86_ops = {
 
 	.has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
 
-	.read_tsc_offset = vmx_read_tsc_offset,
 	.write_tsc_offset = vmx_write_tsc_offset,
 	.adjust_tsc_offset_guest = vmx_adjust_tsc_offset_guest,
 	.read_l1_tsc = vmx_read_l1_tsc,
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index cda4ca5..1651668 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1367,7 +1367,7 @@ static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
 
 static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
 {
-	u64 curr_offset = kvm_x86_ops->read_tsc_offset(vcpu);
+	u64 curr_offset = vcpu->arch.tsc_offset;
 	vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
 }
 
-- 
2.5.5

[PATCH 3/6] kvm: kvm_destroy_vm_debugfs(): check debugfs_stat_data pointer

[Luiz Capitulino]

Otherwise, the kernel panics when kvm_create_vm_debugfs()
fails before assigning this pointer.

Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
---
 virt/kvm/kvm_main.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 1950782..c1dc45e 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -559,9 +559,11 @@ static void kvm_destroy_vm_debugfs(struct kvm *kvm)
 
 	debugfs_remove_recursive(kvm->debugfs_dentry);
 
-	for (i = 0; i < kvm_debugfs_num_entries; i++)
-		kfree(kvm->debugfs_stat_data[i]);
-	kfree(kvm->debugfs_stat_data);
+	if (kvm->debugfs_stat_data) {
+		for (i = 0; i < kvm_debugfs_num_entries; i++)
+			kfree(kvm->debugfs_stat_data[i]);
+		kfree(kvm->debugfs_stat_data);
+	}
 }
 
 static int kvm_create_vm_debugfs(struct kvm *kvm, int fd)
-- 
2.5.5

[PATCH 4/6] kvm: add stubs for arch specific debugfs support

[Luiz Capitulino]

Two stubs are added:

 o kvm_arch_has_vcpu_debugfs(): must return true if the arch
   supports creating debugfs entries in the vcpu debugfs dir
   (which will be implemented by the next commit)

 o kvm_arch_create_vcpu_debugfs(): code that creates debugfs
   entries in the vcpu debugfs dir

For x86, this commit introduces a new file to avoid growing
arch/x86/kvm/x86.c even more.

Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
---
 arch/arm/kvm/arm.c         | 10 ++++++++++
 arch/mips/kvm/mips.c       | 10 ++++++++++
 arch/powerpc/kvm/powerpc.c | 10 ++++++++++
 arch/s390/kvm/kvm-s390.c   | 10 ++++++++++
 arch/x86/kvm/Makefile      |  2 +-
 arch/x86/kvm/debugfs.c     | 20 ++++++++++++++++++++
 include/linux/kvm_host.h   |  3 +++
 7 files changed, 64 insertions(+), 1 deletion(-)
 create mode 100644 arch/x86/kvm/debugfs.c

diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c
index 75f130e..c638935 100644
--- a/arch/arm/kvm/arm.c
+++ b/arch/arm/kvm/arm.c
@@ -144,6 +144,16 @@ out_fail_alloc:
 	return ret;
 }
 
+bool kvm_arch_has_vcpu_debugfs(void)
+{
+	return false;
+}
+
+int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
+{
+	return 0;
+}
+
 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
 {
 	return VM_FAULT_SIGBUS;
diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index a6ea084..49b25e7 100644
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -140,6 +140,16 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	return 0;
 }
 
+bool kvm_arch_has_vcpu_debugfs(void)
+{
+	return false;
+}
+
+int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
+{
+	return 0;
+}
+
 void kvm_mips_free_vcpus(struct kvm *kvm)
 {
 	unsigned int i;
diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c
index 6ce40dd..b1d194d 100644
--- a/arch/powerpc/kvm/powerpc.c
+++ b/arch/powerpc/kvm/powerpc.c
@@ -436,6 +436,16 @@ err_out:
 	return -EINVAL;
 }
 
+bool kvm_arch_has_vcpu_debugfs(void)
+{
+	return false;
+}
+
+int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
+{
+	return 0;
+}
+
 void kvm_arch_destroy_vm(struct kvm *kvm)
 {
 	unsigned int i;
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index f142215..60e7f00 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -1490,6 +1490,16 @@ out_err:
 	return rc;
 }
 
+bool kvm_arch_has_vcpu_debugfs(void)
+{
+	return false;
+}
+
+int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
+{
+	return 0;
+}
+
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 {
 	VCPU_EVENT(vcpu, 3, "%s", "free cpu");
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 464fa47..3bff207 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -13,7 +13,7 @@ kvm-$(CONFIG_KVM_ASYNC_PF)	+= $(KVM)/async_pf.o
 
 kvm-y			+= x86.o mmu.o emulate.o i8259.o irq.o lapic.o \
 			   i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \
-			   hyperv.o page_track.o
+			   hyperv.o page_track.o debugfs.o
 
 kvm-$(CONFIG_KVM_DEVICE_ASSIGNMENT)	+= assigned-dev.o iommu.o
 
diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c
new file mode 100644
index 0000000..bb5e9f6
--- /dev/null
+++ b/arch/x86/kvm/debugfs.c
@@ -0,0 +1,20 @@
+/*
+ * Kernel-based Virtual Machine driver for Linux
+ *
+ * Copyright 2016 Red Hat, Inc. and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ */
+#include <linux/kvm_host.h>
+
+bool kvm_arch_has_vcpu_debugfs(void)
+{
+	return false;
+}
+
+int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
+{
+	return 0;
+}
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 9c28b4d..5486ff9 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -749,6 +749,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu);
 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu);
 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu);
 
+bool kvm_arch_has_vcpu_debugfs(void);
+int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu);
+
 int kvm_arch_hardware_enable(void);
 void kvm_arch_hardware_disable(void);
 int kvm_arch_hardware_setup(void);
-- 
2.5.5

[PATCH 5/6] kvm: create per-vcpu dirs in debugfs

[Luiz Capitulino]

This commit adds the ability for archs to export
per-vcpu information via a new per-vcpu dir in
the VM's debugfs directory.

If kvm_arch_has_vcpu_debugfs() returns true, then KVM
will create a vcpu dir for each vCPU in the VM's
debugfs directory. Then kvm_arch_create_vcpu_debugfs()
is responsible for populating each vcpu directory
with arch specific entries.

The per-vcpu path in debugfs will look like:

/sys/kernel/debug/kvm/29162-10/vcpu0
/sys/kernel/debug/kvm/29162-10/vcpu1

This is all arch specific for now because the only
user of this interface (x86) wants to export x86-specific
per-vcpu information to user-space.

Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
---
 include/linux/kvm_host.h |  1 +
 virt/kvm/kvm_main.c      | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)

diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 5486ff9..01c0b9c 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -265,6 +265,7 @@ struct kvm_vcpu {
 #endif
 	bool preempted;
 	struct kvm_vcpu_arch arch;
+	struct dentry *debugfs_dentry;
 };
 
 static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu)
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index c1dc45e..c598915 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -2371,6 +2371,7 @@ static int kvm_vcpu_release(struct inode *inode, struct file *filp)
 {
 	struct kvm_vcpu *vcpu = filp->private_data;
 
+	debugfs_remove_recursive(vcpu->debugfs_dentry);
 	kvm_put_kvm(vcpu->kvm);
 	return 0;
 }
@@ -2393,6 +2394,32 @@ static int create_vcpu_fd(struct kvm_vcpu *vcpu)
 	return anon_inode_getfd("kvm-vcpu", &kvm_vcpu_fops, vcpu, O_RDWR | O_CLOEXEC);
 }
 
+static int kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
+{
+	char dir_name[ITOA_MAX_LEN * 2];
+	int ret;
+
+	if (!kvm_arch_has_vcpu_debugfs())
+		return 0;
+
+	if (!debugfs_initialized())
+		return 0;
+
+	snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id);
+	vcpu->debugfs_dentry = debugfs_create_dir(dir_name,
+								vcpu->kvm->debugfs_dentry);
+	if (!vcpu->debugfs_dentry)
+		return -ENOMEM;
+
+	ret = kvm_arch_create_vcpu_debugfs(vcpu);
+	if (ret < 0) {
+		debugfs_remove_recursive(vcpu->debugfs_dentry);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * Creates some virtual cpus.  Good luck creating more than one.
  */
@@ -2425,6 +2452,10 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
 	if (r)
 		goto vcpu_destroy;
 
+	r = kvm_create_vcpu_debugfs(vcpu);
+	if (r)
+		goto vcpu_destroy;
+
 	mutex_lock(&kvm->lock);
 	if (kvm_get_vcpu_by_id(kvm, id)) {
 		r = -EEXIST;
@@ -2456,6 +2487,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id)
 
 unlock_vcpu_destroy:
 	mutex_unlock(&kvm->lock);
+	debugfs_remove_recursive(vcpu->debugfs_dentry);
 vcpu_destroy:
 	kvm_arch_vcpu_destroy(vcpu);
 vcpu_decrement:
-- 
2.5.5

[PATCH 6/6] kvm: x86: export TSC information to user-space

[Luiz Capitulino]

This commit exports the following information to
user-space via the newly created per-vcpu debugfs
directory:

 - TSC offset (as a signed number)
 - TSC scaling ratio
 - TSC scaling ratio fractinal bits

The original intention of this commit was to
export only the TSC offset, but the TSC scaling
information is exported for completeness.

We need to retrieve the TSC offset from user-space
in order to support the merging of host and guest
traces in trace-cmd. Today, we use the kvm_write_tsc_offset
tracepoint, but it has a number of problems (mainly,
it requires a running VM to be rebooted, ftrace setup,
and also tracepoints are not supposed to be ABIs).

The merging of host and guest traces is explained
in more detail in this thread:

 [Qemu-devel] [RFC] host and guest kernel trace merging
 https://lists.nongnu.org/archive/html/qemu-devel/2016-03/msg00887.html

This commit creates the following files in debugfs:

/sys/kernel/debug/kvm/66828-10/vcpu0/tsc-offset
/sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio
/sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio-frac-bits

The last two are only created if TSC scaling is supported.

Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
---
 arch/x86/kvm/debugfs.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c
index bb5e9f6..c19c7ed 100644
--- a/arch/x86/kvm/debugfs.c
+++ b/arch/x86/kvm/debugfs.c
@@ -8,13 +8,62 @@
  *
  */
 #include <linux/kvm_host.h>
+#include <linux/debugfs.h>
 
 bool kvm_arch_has_vcpu_debugfs(void)
 {
-	return false;
+	return true;
 }
 
+static int vcpu_get_tsc_offset(void *data, u64 *val)
+{
+	struct kvm_vcpu *vcpu = (struct kvm_vcpu *) data;
+	*val = vcpu->arch.tsc_offset;
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(vcpu_tsc_offset_fops, vcpu_get_tsc_offset, NULL, "%lld\n");
+
+static int vcpu_get_tsc_scaling_ratio(void *data, u64 *val)
+{
+	struct kvm_vcpu *vcpu = (struct kvm_vcpu *) data;
+	*val = vcpu->arch.tsc_scaling_ratio;
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(vcpu_tsc_scaling_fops, vcpu_get_tsc_scaling_ratio, NULL, "%llu\n");
+
+static int vcpu_get_tsc_scaling_frac_bits(void *data, u64 *val)
+{
+	*val = kvm_tsc_scaling_ratio_frac_bits;
+	return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(vcpu_tsc_scaling_frac_fops, vcpu_get_tsc_scaling_frac_bits, NULL, "%llu\n");
+
 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu)
 {
+	struct dentry *ret;
+
+	ret = debugfs_create_file("tsc-offset", 0444,
+							vcpu->debugfs_dentry,
+							vcpu, &vcpu_tsc_offset_fops);
+	if (!ret)
+		return -ENOMEM;
+
+	if (kvm_has_tsc_control) {
+		ret = debugfs_create_file("tsc-scaling-ratio", 0444,
+							vcpu->debugfs_dentry,
+							vcpu, &vcpu_tsc_scaling_fops);
+		if (!ret)
+			return -ENOMEM;
+		ret = debugfs_create_file("tsc-scaling-ratio-frac-bits", 0444,
+							vcpu->debugfs_dentry,
+							vcpu, &vcpu_tsc_scaling_frac_fops);
+		if (!ret)
+			return -ENOMEM;
+
+	}
+
 	return 0;
 }
-- 
2.5.5

Re: [PATCH v2 0/6] kvm: x86: export TSC information to user-space

[Paolo Bonzini]

On 16/09/2016 16:27, Luiz Capitulino wrote:
> [Introduction will follow]
> 
> Changelog
> ---------
> 
> v2
> 
>  - add tsc_offset field to struct kvm_vcpu_arch
>  - drop read_tsc_offset()
>  - add per-vcpu dir entries in debugfs
>  - export TSC scaling info (besides TSC offset)
>  - export the TSC offset as a signed number
>  - drop patch that wrongly tried to improve error
>    handling in kvm_create_vm_debugfs()

I've tested this patch on an AMD machine with TSC scaling, so I'm
pushing it shortly to kvm/queue.

Paolo

> Intro
> -----
> 
> This series exports a VM's TSC offset and TSC scaling
> information to user-space via a new per-vcpu directory
> in debugfs. For example:
> 
>   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-offset
>   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio
>   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio-frac-bits
> 
> The TSC offset in particular is needed in user-space
> in order for tracing tools, such as trace-cmd, to be
> able to merge the host and guest traces using the
> host TSC. This is explained in detail in this thread:
> 
>   [Qemu-devel] [RFC] host and guest kernel trace merging
>   https://lists.nongnu.org/archive/html/qemu-devel/2016-03/msg00887.html
> 
> Luiz Capitulino (6):
>   kvm: x86: add tsc_offset field to struct kvm_vcpu_arch
>   kvm: x86: drop read_tsc_offset()
>   kvm: kvm_destroy_vm_debugfs(): check debugfs_stat_data pointer
>   kvm: add stubs for arch specific debugfs support
>   kvm: create per-vcpu dirs in debugfs
>   kvm: x86: export TSC information to user-space
> 
>  arch/arm/kvm/arm.c              | 10 ++++++
>  arch/mips/kvm/mips.c            | 10 ++++++
>  arch/powerpc/kvm/powerpc.c      | 10 ++++++
>  arch/s390/kvm/kvm-s390.c        | 10 ++++++
>  arch/x86/include/asm/kvm_host.h |  2 +-
>  arch/x86/kvm/Makefile           |  2 +-
>  arch/x86/kvm/debugfs.c          | 69 +++++++++++++++++++++++++++++++++++++++++
>  arch/x86/kvm/svm.c              |  8 -----
>  arch/x86/kvm/vmx.c              |  6 ----
>  arch/x86/kvm/x86.c              | 12 +++++--
>  include/linux/kvm_host.h        |  4 +++
>  virt/kvm/kvm_main.c             | 40 ++++++++++++++++++++++--
>  12 files changed, 161 insertions(+), 22 deletions(-)
>  create mode 100644 arch/x86/kvm/debugfs.c
> 

Re: [PATCH v2 0/6] kvm: x86: export TSC information to user-space

[Luiz Capitulino]

On Fri, 16 Sep 2016 16:56:34 +0200
Paolo Bonzini <pbonzini@redhat.com> wrote:

> On 16/09/2016 16:27, Luiz Capitulino wrote:
> > [Introduction will follow]
> > 
> > Changelog
> > ---------
> > 
> > v2
> > 
> >  - add tsc_offset field to struct kvm_vcpu_arch
> >  - drop read_tsc_offset()
> >  - add per-vcpu dir entries in debugfs
> >  - export TSC scaling info (besides TSC offset)
> >  - export the TSC offset as a signed number
> >  - drop patch that wrongly tried to improve error
> >    handling in kvm_create_vm_debugfs()  
> 
> I've tested this patch on an AMD machine with TSC scaling, so I'm
> pushing it shortly to kvm/queue.

Thanks for the testing! Btw, I don't mind letting it sit on the
list for a few days for review (well, no submitter should mind
this).

> 
> Paolo
> 
> > Intro
> > -----
> > 
> > This series exports a VM's TSC offset and TSC scaling
> > information to user-space via a new per-vcpu directory
> > in debugfs. For example:
> > 
> >   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-offset
> >   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio
> >   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio-frac-bits
> > 
> > The TSC offset in particular is needed in user-space
> > in order for tracing tools, such as trace-cmd, to be
> > able to merge the host and guest traces using the
> > host TSC. This is explained in detail in this thread:
> > 
> >   [Qemu-devel] [RFC] host and guest kernel trace merging
> >   https://lists.nongnu.org/archive/html/qemu-devel/2016-03/msg00887.html
> > 
> > Luiz Capitulino (6):
> >   kvm: x86: add tsc_offset field to struct kvm_vcpu_arch
> >   kvm: x86: drop read_tsc_offset()
> >   kvm: kvm_destroy_vm_debugfs(): check debugfs_stat_data pointer
> >   kvm: add stubs for arch specific debugfs support
> >   kvm: create per-vcpu dirs in debugfs
> >   kvm: x86: export TSC information to user-space
> > 
> >  arch/arm/kvm/arm.c              | 10 ++++++
> >  arch/mips/kvm/mips.c            | 10 ++++++
> >  arch/powerpc/kvm/powerpc.c      | 10 ++++++
> >  arch/s390/kvm/kvm-s390.c        | 10 ++++++
> >  arch/x86/include/asm/kvm_host.h |  2 +-
> >  arch/x86/kvm/Makefile           |  2 +-
> >  arch/x86/kvm/debugfs.c          | 69 +++++++++++++++++++++++++++++++++++++++++
> >  arch/x86/kvm/svm.c              |  8 -----
> >  arch/x86/kvm/vmx.c              |  6 ----
> >  arch/x86/kvm/x86.c              | 12 +++++--
> >  include/linux/kvm_host.h        |  4 +++
> >  virt/kvm/kvm_main.c             | 40 ++++++++++++++++++++++--
> >  12 files changed, 161 insertions(+), 22 deletions(-)
> >  create mode 100644 arch/x86/kvm/debugfs.c
> >   
> 

Re: [PATCH v2 0/6] kvm: x86: export TSC information to user-space

[Paolo Bonzini]

On 16/09/2016 16:59, Luiz Capitulino wrote:
> On Fri, 16 Sep 2016 16:56:34 +0200
> Paolo Bonzini <pbonzini@redhat.com> wrote:
> 
>> On 16/09/2016 16:27, Luiz Capitulino wrote:
>>> [Introduction will follow]
>>>
>>> Changelog
>>> ---------
>>>
>>> v2
>>>
>>>  - add tsc_offset field to struct kvm_vcpu_arch
>>>  - drop read_tsc_offset()
>>>  - add per-vcpu dir entries in debugfs
>>>  - export TSC scaling info (besides TSC offset)
>>>  - export the TSC offset as a signed number
>>>  - drop patch that wrongly tried to improve error
>>>    handling in kvm_create_vm_debugfs()  
>>
>> I've tested this patch on an AMD machine with TSC scaling, so I'm
>> pushing it shortly to kvm/queue.
> 
> Thanks for the testing! Btw, I don't mind letting it sit on the
> list for a few days for review (well, no submitter should mind
> this).

That's what kvm/queue is for. :)

Paolo

>>
>> Paolo
>>
>>> Intro
>>> -----
>>>
>>> This series exports a VM's TSC offset and TSC scaling
>>> information to user-space via a new per-vcpu directory
>>> in debugfs. For example:
>>>
>>>   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-offset
>>>   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio
>>>   /sys/kernel/debug/kvm/66828-10/vcpu0/tsc-scaling-ratio-frac-bits
>>>
>>> The TSC offset in particular is needed in user-space
>>> in order for tracing tools, such as trace-cmd, to be
>>> able to merge the host and guest traces using the
>>> host TSC. This is explained in detail in this thread:
>>>
>>>   [Qemu-devel] [RFC] host and guest kernel trace merging
>>>   https://lists.nongnu.org/archive/html/qemu-devel/2016-03/msg00887.html
>>>
>>> Luiz Capitulino (6):
>>>   kvm: x86: add tsc_offset field to struct kvm_vcpu_arch
>>>   kvm: x86: drop read_tsc_offset()
>>>   kvm: kvm_destroy_vm_debugfs(): check debugfs_stat_data pointer
>>>   kvm: add stubs for arch specific debugfs support
>>>   kvm: create per-vcpu dirs in debugfs
>>>   kvm: x86: export TSC information to user-space
>>>
>>>  arch/arm/kvm/arm.c              | 10 ++++++
>>>  arch/mips/kvm/mips.c            | 10 ++++++
>>>  arch/powerpc/kvm/powerpc.c      | 10 ++++++
>>>  arch/s390/kvm/kvm-s390.c        | 10 ++++++
>>>  arch/x86/include/asm/kvm_host.h |  2 +-
>>>  arch/x86/kvm/Makefile           |  2 +-
>>>  arch/x86/kvm/debugfs.c          | 69 +++++++++++++++++++++++++++++++++++++++++
>>>  arch/x86/kvm/svm.c              |  8 -----
>>>  arch/x86/kvm/vmx.c              |  6 ----
>>>  arch/x86/kvm/x86.c              | 12 +++++--
>>>  include/linux/kvm_host.h        |  4 +++
>>>  virt/kvm/kvm_main.c             | 40 ++++++++++++++++++++++--
>>>  12 files changed, 161 insertions(+), 22 deletions(-)
>>>  create mode 100644 arch/x86/kvm/debugfs.c
>>>   
>>
> 

Re: [PATCH v2 0/6] kvm: x86: export TSC information to user-space

[Luiz Capitulino]

On Fri, 16 Sep 2016 16:59:55 +0200
Paolo Bonzini <pbonzini@redhat.com> wrote:

> On 16/09/2016 16:59, Luiz Capitulino wrote:
> > On Fri, 16 Sep 2016 16:56:34 +0200
> > Paolo Bonzini <pbonzini@redhat.com> wrote:
> >   
> >> On 16/09/2016 16:27, Luiz Capitulino wrote:  
> >>> [Introduction will follow]
> >>>
> >>> Changelog
> >>> ---------
> >>>
> >>> v2
> >>>
> >>>  - add tsc_offset field to struct kvm_vcpu_arch
> >>>  - drop read_tsc_offset()
> >>>  - add per-vcpu dir entries in debugfs
> >>>  - export TSC scaling info (besides TSC offset)
> >>>  - export the TSC offset as a signed number
> >>>  - drop patch that wrongly tried to improve error
> >>>    handling in kvm_create_vm_debugfs()    
> >>
> >> I've tested this patch on an AMD machine with TSC scaling, so I'm
> >> pushing it shortly to kvm/queue.  
> > 
> > Thanks for the testing! Btw, I don't mind letting it sit on the
> > list for a few days for review (well, no submitter should mind
> > this).  
> 
> That's what kvm/queue is for. :)

Makes sense.

Re: [PATCH 2/6] kvm: x86: drop read_tsc_offset()

[Jim Mattson]

vmx_read_tsc_offset has a bug when running nested VMs.  It should really be:

       if (is_guest_mode(vcpu))
               return to_vmx(vcpu)->nested.vmcs01_tsc_offset;
       else
               return vmcs_read64(TSC_OFFSET);

Perhaps a better name woulf be "vmx_get_l1_tsc_offset."

In any case, this does not seem consistent with vcpu->arch.tsc_offset.

On Fri, Sep 16, 2016 at 7:27 AM, Luiz Capitulino <lcapitulino@redhat.com> wrote:
> The TSC offset can now be read directly from struct kvm_arch_vcpu.
>
> Signed-off-by: Luiz Capitulino <lcapitulino@redhat.com>
> ---
>  arch/x86/include/asm/kvm_host.h | 1 -
>  arch/x86/kvm/svm.c              | 8 --------
>  arch/x86/kvm/vmx.c              | 6 ------
>  arch/x86/kvm/x86.c              | 2 +-
>  4 files changed, 1 insertion(+), 16 deletions(-)
>
> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
> index 9b36a31..c4c5ac5 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -954,7 +954,6 @@ struct kvm_x86_ops {
>
>         bool (*has_wbinvd_exit)(void);
>
> -       u64 (*read_tsc_offset)(struct kvm_vcpu *vcpu);
>         void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset);
>
>         u64 (*read_l1_tsc)(struct kvm_vcpu *vcpu, u64 host_tsc);
> diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
> index db77c1c..8023d53 100644
> --- a/arch/x86/kvm/svm.c
> +++ b/arch/x86/kvm/svm.c
> @@ -1119,13 +1119,6 @@ static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
>         seg->base = 0;
>  }
>
> -static u64 svm_read_tsc_offset(struct kvm_vcpu *vcpu)
> -{
> -       struct vcpu_svm *svm = to_svm(vcpu);
> -
> -       return svm->vmcb->control.tsc_offset;
> -}
> -
>  static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
>  {
>         struct vcpu_svm *svm = to_svm(vcpu);
> @@ -5427,7 +5420,6 @@ static struct kvm_x86_ops svm_x86_ops = {
>
>         .has_wbinvd_exit = svm_has_wbinvd_exit,
>
> -       .read_tsc_offset = svm_read_tsc_offset,
>         .write_tsc_offset = svm_write_tsc_offset,
>         .adjust_tsc_offset_guest = svm_adjust_tsc_offset_guest,
>         .read_l1_tsc = svm_read_l1_tsc,
> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
> index 5cede40..29cbd4b 100644
> --- a/arch/x86/kvm/vmx.c
> +++ b/arch/x86/kvm/vmx.c
> @@ -2603,11 +2603,6 @@ static u64 vmx_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
>         return host_tsc + tsc_offset;
>  }
>
> -static u64 vmx_read_tsc_offset(struct kvm_vcpu *vcpu)
> -{
> -       return vmcs_read64(TSC_OFFSET);
> -}
> -
>  /*
>   * writes 'offset' into guest's timestamp counter offset register
>   */
> @@ -11274,7 +11269,6 @@ static struct kvm_x86_ops vmx_x86_ops = {
>
>         .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit,
>
> -       .read_tsc_offset = vmx_read_tsc_offset,
>         .write_tsc_offset = vmx_write_tsc_offset,
>         .adjust_tsc_offset_guest = vmx_adjust_tsc_offset_guest,
>         .read_l1_tsc = vmx_read_l1_tsc,
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index cda4ca5..1651668 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -1367,7 +1367,7 @@ static void kvm_track_tsc_matching(struct kvm_vcpu *vcpu)
>
>  static void update_ia32_tsc_adjust_msr(struct kvm_vcpu *vcpu, s64 offset)
>  {
> -       u64 curr_offset = kvm_x86_ops->read_tsc_offset(vcpu);
> +       u64 curr_offset = vcpu->arch.tsc_offset;
>         vcpu->arch.ia32_tsc_adjust_msr += offset - curr_offset;
>  }
>
> --
> 2.5.5
>
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH 2/6] kvm: x86: drop read_tsc_offset()

[Paolo Bonzini]

[-- Attachment #1.1: Type: text/plain, Size: 496 bytes --]



On 19/09/2016 17:30, Jim Mattson wrote:
> vmx_read_tsc_offset has a bug when running nested VMs.  It should really be:
> 
>        if (is_guest_mode(vcpu))
>                return to_vmx(vcpu)->nested.vmcs01_tsc_offset;
>        else
>                return vmcs_read64(TSC_OFFSET);
> 
> Perhaps a better name woulf be "vmx_get_l1_tsc_offset."

I agree, but doesn't this patch fix the bug too?

Paolo

> In any case, this does not seem consistent with vcpu->arch.tsc_offset.


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 473 bytes --]

Re: [PATCH 2/6] kvm: x86: drop read_tsc_offset()

[Jim Mattson]

Hmmm. Yes, I think it does. With this patch series,
vcpu->arch.tsc_offset appears to contain L1's TSC offset (perhaps
making vmx->nested.vmcs01_tsc_offset redundant).

However, this unfortunately limits the newly added functionality to
merging host and *L1* guest traces. It doesn't work with L2 (or
deeper) guests. Or perhaps I'm missing something?

On Mon, Sep 19, 2016 at 8:34 AM, Paolo Bonzini <pbonzini@redhat.com> wrote:
>
>
> On 19/09/2016 17:30, Jim Mattson wrote:
>> vmx_read_tsc_offset has a bug when running nested VMs.  It should really be:
>>
>>        if (is_guest_mode(vcpu))
>>                return to_vmx(vcpu)->nested.vmcs01_tsc_offset;
>>        else
>>                return vmcs_read64(TSC_OFFSET);
>>
>> Perhaps a better name woulf be "vmx_get_l1_tsc_offset."
>
> I agree, but doesn't this patch fix the bug too?
>
> Paolo
>
>> In any case, this does not seem consistent with vcpu->arch.tsc_offset.
>

Re: [PATCH 2/6] kvm: x86: drop read_tsc_offset()

[Paolo Bonzini]

On 20/09/2016 00:18, Jim Mattson wrote:
> Hmmm. Yes, I think it does. With this patch series,
> vcpu->arch.tsc_offset appears to contain L1's TSC offset (perhaps
> making vmx->nested.vmcs01_tsc_offset redundant).
> 
> However, this unfortunately limits the newly added functionality to
> merging host and *L1* guest traces. It doesn't work with L2 (or
> deeper) guests. Or perhaps I'm missing something?

You can merge L1/L2 first and then host/L1.

Paolo

Re: [PATCH 2/6] kvm: x86: drop read_tsc_offset()

[Jim Mattson]

Doesn't that assume you can run the merge program in L1?

On Mon, Sep 19, 2016 at 10:37 PM, Paolo Bonzini <pbonzini@redhat.com> wrote:
>
>
> On 20/09/2016 00:18, Jim Mattson wrote:
>> Hmmm. Yes, I think it does. With this patch series,
>> vcpu->arch.tsc_offset appears to contain L1's TSC offset (perhaps
>> making vmx->nested.vmcs01_tsc_offset redundant).
>>
>> However, this unfortunately limits the newly added functionality to
>> merging host and *L1* guest traces. It doesn't work with L2 (or
>> deeper) guests. Or perhaps I'm missing something?
>
> You can merge L1/L2 first and then host/L1.
>
> Paolo

Re: [PATCH 2/6] kvm: x86: drop read_tsc_offset()

[Paolo Bonzini]

On 21/09/2016 17:19, Jim Mattson wrote:
> Doesn't that assume you can run the merge program in L1?

You only need the TSC offset, but we should make sure that L0
tracepoints contain enough information to figure out the L0->L2 TSC
offsets (they are the values in VMCS02).

That said, how would you get the trace from L1 if you don't have access
to it?

Paolo

> On Mon, Sep 19, 2016 at 10:37 PM, Paolo Bonzini <pbonzini@redhat.com> wrote:
>>
>>
>> On 20/09/2016 00:18, Jim Mattson wrote:
>>> Hmmm. Yes, I think it does. With this patch series,
>>> vcpu->arch.tsc_offset appears to contain L1's TSC offset (perhaps
>>> making vmx->nested.vmcs01_tsc_offset redundant).
>>>
>>> However, this unfortunately limits the newly added functionality to
>>> merging host and *L1* guest traces. It doesn't work with L2 (or
>>> deeper) guests. Or perhaps I'm missing something?
>>
>> You can merge L1/L2 first and then host/L1.
>>
>> Paolo

Re: [PATCH 2/6] kvm: x86: drop read_tsc_offset()

[Jim Mattson]

I'm thinking about the case where you want to merge traces from L0 and
L2. If the user code in L0 always knew the TSC offset of the current
VMCS, rather than the TSC offset of the L1 VMCS, this would be
trivial, regardless of the nature of L1.

On Wed, Sep 21, 2016 at 8:22 AM, Paolo Bonzini <pbonzini@redhat.com> wrote:
>
>
> On 21/09/2016 17:19, Jim Mattson wrote:
>> Doesn't that assume you can run the merge program in L1?
>
> You only need the TSC offset, but we should make sure that L0
> tracepoints contain enough information to figure out the L0->L2 TSC
> offsets (they are the values in VMCS02).
>
> That said, how would you get the trace from L1 if you don't have access
> to it?
>
> Paolo
>
>> On Mon, Sep 19, 2016 at 10:37 PM, Paolo Bonzini <pbonzini@redhat.com> wrote:
>>>
>>>
>>> On 20/09/2016 00:18, Jim Mattson wrote:
>>>> Hmmm. Yes, I think it does. With this patch series,
>>>> vcpu->arch.tsc_offset appears to contain L1's TSC offset (perhaps
>>>> making vmx->nested.vmcs01_tsc_offset redundant).
>>>>
>>>> However, this unfortunately limits the newly added functionality to
>>>> merging host and *L1* guest traces. It doesn't work with L2 (or
>>>> deeper) guests. Or perhaps I'm missing something?
>>>
>>> You can merge L1/L2 first and then host/L1.
>>>
>>> Paolo