* [RFC PATCH 1/9] psci: export psci conduit get helper.
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
@ 2020-02-10 8:48 ` Jianyong Wu
2020-02-10 8:48 ` [RFC PATCH 2/9] ptp: Reorganize ptp_kvm modules to make it arch-independent Jianyong Wu
` (8 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:48 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
Export arm_smccc_1_1_get_conduit then modules can use smccc helper which
adopts it.
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
---
drivers/firmware/psci/psci.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index b3b6c15e7b36..e0738d6f3d16 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -64,6 +64,7 @@ enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void)
return psci_ops.conduit;
}
+EXPORT_SYMBOL(arm_smccc_1_1_get_conduit);
typedef unsigned long (psci_fn)(unsigned long, unsigned long,
unsigned long, unsigned long);
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* [RFC PATCH 2/9] ptp: Reorganize ptp_kvm modules to make it arch-independent.
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
2020-02-10 8:48 ` [RFC PATCH 1/9] psci: export psci conduit get helper Jianyong Wu
@ 2020-02-10 8:48 ` Jianyong Wu
2020-02-10 8:49 ` [RFC PATCH 3/9] time: Add mechanism to recognize clocksource in time_get_snapshot Jianyong Wu
` (7 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:48 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
Currently, ptp_kvm modules implementation is only for x86 which includs
large part of arch-specific code. This patch move all of those code
into new arch related file in the same directory.
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
---
drivers/ptp/Makefile | 1 +
drivers/ptp/ptp_kvm.h | 11 +++
drivers/ptp/{ptp_kvm.c => ptp_kvm_common.c} | 78 +++++-------------
drivers/ptp/ptp_kvm_x86.c | 87 +++++++++++++++++++++
4 files changed, 118 insertions(+), 59 deletions(-)
create mode 100644 drivers/ptp/ptp_kvm.h
rename drivers/ptp/{ptp_kvm.c => ptp_kvm_common.c} (63%)
create mode 100644 drivers/ptp/ptp_kvm_x86.c
diff --git a/drivers/ptp/Makefile b/drivers/ptp/Makefile
index 8c830336f178..06837d089347 100644
--- a/drivers/ptp/Makefile
+++ b/drivers/ptp/Makefile
@@ -4,6 +4,7 @@
#
ptp-y := ptp_clock.o ptp_chardev.o ptp_sysfs.o
+ptp_kvm-y := ptp_kvm_$(ARCH).o ptp_kvm_common.o
obj-$(CONFIG_PTP_1588_CLOCK) += ptp.o
obj-$(CONFIG_PTP_1588_CLOCK_DTE) += ptp_dte.o
obj-$(CONFIG_PTP_1588_CLOCK_INES) += ptp_ines.o
diff --git a/drivers/ptp/ptp_kvm.h b/drivers/ptp/ptp_kvm.h
new file mode 100644
index 000000000000..4bf1802bbeb8
--- /dev/null
+++ b/drivers/ptp/ptp_kvm.h
@@ -0,0 +1,11 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Virtual PTP 1588 clock for use with KVM guests
+ *
+ * Copyright (C) 2017 Red Hat Inc.
+ */
+
+int kvm_arch_ptp_init(void);
+int kvm_arch_ptp_get_clock(struct timespec64 *ts);
+int kvm_arch_ptp_get_crosststamp(unsigned long *cycle,
+ struct timespec64 *tspec, void *cs);
diff --git a/drivers/ptp/ptp_kvm.c b/drivers/ptp/ptp_kvm_common.c
similarity index 63%
rename from drivers/ptp/ptp_kvm.c
rename to drivers/ptp/ptp_kvm_common.c
index fc7d0b77e118..60442f70d3fc 100644
--- a/drivers/ptp/ptp_kvm.c
+++ b/drivers/ptp/ptp_kvm_common.c
@@ -8,15 +8,16 @@
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
+#include <linux/slab.h>
#include <linux/module.h>
#include <uapi/linux/kvm_para.h>
#include <asm/kvm_para.h>
-#include <asm/pvclock.h>
-#include <asm/kvmclock.h>
#include <uapi/asm/kvm_para.h>
#include <linux/ptp_clock_kernel.h>
+#include "ptp_kvm.h"
+
struct kvm_ptp_clock {
struct ptp_clock *ptp_clock;
struct ptp_clock_info caps;
@@ -24,56 +25,29 @@ struct kvm_ptp_clock {
DEFINE_SPINLOCK(kvm_ptp_lock);
-static struct pvclock_vsyscall_time_info *hv_clock;
-
-static struct kvm_clock_pairing clock_pair;
-static phys_addr_t clock_pair_gpa;
-
static int ptp_kvm_get_time_fn(ktime_t *device_time,
struct system_counterval_t *system_counter,
void *ctx)
{
- unsigned long ret;
+ unsigned long ret, cycle;
struct timespec64 tspec;
- unsigned version;
- int cpu;
- struct pvclock_vcpu_time_info *src;
+ struct clocksource *cs;
spin_lock(&kvm_ptp_lock);
preempt_disable_notrace();
- cpu = smp_processor_id();
- src = &hv_clock[cpu].pvti;
-
- do {
- /*
- * We are using a TSC value read in the hosts
- * kvm_hc_clock_pairing handling.
- * So any changes to tsc_to_system_mul
- * and tsc_shift or any other pvclock
- * data invalidate that measurement.
- */
- version = pvclock_read_begin(src);
-
- ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING,
- clock_pair_gpa,
- KVM_CLOCK_PAIRING_WALLCLOCK);
- if (ret != 0) {
- pr_err_ratelimited("clock pairing hypercall ret %lu\n", ret);
- spin_unlock(&kvm_ptp_lock);
- preempt_enable_notrace();
- return -EOPNOTSUPP;
- }
-
- tspec.tv_sec = clock_pair.sec;
- tspec.tv_nsec = clock_pair.nsec;
- ret = __pvclock_read_cycles(src, clock_pair.tsc);
- } while (pvclock_read_retry(src, version));
+ ret = kvm_arch_ptp_get_crosststamp(&cycle, &tspec, &cs);
+ if (ret != 0) {
+ pr_err_ratelimited("clock pairing hypercall ret %lu\n", ret);
+ spin_unlock(&kvm_ptp_lock);
+ preempt_enable_notrace();
+ return -EOPNOTSUPP;
+ }
preempt_enable_notrace();
- system_counter->cycles = ret;
- system_counter->cs = &kvm_clock;
+ system_counter->cycles = cycle;
+ system_counter->cs = cs;
*device_time = timespec64_to_ktime(tspec);
@@ -116,17 +90,13 @@ static int ptp_kvm_gettime(struct ptp_clock_info *ptp, struct timespec64 *ts)
spin_lock(&kvm_ptp_lock);
- ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING,
- clock_pair_gpa,
- KVM_CLOCK_PAIRING_WALLCLOCK);
+ ret = kvm_arch_ptp_get_clock(&tspec);
if (ret != 0) {
pr_err_ratelimited("clock offset hypercall ret %lu\n", ret);
spin_unlock(&kvm_ptp_lock);
return -EOPNOTSUPP;
}
- tspec.tv_sec = clock_pair.sec;
- tspec.tv_nsec = clock_pair.nsec;
spin_unlock(&kvm_ptp_lock);
memcpy(ts, &tspec, sizeof(struct timespec64));
@@ -166,21 +136,11 @@ static void __exit ptp_kvm_exit(void)
static int __init ptp_kvm_init(void)
{
- long ret;
-
- if (!kvm_para_available())
- return -ENODEV;
+ int ret;
- clock_pair_gpa = slow_virt_to_phys(&clock_pair);
- hv_clock = pvclock_get_pvti_cpu0_va();
-
- if (!hv_clock)
- return -ENODEV;
-
- ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING, clock_pair_gpa,
- KVM_CLOCK_PAIRING_WALLCLOCK);
- if (ret == -KVM_ENOSYS || ret == -KVM_EOPNOTSUPP)
- return -ENODEV;
+ ret = kvm_arch_ptp_init();
+ if (ret)
+ return -EOPNOTSUPP;
kvm_ptp_clock.caps = ptp_kvm_caps;
diff --git a/drivers/ptp/ptp_kvm_x86.c b/drivers/ptp/ptp_kvm_x86.c
new file mode 100644
index 000000000000..6c891d7299c6
--- /dev/null
+++ b/drivers/ptp/ptp_kvm_x86.c
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Virtual PTP 1588 clock for use with KVM guests
+ *
+ * Copyright (C) 2017 Red Hat Inc.
+ */
+
+#include <asm/pvclock.h>
+#include <asm/kvmclock.h>
+#include <linux/module.h>
+#include <uapi/asm/kvm_para.h>
+#include <uapi/linux/kvm_para.h>
+#include <linux/ptp_clock_kernel.h>
+
+phys_addr_t clock_pair_gpa;
+struct kvm_clock_pairing clock_pair;
+struct pvclock_vsyscall_time_info *hv_clock;
+
+int kvm_arch_ptp_init(void)
+{
+ int ret;
+
+ if (!kvm_para_available())
+ return -ENODEV;
+
+ clock_pair_gpa = slow_virt_to_phys(&clock_pair);
+ hv_clock = pvclock_get_pvti_cpu0_va();
+ if (!hv_clock)
+ return -ENODEV;
+
+ ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING, clock_pair_gpa,
+ KVM_CLOCK_PAIRING_WALLCLOCK);
+ if (ret == -KVM_ENOSYS || ret == -KVM_EOPNOTSUPP)
+ return -ENODEV;
+
+ return 0;
+}
+
+int kvm_arch_ptp_get_clock(struct timespec64 *ts)
+{
+ long ret;
+
+ ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING,
+ clock_pair_gpa,
+ KVM_CLOCK_PAIRING_WALLCLOCK);
+ if (ret != 0)
+ return -EOPNOTSUPP;
+
+ ts->tv_sec = clock_pair.sec;
+ ts->tv_nsec = clock_pair.nsec;
+
+ return 0;
+}
+
+int kvm_arch_ptp_get_crosststamp(unsigned long *cycle, struct timespec64 *tspec,
+ struct clocksource **cs)
+{
+ unsigned long ret;
+ unsigned int version;
+ int cpu;
+ struct pvclock_vcpu_time_info *src;
+
+ cpu = smp_processor_id();
+ src = &hv_clock[cpu].pvti;
+
+ do {
+ /*
+ * We are using a TSC value read in the hosts
+ * kvm_hc_clock_pairing handling.
+ * So any changes to tsc_to_system_mul
+ * and tsc_shift or any other pvclock
+ * data invalidate that measurement.
+ */
+ version = pvclock_read_begin(src);
+
+ ret = kvm_hypercall2(KVM_HC_CLOCK_PAIRING,
+ clock_pair_gpa,
+ KVM_CLOCK_PAIRING_WALLCLOCK);
+ tspec->tv_sec = clock_pair.sec;
+ tspec->tv_nsec = clock_pair.nsec;
+ *cycle = __pvclock_read_cycles(src, clock_pair.tsc);
+ } while (pvclock_read_retry(src, version));
+
+ *cs = &kvm_clock;
+
+ return 0;
+}
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* [RFC PATCH 3/9] time: Add mechanism to recognize clocksource in time_get_snapshot
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
2020-02-10 8:48 ` [RFC PATCH 1/9] psci: export psci conduit get helper Jianyong Wu
2020-02-10 8:48 ` [RFC PATCH 2/9] ptp: Reorganize ptp_kvm modules to make it arch-independent Jianyong Wu
@ 2020-02-10 8:49 ` Jianyong Wu
2020-02-10 8:49 ` [RFC PATCH 4/9] clocksource: Add clocksource id for arm arch counter Jianyong Wu
` (6 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:49 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
From: Thomas Gleixner <tglx@linutronix.de>
System time snapshots are not conveying information about the current
clocksource which was used, but callers like the PTP KVM guest
implementation have the requirement to evaluate the clocksource type to
select the appropriate mechanism.
Introduce a clocksource id field in struct clocksource which is by default
set to CSID_GENERIC (0). Clocksource implementations can set that field to
a value which allows to identify the clocksource.
Store the clocksource id of the current clocksource in the
system_time_snapshot so callers can evaluate which clocksource was used to
take the snapshot and act accordingly.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
---
include/linux/clocksource.h | 6 ++++++
include/linux/clocksource_ids.h | 12 ++++++++++++
include/linux/timekeeping.h | 12 +++++++-----
kernel/time/clocksource.c | 3 +++
kernel/time/timekeeping.c | 1 +
5 files changed, 29 insertions(+), 5 deletions(-)
create mode 100644 include/linux/clocksource_ids.h
diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index b21db536fd52..96e85b6f9ca0 100644
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -17,6 +17,7 @@
#include <linux/timer.h>
#include <linux/init.h>
#include <linux/of.h>
+#include <linux/clocksource_ids.h>
#include <asm/div64.h>
#include <asm/io.h>
@@ -49,6 +50,10 @@ struct module;
* 400-499: Perfect
* The ideal clocksource. A must-use where
* available.
+ * @id: Defaults to CSID_GENERIC. The id value is captured
+ * in certain snapshot functions to allow callers to
+ * validate the clocksource from which the snapshot was
+ * taken.
* @read: returns a cycle value, passes clocksource as argument
* @enable: optional function to enable the clocksource
* @disable: optional function to disable the clocksource
@@ -91,6 +96,7 @@ struct clocksource {
const char *name;
struct list_head list;
int rating;
+ enum clocksource_ids id;
int (*enable)(struct clocksource *cs);
void (*disable)(struct clocksource *cs);
unsigned long flags;
diff --git a/include/linux/clocksource_ids.h b/include/linux/clocksource_ids.h
new file mode 100644
index 000000000000..5595ab843c3d
--- /dev/null
+++ b/include/linux/clocksource_ids.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_CLOCKSOURCE_IDS_H
+#define _LINUX_CLOCKSOURCE_IDS_H
+
+/* Enum to give clocksources a unique identifier */
+enum clocksource_ids {
+ CSID_GENERIC = 0,
+ CSID_MAX,
+};
+
+#endif
+
diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h
index b27e2ffa96c1..4ecc32ad3879 100644
--- a/include/linux/timekeeping.h
+++ b/include/linux/timekeeping.h
@@ -2,6 +2,7 @@
#ifndef _LINUX_TIMEKEEPING_H
#define _LINUX_TIMEKEEPING_H
+#include <linux/clocksource_ids.h>
#include <linux/errno.h>
/* Included from linux/ktime.h */
@@ -232,11 +233,12 @@ extern void timekeeping_inject_sleeptime64(const struct timespec64 *delta);
* @cs_was_changed_seq: The sequence number of clocksource change events
*/
struct system_time_snapshot {
- u64 cycles;
- ktime_t real;
- ktime_t raw;
- unsigned int clock_was_set_seq;
- u8 cs_was_changed_seq;
+ u64 cycles;
+ ktime_t real;
+ ktime_t raw;
+ enum clocksource_ids cs_id;
+ unsigned int clock_was_set_seq;
+ u8 cs_was_changed_seq;
};
/*
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index fff5f64981c6..5fe2d61172b1 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -921,6 +921,9 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
clocksource_arch_init(cs);
+ if (WARN_ON_ONCE((unsigned int)cs->id >= CSID_MAX))
+ cs->id = CSID_GENERIC;
+
/* Initialize mult/shift and max_idle_ns */
__clocksource_update_freq_scale(cs, scale, freq);
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index ca69290bee2a..a8b378338b9e 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -979,6 +979,7 @@ void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot)
do {
seq = read_seqcount_begin(&tk_core.seq);
now = tk_clock_read(&tk->tkr_mono);
+ systime_snapshot->cs_id = tk->tkr_mono.clock->id;
systime_snapshot->cs_was_changed_seq = tk->cs_was_changed_seq;
systime_snapshot->clock_was_set_seq = tk->clock_was_set_seq;
base_real = ktime_add(tk->tkr_mono.base,
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* [RFC PATCH 4/9] clocksource: Add clocksource id for arm arch counter
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
` (2 preceding siblings ...)
2020-02-10 8:49 ` [RFC PATCH 3/9] time: Add mechanism to recognize clocksource in time_get_snapshot Jianyong Wu
@ 2020-02-10 8:49 ` Jianyong Wu
2020-02-10 8:49 ` [RFC PATCH 5/9] psci: Add hypercall service for ptp_kvm Jianyong Wu
` (5 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:49 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
Add clocksource id for arm arch counter to let it be identified easily and
elegantly in ptp_kvm implementation for arm.
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
---
drivers/clocksource/arm_arch_timer.c | 2 ++
include/linux/clocksource_ids.h | 1 +
2 files changed, 3 insertions(+)
diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index 9a5464c625b4..4eb014fefa5b 100644
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -16,6 +16,7 @@
#include <linux/cpu_pm.h>
#include <linux/clockchips.h>
#include <linux/clocksource.h>
+#include <linux/clocksource_ids.h>
#include <linux/interrupt.h>
#include <linux/of_irq.h>
#include <linux/of_address.h>
@@ -187,6 +188,7 @@ static u64 arch_counter_read_cc(const struct cyclecounter *cc)
static struct clocksource clocksource_counter = {
.name = "arch_sys_counter",
+ .id = CSID_ARM_ARCH_COUNTER,
.rating = 400,
.read = arch_counter_read,
.mask = CLOCKSOURCE_MASK(56),
diff --git a/include/linux/clocksource_ids.h b/include/linux/clocksource_ids.h
index 5595ab843c3d..93bec8426c44 100644
--- a/include/linux/clocksource_ids.h
+++ b/include/linux/clocksource_ids.h
@@ -5,6 +5,7 @@
/* Enum to give clocksources a unique identifier */
enum clocksource_ids {
CSID_GENERIC = 0,
+ CSID_ARM_ARCH_COUNTER,
CSID_MAX,
};
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* [RFC PATCH 5/9] psci: Add hypercall service for ptp_kvm.
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
` (3 preceding siblings ...)
2020-02-10 8:49 ` [RFC PATCH 4/9] clocksource: Add clocksource id for arm arch counter Jianyong Wu
@ 2020-02-10 8:49 ` Jianyong Wu
2020-02-10 8:49 ` [RFC PATCH 6/9] ptp: arm/arm64: Enable ptp_kvm for arm/arm64 Jianyong Wu
` (4 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:49 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
ptp_kvm modules will get this service through smccc call.
The service offers real time and counter cycle of host for guest.
Also let caller determine which cycle of virtual counter or physical counter
to return.
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
---
include/linux/arm-smccc.h | 21 +++++++++++++++++++
virt/kvm/arm/hypercalls.c | 44 ++++++++++++++++++++++++++++++++++++++-
2 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h
index 59494df0f55b..747b7595d0c6 100644
--- a/include/linux/arm-smccc.h
+++ b/include/linux/arm-smccc.h
@@ -77,6 +77,27 @@
ARM_SMCCC_SMC_32, \
0, 0x7fff)
+/* PTP KVM call requests clock time from guest OS to host */
+#define ARM_SMCCC_HYP_KVM_PTP_FUNC_ID \
+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
+ ARM_SMCCC_SMC_32, \
+ ARM_SMCCC_OWNER_STANDARD_HYP, \
+ 0)
+
+/* request for virtual counter from ptp_kvm guest */
+#define ARM_SMCCC_HYP_KVM_PTP_VIRT \
+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
+ ARM_SMCCC_SMC_32, \
+ ARM_SMCCC_OWNER_STANDARD_HYP, \
+ 1)
+
+/* request for physical counter from ptp_kvm guest */
+#define ARM_SMCCC_HYP_KVM_PTP_PHY \
+ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
+ ARM_SMCCC_SMC_32, \
+ ARM_SMCCC_OWNER_STANDARD_HYP, \
+ 2)
+
#ifndef __ASSEMBLY__
#include <linux/linkage.h>
diff --git a/virt/kvm/arm/hypercalls.c b/virt/kvm/arm/hypercalls.c
index 550dfa3e53cd..a5309c28d4dc 100644
--- a/virt/kvm/arm/hypercalls.c
+++ b/virt/kvm/arm/hypercalls.c
@@ -3,6 +3,7 @@
#include <linux/arm-smccc.h>
#include <linux/kvm_host.h>
+#include <linux/clocksource_ids.h>
#include <asm/kvm_emulate.h>
@@ -11,8 +12,11 @@
int kvm_hvc_call_handler(struct kvm_vcpu *vcpu)
{
- u32 func_id = smccc_get_function(vcpu);
+ struct system_time_snapshot systime_snapshot;
+ long arg[4];
+ u64 cycles;
long val = SMCCC_RET_NOT_SUPPORTED;
+ u32 func_id = smccc_get_function(vcpu);
u32 feature;
gpa_t gpa;
@@ -62,6 +66,44 @@ int kvm_hvc_call_handler(struct kvm_vcpu *vcpu)
if (gpa != GPA_INVALID)
val = gpa;
break;
+ /*
+ * This serves virtual kvm_ptp.
+ * Four values will be passed back.
+ * reg0 stores high 32-bit host ktime;
+ * reg1 stores low 32-bit host ktime;
+ * reg2 stores high 32-bit difference of host cycles and cntvoff;
+ * reg3 stores low 32-bit difference of host cycles and cntvoff.
+ */
+ case ARM_SMCCC_HYP_KVM_PTP_FUNC_ID:
+ /*
+ * system time and counter value must captured in the same
+ * time to keep consistency and precision.
+ */
+ ktime_get_snapshot(&systime_snapshot);
+ if (systime_snapshot.cs_id != CSID_ARM_ARCH_COUNTER)
+ break;
+ arg[0] = upper_32_bits(systime_snapshot.real);
+ arg[1] = lower_32_bits(systime_snapshot.real);
+ /*
+ * which of virtual counter or physical counter being
+ * asked for is decided by the first argument.
+ */
+ feature = smccc_get_arg1(vcpu);
+ switch (feature) {
+ case ARM_SMCCC_HYP_KVM_PTP_PHY:
+ cycles = systime_snapshot.cycles;
+ break;
+ case ARM_SMCCC_HYP_KVM_PTP_VIRT:
+ default:
+ cycles = systime_snapshot.cycles -
+ vcpu_vtimer(vcpu)->cntvoff;
+ }
+ arg[2] = upper_32_bits(cycles);
+ arg[3] = lower_32_bits(cycles);
+
+ smccc_set_retval(vcpu, arg[0], arg[1], arg[2], arg[3]);
+ return 1;
+
default:
return kvm_psci_call(vcpu);
}
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* [RFC PATCH 6/9] ptp: arm/arm64: Enable ptp_kvm for arm/arm64
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
` (4 preceding siblings ...)
2020-02-10 8:49 ` [RFC PATCH 5/9] psci: Add hypercall service for ptp_kvm Jianyong Wu
@ 2020-02-10 8:49 ` Jianyong Wu
2020-02-10 8:49 ` [RFC PATCH 7/9] ptp: extend input argument for getcrosstimestamp API Jianyong Wu
` (3 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:49 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
Currently, there is no mechanism to keep time sync between guest and host
in arm/arm64 virtualization environment. Time in guest will drift compared
with host after boot up as they may both use third party time sources
to correct their time respectively. The time deviation will be in order
of milliseconds. But in some scenarios,like in cloud envirenment, we ask
for higher time precision.
kvm ptp clock, which choose the host clock source clock as a reference
clock to sync time clock between guest and host has been adopted by x86
which makes the time sync order from milliseconds to nanoseconds.
This patch enables kvm ptp on arm/arm64 and improve clock sync precison
significantly.
Test result comparison between with kvm ptp and without it in arm/arm64 is
as follows. This test derived from the result of command 'chronyc
sources'. we should take more care of the last sample column which shows
the offset between the local clock and the source at the last measurement.
no kvm ptp in guest:
MS Name/IP address Stratum Poll Reach LastRx Last sample
========================================================================
^* dns1.synet.edu.cn 2 6 377 13 +1040us[+1581us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 21 +1040us[+1581us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 29 +1040us[+1581us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 37 +1040us[+1581us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 45 +1040us[+1581us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 53 +1040us[+1581us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 61 +1040us[+1581us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 4 -130us[ +796us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 12 -130us[ +796us] +/- 21ms
^* dns1.synet.edu.cn 2 6 377 20 -130us[ +796us] +/- 21ms
in host:
MS Name/IP address Stratum Poll Reach LastRx Last sample
========================================================================
^* 120.25.115.20 2 7 377 72 -470us[ -603us] +/- 18ms
^* 120.25.115.20 2 7 377 92 -470us[ -603us] +/- 18ms
^* 120.25.115.20 2 7 377 112 -470us[ -603us] +/- 18ms
^* 120.25.115.20 2 7 377 2 +872ns[-6808ns] +/- 17ms
^* 120.25.115.20 2 7 377 22 +872ns[-6808ns] +/- 17ms
^* 120.25.115.20 2 7 377 43 +872ns[-6808ns] +/- 17ms
^* 120.25.115.20 2 7 377 63 +872ns[-6808ns] +/- 17ms
^* 120.25.115.20 2 7 377 83 +872ns[-6808ns] +/- 17ms
^* 120.25.115.20 2 7 377 103 +872ns[-6808ns] +/- 17ms
^* 120.25.115.20 2 7 377 123 +872ns[-6808ns] +/- 17ms
The dns1.synet.edu.cn is the network reference clock for guest and
120.25.115.20 is the network reference clock for host. we can't get the
clock error between guest and host directly, but a roughly estimated value
will be in order of hundreds of us to ms.
with kvm ptp in guest:
chrony has been disabled in host to remove the disturb by network clock.
MS Name/IP address Stratum Poll Reach LastRx Last sample
========================================================================
* PHC0 0 3 377 8 -7ns[ +1ns] +/- 3ns
* PHC0 0 3 377 8 +1ns[ +16ns] +/- 3ns
* PHC0 0 3 377 6 -4ns[ -0ns] +/- 6ns
* PHC0 0 3 377 6 -8ns[ -12ns] +/- 5ns
* PHC0 0 3 377 5 +2ns[ +4ns] +/- 4ns
* PHC0 0 3 377 13 +2ns[ +4ns] +/- 4ns
* PHC0 0 3 377 12 -4ns[ -6ns] +/- 4ns
* PHC0 0 3 377 11 -8ns[ -11ns] +/- 6ns
* PHC0 0 3 377 10 -14ns[ -20ns] +/- 4ns
* PHC0 0 3 377 8 +4ns[ +5ns] +/- 4ns
The PHC0 is the ptp clock which choose the host clock as its source
clock. So we can be sure to say that the clock error between host and guest
is in order of ns.
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
---
drivers/clocksource/arm_arch_timer.c | 22 ++++++++++++
drivers/ptp/Kconfig | 2 +-
drivers/ptp/Makefile | 6 +++-
drivers/ptp/ptp_kvm_arm.c | 53 ++++++++++++++++++++++++++++
4 files changed, 81 insertions(+), 2 deletions(-)
create mode 100644 drivers/ptp/ptp_kvm_arm.c
diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index 4eb014fefa5b..4e57ab66165c 100644
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -1625,3 +1625,25 @@ static int __init arch_timer_acpi_init(struct acpi_table_header *table)
}
TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
#endif
+
+#if IS_ENABLED(CONFIG_PTP_1588_CLOCK_KVM)
+#include <linux/arm-smccc.h>
+int kvm_arch_ptp_get_crosststamp(unsigned long *cycle, struct timespec64 *ts,
+ struct clocksource **cs)
+{
+ struct arm_smccc_res hvc_res;
+ ktime_t ktime_overall;
+
+ arm_smccc_1_1_invoke(ARM_SMCCC_HYP_KVM_PTP_FUNC_ID, &hvc_res);
+ if ((int)(hvc_res.a0) < 0)
+ return -EOPNOTSUPP;
+
+ ktime_overall = (long long)hvc_res.a0 << 32 | hvc_res.a1;
+ *ts = ktime_to_timespec64(ktime_overall);
+ *cycle = (long long)hvc_res.a2 << 32 | hvc_res.a3;
+ *cs = &clocksource_counter;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(kvm_arch_ptp_get_crosststamp);
+#endif
diff --git a/drivers/ptp/Kconfig b/drivers/ptp/Kconfig
index 475c60dccaa4..dfb536d14b44 100644
--- a/drivers/ptp/Kconfig
+++ b/drivers/ptp/Kconfig
@@ -106,7 +106,7 @@ config PTP_1588_CLOCK_PCH
config PTP_1588_CLOCK_KVM
tristate "KVM virtual PTP clock"
depends on PTP_1588_CLOCK
- depends on KVM_GUEST && X86
+ depends on KVM_GUEST && X86 || (ARM || ARM64) && ARM_ARCH_TIMER && ARM_PSCI_FW
default y
help
This driver adds support for using kvm infrastructure as a PTP
diff --git a/drivers/ptp/Makefile b/drivers/ptp/Makefile
index 06837d089347..f6688de74a8e 100644
--- a/drivers/ptp/Makefile
+++ b/drivers/ptp/Makefile
@@ -3,8 +3,12 @@
# Makefile for PTP 1588 clock support.
#
+ifeq ($(ARCH), arm64)
+ ptp_kvm-y := ptp_kvm_arm.o ptp_kvm_common.o
+else
+ ptp_kvm-y := ptp_kvm_$(ARCH).o ptp_kvm_common.o
+endif
ptp-y := ptp_clock.o ptp_chardev.o ptp_sysfs.o
-ptp_kvm-y := ptp_kvm_$(ARCH).o ptp_kvm_common.o
obj-$(CONFIG_PTP_1588_CLOCK) += ptp.o
obj-$(CONFIG_PTP_1588_CLOCK_DTE) += ptp_dte.o
obj-$(CONFIG_PTP_1588_CLOCK_INES) += ptp_ines.o
diff --git a/drivers/ptp/ptp_kvm_arm.c b/drivers/ptp/ptp_kvm_arm.c
new file mode 100644
index 000000000000..9752e849e0ae
--- /dev/null
+++ b/drivers/ptp/ptp_kvm_arm.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Virtual PTP 1588 clock for use with KVM guests
+ * Copyright (C) 2019 ARM Ltd.
+ * All Rights Reserved
+ */
+
+#include <linux/kernel.h>
+#include <linux/err.h>
+#include <asm/hypervisor.h>
+#include <linux/module.h>
+#include <linux/psci.h>
+#include <linux/arm-smccc.h>
+#include <linux/timecounter.h>
+#include <linux/sched/clock.h>
+#include <asm/arch_timer.h>
+
+int kvm_arch_ptp_init(void)
+{
+ struct arm_smccc_res hvc_res;
+
+ arm_smccc_1_1_invoke(ARM_SMCCC_HYP_KVM_PTP_FUNC_ID,
+ &hvc_res);
+ if ((int)(hvc_res.a0) < 0)
+ return -EOPNOTSUPP;
+
+ return 0;
+}
+
+int kvm_arch_ptp_get_clock_generic(struct timespec64 *ts,
+ struct arm_smccc_res *hvc_res)
+{
+ ktime_t ktime_overall;
+
+ arm_smccc_1_1_invoke(ARM_SMCCC_HYP_KVM_PTP_FUNC_ID,
+ hvc_res);
+ if ((int)(hvc_res->a0) < 0)
+ return -EOPNOTSUPP;
+
+ ktime_overall = (long long)hvc_res->a0 << 32 | hvc_res->a1;
+ *ts = ktime_to_timespec64(ktime_overall);
+
+ return 0;
+}
+
+int kvm_arch_ptp_get_clock(struct timespec64 *ts)
+{
+ struct arm_smccc_res hvc_res;
+
+ kvm_arch_ptp_get_clock_generic(ts, &hvc_res);
+
+ return 0;
+}
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* [RFC PATCH 7/9] ptp: extend input argument for getcrosstimestamp API
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
` (5 preceding siblings ...)
2020-02-10 8:49 ` [RFC PATCH 6/9] ptp: arm/arm64: Enable ptp_kvm for arm/arm64 Jianyong Wu
@ 2020-02-10 8:49 ` Jianyong Wu
2020-02-10 8:49 ` [RFC PATCH 8/9] arm/arm64: add mechanism to let user choose which counter to return Jianyong Wu
` (2 subsequent siblings)
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:49 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
sometimes we may need tell getcrosstimestamp call back how to perform
itself. Extending input arguments for getcrosstimestamp API to offer more
exquisite control for the operation.
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
---
drivers/clocksource/arm_arch_timer.c | 2 +-
drivers/net/ethernet/intel/e1000e/ptp.c | 3 ++-
drivers/ptp/ptp_chardev.c | 2 +-
drivers/ptp/ptp_kvm.h | 2 +-
drivers/ptp/ptp_kvm_x86.c | 2 +-
include/linux/ptp_clock_kernel.h | 3 ++-
6 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index 4e57ab66165c..b1bc8bd198c2 100644
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -1629,7 +1629,7 @@ TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
#if IS_ENABLED(CONFIG_PTP_1588_CLOCK_KVM)
#include <linux/arm-smccc.h>
int kvm_arch_ptp_get_crosststamp(unsigned long *cycle, struct timespec64 *ts,
- struct clocksource **cs)
+ struct clocksource **cs, long *ctx)
{
struct arm_smccc_res hvc_res;
ktime_t ktime_overall;
diff --git a/drivers/net/ethernet/intel/e1000e/ptp.c b/drivers/net/ethernet/intel/e1000e/ptp.c
index eaa5a0fb99f0..7a45d9634962 100644
--- a/drivers/net/ethernet/intel/e1000e/ptp.c
+++ b/drivers/net/ethernet/intel/e1000e/ptp.c
@@ -150,7 +150,8 @@ static int e1000e_phc_get_syncdevicetime(ktime_t *device,
* clock values in ns.
**/
static int e1000e_phc_getcrosststamp(struct ptp_clock_info *ptp,
- struct system_device_crosststamp *xtstamp)
+ struct system_device_crosststamp *xtstamp,
+ NULL)
{
struct e1000_adapter *adapter = container_of(ptp, struct e1000_adapter,
ptp_clock_info);
diff --git a/drivers/ptp/ptp_chardev.c b/drivers/ptp/ptp_chardev.c
index 9d72ab593f13..09c3f1e5382d 100644
--- a/drivers/ptp/ptp_chardev.c
+++ b/drivers/ptp/ptp_chardev.c
@@ -226,7 +226,7 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
err = -EOPNOTSUPP;
break;
}
- err = ptp->info->getcrosststamp(ptp->info, &xtstamp);
+ err = ptp->info->getcrosststamp(ptp->info, &xtstamp, NULL);
if (err)
break;
diff --git a/drivers/ptp/ptp_kvm.h b/drivers/ptp/ptp_kvm.h
index 4bf1802bbeb8..ccceacbe8398 100644
--- a/drivers/ptp/ptp_kvm.h
+++ b/drivers/ptp/ptp_kvm.h
@@ -8,4 +8,4 @@
int kvm_arch_ptp_init(void);
int kvm_arch_ptp_get_clock(struct timespec64 *ts);
int kvm_arch_ptp_get_crosststamp(unsigned long *cycle,
- struct timespec64 *tspec, void *cs);
+ struct timespec64 *tspec, struct clocksource **cs, long *ctx);
diff --git a/drivers/ptp/ptp_kvm_x86.c b/drivers/ptp/ptp_kvm_x86.c
index 6c891d7299c6..ff95d2791078 100644
--- a/drivers/ptp/ptp_kvm_x86.c
+++ b/drivers/ptp/ptp_kvm_x86.c
@@ -53,7 +53,7 @@ int kvm_arch_ptp_get_clock(struct timespec64 *ts)
}
int kvm_arch_ptp_get_crosststamp(unsigned long *cycle, struct timespec64 *tspec,
- struct clocksource **cs)
+ struct clocksource **cs, void *ctx)
{
unsigned long ret;
unsigned int version;
diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h
index c64a1ef87240..29c5fff7aeda 100644
--- a/include/linux/ptp_clock_kernel.h
+++ b/include/linux/ptp_clock_kernel.h
@@ -133,7 +133,8 @@ struct ptp_clock_info {
int (*gettimex64)(struct ptp_clock_info *ptp, struct timespec64 *ts,
struct ptp_system_timestamp *sts);
int (*getcrosststamp)(struct ptp_clock_info *ptp,
- struct system_device_crosststamp *cts);
+ struct system_device_crosststamp *cts,
+ long *flag);
int (*settime64)(struct ptp_clock_info *p, const struct timespec64 *ts);
int (*enable)(struct ptp_clock_info *ptp,
struct ptp_clock_request *request, int on);
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* [RFC PATCH 8/9] arm/arm64: add mechanism to let user choose which counter to return
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
` (6 preceding siblings ...)
2020-02-10 8:49 ` [RFC PATCH 7/9] ptp: extend input argument for getcrosstimestamp API Jianyong Wu
@ 2020-02-10 8:49 ` Jianyong Wu
2020-02-10 8:49 ` [RFC PATCH 9/9] arm/arm64: Add kvm capability check extension for ptp_kvm Jianyong Wu
2020-02-17 2:28 ` [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:49 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
In general, vm inside will use virtual counter compered with host use
phyical counter. But in some special scenarios, like nested virtualization,
phyical counter maybe used by vm. A interface added in ptp_kvm driver to
offer a mechanism to let user choose which counter should be return from
host.
To use this feature, you should call PTP_EXTTS_REQUEST(2) ioctl with flag
set bit PTP_KVM_ARM_PHY_COUNTER in its argument then call
PTP_SYS_OFFSET_PRECISE(2) ioctl to get the cross timestamp and phyical
counter will return. If the bit not set or no call for PTP_EXTTS_REQUEST2,
virtual counter will return by default.
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
Suggested-by: Marc Zyngier <maz@kernel.org>
---
drivers/clocksource/arm_arch_timer.c | 11 ++++++++++-
drivers/ptp/ptp_chardev.c | 8 +++++++-
drivers/ptp/ptp_kvm_common.c | 7 ++++---
include/uapi/linux/ptp_clock.h | 4 +++-
4 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c
index b1bc8bd198c2..3e43d4e4885e 100644
--- a/drivers/clocksource/arm_arch_timer.c
+++ b/drivers/clocksource/arm_arch_timer.c
@@ -1634,7 +1634,16 @@ int kvm_arch_ptp_get_crosststamp(unsigned long *cycle, struct timespec64 *ts,
struct arm_smccc_res hvc_res;
ktime_t ktime_overall;
- arm_smccc_1_1_invoke(ARM_SMCCC_HYP_KVM_PTP_FUNC_ID, &hvc_res);
+ /*
+ * an argument will be passed by a0 to determine weather virtual
+ * counter or phyical counter should be passed back.
+ */
+ if (*ctx)
+ arm_smccc_1_1_invoke(ARM_SMCCC_HYP_KVM_PTP_FUNC_ID,
+ ARM_SMCCC_HYP_KVM_PTP_PHY, &hvc_res);
+ else
+ arm_smccc_1_1_invoke(ARM_SMCCC_HYP_KVM_PTP_FUNC_ID,
+ ARM_SMCCC_HYP_KVM_PTP_VIRT, &hvc_res);
if ((int)(hvc_res.a0) < 0)
return -EOPNOTSUPP;
diff --git a/drivers/ptp/ptp_chardev.c b/drivers/ptp/ptp_chardev.c
index 09c3f1e5382d..bd7b6a31b6cb 100644
--- a/drivers/ptp/ptp_chardev.c
+++ b/drivers/ptp/ptp_chardev.c
@@ -122,6 +122,7 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
struct ptp_pin_desc pd;
struct timespec64 ts;
int enable, err = 0;
+ static long flag;
switch (cmd) {
@@ -149,6 +150,11 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
err = -EFAULT;
break;
}
+ /*
+ * Tell driver to call for physical counter.
+ * This is only for arm.
+ */
+ flag = req.extts.flags & PTP_KVM_ARM_PHY_COUNTER;
if (cmd == PTP_EXTTS_REQUEST2) {
/* Tell the drivers to check the flags carefully. */
req.extts.flags |= PTP_STRICT_FLAGS;
@@ -226,7 +232,7 @@ long ptp_ioctl(struct posix_clock *pc, unsigned int cmd, unsigned long arg)
err = -EOPNOTSUPP;
break;
}
- err = ptp->info->getcrosststamp(ptp->info, &xtstamp, NULL);
+ err = ptp->info->getcrosststamp(ptp->info, &xtstamp, &flag);
if (err)
break;
diff --git a/drivers/ptp/ptp_kvm_common.c b/drivers/ptp/ptp_kvm_common.c
index 60442f70d3fc..20fbecef1f22 100644
--- a/drivers/ptp/ptp_kvm_common.c
+++ b/drivers/ptp/ptp_kvm_common.c
@@ -36,7 +36,7 @@ static int ptp_kvm_get_time_fn(ktime_t *device_time,
spin_lock(&kvm_ptp_lock);
preempt_disable_notrace();
- ret = kvm_arch_ptp_get_crosststamp(&cycle, &tspec, &cs);
+ ret = kvm_arch_ptp_get_crosststamp(&cycle, &tspec, &cs, ctx);
if (ret != 0) {
pr_err_ratelimited("clock pairing hypercall ret %lu\n", ret);
spin_unlock(&kvm_ptp_lock);
@@ -57,9 +57,10 @@ static int ptp_kvm_get_time_fn(ktime_t *device_time,
}
static int ptp_kvm_getcrosststamp(struct ptp_clock_info *ptp,
- struct system_device_crosststamp *xtstamp)
+ struct system_device_crosststamp *xtstamp,
+ long *flag)
{
- return get_device_system_crosststamp(ptp_kvm_get_time_fn, NULL,
+ return get_device_system_crosststamp(ptp_kvm_get_time_fn, flag,
NULL, xtstamp);
}
diff --git a/include/uapi/linux/ptp_clock.h b/include/uapi/linux/ptp_clock.h
index 9dc9d0079e98..71e388a82244 100644
--- a/include/uapi/linux/ptp_clock.h
+++ b/include/uapi/linux/ptp_clock.h
@@ -32,6 +32,7 @@
#define PTP_RISING_EDGE (1<<1)
#define PTP_FALLING_EDGE (1<<2)
#define PTP_STRICT_FLAGS (1<<3)
+#define PTP_KVM_ARM_PHY_COUNTER (1<<4)
#define PTP_EXTTS_EDGES (PTP_RISING_EDGE | PTP_FALLING_EDGE)
/*
@@ -40,7 +41,8 @@
#define PTP_EXTTS_VALID_FLAGS (PTP_ENABLE_FEATURE | \
PTP_RISING_EDGE | \
PTP_FALLING_EDGE | \
- PTP_STRICT_FLAGS)
+ PTP_STRICT_FLAGS | \
+ PTP_KVM_ARM_PHY_COUNTER)
/*
* flag fields valid for the original PTP_EXTTS_REQUEST ioctl.
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* [RFC PATCH 9/9] arm/arm64: Add kvm capability check extension for ptp_kvm
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
` (7 preceding siblings ...)
2020-02-10 8:49 ` [RFC PATCH 8/9] arm/arm64: add mechanism to let user choose which counter to return Jianyong Wu
@ 2020-02-10 8:49 ` Jianyong Wu
2020-02-17 2:28 ` [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
9 siblings, 0 replies; 13+ messages in thread
From: Jianyong Wu @ 2020-02-10 8:49 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark.Rutland, will,
suzuki.poulose, steven.price
Cc: justin.he, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
Let userspace check if there is kvm ptp service in host.
Before VMs migrate to another host, VMM may check if this
cap is available to determine the next behavior.
Signed-off-by: Jianyong Wu <jianyong.wu@arm.com>
Suggested-by: Marc Zyngier <maz@kernel.org>
---
include/uapi/linux/kvm.h | 1 +
virt/kvm/arm/arm.c | 1 +
2 files changed, 2 insertions(+)
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index f0a16b4adbbd..ca29f6a09c30 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -1009,6 +1009,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_PPC_GUEST_DEBUG_SSTEP 176
#define KVM_CAP_ARM_NISV_TO_USER 177
#define KVM_CAP_ARM_INJECT_EXT_DABT 178
+#define KVM_CAP_ARM_KVM_PTP 179
#ifdef KVM_CAP_IRQ_ROUTING
diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 8de4daf25097..9cad433507df 100644
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -223,6 +223,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2:
case KVM_CAP_ARM_NISV_TO_USER:
case KVM_CAP_ARM_INJECT_EXT_DABT:
+ case KVM_CAP_ARM_KVM_PTP:
r = 1;
break;
case KVM_CAP_ARM_SET_DEVICE_ADDR:
--
2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply related [flat|nested] 13+ messages in thread* RE: [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64
2020-02-10 8:48 [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
` (8 preceding siblings ...)
2020-02-10 8:49 ` [RFC PATCH 9/9] arm/arm64: Add kvm capability check extension for ptp_kvm Jianyong Wu
@ 2020-02-17 2:28 ` Jianyong Wu
2020-03-27 9:35 ` Jianyong Wu
9 siblings, 1 reply; 13+ messages in thread
From: Jianyong Wu @ 2020-02-17 2:28 UTC (permalink / raw)
To: Jianyong Wu, netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark Rutland, will,
Suzuki Poulose, Steven Price
Cc: Justin He, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
Hi,
As this version change a lot, expect more comments. thanks.
Thanks
Jianyong
> -----Original Message-----
> From: Jianyong Wu <jianyong.wu@arm.com>
> Sent: Monday, February 10, 2020 4:49 PM
> To: netdev@vger.kernel.org; yangbo.lu@nxp.com; john.stultz@linaro.org;
> tglx@linutronix.de; pbonzini@redhat.com; sean.j.christopherson@intel.com;
> maz@kernel.org; richardcochran@gmail.com; Mark Rutland
> <Mark.Rutland@arm.com>; will@kernel.org; Suzuki Poulose
> <Suzuki.Poulose@arm.com>; Steven Price <Steven.Price@arm.com>
> Cc: linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
> kvmarm@lists.cs.columbia.edu; kvm@vger.kernel.org; Steve Capper
> <Steve.Capper@arm.com>; Kaly Xin <Kaly.Xin@arm.com>; Justin He
> <Justin.He@arm.com>; Jianyong Wu <Jianyong.Wu@arm.com>; nd
> <nd@arm.com>
> Subject: [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64
>
> kvm ptp targets to provide high precision time sync between guest and host
> in virtualization environment. Here, we enable kvm ptp for arm64.
>
> change log:
> from v10 to v9
> (1) change code base to v5.5.
> (2) enable ptp_kvm both for arm32 and arm64.
> (3) let user choose which of virtual counter or physical counter should
> return when using crosstimestamp mode of ptp_kvm for arm/arm64.
> (4) extend input argument for getcrosstimestamp API.
>
> from v8 to v9:
> (1) move ptp_kvm.h to driver/ptp/
> (2) replace license declaration of ptp_kvm.h the same with other header
> files in the same directory.
>
> from v7 to v8:
> (1) separate adding clocksource id for arm_arch_counter as a single
> patch.
> (2) update commit message for patch 4/8.
> (3) refine patch 7/8 and patch 8/8 to make them more independent.
>
> from v6 to v7:
> (1) include the omitted clocksource_id.h in last version.
> (2) reorder the header file in patch.
> (3) refine some words in commit message to make it more impersonal.
>
> from v5 to v6:
> (1) apply Mark's patch[4] to get SMCCC conduit.
> (2) add mechanism to recognize current clocksource by add
> clocksouce_id value into struct clocksource instead of method in patch-v5.
> (3) rename kvm_arch_ptp_get_clock_fn into
> kvm_arch_ptp_get_crosststamp.
>
> from v4 to v5:
> (1) remove hvc delay compensasion as it should leave to userspace.
> (2) check current clocksource in hvc call service.
> (3) expose current clocksource by adding it to system_time_snapshot.
> (4) add helper to check if clocksource is arm_arch_counter.
> (5) rename kvm_ptp.c to ptp_kvm_common.c
>
> from v3 to v4:
> (1) fix clocksource of ptp_kvm to arch_sys_counter.
> (2) move kvm_arch_ptp_get_clock_fn into arm_arch_timer.c
> (3) subtract cntvoff before return cycles from host.
> (4) use ktime_get_snapshot instead of getnstimeofday and
> get_current_counterval to return time and counter value.
> (5) split ktime and counter into two 32-bit block respectively to avoid
> Y2038-safe issue.
> (6) set time compensation to device time as half of the delay of hvc call.
> (7) add ARM_ARCH_TIMER as dependency of ptp_kvm for arm64.
>
> from v2 to v3:
> (1) fix some issues in commit log.
> (2) add some receivers in send list.
>
> from v1 to v2:
> (1) move arch-specific code from arch/ to driver/ptp/
> (2) offer mechanism to inform userspace if ptp_kvm service is available.
> (3) separate ptp_kvm code for arm64 into hypervisor part and guest part.
> (4) add API to expose monotonic clock and counter value.
> (5) refine code: remove no necessary part and reconsitution.
>
>
> Jianyong Wu (8):
> psci: export psci conduit get helper.
> ptp: Reorganize ptp_kvm modules to make it arch-independent.
> clocksource: Add clocksource id for arm arch counter
> psci: Add hypercall service for ptp_kvm.
> ptp: arm/arm64: Enable ptp_kvm for arm/arm64
> ptp: extend input argument for getcrosstimestamp API
> arm/arm64: add mechanism to let user choose which counter to return
> arm/arm64: Add kvm capability check extension for ptp_kvm
>
> Thomas Gleixner (1):
> time: Add mechanism to recognize clocksource in time_get_snapshot
>
> drivers/clocksource/arm_arch_timer.c | 33 ++++++++
> drivers/firmware/psci/psci.c | 1 +
> drivers/net/ethernet/intel/e1000e/ptp.c | 3 +-
> drivers/ptp/Kconfig | 2 +-
> drivers/ptp/Makefile | 5 ++
> drivers/ptp/ptp_chardev.c | 8 +-
> drivers/ptp/ptp_kvm.h | 11 +++
> drivers/ptp/ptp_kvm_arm.c | 53 +++++++++++++
> drivers/ptp/{ptp_kvm.c => ptp_kvm_common.c} | 83 ++++++--------------
> drivers/ptp/ptp_kvm_x86.c | 87 +++++++++++++++++++++
> include/linux/arm-smccc.h | 21 +++++
> include/linux/clocksource.h | 6 ++
> include/linux/clocksource_ids.h | 13 +++
> include/linux/ptp_clock_kernel.h | 3 +-
> include/linux/timekeeping.h | 12 +--
> include/uapi/linux/kvm.h | 1 +
> include/uapi/linux/ptp_clock.h | 4 +-
> kernel/time/clocksource.c | 3 +
> kernel/time/timekeeping.c | 1 +
> virt/kvm/arm/arm.c | 1 +
> virt/kvm/arm/hypercalls.c | 44 ++++++++++-
> 21 files changed, 323 insertions(+), 72 deletions(-) create mode 100644
> drivers/ptp/ptp_kvm.h create mode 100644 drivers/ptp/ptp_kvm_arm.c
> rename drivers/ptp/{ptp_kvm.c => ptp_kvm_common.c} (60%) create mode
> 100644 drivers/ptp/ptp_kvm_x86.c create mode 100644
> include/linux/clocksource_ids.h
>
> --
> 2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply [flat|nested] 13+ messages in thread* RE: [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64
2020-02-17 2:28 ` [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64 Jianyong Wu
@ 2020-03-27 9:35 ` Jianyong Wu
2020-03-27 9:36 ` Marc Zyngier
0 siblings, 1 reply; 13+ messages in thread
From: Jianyong Wu @ 2020-03-27 9:35 UTC (permalink / raw)
To: netdev, yangbo.lu, john.stultz, tglx, pbonzini,
sean.j.christopherson, maz, richardcochran, Mark Rutland, will,
Suzuki Poulose, Steven Price
Cc: Justin He, kvm, linux-kernel, nd, kvmarm, linux-arm-kernel
Hi ,
Ping ...
Is there any comments.
Need some comments to move forward.
Thanks
Jianyong
> -----Original Message-----
> From: Jianyong Wu <Jianyong.Wu@arm.com>
> Sent: Monday, February 17, 2020 10:29 AM
> To: Jianyong Wu <Jianyong.Wu@arm.com>; netdev@vger.kernel.org;
> yangbo.lu@nxp.com; john.stultz@linaro.org; tglx@linutronix.de;
> pbonzini@redhat.com; sean.j.christopherson@intel.com; maz@kernel.org;
> richardcochran@gmail.com; Mark Rutland <Mark.Rutland@arm.com>;
> will@kernel.org; Suzuki Poulose <Suzuki.Poulose@arm.com>; Steven Price
> <Steven.Price@arm.com>
> Cc: linux-kernel@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
> kvmarm@lists.cs.columbia.edu; kvm@vger.kernel.org; Steve Capper
> <Steve.Capper@arm.com>; Kaly Xin <Kaly.Xin@arm.com>; Justin He
> <Justin.He@arm.com>; nd <nd@arm.com>
> Subject: RE: [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64
>
> Hi,
> As this version change a lot, expect more comments. thanks.
>
> Thanks
> Jianyong
> > -----Original Message-----
> > From: Jianyong Wu <jianyong.wu@arm.com>
> > Sent: Monday, February 10, 2020 4:49 PM
> > To: netdev@vger.kernel.org; yangbo.lu@nxp.com; john.stultz@linaro.org;
> > tglx@linutronix.de; pbonzini@redhat.com;
> > sean.j.christopherson@intel.com; maz@kernel.org;
> > richardcochran@gmail.com; Mark Rutland <Mark.Rutland@arm.com>;
> > will@kernel.org; Suzuki Poulose <Suzuki.Poulose@arm.com>; Steven Price
> > <Steven.Price@arm.com>
> > Cc: linux-kernel@vger.kernel.org;
> > linux-arm-kernel@lists.infradead.org;
> > kvmarm@lists.cs.columbia.edu; kvm@vger.kernel.org; Steve Capper
> > <Steve.Capper@arm.com>; Kaly Xin <Kaly.Xin@arm.com>; Justin He
> > <Justin.He@arm.com>; Jianyong Wu <Jianyong.Wu@arm.com>; nd
> > <nd@arm.com>
> > Subject: [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64
> >
> > kvm ptp targets to provide high precision time sync between guest and
> > host in virtualization environment. Here, we enable kvm ptp for arm64.
> >
> > change log:
> > from v10 to v9
> > (1) change code base to v5.5.
> > (2) enable ptp_kvm both for arm32 and arm64.
> > (3) let user choose which of virtual counter or physical
> > counter should return when using crosstimestamp mode of ptp_kvm for
> arm/arm64.
> > (4) extend input argument for getcrosstimestamp API.
> >
> > from v8 to v9:
> > (1) move ptp_kvm.h to driver/ptp/
> > (2) replace license declaration of ptp_kvm.h the same with
> > other header files in the same directory.
> >
> > from v7 to v8:
> > (1) separate adding clocksource id for arm_arch_counter as a
> > single patch.
> > (2) update commit message for patch 4/8.
> > (3) refine patch 7/8 and patch 8/8 to make them more independent.
> >
> > from v6 to v7:
> > (1) include the omitted clocksource_id.h in last version.
> > (2) reorder the header file in patch.
> > (3) refine some words in commit message to make it more impersonal.
> >
> > from v5 to v6:
> > (1) apply Mark's patch[4] to get SMCCC conduit.
> > (2) add mechanism to recognize current clocksource by add
> > clocksouce_id value into struct clocksource instead of method in patch-v5.
> > (3) rename kvm_arch_ptp_get_clock_fn into
> > kvm_arch_ptp_get_crosststamp.
> >
> > from v4 to v5:
> > (1) remove hvc delay compensasion as it should leave to userspace.
> > (2) check current clocksource in hvc call service.
> > (3) expose current clocksource by adding it to system_time_snapshot.
> > (4) add helper to check if clocksource is arm_arch_counter.
> > (5) rename kvm_ptp.c to ptp_kvm_common.c
> >
> > from v3 to v4:
> > (1) fix clocksource of ptp_kvm to arch_sys_counter.
> > (2) move kvm_arch_ptp_get_clock_fn into arm_arch_timer.c
> > (3) subtract cntvoff before return cycles from host.
> > (4) use ktime_get_snapshot instead of getnstimeofday and
> > get_current_counterval to return time and counter value.
> > (5) split ktime and counter into two 32-bit block respectively
> > to avoid Y2038-safe issue.
> > (6) set time compensation to device time as half of the delay of hvc call.
> > (7) add ARM_ARCH_TIMER as dependency of ptp_kvm for arm64.
> >
> > from v2 to v3:
> > (1) fix some issues in commit log.
> > (2) add some receivers in send list.
> >
> > from v1 to v2:
> > (1) move arch-specific code from arch/ to driver/ptp/
> > (2) offer mechanism to inform userspace if ptp_kvm service is available.
> > (3) separate ptp_kvm code for arm64 into hypervisor part and guest
> part.
> > (4) add API to expose monotonic clock and counter value.
> > (5) refine code: remove no necessary part and reconsitution.
> >
> >
> > Jianyong Wu (8):
> > psci: export psci conduit get helper.
> > ptp: Reorganize ptp_kvm modules to make it arch-independent.
> > clocksource: Add clocksource id for arm arch counter
> > psci: Add hypercall service for ptp_kvm.
> > ptp: arm/arm64: Enable ptp_kvm for arm/arm64
> > ptp: extend input argument for getcrosstimestamp API
> > arm/arm64: add mechanism to let user choose which counter to return
> > arm/arm64: Add kvm capability check extension for ptp_kvm
> >
> > Thomas Gleixner (1):
> > time: Add mechanism to recognize clocksource in time_get_snapshot
> >
> > drivers/clocksource/arm_arch_timer.c | 33 ++++++++
> > drivers/firmware/psci/psci.c | 1 +
> > drivers/net/ethernet/intel/e1000e/ptp.c | 3 +-
> > drivers/ptp/Kconfig | 2 +-
> > drivers/ptp/Makefile | 5 ++
> > drivers/ptp/ptp_chardev.c | 8 +-
> > drivers/ptp/ptp_kvm.h | 11 +++
> > drivers/ptp/ptp_kvm_arm.c | 53 +++++++++++++
> > drivers/ptp/{ptp_kvm.c => ptp_kvm_common.c} | 83 ++++++--------------
> > drivers/ptp/ptp_kvm_x86.c | 87 +++++++++++++++++++++
> > include/linux/arm-smccc.h | 21 +++++
> > include/linux/clocksource.h | 6 ++
> > include/linux/clocksource_ids.h | 13 +++
> > include/linux/ptp_clock_kernel.h | 3 +-
> > include/linux/timekeeping.h | 12 +--
> > include/uapi/linux/kvm.h | 1 +
> > include/uapi/linux/ptp_clock.h | 4 +-
> > kernel/time/clocksource.c | 3 +
> > kernel/time/timekeeping.c | 1 +
> > virt/kvm/arm/arm.c | 1 +
> > virt/kvm/arm/hypercalls.c | 44 ++++++++++-
> > 21 files changed, 323 insertions(+), 72 deletions(-) create mode
> > 100644 drivers/ptp/ptp_kvm.h create mode 100644
> > drivers/ptp/ptp_kvm_arm.c rename drivers/ptp/{ptp_kvm.c =>
> > ptp_kvm_common.c} (60%) create mode
> > 100644 drivers/ptp/ptp_kvm_x86.c create mode 100644
> > include/linux/clocksource_ids.h
> >
> > --
> > 2.17.1
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply [flat|nested] 13+ messages in thread* Re: [RFC PATCH v10 0/9] Enable ptp_kvm for arm/arm64
2020-03-27 9:35 ` Jianyong Wu
@ 2020-03-27 9:36 ` Marc Zyngier
0 siblings, 0 replies; 13+ messages in thread
From: Marc Zyngier @ 2020-03-27 9:36 UTC (permalink / raw)
To: Jianyong Wu
Cc: Justin He, kvm, netdev, richardcochran, linux-kernel,
sean.j.christopherson, Steven Price, john.stultz, yangbo.lu,
pbonzini, tglx, nd, will, kvmarm, linux-arm-kernel
On 2020-03-27 09:35, Jianyong Wu wrote:
> Hi ,
>
> Ping ...
> Is there any comments.
> Need some comments to move forward.
Damn, I dropped the ball on this. Sorry about that.
I'll have a look over the weekend.
Thanks,
M.
--
Jazz is not dead. It just smells funny...
_______________________________________________
kvmarm mailing list
kvmarm@lists.cs.columbia.edu
https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
^ permalink raw reply [flat|nested] 13+ messages in thread