[PATCH 0/3] x86/kvm/hyper-v: Implement Direct Mode for synthetic timers

[PATCH 0/3] x86/kvm/hyper-v: Implement Direct Mode for synthetic timers

[Vitaly Kuznetsov]

Turns out Hyper-V on KVM (as of 2016) will only use synthetic timers
if direct mode is available. With direct mode we notify the guest by
asserting APIC irq instead of sending a SynIC message.

The series was tested with Hyper-V on KVM as well as kvm-unit-tests.
I'm including Qemu a kvm-unit-tests patches for testing purposes.

Vitaly Kuznetsov (3):
  x86/hyper-v: move synic/stimer control structures definitions to
    hyperv-tlfs.h
  x86/kvm/hyper-v: use stimer config definition from hyperv-tlfs.h
  x86/kvm/hyper-v: direct mode for synthetic timers

 arch/x86/include/asm/hyperv-tlfs.h |  73 +++++++++++++++++++--
 arch/x86/include/asm/kvm_host.h    |   2 +-
 arch/x86/kvm/hyperv.c              | 100 ++++++++++++++++++++++-------
 arch/x86/kvm/trace.h               |  10 +--
 arch/x86/kvm/x86.c                 |   1 +
 drivers/hv/hv.c                    |   2 +-
 drivers/hv/hyperv_vmbus.h          |  68 --------------------
 include/uapi/linux/kvm.h           |   1 +
 8 files changed, 155 insertions(+), 102 deletions(-)

-- 
2.17.2

[PATCH 1/3] x86/hyper-v: move synic/stimer control structures definitions to hyperv-tlfs.h

[Vitaly Kuznetsov]

We implement Hyper-V SynIC and synthetic timers in KVM too so there's some
room for code sharing.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 arch/x86/include/asm/hyperv-tlfs.h | 69 ++++++++++++++++++++++++++++++
 drivers/hv/hv.c                    |  2 +-
 drivers/hv/hyperv_vmbus.h          | 68 -----------------------------
 3 files changed, 70 insertions(+), 69 deletions(-)

diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
index 4139f7650fe5..b032c05cd3ee 100644
--- a/arch/x86/include/asm/hyperv-tlfs.h
+++ b/arch/x86/include/asm/hyperv-tlfs.h
@@ -731,6 +731,75 @@ struct hv_enlightened_vmcs {
 #define HV_STIMER_AUTOENABLE		(1ULL << 3)
 #define HV_STIMER_SINT(config)		(__u8)(((config) >> 16) & 0x0F)
 
+/* Define synthetic interrupt controller flag constants. */
+#define HV_EVENT_FLAGS_COUNT		(256 * 8)
+#define HV_EVENT_FLAGS_LONG_COUNT	(256 / sizeof(unsigned long))
+
+/*
+ * Synthetic timer configuration.
+ */
+union hv_stimer_config {
+	u64 as_uint64;
+	struct {
+		u64 enable:1;
+		u64 periodic:1;
+		u64 lazy:1;
+		u64 auto_enable:1;
+		u64 apic_vector:8;
+		u64 direct_mode:1;
+		u64 reserved_z0:3;
+		u64 sintx:4;
+		u64 reserved_z1:44;
+	};
+};
+
+
+/* Define the synthetic interrupt controller event flags format. */
+union hv_synic_event_flags {
+	unsigned long flags[HV_EVENT_FLAGS_LONG_COUNT];
+};
+
+/* Define SynIC control register. */
+union hv_synic_scontrol {
+	u64 as_uint64;
+	struct {
+		u64 enable:1;
+		u64 reserved:63;
+	};
+};
+
+/* Define synthetic interrupt source. */
+union hv_synic_sint {
+	u64 as_uint64;
+	struct {
+		u64 vector:8;
+		u64 reserved1:8;
+		u64 masked:1;
+		u64 auto_eoi:1;
+		u64 reserved2:46;
+	};
+};
+
+/* Define the format of the SIMP register */
+union hv_synic_simp {
+	u64 as_uint64;
+	struct {
+		u64 simp_enabled:1;
+		u64 preserved:11;
+		u64 base_simp_gpa:52;
+	};
+};
+
+/* Define the format of the SIEFP register */
+union hv_synic_siefp {
+	u64 as_uint64;
+	struct {
+		u64 siefp_enabled:1;
+		u64 preserved:11;
+		u64 base_siefp_gpa:52;
+	};
+};
+
 struct hv_vpset {
 	u64 format;
 	u64 valid_bank_mask;
diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c
index 332d7c34be5c..11273cd384d6 100644
--- a/drivers/hv/hv.c
+++ b/drivers/hv/hv.c
@@ -143,7 +143,7 @@ static int hv_ce_shutdown(struct clock_event_device *evt)
 
 static int hv_ce_set_oneshot(struct clock_event_device *evt)
 {
-	union hv_timer_config timer_cfg;
+	union hv_stimer_config timer_cfg;
 
 	timer_cfg.as_uint64 = 0;
 	timer_cfg.enable = 1;
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index 72eaba3d50fc..8df4f45f4b6d 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -44,74 +44,6 @@
  */
 #define HV_UTIL_NEGO_TIMEOUT 55
 
-/* Define synthetic interrupt controller flag constants. */
-#define HV_EVENT_FLAGS_COUNT		(256 * 8)
-#define HV_EVENT_FLAGS_LONG_COUNT	(256 / sizeof(unsigned long))
-
-/*
- * Timer configuration register.
- */
-union hv_timer_config {
-	u64 as_uint64;
-	struct {
-		u64 enable:1;
-		u64 periodic:1;
-		u64 lazy:1;
-		u64 auto_enable:1;
-		u64 apic_vector:8;
-		u64 direct_mode:1;
-		u64 reserved_z0:3;
-		u64 sintx:4;
-		u64 reserved_z1:44;
-	};
-};
-
-
-/* Define the synthetic interrupt controller event flags format. */
-union hv_synic_event_flags {
-	unsigned long flags[HV_EVENT_FLAGS_LONG_COUNT];
-};
-
-/* Define SynIC control register. */
-union hv_synic_scontrol {
-	u64 as_uint64;
-	struct {
-		u64 enable:1;
-		u64 reserved:63;
-	};
-};
-
-/* Define synthetic interrupt source. */
-union hv_synic_sint {
-	u64 as_uint64;
-	struct {
-		u64 vector:8;
-		u64 reserved1:8;
-		u64 masked:1;
-		u64 auto_eoi:1;
-		u64 reserved2:46;
-	};
-};
-
-/* Define the format of the SIMP register */
-union hv_synic_simp {
-	u64 as_uint64;
-	struct {
-		u64 simp_enabled:1;
-		u64 preserved:11;
-		u64 base_simp_gpa:52;
-	};
-};
-
-/* Define the format of the SIEFP register */
-union hv_synic_siefp {
-	u64 as_uint64;
-	struct {
-		u64 siefp_enabled:1;
-		u64 preserved:11;
-		u64 base_siefp_gpa:52;
-	};
-};
 
 /* Definitions for the monitored notification facility */
 union hv_monitor_trigger_group {
-- 
2.17.2

[PATCH 2/3] x86/kvm/hyper-v: use stimer config definition from hyperv-tlfs.h

[Vitaly Kuznetsov]

As a preparation to implementing Direct Mode for Hyper-V synthetic
timers switch to using stimer config definition from hyperv-tlfs.h.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 arch/x86/include/asm/hyperv-tlfs.h |  6 ------
 arch/x86/include/asm/kvm_host.h    |  2 +-
 arch/x86/kvm/hyperv.c              | 33 +++++++++++++++---------------
 3 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
index b032c05cd3ee..ebfed56976d2 100644
--- a/arch/x86/include/asm/hyperv-tlfs.h
+++ b/arch/x86/include/asm/hyperv-tlfs.h
@@ -725,12 +725,6 @@ struct hv_enlightened_vmcs {
 
 #define HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL			0xFFFF
 
-#define HV_STIMER_ENABLE		(1ULL << 0)
-#define HV_STIMER_PERIODIC		(1ULL << 1)
-#define HV_STIMER_LAZY			(1ULL << 2)
-#define HV_STIMER_AUTOENABLE		(1ULL << 3)
-#define HV_STIMER_SINT(config)		(__u8)(((config) >> 16) & 0x0F)
-
 /* Define synthetic interrupt controller flag constants. */
 #define HV_EVENT_FLAGS_COUNT		(256 * 8)
 #define HV_EVENT_FLAGS_LONG_COUNT	(256 / sizeof(unsigned long))
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 55e51ff7e421..e1a40e649cdc 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -497,7 +497,7 @@ struct kvm_mtrr {
 struct kvm_vcpu_hv_stimer {
 	struct hrtimer timer;
 	int index;
-	u64 config;
+	union hv_stimer_config config;
 	u64 count;
 	u64 exp_time;
 	struct hv_message msg;
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index 4e80080f277a..eaec15c738df 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -201,9 +201,8 @@ static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)
 	stimers_pending = 0;
 	for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) {
 		stimer = &hv_vcpu->stimer[idx];
-		if (stimer->msg_pending &&
-		    (stimer->config & HV_STIMER_ENABLE) &&
-		    HV_STIMER_SINT(stimer->config) == sint) {
+		if (stimer->msg_pending && stimer->config.enable &&
+		    stimer->config.sintx == sint) {
 			set_bit(stimer->index,
 				hv_vcpu->stimer_pending_bitmap);
 			stimers_pending++;
@@ -497,7 +496,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
 	time_now = get_time_ref_counter(stimer_to_vcpu(stimer)->kvm);
 	ktime_now = ktime_get();
 
-	if (stimer->config & HV_STIMER_PERIODIC) {
+	if (stimer->config.periodic) {
 		if (stimer->exp_time) {
 			if (time_now >= stimer->exp_time) {
 				u64 remainder;
@@ -546,13 +545,15 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
 static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config,
 			     bool host)
 {
+	union hv_stimer_config new_config = {.as_uint64 = config};
+
 	trace_kvm_hv_stimer_set_config(stimer_to_vcpu(stimer)->vcpu_id,
 				       stimer->index, config, host);
 
 	stimer_cleanup(stimer);
-	if ((stimer->config & HV_STIMER_ENABLE) && HV_STIMER_SINT(config) == 0)
-		config &= ~HV_STIMER_ENABLE;
-	stimer->config = config;
+	if (stimer->config.enable && new_config.sintx == 0)
+		new_config.enable = 0;
+	stimer->config.as_uint64 = new_config.as_uint64;
 	stimer_mark_pending(stimer, false);
 	return 0;
 }
@@ -566,16 +567,16 @@ static int stimer_set_count(struct kvm_vcpu_hv_stimer *stimer, u64 count,
 	stimer_cleanup(stimer);
 	stimer->count = count;
 	if (stimer->count == 0)
-		stimer->config &= ~HV_STIMER_ENABLE;
-	else if (stimer->config & HV_STIMER_AUTOENABLE)
-		stimer->config |= HV_STIMER_ENABLE;
+		stimer->config.enable = 0;
+	else if (stimer->config.auto_enable)
+		stimer->config.enable = 1;
 	stimer_mark_pending(stimer, false);
 	return 0;
 }
 
 static int stimer_get_config(struct kvm_vcpu_hv_stimer *stimer, u64 *pconfig)
 {
-	*pconfig = stimer->config;
+	*pconfig = stimer->config.as_uint64;
 	return 0;
 }
 
@@ -636,7 +637,7 @@ static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer)
 	payload->expiration_time = stimer->exp_time;
 	payload->delivery_time = get_time_ref_counter(vcpu->kvm);
 	return synic_deliver_msg(vcpu_to_synic(vcpu),
-				 HV_STIMER_SINT(stimer->config), msg);
+				 stimer->config.sintx, msg);
 }
 
 static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer)
@@ -649,8 +650,8 @@ static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer)
 				       stimer->index, r);
 	if (!r) {
 		stimer->msg_pending = false;
-		if (!(stimer->config & HV_STIMER_PERIODIC))
-			stimer->config &= ~HV_STIMER_ENABLE;
+		if (!(stimer->config.periodic))
+			stimer->config.enable = 0;
 	}
 }
 
@@ -664,7 +665,7 @@ void kvm_hv_process_stimers(struct kvm_vcpu *vcpu)
 	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++)
 		if (test_and_clear_bit(i, hv_vcpu->stimer_pending_bitmap)) {
 			stimer = &hv_vcpu->stimer[i];
-			if (stimer->config & HV_STIMER_ENABLE) {
+			if (stimer->config.enable) {
 				exp_time = stimer->exp_time;
 
 				if (exp_time) {
@@ -674,7 +675,7 @@ void kvm_hv_process_stimers(struct kvm_vcpu *vcpu)
 						stimer_expiration(stimer);
 				}
 
-				if ((stimer->config & HV_STIMER_ENABLE) &&
+				if ((stimer->config.enable) &&
 				    stimer->count) {
 					if (!stimer->msg_pending)
 						stimer_start(stimer);
-- 
2.17.2

[PATCH 3/3] x86/kvm/hyper-v: direct mode for synthetic timers

[Vitaly Kuznetsov]

Turns out Hyper-V on KVM (as of 2016) will only use synthetic timers
if direct mode is available. With direct mode we notify the guest by
asserting APIC irq instead of sending a SynIC message.

The implementation uses existing vec_bitmap for letting lapic code
know that we're interested in the particular IRQ's EOI request. We assume
that the same APIC irq won't be used by the guest for both direct mode
stimer and as sint source (especially with AutoEOI semantics). It is
unclear how things should be handled if that's not true.

Direct mode is also somewhat less expensive; in my testing
stimer_send_msg() takes not less than 1500 cpu cycles and
stimer_notify_direct() can usually be done in 300-400. WS2016 without
Hyper-V, however, always sticks to non-direct version.

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 arch/x86/kvm/hyperv.c    | 71 +++++++++++++++++++++++++++++++++++-----
 arch/x86/kvm/trace.h     | 10 +++---
 arch/x86/kvm/x86.c       |  1 +
 include/uapi/linux/kvm.h |  1 +
 4 files changed, 70 insertions(+), 13 deletions(-)

diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index eaec15c738df..c451793bf758 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -53,8 +53,21 @@ static inline int synic_get_sint_vector(u64 sint_value)
 static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic *synic,
 				      int vector)
 {
+	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
+	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
+	struct kvm_vcpu_hv_stimer *stimer;
 	int i;
 
+	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) {
+		stimer = &hv_vcpu->stimer[i];
+		if (stimer->config.enable && stimer->config.direct_mode &&
+		    stimer->config.apic_vector == vector)
+			return true;
+	}
+
+	if (vector < HV_SYNIC_FIRST_VALID_VECTOR)
+		return false;
+
 	for (i = 0; i < ARRAY_SIZE(synic->sint); i++) {
 		if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector)
 			return true;
@@ -80,14 +93,14 @@ static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic *synic,
 static void synic_update_vector(struct kvm_vcpu_hv_synic *synic,
 				int vector)
 {
-	if (vector < HV_SYNIC_FIRST_VALID_VECTOR)
-		return;
-
 	if (synic_has_vector_connected(synic, vector))
 		__set_bit(vector, synic->vec_bitmap);
 	else
 		__clear_bit(vector, synic->vec_bitmap);
 
+	if (vector < HV_SYNIC_FIRST_VALID_VECTOR)
+		return;
+
 	if (synic_has_vector_auto_eoi(synic, vector))
 		__set_bit(vector, synic->auto_eoi_bitmap);
 	else
@@ -202,6 +215,7 @@ static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)
 	for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) {
 		stimer = &hv_vcpu->stimer[idx];
 		if (stimer->msg_pending && stimer->config.enable &&
+		    !stimer->config.direct_mode &&
 		    stimer->config.sintx == sint) {
 			set_bit(stimer->index,
 				hv_vcpu->stimer_pending_bitmap);
@@ -371,14 +385,29 @@ int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vpidx, u32 sint)
 
 void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector)
 {
+	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
 	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
-	int i;
+	struct kvm_vcpu_hv_stimer *stimer;
+	int i, stimers_pending = 0;
 
 	trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector);
 
 	for (i = 0; i < ARRAY_SIZE(synic->sint); i++)
 		if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector)
 			kvm_hv_notify_acked_sint(vcpu, i);
+
+	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) {
+		stimer = &hv_vcpu->stimer[i];
+		if (stimer->msg_pending && stimer->config.enable &&
+		    stimer->config.direct_mode &&
+		    stimer->config.apic_vector == vector) {
+			set_bit(stimer->index,
+				hv_vcpu->stimer_pending_bitmap);
+			stimers_pending++;
+		}
+	}
+	if (stimers_pending)
+		kvm_make_request(KVM_REQ_HV_STIMER, vcpu);
 }
 
 static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vpidx, u32 sint, int gsi)
@@ -545,15 +574,25 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
 static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config,
 			     bool host)
 {
-	union hv_stimer_config new_config = {.as_uint64 = config};
+	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
+	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
+	union hv_stimer_config new_config = {.as_uint64 = config},
+		old_config = {.as_uint64 = stimer->config.as_uint64};
 
 	trace_kvm_hv_stimer_set_config(stimer_to_vcpu(stimer)->vcpu_id,
 				       stimer->index, config, host);
 
 	stimer_cleanup(stimer);
-	if (stimer->config.enable && new_config.sintx == 0)
+	if (old_config.enable &&
+	    !new_config.direct_mode && new_config.sintx == 0)
 		new_config.enable = 0;
 	stimer->config.as_uint64 = new_config.as_uint64;
+
+	if (old_config.direct_mode)
+		synic_update_vector(&hv_vcpu->synic, old_config.apic_vector);
+	if (new_config.direct_mode)
+		synic_update_vector(&hv_vcpu->synic, new_config.apic_vector);
+
 	stimer_mark_pending(stimer, false);
 	return 0;
 }
@@ -640,14 +679,28 @@ static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer)
 				 stimer->config.sintx, msg);
 }
 
+static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer)
+{
+	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
+	struct kvm_lapic_irq irq = {
+		.delivery_mode = APIC_DM_FIXED,
+		.vector = stimer->config.apic_vector
+	};
+
+	return !kvm_apic_set_irq(vcpu, &irq, NULL);
+}
+
 static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer)
 {
-	int r;
+	int r, direct = stimer->config.direct_mode;
 
 	stimer->msg_pending = true;
-	r = stimer_send_msg(stimer);
+	if (!direct)
+		r = stimer_send_msg(stimer);
+	else
+		r = stimer_notify_direct(stimer);
 	trace_kvm_hv_stimer_expiration(stimer_to_vcpu(stimer)->vcpu_id,
-				       stimer->index, r);
+				       stimer->index, direct, r);
 	if (!r) {
 		stimer->msg_pending = false;
 		if (!(stimer->config.periodic))
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
index 0659465a745c..705f40ae2532 100644
--- a/arch/x86/kvm/trace.h
+++ b/arch/x86/kvm/trace.h
@@ -1254,24 +1254,26 @@ TRACE_EVENT(kvm_hv_stimer_callback,
  * Tracepoint for stimer_expiration.
  */
 TRACE_EVENT(kvm_hv_stimer_expiration,
-	TP_PROTO(int vcpu_id, int timer_index, int msg_send_result),
-	TP_ARGS(vcpu_id, timer_index, msg_send_result),
+	TP_PROTO(int vcpu_id, int timer_index, int direct, int msg_send_result),
+	TP_ARGS(vcpu_id, timer_index, direct, msg_send_result),
 
 	TP_STRUCT__entry(
 		__field(int, vcpu_id)
 		__field(int, timer_index)
+		__field(int, direct)
 		__field(int, msg_send_result)
 	),
 
 	TP_fast_assign(
 		__entry->vcpu_id = vcpu_id;
 		__entry->timer_index = timer_index;
+		__entry->direct = direct;
 		__entry->msg_send_result = msg_send_result;
 	),
 
-	TP_printk("vcpu_id %d timer %d msg send result %d",
+	TP_printk("vcpu_id %d timer %d direct %d send result %d",
 		  __entry->vcpu_id, __entry->timer_index,
-		  __entry->msg_send_result)
+		  __entry->direct, __entry->msg_send_result)
 );
 
 /*
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 5cd5647120f2..b21b5ceb8d26 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -2997,6 +2997,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 	case KVM_CAP_HYPERV_TLBFLUSH:
 	case KVM_CAP_HYPERV_SEND_IPI:
 	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
+	case KVM_CAP_HYPERV_STIMER_DIRECT:
 	case KVM_CAP_PCI_SEGMENT:
 	case KVM_CAP_DEBUGREGS:
 	case KVM_CAP_X86_ROBUST_SINGLESTEP:
diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
index 2b7a652c9fa4..b8da14cee8e5 100644
--- a/include/uapi/linux/kvm.h
+++ b/include/uapi/linux/kvm.h
@@ -975,6 +975,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_HYPERV_ENLIGHTENED_VMCS 163
 #define KVM_CAP_EXCEPTION_PAYLOAD 164
 #define KVM_CAP_ARM_VM_IPA_SIZE 165
+#define KVM_CAP_HYPERV_STIMER_DIRECT 166
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
-- 
2.17.2

[QEMU PATCH] i386/kvm: add support for Direct Mode for Hyper-V synthetic timers

[Vitaly Kuznetsov]

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 linux-headers/linux/kvm.h  |  1 +
 target/i386/cpu.c          |  1 +
 target/i386/cpu.h          |  1 +
 target/i386/hyperv-proto.h |  1 +
 target/i386/kvm.c          | 11 +++++++++++
 5 files changed, 15 insertions(+)

diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index f11a7eb49c..43fb240b60 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -965,6 +965,7 @@ struct kvm_ppc_resize_hpt {
 #define KVM_CAP_COALESCED_PIO 162
 #define KVM_CAP_HYPERV_ENLIGHTENED_VMCS 163
 #define KVM_CAP_EXCEPTION_PAYLOAD 164
+#define KVM_CAP_HYPERV_STIMER_DIRECT 166
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
diff --git a/target/i386/cpu.c b/target/i386/cpu.c
index af7e9f09cc..cd42fb81dd 100644
--- a/target/i386/cpu.c
+++ b/target/i386/cpu.c
@@ -5733,6 +5733,7 @@ static Property x86_cpu_properties[] = {
     DEFINE_PROP_BOOL("hv-reenlightenment", X86CPU, hyperv_reenlightenment, false),
     DEFINE_PROP_BOOL("hv-tlbflush", X86CPU, hyperv_tlbflush, false),
     DEFINE_PROP_BOOL("hv-ipi", X86CPU, hyperv_ipi, false),
+    DEFINE_PROP_BOOL("hv-stimer-direct", X86CPU, hyperv_stimer_direct, false),
     DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, true),
     DEFINE_PROP_BOOL("enforce", X86CPU, enforce_cpuid, false),
     DEFINE_PROP_BOOL("kvm", X86CPU, expose_kvm, true),
diff --git a/target/i386/cpu.h b/target/i386/cpu.h
index ad0e0b4534..a3d7cac8be 100644
--- a/target/i386/cpu.h
+++ b/target/i386/cpu.h
@@ -1392,6 +1392,7 @@ struct X86CPU {
     bool hyperv_reenlightenment;
     bool hyperv_tlbflush;
     bool hyperv_ipi;
+    bool hyperv_stimer_direct;
     bool check_cpuid;
     bool enforce_cpuid;
     bool expose_kvm;
diff --git a/target/i386/hyperv-proto.h b/target/i386/hyperv-proto.h
index 8c572cd7c2..4543112cf5 100644
--- a/target/i386/hyperv-proto.h
+++ b/target/i386/hyperv-proto.h
@@ -48,6 +48,7 @@
 #define HV_GUEST_IDLE_STATE_AVAILABLE           (1u << 5)
 #define HV_FREQUENCY_MSRS_AVAILABLE             (1u << 8)
 #define HV_GUEST_CRASH_MSR_AVAILABLE            (1u << 10)
+#define HV_STIMER_DIRECT_MODE_AVAILABLE		(1u << 19)
 
 /*
  * HV_CPUID_ENLIGHTMENT_INFO.EAX bits
diff --git a/target/i386/kvm.c b/target/i386/kvm.c
index 796a049a0d..f12321e0f7 100644
--- a/target/i386/kvm.c
+++ b/target/i386/kvm.c
@@ -797,6 +797,17 @@ static int hyperv_handle_properties(CPUState *cs)
         }
         env->features[FEAT_HYPERV_EAX] |= HV_SYNTIMERS_AVAILABLE;
     }
+    if (cpu->hyperv_stimer_direct) {
+        if (kvm_check_extension(cs->kvm_state,
+                                KVM_CAP_HYPERV_STIMER_DIRECT) <= 0) {
+            fprintf(stderr, "Hyper-V timers direct mode "
+                        "(requested by 'hv-stimer-direct' cpu flag) "
+                        " is not supported by kernel\n");
+                return -ENOSYS;
+        }
+        env->features[FEAT_HYPERV_EDX] |= HV_STIMER_DIRECT_MODE_AVAILABLE;
+    }
+
     return 0;
 }
 
-- 
2.17.2

[kvm-unit-tests PATCH] x86/hyper-v: stimer_direct test

[Vitaly Kuznetsov]

Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
---
 x86/Makefile.common        |   3 +
 x86/hyperv.h               |   7 ++
 x86/hyperv_stimer_direct.c | 250 +++++++++++++++++++++++++++++++++++++
 x86/unittests.cfg          |   6 +
 4 files changed, 266 insertions(+)
 create mode 100644 x86/hyperv_stimer_direct.c

diff --git a/x86/Makefile.common b/x86/Makefile.common
index e612dbe..115189a 100644
--- a/x86/Makefile.common
+++ b/x86/Makefile.common
@@ -57,6 +57,7 @@ tests-common = $(TEST_DIR)/vmexit.flat $(TEST_DIR)/tsc.flat \
                $(TEST_DIR)/tsc_adjust.flat $(TEST_DIR)/asyncpf.flat \
                $(TEST_DIR)/init.flat $(TEST_DIR)/smap.flat \
                $(TEST_DIR)/hyperv_synic.flat $(TEST_DIR)/hyperv_stimer.flat \
+               $(TEST_DIR)/hyperv_stimer_direct.flat \
                $(TEST_DIR)/hyperv_connections.flat \
                $(TEST_DIR)/umip.flat
 
@@ -81,6 +82,8 @@ $(TEST_DIR)/hyperv_synic.elf: $(TEST_DIR)/hyperv.o
 
 $(TEST_DIR)/hyperv_stimer.elf: $(TEST_DIR)/hyperv.o
 
+$(TEST_DIR)/hyperv_stimer_direct.elf: $(TEST_DIR)/hyperv.o
+
 $(TEST_DIR)/hyperv_connections.elf: $(TEST_DIR)/hyperv.o
 
 arch_clean:
diff --git a/x86/hyperv.h b/x86/hyperv.h
index 9a83da4..718744c 100644
--- a/x86/hyperv.h
+++ b/x86/hyperv.h
@@ -10,6 +10,8 @@
 #define HV_X64_MSR_SYNIC_AVAILABLE              (1 << 2)
 #define HV_X64_MSR_SYNTIMER_AVAILABLE           (1 << 3)
 
+#define HV_STIMER_DIRECT_MODE_AVAILABLE		(1 << 19)
+
 #define HV_X64_MSR_GUEST_OS_ID                  0x40000000
 #define HV_X64_MSR_HYPERCALL                    0x40000001
 
@@ -205,6 +207,11 @@ static inline bool stimer_supported(void)
     return cpuid(HYPERV_CPUID_FEATURES).a & HV_X64_MSR_SYNIC_AVAILABLE;
 }
 
+static inline bool stimer_direct_supported(void)
+{
+    return cpuid(HYPERV_CPUID_FEATURES).d & HV_STIMER_DIRECT_MODE_AVAILABLE;
+}
+
 static inline bool hv_time_ref_counter_supported(void)
 {
     return cpuid(HYPERV_CPUID_FEATURES).a & HV_X64_MSR_TIME_REF_COUNT_AVAILABLE;
diff --git a/x86/hyperv_stimer_direct.c b/x86/hyperv_stimer_direct.c
new file mode 100644
index 0000000..e93f633
--- /dev/null
+++ b/x86/hyperv_stimer_direct.c
@@ -0,0 +1,250 @@
+#include "libcflat.h"
+#include "processor.h"
+#include "msr.h"
+#include "isr.h"
+#include "vm.h"
+#include "apic.h"
+#include "desc.h"
+#include "smp.h"
+#include "atomic.h"
+#include "hyperv.h"
+#include "asm/barrier.h"
+#include "alloc_page.h"
+
+#define MAX_CPUS 4
+
+#define INT1_VEC 0xF1
+#define INT2_VEC 0xF2
+
+#define ONE_MS_IN_100NS 10000
+
+struct stimer {
+    int sint;
+    int index;
+    atomic_t fire_count;
+};
+
+struct svcpu {
+    int vcpu;
+    struct stimer timer[HV_SYNIC_STIMER_COUNT];
+};
+
+static struct svcpu g_synic_vcpu[MAX_CPUS];
+
+static void stimer_init(struct stimer *timer, int index)
+{
+    memset(timer, 0, sizeof(*timer));
+    timer->index = index;
+}
+
+static void synic_enable(void)
+{
+    int vcpu = smp_id(), i;
+    struct svcpu *svcpu = &g_synic_vcpu[vcpu];
+
+    memset(svcpu, 0, sizeof(*svcpu));
+    svcpu->vcpu = vcpu;
+    for (i = 0; i < ARRAY_SIZE(svcpu->timer); i++) {
+        stimer_init(&svcpu->timer[i], i);
+    }
+}
+
+static void stimer_shutdown(struct stimer *timer)
+{
+    wrmsr(HV_X64_MSR_STIMER0_CONFIG + 2*timer->index, 0);
+}
+
+static void process_stimer_expired(struct stimer *timer)
+{
+    atomic_inc(&timer->fire_count);
+}
+
+static void __stimer_isr_direct(int vcpu, int timer_index)
+{
+    struct svcpu *svcpu = &g_synic_vcpu[vcpu];
+    struct stimer *timer = &svcpu->timer[timer_index];
+
+    process_stimer_expired(timer);
+}
+
+static void stimer_isr_direct1(isr_regs_t *regs)
+{
+    int vcpu = smp_id();
+
+    __stimer_isr_direct(vcpu, 0);
+
+    eoi();
+}
+
+static void stimer_isr_direct2(isr_regs_t *regs)
+{
+    int vcpu = smp_id();
+
+    __stimer_isr_direct(vcpu, 1);
+
+    eoi();
+}
+
+static void stimer_start(struct stimer *timer,
+                         bool auto_enable, bool periodic,
+                         u64 tick_100ns, int apic_vector)
+{
+    u64 count;
+    union hv_stimer_config config = {.as_uint64 = 0};
+
+    timer->sint = 0;
+    atomic_set(&timer->fire_count, 0);
+
+    config.periodic = periodic;
+    config.enable = 1;
+    config.auto_enable = auto_enable;
+    config.direct_mode = 1;
+    config.sintx = 0;
+    config.apic_vector = apic_vector;
+
+    if (periodic) {
+        count = tick_100ns;
+    } else {
+        count = rdmsr(HV_X64_MSR_TIME_REF_COUNT) + tick_100ns;
+    }
+
+    if (!auto_enable) {
+        wrmsr(HV_X64_MSR_STIMER0_COUNT + timer->index*2, count);
+        wrmsr(HV_X64_MSR_STIMER0_CONFIG + timer->index*2, config.as_uint64);
+    } else {
+        wrmsr(HV_X64_MSR_STIMER0_CONFIG + timer->index*2, config.as_uint64);
+        wrmsr(HV_X64_MSR_STIMER0_COUNT + timer->index*2, count);
+    }
+}
+
+static void stimers_shutdown(void)
+{
+    int vcpu = smp_id(), i;
+    struct svcpu *svcpu = &g_synic_vcpu[vcpu];
+
+    for (i = 0; i < ARRAY_SIZE(svcpu->timer); i++) {
+        stimer_shutdown(&svcpu->timer[i]);
+    }
+}
+
+static void stimer_test_prepare(void *ctx)
+{
+    write_cr3((ulong)ctx);
+    synic_enable();
+}
+
+static void stimer_test_periodic(int vcpu, struct stimer *timer1,
+                                 struct stimer *timer2)
+{
+    /* Check periodic timers */
+    stimer_start(timer1, false, true, ONE_MS_IN_100NS, INT1_VEC);
+    stimer_start(timer2, false, true, ONE_MS_IN_100NS, INT2_VEC);
+    while ((atomic_read(&timer1->fire_count) < 1000) ||
+           (atomic_read(&timer2->fire_count) < 1000)) {
+        pause();
+    }
+    report("Hyper-V SynIC periodic timers test vcpu %d", true, vcpu);
+    stimer_shutdown(timer1);
+    stimer_shutdown(timer2);
+}
+
+static void stimer_test_one_shot(int vcpu, struct stimer *timer)
+{
+    /* Check one-shot timer */
+    stimer_start(timer, false, false, ONE_MS_IN_100NS, INT1_VEC);
+    while (atomic_read(&timer->fire_count) < 1) {
+        pause();
+    }
+    report("Hyper-V SynIC one-shot test vcpu %d", true, vcpu);
+    stimer_shutdown(timer);
+}
+
+static void stimer_test_auto_enable_one_shot(int vcpu, struct stimer *timer)
+{
+    /* Check auto-enable one-shot timer */
+    stimer_start(timer, true, false, ONE_MS_IN_100NS, INT2_VEC);
+    while (atomic_read(&timer->fire_count) < 1) {
+        pause();
+    }
+    report("Hyper-V SynIC auto-enable one-shot timer test vcpu %d", true, vcpu);
+    stimer_shutdown(timer);
+}
+
+static void stimer_test_auto_enable_periodic(int vcpu, struct stimer *timer)
+{
+    /* Check auto-enable periodic timer */
+    stimer_start(timer, true, true, ONE_MS_IN_100NS, INT1_VEC);
+    while (atomic_read(&timer->fire_count) < 1000) {
+        pause();
+    }
+    report("Hyper-V SynIC auto-enable periodic timer test vcpu %d", true, vcpu);
+    stimer_shutdown(timer);
+}
+
+static void stimer_test(void *ctx)
+{
+    int vcpu = smp_id();
+    struct svcpu *svcpu = &g_synic_vcpu[vcpu];
+    struct stimer *timer1, *timer2;
+
+    irq_enable();
+
+    timer1 = &svcpu->timer[0];
+    timer2 = &svcpu->timer[1];
+
+    stimer_test_periodic(vcpu, timer1, timer2);
+    stimer_test_one_shot(vcpu, timer1);
+    stimer_test_auto_enable_one_shot(vcpu, timer2);
+    stimer_test_auto_enable_periodic(vcpu, timer1);
+
+    irq_disable();
+}
+
+static void stimer_test_cleanup(void *ctx)
+{
+    stimers_shutdown();
+}
+
+static void stimer_test_all(void)
+{
+    int ncpus;
+
+    setup_vm();
+    smp_init();
+    enable_apic();
+
+    ncpus = cpu_count();
+    if (ncpus > MAX_CPUS)
+        report_abort("number cpus exceeds %d", MAX_CPUS);
+    printf("cpus = %d\n", ncpus);
+
+    handle_irq(INT1_VEC, stimer_isr_direct1);
+    handle_irq(INT2_VEC, stimer_isr_direct2);
+
+    on_cpus(stimer_test_prepare, (void *)read_cr3());
+    on_cpus(stimer_test, NULL);
+    on_cpus(stimer_test_cleanup, NULL);
+}
+
+int main(int ac, char **av)
+{
+
+    if (!stimer_supported()) {
+        report("Hyper-V SynIC timers are not supported", true);
+        goto done;
+    }
+
+    if (!hv_time_ref_counter_supported()) {
+        report("Hyper-V time reference counter is not supported", true);
+        goto done;
+    }
+
+    if (!stimer_direct_supported()) {
+        report("Hyper-V direct mode for synthetic timers is not supported", true);
+        goto done;
+    }
+
+    stimer_test_all();
+done:
+    return report_summary();
+}
diff --git a/x86/unittests.cfg b/x86/unittests.cfg
index c744a6d..36c0657 100644
--- a/x86/unittests.cfg
+++ b/x86/unittests.cfg
@@ -635,6 +635,12 @@ smp = 2
 extra_params = -cpu kvm64,hv_vpindex,hv_time,hv_synic,hv_stimer -device hyperv-testdev
 groups = hyperv
 
+[hyperv_stimer_direct]
+file = hyperv_stimer_direct.flat
+smp = 2
+extra_params = -cpu kvm64,hv_vpindex,hv_time,hv_synic,hv_stimer,hv_stimer_direct -device hyperv-testdev
+groups = hyperv
+
 [hyperv_clock]
 file = hyperv_clock.flat
 smp = 2
-- 
2.17.2

Re: [PATCH 3/3] x86/kvm/hyper-v: direct mode for synthetic timers

[Paolo Bonzini]

On 13/11/18 16:22, Vitaly Kuznetsov wrote:
> Turns out Hyper-V on KVM (as of 2016) will only use synthetic timers
> if direct mode is available. With direct mode we notify the guest by
> asserting APIC irq instead of sending a SynIC message.
> 
> The implementation uses existing vec_bitmap for letting lapic code
> know that we're interested in the particular IRQ's EOI request. We assume
> that the same APIC irq won't be used by the guest for both direct mode
> stimer and as sint source (especially with AutoEOI semantics). It is
> unclear how things should be handled if that's not true.
> 
> Direct mode is also somewhat less expensive; in my testing
> stimer_send_msg() takes not less than 1500 cpu cycles and
> stimer_notify_direct() can usually be done in 300-400. WS2016 without
> Hyper-V, however, always sticks to non-direct version.
> 
> Signed-off-by: Vitaly Kuznetsov <vkuznets@redhat.com>
> ---
>  arch/x86/kvm/hyperv.c    | 71 +++++++++++++++++++++++++++++++++++-----
>  arch/x86/kvm/trace.h     | 10 +++---
>  arch/x86/kvm/x86.c       |  1 +
>  include/uapi/linux/kvm.h |  1 +
>  4 files changed, 70 insertions(+), 13 deletions(-)
> 
> diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
> index eaec15c738df..c451793bf758 100644
> --- a/arch/x86/kvm/hyperv.c
> +++ b/arch/x86/kvm/hyperv.c
> @@ -53,8 +53,21 @@ static inline int synic_get_sint_vector(u64 sint_value)
>  static bool synic_has_vector_connected(struct kvm_vcpu_hv_synic *synic,
>  				      int vector)
>  {
> +	struct kvm_vcpu *vcpu = synic_to_vcpu(synic);
> +	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
> +	struct kvm_vcpu_hv_stimer *stimer;
>  	int i;
>  
> +	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) {
> +		stimer = &hv_vcpu->stimer[i];
> +		if (stimer->config.enable && stimer->config.direct_mode &&
> +		    stimer->config.apic_vector == vector)
> +			return true;
> +	}
> +
> +	if (vector < HV_SYNIC_FIRST_VALID_VECTOR)
> +		return false;
> +
>  	for (i = 0; i < ARRAY_SIZE(synic->sint); i++) {
>  		if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector)
>  			return true;
> @@ -80,14 +93,14 @@ static bool synic_has_vector_auto_eoi(struct kvm_vcpu_hv_synic *synic,
>  static void synic_update_vector(struct kvm_vcpu_hv_synic *synic,
>  				int vector)
>  {
> -	if (vector < HV_SYNIC_FIRST_VALID_VECTOR)
> -		return;
> -
>  	if (synic_has_vector_connected(synic, vector))
>  		__set_bit(vector, synic->vec_bitmap);
>  	else
>  		__clear_bit(vector, synic->vec_bitmap);
>  
> +	if (vector < HV_SYNIC_FIRST_VALID_VECTOR)
> +		return;
> +
>  	if (synic_has_vector_auto_eoi(synic, vector))
>  		__set_bit(vector, synic->auto_eoi_bitmap);
>  	else
> @@ -202,6 +215,7 @@ static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint)
>  	for (idx = 0; idx < ARRAY_SIZE(hv_vcpu->stimer); idx++) {
>  		stimer = &hv_vcpu->stimer[idx];
>  		if (stimer->msg_pending && stimer->config.enable &&
> +		    !stimer->config.direct_mode &&
>  		    stimer->config.sintx == sint) {
>  			set_bit(stimer->index,
>  				hv_vcpu->stimer_pending_bitmap);
> @@ -371,14 +385,29 @@ int kvm_hv_synic_set_irq(struct kvm *kvm, u32 vpidx, u32 sint)
>  
>  void kvm_hv_synic_send_eoi(struct kvm_vcpu *vcpu, int vector)
>  {
> +	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
>  	struct kvm_vcpu_hv_synic *synic = vcpu_to_synic(vcpu);
> -	int i;
> +	struct kvm_vcpu_hv_stimer *stimer;
> +	int i, stimers_pending = 0;
>  
>  	trace_kvm_hv_synic_send_eoi(vcpu->vcpu_id, vector);
>  
>  	for (i = 0; i < ARRAY_SIZE(synic->sint); i++)
>  		if (synic_get_sint_vector(synic_read_sint(synic, i)) == vector)
>  			kvm_hv_notify_acked_sint(vcpu, i);
> +
> +	for (i = 0; i < ARRAY_SIZE(hv_vcpu->stimer); i++) {
> +		stimer = &hv_vcpu->stimer[i];
> +		if (stimer->msg_pending && stimer->config.enable &&
> +		    stimer->config.direct_mode &&
> +		    stimer->config.apic_vector == vector) {
> +			set_bit(stimer->index,
> +				hv_vcpu->stimer_pending_bitmap);
> +			stimers_pending++;
> +		}
> +	}
> +	if (stimers_pending)
> +		kvm_make_request(KVM_REQ_HV_STIMER, vcpu);

Hmm, I thought I had replied to this patch but I didn't.  You don't need
stimers_pending, neither here nor in kvm_hv_notify_acked_sint, if you
just call stimer_mark_pending.  The likelihood of having >1 timer EOI is
probably low enough that calling kvm_make_request more than once is not
an appreciable pessimization.

Otherwise looks good.

Paolo
>  }
>  
>  static int kvm_hv_set_sint_gsi(struct kvm *kvm, u32 vpidx, u32 sint, int gsi)
> @@ -545,15 +574,25 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer)
>  static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config,
>  			     bool host)
>  {
> -	union hv_stimer_config new_config = {.as_uint64 = config};
> +	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
> +	struct kvm_vcpu_hv *hv_vcpu = vcpu_to_hv_vcpu(vcpu);
> +	union hv_stimer_config new_config = {.as_uint64 = config},
> +		old_config = {.as_uint64 = stimer->config.as_uint64};
>  
>  	trace_kvm_hv_stimer_set_config(stimer_to_vcpu(stimer)->vcpu_id,
>  				       stimer->index, config, host);
>  
>  	stimer_cleanup(stimer);
> -	if (stimer->config.enable && new_config.sintx == 0)
> +	if (old_config.enable &&
> +	    !new_config.direct_mode && new_config.sintx == 0)
>  		new_config.enable = 0;
>  	stimer->config.as_uint64 = new_config.as_uint64;
> +
> +	if (old_config.direct_mode)
> +		synic_update_vector(&hv_vcpu->synic, old_config.apic_vector);
> +	if (new_config.direct_mode)
> +		synic_update_vector(&hv_vcpu->synic, new_config.apic_vector);
> +
>  	stimer_mark_pending(stimer, false);
>  	return 0;
>  }
> @@ -640,14 +679,28 @@ static int stimer_send_msg(struct kvm_vcpu_hv_stimer *stimer)
>  				 stimer->config.sintx, msg);
>  }
>  
> +static int stimer_notify_direct(struct kvm_vcpu_hv_stimer *stimer)
> +{
> +	struct kvm_vcpu *vcpu = stimer_to_vcpu(stimer);
> +	struct kvm_lapic_irq irq = {
> +		.delivery_mode = APIC_DM_FIXED,
> +		.vector = stimer->config.apic_vector
> +	};
> +
> +	return !kvm_apic_set_irq(vcpu, &irq, NULL);
> +}
> +
>  static void stimer_expiration(struct kvm_vcpu_hv_stimer *stimer)
>  {
> -	int r;
> +	int r, direct = stimer->config.direct_mode;
>  
>  	stimer->msg_pending = true;
> -	r = stimer_send_msg(stimer);
> +	if (!direct)
> +		r = stimer_send_msg(stimer);
> +	else
> +		r = stimer_notify_direct(stimer);
>  	trace_kvm_hv_stimer_expiration(stimer_to_vcpu(stimer)->vcpu_id,
> -				       stimer->index, r);
> +				       stimer->index, direct, r);
>  	if (!r) {
>  		stimer->msg_pending = false;
>  		if (!(stimer->config.periodic))
> diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
> index 0659465a745c..705f40ae2532 100644
> --- a/arch/x86/kvm/trace.h
> +++ b/arch/x86/kvm/trace.h
> @@ -1254,24 +1254,26 @@ TRACE_EVENT(kvm_hv_stimer_callback,
>   * Tracepoint for stimer_expiration.
>   */
>  TRACE_EVENT(kvm_hv_stimer_expiration,
> -	TP_PROTO(int vcpu_id, int timer_index, int msg_send_result),
> -	TP_ARGS(vcpu_id, timer_index, msg_send_result),
> +	TP_PROTO(int vcpu_id, int timer_index, int direct, int msg_send_result),
> +	TP_ARGS(vcpu_id, timer_index, direct, msg_send_result),
>  
>  	TP_STRUCT__entry(
>  		__field(int, vcpu_id)
>  		__field(int, timer_index)
> +		__field(int, direct)
>  		__field(int, msg_send_result)
>  	),
>  
>  	TP_fast_assign(
>  		__entry->vcpu_id = vcpu_id;
>  		__entry->timer_index = timer_index;
> +		__entry->direct = direct;
>  		__entry->msg_send_result = msg_send_result;
>  	),
>  
> -	TP_printk("vcpu_id %d timer %d msg send result %d",
> +	TP_printk("vcpu_id %d timer %d direct %d send result %d",
>  		  __entry->vcpu_id, __entry->timer_index,
> -		  __entry->msg_send_result)
> +		  __entry->direct, __entry->msg_send_result)
>  );
>  
>  /*
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 5cd5647120f2..b21b5ceb8d26 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -2997,6 +2997,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
>  	case KVM_CAP_HYPERV_TLBFLUSH:
>  	case KVM_CAP_HYPERV_SEND_IPI:
>  	case KVM_CAP_HYPERV_ENLIGHTENED_VMCS:
> +	case KVM_CAP_HYPERV_STIMER_DIRECT:
>  	case KVM_CAP_PCI_SEGMENT:
>  	case KVM_CAP_DEBUGREGS:
>  	case KVM_CAP_X86_ROBUST_SINGLESTEP:
> diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h
> index 2b7a652c9fa4..b8da14cee8e5 100644
> --- a/include/uapi/linux/kvm.h
> +++ b/include/uapi/linux/kvm.h
> @@ -975,6 +975,7 @@ struct kvm_ppc_resize_hpt {
>  #define KVM_CAP_HYPERV_ENLIGHTENED_VMCS 163
>  #define KVM_CAP_EXCEPTION_PAYLOAD 164
>  #define KVM_CAP_ARM_VM_IPA_SIZE 165
> +#define KVM_CAP_HYPERV_STIMER_DIRECT 166
>  
>  #ifdef KVM_CAP_IRQ_ROUTING
>  
>