[RFC Patch 0/3] KVM/x86/hyper-V: Introduce PV guest address space mapping flush support

[RFC Patch 0/3] KVM/x86/hyper-V: Introduce PV guest address space mapping flush support

[Tianyu Lan]

Hyper-V provides a para-virtualization hypercall HvFlushGuestPhysicalAddressSpace
to flush nested VM address space mapping in l1 hypervisor and it's to reduce overhead
of flushing ept tlb among vcpus. The tradition way is to send IPIs to all affected
vcpus and executes INVEPT on each vcpus. It will trigger several vmexits for IPI and
INVEPT emulation. The pv hypercall can help to flush specified ept table on all vcpus
via one single hypercall.

Lan Tianyu (3):
  X86/Hyper-V: Add flush HvFlushGuestPhysicalAddressSpace hypercall
    support
  KVM: Add tlb remote flush callback in kvm_x86_ops.
  KVM/x86: Add tlb_remote_flush callback support for vmcs

 arch/x86/hyperv/Makefile           |  2 +-
 arch/x86/hyperv/nested.c           | 64 ++++++++++++++++++++++++++++++++++++++
 arch/x86/include/asm/hyperv-tlfs.h |  8 +++++
 arch/x86/include/asm/kvm_host.h    |  1 +
 arch/x86/include/asm/mshyperv.h    |  2 ++
 arch/x86/kvm/vmx.c                 | 15 +++++++++
 virt/kvm/kvm_main.c                | 12 ++++++-
 7 files changed, 102 insertions(+), 2 deletions(-)
 create mode 100644 arch/x86/hyperv/nested.c

-- 
2.14.3

[RFC Patch 1/3] X86/Hyper-V: Add flush HvFlushGuestPhysicalAddressSpace hypercall support

[Tianyu Lan]

Hyper-V provides a pv hypercall HvFlushGuestPhysicalAddressSpace to flush
nested VM address space mapping in l1 hypervisor and it's to reduce overhead
of flushing ept tlb among vcpus. This patch is to implement it.

Signed-off-by: Lan Tianyu <Tianyu.Lan@microsoft.com>
---
 arch/x86/hyperv/Makefile           |  2 +-
 arch/x86/hyperv/nested.c           | 64 ++++++++++++++++++++++++++++++++++++++
 arch/x86/include/asm/hyperv-tlfs.h |  8 +++++
 arch/x86/include/asm/mshyperv.h    |  2 ++
 4 files changed, 75 insertions(+), 1 deletion(-)
 create mode 100644 arch/x86/hyperv/nested.c

diff --git a/arch/x86/hyperv/Makefile b/arch/x86/hyperv/Makefile
index b173d404e3df..b21ee65c4101 100644
--- a/arch/x86/hyperv/Makefile
+++ b/arch/x86/hyperv/Makefile
@@ -1,2 +1,2 @@
-obj-y			:= hv_init.o mmu.o
+obj-y			:= hv_init.o mmu.o nested.o
 obj-$(CONFIG_X86_64)	+= hv_apic.o
diff --git a/arch/x86/hyperv/nested.c b/arch/x86/hyperv/nested.c
new file mode 100644
index 000000000000..17f7c288eccc
--- /dev/null
+++ b/arch/x86/hyperv/nested.c
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Hyper-V nested virtualization code.
+ *
+ * Copyright (C) 2018, Microsoft, Inc.
+ *
+ * Author : Lan Tianyu <Tianyu.Lan@microsoft.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
+ * NON INFRINGEMENT.  See the GNU General Public License for more
+ * details.
+ *
+ */
+
+
+#include <linux/types.h>
+#include <asm/hyperv-tlfs.h>
+#include <asm/mshyperv.h>
+#include <asm/tlbflush.h>
+
+int hyperv_flush_guest_mapping(u64 as)
+{
+	struct hv_guest_mapping_flush **flush_pcpu;
+	struct hv_guest_mapping_flush *flush;
+	u64 status = U64_MAX;
+	unsigned long flags;
+	int ret = -EFAULT;
+
+	if (!hv_hypercall_pg)
+		goto fault;
+
+	local_irq_save(flags);
+
+	flush_pcpu = (struct hv_guest_mapping_flush **)
+		this_cpu_ptr(hyperv_pcpu_input_arg);
+
+	flush = *flush_pcpu;
+
+	if (unlikely(!flush)) {
+		local_irq_restore(flags);
+		goto fault;
+	}
+
+	flush->address_space = as;
+	flush->flags = 0;
+
+	status = hv_do_hypercall(HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE,
+				 flush, NULL);
+	local_irq_restore(flags);
+
+	if (!(status & HV_HYPERCALL_RESULT_MASK))
+		ret = 0;
+
+fault:
+	return ret;
+}
+EXPORT_SYMBOL_GPL(hyperv_flush_guest_mapping);
diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
index b8c89265baf0..53bbeb08faea 100644
--- a/arch/x86/include/asm/hyperv-tlfs.h
+++ b/arch/x86/include/asm/hyperv-tlfs.h
@@ -309,6 +309,7 @@ struct ms_hyperv_tsc_page {
 #define HV_X64_MSR_REENLIGHTENMENT_CONTROL	0x40000106
 
 /* Nested features (CPUID 0x4000000A) EAX */
+#define HV_X64_NESTED_GUSET_MAPPING_FLUSH	BIT(18)
 #define HV_X64_NESTED_MSR_BITMAP		BIT(19)
 
 struct hv_reenlightenment_control {
@@ -350,6 +351,7 @@ struct hv_tsc_emulation_status {
 #define HVCALL_SEND_IPI_EX			0x0015
 #define HVCALL_POST_MESSAGE			0x005c
 #define HVCALL_SIGNAL_EVENT			0x005d
+#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af
 
 #define HV_X64_MSR_VP_ASSIST_PAGE_ENABLE	0x00000001
 #define HV_X64_MSR_VP_ASSIST_PAGE_ADDRESS_SHIFT	12
@@ -741,6 +743,12 @@ struct ipi_arg_ex {
 	struct hv_vpset vp_set;
 };
 
+/* HvFlushGuestPhysicalAddressSpace hypercalls */
+struct hv_guest_mapping_flush {
+	u64 address_space;
+	u64 flags;
+};
+
 /* HvFlushVirtualAddressSpace, HvFlushVirtualAddressList hypercalls */
 struct hv_tlb_flush {
 	u64 address_space;
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index 3cd14311edfa..a6a615b49876 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -302,6 +302,7 @@ void hyperv_reenlightenment_intr(struct pt_regs *regs);
 void set_hv_tscchange_cb(void (*cb)(void));
 void clear_hv_tscchange_cb(void);
 void hyperv_stop_tsc_emulation(void);
+int hyperv_flush_guest_mapping(u64 as);
 
 #ifdef CONFIG_X86_64
 void hv_apic_init(void);
@@ -321,6 +322,7 @@ static inline struct hv_vp_assist_page *hv_get_vp_assist_page(unsigned int cpu)
 {
 	return NULL;
 }
+static inline int hyperv_flush_guest_mapping(u64 as) { return -1; }
 #endif /* CONFIG_HYPERV */
 
 #ifdef CONFIG_HYPERV_TSCPAGE
-- 
2.14.3

[RFC Patch 2/3] KVM: Add tlb remote flush callback in kvm_x86_ops.

[Tianyu Lan]

This patch is to provide a way for platforms to register tlb remote flush
callback and this helps to optimize operation of tlb flush among vcpus for
nested virtualization case.

Signed-off-by: Lan Tianyu <Tianyu.Lan@microsoft.com>
---
 arch/x86/include/asm/kvm_host.h |  1 +
 virt/kvm/kvm_main.c             | 12 +++++++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 0ebe659f2802..aaad43916f9d 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -973,6 +973,7 @@ struct kvm_x86_ops {
 	void (*set_rflags)(struct kvm_vcpu *vcpu, unsigned long rflags);
 
 	void (*tlb_flush)(struct kvm_vcpu *vcpu, bool invalidate_gpa);
+	int  (*tlb_remote_flush)(struct kvm *kvm);
 
 	void (*run)(struct kvm_vcpu *vcpu);
 	int (*handle_exit)(struct kvm_vcpu *vcpu);
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index c5f6a552e486..31e2b54c6049 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -251,11 +251,21 @@ bool kvm_make_all_cpus_request(struct kvm *kvm, unsigned int req)
 #ifndef CONFIG_HAVE_KVM_ARCH_TLB_FLUSH_ALL
 void kvm_flush_remote_tlbs(struct kvm *kvm)
 {
+	long dirty_count;
+
+	/*
+	 * Call tlb_remote_flush first and go back old way when
+	 * return failure.
+	 */
+	if (kvm_x86_ops->tlb_remote_flush &&
+	    !kvm_x86_ops->tlb_remote_flush(kvm))
+		return;
+
 	/*
 	 * Read tlbs_dirty before setting KVM_REQ_TLB_FLUSH in
 	 * kvm_make_all_cpus_request.
 	 */
-	long dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
+	dirty_count = smp_load_acquire(&kvm->tlbs_dirty);
 
 	/*
 	 * We want to publish modifications to the page tables before reading
-- 
2.14.3

[RFC Patch 3/3] KVM/x86: Add tlb_remote_flush callback support for vmcs

[Tianyu Lan]

Register tlb_remote_flush callback for vmcs when hyperv capability of
nested guest mapping flush is detected. The interface can help to reduce
overhead when flush ept table among vcpus for nested VM. The tradition way
is to send IPIs to all affected vcpus and executes INVEPT on each vcpus.
It will trigger several vmexits for IPI and INVEPT emulation. Hyperv provides
such hypercall to do flush for all vcpus.

Signed-off-by: Lan Tianyu <Tianyu.Lan@microsoft.com>
---
 arch/x86/kvm/vmx.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index e50beb76d846..6cb241c05690 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -4737,6 +4737,17 @@ static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid,
 	}
 }
 
+static int vmx_remote_flush_tlb(struct kvm *kvm)
+{
+	struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, 0);
+
+	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
+		return -1;
+
+	return hyperv_flush_guest_mapping(construct_eptp(vcpu,
+		vcpu->arch.mmu.root_hpa));
+}
+
 static void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
 {
 	__vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid, invalidate_gpa);
@@ -7495,6 +7506,10 @@ static __init int hardware_setup(void)
 	if (enable_ept && !cpu_has_vmx_ept_2m_page())
 		kvm_disable_largepages();
 
+	if (ms_hyperv.nested_features & HV_X64_NESTED_GUSET_MAPPING_FLUSH
+	    && enable_ept)
+		kvm_x86_ops->tlb_remote_flush = vmx_remote_flush_tlb;
+
 	if (!cpu_has_vmx_ple()) {
 		ple_gap = 0;
 		ple_window = 0;
-- 
2.14.3

RE: [RFC Patch 1/3] X86/Hyper-V: Add flush HvFlushGuestPhysicalAddressSpace hypercall support

[Michael Kelley (EOSG)]

> -----Original Message-----
> From: linux-kernel-owner@vger.kernel.org <linux-kernel-owner@vger.kernel.org> On Behalf
> Of Tianyu Lan
> Sent: Monday, June 4, 2018 2:08 AM
> Cc: Tianyu Lan <Tianyu.Lan@microsoft.com>; KY Srinivasan <kys@microsoft.com>; Haiyang
> Zhang <haiyangz@microsoft.com>; Stephen Hemminger <sthemmin@microsoft.com>;
> tglx@linutronix.de; mingo@redhat.com; hpa@zytor.com; x86@kernel.org;
> pbonzini@redhat.com; rkrcmar@redhat.com; devel@linuxdriverproject.org; linux-
> kernel@vger.kernel.org; kvm@vger.kernel.org; vkuznets@redhat.com
> Subject: [RFC Patch 1/3] X86/Hyper-V: Add flush HvFlushGuestPhysicalAddressSpace hypercall
> support
> 
> Hyper-V provides a pv hypercall HvFlushGuestPhysicalAddressSpace to flush
> nested VM address space mapping in l1 hypervisor and it's to reduce overhead
> of flushing ept tlb among vcpus. This patch is to implement it.
> 
> Signed-off-by: Lan Tianyu <Tianyu.Lan@microsoft.com>
> ---
> diff --git a/arch/x86/hyperv/nested.c b/arch/x86/hyperv/nested.c
> new file mode 100644
> index 000000000000..17f7c288eccc
> --- /dev/null
> +++ b/arch/x86/hyperv/nested.c
> +#include <linux/types.h>
> +#include <asm/hyperv-tlfs.h>
> +#include <asm/mshyperv.h>
> +#include <asm/tlbflush.h>
> +
> +int hyperv_flush_guest_mapping(u64 as)
> +{
> +	struct hv_guest_mapping_flush **flush_pcpu;
> +	struct hv_guest_mapping_flush *flush;
> +	u64 status = U64_MAX;

Initializing status to U64_MAX doesn't seem necessary.

> +	unsigned long flags;
> +	int ret = -EFAULT;
> +
> +	if (!hv_hypercall_pg)
> +		goto fault;
> +
> +	local_irq_save(flags);
> +
> +	flush_pcpu = (struct hv_guest_mapping_flush **)
> +		this_cpu_ptr(hyperv_pcpu_input_arg);
> +
> +	flush = *flush_pcpu;
> +
> +	if (unlikely(!flush)) {
> +		local_irq_restore(flags);
> +		goto fault;
> +	}
> +
> +	flush->address_space = as;
> +	flush->flags = 0;
> +
> +	status = hv_do_hypercall(HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE,
> +				 flush, NULL);
> +	local_irq_restore(flags);
> +
> +	if (!(status & HV_HYPERCALL_RESULT_MASK))
> +		ret = 0;
> +
> +fault:
> +	return ret;
> +}
> +EXPORT_SYMBOL_GPL(hyperv_flush_guest_mapping);
> diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
> index b8c89265baf0..53bbeb08faea 100644
> --- a/arch/x86/include/asm/hyperv-tlfs.h
> +++ b/arch/x86/include/asm/hyperv-tlfs.h
> @@ -309,6 +309,7 @@ struct ms_hyperv_tsc_page {
>  #define HV_X64_MSR_REENLIGHTENMENT_CONTROL	0x40000106
> 
>  /* Nested features (CPUID 0x4000000A) EAX */
> +#define HV_X64_NESTED_GUSET_MAPPING_FLUSH	BIT(18)

The #define name is misspelled.  "_GUSET_" should be "_GUEST_".
And the matching usage in patch 3/3 will need to be updated as well.

Michael

Re: [RFC Patch 1/3] X86/Hyper-V: Add flush HvFlushGuestPhysicalAddressSpace hypercall support

[Tianyu Lan]

Hi Michael:
	Thanks for your review.

On 6/6/2018 12:59 AM, Michael Kelley (EOSG) wrote:
>> -----Original Message-----
>> From: linux-kernel-owner@vger.kernel.org <linux-kernel-owner@vger.kernel.org> On Behalf
>> Of Tianyu Lan
>> Sent: Monday, June 4, 2018 2:08 AM
>> Cc: Tianyu Lan <Tianyu.Lan@microsoft.com>; KY Srinivasan <kys@microsoft.com>; Haiyang
>> Zhang <haiyangz@microsoft.com>; Stephen Hemminger <sthemmin@microsoft.com>;
>> tglx@linutronix.de; mingo@redhat.com; hpa@zytor.com; x86@kernel.org;
>> pbonzini@redhat.com; rkrcmar@redhat.com; devel@linuxdriverproject.org; linux-
>> kernel@vger.kernel.org; kvm@vger.kernel.org; vkuznets@redhat.com
>> Subject: [RFC Patch 1/3] X86/Hyper-V: Add flush HvFlushGuestPhysicalAddressSpace hypercall
>> support
>>
>> Hyper-V provides a pv hypercall HvFlushGuestPhysicalAddressSpace to flush
>> nested VM address space mapping in l1 hypervisor and it's to reduce overhead
>> of flushing ept tlb among vcpus. This patch is to implement it.
>>
>> Signed-off-by: Lan Tianyu <Tianyu.Lan@microsoft.com>
>> ---
>> diff --git a/arch/x86/hyperv/nested.c b/arch/x86/hyperv/nested.c
>> new file mode 100644
>> index 000000000000..17f7c288eccc
>> --- /dev/null
>> +++ b/arch/x86/hyperv/nested.c
>> +#include <linux/types.h>
>> +#include <asm/hyperv-tlfs.h>
>> +#include <asm/mshyperv.h>
>> +#include <asm/tlbflush.h>
>> +
>> +int hyperv_flush_guest_mapping(u64 as)
>> +{
>> +	struct hv_guest_mapping_flush **flush_pcpu;
>> +	struct hv_guest_mapping_flush *flush;
>> +	u64 status = U64_MAX;
> 
> Initializing status to U64_MAX doesn't seem necessary.
> 
>> +	unsigned long flags;
>> +	int ret = -EFAULT;
>> +
>> +	if (!hv_hypercall_pg)
>> +		goto fault;
>> +
>> +	local_irq_save(flags);
>> +
>> +	flush_pcpu = (struct hv_guest_mapping_flush **)
>> +		this_cpu_ptr(hyperv_pcpu_input_arg);
>> +
>> +	flush = *flush_pcpu;
>> +
>> +	if (unlikely(!flush)) {
>> +		local_irq_restore(flags);
>> +		goto fault;
>> +	}
>> +
>> +	flush->address_space = as;
>> +	flush->flags = 0;
>> +
>> +	status = hv_do_hypercall(HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE,
>> +				 flush, NULL);
>> +	local_irq_restore(flags);
>> +
>> +	if (!(status & HV_HYPERCALL_RESULT_MASK))
>> +		ret = 0;
>> +
>> +fault:
>> +	return ret;
>> +}
>> +EXPORT_SYMBOL_GPL(hyperv_flush_guest_mapping);
>> diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h
>> index b8c89265baf0..53bbeb08faea 100644
>> --- a/arch/x86/include/asm/hyperv-tlfs.h
>> +++ b/arch/x86/include/asm/hyperv-tlfs.h
>> @@ -309,6 +309,7 @@ struct ms_hyperv_tsc_page {
>>   #define HV_X64_MSR_REENLIGHTENMENT_CONTROL	0x40000106
>>
>>   /* Nested features (CPUID 0x4000000A) EAX */
>> +#define HV_X64_NESTED_GUSET_MAPPING_FLUSH	BIT(18)
> 
> The #define name is misspelled.  "_GUSET_" should be "_GUEST_".
> And the matching usage in patch 3/3 will need to be updated as well.
> 
> Michael
> 

Nice catch! Will update.

Re: [RFC Patch 3/3] KVM/x86: Add tlb_remote_flush callback support for vmcs

[Vitaly Kuznetsov]

Tianyu Lan <Tianyu.Lan@microsoft.com> writes:

> Register tlb_remote_flush callback for vmcs when hyperv capability of
> nested guest mapping flush is detected. The interface can help to reduce
> overhead when flush ept table among vcpus for nested VM. The tradition way
> is to send IPIs to all affected vcpus and executes INVEPT on each vcpus.
> It will trigger several vmexits for IPI and INVEPT emulation. Hyperv provides
> such hypercall to do flush for all vcpus.
>
> Signed-off-by: Lan Tianyu <Tianyu.Lan@microsoft.com>
> ---
>  arch/x86/kvm/vmx.c | 15 +++++++++++++++
>  1 file changed, 15 insertions(+)
>
> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
> index e50beb76d846..6cb241c05690 100644
> --- a/arch/x86/kvm/vmx.c
> +++ b/arch/x86/kvm/vmx.c
> @@ -4737,6 +4737,17 @@ static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid,
>  	}
>  }
>
> +static int vmx_remote_flush_tlb(struct kvm *kvm)
> +{
> +	struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, 0);
> +
> +	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
> +		return -1;
> +
> +	return hyperv_flush_guest_mapping(construct_eptp(vcpu,
> +		vcpu->arch.mmu.root_hpa));
> +}
> +
>  static void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
>  {
>  	__vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid, invalidate_gpa);
> @@ -7495,6 +7506,10 @@ static __init int hardware_setup(void)
>  	if (enable_ept && !cpu_has_vmx_ept_2m_page())
>  		kvm_disable_largepages();
>
> +	if (ms_hyperv.nested_features & HV_X64_NESTED_GUSET_MAPPING_FLUSH
> +	    && enable_ept)
> +		kvm_x86_ops->tlb_remote_flush = vmx_remote_flush_tlb;
> +

You'll need to surround the above with #if IS_ENABLED(CONFIG_HYPERV) as
ms_hyperv is otherwise unavailable.

>  	if (!cpu_has_vmx_ple()) {
>  		ple_gap = 0;
>  		ple_window = 0;

-- 
  Vitaly

Re: [RFC Patch 3/3] KVM/x86: Add tlb_remote_flush callback support for vmcs

[Vitaly Kuznetsov]

Tianyu Lan <Tianyu.Lan@microsoft.com> writes:

> Register tlb_remote_flush callback for vmcs when hyperv capability of
> nested guest mapping flush is detected. The interface can help to reduce
> overhead when flush ept table among vcpus for nested VM. The tradition way
> is to send IPIs to all affected vcpus and executes INVEPT on each vcpus.
> It will trigger several vmexits for IPI and INVEPT emulation. Hyperv provides
> such hypercall to do flush for all vcpus.
>
> Signed-off-by: Lan Tianyu <Tianyu.Lan@microsoft.com>
> ---
>  arch/x86/kvm/vmx.c | 15 +++++++++++++++
>  1 file changed, 15 insertions(+)
>
> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
> index e50beb76d846..6cb241c05690 100644
> --- a/arch/x86/kvm/vmx.c
> +++ b/arch/x86/kvm/vmx.c
> @@ -4737,6 +4737,17 @@ static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid,
>  	}
>  }
>
> +static int vmx_remote_flush_tlb(struct kvm *kvm)
> +{
> +	struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, 0);
> +
> +	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
> +		return -1;

Why vcpu0? Can arch.mmu.root_hpa-s differ across vCPUs? What happens if
they do?

> +
> +	return hyperv_flush_guest_mapping(construct_eptp(vcpu,
> +		vcpu->arch.mmu.root_hpa));
> +}

The 'vmx_remote_flush_tlb' name looks generic enough but it is actually
Hyper-V-specific. I'd suggest renaming to something like
hv_remote_flush_tlb().

> +
>  static void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
>  {
>  	__vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid, invalidate_gpa);
> @@ -7495,6 +7506,10 @@ static __init int hardware_setup(void)
>  	if (enable_ept && !cpu_has_vmx_ept_2m_page())
>  		kvm_disable_largepages();
>
> +	if (ms_hyperv.nested_features & HV_X64_NESTED_GUSET_MAPPING_FLUSH
> +	    && enable_ept)
> +		kvm_x86_ops->tlb_remote_flush = vmx_remote_flush_tlb;
> +
>  	if (!cpu_has_vmx_ple()) {
>  		ple_gap = 0;
>  		ple_window = 0;

-- 
  Vitaly

Re: [RFC Patch 3/3] KVM/x86: Add tlb_remote_flush callback support for vmcs

[Tianyu Lan]

Hi Vitaly:
	Thanks for your review.

On 6/12/2018 11:12 PM, Vitaly Kuznetsov wrote:
> Tianyu Lan <Tianyu.Lan@microsoft.com> writes:
> 
>> Register tlb_remote_flush callback for vmcs when hyperv capability of
>> nested guest mapping flush is detected. The interface can help to reduce
>> overhead when flush ept table among vcpus for nested VM. The tradition way
>> is to send IPIs to all affected vcpus and executes INVEPT on each vcpus.
>> It will trigger several vmexits for IPI and INVEPT emulation. Hyperv provides
>> such hypercall to do flush for all vcpus.
>>
>> Signed-off-by: Lan Tianyu <Tianyu.Lan@microsoft.com>
>> ---
>>   arch/x86/kvm/vmx.c | 15 +++++++++++++++
>>   1 file changed, 15 insertions(+)
>>
>> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
>> index e50beb76d846..6cb241c05690 100644
>> --- a/arch/x86/kvm/vmx.c
>> +++ b/arch/x86/kvm/vmx.c
>> @@ -4737,6 +4737,17 @@ static inline void __vmx_flush_tlb(struct kvm_vcpu *vcpu, int vpid,
>>   	}
>>   }
>>
>> +static int vmx_remote_flush_tlb(struct kvm *kvm)
>> +{
>> +	struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, 0);
>> +
>> +	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
>> +		return -1;
> 
> Why vcpu0? Can arch.mmu.root_hpa-s differ across vCPUs? What happens if
> they do?

Yes, it may take place that arch.mmu.root_hpa is differ across vCPUs.
We may check all vcpu root_hpa and use the hypercall when there is only 
one validated ept table. If not, go back to current way.

static int hv_remote_flush_tlb()
{
         struct kvm_vcpu *vcpu;
         u64 root_hpa = INVALID_PAGE;
         int i;

         kvm_for_each_vcpu(i, vcpu, kvm) {
                 if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
                         continue;

                 if (!VALID_PAGE(root_hpa))
                         root_hpa = vcpu->arch.mmu.root_hpa;
                 else if (root_hpa != vcpu->arch.mmu.root_hpa) {
                         return -1;
                 }
         }

         if (!VALID_PAGE(root_hpa))
                 return -1;

         return hyperv_flush_guest_mapping(construct_eptp(vcpu,
                 root_hpa));
}


> 
>> +
>> +	return hyperv_flush_guest_mapping(construct_eptp(vcpu,
>> +		vcpu->arch.mmu.root_hpa));
>> +}
> 
> The 'vmx_remote_flush_tlb' name looks generic enough but it is actually
> Hyper-V-specific. I'd suggest renaming to something like
> hv_remote_flush_tlb().
> 
>> +
>>   static void vmx_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
>>   {
>>   	__vmx_flush_tlb(vcpu, to_vmx(vcpu)->vpid, invalidate_gpa);
>> @@ -7495,6 +7506,10 @@ static __init int hardware_setup(void)
>>   	if (enable_ept && !cpu_has_vmx_ept_2m_page())
>>   		kvm_disable_largepages();
>>
>> +	if (ms_hyperv.nested_features & HV_X64_NESTED_GUSET_MAPPING_FLUSH
>> +	    && enable_ept)
>> +		kvm_x86_ops->tlb_remote_flush = vmx_remote_flush_tlb;
>> +
>>   	if (!cpu_has_vmx_ple()) {
>>   		ple_gap = 0;
>>   		ple_window = 0;
> 

Re: [RFC Patch 3/3] KVM/x86: Add tlb_remote_flush callback support for vmcs

[Vitaly Kuznetsov]

Tianyu Lan <Tianyu.Lan@microsoft.com> writes:

> On 6/12/2018 11:12 PM, Vitaly Kuznetsov wrote:
>> Tianyu Lan <Tianyu.Lan@microsoft.com> writes:
>> 
>>>
>>> +static int vmx_remote_flush_tlb(struct kvm *kvm)
>>> +{
>>> +	struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, 0);
>>> +
>>> +	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
>>> +		return -1;
>> 
>> Why vcpu0? Can arch.mmu.root_hpa-s differ across vCPUs? What happens if
>> they do?
>
> Yes, it may take place that arch.mmu.root_hpa is differ across vCPUs.
> We may check all vcpu root_hpa and use the hypercall when there is only 
> one validated ept table. If not, go back to current way.
>

I'd suggest an optimization: keep track of wether root_hpas are equal
across all vcpus (check this on change).

-- 
  Vitaly

Re: [RFC Patch 3/3] KVM/x86: Add tlb_remote_flush callback support for vmcs

[Tianyu Lan]

On 6/14/2018 6:33 PM, Vitaly Kuznetsov wrote:
> Tianyu Lan <Tianyu.Lan@microsoft.com> writes:
> 
>> On 6/12/2018 11:12 PM, Vitaly Kuznetsov wrote:
>>> Tianyu Lan <Tianyu.Lan@microsoft.com> writes:
>>>
>>>>
>>>> +static int vmx_remote_flush_tlb(struct kvm *kvm)
>>>> +{
>>>> +	struct kvm_vcpu *vcpu = kvm_get_vcpu(kvm, 0);
>>>> +
>>>> +	if (!VALID_PAGE(vcpu->arch.mmu.root_hpa))
>>>> +		return -1;
>>>
>>> Why vcpu0? Can arch.mmu.root_hpa-s differ across vCPUs? What happens if
>>> they do?
>>
>> Yes, it may take place that arch.mmu.root_hpa is differ across vCPUs.
>> We may check all vcpu root_hpa and use the hypercall when there is only
>> one validated ept table. If not, go back to current way.
>>
> 
> I'd suggest an optimization: keep track of wether root_hpas are equal
> across all vcpus (check this on change).
> 

Good suggestion. I will try it. Thanks.