[PATCH 0/4] KVM: x86: Let the guest write to multiple debug registers with one vmexit

[PATCH 0/4] KVM: x86: Let the guest write to multiple debug registers with one vmexit

[Paolo Bonzini]

Alex Williamson reported that a Windows game does something weird that
makes the guest save and restore debug registers on each context switch.
This cause several hundred thousands vmexits per second, and basically
cuts performance in half when running under KVM.

However, when not running in guest-debug mode, the guest controls the
debug registers and having to take an exit for each DR access is a waste
of time.  We just need one vmexit to load any stale values of DR0-DR6,
and then we can let the guest run freely.  On the next vmexit (whatever
the reason) we will read out whatever changes the guest made to the
debug registers.

On top of this, we can implement SVM support and let nested guests run
with dirty debug registers too.

Paolo Bonzini (4):
  KVM: vmx: we do rely on loading DR7 on entry
  KVM: x86: change vcpu->arch.switch_db_regs to a bit mask
  KVM: x86: Allow the guest to run with dirty debug registers
  KVM: vmx: Allow the guest to run with dirty debug registers

 arch/x86/include/asm/kvm_host.h |  8 ++++++-
 arch/x86/kvm/vmx.c              | 48 +++++++++++++++++++++++++++++++++++++----
 arch/x86/kvm/x86.c              | 29 +++++++++++++++++++++++--
 3 files changed, 78 insertions(+), 7 deletions(-)

-- 
1.8.3.1

[PATCH 1/4] KVM: vmx: we do rely on loading DR7 on entry

[Paolo Bonzini]

Currently, this works even if the bit is not in "min", because the bit is always
set in MSR_IA32_VMX_ENTRY_CTLS.  Mention it for the sake of documentation, and
to avoid surprises if we later switch to MSR_IA32_VMX_TRUE_ENTRY_CTLS.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/kvm/vmx.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index daca80f0eae2..6e57e1434cf3 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -2872,7 +2872,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 		!(_vmexit_control & VM_EXIT_ACK_INTR_ON_EXIT))
 		_pin_based_exec_control &= ~PIN_BASED_POSTED_INTR;
 
-	min = 0;
+	min = VM_ENTRY_LOAD_DEBUG_CONTROLS;
 	opt = VM_ENTRY_LOAD_IA32_PAT | VM_ENTRY_LOAD_BNDCFGS;
 	if (adjust_vmx_controls(min, opt, MSR_IA32_VMX_ENTRY_CTLS,
 				&_vmentry_control) < 0)
-- 
1.8.3.1

[PATCH 2/4] KVM: x86: change vcpu->arch.switch_db_regs to a bit mask

[Paolo Bonzini]

The next patch will add another bit that we can test with the
same "if".

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/include/asm/kvm_host.h | 6 +++++-
 arch/x86/kvm/x86.c              | 6 ++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index e714f8c08ccf..6b3b067f4780 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -337,6 +337,10 @@ struct kvm_pmu {
 	u64 reprogram_pmi;
 };
 
+enum {
+	KVM_DEBUGREG_BP_ENABLED = 1,
+};
+
 struct kvm_vcpu_arch {
 	/*
 	 * rip and regs accesses must go through
@@ -463,7 +467,7 @@ struct kvm_vcpu_arch {
 	struct mtrr_state_type mtrr_state;
 	u32 pat;
 
-	int switch_db_regs;
+	unsigned switch_db_regs;
 	unsigned long db[KVM_NR_DB_REGS];
 	unsigned long dr6;
 	unsigned long dr7;
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 109985016389..c01dddce6df0 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -759,7 +759,9 @@ static void kvm_update_dr7(struct kvm_vcpu *vcpu)
 	else
 		dr7 = vcpu->arch.dr7;
 	kvm_x86_ops->set_dr7(vcpu, dr7);
-	vcpu->arch.switch_db_regs = (dr7 & DR7_BP_EN_MASK);
+	vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_BP_ENABLED;
+	if (dr7 & DR7_BP_EN_MASK)
+		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_BP_ENABLED;
 }
 
 static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
-- 
1.8.3.1

[PATCH 3/4] KVM: x86: Allow the guest to run with dirty debug registers

[Paolo Bonzini]

When not running in guest-debug mode, the guest controls the debug
registers and having to take an exit for each DR access is a waste
of time.  If the guest gets into a state where each context switch
causes DR to be saved and restored, this can take away as much as 40%
of the execution time from the guest.

After this patch, VMX- and SVM-specific code can set a flag in
switch_db_regs, telling vcpu_enter_guest that on the next exit the debug
registers might be dirty and need to be reloaded.  This flag can be set
on the first access to a debug registers, so that multiple accesses to
the debug registers only cause one vmexit.

Note that since the guest will be able to read debug registers and
enable breakpoints in DR7, we need to ensure that they are synchronized
on entry to the guest---including DR6 that was not synced before.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/include/asm/kvm_host.h |  2 ++
 arch/x86/kvm/x86.c              | 25 ++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 6b3b067f4780..f4b34e071d14 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -339,6 +339,7 @@ struct kvm_pmu {
 
 enum {
 	KVM_DEBUGREG_BP_ENABLED = 1,
+	KVM_DEBUGREG_WONT_EXIT = 2,
 };
 
 struct kvm_vcpu_arch {
@@ -705,6 +706,7 @@ struct kvm_x86_ops {
 	void (*set_gdt)(struct kvm_vcpu *vcpu, struct desc_ptr *dt);
 	u64 (*get_dr6)(struct kvm_vcpu *vcpu);
 	void (*set_dr6)(struct kvm_vcpu *vcpu, unsigned long value);
+	u64 (*get_dr7)(struct kvm_vcpu *vcpu);
 	void (*set_dr7)(struct kvm_vcpu *vcpu, unsigned long value);
 	void (*cache_reg)(struct kvm_vcpu *vcpu, enum kvm_reg reg);
 	unsigned long (*get_rflags)(struct kvm_vcpu *vcpu);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index c01dddce6df0..b4b802cad8fc 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -5998,12 +5998,35 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 		set_debugreg(vcpu->arch.eff_db[1], 1);
 		set_debugreg(vcpu->arch.eff_db[2], 2);
 		set_debugreg(vcpu->arch.eff_db[3], 3);
+		set_debugreg(vcpu->arch.dr6, 6);
 	}
 
 	trace_kvm_entry(vcpu->vcpu_id);
 	kvm_x86_ops->run(vcpu);
 
 	/*
+	 * Do this here before restoring debug registers on the host.  And
+	 * since we do this before handling the vmexit, a DR access vmexit
+	 * can (a) read the correct value of the debug registers, (b) set
+	 * KVM_DEBUGREG_WONT_EXIT again.
+	 */
+	if (unlikely(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT)) {
+		int i;
+
+		get_debugreg(vcpu->arch.db[0], 0);
+		get_debugreg(vcpu->arch.db[1], 1);
+		get_debugreg(vcpu->arch.db[2], 2);
+		get_debugreg(vcpu->arch.db[3], 3);
+		get_debugreg(vcpu->arch.dr6, 6);
+		vcpu->arch.dr7 = kvm_x86_ops->get_dr7(vcpu);
+		vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
+
+		WARN_ON(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP);
+		for (i = 0; i < KVM_NR_DB_REGS; i++)
+			vcpu->arch.eff_db[i] = vcpu->arch.db[i];
+	}
+
+	/*
 	 * If the guest has used debug registers, at least dr7
 	 * will be disabled while returning to the host.
 	 * If we don't have active breakpoints in the host, we don't
-- 
1.8.3.1

[PATCH 4/4] KVM: vmx: Allow the guest to run with dirty debug registers

[Paolo Bonzini]

When not running in guest-debug mode (i.e. the guest controls the debug
registers, having to take an exit for each DR access is a waste of time.
If the guest gets into a state where each context switch causes DR to be
saved and restored, this can take away as much as 40% of the execution
time from the guest.

If the guest is running with vcpu->arch.db == vcpu->arch.eff_db, we
can let it write freely to the debug registers and reload them on the
next exit.  We still need to exit on the first access, so that the
KVM_DEBUGREG_WONT_EXIT flag is set in switch_db_regs; after that, further
accesses to the debug registers will not cause a vmexit.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/kvm/vmx.c | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 6e57e1434cf3..71c57ec48d8f 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -2851,7 +2851,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
 		      vmx_capability.ept, vmx_capability.vpid);
 	}
 
-	min = 0;
+	min = VM_EXIT_SAVE_DEBUG_CONTROLS;
 #ifdef CONFIG_X86_64
 	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
 #endif
@@ -5121,6 +5121,22 @@ static int handle_dr(struct kvm_vcpu *vcpu)
 		}
 	}
 
+	if (vcpu->guest_debug == 0) {
+		u32 cpu_based_vm_exec_control;
+
+		cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
+		cpu_based_vm_exec_control &= ~CPU_BASED_MOV_DR_EXITING;
+		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
+
+		/*
+		 * No more DR vmexits; force a reload of the debug registers
+		 * and reenter on this instruction.  The next vmexit will
+		 * retrieve the full state of the debug registers.
+		 */
+		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
+		return 1;
+	}
+
 	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
 	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
 	reg = DEBUG_REG_ACCESS_REG(exit_qualification);
@@ -5147,6 +5163,18 @@ static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
 {
 }
 
+static u64 vmx_get_dr7(struct kvm_vcpu *vcpu)
+{
+	/* DRs are being synced back to vcpu->arch, exit on DR access.  */
+	u32 cpu_based_vm_exec_control;
+
+	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
+	cpu_based_vm_exec_control |= CPU_BASED_MOV_DR_EXITING;
+	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
+
+	return vmcs_readl(GUEST_DR7);
+}
+
 static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
 {
 	vmcs_writel(GUEST_DR7, val);
@@ -8606,6 +8634,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
 	.set_gdt = vmx_set_gdt,
 	.get_dr6 = vmx_get_dr6,
 	.set_dr6 = vmx_set_dr6,
+	.get_dr7 = vmx_get_dr7,
 	.set_dr7 = vmx_set_dr7,
 	.cache_reg = vmx_cache_reg,
 	.get_rflags = vmx_get_rflags,
-- 
1.8.3.1

Re: [PATCH 4/4] KVM: vmx: Allow the guest to run with dirty debug registers

[Jan Kiszka]

On 2014-02-26 16:49, Paolo Bonzini wrote:
> When not running in guest-debug mode (i.e. the guest controls the debug
> registers, having to take an exit for each DR access is a waste of time.
> If the guest gets into a state where each context switch causes DR to be
> saved and restored, this can take away as much as 40% of the execution
> time from the guest.
> 
> If the guest is running with vcpu->arch.db == vcpu->arch.eff_db, we
> can let it write freely to the debug registers and reload them on the
> next exit.  We still need to exit on the first access, so that the
> KVM_DEBUGREG_WONT_EXIT flag is set in switch_db_regs; after that, further
> accesses to the debug registers will not cause a vmexit.
> 
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> ---
>  arch/x86/kvm/vmx.c | 31 ++++++++++++++++++++++++++++++-
>  1 file changed, 30 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
> index 6e57e1434cf3..71c57ec48d8f 100644
> --- a/arch/x86/kvm/vmx.c
> +++ b/arch/x86/kvm/vmx.c
> @@ -2851,7 +2851,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
>  		      vmx_capability.ept, vmx_capability.vpid);
>  	}
>  
> -	min = 0;
> +	min = VM_EXIT_SAVE_DEBUG_CONTROLS;
>  #ifdef CONFIG_X86_64
>  	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
>  #endif
> @@ -5121,6 +5121,22 @@ static int handle_dr(struct kvm_vcpu *vcpu)
>  		}
>  	}
>  
> +	if (vcpu->guest_debug == 0) {
> +		u32 cpu_based_vm_exec_control;
> +
> +		cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
> +		cpu_based_vm_exec_control &= ~CPU_BASED_MOV_DR_EXITING;
> +		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
> +
> +		/*
> +		 * No more DR vmexits; force a reload of the debug registers
> +		 * and reenter on this instruction.  The next vmexit will
> +		 * retrieve the full state of the debug registers.
> +		 */
> +		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
> +		return 1;
> +	}
> +
>  	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
>  	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
>  	reg = DEBUG_REG_ACCESS_REG(exit_qualification);
> @@ -5147,6 +5163,18 @@ static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
>  {
>  }
>  
> +static u64 vmx_get_dr7(struct kvm_vcpu *vcpu)
> +{
> +	/* DRs are being synced back to vcpu->arch, exit on DR access.  */
> +	u32 cpu_based_vm_exec_control;
> +
> +	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
> +	cpu_based_vm_exec_control |= CPU_BASED_MOV_DR_EXITING;
> +	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
> +
> +	return vmcs_readl(GUEST_DR7);
> +}

The general idea looks ok (It passes x86/debug.flat unit test, right?).
But this side effect of get_dr7 seems a bit ugly to me. Also the
imbalanced updates of arch.switch_db_regs: KVM_DEBUGREG_WONT_EXIT is set
by the vendor code but cleared in a common x86 path. Can't you make this
more regular and explicit?

Jan

> +
>  static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
>  {
>  	vmcs_writel(GUEST_DR7, val);
> @@ -8606,6 +8634,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
>  	.set_gdt = vmx_set_gdt,
>  	.get_dr6 = vmx_get_dr6,
>  	.set_dr6 = vmx_set_dr6,
> +	.get_dr7 = vmx_get_dr7,
>  	.set_dr7 = vmx_set_dr7,
>  	.cache_reg = vmx_cache_reg,
>  	.get_rflags = vmx_get_rflags,
> 

-- 
Siemens AG, Corporate Technology, CT RTC ITP SES-DE
Corporate Competence Center Embedded Linux

Re: [PATCH 4/4] KVM: vmx: Allow the guest to run with dirty debug registers

[Paolo Bonzini]

Il 27/02/2014 12:25, Jan Kiszka ha scritto:
> On 2014-02-26 16:49, Paolo Bonzini wrote:
>> When not running in guest-debug mode (i.e. the guest controls the debug
>> registers, having to take an exit for each DR access is a waste of time.
>> If the guest gets into a state where each context switch causes DR to be
>> saved and restored, this can take away as much as 40% of the execution
>> time from the guest.
>>
>> If the guest is running with vcpu->arch.db == vcpu->arch.eff_db, we
>> can let it write freely to the debug registers and reload them on the
>> next exit.  We still need to exit on the first access, so that the
>> KVM_DEBUGREG_WONT_EXIT flag is set in switch_db_regs; after that, further
>> accesses to the debug registers will not cause a vmexit.
>>
>> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
>> ---
>>  arch/x86/kvm/vmx.c | 31 ++++++++++++++++++++++++++++++-
>>  1 file changed, 30 insertions(+), 1 deletion(-)
>>
>> diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
>> index 6e57e1434cf3..71c57ec48d8f 100644
>> --- a/arch/x86/kvm/vmx.c
>> +++ b/arch/x86/kvm/vmx.c
>> @@ -2851,7 +2851,7 @@ static __init int setup_vmcs_config(struct vmcs_config *vmcs_conf)
>>  		      vmx_capability.ept, vmx_capability.vpid);
>>  	}
>>
>> -	min = 0;
>> +	min = VM_EXIT_SAVE_DEBUG_CONTROLS;
>>  #ifdef CONFIG_X86_64
>>  	min |= VM_EXIT_HOST_ADDR_SPACE_SIZE;
>>  #endif
>> @@ -5121,6 +5121,22 @@ static int handle_dr(struct kvm_vcpu *vcpu)
>>  		}
>>  	}
>>
>> +	if (vcpu->guest_debug == 0) {
>> +		u32 cpu_based_vm_exec_control;
>> +
>> +		cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
>> +		cpu_based_vm_exec_control &= ~CPU_BASED_MOV_DR_EXITING;
>> +		vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
>> +
>> +		/*
>> +		 * No more DR vmexits; force a reload of the debug registers
>> +		 * and reenter on this instruction.  The next vmexit will
>> +		 * retrieve the full state of the debug registers.
>> +		 */
>> +		vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
>> +		return 1;
>> +	}
>> +
>>  	exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
>>  	dr = exit_qualification & DEBUG_REG_ACCESS_NUM;
>>  	reg = DEBUG_REG_ACCESS_REG(exit_qualification);
>> @@ -5147,6 +5163,18 @@ static void vmx_set_dr6(struct kvm_vcpu *vcpu, unsigned long val)
>>  {
>>  }
>>
>> +static u64 vmx_get_dr7(struct kvm_vcpu *vcpu)
>> +{
>> +	/* DRs are being synced back to vcpu->arch, exit on DR access.  */
>> +	u32 cpu_based_vm_exec_control;
>> +
>> +	cpu_based_vm_exec_control = vmcs_read32(CPU_BASED_VM_EXEC_CONTROL);
>> +	cpu_based_vm_exec_control |= CPU_BASED_MOV_DR_EXITING;
>> +	vmcs_write32(CPU_BASED_VM_EXEC_CONTROL, cpu_based_vm_exec_control);
>> +
>> +	return vmcs_readl(GUEST_DR7);
>> +}
>
> The general idea looks ok (It passes x86/debug.flat unit test, right?).

Yes, of course.

> But this side effect of get_dr7 seems a bit ugly to me. Also the
> imbalanced updates of arch.switch_db_regs: KVM_DEBUGREG_WONT_EXIT is set
> by the vendor code but cleared in a common x86 path.

I can certainly remove the difference in the updates of 
KVM_DEBUGREG_WONT_EXIT.  It made some sense when the constant was called 
KVM_DEBUGREG_DIRTY but not now that I renamed it.

I don't like the side effect particularly, either, but I don't have any 
better idea.

Paolo

> Can't you make this more regular and explicit?
>
> Jan
>
>> +
>>  static void vmx_set_dr7(struct kvm_vcpu *vcpu, unsigned long val)
>>  {
>>  	vmcs_writel(GUEST_DR7, val);
>> @@ -8606,6 +8634,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
>>  	.set_gdt = vmx_set_gdt,
>>  	.get_dr6 = vmx_get_dr6,
>>  	.set_dr6 = vmx_set_dr6,
>> +	.get_dr7 = vmx_get_dr7,
>>  	.set_dr7 = vmx_set_dr7,
>>  	.cache_reg = vmx_cache_reg,
>>  	.get_rflags = vmx_get_rflags,
>>
>

Re: [PATCH 0/4] KVM: x86: Let the guest write to multiple debug registers with one vmexit

[Alex Williamson]

On Wed, 2014-02-26 at 16:49 +0100, Paolo Bonzini wrote:
> Alex Williamson reported that a Windows game does something weird that
> makes the guest save and restore debug registers on each context switch.
> This cause several hundred thousands vmexits per second, and basically
> cuts performance in half when running under KVM.
> 
> However, when not running in guest-debug mode, the guest controls the
> debug registers and having to take an exit for each DR access is a waste
> of time.  We just need one vmexit to load any stale values of DR0-DR6,
> and then we can let the guest run freely.  On the next vmexit (whatever
> the reason) we will read out whatever changes the guest made to the
> debug registers.
> 
> On top of this, we can implement SVM support and let nested guests run
> with dirty debug registers too.
> 
> Paolo Bonzini (4):
>   KVM: vmx: we do rely on loading DR7 on entry
>   KVM: x86: change vcpu->arch.switch_db_regs to a bit mask
>   KVM: x86: Allow the guest to run with dirty debug registers
>   KVM: vmx: Allow the guest to run with dirty debug registers
> 
>  arch/x86/include/asm/kvm_host.h |  8 ++++++-
>  arch/x86/kvm/vmx.c              | 48 +++++++++++++++++++++++++++++++++++++----
>  arch/x86/kvm/x86.c              | 29 +++++++++++++++++++++++--
>  3 files changed, 78 insertions(+), 7 deletions(-)
> 

I see a slightly better than 100% improvement in the frame rate in
Borderlands2 running in a VM with assigned Nvidia GPU with this series.

Tested-by: Alex Williamson <alex.williamson@redhat.com>