[PATCH 0/3] KVM: x86: MSR completion refactoring for SEV-ES

[PATCH 0/3] KVM: x86: MSR completion refactoring for SEV-ES

[Paolo Bonzini]

These patches remove kvm_inject_gp from the RDMSR/WRMSR emulation
path, with the purpose of letting SEV-ES inject the #GP through
the GHCB instead.

The main idea is to introduce a complete_emulated_msr callback
that is call-compatible with kvm_complete_insn_gp, so that svm.c
can just call kvm_complete_insn_gp in the common case.

I have more patches to use kvm_complete_insn_gp instead of
kvm_inject_gp in other paths, but they are not necessary for
SEV-ES so they can be delayed to 5.12.

Paolo

Paolo Bonzini (3):
  KVM: x86: remove bogus #GP injection
  KVM: x86: use kvm_complete_insn_gp in emulating RDMSR/WRMSR
  KVM: x86: introduce complete_emulated_msr callback

 arch/x86/include/asm/kvm_host.h |  1 +
 arch/x86/kvm/mtrr.c             |  6 +----
 arch/x86/kvm/svm/svm.c          |  1 +
 arch/x86/kvm/vmx/vmx.c          |  1 +
 arch/x86/kvm/x86.c              | 42 +++++++++++++--------------------
 5 files changed, 20 insertions(+), 31 deletions(-)

-- 
2.26.2

[PATCH 1/3] KVM: x86: remove bogus #GP injection

[Paolo Bonzini]

There is no need to inject a #GP from kvm_mtrr_set_msr, kvm_emulate_wrmsr will
handle it.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/kvm/mtrr.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/arch/x86/kvm/mtrr.c b/arch/x86/kvm/mtrr.c
index 7f0059aa30e1..f472fdb6ae7e 100644
--- a/arch/x86/kvm/mtrr.c
+++ b/arch/x86/kvm/mtrr.c
@@ -84,12 +84,8 @@ bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
 	} else
 		/* MTRR mask */
 		mask |= 0x7ff;
-	if (data & mask) {
-		kvm_inject_gp(vcpu, 0);
-		return false;
-	}
 
-	return true;
+	return (data & mask) == 0;
 }
 EXPORT_SYMBOL_GPL(kvm_mtrr_valid);
 
-- 
2.26.2

Re: [PATCH 1/3] KVM: x86: remove bogus #GP injection

[Tom Lendacky]

On 12/14/20 12:32 PM, Paolo Bonzini wrote:
> There is no need to inject a #GP from kvm_mtrr_set_msr, kvm_emulate_wrmsr will
> handle it.
> 
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>

> ---
>  arch/x86/kvm/mtrr.c | 6 +-----
>  1 file changed, 1 insertion(+), 5 deletions(-)
> 
> diff --git a/arch/x86/kvm/mtrr.c b/arch/x86/kvm/mtrr.c
> index 7f0059aa30e1..f472fdb6ae7e 100644
> --- a/arch/x86/kvm/mtrr.c
> +++ b/arch/x86/kvm/mtrr.c
> @@ -84,12 +84,8 @@ bool kvm_mtrr_valid(struct kvm_vcpu *vcpu, u32 msr, u64 data)
>  	} else
>  		/* MTRR mask */
>  		mask |= 0x7ff;
> -	if (data & mask) {
> -		kvm_inject_gp(vcpu, 0);
> -		return false;
> -	}
>  
> -	return true;
> +	return (data & mask) == 0;
>  }
>  EXPORT_SYMBOL_GPL(kvm_mtrr_valid);
>  
> 

[PATCH 2/3] KVM: x86: use kvm_complete_insn_gp in emulating RDMSR/WRMSR

[Paolo Bonzini]

Simplify the four functions that handle {kernel,user} {rd,wr}msr, there
is still some repetition between the two instances of rdmsr but the
whole business of calling kvm_inject_gp and kvm_skip_emulated_instruction
can be unified nicely.

Because complete_emulated_wrmsr now becomes essentially a call to
kvm_complete_insn_gp, remove complete_emulated_msr.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/kvm/x86.c | 42 ++++++++++++++++--------------------------
 1 file changed, 16 insertions(+), 26 deletions(-)

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index a3fdc16cfd6f..2f1bc52e70c0 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1634,27 +1634,20 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
 }
 EXPORT_SYMBOL_GPL(kvm_set_msr);
 
-static int complete_emulated_msr(struct kvm_vcpu *vcpu, bool is_read)
+static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
 {
-	if (vcpu->run->msr.error) {
-		kvm_inject_gp(vcpu, 0);
-		return 1;
-	} else if (is_read) {
+	int err = vcpu->run->msr.error;
+	if (!err) {
 		kvm_rax_write(vcpu, (u32)vcpu->run->msr.data);
 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
 	}
 
-	return kvm_skip_emulated_instruction(vcpu);
-}
-
-static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
-{
-	return complete_emulated_msr(vcpu, true);
+	return kvm_complete_insn_gp(vcpu, err);
 }
 
 static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)
 {
-	return complete_emulated_msr(vcpu, false);
+	return kvm_complete_insn_gp(vcpu, vcpu->run->msr.error);
 }
 
 static u64 kvm_msr_reason(int r)
@@ -1718,17 +1711,16 @@ int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
 	}
 
 	/* MSR read failed? Inject a #GP */
-	if (r) {
+	if (!r) {
+		trace_kvm_msr_read(ecx, data);
+
+		kvm_rax_write(vcpu, data & -1u);
+		kvm_rdx_write(vcpu, (data >> 32) & -1u);
+	} else {
 		trace_kvm_msr_read_ex(ecx);
-		kvm_inject_gp(vcpu, 0);
-		return 1;
 	}
 
-	trace_kvm_msr_read(ecx, data);
-
-	kvm_rax_write(vcpu, data & -1u);
-	kvm_rdx_write(vcpu, (data >> 32) & -1u);
-	return kvm_skip_emulated_instruction(vcpu);
+	return kvm_complete_insn_gp(vcpu, r);
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
 
@@ -1750,14 +1742,12 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
 		return r;
 
 	/* MSR write failed? Inject a #GP */
-	if (r > 0) {
+	if (!r)
+		trace_kvm_msr_write(ecx, data);
+	else
 		trace_kvm_msr_write_ex(ecx, data);
-		kvm_inject_gp(vcpu, 0);
-		return 1;
-	}
 
-	trace_kvm_msr_write(ecx, data);
-	return kvm_skip_emulated_instruction(vcpu);
+	return kvm_complete_insn_gp(vcpu, r);
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
 
-- 
2.26.2

Re: [PATCH 2/3] KVM: x86: use kvm_complete_insn_gp in emulating RDMSR/WRMSR

[Tom Lendacky]

On 12/14/20 12:32 PM, Paolo Bonzini wrote:
> Simplify the four functions that handle {kernel,user} {rd,wr}msr, there
> is still some repetition between the two instances of rdmsr but the
> whole business of calling kvm_inject_gp and kvm_skip_emulated_instruction
> can be unified nicely.
> 
> Because complete_emulated_wrmsr now becomes essentially a call to
> kvm_complete_insn_gp, remove complete_emulated_msr.
> 
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Just two minor nits below.

Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>

> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index a3fdc16cfd6f..2f1bc52e70c0 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -1634,27 +1634,20 @@ int kvm_set_msr(struct kvm_vcpu *vcpu, u32 index, u64 data)
>  }
>  EXPORT_SYMBOL_GPL(kvm_set_msr);
>  
>  
>  	/* MSR read failed? Inject a #GP */

This comment isn't accurate any more, maybe just delete it?

> -	if (r) {
> +	if (!r) {
> +		trace_kvm_msr_read(ecx, data);
> +
> +		kvm_rax_write(vcpu, data & -1u);
> +		kvm_rdx_write(vcpu, (data >> 32) & -1u);
> +	} else {
>  		trace_kvm_msr_read_ex(ecx);
> -		kvm_inject_gp(vcpu, 0);
> -		return 1;
>  	}
>  
> -	trace_kvm_msr_read(ecx, data);
> -
> -	kvm_rax_write(vcpu, data & -1u);
> -	kvm_rdx_write(vcpu, (data >> 32) & -1u);
> -	return kvm_skip_emulated_instruction(vcpu);
> +	return kvm_complete_insn_gp(vcpu, r);
>  }
>  EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
>  
> @@ -1750,14 +1742,12 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
>  		return r;
>  
>  	/* MSR write failed? Inject a #GP */

Ditto on this comment.

Thanks,
Tom

> -	if (r > 0) {
> +	if (!r)
> +		trace_kvm_msr_write(ecx, data);
> +	else
>  		trace_kvm_msr_write_ex(ecx, data);
> -		kvm_inject_gp(vcpu, 0);
> -		return 1;
> -	}
>  
> -	trace_kvm_msr_write(ecx, data);
> -	return kvm_skip_emulated_instruction(vcpu);
> +	return kvm_complete_insn_gp(vcpu, r);
>  }
>  EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
>  
> 

[PATCH 3/3] KVM: x86: introduce complete_emulated_msr callback

[Paolo Bonzini]

This will be used by SEV-ES to inject MSR failure via the GHCB.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
---
 arch/x86/include/asm/kvm_host.h | 1 +
 arch/x86/kvm/svm/svm.c          | 1 +
 arch/x86/kvm/vmx/vmx.c          | 1 +
 arch/x86/kvm/x86.c              | 8 ++++----
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 8cf6b0493d49..18aa15e6fadd 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1285,6 +1285,7 @@ struct kvm_x86_ops {
 
 	void (*migrate_timers)(struct kvm_vcpu *vcpu);
 	void (*msr_filter_changed)(struct kvm_vcpu *vcpu);
+	int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err);
 };
 
 struct kvm_x86_nested_ops {
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index 801e0a641258..4067d511be08 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -4306,6 +4306,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
 	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
 
 	.msr_filter_changed = svm_msr_filter_changed,
+	.complete_emulated_msr = kvm_complete_insn_gp,
 };
 
 static struct kvm_x86_init_ops svm_init_ops __initdata = {
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 849be2a9f260..55fa51c0cd9d 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -7701,6 +7701,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
 	.migrate_timers = vmx_migrate_timers,
 
 	.msr_filter_changed = vmx_msr_filter_changed,
+	.complete_emulated_msr = kvm_complete_insn_gp,
 	.cpu_dirty_log_size = vmx_cpu_dirty_log_size,
 };
 
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 2f1bc52e70c0..6c4482b97c91 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1642,12 +1642,12 @@ static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
 		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
 	}
 
-	return kvm_complete_insn_gp(vcpu, err);
+	return kvm_x86_ops.complete_emulated_msr(vcpu, err);
 }
 
 static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)
 {
-	return kvm_complete_insn_gp(vcpu, vcpu->run->msr.error);
+	return kvm_x86_ops.complete_emulated_msr(vcpu, vcpu->run->msr.error);
 }
 
 static u64 kvm_msr_reason(int r)
@@ -1720,7 +1720,7 @@ int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
 		trace_kvm_msr_read_ex(ecx);
 	}
 
-	return kvm_complete_insn_gp(vcpu, r);
+	return kvm_x86_ops.complete_emulated_msr(vcpu, r);
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
 
@@ -1747,7 +1747,7 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
 	else
 		trace_kvm_msr_write_ex(ecx, data);
 
-	return kvm_complete_insn_gp(vcpu, r);
+	return kvm_x86_ops.complete_emulated_msr(vcpu, r);
 }
 EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
 
-- 
2.26.2

Re: [PATCH 3/3] KVM: x86: introduce complete_emulated_msr callback

[Tom Lendacky]

On 12/14/20 12:32 PM, Paolo Bonzini wrote:
> This will be used by SEV-ES to inject MSR failure via the GHCB.
> 
> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>

(Changed Sean's email on this reply, but missed the others...)

> ---
>  arch/x86/include/asm/kvm_host.h | 1 +
>  arch/x86/kvm/svm/svm.c          | 1 +
>  arch/x86/kvm/vmx/vmx.c          | 1 +
>  arch/x86/kvm/x86.c              | 8 ++++----
>  4 files changed, 7 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
> index 8cf6b0493d49..18aa15e6fadd 100644
> --- a/arch/x86/include/asm/kvm_host.h
> +++ b/arch/x86/include/asm/kvm_host.h
> @@ -1285,6 +1285,7 @@ struct kvm_x86_ops {
>  
>  	void (*migrate_timers)(struct kvm_vcpu *vcpu);
>  	void (*msr_filter_changed)(struct kvm_vcpu *vcpu);
> +	int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err);
>  };
>  
>  struct kvm_x86_nested_ops {
> diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
> index 801e0a641258..4067d511be08 100644
> --- a/arch/x86/kvm/svm/svm.c
> +++ b/arch/x86/kvm/svm/svm.c
> @@ -4306,6 +4306,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
>  	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
>  
>  	.msr_filter_changed = svm_msr_filter_changed,
> +	.complete_emulated_msr = kvm_complete_insn_gp,
>  };
>  
>  static struct kvm_x86_init_ops svm_init_ops __initdata = {
> diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
> index 849be2a9f260..55fa51c0cd9d 100644
> --- a/arch/x86/kvm/vmx/vmx.c
> +++ b/arch/x86/kvm/vmx/vmx.c
> @@ -7701,6 +7701,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
>  	.migrate_timers = vmx_migrate_timers,
>  
>  	.msr_filter_changed = vmx_msr_filter_changed,
> +	.complete_emulated_msr = kvm_complete_insn_gp,
>  	.cpu_dirty_log_size = vmx_cpu_dirty_log_size,
>  };
>  
> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
> index 2f1bc52e70c0..6c4482b97c91 100644
> --- a/arch/x86/kvm/x86.c
> +++ b/arch/x86/kvm/x86.c
> @@ -1642,12 +1642,12 @@ static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
>  		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
>  	}
>  
> -	return kvm_complete_insn_gp(vcpu, err);
> +	return kvm_x86_ops.complete_emulated_msr(vcpu, err);
>  }
>  
>  static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)
>  {
> -	return kvm_complete_insn_gp(vcpu, vcpu->run->msr.error);
> +	return kvm_x86_ops.complete_emulated_msr(vcpu, vcpu->run->msr.error);
>  }
>  
>  static u64 kvm_msr_reason(int r)
> @@ -1720,7 +1720,7 @@ int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
>  		trace_kvm_msr_read_ex(ecx);
>  	}
>  
> -	return kvm_complete_insn_gp(vcpu, r);
> +	return kvm_x86_ops.complete_emulated_msr(vcpu, r);
>  }
>  EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
>  
> @@ -1747,7 +1747,7 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
>  	else
>  		trace_kvm_msr_write_ex(ecx, data);
>  
> -	return kvm_complete_insn_gp(vcpu, r);
> +	return kvm_x86_ops.complete_emulated_msr(vcpu, r);
>  }
>  EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
>  
> 

Re: [PATCH 3/3] KVM: x86: introduce complete_emulated_msr callback

[Paolo Bonzini]

On 14/12/20 21:55, Tom Lendacky wrote:
> On 12/14/20 12:32 PM, Paolo Bonzini wrote:
>> This will be used by SEV-ES to inject MSR failure via the GHCB.
>>
>> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
> 
> Reviewed-by: Tom Lendacky <thomas.lendacky@amd.com>
> 
> (Changed Sean's email on this reply, but missed the others...)

Thanks for the review, I pushed to kvm/queue now.

Paolo

>> ---
>>   arch/x86/include/asm/kvm_host.h | 1 +
>>   arch/x86/kvm/svm/svm.c          | 1 +
>>   arch/x86/kvm/vmx/vmx.c          | 1 +
>>   arch/x86/kvm/x86.c              | 8 ++++----
>>   4 files changed, 7 insertions(+), 4 deletions(-)
>>
>> diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
>> index 8cf6b0493d49..18aa15e6fadd 100644
>> --- a/arch/x86/include/asm/kvm_host.h
>> +++ b/arch/x86/include/asm/kvm_host.h
>> @@ -1285,6 +1285,7 @@ struct kvm_x86_ops {
>>   
>>   	void (*migrate_timers)(struct kvm_vcpu *vcpu);
>>   	void (*msr_filter_changed)(struct kvm_vcpu *vcpu);
>> +	int (*complete_emulated_msr)(struct kvm_vcpu *vcpu, int err);
>>   };
>>   
>>   struct kvm_x86_nested_ops {
>> diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
>> index 801e0a641258..4067d511be08 100644
>> --- a/arch/x86/kvm/svm/svm.c
>> +++ b/arch/x86/kvm/svm/svm.c
>> @@ -4306,6 +4306,7 @@ static struct kvm_x86_ops svm_x86_ops __initdata = {
>>   	.apic_init_signal_blocked = svm_apic_init_signal_blocked,
>>   
>>   	.msr_filter_changed = svm_msr_filter_changed,
>> +	.complete_emulated_msr = kvm_complete_insn_gp,
>>   };
>>   
>>   static struct kvm_x86_init_ops svm_init_ops __initdata = {
>> diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
>> index 849be2a9f260..55fa51c0cd9d 100644
>> --- a/arch/x86/kvm/vmx/vmx.c
>> +++ b/arch/x86/kvm/vmx/vmx.c
>> @@ -7701,6 +7701,7 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = {
>>   	.migrate_timers = vmx_migrate_timers,
>>   
>>   	.msr_filter_changed = vmx_msr_filter_changed,
>> +	.complete_emulated_msr = kvm_complete_insn_gp,
>>   	.cpu_dirty_log_size = vmx_cpu_dirty_log_size,
>>   };
>>   
>> diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
>> index 2f1bc52e70c0..6c4482b97c91 100644
>> --- a/arch/x86/kvm/x86.c
>> +++ b/arch/x86/kvm/x86.c
>> @@ -1642,12 +1642,12 @@ static int complete_emulated_rdmsr(struct kvm_vcpu *vcpu)
>>   		kvm_rdx_write(vcpu, vcpu->run->msr.data >> 32);
>>   	}
>>   
>> -	return kvm_complete_insn_gp(vcpu, err);
>> +	return kvm_x86_ops.complete_emulated_msr(vcpu, err);
>>   }
>>   
>>   static int complete_emulated_wrmsr(struct kvm_vcpu *vcpu)
>>   {
>> -	return kvm_complete_insn_gp(vcpu, vcpu->run->msr.error);
>> +	return kvm_x86_ops.complete_emulated_msr(vcpu, vcpu->run->msr.error);
>>   }
>>   
>>   static u64 kvm_msr_reason(int r)
>> @@ -1720,7 +1720,7 @@ int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu)
>>   		trace_kvm_msr_read_ex(ecx);
>>   	}
>>   
>> -	return kvm_complete_insn_gp(vcpu, r);
>> +	return kvm_x86_ops.complete_emulated_msr(vcpu, r);
>>   }
>>   EXPORT_SYMBOL_GPL(kvm_emulate_rdmsr);
>>   
>> @@ -1747,7 +1747,7 @@ int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu)
>>   	else
>>   		trace_kvm_msr_write_ex(ecx, data);
>>   
>> -	return kvm_complete_insn_gp(vcpu, r);
>> +	return kvm_x86_ops.complete_emulated_msr(vcpu, r);
>>   }
>>   EXPORT_SYMBOL_GPL(kvm_emulate_wrmsr);
>>   
>>
>