[PATCH 0/2] KVM: nVMX: Fix nested posted interrupts delivery

[PATCH 0/2] KVM: nVMX: Fix nested posted interrupts delivery

[fanwenyi0529]

From: Wincy Van <fanwenyi0529@gmail.com>

The nested/non-nested PI vector is the same currently, this is wrong
since the non-nested PI may not be recognized if dest vcpu is in L2.

This patch series introduce a new vector for nested PI to solve the
problem.

Wincy Van (2):
  x86: irq: Define a global vector for nested posted interrupts
  KVM: nVMX: Fix posted intr delivery when vcpu is in guest mode

 arch/x86/entry/entry_64.S          |  1 +
 arch/x86/include/asm/entry_arch.h  |  2 ++
 arch/x86/include/asm/hardirq.h     |  1 +
 arch/x86/include/asm/hw_irq.h      |  2 ++
 arch/x86/include/asm/irq_vectors.h |  1 +
 arch/x86/kernel/irq.c              | 13 +++++++++++++
 arch/x86/kernel/irqinit.c          |  2 ++
 arch/x86/kvm/vmx.c                 | 22 +++++++++++-----------
 8 files changed, 33 insertions(+), 11 deletions(-)

-- 
1.8.3.1

[PATCH 1/2] x86: irq: Define a global vector for nested posted interrupts

[fanwenyi0529]

From: Wincy Van <fanwenyi0529@gmail.com>

We are using the same vector for nested/non-nested posted
interrupts delivery, this may cause interrupts latency in
L1 since we can't kick the L2 vcpu out of vmx-nonroot mode.

This patch introduces a new vector which is only for nested
posted interrupts to solve the problems above.

Signed-off-by: Wincy Van <fanwenyi0529@gmail.com>
---
 arch/x86/entry/entry_64.S          |  1 +
 arch/x86/include/asm/entry_arch.h  |  2 ++
 arch/x86/include/asm/hardirq.h     |  1 +
 arch/x86/include/asm/hw_irq.h      |  2 ++
 arch/x86/include/asm/irq_vectors.h |  1 +
 arch/x86/kernel/irq.c              | 13 +++++++++++++
 arch/x86/kernel/irqinit.c          |  2 ++
 7 files changed, 22 insertions(+)

diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 044d18e..c01e62c 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -710,6 +710,7 @@ apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
 #ifdef CONFIG_HAVE_KVM
 apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
 apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
+apicinterrupt3 POSTED_INTR_NESTED_VECTOR	kvm_posted_intr_nested_ipi	smp_kvm_posted_intr_nested_ipi
 #endif
 
 #ifdef CONFIG_X86_MCE_THRESHOLD
diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
index df00299..07b0695 100644
--- a/arch/x86/include/asm/entry_arch.h
+++ b/arch/x86/include/asm/entry_arch.h
@@ -25,6 +25,8 @@
 		 smp_kvm_posted_intr_ipi)
 BUILD_INTERRUPT3(kvm_posted_intr_wakeup_ipi, POSTED_INTR_WAKEUP_VECTOR,
 		 smp_kvm_posted_intr_wakeup_ipi)
+BUILD_INTERRUPT3(kvm_posted_intr_nested_ipi, POSTED_INTR_NESTED_VECTOR,
+		 smp_kvm_posted_intr_nested_ipi)
 #endif
 
 /*
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index 59405a2..3a81b21 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -15,6 +15,7 @@
 #ifdef CONFIG_HAVE_KVM
 	unsigned int kvm_posted_intr_ipis;
 	unsigned int kvm_posted_intr_wakeup_ipis;
+	unsigned int kvm_posted_intr_nested_ipis;
 #endif
 	unsigned int x86_platform_ipis;	/* arch dependent */
 	unsigned int apic_perf_irqs;
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
index b90e105..d6dbafb 100644
--- a/arch/x86/include/asm/hw_irq.h
+++ b/arch/x86/include/asm/hw_irq.h
@@ -30,6 +30,7 @@
 extern asmlinkage void x86_platform_ipi(void);
 extern asmlinkage void kvm_posted_intr_ipi(void);
 extern asmlinkage void kvm_posted_intr_wakeup_ipi(void);
+extern asmlinkage void kvm_posted_intr_nested_ipi(void);
 extern asmlinkage void error_interrupt(void);
 extern asmlinkage void irq_work_interrupt(void);
 
@@ -62,6 +63,7 @@
 #define trace_reboot_interrupt  reboot_interrupt
 #define trace_kvm_posted_intr_ipi kvm_posted_intr_ipi
 #define trace_kvm_posted_intr_wakeup_ipi kvm_posted_intr_wakeup_ipi
+#define trace_kvm_posted_intr_nested_ipi kvm_posted_intr_nested_ipi
 #endif /* CONFIG_TRACING */
 
 #ifdef	CONFIG_X86_LOCAL_APIC
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 6ca9fd6..30f2bc5 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -98,6 +98,7 @@
 /* Vector for KVM to deliver posted interrupt IPI */
 #ifdef CONFIG_HAVE_KVM
 #define POSTED_INTR_VECTOR		0xf2
+#define POSTED_INTR_NESTED_VECTOR	0xf0
 #endif
 
 /*
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 4d8183b..9fed1e1 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -313,6 +313,19 @@ __visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
 	exiting_irq();
 	set_irq_regs(old_regs);
 }
+
+/*
+ * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
+ */
+__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
+{
+	struct pt_regs *old_regs = set_irq_regs(regs);
+
+	entering_ack_irq();
+	inc_irq_stat(kvm_posted_intr_nested_ipis);
+	exiting_irq();
+	set_irq_regs(old_regs);
+}
 #endif
 
 __visible void __irq_entry smp_trace_x86_platform_ipi(struct pt_regs *regs)
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 1423ab1..4c86c92 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -150,6 +150,8 @@ static void __init apic_intr_init(void)
 	alloc_intr_gate(POSTED_INTR_VECTOR, kvm_posted_intr_ipi);
 	/* IPI for KVM to deliver interrupt to wake up tasks */
 	alloc_intr_gate(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi);
+	/* IPI for KVM to deliver nested posted interrupt */
+	alloc_intr_gate(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi);
 #endif
 
 	/* IPI vectors for APIC spurious and error interrupts */
-- 
1.8.3.1

[PATCH 2/2] KVM: nVMX: Fix posted intr delivery when vcpu is in guest mode

[fanwenyi0529]

From: Wincy Van <fanwenyi0529@gmail.com>

The PI vector for L0 and L1 must be different. If dest vcpu0
is in guest mode while vcpu1 is delivering a non-nested PI to
vcpu0, there wont't be any vmexit so that the non-nested interrupt
will be delayed.

Signed-off-by: Wincy Van <fanwenyi0529@gmail.com>
---
 arch/x86/kvm/vmx.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 75e6cc9..f269f9c 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -4913,9 +4913,12 @@ static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
 	}
 }
 
-static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu)
+static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu,
+						     bool nested)
 {
 #ifdef CONFIG_SMP
+	int pi_vec = nested ? POSTED_INTR_NESTED_VECTOR : POSTED_INTR_VECTOR;
+
 	if (vcpu->mode == IN_GUEST_MODE) {
 		struct vcpu_vmx *vmx = to_vmx(vcpu);
 
@@ -4933,8 +4936,7 @@ static inline bool kvm_vcpu_trigger_posted_interrupt(struct kvm_vcpu *vcpu)
 		 */
 		WARN_ON_ONCE(pi_test_sn(&vmx->pi_desc));
 
-		apic->send_IPI_mask(get_cpu_mask(vcpu->cpu),
-				POSTED_INTR_VECTOR);
+		apic->send_IPI_mask(get_cpu_mask(vcpu->cpu), pi_vec);
 		return true;
 	}
 #endif
@@ -4949,7 +4951,7 @@ static int vmx_deliver_nested_posted_interrupt(struct kvm_vcpu *vcpu,
 	if (is_guest_mode(vcpu) &&
 	    vector == vmx->nested.posted_intr_nv) {
 		/* the PIR and ON have been set by L1. */
-		kvm_vcpu_trigger_posted_interrupt(vcpu);
+		kvm_vcpu_trigger_posted_interrupt(vcpu, true);
 		/*
 		 * If a posted intr is not recognized by hardware,
 		 * we will accomplish it in the next vmentry.
@@ -4983,7 +4985,7 @@ static void vmx_deliver_posted_interrupt(struct kvm_vcpu *vcpu, int vector)
 	if (pi_test_and_set_on(&vmx->pi_desc))
 		return;
 
-	if (!kvm_vcpu_trigger_posted_interrupt(vcpu))
+	if (!kvm_vcpu_trigger_posted_interrupt(vcpu, false))
 		kvm_vcpu_kick(vcpu);
 }
 
@@ -9967,13 +9969,9 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 
 	/* Posted interrupts setting is only taken from vmcs12.  */
 	if (nested_cpu_has_posted_intr(vmcs12)) {
-		/*
-		 * Note that we use L0's vector here and in
-		 * vmx_deliver_nested_posted_interrupt.
-		 */
 		vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
 		vmx->nested.pi_pending = false;
-		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
+		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
 	} else {
 		exec_control &= ~PIN_BASED_POSTED_INTR;
 	}
@@ -10815,7 +10813,9 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
 		 */
 		vmx_flush_tlb(vcpu);
 	}
-
+	/* Restore posted intr vector. */
+	if (nested_cpu_has_posted_intr(vmcs12))
+		vmcs_write16(POSTED_INTR_NV, POSTED_INTR_VECTOR);
 
 	vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
 	vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
-- 
1.8.3.1

Re: [PATCH 1/2] x86: irq: Define a global vector for nested posted interrupts

[Paolo Bonzini]

Ingo,

can you please ack this patch?

Thanks,

Paolo

On 28/04/2017 07:13, fanwenyi0529@gmail.com wrote:
> From: Wincy Van <fanwenyi0529@gmail.com>
> 
> We are using the same vector for nested/non-nested posted
> interrupts delivery, this may cause interrupts latency in
> L1 since we can't kick the L2 vcpu out of vmx-nonroot mode.
> 
> This patch introduces a new vector which is only for nested
> posted interrupts to solve the problems above.
> 
> Signed-off-by: Wincy Van <fanwenyi0529@gmail.com>
> ---
>  arch/x86/entry/entry_64.S          |  1 +
>  arch/x86/include/asm/entry_arch.h  |  2 ++
>  arch/x86/include/asm/hardirq.h     |  1 +
>  arch/x86/include/asm/hw_irq.h      |  2 ++
>  arch/x86/include/asm/irq_vectors.h |  1 +
>  arch/x86/kernel/irq.c              | 13 +++++++++++++
>  arch/x86/kernel/irqinit.c          |  2 ++
>  7 files changed, 22 insertions(+)
> 
> diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
> index 044d18e..c01e62c 100644
> --- a/arch/x86/entry/entry_64.S
> +++ b/arch/x86/entry/entry_64.S
> @@ -710,6 +710,7 @@ apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
>  #ifdef CONFIG_HAVE_KVM
>  apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
>  apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
> +apicinterrupt3 POSTED_INTR_NESTED_VECTOR	kvm_posted_intr_nested_ipi	smp_kvm_posted_intr_nested_ipi
>  #endif
>  
>  #ifdef CONFIG_X86_MCE_THRESHOLD
> diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
> index df00299..07b0695 100644
> --- a/arch/x86/include/asm/entry_arch.h
> +++ b/arch/x86/include/asm/entry_arch.h
> @@ -25,6 +25,8 @@
>  		 smp_kvm_posted_intr_ipi)
>  BUILD_INTERRUPT3(kvm_posted_intr_wakeup_ipi, POSTED_INTR_WAKEUP_VECTOR,
>  		 smp_kvm_posted_intr_wakeup_ipi)
> +BUILD_INTERRUPT3(kvm_posted_intr_nested_ipi, POSTED_INTR_NESTED_VECTOR,
> +		 smp_kvm_posted_intr_nested_ipi)
>  #endif
>  
>  /*
> diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
> index 59405a2..3a81b21 100644
> --- a/arch/x86/include/asm/hardirq.h
> +++ b/arch/x86/include/asm/hardirq.h
> @@ -15,6 +15,7 @@
>  #ifdef CONFIG_HAVE_KVM
>  	unsigned int kvm_posted_intr_ipis;
>  	unsigned int kvm_posted_intr_wakeup_ipis;
> +	unsigned int kvm_posted_intr_nested_ipis;
>  #endif
>  	unsigned int x86_platform_ipis;	/* arch dependent */
>  	unsigned int apic_perf_irqs;
> diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
> index b90e105..d6dbafb 100644
> --- a/arch/x86/include/asm/hw_irq.h
> +++ b/arch/x86/include/asm/hw_irq.h
> @@ -30,6 +30,7 @@
>  extern asmlinkage void x86_platform_ipi(void);
>  extern asmlinkage void kvm_posted_intr_ipi(void);
>  extern asmlinkage void kvm_posted_intr_wakeup_ipi(void);
> +extern asmlinkage void kvm_posted_intr_nested_ipi(void);
>  extern asmlinkage void error_interrupt(void);
>  extern asmlinkage void irq_work_interrupt(void);
>  
> @@ -62,6 +63,7 @@
>  #define trace_reboot_interrupt  reboot_interrupt
>  #define trace_kvm_posted_intr_ipi kvm_posted_intr_ipi
>  #define trace_kvm_posted_intr_wakeup_ipi kvm_posted_intr_wakeup_ipi
> +#define trace_kvm_posted_intr_nested_ipi kvm_posted_intr_nested_ipi
>  #endif /* CONFIG_TRACING */
>  
>  #ifdef	CONFIG_X86_LOCAL_APIC
> diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
> index 6ca9fd6..30f2bc5 100644
> --- a/arch/x86/include/asm/irq_vectors.h
> +++ b/arch/x86/include/asm/irq_vectors.h
> @@ -98,6 +98,7 @@
>  /* Vector for KVM to deliver posted interrupt IPI */
>  #ifdef CONFIG_HAVE_KVM
>  #define POSTED_INTR_VECTOR		0xf2
> +#define POSTED_INTR_NESTED_VECTOR	0xf0
>  #endif
>  
>  /*
> diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
> index 4d8183b..9fed1e1 100644
> --- a/arch/x86/kernel/irq.c
> +++ b/arch/x86/kernel/irq.c
> @@ -313,6 +313,19 @@ __visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
>  	exiting_irq();
>  	set_irq_regs(old_regs);
>  }
> +
> +/*
> + * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
> + */
> +__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
> +{
> +	struct pt_regs *old_regs = set_irq_regs(regs);
> +
> +	entering_ack_irq();
> +	inc_irq_stat(kvm_posted_intr_nested_ipis);
> +	exiting_irq();
> +	set_irq_regs(old_regs);
> +}
>  #endif
>  
>  __visible void __irq_entry smp_trace_x86_platform_ipi(struct pt_regs *regs)
> diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
> index 1423ab1..4c86c92 100644
> --- a/arch/x86/kernel/irqinit.c
> +++ b/arch/x86/kernel/irqinit.c
> @@ -150,6 +150,8 @@ static void __init apic_intr_init(void)
>  	alloc_intr_gate(POSTED_INTR_VECTOR, kvm_posted_intr_ipi);
>  	/* IPI for KVM to deliver interrupt to wake up tasks */
>  	alloc_intr_gate(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi);
> +	/* IPI for KVM to deliver nested posted interrupt */
> +	alloc_intr_gate(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi);
>  #endif
>  
>  	/* IPI vectors for APIC spurious and error interrupts */
> 

Re: [PATCH 1/2] x86: irq: Define a global vector for nested posted interrupts

[Paolo Bonzini]

On 28/04/2017 12:16, Paolo Bonzini wrote:
> Ingo,
> 
> can you please ack this patch?

Ping.  We would like to have this in 4.12.

Thanks,

Paolo

> Thanks,
> 
> Paolo
> 
> On 28/04/2017 07:13, fanwenyi0529@gmail.com wrote:
>> From: Wincy Van <fanwenyi0529@gmail.com>
>>
>> We are using the same vector for nested/non-nested posted
>> interrupts delivery, this may cause interrupts latency in
>> L1 since we can't kick the L2 vcpu out of vmx-nonroot mode.
>>
>> This patch introduces a new vector which is only for nested
>> posted interrupts to solve the problems above.
>>
>> Signed-off-by: Wincy Van <fanwenyi0529@gmail.com>
>> ---
>>  arch/x86/entry/entry_64.S          |  1 +
>>  arch/x86/include/asm/entry_arch.h  |  2 ++
>>  arch/x86/include/asm/hardirq.h     |  1 +
>>  arch/x86/include/asm/hw_irq.h      |  2 ++
>>  arch/x86/include/asm/irq_vectors.h |  1 +
>>  arch/x86/kernel/irq.c              | 13 +++++++++++++
>>  arch/x86/kernel/irqinit.c          |  2 ++
>>  7 files changed, 22 insertions(+)
>>
>> diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
>> index 044d18e..c01e62c 100644
>> --- a/arch/x86/entry/entry_64.S
>> +++ b/arch/x86/entry/entry_64.S
>> @@ -710,6 +710,7 @@ apicinterrupt X86_PLATFORM_IPI_VECTOR		x86_platform_ipi		smp_x86_platform_ipi
>>  #ifdef CONFIG_HAVE_KVM
>>  apicinterrupt3 POSTED_INTR_VECTOR		kvm_posted_intr_ipi		smp_kvm_posted_intr_ipi
>>  apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR	kvm_posted_intr_wakeup_ipi	smp_kvm_posted_intr_wakeup_ipi
>> +apicinterrupt3 POSTED_INTR_NESTED_VECTOR	kvm_posted_intr_nested_ipi	smp_kvm_posted_intr_nested_ipi
>>  #endif
>>  
>>  #ifdef CONFIG_X86_MCE_THRESHOLD
>> diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
>> index df00299..07b0695 100644
>> --- a/arch/x86/include/asm/entry_arch.h
>> +++ b/arch/x86/include/asm/entry_arch.h
>> @@ -25,6 +25,8 @@
>>  		 smp_kvm_posted_intr_ipi)
>>  BUILD_INTERRUPT3(kvm_posted_intr_wakeup_ipi, POSTED_INTR_WAKEUP_VECTOR,
>>  		 smp_kvm_posted_intr_wakeup_ipi)
>> +BUILD_INTERRUPT3(kvm_posted_intr_nested_ipi, POSTED_INTR_NESTED_VECTOR,
>> +		 smp_kvm_posted_intr_nested_ipi)
>>  #endif
>>  
>>  /*
>> diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
>> index 59405a2..3a81b21 100644
>> --- a/arch/x86/include/asm/hardirq.h
>> +++ b/arch/x86/include/asm/hardirq.h
>> @@ -15,6 +15,7 @@
>>  #ifdef CONFIG_HAVE_KVM
>>  	unsigned int kvm_posted_intr_ipis;
>>  	unsigned int kvm_posted_intr_wakeup_ipis;
>> +	unsigned int kvm_posted_intr_nested_ipis;
>>  #endif
>>  	unsigned int x86_platform_ipis;	/* arch dependent */
>>  	unsigned int apic_perf_irqs;
>> diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
>> index b90e105..d6dbafb 100644
>> --- a/arch/x86/include/asm/hw_irq.h
>> +++ b/arch/x86/include/asm/hw_irq.h
>> @@ -30,6 +30,7 @@
>>  extern asmlinkage void x86_platform_ipi(void);
>>  extern asmlinkage void kvm_posted_intr_ipi(void);
>>  extern asmlinkage void kvm_posted_intr_wakeup_ipi(void);
>> +extern asmlinkage void kvm_posted_intr_nested_ipi(void);
>>  extern asmlinkage void error_interrupt(void);
>>  extern asmlinkage void irq_work_interrupt(void);
>>  
>> @@ -62,6 +63,7 @@
>>  #define trace_reboot_interrupt  reboot_interrupt
>>  #define trace_kvm_posted_intr_ipi kvm_posted_intr_ipi
>>  #define trace_kvm_posted_intr_wakeup_ipi kvm_posted_intr_wakeup_ipi
>> +#define trace_kvm_posted_intr_nested_ipi kvm_posted_intr_nested_ipi
>>  #endif /* CONFIG_TRACING */
>>  
>>  #ifdef	CONFIG_X86_LOCAL_APIC
>> diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
>> index 6ca9fd6..30f2bc5 100644
>> --- a/arch/x86/include/asm/irq_vectors.h
>> +++ b/arch/x86/include/asm/irq_vectors.h
>> @@ -98,6 +98,7 @@
>>  /* Vector for KVM to deliver posted interrupt IPI */
>>  #ifdef CONFIG_HAVE_KVM
>>  #define POSTED_INTR_VECTOR		0xf2
>> +#define POSTED_INTR_NESTED_VECTOR	0xf0
>>  #endif
>>  
>>  /*
>> diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
>> index 4d8183b..9fed1e1 100644
>> --- a/arch/x86/kernel/irq.c
>> +++ b/arch/x86/kernel/irq.c
>> @@ -313,6 +313,19 @@ __visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs)
>>  	exiting_irq();
>>  	set_irq_regs(old_regs);
>>  }
>> +
>> +/*
>> + * Handler for POSTED_INTERRUPT_NESTED_VECTOR.
>> + */
>> +__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs)
>> +{
>> +	struct pt_regs *old_regs = set_irq_regs(regs);
>> +
>> +	entering_ack_irq();
>> +	inc_irq_stat(kvm_posted_intr_nested_ipis);
>> +	exiting_irq();
>> +	set_irq_regs(old_regs);
>> +}
>>  #endif
>>  
>>  __visible void __irq_entry smp_trace_x86_platform_ipi(struct pt_regs *regs)
>> diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
>> index 1423ab1..4c86c92 100644
>> --- a/arch/x86/kernel/irqinit.c
>> +++ b/arch/x86/kernel/irqinit.c
>> @@ -150,6 +150,8 @@ static void __init apic_intr_init(void)
>>  	alloc_intr_gate(POSTED_INTR_VECTOR, kvm_posted_intr_ipi);
>>  	/* IPI for KVM to deliver interrupt to wake up tasks */
>>  	alloc_intr_gate(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi);
>> +	/* IPI for KVM to deliver nested posted interrupt */
>> +	alloc_intr_gate(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi);
>>  #endif
>>  
>>  	/* IPI vectors for APIC spurious and error interrupts */
>>
> 

Re: [PATCH 0/2] KVM: nVMX: Fix nested posted interrupts delivery

[Paolo Bonzini]

On 28/04/2017 07:13, fanwenyi0529@gmail.com wrote:
> From: Wincy Van <fanwenyi0529@gmail.com>
> 
> The nested/non-nested PI vector is the same currently, this is wrong
> since the non-nested PI may not be recognized if dest vcpu is in L2.
> 
> This patch series introduce a new vector for nested PI to solve the
> problem.
> 
> Wincy Van (2):
>   x86: irq: Define a global vector for nested posted interrupts
>   KVM: nVMX: Fix posted intr delivery when vcpu is in guest mode
> 
>  arch/x86/entry/entry_64.S          |  1 +
>  arch/x86/include/asm/entry_arch.h  |  2 ++
>  arch/x86/include/asm/hardirq.h     |  1 +
>  arch/x86/include/asm/hw_irq.h      |  2 ++
>  arch/x86/include/asm/irq_vectors.h |  1 +
>  arch/x86/kernel/irq.c              | 13 +++++++++++++
>  arch/x86/kernel/irqinit.c          |  2 ++
>  arch/x86/kvm/vmx.c                 | 22 +++++++++++-----------
>  8 files changed, 33 insertions(+), 11 deletions(-)
> 

Belatedly applying this, with this change:

diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index e3d4718faa95..6b85958ea2ec 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -155,6 +155,12 @@ int arch_show_interrupts(struct seq_file *p, int prec)
 		seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
 	seq_puts(p, "  Posted-interrupt notification event\n");
 
+	seq_printf(p, "%*s: ", prec, "NPI");
+	for_each_online_cpu(j)
+		seq_printf(p, "%10u ",
+			   irq_stats(j)->kvm_posted_intr_nested_ipis);
+	seq_puts(p, "  Nested posted-interrupt wakeup event\n");
+
 	seq_printf(p, "%*s: ", prec, "PIW");
 	for_each_online_cpu(j)
 		seq_printf(p, "%10u ",

on top of patch 1.

Paolo

Re: [PATCH 0/2] KVM: nVMX: Fix nested posted interrupts delivery

[Wincy Van]

On Tue, Jul 25, 2017 at 1:08 AM, Paolo Bonzini <pbonzini@redhat.com> wrote:
> On 28/04/2017 07:13, fanwenyi0529@gmail.com wrote:
>> From: Wincy Van <fanwenyi0529@gmail.com>
>>
>> The nested/non-nested PI vector is the same currently, this is wrong
>> since the non-nested PI may not be recognized if dest vcpu is in L2.
>>
>> This patch series introduce a new vector for nested PI to solve the
>> problem.
>>
>> Wincy Van (2):
>>   x86: irq: Define a global vector for nested posted interrupts
>>   KVM: nVMX: Fix posted intr delivery when vcpu is in guest mode
>>
>>  arch/x86/entry/entry_64.S          |  1 +
>>  arch/x86/include/asm/entry_arch.h  |  2 ++
>>  arch/x86/include/asm/hardirq.h     |  1 +
>>  arch/x86/include/asm/hw_irq.h      |  2 ++
>>  arch/x86/include/asm/irq_vectors.h |  1 +
>>  arch/x86/kernel/irq.c              | 13 +++++++++++++
>>  arch/x86/kernel/irqinit.c          |  2 ++
>>  arch/x86/kvm/vmx.c                 | 22 +++++++++++-----------
>>  8 files changed, 33 insertions(+), 11 deletions(-)
>>
>
> Belatedly applying this, with this change:
>
> diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
> index e3d4718faa95..6b85958ea2ec 100644
> --- a/arch/x86/kernel/irq.c
> +++ b/arch/x86/kernel/irq.c
> @@ -155,6 +155,12 @@ int arch_show_interrupts(struct seq_file *p, int prec)
>                 seq_printf(p, "%10u ", irq_stats(j)->kvm_posted_intr_ipis);
>         seq_puts(p, "  Posted-interrupt notification event\n");
>
> +       seq_printf(p, "%*s: ", prec, "NPI");
> +       for_each_online_cpu(j)
> +               seq_printf(p, "%10u ",
> +                          irq_stats(j)->kvm_posted_intr_nested_ipis);
> +       seq_puts(p, "  Nested posted-interrupt wakeup event\n");
> +
>         seq_printf(p, "%*s: ", prec, "PIW");
>         for_each_online_cpu(j)
>                 seq_printf(p, "%10u ",
>
> on top of patch 1.
>

It's nice, thank you :)

Wincy