[PATCH] enable x2APIC without interrupt remapping under KVM

[PATCH] enable x2APIC without interrupt remapping under KVM

[Gleb Natapov]

KVM would like to provide x2APIC interface to a guest without emulating
interrupt remapping device. The reason KVM prefers guest to use x2APIC
is that x2APIC interface is better virtualizable and provides better
performance than mmio xAPIC interface:

- msr exits are faster than mmio (no page table walk, emulation)
- no need to read back ICR to look at the busy bit
- one 64 bit ICR write instead of two 32 bit writes
- shared code with the Hyper-V paravirt interface

Included patch changes x2APIC enabling logic to enable it even if IR
initialization failed, but kernel runs under KVM and no apic id is
greater than 255 (if there is one spec requires BIOS to move to x2apic
mode before starting an OS).

Signed-off-by: Gleb Natapov <gleb@redhat.com>
Acked-by: Suresh Siddha <suresh.b.siddha@intel.com>
---

This is based on x86 tree commit ff78d570fae with Ingo comments
addressed (comment formatting, print error on alloc failure, shorter
error messages).

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 0a1c283..a79e8a9 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -49,6 +49,7 @@
 #include <asm/mtrr.h>
 #include <asm/smp.h>
 #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
 unsigned int num_processors;
 
@@ -1361,52 +1362,76 @@ void enable_x2apic(void)
 }
 #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
 {
 #ifdef CONFIG_INTR_REMAP
 	int ret;
-	unsigned long flags;
-	struct IO_APIC_route_entry **ioapic_entries = NULL;
 
 	ret = dmar_table_init();
 	if (ret) {
 		pr_debug("dmar_table_init() failed with %d:\n", ret);
-		goto ir_failed;
+		return 0;
 	}
 
 	if (!intr_remapping_supported()) {
 		pr_debug("intr-remapping not supported\n");
-		goto ir_failed;
+		return 0;
 	}
 
-
 	if (!x2apic_preenabled && skip_ioapic_setup) {
 		pr_info("Skipped enabling intr-remap because of skipping "
 			"io-apic setup\n");
-		return;
+		return 0;
 	}
 
+	if (enable_intr_remapping(x2apic_supported()))
+		return 0;
+
+	pr_info("Enabled Interrupt-remapping\n");
+
+	return 1;
+
+#endif
+	return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+	unsigned long flags;
+	struct IO_APIC_route_entry **ioapic_entries = NULL;
+	int ret, x2apic_enabled = 0;
+
 	ioapic_entries = alloc_ioapic_entries();
 	if (!ioapic_entries) {
-		pr_info("Allocate ioapic_entries failed: %d\n", ret);
-		goto end;
+		pr_err("Allocate ioapic_entries failed\n");
+		goto out;
 	}
 
 	ret = save_IO_APIC_setup(ioapic_entries);
 	if (ret) {
 		pr_info("Saving IO-APIC state failed: %d\n", ret);
-		goto end;
+		goto out;
 	}
 
 	local_irq_save(flags);
-	mask_IO_APIC_setup(ioapic_entries);
 	mask_8259A();
+	mask_IO_APIC_setup(ioapic_entries);
 
-	ret = enable_intr_remapping(x2apic_supported());
-	if (ret)
-		goto end_restore;
+	ret = enable_IR();
+	if (!ret) {
+		/* IR is required if there is APIC ID > 255 even when running
+		 * under KVM
+		 */
+		if (max_physical_apicid > 255 || !kvm_para_available())
+			goto nox2apic;
+		/*
+		 * without IR all CPUs can be addressed by IOAPIC/MSI
+		 * only in physical mode
+		 */
+		x2apic_phys = 1;
+	}
 
-	pr_info("Enabled Interrupt-remapping\n");
+	x2apic_enabled = 1;
 
 	if (x2apic_supported() && !x2apic_mode) {
 		x2apic_mode = 1;
@@ -1414,41 +1439,25 @@ void __init enable_IR_x2apic(void)
 		pr_info("Enabled x2apic\n");
 	}
 
-end_restore:
-	if (ret)
-		/*
-		 * IR enabling failed
-		 */
+nox2apic:
+	if (!ret) /* IR enabling failed */
 		restore_IO_APIC_setup(ioapic_entries);
-
 	unmask_8259A();
 	local_irq_restore(flags);
 
-end:
+out:
 	if (ioapic_entries)
 		free_ioapic_entries(ioapic_entries);
 
-	if (!ret)
+	if (x2apic_enabled)
 		return;
 
-ir_failed:
 	if (x2apic_preenabled)
-		panic("x2apic enabled by bios. But IR enabling failed");
+		panic("x2apic: enabled by BIOS but kernel init failed.");
 	else if (cpu_has_x2apic)
-		pr_info("Not enabling x2apic,Intr-remapping\n");
-#else
-	if (!cpu_has_x2apic)
-		return;
-
-	if (x2apic_preenabled)
-		panic("x2apic enabled prior OS handover,"
-		      " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
-#endif
-
-	return;
+		pr_info("Not enabling x2apic, Intr-remapping init failed.\n");
 }
 
-
 #ifdef CONFIG_X86_64
 /*
  * Detect and enable local APICs on non-SMP boards.
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index bc3e880..f3b1037 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -50,11 +50,11 @@ static struct apic *apic_probe[] __initdata = {
 void __init default_setup_apic_routing(void)
 {
 #ifdef CONFIG_X86_X2APIC
-	if (x2apic_mode && (apic != &apic_x2apic_phys &&
+	if (x2apic_mode
 #ifdef CONFIG_X86_UV
-		       apic != &apic_x2apic_uv_x &&
+		       && apic != &apic_x2apic_uv_x
 #endif
-		       apic != &apic_x2apic_cluster)) {
+		       ) {
 		if (x2apic_phys)
 			apic = &apic_x2apic_phys;
 		else
--
			Gleb.

[PATCH] enable x2APIC without interrupt remapping under KVM

[Gleb Natapov]

KVM would like to provide x2APIC interface to a guest without emulating
interrupt remapping device. The reason KVM prefers guest to use x2APIC
is that x2APIC interface is better virtualizable and provides better
performance than mmio xAPIC interface:

- msr exits are faster than mmio (no page table walk, emulation)
- no need to read back ICR to look at the busy bit
- one 64 bit ICR write instead of two 32 bit writes
- shared code with the Hyper-V paravirt interface

Included patch changes x2APIC enabling logic to enable it even if IR
initialization failed, but kernel runs under KVM and no apic id is
greater than 255 (if there is one spec requires BIOS to move to x2apic
mode before starting an OS).

Signed-off-by: Gleb Natapov <gleb@redhat.com>
Acked-by: Suresh Siddha <suresh.b.siddha@intel.com>
---

This is based on x86 tree commit d7ccd190e5107 with Ingo comments
addressed (comment formatting, print error on alloc failure).

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 0a1c283..a010af4 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -49,6 +49,7 @@
 #include <asm/mtrr.h>
 #include <asm/smp.h>
 #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
 unsigned int num_processors;
 
@@ -1361,52 +1362,76 @@ void enable_x2apic(void)
 }
 #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
 {
 #ifdef CONFIG_INTR_REMAP
 	int ret;
-	unsigned long flags;
-	struct IO_APIC_route_entry **ioapic_entries = NULL;
 
 	ret = dmar_table_init();
 	if (ret) {
 		pr_debug("dmar_table_init() failed with %d:\n", ret);
-		goto ir_failed;
+		return 0;
 	}
 
 	if (!intr_remapping_supported()) {
 		pr_debug("intr-remapping not supported\n");
-		goto ir_failed;
+		return 0;
 	}
 
-
 	if (!x2apic_preenabled && skip_ioapic_setup) {
 		pr_info("Skipped enabling intr-remap because of skipping "
 			"io-apic setup\n");
-		return;
+		return 0;
 	}
 
+	if (enable_intr_remapping(x2apic_supported()))
+		return 0;
+
+	pr_info("Enabled Interrupt-remapping\n");
+
+	return 1;
+
+#endif
+	return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+	unsigned long flags;
+	struct IO_APIC_route_entry **ioapic_entries = NULL;
+	int ret, x2apic_enabled = 0;
+
 	ioapic_entries = alloc_ioapic_entries();
 	if (!ioapic_entries) {
-		pr_info("Allocate ioapic_entries failed: %d\n", ret);
-		goto end;
+		pr_err("Allocate ioapic_entries failed\n");
+		goto out;
 	}
 
 	ret = save_IO_APIC_setup(ioapic_entries);
 	if (ret) {
 		pr_info("Saving IO-APIC state failed: %d\n", ret);
-		goto end;
+		goto out;
 	}
 
 	local_irq_save(flags);
-	mask_IO_APIC_setup(ioapic_entries);
 	mask_8259A();
+	mask_IO_APIC_setup(ioapic_entries);
 
-	ret = enable_intr_remapping(x2apic_supported());
-	if (ret)
-		goto end_restore;
+	ret = enable_IR();
+	if (!ret) {
+		/* IR is required if there is APIC ID > 255 even when running
+		 * under KVM
+		*/
+		if (max_physical_apicid > 255 || !kvm_para_available())
+			goto nox2apic;
+		/*
+		 * without IR all CPUs can be addressed by IOAPIC/MSI
+		 * only in physical mode
+		 */
+		x2apic_phys = 1;
+	}
 
-	pr_info("Enabled Interrupt-remapping\n");
+	x2apic_enabled = 1;
 
 	if (x2apic_supported() && !x2apic_mode) {
 		x2apic_mode = 1;
@@ -1414,41 +1439,27 @@ void __init enable_IR_x2apic(void)
 		pr_info("Enabled x2apic\n");
 	}
 
-end_restore:
-	if (ret)
-		/*
-		 * IR enabling failed
-		 */
+nox2apic:
+	if (!ret) /* IR enabling failed */
 		restore_IO_APIC_setup(ioapic_entries);
-
 	unmask_8259A();
 	local_irq_restore(flags);
 
-end:
+out:
 	if (ioapic_entries)
 		free_ioapic_entries(ioapic_entries);
 
-	if (!ret)
+	if (x2apic_enabled)
 		return;
 
-ir_failed:
-	if (x2apic_preenabled)
-		panic("x2apic enabled by bios. But IR enabling failed");
-	else if (cpu_has_x2apic)
-		pr_info("Not enabling x2apic,Intr-remapping\n");
-#else
-	if (!cpu_has_x2apic)
-		return;
-
-	if (x2apic_preenabled)
-		panic("x2apic enabled prior OS handover,"
-		      " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
-#endif
-
-	return;
+	if (x2apic_preenabled) {
+		panic("x2apic enabled by bios. But IR enabling failed."
+		      " Check that CONFIG_X86_X2APIC and CONFIG_INTR_REMAP are"
+		      " enabled.");
+	} else if (cpu_has_x2apic)
+		pr_info("Not enabling x2apic, Intr-remapping\n");
 }
 
-
 #ifdef CONFIG_X86_64
 /*
  * Detect and enable local APICs on non-SMP boards.
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index bc3e880..f3b1037 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -50,11 +50,11 @@ static struct apic *apic_probe[] __initdata = {
 void __init default_setup_apic_routing(void)
 {
 #ifdef CONFIG_X86_X2APIC
-	if (x2apic_mode && (apic != &apic_x2apic_phys &&
+	if (x2apic_mode
 #ifdef CONFIG_X86_UV
-		       apic != &apic_x2apic_uv_x &&
+		       && apic != &apic_x2apic_uv_x
 #endif
-		       apic != &apic_x2apic_cluster)) {
+		       ) {
 		if (x2apic_phys)
 			apic = &apic_x2apic_phys;
 		else
--
			Gleb.

Re: [PATCH] enable x2APIC without interrupt remapping under KVM

[Ingo Molnar]

* Gleb Natapov <gleb@redhat.com> wrote:

> +	if (!ret) {
> +		/* IR is required if there is APIC ID > 255 even when running
> +		 * under KVM
> +		*/

Eeek.

> +	if (x2apic_preenabled) {
> +		panic("x2apic enabled by bios. But IR enabling failed."
> +		      " Check that CONFIG_X86_X2APIC and CONFIG_INTR_REMAP are"
> +		      " enabled.");

We dont want to break such messages mid-line, it makes it hard to 
git-grep for them. Also, the '. But' looks weird.

	Ingo

Re: [PATCH] enable x2APIC without interrupt remapping under KVM

[Gleb Natapov]

On Sun, Jul 19, 2009 at 06:27:35PM +0200, Ingo Molnar wrote:
> 
> * Gleb Natapov <gleb@redhat.com> wrote:
> 
> > +	if (!ret) {
> > +		/* IR is required if there is APIC ID > 255 even when running
> > +		 * under KVM
> > +		*/
> 
> Eeek.
> 
> > +	if (x2apic_preenabled) {
> > +		panic("x2apic enabled by bios. But IR enabling failed."
> > +		      " Check that CONFIG_X86_X2APIC and CONFIG_INTR_REMAP are"
> > +		      " enabled.");
> 
> We dont want to break such messages mid-line, it makes it hard to 
> git-grep for them. Also, the '. But' looks weird.

So what should I do? Make really long line?

--
			Gleb.

Re: [PATCH] enable x2APIC without interrupt remapping under KVM

[Ingo Molnar]

* Gleb Natapov <gleb@redhat.com> wrote:

> On Sun, Jul 19, 2009 at 06:27:35PM +0200, Ingo Molnar wrote:
> > 
> > * Gleb Natapov <gleb@redhat.com> wrote:
> > 
> > > +	if (!ret) {
> > > +		/* IR is required if there is APIC ID > 255 even when running
> > > +		 * under KVM
> > > +		*/
> > 
> > Eeek.
> > 
> > > +	if (x2apic_preenabled) {
> > > +		panic("x2apic enabled by bios. But IR enabling failed."
> > > +		      " Check that CONFIG_X86_X2APIC and CONFIG_INTR_REMAP are"
> > > +		      " enabled.");
> > 
> > We dont want to break such messages mid-line, it makes it hard to 
> > git-grep for them. Also, the '. But' looks weird.
> 
> So what should I do? Make really long line?

i'd suggest short ones generally, like:

	"x2apic: enabled by BIOS but kernel init failed."

also, referring to whether the config are enabled is a bit lame - we 
are in the kernel so we should know whether they are enabled.

	Ingo

[PATCH v5] enable x2APIC without interrupt remapping under KVM

[Gleb Natapov]

KVM would like to provide x2APIC interface to a guest without emulating
interrupt remapping device. The reason KVM prefers guest to use x2APIC
is that x2APIC interface is better virtualizable and provides better
performance than mmio xAPIC interface:
    
- msr exits are faster than mmio (no page table walk, emulation)
- no need to read back ICR to look at the busy bit
- one 64 bit ICR write instead of two 32 bit writes
- shared code with the Hyper-V paravirt interface
    
Included patch changes x2APIC enabling logic to enable it even if IR
initialization failed, but kernel runs under KVM and no apic id is
greater than 255 (if there is one spec requires BIOS to move to x2apic
mode before starting an OS).

Signed-off-by: Gleb Natapov <gleb@redhat.com>
---

This version does not rely on x2apic_preenabled to determine max APIC id
since this logic will break after kexec. Use max_physical_apicid instead.
Also allow to override apic ops after call to enable_IR_x2apic() since
we may decide to use physical mode instead of cluster.

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 8c7c042..db0c07f 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -49,6 +49,7 @@
 #include <asm/mtrr.h>
 #include <asm/smp.h>
 #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
 unsigned int num_processors;
 
@@ -1363,52 +1364,74 @@ void enable_x2apic(void)
 }
 #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
 {
 #ifdef CONFIG_INTR_REMAP
 	int ret;
-	unsigned long flags;
-	struct IO_APIC_route_entry **ioapic_entries = NULL;
 
 	ret = dmar_table_init();
 	if (ret) {
 		pr_debug("dmar_table_init() failed with %d:\n", ret);
-		goto ir_failed;
+		return 0;
 	}
 
 	if (!intr_remapping_supported()) {
 		pr_debug("intr-remapping not supported\n");
-		goto ir_failed;
+		return 0;
 	}
 
-
 	if (!x2apic_preenabled && skip_ioapic_setup) {
 		pr_info("Skipped enabling intr-remap because of skipping "
 			"io-apic setup\n");
-		return;
+		return 0;
 	}
 
+	if (enable_intr_remapping(x2apic_supported()))
+		return 0;
+
+	pr_info("Enabled Interrupt-remapping\n");
+
+	return 1;
+
+#endif
+	return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+	unsigned long flags;
+	struct IO_APIC_route_entry **ioapic_entries = NULL;
+	int ret, x2apic_enabled = 0;
+
 	ioapic_entries = alloc_ioapic_entries();
 	if (!ioapic_entries) {
-		pr_info("Allocate ioapic_entries failed: %d\n", ret);
-		goto end;
+		 pr_info("Allocate ioapic_entries failed\n");
+		 goto out;
 	}
 
 	ret = save_IO_APIC_setup(ioapic_entries);
 	if (ret) {
 		pr_info("Saving IO-APIC state failed: %d\n", ret);
-		goto end;
+		goto out;
 	}
 
 	local_irq_save(flags);
-	mask_IO_APIC_setup(ioapic_entries);
 	mask_8259A();
+	mask_IO_APIC_setup(ioapic_entries);
 
-	ret = enable_intr_remapping(x2apic_supported());
-	if (ret)
-		goto end_restore;
+	ret = enable_IR();
+	if (!ret) {
+		/* IR is required if x2apic is enabled by BIOS even
+		 * when running in kvm since this indicates present
+		 * of APIC ID > 255 */
+	       	if (max_physical_apicid > 255 || !kvm_para_available())
+			goto nox2apic;
+		/* without IR all CPUs can be addressed by IOAPIC/MSI
+		 * only in physical mode */
+		x2apic_phys = 1;
+	}
 
-	pr_info("Enabled Interrupt-remapping\n");
+	x2apic_enabled = 1;
 
 	if (x2apic_supported() && !x2apic_mode) {
 		x2apic_mode = 1;
@@ -1416,41 +1439,30 @@ void __init enable_IR_x2apic(void)
 		pr_info("Enabled x2apic\n");
 	}
 
-end_restore:
-	if (ret)
-		/*
-		 * IR enabling failed
-		 */
+nox2apic:
+	if (!ret) /* IR enabling failed */
 		restore_IO_APIC_setup(ioapic_entries);
-
 	unmask_8259A();
 	local_irq_restore(flags);
 
-end:
+out:
 	if (ioapic_entries)
 		free_ioapic_entries(ioapic_entries);
 
-	if (!ret)
+	if (x2apic_enabled)
 		return;
 
-ir_failed:
-	if (x2apic_preenabled)
+	if (x2apic_preenabled) {
+#ifdef CONFIG_INTR_REMAP
 		panic("x2apic enabled by bios. But IR enabling failed");
-	else if (cpu_has_x2apic)
-		pr_info("Not enabling x2apic,Intr-remapping\n");
 #else
-	if (!cpu_has_x2apic)
-		return;
-
-	if (x2apic_preenabled)
 		panic("x2apic enabled prior OS handover,"
-		      " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
+				" enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
 #endif
-
-	return;
+	} else if (cpu_has_x2apic)
+		pr_info("Not enabling x2apic,Intr-remapping\n");
 }
 
-
 #ifdef CONFIG_X86_64
 /*
  * Detect and enable local APICs on non-SMP boards.
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index bc3e880..f3b1037 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -50,11 +50,11 @@ static struct apic *apic_probe[] __initdata = {
 void __init default_setup_apic_routing(void)
 {
 #ifdef CONFIG_X86_X2APIC
-	if (x2apic_mode && (apic != &apic_x2apic_phys &&
+	if (x2apic_mode
 #ifdef CONFIG_X86_UV
-		       apic != &apic_x2apic_uv_x &&
+		       && apic != &apic_x2apic_uv_x
 #endif
-		       apic != &apic_x2apic_cluster)) {
+		       ) {
 		if (x2apic_phys)
 			apic = &apic_x2apic_phys;
 		else
--
			Gleb.

Re: [PATCH v5] enable x2APIC without interrupt remapping under KVM

[Suresh Siddha]

On Wed, 2009-07-01 at 06:30 -0700, Gleb Natapov wrote:
> KVM would like to provide x2APIC interface to a guest without emulating
> interrupt remapping device. The reason KVM prefers guest to use x2APIC
> is that x2APIC interface is better virtualizable and provides better
> performance than mmio xAPIC interface:
>     
> - msr exits are faster than mmio (no page table walk, emulation)
> - no need to read back ICR to look at the busy bit
> - one 64 bit ICR write instead of two 32 bit writes
> - shared code with the Hyper-V paravirt interface
>     
> Included patch changes x2APIC enabling logic to enable it even if IR
> initialization failed, but kernel runs under KVM and no apic id is
> greater than 255 (if there is one spec requires BIOS to move to x2apic
> mode before starting an OS).
> 
> Signed-off-by: Gleb Natapov <gleb@redhat.com>

Acked-by: Suresh Siddha <suresh.b.siddha@intel.com>

Re: [PATCH v5] enable x2APIC without interrupt remapping under KVM

[Ingo Molnar]

* Suresh Siddha <suresh.b.siddha@intel.com> wrote:

> On Wed, 2009-07-01 at 06:30 -0700, Gleb Natapov wrote:
> > KVM would like to provide x2APIC interface to a guest without emulating
> > interrupt remapping device. The reason KVM prefers guest to use x2APIC
> > is that x2APIC interface is better virtualizable and provides better
> > performance than mmio xAPIC interface:
> >     
> > - msr exits are faster than mmio (no page table walk, emulation)
> > - no need to read back ICR to look at the busy bit
> > - one 64 bit ICR write instead of two 32 bit writes
> > - shared code with the Hyper-V paravirt interface
> >     
> > Included patch changes x2APIC enabling logic to enable it even if IR
> > initialization failed, but kernel runs under KVM and no apic id is
> > greater than 255 (if there is one spec requires BIOS to move to x2apic
> > mode before starting an OS).
> > 
> > Signed-off-by: Gleb Natapov <gleb@redhat.com>
> 
> Acked-by: Suresh Siddha <suresh.b.siddha@intel.com>

Now, since this affects core x86 APIC code non-trivially so should 
submitted to and go via the x86 tree. (Can prepare a special branch 
with just this change if KVM tree wants/needs to pull it before 
v2.6.32.)

	Ingo

[PATCH] enable x2APIC without interrupt remapping under KVM

[Gleb Natapov]

KVM would like to provide x2APIC interface to a guest without emulating
interrupt remapping device. The reason KVM prefers guest to use x2APIC
is that x2APIC interface is better virtualizable and provides better
performance than mmio xAPIC interface:
    
- msr exits are faster than mmio (no page table walk, emulation)
- no need to read back ICR to look at the busy bit
- one 64 bit ICR write instead of two 32 bit writes
- shared code with the Hyper-V paravirt interface
    
Included patch changes x2APIC enabling logic to enable it even if IR
initialization failed, but kernel runs under KVM and no apic id is
greater than 255 (if there is one spec requires BIOS to move to x2apic
mode before starting an OS).

Signed-off-by: Gleb Natapov <gleb@redhat.com>
Acked-by: Suresh Siddha <suresh.b.siddha@intel.com>
---

This is the same as v5 only rebased on x86 tree (a74d2cea).

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 0a1c283..9e0b37a 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -49,6 +49,7 @@
 #include <asm/mtrr.h>
 #include <asm/smp.h>
 #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
 unsigned int num_processors;
 
@@ -1361,52 +1362,74 @@ void enable_x2apic(void)
 }
 #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
 {
 #ifdef CONFIG_INTR_REMAP
 	int ret;
-	unsigned long flags;
-	struct IO_APIC_route_entry **ioapic_entries = NULL;
 
 	ret = dmar_table_init();
 	if (ret) {
 		pr_debug("dmar_table_init() failed with %d:\n", ret);
-		goto ir_failed;
+		return 0;
 	}
 
 	if (!intr_remapping_supported()) {
 		pr_debug("intr-remapping not supported\n");
-		goto ir_failed;
+		return 0;
 	}
 
-
 	if (!x2apic_preenabled && skip_ioapic_setup) {
 		pr_info("Skipped enabling intr-remap because of skipping "
 			"io-apic setup\n");
-		return;
+		return 0;
 	}
 
+	if (enable_intr_remapping(x2apic_supported()))
+		return 0;
+
+	pr_info("Enabled Interrupt-remapping\n");
+
+	return 1;
+
+#endif
+	return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+	unsigned long flags;
+	struct IO_APIC_route_entry **ioapic_entries = NULL;
+	int ret, x2apic_enabled = 0;
+
 	ioapic_entries = alloc_ioapic_entries();
 	if (!ioapic_entries) {
-		pr_info("Allocate ioapic_entries failed: %d\n", ret);
-		goto end;
+		 pr_info("Allocate ioapic_entries failed\n");
+		 goto out;
 	}
 
 	ret = save_IO_APIC_setup(ioapic_entries);
 	if (ret) {
 		pr_info("Saving IO-APIC state failed: %d\n", ret);
-		goto end;
+		goto out;
 	}
 
 	local_irq_save(flags);
-	mask_IO_APIC_setup(ioapic_entries);
 	mask_8259A();
+	mask_IO_APIC_setup(ioapic_entries);
 
-	ret = enable_intr_remapping(x2apic_supported());
-	if (ret)
-		goto end_restore;
+	ret = enable_IR();
+	if (!ret) {
+		/* IR is required if x2apic is enabled by BIOS even
+		 * when running in kvm since this indicates present
+		 * of APIC ID > 255 */
+	       	if (max_physical_apicid > 255 || !kvm_para_available())
+			goto nox2apic;
+		/* without IR all CPUs can be addressed by IOAPIC/MSI
+		 * only in physical mode */
+		x2apic_phys = 1;
+	}
 
-	pr_info("Enabled Interrupt-remapping\n");
+	x2apic_enabled = 1;
 
 	if (x2apic_supported() && !x2apic_mode) {
 		x2apic_mode = 1;
@@ -1414,41 +1437,30 @@ void __init enable_IR_x2apic(void)
 		pr_info("Enabled x2apic\n");
 	}
 
-end_restore:
-	if (ret)
-		/*
-		 * IR enabling failed
-		 */
+nox2apic:
+	if (!ret) /* IR enabling failed */
 		restore_IO_APIC_setup(ioapic_entries);
-
 	unmask_8259A();
 	local_irq_restore(flags);
 
-end:
+out:
 	if (ioapic_entries)
 		free_ioapic_entries(ioapic_entries);
 
-	if (!ret)
+	if (x2apic_enabled)
 		return;
 
-ir_failed:
-	if (x2apic_preenabled)
+	if (x2apic_preenabled) {
+#ifdef CONFIG_INTR_REMAP
 		panic("x2apic enabled by bios. But IR enabling failed");
-	else if (cpu_has_x2apic)
-		pr_info("Not enabling x2apic,Intr-remapping\n");
 #else
-	if (!cpu_has_x2apic)
-		return;
-
-	if (x2apic_preenabled)
 		panic("x2apic enabled prior OS handover,"
-		      " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
+				" enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
 #endif
-
-	return;
+	} else if (cpu_has_x2apic)
+		pr_info("Not enabling x2apic,Intr-remapping\n");
 }
 
-
 #ifdef CONFIG_X86_64
 /*
  * Detect and enable local APICs on non-SMP boards.
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index bc3e880..f3b1037 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -50,11 +50,11 @@ static struct apic *apic_probe[] __initdata = {
 void __init default_setup_apic_routing(void)
 {
 #ifdef CONFIG_X86_X2APIC
-	if (x2apic_mode && (apic != &apic_x2apic_phys &&
+	if (x2apic_mode
 #ifdef CONFIG_X86_UV
-		       apic != &apic_x2apic_uv_x &&
+		       && apic != &apic_x2apic_uv_x
 #endif
-		       apic != &apic_x2apic_cluster)) {
+		       ) {
 		if (x2apic_phys)
 			apic = &apic_x2apic_phys;
 		else
--
			Gleb.

Re: [PATCH] enable x2APIC without interrupt remapping under KVM

[Ingo Molnar]

* Gleb Natapov <gleb@redhat.com> wrote:

> KVM would like to provide x2APIC interface to a guest without emulating
> interrupt remapping device. The reason KVM prefers guest to use x2APIC
> is that x2APIC interface is better virtualizable and provides better
> performance than mmio xAPIC interface:
>     
> - msr exits are faster than mmio (no page table walk, emulation)
> - no need to read back ICR to look at the busy bit
> - one 64 bit ICR write instead of two 32 bit writes
> - shared code with the Hyper-V paravirt interface
>     
> Included patch changes x2APIC enabling logic to enable it even if IR
> initialization failed, but kernel runs under KVM and no apic id is
> greater than 255 (if there is one spec requires BIOS to move to x2apic
> mode before starting an OS).
> 
> Signed-off-by: Gleb Natapov <gleb@redhat.com>
> Acked-by: Suresh Siddha <suresh.b.siddha@intel.com>
> ---
> 
> This is the same as v5 only rebased on x86 tree (a74d2cea).

Looks good.

Small detail: please run the patch through scripts/checkpatch.pl and 
fix the complaints it has - all 3 details it points out seem like 
valid complaints to me (at a quick glance) that should be fixed.

Thanks,

	Ingo

[PATCH] enable x2APIC without interrupt remapping under KVM

[Gleb Natapov]

KVM would like to provide x2APIC interface to a guest without emulating
interrupt remapping device. The reason KVM prefers guest to use x2APIC
is that x2APIC interface is better virtualizable and provides better
performance than mmio xAPIC interface:
    
- msr exits are faster than mmio (no page table walk, emulation)
- no need to read back ICR to look at the busy bit
- one 64 bit ICR write instead of two 32 bit writes
- shared code with the Hyper-V paravirt interface
    
Included patch changes x2APIC enabling logic to enable it even if IR
initialization failed, but kernel runs under KVM and no apic id is
greater than 255 (if there is one spec requires BIOS to move to x2apic
mode before starting an OS).

Signed-off-by: Gleb Natapov <gleb@redhat.com>
Acked-by: Suresh Siddha <suresh.b.siddha@intel.com>
---

This is based on x86 tree commit 5b9eea3e7 with checkpatch complains
fixed.

diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 0a1c283..6e67916 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -49,6 +49,7 @@
 #include <asm/mtrr.h>
 #include <asm/smp.h>
 #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
 unsigned int num_processors;
 
@@ -1361,52 +1362,74 @@ void enable_x2apic(void)
 }
 #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
 {
 #ifdef CONFIG_INTR_REMAP
 	int ret;
-	unsigned long flags;
-	struct IO_APIC_route_entry **ioapic_entries = NULL;
 
 	ret = dmar_table_init();
 	if (ret) {
 		pr_debug("dmar_table_init() failed with %d:\n", ret);
-		goto ir_failed;
+		return 0;
 	}
 
 	if (!intr_remapping_supported()) {
 		pr_debug("intr-remapping not supported\n");
-		goto ir_failed;
+		return 0;
 	}
 
-
 	if (!x2apic_preenabled && skip_ioapic_setup) {
 		pr_info("Skipped enabling intr-remap because of skipping "
 			"io-apic setup\n");
-		return;
+		return 0;
 	}
 
+	if (enable_intr_remapping(x2apic_supported()))
+		return 0;
+
+	pr_info("Enabled Interrupt-remapping\n");
+
+	return 1;
+
+#endif
+	return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+	unsigned long flags;
+	struct IO_APIC_route_entry **ioapic_entries = NULL;
+	int ret, x2apic_enabled = 0;
+
 	ioapic_entries = alloc_ioapic_entries();
 	if (!ioapic_entries) {
-		pr_info("Allocate ioapic_entries failed: %d\n", ret);
-		goto end;
+		pr_info("Allocate ioapic_entries failed\n");
+		goto out;
 	}
 
 	ret = save_IO_APIC_setup(ioapic_entries);
 	if (ret) {
 		pr_info("Saving IO-APIC state failed: %d\n", ret);
-		goto end;
+		goto out;
 	}
 
 	local_irq_save(flags);
-	mask_IO_APIC_setup(ioapic_entries);
 	mask_8259A();
+	mask_IO_APIC_setup(ioapic_entries);
 
-	ret = enable_intr_remapping(x2apic_supported());
-	if (ret)
-		goto end_restore;
+	ret = enable_IR();
+	if (!ret) {
+		/* IR is required if x2apic is enabled by BIOS even
+		 * when running in kvm since this indicates present
+		 * of APIC ID > 255 */
+		if (max_physical_apicid > 255 || !kvm_para_available())
+			goto nox2apic;
+		/* without IR all CPUs can be addressed by IOAPIC/MSI
+		 * only in physical mode */
+		x2apic_phys = 1;
+	}
 
-	pr_info("Enabled Interrupt-remapping\n");
+	x2apic_enabled = 1;
 
 	if (x2apic_supported() && !x2apic_mode) {
 		x2apic_mode = 1;
@@ -1414,41 +1437,30 @@ void __init enable_IR_x2apic(void)
 		pr_info("Enabled x2apic\n");
 	}
 
-end_restore:
-	if (ret)
-		/*
-		 * IR enabling failed
-		 */
+nox2apic:
+	if (!ret) /* IR enabling failed */
 		restore_IO_APIC_setup(ioapic_entries);
-
 	unmask_8259A();
 	local_irq_restore(flags);
 
-end:
+out:
 	if (ioapic_entries)
 		free_ioapic_entries(ioapic_entries);
 
-	if (!ret)
+	if (x2apic_enabled)
 		return;
 
-ir_failed:
-	if (x2apic_preenabled)
+	if (x2apic_preenabled) {
+#ifdef CONFIG_INTR_REMAP
 		panic("x2apic enabled by bios. But IR enabling failed");
-	else if (cpu_has_x2apic)
-		pr_info("Not enabling x2apic,Intr-remapping\n");
 #else
-	if (!cpu_has_x2apic)
-		return;
-
-	if (x2apic_preenabled)
 		panic("x2apic enabled prior OS handover,"
-		      " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
+				" enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
 #endif
-
-	return;
+	} else if (cpu_has_x2apic)
+		pr_info("Not enabling x2apic,Intr-remapping\n");
 }
 
-
 #ifdef CONFIG_X86_64
 /*
  * Detect and enable local APICs on non-SMP boards.
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index bc3e880..f3b1037 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -50,11 +50,11 @@ static struct apic *apic_probe[] __initdata = {
 void __init default_setup_apic_routing(void)
 {
 #ifdef CONFIG_X86_X2APIC
-	if (x2apic_mode && (apic != &apic_x2apic_phys &&
+	if (x2apic_mode
 #ifdef CONFIG_X86_UV
-		       apic != &apic_x2apic_uv_x &&
+		       && apic != &apic_x2apic_uv_x
 #endif
-		       apic != &apic_x2apic_cluster)) {
+		       ) {
 		if (x2apic_phys)
 			apic = &apic_x2apic_phys;
 		else
--
			Gleb.

Re: [PATCH] enable x2APIC without interrupt remapping under KVM

[Ingo Molnar]

i've only got some small stylistic gripes:

> +void __init enable_IR_x2apic(void)
> +{
> +	unsigned long flags;
> +	struct IO_APIC_route_entry **ioapic_entries = NULL;
> +	int ret, x2apic_enabled = 0;
> +
>  	ioapic_entries = alloc_ioapic_entries();
>  	if (!ioapic_entries) {
> -		pr_info("Allocate ioapic_entries failed: %d\n", ret);
> -		goto end;
> +		pr_info("Allocate ioapic_entries failed\n");
> +		goto out;

That's probably a fatal condition so perhaps a WARN() or pr_err() 
would be more appropriate.

> +	ret = enable_IR();
> +	if (!ret) {
> +		/* IR is required if x2apic is enabled by BIOS even
> +		 * when running in kvm since this indicates present
> +		 * of APIC ID > 255 */

please use the customary (multi-line) comment style:

  /*
   * Comment .....
   * ...... goes here.
   */

specified in Documentation/CodingStyle.

> +		if (max_physical_apicid > 255 || !kvm_para_available())
> +			goto nox2apic;
> +		/* without IR all CPUs can be addressed by IOAPIC/MSI
> +		 * only in physical mode */

ditto.

> +		x2apic_phys = 1;
> +	}
>  
> -	pr_info("Enabled Interrupt-remapping\n");
> +	x2apic_enabled = 1;
>  
>  	if (x2apic_supported() && !x2apic_mode) {
>  		x2apic_mode = 1;
> @@ -1414,41 +1437,30 @@ void __init enable_IR_x2apic(void)
>  		pr_info("Enabled x2apic\n");
>  	}
>  
> -end_restore:
> -	if (ret)
> -		/*
> -		 * IR enabling failed
> -		 */
> +nox2apic:
> +	if (!ret) /* IR enabling failed */
>  		restore_IO_APIC_setup(ioapic_entries);
> -
>  	unmask_8259A();
>  	local_irq_restore(flags);
>  
> -end:
> +out:
>  	if (ioapic_entries)
>  		free_ioapic_entries(ioapic_entries);
>  
> -	if (!ret)
> +	if (x2apic_enabled)
>  		return;
>  
> -ir_failed:
> -	if (x2apic_preenabled)
> +	if (x2apic_preenabled) {
> +#ifdef CONFIG_INTR_REMAP
>  		panic("x2apic enabled by bios. But IR enabling failed");
> -	else if (cpu_has_x2apic)
> -		pr_info("Not enabling x2apic,Intr-remapping\n");
>  #else
> -	if (!cpu_has_x2apic)
> -		return;
> -
> -	if (x2apic_preenabled)
>  		panic("x2apic enabled prior OS handover,"
> -		      " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
> +				" enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
>  #endif

can this #ifdef be avoided?

> -
> -	return;
> +	} else if (cpu_has_x2apic)
> +		pr_info("Not enabling x2apic,Intr-remapping\n");

please put a space after commas.

Thanks,

	Ingo

[PATCH] enable x2APIC without interrupt remapping under KVM

[Gleb Natapov]

KVM would like to provide x2APIC interface to a guest without emulating
interrupt remapping device. The reason KVM prefers guest to use x2APIC
is that x2APIC interface is better virtualizable and provides better
performance than mmio xAPIC interface:

- msr exits are faster than mmio (no page table walk, emulation)
- no need to read back ICR to look at the busy bit
- one 64 bit ICR write instead of two 32 bit writes
- shared code with the Hyper-V paravirt interface

Included patch changes x2APIC enabling logic to enable it even if IR
initialization failed, but kernel runs under KVM and no apic id is
greater than 255 (if there is one spec requires BIOS to move to x2apic
mode before starting an OS).

Signed-off-by: Gleb Natapov <gleb@redhat.com>
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index d1430ef..7438c5c 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -260,7 +260,7 @@ config SMP
 
 config X86_X2APIC
 	bool "Support x2apic"
-	depends on X86_LOCAL_APIC && X86_64 && INTR_REMAP
+	depends on X86_LOCAL_APIC && X86_64
 	---help---
 	  This enables x2apic support on CPUs that have this feature.
 
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 8c7c042..48a63b6 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -49,6 +49,7 @@
 #include <asm/mtrr.h>
 #include <asm/smp.h>
 #include <asm/mce.h>
+#include <asm/kvm_para.h>
 
 unsigned int num_processors;
 
@@ -1363,11 +1364,10 @@ void enable_x2apic(void)
 }
 #endif /* CONFIG_X86_X2APIC */
 
-void __init enable_IR_x2apic(void)
+int __init enable_IR(void)
 {
 #ifdef CONFIG_INTR_REMAP
 	int ret;
-	unsigned long flags;
 	struct IO_APIC_route_entry **ioapic_entries = NULL;
 
 	ret = dmar_table_init();
@@ -1381,11 +1381,10 @@ void __init enable_IR_x2apic(void)
 		goto ir_failed;
 	}
 
-
 	if (!x2apic_preenabled && skip_ioapic_setup) {
 		pr_info("Skipped enabling intr-remap because of skipping "
 			"io-apic setup\n");
-		return;
+		goto ir_failed;
 	}
 
 	ioapic_entries = alloc_ioapic_entries();
@@ -1400,22 +1399,14 @@ void __init enable_IR_x2apic(void)
 		goto end;
 	}
 
-	local_irq_save(flags);
 	mask_IO_APIC_setup(ioapic_entries);
-	mask_8259A();
 
 	ret = enable_intr_remapping(x2apic_supported());
 	if (ret)
 		goto end_restore;
-
 	pr_info("Enabled Interrupt-remapping\n");
 
-	if (x2apic_supported() && !x2apic_mode) {
-		x2apic_mode = 1;
-		enable_x2apic();
-		pr_info("Enabled x2apic\n");
-	}
-
+	return 1;
 end_restore:
 	if (ret)
 		/*
@@ -1423,30 +1414,50 @@ end_restore:
 		 */
 		restore_IO_APIC_setup(ioapic_entries);
 
-	unmask_8259A();
-	local_irq_restore(flags);
-
 end:
 	if (ioapic_entries)
 		free_ioapic_entries(ioapic_entries);
 
-	if (!ret)
-		return;
-
 ir_failed:
-	if (x2apic_preenabled)
+#endif
+	return 0;
+}
+
+void __init enable_IR_x2apic(void)
+{
+	unsigned long flags;
+
+	local_irq_save(flags);
+	mask_8259A();
+
+	/* IR is required if x2apic is enabled by BIOS even when running in kvm
+	 * since this indicates present of APIC ID > 255 */
+	if (!enable_IR() && (x2apic_preenabled || !kvm_para_available()))
+		goto nox2apic;
+
+	if (x2apic_supported() && !x2apic_mode) {
+		x2apic_mode = 1;
+		enable_x2apic();
+		pr_info("Enabled x2apic\n");
+	}
+
+	unmask_8259A();
+	local_irq_restore(flags);
+	return;
+
+nox2apic:
+	unmask_8259A();
+	local_irq_restore(flags);
+
+	if (x2apic_preenabled) {
+#ifdef CONFIG_INTR_REMAP
 		panic("x2apic enabled by bios. But IR enabling failed");
-	else if (cpu_has_x2apic)
-		pr_info("Not enabling x2apic,Intr-remapping\n");
 #else
-	if (!cpu_has_x2apic)
-		return;
-
-	if (x2apic_preenabled)
 		panic("x2apic enabled prior OS handover,"
 		      " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
 #endif
-
+	} else if (cpu_has_x2apic)
+		pr_info("Not enabling x2apic,Intr-remapping\n");
 	return;
 }
 
--
			Gleb.

Re: [PATCH] enable x2APIC without interrupt remapping under KVM

[Sheng Yang]

On Sunday 28 June 2009 20:51:14 Gleb Natapov wrote:
 > KVM would like to provide x2APIC interface to a guest without emulating
 > interrupt remapping device. The reason KVM prefers guest to use x2APIC
 > is that x2APIC interface is better virtualizable and provides better
 > performance than mmio xAPIC interface:
 >
 > - msr exits are faster than mmio (no page table walk, emulation)
 > - no need to read back ICR to look at the busy bit
 > - one 64 bit ICR write instead of two 32 bit writes
 > - shared code with the Hyper-V paravirt interface
 >
 > Included patch changes x2APIC enabling logic to enable it even if IR
 > initialization failed, but kernel runs under KVM and no apic id is
 > greater than 255 (if there is one spec requires BIOS to move to x2apic
 > mode before starting an OS).
 
[Resend, sorry for html noise...]

Add Suresh here. And some comments and unclear points below.

> Signed-off-by: Gleb Natapov <gleb@redhat.com>
 > diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
 > index d1430ef..7438c5c 100644
 > --- a/arch/x86/Kconfig
 > +++ b/arch/x86/Kconfig
 > @@ -260,7 +260,7 @@ config SMP
 >
 > config X86_X2APIC
 > bool "Support x2apic"
 > - depends on X86_LOCAL_APIC && X86_64 && INTR_REMAP
 > + depends on X86_LOCAL_APIC && X86_64
 > ---help---
 > This enables x2apic support on CPUs that have this feature.
 

We may need to note x2apic won't work without interrupt remapping or as a 
guest in KVM here?
 >
 > diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
 > index 8c7c042..48a63b6 100644
 > --- a/arch/x86/kernel/apic/apic.c
 > +++ b/arch/x86/kernel/apic/apic.c
 > @@ -49,6 +49,7 @@
 > #include <asm/mtrr.h>
 > #include <asm/smp.h>
 > #include <asm/mce.h>
 > +#include <asm/kvm_para.h>
 >
 > unsigned int num_processors;
 >
 > @@ -1363,11 +1364,10 @@ void enable_x2apic(void)
 > }
 > #endif /* CONFIG_X86_X2APIC */
 >
 > -void __init enable_IR_x2apic(void)
 > +int __init enable_IR(void)
 > {
 > #ifdef CONFIG_INTR_REMAP
 > int ret;
 > - unsigned long flags;
 > struct IO_APIC_route_entry **ioapic_entries = NULL;
 >
 > ret = dmar_table_init();
 > @@ -1381,11 +1381,10 @@ void __init enable_IR_x2apic(void)
 > goto ir_failed;
 > }
 >
 > -
 > if (!x2apic_preenabled && skip_ioapic_setup) {
 > pr_info("Skipped enabling intr-remap because of skipping "
 > "io-apic setup\n");
 > - return;
 > + goto ir_failed;
 > }
 >
 > ioapic_entries = alloc_ioapic_entries();
 > @@ -1400,22 +1399,14 @@ void __init enable_IR_x2apic(void)
 > goto end;
 > }
 >
 > - local_irq_save(flags);
 > mask_IO_APIC_setup(ioapic_entries);
 > - mask_8259A();
 >
 > ret = enable_intr_remapping(x2apic_supported());
 > if (ret)
 > goto end_restore;
 > -

Unrelated...

> pr_info("Enabled Interrupt-remapping\n");
 >
 > - if (x2apic_supported() && !x2apic_mode) {
 > - x2apic_mode = 1;
 > - enable_x2apic();
 > - pr_info("Enabled x2apic\n");
 > - }
 > -
 > + return 1;
 > end_restore:
 > if (ret)
 > /*
 > @@ -1423,30 +1414,50 @@ end_restore:
 > */
 > restore_IO_APIC_setup(ioapic_entries);
 >
 > - unmask_8259A();
 > - local_irq_restore(flags);
 > -
 > end:
 > if (ioapic_entries)
 > free_ioapic_entries(ioapic_entries);
 >
 > - if (!ret)
 > - return;
 > -
 > ir_failed:
 > - if (x2apic_preenabled)
 > +#endif
 > + return 0;
 > +}
 > +
 > +void __init enable_IR_x2apic(void)
 > +{
 > + unsigned long flags;
 > +
 > + local_irq_save(flags);
 > + mask_8259A();
 > +
 > + /* IR is required if x2apic is enabled by BIOS even when running in kvm
 > + * since this indicates present of APIC ID > 255 */
 > + if (!enable_IR() && (x2apic_preenabled || !kvm_para_available()))
 > + goto nox2apic;
 > +

Not quite understand the comment here. Could you explain why "since this 
indicates present of APIC ID > 255?" In another word, why enable_IR() needed 
for KVM even it would be fail at dmar_table_init()?
 
Another question is, we supposed KVM would use physical mode of x2apic, for 
IOAPIC and MSI won't support APIC ID > 255. But seems not, or I miss 
something?

-- 
regards
Yang, Sheng
 

> + if (x2apic_supported() && !x2apic_mode) {
 > + x2apic_mode = 1;
 > + enable_x2apic();
 > + pr_info("Enabled x2apic\n");
 > + }
 > +
 > + unmask_8259A();
 > + local_irq_restore(flags);
 > + return;
 > +
 > +nox2apic:
 > + unmask_8259A();
 > + local_irq_restore(flags);
 > +
 > + if (x2apic_preenabled) {
 > +#ifdef CONFIG_INTR_REMAP
 > panic("x2apic enabled by bios. But IR enabling failed");
 > - else if (cpu_has_x2apic)
 > - pr_info("Not enabling x2apic,Intr-remapping\n");
 > #else
 > - if (!cpu_has_x2apic)
 > - return;
 > -
 > - if (x2apic_preenabled)
 > panic("x2apic enabled prior OS handover,"
 > " enable CONFIG_X86_X2APIC, CONFIG_INTR_REMAP");
 > #endif
 > -
 > + } else if (cpu_has_x2apic)
 > + pr_info("Not enabling x2apic,Intr-remapping\n");
 > return;
 > }
 >
 > --
 > Gleb.
 > --
 > To unsubscribe from this list: send the line "unsubscribe kvm" in
 > the body of a message to majordomo@vger.kernel.org
 > More majordomo info at http://vger.kernel.org/majordomo-info.html
 

Re: [PATCH] enable x2APIC without interrupt remapping under KVM

[Gleb Natapov]

On Mon, Jun 29, 2009 at 03:49:45PM +0800, Sheng Yang wrote:
> > Signed-off-by: Gleb Natapov <gleb@redhat.com>
> > diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> > index d1430ef..7438c5c 100644
> > --- a/arch/x86/Kconfig
> > +++ b/arch/x86/Kconfig
> > @@ -260,7 +260,7 @@ config SMP
> >
> >  config X86_X2APIC
> >  	bool "Support x2apic"
> > -	depends on X86_LOCAL_APIC && X86_64 && INTR_REMAP
> > +	depends on X86_LOCAL_APIC && X86_64
> >  	---help---
> >  	  This enables x2apic support on CPUs that have this feature.
> 
> We may need to note x2apic won't work without interrupt remapping or as a 
> guest in KVM here?
Will add help text to say this.

> >
> > diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
> > index 8c7c042..48a63b6 100644
> > --- a/arch/x86/kernel/apic/apic.c
> > +++ b/arch/x86/kernel/apic/apic.c
> > @@ -49,6 +49,7 @@
> >  #include <asm/mtrr.h>
> >  #include <asm/smp.h>
> >  #include <asm/mce.h>
> > +#include <asm/kvm_para.h>
> >
> >  unsigned int num_processors;
> >
> > @@ -1363,11 +1364,10 @@ void enable_x2apic(void)
> >  }
> >  #endif /* CONFIG_X86_X2APIC */
> >
> > -void __init enable_IR_x2apic(void)
> > +int __init enable_IR(void)
> >  {
> >  #ifdef CONFIG_INTR_REMAP
> >  	int ret;
> > -	unsigned long flags;
> >  	struct IO_APIC_route_entry **ioapic_entries = NULL;
> >
> >  	ret = dmar_table_init();
> > @@ -1381,11 +1381,10 @@ void __init enable_IR_x2apic(void)
> >  		goto ir_failed;
> >  	}
> >
> > -
> >  	if (!x2apic_preenabled && skip_ioapic_setup) {
> >  		pr_info("Skipped enabling intr-remap because of skipping "
> >  			"io-apic setup\n");
> > -		return;
> > +		goto ir_failed;
> >  	}
> >
> >  	ioapic_entries = alloc_ioapic_entries();
> > @@ -1400,22 +1399,14 @@ void __init enable_IR_x2apic(void)
> >  		goto end;
> >  	}
> >
> > -	local_irq_save(flags);
> >  	mask_IO_APIC_setup(ioapic_entries);
> > -	mask_8259A();
> >
> >  	ret = enable_intr_remapping(x2apic_supported());
> >  	if (ret)
> >  		goto end_restore;
> > -
> 
> Unrelated...
> 
> >  	pr_info("Enabled Interrupt-remapping\n");
> >
> > -	if (x2apic_supported() && !x2apic_mode) {
> > -		x2apic_mode = 1;
> > -		enable_x2apic();
> > -		pr_info("Enabled x2apic\n");
> > -	}
> > -
> > +	return 1;
> >  end_restore:
> >  	if (ret)
> >  		/*
> > @@ -1423,30 +1414,50 @@ end_restore:
> >  		 */
> >  		restore_IO_APIC_setup(ioapic_entries);
> >
> > -	unmask_8259A();
> > -	local_irq_restore(flags);
> > -
> >  end:
> >  	if (ioapic_entries)
> >  		free_ioapic_entries(ioapic_entries);
> >
> > -	if (!ret)
> > -		return;
> > -
> >  ir_failed:
> > -	if (x2apic_preenabled)
> > +#endif
> > +	return 0;
> > +}
> > +
> > +void __init enable_IR_x2apic(void)
> > +{
> > +	unsigned long flags;
> > +
> > +	local_irq_save(flags);
> > +	mask_8259A();
> > +
> > +	/* IR is required if x2apic is enabled by BIOS even when running in kvm
> > +	 * since this indicates present of APIC ID > 255 */
> > +	if (!enable_IR() && (x2apic_preenabled || !kvm_para_available()))
> > +		goto nox2apic;
> > +
> 
> Not quite understand the comment here. Could you explain why "since this 
> indicates present of APIC ID > 255?" In another word, why enable_IR() needed 
> for KVM even it would be fail at dmar_table_init()?
> 
The x2APIC doc says:

  The default will be for the BIOS to pass the control to the OS with the
  local x2APICs in xAPIC mode if all x2APIC IDs reported by CPUID.0BH:EDX
  are less than 255, and in x2APIC mode if there are any logical processor
  reporting its x2APIC ID at 255 or greater.

So x2apic_preenabled will be true if there is an APIC ID greater then 255
and in this case we don't want to allow x2APIC use without IR even in KVM
for the reason you mention: IOAPIC and MSI will not be able to address
all CPUs. The reason to try to enable IR first is so that when KVM will
want to support more then 255 CPUs it will be possible to implement IR
device and use existing kernel without additional changes.

> Another question is, we supposed KVM would use physical mode of x2apic, for 
> IOAPIC and MSI won't support APIC ID > 255. But seems not, or I miss 
> something?
> 
Good point. The patch should be fixed to require physical mode for x2apic.
 
--
			Gleb.