[PATCH 1/2] vsmp: Fix number of CPUs when vsmp is disabled

[PATCH 1/2] vsmp: Fix number of CPUs when vsmp is disabled

[Ido Yariv]

From: Shai Fultheim <shai@scalemp.com>

In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
number of CPUs of the first board.

Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, fixed minor coding-style issues]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
 arch/x86/kernel/vsmp_64.c |   48 +++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 48 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index a1d804b..61571fd 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
+#include <linux/smp.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -22,6 +23,8 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 
+#define TOPOLOGY_REGISTER_OFFSET 0x10
+
 #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
 /*
  * Interrupt control on vSMPowered systems:
@@ -149,12 +152,57 @@ int is_vsmp_box(void)
 	return 0;
 }
 #endif
+
+#ifdef CONFIG_X86_VSMP
+
+static void __init vsmp_cap_cpus(void)
+{
+	/* VSMP is enabled, no need to cap cpus */
+}
+
+#else
+
+static void __init vsmp_cap_cpus(void)
+{
+	/*
+	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
+	 * ones present in the first board, unless explicitly overridden by
+	 * setup_max_cpus
+	 */
+	if (setup_max_cpus == NR_CPUS) {
+		void __iomem *address;
+		unsigned int cfg, topology, node_shift, maxcpus;
+
+		/* Read the vSMP Foundation topology register */
+		cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
+		address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
+		if (WARN_ON(!address))
+			return;
+
+		topology = readl(address);
+		node_shift = (topology >> 16) & 0x7;
+		if (!node_shift)
+			/* The value 0 should be decoded as 8 */
+			node_shift = 8;
+		maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
+
+		printk(KERN_INFO "vSMP CTL: Capping CPUs to %d "
+				 "(CONFIG_X86_VSMP is unset)\n", maxcpus);
+		setup_max_cpus = maxcpus;
+		early_iounmap(address, 4);
+	}
+}
+
+#endif
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	vsmp_cap_cpus();
+
 	set_vsmp_pv_ops();
 	return;
 }
-- 
1.7.7.6

[PATCH 2/2] vsmp: Ignore IOAPIC IRQ affinity if possible

[Ido Yariv]

From: Ravikiran Thirumalai <ravikiran.thirumalai@amd.com>

vSMP can route interrupts more optimally based on internal knowledge the
OS does not have. In order to support this optimization, all CPUs must
be able to handle all possible IOAPIC interrupts.

Fix this by setting the vector allocation domain for all CPUs and by
enabling this feature in vSMP.

Signed-off-by: Ravikiran Thirumalai <ravikiran.thirumalai@amd.com>
Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, simplified, and reworded the commit message]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
 arch/x86/kernel/apic/probe_64.c |   10 ++++++++++
 arch/x86/kernel/vsmp_64.c       |   15 +++++++++++----
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index 3fe9866..0f96703 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -29,6 +29,15 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
 }
 
 /*
+ * In vSMP, all cpus should be capable of handling interrupts, regardless of
+ * the apic used.
+ */
+static void fill_vector_allocation_domain(int cpu, struct cpumask *retmask)
+{
+	cpumask_setall(retmask);
+}
+
+/*
  * Check the APIC IDs in bios_cpu_apicid and choose the APIC mode.
  */
 void __init default_setup_apic_routing(void)
@@ -51,6 +60,7 @@ void __init default_setup_apic_routing(void)
 	if (is_vsmp_box()) {
 		/* need to update phys_pkg_id */
 		apic->phys_pkg_id = apicid_phys_pkg_id;
+		apic->vector_allocation_domain = fill_vector_allocation_domain;
 	}
 }
 
diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 61571fd..c91b08f 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -16,6 +16,7 @@
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
 #include <linux/smp.h>
+#include <linux/irq.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -95,6 +96,13 @@ static void __init set_vsmp_pv_ops(void)
 	ctl = readl(address + 4);
 	printk(KERN_INFO "vSMP CTL: capabilities:0x%08x  control:0x%08x\n",
 	       cap, ctl);
+	/*
+	 * If possible, let the vSMP foundation route the interrupt optimally */
+	if (cap & ctl & BIT(8)) {
+		ctl &= ~BIT(8);
+		no_irq_affinity = 1;
+	}
+
 	if (cap & ctl & (1 << 4)) {
 		/* Setup irq ops and turn on vSMP  IRQ fastpath handling */
 		pv_irq_ops.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable);
@@ -102,12 +110,11 @@ static void __init set_vsmp_pv_ops(void)
 		pv_irq_ops.save_fl  = PV_CALLEE_SAVE(vsmp_save_fl);
 		pv_irq_ops.restore_fl  = PV_CALLEE_SAVE(vsmp_restore_fl);
 		pv_init_ops.patch = vsmp_patch;
-
 		ctl &= ~(1 << 4);
-		writel(ctl, address + 4);
-		ctl = readl(address + 4);
-		printk(KERN_INFO "vSMP CTL: control set to:0x%08x\n", ctl);
 	}
+	writel(ctl, address + 4);
+	ctl = readl(address + 4);
+	printk(KERN_INFO "vSMP CTL: control set to:0x%08x\n", ctl);
 
 	early_iounmap(address, 8);
 }
-- 
1.7.7.6

[PATCH v2 1/2] vsmp: Fix number of CPUs when vsmp is disabled

[Ido Yariv]

From: Shai Fultheim <shai@scalemp.com>

In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
number of CPUs of the first board.

Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, fixed minor coding-style issues]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
 arch/x86/kernel/vsmp_64.c |   48 +++++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 48 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index a1d804b..61571fd 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
+#include <linux/smp.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -22,6 +23,8 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 
+#define TOPOLOGY_REGISTER_OFFSET 0x10
+
 #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
 /*
  * Interrupt control on vSMPowered systems:
@@ -149,12 +152,57 @@ int is_vsmp_box(void)
 	return 0;
 }
 #endif
+
+#ifdef CONFIG_X86_VSMP
+
+static void __init vsmp_cap_cpus(void)
+{
+	/* VSMP is enabled, no need to cap cpus */
+}
+
+#else
+
+static void __init vsmp_cap_cpus(void)
+{
+	/*
+	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
+	 * ones present in the first board, unless explicitly overridden by
+	 * setup_max_cpus
+	 */
+	if (setup_max_cpus == NR_CPUS) {
+		void __iomem *address;
+		unsigned int cfg, topology, node_shift, maxcpus;
+
+		/* Read the vSMP Foundation topology register */
+		cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
+		address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
+		if (WARN_ON(!address))
+			return;
+
+		topology = readl(address);
+		node_shift = (topology >> 16) & 0x7;
+		if (!node_shift)
+			/* The value 0 should be decoded as 8 */
+			node_shift = 8;
+		maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
+
+		printk(KERN_INFO "vSMP CTL: Capping CPUs to %d "
+				 "(CONFIG_X86_VSMP is unset)\n", maxcpus);
+		setup_max_cpus = maxcpus;
+		early_iounmap(address, 4);
+	}
+}
+
+#endif
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	vsmp_cap_cpus();
+
 	set_vsmp_pv_ops();
 	return;
 }
-- 
1.7.7.6

[PATCH v2 2/2] vsmp: Ignore IOAPIC IRQ affinity if possible

[Ido Yariv]

From: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>

vSMP can route interrupts more optimally based on internal knowledge the
OS does not have. In order to support this optimization, all CPUs must
be able to handle all possible IOAPIC interrupts.

Fix this by setting the vector allocation domain for all CPUs and by
enabling this feature in vSMP.

Signed-off-by: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, simplified, and reworded the commit message]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
Changes from v1:
- Updated Kiran's email address

 arch/x86/kernel/apic/probe_64.c |   10 ++++++++++
 arch/x86/kernel/vsmp_64.c       |   15 +++++++++++----
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index 3fe9866..0f96703 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -29,6 +29,15 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
 }
 
 /*
+ * In vSMP, all cpus should be capable of handling interrupts, regardless of
+ * the apic used.
+ */
+static void fill_vector_allocation_domain(int cpu, struct cpumask *retmask)
+{
+	cpumask_setall(retmask);
+}
+
+/*
  * Check the APIC IDs in bios_cpu_apicid and choose the APIC mode.
  */
 void __init default_setup_apic_routing(void)
@@ -51,6 +60,7 @@ void __init default_setup_apic_routing(void)
 	if (is_vsmp_box()) {
 		/* need to update phys_pkg_id */
 		apic->phys_pkg_id = apicid_phys_pkg_id;
+		apic->vector_allocation_domain = fill_vector_allocation_domain;
 	}
 }
 
diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 61571fd..c91b08f 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -16,6 +16,7 @@
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
 #include <linux/smp.h>
+#include <linux/irq.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -95,6 +96,13 @@ static void __init set_vsmp_pv_ops(void)
 	ctl = readl(address + 4);
 	printk(KERN_INFO "vSMP CTL: capabilities:0x%08x  control:0x%08x\n",
 	       cap, ctl);
+	/*
+	 * If possible, let the vSMP foundation route the interrupt optimally */
+	if (cap & ctl & BIT(8)) {
+		ctl &= ~BIT(8);
+		no_irq_affinity = 1;
+	}
+
 	if (cap & ctl & (1 << 4)) {
 		/* Setup irq ops and turn on vSMP  IRQ fastpath handling */
 		pv_irq_ops.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable);
@@ -102,12 +110,11 @@ static void __init set_vsmp_pv_ops(void)
 		pv_irq_ops.save_fl  = PV_CALLEE_SAVE(vsmp_save_fl);
 		pv_irq_ops.restore_fl  = PV_CALLEE_SAVE(vsmp_restore_fl);
 		pv_init_ops.patch = vsmp_patch;
-
 		ctl &= ~(1 << 4);
-		writel(ctl, address + 4);
-		ctl = readl(address + 4);
-		printk(KERN_INFO "vSMP CTL: control set to:0x%08x\n", ctl);
 	}
+	writel(ctl, address + 4);
+	ctl = readl(address + 4);
+	printk(KERN_INFO "vSMP CTL: control set to:0x%08x\n", ctl);
 
 	early_iounmap(address, 8);
 }
-- 
1.7.7.6

Re: [PATCH v2 1/2] vsmp: Fix number of CPUs when vsmp is disabled

[Ido Yariv]

Hi,

On Mon, Apr 16, 2012 at 10:39 AM, Ido Yariv <ido@wizery.com> wrote:
> From: Shai Fultheim <shai@scalemp.com>
>
> In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
> number of CPUs of the first board.
>
> Signed-off-by: Shai Fultheim <shai@scalemp.com>
> [ido@wizery.com: rebased, fixed minor coding-style issues]
> Signed-off-by: Ido Yariv <ido@wizery.com>
> ---
>  arch/x86/kernel/vsmp_64.c |   48 +++++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 48 insertions(+), 0 deletions(-)

Any thoughts on this set?

Thanks,
Ido.

RE: [PATCH v2 1/2] vsmp: Fix number of CPUs when vsmp is disabled

[Shai Fultheim (Shai@ScaleMP.com)]

I'm all in favor of this - can we get this in, please ?


--Shai 


-----Original Message-----
From: Ido Yariv [mailto:ido@wizery.com] 
Sent: Sunday, April 29, 2012 00:33
To: linux-kernel@vger.kernel.org; Thomas Gleixner; Ingo Molnar; H. Peter Anvin
Cc: Shai Fultheim (Shai@ScaleMP.com); Ido Yariv
Subject: Re: [PATCH v2 1/2] vsmp: Fix number of CPUs when vsmp is disabled

Hi,

On Mon, Apr 16, 2012 at 10:39 AM, Ido Yariv <ido@wizery.com> wrote:
> From: Shai Fultheim <shai@scalemp.com>
>
> In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
> number of CPUs of the first board.
>
> Signed-off-by: Shai Fultheim <shai@scalemp.com>
> [ido@wizery.com: rebased, fixed minor coding-style issues]
> Signed-off-by: Ido Yariv <ido@wizery.com>
> ---
>  arch/x86/kernel/vsmp_64.c |   48 +++++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 48 insertions(+), 0 deletions(-)

Any thoughts on this set?

Thanks,
Ido.

Re: [PATCH v2 2/2] vsmp: Ignore IOAPIC IRQ affinity if possible

[Ingo Molnar]

* Ido Yariv <ido@wizery.com> wrote:

> From: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
> 
> vSMP can route interrupts more optimally based on internal knowledge the
> OS does not have. In order to support this optimization, all CPUs must
> be able to handle all possible IOAPIC interrupts.
> 
> Fix this by setting the vector allocation domain for all CPUs and by
> enabling this feature in vSMP.
> 
> Signed-off-by: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
> Signed-off-by: Shai Fultheim <shai@scalemp.com>
> [ido@wizery.com: rebased, simplified, and reworded the commit message]
> Signed-off-by: Ido Yariv <ido@wizery.com>
> ---
> Changes from v1:
> - Updated Kiran's email address
> 
>  arch/x86/kernel/apic/probe_64.c |   10 ++++++++++
>  arch/x86/kernel/vsmp_64.c       |   15 +++++++++++----
>  2 files changed, 21 insertions(+), 4 deletions(-)
> 
> diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
> index 3fe9866..0f96703 100644
> --- a/arch/x86/kernel/apic/probe_64.c
> +++ b/arch/x86/kernel/apic/probe_64.c
> @@ -29,6 +29,15 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
>  }
>  
>  /*
> + * In vSMP, all cpus should be capable of handling interrupts, regardless of
> + * the apic used.
> + */
> +static void fill_vector_allocation_domain(int cpu, struct cpumask *retmask)
> +{
> +	cpumask_setall(retmask);
> +}
> +
> +/*
>   * Check the APIC IDs in bios_cpu_apicid and choose the APIC mode.
>   */
>  void __init default_setup_apic_routing(void)
> @@ -51,6 +60,7 @@ void __init default_setup_apic_routing(void)
>  	if (is_vsmp_box()) {
>  		/* need to update phys_pkg_id */
>  		apic->phys_pkg_id = apicid_phys_pkg_id;
> +		apic->vector_allocation_domain = fill_vector_allocation_domain;
>  	}
>  }

This is_vsmp_box() special case should really move into its own 
apic handler.

Thanks,

	Ingo

Re: [PATCH v2 1/2] vsmp: Fix number of CPUs when vsmp is disabled

[Ingo Molnar]

* Ido Yariv <ido@wizery.com> wrote:

> From: Shai Fultheim <shai@scalemp.com>
> 
> In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
> number of CPUs of the first board.
> 
> Signed-off-by: Shai Fultheim <shai@scalemp.com>
> [ido@wizery.com: rebased, fixed minor coding-style issues]
> Signed-off-by: Ido Yariv <ido@wizery.com>
> ---
>  arch/x86/kernel/vsmp_64.c |   48 +++++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 48 insertions(+), 0 deletions(-)
> 
> diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
> index a1d804b..61571fd 100644
> --- a/arch/x86/kernel/vsmp_64.c
> +++ b/arch/x86/kernel/vsmp_64.c
> @@ -15,6 +15,7 @@
>  #include <linux/init.h>
>  #include <linux/pci_ids.h>
>  #include <linux/pci_regs.h>
> +#include <linux/smp.h>
>  
>  #include <asm/apic.h>
>  #include <asm/pci-direct.h>
> @@ -22,6 +23,8 @@
>  #include <asm/paravirt.h>
>  #include <asm/setup.h>
>  
> +#define TOPOLOGY_REGISTER_OFFSET 0x10
> +
>  #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
>  /*
>   * Interrupt control on vSMPowered systems:
> @@ -149,12 +152,57 @@ int is_vsmp_box(void)
>  	return 0;
>  }
>  #endif
> +
> +#ifdef CONFIG_X86_VSMP
> +
> +static void __init vsmp_cap_cpus(void)
> +{
> +	/* VSMP is enabled, no need to cap cpus */
> +}
> +
> +#else


Please move this int the vsmp_cap_cpus() function and you can 
simplify the #ifdef block to:

 #ifdef CONFIG_X86_VSMP
	/* VSMP is enabled, no need to cap cpus */
	return;
 #endif

> +
> +static void __init vsmp_cap_cpus(void)
> +{
> +	/*
> +	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
> +	 * ones present in the first board, unless explicitly overridden by
> +	 * setup_max_cpus
> +	 */
> +	if (setup_max_cpus == NR_CPUS) {


you can simplify the function and save a level of indentation by 
turning this into:

	if (setup_max_cpus != NR_CPUS)
		return;


> +		void __iomem *address;
> +		unsigned int cfg, topology, node_shift, maxcpus;
> +
> +		/* Read the vSMP Foundation topology register */
> +		cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
> +		address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
> +		if (WARN_ON(!address))
> +			return;
> +
> +		topology = readl(address);
> +		node_shift = (topology >> 16) & 0x7;
> +		if (!node_shift)
> +			/* The value 0 should be decoded as 8 */
> +			node_shift = 8;
> +		maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
> +
> +		printk(KERN_INFO "vSMP CTL: Capping CPUs to %d "
> +				 "(CONFIG_X86_VSMP is unset)\n", maxcpus);

Please don't break up format strings - just leave them long and 
use pr_info().

> +		setup_max_cpus = maxcpus;
> +		early_iounmap(address, 4);
> +	}
> +}
> +
> +#endif

Looks good and useful otherwise.

Thanks,

	Ingo

RE: [PATCH v2 2/2] vsmp: Ignore IOAPIC IRQ affinity if possible

[Shai Fultheim (Shai@ScaleMP.com)]

Ingo Molnar <mingo.kernel.org@gmail.com> wrote:
> * Ido Yariv <ido@wizery.com> wrote:

> > @@ -51,6 +60,7 @@ void __init default_setup_apic_routing(void)
> >  	if (is_vsmp_box()) {
> >  		/* need to update phys_pkg_id */
> >  		apic->phys_pkg_id = apicid_phys_pkg_id;
> > +		apic->vector_allocation_domain =
> fill_vector_allocation_domain;
> >  	}
> >  }
> 
> This is_vsmp_box() special case should really move into its own
> apic handler.
> 
> Thanks,

Ingo,

vSMP Foundation virtualize multiple systems as one.  The Guest OS is using the underlying hardware APIC driver, and therefore we must allow the various APIC drivers to kick in.  When using vSMP Foundation APIC driver is selected by the kernel based on the hardware that is being virtualized, and that can be APIC, X2APIC phys, X2APIC cluster, summit, etc.

If we would to implement an APIC handler, we would not be able to use the native hardware APIC driver (unless we do an ugly hack).  

The above optimization applies to all APIC drivers.  It allow vSMP Foundation to route interrupts to the "closest" processor.  For this we need to override the vector_allocation_domain of any APIC that selected (as noted, selection depends on real HW).  

I hope this will allow you to ack our approach.

Regards,
Shai.

[PATCH v3] vsmp: Fix number of CPUs when vsmp is disabled

[Ido Yariv]

From: Shai Fultheim <shai@scalemp.com>

In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
number of CPUs of the first board.

Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, fixed minor coding-style issues]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
Changes from v2:
- Addressed Ingo's comments

 arch/x86/kernel/vsmp_64.c |   43 +++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 43 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index a1d804b..6aa0d59 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
+#include <linux/smp.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -22,6 +23,8 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 
+#define TOPOLOGY_REGISTER_OFFSET 0x10
+
 #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
 /*
  * Interrupt control on vSMPowered systems:
@@ -149,12 +152,52 @@ int is_vsmp_box(void)
 	return 0;
 }
 #endif
+
+static void __init vsmp_cap_cpus(void)
+{
+	void __iomem *address;
+	unsigned int cfg, topology, node_shift, maxcpus;
+
+#ifdef CONFIG_X86_VSMP
+	/* VSMP is enabled, no need to cap cpus */
+	return;
+#endif
+
+	/*
+	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
+	 * ones present in the first board, unless explicitly overridden by
+	 * setup_max_cpus
+	 */
+	if (setup_max_cpus != NR_CPUS)
+		return;
+
+	/* Read the vSMP Foundation topology register */
+	cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
+	address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
+	if (WARN_ON(!address))
+		return;
+
+	topology = readl(address);
+	node_shift = (topology >> 16) & 0x7;
+	if (!node_shift)
+		/* The value 0 should be decoded as 8 */
+		node_shift = 8;
+	maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
+
+	pr_info("vSMP CTL: Capping CPUs to %d (CONFIG_X86_VSMP is unset)\n",
+		maxcpus);
+	setup_max_cpus = maxcpus;
+	early_iounmap(address, 4);
+}
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	vsmp_cap_cpus();
+
 	set_vsmp_pv_ops();
 	return;
 }
-- 
1.7.6.5

[tip:x86/platform] vsmp: Fix number of CPUs when vsmp is disabled

[tip-bot for Shai Fultheim]

Commit-ID:  824a6ae65f085255464155760cb9628a2f0deb29
Gitweb:     http://git.kernel.org/tip/824a6ae65f085255464155760cb9628a2f0deb29
Author:     Shai Fultheim <shai@scalemp.com>
AuthorDate: Tue, 8 May 2012 10:34:15 +0300
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Tue, 8 May 2012 11:24:45 +0200

vsmp: Fix number of CPUs when vsmp is disabled

In case CONFIG_X86_VSMP is not set, limit the number of CPUs to
the number of CPUs of the first board.

Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, fixed minor coding-style issues]
Signed-off-by: Ido Yariv <ido@wizery.com>
Link: http://lkml.kernel.org/r/1336462455-22544-1-git-send-email-ido@wizery.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/vsmp_64.c |   43 +++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 43 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index a1d804b..6aa0d59 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
+#include <linux/smp.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -22,6 +23,8 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 
+#define TOPOLOGY_REGISTER_OFFSET 0x10
+
 #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
 /*
  * Interrupt control on vSMPowered systems:
@@ -149,12 +152,52 @@ int is_vsmp_box(void)
 	return 0;
 }
 #endif
+
+static void __init vsmp_cap_cpus(void)
+{
+	void __iomem *address;
+	unsigned int cfg, topology, node_shift, maxcpus;
+
+#ifdef CONFIG_X86_VSMP
+	/* VSMP is enabled, no need to cap cpus */
+	return;
+#endif
+
+	/*
+	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
+	 * ones present in the first board, unless explicitly overridden by
+	 * setup_max_cpus
+	 */
+	if (setup_max_cpus != NR_CPUS)
+		return;
+
+	/* Read the vSMP Foundation topology register */
+	cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
+	address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
+	if (WARN_ON(!address))
+		return;
+
+	topology = readl(address);
+	node_shift = (topology >> 16) & 0x7;
+	if (!node_shift)
+		/* The value 0 should be decoded as 8 */
+		node_shift = 8;
+	maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
+
+	pr_info("vSMP CTL: Capping CPUs to %d (CONFIG_X86_VSMP is unset)\n",
+		maxcpus);
+	setup_max_cpus = maxcpus;
+	early_iounmap(address, 4);
+}
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	vsmp_cap_cpus();
+
 	set_vsmp_pv_ops();
 	return;
 }

Re: [tip:x86/platform] vsmp: Fix number of CPUs when vsmp is disabled

[Ingo Molnar]

* tip-bot for Shai Fultheim <shai@scalemp.com> wrote:

> Commit-ID:  824a6ae65f085255464155760cb9628a2f0deb29
> Gitweb:     http://git.kernel.org/tip/824a6ae65f085255464155760cb9628a2f0deb29
> Author:     Shai Fultheim <shai@scalemp.com>
> AuthorDate: Tue, 8 May 2012 10:34:15 +0300
> Committer:  Ingo Molnar <mingo@kernel.org>
> CommitDate: Tue, 8 May 2012 11:24:45 +0200
> 
> vsmp: Fix number of CPUs when vsmp is disabled
> 
> In case CONFIG_X86_VSMP is not set, limit the number of CPUs to
> the number of CPUs of the first board.
> 
> Signed-off-by: Shai Fultheim <shai@scalemp.com>
> [ido@wizery.com: rebased, fixed minor coding-style issues]
> Signed-off-by: Ido Yariv <ido@wizery.com>
> Link: http://lkml.kernel.org/r/1336462455-22544-1-git-send-email-ido@wizery.com
> Signed-off-by: Ingo Molnar <mingo@kernel.org>
> ---
>  arch/x86/kernel/vsmp_64.c |   43 +++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 43 insertions(+), 0 deletions(-)
> 
> diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
> index a1d804b..6aa0d59 100644
> --- a/arch/x86/kernel/vsmp_64.c
> +++ b/arch/x86/kernel/vsmp_64.c
> @@ -15,6 +15,7 @@
>  #include <linux/init.h>
>  #include <linux/pci_ids.h>
>  #include <linux/pci_regs.h>
> +#include <linux/smp.h>
>  
>  #include <asm/apic.h>
>  #include <asm/pci-direct.h>
> @@ -22,6 +23,8 @@
>  #include <asm/paravirt.h>
>  #include <asm/setup.h>
>  
> +#define TOPOLOGY_REGISTER_OFFSET 0x10
> +
>  #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
>  /*
>   * Interrupt control on vSMPowered systems:
> @@ -149,12 +152,52 @@ int is_vsmp_box(void)
>  	return 0;
>  }
>  #endif
> +
> +static void __init vsmp_cap_cpus(void)
> +{
> +	void __iomem *address;
> +	unsigned int cfg, topology, node_shift, maxcpus;
> +
> +#ifdef CONFIG_X86_VSMP
> +	/* VSMP is enabled, no need to cap cpus */
> +	return;
> +#endif
> +
> +	/*
> +	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
> +	 * ones present in the first board, unless explicitly overridden by
> +	 * setup_max_cpus
> +	 */
> +	if (setup_max_cpus != NR_CPUS)
> +		return;
> +
> +	/* Read the vSMP Foundation topology register */
> +	cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
> +	address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
> +	if (WARN_ON(!address))
> +		return;
> +
> +	topology = readl(address);
> +	node_shift = (topology >> 16) & 0x7;
> +	if (!node_shift)
> +		/* The value 0 should be decoded as 8 */
> +		node_shift = 8;
> +	maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
> +
> +	pr_info("vSMP CTL: Capping CPUs to %d (CONFIG_X86_VSMP is unset)\n",
> +		maxcpus);
> +	setup_max_cpus = maxcpus;
> +	early_iounmap(address, 4);

On !CONFIG_SMP this produces the following build failure:

 arch/x86/kernel/vsmp_64.c:171:6: error: ‘setup_max_cpus’ undeclared (first use in this function)

Do you want to keep vSMP on UP? If not then you could add a 
'depends on SMP' to the vSMP Kconfig entry, avoiding the whole 
UP problem space.

Thanks,

	Ingo

Re: [tip:x86/platform] vsmp: Fix number of CPUs when vsmp is disabled

[Ido Yariv]

Hi Ingo,

On Tue, May 08, 2012 at 02:25:49PM +0200, Ingo Molnar wrote:
> 
> On !CONFIG_SMP this produces the following build failure:
> 
>  arch/x86/kernel/vsmp_64.c:171:6: error: ‘setup_max_cpus’ undeclared (first use in this function)
> 
> Do you want to keep vSMP on UP? If not then you could add a 
> 'depends on SMP' to the vSMP Kconfig entry, avoiding the whole 
> UP problem space.

Oops, sorry for that.

vsmp_64.c is built even if CONFIG_X86_VSMP is not set, so even if we set
this dependency, it will not fix this issue.
How about the following CONFIG_SMP check instead?

Thanks,
Ido.

>From b3c6d985b101364dd74f0efaf42c6b34fc7615f9 Mon Sep 17 00:00:00 2001
From: Shai Fultheim <shai@scalemp.com>
Date: Mon, 16 Apr 2012 10:39:35 +0300
Subject: [PATCH] vsmp: Fix number of CPUs when vsmp is disabled

In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
number of CPUs of the first board.

Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, fixed minor coding-style issues]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
 arch/x86/kernel/vsmp_64.c |   43 +++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 43 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index a1d804b..06779b7 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
+#include <linux/smp.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -22,6 +23,8 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 
+#define TOPOLOGY_REGISTER_OFFSET 0x10
+
 #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
 /*
  * Interrupt control on vSMPowered systems:
@@ -149,12 +152,52 @@ int is_vsmp_box(void)
 	return 0;
 }
 #endif
+
+static void __init vsmp_cap_cpus(void)
+{
+	void __iomem *address;
+	unsigned int cfg, topology, node_shift, maxcpus;
+
+#ifdef CONFIG_X86_VSMP
+	/* VSMP is enabled, no need to cap cpus */
+	return;
+#elif CONFIG_SMP
+	/*
+	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
+	 * ones present in the first board, unless explicitly overridden by
+	 * setup_max_cpus
+	 */
+	if (setup_max_cpus != NR_CPUS)
+		return;
+
+	/* Read the vSMP Foundation topology register */
+	cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
+	address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
+	if (WARN_ON(!address))
+		return;
+
+	topology = readl(address);
+	node_shift = (topology >> 16) & 0x7;
+	if (!node_shift)
+		/* The value 0 should be decoded as 8 */
+		node_shift = 8;
+	maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
+
+	pr_info("vSMP CTL: Capping CPUs to %d (CONFIG_X86_VSMP is unset)\n",
+		maxcpus);
+	setup_max_cpus = maxcpus;
+	early_iounmap(address, 4);
+#endif
+}
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	vsmp_cap_cpus();
+
 	set_vsmp_pv_ops();
 	return;
 }
-- 
1.7.6.5

Re: [tip:x86/platform] vsmp: Fix number of CPUs when vsmp is disabled

[Ido Yariv]

On Wed, May 09, 2012 at 11:03:35AM +0300, Ido Yariv wrote:
> +#ifdef CONFIG_X86_VSMP
> +	/* VSMP is enabled, no need to cap cpus */
> +	return;
> +#elif CONFIG_SMP

I've missed a 'defined' here. I'll send a new version right away.

Thanks,
Ido.

[PATCH v4] vsmp: Fix number of CPUs when vsmp is disabled

[Ido Yariv]

From: Shai Fultheim <shai@scalemp.com>

In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
number of CPUs of the first board.

Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, fixed minor coding-style issues]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
Changes from v3:
- Don't reference setup_max_cpus if CONFIG_SMP isn't set

 arch/x86/kernel/vsmp_64.c |   43 +++++++++++++++++++++++++++++++++++++++++++
 1 files changed, 43 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index a1d804b..5234c83 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
+#include <linux/smp.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -22,6 +23,8 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 
+#define TOPOLOGY_REGISTER_OFFSET 0x10
+
 #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
 /*
  * Interrupt control on vSMPowered systems:
@@ -149,12 +152,52 @@ int is_vsmp_box(void)
 	return 0;
 }
 #endif
+
+static void __init vsmp_cap_cpus(void)
+{
+	void __iomem *address;
+	unsigned int cfg, topology, node_shift, maxcpus;
+
+#ifdef CONFIG_X86_VSMP
+	/* VSMP is enabled, no need to cap cpus */
+	return;
+#elif defined(CONFIG_SMP)
+	/*
+	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
+	 * ones present in the first board, unless explicitly overridden by
+	 * setup_max_cpus
+	 */
+	if (setup_max_cpus != NR_CPUS)
+		return;
+
+	/* Read the vSMP Foundation topology register */
+	cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
+	address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
+	if (WARN_ON(!address))
+		return;
+
+	topology = readl(address);
+	node_shift = (topology >> 16) & 0x7;
+	if (!node_shift)
+		/* The value 0 should be decoded as 8 */
+		node_shift = 8;
+	maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
+
+	pr_info("vSMP CTL: Capping CPUs to %d (CONFIG_X86_VSMP is unset)\n",
+		maxcpus);
+	setup_max_cpus = maxcpus;
+	early_iounmap(address, 4);
+#endif
+}
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	vsmp_cap_cpus();
+
 	set_vsmp_pv_ops();
 	return;
 }
-- 
1.7.6.5

Re: [tip:x86/platform] vsmp: Fix number of CPUs when vsmp is disabled

[Ingo Molnar]

* Ido Yariv <ido@wizery.com> wrote:

> Hi Ingo,
> 
> On Tue, May 08, 2012 at 02:25:49PM +0200, Ingo Molnar wrote:
> > 
> > On !CONFIG_SMP this produces the following build failure:
> > 
> >  arch/x86/kernel/vsmp_64.c:171:6: error: ‘setup_max_cpus’ undeclared (first use in this function)
> > 
> > Do you want to keep vSMP on UP? If not then you could add a 
> > 'depends on SMP' to the vSMP Kconfig entry, avoiding the whole 
> > UP problem space.
> 
> Oops, sorry for that.
> 
> vsmp_64.c is built even if CONFIG_X86_VSMP is not set, so even 
> if we set this dependency, it will not fix this issue.

Well, then fix the Makefile rule as well. Is there any point to 
using vSMP specific glue on UP kernels?

> How about the following CONFIG_SMP check instead?

Looks pretty ugly, because it just increases the config jungle 
instead of reducing it.

Thanks,

	Ingo

Re: [PATCH v2 2/2] vsmp: Ignore IOAPIC IRQ affinity if possible

[Ingo Molnar]

* Shai Fultheim (Shai@ScaleMP.com) <Shai@ScaleMP.com> wrote:

> Ingo Molnar <mingo.kernel.org@gmail.com> wrote:
> > * Ido Yariv <ido@wizery.com> wrote:
> 
> > > @@ -51,6 +60,7 @@ void __init default_setup_apic_routing(void)
> > >  	if (is_vsmp_box()) {
> > >  		/* need to update phys_pkg_id */
> > >  		apic->phys_pkg_id = apicid_phys_pkg_id;
> > > +		apic->vector_allocation_domain =
> > fill_vector_allocation_domain;
> > >  	}
> > >  }
> > 
> > This is_vsmp_box() special case should really move into its own
> > apic handler.
> > 
> > Thanks,
> 
> Ingo,
> 
> vSMP Foundation virtualize multiple systems as one.  The Guest 
> OS is using the underlying hardware APIC driver, and therefore 
> we must allow the various APIC drivers to kick in.  When using 
> vSMP Foundation APIC driver is selected by the kernel based on 
> the hardware that is being virtualized, and that can be APIC, 
> X2APIC phys, X2APIC cluster, summit, etc.
> 
> If we would to implement an APIC handler, we would not be able 
> to use the native hardware APIC driver (unless we do an ugly 
> hack).

If you need a callback on that level then there's a cleaner 
mechanism than open-coding vsmp handlers into generic code:

 - create a new platform callback in x86_platform_ops
 - call it from apic_64.c if set
 - fill in the vSMP specific handler from the vSMP init code

Thanks,

	Ingo

RE: [tip:x86/platform] vsmp: Fix number of CPUs when vsmp is disabled

[Shai Fultheim (Shai@ScaleMP.com)]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="utf-8", Size: 621 bytes --]

Ingo Molnar wrote:
> Well, then fix the Makefile rule as well. Is there any point to
> using vSMP specific glue on UP kernels?

Yes - vsmp_64.c is needed even on UP kernel. 
This allows single CPU kernels, running on vSMP Foundation (such as installer kernels), to make use of the other virtualization-specific (under CONFIG_PARAVIRT )codes such as set_vsmp_pv_ops().

... thinking about it, I do not see other way to resolve this rather than Ido's suggestion.
ÿôèº{.nÇ+‰·Ÿ®‰­†+%ŠËÿ±éݶ\x17¥Šwÿº{.nÇ+‰·¥Š{±þG«éÿŠ{ayº\x1dʇڙë,j\a­¢f£¢·hšïêÿ‘êçz_è®\x03(­éšŽŠÝ¢j"ú\x1a¶^[m§ÿÿ¾\a«þG«éÿ¢¸?™¨è­Ú&£ø§~á¶iO•æ¬z·švØ^\x14\x04\x1a¶^[m§ÿÿÃ\fÿ¶ìÿ¢¸?–I¥

Re: [PATCH v4] vsmp: Fix number of CPUs when vsmp is disabled

[Ingo Molnar]

* Ido Yariv <ido@wizery.com> wrote:

> +static void __init vsmp_cap_cpus(void)
> +{
> +	void __iomem *address;
> +	unsigned int cfg, topology, node_shift, maxcpus;
> +
> +#ifdef CONFIG_X86_VSMP
> +	/* VSMP is enabled, no need to cap cpus */
> +	return;
> +#elif defined(CONFIG_SMP)
> +	/*
> +	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the

I suspect this will throw compiler warnings in the 
CONFIG_X86_VSMP && !CONFIG_SMP case.

Why not do something like:

static void __init vsmp_cap_cpus(void)
{
#if !defined(CONFIG_X86_VSMP) && defined(CONFIG_SMP)
   ...
#endif
}

This keeps the #ifdef complexity reasonably concentrated.

Thanks,

	Ingo

Re: [PATCH v4] vsmp: Fix number of CPUs when vsmp is disabled

[H. Peter Anvin]

On 05/09/2012 08:44 AM, Ingo Molnar wrote:
> 
> * Ido Yariv <ido@wizery.com> wrote:
> 
>> +static void __init vsmp_cap_cpus(void)
>> +{
>> +	void __iomem *address;
>> +	unsigned int cfg, topology, node_shift, maxcpus;
>> +
>> +#ifdef CONFIG_X86_VSMP
>> +	/* VSMP is enabled, no need to cap cpus */
>> +	return;
>> +#elif defined(CONFIG_SMP)
>> +	/*
>> +	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
> 
> I suspect this will throw compiler warnings in the 
> CONFIG_X86_VSMP && !CONFIG_SMP case.
> 

What on Earth is the point of allowing that combination?  Why not make
X86_VSMP depend on SMP and reduce the testing matrix?

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

Re: [PATCH v2 2/2] vsmp: Ignore IOAPIC IRQ affinity if possible

[Ido Yariv]

Hi Ingo,

On Wed, May 09, 2012 at 11:18:27AM +0200, Ingo Molnar wrote:
> 
> * Shai Fultheim (Shai@ScaleMP.com) <Shai@ScaleMP.com> wrote:
> 
> > Ingo Molnar <mingo.kernel.org@gmail.com> wrote:
> > > * Ido Yariv <ido@wizery.com> wrote:
> > 
> > > > @@ -51,6 +60,7 @@ void __init default_setup_apic_routing(void)
> > > >  	if (is_vsmp_box()) {
> > > >  		/* need to update phys_pkg_id */
> > > >  		apic->phys_pkg_id = apicid_phys_pkg_id;
> > > > +		apic->vector_allocation_domain =
> > > fill_vector_allocation_domain;
> > > >  	}
> > > >  }
> > > 
> > > This is_vsmp_box() special case should really move into its own
> > > apic handler.
> > > 
> > > Thanks,
> > 
> > Ingo,
> > 
> > vSMP Foundation virtualize multiple systems as one.  The Guest 
> > OS is using the underlying hardware APIC driver, and therefore 
> > we must allow the various APIC drivers to kick in.  When using 
> > vSMP Foundation APIC driver is selected by the kernel based on 
> > the hardware that is being virtualized, and that can be APIC, 
> > X2APIC phys, X2APIC cluster, summit, etc.
> > 
> > If we would to implement an APIC handler, we would not be able 
> > to use the native hardware APIC driver (unless we do an ugly 
> > hack).
> 
> If you need a callback on that level then there's a cleaner 
> mechanism than open-coding vsmp handlers into generic code:
> 
>  - create a new platform callback in x86_platform_ops
>  - call it from apic_64.c if set
>  - fill in the vSMP specific handler from the vSMP init code

Sounds good.

We'll look into it and post an alternative patch to this one.

Thanks,
Ido.

Re: [PATCH v4] vsmp: Fix number of CPUs when vsmp is disabled

[Ido Yariv]

Hi,

On Wed, May 09, 2012 at 08:55:54AM -0700, H. Peter Anvin wrote:
> On 05/09/2012 08:44 AM, Ingo Molnar wrote:
> > 
> > * Ido Yariv <ido@wizery.com> wrote:
> > 
> >> +static void __init vsmp_cap_cpus(void)
> >> +{
> >> +	void __iomem *address;
> >> +	unsigned int cfg, topology, node_shift, maxcpus;
> >> +
> >> +#ifdef CONFIG_X86_VSMP
> >> +	/* VSMP is enabled, no need to cap cpus */
> >> +	return;
> >> +#elif defined(CONFIG_SMP)
> >> +	/*
> >> +	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
> > 
> > I suspect this will throw compiler warnings in the 
> > CONFIG_X86_VSMP && !CONFIG_SMP case.
> > 
> 
> What on Earth is the point of allowing that combination?  Why not make
> X86_VSMP depend on SMP and reduce the testing matrix?

CONFIG_X86_VSMP shouldn't be used without CONFIG_SMP, so we can safely
add this dependency.
Ingo's suggestion will still fix unreferenced variables warnings, so how
about the following patch?

Thanks,
Ido.

>From 6fa023be44749092088618379f13d7a3e086d692 Mon Sep 17 00:00:00 2001
From: Shai Fultheim <shai@scalemp.com>
Date: Mon, 16 Apr 2012 10:39:35 +0300
Subject: [PATCH v5] vsmp: Fix number of CPUs when vsmp is disabled

In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
number of CPUs of the first board.

Also make CONFIG_X86_VSMP depend on CONFIG_SMP, as there's little
point in having a vsmp machine with a single CPU.

Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, fixed minor coding-style issues]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
 arch/x86/Kconfig          |    1 +
 arch/x86/kernel/vsmp_64.c |   40 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 0 deletions(-)

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index c9866b0..b1e98c9 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -374,6 +374,7 @@ config X86_VSMP
 	select PARAVIRT
 	depends on X86_64 && PCI
 	depends on X86_EXTENDED_PLATFORM
+	depends on SMP
 	---help---
 	  Support for ScaleMP vSMP systems.  Say 'Y' here if this kernel is
 	  supposed to run on these EM64T-based machines.  Only choose this option
diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index a1d804b..8eeb55a 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
+#include <linux/smp.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -22,6 +23,8 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 
+#define TOPOLOGY_REGISTER_OFFSET 0x10
+
 #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
 /*
  * Interrupt control on vSMPowered systems:
@@ -149,12 +152,49 @@ int is_vsmp_box(void)
 	return 0;
 }
 #endif
+
+static void __init vsmp_cap_cpus(void)
+{
+#if !defined(CONFIG_X86_VSMP) && defined(CONFIG_SMP)
+	void __iomem *address;
+	unsigned int cfg, topology, node_shift, maxcpus;
+
+	/*
+	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to the
+	 * ones present in the first board, unless explicitly overridden by
+	 * setup_max_cpus
+	 */
+	if (setup_max_cpus != NR_CPUS)
+		return;
+
+	/* Read the vSMP Foundation topology register */
+	cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
+	address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
+	if (WARN_ON(!address))
+		return;
+
+	topology = readl(address);
+	node_shift = (topology >> 16) & 0x7;
+	if (!node_shift)
+		/* The value 0 should be decoded as 8 */
+		node_shift = 8;
+	maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
+
+	pr_info("vSMP CTL: Capping CPUs to %d (CONFIG_X86_VSMP is unset)\n",
+		maxcpus);
+	setup_max_cpus = maxcpus;
+	early_iounmap(address, 4);
+#endif
+}
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	vsmp_cap_cpus();
+
 	set_vsmp_pv_ops();
 	return;
 }
-- 
1.7.6.5

RE: [PATCH v4] vsmp: Fix number of CPUs when vsmp is disabled

[Shai Fultheim (Shai@ScaleMP.com)]

Looks good.


--Shai 


> -----Original Message-----
> From: Ido Yariv [mailto:ido@wizery.com]
> Sent: Friday, May 11, 2012 12:27
> To: H. Peter Anvin
> Cc: Ingo Molnar; linux-kernel@vger.kernel.org; Ingo Molnar; Thomas
> Gleixner; Shai Fultheim (Shai@ScaleMP.com)
> Subject: Re: [PATCH v4] vsmp: Fix number of CPUs when vsmp is disabled
> 
> Hi,
> 
> On Wed, May 09, 2012 at 08:55:54AM -0700, H. Peter Anvin wrote:
> > On 05/09/2012 08:44 AM, Ingo Molnar wrote:
> > >
> > > * Ido Yariv <ido@wizery.com> wrote:
> > >
> > >> +static void __init vsmp_cap_cpus(void)
> > >> +{
> > >> +	void __iomem *address;
> > >> +	unsigned int cfg, topology, node_shift, maxcpus;
> > >> +
> > >> +#ifdef CONFIG_X86_VSMP
> > >> +	/* VSMP is enabled, no need to cap cpus */
> > >> +	return;
> > >> +#elif defined(CONFIG_SMP)
> > >> +	/*
> > >> +	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to
> the
> > >
> > > I suspect this will throw compiler warnings in the
> > > CONFIG_X86_VSMP && !CONFIG_SMP case.
> > >
> >
> > What on Earth is the point of allowing that combination?  Why not make
> > X86_VSMP depend on SMP and reduce the testing matrix?
> 
> CONFIG_X86_VSMP shouldn't be used without CONFIG_SMP, so we can
> safely
> add this dependency.
> Ingo's suggestion will still fix unreferenced variables warnings, so how
> about the following patch?
> 
> Thanks,
> Ido.
> 
> From 6fa023be44749092088618379f13d7a3e086d692 Mon Sep 17 00:00:00
> 2001
> From: Shai Fultheim <shai@scalemp.com>
> Date: Mon, 16 Apr 2012 10:39:35 +0300
> Subject: [PATCH v5] vsmp: Fix number of CPUs when vsmp is disabled
> 
> In case CONFIG_X86_VSMP is not set, limit the number of CPUs to the
> number of CPUs of the first board.
> 
> Also make CONFIG_X86_VSMP depend on CONFIG_SMP, as there's little
> point in having a vsmp machine with a single CPU.
> 
> Signed-off-by: Shai Fultheim <shai@scalemp.com>
> [ido@wizery.com: rebased, fixed minor coding-style issues]
> Signed-off-by: Ido Yariv <ido@wizery.com>
> ---
>  arch/x86/Kconfig          |    1 +
>  arch/x86/kernel/vsmp_64.c |   40
> ++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 41 insertions(+), 0 deletions(-)
> 
> diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
> index c9866b0..b1e98c9 100644
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -374,6 +374,7 @@ config X86_VSMP
>  	select PARAVIRT
>  	depends on X86_64 && PCI
>  	depends on X86_EXTENDED_PLATFORM
> +	depends on SMP
>  	---help---
>  	  Support for ScaleMP vSMP systems.  Say 'Y' here if this kernel is
>  	  supposed to run on these EM64T-based machines.  Only choose this
> option
> diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
> index a1d804b..8eeb55a 100644
> --- a/arch/x86/kernel/vsmp_64.c
> +++ b/arch/x86/kernel/vsmp_64.c
> @@ -15,6 +15,7 @@
>  #include <linux/init.h>
>  #include <linux/pci_ids.h>
>  #include <linux/pci_regs.h>
> +#include <linux/smp.h>
> 
>  #include <asm/apic.h>
>  #include <asm/pci-direct.h>
> @@ -22,6 +23,8 @@
>  #include <asm/paravirt.h>
>  #include <asm/setup.h>
> 
> +#define TOPOLOGY_REGISTER_OFFSET 0x10
> +
>  #if defined CONFIG_PCI && defined CONFIG_PARAVIRT
>  /*
>   * Interrupt control on vSMPowered systems:
> @@ -149,12 +152,49 @@ int is_vsmp_box(void)
>  	return 0;
>  }
>  #endif
> +
> +static void __init vsmp_cap_cpus(void)
> +{
> +#if !defined(CONFIG_X86_VSMP) && defined(CONFIG_SMP)
> +	void __iomem *address;
> +	unsigned int cfg, topology, node_shift, maxcpus;
> +
> +	/*
> +	 * CONFIG_X86_VSMP is not configured, so limit the number CPUs to
> the
> +	 * ones present in the first board, unless explicitly overridden by
> +	 * setup_max_cpus
> +	 */
> +	if (setup_max_cpus != NR_CPUS)
> +		return;
> +
> +	/* Read the vSMP Foundation topology register */
> +	cfg = read_pci_config(0, 0x1f, 0, PCI_BASE_ADDRESS_0);
> +	address = early_ioremap(cfg + TOPOLOGY_REGISTER_OFFSET, 4);
> +	if (WARN_ON(!address))
> +		return;
> +
> +	topology = readl(address);
> +	node_shift = (topology >> 16) & 0x7;
> +	if (!node_shift)
> +		/* The value 0 should be decoded as 8 */
> +		node_shift = 8;
> +	maxcpus = (topology & ((1 << node_shift) - 1)) + 1;
> +
> +	pr_info("vSMP CTL: Capping CPUs to %d (CONFIG_X86_VSMP is
> unset)\n",
> +		maxcpus);
> +	setup_max_cpus = maxcpus;
> +	early_iounmap(address, 4);
> +#endif
> +}
> +
>  void __init vsmp_init(void)
>  {
>  	detect_vsmp_box();
>  	if (!is_vsmp_box())
>  		return;
> 
> +	vsmp_cap_cpus();
> +
>  	set_vsmp_pv_ops();
>  	return;
>  }
> --
> 1.7.6.5

[PATCH 1/2] x86: Introduce apic post-initialization callback

[Ido Yariv]

Some architectures (such as vSMP) need to slightly adjust the underlying
apic structure. Add an apic post-initialization to x86_platform_ops for
this purpose and use it for adjusting the apic structure on vSMP
systems.

Signed-off-by: Ido Yariv <ido@wizery.com>
Acked-by: Shai Fultheim <shai@scalemp.com>
---
 arch/x86/include/asm/x86_init.h |    2 ++
 arch/x86/kernel/apic/probe_32.c |    3 +++
 arch/x86/kernel/apic/probe_64.c |   11 ++---------
 arch/x86/kernel/vsmp_64.c       |   13 +++++++++++++
 4 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index c090af1..c377d9c 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -164,6 +164,7 @@ struct x86_cpuinit_ops {
  * @i8042_detect		pre-detect if i8042 controller exists
  * @save_sched_clock_state:	save state for sched_clock() on suspend
  * @restore_sched_clock_state:	restore state for sched_clock() on resume
+ * @apic_post_init:		adjust apic if neeeded
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -177,6 +178,7 @@ struct x86_platform_ops {
 	int (*i8042_detect)(void);
 	void (*save_sched_clock_state)(void);
 	void (*restore_sched_clock_state)(void);
+	void (*apic_post_init)(void);
 };
 
 struct pci_dev;
diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c
index 1b291da..8616d51 100644
--- a/arch/x86/kernel/apic/probe_32.c
+++ b/arch/x86/kernel/apic/probe_32.c
@@ -208,6 +208,9 @@ void __init default_setup_apic_routing(void)
 
 	if (apic->setup_apic_routing)
 		apic->setup_apic_routing();
+
+	if (x86_platform.apic_post_init)
+		x86_platform.apic_post_init();
 }
 
 void __init generic_apic_probe(void)
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index 3fe9866..1793dba 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -23,11 +23,6 @@
 #include <asm/ipi.h>
 #include <asm/setup.h>
 
-static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
-{
-	return hard_smp_processor_id() >> index_msb;
-}
-
 /*
  * Check the APIC IDs in bios_cpu_apicid and choose the APIC mode.
  */
@@ -48,10 +43,8 @@ void __init default_setup_apic_routing(void)
 		}
 	}
 
-	if (is_vsmp_box()) {
-		/* need to update phys_pkg_id */
-		apic->phys_pkg_id = apicid_phys_pkg_id;
-	}
+	if (x86_platform.apic_post_init)
+		x86_platform.apic_post_init();
 }
 
 /* Same for both flat and physical. */
diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 8eeb55a..59eea85 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -187,12 +187,25 @@ static void __init vsmp_cap_cpus(void)
 #endif
 }
 
+static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
+{
+	return hard_smp_processor_id() >> index_msb;
+}
+
+static void vsmp_apic_post_init(void)
+{
+	/* need to update phys_pkg_id */
+	apic->phys_pkg_id = apicid_phys_pkg_id;
+}
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	x86_platform.apic_post_init = vsmp_apic_post_init;
+
 	vsmp_cap_cpus();
 
 	set_vsmp_pv_ops();
-- 
1.7.7.6

[PATCH v3 2/2] vsmp: Ignore IOAPIC IRQ affinity if possible

[Ido Yariv]

From: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>

vSMP can route interrupts more optimally based on internal knowledge the
OS does not have. In order to support this optimization, all CPUs must
be able to handle all possible IOAPIC interrupts.

Fix this by setting the vector allocation domain for all CPUs and by
enabling this feature in vSMP.

Signed-off-by: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, simplified, and reworded the commit message]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
Changes from v2:
- Set the vector allocation domain in the apic_post_init callback

 arch/x86/kernel/vsmp_64.c |   25 +++++++++++++++++++++----
 1 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 59eea85..a24cf34 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -16,6 +16,7 @@
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
 #include <linux/smp.h>
+#include <linux/irq.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -95,6 +96,13 @@ static void __init set_vsmp_pv_ops(void)
 	ctl = readl(address + 4);
 	printk(KERN_INFO "vSMP CTL: capabilities:0x%08x  control:0x%08x\n",
 	       cap, ctl);
+
+	/* If possible, let the vSMP foundation route the interrupt optimally */
+	if (cap & ctl & BIT(8)) {
+		ctl &= ~BIT(8);
+		no_irq_affinity = 1;
+	}
+
 	if (cap & ctl & (1 << 4)) {
 		/* Setup irq ops and turn on vSMP  IRQ fastpath handling */
 		pv_irq_ops.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable);
@@ -102,12 +110,11 @@ static void __init set_vsmp_pv_ops(void)
 		pv_irq_ops.save_fl  = PV_CALLEE_SAVE(vsmp_save_fl);
 		pv_irq_ops.restore_fl  = PV_CALLEE_SAVE(vsmp_restore_fl);
 		pv_init_ops.patch = vsmp_patch;
-
 		ctl &= ~(1 << 4);
-		writel(ctl, address + 4);
-		ctl = readl(address + 4);
-		printk(KERN_INFO "vSMP CTL: control set to:0x%08x\n", ctl);
 	}
+	writel(ctl, address + 4);
+	ctl = readl(address + 4);
+	pr_info("vSMP CTL: control set to:0x%08x\n", ctl);
 
 	early_iounmap(address, 8);
 }
@@ -192,10 +199,20 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
 	return hard_smp_processor_id() >> index_msb;
 }
 
+/*
+ * In vSMP, all cpus should be capable of handling interrupts, regardless of
+ * the apic used.
+ */
+static void fill_vector_allocation_domain(int cpu, struct cpumask *retmask)
+{
+	cpumask_setall(retmask);
+}
+
 static void vsmp_apic_post_init(void)
 {
 	/* need to update phys_pkg_id */
 	apic->phys_pkg_id = apicid_phys_pkg_id;
+	apic->vector_allocation_domain = fill_vector_allocation_domain;
 }
 
 void __init vsmp_init(void)
-- 
1.7.7.6

[tip:x86/platform] x86/platform: Introduce APIC post-initialization callback

[tip-bot for Ido Yariv]

Commit-ID:  7db971b235480849aa5b9209b67b62e987b3181b
Gitweb:     http://git.kernel.org/tip/7db971b235480849aa5b9209b67b62e987b3181b
Author:     Ido Yariv <ido@wizery.com>
AuthorDate: Sun, 3 Jun 2012 01:11:34 +0300
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 6 Jun 2012 09:06:19 +0200

x86/platform: Introduce APIC post-initialization callback

Some subarchitectures (such as vSMP) need to slightly adjust the
underlying APIC structure. Add an APIC post-initialization callback
to 'struct x86_platform_ops' for this purpose and use it for
adjusting the APIC structure on vSMP systems.

Signed-off-by: Ido Yariv <ido@wizery.com>
Acked-by: Shai Fultheim <shai@scalemp.com>
Link: http://lkml.kernel.org/r/1338675095-27260-1-git-send-email-ido@wizery.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/include/asm/x86_init.h |    2 ++
 arch/x86/kernel/apic/probe_32.c |    3 +++
 arch/x86/kernel/apic/probe_64.c |   11 ++---------
 arch/x86/kernel/vsmp_64.c       |   13 +++++++++++++
 4 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h
index c090af1..c377d9c 100644
--- a/arch/x86/include/asm/x86_init.h
+++ b/arch/x86/include/asm/x86_init.h
@@ -164,6 +164,7 @@ struct x86_cpuinit_ops {
  * @i8042_detect		pre-detect if i8042 controller exists
  * @save_sched_clock_state:	save state for sched_clock() on suspend
  * @restore_sched_clock_state:	restore state for sched_clock() on resume
+ * @apic_post_init:		adjust apic if neeeded
  */
 struct x86_platform_ops {
 	unsigned long (*calibrate_tsc)(void);
@@ -177,6 +178,7 @@ struct x86_platform_ops {
 	int (*i8042_detect)(void);
 	void (*save_sched_clock_state)(void);
 	void (*restore_sched_clock_state)(void);
+	void (*apic_post_init)(void);
 };
 
 struct pci_dev;
diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c
index 1b291da..8616d51 100644
--- a/arch/x86/kernel/apic/probe_32.c
+++ b/arch/x86/kernel/apic/probe_32.c
@@ -208,6 +208,9 @@ void __init default_setup_apic_routing(void)
 
 	if (apic->setup_apic_routing)
 		apic->setup_apic_routing();
+
+	if (x86_platform.apic_post_init)
+		x86_platform.apic_post_init();
 }
 
 void __init generic_apic_probe(void)
diff --git a/arch/x86/kernel/apic/probe_64.c b/arch/x86/kernel/apic/probe_64.c
index 3fe9866..1793dba 100644
--- a/arch/x86/kernel/apic/probe_64.c
+++ b/arch/x86/kernel/apic/probe_64.c
@@ -23,11 +23,6 @@
 #include <asm/ipi.h>
 #include <asm/setup.h>
 
-static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
-{
-	return hard_smp_processor_id() >> index_msb;
-}
-
 /*
  * Check the APIC IDs in bios_cpu_apicid and choose the APIC mode.
  */
@@ -48,10 +43,8 @@ void __init default_setup_apic_routing(void)
 		}
 	}
 
-	if (is_vsmp_box()) {
-		/* need to update phys_pkg_id */
-		apic->phys_pkg_id = apicid_phys_pkg_id;
-	}
+	if (x86_platform.apic_post_init)
+		x86_platform.apic_post_init();
 }
 
 /* Same for both flat and physical. */
diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 8eeb55a..59eea85 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -187,12 +187,25 @@ static void __init vsmp_cap_cpus(void)
 #endif
 }
 
+static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
+{
+	return hard_smp_processor_id() >> index_msb;
+}
+
+static void vsmp_apic_post_init(void)
+{
+	/* need to update phys_pkg_id */
+	apic->phys_pkg_id = apicid_phys_pkg_id;
+}
+
 void __init vsmp_init(void)
 {
 	detect_vsmp_box();
 	if (!is_vsmp_box())
 		return;
 
+	x86_platform.apic_post_init = vsmp_apic_post_init;
+
 	vsmp_cap_cpus();
 
 	set_vsmp_pv_ops();

[tip:x86/platform] x86/vsmp: Ignore IOAPIC IRQ affinity if possible

[tip-bot for Ravikiran Thirumalai]

Commit-ID:  3aabb53ce5849605cee731bbc32f37120b4c4ceb
Gitweb:     http://git.kernel.org/tip/3aabb53ce5849605cee731bbc32f37120b4c4ceb
Author:     Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
AuthorDate: Sun, 3 Jun 2012 01:11:35 +0300
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Wed, 6 Jun 2012 09:06:20 +0200

x86/vsmp: Ignore IOAPIC IRQ affinity if possible

vSMP can route interrupts more optimally based on internal
knowledge the OS does not have. In order to support this
optimization, all CPUs must be able to handle all possible
IOAPIC interrupts.

Fix this by setting the vector allocation domain for all CPUs
and by enabling this feature in vSMP.

Signed-off-by: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ rebased, simplified, and reworded the commit message ]
Signed-off-by: Ido Yariv <ido@wizery.com>
Link: http://lkml.kernel.org/r/1338675095-27260-2-git-send-email-ido@wizery.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/vsmp_64.c |   25 +++++++++++++++++++++----
 1 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 59eea85..c72c0d3 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -16,6 +16,7 @@
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
 #include <linux/smp.h>
+#include <linux/irq.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -95,6 +96,13 @@ static void __init set_vsmp_pv_ops(void)
 	ctl = readl(address + 4);
 	printk(KERN_INFO "vSMP CTL: capabilities:0x%08x  control:0x%08x\n",
 	       cap, ctl);
+
+	/* If possible, let the vSMP foundation route the interrupt optimally */
+	if (cap & ctl & BIT(8)) {
+		ctl &= ~BIT(8);
+		no_irq_affinity = 1;
+	}
+
 	if (cap & ctl & (1 << 4)) {
 		/* Setup irq ops and turn on vSMP  IRQ fastpath handling */
 		pv_irq_ops.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable);
@@ -102,12 +110,11 @@ static void __init set_vsmp_pv_ops(void)
 		pv_irq_ops.save_fl  = PV_CALLEE_SAVE(vsmp_save_fl);
 		pv_irq_ops.restore_fl  = PV_CALLEE_SAVE(vsmp_restore_fl);
 		pv_init_ops.patch = vsmp_patch;
-
 		ctl &= ~(1 << 4);
-		writel(ctl, address + 4);
-		ctl = readl(address + 4);
-		printk(KERN_INFO "vSMP CTL: control set to:0x%08x\n", ctl);
 	}
+	writel(ctl, address + 4);
+	ctl = readl(address + 4);
+	pr_info("vSMP CTL: control set to:0x%08x\n", ctl);
 
 	early_iounmap(address, 8);
 }
@@ -192,10 +199,20 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
 	return hard_smp_processor_id() >> index_msb;
 }
 
+/*
+ * In vSMP, all cpus should be capable of handling interrupts, regardless of
+ * the APIC used.
+ */
+static void fill_vector_allocation_domain(int cpu, struct cpumask *retmask)
+{
+	cpumask_setall(retmask);
+}
+
 static void vsmp_apic_post_init(void)
 {
 	/* need to update phys_pkg_id */
 	apic->phys_pkg_id = apicid_phys_pkg_id;
+	apic->vector_allocation_domain = fill_vector_allocation_domain;
 }
 
 void __init vsmp_init(void)

Re: [tip:x86/platform] x86/vsmp: Ignore IOAPIC IRQ affinity if possible

[Ingo Molnar]

* tip-bot for Ravikiran Thirumalai <kiran.thirumalai@gmail.com> wrote:

> Commit-ID:  3aabb53ce5849605cee731bbc32f37120b4c4ceb
> Gitweb:     http://git.kernel.org/tip/3aabb53ce5849605cee731bbc32f37120b4c4ceb
> Author:     Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
> AuthorDate: Sun, 3 Jun 2012 01:11:35 +0300
> Committer:  Ingo Molnar <mingo@kernel.org>
> CommitDate: Wed, 6 Jun 2012 09:06:20 +0200
> 
> x86/vsmp: Ignore IOAPIC IRQ affinity if possible

This patch causes the following build failure:

 (.init.text+0x10626): undefined reference to `no_irq_affinity'

on UP kernels.

Thanks,

	Ingo

Re: [tip:x86/platform] x86/vsmp: Ignore IOAPIC IRQ affinity if possible

[Ido Yariv]

Hi Ingo,

On Fri, Jun 08, 2012 at 10:52:29AM +0200, Ingo Molnar wrote:
> 
> * tip-bot for Ravikiran Thirumalai <kiran.thirumalai@gmail.com> wrote:
> 
> > Commit-ID:  3aabb53ce5849605cee731bbc32f37120b4c4ceb
> > Gitweb:     http://git.kernel.org/tip/3aabb53ce5849605cee731bbc32f37120b4c4ceb
> > Author:     Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
> > AuthorDate: Sun, 3 Jun 2012 01:11:35 +0300
> > Committer:  Ingo Molnar <mingo@kernel.org>
> > CommitDate: Wed, 6 Jun 2012 09:06:20 +0200
> > 
> > x86/vsmp: Ignore IOAPIC IRQ affinity if possible
> 
> This patch causes the following build failure:
> 
>  (.init.text+0x10626): undefined reference to `no_irq_affinity'
> 
> on UP kernels.

This seems to break when CONFIG_PARAVIRT is set but CONFIG_SMP isn't.
Since there's little point in optimizing IOAPIC routing for UP kernels,
how about the following fix?

Thanks,
Ido.

>From a67a9f5b0efa20c4da0fa85848d6c9e46f8f2665 Mon Sep 17 00:00:00 2001
From: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
Date: Sun, 3 Jun 2012 01:11:35 +0300
Subject: [PATCH] x86/vsmp: Ignore IOAPIC IRQ affinity if possible

vSMP can route interrupts more optimally based on internal
knowledge the OS does not have. In order to support this
optimization, all CPUs must be able to handle all possible
IOAPIC interrupts.

Fix this by setting the vector allocation domain for all CPUs
and by enabling this feature in vSMP.

Signed-off-by: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, simplified, and reworded the commit message]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
 arch/x86/kernel/vsmp_64.c |   27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 59eea85..6b96a73 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -16,6 +16,7 @@
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
 #include <linux/smp.h>
+#include <linux/irq.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -95,6 +96,15 @@ static void __init set_vsmp_pv_ops(void)
 	ctl = readl(address + 4);
 	printk(KERN_INFO "vSMP CTL: capabilities:0x%08x  control:0x%08x\n",
 	       cap, ctl);
+
+	/* If possible, let the vSMP foundation route the interrupt optimally */
+#ifdef CONFIG_SMP
+	if (cap & ctl & BIT(8)) {
+		ctl &= ~BIT(8);
+		no_irq_affinity = 1;
+	}
+#endif
+
 	if (cap & ctl & (1 << 4)) {
 		/* Setup irq ops and turn on vSMP  IRQ fastpath handling */
 		pv_irq_ops.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable);
@@ -102,12 +112,11 @@ static void __init set_vsmp_pv_ops(void)
 		pv_irq_ops.save_fl  = PV_CALLEE_SAVE(vsmp_save_fl);
 		pv_irq_ops.restore_fl  = PV_CALLEE_SAVE(vsmp_restore_fl);
 		pv_init_ops.patch = vsmp_patch;
-
 		ctl &= ~(1 << 4);
-		writel(ctl, address + 4);
-		ctl = readl(address + 4);
-		printk(KERN_INFO "vSMP CTL: control set to:0x%08x\n", ctl);
 	}
+	writel(ctl, address + 4);
+	ctl = readl(address + 4);
+	pr_info("vSMP CTL: control set to:0x%08x\n", ctl);
 
 	early_iounmap(address, 8);
 }
@@ -192,10 +201,20 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
 	return hard_smp_processor_id() >> index_msb;
 }
 
+/*
+ * In vSMP, all cpus should be capable of handling interrupts, regardless of
+ * the APIC used.
+ */
+static void fill_vector_allocation_domain(int cpu, struct cpumask *retmask)
+{
+	cpumask_setall(retmask);
+}
+
 static void vsmp_apic_post_init(void)
 {
 	/* need to update phys_pkg_id */
 	apic->phys_pkg_id = apicid_phys_pkg_id;
+	apic->vector_allocation_domain = fill_vector_allocation_domain;
 }
 
 void __init vsmp_init(void)
-- 
1.7.10.2

Re: [tip:x86/platform] x86/vsmp: Ignore IOAPIC IRQ affinity if possible

[Ingo Molnar]

* Ido Yariv <ido@wizery.com> wrote:

> This seems to break when CONFIG_PARAVIRT is set but CONFIG_SMP 
> isn't. Since there's little point in optimizing IOAPIC routing 
> for UP kernels, how about the following fix?

Looks good to me, I'll give it some testing.

Thanks,

	Ingo

Re: [tip:x86/platform] x86/vsmp: Ignore IOAPIC IRQ affinity if possible

[Ido Yariv]

Hi Ingo,

On Mon, Jun 11, 2012 at 10:58:43AM +0200, Ingo Molnar wrote:
> 
> * Ido Yariv <ido@wizery.com> wrote:
> 
> > This seems to break when CONFIG_PARAVIRT is set but CONFIG_SMP 
> > isn't. Since there's little point in optimizing IOAPIC routing 
> > for UP kernels, how about the following fix?
> 
> Looks good to me, I'll give it some testing.

I'm afraid this patch is missing one more config dependency. Since
no_irq_affinity is declared in proc.c, we need to verify that
CONFIG_PROC_FS is set before referencing this variable. The below patch
fixes this.

Thanks to Fengguang Wu for reporting this.

Thanks,
Ido.

>From 2a865725447a84a7e3921b01b3683b665e896971 Mon Sep 17 00:00:00 2001
From: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
Date: Sun, 3 Jun 2012 01:11:35 +0300
Subject: [PATCH] x86/vsmp: Ignore IOAPIC IRQ affinity if possible

vSMP can route interrupts more optimally based on internal
knowledge the OS does not have. In order to support this
optimization, all CPUs must be able to handle all possible
IOAPIC interrupts.

Fix this by setting the vector allocation domain for all CPUs
and by enabling this feature in vSMP.

Signed-off-by: Ravikiran Thirumalai <kiran.thirumalai@gmail.com>
Signed-off-by: Shai Fultheim <shai@scalemp.com>
[ido@wizery.com: rebased, simplified, and reworded the commit message]
Signed-off-by: Ido Yariv <ido@wizery.com>
---
 arch/x86/kernel/vsmp_64.c |   30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 59eea85..3f0285a 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -16,6 +16,7 @@
 #include <linux/pci_ids.h>
 #include <linux/pci_regs.h>
 #include <linux/smp.h>
+#include <linux/irq.h>
 
 #include <asm/apic.h>
 #include <asm/pci-direct.h>
@@ -95,6 +96,18 @@ static void __init set_vsmp_pv_ops(void)
 	ctl = readl(address + 4);
 	printk(KERN_INFO "vSMP CTL: capabilities:0x%08x  control:0x%08x\n",
 	       cap, ctl);
+
+	/* If possible, let the vSMP foundation route the interrupt optimally */
+#ifdef CONFIG_SMP
+	if (cap & ctl & BIT(8)) {
+		ctl &= ~BIT(8);
+#ifdef CONFIG_PROC_FS
+		/* Don't let users change irq affinity via procfs */
+		no_irq_affinity = 1;
+#endif
+	}
+#endif
+
 	if (cap & ctl & (1 << 4)) {
 		/* Setup irq ops and turn on vSMP  IRQ fastpath handling */
 		pv_irq_ops.irq_disable = PV_CALLEE_SAVE(vsmp_irq_disable);
@@ -102,12 +115,11 @@ static void __init set_vsmp_pv_ops(void)
 		pv_irq_ops.save_fl  = PV_CALLEE_SAVE(vsmp_save_fl);
 		pv_irq_ops.restore_fl  = PV_CALLEE_SAVE(vsmp_restore_fl);
 		pv_init_ops.patch = vsmp_patch;
-
 		ctl &= ~(1 << 4);
-		writel(ctl, address + 4);
-		ctl = readl(address + 4);
-		printk(KERN_INFO "vSMP CTL: control set to:0x%08x\n", ctl);
 	}
+	writel(ctl, address + 4);
+	ctl = readl(address + 4);
+	pr_info("vSMP CTL: control set to:0x%08x\n", ctl);
 
 	early_iounmap(address, 8);
 }
@@ -192,10 +204,20 @@ static int apicid_phys_pkg_id(int initial_apic_id, int index_msb)
 	return hard_smp_processor_id() >> index_msb;
 }
 
+/*
+ * In vSMP, all cpus should be capable of handling interrupts, regardless of
+ * the APIC used.
+ */
+static void fill_vector_allocation_domain(int cpu, struct cpumask *retmask)
+{
+	cpumask_setall(retmask);
+}
+
 static void vsmp_apic_post_init(void)
 {
 	/* need to update phys_pkg_id */
 	apic->phys_pkg_id = apicid_phys_pkg_id;
+	apic->vector_allocation_domain = fill_vector_allocation_domain;
 }
 
 void __init vsmp_init(void)
-- 
1.7.10.2

Re: [tip:x86/platform] x86/vsmp: Ignore IOAPIC IRQ affinity if possible

[Ingo Molnar]

* Ido Yariv <ido@wizery.com> wrote:

> Hi Ingo,
> 
> On Mon, Jun 11, 2012 at 10:58:43AM +0200, Ingo Molnar wrote:
> > 
> > * Ido Yariv <ido@wizery.com> wrote:
> > 
> > > This seems to break when CONFIG_PARAVIRT is set but CONFIG_SMP 
> > > isn't. Since there's little point in optimizing IOAPIC routing 
> > > for UP kernels, how about the following fix?
> > 
> > Looks good to me, I'll give it some testing.
> 
> I'm afraid this patch is missing one more config dependency. Since
> no_irq_affinity is declared in proc.c, we need to verify that
> CONFIG_PROC_FS is set before referencing this variable. The below patch
> fixes this.
> 
> Thanks to Fengguang Wu for reporting this.

Please send a delta fix patch on top of latest -tip to fix this.

Thanks,

	Ingo

[PATCH] x86/vsmp: Fix linker error when CONFIG_PROC_FS is not set

[Ido Yariv]

set_vsmp_pv_ops() references no_irq_affinity which is undeclared if
CONFIG_PROC_FS isn't set. Fix this by adding an ifdef around this
variable's access.

Reported-by: Fengguang Wu <wfg@linux.intel.com>
Signed-off-by: Ido Yariv <ido@wizery.com>
Acked-by: Shai Fultheim <shai@scalemp.com>
---
 arch/x86/kernel/vsmp_64.c |    3 +++
 1 file changed, 3 insertions(+)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 6b96a73..3f0285a 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -101,7 +101,10 @@ static void __init set_vsmp_pv_ops(void)
 #ifdef CONFIG_SMP
 	if (cap & ctl & BIT(8)) {
 		ctl &= ~BIT(8);
+#ifdef CONFIG_PROC_FS
+		/* Don't let users change irq affinity via procfs */
 		no_irq_affinity = 1;
+#endif
 	}
 #endif
 
-- 
1.7.10.2

Re: [tip:x86/platform] x86/platform: Introduce APIC post-initialization callback

[H. Peter Anvin]

On 06/06/2012 08:01 AM, tip-bot for Ido Yariv wrote:
> Some subarchitectures (such as vSMP)

Ido, could you please stop using this formula unless you actually have
evidence of anything other than vSMP having use for it?  It chronically
comes across as dishonest.

	-hpa

-- 
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel.  I don't speak on their behalf.

[tip:x86/platform] x86/vsmp: Fix linker error when CONFIG_PROC_FS is not set

[tip-bot for Ido Yariv]

Commit-ID:  d48daf37a3d2e2b28a61e615c0fc538301edb0dd
Gitweb:     http://git.kernel.org/tip/d48daf37a3d2e2b28a61e615c0fc538301edb0dd
Author:     Ido Yariv <ido@wizery.com>
AuthorDate: Thu, 14 Jun 2012 18:43:08 +0300
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Fri, 15 Jun 2012 13:54:11 +0200

x86/vsmp: Fix linker error when CONFIG_PROC_FS is not set

set_vsmp_pv_ops() references no_irq_affinity which is undeclared
if CONFIG_PROC_FS isn't set. Fix this by adding an #ifdef around
this variable's access.

Reported-by: Fengguang Wu <wfg@linux.intel.com>
Signed-off-by: Ido Yariv <ido@wizery.com>
Acked-by: Shai Fultheim <shai@scalemp.com>
Link: http://lkml.kernel.org/r/1339688588-12674-1-git-send-email-ido@wizery.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/vsmp_64.c |    3 +++
 1 files changed, 3 insertions(+), 0 deletions(-)

diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c
index 6b96a73..3f0285a 100644
--- a/arch/x86/kernel/vsmp_64.c
+++ b/arch/x86/kernel/vsmp_64.c
@@ -101,7 +101,10 @@ static void __init set_vsmp_pv_ops(void)
 #ifdef CONFIG_SMP
 	if (cap & ctl & BIT(8)) {
 		ctl &= ~BIT(8);
+#ifdef CONFIG_PROC_FS
+		/* Don't let users change irq affinity via procfs */
 		no_irq_affinity = 1;
+#endif
 	}
 #endif
 

Re: [tip:x86/platform] x86/platform: Introduce APIC post-initialization callback

[Ido Yariv]

Hi Peter,

On Thu, Jun 14, 2012 at 10:34:03AM -0700, H. Peter Anvin wrote:
> On 06/06/2012 08:01 AM, tip-bot for Ido Yariv wrote:
> > Some subarchitectures (such as vSMP)
> 
> Ido, could you please stop using this formula unless you actually have
> evidence of anything other than vSMP having use for it?  It chronically
> comes across as dishonest.

It was not my intention to disguise vSMP specific features as generic
ones, and I'm sorry if it seemed that way. I assumed other
virtualization architectures might benefit from these features, but at
least initially vSMP is the sole user.

Having said that, I'll be more specific in the future.

Thanks,
Ido.