[PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Len Brown]

From: Len Brown <len.brown@intel.com>

In Linux-3.9 we removed the mwait_idle() loop:
'x86 idle: remove mwait_idle() and "idle=mwait" cmdline param'
(69fb3676df3329a7142803bb3502fa59dc0db2e3)

The reasoning was that modern machines should be sufficiently
happy during the boot process using the default_idle() HALT loop,
until cpuidle loads and either acpi_idle or intel_idle
invoke the newer MWAIT-with-hints idle loop.

But two machines reported problems:
1. Certain Core2-era machines support MWAIT-C1 and HALT only.
   MWAIT-C1 is preferred for optimal power and performance.
   But if they support just C1, cpuidle never loads and
   so they use the boot-time default idle loop forever.

2. Some laptops will boot-hang if HALT is used,
   but will boot successfully if MWAIT is used.
   This appears to be a hidden assumption in BIOS SMI,
   that is presumably valid on the proprietary OS
   where the BIOS was validated.

   https://bugzilla.kernel.org/show_bug.cgi?id=60770

So here we effectively revert the patch above, restoring
the mwait_idle() loop.  However, we don't bother restoring
the idle=mwait cmdline parameter, since it appears to add
no value.

Maintainer notes:
For 3.9, simply revert 69fb3676df
for 3.10, patch -F3 applies, fuzz needed due to __cpuinit use in context
For 3.11, 3.12, 3.13, this patch applies cleanly

Cc: Mike Galbraith <bitbucket@online.de>
Cc: Ian Malone <ibmalone@gmail.com>
Cc: Josh Boyer <jwboyer@redhat.com>
Cc: <stable@vger.kernel.org> # 3.9, 3.10, 3.11, 3.12, 3.13
Signed-off-by: Len Brown <len.brown@intel.com>
---
 arch/x86/kernel/process.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 3fb8d95..db471a8 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -398,6 +398,49 @@ static void amd_e400_idle(void)
 		default_idle();
 }
 
+/*
+ * Intel Core2 and older machines prefer MWAIT over HALT for C1.
+ * We can't rely on cpuidle installing MWAIT, because it will not load
+ * on systems that support only C1 -- so the boot default must be MWAIT.
+ *  
+ * Some AMD machines are the opposite, they depend on using HALT.
+ *
+ * So for default C1, which is used during boot until cpuidle loads,
+ * use MWAIT-C1 on Intel HW that has it, else use HALT.
+ */
+static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
+{
+	if (c->x86_vendor != X86_VENDOR_INTEL)
+		return 0;
+
+	if (!cpu_has(c, X86_FEATURE_MWAIT))
+		return 0;
+
+	return 1;
+}
+
+/*
+ * MONITOR/MWAIT with no hints, used for default default C1 state.
+ * This invokes MWAIT with interrutps enabled and no flags,
+ * which is backwards compatible with the original MWAIT implementation.
+ */
+
+static void mwait_idle(void)
+{
+	if (!need_resched()) {
+		if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
+			clflush((void *)&current_thread_info()->flags);
+
+		__monitor((void *)&current_thread_info()->flags, 0, 0);
+		smp_mb();
+		if (!need_resched())
+			__sti_mwait(0, 0);
+		else
+			local_irq_enable();
+	} else
+		local_irq_enable();
+}
+
 void select_idle_routine(const struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_SMP
@@ -411,6 +454,9 @@ void select_idle_routine(const struct cpuinfo_x86 *c)
 		/* E400: APIC timer interrupt does not wake up CPU from C1e */
 		pr_info("using AMD E400 aware idle routine\n");
 		x86_idle = amd_e400_idle;
+	} else if (prefer_mwait_c1_over_halt(c)) {
+		pr_info("using mwait in idle threads\n");
+		x86_idle = mwait_idle;
 	} else
 		x86_idle = default_idle;
 }
-- 
1.8.5.2.309.ga25014b

[PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Len Brown]

From: Len Brown <len.brown@intel.com>

In Linux-3.9 we removed the mwait_idle() loop:
'x86 idle: remove mwait_idle() and "idle=mwait" cmdline param'
(69fb3676df3329a7142803bb3502fa59dc0db2e3)

The reasoning was that modern machines should be sufficiently
happy during the boot process using the default_idle() HALT loop,
until cpuidle loads and either acpi_idle or intel_idle
invoke the newer MWAIT-with-hints idle loop.

But two machines reported problems:
1. Certain Core2-era machines support MWAIT-C1 and HALT only.
   MWAIT-C1 is preferred for optimal power and performance.
   But if they support just C1, cpuidle never loads and
   so they use the boot-time default idle loop forever.

2. Some laptops will boot-hang if HALT is used,
   but will boot successfully if MWAIT is used.
   This appears to be a hidden assumption in BIOS SMI,
   that is presumably valid on the proprietary OS
   where the BIOS was validated.

   https://bugzilla.kernel.org/show_bug.cgi?id=60770

So here we effectively revert the patch above, restoring
the mwait_idle() loop.  However, we don't bother restoring
the idle=mwait cmdline parameter, since it appears to add
no value.

Maintainer notes:
For 3.9, simply revert 69fb3676df
for 3.10, patch -F3 applies, fuzz needed due to __cpuinit use in context
For 3.11, 3.12, 3.13, this patch applies cleanly

Cc: Mike Galbraith <bitbucket@online.de>
Cc: Ian Malone <ibmalone@gmail.com>
Cc: Josh Boyer <jwboyer@redhat.com>
Cc: <stable@vger.kernel.org> # 3.9, 3.10, 3.11, 3.12, 3.13
Signed-off-by: Len Brown <len.brown@intel.com>
---
 arch/x86/kernel/process.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index 3fb8d95..db471a8 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -398,6 +398,49 @@ static void amd_e400_idle(void)
 		default_idle();
 }
 
+/*
+ * Intel Core2 and older machines prefer MWAIT over HALT for C1.
+ * We can't rely on cpuidle installing MWAIT, because it will not load
+ * on systems that support only C1 -- so the boot default must be MWAIT.
+ *  
+ * Some AMD machines are the opposite, they depend on using HALT.
+ *
+ * So for default C1, which is used during boot until cpuidle loads,
+ * use MWAIT-C1 on Intel HW that has it, else use HALT.
+ */
+static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
+{
+	if (c->x86_vendor != X86_VENDOR_INTEL)
+		return 0;
+
+	if (!cpu_has(c, X86_FEATURE_MWAIT))
+		return 0;
+
+	return 1;
+}
+
+/*
+ * MONITOR/MWAIT with no hints, used for default default C1 state.
+ * This invokes MWAIT with interrutps enabled and no flags,
+ * which is backwards compatible with the original MWAIT implementation.
+ */
+
+static void mwait_idle(void)
+{
+	if (!need_resched()) {
+		if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
+			clflush((void *)&current_thread_info()->flags);
+
+		__monitor((void *)&current_thread_info()->flags, 0, 0);
+		smp_mb();
+		if (!need_resched())
+			__sti_mwait(0, 0);
+		else
+			local_irq_enable();
+	} else
+		local_irq_enable();
+}
+
 void select_idle_routine(const struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_SMP
@@ -411,6 +454,9 @@ void select_idle_routine(const struct cpuinfo_x86 *c)
 		/* E400: APIC timer interrupt does not wake up CPU from C1e */
 		pr_info("using AMD E400 aware idle routine\n");
 		x86_idle = amd_e400_idle;
+	} else if (prefer_mwait_c1_over_halt(c)) {
+		pr_info("using mwait in idle threads\n");
+		x86_idle = mwait_idle;
 	} else
 		x86_idle = default_idle;
 }
-- 
1.8.5.2.309.ga25014b

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Mike Galbraith]

On Wed, 2014-01-15 at 00:37 -0500, Len Brown wrote: 
> From: Len Brown <len.brown@intel.com>
> 
> In Linux-3.9 we removed the mwait_idle() loop:
> 'x86 idle: remove mwait_idle() and "idle=mwait" cmdline param'
> (69fb3676df3329a7142803bb3502fa59dc0db2e3)
> 
> The reasoning was that modern machines should be sufficiently
> happy during the boot process using the default_idle() HALT loop,
> until cpuidle loads and either acpi_idle or intel_idle
> invoke the newer MWAIT-with-hints idle loop.
> 
> But two machines reported problems:
> 1. Certain Core2-era machines support MWAIT-C1 and HALT only.
>    MWAIT-C1 is preferred for optimal power and performance.
>    But if they support just C1, cpuidle never loads and
>    so they use the boot-time default idle loop forever.

Q6600 box (allegedly) has a slightly greenish tinge again.

Tested-by: Mike Galbraith <bitbucket@online.de>

> 2. Some laptops will boot-hang if HALT is used,
>    but will boot successfully if MWAIT is used.
>    This appears to be a hidden assumption in BIOS SMI,
>    that is presumably valid on the proprietary OS
>    where the BIOS was validated.
> 
>    https://bugzilla.kernel.org/show_bug.cgi?id=60770
> 
> So here we effectively revert the patch above, restoring
> the mwait_idle() loop.  However, we don't bother restoring
> the idle=mwait cmdline parameter, since it appears to add
> no value.
> 
> Maintainer notes:
> For 3.9, simply revert 69fb3676df
> for 3.10, patch -F3 applies, fuzz needed due to __cpuinit use in context
> For 3.11, 3.12, 3.13, this patch applies cleanly
> 
> Cc: Mike Galbraith <bitbucket@online.de>
> Cc: Ian Malone <ibmalone@gmail.com>
> Cc: Josh Boyer <jwboyer@redhat.com>
> Cc: <stable@vger.kernel.org> # 3.9, 3.10, 3.11, 3.12, 3.13
> Signed-off-by: Len Brown <len.brown@intel.com>
> ---
>  arch/x86/kernel/process.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 46 insertions(+)
> 
> diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
> index 3fb8d95..db471a8 100644
> --- a/arch/x86/kernel/process.c
> +++ b/arch/x86/kernel/process.c
> @@ -398,6 +398,49 @@ static void amd_e400_idle(void)
>  		default_idle();
>  }
>  
> +/*
> + * Intel Core2 and older machines prefer MWAIT over HALT for C1.
> + * We can't rely on cpuidle installing MWAIT, because it will not load
> + * on systems that support only C1 -- so the boot default must be MWAIT.
> + *  
> + * Some AMD machines are the opposite, they depend on using HALT.
> + *
> + * So for default C1, which is used during boot until cpuidle loads,
> + * use MWAIT-C1 on Intel HW that has it, else use HALT.
> + */
> +static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
> +{
> +	if (c->x86_vendor != X86_VENDOR_INTEL)
> +		return 0;
> +
> +	if (!cpu_has(c, X86_FEATURE_MWAIT))
> +		return 0;
> +
> +	return 1;
> +}
> +
> +/*
> + * MONITOR/MWAIT with no hints, used for default default C1 state.
> + * This invokes MWAIT with interrutps enabled and no flags,
> + * which is backwards compatible with the original MWAIT implementation.
> + */
> +
> +static void mwait_idle(void)
> +{
> +	if (!need_resched()) {
> +		if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
> +			clflush((void *)&current_thread_info()->flags);
> +
> +		__monitor((void *)&current_thread_info()->flags, 0, 0);
> +		smp_mb();
> +		if (!need_resched())
> +			__sti_mwait(0, 0);
> +		else
> +			local_irq_enable();
> +	} else
> +		local_irq_enable();
> +}
> +
>  void select_idle_routine(const struct cpuinfo_x86 *c)
>  {
>  #ifdef CONFIG_SMP
> @@ -411,6 +454,9 @@ void select_idle_routine(const struct cpuinfo_x86 *c)
>  		/* E400: APIC timer interrupt does not wake up CPU from C1e */
>  		pr_info("using AMD E400 aware idle routine\n");
>  		x86_idle = amd_e400_idle;
> +	} else if (prefer_mwait_c1_over_halt(c)) {
> +		pr_info("using mwait in idle threads\n");
> +		x86_idle = mwait_idle;
>  	} else
>  		x86_idle = default_idle;
>  }

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Andy Lutomirski]

On 01/14/2014 09:37 PM, Len Brown wrote:
> From: Len Brown <len.brown@intel.com>
> 
> In Linux-3.9 we removed the mwait_idle() loop:
> 'x86 idle: remove mwait_idle() and "idle=mwait" cmdline param'
> (69fb3676df3329a7142803bb3502fa59dc0db2e3)
> 
> The reasoning was that modern machines should be sufficiently
> happy during the boot process using the default_idle() HALT loop,
> until cpuidle loads and either acpi_idle or intel_idle
> invoke the newer MWAIT-with-hints idle loop.
> 
> But two machines reported problems:
> 1. Certain Core2-era machines support MWAIT-C1 and HALT only.
>    MWAIT-C1 is preferred for optimal power and performance.
>    But if they support just C1, cpuidle never loads and
>    so they use the boot-time default idle loop forever.
> 
> 2. Some laptops will boot-hang if HALT is used,
>    but will boot successfully if MWAIT is used.
>    This appears to be a hidden assumption in BIOS SMI,
>    that is presumably valid on the proprietary OS
>    where the BIOS was validated.
> 
>    https://bugzilla.kernel.org/show_bug.cgi?id=60770
> 
> So here we effectively revert the patch above, restoring
> the mwait_idle() loop.  However, we don't bother restoring
> the idle=mwait cmdline parameter, since it appears to add
> no value.
> 
> Maintainer notes:
> For 3.9, simply revert 69fb3676df
> for 3.10, patch -F3 applies, fuzz needed due to __cpuinit use in context
> For 3.11, 3.12, 3.13, this patch applies cleanly
> 
> Cc: Mike Galbraith <bitbucket@online.de>
> Cc: Ian Malone <ibmalone@gmail.com>
> Cc: Josh Boyer <jwboyer@redhat.com>
> Cc: <stable@vger.kernel.org> # 3.9, 3.10, 3.11, 3.12, 3.13
> Signed-off-by: Len Brown <len.brown@intel.com>
> ---
>  arch/x86/kernel/process.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 46 insertions(+)
> 
> diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
> index 3fb8d95..db471a8 100644
> --- a/arch/x86/kernel/process.c
> +++ b/arch/x86/kernel/process.c
> @@ -398,6 +398,49 @@ static void amd_e400_idle(void)
>  		default_idle();
>  }
>  
> +/*
> + * Intel Core2 and older machines prefer MWAIT over HALT for C1.
> + * We can't rely on cpuidle installing MWAIT, because it will not load
> + * on systems that support only C1 -- so the boot default must be MWAIT.
> + *  
> + * Some AMD machines are the opposite, they depend on using HALT.
> + *
> + * So for default C1, which is used during boot until cpuidle loads,
> + * use MWAIT-C1 on Intel HW that has it, else use HALT.
> + */
> +static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
> +{
> +	if (c->x86_vendor != X86_VENDOR_INTEL)
> +		return 0;
> +
> +	if (!cpu_has(c, X86_FEATURE_MWAIT))
> +		return 0;
> +
> +	return 1;
> +}
> +
> +/*
> + * MONITOR/MWAIT with no hints, used for default default C1 state.
> + * This invokes MWAIT with interrutps enabled and no flags,
> + * which is backwards compatible with the original MWAIT implementation.
> + */
> +
> +static void mwait_idle(void)
> +{
> +	if (!need_resched()) {
> +		if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
> +			clflush((void *)&current_thread_info()->flags);
> +
> +		__monitor((void *)&current_thread_info()->flags, 0, 0);
> +		smp_mb();
> +		if (!need_resched())
> +			__sti_mwait(0, 0);
> +		else
> +			local_irq_enable();
> +	} else
> +		local_irq_enable();
> +}

Admittedly, there may be relatively few users left, but SMP users on
C1-only Core 2 machines can, in principle, benefit from the monitor
functionality of mwait to avoid rescheduling IPIs.  This stuff changed
recently so it now works with the cpuidle drivers (it used to be
terminally broken).  Should something be twiddling TS_POLLING
differently so that HLT gets the IPIs but mwait doesn't?

--Andy

> +
>  void select_idle_routine(const struct cpuinfo_x86 *c)
>  {
>  #ifdef CONFIG_SMP
> @@ -411,6 +454,9 @@ void select_idle_routine(const struct cpuinfo_x86 *c)
>  		/* E400: APIC timer interrupt does not wake up CPU from C1e */
>  		pr_info("using AMD E400 aware idle routine\n");
>  		x86_idle = amd_e400_idle;
> +	} else if (prefer_mwait_c1_over_halt(c)) {
> +		pr_info("using mwait in idle threads\n");
> +		x86_idle = mwait_idle;
>  	} else
>  		x86_idle = default_idle;
>  }
> 

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Mike Galbraith]

On Thu, 2014-01-16 at 14:00 -0800, Andy Lutomirski wrote:
> On 01/14/2014 09:37 PM, Len Brown wrote:
> > From: Len Brown <len.brown@intel.com>
> > 
> > In Linux-3.9 we removed the mwait_idle() loop:
> > 'x86 idle: remove mwait_idle() and "idle=mwait" cmdline param'
> > (69fb3676df3329a7142803bb3502fa59dc0db2e3)
> > 
> > The reasoning was that modern machines should be sufficiently
> > happy during the boot process using the default_idle() HALT loop,
> > until cpuidle loads and either acpi_idle or intel_idle
> > invoke the newer MWAIT-with-hints idle loop.
> > 
> > But two machines reported problems:
> > 1. Certain Core2-era machines support MWAIT-C1 and HALT only.
> >    MWAIT-C1 is preferred for optimal power and performance.
> >    But if they support just C1, cpuidle never loads and
> >    so they use the boot-time default idle loop forever.
> > 
> > 2. Some laptops will boot-hang if HALT is used,
> >    but will boot successfully if MWAIT is used.
> >    This appears to be a hidden assumption in BIOS SMI,
> >    that is presumably valid on the proprietary OS
> >    where the BIOS was validated.
> > 
> >    https://bugzilla.kernel.org/show_bug.cgi?id=60770
> > 
> > So here we effectively revert the patch above, restoring
> > the mwait_idle() loop.  However, we don't bother restoring
> > the idle=mwait cmdline parameter, since it appears to add
> > no value.
> > 
> > Maintainer notes:
> > For 3.9, simply revert 69fb3676df
> > for 3.10, patch -F3 applies, fuzz needed due to __cpuinit use in context
> > For 3.11, 3.12, 3.13, this patch applies cleanly
> > 
> > Cc: Mike Galbraith <bitbucket@online.de>
> > Cc: Ian Malone <ibmalone@gmail.com>
> > Cc: Josh Boyer <jwboyer@redhat.com>
> > Cc: <stable@vger.kernel.org> # 3.9, 3.10, 3.11, 3.12, 3.13
> > Signed-off-by: Len Brown <len.brown@intel.com>
> > ---
> >  arch/x86/kernel/process.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 46 insertions(+)
> > 
> > diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
> > index 3fb8d95..db471a8 100644
> > --- a/arch/x86/kernel/process.c
> > +++ b/arch/x86/kernel/process.c
> > @@ -398,6 +398,49 @@ static void amd_e400_idle(void)
> >  		default_idle();
> >  }
> >  
> > +/*
> > + * Intel Core2 and older machines prefer MWAIT over HALT for C1.
> > + * We can't rely on cpuidle installing MWAIT, because it will not load
> > + * on systems that support only C1 -- so the boot default must be MWAIT.
> > + *  
> > + * Some AMD machines are the opposite, they depend on using HALT.
> > + *
> > + * So for default C1, which is used during boot until cpuidle loads,
> > + * use MWAIT-C1 on Intel HW that has it, else use HALT.
> > + */
> > +static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
> > +{
> > +	if (c->x86_vendor != X86_VENDOR_INTEL)
> > +		return 0;
> > +
> > +	if (!cpu_has(c, X86_FEATURE_MWAIT))
> > +		return 0;
> > +
> > +	return 1;
> > +}
> > +
> > +/*
> > + * MONITOR/MWAIT with no hints, used for default default C1 state.
> > + * This invokes MWAIT with interrutps enabled and no flags,
> > + * which is backwards compatible with the original MWAIT implementation.
> > + */
> > +
> > +static void mwait_idle(void)
> > +{
> > +	if (!need_resched()) {
> > +		if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
> > +			clflush((void *)&current_thread_info()->flags);
> > +
> > +		__monitor((void *)&current_thread_info()->flags, 0, 0);
> > +		smp_mb();
> > +		if (!need_resched())
> > +			__sti_mwait(0, 0);
> > +		else
> > +			local_irq_enable();
> > +	} else
> > +		local_irq_enable();
> > +}
> 
> Admittedly, there may be relatively few users left, but SMP users on
> C1-only Core 2 machines can, in principle, benefit from the monitor
                                                    ^hugely
> functionality of mwait to avoid rescheduling IPIs.  This stuff changed
> recently so it now works with the cpuidle drivers (it used to be
> terminally broken).  Should something be twiddling TS_POLLING
> differently so that HLT gets the IPIs but mwait doesn't?

Urk, definitely.  The IPI is the primary cause of the size large cross
core scheduling performance regression for my aging, but lovely Q6600.

taskset 0xc pipe-test 1

3.8.13     3.397977 usecs/loop -- avg 3.400336 588.2 KHz
master+    4.798547 usecs/loop -- avg 4.791692 417.4 KHz

-Mike

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Mike Galbraith]

On Fri, 2014-01-17 at 05:20 +0100, Mike Galbraith wrote:

> taskset 0xc pipe-test 1
> 
> 3.8.13     3.397977 usecs/loop -- avg 3.400336 588.2 KHz
> master+    4.798547 usecs/loop -- avg 4.791692 417.4 KHz

Bah, those are apple/grape, these are apple/apple.

idle: kill unnecessary mwait_idle() resched IPIs

Set/clear polling instead.

Q6600, pipe-test scheduling cross core:
3.8.13                   487.2 KHz    1.000
3.13.0-master            415.5 KHz     .852
3.13.0-master+           415.2 KHz     .852     + restore mwait_idle
3.13.0-master++          488.5 KHz    1.002     + restore mwait_idle + IPI fix

Signed-off-by: Mike Galbraith <bitbucket@online.de>
---
 arch/x86/kernel/process.c |    4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -427,18 +427,18 @@ static int prefer_mwait_c1_over_halt(con
 
 static void mwait_idle(void)
 {
-	if (!need_resched()) {
+	if (!current_set_polling_and_test()) {
 		if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
 			clflush((void *)&current_thread_info()->flags);
 
 		__monitor((void *)&current_thread_info()->flags, 0, 0);
-		smp_mb();
 		if (!need_resched())
 			__sti_mwait(0, 0);
 		else
 			local_irq_enable();
 	} else
 		local_irq_enable();
+	__current_clr_polling();
 }
 
 void select_idle_routine(const struct cpuinfo_x86 *c)

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Mike Galbraith]

On Sat, 2014-01-18 at 10:33 +0100, Mike Galbraith wrote: 
> On Fri, 2014-01-17 at 05:20 +0100, Mike Galbraith wrote:
> 
> > taskset 0xc pipe-test 1
> > 
> > 3.8.13     3.397977 usecs/loop -- avg 3.400336 588.2 KHz
> > master+    4.798547 usecs/loop -- avg 4.791692 417.4 KHz
> 
> Bah, those are apple/grape, these are apple/apple.

Or, to make it more correct for 3.10..13, add the clflush barriers as
well for afflicted CPUs.

idle: kill unnecessary mwait_idle() resched IPIs

Set/clear polling instead.

Q6600, pipe-test scheduling cross core:
3.8.13                   487.2 KHz    1.000
3.13.0-master            415.5 KHz     .852
3.13.0-master+           415.2 KHz     .852     + restore mwait_idle
3.13.0-master++          488.5 KHz    1.002     + restore mwait_idle + IPI fix

Signed-off-by: Mike Galbraith <bitbucket@online.de>
Cc: <stable@vger.kernel.org> # 3.10+
---
 arch/x86/kernel/process.c |    9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -427,18 +427,21 @@ static int prefer_mwait_c1_over_halt(con
 
 static void mwait_idle(void)
 {
-	if (!need_resched()) {
-		if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR))
+	if (!current_set_polling_and_test()) {
+		if (this_cpu_has(X86_FEATURE_CLFLUSH_MONITOR)) {
+			mb();
 			clflush((void *)&current_thread_info()->flags);
+			mb();
+		}
 
 		__monitor((void *)&current_thread_info()->flags, 0, 0);
-		smp_mb();
 		if (!need_resched())
 			__sti_mwait(0, 0);
 		else
 			local_irq_enable();
 	} else
 		local_irq_enable();
+	__current_clr_polling();
 }
 
 void select_idle_routine(const struct cpuinfo_x86 *c)

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Ian Malone]

On 18 January 2014 at 16:14, Mike Galbraith <bitbucket@online.de> wrote:
> On Sat, 2014-01-18 at 10:33 +0100, Mike Galbraith wrote:
>> On Fri, 2014-01-17 at 05:20 +0100, Mike Galbraith wrote:

> Signed-off-by: Mike Galbraith <bitbucket@online.de>
> Cc: <stable@vger.kernel.org> # 3.10+
> ---
>  arch/x86/kernel/process.c |    9 ++++++---
>  1 file changed, 6 insertions(+), 3 deletions(-)
>
> --- a/arch/x86/kernel/process.c
> +++ b/arch/x86/kernel/process.c
> @@ -427,18 +427,21 @@ static int prefer_mwait_c1_over_halt(con

Hi, this series of patches never seem to have made it as far as the
mainline kernel, anyone know what needs to happen next?

-- 
imalone
http://ibmalone.blogspot.co.uk

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Mike Galbraith]

On Sat, 2015-03-14 at 23:44 +0000, Ian Malone wrote:
> On 18 January 2014 at 16:14, Mike Galbraith <bitbucket@online.de> wrote:
> > On Sat, 2014-01-18 at 10:33 +0100, Mike Galbraith wrote:
> >> On Fri, 2014-01-17 at 05:20 +0100, Mike Galbraith wrote:
> 
> > Signed-off-by: Mike Galbraith <bitbucket@online.de>
> > Cc: <stable@vger.kernel.org> # 3.10+
> > ---
> >  arch/x86/kernel/process.c |    9 ++++++---
> >  1 file changed, 6 insertions(+), 3 deletions(-)
> >
> > --- a/arch/x86/kernel/process.c
> > +++ b/arch/x86/kernel/process.c
> > @@ -427,18 +427,21 @@ static int prefer_mwait_c1_over_halt(con
> 
> Hi, this series of patches never seem to have made it as far as the
> mainline kernel, anyone know what needs to happen next?

My plan is to keep on carrying it locally for as long as I run new
kernels on crusty ole core2 boxen, then stop caring about them entirely
like the rest of the planet :)

	-Mike

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Ian Malone]

On 15 March 2015 at 04:53, Mike Galbraith <umgwanakikbuti@gmail.com> wrote:
> On Sat, 2015-03-14 at 23:44 +0000, Ian Malone wrote:
>> On 18 January 2014 at 16:14, Mike Galbraith <bitbucket@online.de> wrote:
>> > On Sat, 2014-01-18 at 10:33 +0100, Mike Galbraith wrote:
>> >> On Fri, 2014-01-17 at 05:20 +0100, Mike Galbraith wrote:
>>
>> > Signed-off-by: Mike Galbraith <bitbucket@online.de>
>> > Cc: <stable@vger.kernel.org> # 3.10+
>> > ---
>> >  arch/x86/kernel/process.c |    9 ++++++---
>> >  1 file changed, 6 insertions(+), 3 deletions(-)
>> >
>> > --- a/arch/x86/kernel/process.c
>> > +++ b/arch/x86/kernel/process.c
>> > @@ -427,18 +427,21 @@ static int prefer_mwait_c1_over_halt(con
>>
>> Hi, this series of patches never seem to have made it as far as the
>> mainline kernel, anyone know what needs to happen next?
>
> My plan is to keep on carrying it locally for as long as I run new
> kernels on crusty ole core2 boxen, then stop caring about them entirely
> like the rest of the planet :)
>

Looks like Ingo Molnar has committed to tip which is probably a good
sign, thanks all.
(Have to hand this system on to someone who wont be patching kernels...)

-- 
imalone
http://ibmalone.blogspot.co.uk

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Ingo Molnar]

* Ian Malone <ibmalone@gmail.com> wrote:

> On 15 March 2015 at 04:53, Mike Galbraith <umgwanakikbuti@gmail.com> wrote:
> > On Sat, 2015-03-14 at 23:44 +0000, Ian Malone wrote:
> >> On 18 January 2014 at 16:14, Mike Galbraith <bitbucket@online.de> wrote:
> >> > On Sat, 2014-01-18 at 10:33 +0100, Mike Galbraith wrote:
> >> >> On Fri, 2014-01-17 at 05:20 +0100, Mike Galbraith wrote:
> >>
> >> > Signed-off-by: Mike Galbraith <bitbucket@online.de>
> >> > Cc: <stable@vger.kernel.org> # 3.10+
> >> > ---
> >> >  arch/x86/kernel/process.c |    9 ++++++---
> >> >  1 file changed, 6 insertions(+), 3 deletions(-)
> >> >
> >> > --- a/arch/x86/kernel/process.c
> >> > +++ b/arch/x86/kernel/process.c
> >> > @@ -427,18 +427,21 @@ static int prefer_mwait_c1_over_halt(con
> >>
> >> Hi, this series of patches never seem to have made it as far as the
> >> mainline kernel, anyone know what needs to happen next?
> >
> > My plan is to keep on carrying it locally for as long as I run new
> > kernels on crusty ole core2 boxen, then stop caring about them entirely
> > like the rest of the planet :)
> >
> 
> Looks like Ingo Molnar has committed to tip which is probably a good
> sign, thanks all.
> (Have to hand this system on to someone who wont be patching kernels...)

Guys, since I don't have the affected hardware, mind testing the 
latest sched/core tree:

  git clone git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
  cd tip
  git checkout sched/core
  # build a test kernel and boot it

Or if you already have a kernel git tree, do something like this to 
pick up the scheduler development tree:

  cd linux.git
  git remote add git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip
  git remote update
  git checkout tip/sched/core
  # build a test kernel and boot it

and check whether the mwait related bugs are now fixed for good?

Thanks,

	Ingo

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Mike Galbraith]

On Tue, 2015-03-17 at 09:22 +0100, Ingo Molnar wrote:
> * Ian Malone <ibmalone@gmail.com> wrote:
> 
> > On 15 March 2015 at 04:53, Mike Galbraith <umgwanakikbuti@gmail.com> wrote:
> > > On Sat, 2015-03-14 at 23:44 +0000, Ian Malone wrote:
> > >> On 18 January 2014 at 16:14, Mike Galbraith <bitbucket@online.de> wrote:
> > >> > On Sat, 2014-01-18 at 10:33 +0100, Mike Galbraith wrote:
> > >> >> On Fri, 2014-01-17 at 05:20 +0100, Mike Galbraith wrote:
> > >>
> > >> > Signed-off-by: Mike Galbraith <bitbucket@online.de>
> > >> > Cc: <stable@vger.kernel.org> # 3.10+
> > >> > ---
> > >> >  arch/x86/kernel/process.c |    9 ++++++---
> > >> >  1 file changed, 6 insertions(+), 3 deletions(-)
> > >> >
> > >> > --- a/arch/x86/kernel/process.c
> > >> > +++ b/arch/x86/kernel/process.c
> > >> > @@ -427,18 +427,21 @@ static int prefer_mwait_c1_over_halt(con
> > >>
> > >> Hi, this series of patches never seem to have made it as far as the
> > >> mainline kernel, anyone know what needs to happen next?
> > >
> > > My plan is to keep on carrying it locally for as long as I run new
> > > kernels on crusty ole core2 boxen, then stop caring about them entirely
> > > like the rest of the planet :)
> > >
> > 
> > Looks like Ingo Molnar has committed to tip which is probably a good
> > sign, thanks all.
> > (Have to hand this system on to someone who wont be patching kernels...)
> 
> Guys, since I don't have the affected hardware, mind testing the 
> latest sched/core tree:
> 
>   git clone git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
>   cd tip
>   git checkout sched/core
>   # build a test kernel and boot it
> 
> Or if you already have a kernel git tree, do something like this to 
> pick up the scheduler development tree:
> 
>   cd linux.git
>   git remote add git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip
>   git remote update
>   git checkout tip/sched/core
>   # build a test kernel and boot it
> 
> and check whether the mwait related bugs are now fixed for good?

"marge" (Q6600 box) is mostly retired now, but I'll kick her out of her
rocking chair as soon as I can escape from bugzilla.  All should be
well, as she's been running it for ages, and ran it in master just a
couple days ago.

	-Mike

Re: [PATCH REGRESSION FIX] x86 idle: restore mwait_idle()

[Ian Malone]

On 17 March 2015 at 08:22, Ingo Molnar <mingo@kernel.org> wrote:
>
> * Ian Malone <ibmalone@gmail.com> wrote:
>

>> Looks like Ingo Molnar has committed to tip which is probably a good
>> sign, thanks all.
>> (Have to hand this system on to someone who wont be patching kernels...)
>
> Guys, since I don't have the affected hardware, mind testing the
> latest sched/core tree:
>
>   git clone git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
>   cd tip
>   git checkout sched/core
>   # build a test kernel and boot it
>
> Or if you already have a kernel git tree, do something like this to
> pick up the scheduler development tree:
>
>   cd linux.git
>   git remote add git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip
>   git remote update
>   git checkout tip/sched/core
>   # build a test kernel and boot it
>
> and check whether the mwait related bugs are now fixed for good?
>

Fixes https://bugzilla.kernel.org/show_bug.cgi?id=60770, thank you.

-- 
imalone
http://ibmalone.blogspot.co.uk

[tip:sched/core] sched/idle/x86: Optimize unnecessary mwait_idle( ) resched IPIs

[tip-bot for Mike Galbraith]

Commit-ID:  f8e617f4582995f7c25ef25b4167213120ad122b
Gitweb:     http://git.kernel.org/tip/f8e617f4582995f7c25ef25b4167213120ad122b
Author:     Mike Galbraith <bitbucket@online.de>
AuthorDate: Sat, 18 Jan 2014 17:14:44 +0100
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Mon, 16 Mar 2015 11:14:22 +0100

sched/idle/x86: Optimize unnecessary mwait_idle() resched IPIs

To fully take advantage of MWAIT, apparently the CLFLUSH instruction needs
another quirk on certain CPUs: proper barriers around it on certain machines.

On a Q6600 SMP system, pipe-test scheduling performance, cross core,
improves significantly:

  3.8.13                   487.2 KHz    1.000
  3.13.0-master            415.5 KHz     .852
  3.13.0-master+           415.2 KHz     .852     + restore mwait_idle
  3.13.0-master++          488.5 KHz    1.002     + restore mwait_idle + IPI fix

Since X86_BUG_CLFLUSH_MONITOR is already a quirk, don't create a separate
quirk for the extra smp_mb()s.

Signed-off-by: Mike Galbraith <bitbucket@online.de>
Cc: <stable@vger.kernel.org> # 3.10+
Cc: Borislav Petkov <bp@alien8.de>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ian Malone <ibmalone@gmail.com>
Cc: Josh Boyer <jwboyer@redhat.com>
Cc: Len Brown <len.brown@intel.com>
Cc: Len Brown <lenb@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/1390061684.5566.4.camel@marge.simpson.net
[ Ported to recent kernel, added comments about the quirk. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/kernel/process.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index da06f74..6ad8a63 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -428,18 +428,22 @@ static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
 
 static void mwait_idle(void)
 {
-	if (!need_resched()) {
-		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR))
+	if (!current_set_polling_and_test()) {
+		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR)) {
+			smp_mb(); /* quirk */
 			clflush((void *)&current_thread_info()->flags);
+			smp_mb(); /* quirk */
+		}
 
 		__monitor((void *)&current_thread_info()->flags, 0, 0);
-		smp_mb();
 		if (!need_resched())
 			__sti_mwait(0, 0);
 		else
 			local_irq_enable();
-	} else
+	} else {
 		local_irq_enable();
+	}
+	__current_clr_polling();
 }
 
 void select_idle_routine(const struct cpuinfo_x86 *c)

[tip:sched/core] sched/idle/x86: Restore mwait_idle() to fix boot hangs, to improve power savings and to improve performance

[tip-bot for Len Brown]

Commit-ID:  b253149b843f89cd300cbdbea27ce1f847506f99
Gitweb:     http://git.kernel.org/tip/b253149b843f89cd300cbdbea27ce1f847506f99
Author:     Len Brown <len.brown@intel.com>
AuthorDate: Wed, 15 Jan 2014 00:37:34 -0500
Committer:  Ingo Molnar <mingo@kernel.org>
CommitDate: Mon, 16 Mar 2015 11:14:21 +0100

sched/idle/x86: Restore mwait_idle() to fix boot hangs, to improve power savings and to improve performance

In Linux-3.9 we removed the mwait_idle() loop:

  69fb3676df33 ("x86 idle: remove mwait_idle() and "idle=mwait" cmdline param")

The reasoning was that modern machines should be sufficiently
happy during the boot process using the default_idle() HALT
loop, until cpuidle loads and either acpi_idle or intel_idle
invoke the newer MWAIT-with-hints idle loop.

But two machines reported problems:

 1. Certain Core2-era machines support MWAIT-C1 and HALT only.
    MWAIT-C1 is preferred for optimal power and performance.
    But if they support just C1, cpuidle never loads and
    so they use the boot-time default idle loop forever.

 2. Some laptops will boot-hang if HALT is used,
    but will boot successfully if MWAIT is used.
    This appears to be a hidden assumption in BIOS SMI,
    that is presumably valid on the proprietary OS
    where the BIOS was validated.

       https://bugzilla.kernel.org/show_bug.cgi?id=60770

So here we effectively revert the patch above, restoring
the mwait_idle() loop.  However, we don't bother restoring
the idle=mwait cmdline parameter, since it appears to add
no value.

Maintainer notes:

  For 3.9, simply revert 69fb3676df
  for 3.10, patch -F3 applies, fuzz needed due to __cpuinit use in
  context For 3.11, 3.12, 3.13, this patch applies cleanly

Tested-by: Mike Galbraith <bitbucket@online.de>
Signed-off-by: Len Brown <len.brown@intel.com>
Acked-by: Mike Galbraith <bitbucket@online.de>
Cc: <stable@vger.kernel.org> # 3.9+
Cc: Borislav Petkov <bp@alien8.de>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Ian Malone <ibmalone@gmail.com>
Cc: Josh Boyer <jwboyer@redhat.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/345254a551eb5a6a866e048d7ab570fd2193aca4.1389763084.git.len.brown@intel.com
[ Ported to recent kernels. ]
Signed-off-by: Ingo Molnar <mingo@kernel.org>
---
 arch/x86/include/asm/mwait.h |  8 ++++++++
 arch/x86/kernel/process.c    | 47 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index a1410db..653dfa7 100644
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -30,6 +30,14 @@ static inline void __mwait(unsigned long eax, unsigned long ecx)
 		     :: "a" (eax), "c" (ecx));
 }
 
+static inline void __sti_mwait(unsigned long eax, unsigned long ecx)
+{
+	trace_hardirqs_on();
+	/* "mwait %eax, %ecx;" */
+	asm volatile("sti; .byte 0x0f, 0x01, 0xc9;"
+		     :: "a" (eax), "c" (ecx));
+}
+
 /*
  * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
  * which can obviate IPI to trigger checking of need_resched.
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index e127dda..da06f74 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -24,6 +24,7 @@
 #include <asm/syscalls.h>
 #include <asm/idle.h>
 #include <asm/uaccess.h>
+#include <asm/mwait.h>
 #include <asm/i387.h>
 #include <asm/fpu-internal.h>
 #include <asm/debugreg.h>
@@ -398,6 +399,49 @@ static void amd_e400_idle(void)
 		default_idle();
 }
 
+/*
+ * Intel Core2 and older machines prefer MWAIT over HALT for C1.
+ * We can't rely on cpuidle installing MWAIT, because it will not load
+ * on systems that support only C1 -- so the boot default must be MWAIT.
+ *
+ * Some AMD machines are the opposite, they depend on using HALT.
+ *
+ * So for default C1, which is used during boot until cpuidle loads,
+ * use MWAIT-C1 on Intel HW that has it, else use HALT.
+ */
+static int prefer_mwait_c1_over_halt(const struct cpuinfo_x86 *c)
+{
+	if (c->x86_vendor != X86_VENDOR_INTEL)
+		return 0;
+
+	if (!cpu_has(c, X86_FEATURE_MWAIT))
+		return 0;
+
+	return 1;
+}
+
+/*
+ * MONITOR/MWAIT with no hints, used for default default C1 state.
+ * This invokes MWAIT with interrutps enabled and no flags,
+ * which is backwards compatible with the original MWAIT implementation.
+ */
+
+static void mwait_idle(void)
+{
+	if (!need_resched()) {
+		if (this_cpu_has(X86_BUG_CLFLUSH_MONITOR))
+			clflush((void *)&current_thread_info()->flags);
+
+		__monitor((void *)&current_thread_info()->flags, 0, 0);
+		smp_mb();
+		if (!need_resched())
+			__sti_mwait(0, 0);
+		else
+			local_irq_enable();
+	} else
+		local_irq_enable();
+}
+
 void select_idle_routine(const struct cpuinfo_x86 *c)
 {
 #ifdef CONFIG_SMP
@@ -411,6 +455,9 @@ void select_idle_routine(const struct cpuinfo_x86 *c)
 		/* E400: APIC timer interrupt does not wake up CPU from C1e */
 		pr_info("using AMD E400 aware idle routine\n");
 		x86_idle = amd_e400_idle;
+	} else if (prefer_mwait_c1_over_halt(c)) {
+		pr_info("using mwait in idle threads\n");
+		x86_idle = mwait_idle;
 	} else
 		x86_idle = default_idle;
 }