[PATCH 0/2] cpuidle: psci: Enable s2idle when using PSCI OSI

[PATCH 0/2] cpuidle: psci: Enable s2idle when using PSCI OSI

[Ulf Hansson]

When using PSCI OSI together with the PM domain topology, the domain idle
states becomes selected via calls to pm_runtime_put_sync_suspend(). This works
fine for the regular idle path, but not for system wide suspend-to-idle.

This problem is because runtime PM gets disabled by the PM core, during system
wide suspend. In this small series, these issues are being fixed.

Kind regards
Ulf Hansson

Ulf Hansson (2):
  PM / Domains: Enable locking for syscore devices for IRQ safe genpds
  cpuidle: psci: Enable s2idle when using OSI with the PM domain
    topology

 drivers/base/power/domain.c           | 13 ++++++++++--
 drivers/cpuidle/cpuidle-psci-domain.c |  2 ++
 drivers/cpuidle/cpuidle-psci.c        | 30 +++++++++++++++++++++++----
 3 files changed, 39 insertions(+), 6 deletions(-)

-- 
2.25.1

[PATCH 1/2] PM / Domains: Enable locking for syscore devices for IRQ safe genpds

[Ulf Hansson]

The genpd lock is currently not being used, while suspending/resuming
syscore devices through genpd. This because we need to avoid using a mutex
when running in the syscore phase.

However, the locking can be useful under special circumstances (as shown in
subsequent changes) and for a genpd having the flag GENPD_FLAG_IRQ_SAFE
set. Therefore, let's make use of the lock when it's possible.

Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>
---
 drivers/base/power/domain.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/drivers/base/power/domain.c b/drivers/base/power/domain.c
index 2cb5e04cf86c..55d99a36bf6b 100644
--- a/drivers/base/power/domain.c
+++ b/drivers/base/power/domain.c
@@ -1342,18 +1342,27 @@ static void genpd_complete(struct device *dev)
 static void genpd_syscore_switch(struct device *dev, bool suspend)
 {
 	struct generic_pm_domain *genpd;
+	bool use_lock;
 
 	genpd = dev_to_genpd_safe(dev);
 	if (!genpd)
 		return;
 
+	use_lock = genpd_is_irq_safe(genpd);
+
+	if (use_lock)
+		genpd_lock(genpd);
+
 	if (suspend) {
 		genpd->suspended_count++;
-		genpd_sync_power_off(genpd, false, 0);
+		genpd_sync_power_off(genpd, use_lock, 0);
 	} else {
-		genpd_sync_power_on(genpd, false, 0);
+		genpd_sync_power_on(genpd, use_lock, 0);
 		genpd->suspended_count--;
 	}
+
+	if (use_lock)
+		genpd_unlock(genpd);
 }
 
 void pm_genpd_syscore_poweroff(struct device *dev)
-- 
2.25.1

[PATCH 2/2] cpuidle: psci: Enable s2idle when using OSI with the PM domain topology

[Ulf Hansson]

To select domain idle states for cpuidle-psci, the PM domains via genpd are
being managed by using runtime PM. This works fine for the regular idle
path, but it doesn't when doing s2idle.

More precisely, the domain idle states becomes temporarily disabled, which
is because the PM core disables runtime PM for devices during system
suspend. Even if genpd tries to power off the PM domain in the
suspend_noirq phase, that doesn't help to properly select a domain idle
state, as this needs to be done on per CPU basis.

Let's address the issue by enabling the syscore flag for the attached CPU
devices. This prevents genpd from trying to power off the corresponding PM
domains in the suspend_noirq phase. Moreover, let's assign a specific
->enter_s2idle() callback for the corresponding domain idle state and let
it invoke pm_genpd_syscore_poweroff|poweron(), rather than using runtime
PM.

Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>
---
 drivers/cpuidle/cpuidle-psci-domain.c |  2 ++
 drivers/cpuidle/cpuidle-psci.c        | 30 +++++++++++++++++++++++----
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c
index b6e9649ab0da..65437ba5fa78 100644
--- a/drivers/cpuidle/cpuidle-psci-domain.c
+++ b/drivers/cpuidle/cpuidle-psci-domain.c
@@ -324,6 +324,8 @@ struct device *psci_dt_attach_cpu(int cpu)
 	if (cpu_online(cpu))
 		pm_runtime_get_sync(dev);
 
+	dev_pm_syscore_device(dev, true);
+
 	return dev;
 }
 
diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
index 74463841805f..6322d55a0a7d 100644
--- a/drivers/cpuidle/cpuidle-psci.c
+++ b/drivers/cpuidle/cpuidle-psci.c
@@ -19,6 +19,7 @@
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/psci.h>
+#include <linux/pm_domain.h>
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/string.h>
@@ -52,8 +53,9 @@ static inline int psci_enter_state(int idx, u32 state)
 	return CPU_PM_CPU_IDLE_ENTER_PARAM(psci_cpu_suspend_enter, idx, state);
 }
 
-static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
-					struct cpuidle_driver *drv, int idx)
+static int __psci_enter_domain_idle_state(struct cpuidle_device *dev,
+					  struct cpuidle_driver *drv, int idx,
+					  bool s2idle)
 {
 	struct psci_cpuidle_data *data = this_cpu_ptr(&psci_cpuidle_data);
 	u32 *states = data->psci_states;
@@ -66,7 +68,10 @@ static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
 		return -1;
 
 	/* Do runtime PM to manage a hierarchical CPU toplogy. */
-	pm_runtime_put_sync_suspend(pd_dev);
+	if (s2idle)
+		pm_genpd_syscore_poweroff(pd_dev);
+	else
+		pm_runtime_put_sync_suspend(pd_dev);
 
 	state = psci_get_domain_state();
 	if (!state)
@@ -74,7 +79,10 @@ static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
 
 	ret = psci_cpu_suspend_enter(state) ? -1 : idx;
 
-	pm_runtime_get_sync(pd_dev);
+	if (s2idle)
+		pm_genpd_syscore_poweron(pd_dev);
+	else
+		pm_runtime_get_sync(pd_dev);
 
 	cpu_pm_exit();
 
@@ -83,6 +91,19 @@ static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
 	return ret;
 }
 
+static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
+					struct cpuidle_driver *drv, int idx)
+{
+	return __psci_enter_domain_idle_state(dev, drv, idx, false);
+}
+
+static int psci_enter_s2idle_domain_idle_state(struct cpuidle_device *dev,
+					       struct cpuidle_driver *drv,
+					       int idx)
+{
+	return __psci_enter_domain_idle_state(dev, drv, idx, true);
+}
+
 static int psci_idle_cpuhp_up(unsigned int cpu)
 {
 	struct device *pd_dev = __this_cpu_read(psci_cpuidle_data.dev);
@@ -170,6 +191,7 @@ static int psci_dt_cpu_init_topology(struct cpuidle_driver *drv,
 	 * deeper states.
 	 */
 	drv->states[state_count - 1].enter = psci_enter_domain_idle_state;
+	drv->states[state_count - 1].enter_s2idle = psci_enter_s2idle_domain_idle_state;
 	psci_cpuidle_use_cpuhp = true;
 
 	return 0;
-- 
2.25.1

Re: [PATCH 2/2] cpuidle: psci: Enable s2idle when using OSI with the PM domain topology

[Sudeep Holla]

On Tue, Sep 01, 2020 at 10:27:07AM +0200, Ulf Hansson wrote:
> To select domain idle states for cpuidle-psci, the PM domains via genpd are
> being managed by using runtime PM. This works fine for the regular idle
> path, but it doesn't when doing s2idle.
>
> More precisely, the domain idle states becomes temporarily disabled, which
> is because the PM core disables runtime PM for devices during system
> suspend.

When you refer system suspend above, you mean both S2R and S2I ?

> Even if genpd tries to power off the PM domain in the
> suspend_noirq phase, that doesn't help to properly select a domain idle
> state, as this needs to be done on per CPU basis.
>

And what prevents doing per CPU basis ?

> Let's address the issue by enabling the syscore flag for the attached CPU
> devices. This prevents genpd from trying to power off the corresponding PM
> domains in the suspend_noirq phase. Moreover, let's assign a specific
> ->enter_s2idle() callback for the corresponding domain idle state and let
> it invoke pm_genpd_syscore_poweroff|poweron(), rather than using runtime
> PM.
>

The syscore_suspend is not executed for S2I and using syscore APIs here
is bit confusing IMO. If Rafael is fine, I have no objections.

> Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>
> ---
>  drivers/cpuidle/cpuidle-psci-domain.c |  2 ++
>  drivers/cpuidle/cpuidle-psci.c        | 30 +++++++++++++++++++++++----
>  2 files changed, 28 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c
> index b6e9649ab0da..65437ba5fa78 100644
> --- a/drivers/cpuidle/cpuidle-psci-domain.c
> +++ b/drivers/cpuidle/cpuidle-psci-domain.c
> @@ -324,6 +324,8 @@ struct device *psci_dt_attach_cpu(int cpu)
>  	if (cpu_online(cpu))
>  		pm_runtime_get_sync(dev);
>
> +	dev_pm_syscore_device(dev, true);
> +
>  	return dev;
>  }
>
> diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
> index 74463841805f..6322d55a0a7d 100644
> --- a/drivers/cpuidle/cpuidle-psci.c
> +++ b/drivers/cpuidle/cpuidle-psci.c
> @@ -19,6 +19,7 @@
>  #include <linux/of_device.h>
>  #include <linux/platform_device.h>
>  #include <linux/psci.h>
> +#include <linux/pm_domain.h>
>  #include <linux/pm_runtime.h>
>  #include <linux/slab.h>
>  #include <linux/string.h>
> @@ -52,8 +53,9 @@ static inline int psci_enter_state(int idx, u32 state)
>  	return CPU_PM_CPU_IDLE_ENTER_PARAM(psci_cpu_suspend_enter, idx, state);
>  }
>
> -static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
> -					struct cpuidle_driver *drv, int idx)
> +static int __psci_enter_domain_idle_state(struct cpuidle_device *dev,
> +					  struct cpuidle_driver *drv, int idx,
> +					  bool s2idle)
>  {
>  	struct psci_cpuidle_data *data = this_cpu_ptr(&psci_cpuidle_data);
>  	u32 *states = data->psci_states;
> @@ -66,7 +68,10 @@ static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
>  		return -1;
>
>  	/* Do runtime PM to manage a hierarchical CPU toplogy. */
> -	pm_runtime_put_sync_suspend(pd_dev);
> +	if (s2idle)
> +		pm_genpd_syscore_poweroff(pd_dev);
> +	else
> +		pm_runtime_put_sync_suspend(pd_dev);

Since CPU genpd is special and handled so in core, can this be moved to core ?
I mean pm_runtime_put_sync_suspend handle that based genpd_is_cpu_domain.

--
Regards,
Sudeep

Re: [PATCH 2/2] cpuidle: psci: Enable s2idle when using OSI with the PM domain topology

[Rafael J. Wysocki]

On Thu, Oct 1, 2020 at 12:18 PM Sudeep Holla <sudeep.holla@arm.com> wrote:
>
> On Tue, Sep 01, 2020 at 10:27:07AM +0200, Ulf Hansson wrote:
> > To select domain idle states for cpuidle-psci, the PM domains via genpd are
> > being managed by using runtime PM. This works fine for the regular idle
> > path, but it doesn't when doing s2idle.
> >
> > More precisely, the domain idle states becomes temporarily disabled, which
> > is because the PM core disables runtime PM for devices during system
> > suspend.
>
> When you refer system suspend above, you mean both S2R and S2I ?
>
> > Even if genpd tries to power off the PM domain in the
> > suspend_noirq phase, that doesn't help to properly select a domain idle
> > state, as this needs to be done on per CPU basis.
> >
>
> And what prevents doing per CPU basis ?
>
> > Let's address the issue by enabling the syscore flag for the attached CPU
> > devices. This prevents genpd from trying to power off the corresponding PM
> > domains in the suspend_noirq phase. Moreover, let's assign a specific
> > ->enter_s2idle() callback for the corresponding domain idle state and let
> > it invoke pm_genpd_syscore_poweroff|poweron(), rather than using runtime
> > PM.
> >
>
> The syscore_suspend is not executed for S2I and using syscore APIs here
> is bit confusing IMO.

Right.

> If Rafael is fine,

No, I'm not.

Cheers!

> I have no objections.
>
> > Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>
> > ---
> >  drivers/cpuidle/cpuidle-psci-domain.c |  2 ++
> >  drivers/cpuidle/cpuidle-psci.c        | 30 +++++++++++++++++++++++----
> >  2 files changed, 28 insertions(+), 4 deletions(-)
> >
> > diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c
> > index b6e9649ab0da..65437ba5fa78 100644
> > --- a/drivers/cpuidle/cpuidle-psci-domain.c
> > +++ b/drivers/cpuidle/cpuidle-psci-domain.c
> > @@ -324,6 +324,8 @@ struct device *psci_dt_attach_cpu(int cpu)
> >       if (cpu_online(cpu))
> >               pm_runtime_get_sync(dev);
> >
> > +     dev_pm_syscore_device(dev, true);
> > +
> >       return dev;
> >  }
> >
> > diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
> > index 74463841805f..6322d55a0a7d 100644
> > --- a/drivers/cpuidle/cpuidle-psci.c
> > +++ b/drivers/cpuidle/cpuidle-psci.c
> > @@ -19,6 +19,7 @@
> >  #include <linux/of_device.h>
> >  #include <linux/platform_device.h>
> >  #include <linux/psci.h>
> > +#include <linux/pm_domain.h>
> >  #include <linux/pm_runtime.h>
> >  #include <linux/slab.h>
> >  #include <linux/string.h>
> > @@ -52,8 +53,9 @@ static inline int psci_enter_state(int idx, u32 state)
> >       return CPU_PM_CPU_IDLE_ENTER_PARAM(psci_cpu_suspend_enter, idx, state);
> >  }
> >
> > -static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
> > -                                     struct cpuidle_driver *drv, int idx)
> > +static int __psci_enter_domain_idle_state(struct cpuidle_device *dev,
> > +                                       struct cpuidle_driver *drv, int idx,
> > +                                       bool s2idle)
> >  {
> >       struct psci_cpuidle_data *data = this_cpu_ptr(&psci_cpuidle_data);
> >       u32 *states = data->psci_states;
> > @@ -66,7 +68,10 @@ static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
> >               return -1;
> >
> >       /* Do runtime PM to manage a hierarchical CPU toplogy. */
> > -     pm_runtime_put_sync_suspend(pd_dev);
> > +     if (s2idle)
> > +             pm_genpd_syscore_poweroff(pd_dev);
> > +     else
> > +             pm_runtime_put_sync_suspend(pd_dev);
>
> Since CPU genpd is special and handled so in core, can this be moved to core ?
> I mean pm_runtime_put_sync_suspend handle that based genpd_is_cpu_domain.
>
> --
> Regards,
> Sudeep

Re: [PATCH 2/2] cpuidle: psci: Enable s2idle when using OSI with the PM domain topology

[Ulf Hansson]

On Thu, 1 Oct 2020 at 12:18, Sudeep Holla <sudeep.holla@arm.com> wrote:
>
> On Tue, Sep 01, 2020 at 10:27:07AM +0200, Ulf Hansson wrote:
> > To select domain idle states for cpuidle-psci, the PM domains via genpd are
> > being managed by using runtime PM. This works fine for the regular idle
> > path, but it doesn't when doing s2idle.
> >
> > More precisely, the domain idle states becomes temporarily disabled, which
> > is because the PM core disables runtime PM for devices during system
> > suspend.
>
> When you refer system suspend above, you mean both S2R and S2I ?

Correct.

Although, there is no problem with S2R to reach the proper idlestate,
because of the way we offline all but the boot CPU.

>
> > Even if genpd tries to power off the PM domain in the
> > suspend_noirq phase, that doesn't help to properly select a domain idle
> > state, as this needs to be done on per CPU basis.
> >
>
> And what prevents doing per CPU basis ?

The PM core doesn't execute the system suspend callbacks on a per CPU basis.

>
> > Let's address the issue by enabling the syscore flag for the attached CPU
> > devices. This prevents genpd from trying to power off the corresponding PM
> > domains in the suspend_noirq phase. Moreover, let's assign a specific
> > ->enter_s2idle() callback for the corresponding domain idle state and let
> > it invoke pm_genpd_syscore_poweroff|poweron(), rather than using runtime
> > PM.
> >
>
> The syscore_suspend is not executed for S2I and using syscore APIs here
> is bit confusing IMO. If Rafael is fine, I have no objections.

That's correct, the syscore phase doesn't exist in the S2I path.

However, in some cases the same functions that are being called in the
syscore phase, are also being called for S2I. For example, have a look
at timekeeping_suspend(), which is being called from both paths.

In the end, I think the confusing part is the name of the genpd functions.

Maybe we should rename pm_genpd_syscore_poweroff|poweron() to
pm_genpd_suspend|resume() - or something along those lines.

Kind regards
Uffe

>
> > Signed-off-by: Ulf Hansson <ulf.hansson@linaro.org>
> > ---
> >  drivers/cpuidle/cpuidle-psci-domain.c |  2 ++
> >  drivers/cpuidle/cpuidle-psci.c        | 30 +++++++++++++++++++++++----
> >  2 files changed, 28 insertions(+), 4 deletions(-)
> >
> > diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c
> > index b6e9649ab0da..65437ba5fa78 100644
> > --- a/drivers/cpuidle/cpuidle-psci-domain.c
> > +++ b/drivers/cpuidle/cpuidle-psci-domain.c
> > @@ -324,6 +324,8 @@ struct device *psci_dt_attach_cpu(int cpu)
> >       if (cpu_online(cpu))
> >               pm_runtime_get_sync(dev);
> >
> > +     dev_pm_syscore_device(dev, true);
> > +
> >       return dev;
> >  }
> >
> > diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c
> > index 74463841805f..6322d55a0a7d 100644
> > --- a/drivers/cpuidle/cpuidle-psci.c
> > +++ b/drivers/cpuidle/cpuidle-psci.c
> > @@ -19,6 +19,7 @@
> >  #include <linux/of_device.h>
> >  #include <linux/platform_device.h>
> >  #include <linux/psci.h>
> > +#include <linux/pm_domain.h>
> >  #include <linux/pm_runtime.h>
> >  #include <linux/slab.h>
> >  #include <linux/string.h>
> > @@ -52,8 +53,9 @@ static inline int psci_enter_state(int idx, u32 state)
> >       return CPU_PM_CPU_IDLE_ENTER_PARAM(psci_cpu_suspend_enter, idx, state);
> >  }
> >
> > -static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
> > -                                     struct cpuidle_driver *drv, int idx)
> > +static int __psci_enter_domain_idle_state(struct cpuidle_device *dev,
> > +                                       struct cpuidle_driver *drv, int idx,
> > +                                       bool s2idle)
> >  {
> >       struct psci_cpuidle_data *data = this_cpu_ptr(&psci_cpuidle_data);
> >       u32 *states = data->psci_states;
> > @@ -66,7 +68,10 @@ static int psci_enter_domain_idle_state(struct cpuidle_device *dev,
> >               return -1;
> >
> >       /* Do runtime PM to manage a hierarchical CPU toplogy. */
> > -     pm_runtime_put_sync_suspend(pd_dev);
> > +     if (s2idle)
> > +             pm_genpd_syscore_poweroff(pd_dev);
> > +     else
> > +             pm_runtime_put_sync_suspend(pd_dev);
>
> Since CPU genpd is special and handled so in core, can this be moved to core ?
> I mean pm_runtime_put_sync_suspend handle that based genpd_is_cpu_domain.
>
> --
> Regards,
> Sudeep

Re: [PATCH 2/2] cpuidle: psci: Enable s2idle when using OSI with the PM domain topology

[Ulf Hansson]

On Thu, 1 Oct 2020 at 13:32, Ulf Hansson <ulf.hansson@linaro.org> wrote:
>
> On Thu, 1 Oct 2020 at 12:18, Sudeep Holla <sudeep.holla@arm.com> wrote:
> >
> > On Tue, Sep 01, 2020 at 10:27:07AM +0200, Ulf Hansson wrote:
> > > To select domain idle states for cpuidle-psci, the PM domains via genpd are
> > > being managed by using runtime PM. This works fine for the regular idle
> > > path, but it doesn't when doing s2idle.
> > >
> > > More precisely, the domain idle states becomes temporarily disabled, which
> > > is because the PM core disables runtime PM for devices during system
> > > suspend.
> >
> > When you refer system suspend above, you mean both S2R and S2I ?
>
> Correct.
>
> Although, there is no problem with S2R to reach the proper idlestate,
> because of the way we offline all but the boot CPU.
>
> >
> > > Even if genpd tries to power off the PM domain in the
> > > suspend_noirq phase, that doesn't help to properly select a domain idle
> > > state, as this needs to be done on per CPU basis.
> > >
> >
> > And what prevents doing per CPU basis ?
>
> The PM core doesn't execute the system suspend callbacks on a per CPU basis.
>
> >
> > > Let's address the issue by enabling the syscore flag for the attached CPU
> > > devices. This prevents genpd from trying to power off the corresponding PM
> > > domains in the suspend_noirq phase. Moreover, let's assign a specific
> > > ->enter_s2idle() callback for the corresponding domain idle state and let
> > > it invoke pm_genpd_syscore_poweroff|poweron(), rather than using runtime
> > > PM.
> > >
> >
> > The syscore_suspend is not executed for S2I and using syscore APIs here
> > is bit confusing IMO. If Rafael is fine, I have no objections.
>
> That's correct, the syscore phase doesn't exist in the S2I path.
>
> However, in some cases the same functions that are being called in the
> syscore phase, are also being called for S2I. For example, have a look
> at timekeeping_suspend(), which is being called from both paths.
>
> In the end, I think the confusing part is the name of the genpd functions.
>
> Maybe we should rename pm_genpd_syscore_poweroff|poweron() to
> pm_genpd_suspend|resume() - or something along those lines.
>

[...]

Rafael, I understand you have objections to the approach. Would
renaming the genpd APIs be a way forward? Another option is to add two
new genpd APIs along the side of the existing, not sure what would be
the best.

Kind regards
Uffe