[RFC V2 0/2] cpufreq: cppc: Add support for frequency invariance

[RFC V2 0/2] cpufreq: cppc: Add support for frequency invariance

[Viresh Kumar]

Hello,

CPPC cpufreq driver is used for ARM servers and this patch series tries
to provide frequency invariance support for them.

This is tested with some hacks, as I didn't have access to the right
hardware, on the ARM64 hikey board to check the overall functionality
and that works fine.

Ionela/Peter Puhov, it would be nice if you guys can give this a shot.

This is based of pm/linux-next and patches [1] and [2] which I sent
recently to cleanup arm64 topology stuff.

Changes since V1:
- The interface for setting the callbacks is improved, so different
  parts looking to provide their callbacks don't need to think about
  each other.

- Moved to per-cpu storage for storing the callback related data, AMU
  counters have higher priority with this.

--
viresh

[1] https://lore.kernel.org/lkml/7a171f710cdc0f808a2bfbd7db839c0d265527e7.1607579234.git.viresh.kumar@linaro.org/
[2] https://lore.kernel.org/lkml/5ffc7b9ed03c6301ac2f710f609282959491b526.1608010334.git.viresh.kumar@linaro.org/

Viresh Kumar (2):
  topology: Allow multiple entities to provide sched_freq_tick()
    callback
  cpufreq: cppc: Add support for frequency invariance

 arch/arm64/include/asm/topology.h |   8 +-
 arch/arm64/kernel/topology.c      |  89 +++++++++----------
 drivers/base/arch_topology.c      |  56 +++++++++++-
 drivers/cpufreq/cppc_cpufreq.c    | 140 +++++++++++++++++++++++++++++-
 include/linux/arch_topology.h     |  14 ++-
 kernel/sched/core.c               |   1 +
 6 files changed, 244 insertions(+), 64 deletions(-)

-- 
2.25.0.rc1.19.g042ed3e048af

[RFC V2 1/2] topology: Allow multiple entities to provide sched_freq_tick() callback

[Viresh Kumar]

This patch attempts to make it generic enough so other parts of the
kernel can also provide their own implementation of scale_freq_tick()
callback, which is called by the scheduler periodically to update the
per-cpu freq_scale variable.

The implementations now need to provide struct scale_freq_tick_data for
the CPUs for which they have hardware counters available, and a callback
gets registered for each possible CPU in a per-cpu variable.

The AMU counters are updated to adapt to this and they take the highest
priority if they are available, i.e. they will be used instead of CPPC
based counters for example.

Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
---
 arch/arm64/include/asm/topology.h |  8 +--
 arch/arm64/kernel/topology.c      | 89 ++++++++++++++-----------------
 drivers/base/arch_topology.c      | 56 +++++++++++++++++--
 include/linux/arch_topology.h     | 13 ++++-
 4 files changed, 104 insertions(+), 62 deletions(-)

diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h
index 3b8dca4eb08d..be6a53ba3e2d 100644
--- a/arch/arm64/include/asm/topology.h
+++ b/arch/arm64/include/asm/topology.h
@@ -17,15 +17,9 @@ int pcibus_to_node(struct pci_bus *bus);
 #include <linux/arch_topology.h>
 
 void update_freq_counters_refs(void);
-void topology_scale_freq_tick(void);
 
-#ifdef CONFIG_ARM64_AMU_EXTN
-/*
- * Replace task scheduler's default counter-based
- * frequency-invariance scale factor setting.
- */
+/* Replace task scheduler's default frequency-invariance scale factor setting */
 #define arch_scale_freq_tick topology_scale_freq_tick
-#endif /* CONFIG_ARM64_AMU_EXTN */
 
 /* Replace task scheduler's default frequency-invariant accounting */
 #define arch_set_freq_scale topology_set_freq_scale
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index 0fc2b32ddb45..22f056821585 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -199,8 +199,44 @@ static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate)
 	return 0;
 }
 
-static DEFINE_STATIC_KEY_FALSE(amu_fie_key);
-#define amu_freq_invariant() static_branch_unlikely(&amu_fie_key)
+static void amu_scale_freq_tick(void)
+{
+	u64 prev_core_cnt, prev_const_cnt;
+	u64 core_cnt, const_cnt, scale;
+
+	prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
+	prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
+
+	update_freq_counters_refs();
+
+	const_cnt = this_cpu_read(arch_const_cycles_prev);
+	core_cnt = this_cpu_read(arch_core_cycles_prev);
+
+	if (unlikely(core_cnt <= prev_core_cnt ||
+		     const_cnt <= prev_const_cnt))
+		return;
+
+	/*
+	 *	    /\core    arch_max_freq_scale
+	 * scale =  ------- * --------------------
+	 *	    /\const   SCHED_CAPACITY_SCALE
+	 *
+	 * See validate_cpu_freq_invariance_counters() for details on
+	 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
+	 */
+	scale = core_cnt - prev_core_cnt;
+	scale *= this_cpu_read(arch_max_freq_scale);
+	scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
+			  const_cnt - prev_const_cnt);
+
+	scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
+	this_cpu_write(freq_scale, (unsigned long)scale);
+}
+
+static struct scale_freq_tick_data amu_sftd = {
+	.source = SCALE_FREQ_SOURCE_AMU,
+	.scale_freq = amu_scale_freq_tick,
+};
 
 static void amu_fie_setup(const struct cpumask *cpus)
 {
@@ -227,7 +263,7 @@ static void amu_fie_setup(const struct cpumask *cpus)
 	if (!invariant && !cpumask_equal(amu_fie_cpus, cpu_present_mask))
 		return;
 
-	static_branch_enable(&amu_fie_key);
+	topology_set_scale_freq_tick(&amu_sftd, amu_fie_cpus);
 
 	pr_info("CPUs[%*pbl]: counters will be used for FIE.",
 		cpumask_pr_args(amu_fie_cpus));
@@ -283,53 +319,6 @@ static int __init init_amu_fie(void)
 }
 core_initcall(init_amu_fie);
 
-bool arch_freq_counters_available(const struct cpumask *cpus)
-{
-	return amu_freq_invariant() &&
-	       cpumask_subset(cpus, amu_fie_cpus);
-}
-
-void topology_scale_freq_tick(void)
-{
-	u64 prev_core_cnt, prev_const_cnt;
-	u64 core_cnt, const_cnt, scale;
-	int cpu = smp_processor_id();
-
-	if (!amu_freq_invariant())
-		return;
-
-	if (!cpumask_test_cpu(cpu, amu_fie_cpus))
-		return;
-
-	prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
-	prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
-
-	update_freq_counters_refs();
-
-	const_cnt = this_cpu_read(arch_const_cycles_prev);
-	core_cnt = this_cpu_read(arch_core_cycles_prev);
-
-	if (unlikely(core_cnt <= prev_core_cnt ||
-		     const_cnt <= prev_const_cnt))
-		return;
-
-	/*
-	 *	    /\core    arch_max_freq_scale
-	 * scale =  ------- * --------------------
-	 *	    /\const   SCHED_CAPACITY_SCALE
-	 *
-	 * See validate_cpu_freq_invariance_counters() for details on
-	 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
-	 */
-	scale = core_cnt - prev_core_cnt;
-	scale *= this_cpu_read(arch_max_freq_scale);
-	scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
-			  const_cnt - prev_const_cnt);
-
-	scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
-	this_cpu_write(freq_scale, (unsigned long)scale);
-}
-
 #ifdef CONFIG_ACPI_CPPC_LIB
 #include <acpi/cppc_acpi.h>
 
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index de8587cc119e..07774fb47bc4 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -21,17 +21,65 @@
 #include <linux/sched.h>
 #include <linux/smp.h>
 
+static DEFINE_PER_CPU(struct scale_freq_tick_data *, sft_data);
+static struct cpumask scale_freq_tick_mask;
+
+static bool supports_scale_freq_tick(const struct cpumask *cpus)
+{
+	return cpumask_subset(cpus, &scale_freq_tick_mask);
+}
+
 bool topology_scale_freq_invariant(void)
 {
 	return cpufreq_supports_freq_invariance() ||
-	       arch_freq_counters_available(cpu_online_mask);
+	       supports_scale_freq_tick(cpu_online_mask);
+}
+
+void topology_set_scale_freq_tick(struct scale_freq_tick_data *data,
+				  const struct cpumask *cpus)
+{
+	struct scale_freq_tick_data *sftd;
+	int cpu;
+
+	for_each_cpu(cpu, cpus) {
+		sftd = per_cpu(sft_data, cpu);
+
+		/* Use AMU counters whenever possible */
+		if (!sftd || sftd->source != SCALE_FREQ_SOURCE_AMU) {
+			per_cpu(sft_data, cpu) = data;
+			cpumask_set_cpu(cpu, &scale_freq_tick_mask);
+		}
+	}
 }
+EXPORT_SYMBOL_GPL(topology_set_scale_freq_tick);
 
-__weak bool arch_freq_counters_available(const struct cpumask *cpus)
+void topology_clear_scale_freq_tick(enum scale_freq_tick_source source,
+				    const struct cpumask *cpus)
 {
-	return false;
+	struct scale_freq_tick_data *sftd;
+	int cpu;
+
+	for_each_cpu(cpu, cpus) {
+		sftd = per_cpu(sft_data, cpu);
+
+		if (sftd && sftd->source == source) {
+			per_cpu(sft_data, cpu) = NULL;
+			cpumask_clear_cpu(cpu, &scale_freq_tick_mask);
+		}
+	}
 }
+EXPORT_SYMBOL_GPL(topology_clear_scale_freq_tick);
+
+void topology_scale_freq_tick(void)
+{
+	struct scale_freq_tick_data *sftd = *this_cpu_ptr(&sft_data);
+
+	if (sftd)
+		sftd->scale_freq();
+}
+
 DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
+EXPORT_SYMBOL_GPL(freq_scale);
 
 void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
 			     unsigned long max_freq)
@@ -47,7 +95,7 @@ void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
 	 * want to update the scale factor with information from CPUFREQ.
 	 * Instead the scale factor will be updated from arch_scale_freq_tick.
 	 */
-	if (arch_freq_counters_available(cpus))
+	if (supports_scale_freq_tick(cpus))
 		return;
 
 	scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq;
diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
index 0f6cd6b73a61..b2422ebef2dd 100644
--- a/include/linux/arch_topology.h
+++ b/include/linux/arch_topology.h
@@ -34,7 +34,18 @@ void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
 			     unsigned long max_freq);
 bool topology_scale_freq_invariant(void);
 
-bool arch_freq_counters_available(const struct cpumask *cpus);
+enum scale_freq_tick_source {
+	SCALE_FREQ_SOURCE_AMU,
+};
+
+struct scale_freq_tick_data {
+	enum scale_freq_tick_source source;
+	void (*scale_freq)(void);
+};
+
+void topology_scale_freq_tick(void);
+void topology_set_scale_freq_tick(struct scale_freq_tick_data *data, const struct cpumask *cpus);
+void topology_clear_scale_freq_tick(enum scale_freq_tick_source source, const struct cpumask *cpus);
 
 DECLARE_PER_CPU(unsigned long, thermal_pressure);
 
-- 
2.25.0.rc1.19.g042ed3e048af

[RFC V2 2/2] cpufreq: cppc: Add support for frequency invariance

[Viresh Kumar]

The Frequency Invariance Engine (FIE) is providing a frequency scaling
correction factor that helps achieve more accurate load-tracking.

Normally, this scaling factor can be obtained directly with the help of
the cpufreq drivers as they know the exact frequency the hardware is
running at. But that isn't the case for CPPC cpufreq driver.

Another way of obtaining that is using the AMU counter support, which is
already present in kernel, but that hardware is optional for platforms.

This patch thus obtains this scaling factor using the existing logic
present in the cppc driver. Note that the AMUs have higher priority than
CPPC counters if available, though the CPPC driver doesn't need to have
any special handling for that.

This also exports sched_setattr_nocheck() as the CPPC driver can be
built as a module.

Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
---
 drivers/cpufreq/cppc_cpufreq.c | 140 ++++++++++++++++++++++++++++++++-
 include/linux/arch_topology.h  |   1 +
 kernel/sched/core.c            |   1 +
 3 files changed, 140 insertions(+), 2 deletions(-)

diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 7cc9bd8568de..a739e20aefd6 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -10,14 +10,18 @@
 
 #define pr_fmt(fmt)	"CPPC Cpufreq:"	fmt
 
+#include <linux/arch_topology.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/delay.h>
 #include <linux/cpu.h>
 #include <linux/cpufreq.h>
 #include <linux/dmi.h>
+#include <linux/irq_work.h>
+#include <linux/kthread.h>
 #include <linux/time.h>
 #include <linux/vmalloc.h>
+#include <uapi/linux/sched/types.h>
 
 #include <asm/unaligned.h>
 
@@ -39,6 +43,15 @@
 static struct cppc_cpudata **all_cpu_data;
 static bool boost_supported;
 
+struct cppc_freq_invariance {
+	struct kthread_worker *worker;
+	struct irq_work irq_work;
+	struct kthread_work work;
+	struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
+	unsigned int max_freq;
+};
+static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_f_i);
+
 struct cppc_workaround_oem_info {
 	char oem_id[ACPI_OEM_ID_SIZE + 1];
 	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
@@ -243,7 +256,7 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 	struct cppc_cpudata *cpu_data = all_cpu_data[policy->cpu];
 	struct cppc_perf_caps *caps = &cpu_data->perf_caps;
 	unsigned int cpu = policy->cpu;
-	int ret = 0;
+	int ret = 0, i;
 
 	cpu_data->cpu = cpu;
 	ret = cppc_get_perf_caps(cpu, caps);
@@ -300,6 +313,9 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
 
 	cpu_data->cur_policy = policy;
 
+	for_each_cpu(i, policy->cpus)
+		per_cpu(cppc_f_i, i).max_freq = policy->cpuinfo.max_freq;
+
 	/*
 	 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost
 	 * is supported.
@@ -374,7 +390,7 @@ static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
 {
 	struct cppc_cpudata *cpu_data = all_cpu_data[policy->cpu];
 	struct cppc_perf_caps *caps = &cpu_data->perf_caps;
-	int ret;
+	int ret, i;
 
 	if (!boost_supported) {
 		pr_err("BOOST not supported by CPU or firmware\n");
@@ -389,6 +405,9 @@ static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
 						       caps->nominal_perf);
 	policy->cpuinfo.max_freq = policy->max;
 
+	for_each_cpu(i, policy->related_cpus)
+		per_cpu(cppc_f_i, i).max_freq = policy->cpuinfo.max_freq;
+
 	ret = freq_qos_update_request(policy->max_freq_req, policy->max);
 	if (ret < 0)
 		return ret;
@@ -449,6 +468,120 @@ static void cppc_check_hisi_workaround(void)
 	acpi_put_table(tbl);
 }
 
+static void cppc_scale_freq_tick_workfn(struct kthread_work *work)
+{
+	struct cppc_freq_invariance *cppc_fi;
+	struct cppc_perf_fb_ctrs fb_ctrs = {0};
+	int cpu = raw_smp_processor_id();
+	struct cppc_cpudata *cpudata = all_cpu_data[cpu];
+	u64 rate;
+
+	cppc_fi = container_of(work, struct cppc_freq_invariance, work);
+
+	if (cppc_get_perf_ctrs(cpu, &fb_ctrs)) {
+		pr_info("%s: cppc_get_perf_ctrs() failed\n", __func__);
+		return;
+	}
+
+	rate = cppc_get_rate_from_fbctrs(cpudata, cppc_fi->prev_perf_fb_ctrs, fb_ctrs);
+	cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
+
+	rate <<= SCHED_CAPACITY_SHIFT;
+	per_cpu(freq_scale, cpu) = div64_u64(rate, cppc_fi->max_freq);
+}
+
+static void cppc_irq_work(struct irq_work *irq_work)
+{
+	struct cppc_freq_invariance *cppc_fi;
+
+	cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);
+	kthread_queue_work(cppc_fi->worker, &cppc_fi->work);
+}
+
+static void cppc_scale_freq_tick(void)
+{
+	struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_f_i, raw_smp_processor_id());
+
+	/*
+	 * cppc_get_perf_ctrs() can potentially sleep, call that from the right
+	 * context.
+	 */
+	irq_work_queue(&cppc_fi->irq_work);
+}
+
+static struct scale_freq_tick_data cppc_sftd = {
+	.source = SCALE_FREQ_SOURCE_CPPC,
+	.scale_freq = cppc_scale_freq_tick,
+};
+
+static void cppc_freq_invariance_exit(void)
+{
+	struct cppc_freq_invariance *cppc_fi;
+	int i;
+
+	if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
+		return;
+
+	topology_clear_scale_freq_tick(SCALE_FREQ_SOURCE_CPPC, cpu_present_mask);
+
+	for_each_possible_cpu(i) {
+		cppc_fi = &per_cpu(cppc_f_i, i);
+		if (cppc_fi->worker) {
+			irq_work_sync(&cppc_fi->irq_work);
+			kthread_destroy_worker(cppc_fi->worker);
+			cppc_fi->worker = NULL;
+		}
+	}
+}
+
+static void __init cppc_freq_invariance_init(void)
+{
+	struct cppc_perf_fb_ctrs fb_ctrs = {0};
+	struct cppc_freq_invariance *cppc_fi;
+	struct sched_attr attr = {
+		.size		= sizeof(struct sched_attr),
+		.sched_policy	= SCHED_DEADLINE,
+		.sched_nice	= 0,
+		.sched_priority	= 0,
+		/*
+		 * Fake (unused) bandwidth; workaround to "fix"
+		 * priority inheritance.
+		 */
+		.sched_runtime	= 1000000,
+		.sched_deadline = 10000000,
+		.sched_period	= 10000000,
+	};
+	struct kthread_worker *worker;
+	int i, ret;
+
+	if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
+		return;
+
+	for_each_possible_cpu(i) {
+		cppc_fi = &per_cpu(cppc_f_i, i);
+
+		kthread_init_work(&cppc_fi->work, cppc_scale_freq_tick_workfn);
+		init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
+		worker = kthread_create_worker_on_cpu(i, 0, "cppc:%d", i);
+		if (IS_ERR(worker))
+			return cppc_freq_invariance_exit();
+
+		cppc_fi->worker = worker;
+		ret = sched_setattr_nocheck(worker->task, &attr);
+		if (ret) {
+			pr_warn("%s: failed to set SCHED_DEADLINE\n", __func__);
+			return cppc_freq_invariance_exit();
+		}
+
+		ret = cppc_get_perf_ctrs(i, &fb_ctrs);
+		if (!ret)
+			per_cpu(cppc_fi->prev_perf_fb_ctrs, i) = fb_ctrs;
+	}
+
+	/* Register for freq-invariance */
+	topology_set_scale_freq_tick(&cppc_sftd, cpu_present_mask);
+}
+
 static int __init cppc_cpufreq_init(void)
 {
 	struct cppc_cpudata *cpu_data;
@@ -484,6 +617,8 @@ static int __init cppc_cpufreq_init(void)
 	if (ret)
 		goto out;
 
+	cppc_freq_invariance_init();
+
 	return ret;
 
 out:
@@ -504,6 +639,7 @@ static void __exit cppc_cpufreq_exit(void)
 	struct cppc_cpudata *cpu_data;
 	int i;
 
+	cppc_freq_invariance_exit();
 	cpufreq_unregister_driver(&cppc_cpufreq_driver);
 
 	for_each_possible_cpu(i) {
diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
index b2422ebef2dd..09205b584ca5 100644
--- a/include/linux/arch_topology.h
+++ b/include/linux/arch_topology.h
@@ -36,6 +36,7 @@ bool topology_scale_freq_invariant(void);
 
 enum scale_freq_tick_source {
 	SCALE_FREQ_SOURCE_AMU,
+	SCALE_FREQ_SOURCE_CPPC,
 };
 
 struct scale_freq_tick_data {
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 51514eef0a9d..76b2fa1a7aaa 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6075,6 +6075,7 @@ int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr)
 {
 	return __sched_setscheduler(p, attr, false, true);
 }
+EXPORT_SYMBOL_GPL(sched_setattr_nocheck);
 
 /**
  * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
-- 
2.25.0.rc1.19.g042ed3e048af

Re: [RFC V2 0/2] cpufreq: cppc: Add support for frequency invariance

[Ionela Voinescu]

Hi,

On Tuesday 15 Dec 2020 at 16:46:34 (+0530), Viresh Kumar wrote:
> Hello,
> 
> CPPC cpufreq driver is used for ARM servers and this patch series tries
> to provide frequency invariance support for them.
> 

Nitpick: cppc_cpufreq already supports cpufreq-based frequency
invariance, but not counter-based frequency invariance, when lacking
AMUs :).

> This is tested with some hacks, as I didn't have access to the right
> hardware, on the ARM64 hikey board to check the overall functionality
> and that works fine.
> 
> Ionela/Peter Puhov, it would be nice if you guys can give this a shot.
> 

I'll be on holiday starting tomorrow until January. I fully intend to
review and test this but I'll have to set myself a deadline for January
to do it.

Sorry for the delay,
Ionela.

> This is based of pm/linux-next and patches [1] and [2] which I sent
> recently to cleanup arm64 topology stuff.
> 
> Changes since V1:
> - The interface for setting the callbacks is improved, so different
>   parts looking to provide their callbacks don't need to think about
>   each other.
> 
> - Moved to per-cpu storage for storing the callback related data, AMU
>   counters have higher priority with this.
> 
> --
> viresh
> 
> [1] https://lore.kernel.org/lkml/7a171f710cdc0f808a2bfbd7db839c0d265527e7.1607579234.git.viresh.kumar@linaro.org/
> [2] https://lore.kernel.org/lkml/5ffc7b9ed03c6301ac2f710f609282959491b526.1608010334.git.viresh.kumar@linaro.org/
> 
> Viresh Kumar (2):
>   topology: Allow multiple entities to provide sched_freq_tick()
>     callback
>   cpufreq: cppc: Add support for frequency invariance
> 
>  arch/arm64/include/asm/topology.h |   8 +-
>  arch/arm64/kernel/topology.c      |  89 +++++++++----------
>  drivers/base/arch_topology.c      |  56 +++++++++++-
>  drivers/cpufreq/cppc_cpufreq.c    | 140 +++++++++++++++++++++++++++++-
>  include/linux/arch_topology.h     |  14 ++-
>  kernel/sched/core.c               |   1 +
>  6 files changed, 244 insertions(+), 64 deletions(-)
> 
> -- 
> 2.25.0.rc1.19.g042ed3e048af
> 

Re: [RFC V2 1/2] topology: Allow multiple entities to provide sched_freq_tick() callback

[Ionela Voinescu]

Hi,

I focused for now on the design of the solution and mostly on the
generic code in arch_topology.c, as that will then impact the arm64
code. 

On Tuesday 15 Dec 2020 at 16:46:35 (+0530), Viresh Kumar wrote:
[..]
>  
> diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
> index de8587cc119e..07774fb47bc4 100644
> --- a/drivers/base/arch_topology.c
> +++ b/drivers/base/arch_topology.c
> @@ -21,17 +21,65 @@
>  #include <linux/sched.h>
>  #include <linux/smp.h>
>  
> +static DEFINE_PER_CPU(struct scale_freq_tick_data *, sft_data);
> +static struct cpumask scale_freq_tick_mask;
> +
> +static bool supports_scale_freq_tick(const struct cpumask *cpus)
> +{
> +	return cpumask_subset(cpus, &scale_freq_tick_mask);
> +}
> +
>  bool topology_scale_freq_invariant(void)
>  {
>  	return cpufreq_supports_freq_invariance() ||
> -	       arch_freq_counters_available(cpu_online_mask);
> +	       supports_scale_freq_tick(cpu_online_mask);
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
I think the names that involve "scale_freq_tick" are unfortunate and
unclear here. arch_scale_freq_tick() is a good wrapper as it refers to
some work being done for FIE on the tick, but for other functions and
structures here, any similar naming makes things unclear.

Basically all FIE related activity on the tick involves counters that
need regular sampling. So I'd suggest to keep to names that give a hint
about the source of information for frequency invariance, as it makes it
more clear what are the options you're choosing between for FIE.

s/supports_scale_freq_tick/supports_scale_freq_counters
s/scale_freq_tick_data/scale_freq_data
s/topology_set_scale_freq_tick/topology_set_scale_freq_source
s/topology_clear_scale_freq_tick/topology_clear_scale_freq_source
s/scale_freq_tick_mask/scale_freq_counters_mask

.. or similar.

> +}
> +
> +void topology_set_scale_freq_tick(struct scale_freq_tick_data *data,
> +				  const struct cpumask *cpus)
> +{
> +	struct scale_freq_tick_data *sftd;
> +	int cpu;
> +
> +	for_each_cpu(cpu, cpus) {
> +		sftd = per_cpu(sft_data, cpu);
> +
> +		/* Use AMU counters whenever possible */
> +		if (!sftd || sftd->source != SCALE_FREQ_SOURCE_AMU) {

I don't think the arch_topology driver should know anything about AMUs, as
they are an arm64 feature. SCALE_FREQ_SOURCE_ARCH might be better.
This way any architecture that uses the arch topology driver can register
some architected method which should take priority over all other
methods (at least until someone shows up wanting otherwise).

> +			per_cpu(sft_data, cpu) = data;
> +			cpumask_set_cpu(cpu, &scale_freq_tick_mask);
> +		}
> +	}
>  }
> +EXPORT_SYMBOL_GPL(topology_set_scale_freq_tick);
>  
> -__weak bool arch_freq_counters_available(const struct cpumask *cpus)
> +void topology_clear_scale_freq_tick(enum scale_freq_tick_source source,
> +				    const struct cpumask *cpus)
>  {
> -	return false;
> +	struct scale_freq_tick_data *sftd;
> +	int cpu;
> +
> +	for_each_cpu(cpu, cpus) {
> +		sftd = per_cpu(sft_data, cpu);
> +
> +		if (sftd && sftd->source == source) {
> +			per_cpu(sft_data, cpu) = NULL;
> +			cpumask_clear_cpu(cpu, &scale_freq_tick_mask);
> +		}
> +	}
>  }
> +EXPORT_SYMBOL_GPL(topology_clear_scale_freq_tick);
> +
> +void topology_scale_freq_tick(void)
> +{
> +	struct scale_freq_tick_data *sftd = *this_cpu_ptr(&sft_data);
> +
> +	if (sftd)
> +		sftd->scale_freq();
> +}

What do you think about having a single topology function that handles
all sources of invariance (cpufreq, arch counters, platform counters)?

Snippet of possible code:

"""
diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h
index be6a53ba3e2d..d9b7221b8ea9 100644
--- a/arch/arm64/include/asm/topology.h
+++ b/arch/arm64/include/asm/topology.h
@@ -19,7 +19,7 @@ int pcibus_to_node(struct pci_bus *bus);
 void update_freq_counters_refs(void);
 
 /* Replace task scheduler's default frequency-invariance scale factor setting */
-#define arch_scale_freq_tick topology_scale_freq_tick
+#define arch_scale_freq_tick() topology_set_freq_scale(NULL, 0, 0)
 
 /* Replace task scheduler's default frequency-invariant accounting */
 #define arch_set_freq_scale topology_set_freq_scale
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index 07774fb47bc4..a669963e7b01 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -70,38 +70,27 @@ void topology_clear_scale_freq_tick(enum scale_freq_tick_source source,
 }
 EXPORT_SYMBOL_GPL(topology_clear_scale_freq_tick);
 
-void topology_scale_freq_tick(void)
-{
-	struct scale_freq_tick_data *sftd = *this_cpu_ptr(&sft_data);
-
-	if (sftd)
-		sftd->scale_freq();
-}
-
 DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
 EXPORT_SYMBOL_GPL(freq_scale);
 
 void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
 			     unsigned long max_freq)
 {
+	struct scale_freq_tick_data *sftd;
 	unsigned long scale;
 	int i;
 
-	if (WARN_ON_ONCE(!cur_freq || !max_freq))
-		return;
-
-	/*
-	 * If the use of counters for FIE is enabled, just return as we don't
-	 * want to update the scale factor with information from CPUFREQ.
-	 * Instead the scale factor will be updated from arch_scale_freq_tick.
-	 */
-	if (supports_scale_freq_tick(cpus))
-		return;
-
-	scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq;
-
-	for_each_cpu(i, cpus)
-		per_cpu(freq_scale, i) = scale;
+	/* Retrieve provided method for scale factor setting for current CPU */
+	if (!cpus) {
+		sftd = *this_cpu_ptr(&sft_data);
+		if (sftd)
+			sftd->scale_freq();
+	} else if (cur_freq && max_freq) {
+		/* Use arguments to compute and set scale factor for CPUs */
+		scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq;
+		for_each_cpu(i, cpus)
+			per_cpu(freq_scale, i) = scale;
+	}
 }
 
 DEFINE_PER_CPU(unsigned long, cpu_scale) = SCHED_CAPACITY_SCALE;
diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
index 09205b584ca5..9b41a90fc955 100644
--- a/include/linux/arch_topology.h
+++ b/include/linux/arch_topology.h
@@ -44,7 +44,6 @@ struct scale_freq_tick_data {
 	void (*scale_freq)(void);
 };
 
-void topology_scale_freq_tick(void);
 void topology_set_scale_freq_tick(struct scale_freq_tick_data *data, const struct cpumask *cpus);
 void topology_clear_scale_freq_tick(enum scale_freq_tick_source source, const struct cpumask *cpus);
 
"""

This basically says that if there is a method registered for a certain
CPU to set the scale factor, that method should be used. Otherwise,
rely on this default math of obtaining the scale factor based on the
current and maximum frequency arguments, if they are provided.

This could get us rid of supports_scale_freq_tick() and possibly
even the scale_freq_tick_mask. topology_set_scale_freq_tick() and
topology_clear_scale_freq_tick() could also be used to enable and
disable system level invariance (through a static key) so we don't have
to do cpumask checks in topology_scale_freq_invariant().

We might then add a static key to prevent those useless conditions being
evaluated on the tick. But we'll see possible optimisations later.

> +
>  DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
> +EXPORT_SYMBOL_GPL(freq_scale);
>  
>  void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
>  			     unsigned long max_freq)
> @@ -47,7 +95,7 @@ void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
>  	 * want to update the scale factor with information from CPUFREQ.
>  	 * Instead the scale factor will be updated from arch_scale_freq_tick.
>  	 */
> -	if (arch_freq_counters_available(cpus))
> +	if (supports_scale_freq_tick(cpus))
>  		return;
>  
>  	scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq;
> diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
> index 0f6cd6b73a61..b2422ebef2dd 100644
> --- a/include/linux/arch_topology.h
> +++ b/include/linux/arch_topology.h
> @@ -34,7 +34,18 @@ void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
>  			     unsigned long max_freq);
>  bool topology_scale_freq_invariant(void);
>  
> -bool arch_freq_counters_available(const struct cpumask *cpus);
> +enum scale_freq_tick_source {
> +	SCALE_FREQ_SOURCE_AMU,
> +};
> +
> +struct scale_freq_tick_data {
> +	enum scale_freq_tick_source source;
> +	void (*scale_freq)(void);
> +};

s/scale_freq/set_freq_scale?

Hope it helps,
Ionela.

> +
> +void topology_scale_freq_tick(void);
> +void topology_set_scale_freq_tick(struct scale_freq_tick_data *data, const struct cpumask *cpus);
> +void topology_clear_scale_freq_tick(enum scale_freq_tick_source source, const struct cpumask *cpus);
>  
>  DECLARE_PER_CPU(unsigned long, thermal_pressure);
>  
> -- 
> 2.25.0.rc1.19.g042ed3e048af
> 

Re: [RFC V2 1/2] topology: Allow multiple entities to provide sched_freq_tick() callback

[Viresh Kumar]

On 13-01-21, 16:18, Ionela Voinescu wrote:
> On Tuesday 15 Dec 2020 at 16:46:35 (+0530), Viresh Kumar wrote:
> > +void topology_scale_freq_tick(void)
> > +{
> > +	struct scale_freq_tick_data *sftd = *this_cpu_ptr(&sft_data);
> > +
> > +	if (sftd)
> > +		sftd->scale_freq();
> > +}
> 
> What do you think about having a single topology function that handles
> all sources of invariance (cpufreq, arch counters, platform counters)?

I think keeping them separate is better, both of these are called from
scheduler's context (hotpath) and adding any more unnecessary
conditionals there should be rather avoided. We could have given a
though to merging them if they were going to share some code, but that
is not the case here clearly. They are quite different.

This is how it looks now:

diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h
index 3b8dca4eb08d..be6a53ba3e2d 100644
--- a/arch/arm64/include/asm/topology.h
+++ b/arch/arm64/include/asm/topology.h
@@ -17,15 +17,9 @@ int pcibus_to_node(struct pci_bus *bus);
 #include <linux/arch_topology.h>
 
 void update_freq_counters_refs(void);
-void topology_scale_freq_tick(void);
 
-#ifdef CONFIG_ARM64_AMU_EXTN
-/*
- * Replace task scheduler's default counter-based
- * frequency-invariance scale factor setting.
- */
+/* Replace task scheduler's default frequency-invariance scale factor setting */
 #define arch_scale_freq_tick topology_scale_freq_tick
-#endif /* CONFIG_ARM64_AMU_EXTN */
 
 /* Replace task scheduler's default frequency-invariant accounting */
 #define arch_set_freq_scale topology_set_freq_scale
diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
index e08a4126453a..1e47dfd465f8 100644
--- a/arch/arm64/kernel/topology.c
+++ b/arch/arm64/kernel/topology.c
@@ -199,8 +199,44 @@ static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate)
 	return 0;
 }
 
-static DEFINE_STATIC_KEY_FALSE(amu_fie_key);
-#define amu_freq_invariant() static_branch_unlikely(&amu_fie_key)
+static void amu_scale_freq_tick(void)
+{
+	u64 prev_core_cnt, prev_const_cnt;
+	u64 core_cnt, const_cnt, scale;
+
+	prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
+	prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
+
+	update_freq_counters_refs();
+
+	const_cnt = this_cpu_read(arch_const_cycles_prev);
+	core_cnt = this_cpu_read(arch_core_cycles_prev);
+
+	if (unlikely(core_cnt <= prev_core_cnt ||
+		     const_cnt <= prev_const_cnt))
+		return;
+
+	/*
+	 *	    /\core    arch_max_freq_scale
+	 * scale =  ------- * --------------------
+	 *	    /\const   SCHED_CAPACITY_SCALE
+	 *
+	 * See validate_cpu_freq_invariance_counters() for details on
+	 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
+	 */
+	scale = core_cnt - prev_core_cnt;
+	scale *= this_cpu_read(arch_max_freq_scale);
+	scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
+			  const_cnt - prev_const_cnt);
+
+	scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
+	this_cpu_write(freq_scale, (unsigned long)scale);
+}
+
+static struct scale_freq_data amu_sfd = {
+	.source = SCALE_FREQ_SOURCE_ARCH,
+	.set_freq_scale = amu_scale_freq_tick,
+};
 
 static void amu_fie_setup(const struct cpumask *cpus)
 {
@@ -227,7 +263,7 @@ static void amu_fie_setup(const struct cpumask *cpus)
 	if (!invariant && !cpumask_equal(amu_fie_cpus, cpu_present_mask))
 		return;
 
-	static_branch_enable(&amu_fie_key);
+	topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus);
 
 	pr_debug("CPUs[%*pbl]: counters will be used for FIE.",
 		 cpumask_pr_args(cpus));
@@ -283,53 +319,6 @@ static int __init init_amu_fie(void)
 }
 core_initcall(init_amu_fie);
 
-bool arch_freq_counters_available(const struct cpumask *cpus)
-{
-	return amu_freq_invariant() &&
-	       cpumask_subset(cpus, amu_fie_cpus);
-}
-
-void topology_scale_freq_tick(void)
-{
-	u64 prev_core_cnt, prev_const_cnt;
-	u64 core_cnt, const_cnt, scale;
-	int cpu = smp_processor_id();
-
-	if (!amu_freq_invariant())
-		return;
-
-	if (!cpumask_test_cpu(cpu, amu_fie_cpus))
-		return;
-
-	prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
-	prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
-
-	update_freq_counters_refs();
-
-	const_cnt = this_cpu_read(arch_const_cycles_prev);
-	core_cnt = this_cpu_read(arch_core_cycles_prev);
-
-	if (unlikely(core_cnt <= prev_core_cnt ||
-		     const_cnt <= prev_const_cnt))
-		return;
-
-	/*
-	 *	    /\core    arch_max_freq_scale
-	 * scale =  ------- * --------------------
-	 *	    /\const   SCHED_CAPACITY_SCALE
-	 *
-	 * See validate_cpu_freq_invariance_counters() for details on
-	 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
-	 */
-	scale = core_cnt - prev_core_cnt;
-	scale *= this_cpu_read(arch_max_freq_scale);
-	scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
-			  const_cnt - prev_const_cnt);
-
-	scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
-	this_cpu_write(freq_scale, (unsigned long)scale);
-}
-
 #ifdef CONFIG_ACPI_CPPC_LIB
 #include <acpi/cppc_acpi.h>
 
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index de8587cc119e..e2115ea348dc 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -21,17 +21,65 @@
 #include <linux/sched.h>
 #include <linux/smp.h>
 
+static DEFINE_PER_CPU(struct scale_freq_data *, sft_data);
+static struct cpumask scale_freq_counters_mask;
+
+static bool supports_scale_freq_counters(const struct cpumask *cpus)
+{
+	return cpumask_subset(cpus, &scale_freq_counters_mask);
+}
+
 bool topology_scale_freq_invariant(void)
 {
 	return cpufreq_supports_freq_invariance() ||
-	       arch_freq_counters_available(cpu_online_mask);
+	       supports_scale_freq_counters(cpu_online_mask);
+}
+
+void topology_set_scale_freq_source(struct scale_freq_data *data,
+				  const struct cpumask *cpus)
+{
+	struct scale_freq_data *sfd;
+	int cpu;
+
+	for_each_cpu(cpu, cpus) {
+		sfd = per_cpu(sft_data, cpu);
+
+		/* Use ARCH provided counters whenever possible */
+		if (!sfd || sfd->source != SCALE_FREQ_SOURCE_ARCH) {
+			per_cpu(sft_data, cpu) = data;
+			cpumask_set_cpu(cpu, &scale_freq_counters_mask);
+		}
+	}
 }
+EXPORT_SYMBOL_GPL(topology_set_scale_freq_source);
 
-__weak bool arch_freq_counters_available(const struct cpumask *cpus)
+void topology_clear_scale_freq_source(enum scale_freq_source source,
+				    const struct cpumask *cpus)
 {
-	return false;
+	struct scale_freq_data *sfd;
+	int cpu;
+
+	for_each_cpu(cpu, cpus) {
+		sfd = per_cpu(sft_data, cpu);
+
+		if (sfd && sfd->source == source) {
+			per_cpu(sft_data, cpu) = NULL;
+			cpumask_clear_cpu(cpu, &scale_freq_counters_mask);
+		}
+	}
 }
+EXPORT_SYMBOL_GPL(topology_clear_scale_freq_source);
+
+void topology_scale_freq_tick(void)
+{
+	struct scale_freq_data *sfd = *this_cpu_ptr(&sft_data);
+
+	if (sfd)
+		sfd->set_freq_scale();
+}
+
 DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
+EXPORT_SYMBOL_GPL(freq_scale);
 
 void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
 			     unsigned long max_freq)
@@ -47,7 +95,7 @@ void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
 	 * want to update the scale factor with information from CPUFREQ.
 	 * Instead the scale factor will be updated from arch_scale_freq_tick.
 	 */
-	if (arch_freq_counters_available(cpus))
+	if (supports_scale_freq_counters(cpus))
 		return;
 
 	scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq;
diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
index 0f6cd6b73a61..1bcbf8eff991 100644
--- a/include/linux/arch_topology.h
+++ b/include/linux/arch_topology.h
@@ -34,7 +34,18 @@ void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
 			     unsigned long max_freq);
 bool topology_scale_freq_invariant(void);
 
-bool arch_freq_counters_available(const struct cpumask *cpus);
+enum scale_freq_source {
+	SCALE_FREQ_SOURCE_ARCH,
+};
+
+struct scale_freq_data {
+	enum scale_freq_source source;
+	void (*set_freq_scale)(void);
+};
+
+void topology_scale_freq_tick(void);
+void topology_set_scale_freq_source(struct scale_freq_data *data, const struct cpumask *cpus);
+void topology_clear_scale_freq_source(enum scale_freq_source source, const struct cpumask *cpus);
 
 DECLARE_PER_CPU(unsigned long, thermal_pressure);
 

-- 
viresh

Re: [RFC V2 1/2] topology: Allow multiple entities to provide sched_freq_tick() callback

[Ionela Voinescu]

Hi,

On Friday 15 Jan 2021 at 13:18:47 (+0530), Viresh Kumar wrote:
> On 13-01-21, 16:18, Ionela Voinescu wrote:
> > On Tuesday 15 Dec 2020 at 16:46:35 (+0530), Viresh Kumar wrote:
> > > +void topology_scale_freq_tick(void)
> > > +{
> > > +	struct scale_freq_tick_data *sftd = *this_cpu_ptr(&sft_data);
> > > +
> > > +	if (sftd)
> > > +		sftd->scale_freq();
> > > +}
> > 
> > What do you think about having a single topology function that handles
> > all sources of invariance (cpufreq, arch counters, platform counters)?
> 
> I think keeping them separate is better, both of these are called from
> scheduler's context (hotpath) and adding any more unnecessary
> conditionals there should be rather avoided. We could have given a

It would be a single added condition on the hotpath for !cpus and I
think CPUs nowadays do a very good job optimising those.
Also, you lose a branch and a cpumask operation on the
arch_set_freq_scale path.

> though to merging them if they were going to share some code, but that
> is not the case here clearly. They are quite different.
> 

I think it would improve clarity a lot by having a single topology
function, with clear comments on which path is taken when.

In regards to them sharing code, there's not much code they could be
sharing, as one just does some simple math on provided values.
I attempted to have a single function for frequency invariance at some
point [1], as the logic for the computation is the same, but I could
never convince myself the clarity gained was worth the changes. But
I really like this version of your code for this purpose as well.

But I'll leave that to your judgement.

[1] https://gitlab.arm.com/linux-arm/linux-power/-/commit/b9277295475307051c34ca31aef28eac50115e1a

> This is how it looks now:
> 
> diff --git a/arch/arm64/include/asm/topology.h b/arch/arm64/include/asm/topology.h
> index 3b8dca4eb08d..be6a53ba3e2d 100644
> --- a/arch/arm64/include/asm/topology.h
> +++ b/arch/arm64/include/asm/topology.h
> @@ -17,15 +17,9 @@ int pcibus_to_node(struct pci_bus *bus);
>  #include <linux/arch_topology.h>
>  
>  void update_freq_counters_refs(void);
> -void topology_scale_freq_tick(void);
>  
> -#ifdef CONFIG_ARM64_AMU_EXTN
> -/*
> - * Replace task scheduler's default counter-based
> - * frequency-invariance scale factor setting.
> - */
> +/* Replace task scheduler's default frequency-invariance scale factor setting */
>  #define arch_scale_freq_tick topology_scale_freq_tick

You could probably move this define under the comment below, as both
arch functions do frequency-invariance accounting, but from different
sources.

> -#endif /* CONFIG_ARM64_AMU_EXTN */
>  
>  /* Replace task scheduler's default frequency-invariant accounting */
>  #define arch_set_freq_scale topology_set_freq_scale
> diff --git a/arch/arm64/kernel/topology.c b/arch/arm64/kernel/topology.c
> index e08a4126453a..1e47dfd465f8 100644
> --- a/arch/arm64/kernel/topology.c
> +++ b/arch/arm64/kernel/topology.c
> @@ -199,8 +199,44 @@ static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate)
>  	return 0;
>  }
>  
> -static DEFINE_STATIC_KEY_FALSE(amu_fie_key);
> -#define amu_freq_invariant() static_branch_unlikely(&amu_fie_key)
> +static void amu_scale_freq_tick(void)
> +{
> +	u64 prev_core_cnt, prev_const_cnt;
> +	u64 core_cnt, const_cnt, scale;
> +
> +	prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
> +	prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
> +
> +	update_freq_counters_refs();
> +
> +	const_cnt = this_cpu_read(arch_const_cycles_prev);
> +	core_cnt = this_cpu_read(arch_core_cycles_prev);
> +
> +	if (unlikely(core_cnt <= prev_core_cnt ||
> +		     const_cnt <= prev_const_cnt))
> +		return;
> +
> +	/*
> +	 *	    /\core    arch_max_freq_scale
> +	 * scale =  ------- * --------------------
> +	 *	    /\const   SCHED_CAPACITY_SCALE
> +	 *
> +	 * See validate_cpu_freq_invariance_counters() for details on
> +	 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
> +	 */
> +	scale = core_cnt - prev_core_cnt;
> +	scale *= this_cpu_read(arch_max_freq_scale);
> +	scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
> +			  const_cnt - prev_const_cnt);
> +
> +	scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
> +	this_cpu_write(freq_scale, (unsigned long)scale);
> +}
> +
> +static struct scale_freq_data amu_sfd = {
> +	.source = SCALE_FREQ_SOURCE_ARCH,
> +	.set_freq_scale = amu_scale_freq_tick,
> +};
>  
>  static void amu_fie_setup(const struct cpumask *cpus)
>  {
> @@ -227,7 +263,7 @@ static void amu_fie_setup(const struct cpumask *cpus)
>  	if (!invariant && !cpumask_equal(amu_fie_cpus, cpu_present_mask))
>  		return;
>  
> -	static_branch_enable(&amu_fie_key);
> +	topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus);
>  
>  	pr_debug("CPUs[%*pbl]: counters will be used for FIE.",
>  		 cpumask_pr_args(cpus));
> @@ -283,53 +319,6 @@ static int __init init_amu_fie(void)
>  }
>  core_initcall(init_amu_fie);
>  
> -bool arch_freq_counters_available(const struct cpumask *cpus)
> -{
> -	return amu_freq_invariant() &&
> -	       cpumask_subset(cpus, amu_fie_cpus);
> -}
> -
> -void topology_scale_freq_tick(void)
> -{
> -	u64 prev_core_cnt, prev_const_cnt;
> -	u64 core_cnt, const_cnt, scale;
> -	int cpu = smp_processor_id();
> -
> -	if (!amu_freq_invariant())
> -		return;
> -
> -	if (!cpumask_test_cpu(cpu, amu_fie_cpus))
> -		return;
> -
> -	prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
> -	prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
> -
> -	update_freq_counters_refs();
> -
> -	const_cnt = this_cpu_read(arch_const_cycles_prev);
> -	core_cnt = this_cpu_read(arch_core_cycles_prev);
> -
> -	if (unlikely(core_cnt <= prev_core_cnt ||
> -		     const_cnt <= prev_const_cnt))
> -		return;
> -
> -	/*
> -	 *	    /\core    arch_max_freq_scale
> -	 * scale =  ------- * --------------------
> -	 *	    /\const   SCHED_CAPACITY_SCALE
> -	 *
> -	 * See validate_cpu_freq_invariance_counters() for details on
> -	 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
> -	 */
> -	scale = core_cnt - prev_core_cnt;
> -	scale *= this_cpu_read(arch_max_freq_scale);
> -	scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
> -			  const_cnt - prev_const_cnt);
> -
> -	scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
> -	this_cpu_write(freq_scale, (unsigned long)scale);
> -}
> -
>  #ifdef CONFIG_ACPI_CPPC_LIB
>  #include <acpi/cppc_acpi.h>
>  
> diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
> index de8587cc119e..e2115ea348dc 100644
> --- a/drivers/base/arch_topology.c
> +++ b/drivers/base/arch_topology.c
> @@ -21,17 +21,65 @@
>  #include <linux/sched.h>
>  #include <linux/smp.h>
>  
> +static DEFINE_PER_CPU(struct scale_freq_data *, sft_data);
> +static struct cpumask scale_freq_counters_mask;
> +
> +static bool supports_scale_freq_counters(const struct cpumask *cpus)
> +{
> +	return cpumask_subset(cpus, &scale_freq_counters_mask);
> +}
> +
>  bool topology_scale_freq_invariant(void)
>  {
>  	return cpufreq_supports_freq_invariance() ||
> -	       arch_freq_counters_available(cpu_online_mask);
> +	       supports_scale_freq_counters(cpu_online_mask);
> +}
> +
> +void topology_set_scale_freq_source(struct scale_freq_data *data,
> +				  const struct cpumask *cpus)
> +{
> +	struct scale_freq_data *sfd;
> +	int cpu;
> +
> +	for_each_cpu(cpu, cpus) {
> +		sfd = per_cpu(sft_data, cpu);
> +
> +		/* Use ARCH provided counters whenever possible */
> +		if (!sfd || sfd->source != SCALE_FREQ_SOURCE_ARCH) {
> +			per_cpu(sft_data, cpu) = data;
> +			cpumask_set_cpu(cpu, &scale_freq_counters_mask);
> +		}
> +	}
>  }
> +EXPORT_SYMBOL_GPL(topology_set_scale_freq_source);
>  
> -__weak bool arch_freq_counters_available(const struct cpumask *cpus)
> +void topology_clear_scale_freq_source(enum scale_freq_source source,
> +				    const struct cpumask *cpus)
>  {
> -	return false;
> +	struct scale_freq_data *sfd;
> +	int cpu;
> +
> +	for_each_cpu(cpu, cpus) {
> +		sfd = per_cpu(sft_data, cpu);
> +
> +		if (sfd && sfd->source == source) {
> +			per_cpu(sft_data, cpu) = NULL;
> +			cpumask_clear_cpu(cpu, &scale_freq_counters_mask);
> +		}
> +	}
>  }
> +EXPORT_SYMBOL_GPL(topology_clear_scale_freq_source);
> +
> +void topology_scale_freq_tick(void)
> +{
> +	struct scale_freq_data *sfd = *this_cpu_ptr(&sft_data);
> +
> +	if (sfd)
> +		sfd->set_freq_scale();
> +}
> +
>  DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
> +EXPORT_SYMBOL_GPL(freq_scale);
>  
>  void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
>  			     unsigned long max_freq)
> @@ -47,7 +95,7 @@ void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
>  	 * want to update the scale factor with information from CPUFREQ.
>  	 * Instead the scale factor will be updated from arch_scale_freq_tick.
>  	 */
> -	if (arch_freq_counters_available(cpus))
> +	if (supports_scale_freq_counters(cpus))
>  		return;
>  
>  	scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq;
> diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
> index 0f6cd6b73a61..1bcbf8eff991 100644
> --- a/include/linux/arch_topology.h
> +++ b/include/linux/arch_topology.h
> @@ -34,7 +34,18 @@ void topology_set_freq_scale(const struct cpumask *cpus, unsigned long cur_freq,
>  			     unsigned long max_freq);
>  bool topology_scale_freq_invariant(void);
>  
> -bool arch_freq_counters_available(const struct cpumask *cpus);
> +enum scale_freq_source {

Maybe you can add SCALE_FREQ_SOURCE_CPUFREQ as the first value (=0).
That will give a hint of CPUFREQ being a source of information for FIE
and the default when counters are not supported, even if nothing gets
done with that value (for now at least).

> +	SCALE_FREQ_SOURCE_ARCH,
> +};
> +
> +struct scale_freq_data {
> +	enum scale_freq_source source;
> +	void (*set_freq_scale)(void);
> +};
> +
> +void topology_scale_freq_tick(void);
> +void topology_set_scale_freq_source(struct scale_freq_data *data, const struct cpumask *cpus);
> +void topology_clear_scale_freq_source(enum scale_freq_source source, const struct cpumask *cpus);
>  
>  DECLARE_PER_CPU(unsigned long, thermal_pressure);
>  

+1 on the rewrite!

Many thanks,
Ionela.

> 
> -- 
> viresh

Re: [RFC V2 2/2] cpufreq: cppc: Add support for frequency invariance

[Ionela Voinescu]

Hi,

Do you know of a current platform that would benefit from this, that we
could run some tests on?

I've Cc-ed Jeremy as well, as he might be interested in this.

Also, please find some initial comments below:

On Tuesday 15 Dec 2020 at 16:46:36 (+0530), Viresh Kumar wrote:
> The Frequency Invariance Engine (FIE) is providing a frequency scaling
> correction factor that helps achieve more accurate load-tracking.
> 
> Normally, this scaling factor can be obtained directly with the help of
> the cpufreq drivers as they know the exact frequency the hardware is
> running at. But that isn't the case for CPPC cpufreq driver.
> 
> Another way of obtaining that is using the AMU counter support, which is
> already present in kernel, but that hardware is optional for platforms.
> 
> This patch thus obtains this scaling factor using the existing logic
> present in the cppc driver. Note that the AMUs have higher priority than
> CPPC counters if available, though the CPPC driver doesn't need to have
> any special handling for that.
> 

Probably best to replace "AMU counters" with "architectural counters" as
the use of cppc_cpufreq is not limited to arm64.

> This also exports sched_setattr_nocheck() as the CPPC driver can be
> built as a module.
> 
> Signed-off-by: Viresh Kumar <viresh.kumar@linaro.org>
> ---
>  drivers/cpufreq/cppc_cpufreq.c | 140 ++++++++++++++++++++++++++++++++-
>  include/linux/arch_topology.h  |   1 +
>  kernel/sched/core.c            |   1 +
>  3 files changed, 140 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
> index 7cc9bd8568de..a739e20aefd6 100644
> --- a/drivers/cpufreq/cppc_cpufreq.c
> +++ b/drivers/cpufreq/cppc_cpufreq.c
> @@ -10,14 +10,18 @@
>  
>  #define pr_fmt(fmt)	"CPPC Cpufreq:"	fmt
>  
> +#include <linux/arch_topology.h>
>  #include <linux/kernel.h>
>  #include <linux/module.h>
>  #include <linux/delay.h>
>  #include <linux/cpu.h>
>  #include <linux/cpufreq.h>
>  #include <linux/dmi.h>
> +#include <linux/irq_work.h>
> +#include <linux/kthread.h>
>  #include <linux/time.h>
>  #include <linux/vmalloc.h>
> +#include <uapi/linux/sched/types.h>
>  
>  #include <asm/unaligned.h>
>  
> @@ -39,6 +43,15 @@
>  static struct cppc_cpudata **all_cpu_data;
>  static bool boost_supported;
>  
> +struct cppc_freq_invariance {
> +	struct kthread_worker *worker;
> +	struct irq_work irq_work;
> +	struct kthread_work work;
> +	struct cppc_perf_fb_ctrs prev_perf_fb_ctrs;
> +	unsigned int max_freq;
> +};
> +static DEFINE_PER_CPU(struct cppc_freq_invariance, cppc_f_i);
> +
>  struct cppc_workaround_oem_info {
>  	char oem_id[ACPI_OEM_ID_SIZE + 1];
>  	char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
> @@ -243,7 +256,7 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
>  	struct cppc_cpudata *cpu_data = all_cpu_data[policy->cpu];
>  	struct cppc_perf_caps *caps = &cpu_data->perf_caps;
>  	unsigned int cpu = policy->cpu;
> -	int ret = 0;
> +	int ret = 0, i;
>  
>  	cpu_data->cpu = cpu;
>  	ret = cppc_get_perf_caps(cpu, caps);
> @@ -300,6 +313,9 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
>  
>  	cpu_data->cur_policy = policy;
>  
> +	for_each_cpu(i, policy->cpus)
> +		per_cpu(cppc_f_i, i).max_freq = policy->cpuinfo.max_freq;
> +

Is policy->cpuinfo populated at this point?

>  	/*
>  	 * If 'highest_perf' is greater than 'nominal_perf', we assume CPU Boost
>  	 * is supported.
> @@ -374,7 +390,7 @@ static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
>  {
>  	struct cppc_cpudata *cpu_data = all_cpu_data[policy->cpu];
>  	struct cppc_perf_caps *caps = &cpu_data->perf_caps;
> -	int ret;
> +	int ret, i;
>  
>  	if (!boost_supported) {
>  		pr_err("BOOST not supported by CPU or firmware\n");
> @@ -389,6 +405,9 @@ static int cppc_cpufreq_set_boost(struct cpufreq_policy *policy, int state)
>  						       caps->nominal_perf);
>  	policy->cpuinfo.max_freq = policy->max;
>  
> +	for_each_cpu(i, policy->related_cpus)
> +		per_cpu(cppc_f_i, i).max_freq = policy->cpuinfo.max_freq;
> +
>  	ret = freq_qos_update_request(policy->max_freq_req, policy->max);
>  	if (ret < 0)
>  		return ret;
> @@ -449,6 +468,120 @@ static void cppc_check_hisi_workaround(void)
>  	acpi_put_table(tbl);
>  }
>  
> +static void cppc_scale_freq_tick_workfn(struct kthread_work *work)
> +{
> +	struct cppc_freq_invariance *cppc_fi;
> +	struct cppc_perf_fb_ctrs fb_ctrs = {0};
> +	int cpu = raw_smp_processor_id();
> +	struct cppc_cpudata *cpudata = all_cpu_data[cpu];
> +	u64 rate;
> +
> +	cppc_fi = container_of(work, struct cppc_freq_invariance, work);
> +
> +	if (cppc_get_perf_ctrs(cpu, &fb_ctrs)) {
> +		pr_info("%s: cppc_get_perf_ctrs() failed\n", __func__);
> +		return;
> +	}
> +
> +	rate = cppc_get_rate_from_fbctrs(cpudata, cppc_fi->prev_perf_fb_ctrs, fb_ctrs);
> +	cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
> +
> +	rate <<= SCHED_CAPACITY_SHIFT;
> +	per_cpu(freq_scale, cpu) = div64_u64(rate, cppc_fi->max_freq);

It will save you some computation by skipping the intermediary frequency
scale transition. For this computation you're obtaining current
performance from counters, on the CPPC abstract performance scale,
then you're converting it to a current frequency, which then gets
translated again to a scale factor on the [0,1024] scale.

You probably want to keep the sanitation done in
cppc_get_rate_from_fbctrs() on the counter values, but you could skip
the call to cppc_cpufreq_perf_to_khz(), and use obtained performance
together with caps->highest_perf, or caps->nominal_perf instead of
cppc_fi->max_freq, in this function.

Also, to optimise it further, you can compute a reference scale (from
reference performance and highest/nominal performance as done in
freq_inv_set_max_ratio() - arch/arm64/kernel/topology.c, and use that
instead in further freq scale computations.

I've resurrected my Juno setup and I'll do further review and testing
tomorrow.

Hope it helps,
Ionela.

> +}
> +
> +static void cppc_irq_work(struct irq_work *irq_work)
> +{
> +	struct cppc_freq_invariance *cppc_fi;
> +
> +	cppc_fi = container_of(irq_work, struct cppc_freq_invariance, irq_work);
> +	kthread_queue_work(cppc_fi->worker, &cppc_fi->work);
> +}
> +
> +static void cppc_scale_freq_tick(void)
> +{
> +	struct cppc_freq_invariance *cppc_fi = &per_cpu(cppc_f_i, raw_smp_processor_id());
> +
> +	/*
> +	 * cppc_get_perf_ctrs() can potentially sleep, call that from the right
> +	 * context.
> +	 */
> +	irq_work_queue(&cppc_fi->irq_work);
> +}
> +
> +static struct scale_freq_tick_data cppc_sftd = {
> +	.source = SCALE_FREQ_SOURCE_CPPC,
> +	.scale_freq = cppc_scale_freq_tick,
> +};
> +
> +static void cppc_freq_invariance_exit(void)
> +{
> +	struct cppc_freq_invariance *cppc_fi;
> +	int i;
> +
> +	if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
> +		return;
> +
> +	topology_clear_scale_freq_tick(SCALE_FREQ_SOURCE_CPPC, cpu_present_mask);
> +
> +	for_each_possible_cpu(i) {
> +		cppc_fi = &per_cpu(cppc_f_i, i);
> +		if (cppc_fi->worker) {
> +			irq_work_sync(&cppc_fi->irq_work);
> +			kthread_destroy_worker(cppc_fi->worker);
> +			cppc_fi->worker = NULL;
> +		}
> +	}
> +}
> +
> +static void __init cppc_freq_invariance_init(void)
> +{
> +	struct cppc_perf_fb_ctrs fb_ctrs = {0};
> +	struct cppc_freq_invariance *cppc_fi;
> +	struct sched_attr attr = {
> +		.size		= sizeof(struct sched_attr),
> +		.sched_policy	= SCHED_DEADLINE,
> +		.sched_nice	= 0,
> +		.sched_priority	= 0,
> +		/*
> +		 * Fake (unused) bandwidth; workaround to "fix"
> +		 * priority inheritance.
> +		 */
> +		.sched_runtime	= 1000000,
> +		.sched_deadline = 10000000,
> +		.sched_period	= 10000000,
> +	};
> +	struct kthread_worker *worker;
> +	int i, ret;
> +
> +	if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
> +		return;
> +
> +	for_each_possible_cpu(i) {
> +		cppc_fi = &per_cpu(cppc_f_i, i);
> +
> +		kthread_init_work(&cppc_fi->work, cppc_scale_freq_tick_workfn);
> +		init_irq_work(&cppc_fi->irq_work, cppc_irq_work);
> +		worker = kthread_create_worker_on_cpu(i, 0, "cppc:%d", i);
> +		if (IS_ERR(worker))
> +			return cppc_freq_invariance_exit();
> +
> +		cppc_fi->worker = worker;
> +		ret = sched_setattr_nocheck(worker->task, &attr);
> +		if (ret) {
> +			pr_warn("%s: failed to set SCHED_DEADLINE\n", __func__);
> +			return cppc_freq_invariance_exit();
> +		}
> +
> +		ret = cppc_get_perf_ctrs(i, &fb_ctrs);
> +		if (!ret)
> +			per_cpu(cppc_fi->prev_perf_fb_ctrs, i) = fb_ctrs;
> +	}
> +
> +	/* Register for freq-invariance */
> +	topology_set_scale_freq_tick(&cppc_sftd, cpu_present_mask);
> +}
> +
>  static int __init cppc_cpufreq_init(void)
>  {
>  	struct cppc_cpudata *cpu_data;
> @@ -484,6 +617,8 @@ static int __init cppc_cpufreq_init(void)
>  	if (ret)
>  		goto out;
>  
> +	cppc_freq_invariance_init();
> +
>  	return ret;
>  
>  out:
> @@ -504,6 +639,7 @@ static void __exit cppc_cpufreq_exit(void)
>  	struct cppc_cpudata *cpu_data;
>  	int i;
>  
> +	cppc_freq_invariance_exit();
>  	cpufreq_unregister_driver(&cppc_cpufreq_driver);
>  
>  	for_each_possible_cpu(i) {
> diff --git a/include/linux/arch_topology.h b/include/linux/arch_topology.h
> index b2422ebef2dd..09205b584ca5 100644
> --- a/include/linux/arch_topology.h
> +++ b/include/linux/arch_topology.h
> @@ -36,6 +36,7 @@ bool topology_scale_freq_invariant(void);
>  
>  enum scale_freq_tick_source {
>  	SCALE_FREQ_SOURCE_AMU,
> +	SCALE_FREQ_SOURCE_CPPC,
>  };
>  
>  struct scale_freq_tick_data {
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 51514eef0a9d..76b2fa1a7aaa 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -6075,6 +6075,7 @@ int sched_setattr_nocheck(struct task_struct *p, const struct sched_attr *attr)
>  {
>  	return __sched_setscheduler(p, attr, false, true);
>  }
> +EXPORT_SYMBOL_GPL(sched_setattr_nocheck);
>  
>  /**
>   * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
> -- 
> 2.25.0.rc1.19.g042ed3e048af
> 

Re: [RFC V2 1/2] topology: Allow multiple entities to provide sched_freq_tick() callback

[Viresh Kumar]

On 19-01-21, 19:05, Ionela Voinescu wrote:
> I think it would improve clarity a lot by having a single topology
> function, with clear comments on which path is taken when.
> 
> In regards to them sharing code, there's not much code they could be
> sharing, as one just does some simple math on provided values.
> I attempted to have a single function for frequency invariance at some
> point [1], as the logic for the computation is the same, but I could
> never convince myself the clarity gained was worth the changes. But
> I really like this version of your code for this purpose as well.
> 
> But I'll leave that to your judgement.

Hmm, I don't really agree to merging those two paths into a single
routine unless we share something there. So I would like to skip that
for now :)

All other comments accepted.

Thanks.

-- 
viresh

Re: [RFC V2 2/2] cpufreq: cppc: Add support for frequency invariance

[Viresh Kumar]

On 19-01-21, 19:17, Ionela Voinescu wrote:
> Hi,
> 
> Do you know of a current platform that would benefit from this, that we
> could run some tests on?

Thunderx2 is one.

> On Tuesday 15 Dec 2020 at 16:46:36 (+0530), Viresh Kumar wrote:
> > @@ -243,7 +256,7 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
> >  	struct cppc_cpudata *cpu_data = all_cpu_data[policy->cpu];
> >  	struct cppc_perf_caps *caps = &cpu_data->perf_caps;
> >  	unsigned int cpu = policy->cpu;
> > -	int ret = 0;
> > +	int ret = 0, i;
> >  
> >  	cpu_data->cpu = cpu;
> >  	ret = cppc_get_perf_caps(cpu, caps);
> > @@ -300,6 +313,9 @@ static int cppc_cpufreq_cpu_init(struct cpufreq_policy *policy)
> >  
> >  	cpu_data->cur_policy = policy;
> >  
> > +	for_each_cpu(i, policy->cpus)
> > +		per_cpu(cppc_f_i, i).max_freq = policy->cpuinfo.max_freq;
> > +
> 
> Is policy->cpuinfo populated at this point?

The base has changed since the time I posted the patch, but yes this
routine itself updates min/max freq in cpuinfo at an earlier point.

> > +static void cppc_scale_freq_tick_workfn(struct kthread_work *work)
> > +{
> > +	struct cppc_freq_invariance *cppc_fi;
> > +	struct cppc_perf_fb_ctrs fb_ctrs = {0};
> > +	int cpu = raw_smp_processor_id();
> > +	struct cppc_cpudata *cpudata = all_cpu_data[cpu];
> > +	u64 rate;
> > +
> > +	cppc_fi = container_of(work, struct cppc_freq_invariance, work);
> > +
> > +	if (cppc_get_perf_ctrs(cpu, &fb_ctrs)) {
> > +		pr_info("%s: cppc_get_perf_ctrs() failed\n", __func__);
> > +		return;
> > +	}
> > +
> > +	rate = cppc_get_rate_from_fbctrs(cpudata, cppc_fi->prev_perf_fb_ctrs, fb_ctrs);
> > +	cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
> > +
> > +	rate <<= SCHED_CAPACITY_SHIFT;
> > +	per_cpu(freq_scale, cpu) = div64_u64(rate, cppc_fi->max_freq);
> 
> It will save you some computation by skipping the intermediary frequency
> scale transition. For this computation you're obtaining current
> performance from counters, on the CPPC abstract performance scale,
> then you're converting it to a current frequency, which then gets
> translated again to a scale factor on the [0,1024] scale.
> 
> You probably want to keep the sanitation done in
> cppc_get_rate_from_fbctrs() on the counter values, but you could skip
> the call to cppc_cpufreq_perf_to_khz(), and use obtained performance
> together with caps->highest_perf, or caps->nominal_perf instead of
> cppc_fi->max_freq, in this function.

Something like this ?

diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 0d7a950f3c9f..d4d7fb0dc918 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -389,9 +389,9 @@ static inline u64 get_delta(u64 t1, u64 t0)
        return (u32)t1 - (u32)t0;
 }
 
-static int cppc_get_rate_from_fbctrs(struct cppc_cpudata *cpu_data,
-                                    struct cppc_perf_fb_ctrs fb_ctrs_t0,
-                                    struct cppc_perf_fb_ctrs fb_ctrs_t1)
+static int cppc_perf_from_fbctrs(struct cppc_cpudata *cpu_data,
+                                struct cppc_perf_fb_ctrs fb_ctrs_t0,
+                                struct cppc_perf_fb_ctrs fb_ctrs_t1)
 {
        u64 delta_reference, delta_delivered;
        u64 reference_perf, delivered_perf;
@@ -404,11 +404,20 @@ static int cppc_get_rate_from_fbctrs(struct cppc_cpudata *cpu_data,
                                    fb_ctrs_t0.delivered);
 
        /* Check to avoid divide-by zero */
-       if (delta_reference || delta_delivered)
-               delivered_perf = (reference_perf * delta_delivered) /
-                                       delta_reference;
-       else
-               delivered_perf = cpu_data->perf_ctrls.desired_perf;
+       if (!delta_reference && !delta_delivered)
+               return cpu_data->perf_ctrls.desired_perf;
+
+       return (reference_perf * delta_delivered) / delta_reference;
+}
+
+static int cppc_get_rate_from_fbctrs(struct cppc_cpudata *cpu_data,
+                                    struct cppc_perf_fb_ctrs fb_ctrs_t0,
+                                    struct cppc_perf_fb_ctrs fb_ctrs_t1)
+{
+       u64 delivered_perf;
+
+       delivered_perf = cppc_perf_from_fbctrs(cpu_data, fb_ctrs_t0,
+                                              fb_ctrs_t1);
 
        return cppc_cpufreq_perf_to_khz(cpu_data, delivered_perf);
 }
@@ -539,21 +548,23 @@ static void cppc_scale_freq_workfn(struct kthread_work *work)
        struct cppc_freq_invariance *cppc_fi;
        struct cppc_perf_fb_ctrs fb_ctrs = {0};
        int cpu = raw_smp_processor_id();
-       u64 rate;
+       struct cppc_cpudata *cpu_data;
+       u64 perf;
 
        cppc_fi = container_of(work, struct cppc_freq_invariance, work);
+       cpu_data = cppc_fi->cpu_data;
 
        if (cppc_get_perf_ctrs(cpu, &fb_ctrs)) {
                pr_info("%s: cppc_get_perf_ctrs() failed\n", __func__);
                return;
        }
 
-       rate = cppc_get_rate_from_fbctrs(cppc_fi->cpu_data,
-                                        cppc_fi->prev_perf_fb_ctrs, fb_ctrs);
        cppc_fi->prev_perf_fb_ctrs = fb_ctrs;
+       perf = cppc_perf_from_fbctrs(cpu_data, cppc_fi->prev_perf_fb_ctrs,
+                                    fb_ctrs);
 
-       rate <<= SCHED_CAPACITY_SHIFT;
-       per_cpu(freq_scale, cpu) = div64_u64(rate, cppc_fi->max_freq);
+       perf <<= SCHED_CAPACITY_SHIFT;
+       per_cpu(freq_scale, cpu) = div64_u64(perf , cpu_data->perf_caps->highest_perf);
 }
 
 static void cppc_irq_work(struct irq_work *irq_work)

> Also, to optimise it further, you can compute a reference scale (from
> reference performance and highest/nominal performance as done in
> freq_inv_set_max_ratio() - arch/arm64/kernel/topology.c, and use that
> instead in further freq scale computations.

-- 
viresh