All of lore.kernel.org
 help / color / mirror / Atom feed
* [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
@ 2022-11-08 19:06 Ashutosh Dixit
  2022-11-08 20:36 ` [igt-dev] ✓ Fi.CI.BAT: success for tests/perf_pmu: Compare against requested freq in frequency subtest (rev2) Patchwork
  2022-11-09  3:51 ` [igt-dev] ✓ Fi.CI.IGT: " Patchwork
  0 siblings, 2 replies; 34+ messages in thread
From: Ashutosh Dixit @ 2022-11-08 19:06 UTC (permalink / raw)
  To: igt-dev

In igt@perf_pmu@frequency subtest, compare the requested freq reported by
PMU not against the set freq but against the requested freq reported in
sysfs. If requested freq differs from the set freq (even when min == max
freq) for whatever reason, the sysfs requested freq should be closer to the
PMU measured requested freq which should minimize sporadic failures.

v2: Introduce a 100 ms delay after setting freq

Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
---
 tests/i915/perf_pmu.c                 | 23 +++++++++++++++--------
 tests/intel-ci/fast-feedback.testlist |  1 +
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
index f363db2ba13..02f6ae989b1 100644
--- a/tests/i915/perf_pmu.c
+++ b/tests/i915/perf_pmu.c
@@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
 	igt_assert_lte(target, busy);
 }
 
+/* Wait for GuC SLPC freq changes to take effect */
+#define wait_freq_set()		usleep(100000)
+
 static void
 test_frequency(int gem_fd)
 {
-	uint32_t min_freq, max_freq, boost_freq;
+	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
 	uint64_t val[2], start[2], slept;
 	double min[2], max[2];
 	igt_spin_t *spin;
@@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
 	 * Set GPU to min frequency and read PMU counters.
 	 */
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
-	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
-	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
+	wait_freq_set();
+	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
+	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
 
 	gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes effect */
@@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
 
 	min[0] = 1e9*(val[0] - start[0]) / slept;
 	min[1] = 1e9*(val[1] - start[1]) / slept;
+	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
 
 	igt_spin_free(gem_fd, spin);
 	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
@@ -1597,11 +1602,11 @@ test_frequency(int gem_fd)
 	 * Set GPU to max frequency and read PMU counters.
 	 */
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", max_freq));
-	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", boost_freq));
-	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
-
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", max_freq));
+	wait_freq_set();
+	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
+	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
 	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == max_freq);
 
 	gem_quiescent_gpu(gem_fd);
@@ -1613,6 +1618,7 @@ test_frequency(int gem_fd)
 
 	max[0] = 1e9*(val[0] - start[0]) / slept;
 	max[1] = 1e9*(val[1] - start[1]) / slept;
+	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
 
 	igt_spin_free(gem_fd, spin);
 	gem_quiescent_gpu(gem_fd);
@@ -1621,6 +1627,7 @@ test_frequency(int gem_fd)
 	 * Restore min/max.
 	 */
 	igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq);
+	wait_freq_set();
 	if (igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") != min_freq)
 		igt_warn("Unable to restore min frequency to saved value [%u MHz], now %u MHz\n",
 			 min_freq, igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz"));
@@ -1633,12 +1640,12 @@ test_frequency(int gem_fd)
 	igt_info("Max frequency: requested %.1f, actual %.1f\n",
 		 max[0], max[1]);
 
-	assert_within_epsilon(min[0], min_freq, tolerance);
+	assert_within_epsilon(min[0], min_req, tolerance);
 	/*
 	 * On thermally throttled devices we cannot be sure maximum frequency
 	 * can be reached so use larger tolerance downards.
 	 */
-	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
+	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);
 }
 
 static void
diff --git a/tests/intel-ci/fast-feedback.testlist b/tests/intel-ci/fast-feedback.testlist
index f57f8ff3be1..57cc4cf0a88 100644
--- a/tests/intel-ci/fast-feedback.testlist
+++ b/tests/intel-ci/fast-feedback.testlist
@@ -129,6 +129,7 @@ igt@i915_pm_rpm@basic-pci-d3-state
 igt@i915_pm_rpm@basic-rte
 igt@i915_pm_rps@basic-api
 igt@prime_self_import@basic-llseek-bad
+igt@perf_pmu@frequency
 igt@prime_self_import@basic-llseek-size
 igt@prime_self_import@basic-with_fd_dup
 igt@prime_self_import@basic-with_one_bo
-- 
2.38.0

^ permalink raw reply related	[flat|nested] 34+ messages in thread

* [igt-dev] ✓ Fi.CI.BAT: success for tests/perf_pmu: Compare against requested freq in frequency subtest (rev2)
  2022-11-08 19:06 [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest Ashutosh Dixit
@ 2022-11-08 20:36 ` Patchwork
  2022-11-09  3:51 ` [igt-dev] ✓ Fi.CI.IGT: " Patchwork
  1 sibling, 0 replies; 34+ messages in thread
From: Patchwork @ 2022-11-08 20:36 UTC (permalink / raw)
  To: Ashutosh Dixit; +Cc: igt-dev

[-- Attachment #1: Type: text/plain, Size: 4289 bytes --]

== Series Details ==

Series: tests/perf_pmu: Compare against requested freq in frequency subtest (rev2)
URL   : https://patchwork.freedesktop.org/series/110574/
State : success

== Summary ==

CI Bug Log - changes from CI_DRM_12356 -> IGTPW_8069
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/index.html

Participating hosts (42 -> 37)
------------------------------

  Missing    (5): fi-ilk-m540 fi-rkl-11600 bat-adlp-4 fi-ctg-p8600 fi-bdw-samus 

Known issues
------------

  Here are the changes found in IGTPW_8069 that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@i915_selftest@live@execlists:
    - fi-bsw-nick:        NOTRUN -> [INCOMPLETE][1] ([i915#7120])
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-bsw-nick/igt@i915_selftest@live@execlists.html

  * igt@perf_pmu@frequency:
    - fi-ilk-650:         NOTRUN -> [SKIP][2] ([fdo#109271])
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-ilk-650/igt@perf_pmu@frequency.html
    - fi-blb-e6850:       NOTRUN -> [SKIP][3] ([fdo#109271])
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-blb-e6850/igt@perf_pmu@frequency.html
    - fi-bsw-nick:        NOTRUN -> [SKIP][4] ([fdo#109271])
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-bsw-nick/igt@perf_pmu@frequency.html
    - fi-bsw-kefka:       NOTRUN -> [SKIP][5] ([fdo#109271])
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-bsw-kefka/igt@perf_pmu@frequency.html
    - fi-elk-e7500:       NOTRUN -> [SKIP][6] ([fdo#109271])
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-elk-e7500/igt@perf_pmu@frequency.html
    - fi-pnv-d510:        NOTRUN -> [SKIP][7] ([fdo#109271])
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-pnv-d510/igt@perf_pmu@frequency.html

  
#### Possible fixes ####

  * igt@i915_pm_rpm@module-reload:
    - {bat-rpls-2}:       [WARN][8] ([i915#7346]) -> [PASS][9]
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/bat-rpls-2/igt@i915_pm_rpm@module-reload.html
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/bat-rpls-2/igt@i915_pm_rpm@module-reload.html

  * igt@i915_selftest@live@gt_heartbeat:
    - fi-apl-guc:         [DMESG-FAIL][10] ([i915#5334]) -> [PASS][11]
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/fi-apl-guc/igt@i915_selftest@live@gt_heartbeat.html
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-apl-guc/igt@i915_selftest@live@gt_heartbeat.html

  * igt@i915_selftest@live@hugepages:
    - fi-bsw-nick:        [DMESG-FAIL][12] ([i915#6217]) -> [PASS][13]
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/fi-bsw-nick/igt@i915_selftest@live@hugepages.html
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/fi-bsw-nick/igt@i915_selftest@live@hugepages.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
  [i915#2582]: https://gitlab.freedesktop.org/drm/intel/issues/2582
  [i915#5278]: https://gitlab.freedesktop.org/drm/intel/issues/5278
  [i915#5334]: https://gitlab.freedesktop.org/drm/intel/issues/5334
  [i915#6217]: https://gitlab.freedesktop.org/drm/intel/issues/6217
  [i915#6257]: https://gitlab.freedesktop.org/drm/intel/issues/6257
  [i915#6367]: https://gitlab.freedesktop.org/drm/intel/issues/6367
  [i915#6997]: https://gitlab.freedesktop.org/drm/intel/issues/6997
  [i915#7120]: https://gitlab.freedesktop.org/drm/intel/issues/7120
  [i915#7346]: https://gitlab.freedesktop.org/drm/intel/issues/7346


Build changes
-------------

  * CI: CI-20190529 -> None
  * IGT: IGT_7048 -> IGTPW_8069

  CI-20190529: 20190529
  CI_DRM_12356: 1278975de8debde9eb1f5d86fd2fbe533361e456 @ git://anongit.freedesktop.org/gfx-ci/linux
  IGTPW_8069: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/index.html
  IGT_7048: 5edd5c539f1fdf1c02157bf43fa1fd22d4ad2c75 @ https://gitlab.freedesktop.org/drm/igt-gpu-tools.git

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/index.html

[-- Attachment #2: Type: text/html, Size: 5074 bytes --]

^ permalink raw reply	[flat|nested] 34+ messages in thread

* [igt-dev] ✓ Fi.CI.IGT: success for tests/perf_pmu: Compare against requested freq in frequency subtest (rev2)
  2022-11-08 19:06 [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest Ashutosh Dixit
  2022-11-08 20:36 ` [igt-dev] ✓ Fi.CI.BAT: success for tests/perf_pmu: Compare against requested freq in frequency subtest (rev2) Patchwork
@ 2022-11-09  3:51 ` Patchwork
  1 sibling, 0 replies; 34+ messages in thread
From: Patchwork @ 2022-11-09  3:51 UTC (permalink / raw)
  To: Ashutosh Dixit; +Cc: igt-dev

[-- Attachment #1: Type: text/plain, Size: 46849 bytes --]

== Series Details ==

Series: tests/perf_pmu: Compare against requested freq in frequency subtest (rev2)
URL   : https://patchwork.freedesktop.org/series/110574/
State : success

== Summary ==

CI Bug Log - changes from CI_DRM_12356_full -> IGTPW_8069_full
====================================================

Summary
-------

  **SUCCESS**

  No regressions found.

  External URL: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/index.html

Participating hosts (11 -> 8)
------------------------------

  Missing    (3): pig-skl-6260u pig-kbl-iris pig-glk-j5005 

Possible new issues
-------------------

  Here are the unknown changes that may have been introduced in IGTPW_8069_full:

### IGT changes ###

#### Suppressed ####

  The following results come from untrusted machines, tests, or statuses.
  They do not affect the overall result.

  * igt@perf_pmu@busy-check-all@vcs1:
    - {shard-dg1}:        NOTRUN -> [FAIL][1] +3 similar issues
   [1]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-dg1-12/igt@perf_pmu@busy-check-all@vcs1.html

  
Known issues
------------

  Here are the changes found in IGTPW_8069_full that come from known issues:

### IGT changes ###

#### Issues hit ####

  * igt@feature_discovery@display-4x:
    - shard-iclb:         NOTRUN -> [SKIP][2] ([i915#1839])
   [2]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb7/igt@feature_discovery@display-4x.html
    - shard-tglb:         NOTRUN -> [SKIP][3] ([i915#1839])
   [3]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb6/igt@feature_discovery@display-4x.html

  * igt@gem_create@create-massive:
    - shard-apl:          NOTRUN -> [DMESG-WARN][4] ([i915#4991])
   [4]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl2/igt@gem_create@create-massive.html

  * igt@gem_ctx_persistence@hang:
    - shard-snb:          NOTRUN -> [SKIP][5] ([fdo#109271] / [i915#1099]) +2 similar issues
   [5]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-snb5/igt@gem_ctx_persistence@hang.html

  * igt@gem_ctx_sseu@mmap-args:
    - shard-tglb:         NOTRUN -> [SKIP][6] ([i915#280])
   [6]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb7/igt@gem_ctx_sseu@mmap-args.html

  * igt@gem_exec_fair@basic-none@vecs0:
    - shard-glk:          [PASS][7] -> [FAIL][8] ([i915#2842])
   [7]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-glk3/igt@gem_exec_fair@basic-none@vecs0.html
   [8]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk3/igt@gem_exec_fair@basic-none@vecs0.html

  * igt@gem_exec_fair@basic-pace@vcs1:
    - shard-iclb:         NOTRUN -> [FAIL][9] ([i915#2842])
   [9]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb1/igt@gem_exec_fair@basic-pace@vcs1.html

  * igt@gem_huc_copy@huc-copy:
    - shard-apl:          NOTRUN -> [SKIP][10] ([fdo#109271] / [i915#2190])
   [10]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl1/igt@gem_huc_copy@huc-copy.html

  * igt@gem_lmem_swapping@parallel-random:
    - shard-apl:          NOTRUN -> [SKIP][11] ([fdo#109271] / [i915#4613]) +2 similar issues
   [11]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl1/igt@gem_lmem_swapping@parallel-random.html

  * igt@gem_lmem_swapping@verify:
    - shard-iclb:         NOTRUN -> [SKIP][12] ([i915#4613])
   [12]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb7/igt@gem_lmem_swapping@verify.html
    - shard-tglb:         NOTRUN -> [SKIP][13] ([i915#4613])
   [13]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb6/igt@gem_lmem_swapping@verify.html
    - shard-glk:          NOTRUN -> [SKIP][14] ([fdo#109271] / [i915#4613])
   [14]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk6/igt@gem_lmem_swapping@verify.html

  * igt@gem_pxp@protected-raw-src-copy-not-readible:
    - shard-iclb:         NOTRUN -> [SKIP][15] ([i915#4270])
   [15]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb8/igt@gem_pxp@protected-raw-src-copy-not-readible.html
    - shard-tglb:         NOTRUN -> [SKIP][16] ([i915#4270])
   [16]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb5/igt@gem_pxp@protected-raw-src-copy-not-readible.html

  * igt@gem_render_copy@y-tiled-mc-ccs-to-yf-tiled-ccs:
    - shard-iclb:         NOTRUN -> [SKIP][17] ([i915#768])
   [17]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb6/igt@gem_render_copy@y-tiled-mc-ccs-to-yf-tiled-ccs.html

  * igt@gem_softpin@evict-single-offset:
    - shard-iclb:         [PASS][18] -> [FAIL][19] ([i915#4171])
   [18]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb2/igt@gem_softpin@evict-single-offset.html
   [19]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb5/igt@gem_softpin@evict-single-offset.html

  * igt@gem_userptr_blits@coherency-sync:
    - shard-iclb:         NOTRUN -> [SKIP][20] ([fdo#109290])
   [20]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb2/igt@gem_userptr_blits@coherency-sync.html
    - shard-tglb:         NOTRUN -> [SKIP][21] ([fdo#110542])
   [21]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb3/igt@gem_userptr_blits@coherency-sync.html

  * igt@gem_userptr_blits@readonly-pwrite-unsync:
    - shard-tglb:         NOTRUN -> [SKIP][22] ([i915#3297])
   [22]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb7/igt@gem_userptr_blits@readonly-pwrite-unsync.html
    - shard-iclb:         NOTRUN -> [SKIP][23] ([i915#3297])
   [23]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb6/igt@gem_userptr_blits@readonly-pwrite-unsync.html

  * igt@gen9_exec_parse@cmd-crossing-page:
    - shard-tglb:         NOTRUN -> [SKIP][24] ([i915#2527] / [i915#2856]) +2 similar issues
   [24]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb8/igt@gen9_exec_parse@cmd-crossing-page.html
    - shard-iclb:         NOTRUN -> [SKIP][25] ([i915#2856]) +2 similar issues
   [25]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb3/igt@gen9_exec_parse@cmd-crossing-page.html

  * igt@i915_pipe_stress@stress-xrgb8888-untiled:
    - shard-apl:          NOTRUN -> [FAIL][26] ([i915#7036])
   [26]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl8/igt@i915_pipe_stress@stress-xrgb8888-untiled.html

  * igt@i915_pm_dc@dc9-dpms:
    - shard-iclb:         [PASS][27] -> [SKIP][28] ([i915#4281])
   [27]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb2/igt@i915_pm_dc@dc9-dpms.html
   [28]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb3/igt@i915_pm_dc@dc9-dpms.html
    - shard-apl:          [PASS][29] -> [SKIP][30] ([fdo#109271])
   [29]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-apl7/igt@i915_pm_dc@dc9-dpms.html
   [30]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl1/igt@i915_pm_dc@dc9-dpms.html

  * igt@i915_pm_rc6_residency@rc6-idle@rcs0:
    - shard-tglb:         NOTRUN -> [WARN][31] ([i915#2681]) +3 similar issues
   [31]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb6/igt@i915_pm_rc6_residency@rc6-idle@rcs0.html

  * igt@i915_pm_rpm@dpms-non-lpsp:
    - shard-tglb:         NOTRUN -> [SKIP][32] ([fdo#111644] / [i915#1397] / [i915#2411])
   [32]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb5/igt@i915_pm_rpm@dpms-non-lpsp.html
    - shard-iclb:         NOTRUN -> [SKIP][33] ([fdo#110892])
   [33]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb1/igt@i915_pm_rpm@dpms-non-lpsp.html

  * igt@i915_pm_sseu@full-enable:
    - shard-tglb:         NOTRUN -> [SKIP][34] ([i915#4387])
   [34]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb6/igt@i915_pm_sseu@full-enable.html
    - shard-iclb:         NOTRUN -> [SKIP][35] ([i915#4387])
   [35]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb1/igt@i915_pm_sseu@full-enable.html

  * igt@i915_selftest@live@hangcheck:
    - shard-tglb:         [PASS][36] -> [DMESG-WARN][37] ([i915#5591])
   [36]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-tglb8/igt@i915_selftest@live@hangcheck.html
   [37]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb3/igt@i915_selftest@live@hangcheck.html

  * igt@kms_big_fb@4-tiled-16bpp-rotate-0:
    - shard-tglb:         NOTRUN -> [SKIP][38] ([i915#5286])
   [38]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb1/igt@kms_big_fb@4-tiled-16bpp-rotate-0.html
    - shard-iclb:         NOTRUN -> [SKIP][39] ([i915#5286])
   [39]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb7/igt@kms_big_fb@4-tiled-16bpp-rotate-0.html

  * igt@kms_big_fb@linear-8bpp-rotate-90:
    - shard-iclb:         NOTRUN -> [SKIP][40] ([fdo#110725] / [fdo#111614]) +1 similar issue
   [40]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb7/igt@kms_big_fb@linear-8bpp-rotate-90.html

  * igt@kms_big_fb@x-tiled-32bpp-rotate-270:
    - shard-tglb:         NOTRUN -> [SKIP][41] ([fdo#111614]) +1 similar issue
   [41]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb2/igt@kms_big_fb@x-tiled-32bpp-rotate-270.html

  * igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-180-async-flip:
    - shard-tglb:         [PASS][42] -> [FAIL][43] ([i915#3743])
   [42]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-tglb5/igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-180-async-flip.html
   [43]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb6/igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-180-async-flip.html

  * igt@kms_big_fb@yf-tiled-max-hw-stride-32bpp-rotate-0:
    - shard-tglb:         NOTRUN -> [SKIP][44] ([fdo#111615]) +1 similar issue
   [44]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb3/igt@kms_big_fb@yf-tiled-max-hw-stride-32bpp-rotate-0.html

  * igt@kms_big_fb@yf-tiled-max-hw-stride-64bpp-rotate-0-hflip-async-flip:
    - shard-iclb:         NOTRUN -> [SKIP][45] ([fdo#110723])
   [45]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb8/igt@kms_big_fb@yf-tiled-max-hw-stride-64bpp-rotate-0-hflip-async-flip.html

  * igt@kms_ccs@pipe-a-bad-pixel-format-4_tiled_dg2_rc_ccs:
    - shard-iclb:         NOTRUN -> [SKIP][46] ([fdo#109278]) +8 similar issues
   [46]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb7/igt@kms_ccs@pipe-a-bad-pixel-format-4_tiled_dg2_rc_ccs.html
    - shard-tglb:         NOTRUN -> [SKIP][47] ([i915#3689] / [i915#6095])
   [47]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb1/igt@kms_ccs@pipe-a-bad-pixel-format-4_tiled_dg2_rc_ccs.html

  * igt@kms_ccs@pipe-b-crc-primary-basic-yf_tiled_ccs:
    - shard-tglb:         NOTRUN -> [SKIP][48] ([fdo#111615] / [i915#3689]) +2 similar issues
   [48]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb7/igt@kms_ccs@pipe-b-crc-primary-basic-yf_tiled_ccs.html

  * igt@kms_ccs@pipe-b-crc-sprite-planes-basic-4_tiled_dg2_rc_ccs:
    - shard-tglb:         NOTRUN -> [SKIP][49] ([i915#6095]) +2 similar issues
   [49]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb1/igt@kms_ccs@pipe-b-crc-sprite-planes-basic-4_tiled_dg2_rc_ccs.html

  * igt@kms_ccs@pipe-c-bad-pixel-format-y_tiled_gen12_mc_ccs:
    - shard-apl:          NOTRUN -> [SKIP][50] ([fdo#109271] / [i915#3886]) +7 similar issues
   [50]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl2/igt@kms_ccs@pipe-c-bad-pixel-format-y_tiled_gen12_mc_ccs.html

  * igt@kms_ccs@pipe-c-bad-rotation-90-y_tiled_gen12_mc_ccs:
    - shard-glk:          NOTRUN -> [SKIP][51] ([fdo#109271] / [i915#3886]) +4 similar issues
   [51]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk7/igt@kms_ccs@pipe-c-bad-rotation-90-y_tiled_gen12_mc_ccs.html

  * igt@kms_ccs@pipe-c-crc-sprite-planes-basic-y_tiled_gen12_mc_ccs:
    - shard-tglb:         NOTRUN -> [SKIP][52] ([i915#3689] / [i915#3886])
   [52]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb7/igt@kms_ccs@pipe-c-crc-sprite-planes-basic-y_tiled_gen12_mc_ccs.html
    - shard-iclb:         NOTRUN -> [SKIP][53] ([fdo#109278] / [i915#3886]) +1 similar issue
   [53]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb6/igt@kms_ccs@pipe-c-crc-sprite-planes-basic-y_tiled_gen12_mc_ccs.html

  * igt@kms_ccs@pipe-d-missing-ccs-buffer-y_tiled_gen12_mc_ccs:
    - shard-tglb:         NOTRUN -> [SKIP][54] ([i915#3689]) +3 similar issues
   [54]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb6/igt@kms_ccs@pipe-d-missing-ccs-buffer-y_tiled_gen12_mc_ccs.html

  * igt@kms_chamelium@common-hpd-after-suspend:
    - shard-apl:          NOTRUN -> [SKIP][55] ([fdo#109271] / [fdo#111827]) +9 similar issues
   [55]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl2/igt@kms_chamelium@common-hpd-after-suspend.html

  * igt@kms_chamelium@dp-edid-read:
    - shard-glk:          NOTRUN -> [SKIP][56] ([fdo#109271] / [fdo#111827]) +2 similar issues
   [56]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk5/igt@kms_chamelium@dp-edid-read.html

  * igt@kms_color_chamelium@ctm-0-75:
    - shard-iclb:         NOTRUN -> [SKIP][57] ([fdo#109284] / [fdo#111827])
   [57]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb8/igt@kms_color_chamelium@ctm-0-75.html
    - shard-tglb:         NOTRUN -> [SKIP][58] ([fdo#109284] / [fdo#111827])
   [58]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb5/igt@kms_color_chamelium@ctm-0-75.html

  * igt@kms_color_chamelium@ctm-red-to-blue:
    - shard-snb:          NOTRUN -> [SKIP][59] ([fdo#109271] / [fdo#111827]) +3 similar issues
   [59]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-snb6/igt@kms_color_chamelium@ctm-red-to-blue.html

  * igt@kms_content_protection@atomic-dpms@pipe-a-dp-1:
    - shard-apl:          NOTRUN -> [INCOMPLETE][60] ([i915#7121] / [i915#7173])
   [60]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl6/igt@kms_content_protection@atomic-dpms@pipe-a-dp-1.html

  * igt@kms_cursor_crc@cursor-onscreen-128x128@pipe-a-edp-1:
    - shard-iclb:         [PASS][61] -> [DMESG-WARN][62] ([i915#402])
   [61]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb5/igt@kms_cursor_crc@cursor-onscreen-128x128@pipe-a-edp-1.html
   [62]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb8/igt@kms_cursor_crc@cursor-onscreen-128x128@pipe-a-edp-1.html

  * igt@kms_cursor_legacy@2x-long-nonblocking-modeset-vs-cursor-atomic:
    - shard-tglb:         NOTRUN -> [SKIP][63] ([fdo#109274] / [fdo#111825])
   [63]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb8/igt@kms_cursor_legacy@2x-long-nonblocking-modeset-vs-cursor-atomic.html

  * igt@kms_fbcon_fbt@fbc:
    - shard-tglb:         NOTRUN -> [FAIL][64] ([i915#4767])
   [64]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb6/igt@kms_fbcon_fbt@fbc.html
    - shard-glk:          NOTRUN -> [FAIL][65] ([i915#4767])
   [65]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk3/igt@kms_fbcon_fbt@fbc.html
    - shard-iclb:         NOTRUN -> [FAIL][66] ([i915#4767])
   [66]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb1/igt@kms_fbcon_fbt@fbc.html

  * igt@kms_fbcon_fbt@fbc-suspend:
    - shard-apl:          NOTRUN -> [FAIL][67] ([i915#4767]) +1 similar issue
   [67]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl6/igt@kms_fbcon_fbt@fbc-suspend.html

  * igt@kms_flip@2x-modeset-vs-vblank-race:
    - shard-tglb:         NOTRUN -> [SKIP][68] ([fdo#109274] / [fdo#111825] / [i915#3637]) +2 similar issues
   [68]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb5/igt@kms_flip@2x-modeset-vs-vblank-race.html

  * igt@kms_flip@2x-wf_vblank-ts-check:
    - shard-iclb:         NOTRUN -> [SKIP][69] ([fdo#109274]) +3 similar issues
   [69]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb6/igt@kms_flip@2x-wf_vblank-ts-check.html

  * igt@kms_flip@plain-flip-fb-recreate@b-hdmi-a1:
    - shard-glk:          [PASS][70] -> [FAIL][71] ([i915#2122])
   [70]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-glk9/igt@kms_flip@plain-flip-fb-recreate@b-hdmi-a1.html
   [71]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk5/igt@kms_flip@plain-flip-fb-recreate@b-hdmi-a1.html

  * igt@kms_flip_scaled_crc@flip-32bpp-4tile-to-64bpp-4tile-downscaling@pipe-a-valid-mode:
    - shard-iclb:         NOTRUN -> [SKIP][72] ([i915#2587] / [i915#2672]) +7 similar issues
   [72]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb6/igt@kms_flip_scaled_crc@flip-32bpp-4tile-to-64bpp-4tile-downscaling@pipe-a-valid-mode.html

  * igt@kms_flip_scaled_crc@flip-32bpp-yftile-to-64bpp-yftile-downscaling@pipe-a-default-mode:
    - shard-iclb:         NOTRUN -> [SKIP][73] ([i915#2672]) +5 similar issues
   [73]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb2/igt@kms_flip_scaled_crc@flip-32bpp-yftile-to-64bpp-yftile-downscaling@pipe-a-default-mode.html

  * igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytileccs-downscaling@pipe-a-default-mode:
    - shard-iclb:         NOTRUN -> [SKIP][74] ([i915#3555]) +1 similar issue
   [74]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb2/igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytileccs-downscaling@pipe-a-default-mode.html

  * igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytilegen12rcccs-downscaling@pipe-a-default-mode:
    - shard-iclb:         NOTRUN -> [SKIP][75] ([i915#2672] / [i915#3555])
   [75]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb3/igt@kms_flip_scaled_crc@flip-32bpp-ytile-to-32bpp-ytilegen12rcccs-downscaling@pipe-a-default-mode.html

  * igt@kms_frontbuffer_tracking@fbc-2p-scndscrn-indfb-pgflip-blt:
    - shard-iclb:         NOTRUN -> [SKIP][76] ([fdo#109280]) +7 similar issues
   [76]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb6/igt@kms_frontbuffer_tracking@fbc-2p-scndscrn-indfb-pgflip-blt.html

  * igt@kms_frontbuffer_tracking@fbcpsr-1p-primscrn-spr-indfb-move:
    - shard-tglb:         NOTRUN -> [SKIP][77] ([i915#6497]) +7 similar issues
   [77]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb8/igt@kms_frontbuffer_tracking@fbcpsr-1p-primscrn-spr-indfb-move.html

  * igt@kms_frontbuffer_tracking@fbcpsr-2p-primscrn-spr-indfb-draw-mmap-wc:
    - shard-tglb:         NOTRUN -> [SKIP][78] ([fdo#109280] / [fdo#111825]) +8 similar issues
   [78]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb1/igt@kms_frontbuffer_tracking@fbcpsr-2p-primscrn-spr-indfb-draw-mmap-wc.html

  * igt@kms_frontbuffer_tracking@fbcpsr-2p-scndscrn-spr-indfb-draw-mmap-wc:
    - shard-glk:          NOTRUN -> [SKIP][79] ([fdo#109271]) +101 similar issues
   [79]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk8/igt@kms_frontbuffer_tracking@fbcpsr-2p-scndscrn-spr-indfb-draw-mmap-wc.html

  * igt@kms_frontbuffer_tracking@psr-2p-scndscrn-spr-indfb-draw-render:
    - shard-apl:          NOTRUN -> [SKIP][80] ([fdo#109271]) +196 similar issues
   [80]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl3/igt@kms_frontbuffer_tracking@psr-2p-scndscrn-spr-indfb-draw-render.html

  * igt@kms_frontbuffer_tracking@psr-rgb101010-draw-render:
    - shard-iclb:         [PASS][81] -> [FAIL][82] ([i915#2546])
   [81]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb7/igt@kms_frontbuffer_tracking@psr-rgb101010-draw-render.html
   [82]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb8/igt@kms_frontbuffer_tracking@psr-rgb101010-draw-render.html

  * igt@kms_hdr@bpc-switch:
    - shard-tglb:         NOTRUN -> [SKIP][83] ([i915#3555])
   [83]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb5/igt@kms_hdr@bpc-switch.html

  * igt@kms_pipe_b_c_ivb@disable-pipe-b-enable-pipe-c:
    - shard-tglb:         NOTRUN -> [SKIP][84] ([fdo#109289])
   [84]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb1/igt@kms_pipe_b_c_ivb@disable-pipe-b-enable-pipe-c.html

  * igt@kms_plane@plane-panning-bottom-right-suspend@pipe-b-planes:
    - shard-iclb:         [PASS][85] -> [DMESG-WARN][86] ([i915#2867])
   [85]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb1/igt@kms_plane@plane-panning-bottom-right-suspend@pipe-b-planes.html
   [86]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb5/igt@kms_plane@plane-panning-bottom-right-suspend@pipe-b-planes.html

  * igt@kms_plane_alpha_blend@alpha-opaque-fb@pipe-a-dp-1:
    - shard-apl:          NOTRUN -> [FAIL][87] ([i915#4573]) +2 similar issues
   [87]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl3/igt@kms_plane_alpha_blend@alpha-opaque-fb@pipe-a-dp-1.html

  * igt@kms_plane_scaling@plane-scaler-with-clipping-clamping-rotation@pipe-a-vga-1:
    - shard-snb:          NOTRUN -> [SKIP][88] ([fdo#109271]) +157 similar issues
   [88]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-snb7/igt@kms_plane_scaling@plane-scaler-with-clipping-clamping-rotation@pipe-a-vga-1.html

  * igt@kms_plane_scaling@plane-scaler-with-rotation-unity-scaling@pipe-a-edp-1:
    - shard-iclb:         NOTRUN -> [SKIP][89] ([i915#5176]) +2 similar issues
   [89]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb2/igt@kms_plane_scaling@plane-scaler-with-rotation-unity-scaling@pipe-a-edp-1.html

  * igt@kms_plane_scaling@plane-scaler-with-rotation-unity-scaling@pipe-b-edp-1:
    - shard-tglb:         NOTRUN -> [SKIP][90] ([i915#5176]) +3 similar issues
   [90]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb2/igt@kms_plane_scaling@plane-scaler-with-rotation-unity-scaling@pipe-b-edp-1.html

  * igt@kms_psr2_sf@overlay-plane-move-continuous-exceed-sf:
    - shard-iclb:         NOTRUN -> [SKIP][91] ([i915#658])
   [91]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb5/igt@kms_psr2_sf@overlay-plane-move-continuous-exceed-sf.html
    - shard-apl:          NOTRUN -> [SKIP][92] ([fdo#109271] / [i915#658]) +1 similar issue
   [92]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl2/igt@kms_psr2_sf@overlay-plane-move-continuous-exceed-sf.html
    - shard-tglb:         NOTRUN -> [SKIP][93] ([i915#2920])
   [93]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb3/igt@kms_psr2_sf@overlay-plane-move-continuous-exceed-sf.html
    - shard-glk:          NOTRUN -> [SKIP][94] ([fdo#109271] / [i915#658])
   [94]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk6/igt@kms_psr2_sf@overlay-plane-move-continuous-exceed-sf.html

  * igt@kms_psr2_su@page_flip-nv12@pipe-b-edp-1:
    - shard-iclb:         NOTRUN -> [FAIL][95] ([i915#5939]) +2 similar issues
   [95]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb2/igt@kms_psr2_su@page_flip-nv12@pipe-b-edp-1.html

  * igt@kms_psr@psr2_no_drrs:
    - shard-iclb:         [PASS][96] -> [SKIP][97] ([fdo#109441]) +1 similar issue
   [96]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb2/igt@kms_psr@psr2_no_drrs.html
   [97]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb8/igt@kms_psr@psr2_no_drrs.html

  * igt@kms_psr@psr2_primary_mmap_cpu:
    - shard-iclb:         NOTRUN -> [SKIP][98] ([fdo#109441]) +1 similar issue
   [98]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb1/igt@kms_psr@psr2_primary_mmap_cpu.html

  * igt@kms_psr@psr2_suspend:
    - shard-tglb:         NOTRUN -> [FAIL][99] ([i915#132] / [i915#3467]) +1 similar issue
   [99]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb7/igt@kms_psr@psr2_suspend.html

  * igt@kms_psr_stress_test@flip-primary-invalidate-overlay:
    - shard-iclb:         NOTRUN -> [SKIP][100] ([i915#5519])
   [100]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb3/igt@kms_psr_stress_test@flip-primary-invalidate-overlay.html

  * igt@kms_psr_stress_test@invalidate-primary-flip-overlay:
    - shard-tglb:         [PASS][101] -> [SKIP][102] ([i915#5519])
   [101]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-tglb7/igt@kms_psr_stress_test@invalidate-primary-flip-overlay.html
   [102]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb7/igt@kms_psr_stress_test@invalidate-primary-flip-overlay.html
    - shard-iclb:         [PASS][103] -> [SKIP][104] ([i915#5519])
   [103]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb3/igt@kms_psr_stress_test@invalidate-primary-flip-overlay.html
   [104]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb6/igt@kms_psr_stress_test@invalidate-primary-flip-overlay.html

  * igt@kms_setmode@basic@pipe-a-vga-1:
    - shard-snb:          NOTRUN -> [FAIL][105] ([i915#5465]) +1 similar issue
   [105]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-snb4/igt@kms_setmode@basic@pipe-a-vga-1.html

  * igt@kms_vblank@pipe-a-accuracy-idle:
    - shard-glk:          [PASS][106] -> [FAIL][107] ([i915#43])
   [106]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-glk1/igt@kms_vblank@pipe-a-accuracy-idle.html
   [107]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk3/igt@kms_vblank@pipe-a-accuracy-idle.html

  * igt@perf@gen12-mi-rpc:
    - shard-iclb:         NOTRUN -> [SKIP][108] ([fdo#109289]) +2 similar issues
   [108]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb2/igt@perf@gen12-mi-rpc.html

  * igt@sysfs_clients@fair-0:
    - shard-glk:          NOTRUN -> [SKIP][109] ([fdo#109271] / [i915#2994])
   [109]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk2/igt@sysfs_clients@fair-0.html

  
#### Possible fixes ####

  * igt@gem_bad_reloc@negative-reloc-lut:
    - {shard-rkl}:        [SKIP][110] ([i915#3281]) -> [PASS][111] +3 similar issues
   [110]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-4/igt@gem_bad_reloc@negative-reloc-lut.html
   [111]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-5/igt@gem_bad_reloc@negative-reloc-lut.html

  * igt@gem_exec_fair@basic-deadline:
    - {shard-rkl}:        [FAIL][112] ([i915#2846]) -> [PASS][113]
   [112]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-2/igt@gem_exec_fair@basic-deadline.html
   [113]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-3/igt@gem_exec_fair@basic-deadline.html

  * igt@gem_exec_fair@basic-none-rrul@rcs0:
    - shard-glk:          [FAIL][114] ([i915#2842]) -> [PASS][115] +1 similar issue
   [114]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-glk7/igt@gem_exec_fair@basic-none-rrul@rcs0.html
   [115]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk5/igt@gem_exec_fair@basic-none-rrul@rcs0.html

  * igt@gem_exec_fair@basic-pace@vecs0:
    - shard-iclb:         [FAIL][116] ([i915#2842]) -> [PASS][117]
   [116]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb6/igt@gem_exec_fair@basic-pace@vecs0.html
   [117]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb1/igt@gem_exec_fair@basic-pace@vecs0.html

  * igt@gem_huc_copy@huc-copy:
    - shard-tglb:         [SKIP][118] ([i915#2190]) -> [PASS][119]
   [118]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-tglb7/igt@gem_huc_copy@huc-copy.html
   [119]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb8/igt@gem_huc_copy@huc-copy.html

  * igt@gem_lmem_swapping@smem-oom@lmem0:
    - {shard-dg1}:        [DMESG-WARN][120] ([i915#4936]) -> [PASS][121]
   [120]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-dg1-18/igt@gem_lmem_swapping@smem-oom@lmem0.html
   [121]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-dg1-19/igt@gem_lmem_swapping@smem-oom@lmem0.html

  * igt@gem_partial_pwrite_pread@reads-uncached:
    - {shard-rkl}:        [SKIP][122] ([i915#3282]) -> [PASS][123]
   [122]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-6/igt@gem_partial_pwrite_pread@reads-uncached.html
   [123]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-5/igt@gem_partial_pwrite_pread@reads-uncached.html

  * igt@i915_pm_backlight@fade:
    - {shard-rkl}:        [SKIP][124] ([i915#3012]) -> [PASS][125]
   [124]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-1/igt@i915_pm_backlight@fade.html
   [125]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-6/igt@i915_pm_backlight@fade.html

  * igt@i915_pm_dc@dc9-dpms:
    - {shard-rkl}:        [SKIP][126] ([i915#3361]) -> [PASS][127]
   [126]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-5/igt@i915_pm_dc@dc9-dpms.html
   [127]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-4/igt@i915_pm_dc@dc9-dpms.html

  * igt@i915_pm_rps@min-max-config-idle:
    - {shard-rkl}:        [FAIL][128] ([i915#7142]) -> [PASS][129]
   [128]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-1/igt@i915_pm_rps@min-max-config-idle.html
   [129]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-5/igt@i915_pm_rps@min-max-config-idle.html

  * igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-0-hflip-async-flip:
    - {shard-rkl}:        [SKIP][130] ([i915#1845] / [i915#4098]) -> [PASS][131] +17 similar issues
   [130]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-2/igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-0-hflip-async-flip.html
   [131]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-6/igt@kms_big_fb@y-tiled-max-hw-stride-32bpp-rotate-0-hflip-async-flip.html

  * igt@kms_cursor_legacy@flip-vs-cursor@atomic-transitions:
    - shard-glk:          [FAIL][132] ([i915#2346]) -> [PASS][133]
   [132]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-glk6/igt@kms_cursor_legacy@flip-vs-cursor@atomic-transitions.html
   [133]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-glk2/igt@kms_cursor_legacy@flip-vs-cursor@atomic-transitions.html

  * igt@kms_frontbuffer_tracking@fbc-suspend:
    - shard-apl:          [DMESG-WARN][134] ([i915#180]) -> [PASS][135] +1 similar issue
   [134]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-apl8/igt@kms_frontbuffer_tracking@fbc-suspend.html
   [135]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl1/igt@kms_frontbuffer_tracking@fbc-suspend.html

  * igt@kms_frontbuffer_tracking@psr-1p-primscrn-shrfb-msflip-blt:
    - {shard-rkl}:        [SKIP][136] ([i915#1849] / [i915#4098]) -> [PASS][137] +11 similar issues
   [136]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-4/igt@kms_frontbuffer_tracking@psr-1p-primscrn-shrfb-msflip-blt.html
   [137]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-6/igt@kms_frontbuffer_tracking@psr-1p-primscrn-shrfb-msflip-blt.html

  * igt@kms_psr@cursor_blt:
    - {shard-rkl}:        [SKIP][138] ([i915#1072]) -> [PASS][139] +1 similar issue
   [138]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-rkl-4/igt@kms_psr@cursor_blt.html
   [139]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-rkl-6/igt@kms_psr@cursor_blt.html

  * igt@kms_psr@psr2_dpms:
    - shard-iclb:         [SKIP][140] ([fdo#109441]) -> [PASS][141]
   [140]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb5/igt@kms_psr@psr2_dpms.html
   [141]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb2/igt@kms_psr@psr2_dpms.html

  * igt@perf_pmu@busy-double-start@vcs0:
    - {shard-dg1}:        [FAIL][142] ([i915#4349]) -> [PASS][143] +1 similar issue
   [142]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-dg1-12/igt@perf_pmu@busy-double-start@vcs0.html
   [143]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-dg1-17/igt@perf_pmu@busy-double-start@vcs0.html

  * igt@sysfs_timeslice_duration@timeout@vcs1:
    - {shard-dg1}:        [FAIL][144] ([i915#1755]) -> [PASS][145] +3 similar issues
   [144]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-dg1-15/igt@sysfs_timeslice_duration@timeout@vcs1.html
   [145]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-dg1-17/igt@sysfs_timeslice_duration@timeout@vcs1.html

  
#### Warnings ####

  * igt@gem_ctx_isolation@preservation-s3@rcs0:
    - shard-snb:          [SKIP][146] ([fdo#109271]) -> [DMESG-WARN][147] ([i915#5090])
   [146]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-snb6/igt@gem_ctx_isolation@preservation-s3@rcs0.html
   [147]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-snb2/igt@gem_ctx_isolation@preservation-s3@rcs0.html

  * igt@gem_pread@exhaustion:
    - shard-apl:          [INCOMPLETE][148] ([i915#7248]) -> [WARN][149] ([i915#2658])
   [148]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-apl7/igt@gem_pread@exhaustion.html
   [149]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl6/igt@gem_pread@exhaustion.html

  * igt@gem_pwrite@basic-exhaustion:
    - shard-tglb:         [WARN][150] ([i915#2658]) -> [INCOMPLETE][151] ([i915#7248])
   [150]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-tglb8/igt@gem_pwrite@basic-exhaustion.html
   [151]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-tglb8/igt@gem_pwrite@basic-exhaustion.html

  * igt@kms_psr2_sf@overlay-plane-update-sf-dmg-area:
    - shard-iclb:         [SKIP][152] ([i915#2920]) -> [SKIP][153] ([fdo#111068] / [i915#658])
   [152]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-iclb2/igt@kms_psr2_sf@overlay-plane-update-sf-dmg-area.html
   [153]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-iclb3/igt@kms_psr2_sf@overlay-plane-update-sf-dmg-area.html

  * igt@runner@aborted:
    - shard-apl:          ([FAIL][154], [FAIL][155], [FAIL][156], [FAIL][157], [FAIL][158]) ([fdo#109271] / [i915#180] / [i915#3002] / [i915#4312]) -> ([FAIL][159], [FAIL][160], [FAIL][161]) ([i915#3002] / [i915#4312])
   [154]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-apl8/igt@runner@aborted.html
   [155]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-apl3/igt@runner@aborted.html
   [156]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-apl2/igt@runner@aborted.html
   [157]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-apl1/igt@runner@aborted.html
   [158]: https://intel-gfx-ci.01.org/tree/drm-tip/CI_DRM_12356/shard-apl8/igt@runner@aborted.html
   [159]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl2/igt@runner@aborted.html
   [160]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl2/igt@runner@aborted.html
   [161]: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/shard-apl3/igt@runner@aborted.html

  
  {name}: This element is suppressed. This means it is ignored when computing
          the status of the difference (SUCCESS, WARNING, or FAILURE).

  [fdo#109271]: https://bugs.freedesktop.org/show_bug.cgi?id=109271
  [fdo#109274]: https://bugs.freedesktop.org/show_bug.cgi?id=109274
  [fdo#109278]: https://bugs.freedesktop.org/show_bug.cgi?id=109278
  [fdo#109280]: https://bugs.freedesktop.org/show_bug.cgi?id=109280
  [fdo#109283]: https://bugs.freedesktop.org/show_bug.cgi?id=109283
  [fdo#109284]: https://bugs.freedesktop.org/show_bug.cgi?id=109284
  [fdo#109289]: https://bugs.freedesktop.org/show_bug.cgi?id=109289
  [fdo#109290]: https://bugs.freedesktop.org/show_bug.cgi?id=109290
  [fdo#109295]: https://bugs.freedesktop.org/show_bug.cgi?id=109295
  [fdo#109303]: https://bugs.freedesktop.org/show_bug.cgi?id=109303
  [fdo#109307]: https://bugs.freedesktop.org/show_bug.cgi?id=109307
  [fdo#109308]: https://bugs.freedesktop.org/show_bug.cgi?id=109308
  [fdo#109441]: https://bugs.freedesktop.org/show_bug.cgi?id=109441
  [fdo#110189]: https://bugs.freedesktop.org/show_bug.cgi?id=110189
  [fdo#110542]: https://bugs.freedesktop.org/show_bug.cgi?id=110542
  [fdo#110723]: https://bugs.freedesktop.org/show_bug.cgi?id=110723
  [fdo#110725]: https://bugs.freedesktop.org/show_bug.cgi?id=110725
  [fdo#110892]: https://bugs.freedesktop.org/show_bug.cgi?id=110892
  [fdo#111068]: https://bugs.freedesktop.org/show_bug.cgi?id=111068
  [fdo#111614]: https://bugs.freedesktop.org/show_bug.cgi?id=111614
  [fdo#111615]: https://bugs.freedesktop.org/show_bug.cgi?id=111615
  [fdo#111644]: https://bugs.freedesktop.org/show_bug.cgi?id=111644
  [fdo#111656]: https://bugs.freedesktop.org/show_bug.cgi?id=111656
  [fdo#111825]: https://bugs.freedesktop.org/show_bug.cgi?id=111825
  [fdo#111827]: https://bugs.freedesktop.org/show_bug.cgi?id=111827
  [fdo#112054]: https://bugs.freedesktop.org/show_bug.cgi?id=112054
  [fdo#112283]: https://bugs.freedesktop.org/show_bug.cgi?id=112283
  [i915#1072]: https://gitlab.freedesktop.org/drm/intel/issues/1072
  [i915#1099]: https://gitlab.freedesktop.org/drm/intel/issues/1099
  [i915#1155]: https://gitlab.freedesktop.org/drm/intel/issues/1155
  [i915#132]: https://gitlab.freedesktop.org/drm/intel/issues/132
  [i915#1397]: https://gitlab.freedesktop.org/drm/intel/issues/1397
  [i915#1755]: https://gitlab.freedesktop.org/drm/intel/issues/1755
  [i915#1769]: https://gitlab.freedesktop.org/drm/intel/issues/1769
  [i915#180]: https://gitlab.freedesktop.org/drm/intel/issues/180
  [i915#1825]: https://gitlab.freedesktop.org/drm/intel/issues/1825
  [i915#1839]: https://gitlab.freedesktop.org/drm/intel/issues/1839
  [i915#1845]: https://gitlab.freedesktop.org/drm/intel/issues/1845
  [i915#1849]: https://gitlab.freedesktop.org/drm/intel/issues/1849
  [i915#1850]: https://gitlab.freedesktop.org/drm/intel/issues/1850
  [i915#2122]: https://gitlab.freedesktop.org/drm/intel/issues/2122
  [i915#2190]: https://gitlab.freedesktop.org/drm/intel/issues/2190
  [i915#2346]: https://gitlab.freedesktop.org/drm/intel/issues/2346
  [i915#2410]: https://gitlab.freedesktop.org/drm/intel/issues/2410
  [i915#2411]: https://gitlab.freedesktop.org/drm/intel/issues/2411
  [i915#2433]: https://gitlab.freedesktop.org/drm/intel/issues/2433
  [i915#2434]: https://gitlab.freedesktop.org/drm/intel/issues/2434
  [i915#2437]: https://gitlab.freedesktop.org/drm/intel/issues/2437
  [i915#2527]: https://gitlab.freedesktop.org/drm/intel/issues/2527
  [i915#2532]: https://gitlab.freedesktop.org/drm/intel/issues/2532
  [i915#2546]: https://gitlab.freedesktop.org/drm/intel/issues/2546
  [i915#2582]: https://gitlab.freedesktop.org/drm/intel/issues/2582
  [i915#2587]: https://gitlab.freedesktop.org/drm/intel/issues/2587
  [i915#2658]: https://gitlab.freedesktop.org/drm/intel/issues/2658
  [i915#2672]: https://gitlab.freedesktop.org/drm/intel/issues/2672
  [i915#2681]: https://gitlab.freedesktop.org/drm/intel/issues/2681
  [i915#280]: https://gitlab.freedesktop.org/drm/intel/issues/280
  [i915#284]: https://gitlab.freedesktop.org/drm/intel/issues/284
  [i915#2842]: https://gitlab.freedesktop.org/drm/intel/issues/2842
  [i915#2846]: https://gitlab.freedesktop.org/drm/intel/issues/2846
  [i915#2856]: https://gitlab.freedesktop.org/drm/intel/issues/2856
  [i915#2867]: https://gitlab.freedesktop.org/drm/intel/issues/2867
  [i915#2920]: https://gitlab.freedesktop.org/drm/intel/issues/2920
  [i915#2994]: https://gitlab.freedesktop.org/drm/intel/issues/2994
  [i915#3002]: https://gitlab.freedesktop.org/drm/intel/issues/3002
  [i915#3012]: https://gitlab.freedesktop.org/drm/intel/issues/3012
  [i915#3281]: https://gitlab.freedesktop.org/drm/intel/issues/3281
  [i915#3282]: https://gitlab.freedesktop.org/drm/intel/issues/3282
  [i915#3291]: https://gitlab.freedesktop.org/drm/intel/issues/3291
  [i915#3297]: https://gitlab.freedesktop.org/drm/intel/issues/3297
  [i915#3299]: https://gitlab.freedesktop.org/drm/intel/issues/3299
  [i915#3359]: https://gitlab.freedesktop.org/drm/intel/issues/3359
  [i915#3361]: https://gitlab.freedesktop.org/drm/intel/issues/3361
  [i915#3458]: https://gitlab.freedesktop.org/drm/intel/issues/3458
  [i915#3467]: https://gitlab.freedesktop.org/drm/intel/issues/3467
  [i915#3539]: https://gitlab.freedesktop.org/drm/intel/issues/3539
  [i915#3546]: https://gitlab.freedesktop.org/drm/intel/issues/3546
  [i915#3555]: https://gitlab.freedesktop.org/drm/intel/issues/3555
  [i915#3558]: https://gitlab.freedesktop.org/drm/intel/issues/3558
  [i915#3637]: https://gitlab.freedesktop.org/drm/intel/issues/3637
  [i915#3638]: https://gitlab.freedesktop.org/drm/intel/issues/3638
  [i915#3689]: https://gitlab.freedesktop.org/drm/intel/issues/3689
  [i915#3708]: https://gitlab.freedesktop.org/drm/intel/issues/3708
  [i915#3734]: https://gitlab.freedesktop.org/drm/intel/issues/3734
  [i915#3743]: https://gitlab.freedesktop.org/drm/intel/issues/3743
  [i915#3886]: https://gitlab.freedesktop.org/drm/intel/issues/3886
  [i915#3952]: https://gitlab.freedesktop.org/drm/intel/issues/3952
  [i915#3955]: https://gitlab.freedesktop.org/drm/intel/issues/3955
  [i915#402]: https://gitlab.freedesktop.org/drm/intel/issues/402
  [i915#4036]: https://gitlab.freedesktop.org/drm/intel/issues/4036
  [i915#404]: https://gitlab.freedesktop.org/drm/intel/issues/404
  [i915#4070]: https://gitlab.freedesktop.org/drm/intel/issues/4070
  [i915#4077]: https://gitlab.freedesktop.org/drm/intel/issues/4077
  [i915#4078]: https://gitlab.freedesktop.org/drm/intel/issues/4078
  [i915#4079]: https://gitlab.freedesktop.org/drm/intel/issues/4079
  [i915#4083]: https://gitlab.freedesktop.org/drm/intel/issues/4083
  [i915#4098]: https://gitlab.freedesktop.org/drm/intel/issues/4098
  [i915#4171]: https://gitlab.freedesktop.org/drm/intel/issues/4171
  [i915#4212]: https://gitlab.freedesktop.org/drm/intel/issues/4212
  [i915#4215]: https://gitlab.freedesktop.org/drm/intel/issues/4215
  [i915#426]: https://gitlab.freedesktop.org/drm/intel/issues/426
  [i915#4270]: https://gitlab.freedesktop.org/drm/intel/issues/4270
  [i915#4281]: https://gitlab.freedesktop.org/drm/intel/issues/4281
  [i915#43]: https://gitlab.freedesktop.org/drm/intel/issues/43
  [i915#4312]: https://gitlab.freedesktop.org/drm/intel/issues/4312
  [i915#4349]: https://gitlab.freedesktop.org/drm/intel/issues/4349
  [i915#4387]: https://gitlab.freedesktop.org/drm/intel/issues/4387
  [i915#4525]: https://gitlab.freedesktop.org/drm/intel/issues/4525
  [i915#4538]: https://gitlab.freedesktop.org/drm/intel/issues/4538
  [i915#4573]: https://gitlab.freedesktop.org/drm/intel/issues/4573
  [i915#4613]: https://gitlab.freedesktop.org/drm/intel/issues/4613
  [i915#4767]: https://gitlab.freedesktop.org/drm/intel/issues/4767
  [i915#4771]: https://gitlab.freedesktop.org/drm/intel/issues/4771
  [i915#4812]: https://gitlab.freedesktop.org/drm/intel/issues/4812
  [i915#4818]: https://gitlab.freedesktop.org/drm/intel/issues/4818
  [i915#4833]: https://gitlab.freedesktop.org/drm/intel/issues/4833
  [i915#4852]: https://gitlab.freedesktop.org/drm/intel/issues/4852
  [i915#4855]: https://gitlab.freedesktop.org/drm/intel/issues/4855
  [i915#4859]: https://gitlab.freedesktop.org/drm/intel/issues/4859
  [i915#4860]: https://gitlab.freedesktop.org/drm/intel/issues/4860
  [i915#4874]: https://gitlab.freedesktop.org/drm/intel/issues/4874
  [i915#4880]: https://gitlab.freedesktop.org/drm/intel/issues/4880
  [i915#4881]: https://gitlab.freedesktop.org/drm/intel/issues/4881
  [i915#4936]: https://gitlab.freedesktop.org/drm/intel/issues/4936
  [i915#4991]: https://gitlab.freedesktop.org/drm/intel/issues/4991
  [i915#5090]: https://gitlab.freedesktop.org/drm/intel/issues/5090
  [i915#5176]: https://gitlab.freedesktop.org/drm/intel/issues/5176
  [i915#5235]: https://gitlab.freedesktop.org/drm/intel/issues/5235
  [i915#5286]: https://gitlab.freedesktop.org/drm/intel/issues/5286
  [i915#5288]: https://gitlab.freedesktop.org/drm/intel/issues/5288
  [i915#5289]: https://gitlab.freedesktop.org/drm/intel/issues/5289
  [i915#5325]: https://gitlab.freedesktop.org/drm/intel/issues/5325
  [i915#5327]: https://gitlab.freedesktop.org/drm/intel/issues/5327
  [i915#533]: https://gitlab.freedesktop.org/drm/intel/issues/533
  [i915#5439]: https://gitlab.freedesktop.org/drm/intel/issues/5439
  [i915#5461]: https://gitlab.freedesktop.org/drm/intel/issues/5461
  [i915#5465]: https://gitlab.freedesktop.org/drm/intel/issues/5465
  [i915#5519]: https://gitlab.freedesktop.org/drm/intel/issues/5519
  [i915#5563]: https://gitlab.freedesktop.org/drm/intel/issues/5563
  [i915#5566]: https://gitlab.freedesktop.org/drm/intel/issues/5566
  [i915#5591]: https://gitlab.freedesktop.org/drm/intel/issues/5591
  [i915#5723]: https://gitlab.freedesktop.org/drm/intel/issues/5723
  [i915#5784]: https://gitlab.freedesktop.org/drm/intel/issues/5784
  [i915#5939]: https://gitlab.freedesktop.org/drm/intel/issues/5939
  [i915#6095]: https://gitlab.freedesktop.org/drm/intel/issues/6095
  [i915#6227]: https://gitlab.freedesktop.org/drm/intel/issues/6227
  [i915#6230]: https://gitlab.freedesktop.org/drm/intel/issues/6230
  [i915#6248]: https://gitlab.freedesktop.org/drm/intel/issues/6248
  [i915#6258]: https://gitlab.freedesktop.org/drm/intel/issues/6258
  [i915#6301]: https://gitlab.freedesktop.org/drm/intel/issues/6301
  [i915#6412]: https://gitlab.freedesktop.org/drm/intel/issues/6412
  [i915#6433]: https://gitlab.freedesktop.org/drm/intel/issues/6433
  [i915#6497]: https://gitlab.freedesktop.org/drm/intel/issues/6497
  [i915#6524]: https://gitlab.freedesktop.org/drm/intel/issues/6524
  [i915#658]: https://gitlab.freedesktop.org/drm/intel/issues/658
  [i915#6768]: https://gitlab.freedesktop.org/drm/intel/issues/6768
  [i915#6946]: https://gitlab.freedesktop.org/drm/intel/issues/6946
  [i915#7036]: https://gitlab.freedesktop.org/drm/intel/issues/7036
  [i915#7037]: https://gitlab.freedesktop.org/drm/intel/issues/7037
  [i915#7116]: https://gitlab.freedesktop.org/drm/intel/issues/7116
  [i915#7118]: https://gitlab.freedesktop.org/drm/intel/issues/7118
  [i915#7121]: https://gitlab.freedesktop.org/drm/intel/issues/7121
  [i915#7142]: https://gitlab.freedesktop.org/drm/intel/issues/7142
  [i915#7173]: https://gitlab.freedesktop.org/drm/intel/issues/7173
  [i915#7178]: https://gitlab.freedesktop.org/drm/intel/issues/7178
  [i915#7248]: https://gitlab.freedesktop.org/drm/intel/issues/7248
  [i915#7390]: https://gitlab.freedesktop.org/drm/intel/issues/7390
  [i915#7456]: https://gitlab.freedesktop.org/drm/intel/issues/7456
  [i915#7468]: https://gitlab.freedesktop.org/drm/intel/issues/7468
  [i915#768]: https://gitlab.freedesktop.org/drm/intel/issues/768


Build changes
-------------

  * CI: CI-20190529 -> None
  * IGT: IGT_7048 -> IGTPW_8069
  * Piglit: piglit_4509 -> None

  CI-20190529: 20190529
  CI_DRM_12356: 1278975de8debde9eb1f5d86fd2fbe533361e456 @ git://anongit.freedesktop.org/gfx-ci/linux
  IGTPW_8069: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/index.html
  IGT_7048: 5edd5c539f1fdf1c02157bf43fa1fd22d4ad2c75 @ https://gitlab.freedesktop.org/drm/igt-gpu-tools.git
  piglit_4509: fdc5a4ca11124ab8413c7988896eec4c97336694 @ git://anongit.freedesktop.org/piglit

== Logs ==

For more details see: https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8069/index.html

[-- Attachment #2: Type: text/html, Size: 50048 bytes --]

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2023-01-09 21:01                               ` Rodrigo Vivi
@ 2023-01-10 19:49                                 ` Dixit, Ashutosh
  0 siblings, 0 replies; 34+ messages in thread
From: Dixit, Ashutosh @ 2023-01-10 19:49 UTC (permalink / raw)
  To: Rodrigo Vivi; +Cc: igt-dev

On Mon, 09 Jan 2023 13:01:47 -0800, Rodrigo Vivi wrote:
>
> On Fri, Jan 06, 2023 at 01:38:32PM -0800, Dixit, Ashutosh wrote:
> > On Fri, 06 Jan 2023 12:39:27 -0800, Belgaumkar, Vinay wrote:
> > > On 1/6/2023 12:12 PM, Rodrigo Vivi wrote:
> > > > On Thu, Jan 05, 2023 at 01:26:34PM -0800, Dixit, Ashutosh wrote:
> > > >> On Fri, 23 Dec 2022 07:39:29 -0800, Rodrigo Vivi wrote:
> > > >>> On Fri, Dec 23, 2022 at 09:22:53AM +0000, Tvrtko Ursulin wrote:
> > > >>>> On 22/12/2022 20:28, Rodrigo Vivi wrote:
> > > >>>>> On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
> > > >>>>>> On 17/12/2022 02:49, Dixit, Ashutosh wrote:
> > > >>>>>>> On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
> > > >>>>>>>> On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> > > >>>>>>>>> On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> > > >>>>>>>>>> On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> > > >>>>>>>>>>> On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > > >>>>>>>>>>>> On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> > > >>>>>>>>>> Hi Tvrtko,
> > > >>>>>>>>>>
> > > >>>>>>>>>> Sorry for the delay in replying on this. We are discussing this but still
> > > >>>>>>>>>> no conclusion. I have replied what I can below, will update again after I
> > > >>>>>>>>>> know more. Wanted to send a reply before disappearing for the
> > > >>>>>>>>>> holidays. Please see below.
> > > >>>>>>>>>>
> > > >>>>>>>>>>>> Hi Tvrtko,
> > > >>>>>>>>>>>>
> > > >>>>>>>>>>>> I am only offering an overall clarification below first, not answering all
> > > >>>>>>>>>>>> of your points for now.
> > > >>>>>>>>>>>>
> > > >>>>>>>>>>>>> On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > > >>>>>>>>>>>>>> With SLPC, even when we set the same min and max freq's, the requested and
> > > >>>>>>>>>>>>>> actual freq's can differ from the min/max freq set. For example "efficient
> > > >>>>>>>>>>>>>> freq" (when in effect) can override set min/max freq. In general FW is the
> > > >>>>>>>>>>>>>> final arbiter in determining freq and can override set values.
> > > >>>>>>>>>>>>>>
> > > >>>>>>>>>>>>>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > > >>>>>>>>>>>>>> reported by PMU not against the set freq's but against the requested freq
> > > >>>>>>>>>>>>>> reported in sysfs. Also add a delay after setting freq's to account for
> > > >>>>>>>>>>>>>> messaging delays in setting freq's in GuC.
> > > >>>>>>>>>>>>>>
> > > >>>>>>>>>>>>>> v2: Introduce a 100 ms delay after setting freq
> > > >>>>>>>>>>>>>> v3: Update commit message, code identical to v2
> > > >>>>>>>>>>>>>>
> > > >>>>>>>>>>>>>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > > >>>>>>>>>>>>>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > > >>>>>>>>>>>>>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > > >>>>>>>>>>>>>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > > >>>>>>>>>>>>>> ---
> > > >>>>>>>>>>>>>>        tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> > > >>>>>>>>>>>>>>        1 file changed, 15 insertions(+), 8 deletions(-)
> > > >>>>>>>>>>>>>>
> > > >>>>>>>>>>>>>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > > >>>>>>>>>>>>>> index f363db2ba13..02f6ae989b1 100644
> > > >>>>>>>>>>>>>> --- a/tests/i915/perf_pmu.c
> > > >>>>>>>>>>>>>> +++ b/tests/i915/perf_pmu.c
> > > >>>>>>>>>>>>>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> > > >>>>>>>>>>>>>>		igt_assert_lte(target, busy);
> > > >>>>>>>>>>>>>>        }
> > > >>>>>>>>>>>>>>        +/* Wait for GuC SLPC freq changes to take effect */
> > > >>>>>>>>>>>>>> +#define wait_freq_set()		usleep(100000)
> > > >>>>>>>>>>>>> A future task of maybe adding a drop_caches flag to wait for this?
> > > >>>>>>>>>>>>>
> > > >>>>>>>>>>>>> Alternatively, if we have a sysfs or debugfs file which reflects the value
> > > >>>>>>>>>>>>> once GuC has processed it, keep reading it until it is updated? Even if to
> > > >>>>>>>>>>>>> something other than it was before the write, to handle the inability to
> > > >>>>>>>>>>>>> set the exact requested frequency sometimes?
> > > >>>>>>>>>>>>>> +
> > > >>>>>>>>>>>>>>        static void
> > > >>>>>>>>>>>>>>        test_frequency(int gem_fd)
> > > >>>>>>>>>>>>>>        {
> > > >>>>>>>>>>>>>> -	uint32_t min_freq, max_freq, boost_freq;
> > > >>>>>>>>>>>>>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> > > >>>>>>>>>>>>>>		uint64_t val[2], start[2], slept;
> > > >>>>>>>>>>>>>>		double min[2], max[2];
> > > >>>>>>>>>>>>>>		igt_spin_t *spin;
> > > >>>>>>>>>>>>>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> > > >>>>>>>>>>>>>>		 * Set GPU to min frequency and read PMU counters.
> > > >>>>>>>>>>>>>>		 */
> > > >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > > >>>>>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > > >>>>>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > > >>>>>>>>>>>>>> +	wait_freq_set();
> > > >>>>>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > >>>>>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > >>>>>>>>>>>>>>		igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> > > >>>>>>>>>>>>>>			gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > > >>>>>>>>>>>>>> effect */
> > > >>>>>>>>>>>>>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> > > >>>>>>>>>>>>>>			min[0] = 1e9*(val[0] - start[0]) / slept;
> > > >>>>>>>>>>>>>>		min[1] = 1e9*(val[1] - start[1]) / slept;
> > > >>>>>>>>>>>>>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> > > >>>>>>>>>>>>> This really leaves a bad taste for me still, sorry. First of all, it may
> > > >>>>>>>>>>>>> make it more palatable if readback was immediately after wait_freq_set(),
> > > >>>>>>>>>>>>> or as part of it. Otherwise it seems like even more confused test and sysfs
> > > >>>>>>>>>>>>> ABI.
> > > >>>>>>>>>>>>>
> > > >>>>>>>>>>>>> Did we close on my question of whether we can make readback of
> > > >>>>>>>>>>>>> gt_max_freq_mhz actually represent the real max? Correct me please if I got
> > > >>>>>>>>>>>>> confused - with min freq checking here we write max=300 and get 350? If so
> > > >>>>>>>>>>>>> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> > > >>>>>>>>>>>>> error?
> > > >>>>>>>>>>>> Did you read Vinay's email on v1 here:
> > > >>>>>>>>>>>>
> > > >>>>>>>>>>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> > > >>>>>>>>>>>>
> > > >>>>>>>>>>>> As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > > >>>>>>>>>>>> freq rather than the set 'min == max freq') and no amount of waiting will
> > > >>>>>>>>>>>> result in the requested freq being the same as the set 'min == max
> > > >>>>>>>>>>>> freq'. Also the efficient freq cannot be controlled from user space.
> > > >>>>>>>>>>> Why is that not a break in our ABI? What is the point in allowing users to
> > > >>>>>>>>>>> write into this field if the desired value can be silently ignored?
> > > >>>>>>>>>> Yes, this is in discussion. FW is doing this and we need to see if FW can
> > > >>>>>>>>>> be made to obey the i915 ABI. IMHO what matters is performance per watt and
> > > >>>>>>>>>> if the ABI has to be tossed so be it. I will keep this list posted about
> > > >>>>>>>>>> what happens here. Merging any patches for these issues (GL #6806 and
> > > >>>>>>>>>> #6786) is blocked till the ABI breakage issue is resolved one way or
> > > >>>>>>>>>> another.
> > > >>>>>>>>> Two thoughts here, one has been mentioned before.
> > > >>>>>>>>>
> > > >>>>>>>>> New one is that performance per watt perhaps does not matter (is not a
> > > >>>>>>>>> primary concern at least) for use cases where "users" (for some definition)
> > > >>>>>>>>> tweak the min/max in sysfs. Presumably in those cases user wants control,
> > > >>>>>>>>> for some reason, so as long as it is safe and within the limits why not give
> > > >>>>>>>>> it to them.
> > > >>>>>>>>>
> > > >>>>>>>>> Same argument from a different angle - what is the use case for allowing
> > > >>>>>>>>> min/max control which does not really do what it says on the tin?
> > > >>>>>>>>>
> > > >>>>>>>>> Old thought is this - if SLPC refuses to be controlled using our existing
> > > >>>>>>>>> controls then we could just say -ENODEV on attempts to modify them. Whether
> > > >>>>>>>>> for any input value, or some, depending on what SLPC can support (or wants
> > > >>>>>>>>> to support).
> > > >>>>>>>> The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
> > > >>>>>>>> has sudo and all the rights to disrespect what drivers (including SLPC as a
> > > >>>>>>>> a driver from PCODE perspective) are requesting.
> > > >>>>>>>>
> > > >>>>>>>>> Important to note I am primarily focusing on the min freq here, which AFAIR
> > > >>>>>>>>> we concluded can behave that user writes in 300, but SLPC decides it will
> > > >>>>>>>>> only go as low as 350.
> > > >>>>>>>> The other caveat here with the efficient freq is that it is dynamic and PCODE
> > > >>>>>>>> decides it with runtime conditions. It may start with 300, then move to 350,
> > > >>>>>>>> then back to 300. i915 is only stashing the initial value.
> > > >>>>>>>>
> > > >>>>>>>> We could potentially update our rpe everytime any freq control is
> > > >>>>>>>> touched.
> > > >>>>>>> We thought of doing this but two points:
> > > >>>>>>>
> > > >>>>>>> 1. i915 now reads the same register as GuC for rpe but I never see the
> > > >>>>>>>       value in this register change.
> > > >>>>>>> 2. Even if we could know the real rpe, the freq's might be set before a
> > > >>>>>>>       workload start running and rpe might change after the workload starts
> > > >>>>>>>       running. So knowing rpe before the workload might not be very useful.
> > > >>>>>>>
> > > >>>>>>> I am of the opposite opinion that anyway the previous kernel ABI is
> > > >>>>>>> probably now irrecoverably bust, so we should give FW free reign and stop
> > > >>>>>>> disabling efficient freq the way we do now when min_freq < rpe_freq.
> > > >>>>>> We have to answer the question of why we need to keep exposing the sysfs
> > > >>>>>> controls if they don't/can't/won't do what they say on the tin. Could we
> > > >>>>>> make a break with DG2 for instance? Some use cases might suffer but it's
> > > >>>>>> probably better than having them think it does what they think it does, or
> > > >>>>>> file bugs when it does not.
> > > >>>>> That's a good question. But if we have no control at all people will still
> > > >>>>> file requests for something our hardware and low level firmwares cannot promise.
> > > >>>>>
> > > >>>>> There are many folks who run important/critical experiments using these
> > > >>>>> interfaces at the same time they are monitoring throttle conditions and all.
> > > >>>> Aren't the two paragraphs a bit in contradiction? Important and critical
> > > >>>> experiments using controls which are not doing what they think they are.
> > > >>> Well, not really.
> > > >>>
> > > >>> Most of the times it will work. We just don't have a warranty.
> > > >> I think it would help for the user to know when it will work and when it
> > > >> will not.
> > > > Yeap. But mostly this lays on improving our throttle reasons infra.
> > > >
> > > > I'm also wondering now if upon freq reads if we also couldn't internally
> > > > check the throttle reason and put a debug message?!
> > > >
> > > >>> We have some indications when this were disrespected like the
> > > >>> throttle_reasons.
> > > >> There is no indication related to rpe_freq in throttle_reasons, isn't it?
> > > > no.
> > > >
> > > >>> Those limited times doesn't exclude the importance of the usage.
> > > >>>
> > > >>>> I don't claim I know for what all use cases do they get used. I *think*
> > > >>>> there was one in the media transcoding world one where people wanted truly
> > > >>>> fixed frequency so they can compare like-for-like between something.
> > > >>>>
> > > >>>> So bah.. but okay for the perf_pmu changes, how about instead of this:
> > > >>>>
> > > >>>> rpN = read RPn from sysfs
> > > >>>> set_min_freq(rpN)
> > > >>>> sleep
> > > >>>> real_min = read actual from sysfs
> > > >>>> actual = sample pmu for t
> > > >>>> assert real_min is actual

I have resubmitted the previous IGT patches based on this approach (with
RPe enabled) after incorporating all the previous findings (mentioned
previously in this thread) and copied all of you on this mail thread:

https://patchwork.freedesktop.org/series/111282/#rev8

> > > >>>>
> > > >>>> We do this:
> > > >>>>
> > > >>>> rpN = read RPn from sysfs
> > > >>>> set_min_freq(rpN)
> > > >>>> sleep
> > > >>>> fw_min = read actual from sysfs
> > > >>>> set_min_freq(fw_min)
> > > >>>> sleep
> > > >>>> actual = sample pmu for t
> > > >>>> assert fw_min is actual
> > > >>>>
> > > >>>> That's a tiny difference which both "documents" and verifies that even if we
> > > >>>> cannot set the min to RPn, firmware will somewhat consistently (stable)
> > > >>>> respect the minimum it prefers.
> > > >> Note it's not the min but the max freq which is not honored.

I decided not to do this because as mentioned previously we already know
that FW does not respect max_freq. But yes by way of documenting, I have
explained reasons for the changes in comments in the code (and the commit
message history).

> > > >>
> > > >>>> Is that somewhat acceptable?
> > > >>> yeap, this sounds a better approach for a sane test.
> > > >> We talked with the GuC guys and learnt a few things:
> > > >>
> > > >> * PCODE continuously calculates and updates the current RPe (efficient
> > > >>    freq) value
> > > >> * GuC samples the RPe values from PCODE every 1 ms and uses these to
> > > >>    request operating freq. So the requested freq is always set by GuC but
> > > >>    can be overridden say in cases of thermal throttling etc.
> > > >> * GuC ignores the max freq and directly uses RPe freq when requesting an
> > > >>    operating freq. This is beacuse RPe is supposedly more efficient that a
> > > >>    lower max freq.
> > > >> * GuC does have a separate control to ignore RPe entirely.
> > > >> * Regarding the delays I added in the patch, some time back GuC FW was
> > > >>    changed to move setting min/max freq's to the top half. As a result the
> > > >>    delays are no longer required.
> > > >>
> > > >> The most recent version the patches in light of above points are here:
> > > >>
> > > >> https://patchwork.freedesktop.org/series/111282/#rev3
> > > >>
> > > >> (The patches are similar to Tvrtko's first (rather than the second)
> > > >> pseudo-code above).
> > > >>
> > > >> These patches improve things but even with these patches unfortunately
> > > >> there are still sporadic failures. For example:
> > > >>
> > > >> Original failure:
> > > >> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8303/bat-dg2-11/igt@perf_pmu@frequency.html
> > > >> * RPn 300 MHz, PMU measures 397 MHz
> > > >>
> > > >> New failure:
> > > >> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8304/bat-dg2-11/igt@perf_pmu@frequency.html
> > > >> * PMU measures 371 MHz, requested freq measured after PMU measurement 400 MHz
> > > >>
> > > >> So these sporadic failures seems to be due to dynamically changing RPe.
> > > > Yes, they are.
> > > >
> > > >> It may be possible to get rid of the failures by either increasing the
> > > >> tolerance or sampling the requested freq sysfs more often (rather than just
> > > >> once after the PMU measurement) though maybe it's too much.
> > > > On Xe driver I workarounded it by 'constantly' updating our cached rpe.
> > > > By constantly I mean whenever anyone is touching (reading or writing) any
> > > > frequency.
> > > >
> > > > However I'm aware that this can be racy. Specially for the PMU cases or
> > > > with some out of band checks.
> >
> > Consider 2 cases:
> >
> > 1. We start a spinner load and then update min/max freq
> > 2. We update min/max freq first and then start spinner load
> >
> > Doing the above, we likely pick up the correct (RPe) freq being used for
> > case 1 but not for case 2. And even for case 1 RPe might be dynamically
> > changing.
>
> Indeed. It is racy no matter how you look at it. 1 component deciding and
> 2 other reading and using to take decisions without any lock.
>
> >
> > > >> So the best way to fix such sporadic failures seems to be to expose a
> > > >> 'rpe_enable' sysfs to turn off RPe for such tests/usages where strict
> > > >> userspace control of freq limits is required.
> > > >>
> > > >> @Rodrigo Vivi: so we need to see if we can make this uapi change and then
> > > >> go from there I think.
> > > > I believe this is the most sane option right now. We need to document this
> > > > well and have test cases for both.

So as mentioned above the current patches are for the case with RPe
enabled. Let's try to implement the rpe_enable sysfs and then have separate
patches for the RPe disabled case as a next step?

Thanks.
--
Ashutosh


> > > Either that, or we turn off efficient frequency when user updates max
> > > frequency as well (similar to what we do for min).
> >
> > I think I disagree here, turning off RPe will affect PnP. I think we should
> > have explicit disable of RPe when that is needed and should actually remove
> > RPe disable when min is updated. Basically IMO the idea should be to
> > provide the best PnP and then tailor the ABI for the best PnP.
>
> We don't have any api problem when rpe is enabled but min <= rpe < max.
> So Vinay is wondering if we can get this scenario to enable the rpe in guc
> we don't need to create another api. Then on any read or write we would check
> this condition and decide on the efficient enabled or not.
>
> But as I told, it is racy anyway.
>
> So, in the end if we don't disable rpe we don't have how to guarantee the freq
> stable freq api. But if we disable we have the pnp impact. If we try any magic
> it will be racy.
>
> I'm tending to agree with Ashutosh and that adding the rpe_enabled is the best
> option. But document that very well. And yes, update the cached actual rpe anytime
> that we try to read that.
>
> >
> > > >
> > > > Probably this plus the constant update of the cached RPe.
> > >
> > > Caching might be easier if GuC does the same. It can update the task state
> > > data with the new value for everyone else to read.
> >
> > i915 will either need to read the register or read the task state from
> > GuC. You mean reading the task state is more efficient than reading the
> > register?
>
> oh, indeed... we can remove the cache and read from pcode directly anytime
> we need that. Safer indeed.
>
> >
> > Thanks.
> > --
> > Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2023-01-06 21:38                             ` Dixit, Ashutosh
@ 2023-01-09 21:01                               ` Rodrigo Vivi
  2023-01-10 19:49                                 ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Rodrigo Vivi @ 2023-01-09 21:01 UTC (permalink / raw)
  To: Dixit, Ashutosh; +Cc: igt-dev

On Fri, Jan 06, 2023 at 01:38:32PM -0800, Dixit, Ashutosh wrote:
> On Fri, 06 Jan 2023 12:39:27 -0800, Belgaumkar, Vinay wrote:
> > On 1/6/2023 12:12 PM, Rodrigo Vivi wrote:
> > > On Thu, Jan 05, 2023 at 01:26:34PM -0800, Dixit, Ashutosh wrote:
> > >> On Fri, 23 Dec 2022 07:39:29 -0800, Rodrigo Vivi wrote:
> > >>> On Fri, Dec 23, 2022 at 09:22:53AM +0000, Tvrtko Ursulin wrote:
> > >>>> On 22/12/2022 20:28, Rodrigo Vivi wrote:
> > >>>>> On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
> > >>>>>> On 17/12/2022 02:49, Dixit, Ashutosh wrote:
> > >>>>>>> On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
> > >>>>>>>> On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> > >>>>>>>>> On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> > >>>>>>>>>> On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> > >>>>>>>>>>> On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > >>>>>>>>>>>> On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> > >>>>>>>>>> Hi Tvrtko,
> > >>>>>>>>>>
> > >>>>>>>>>> Sorry for the delay in replying on this. We are discussing this but still
> > >>>>>>>>>> no conclusion. I have replied what I can below, will update again after I
> > >>>>>>>>>> know more. Wanted to send a reply before disappearing for the
> > >>>>>>>>>> holidays. Please see below.
> > >>>>>>>>>>
> > >>>>>>>>>>>> Hi Tvrtko,
> > >>>>>>>>>>>>
> > >>>>>>>>>>>> I am only offering an overall clarification below first, not answering all
> > >>>>>>>>>>>> of your points for now.
> > >>>>>>>>>>>>
> > >>>>>>>>>>>>> On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > >>>>>>>>>>>>>> With SLPC, even when we set the same min and max freq's, the requested and
> > >>>>>>>>>>>>>> actual freq's can differ from the min/max freq set. For example "efficient
> > >>>>>>>>>>>>>> freq" (when in effect) can override set min/max freq. In general FW is the
> > >>>>>>>>>>>>>> final arbiter in determining freq and can override set values.
> > >>>>>>>>>>>>>>
> > >>>>>>>>>>>>>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > >>>>>>>>>>>>>> reported by PMU not against the set freq's but against the requested freq
> > >>>>>>>>>>>>>> reported in sysfs. Also add a delay after setting freq's to account for
> > >>>>>>>>>>>>>> messaging delays in setting freq's in GuC.
> > >>>>>>>>>>>>>>
> > >>>>>>>>>>>>>> v2: Introduce a 100 ms delay after setting freq
> > >>>>>>>>>>>>>> v3: Update commit message, code identical to v2
> > >>>>>>>>>>>>>>
> > >>>>>>>>>>>>>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > >>>>>>>>>>>>>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > >>>>>>>>>>>>>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > >>>>>>>>>>>>>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > >>>>>>>>>>>>>> ---
> > >>>>>>>>>>>>>>        tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> > >>>>>>>>>>>>>>        1 file changed, 15 insertions(+), 8 deletions(-)
> > >>>>>>>>>>>>>>
> > >>>>>>>>>>>>>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > >>>>>>>>>>>>>> index f363db2ba13..02f6ae989b1 100644
> > >>>>>>>>>>>>>> --- a/tests/i915/perf_pmu.c
> > >>>>>>>>>>>>>> +++ b/tests/i915/perf_pmu.c
> > >>>>>>>>>>>>>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> > >>>>>>>>>>>>>>		igt_assert_lte(target, busy);
> > >>>>>>>>>>>>>>        }
> > >>>>>>>>>>>>>>        +/* Wait for GuC SLPC freq changes to take effect */
> > >>>>>>>>>>>>>> +#define wait_freq_set()		usleep(100000)
> > >>>>>>>>>>>>> A future task of maybe adding a drop_caches flag to wait for this?
> > >>>>>>>>>>>>>
> > >>>>>>>>>>>>> Alternatively, if we have a sysfs or debugfs file which reflects the value
> > >>>>>>>>>>>>> once GuC has processed it, keep reading it until it is updated? Even if to
> > >>>>>>>>>>>>> something other than it was before the write, to handle the inability to
> > >>>>>>>>>>>>> set the exact requested frequency sometimes?
> > >>>>>>>>>>>>>> +
> > >>>>>>>>>>>>>>        static void
> > >>>>>>>>>>>>>>        test_frequency(int gem_fd)
> > >>>>>>>>>>>>>>        {
> > >>>>>>>>>>>>>> -	uint32_t min_freq, max_freq, boost_freq;
> > >>>>>>>>>>>>>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> > >>>>>>>>>>>>>>		uint64_t val[2], start[2], slept;
> > >>>>>>>>>>>>>>		double min[2], max[2];
> > >>>>>>>>>>>>>>		igt_spin_t *spin;
> > >>>>>>>>>>>>>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> > >>>>>>>>>>>>>>		 * Set GPU to min frequency and read PMU counters.
> > >>>>>>>>>>>>>>		 */
> > >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > >>>>>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > >>>>>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > >>>>>>>>>>>>>> +	wait_freq_set();
> > >>>>>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > >>>>>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > >>>>>>>>>>>>>>		igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> > >>>>>>>>>>>>>>			gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > >>>>>>>>>>>>>> effect */
> > >>>>>>>>>>>>>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> > >>>>>>>>>>>>>>			min[0] = 1e9*(val[0] - start[0]) / slept;
> > >>>>>>>>>>>>>>		min[1] = 1e9*(val[1] - start[1]) / slept;
> > >>>>>>>>>>>>>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> > >>>>>>>>>>>>> This really leaves a bad taste for me still, sorry. First of all, it may
> > >>>>>>>>>>>>> make it more palatable if readback was immediately after wait_freq_set(),
> > >>>>>>>>>>>>> or as part of it. Otherwise it seems like even more confused test and sysfs
> > >>>>>>>>>>>>> ABI.
> > >>>>>>>>>>>>>
> > >>>>>>>>>>>>> Did we close on my question of whether we can make readback of
> > >>>>>>>>>>>>> gt_max_freq_mhz actually represent the real max? Correct me please if I got
> > >>>>>>>>>>>>> confused - with min freq checking here we write max=300 and get 350? If so
> > >>>>>>>>>>>>> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> > >>>>>>>>>>>>> error?
> > >>>>>>>>>>>> Did you read Vinay's email on v1 here:
> > >>>>>>>>>>>>
> > >>>>>>>>>>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> > >>>>>>>>>>>>
> > >>>>>>>>>>>> As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > >>>>>>>>>>>> freq rather than the set 'min == max freq') and no amount of waiting will
> > >>>>>>>>>>>> result in the requested freq being the same as the set 'min == max
> > >>>>>>>>>>>> freq'. Also the efficient freq cannot be controlled from user space.
> > >>>>>>>>>>> Why is that not a break in our ABI? What is the point in allowing users to
> > >>>>>>>>>>> write into this field if the desired value can be silently ignored?
> > >>>>>>>>>> Yes, this is in discussion. FW is doing this and we need to see if FW can
> > >>>>>>>>>> be made to obey the i915 ABI. IMHO what matters is performance per watt and
> > >>>>>>>>>> if the ABI has to be tossed so be it. I will keep this list posted about
> > >>>>>>>>>> what happens here. Merging any patches for these issues (GL #6806 and
> > >>>>>>>>>> #6786) is blocked till the ABI breakage issue is resolved one way or
> > >>>>>>>>>> another.
> > >>>>>>>>> Two thoughts here, one has been mentioned before.
> > >>>>>>>>>
> > >>>>>>>>> New one is that performance per watt perhaps does not matter (is not a
> > >>>>>>>>> primary concern at least) for use cases where "users" (for some definition)
> > >>>>>>>>> tweak the min/max in sysfs. Presumably in those cases user wants control,
> > >>>>>>>>> for some reason, so as long as it is safe and within the limits why not give
> > >>>>>>>>> it to them.
> > >>>>>>>>>
> > >>>>>>>>> Same argument from a different angle - what is the use case for allowing
> > >>>>>>>>> min/max control which does not really do what it says on the tin?
> > >>>>>>>>>
> > >>>>>>>>> Old thought is this - if SLPC refuses to be controlled using our existing
> > >>>>>>>>> controls then we could just say -ENODEV on attempts to modify them. Whether
> > >>>>>>>>> for any input value, or some, depending on what SLPC can support (or wants
> > >>>>>>>>> to support).
> > >>>>>>>> The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
> > >>>>>>>> has sudo and all the rights to disrespect what drivers (including SLPC as a
> > >>>>>>>> a driver from PCODE perspective) are requesting.
> > >>>>>>>>
> > >>>>>>>>> Important to note I am primarily focusing on the min freq here, which AFAIR
> > >>>>>>>>> we concluded can behave that user writes in 300, but SLPC decides it will
> > >>>>>>>>> only go as low as 350.
> > >>>>>>>> The other caveat here with the efficient freq is that it is dynamic and PCODE
> > >>>>>>>> decides it with runtime conditions. It may start with 300, then move to 350,
> > >>>>>>>> then back to 300. i915 is only stashing the initial value.
> > >>>>>>>>
> > >>>>>>>> We could potentially update our rpe everytime any freq control is
> > >>>>>>>> touched.
> > >>>>>>> We thought of doing this but two points:
> > >>>>>>>
> > >>>>>>> 1. i915 now reads the same register as GuC for rpe but I never see the
> > >>>>>>>       value in this register change.
> > >>>>>>> 2. Even if we could know the real rpe, the freq's might be set before a
> > >>>>>>>       workload start running and rpe might change after the workload starts
> > >>>>>>>       running. So knowing rpe before the workload might not be very useful.
> > >>>>>>>
> > >>>>>>> I am of the opposite opinion that anyway the previous kernel ABI is
> > >>>>>>> probably now irrecoverably bust, so we should give FW free reign and stop
> > >>>>>>> disabling efficient freq the way we do now when min_freq < rpe_freq.
> > >>>>>> We have to answer the question of why we need to keep exposing the sysfs
> > >>>>>> controls if they don't/can't/won't do what they say on the tin. Could we
> > >>>>>> make a break with DG2 for instance? Some use cases might suffer but it's
> > >>>>>> probably better than having them think it does what they think it does, or
> > >>>>>> file bugs when it does not.
> > >>>>> That's a good question. But if we have no control at all people will still
> > >>>>> file requests for something our hardware and low level firmwares cannot promise.
> > >>>>>
> > >>>>> There are many folks who run important/critical experiments using these
> > >>>>> interfaces at the same time they are monitoring throttle conditions and all.
> > >>>> Aren't the two paragraphs a bit in contradiction? Important and critical
> > >>>> experiments using controls which are not doing what they think they are.
> > >>> Well, not really.
> > >>>
> > >>> Most of the times it will work. We just don't have a warranty.
> > >> I think it would help for the user to know when it will work and when it
> > >> will not.
> > > Yeap. But mostly this lays on improving our throttle reasons infra.
> > >
> > > I'm also wondering now if upon freq reads if we also couldn't internally
> > > check the throttle reason and put a debug message?!
> > >
> > >>> We have some indications when this were disrespected like the
> > >>> throttle_reasons.
> > >> There is no indication related to rpe_freq in throttle_reasons, isn't it?
> > > no.
> > >
> > >>> Those limited times doesn't exclude the importance of the usage.
> > >>>
> > >>>> I don't claim I know for what all use cases do they get used. I *think*
> > >>>> there was one in the media transcoding world one where people wanted truly
> > >>>> fixed frequency so they can compare like-for-like between something.
> > >>>>
> > >>>> So bah.. but okay for the perf_pmu changes, how about instead of this:
> > >>>>
> > >>>> rpN = read RPn from sysfs
> > >>>> set_min_freq(rpN)
> > >>>> sleep
> > >>>> real_min = read actual from sysfs
> > >>>> actual = sample pmu for t
> > >>>> assert real_min is actual
> > >>>>
> > >>>> We do this:
> > >>>>
> > >>>> rpN = read RPn from sysfs
> > >>>> set_min_freq(rpN)
> > >>>> sleep
> > >>>> fw_min = read actual from sysfs
> > >>>> set_min_freq(fw_min)
> > >>>> sleep
> > >>>> actual = sample pmu for t
> > >>>> assert fw_min is actual
> > >>>>
> > >>>> That's a tiny difference which both "documents" and verifies that even if we
> > >>>> cannot set the min to RPn, firmware will somewhat consistently (stable)
> > >>>> respect the minimum it prefers.
> > >> Note it's not the min but the max freq which is not honored.
> > >>
> > >>>> Is that somewhat acceptable?
> > >>> yeap, this sounds a better approach for a sane test.
> > >> We talked with the GuC guys and learnt a few things:
> > >>
> > >> * PCODE continuously calculates and updates the current RPe (efficient
> > >>    freq) value
> > >> * GuC samples the RPe values from PCODE every 1 ms and uses these to
> > >>    request operating freq. So the requested freq is always set by GuC but
> > >>    can be overridden say in cases of thermal throttling etc.
> > >> * GuC ignores the max freq and directly uses RPe freq when requesting an
> > >>    operating freq. This is beacuse RPe is supposedly more efficient that a
> > >>    lower max freq.
> > >> * GuC does have a separate control to ignore RPe entirely.
> > >> * Regarding the delays I added in the patch, some time back GuC FW was
> > >>    changed to move setting min/max freq's to the top half. As a result the
> > >>    delays are no longer required.
> > >>
> > >> The most recent version the patches in light of above points are here:
> > >>
> > >> https://patchwork.freedesktop.org/series/111282/#rev3
> > >>
> > >> (The patches are similar to Tvrtko's first (rather than the second)
> > >> pseudo-code above).
> > >>
> > >> These patches improve things but even with these patches unfortunately
> > >> there are still sporadic failures. For example:
> > >>
> > >> Original failure:
> > >> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8303/bat-dg2-11/igt@perf_pmu@frequency.html
> > >> * RPn 300 MHz, PMU measures 397 MHz
> > >>
> > >> New failure:
> > >> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8304/bat-dg2-11/igt@perf_pmu@frequency.html
> > >> * PMU measures 371 MHz, requested freq measured after PMU measurement 400 MHz
> > >>
> > >> So these sporadic failures seems to be due to dynamically changing RPe.
> > > Yes, they are.
> > >
> > >> It may be possible to get rid of the failures by either increasing the
> > >> tolerance or sampling the requested freq sysfs more often (rather than just
> > >> once after the PMU measurement) though maybe it's too much.
> > > On Xe driver I workarounded it by 'constantly' updating our cached rpe.
> > > By constantly I mean whenever anyone is touching (reading or writing) any
> > > frequency.
> > >
> > > However I'm aware that this can be racy. Specially for the PMU cases or
> > > with some out of band checks.
> 
> Consider 2 cases:
> 
> 1. We start a spinner load and then update min/max freq
> 2. We update min/max freq first and then start spinner load
> 
> Doing the above, we likely pick up the correct (RPe) freq being used for
> case 1 but not for case 2. And even for case 1 RPe might be dynamically
> changing.

Indeed. It is racy no matter how you look at it. 1 component deciding and
2 other reading and using to take decisions without any lock.

> 
> > >> So the best way to fix such sporadic failures seems to be to expose a
> > >> 'rpe_enable' sysfs to turn off RPe for such tests/usages where strict
> > >> userspace control of freq limits is required.
> > >>
> > >> @Rodrigo Vivi: so we need to see if we can make this uapi change and then
> > >> go from there I think.
> > > I believe this is the most sane option right now. We need to document this
> > > well and have test cases for both.
> > Either that, or we turn off efficient frequency when user updates max
> > frequency as well (similar to what we do for min).
> 
> I think I disagree here, turning off RPe will affect PnP. I think we should
> have explicit disable of RPe when that is needed and should actually remove
> RPe disable when min is updated. Basically IMO the idea should be to
> provide the best PnP and then tailor the ABI for the best PnP.

We don't have any api problem when rpe is enabled but min <= rpe < max.
So Vinay is wondering if we can get this scenario to enable the rpe in guc
we don't need to create another api. Then on any read or write we would check
this condition and decide on the efficient enabled or not.

But as I told, it is racy anyway.

So, in the end if we don't disable rpe we don't have how to guarantee the freq
stable freq api. But if we disable we have the pnp impact. If we try any magic
it will be racy.

I'm tending to agree with Ashutosh and that adding the rpe_enabled is the best
option. But document that very well. And yes, update the cached actual rpe anytime
that we try to read that.

> 
> > >
> > > Probably this plus the constant update of the cached RPe.
> >
> > Caching might be easier if GuC does the same. It can update the task state
> > data with the new value for everyone else to read.
> 
> i915 will either need to read the register or read the task state from
> GuC. You mean reading the task state is more efficient than reading the
> register?

oh, indeed... we can remove the cache and read from pcode directly anytime
we need that. Safer indeed.

> 
> Thanks.
> --
> Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2023-01-06 20:39                           ` Belgaumkar, Vinay
@ 2023-01-06 21:38                             ` Dixit, Ashutosh
  2023-01-09 21:01                               ` Rodrigo Vivi
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2023-01-06 21:38 UTC (permalink / raw)
  To: Belgaumkar, Vinay; +Cc: igt-dev, Rodrigo Vivi

On Fri, 06 Jan 2023 12:39:27 -0800, Belgaumkar, Vinay wrote:
> On 1/6/2023 12:12 PM, Rodrigo Vivi wrote:
> > On Thu, Jan 05, 2023 at 01:26:34PM -0800, Dixit, Ashutosh wrote:
> >> On Fri, 23 Dec 2022 07:39:29 -0800, Rodrigo Vivi wrote:
> >>> On Fri, Dec 23, 2022 at 09:22:53AM +0000, Tvrtko Ursulin wrote:
> >>>> On 22/12/2022 20:28, Rodrigo Vivi wrote:
> >>>>> On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
> >>>>>> On 17/12/2022 02:49, Dixit, Ashutosh wrote:
> >>>>>>> On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
> >>>>>>>> On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> >>>>>>>>> On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> >>>>>>>>>> On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> >>>>>>>>>>> On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> >>>>>>>>>>>> On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> >>>>>>>>>> Hi Tvrtko,
> >>>>>>>>>>
> >>>>>>>>>> Sorry for the delay in replying on this. We are discussing this but still
> >>>>>>>>>> no conclusion. I have replied what I can below, will update again after I
> >>>>>>>>>> know more. Wanted to send a reply before disappearing for the
> >>>>>>>>>> holidays. Please see below.
> >>>>>>>>>>
> >>>>>>>>>>>> Hi Tvrtko,
> >>>>>>>>>>>>
> >>>>>>>>>>>> I am only offering an overall clarification below first, not answering all
> >>>>>>>>>>>> of your points for now.
> >>>>>>>>>>>>
> >>>>>>>>>>>>> On 19/11/2022 02:00, Ashutosh Dixit wrote:
> >>>>>>>>>>>>>> With SLPC, even when we set the same min and max freq's, the requested and
> >>>>>>>>>>>>>> actual freq's can differ from the min/max freq set. For example "efficient
> >>>>>>>>>>>>>> freq" (when in effect) can override set min/max freq. In general FW is the
> >>>>>>>>>>>>>> final arbiter in determining freq and can override set values.
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> >>>>>>>>>>>>>> reported by PMU not against the set freq's but against the requested freq
> >>>>>>>>>>>>>> reported in sysfs. Also add a delay after setting freq's to account for
> >>>>>>>>>>>>>> messaging delays in setting freq's in GuC.
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> v2: Introduce a 100 ms delay after setting freq
> >>>>>>>>>>>>>> v3: Update commit message, code identical to v2
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> >>>>>>>>>>>>>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> >>>>>>>>>>>>>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> >>>>>>>>>>>>>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> >>>>>>>>>>>>>> ---
> >>>>>>>>>>>>>>        tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> >>>>>>>>>>>>>>        1 file changed, 15 insertions(+), 8 deletions(-)
> >>>>>>>>>>>>>>
> >>>>>>>>>>>>>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> >>>>>>>>>>>>>> index f363db2ba13..02f6ae989b1 100644
> >>>>>>>>>>>>>> --- a/tests/i915/perf_pmu.c
> >>>>>>>>>>>>>> +++ b/tests/i915/perf_pmu.c
> >>>>>>>>>>>>>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> >>>>>>>>>>>>>>		igt_assert_lte(target, busy);
> >>>>>>>>>>>>>>        }
> >>>>>>>>>>>>>>        +/* Wait for GuC SLPC freq changes to take effect */
> >>>>>>>>>>>>>> +#define wait_freq_set()		usleep(100000)
> >>>>>>>>>>>>> A future task of maybe adding a drop_caches flag to wait for this?
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> Alternatively, if we have a sysfs or debugfs file which reflects the value
> >>>>>>>>>>>>> once GuC has processed it, keep reading it until it is updated? Even if to
> >>>>>>>>>>>>> something other than it was before the write, to handle the inability to
> >>>>>>>>>>>>> set the exact requested frequency sometimes?
> >>>>>>>>>>>>>> +
> >>>>>>>>>>>>>>        static void
> >>>>>>>>>>>>>>        test_frequency(int gem_fd)
> >>>>>>>>>>>>>>        {
> >>>>>>>>>>>>>> -	uint32_t min_freq, max_freq, boost_freq;
> >>>>>>>>>>>>>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> >>>>>>>>>>>>>>		uint64_t val[2], start[2], slept;
> >>>>>>>>>>>>>>		double min[2], max[2];
> >>>>>>>>>>>>>>		igt_spin_t *spin;
> >>>>>>>>>>>>>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> >>>>>>>>>>>>>>		 * Set GPU to min frequency and read PMU counters.
> >>>>>>>>>>>>>>		 */
> >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> >>>>>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> >>>>>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> >>>>>>>>>>>>>>		igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> >>>>>>>>>>>>>> +	wait_freq_set();
> >>>>>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> >>>>>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> >>>>>>>>>>>>>>		igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> >>>>>>>>>>>>>>			gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> >>>>>>>>>>>>>> effect */
> >>>>>>>>>>>>>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> >>>>>>>>>>>>>>			min[0] = 1e9*(val[0] - start[0]) / slept;
> >>>>>>>>>>>>>>		min[1] = 1e9*(val[1] - start[1]) / slept;
> >>>>>>>>>>>>>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> >>>>>>>>>>>>> This really leaves a bad taste for me still, sorry. First of all, it may
> >>>>>>>>>>>>> make it more palatable if readback was immediately after wait_freq_set(),
> >>>>>>>>>>>>> or as part of it. Otherwise it seems like even more confused test and sysfs
> >>>>>>>>>>>>> ABI.
> >>>>>>>>>>>>>
> >>>>>>>>>>>>> Did we close on my question of whether we can make readback of
> >>>>>>>>>>>>> gt_max_freq_mhz actually represent the real max? Correct me please if I got
> >>>>>>>>>>>>> confused - with min freq checking here we write max=300 and get 350? If so
> >>>>>>>>>>>>> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> >>>>>>>>>>>>> error?
> >>>>>>>>>>>> Did you read Vinay's email on v1 here:
> >>>>>>>>>>>>
> >>>>>>>>>>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> >>>>>>>>>>>>
> >>>>>>>>>>>> As Vinay says, SLPC is requesting an entirely different freq (the effecient
> >>>>>>>>>>>> freq rather than the set 'min == max freq') and no amount of waiting will
> >>>>>>>>>>>> result in the requested freq being the same as the set 'min == max
> >>>>>>>>>>>> freq'. Also the efficient freq cannot be controlled from user space.
> >>>>>>>>>>> Why is that not a break in our ABI? What is the point in allowing users to
> >>>>>>>>>>> write into this field if the desired value can be silently ignored?
> >>>>>>>>>> Yes, this is in discussion. FW is doing this and we need to see if FW can
> >>>>>>>>>> be made to obey the i915 ABI. IMHO what matters is performance per watt and
> >>>>>>>>>> if the ABI has to be tossed so be it. I will keep this list posted about
> >>>>>>>>>> what happens here. Merging any patches for these issues (GL #6806 and
> >>>>>>>>>> #6786) is blocked till the ABI breakage issue is resolved one way or
> >>>>>>>>>> another.
> >>>>>>>>> Two thoughts here, one has been mentioned before.
> >>>>>>>>>
> >>>>>>>>> New one is that performance per watt perhaps does not matter (is not a
> >>>>>>>>> primary concern at least) for use cases where "users" (for some definition)
> >>>>>>>>> tweak the min/max in sysfs. Presumably in those cases user wants control,
> >>>>>>>>> for some reason, so as long as it is safe and within the limits why not give
> >>>>>>>>> it to them.
> >>>>>>>>>
> >>>>>>>>> Same argument from a different angle - what is the use case for allowing
> >>>>>>>>> min/max control which does not really do what it says on the tin?
> >>>>>>>>>
> >>>>>>>>> Old thought is this - if SLPC refuses to be controlled using our existing
> >>>>>>>>> controls then we could just say -ENODEV on attempts to modify them. Whether
> >>>>>>>>> for any input value, or some, depending on what SLPC can support (or wants
> >>>>>>>>> to support).
> >>>>>>>> The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
> >>>>>>>> has sudo and all the rights to disrespect what drivers (including SLPC as a
> >>>>>>>> a driver from PCODE perspective) are requesting.
> >>>>>>>>
> >>>>>>>>> Important to note I am primarily focusing on the min freq here, which AFAIR
> >>>>>>>>> we concluded can behave that user writes in 300, but SLPC decides it will
> >>>>>>>>> only go as low as 350.
> >>>>>>>> The other caveat here with the efficient freq is that it is dynamic and PCODE
> >>>>>>>> decides it with runtime conditions. It may start with 300, then move to 350,
> >>>>>>>> then back to 300. i915 is only stashing the initial value.
> >>>>>>>>
> >>>>>>>> We could potentially update our rpe everytime any freq control is
> >>>>>>>> touched.
> >>>>>>> We thought of doing this but two points:
> >>>>>>>
> >>>>>>> 1. i915 now reads the same register as GuC for rpe but I never see the
> >>>>>>>       value in this register change.
> >>>>>>> 2. Even if we could know the real rpe, the freq's might be set before a
> >>>>>>>       workload start running and rpe might change after the workload starts
> >>>>>>>       running. So knowing rpe before the workload might not be very useful.
> >>>>>>>
> >>>>>>> I am of the opposite opinion that anyway the previous kernel ABI is
> >>>>>>> probably now irrecoverably bust, so we should give FW free reign and stop
> >>>>>>> disabling efficient freq the way we do now when min_freq < rpe_freq.
> >>>>>> We have to answer the question of why we need to keep exposing the sysfs
> >>>>>> controls if they don't/can't/won't do what they say on the tin. Could we
> >>>>>> make a break with DG2 for instance? Some use cases might suffer but it's
> >>>>>> probably better than having them think it does what they think it does, or
> >>>>>> file bugs when it does not.
> >>>>> That's a good question. But if we have no control at all people will still
> >>>>> file requests for something our hardware and low level firmwares cannot promise.
> >>>>>
> >>>>> There are many folks who run important/critical experiments using these
> >>>>> interfaces at the same time they are monitoring throttle conditions and all.
> >>>> Aren't the two paragraphs a bit in contradiction? Important and critical
> >>>> experiments using controls which are not doing what they think they are.
> >>> Well, not really.
> >>>
> >>> Most of the times it will work. We just don't have a warranty.
> >> I think it would help for the user to know when it will work and when it
> >> will not.
> > Yeap. But mostly this lays on improving our throttle reasons infra.
> >
> > I'm also wondering now if upon freq reads if we also couldn't internally
> > check the throttle reason and put a debug message?!
> >
> >>> We have some indications when this were disrespected like the
> >>> throttle_reasons.
> >> There is no indication related to rpe_freq in throttle_reasons, isn't it?
> > no.
> >
> >>> Those limited times doesn't exclude the importance of the usage.
> >>>
> >>>> I don't claim I know for what all use cases do they get used. I *think*
> >>>> there was one in the media transcoding world one where people wanted truly
> >>>> fixed frequency so they can compare like-for-like between something.
> >>>>
> >>>> So bah.. but okay for the perf_pmu changes, how about instead of this:
> >>>>
> >>>> rpN = read RPn from sysfs
> >>>> set_min_freq(rpN)
> >>>> sleep
> >>>> real_min = read actual from sysfs
> >>>> actual = sample pmu for t
> >>>> assert real_min is actual
> >>>>
> >>>> We do this:
> >>>>
> >>>> rpN = read RPn from sysfs
> >>>> set_min_freq(rpN)
> >>>> sleep
> >>>> fw_min = read actual from sysfs
> >>>> set_min_freq(fw_min)
> >>>> sleep
> >>>> actual = sample pmu for t
> >>>> assert fw_min is actual
> >>>>
> >>>> That's a tiny difference which both "documents" and verifies that even if we
> >>>> cannot set the min to RPn, firmware will somewhat consistently (stable)
> >>>> respect the minimum it prefers.
> >> Note it's not the min but the max freq which is not honored.
> >>
> >>>> Is that somewhat acceptable?
> >>> yeap, this sounds a better approach for a sane test.
> >> We talked with the GuC guys and learnt a few things:
> >>
> >> * PCODE continuously calculates and updates the current RPe (efficient
> >>    freq) value
> >> * GuC samples the RPe values from PCODE every 1 ms and uses these to
> >>    request operating freq. So the requested freq is always set by GuC but
> >>    can be overridden say in cases of thermal throttling etc.
> >> * GuC ignores the max freq and directly uses RPe freq when requesting an
> >>    operating freq. This is beacuse RPe is supposedly more efficient that a
> >>    lower max freq.
> >> * GuC does have a separate control to ignore RPe entirely.
> >> * Regarding the delays I added in the patch, some time back GuC FW was
> >>    changed to move setting min/max freq's to the top half. As a result the
> >>    delays are no longer required.
> >>
> >> The most recent version the patches in light of above points are here:
> >>
> >> https://patchwork.freedesktop.org/series/111282/#rev3
> >>
> >> (The patches are similar to Tvrtko's first (rather than the second)
> >> pseudo-code above).
> >>
> >> These patches improve things but even with these patches unfortunately
> >> there are still sporadic failures. For example:
> >>
> >> Original failure:
> >> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8303/bat-dg2-11/igt@perf_pmu@frequency.html
> >> * RPn 300 MHz, PMU measures 397 MHz
> >>
> >> New failure:
> >> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8304/bat-dg2-11/igt@perf_pmu@frequency.html
> >> * PMU measures 371 MHz, requested freq measured after PMU measurement 400 MHz
> >>
> >> So these sporadic failures seems to be due to dynamically changing RPe.
> > Yes, they are.
> >
> >> It may be possible to get rid of the failures by either increasing the
> >> tolerance or sampling the requested freq sysfs more often (rather than just
> >> once after the PMU measurement) though maybe it's too much.
> > On Xe driver I workarounded it by 'constantly' updating our cached rpe.
> > By constantly I mean whenever anyone is touching (reading or writing) any
> > frequency.
> >
> > However I'm aware that this can be racy. Specially for the PMU cases or
> > with some out of band checks.

Consider 2 cases:

1. We start a spinner load and then update min/max freq
2. We update min/max freq first and then start spinner load

Doing the above, we likely pick up the correct (RPe) freq being used for
case 1 but not for case 2. And even for case 1 RPe might be dynamically
changing.

> >> So the best way to fix such sporadic failures seems to be to expose a
> >> 'rpe_enable' sysfs to turn off RPe for such tests/usages where strict
> >> userspace control of freq limits is required.
> >>
> >> @Rodrigo Vivi: so we need to see if we can make this uapi change and then
> >> go from there I think.
> > I believe this is the most sane option right now. We need to document this
> > well and have test cases for both.
> Either that, or we turn off efficient frequency when user updates max
> frequency as well (similar to what we do for min).

I think I disagree here, turning off RPe will affect PnP. I think we should
have explicit disable of RPe when that is needed and should actually remove
RPe disable when min is updated. Basically IMO the idea should be to
provide the best PnP and then tailor the ABI for the best PnP.

> >
> > Probably this plus the constant update of the cached RPe.
>
> Caching might be easier if GuC does the same. It can update the task state
> data with the new value for everyone else to read.

i915 will either need to read the register or read the task state from
GuC. You mean reading the task state is more efficient than reading the
register?

Thanks.
--
Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2023-01-06 20:12                         ` Rodrigo Vivi
@ 2023-01-06 20:39                           ` Belgaumkar, Vinay
  2023-01-06 21:38                             ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Belgaumkar, Vinay @ 2023-01-06 20:39 UTC (permalink / raw)
  To: Rodrigo Vivi, Dixit, Ashutosh; +Cc: igt-dev


On 1/6/2023 12:12 PM, Rodrigo Vivi wrote:
> On Thu, Jan 05, 2023 at 01:26:34PM -0800, Dixit, Ashutosh wrote:
>> On Fri, 23 Dec 2022 07:39:29 -0800, Rodrigo Vivi wrote:
>>> On Fri, Dec 23, 2022 at 09:22:53AM +0000, Tvrtko Ursulin wrote:
>>>> On 22/12/2022 20:28, Rodrigo Vivi wrote:
>>>>> On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
>>>>>> On 17/12/2022 02:49, Dixit, Ashutosh wrote:
>>>>>>> On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
>>>>>>>> On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
>>>>>>>>> On 16/12/2022 06:21, Dixit, Ashutosh wrote:
>>>>>>>>>> On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
>>>>>>>>>>> On 23/11/2022 06:03, Dixit, Ashutosh wrote:
>>>>>>>>>>>> On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
>>>>>>>>>> Hi Tvrtko,
>>>>>>>>>>
>>>>>>>>>> Sorry for the delay in replying on this. We are discussing this but still
>>>>>>>>>> no conclusion. I have replied what I can below, will update again after I
>>>>>>>>>> know more. Wanted to send a reply before disappearing for the
>>>>>>>>>> holidays. Please see below.
>>>>>>>>>>
>>>>>>>>>>>> Hi Tvrtko,
>>>>>>>>>>>>
>>>>>>>>>>>> I am only offering an overall clarification below first, not answering all
>>>>>>>>>>>> of your points for now.
>>>>>>>>>>>>
>>>>>>>>>>>>> On 19/11/2022 02:00, Ashutosh Dixit wrote:
>>>>>>>>>>>>>> With SLPC, even when we set the same min and max freq's, the requested and
>>>>>>>>>>>>>> actual freq's can differ from the min/max freq set. For example "efficient
>>>>>>>>>>>>>> freq" (when in effect) can override set min/max freq. In general FW is the
>>>>>>>>>>>>>> final arbiter in determining freq and can override set values.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
>>>>>>>>>>>>>> reported by PMU not against the set freq's but against the requested freq
>>>>>>>>>>>>>> reported in sysfs. Also add a delay after setting freq's to account for
>>>>>>>>>>>>>> messaging delays in setting freq's in GuC.
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> v2: Introduce a 100 ms delay after setting freq
>>>>>>>>>>>>>> v3: Update commit message, code identical to v2
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
>>>>>>>>>>>>>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
>>>>>>>>>>>>>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
>>>>>>>>>>>>>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
>>>>>>>>>>>>>> ---
>>>>>>>>>>>>>>        tests/i915/perf_pmu.c | 23 +++++++++++++++--------
>>>>>>>>>>>>>>        1 file changed, 15 insertions(+), 8 deletions(-)
>>>>>>>>>>>>>>
>>>>>>>>>>>>>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
>>>>>>>>>>>>>> index f363db2ba13..02f6ae989b1 100644
>>>>>>>>>>>>>> --- a/tests/i915/perf_pmu.c
>>>>>>>>>>>>>> +++ b/tests/i915/perf_pmu.c
>>>>>>>>>>>>>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
>>>>>>>>>>>>>> 		igt_assert_lte(target, busy);
>>>>>>>>>>>>>>        }
>>>>>>>>>>>>>>        +/* Wait for GuC SLPC freq changes to take effect */
>>>>>>>>>>>>>> +#define wait_freq_set()		usleep(100000)
>>>>>>>>>>>>> A future task of maybe adding a drop_caches flag to wait for this?
>>>>>>>>>>>>>
>>>>>>>>>>>>> Alternatively, if we have a sysfs or debugfs file which reflects the value
>>>>>>>>>>>>> once GuC has processed it, keep reading it until it is updated? Even if to
>>>>>>>>>>>>> something other than it was before the write, to handle the inability to
>>>>>>>>>>>>> set the exact requested frequency sometimes?
>>>>>>>>>>>>>> +
>>>>>>>>>>>>>>        static void
>>>>>>>>>>>>>>        test_frequency(int gem_fd)
>>>>>>>>>>>>>>        {
>>>>>>>>>>>>>> -	uint32_t min_freq, max_freq, boost_freq;
>>>>>>>>>>>>>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
>>>>>>>>>>>>>> 		uint64_t val[2], start[2], slept;
>>>>>>>>>>>>>> 		double min[2], max[2];
>>>>>>>>>>>>>> 		igt_spin_t *spin;
>>>>>>>>>>>>>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
>>>>>>>>>>>>>> 		 * Set GPU to min frequency and read PMU counters.
>>>>>>>>>>>>>> 		 */
>>>>>>>>>>>>>> 		igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
>>>>>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>>>>>>>>>>>>> 		igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
>>>>>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>>>>>>>>>>>>> 		igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
>>>>>>>>>>>>>> +	wait_freq_set();
>>>>>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>>>>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>>>>>>>>>>>>> 		igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
>>>>>>>>>>>>>> 			gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
>>>>>>>>>>>>>> effect */
>>>>>>>>>>>>>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
>>>>>>>>>>>>>> 			min[0] = 1e9*(val[0] - start[0]) / slept;
>>>>>>>>>>>>>> 		min[1] = 1e9*(val[1] - start[1]) / slept;
>>>>>>>>>>>>>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>>>>>>>>>>>>> This really leaves a bad taste for me still, sorry. First of all, it may
>>>>>>>>>>>>> make it more palatable if readback was immediately after wait_freq_set(),
>>>>>>>>>>>>> or as part of it. Otherwise it seems like even more confused test and sysfs
>>>>>>>>>>>>> ABI.
>>>>>>>>>>>>>
>>>>>>>>>>>>> Did we close on my question of whether we can make readback of
>>>>>>>>>>>>> gt_max_freq_mhz actually represent the real max? Correct me please if I got
>>>>>>>>>>>>> confused - with min freq checking here we write max=300 and get 350? If so
>>>>>>>>>>>>> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
>>>>>>>>>>>>> error?
>>>>>>>>>>>> Did you read Vinay's email on v1 here:
>>>>>>>>>>>>
>>>>>>>>>>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
>>>>>>>>>>>>
>>>>>>>>>>>> As Vinay says, SLPC is requesting an entirely different freq (the effecient
>>>>>>>>>>>> freq rather than the set 'min == max freq') and no amount of waiting will
>>>>>>>>>>>> result in the requested freq being the same as the set 'min == max
>>>>>>>>>>>> freq'. Also the efficient freq cannot be controlled from user space.
>>>>>>>>>>> Why is that not a break in our ABI? What is the point in allowing users to
>>>>>>>>>>> write into this field if the desired value can be silently ignored?
>>>>>>>>>> Yes, this is in discussion. FW is doing this and we need to see if FW can
>>>>>>>>>> be made to obey the i915 ABI. IMHO what matters is performance per watt and
>>>>>>>>>> if the ABI has to be tossed so be it. I will keep this list posted about
>>>>>>>>>> what happens here. Merging any patches for these issues (GL #6806 and
>>>>>>>>>> #6786) is blocked till the ABI breakage issue is resolved one way or
>>>>>>>>>> another.
>>>>>>>>> Two thoughts here, one has been mentioned before.
>>>>>>>>>
>>>>>>>>> New one is that performance per watt perhaps does not matter (is not a
>>>>>>>>> primary concern at least) for use cases where "users" (for some definition)
>>>>>>>>> tweak the min/max in sysfs. Presumably in those cases user wants control,
>>>>>>>>> for some reason, so as long as it is safe and within the limits why not give
>>>>>>>>> it to them.
>>>>>>>>>
>>>>>>>>> Same argument from a different angle - what is the use case for allowing
>>>>>>>>> min/max control which does not really do what it says on the tin?
>>>>>>>>>
>>>>>>>>> Old thought is this - if SLPC refuses to be controlled using our existing
>>>>>>>>> controls then we could just say -ENODEV on attempts to modify them. Whether
>>>>>>>>> for any input value, or some, depending on what SLPC can support (or wants
>>>>>>>>> to support).
>>>>>>>> The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
>>>>>>>> has sudo and all the rights to disrespect what drivers (including SLPC as a
>>>>>>>> a driver from PCODE perspective) are requesting.
>>>>>>>>
>>>>>>>>> Important to note I am primarily focusing on the min freq here, which AFAIR
>>>>>>>>> we concluded can behave that user writes in 300, but SLPC decides it will
>>>>>>>>> only go as low as 350.
>>>>>>>> The other caveat here with the efficient freq is that it is dynamic and PCODE
>>>>>>>> decides it with runtime conditions. It may start with 300, then move to 350,
>>>>>>>> then back to 300. i915 is only stashing the initial value.
>>>>>>>>
>>>>>>>> We could potentially update our rpe everytime any freq control is
>>>>>>>> touched.
>>>>>>> We thought of doing this but two points:
>>>>>>>
>>>>>>> 1. i915 now reads the same register as GuC for rpe but I never see the
>>>>>>>       value in this register change.
>>>>>>> 2. Even if we could know the real rpe, the freq's might be set before a
>>>>>>>       workload start running and rpe might change after the workload starts
>>>>>>>       running. So knowing rpe before the workload might not be very useful.
>>>>>>>
>>>>>>> I am of the opposite opinion that anyway the previous kernel ABI is
>>>>>>> probably now irrecoverably bust, so we should give FW free reign and stop
>>>>>>> disabling efficient freq the way we do now when min_freq < rpe_freq.
>>>>>> We have to answer the question of why we need to keep exposing the sysfs
>>>>>> controls if they don't/can't/won't do what they say on the tin. Could we
>>>>>> make a break with DG2 for instance? Some use cases might suffer but it's
>>>>>> probably better than having them think it does what they think it does, or
>>>>>> file bugs when it does not.
>>>>> That's a good question. But if we have no control at all people will still
>>>>> file requests for something our hardware and low level firmwares cannot promise.
>>>>>
>>>>> There are many folks who run important/critical experiments using these
>>>>> interfaces at the same time they are monitoring throttle conditions and all.
>>>> Aren't the two paragraphs a bit in contradiction? Important and critical
>>>> experiments using controls which are not doing what they think they are.
>>> Well, not really.
>>>
>>> Most of the times it will work. We just don't have a warranty.
>> I think it would help for the user to know when it will work and when it
>> will not.
> Yeap. But mostly this lays on improving our throttle reasons infra.
>
> I'm also wondering now if upon freq reads if we also couldn't internally
> check the throttle reason and put a debug message?!
>
>>> We have some indications when this were disrespected like the
>>> throttle_reasons.
>> There is no indication related to rpe_freq in throttle_reasons, isn't it?
> no.
>
>>> Those limited times doesn't exclude the importance of the usage.
>>>
>>>> I don't claim I know for what all use cases do they get used. I *think*
>>>> there was one in the media transcoding world one where people wanted truly
>>>> fixed frequency so they can compare like-for-like between something.
>>>>
>>>> So bah.. but okay for the perf_pmu changes, how about instead of this:
>>>>
>>>> rpN = read RPn from sysfs
>>>> set_min_freq(rpN)
>>>> sleep
>>>> real_min = read actual from sysfs
>>>> actual = sample pmu for t
>>>> assert real_min is actual
>>>>
>>>> We do this:
>>>>
>>>> rpN = read RPn from sysfs
>>>> set_min_freq(rpN)
>>>> sleep
>>>> fw_min = read actual from sysfs
>>>> set_min_freq(fw_min)
>>>> sleep
>>>> actual = sample pmu for t
>>>> assert fw_min is actual
>>>>
>>>> That's a tiny difference which both "documents" and verifies that even if we
>>>> cannot set the min to RPn, firmware will somewhat consistently (stable)
>>>> respect the minimum it prefers.
>> Note it's not the min but the max freq which is not honored.
>>
>>>> Is that somewhat acceptable?
>>> yeap, this sounds a better approach for a sane test.
>> We talked with the GuC guys and learnt a few things:
>>
>> * PCODE continuously calculates and updates the current RPe (efficient
>>    freq) value
>> * GuC samples the RPe values from PCODE every 1 ms and uses these to
>>    request operating freq. So the requested freq is always set by GuC but
>>    can be overridden say in cases of thermal throttling etc.
>> * GuC ignores the max freq and directly uses RPe freq when requesting an
>>    operating freq. This is beacuse RPe is supposedly more efficient that a
>>    lower max freq.
>> * GuC does have a separate control to ignore RPe entirely.
>> * Regarding the delays I added in the patch, some time back GuC FW was
>>    changed to move setting min/max freq's to the top half. As a result the
>>    delays are no longer required.
>>
>> The most recent version the patches in light of above points are here:
>>
>> https://patchwork.freedesktop.org/series/111282/#rev3
>>
>> (The patches are similar to Tvrtko's first (rather than the second)
>> pseudo-code above).
>>
>> These patches improve things but even with these patches unfortunately
>> there are still sporadic failures. For example:
>>
>> Original failure:
>> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8303/bat-dg2-11/igt@perf_pmu@frequency.html
>> * RPn 300 MHz, PMU measures 397 MHz
>>
>> New failure:
>> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8304/bat-dg2-11/igt@perf_pmu@frequency.html
>> * PMU measures 371 MHz, requested freq measured after PMU measurement 400 MHz
>>
>> So these sporadic failures seems to be due to dynamically changing RPe.
> Yes, they are.
>
>> It may be possible to get rid of the failures by either increasing the
>> tolerance or sampling the requested freq sysfs more often (rather than just
>> once after the PMU measurement) though maybe it's too much.
> On Xe driver I workarounded it by 'constantly' updating our cached rpe.
> By constantly I mean whenever anyone is touching (reading or writing) any
> frequency.
>
> However I'm aware that this can be racy. Specially for the PMU cases or
> with some out of band checks.
>
>> So the best way to fix such sporadic failures seems to be to expose a
>> 'rpe_enable' sysfs to turn off RPe for such tests/usages where strict
>> userspace control of freq limits is required.
>>
>> @Rodrigo Vivi: so we need to see if we can make this uapi change and then
>> go from there I think.
> I believe this is the most sane option right now. We need to document this
> well and have test cases for both.
Either that, or we turn off efficient frequency when user updates max 
frequency as well (similar to what we do for min).
>
> Probably this plus the constant update of the cached RPe.

Caching might be easier if GuC does the same. It can update the task 
state data with the new value for everyone else to read.

Thanks,

Vinay.

>
>> Thanks.
>> --
>> Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2023-01-05 21:26                       ` Dixit, Ashutosh
@ 2023-01-06 20:12                         ` Rodrigo Vivi
  2023-01-06 20:39                           ` Belgaumkar, Vinay
  0 siblings, 1 reply; 34+ messages in thread
From: Rodrigo Vivi @ 2023-01-06 20:12 UTC (permalink / raw)
  To: Dixit, Ashutosh; +Cc: igt-dev

On Thu, Jan 05, 2023 at 01:26:34PM -0800, Dixit, Ashutosh wrote:
> On Fri, 23 Dec 2022 07:39:29 -0800, Rodrigo Vivi wrote:
> >
> > On Fri, Dec 23, 2022 at 09:22:53AM +0000, Tvrtko Ursulin wrote:
> > >
> > > On 22/12/2022 20:28, Rodrigo Vivi wrote:
> > > > On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
> > > > >
> > > > > On 17/12/2022 02:49, Dixit, Ashutosh wrote:
> > > > > > On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
> > > > > > > On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> > > > > > > >
> > > > > > > > On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> > > > > > > > > On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> > > > > > > > > >
> > > > > > > > > > On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > > > > > > > > > > On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> > > > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > >
> > > > > > > > > Hi Tvrtko,
> > > > > > > > >
> > > > > > > > > Sorry for the delay in replying on this. We are discussing this but still
> > > > > > > > > no conclusion. I have replied what I can below, will update again after I
> > > > > > > > > know more. Wanted to send a reply before disappearing for the
> > > > > > > > > holidays. Please see below.
> > > > > > > > >
> > > > > > > > > > > Hi Tvrtko,
> > > > > > > > > > >
> > > > > > > > > > > I am only offering an overall clarification below first, not answering all
> > > > > > > > > > > of your points for now.
> > > > > > > > > > >
> > > > > > > > > > > >
> > > > > > > > > > > > On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > > > > > > > > > > > > With SLPC, even when we set the same min and max freq's, the requested and
> > > > > > > > > > > > > actual freq's can differ from the min/max freq set. For example "efficient
> > > > > > > > > > > > > freq" (when in effect) can override set min/max freq. In general FW is the
> > > > > > > > > > > > > final arbiter in determining freq and can override set values.
> > > > > > > > > > > > >
> > > > > > > > > > > > > Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > > > > > > > > > > > > reported by PMU not against the set freq's but against the requested freq
> > > > > > > > > > > > > reported in sysfs. Also add a delay after setting freq's to account for
> > > > > > > > > > > > > messaging delays in setting freq's in GuC.
> > > > > > > > > > > > >
> > > > > > > > > > > > > v2: Introduce a 100 ms delay after setting freq
> > > > > > > > > > > > > v3: Update commit message, code identical to v2
> > > > > > > > > > > > >
> > > > > > > > > > > > > Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > > > > > > > > > > > > Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > > > > > > > > > > > > Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > > > > > > > > > > > > Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > > > > > > > > > > > > ---
> > > > > > > > > > > > >       tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> > > > > > > > > > > > >       1 file changed, 15 insertions(+), 8 deletions(-)
> > > > > > > > > > > > >
> > > > > > > > > > > > > diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > > > > > > > > > > > > index f363db2ba13..02f6ae989b1 100644
> > > > > > > > > > > > > --- a/tests/i915/perf_pmu.c
> > > > > > > > > > > > > +++ b/tests/i915/perf_pmu.c
> > > > > > > > > > > > > @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> > > > > > > > > > > > >		igt_assert_lte(target, busy);
> > > > > > > > > > > > >       }
> > > > > > > > > > > > >       +/* Wait for GuC SLPC freq changes to take effect */
> > > > > > > > > > > > > +#define wait_freq_set()		usleep(100000)
> > > > > > > > > > > >
> > > > > > > > > > > > A future task of maybe adding a drop_caches flag to wait for this?
> > > > > > > > > > > >
> > > > > > > > > > > > Alternatively, if we have a sysfs or debugfs file which reflects the value
> > > > > > > > > > > > once GuC has processed it, keep reading it until it is updated? Even if to
> > > > > > > > > > > > something other than it was before the write, to handle the inability to
> > > > > > > > > > > > set the exact requested frequency sometimes?
> > > > > > > > > > > > >
> > > > > > > > > > > > > +
> > > > > > > > > > > > >       static void
> > > > > > > > > > > > >       test_frequency(int gem_fd)
> > > > > > > > > > > > >       {
> > > > > > > > > > > > > -	uint32_t min_freq, max_freq, boost_freq;
> > > > > > > > > > > > > +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> > > > > > > > > > > > >		uint64_t val[2], start[2], slept;
> > > > > > > > > > > > >		double min[2], max[2];
> > > > > > > > > > > > >		igt_spin_t *spin;
> > > > > > > > > > > > > @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> > > > > > > > > > > > >		 * Set GPU to min frequency and read PMU counters.
> > > > > > > > > > > > >		 */
> > > > > > > > > > > > >		igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > > > > > > > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > > > > > > >		igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > > > > > > > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > > > > > > > >		igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > > > > > > > > > > > > +	wait_freq_set();
> > > > > > > > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > > > > > > > >		igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> > > > > > > > > > > > >			gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > > > > > > > > > > > > effect */
> > > > > > > > > > > > > @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> > > > > > > > > > > > >			min[0] = 1e9*(val[0] - start[0]) / slept;
> > > > > > > > > > > > >		min[1] = 1e9*(val[1] - start[1]) / slept;
> > > > > > > > > > > > > +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> > > > > > > > > > > >
> > > > > > > > > > > > This really leaves a bad taste for me still, sorry. First of all, it may
> > > > > > > > > > > > make it more palatable if readback was immediately after wait_freq_set(),
> > > > > > > > > > > > or as part of it. Otherwise it seems like even more confused test and sysfs
> > > > > > > > > > > > ABI.
> > > > > > > > > > > >
> > > > > > > > > > > > Did we close on my question of whether we can make readback of
> > > > > > > > > > > > gt_max_freq_mhz actually represent the real max? Correct me please if I got
> > > > > > > > > > > > confused - with min freq checking here we write max=300 and get 350? If so
> > > > > > > > > > > > can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> > > > > > > > > > > > error?
> > > > > > > > > > >
> > > > > > > > > > > Did you read Vinay's email on v1 here:
> > > > > > > > > > >
> > > > > > > > > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> > > > > > > > > > >
> > > > > > > > > > > As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > > > > > > > > > > freq rather than the set 'min == max freq') and no amount of waiting will
> > > > > > > > > > > result in the requested freq being the same as the set 'min == max
> > > > > > > > > > > freq'. Also the efficient freq cannot be controlled from user space.
> > > > > > > > > >
> > > > > > > > > > Why is that not a break in our ABI? What is the point in allowing users to
> > > > > > > > > > write into this field if the desired value can be silently ignored?
> > > > > > > > >
> > > > > > > > > Yes, this is in discussion. FW is doing this and we need to see if FW can
> > > > > > > > > be made to obey the i915 ABI. IMHO what matters is performance per watt and
> > > > > > > > > if the ABI has to be tossed so be it. I will keep this list posted about
> > > > > > > > > what happens here. Merging any patches for these issues (GL #6806 and
> > > > > > > > > #6786) is blocked till the ABI breakage issue is resolved one way or
> > > > > > > > > another.
> > > > > > > >
> > > > > > > > Two thoughts here, one has been mentioned before.
> > > > > > > >
> > > > > > > > New one is that performance per watt perhaps does not matter (is not a
> > > > > > > > primary concern at least) for use cases where "users" (for some definition)
> > > > > > > > tweak the min/max in sysfs. Presumably in those cases user wants control,
> > > > > > > > for some reason, so as long as it is safe and within the limits why not give
> > > > > > > > it to them.
> > > > > > > >
> > > > > > > > Same argument from a different angle - what is the use case for allowing
> > > > > > > > min/max control which does not really do what it says on the tin?
> > > > > > > >
> > > > > > > > Old thought is this - if SLPC refuses to be controlled using our existing
> > > > > > > > controls then we could just say -ENODEV on attempts to modify them. Whether
> > > > > > > > for any input value, or some, depending on what SLPC can support (or wants
> > > > > > > > to support).
> > > > > > >
> > > > > > > The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
> > > > > > > has sudo and all the rights to disrespect what drivers (including SLPC as a
> > > > > > > a driver from PCODE perspective) are requesting.
> > > > > > >
> > > > > > > >
> > > > > > > > Important to note I am primarily focusing on the min freq here, which AFAIR
> > > > > > > > we concluded can behave that user writes in 300, but SLPC decides it will
> > > > > > > > only go as low as 350.
> > > > > > >
> > > > > > > The other caveat here with the efficient freq is that it is dynamic and PCODE
> > > > > > > decides it with runtime conditions. It may start with 300, then move to 350,
> > > > > > > then back to 300. i915 is only stashing the initial value.
> > > > > > >
> > > > > > > We could potentially update our rpe everytime any freq control is
> > > > > > > touched.
> > > > > >
> > > > > > We thought of doing this but two points:
> > > > > >
> > > > > > 1. i915 now reads the same register as GuC for rpe but I never see the
> > > > > >      value in this register change.
> > > > > > 2. Even if we could know the real rpe, the freq's might be set before a
> > > > > >      workload start running and rpe might change after the workload starts
> > > > > >      running. So knowing rpe before the workload might not be very useful.
> > > > > >
> > > > > > I am of the opposite opinion that anyway the previous kernel ABI is
> > > > > > probably now irrecoverably bust, so we should give FW free reign and stop
> > > > > > disabling efficient freq the way we do now when min_freq < rpe_freq.
> > > > >
> > > > > We have to answer the question of why we need to keep exposing the sysfs
> > > > > controls if they don't/can't/won't do what they say on the tin. Could we
> > > > > make a break with DG2 for instance? Some use cases might suffer but it's
> > > > > probably better than having them think it does what they think it does, or
> > > > > file bugs when it does not.
> > > >
> > > > That's a good question. But if we have no control at all people will still
> > > > file requests for something our hardware and low level firmwares cannot promise.
> > > >
> > > > There are many folks who run important/critical experiments using these
> > > > interfaces at the same time they are monitoring throttle conditions and all.
> > >
> > > Aren't the two paragraphs a bit in contradiction? Important and critical
> > > experiments using controls which are not doing what they think they are.
> >
> > Well, not really.
> >
> > Most of the times it will work. We just don't have a warranty.
> 
> I think it would help for the user to know when it will work and when it
> will not.

Yeap. But mostly this lays on improving our throttle reasons infra.

I'm also wondering now if upon freq reads if we also couldn't internally
check the throttle reason and put a debug message?!

> 
> > We have some indications when this were disrespected like the
> > throttle_reasons.
> 
> There is no indication related to rpe_freq in throttle_reasons, isn't it?

no.

> 
> > Those limited times doesn't exclude the importance of the usage.
> >
> > >
> > > I don't claim I know for what all use cases do they get used. I *think*
> > > there was one in the media transcoding world one where people wanted truly
> > > fixed frequency so they can compare like-for-like between something.
> > >
> > > So bah.. but okay for the perf_pmu changes, how about instead of this:
> > >
> > > rpN = read RPn from sysfs
> > > set_min_freq(rpN)
> > > sleep
> > > real_min = read actual from sysfs
> > > actual = sample pmu for t
> > > assert real_min is actual
> > >
> > > We do this:
> > >
> > > rpN = read RPn from sysfs
> > > set_min_freq(rpN)
> > > sleep
> > > fw_min = read actual from sysfs
> > > set_min_freq(fw_min)
> > > sleep
> > > actual = sample pmu for t
> > > assert fw_min is actual
> > >
> > > That's a tiny difference which both "documents" and verifies that even if we
> > > cannot set the min to RPn, firmware will somewhat consistently (stable)
> > > respect the minimum it prefers.
> 
> Note it's not the min but the max freq which is not honored.
> 
> > >
> > > Is that somewhat acceptable?
> >
> > yeap, this sounds a better approach for a sane test.
> 
> We talked with the GuC guys and learnt a few things:
> 
> * PCODE continuously calculates and updates the current RPe (efficient
>   freq) value
> * GuC samples the RPe values from PCODE every 1 ms and uses these to
>   request operating freq. So the requested freq is always set by GuC but
>   can be overridden say in cases of thermal throttling etc.
> * GuC ignores the max freq and directly uses RPe freq when requesting an
>   operating freq. This is beacuse RPe is supposedly more efficient that a
>   lower max freq.
> * GuC does have a separate control to ignore RPe entirely.
> * Regarding the delays I added in the patch, some time back GuC FW was
>   changed to move setting min/max freq's to the top half. As a result the
>   delays are no longer required.
> 
> The most recent version the patches in light of above points are here:
> 
> https://patchwork.freedesktop.org/series/111282/#rev3
> 
> (The patches are similar to Tvrtko's first (rather than the second)
> pseudo-code above).
> 
> These patches improve things but even with these patches unfortunately
> there are still sporadic failures. For example:
> 
> Original failure:
> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8303/bat-dg2-11/igt@perf_pmu@frequency.html
> * RPn 300 MHz, PMU measures 397 MHz
> 
> New failure:
> * https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8304/bat-dg2-11/igt@perf_pmu@frequency.html
> * PMU measures 371 MHz, requested freq measured after PMU measurement 400 MHz
> 
> So these sporadic failures seems to be due to dynamically changing RPe.

Yes, they are.

> 
> It may be possible to get rid of the failures by either increasing the
> tolerance or sampling the requested freq sysfs more often (rather than just
> once after the PMU measurement) though maybe it's too much.

On Xe driver I workarounded it by 'constantly' updating our cached rpe.
By constantly I mean whenever anyone is touching (reading or writing) any
frequency.

However I'm aware that this can be racy. Specially for the PMU cases or
with some out of band checks.

> 
> So the best way to fix such sporadic failures seems to be to expose a
> 'rpe_enable' sysfs to turn off RPe for such tests/usages where strict
> userspace control of freq limits is required.
> 
> @Rodrigo Vivi: so we need to see if we can make this uapi change and then
> go from there I think.

I believe this is the most sane option right now. We need to document this
well and have test cases for both.

Probably this plus the constant update of the cached RPe.

> 
> Thanks.
> --
> Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-12-23 15:39                     ` Rodrigo Vivi
@ 2023-01-05 21:26                       ` Dixit, Ashutosh
  2023-01-06 20:12                         ` Rodrigo Vivi
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2023-01-05 21:26 UTC (permalink / raw)
  To: Rodrigo Vivi, Tvrtko Ursulin; +Cc: igt-dev

On Fri, 23 Dec 2022 07:39:29 -0800, Rodrigo Vivi wrote:
>
> On Fri, Dec 23, 2022 at 09:22:53AM +0000, Tvrtko Ursulin wrote:
> >
> > On 22/12/2022 20:28, Rodrigo Vivi wrote:
> > > On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
> > > >
> > > > On 17/12/2022 02:49, Dixit, Ashutosh wrote:
> > > > > On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
> > > > > > On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> > > > > > >
> > > > > > > On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> > > > > > > > On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> > > > > > > > >
> > > > > > > > > On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > > > > > > > > > On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> > > > > > > > > > >
> > > > > > > > > >
> > > > > > > >
> > > > > > > > Hi Tvrtko,
> > > > > > > >
> > > > > > > > Sorry for the delay in replying on this. We are discussing this but still
> > > > > > > > no conclusion. I have replied what I can below, will update again after I
> > > > > > > > know more. Wanted to send a reply before disappearing for the
> > > > > > > > holidays. Please see below.
> > > > > > > >
> > > > > > > > > > Hi Tvrtko,
> > > > > > > > > >
> > > > > > > > > > I am only offering an overall clarification below first, not answering all
> > > > > > > > > > of your points for now.
> > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > > On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > > > > > > > > > > > With SLPC, even when we set the same min and max freq's, the requested and
> > > > > > > > > > > > actual freq's can differ from the min/max freq set. For example "efficient
> > > > > > > > > > > > freq" (when in effect) can override set min/max freq. In general FW is the
> > > > > > > > > > > > final arbiter in determining freq and can override set values.
> > > > > > > > > > > >
> > > > > > > > > > > > Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > > > > > > > > > > > reported by PMU not against the set freq's but against the requested freq
> > > > > > > > > > > > reported in sysfs. Also add a delay after setting freq's to account for
> > > > > > > > > > > > messaging delays in setting freq's in GuC.
> > > > > > > > > > > >
> > > > > > > > > > > > v2: Introduce a 100 ms delay after setting freq
> > > > > > > > > > > > v3: Update commit message, code identical to v2
> > > > > > > > > > > >
> > > > > > > > > > > > Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > > > > > > > > > > > Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > > > > > > > > > > > Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > > > > > > > > > > > Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > > > > > > > > > > > ---
> > > > > > > > > > > >       tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> > > > > > > > > > > >       1 file changed, 15 insertions(+), 8 deletions(-)
> > > > > > > > > > > >
> > > > > > > > > > > > diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > > > > > > > > > > > index f363db2ba13..02f6ae989b1 100644
> > > > > > > > > > > > --- a/tests/i915/perf_pmu.c
> > > > > > > > > > > > +++ b/tests/i915/perf_pmu.c
> > > > > > > > > > > > @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> > > > > > > > > > > >		igt_assert_lte(target, busy);
> > > > > > > > > > > >       }
> > > > > > > > > > > >       +/* Wait for GuC SLPC freq changes to take effect */
> > > > > > > > > > > > +#define wait_freq_set()		usleep(100000)
> > > > > > > > > > >
> > > > > > > > > > > A future task of maybe adding a drop_caches flag to wait for this?
> > > > > > > > > > >
> > > > > > > > > > > Alternatively, if we have a sysfs or debugfs file which reflects the value
> > > > > > > > > > > once GuC has processed it, keep reading it until it is updated? Even if to
> > > > > > > > > > > something other than it was before the write, to handle the inability to
> > > > > > > > > > > set the exact requested frequency sometimes?
> > > > > > > > > > > >
> > > > > > > > > > > > +
> > > > > > > > > > > >       static void
> > > > > > > > > > > >       test_frequency(int gem_fd)
> > > > > > > > > > > >       {
> > > > > > > > > > > > -	uint32_t min_freq, max_freq, boost_freq;
> > > > > > > > > > > > +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> > > > > > > > > > > >		uint64_t val[2], start[2], slept;
> > > > > > > > > > > >		double min[2], max[2];
> > > > > > > > > > > >		igt_spin_t *spin;
> > > > > > > > > > > > @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> > > > > > > > > > > >		 * Set GPU to min frequency and read PMU counters.
> > > > > > > > > > > >		 */
> > > > > > > > > > > >		igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > > > > > > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > > > > > >		igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > > > > > > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > > > > > > >		igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > > > > > > > > > > > +	wait_freq_set();
> > > > > > > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > > > > > > >		igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> > > > > > > > > > > >			gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > > > > > > > > > > > effect */
> > > > > > > > > > > > @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> > > > > > > > > > > >			min[0] = 1e9*(val[0] - start[0]) / slept;
> > > > > > > > > > > >		min[1] = 1e9*(val[1] - start[1]) / slept;
> > > > > > > > > > > > +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> > > > > > > > > > >
> > > > > > > > > > > This really leaves a bad taste for me still, sorry. First of all, it may
> > > > > > > > > > > make it more palatable if readback was immediately after wait_freq_set(),
> > > > > > > > > > > or as part of it. Otherwise it seems like even more confused test and sysfs
> > > > > > > > > > > ABI.
> > > > > > > > > > >
> > > > > > > > > > > Did we close on my question of whether we can make readback of
> > > > > > > > > > > gt_max_freq_mhz actually represent the real max? Correct me please if I got
> > > > > > > > > > > confused - with min freq checking here we write max=300 and get 350? If so
> > > > > > > > > > > can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> > > > > > > > > > > error?
> > > > > > > > > >
> > > > > > > > > > Did you read Vinay's email on v1 here:
> > > > > > > > > >
> > > > > > > > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> > > > > > > > > >
> > > > > > > > > > As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > > > > > > > > > freq rather than the set 'min == max freq') and no amount of waiting will
> > > > > > > > > > result in the requested freq being the same as the set 'min == max
> > > > > > > > > > freq'. Also the efficient freq cannot be controlled from user space.
> > > > > > > > >
> > > > > > > > > Why is that not a break in our ABI? What is the point in allowing users to
> > > > > > > > > write into this field if the desired value can be silently ignored?
> > > > > > > >
> > > > > > > > Yes, this is in discussion. FW is doing this and we need to see if FW can
> > > > > > > > be made to obey the i915 ABI. IMHO what matters is performance per watt and
> > > > > > > > if the ABI has to be tossed so be it. I will keep this list posted about
> > > > > > > > what happens here. Merging any patches for these issues (GL #6806 and
> > > > > > > > #6786) is blocked till the ABI breakage issue is resolved one way or
> > > > > > > > another.
> > > > > > >
> > > > > > > Two thoughts here, one has been mentioned before.
> > > > > > >
> > > > > > > New one is that performance per watt perhaps does not matter (is not a
> > > > > > > primary concern at least) for use cases where "users" (for some definition)
> > > > > > > tweak the min/max in sysfs. Presumably in those cases user wants control,
> > > > > > > for some reason, so as long as it is safe and within the limits why not give
> > > > > > > it to them.
> > > > > > >
> > > > > > > Same argument from a different angle - what is the use case for allowing
> > > > > > > min/max control which does not really do what it says on the tin?
> > > > > > >
> > > > > > > Old thought is this - if SLPC refuses to be controlled using our existing
> > > > > > > controls then we could just say -ENODEV on attempts to modify them. Whether
> > > > > > > for any input value, or some, depending on what SLPC can support (or wants
> > > > > > > to support).
> > > > > >
> > > > > > The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
> > > > > > has sudo and all the rights to disrespect what drivers (including SLPC as a
> > > > > > a driver from PCODE perspective) are requesting.
> > > > > >
> > > > > > >
> > > > > > > Important to note I am primarily focusing on the min freq here, which AFAIR
> > > > > > > we concluded can behave that user writes in 300, but SLPC decides it will
> > > > > > > only go as low as 350.
> > > > > >
> > > > > > The other caveat here with the efficient freq is that it is dynamic and PCODE
> > > > > > decides it with runtime conditions. It may start with 300, then move to 350,
> > > > > > then back to 300. i915 is only stashing the initial value.
> > > > > >
> > > > > > We could potentially update our rpe everytime any freq control is
> > > > > > touched.
> > > > >
> > > > > We thought of doing this but two points:
> > > > >
> > > > > 1. i915 now reads the same register as GuC for rpe but I never see the
> > > > >      value in this register change.
> > > > > 2. Even if we could know the real rpe, the freq's might be set before a
> > > > >      workload start running and rpe might change after the workload starts
> > > > >      running. So knowing rpe before the workload might not be very useful.
> > > > >
> > > > > I am of the opposite opinion that anyway the previous kernel ABI is
> > > > > probably now irrecoverably bust, so we should give FW free reign and stop
> > > > > disabling efficient freq the way we do now when min_freq < rpe_freq.
> > > >
> > > > We have to answer the question of why we need to keep exposing the sysfs
> > > > controls if they don't/can't/won't do what they say on the tin. Could we
> > > > make a break with DG2 for instance? Some use cases might suffer but it's
> > > > probably better than having them think it does what they think it does, or
> > > > file bugs when it does not.
> > >
> > > That's a good question. But if we have no control at all people will still
> > > file requests for something our hardware and low level firmwares cannot promise.
> > >
> > > There are many folks who run important/critical experiments using these
> > > interfaces at the same time they are monitoring throttle conditions and all.
> >
> > Aren't the two paragraphs a bit in contradiction? Important and critical
> > experiments using controls which are not doing what they think they are.
>
> Well, not really.
>
> Most of the times it will work. We just don't have a warranty.

I think it would help for the user to know when it will work and when it
will not.

> We have some indications when this were disrespected like the
> throttle_reasons.

There is no indication related to rpe_freq in throttle_reasons, isn't it?

> Those limited times doesn't exclude the importance of the usage.
>
> >
> > I don't claim I know for what all use cases do they get used. I *think*
> > there was one in the media transcoding world one where people wanted truly
> > fixed frequency so they can compare like-for-like between something.
> >
> > So bah.. but okay for the perf_pmu changes, how about instead of this:
> >
> > rpN = read RPn from sysfs
> > set_min_freq(rpN)
> > sleep
> > real_min = read actual from sysfs
> > actual = sample pmu for t
> > assert real_min is actual
> >
> > We do this:
> >
> > rpN = read RPn from sysfs
> > set_min_freq(rpN)
> > sleep
> > fw_min = read actual from sysfs
> > set_min_freq(fw_min)
> > sleep
> > actual = sample pmu for t
> > assert fw_min is actual
> >
> > That's a tiny difference which both "documents" and verifies that even if we
> > cannot set the min to RPn, firmware will somewhat consistently (stable)
> > respect the minimum it prefers.

Note it's not the min but the max freq which is not honored.

> >
> > Is that somewhat acceptable?
>
> yeap, this sounds a better approach for a sane test.

We talked with the GuC guys and learnt a few things:

* PCODE continuously calculates and updates the current RPe (efficient
  freq) value
* GuC samples the RPe values from PCODE every 1 ms and uses these to
  request operating freq. So the requested freq is always set by GuC but
  can be overridden say in cases of thermal throttling etc.
* GuC ignores the max freq and directly uses RPe freq when requesting an
  operating freq. This is beacuse RPe is supposedly more efficient that a
  lower max freq.
* GuC does have a separate control to ignore RPe entirely.
* Regarding the delays I added in the patch, some time back GuC FW was
  changed to move setting min/max freq's to the top half. As a result the
  delays are no longer required.

The most recent version the patches in light of above points are here:

https://patchwork.freedesktop.org/series/111282/#rev3

(The patches are similar to Tvrtko's first (rather than the second)
pseudo-code above).

These patches improve things but even with these patches unfortunately
there are still sporadic failures. For example:

Original failure:
* https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8303/bat-dg2-11/igt@perf_pmu@frequency.html
* RPn 300 MHz, PMU measures 397 MHz

New failure:
* https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8304/bat-dg2-11/igt@perf_pmu@frequency.html
* PMU measures 371 MHz, requested freq measured after PMU measurement 400 MHz

So these sporadic failures seems to be due to dynamically changing RPe.

It may be possible to get rid of the failures by either increasing the
tolerance or sampling the requested freq sysfs more often (rather than just
once after the PMU measurement) though maybe it's too much.

So the best way to fix such sporadic failures seems to be to expose a
'rpe_enable' sysfs to turn off RPe for such tests/usages where strict
userspace control of freq limits is required.

@Rodrigo Vivi: so we need to see if we can make this uapi change and then
go from there I think.

Thanks.
--
Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-12-23  9:22                   ` Tvrtko Ursulin
@ 2022-12-23 15:39                     ` Rodrigo Vivi
  2023-01-05 21:26                       ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Rodrigo Vivi @ 2022-12-23 15:39 UTC (permalink / raw)
  To: Tvrtko Ursulin; +Cc: igt-dev

On Fri, Dec 23, 2022 at 09:22:53AM +0000, Tvrtko Ursulin wrote:
> 
> On 22/12/2022 20:28, Rodrigo Vivi wrote:
> > On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
> > > 
> > > On 17/12/2022 02:49, Dixit, Ashutosh wrote:
> > > > On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
> > > > > On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> > > > > > 
> > > > > > On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> > > > > > > On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> > > > > > > > 
> > > > > > > > On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > > > > > > > > On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> > > > > > > > > > 
> > > > > > > > > 
> > > > > > > 
> > > > > > > Hi Tvrtko,
> > > > > > > 
> > > > > > > Sorry for the delay in replying on this. We are discussing this but still
> > > > > > > no conclusion. I have replied what I can below, will update again after I
> > > > > > > know more. Wanted to send a reply before disappearing for the
> > > > > > > holidays. Please see below.
> > > > > > > 
> > > > > > > > > Hi Tvrtko,
> > > > > > > > > 
> > > > > > > > > I am only offering an overall clarification below first, not answering all
> > > > > > > > > of your points for now.
> > > > > > > > > 
> > > > > > > > > > 
> > > > > > > > > > On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > > > > > > > > > > With SLPC, even when we set the same min and max freq's, the requested and
> > > > > > > > > > > actual freq's can differ from the min/max freq set. For example "efficient
> > > > > > > > > > > freq" (when in effect) can override set min/max freq. In general FW is the
> > > > > > > > > > > final arbiter in determining freq and can override set values.
> > > > > > > > > > > 
> > > > > > > > > > > Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > > > > > > > > > > reported by PMU not against the set freq's but against the requested freq
> > > > > > > > > > > reported in sysfs. Also add a delay after setting freq's to account for
> > > > > > > > > > > messaging delays in setting freq's in GuC.
> > > > > > > > > > > 
> > > > > > > > > > > v2: Introduce a 100 ms delay after setting freq
> > > > > > > > > > > v3: Update commit message, code identical to v2
> > > > > > > > > > > 
> > > > > > > > > > > Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > > > > > > > > > > Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > > > > > > > > > > Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > > > > > > > > > > Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > > > > > > > > > > ---
> > > > > > > > > > >       tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> > > > > > > > > > >       1 file changed, 15 insertions(+), 8 deletions(-)
> > > > > > > > > > > 
> > > > > > > > > > > diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > > > > > > > > > > index f363db2ba13..02f6ae989b1 100644
> > > > > > > > > > > --- a/tests/i915/perf_pmu.c
> > > > > > > > > > > +++ b/tests/i915/perf_pmu.c
> > > > > > > > > > > @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> > > > > > > > > > > 	igt_assert_lte(target, busy);
> > > > > > > > > > >       }
> > > > > > > > > > >       +/* Wait for GuC SLPC freq changes to take effect */
> > > > > > > > > > > +#define wait_freq_set()		usleep(100000)
> > > > > > > > > > 
> > > > > > > > > > A future task of maybe adding a drop_caches flag to wait for this?
> > > > > > > > > > 
> > > > > > > > > > Alternatively, if we have a sysfs or debugfs file which reflects the value
> > > > > > > > > > once GuC has processed it, keep reading it until it is updated? Even if to
> > > > > > > > > > something other than it was before the write, to handle the inability to
> > > > > > > > > > set the exact requested frequency sometimes?
> > > > > > > > > > > 
> > > > > > > > > > > +
> > > > > > > > > > >       static void
> > > > > > > > > > >       test_frequency(int gem_fd)
> > > > > > > > > > >       {
> > > > > > > > > > > -	uint32_t min_freq, max_freq, boost_freq;
> > > > > > > > > > > +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> > > > > > > > > > > 	uint64_t val[2], start[2], slept;
> > > > > > > > > > > 	double min[2], max[2];
> > > > > > > > > > > 	igt_spin_t *spin;
> > > > > > > > > > > @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> > > > > > > > > > > 	 * Set GPU to min frequency and read PMU counters.
> > > > > > > > > > > 	 */
> > > > > > > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > > > > > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > > > > > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > > > > > > > > > > +	wait_freq_set();
> > > > > > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > > > > > > 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> > > > > > > > > > > 		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > > > > > > > > > > effect */
> > > > > > > > > > > @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> > > > > > > > > > > 		min[0] = 1e9*(val[0] - start[0]) / slept;
> > > > > > > > > > > 	min[1] = 1e9*(val[1] - start[1]) / slept;
> > > > > > > > > > > +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> > > > > > > > > > 
> > > > > > > > > > This really leaves a bad taste for me still, sorry. First of all, it may
> > > > > > > > > > make it more palatable if readback was immediately after wait_freq_set(),
> > > > > > > > > > or as part of it. Otherwise it seems like even more confused test and sysfs
> > > > > > > > > > ABI.
> > > > > > > > > > 
> > > > > > > > > > Did we close on my question of whether we can make readback of
> > > > > > > > > > gt_max_freq_mhz actually represent the real max? Correct me please if I got
> > > > > > > > > > confused - with min freq checking here we write max=300 and get 350? If so
> > > > > > > > > > can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> > > > > > > > > > error?
> > > > > > > > > 
> > > > > > > > > Did you read Vinay's email on v1 here:
> > > > > > > > > 
> > > > > > > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> > > > > > > > > 
> > > > > > > > > As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > > > > > > > > freq rather than the set 'min == max freq') and no amount of waiting will
> > > > > > > > > result in the requested freq being the same as the set 'min == max
> > > > > > > > > freq'. Also the efficient freq cannot be controlled from user space.
> > > > > > > > 
> > > > > > > > Why is that not a break in our ABI? What is the point in allowing users to
> > > > > > > > write into this field if the desired value can be silently ignored?
> > > > > > > 
> > > > > > > Yes, this is in discussion. FW is doing this and we need to see if FW can
> > > > > > > be made to obey the i915 ABI. IMHO what matters is performance per watt and
> > > > > > > if the ABI has to be tossed so be it. I will keep this list posted about
> > > > > > > what happens here. Merging any patches for these issues (GL #6806 and
> > > > > > > #6786) is blocked till the ABI breakage issue is resolved one way or
> > > > > > > another.
> > > > > > 
> > > > > > Two thoughts here, one has been mentioned before.
> > > > > > 
> > > > > > New one is that performance per watt perhaps does not matter (is not a
> > > > > > primary concern at least) for use cases where "users" (for some definition)
> > > > > > tweak the min/max in sysfs. Presumably in those cases user wants control,
> > > > > > for some reason, so as long as it is safe and within the limits why not give
> > > > > > it to them.
> > > > > > 
> > > > > > Same argument from a different angle - what is the use case for allowing
> > > > > > min/max control which does not really do what it says on the tin?
> > > > > > 
> > > > > > Old thought is this - if SLPC refuses to be controlled using our existing
> > > > > > controls then we could just say -ENODEV on attempts to modify them. Whether
> > > > > > for any input value, or some, depending on what SLPC can support (or wants
> > > > > > to support).
> > > > > 
> > > > > The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
> > > > > has sudo and all the rights to disrespect what drivers (including SLPC as a
> > > > > a driver from PCODE perspective) are requesting.
> > > > > 
> > > > > > 
> > > > > > Important to note I am primarily focusing on the min freq here, which AFAIR
> > > > > > we concluded can behave that user writes in 300, but SLPC decides it will
> > > > > > only go as low as 350.
> > > > > 
> > > > > The other caveat here with the efficient freq is that it is dynamic and PCODE
> > > > > decides it with runtime conditions. It may start with 300, then move to 350,
> > > > > then back to 300. i915 is only stashing the initial value.
> > > > > 
> > > > > We could potentially update our rpe everytime any freq control is
> > > > > touched.
> > > > 
> > > > We thought of doing this but two points:
> > > > 
> > > > 1. i915 now reads the same register as GuC for rpe but I never see the
> > > >      value in this register change.
> > > > 2. Even if we could know the real rpe, the freq's might be set before a
> > > >      workload start running and rpe might change after the workload starts
> > > >      running. So knowing rpe before the workload might not be very useful.
> > > > 
> > > > I am of the opposite opinion that anyway the previous kernel ABI is
> > > > probably now irrecoverably bust, so we should give FW free reign and stop
> > > > disabling efficient freq the way we do now when min_freq < rpe_freq.
> > > 
> > > We have to answer the question of why we need to keep exposing the sysfs
> > > controls if they don't/can't/won't do what they say on the tin. Could we
> > > make a break with DG2 for instance? Some use cases might suffer but it's
> > > probably better than having them think it does what they think it does, or
> > > file bugs when it does not.
> > 
> > That's a good question. But if we have no control at all people will still
> > file requests for something our hardware and low level firmwares cannot promise.
> > 
> > There are many folks who run important/critical experiments using these
> > interfaces at the same time they are monitoring throttle conditions and all.
> 
> Aren't the two paragraphs a bit in contradiction? Important and critical
> experiments using controls which are not doing what they think they are.

Well, not really.

Most of the times it will work. We just don't have a warranty.
We have some indications when this were disrespected like the throttle_reasons.

Those limited times doesn't exclude the importance of the usage.

> 
> I don't claim I know for what all use cases do they get used. I *think*
> there was one in the media transcoding world one where people wanted truly
> fixed frequency so they can compare like-for-like between something.
> 
> So bah.. but okay for the perf_pmu changes, how about instead of this:
> 
> rpN = read RPn from sysfs
> set_min_freq(rpN)
> sleep
> real_min = read actual from sysfs
> actual = sample pmu for t
> assert real_min is actual
> 
> We do this:
> 
> rpN = read RPn from sysfs
> set_min_freq(rpN)
> sleep
> fw_min = read actual from sysfs
> set_min_freq(fw_min)
> sleep
> actual = sample pmu for t
> assert fw_min is actual
> 
> That's a tiny difference which both "documents" and verifies that even if we
> cannot set the min to RPn, firmware will somewhat consistently (stable)
> respect the minimum it prefers.
> 
> Is that somewhat acceptable?

yeap, this sounds a better approach for a sane test.

> 
> Regards,
> 
> Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-12-22 20:28                 ` Rodrigo Vivi
@ 2022-12-23  9:22                   ` Tvrtko Ursulin
  2022-12-23 15:39                     ` Rodrigo Vivi
  0 siblings, 1 reply; 34+ messages in thread
From: Tvrtko Ursulin @ 2022-12-23  9:22 UTC (permalink / raw)
  To: Rodrigo Vivi; +Cc: igt-dev


On 22/12/2022 20:28, Rodrigo Vivi wrote:
> On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
>>
>> On 17/12/2022 02:49, Dixit, Ashutosh wrote:
>>> On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
>>>> On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
>>>>>
>>>>> On 16/12/2022 06:21, Dixit, Ashutosh wrote:
>>>>>> On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
>>>>>>>
>>>>>>> On 23/11/2022 06:03, Dixit, Ashutosh wrote:
>>>>>>>> On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
>>>>>>>>>
>>>>>>>>
>>>>>>
>>>>>> Hi Tvrtko,
>>>>>>
>>>>>> Sorry for the delay in replying on this. We are discussing this but still
>>>>>> no conclusion. I have replied what I can below, will update again after I
>>>>>> know more. Wanted to send a reply before disappearing for the
>>>>>> holidays. Please see below.
>>>>>>
>>>>>>>> Hi Tvrtko,
>>>>>>>>
>>>>>>>> I am only offering an overall clarification below first, not answering all
>>>>>>>> of your points for now.
>>>>>>>>
>>>>>>>>>
>>>>>>>>> On 19/11/2022 02:00, Ashutosh Dixit wrote:
>>>>>>>>>> With SLPC, even when we set the same min and max freq's, the requested and
>>>>>>>>>> actual freq's can differ from the min/max freq set. For example "efficient
>>>>>>>>>> freq" (when in effect) can override set min/max freq. In general FW is the
>>>>>>>>>> final arbiter in determining freq and can override set values.
>>>>>>>>>>
>>>>>>>>>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
>>>>>>>>>> reported by PMU not against the set freq's but against the requested freq
>>>>>>>>>> reported in sysfs. Also add a delay after setting freq's to account for
>>>>>>>>>> messaging delays in setting freq's in GuC.
>>>>>>>>>>
>>>>>>>>>> v2: Introduce a 100 ms delay after setting freq
>>>>>>>>>> v3: Update commit message, code identical to v2
>>>>>>>>>>
>>>>>>>>>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
>>>>>>>>>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
>>>>>>>>>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
>>>>>>>>>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
>>>>>>>>>> ---
>>>>>>>>>>       tests/i915/perf_pmu.c | 23 +++++++++++++++--------
>>>>>>>>>>       1 file changed, 15 insertions(+), 8 deletions(-)
>>>>>>>>>>
>>>>>>>>>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
>>>>>>>>>> index f363db2ba13..02f6ae989b1 100644
>>>>>>>>>> --- a/tests/i915/perf_pmu.c
>>>>>>>>>> +++ b/tests/i915/perf_pmu.c
>>>>>>>>>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
>>>>>>>>>> 	igt_assert_lte(target, busy);
>>>>>>>>>>       }
>>>>>>>>>>       +/* Wait for GuC SLPC freq changes to take effect */
>>>>>>>>>> +#define wait_freq_set()		usleep(100000)
>>>>>>>>>
>>>>>>>>> A future task of maybe adding a drop_caches flag to wait for this?
>>>>>>>>>
>>>>>>>>> Alternatively, if we have a sysfs or debugfs file which reflects the value
>>>>>>>>> once GuC has processed it, keep reading it until it is updated? Even if to
>>>>>>>>> something other than it was before the write, to handle the inability to
>>>>>>>>> set the exact requested frequency sometimes?
>>>>>>>>>>
>>>>>>>>>> +
>>>>>>>>>>       static void
>>>>>>>>>>       test_frequency(int gem_fd)
>>>>>>>>>>       {
>>>>>>>>>> -	uint32_t min_freq, max_freq, boost_freq;
>>>>>>>>>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
>>>>>>>>>> 	uint64_t val[2], start[2], slept;
>>>>>>>>>> 	double min[2], max[2];
>>>>>>>>>> 	igt_spin_t *spin;
>>>>>>>>>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
>>>>>>>>>> 	 * Set GPU to min frequency and read PMU counters.
>>>>>>>>>> 	 */
>>>>>>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>>>>>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
>>>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>>>>>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
>>>>>>>>>> +	wait_freq_set();
>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>>>>>>>>> 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
>>>>>>>>>> 		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
>>>>>>>>>> effect */
>>>>>>>>>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
>>>>>>>>>> 		min[0] = 1e9*(val[0] - start[0]) / slept;
>>>>>>>>>> 	min[1] = 1e9*(val[1] - start[1]) / slept;
>>>>>>>>>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>>>>>>>>>
>>>>>>>>> This really leaves a bad taste for me still, sorry. First of all, it may
>>>>>>>>> make it more palatable if readback was immediately after wait_freq_set(),
>>>>>>>>> or as part of it. Otherwise it seems like even more confused test and sysfs
>>>>>>>>> ABI.
>>>>>>>>>
>>>>>>>>> Did we close on my question of whether we can make readback of
>>>>>>>>> gt_max_freq_mhz actually represent the real max? Correct me please if I got
>>>>>>>>> confused - with min freq checking here we write max=300 and get 350? If so
>>>>>>>>> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
>>>>>>>>> error?
>>>>>>>>
>>>>>>>> Did you read Vinay's email on v1 here:
>>>>>>>>
>>>>>>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
>>>>>>>>
>>>>>>>> As Vinay says, SLPC is requesting an entirely different freq (the effecient
>>>>>>>> freq rather than the set 'min == max freq') and no amount of waiting will
>>>>>>>> result in the requested freq being the same as the set 'min == max
>>>>>>>> freq'. Also the efficient freq cannot be controlled from user space.
>>>>>>>
>>>>>>> Why is that not a break in our ABI? What is the point in allowing users to
>>>>>>> write into this field if the desired value can be silently ignored?
>>>>>>
>>>>>> Yes, this is in discussion. FW is doing this and we need to see if FW can
>>>>>> be made to obey the i915 ABI. IMHO what matters is performance per watt and
>>>>>> if the ABI has to be tossed so be it. I will keep this list posted about
>>>>>> what happens here. Merging any patches for these issues (GL #6806 and
>>>>>> #6786) is blocked till the ABI breakage issue is resolved one way or
>>>>>> another.
>>>>>
>>>>> Two thoughts here, one has been mentioned before.
>>>>>
>>>>> New one is that performance per watt perhaps does not matter (is not a
>>>>> primary concern at least) for use cases where "users" (for some definition)
>>>>> tweak the min/max in sysfs. Presumably in those cases user wants control,
>>>>> for some reason, so as long as it is safe and within the limits why not give
>>>>> it to them.
>>>>>
>>>>> Same argument from a different angle - what is the use case for allowing
>>>>> min/max control which does not really do what it says on the tin?
>>>>>
>>>>> Old thought is this - if SLPC refuses to be controlled using our existing
>>>>> controls then we could just say -ENODEV on attempts to modify them. Whether
>>>>> for any input value, or some, depending on what SLPC can support (or wants
>>>>> to support).
>>>>
>>>> The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
>>>> has sudo and all the rights to disrespect what drivers (including SLPC as a
>>>> a driver from PCODE perspective) are requesting.
>>>>
>>>>>
>>>>> Important to note I am primarily focusing on the min freq here, which AFAIR
>>>>> we concluded can behave that user writes in 300, but SLPC decides it will
>>>>> only go as low as 350.
>>>>
>>>> The other caveat here with the efficient freq is that it is dynamic and PCODE
>>>> decides it with runtime conditions. It may start with 300, then move to 350,
>>>> then back to 300. i915 is only stashing the initial value.
>>>>
>>>> We could potentially update our rpe everytime any freq control is
>>>> touched.
>>>
>>> We thought of doing this but two points:
>>>
>>> 1. i915 now reads the same register as GuC for rpe but I never see the
>>>      value in this register change.
>>> 2. Even if we could know the real rpe, the freq's might be set before a
>>>      workload start running and rpe might change after the workload starts
>>>      running. So knowing rpe before the workload might not be very useful.
>>>
>>> I am of the opposite opinion that anyway the previous kernel ABI is
>>> probably now irrecoverably bust, so we should give FW free reign and stop
>>> disabling efficient freq the way we do now when min_freq < rpe_freq.
>>
>> We have to answer the question of why we need to keep exposing the sysfs
>> controls if they don't/can't/won't do what they say on the tin. Could we
>> make a break with DG2 for instance? Some use cases might suffer but it's
>> probably better than having them think it does what they think it does, or
>> file bugs when it does not.
> 
> That's a good question. But if we have no control at all people will still
> file requests for something our hardware and low level firmwares cannot promise.
> 
> There are many folks who run important/critical experiments using these
> interfaces at the same time they are monitoring throttle conditions and all.

Aren't the two paragraphs a bit in contradiction? Important and critical 
experiments using controls which are not doing what they think they are.

I don't claim I know for what all use cases do they get used. I *think* 
there was one in the media transcoding world one where people wanted 
truly fixed frequency so they can compare like-for-like between something.

So bah.. but okay for the perf_pmu changes, how about instead of this:

rpN = read RPn from sysfs
set_min_freq(rpN)
sleep
real_min = read actual from sysfs
actual = sample pmu for t
assert real_min is actual

We do this:

rpN = read RPn from sysfs
set_min_freq(rpN)
sleep
fw_min = read actual from sysfs
set_min_freq(fw_min)
sleep
actual = sample pmu for t
assert fw_min is actual

That's a tiny difference which both "documents" and verifies that even 
if we cannot set the min to RPn, firmware will somewhat consistently 
(stable) respect the minimum it prefers.

Is that somewhat acceptable?

Regards,

Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-12-19  8:46               ` Tvrtko Ursulin
@ 2022-12-22 20:28                 ` Rodrigo Vivi
  2022-12-23  9:22                   ` Tvrtko Ursulin
  0 siblings, 1 reply; 34+ messages in thread
From: Rodrigo Vivi @ 2022-12-22 20:28 UTC (permalink / raw)
  To: Tvrtko Ursulin; +Cc: igt-dev

On Mon, Dec 19, 2022 at 08:46:59AM +0000, Tvrtko Ursulin wrote:
> 
> On 17/12/2022 02:49, Dixit, Ashutosh wrote:
> > On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
> > > On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> > > > 
> > > > On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> > > > > On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> > > > > > 
> > > > > > On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > > > > > > On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> > > > > > > > 
> > > > > > > 
> > > > > 
> > > > > Hi Tvrtko,
> > > > > 
> > > > > Sorry for the delay in replying on this. We are discussing this but still
> > > > > no conclusion. I have replied what I can below, will update again after I
> > > > > know more. Wanted to send a reply before disappearing for the
> > > > > holidays. Please see below.
> > > > > 
> > > > > > > Hi Tvrtko,
> > > > > > > 
> > > > > > > I am only offering an overall clarification below first, not answering all
> > > > > > > of your points for now.
> > > > > > > 
> > > > > > > > 
> > > > > > > > On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > > > > > > > > With SLPC, even when we set the same min and max freq's, the requested and
> > > > > > > > > actual freq's can differ from the min/max freq set. For example "efficient
> > > > > > > > > freq" (when in effect) can override set min/max freq. In general FW is the
> > > > > > > > > final arbiter in determining freq and can override set values.
> > > > > > > > > 
> > > > > > > > > Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > > > > > > > > reported by PMU not against the set freq's but against the requested freq
> > > > > > > > > reported in sysfs. Also add a delay after setting freq's to account for
> > > > > > > > > messaging delays in setting freq's in GuC.
> > > > > > > > > 
> > > > > > > > > v2: Introduce a 100 ms delay after setting freq
> > > > > > > > > v3: Update commit message, code identical to v2
> > > > > > > > > 
> > > > > > > > > Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > > > > > > > > Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > > > > > > > > Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > > > > > > > > Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > > > > > > > > ---
> > > > > > > > >      tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> > > > > > > > >      1 file changed, 15 insertions(+), 8 deletions(-)
> > > > > > > > > 
> > > > > > > > > diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > > > > > > > > index f363db2ba13..02f6ae989b1 100644
> > > > > > > > > --- a/tests/i915/perf_pmu.c
> > > > > > > > > +++ b/tests/i915/perf_pmu.c
> > > > > > > > > @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> > > > > > > > > 	igt_assert_lte(target, busy);
> > > > > > > > >      }
> > > > > > > > >      +/* Wait for GuC SLPC freq changes to take effect */
> > > > > > > > > +#define wait_freq_set()		usleep(100000)
> > > > > > > > 
> > > > > > > > A future task of maybe adding a drop_caches flag to wait for this?
> > > > > > > > 
> > > > > > > > Alternatively, if we have a sysfs or debugfs file which reflects the value
> > > > > > > > once GuC has processed it, keep reading it until it is updated? Even if to
> > > > > > > > something other than it was before the write, to handle the inability to
> > > > > > > > set the exact requested frequency sometimes?
> > > > > > > > > 
> > > > > > > > > +
> > > > > > > > >      static void
> > > > > > > > >      test_frequency(int gem_fd)
> > > > > > > > >      {
> > > > > > > > > -	uint32_t min_freq, max_freq, boost_freq;
> > > > > > > > > +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> > > > > > > > > 	uint64_t val[2], start[2], slept;
> > > > > > > > > 	double min[2], max[2];
> > > > > > > > > 	igt_spin_t *spin;
> > > > > > > > > @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> > > > > > > > > 	 * Set GPU to min frequency and read PMU counters.
> > > > > > > > > 	 */
> > > > > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > > > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > > > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > > > > > > > > +	wait_freq_set();
> > > > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > > > > 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> > > > > > > > > 		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > > > > > > > > effect */
> > > > > > > > > @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> > > > > > > > > 		min[0] = 1e9*(val[0] - start[0]) / slept;
> > > > > > > > > 	min[1] = 1e9*(val[1] - start[1]) / slept;
> > > > > > > > > +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> > > > > > > > 
> > > > > > > > This really leaves a bad taste for me still, sorry. First of all, it may
> > > > > > > > make it more palatable if readback was immediately after wait_freq_set(),
> > > > > > > > or as part of it. Otherwise it seems like even more confused test and sysfs
> > > > > > > > ABI.
> > > > > > > > 
> > > > > > > > Did we close on my question of whether we can make readback of
> > > > > > > > gt_max_freq_mhz actually represent the real max? Correct me please if I got
> > > > > > > > confused - with min freq checking here we write max=300 and get 350? If so
> > > > > > > > can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> > > > > > > > error?
> > > > > > > 
> > > > > > > Did you read Vinay's email on v1 here:
> > > > > > > 
> > > > > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> > > > > > > 
> > > > > > > As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > > > > > > freq rather than the set 'min == max freq') and no amount of waiting will
> > > > > > > result in the requested freq being the same as the set 'min == max
> > > > > > > freq'. Also the efficient freq cannot be controlled from user space.
> > > > > > 
> > > > > > Why is that not a break in our ABI? What is the point in allowing users to
> > > > > > write into this field if the desired value can be silently ignored?
> > > > > 
> > > > > Yes, this is in discussion. FW is doing this and we need to see if FW can
> > > > > be made to obey the i915 ABI. IMHO what matters is performance per watt and
> > > > > if the ABI has to be tossed so be it. I will keep this list posted about
> > > > > what happens here. Merging any patches for these issues (GL #6806 and
> > > > > #6786) is blocked till the ABI breakage issue is resolved one way or
> > > > > another.
> > > > 
> > > > Two thoughts here, one has been mentioned before.
> > > > 
> > > > New one is that performance per watt perhaps does not matter (is not a
> > > > primary concern at least) for use cases where "users" (for some definition)
> > > > tweak the min/max in sysfs. Presumably in those cases user wants control,
> > > > for some reason, so as long as it is safe and within the limits why not give
> > > > it to them.
> > > > 
> > > > Same argument from a different angle - what is the use case for allowing
> > > > min/max control which does not really do what it says on the tin?
> > > > 
> > > > Old thought is this - if SLPC refuses to be controlled using our existing
> > > > controls then we could just say -ENODEV on attempts to modify them. Whether
> > > > for any input value, or some, depending on what SLPC can support (or wants
> > > > to support).
> > > 
> > > The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
> > > has sudo and all the rights to disrespect what drivers (including SLPC as a
> > > a driver from PCODE perspective) are requesting.
> > > 
> > > > 
> > > > Important to note I am primarily focusing on the min freq here, which AFAIR
> > > > we concluded can behave that user writes in 300, but SLPC decides it will
> > > > only go as low as 350.
> > > 
> > > The other caveat here with the efficient freq is that it is dynamic and PCODE
> > > decides it with runtime conditions. It may start with 300, then move to 350,
> > > then back to 300. i915 is only stashing the initial value.
> > > 
> > > We could potentially update our rpe everytime any freq control is
> > > touched.
> > 
> > We thought of doing this but two points:
> > 
> > 1. i915 now reads the same register as GuC for rpe but I never see the
> >     value in this register change.
> > 2. Even if we could know the real rpe, the freq's might be set before a
> >     workload start running and rpe might change after the workload starts
> >     running. So knowing rpe before the workload might not be very useful.
> > 
> > I am of the opposite opinion that anyway the previous kernel ABI is
> > probably now irrecoverably bust, so we should give FW free reign and stop
> > disabling efficient freq the way we do now when min_freq < rpe_freq.
> 
> We have to answer the question of why we need to keep exposing the sysfs
> controls if they don't/can't/won't do what they say on the tin. Could we
> make a break with DG2 for instance? Some use cases might suffer but it's
> probably better than having them think it does what they think it does, or
> file bugs when it does not.

That's a good question. But if we have no control at all people will still
file requests for something our hardware and low level firmwares cannot promise.

There are many folks who run important/critical experiments using these
interfaces at the same time they are monitoring throttle conditions and all.

> 
> Regards,
> 
> Tvrtko
> 
> > 
> > Cheers.
> > --
> > Ashutosh
> > 
> > > Just to highlight how bad this is, I've seen cases where rpe in runtime
> > > goes below the minimal fused freq (our rpn).
> > > 
> > > > Because that is different from the existing, should
> > > > be well known, situation where we can write a max which cannot be achieved
> > > > due thermal throttling.
> > > 
> > > yeap, usually when things got disrespected by PCODE there will be some thermal
> > > throttling information. But I have seen cases where this indicators simply do
> > > not exist. What complicate the api expectation to a very bad level.
> > > 
> > > > 
> > > > Perhaps we need a table, or a summary of some kind, which explains the
> > > > possible behaviours for both min and max settings.
> > > 
> > > This is probably a good idea. At least document everything and what and when
> > > things can be disrespected, etc...
> > > 
> > > (and update our rpe info dynamically)
> > > 
> > > > 
> > > > > > > (So in the example above min and max are set to 300 MHz and SLPC is
> > > > > > > requesting efficient freq which is 350 or 400 MHz. So min and max values
> > > > > > > are correct at 300 MHz and requested/actual freq will show 350/400 MHz).
> > > > > > > 
> > > > > > > Let me also explain what happens in GuC FW when we change min/max freq:
> > > > > > > * i915 receives min/max freq from sysfs and sends a message to GuC to
> > > > > > >      change min/max freq.
> > > > > > > * The message itself might be queued behind other messages in the host to
> > > > > > >      GuC queue. i915 is still blocked in the sysfs handler.
> > > > > > > * Now when GuC receives the message interrupt, in its top-half handler it
> > > > > > >      validates the freq parameters and sends a message back to host saying the
> > > > > > >      freq change has been accepted. But min/max freq is actually changed later
> > > > > > >      in the bottom-half handler in GuC FW. So the top half only validates the
> > > > > > >      freq value.
> > > > > > > * When i915 receives the message from GuC top half, the sysfs handler
> > > > > > >      unblocks and starts reflecting the new min/max freq value.
> > > > > > > 
> > > > > > > So the delay for the new min/max freq values to take effect is between
> > > > > > > returning from the sysfs handler and when the min/max freq are really set
> > > > > > > in GuC bottom half (an indeterminate possibly large amount of time if GuC
> > > > > > > is busy).
> > > > > > 
> > > > > > Do we have a drop_caches flag which would force waiting of idle GuC, which
> > > > > > would cover the bottom half as well? Could we or should we add it if not?
> > > > > > Would beat the usleep approach..
> > > > > 
> > > > > Agree, waiting for GuC to idle would do it, but not sure "how" to do it. As
> > > > > I said there is no notification after the bottom half is done applying the
> > > > > freq settings. We may need to schedule something after the freq setting
> > > > > host to guc command and wait for that to be done. A little bit out of my
> > > > > depth on this, let me know if you have any ideas here. I'll ask the GuC
> > > > > guys too. Currently we don't have a drop_caches flag for this.
> > > > 
> > > > Okay, I was wondering if DROP_IDLE would do it, but perhaps this bottom half
> > > > can run independently from those flows so it was just a thought, not a
> > > > claim.
> > > > 
> > > > > > > So as you see this introduces an indeterminacy due to to GuC interface not
> > > > > > > being synchronous. If this needs to be fixed, it needs to be fixed in GuC
> > > > > > > FW rather than waiting etc. in i915. GuC needs to implement another
> > > > > > > notification to host /after/ the min/max freq change has been applied in
> > > > > > > its bottom half. Doing this is not a priority for the GuC team at this
> > > > > > > point.
> > > > > > 
> > > > > > Is there a method for querying the real min from GuC, which would maybe
> > > > > > work after waiting for GuC to idle?
> > > > > 
> > > > > Will find out about this too. Afaik, at present querying the real min from
> > > > > GuC is not possible. As soon as the top half has completed the freq
> > > > > settings are updated so querying will return the new settings even though
> > > > > they might not have been applied by the bottom half yet.
> > > > 
> > > > And it will return the written value, or the real one? Going back to the 300
> > > > written in, 350 actual min freq example. Presumably 300 and 350 SLPC picks
> > > > is completely invisible to i915 via the FW interface?
> > > > 
> > > > > > > However, this bug is not due to these delays but due to GuC requesting the
> > > > > > > efficient freq instead of the set 'min == max freq' as mentioned above.
> > > > > > > 
> > > > > > > If all these delays are confusing we can revert to v1 of the patch:
> > > > > > > 
> > > > > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1
> > > > > > > 
> > > > > > > v1 is simple and does not introduce these delays but just reads the
> > > > > > > requested freq from sysfs after the 500 ms PMU measurement but while the
> > > > > > > spinner is still running (which we do in v3 too). Without load SLPC will
> > > > > > > only request RPn so to compare with PMU measurement we must read the
> > > > > > > requested freq from sysfs with the spinner load. v1 is sufficient to fix
> > > > > > > the bug but I thought to be completely safe it is better to add
> > > > > > > wait_freq_set() after changing min/max freq (and hence v3).
> > > > > > > 
> > > > > > > I hope this clarifies things somewhat. If not please let me know.
> > > > > > 
> > > > > > It clarifies a large chunk thank you.
> > > > > > 
> > > > > > Another question which you missed is why is it needed for the max mode?
> > > > > > There we allow a 15% error (more than 5% default). Is that not enough with
> > > > > > SLPC?
> > > > > 
> > > > > Yes I think you are right, should not be needed for the max mode. The real
> > > > > issue is that efficient freq can exceed the max freq set but I think that
> > > > > should not be an issue if the max is RP0.
> > > > 
> > > > Ah okay, so SLPC not respecting max same as with min. I think the question
> > > > of what is the use case for min/max controls in the SLPC world is the one to
> > > > answer then.
> > > > 
> > > > Regards,
> > > > 
> > > > Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-12-17  2:49             ` Dixit, Ashutosh
@ 2022-12-19  8:46               ` Tvrtko Ursulin
  2022-12-22 20:28                 ` Rodrigo Vivi
  0 siblings, 1 reply; 34+ messages in thread
From: Tvrtko Ursulin @ 2022-12-19  8:46 UTC (permalink / raw)
  To: Dixit, Ashutosh, Rodrigo Vivi; +Cc: igt-dev


On 17/12/2022 02:49, Dixit, Ashutosh wrote:
> On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
>> On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
>>>
>>> On 16/12/2022 06:21, Dixit, Ashutosh wrote:
>>>> On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
>>>>>
>>>>> On 23/11/2022 06:03, Dixit, Ashutosh wrote:
>>>>>> On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
>>>>>>>
>>>>>>
>>>>
>>>> Hi Tvrtko,
>>>>
>>>> Sorry for the delay in replying on this. We are discussing this but still
>>>> no conclusion. I have replied what I can below, will update again after I
>>>> know more. Wanted to send a reply before disappearing for the
>>>> holidays. Please see below.
>>>>
>>>>>> Hi Tvrtko,
>>>>>>
>>>>>> I am only offering an overall clarification below first, not answering all
>>>>>> of your points for now.
>>>>>>
>>>>>>>
>>>>>>> On 19/11/2022 02:00, Ashutosh Dixit wrote:
>>>>>>>> With SLPC, even when we set the same min and max freq's, the requested and
>>>>>>>> actual freq's can differ from the min/max freq set. For example "efficient
>>>>>>>> freq" (when in effect) can override set min/max freq. In general FW is the
>>>>>>>> final arbiter in determining freq and can override set values.
>>>>>>>>
>>>>>>>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
>>>>>>>> reported by PMU not against the set freq's but against the requested freq
>>>>>>>> reported in sysfs. Also add a delay after setting freq's to account for
>>>>>>>> messaging delays in setting freq's in GuC.
>>>>>>>>
>>>>>>>> v2: Introduce a 100 ms delay after setting freq
>>>>>>>> v3: Update commit message, code identical to v2
>>>>>>>>
>>>>>>>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
>>>>>>>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
>>>>>>>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
>>>>>>>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
>>>>>>>> ---
>>>>>>>>      tests/i915/perf_pmu.c | 23 +++++++++++++++--------
>>>>>>>>      1 file changed, 15 insertions(+), 8 deletions(-)
>>>>>>>>
>>>>>>>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
>>>>>>>> index f363db2ba13..02f6ae989b1 100644
>>>>>>>> --- a/tests/i915/perf_pmu.c
>>>>>>>> +++ b/tests/i915/perf_pmu.c
>>>>>>>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
>>>>>>>> 	igt_assert_lte(target, busy);
>>>>>>>>      }
>>>>>>>>      +/* Wait for GuC SLPC freq changes to take effect */
>>>>>>>> +#define wait_freq_set()		usleep(100000)
>>>>>>>
>>>>>>> A future task of maybe adding a drop_caches flag to wait for this?
>>>>>>>
>>>>>>> Alternatively, if we have a sysfs or debugfs file which reflects the value
>>>>>>> once GuC has processed it, keep reading it until it is updated? Even if to
>>>>>>> something other than it was before the write, to handle the inability to
>>>>>>> set the exact requested frequency sometimes?
>>>>>>>>
>>>>>>>> +
>>>>>>>>      static void
>>>>>>>>      test_frequency(int gem_fd)
>>>>>>>>      {
>>>>>>>> -	uint32_t min_freq, max_freq, boost_freq;
>>>>>>>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
>>>>>>>> 	uint64_t val[2], start[2], slept;
>>>>>>>> 	double min[2], max[2];
>>>>>>>> 	igt_spin_t *spin;
>>>>>>>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
>>>>>>>> 	 * Set GPU to min frequency and read PMU counters.
>>>>>>>> 	 */
>>>>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>>>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
>>>>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>>>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
>>>>>>>> +	wait_freq_set();
>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>>>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>>>>>>> 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
>>>>>>>> 		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
>>>>>>>> effect */
>>>>>>>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
>>>>>>>> 		min[0] = 1e9*(val[0] - start[0]) / slept;
>>>>>>>> 	min[1] = 1e9*(val[1] - start[1]) / slept;
>>>>>>>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>>>>>>>
>>>>>>> This really leaves a bad taste for me still, sorry. First of all, it may
>>>>>>> make it more palatable if readback was immediately after wait_freq_set(),
>>>>>>> or as part of it. Otherwise it seems like even more confused test and sysfs
>>>>>>> ABI.
>>>>>>>
>>>>>>> Did we close on my question of whether we can make readback of
>>>>>>> gt_max_freq_mhz actually represent the real max? Correct me please if I got
>>>>>>> confused - with min freq checking here we write max=300 and get 350? If so
>>>>>>> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
>>>>>>> error?
>>>>>>
>>>>>> Did you read Vinay's email on v1 here:
>>>>>>
>>>>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
>>>>>>
>>>>>> As Vinay says, SLPC is requesting an entirely different freq (the effecient
>>>>>> freq rather than the set 'min == max freq') and no amount of waiting will
>>>>>> result in the requested freq being the same as the set 'min == max
>>>>>> freq'. Also the efficient freq cannot be controlled from user space.
>>>>>
>>>>> Why is that not a break in our ABI? What is the point in allowing users to
>>>>> write into this field if the desired value can be silently ignored?
>>>>
>>>> Yes, this is in discussion. FW is doing this and we need to see if FW can
>>>> be made to obey the i915 ABI. IMHO what matters is performance per watt and
>>>> if the ABI has to be tossed so be it. I will keep this list posted about
>>>> what happens here. Merging any patches for these issues (GL #6806 and
>>>> #6786) is blocked till the ABI breakage issue is resolved one way or
>>>> another.
>>>
>>> Two thoughts here, one has been mentioned before.
>>>
>>> New one is that performance per watt perhaps does not matter (is not a
>>> primary concern at least) for use cases where "users" (for some definition)
>>> tweak the min/max in sysfs. Presumably in those cases user wants control,
>>> for some reason, so as long as it is safe and within the limits why not give
>>> it to them.
>>>
>>> Same argument from a different angle - what is the use case for allowing
>>> min/max control which does not really do what it says on the tin?
>>>
>>> Old thought is this - if SLPC refuses to be controlled using our existing
>>> controls then we could just say -ENODEV on attempts to modify them. Whether
>>> for any input value, or some, depending on what SLPC can support (or wants
>>> to support).
>>
>> The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
>> has sudo and all the rights to disrespect what drivers (including SLPC as a
>> a driver from PCODE perspective) are requesting.
>>
>>>
>>> Important to note I am primarily focusing on the min freq here, which AFAIR
>>> we concluded can behave that user writes in 300, but SLPC decides it will
>>> only go as low as 350.
>>
>> The other caveat here with the efficient freq is that it is dynamic and PCODE
>> decides it with runtime conditions. It may start with 300, then move to 350,
>> then back to 300. i915 is only stashing the initial value.
>>
>> We could potentially update our rpe everytime any freq control is
>> touched.
> 
> We thought of doing this but two points:
> 
> 1. i915 now reads the same register as GuC for rpe but I never see the
>     value in this register change.
> 2. Even if we could know the real rpe, the freq's might be set before a
>     workload start running and rpe might change after the workload starts
>     running. So knowing rpe before the workload might not be very useful.
> 
> I am of the opposite opinion that anyway the previous kernel ABI is
> probably now irrecoverably bust, so we should give FW free reign and stop
> disabling efficient freq the way we do now when min_freq < rpe_freq.

We have to answer the question of why we need to keep exposing the sysfs 
controls if they don't/can't/won't do what they say on the tin. Could we 
make a break with DG2 for instance? Some use cases might suffer but it's 
probably better than having them think it does what they think it does, 
or file bugs when it does not.

Regards,

Tvrtko

> 
> Cheers.
> --
> Ashutosh
> 
>> Just to highlight how bad this is, I've seen cases where rpe in runtime
>> goes below the minimal fused freq (our rpn).
>>
>>> Because that is different from the existing, should
>>> be well known, situation where we can write a max which cannot be achieved
>>> due thermal throttling.
>>
>> yeap, usually when things got disrespected by PCODE there will be some thermal
>> throttling information. But I have seen cases where this indicators simply do
>> not exist. What complicate the api expectation to a very bad level.
>>
>>>
>>> Perhaps we need a table, or a summary of some kind, which explains the
>>> possible behaviours for both min and max settings.
>>
>> This is probably a good idea. At least document everything and what and when
>> things can be disrespected, etc...
>>
>> (and update our rpe info dynamically)
>>
>>>
>>>>>> (So in the example above min and max are set to 300 MHz and SLPC is
>>>>>> requesting efficient freq which is 350 or 400 MHz. So min and max values
>>>>>> are correct at 300 MHz and requested/actual freq will show 350/400 MHz).
>>>>>>
>>>>>> Let me also explain what happens in GuC FW when we change min/max freq:
>>>>>> * i915 receives min/max freq from sysfs and sends a message to GuC to
>>>>>>      change min/max freq.
>>>>>> * The message itself might be queued behind other messages in the host to
>>>>>>      GuC queue. i915 is still blocked in the sysfs handler.
>>>>>> * Now when GuC receives the message interrupt, in its top-half handler it
>>>>>>      validates the freq parameters and sends a message back to host saying the
>>>>>>      freq change has been accepted. But min/max freq is actually changed later
>>>>>>      in the bottom-half handler in GuC FW. So the top half only validates the
>>>>>>      freq value.
>>>>>> * When i915 receives the message from GuC top half, the sysfs handler
>>>>>>      unblocks and starts reflecting the new min/max freq value.
>>>>>>
>>>>>> So the delay for the new min/max freq values to take effect is between
>>>>>> returning from the sysfs handler and when the min/max freq are really set
>>>>>> in GuC bottom half (an indeterminate possibly large amount of time if GuC
>>>>>> is busy).
>>>>>
>>>>> Do we have a drop_caches flag which would force waiting of idle GuC, which
>>>>> would cover the bottom half as well? Could we or should we add it if not?
>>>>> Would beat the usleep approach..
>>>>
>>>> Agree, waiting for GuC to idle would do it, but not sure "how" to do it. As
>>>> I said there is no notification after the bottom half is done applying the
>>>> freq settings. We may need to schedule something after the freq setting
>>>> host to guc command and wait for that to be done. A little bit out of my
>>>> depth on this, let me know if you have any ideas here. I'll ask the GuC
>>>> guys too. Currently we don't have a drop_caches flag for this.
>>>
>>> Okay, I was wondering if DROP_IDLE would do it, but perhaps this bottom half
>>> can run independently from those flows so it was just a thought, not a
>>> claim.
>>>
>>>>>> So as you see this introduces an indeterminacy due to to GuC interface not
>>>>>> being synchronous. If this needs to be fixed, it needs to be fixed in GuC
>>>>>> FW rather than waiting etc. in i915. GuC needs to implement another
>>>>>> notification to host /after/ the min/max freq change has been applied in
>>>>>> its bottom half. Doing this is not a priority for the GuC team at this
>>>>>> point.
>>>>>
>>>>> Is there a method for querying the real min from GuC, which would maybe
>>>>> work after waiting for GuC to idle?
>>>>
>>>> Will find out about this too. Afaik, at present querying the real min from
>>>> GuC is not possible. As soon as the top half has completed the freq
>>>> settings are updated so querying will return the new settings even though
>>>> they might not have been applied by the bottom half yet.
>>>
>>> And it will return the written value, or the real one? Going back to the 300
>>> written in, 350 actual min freq example. Presumably 300 and 350 SLPC picks
>>> is completely invisible to i915 via the FW interface?
>>>
>>>>>> However, this bug is not due to these delays but due to GuC requesting the
>>>>>> efficient freq instead of the set 'min == max freq' as mentioned above.
>>>>>>
>>>>>> If all these delays are confusing we can revert to v1 of the patch:
>>>>>>
>>>>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1
>>>>>>
>>>>>> v1 is simple and does not introduce these delays but just reads the
>>>>>> requested freq from sysfs after the 500 ms PMU measurement but while the
>>>>>> spinner is still running (which we do in v3 too). Without load SLPC will
>>>>>> only request RPn so to compare with PMU measurement we must read the
>>>>>> requested freq from sysfs with the spinner load. v1 is sufficient to fix
>>>>>> the bug but I thought to be completely safe it is better to add
>>>>>> wait_freq_set() after changing min/max freq (and hence v3).
>>>>>>
>>>>>> I hope this clarifies things somewhat. If not please let me know.
>>>>>
>>>>> It clarifies a large chunk thank you.
>>>>>
>>>>> Another question which you missed is why is it needed for the max mode?
>>>>> There we allow a 15% error (more than 5% default). Is that not enough with
>>>>> SLPC?
>>>>
>>>> Yes I think you are right, should not be needed for the max mode. The real
>>>> issue is that efficient freq can exceed the max freq set but I think that
>>>> should not be an issue if the max is RP0.
>>>
>>> Ah okay, so SLPC not respecting max same as with min. I think the question
>>> of what is the use case for min/max controls in the SLPC world is the one to
>>> answer then.
>>>
>>> Regards,
>>>
>>> Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-12-16 15:39           ` Rodrigo Vivi
@ 2022-12-17  2:49             ` Dixit, Ashutosh
  2022-12-19  8:46               ` Tvrtko Ursulin
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-12-17  2:49 UTC (permalink / raw)
  To: Rodrigo Vivi; +Cc: igt-dev

On Fri, 16 Dec 2022 07:39:52 -0800, Rodrigo Vivi wrote:
> On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> >
> > On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> > > On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> > > >
> > > > On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > > > > On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> > > > > >
> > > > >
> > >
> > > Hi Tvrtko,
> > >
> > > Sorry for the delay in replying on this. We are discussing this but still
> > > no conclusion. I have replied what I can below, will update again after I
> > > know more. Wanted to send a reply before disappearing for the
> > > holidays. Please see below.
> > >
> > > > > Hi Tvrtko,
> > > > >
> > > > > I am only offering an overall clarification below first, not answering all
> > > > > of your points for now.
> > > > >
> > > > > >
> > > > > > On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > > > > > > With SLPC, even when we set the same min and max freq's, the requested and
> > > > > > > actual freq's can differ from the min/max freq set. For example "efficient
> > > > > > > freq" (when in effect) can override set min/max freq. In general FW is the
> > > > > > > final arbiter in determining freq and can override set values.
> > > > > > >
> > > > > > > Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > > > > > > reported by PMU not against the set freq's but against the requested freq
> > > > > > > reported in sysfs. Also add a delay after setting freq's to account for
> > > > > > > messaging delays in setting freq's in GuC.
> > > > > > >
> > > > > > > v2: Introduce a 100 ms delay after setting freq
> > > > > > > v3: Update commit message, code identical to v2
> > > > > > >
> > > > > > > Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > > > > > > Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > > > > > > Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > > > > > > Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > > > > > > ---
> > > > > > >     tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> > > > > > >     1 file changed, 15 insertions(+), 8 deletions(-)
> > > > > > >
> > > > > > > diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > > > > > > index f363db2ba13..02f6ae989b1 100644
> > > > > > > --- a/tests/i915/perf_pmu.c
> > > > > > > +++ b/tests/i915/perf_pmu.c
> > > > > > > @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> > > > > > >	igt_assert_lte(target, busy);
> > > > > > >     }
> > > > > > >     +/* Wait for GuC SLPC freq changes to take effect */
> > > > > > > +#define wait_freq_set()		usleep(100000)
> > > > > >
> > > > > > A future task of maybe adding a drop_caches flag to wait for this?
> > > > > >
> > > > > > Alternatively, if we have a sysfs or debugfs file which reflects the value
> > > > > > once GuC has processed it, keep reading it until it is updated? Even if to
> > > > > > something other than it was before the write, to handle the inability to
> > > > > > set the exact requested frequency sometimes?
> > > > > > >
> > > > > > > +
> > > > > > >     static void
> > > > > > >     test_frequency(int gem_fd)
> > > > > > >     {
> > > > > > > -	uint32_t min_freq, max_freq, boost_freq;
> > > > > > > +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> > > > > > >	uint64_t val[2], start[2], slept;
> > > > > > >	double min[2], max[2];
> > > > > > >	igt_spin_t *spin;
> > > > > > > @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> > > > > > >	 * Set GPU to min frequency and read PMU counters.
> > > > > > >	 */
> > > > > > >	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > >	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > >	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > > > > > > +	wait_freq_set();
> > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > >	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> > > > > > >		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > > > > > > effect */
> > > > > > > @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> > > > > > >		min[0] = 1e9*(val[0] - start[0]) / slept;
> > > > > > >	min[1] = 1e9*(val[1] - start[1]) / slept;
> > > > > > > +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> > > > > >
> > > > > > This really leaves a bad taste for me still, sorry. First of all, it may
> > > > > > make it more palatable if readback was immediately after wait_freq_set(),
> > > > > > or as part of it. Otherwise it seems like even more confused test and sysfs
> > > > > > ABI.
> > > > > >
> > > > > > Did we close on my question of whether we can make readback of
> > > > > > gt_max_freq_mhz actually represent the real max? Correct me please if I got
> > > > > > confused - with min freq checking here we write max=300 and get 350? If so
> > > > > > can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> > > > > > error?
> > > > >
> > > > > Did you read Vinay's email on v1 here:
> > > > >
> > > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> > > > >
> > > > > As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > > > > freq rather than the set 'min == max freq') and no amount of waiting will
> > > > > result in the requested freq being the same as the set 'min == max
> > > > > freq'. Also the efficient freq cannot be controlled from user space.
> > > >
> > > > Why is that not a break in our ABI? What is the point in allowing users to
> > > > write into this field if the desired value can be silently ignored?
> > >
> > > Yes, this is in discussion. FW is doing this and we need to see if FW can
> > > be made to obey the i915 ABI. IMHO what matters is performance per watt and
> > > if the ABI has to be tossed so be it. I will keep this list posted about
> > > what happens here. Merging any patches for these issues (GL #6806 and
> > > #6786) is blocked till the ABI breakage issue is resolved one way or
> > > another.
> >
> > Two thoughts here, one has been mentioned before.
> >
> > New one is that performance per watt perhaps does not matter (is not a
> > primary concern at least) for use cases where "users" (for some definition)
> > tweak the min/max in sysfs. Presumably in those cases user wants control,
> > for some reason, so as long as it is safe and within the limits why not give
> > it to them.
> >
> > Same argument from a different angle - what is the use case for allowing
> > min/max control which does not really do what it says on the tin?
> >
> > Old thought is this - if SLPC refuses to be controlled using our existing
> > controls then we could just say -ENODEV on attempts to modify them. Whether
> > for any input value, or some, depending on what SLPC can support (or wants
> > to support).
>
> The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
> has sudo and all the rights to disrespect what drivers (including SLPC as a
> a driver from PCODE perspective) are requesting.
>
> >
> > Important to note I am primarily focusing on the min freq here, which AFAIR
> > we concluded can behave that user writes in 300, but SLPC decides it will
> > only go as low as 350.
>
> The other caveat here with the efficient freq is that it is dynamic and PCODE
> decides it with runtime conditions. It may start with 300, then move to 350,
> then back to 300. i915 is only stashing the initial value.
>
> We could potentially update our rpe everytime any freq control is
> touched.

We thought of doing this but two points:

1. i915 now reads the same register as GuC for rpe but I never see the
   value in this register change.
2. Even if we could know the real rpe, the freq's might be set before a
   workload start running and rpe might change after the workload starts
   running. So knowing rpe before the workload might not be very useful.

I am of the opposite opinion that anyway the previous kernel ABI is
probably now irrecoverably bust, so we should give FW free reign and stop
disabling efficient freq the way we do now when min_freq < rpe_freq.

Cheers.
--
Ashutosh

> Just to highlight how bad this is, I've seen cases where rpe in runtime
> goes below the minimal fused freq (our rpn).
>
> > Because that is different from the existing, should
> > be well known, situation where we can write a max which cannot be achieved
> > due thermal throttling.
>
> yeap, usually when things got disrespected by PCODE there will be some thermal
> throttling information. But I have seen cases where this indicators simply do
> not exist. What complicate the api expectation to a very bad level.
>
> >
> > Perhaps we need a table, or a summary of some kind, which explains the
> > possible behaviours for both min and max settings.
>
> This is probably a good idea. At least document everything and what and when
> things can be disrespected, etc...
>
> (and update our rpe info dynamically)
>
> >
> > > > > (So in the example above min and max are set to 300 MHz and SLPC is
> > > > > requesting efficient freq which is 350 or 400 MHz. So min and max values
> > > > > are correct at 300 MHz and requested/actual freq will show 350/400 MHz).
> > > > >
> > > > > Let me also explain what happens in GuC FW when we change min/max freq:
> > > > > * i915 receives min/max freq from sysfs and sends a message to GuC to
> > > > >     change min/max freq.
> > > > > * The message itself might be queued behind other messages in the host to
> > > > >     GuC queue. i915 is still blocked in the sysfs handler.
> > > > > * Now when GuC receives the message interrupt, in its top-half handler it
> > > > >     validates the freq parameters and sends a message back to host saying the
> > > > >     freq change has been accepted. But min/max freq is actually changed later
> > > > >     in the bottom-half handler in GuC FW. So the top half only validates the
> > > > >     freq value.
> > > > > * When i915 receives the message from GuC top half, the sysfs handler
> > > > >     unblocks and starts reflecting the new min/max freq value.
> > > > >
> > > > > So the delay for the new min/max freq values to take effect is between
> > > > > returning from the sysfs handler and when the min/max freq are really set
> > > > > in GuC bottom half (an indeterminate possibly large amount of time if GuC
> > > > > is busy).
> > > >
> > > > Do we have a drop_caches flag which would force waiting of idle GuC, which
> > > > would cover the bottom half as well? Could we or should we add it if not?
> > > > Would beat the usleep approach..
> > >
> > > Agree, waiting for GuC to idle would do it, but not sure "how" to do it. As
> > > I said there is no notification after the bottom half is done applying the
> > > freq settings. We may need to schedule something after the freq setting
> > > host to guc command and wait for that to be done. A little bit out of my
> > > depth on this, let me know if you have any ideas here. I'll ask the GuC
> > > guys too. Currently we don't have a drop_caches flag for this.
> >
> > Okay, I was wondering if DROP_IDLE would do it, but perhaps this bottom half
> > can run independently from those flows so it was just a thought, not a
> > claim.
> >
> > > > > So as you see this introduces an indeterminacy due to to GuC interface not
> > > > > being synchronous. If this needs to be fixed, it needs to be fixed in GuC
> > > > > FW rather than waiting etc. in i915. GuC needs to implement another
> > > > > notification to host /after/ the min/max freq change has been applied in
> > > > > its bottom half. Doing this is not a priority for the GuC team at this
> > > > > point.
> > > >
> > > > Is there a method for querying the real min from GuC, which would maybe
> > > > work after waiting for GuC to idle?
> > >
> > > Will find out about this too. Afaik, at present querying the real min from
> > > GuC is not possible. As soon as the top half has completed the freq
> > > settings are updated so querying will return the new settings even though
> > > they might not have been applied by the bottom half yet.
> >
> > And it will return the written value, or the real one? Going back to the 300
> > written in, 350 actual min freq example. Presumably 300 and 350 SLPC picks
> > is completely invisible to i915 via the FW interface?
> >
> > > > > However, this bug is not due to these delays but due to GuC requesting the
> > > > > efficient freq instead of the set 'min == max freq' as mentioned above.
> > > > >
> > > > > If all these delays are confusing we can revert to v1 of the patch:
> > > > >
> > > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1
> > > > >
> > > > > v1 is simple and does not introduce these delays but just reads the
> > > > > requested freq from sysfs after the 500 ms PMU measurement but while the
> > > > > spinner is still running (which we do in v3 too). Without load SLPC will
> > > > > only request RPn so to compare with PMU measurement we must read the
> > > > > requested freq from sysfs with the spinner load. v1 is sufficient to fix
> > > > > the bug but I thought to be completely safe it is better to add
> > > > > wait_freq_set() after changing min/max freq (and hence v3).
> > > > >
> > > > > I hope this clarifies things somewhat. If not please let me know.
> > > >
> > > > It clarifies a large chunk thank you.
> > > >
> > > > Another question which you missed is why is it needed for the max mode?
> > > > There we allow a 15% error (more than 5% default). Is that not enough with
> > > > SLPC?
> > >
> > > Yes I think you are right, should not be needed for the max mode. The real
> > > issue is that efficient freq can exceed the max freq set but I think that
> > > should not be an issue if the max is RP0.
> >
> > Ah okay, so SLPC not respecting max same as with min. I think the question
> > of what is the use case for min/max controls in the SLPC world is the one to
> > answer then.
> >
> > Regards,
> >
> > Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-12-16  9:37         ` Tvrtko Ursulin
@ 2022-12-16 15:39           ` Rodrigo Vivi
  2022-12-17  2:49             ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Rodrigo Vivi @ 2022-12-16 15:39 UTC (permalink / raw)
  To: Tvrtko Ursulin; +Cc: igt-dev

On Fri, Dec 16, 2022 at 09:37:31AM +0000, Tvrtko Ursulin wrote:
> 
> On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> > On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
> > > 
> > > On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > > > On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> > > > > 
> > > > 
> > 
> > Hi Tvrtko,
> > 
> > Sorry for the delay in replying on this. We are discussing this but still
> > no conclusion. I have replied what I can below, will update again after I
> > know more. Wanted to send a reply before disappearing for the
> > holidays. Please see below.
> > 
> > > > Hi Tvrtko,
> > > > 
> > > > I am only offering an overall clarification below first, not answering all
> > > > of your points for now.
> > > > 
> > > > > 
> > > > > On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > > > > > With SLPC, even when we set the same min and max freq's, the requested and
> > > > > > actual freq's can differ from the min/max freq set. For example "efficient
> > > > > > freq" (when in effect) can override set min/max freq. In general FW is the
> > > > > > final arbiter in determining freq and can override set values.
> > > > > > 
> > > > > > Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > > > > > reported by PMU not against the set freq's but against the requested freq
> > > > > > reported in sysfs. Also add a delay after setting freq's to account for
> > > > > > messaging delays in setting freq's in GuC.
> > > > > > 
> > > > > > v2: Introduce a 100 ms delay after setting freq
> > > > > > v3: Update commit message, code identical to v2
> > > > > > 
> > > > > > Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > > > > > Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > > > > > Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > > > > > Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > > > > > ---
> > > > > >     tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> > > > > >     1 file changed, 15 insertions(+), 8 deletions(-)
> > > > > > 
> > > > > > diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > > > > > index f363db2ba13..02f6ae989b1 100644
> > > > > > --- a/tests/i915/perf_pmu.c
> > > > > > +++ b/tests/i915/perf_pmu.c
> > > > > > @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> > > > > > 	igt_assert_lte(target, busy);
> > > > > >     }
> > > > > >     +/* Wait for GuC SLPC freq changes to take effect */
> > > > > > +#define wait_freq_set()		usleep(100000)
> > > > > 
> > > > > A future task of maybe adding a drop_caches flag to wait for this?
> > > > > 
> > > > > Alternatively, if we have a sysfs or debugfs file which reflects the value
> > > > > once GuC has processed it, keep reading it until it is updated? Even if to
> > > > > something other than it was before the write, to handle the inability to
> > > > > set the exact requested frequency sometimes?
> > > > > > 
> > > > > > +
> > > > > >     static void
> > > > > >     test_frequency(int gem_fd)
> > > > > >     {
> > > > > > -	uint32_t min_freq, max_freq, boost_freq;
> > > > > > +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> > > > > > 	uint64_t val[2], start[2], slept;
> > > > > > 	double min[2], max[2];
> > > > > > 	igt_spin_t *spin;
> > > > > > @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> > > > > > 	 * Set GPU to min frequency and read PMU counters.
> > > > > > 	 */
> > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > > > > > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > > > > > +	wait_freq_set();
> > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > > > > > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> > > > > > 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> > > > > > 		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > > > > > effect */
> > > > > > @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> > > > > > 		min[0] = 1e9*(val[0] - start[0]) / slept;
> > > > > > 	min[1] = 1e9*(val[1] - start[1]) / slept;
> > > > > > +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> > > > > 
> > > > > This really leaves a bad taste for me still, sorry. First of all, it may
> > > > > make it more palatable if readback was immediately after wait_freq_set(),
> > > > > or as part of it. Otherwise it seems like even more confused test and sysfs
> > > > > ABI.
> > > > > 
> > > > > Did we close on my question of whether we can make readback of
> > > > > gt_max_freq_mhz actually represent the real max? Correct me please if I got
> > > > > confused - with min freq checking here we write max=300 and get 350? If so
> > > > > can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> > > > > error?
> > > > 
> > > > Did you read Vinay's email on v1 here:
> > > > 
> > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> > > > 
> > > > As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > > > freq rather than the set 'min == max freq') and no amount of waiting will
> > > > result in the requested freq being the same as the set 'min == max
> > > > freq'. Also the efficient freq cannot be controlled from user space.
> > > 
> > > Why is that not a break in our ABI? What is the point in allowing users to
> > > write into this field if the desired value can be silently ignored?
> > 
> > Yes, this is in discussion. FW is doing this and we need to see if FW can
> > be made to obey the i915 ABI. IMHO what matters is performance per watt and
> > if the ABI has to be tossed so be it. I will keep this list posted about
> > what happens here. Merging any patches for these issues (GL #6806 and
> > #6786) is blocked till the ABI breakage issue is resolved one way or
> > another.
> 
> Two thoughts here, one has been mentioned before.
> 
> New one is that performance per watt perhaps does not matter (is not a
> primary concern at least) for use cases where "users" (for some definition)
> tweak the min/max in sysfs. Presumably in those cases user wants control,
> for some reason, so as long as it is safe and within the limits why not give
> it to them.
> 
> Same argument from a different angle - what is the use case for allowing
> min/max control which does not really do what it says on the tin?
> 
> Old thought is this - if SLPC refuses to be controlled using our existing
> controls then we could just say -ENODEV on attempts to modify them. Whether
> for any input value, or some, depending on what SLPC can support (or wants
> to support).

The problem is not just	SLPC. The freq is ultimately decided by PCODE and it
has sudo and all the rights to disrespect what drivers (including SLPC as a
a driver from PCODE perspective) are requesting.

> 
> Important to note I am primarily focusing on the min freq here, which AFAIR
> we concluded can behave that user writes in 300, but SLPC decides it will
> only go as low as 350.

The other caveat here with the efficient freq is that it is dynamic and PCODE
decides it with runtime conditions. It may start with 300, then move to 350,
then back to 300. i915 is only stashing the initial value.

We could potentially update our rpe everytime any freq control is touched.

Just to highlight how bad this is, I've seen cases where rpe in runtime
goes below the minimal fused freq (our rpn).

> Because that is different from the existing, should
> be well known, situation where we can write a max which cannot be achieved
> due thermal throttling.

yeap, usually when things got disrespected by PCODE there will be some thermal
throttling information. But I have seen cases where this indicators simply do
not exist. What complicate the api expectation to a very bad level.

> 
> Perhaps we need a table, or a summary of some kind, which explains the
> possible behaviours for both min and max settings.

This is probably a good idea. At least document everything and what and when
things can be disrespected, etc...

(and update our rpe info dynamically)

> 
> > > > (So in the example above min and max are set to 300 MHz and SLPC is
> > > > requesting efficient freq which is 350 or 400 MHz. So min and max values
> > > > are correct at 300 MHz and requested/actual freq will show 350/400 MHz).
> > > > 
> > > > Let me also explain what happens in GuC FW when we change min/max freq:
> > > > * i915 receives min/max freq from sysfs and sends a message to GuC to
> > > >     change min/max freq.
> > > > * The message itself might be queued behind other messages in the host to
> > > >     GuC queue. i915 is still blocked in the sysfs handler.
> > > > * Now when GuC receives the message interrupt, in its top-half handler it
> > > >     validates the freq parameters and sends a message back to host saying the
> > > >     freq change has been accepted. But min/max freq is actually changed later
> > > >     in the bottom-half handler in GuC FW. So the top half only validates the
> > > >     freq value.
> > > > * When i915 receives the message from GuC top half, the sysfs handler
> > > >     unblocks and starts reflecting the new min/max freq value.
> > > > 
> > > > So the delay for the new min/max freq values to take effect is between
> > > > returning from the sysfs handler and when the min/max freq are really set
> > > > in GuC bottom half (an indeterminate possibly large amount of time if GuC
> > > > is busy).
> > > 
> > > Do we have a drop_caches flag which would force waiting of idle GuC, which
> > > would cover the bottom half as well? Could we or should we add it if not?
> > > Would beat the usleep approach..
> > 
> > Agree, waiting for GuC to idle would do it, but not sure "how" to do it. As
> > I said there is no notification after the bottom half is done applying the
> > freq settings. We may need to schedule something after the freq setting
> > host to guc command and wait for that to be done. A little bit out of my
> > depth on this, let me know if you have any ideas here. I'll ask the GuC
> > guys too. Currently we don't have a drop_caches flag for this.
> 
> Okay, I was wondering if DROP_IDLE would do it, but perhaps this bottom half
> can run independently from those flows so it was just a thought, not a
> claim.
> 
> > > > So as you see this introduces an indeterminacy due to to GuC interface not
> > > > being synchronous. If this needs to be fixed, it needs to be fixed in GuC
> > > > FW rather than waiting etc. in i915. GuC needs to implement another
> > > > notification to host /after/ the min/max freq change has been applied in
> > > > its bottom half. Doing this is not a priority for the GuC team at this
> > > > point.
> > > 
> > > Is there a method for querying the real min from GuC, which would maybe
> > > work after waiting for GuC to idle?
> > 
> > Will find out about this too. Afaik, at present querying the real min from
> > GuC is not possible. As soon as the top half has completed the freq
> > settings are updated so querying will return the new settings even though
> > they might not have been applied by the bottom half yet.
> 
> And it will return the written value, or the real one? Going back to the 300
> written in, 350 actual min freq example. Presumably 300 and 350 SLPC picks
> is completely invisible to i915 via the FW interface?
> 
> > > > However, this bug is not due to these delays but due to GuC requesting the
> > > > efficient freq instead of the set 'min == max freq' as mentioned above.
> > > > 
> > > > If all these delays are confusing we can revert to v1 of the patch:
> > > > 
> > > > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1
> > > > 
> > > > v1 is simple and does not introduce these delays but just reads the
> > > > requested freq from sysfs after the 500 ms PMU measurement but while the
> > > > spinner is still running (which we do in v3 too). Without load SLPC will
> > > > only request RPn so to compare with PMU measurement we must read the
> > > > requested freq from sysfs with the spinner load. v1 is sufficient to fix
> > > > the bug but I thought to be completely safe it is better to add
> > > > wait_freq_set() after changing min/max freq (and hence v3).
> > > > 
> > > > I hope this clarifies things somewhat. If not please let me know.
> > > 
> > > It clarifies a large chunk thank you.
> > > 
> > > Another question which you missed is why is it needed for the max mode?
> > > There we allow a 15% error (more than 5% default). Is that not enough with
> > > SLPC?
> > 
> > Yes I think you are right, should not be needed for the max mode. The real
> > issue is that efficient freq can exceed the max freq set but I think that
> > should not be an issue if the max is RP0.
> 
> Ah okay, so SLPC not respecting max same as with min. I think the question
> of what is the use case for min/max controls in the SLPC world is the one to
> answer then.
> 
> Regards,
> 
> Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-12-16  6:21       ` Dixit, Ashutosh
@ 2022-12-16  9:37         ` Tvrtko Ursulin
  2022-12-16 15:39           ` Rodrigo Vivi
  0 siblings, 1 reply; 34+ messages in thread
From: Tvrtko Ursulin @ 2022-12-16  9:37 UTC (permalink / raw)
  To: Dixit, Ashutosh; +Cc: igt-dev


On 16/12/2022 06:21, Dixit, Ashutosh wrote:
> On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
>>
>> On 23/11/2022 06:03, Dixit, Ashutosh wrote:
>>> On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
>>>>
>>>
> 
> Hi Tvrtko,
> 
> Sorry for the delay in replying on this. We are discussing this but still
> no conclusion. I have replied what I can below, will update again after I
> know more. Wanted to send a reply before disappearing for the
> holidays. Please see below.
> 
>>> Hi Tvrtko,
>>>
>>> I am only offering an overall clarification below first, not answering all
>>> of your points for now.
>>>
>>>>
>>>> On 19/11/2022 02:00, Ashutosh Dixit wrote:
>>>>> With SLPC, even when we set the same min and max freq's, the requested and
>>>>> actual freq's can differ from the min/max freq set. For example "efficient
>>>>> freq" (when in effect) can override set min/max freq. In general FW is the
>>>>> final arbiter in determining freq and can override set values.
>>>>>
>>>>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
>>>>> reported by PMU not against the set freq's but against the requested freq
>>>>> reported in sysfs. Also add a delay after setting freq's to account for
>>>>> messaging delays in setting freq's in GuC.
>>>>>
>>>>> v2: Introduce a 100 ms delay after setting freq
>>>>> v3: Update commit message, code identical to v2
>>>>>
>>>>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
>>>>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
>>>>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
>>>>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
>>>>> ---
>>>>>     tests/i915/perf_pmu.c | 23 +++++++++++++++--------
>>>>>     1 file changed, 15 insertions(+), 8 deletions(-)
>>>>>
>>>>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
>>>>> index f363db2ba13..02f6ae989b1 100644
>>>>> --- a/tests/i915/perf_pmu.c
>>>>> +++ b/tests/i915/perf_pmu.c
>>>>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
>>>>> 	igt_assert_lte(target, busy);
>>>>>     }
>>>>>     +/* Wait for GuC SLPC freq changes to take effect */
>>>>> +#define wait_freq_set()		usleep(100000)
>>>>
>>>> A future task of maybe adding a drop_caches flag to wait for this?
>>>>
>>>> Alternatively, if we have a sysfs or debugfs file which reflects the value
>>>> once GuC has processed it, keep reading it until it is updated? Even if to
>>>> something other than it was before the write, to handle the inability to
>>>> set the exact requested frequency sometimes?
>>>>>
>>>>> +
>>>>>     static void
>>>>>     test_frequency(int gem_fd)
>>>>>     {
>>>>> -	uint32_t min_freq, max_freq, boost_freq;
>>>>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
>>>>> 	uint64_t val[2], start[2], slept;
>>>>> 	double min[2], max[2];
>>>>> 	igt_spin_t *spin;
>>>>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
>>>>> 	 * Set GPU to min frequency and read PMU counters.
>>>>> 	 */
>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
>>>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
>>>>> +	wait_freq_set();
>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>>>> 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
>>>>> 		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
>>>>> effect */
>>>>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
>>>>> 		min[0] = 1e9*(val[0] - start[0]) / slept;
>>>>> 	min[1] = 1e9*(val[1] - start[1]) / slept;
>>>>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>>>>
>>>> This really leaves a bad taste for me still, sorry. First of all, it may
>>>> make it more palatable if readback was immediately after wait_freq_set(),
>>>> or as part of it. Otherwise it seems like even more confused test and sysfs
>>>> ABI.
>>>>
>>>> Did we close on my question of whether we can make readback of
>>>> gt_max_freq_mhz actually represent the real max? Correct me please if I got
>>>> confused - with min freq checking here we write max=300 and get 350? If so
>>>> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
>>>> error?
>>>
>>> Did you read Vinay's email on v1 here:
>>>
>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
>>>
>>> As Vinay says, SLPC is requesting an entirely different freq (the effecient
>>> freq rather than the set 'min == max freq') and no amount of waiting will
>>> result in the requested freq being the same as the set 'min == max
>>> freq'. Also the efficient freq cannot be controlled from user space.
>>
>> Why is that not a break in our ABI? What is the point in allowing users to
>> write into this field if the desired value can be silently ignored?
> 
> Yes, this is in discussion. FW is doing this and we need to see if FW can
> be made to obey the i915 ABI. IMHO what matters is performance per watt and
> if the ABI has to be tossed so be it. I will keep this list posted about
> what happens here. Merging any patches for these issues (GL #6806 and
> #6786) is blocked till the ABI breakage issue is resolved one way or
> another.

Two thoughts here, one has been mentioned before.

New one is that performance per watt perhaps does not matter (is not a 
primary concern at least) for use cases where "users" (for some 
definition) tweak the min/max in sysfs. Presumably in those cases user 
wants control, for some reason, so as long as it is safe and within the 
limits why not give it to them.

Same argument from a different angle - what is the use case for allowing 
min/max control which does not really do what it says on the tin?

Old thought is this - if SLPC refuses to be controlled using our 
existing controls then we could just say -ENODEV on attempts to modify 
them. Whether for any input value, or some, depending on what SLPC can 
support (or wants to support).

Important to note I am primarily focusing on the min freq here, which 
AFAIR we concluded can behave that user writes in 300, but SLPC decides 
it will only go as low as 350. Because that is different from the 
existing, should be well known, situation where we can write a max which 
cannot be achieved due thermal throttling.

Perhaps we need a table, or a summary of some kind, which explains the 
possible behaviours for both min and max settings.

>>> (So in the example above min and max are set to 300 MHz and SLPC is
>>> requesting efficient freq which is 350 or 400 MHz. So min and max values
>>> are correct at 300 MHz and requested/actual freq will show 350/400 MHz).
>>>
>>> Let me also explain what happens in GuC FW when we change min/max freq:
>>> * i915 receives min/max freq from sysfs and sends a message to GuC to
>>>     change min/max freq.
>>> * The message itself might be queued behind other messages in the host to
>>>     GuC queue. i915 is still blocked in the sysfs handler.
>>> * Now when GuC receives the message interrupt, in its top-half handler it
>>>     validates the freq parameters and sends a message back to host saying the
>>>     freq change has been accepted. But min/max freq is actually changed later
>>>     in the bottom-half handler in GuC FW. So the top half only validates the
>>>     freq value.
>>> * When i915 receives the message from GuC top half, the sysfs handler
>>>     unblocks and starts reflecting the new min/max freq value.
>>>
>>> So the delay for the new min/max freq values to take effect is between
>>> returning from the sysfs handler and when the min/max freq are really set
>>> in GuC bottom half (an indeterminate possibly large amount of time if GuC
>>> is busy).
>>
>> Do we have a drop_caches flag which would force waiting of idle GuC, which
>> would cover the bottom half as well? Could we or should we add it if not?
>> Would beat the usleep approach..
> 
> Agree, waiting for GuC to idle would do it, but not sure "how" to do it. As
> I said there is no notification after the bottom half is done applying the
> freq settings. We may need to schedule something after the freq setting
> host to guc command and wait for that to be done. A little bit out of my
> depth on this, let me know if you have any ideas here. I'll ask the GuC
> guys too. Currently we don't have a drop_caches flag for this.

Okay, I was wondering if DROP_IDLE would do it, but perhaps this bottom 
half can run independently from those flows so it was just a thought, 
not a claim.

>>> So as you see this introduces an indeterminacy due to to GuC interface not
>>> being synchronous. If this needs to be fixed, it needs to be fixed in GuC
>>> FW rather than waiting etc. in i915. GuC needs to implement another
>>> notification to host /after/ the min/max freq change has been applied in
>>> its bottom half. Doing this is not a priority for the GuC team at this
>>> point.
>>
>> Is there a method for querying the real min from GuC, which would maybe
>> work after waiting for GuC to idle?
> 
> Will find out about this too. Afaik, at present querying the real min from
> GuC is not possible. As soon as the top half has completed the freq
> settings are updated so querying will return the new settings even though
> they might not have been applied by the bottom half yet.

And it will return the written value, or the real one? Going back to the 
300 written in, 350 actual min freq example. Presumably 300 and 350 SLPC 
picks is completely invisible to i915 via the FW interface?

>>> However, this bug is not due to these delays but due to GuC requesting the
>>> efficient freq instead of the set 'min == max freq' as mentioned above.
>>>
>>> If all these delays are confusing we can revert to v1 of the patch:
>>>
>>> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1
>>>
>>> v1 is simple and does not introduce these delays but just reads the
>>> requested freq from sysfs after the 500 ms PMU measurement but while the
>>> spinner is still running (which we do in v3 too). Without load SLPC will
>>> only request RPn so to compare with PMU measurement we must read the
>>> requested freq from sysfs with the spinner load. v1 is sufficient to fix
>>> the bug but I thought to be completely safe it is better to add
>>> wait_freq_set() after changing min/max freq (and hence v3).
>>>
>>> I hope this clarifies things somewhat. If not please let me know.
>>
>> It clarifies a large chunk thank you.
>>
>> Another question which you missed is why is it needed for the max mode?
>> There we allow a 15% error (more than 5% default). Is that not enough with
>> SLPC?
> 
> Yes I think you are right, should not be needed for the max mode. The real
> issue is that efficient freq can exceed the max freq set but I think that
> should not be an issue if the max is RP0.

Ah okay, so SLPC not respecting max same as with min. I think the 
question of what is the use case for min/max controls in the SLPC world 
is the one to answer then.

Regards,

Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-24 12:42     ` Tvrtko Ursulin
@ 2022-12-16  6:21       ` Dixit, Ashutosh
  2022-12-16  9:37         ` Tvrtko Ursulin
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-12-16  6:21 UTC (permalink / raw)
  To: Tvrtko Ursulin; +Cc: igt-dev

On Thu, 24 Nov 2022 04:42:08 -0800, Tvrtko Ursulin wrote:
>
> On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> > On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
> >>
> >

Hi Tvrtko,

Sorry for the delay in replying on this. We are discussing this but still
no conclusion. I have replied what I can below, will update again after I
know more. Wanted to send a reply before disappearing for the
holidays. Please see below.

> > Hi Tvrtko,
> >
> > I am only offering an overall clarification below first, not answering all
> > of your points for now.
> >
> >>
> >> On 19/11/2022 02:00, Ashutosh Dixit wrote:
> >>> With SLPC, even when we set the same min and max freq's, the requested and
> >>> actual freq's can differ from the min/max freq set. For example "efficient
> >>> freq" (when in effect) can override set min/max freq. In general FW is the
> >>> final arbiter in determining freq and can override set values.
> >>>
> >>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> >>> reported by PMU not against the set freq's but against the requested freq
> >>> reported in sysfs. Also add a delay after setting freq's to account for
> >>> messaging delays in setting freq's in GuC.
> >>>
> >>> v2: Introduce a 100 ms delay after setting freq
> >>> v3: Update commit message, code identical to v2
> >>>
> >>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> >>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> >>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> >>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> >>> ---
> >>>    tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> >>>    1 file changed, 15 insertions(+), 8 deletions(-)
> >>>
> >>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> >>> index f363db2ba13..02f6ae989b1 100644
> >>> --- a/tests/i915/perf_pmu.c
> >>> +++ b/tests/i915/perf_pmu.c
> >>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> >>>	igt_assert_lte(target, busy);
> >>>    }
> >>>    +/* Wait for GuC SLPC freq changes to take effect */
> >>> +#define wait_freq_set()		usleep(100000)
> >>
> >> A future task of maybe adding a drop_caches flag to wait for this?
> >>
> >> Alternatively, if we have a sysfs or debugfs file which reflects the value
> >> once GuC has processed it, keep reading it until it is updated? Even if to
> >> something other than it was before the write, to handle the inability to
> >> set the exact requested frequency sometimes?
> >>>
> >>> +
> >>>    static void
> >>>    test_frequency(int gem_fd)
> >>>    {
> >>> -	uint32_t min_freq, max_freq, boost_freq;
> >>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> >>>	uint64_t val[2], start[2], slept;
> >>>	double min[2], max[2];
> >>>	igt_spin_t *spin;
> >>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> >>>	 * Set GPU to min frequency and read PMU counters.
> >>>	 */
> >>>	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> >>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> >>>	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> >>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> >>>	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> >>> +	wait_freq_set();
> >>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> >>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> >>>	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> >>>		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> >>> effect */
> >>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> >>>		min[0] = 1e9*(val[0] - start[0]) / slept;
> >>>	min[1] = 1e9*(val[1] - start[1]) / slept;
> >>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> >>
> >> This really leaves a bad taste for me still, sorry. First of all, it may
> >> make it more palatable if readback was immediately after wait_freq_set(),
> >> or as part of it. Otherwise it seems like even more confused test and sysfs
> >> ABI.
> >>
> >> Did we close on my question of whether we can make readback of
> >> gt_max_freq_mhz actually represent the real max? Correct me please if I got
> >> confused - with min freq checking here we write max=300 and get 350? If so
> >> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> >> error?
> >
> > Did you read Vinay's email on v1 here:
> >
> > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> >
> > As Vinay says, SLPC is requesting an entirely different freq (the effecient
> > freq rather than the set 'min == max freq') and no amount of waiting will
> > result in the requested freq being the same as the set 'min == max
> > freq'. Also the efficient freq cannot be controlled from user space.
>
> Why is that not a break in our ABI? What is the point in allowing users to
> write into this field if the desired value can be silently ignored?

Yes, this is in discussion. FW is doing this and we need to see if FW can
be made to obey the i915 ABI. IMHO what matters is performance per watt and
if the ABI has to be tossed so be it. I will keep this list posted about
what happens here. Merging any patches for these issues (GL #6806 and
#6786) is blocked till the ABI breakage issue is resolved one way or
another.

>
> > (So in the example above min and max are set to 300 MHz and SLPC is
> > requesting efficient freq which is 350 or 400 MHz. So min and max values
> > are correct at 300 MHz and requested/actual freq will show 350/400 MHz).
> >
> > Let me also explain what happens in GuC FW when we change min/max freq:
> > * i915 receives min/max freq from sysfs and sends a message to GuC to
> >    change min/max freq.
> > * The message itself might be queued behind other messages in the host to
> >    GuC queue. i915 is still blocked in the sysfs handler.
> > * Now when GuC receives the message interrupt, in its top-half handler it
> >    validates the freq parameters and sends a message back to host saying the
> >    freq change has been accepted. But min/max freq is actually changed later
> >    in the bottom-half handler in GuC FW. So the top half only validates the
> >    freq value.
> > * When i915 receives the message from GuC top half, the sysfs handler
> >    unblocks and starts reflecting the new min/max freq value.
> >
> > So the delay for the new min/max freq values to take effect is between
> > returning from the sysfs handler and when the min/max freq are really set
> > in GuC bottom half (an indeterminate possibly large amount of time if GuC
> > is busy).
>
> Do we have a drop_caches flag which would force waiting of idle GuC, which
> would cover the bottom half as well? Could we or should we add it if not?
> Would beat the usleep approach..

Agree, waiting for GuC to idle would do it, but not sure "how" to do it. As
I said there is no notification after the bottom half is done applying the
freq settings. We may need to schedule something after the freq setting
host to guc command and wait for that to be done. A little bit out of my
depth on this, let me know if you have any ideas here. I'll ask the GuC
guys too. Currently we don't have a drop_caches flag for this.

>
> > So as you see this introduces an indeterminacy due to to GuC interface not
> > being synchronous. If this needs to be fixed, it needs to be fixed in GuC
> > FW rather than waiting etc. in i915. GuC needs to implement another
> > notification to host /after/ the min/max freq change has been applied in
> > its bottom half. Doing this is not a priority for the GuC team at this
> > point.
>
> Is there a method for querying the real min from GuC, which would maybe
> work after waiting for GuC to idle?

Will find out about this too. Afaik, at present querying the real min from
GuC is not possible. As soon as the top half has completed the freq
settings are updated so querying will return the new settings even though
they might not have been applied by the bottom half yet.

>
> >
> > However, this bug is not due to these delays but due to GuC requesting the
> > efficient freq instead of the set 'min == max freq' as mentioned above.
> >
> > If all these delays are confusing we can revert to v1 of the patch:
> >
> > https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1
> >
> > v1 is simple and does not introduce these delays but just reads the
> > requested freq from sysfs after the 500 ms PMU measurement but while the
> > spinner is still running (which we do in v3 too). Without load SLPC will
> > only request RPn so to compare with PMU measurement we must read the
> > requested freq from sysfs with the spinner load. v1 is sufficient to fix
> > the bug but I thought to be completely safe it is better to add
> > wait_freq_set() after changing min/max freq (and hence v3).
> >
> > I hope this clarifies things somewhat. If not please let me know.
>
> It clarifies a large chunk thank you.
>
> Another question which you missed is why is it needed for the max mode?
> There we allow a 15% error (more than 5% default). Is that not enough with
> SLPC?

Yes I think you are right, should not be needed for the max mode. The real
issue is that efficient freq can exceed the max freq set but I think that
should not be an issue if the max is RP0.

Thanks.
--
Ashutosh


> >
> >>>	igt_spin_free(gem_fd, spin);
> >>>	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
> >>> @@ -1597,11 +1602,11 @@ test_frequency(int gem_fd)
> >>>	 * Set GPU to max frequency and read PMU counters.
> >>>	 */
> >>>	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", max_freq));
> >>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
> >>>	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", boost_freq));
> >>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
> >>> -
> >>>	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", max_freq));
> >>> +	wait_freq_set();
> >>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
> >>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
> >>>	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == max_freq);
> >>>		gem_quiescent_gpu(gem_fd);
> >>> @@ -1613,6 +1618,7 @@ test_frequency(int gem_fd)
> >>>		max[0] = 1e9*(val[0] - start[0]) / slept;
> >>>	max[1] = 1e9*(val[1] - start[1]) / slept;
> >>> +	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> >>
> >> For the max freq use case we had 15% downward tolerance to account for
> >> thermally challenged.
> >>
> >> a) Is that not enough with SLPC? Like there is a new "failure" mode?
> >>
> >> b) Is the extra 10% needed with the sysfs readback added? Aka do we need
> >> both?
> >>
> >>>		igt_spin_free(gem_fd, spin);
> >>>	gem_quiescent_gpu(gem_fd);
> >>> @@ -1621,6 +1627,7 @@ test_frequency(int gem_fd)
> >>>	 * Restore min/max.
> >>>	 */
> >>>	igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq);
> >>> +	wait_freq_set();
> >>>	if (igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") != min_freq)
> >>>		igt_warn("Unable to restore min frequency to saved value [%u MHz], now %u MHz\n",
> >>>			 min_freq, igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz"));
> >>> @@ -1633,12 +1640,12 @@ test_frequency(int gem_fd)
> >>>	igt_info("Max frequency: requested %.1f, actual %.1f\n",
> >>>		 max[0], max[1]);
> >>>    -	assert_within_epsilon(min[0], min_freq, tolerance);
> >>> +	assert_within_epsilon(min[0], min_req, tolerance);
> >>>	/*
> >>>	 * On thermally throttled devices we cannot be sure maximum frequency
> >>>	 * can be reached so use larger tolerance downards.
> >>>	 */
> >>> -	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
> >>> +	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);
> >>
> >>
> >> This 0.15f is what I was talking about above.
> >>
> >> Regards,
> >>
> >> Tvrtko
> >>
> >>>    }
> >>>      static void

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-23  6:03   ` Dixit, Ashutosh
@ 2022-11-24 12:42     ` Tvrtko Ursulin
  2022-12-16  6:21       ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Tvrtko Ursulin @ 2022-11-24 12:42 UTC (permalink / raw)
  To: Dixit, Ashutosh; +Cc: igt-dev


On 23/11/2022 06:03, Dixit, Ashutosh wrote:
> On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
>>
> 
> Hi Tvrtko,
> 
> I am only offering an overall clarification below first, not answering all
> of your points for now.
> 
>>
>> On 19/11/2022 02:00, Ashutosh Dixit wrote:
>>> With SLPC, even when we set the same min and max freq's, the requested and
>>> actual freq's can differ from the min/max freq set. For example "efficient
>>> freq" (when in effect) can override set min/max freq. In general FW is the
>>> final arbiter in determining freq and can override set values.
>>>
>>> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
>>> reported by PMU not against the set freq's but against the requested freq
>>> reported in sysfs. Also add a delay after setting freq's to account for
>>> messaging delays in setting freq's in GuC.
>>>
>>> v2: Introduce a 100 ms delay after setting freq
>>> v3: Update commit message, code identical to v2
>>>
>>> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
>>> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
>>> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
>>> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
>>> ---
>>>    tests/i915/perf_pmu.c | 23 +++++++++++++++--------
>>>    1 file changed, 15 insertions(+), 8 deletions(-)
>>>
>>> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
>>> index f363db2ba13..02f6ae989b1 100644
>>> --- a/tests/i915/perf_pmu.c
>>> +++ b/tests/i915/perf_pmu.c
>>> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
>>> 	igt_assert_lte(target, busy);
>>>    }
>>>    +/* Wait for GuC SLPC freq changes to take effect */
>>> +#define wait_freq_set()		usleep(100000)
>>
>> A future task of maybe adding a drop_caches flag to wait for this?
>>
>> Alternatively, if we have a sysfs or debugfs file which reflects the value
>> once GuC has processed it, keep reading it until it is updated? Even if to
>> something other than it was before the write, to handle the inability to
>> set the exact requested frequency sometimes?
>>>
>>> +
>>>    static void
>>>    test_frequency(int gem_fd)
>>>    {
>>> -	uint32_t min_freq, max_freq, boost_freq;
>>> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
>>> 	uint64_t val[2], start[2], slept;
>>> 	double min[2], max[2];
>>> 	igt_spin_t *spin;
>>> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
>>> 	 * Set GPU to min frequency and read PMU counters.
>>> 	 */
>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
>>> +	wait_freq_set();
>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>>> 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
>>> 		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
>>> effect */
>>> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
>>> 		min[0] = 1e9*(val[0] - start[0]) / slept;
>>> 	min[1] = 1e9*(val[1] - start[1]) / slept;
>>> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>>
>> This really leaves a bad taste for me still, sorry. First of all, it may
>> make it more palatable if readback was immediately after wait_freq_set(),
>> or as part of it. Otherwise it seems like even more confused test and sysfs
>> ABI.
>>
>> Did we close on my question of whether we can make readback of
>> gt_max_freq_mhz actually represent the real max? Correct me please if I got
>> confused - with min freq checking here we write max=300 and get 350? If so
>> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
>> error?
> 
> Did you read Vinay's email on v1 here:
> 
> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571
> 
> As Vinay says, SLPC is requesting an entirely different freq (the effecient
> freq rather than the set 'min == max freq') and no amount of waiting will
> result in the requested freq being the same as the set 'min == max
> freq'. Also the efficient freq cannot be controlled from user space.

Why is that not a break in our ABI? What is the point in allowing users 
to write into this field if the desired value can be silently ignored?

> (So in the example above min and max are set to 300 MHz and SLPC is
> requesting efficient freq which is 350 or 400 MHz. So min and max values
> are correct at 300 MHz and requested/actual freq will show 350/400 MHz).
> 
> Let me also explain what happens in GuC FW when we change min/max freq:
> * i915 receives min/max freq from sysfs and sends a message to GuC to
>    change min/max freq.
> * The message itself might be queued behind other messages in the host to
>    GuC queue. i915 is still blocked in the sysfs handler.
> * Now when GuC receives the message interrupt, in its top-half handler it
>    validates the freq parameters and sends a message back to host saying the
>    freq change has been accepted. But min/max freq is actually changed later
>    in the bottom-half handler in GuC FW. So the top half only validates the
>    freq value.
> * When i915 receives the message from GuC top half, the sysfs handler
>    unblocks and starts reflecting the new min/max freq value.
> 
> So the delay for the new min/max freq values to take effect is between
> returning from the sysfs handler and when the min/max freq are really set
> in GuC bottom half (an indeterminate possibly large amount of time if GuC
> is busy).

Do we have a drop_caches flag which would force waiting of idle GuC, 
which would cover the bottom half as well? Could we or should we add it 
if not? Would beat the usleep approach..

> So as you see this introduces an indeterminacy due to to GuC interface not
> being synchronous. If this needs to be fixed, it needs to be fixed in GuC
> FW rather than waiting etc. in i915. GuC needs to implement another
> notification to host /after/ the min/max freq change has been applied in
> its bottom half. Doing this is not a priority for the GuC team at this
> point.

Is there a method for querying the real min from GuC, which would maybe 
work after waiting for GuC to idle?

> 
> However, this bug is not due to these delays but due to GuC requesting the
> efficient freq instead of the set 'min == max freq' as mentioned above.
> 
> If all these delays are confusing we can revert to v1 of the patch:
> 
> https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1
> 
> v1 is simple and does not introduce these delays but just reads the
> requested freq from sysfs after the 500 ms PMU measurement but while the
> spinner is still running (which we do in v3 too). Without load SLPC will
> only request RPn so to compare with PMU measurement we must read the
> requested freq from sysfs with the spinner load. v1 is sufficient to fix
> the bug but I thought to be completely safe it is better to add
> wait_freq_set() after changing min/max freq (and hence v3).
> 
> I hope this clarifies things somewhat. If not please let me know.

It clarifies a large chunk thank you.

Another question which you missed is why is it needed for the max mode? 
There we allow a 15% error (more than 5% default). Is that not enough 
with SLPC?

Regards,

Tvrtko

> 
> Thanks.
> --
> Ashutosh
> 
> 
>>> 	igt_spin_free(gem_fd, spin);
>>> 	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
>>> @@ -1597,11 +1602,11 @@ test_frequency(int gem_fd)
>>> 	 * Set GPU to max frequency and read PMU counters.
>>> 	 */
>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", max_freq));
>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", boost_freq));
>>> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
>>> -
>>> 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", max_freq));
>>> +	wait_freq_set();
>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
>>> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
>>> 	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == max_freq);
>>> 		gem_quiescent_gpu(gem_fd);
>>> @@ -1613,6 +1618,7 @@ test_frequency(int gem_fd)
>>> 		max[0] = 1e9*(val[0] - start[0]) / slept;
>>> 	max[1] = 1e9*(val[1] - start[1]) / slept;
>>> +	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>>
>> For the max freq use case we had 15% downward tolerance to account for
>> thermally challenged.
>>
>> a) Is that not enough with SLPC? Like there is a new "failure" mode?
>>
>> b) Is the extra 10% needed with the sysfs readback added? Aka do we need
>> both?
>>
>>> 		igt_spin_free(gem_fd, spin);
>>> 	gem_quiescent_gpu(gem_fd);
>>> @@ -1621,6 +1627,7 @@ test_frequency(int gem_fd)
>>> 	 * Restore min/max.
>>> 	 */
>>> 	igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq);
>>> +	wait_freq_set();
>>> 	if (igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") != min_freq)
>>> 		igt_warn("Unable to restore min frequency to saved value [%u MHz], now %u MHz\n",
>>> 			 min_freq, igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz"));
>>> @@ -1633,12 +1640,12 @@ test_frequency(int gem_fd)
>>> 	igt_info("Max frequency: requested %.1f, actual %.1f\n",
>>> 		 max[0], max[1]);
>>>    -	assert_within_epsilon(min[0], min_freq, tolerance);
>>> +	assert_within_epsilon(min[0], min_req, tolerance);
>>> 	/*
>>> 	 * On thermally throttled devices we cannot be sure maximum frequency
>>> 	 * can be reached so use larger tolerance downards.
>>> 	 */
>>> -	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
>>> +	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);
>>
>>
>> This 0.15f is what I was talking about above.
>>
>> Regards,
>>
>> Tvrtko
>>
>>>    }
>>>      static void

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-21  9:09 ` Tvrtko Ursulin
@ 2022-11-23  6:03   ` Dixit, Ashutosh
  2022-11-24 12:42     ` Tvrtko Ursulin
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-11-23  6:03 UTC (permalink / raw)
  To: Tvrtko Ursulin; +Cc: igt-dev

On Mon, 21 Nov 2022 01:09:55 -0800, Tvrtko Ursulin wrote:
>

Hi Tvrtko,

I am only offering an overall clarification below first, not answering all
of your points for now.

>
> On 19/11/2022 02:00, Ashutosh Dixit wrote:
> > With SLPC, even when we set the same min and max freq's, the requested and
> > actual freq's can differ from the min/max freq set. For example "efficient
> > freq" (when in effect) can override set min/max freq. In general FW is the
> > final arbiter in determining freq and can override set values.
> >
> > Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> > reported by PMU not against the set freq's but against the requested freq
> > reported in sysfs. Also add a delay after setting freq's to account for
> > messaging delays in setting freq's in GuC.
> >
> > v2: Introduce a 100 ms delay after setting freq
> > v3: Update commit message, code identical to v2
> >
> > Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> > Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> > Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > ---
> >   tests/i915/perf_pmu.c | 23 +++++++++++++++--------
> >   1 file changed, 15 insertions(+), 8 deletions(-)
> >
> > diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > index f363db2ba13..02f6ae989b1 100644
> > --- a/tests/i915/perf_pmu.c
> > +++ b/tests/i915/perf_pmu.c
> > @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
> >	igt_assert_lte(target, busy);
> >   }
> >   +/* Wait for GuC SLPC freq changes to take effect */
> > +#define wait_freq_set()		usleep(100000)
>
> A future task of maybe adding a drop_caches flag to wait for this?
>
> Alternatively, if we have a sysfs or debugfs file which reflects the value
> once GuC has processed it, keep reading it until it is updated? Even if to
> something other than it was before the write, to handle the inability to
> set the exact requested frequency sometimes?
> >
> > +
> >   static void
> >   test_frequency(int gem_fd)
> >   {
> > -	uint32_t min_freq, max_freq, boost_freq;
> > +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> >	uint64_t val[2], start[2], slept;
> >	double min[2], max[2];
> >	igt_spin_t *spin;
> > @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
> >	 * Set GPU to min frequency and read PMU counters.
> >	 */
> >	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> >	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> >	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> > +	wait_freq_set();
> > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
> >	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> >		gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes
> > effect */
> > @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
> >		min[0] = 1e9*(val[0] - start[0]) / slept;
> >	min[1] = 1e9*(val[1] - start[1]) / slept;
> > +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>
> This really leaves a bad taste for me still, sorry. First of all, it may
> make it more palatable if readback was immediately after wait_freq_set(),
> or as part of it. Otherwise it seems like even more confused test and sysfs
> ABI.
>
> Did we close on my question of whether we can make readback of
> gt_max_freq_mhz actually represent the real max? Correct me please if I got
> confused - with min freq checking here we write max=300 and get 350? If so
> can we make gt_max_freq_mhz return 350 after the write of 300? Or return an
> error?

Did you read Vinay's email on v1 here:

https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1#comment_921571

As Vinay says, SLPC is requesting an entirely different freq (the effecient
freq rather than the set 'min == max freq') and no amount of waiting will
result in the requested freq being the same as the set 'min == max
freq'. Also the efficient freq cannot be controlled from user space.

(So in the example above min and max are set to 300 MHz and SLPC is
requesting efficient freq which is 350 or 400 MHz. So min and max values
are correct at 300 MHz and requested/actual freq will show 350/400 MHz).

Let me also explain what happens in GuC FW when we change min/max freq:
* i915 receives min/max freq from sysfs and sends a message to GuC to
  change min/max freq.
* The message itself might be queued behind other messages in the host to
  GuC queue. i915 is still blocked in the sysfs handler.
* Now when GuC receives the message interrupt, in its top-half handler it
  validates the freq parameters and sends a message back to host saying the
  freq change has been accepted. But min/max freq is actually changed later
  in the bottom-half handler in GuC FW. So the top half only validates the
  freq value.
* When i915 receives the message from GuC top half, the sysfs handler
  unblocks and starts reflecting the new min/max freq value.

So the delay for the new min/max freq values to take effect is between
returning from the sysfs handler and when the min/max freq are really set
in GuC bottom half (an indeterminate possibly large amount of time if GuC
is busy).

So as you see this introduces an indeterminacy due to to GuC interface not
being synchronous. If this needs to be fixed, it needs to be fixed in GuC
FW rather than waiting etc. in i915. GuC needs to implement another
notification to host /after/ the min/max freq change has been applied in
its bottom half. Doing this is not a priority for the GuC team at this
point.

However, this bug is not due to these delays but due to GuC requesting the
efficient freq instead of the set 'min == max freq' as mentioned above.

If all these delays are confusing we can revert to v1 of the patch:

https://patchwork.freedesktop.org/patch/510304/?series=110574&rev=1

v1 is simple and does not introduce these delays but just reads the
requested freq from sysfs after the 500 ms PMU measurement but while the
spinner is still running (which we do in v3 too). Without load SLPC will
only request RPn so to compare with PMU measurement we must read the
requested freq from sysfs with the spinner load. v1 is sufficient to fix
the bug but I thought to be completely safe it is better to add
wait_freq_set() after changing min/max freq (and hence v3).

I hope this clarifies things somewhat. If not please let me know.

Thanks.
--
Ashutosh


> >	igt_spin_free(gem_fd, spin);
> >	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
> > @@ -1597,11 +1602,11 @@ test_frequency(int gem_fd)
> >	 * Set GPU to max frequency and read PMU counters.
> >	 */
> >	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", max_freq));
> > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
> >	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", boost_freq));
> > -	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
> > -
> >	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", max_freq));
> > +	wait_freq_set();
> > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
> > +	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
> >	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == max_freq);
> >		gem_quiescent_gpu(gem_fd);
> > @@ -1613,6 +1618,7 @@ test_frequency(int gem_fd)
> >		max[0] = 1e9*(val[0] - start[0]) / slept;
> >	max[1] = 1e9*(val[1] - start[1]) / slept;
> > +	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>
> For the max freq use case we had 15% downward tolerance to account for
> thermally challenged.
>
> a) Is that not enough with SLPC? Like there is a new "failure" mode?
>
> b) Is the extra 10% needed with the sysfs readback added? Aka do we need
> both?
>
> >		igt_spin_free(gem_fd, spin);
> >	gem_quiescent_gpu(gem_fd);
> > @@ -1621,6 +1627,7 @@ test_frequency(int gem_fd)
> >	 * Restore min/max.
> >	 */
> >	igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq);
> > +	wait_freq_set();
> >	if (igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") != min_freq)
> >		igt_warn("Unable to restore min frequency to saved value [%u MHz], now %u MHz\n",
> >			 min_freq, igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz"));
> > @@ -1633,12 +1640,12 @@ test_frequency(int gem_fd)
> >	igt_info("Max frequency: requested %.1f, actual %.1f\n",
> >		 max[0], max[1]);
> >   -	assert_within_epsilon(min[0], min_freq, tolerance);
> > +	assert_within_epsilon(min[0], min_req, tolerance);
> >	/*
> >	 * On thermally throttled devices we cannot be sure maximum frequency
> >	 * can be reached so use larger tolerance downards.
> >	 */
> > -	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
> > +	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);
>
>
> This 0.15f is what I was talking about above.
>
> Regards,
>
> Tvrtko
>
> >   }
> >     static void

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-19  2:00 [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest Ashutosh Dixit
@ 2022-11-21  9:09 ` Tvrtko Ursulin
  2022-11-23  6:03   ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Tvrtko Ursulin @ 2022-11-21  9:09 UTC (permalink / raw)
  To: Ashutosh Dixit, igt-dev


On 19/11/2022 02:00, Ashutosh Dixit wrote:
> With SLPC, even when we set the same min and max freq's, the requested and
> actual freq's can differ from the min/max freq set. For example "efficient
> freq" (when in effect) can override set min/max freq. In general FW is the
> final arbiter in determining freq and can override set values.
> 
> Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
> reported by PMU not against the set freq's but against the requested freq
> reported in sysfs. Also add a delay after setting freq's to account for
> messaging delays in setting freq's in GuC.
> 
> v2: Introduce a 100 ms delay after setting freq
> v3: Update commit message, code identical to v2
> 
> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
> Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> ---
>   tests/i915/perf_pmu.c | 23 +++++++++++++++--------
>   1 file changed, 15 insertions(+), 8 deletions(-)
> 
> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> index f363db2ba13..02f6ae989b1 100644
> --- a/tests/i915/perf_pmu.c
> +++ b/tests/i915/perf_pmu.c
> @@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
>   	igt_assert_lte(target, busy);
>   }
>   
> +/* Wait for GuC SLPC freq changes to take effect */
> +#define wait_freq_set()		usleep(100000)

A future task of maybe adding a drop_caches flag to wait for this?

Alternatively, if we have a sysfs or debugfs file which reflects the 
value once GuC has processed it, keep reading it until it is updated? 
Even if to something other than it was before the write, to handle the 
inability to set the exact requested frequency sometimes?

> +
>   static void
>   test_frequency(int gem_fd)
>   {
> -	uint32_t min_freq, max_freq, boost_freq;
> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
>   	uint64_t val[2], start[2], slept;
>   	double min[2], max[2];
>   	igt_spin_t *spin;
> @@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
>   	 * Set GPU to min frequency and read PMU counters.
>   	 */
>   	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>   	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>   	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
> +	wait_freq_set();
> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>   	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
>   
>   	gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes effect */
> @@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
>   
>   	min[0] = 1e9*(val[0] - start[0]) / slept;
>   	min[1] = 1e9*(val[1] - start[1]) / slept;
> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");

This really leaves a bad taste for me still, sorry. First of all, it may 
make it more palatable if readback was immediately after 
wait_freq_set(), or as part of it. Otherwise it seems like even more 
confused test and sysfs ABI.

Did we close on my question of whether we can make readback of 
gt_max_freq_mhz actually represent the real max? Correct me please if I 
got confused - with min freq checking here we write max=300 and get 350? 
If so can we make gt_max_freq_mhz return 350 after the write of 300? Or 
return an error?

>   	igt_spin_free(gem_fd, spin);
>   	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
> @@ -1597,11 +1602,11 @@ test_frequency(int gem_fd)
>   	 * Set GPU to max frequency and read PMU counters.
>   	 */
>   	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", max_freq));
> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
>   	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", boost_freq));
> -	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
> -
>   	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", max_freq));
> +	wait_freq_set();
> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
> +	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
>   	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == max_freq);
>   
>   	gem_quiescent_gpu(gem_fd);
> @@ -1613,6 +1618,7 @@ test_frequency(int gem_fd)
>   
>   	max[0] = 1e9*(val[0] - start[0]) / slept;
>   	max[1] = 1e9*(val[1] - start[1]) / slept;
> +	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");

For the max freq use case we had 15% downward tolerance to account for 
thermally challenged.

a) Is that not enough with SLPC? Like there is a new "failure" mode?

b) Is the extra 10% needed with the sysfs readback added? Aka do we need 
both?

>   
>   	igt_spin_free(gem_fd, spin);
>   	gem_quiescent_gpu(gem_fd);
> @@ -1621,6 +1627,7 @@ test_frequency(int gem_fd)
>   	 * Restore min/max.
>   	 */
>   	igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq);
> +	wait_freq_set();
>   	if (igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") != min_freq)
>   		igt_warn("Unable to restore min frequency to saved value [%u MHz], now %u MHz\n",
>   			 min_freq, igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz"));
> @@ -1633,12 +1640,12 @@ test_frequency(int gem_fd)
>   	igt_info("Max frequency: requested %.1f, actual %.1f\n",
>   		 max[0], max[1]);
>   
> -	assert_within_epsilon(min[0], min_freq, tolerance);
> +	assert_within_epsilon(min[0], min_req, tolerance);
>   	/*
>   	 * On thermally throttled devices we cannot be sure maximum frequency
>   	 * can be reached so use larger tolerance downards.
>   	 */
> -	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
> +	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);


This 0.15f is what I was talking about above.

Regards,

Tvrtko

>   }
>   
>   static void

^ permalink raw reply	[flat|nested] 34+ messages in thread

* [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
@ 2022-11-19  2:00 Ashutosh Dixit
  2022-11-21  9:09 ` Tvrtko Ursulin
  0 siblings, 1 reply; 34+ messages in thread
From: Ashutosh Dixit @ 2022-11-19  2:00 UTC (permalink / raw)
  To: igt-dev

With SLPC, even when we set the same min and max freq's, the requested and
actual freq's can differ from the min/max freq set. For example "efficient
freq" (when in effect) can override set min/max freq. In general FW is the
final arbiter in determining freq and can override set values.

Therefore in igt@perf_pmu@frequency subtest, compare the requested freq
reported by PMU not against the set freq's but against the requested freq
reported in sysfs. Also add a delay after setting freq's to account for
messaging delays in setting freq's in GuC.

v2: Introduce a 100 ms delay after setting freq
v3: Update commit message, code identical to v2

Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
Cc: Vinay Belgaumkar <vinay.belgaumkar@intel.com>
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
---
 tests/i915/perf_pmu.c | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
index f363db2ba13..02f6ae989b1 100644
--- a/tests/i915/perf_pmu.c
+++ b/tests/i915/perf_pmu.c
@@ -1543,10 +1543,13 @@ test_interrupts_sync(int gem_fd)
 	igt_assert_lte(target, busy);
 }
 
+/* Wait for GuC SLPC freq changes to take effect */
+#define wait_freq_set()		usleep(100000)
+
 static void
 test_frequency(int gem_fd)
 {
-	uint32_t min_freq, max_freq, boost_freq;
+	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
 	uint64_t val[2], start[2], slept;
 	double min[2], max[2];
 	igt_spin_t *spin;
@@ -1572,10 +1575,11 @@ test_frequency(int gem_fd)
 	 * Set GPU to min frequency and read PMU counters.
 	 */
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
-	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
-	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
+	wait_freq_set();
+	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
+	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
 	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
 
 	gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes effect */
@@ -1587,6 +1591,7 @@ test_frequency(int gem_fd)
 
 	min[0] = 1e9*(val[0] - start[0]) / slept;
 	min[1] = 1e9*(val[1] - start[1]) / slept;
+	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
 
 	igt_spin_free(gem_fd, spin);
 	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
@@ -1597,11 +1602,11 @@ test_frequency(int gem_fd)
 	 * Set GPU to max frequency and read PMU counters.
 	 */
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", max_freq));
-	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", boost_freq));
-	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
-
 	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", max_freq));
+	wait_freq_set();
+	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == max_freq);
+	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == boost_freq);
 	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == max_freq);
 
 	gem_quiescent_gpu(gem_fd);
@@ -1613,6 +1618,7 @@ test_frequency(int gem_fd)
 
 	max[0] = 1e9*(val[0] - start[0]) / slept;
 	max[1] = 1e9*(val[1] - start[1]) / slept;
+	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
 
 	igt_spin_free(gem_fd, spin);
 	gem_quiescent_gpu(gem_fd);
@@ -1621,6 +1627,7 @@ test_frequency(int gem_fd)
 	 * Restore min/max.
 	 */
 	igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq);
+	wait_freq_set();
 	if (igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") != min_freq)
 		igt_warn("Unable to restore min frequency to saved value [%u MHz], now %u MHz\n",
 			 min_freq, igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz"));
@@ -1633,12 +1640,12 @@ test_frequency(int gem_fd)
 	igt_info("Max frequency: requested %.1f, actual %.1f\n",
 		 max[0], max[1]);
 
-	assert_within_epsilon(min[0], min_freq, tolerance);
+	assert_within_epsilon(min[0], min_req, tolerance);
 	/*
 	 * On thermally throttled devices we cannot be sure maximum frequency
 	 * can be reached so use larger tolerance downards.
 	 */
-	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
+	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);
 }
 
 static void
-- 
2.38.1

^ permalink raw reply related	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-10  1:37                 ` Belgaumkar, Vinay
@ 2022-11-10  4:20                   ` Dixit, Ashutosh
  0 siblings, 0 replies; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-11-10  4:20 UTC (permalink / raw)
  To: Belgaumkar, Vinay; +Cc: igt-dev

On Wed, 09 Nov 2022 17:37:18 -0800, Belgaumkar, Vinay wrote:
>
> On 11/8/2022 5:53 PM, Dixit, Ashutosh wrote:
> > On Tue, 08 Nov 2022 13:02:33 -0800, Belgaumkar, Vinay wrote:
> >>
> >> On 11/8/2022 1:24 AM, Tvrtko Ursulin wrote:
> >>> On 08/11/2022 01:31, Dixit, Ashutosh wrote:
> >>>> On Mon, 07 Nov 2022 16:57:24 -0800, Belgaumkar, Vinay wrote:
> >>>>> On 11/7/2022 4:22 PM, Dixit, Ashutosh wrote:
> >>>>>> On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
> >>>>>> Hi Vinay,
> >>>>>>
> >>>>>> A question for you below.
> >>>>>>
> >>>>>>> So I submitted this patch to repro the issue and to print out the
> >>>>>>> requested
> >>>>>>> freq from sysfs:
> >>>>>>>
> >>>>>>> https://patchwork.freedesktop.org/series/110630/
> >>>>>>>
> >>>>>>> And we can see the output here:
> >>>>>>>
> >>>>>>> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html
> >>>>>>>
> >>>>>>> ```
> >>>>>>> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux:
> >>>>>>> 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
> >>>>>>> Starting subtest: frequency
> >>>>>>> Frequency: min=300, max=2050, boost=2050 MHz
> >>>>>>> Min frequency: requested 349.7, actual 349.7
> >>>>>>> Max frequency: requested 2048.0, actual 2048.0
> >>>>>>> Sysfs requested: min 350, max 2050
> >>>>>>> Stack trace:
> >>>>>>>      #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908
> >>>>>>> __igt_fail_assert()
> >>>>>>>      #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656
> >>>>>>> __igt_unique____real_main2147()
> >>>>>>>      #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147
> >>>>>>> main()
> >>>>>>>      #3 [__libc_start_main+0xf3]
> >>>>>>>      #4 [_start+0x2e]
> >>>>>>> Subtest frequency: FAIL (2.212s)
> >>>>>>> ```
> >>>>>>>
> >>>>>>> So we clearly see the requested freq from sysfs is indeed 350 MHz so
> >>>>>>> SLPC/PCODE is not honoring the set min == max == boost freq (and PMU
> >>>>>>> is
> >>>>>>> measuring what sysfs is showing). In general PCODE is the final
> >>>>>>> arbiter in
> >>>>>>> such cases and we do occasionally see instances where set freq limits
> >>>>>>> are
> >>>>>>> not honored.
> >>>>>>>
> >>>>>>> I would say if igt@perf_pmu@frequency is testing freq measured by PMU
> >>>>>>> then
> >>>>>>> the patch below is correct. Whether SLPC/PCODE is honoring the set
> >>>>>>> freq
> >>>>>>> limits should be tested in a SLPC test (which we also have).
> >>>>>> igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we
> >>>>>> still
> >>>>>> see the requested freq to be 350 MHz. Do we have a SLPC test covering
> >>>>>> this
> >>>>>> scenario or should we add one? This is failing on one of the DG2's.
> >>>>> Does adding a delay help (around 20 ms for the h2g to go through
> >>>>> typically)?
> >>>> There is no delay but the test calls gem_quiescent_gpu() after setting
> >>>> 'min
> >>>> == max == boost == 300 MHz' and then launches a spinner. We are checking
> >>>> the requested freq 500 ms after the spinner is started (so plenty of
> >>>> time
> >>>> for the h2g) and still the requested freq is 350 MHz.
> >>>>
> >>>>> Also, is there a workload running when we change the min=max=boost to
> >>>>> 300?
> >>>> No, the workload is started after setting the freq and calling
> >>>> gem_quiescent_gpu().
> >>> Implication here seems to be that gem_quiescent_gpu() would need to cover
> >>> H2G communication - does it?
> >> No, more like there needs to be a WL running in order for SLPC to actively
> >> make changes to the requested frequency.
> > Well here we set the freq's first and later when the WL runs FW should
> > select an appropriate requested freq.
> >
> >>>>> We already check these things in our SLPC selftests.
> >>> Is it then expected to respect the 300MHz max in this case? Or if it
> >>> can't, should it be reflected in the sysfs readback?
> >> It should respect the 300 Mhz. The only question in my mind is regarding
> >> efficient frequency. Can we print out what the RP1 is here?
> > RP1 is also 300. We can see it here:
> >
> > https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8071/bat-dg2-11/igt@perf_pmu@frequency.html
> >
> > with this patch:
> >
> > https://patchwork.freedesktop.org/series/110630/#rev3
> >
> > In this case, min == max == 300, but requested freq is 400 (previously it
> > was 350).
>
> Ok, this might be happening due to the following -
>
> 1. We have efficient frequency enabled now, so GuC will use that instead of
> min even on light loads etc.
>
> 2. It is also known that this efficient frequency is "dynamic", especially
> on DG2.
>
> 3. When we set min freq to a value lower than efficient, we will turn off
> efficient frequency usage. But, here min = efficient = 300, so it will
> remain ON.
>
> This is why we see 350 or even 400 sometimes as we are not bound to the min
> (or even a single freq level) when efficient freq usage is allowed.

Hi Vinay, thanks for the explanation, makes sense.

> The solution for this case may be to turn off efficient frequency
> forcibly for this test and see if that helps. We do that in the selftests
> to ensure proper frequency bounds.

I don't believe we can do this from userland, can we?

Thanks.
--
Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-09  1:53               ` Dixit, Ashutosh
@ 2022-11-10  1:37                 ` Belgaumkar, Vinay
  2022-11-10  4:20                   ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Belgaumkar, Vinay @ 2022-11-10  1:37 UTC (permalink / raw)
  To: Dixit, Ashutosh; +Cc: igt-dev


On 11/8/2022 5:53 PM, Dixit, Ashutosh wrote:
> On Tue, 08 Nov 2022 13:02:33 -0800, Belgaumkar, Vinay wrote:
>>
>> On 11/8/2022 1:24 AM, Tvrtko Ursulin wrote:
>>> On 08/11/2022 01:31, Dixit, Ashutosh wrote:
>>>> On Mon, 07 Nov 2022 16:57:24 -0800, Belgaumkar, Vinay wrote:
>>>>> On 11/7/2022 4:22 PM, Dixit, Ashutosh wrote:
>>>>>> On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
>>>>>> Hi Vinay,
>>>>>>
>>>>>> A question for you below.
>>>>>>
>>>>>>> So I submitted this patch to repro the issue and to print out the
>>>>>>> requested
>>>>>>> freq from sysfs:
>>>>>>>
>>>>>>> https://patchwork.freedesktop.org/series/110630/
>>>>>>>
>>>>>>> And we can see the output here:
>>>>>>>
>>>>>>> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html
>>>>>>>
>>>>>>> ```
>>>>>>> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux:
>>>>>>> 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
>>>>>>> Starting subtest: frequency
>>>>>>> Frequency: min=300, max=2050, boost=2050 MHz
>>>>>>> Min frequency: requested 349.7, actual 349.7
>>>>>>> Max frequency: requested 2048.0, actual 2048.0
>>>>>>> Sysfs requested: min 350, max 2050
>>>>>>> Stack trace:
>>>>>>>      #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908
>>>>>>> __igt_fail_assert()
>>>>>>>      #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656
>>>>>>> __igt_unique____real_main2147()
>>>>>>>      #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147
>>>>>>> main()
>>>>>>>      #3 [__libc_start_main+0xf3]
>>>>>>>      #4 [_start+0x2e]
>>>>>>> Subtest frequency: FAIL (2.212s)
>>>>>>> ```
>>>>>>>
>>>>>>> So we clearly see the requested freq from sysfs is indeed 350 MHz so
>>>>>>> SLPC/PCODE is not honoring the set min == max == boost freq (and PMU
>>>>>>> is
>>>>>>> measuring what sysfs is showing). In general PCODE is the final
>>>>>>> arbiter in
>>>>>>> such cases and we do occasionally see instances where set freq limits
>>>>>>> are
>>>>>>> not honored.
>>>>>>>
>>>>>>> I would say if igt@perf_pmu@frequency is testing freq measured by PMU
>>>>>>> then
>>>>>>> the patch below is correct. Whether SLPC/PCODE is honoring the set
>>>>>>> freq
>>>>>>> limits should be tested in a SLPC test (which we also have).
>>>>>> igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we
>>>>>> still
>>>>>> see the requested freq to be 350 MHz. Do we have a SLPC test covering
>>>>>> this
>>>>>> scenario or should we add one? This is failing on one of the DG2's.
>>>>> Does adding a delay help (around 20 ms for the h2g to go through
>>>>> typically)?
>>>> There is no delay but the test calls gem_quiescent_gpu() after setting
>>>> 'min
>>>> == max == boost == 300 MHz' and then launches a spinner. We are checking
>>>> the requested freq 500 ms after the spinner is started (so plenty of
>>>> time
>>>> for the h2g) and still the requested freq is 350 MHz.
>>>>
>>>>> Also, is there a workload running when we change the min=max=boost to
>>>>> 300?
>>>> No, the workload is started after setting the freq and calling
>>>> gem_quiescent_gpu().
>>> Implication here seems to be that gem_quiescent_gpu() would need to cover
>>> H2G communication - does it?
>> No, more like there needs to be a WL running in order for SLPC to actively
>> make changes to the requested frequency.
> Well here we set the freq's first and later when the WL runs FW should
> select an appropriate requested freq.
>
>>>>> We already check these things in our SLPC selftests.
>>> Is it then expected to respect the 300MHz max in this case? Or if it
>>> can't, should it be reflected in the sysfs readback?
>> It should respect the 300 Mhz. The only question in my mind is regarding
>> efficient frequency. Can we print out what the RP1 is here?
> RP1 is also 300. We can see it here:
>
> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8071/bat-dg2-11/igt@perf_pmu@frequency.html
>
> with this patch:
>
> https://patchwork.freedesktop.org/series/110630/#rev3
>
> In this case, min == max == 300, but requested freq is 400 (previously it
> was 350).

Ok, this might be happening due to the following -

1. We have efficient frequency enabled now, so GuC will use that instead 
of min even on light loads etc.

2. It is also known that this efficient frequency is "dynamic", 
especially on DG2.

3. When we set min freq to a value lower than efficient, we will turn 
off efficient frequency usage. But, here min = efficient = 300, so it 
will remain ON.

This is why we see 350 or even 400 sometimes as we are not bound to the 
min (or even a single freq level) when efficient freq usage is allowed. 
The solution for this case may be to turn off efficient frequency 
forcibly for this test and see if that helps. We do that in the 
selftests to ensure proper frequency bounds.

Thanks,

Vinay.

>
> Thanks.
> --
> Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-09  1:49             ` Dixit, Ashutosh
@ 2022-11-09  6:03               ` Dixit, Ashutosh
  0 siblings, 0 replies; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-11-09  6:03 UTC (permalink / raw)
  To: Tvrtko Ursulin; +Cc: igt-dev

On Tue, 08 Nov 2022 17:49:04 -0800, Dixit, Ashutosh wrote:
>
> > Is it then expected to respect the 300MHz max in this case? Or if it can't,
> > should it be reflected in the sysfs readback?
>
> The max is set to 300 MHz but is not respected. So the sysfs readback shows
> max as 300 MHz but actual requested freq is higher. The flow is that i915
> sets min/max using h2g. GuC FW validates the parameters in the top-half and
> returns success (which results in max freq being set in i915 to 300 MHz)
> but the actual min/max freq change in PCODE is done later in the
> bottom-half because of which the delay is needed.
>
> Though Vinay we never read the max freq back from FW, that is we call
> intel_rps_get_max_frequency and not intel_guc_slpc_get_max_freq when
> displaying in sysfs. That is we display i915 cached values rather than
> interrogate FW, assuming that FW min/max values are same as the cached
> values. I can add a couple more prints and see what actual FW values are.

So I submitted this patch which queries freq's from FW rather than
displaying cached values:

https://patchwork.freedesktop.org/series/110685/

But here also we see the same min/max freq's:

https://intel-gfx-ci.01.org/tree/drm-tip/Patchwork_110685v1/bat-dg2-11/igt@perf_pmu@frequency.html

So this means freq's set in FW match the cached values in i915.

(Compare the above results against:
https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8071/bat-dg2-11/igt@perf_pmu@frequency.html
which uses cached values).

So again we are back to the earlier conclusion that "max is set to 300 MHz
but is not respected".

So finally let's think about whether v1 or v2 of the IGT patch makes sense:

https://patchwork.freedesktop.org/series/110574/

Thanks.
--
Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-08 21:02             ` Belgaumkar, Vinay
@ 2022-11-09  1:53               ` Dixit, Ashutosh
  2022-11-10  1:37                 ` Belgaumkar, Vinay
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-11-09  1:53 UTC (permalink / raw)
  To: Belgaumkar, Vinay; +Cc: igt-dev

On Tue, 08 Nov 2022 13:02:33 -0800, Belgaumkar, Vinay wrote:
>
>
> On 11/8/2022 1:24 AM, Tvrtko Ursulin wrote:
> >
> > On 08/11/2022 01:31, Dixit, Ashutosh wrote:
> >> On Mon, 07 Nov 2022 16:57:24 -0800, Belgaumkar, Vinay wrote:
> >>> On 11/7/2022 4:22 PM, Dixit, Ashutosh wrote:
> >>>> On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
> >>>> Hi Vinay,
> >>>>
> >>>> A question for you below.
> >>>>
> >>>>> So I submitted this patch to repro the issue and to print out the
> >>>>> requested
> >>>>> freq from sysfs:
> >>>>>
> >>>>> https://patchwork.freedesktop.org/series/110630/
> >>>>>
> >>>>> And we can see the output here:
> >>>>>
> >>>>> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html
> >>>>>
> >>>>> ```
> >>>>> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux:
> >>>>> 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
> >>>>> Starting subtest: frequency
> >>>>> Frequency: min=300, max=2050, boost=2050 MHz
> >>>>> Min frequency: requested 349.7, actual 349.7
> >>>>> Max frequency: requested 2048.0, actual 2048.0
> >>>>> Sysfs requested: min 350, max 2050
> >>>>> Stack trace:
> >>>>>     #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908
> >>>>> __igt_fail_assert()
> >>>>>     #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656
> >>>>> __igt_unique____real_main2147()
> >>>>>     #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147
> >>>>> main()
> >>>>>     #3 [__libc_start_main+0xf3]
> >>>>>     #4 [_start+0x2e]
> >>>>> Subtest frequency: FAIL (2.212s)
> >>>>> ```
> >>>>>
> >>>>> So we clearly see the requested freq from sysfs is indeed 350 MHz so
> >>>>> SLPC/PCODE is not honoring the set min == max == boost freq (and PMU
> >>>>> is
> >>>>> measuring what sysfs is showing). In general PCODE is the final
> >>>>> arbiter in
> >>>>> such cases and we do occasionally see instances where set freq limits
> >>>>> are
> >>>>> not honored.
> >>>>>
> >>>>> I would say if igt@perf_pmu@frequency is testing freq measured by PMU
> >>>>> then
> >>>>> the patch below is correct. Whether SLPC/PCODE is honoring the set
> >>>>> freq
> >>>>> limits should be tested in a SLPC test (which we also have).
> >>>>
> >>>> igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we
> >>>> still
> >>>> see the requested freq to be 350 MHz. Do we have a SLPC test covering
> >>>> this
> >>>> scenario or should we add one? This is failing on one of the DG2's.
> >>>
> >>> Does adding a delay help (around 20 ms for the h2g to go through
> >>> typically)?
> >>
> >> There is no delay but the test calls gem_quiescent_gpu() after setting
> >> 'min
> >> == max == boost == 300 MHz' and then launches a spinner. We are checking
> >> the requested freq 500 ms after the spinner is started (so plenty of
> >> time
> >> for the h2g) and still the requested freq is 350 MHz.
> >>
> >>> Also, is there a workload running when we change the min=max=boost to
> >>> 300?
> >>
> >> No, the workload is started after setting the freq and calling
> >> gem_quiescent_gpu().
> >
> > Implication here seems to be that gem_quiescent_gpu() would need to cover
> > H2G communication - does it?
> No, more like there needs to be a WL running in order for SLPC to actively
> make changes to the requested frequency.

Well here we set the freq's first and later when the WL runs FW should
select an appropriate requested freq.

> >
> >>> We already check these things in our SLPC selftests.
> >
> > Is it then expected to respect the 300MHz max in this case? Or if it
> > can't, should it be reflected in the sysfs readback?
>
> It should respect the 300 Mhz. The only question in my mind is regarding
> efficient frequency. Can we print out what the RP1 is here?

RP1 is also 300. We can see it here:

https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8071/bat-dg2-11/igt@perf_pmu@frequency.html

with this patch:

https://patchwork.freedesktop.org/series/110630/#rev3

In this case, min == max == 300, but requested freq is 400 (previously it
was 350).

Thanks.
--
Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-08  9:24           ` Tvrtko Ursulin
  2022-11-08 21:02             ` Belgaumkar, Vinay
@ 2022-11-09  1:49             ` Dixit, Ashutosh
  2022-11-09  6:03               ` Dixit, Ashutosh
  1 sibling, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-11-09  1:49 UTC (permalink / raw)
  To: Tvrtko Ursulin; +Cc: igt-dev

On Tue, 08 Nov 2022 01:24:14 -0800, Tvrtko Ursulin wrote:
>
> On 08/11/2022 01:31, Dixit, Ashutosh wrote:
> > On Mon, 07 Nov 2022 16:57:24 -0800, Belgaumkar, Vinay wrote:
> >> On 11/7/2022 4:22 PM, Dixit, Ashutosh wrote:
> >>> On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
> >>> Hi Vinay,
> >>>
> >>> A question for you below.
> >>>
> >>>> So I submitted this patch to repro the issue and to print out the requested
> >>>> freq from sysfs:
> >>>>
> >>>> https://patchwork.freedesktop.org/series/110630/
> >>>>
> >>>> And we can see the output here:
> >>>>
> >>>> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html
> >>>>
> >>>> ```
> >>>> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux: 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
> >>>> Starting subtest: frequency
> >>>> Frequency: min=300, max=2050, boost=2050 MHz
> >>>> Min frequency: requested 349.7, actual 349.7
> >>>> Max frequency: requested 2048.0, actual 2048.0
> >>>> Sysfs requested: min 350, max 2050
> >>>> Stack trace:
> >>>>     #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908 __igt_fail_assert()
> >>>>     #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656 __igt_unique____real_main2147()
> >>>>     #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147 main()
> >>>>     #3 [__libc_start_main+0xf3]
> >>>>     #4 [_start+0x2e]
> >>>> Subtest frequency: FAIL (2.212s)
> >>>> ```
> >>>>
> >>>> So we clearly see the requested freq from sysfs is indeed 350 MHz so
> >>>> SLPC/PCODE is not honoring the set min == max == boost freq (and PMU is
> >>>> measuring what sysfs is showing). In general PCODE is the final arbiter in
> >>>> such cases and we do occasionally see instances where set freq limits are
> >>>> not honored.
> >>>>
> >>>> I would say if igt@perf_pmu@frequency is testing freq measured by PMU then
> >>>> the patch below is correct. Whether SLPC/PCODE is honoring the set freq
> >>>> limits should be tested in a SLPC test (which we also have).
> >>>
> >>> igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we still
> >>> see the requested freq to be 350 MHz. Do we have a SLPC test covering this
> >>> scenario or should we add one? This is failing on one of the DG2's.
> >>
> >> Does adding a delay help (around 20 ms for the h2g to go through
> >> typically)?
> >
> > There is no delay but the test calls gem_quiescent_gpu() after setting 'min
> > == max == boost == 300 MHz' and then launches a spinner. We are checking
> > the requested freq 500 ms after the spinner is started (so plenty of time
> > for the h2g) and still the requested freq is 350 MHz.
> >
> >> Also, is there a workload running when we change the min=max=boost to
> >> 300?
> >
> > No, the workload is started after setting the freq and calling
> > gem_quiescent_gpu().
>
> Implication here seems to be that gem_quiescent_gpu() would need to cover
> H2G communication - does it?

Don't think gem_quiescent_gpu should cover h2g delays. I have added a
separate 100 ms delay (which I've used elsewhere too) in v2. I am not sure
what is the delay caused by gem_quiescent_gpu and if that is sufficient we
don't need this additional 100 ms delay. Anyway the delay will make PMU
measurements a little more accurate but does not affect the overall result.

> >> We already check these things in our SLPC selftests.
>
> Is it then expected to respect the 300MHz max in this case? Or if it can't,
> should it be reflected in the sysfs readback?

The max is set to 300 MHz but is not respected. So the sysfs readback shows
max as 300 MHz but actual requested freq is higher. The flow is that i915
sets min/max using h2g. GuC FW validates the parameters in the top-half and
returns success (which results in max freq being set in i915 to 300 MHz)
but the actual min/max freq change in PCODE is done later in the
bottom-half because of which the delay is needed.

Though Vinay we never read the max freq back from FW, that is we call
intel_rps_get_max_frequency and not intel_guc_slpc_get_max_freq when
displaying in sysfs. That is we display i915 cached values rather than
interrogate FW, assuming that FW min/max values are same as the cached
values. I can add a couple more prints and see what actual FW values are.

Thanks.
--
Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-08  9:24           ` Tvrtko Ursulin
@ 2022-11-08 21:02             ` Belgaumkar, Vinay
  2022-11-09  1:53               ` Dixit, Ashutosh
  2022-11-09  1:49             ` Dixit, Ashutosh
  1 sibling, 1 reply; 34+ messages in thread
From: Belgaumkar, Vinay @ 2022-11-08 21:02 UTC (permalink / raw)
  To: Tvrtko Ursulin, Dixit, Ashutosh; +Cc: igt-dev


On 11/8/2022 1:24 AM, Tvrtko Ursulin wrote:
>
> On 08/11/2022 01:31, Dixit, Ashutosh wrote:
>> On Mon, 07 Nov 2022 16:57:24 -0800, Belgaumkar, Vinay wrote:
>>> On 11/7/2022 4:22 PM, Dixit, Ashutosh wrote:
>>>> On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
>>>> Hi Vinay,
>>>>
>>>> A question for you below.
>>>>
>>>>> So I submitted this patch to repro the issue and to print out the 
>>>>> requested
>>>>> freq from sysfs:
>>>>>
>>>>> https://patchwork.freedesktop.org/series/110630/
>>>>>
>>>>> And we can see the output here:
>>>>>
>>>>> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html 
>>>>>
>>>>>
>>>>> ```
>>>>> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux: 
>>>>> 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
>>>>> Starting subtest: frequency
>>>>> Frequency: min=300, max=2050, boost=2050 MHz
>>>>> Min frequency: requested 349.7, actual 349.7
>>>>> Max frequency: requested 2048.0, actual 2048.0
>>>>> Sysfs requested: min 350, max 2050
>>>>> Stack trace:
>>>>>     #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908 
>>>>> __igt_fail_assert()
>>>>>     #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656 
>>>>> __igt_unique____real_main2147()
>>>>>     #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147 
>>>>> main()
>>>>>     #3 [__libc_start_main+0xf3]
>>>>>     #4 [_start+0x2e]
>>>>> Subtest frequency: FAIL (2.212s)
>>>>> ```
>>>>>
>>>>> So we clearly see the requested freq from sysfs is indeed 350 MHz so
>>>>> SLPC/PCODE is not honoring the set min == max == boost freq (and 
>>>>> PMU is
>>>>> measuring what sysfs is showing). In general PCODE is the final 
>>>>> arbiter in
>>>>> such cases and we do occasionally see instances where set freq 
>>>>> limits are
>>>>> not honored.
>>>>>
>>>>> I would say if igt@perf_pmu@frequency is testing freq measured by 
>>>>> PMU then
>>>>> the patch below is correct. Whether SLPC/PCODE is honoring the set 
>>>>> freq
>>>>> limits should be tested in a SLPC test (which we also have).
>>>>
>>>> igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we 
>>>> still
>>>> see the requested freq to be 350 MHz. Do we have a SLPC test 
>>>> covering this
>>>> scenario or should we add one? This is failing on one of the DG2's.
>>>
>>> Does adding a delay help (around 20 ms for the h2g to go through
>>> typically)?
>>
>> There is no delay but the test calls gem_quiescent_gpu() after 
>> setting 'min
>> == max == boost == 300 MHz' and then launches a spinner. We are checking
>> the requested freq 500 ms after the spinner is started (so plenty of 
>> time
>> for the h2g) and still the requested freq is 350 MHz.
>>
>>> Also, is there a workload running when we change the min=max=boost to
>>> 300?
>>
>> No, the workload is started after setting the freq and calling
>> gem_quiescent_gpu().
>
> Implication here seems to be that gem_quiescent_gpu() would need to 
> cover H2G communication - does it?
No, more like there needs to be a WL running in order for SLPC to 
actively make changes to the requested frequency.
>
>>> We already check these things in our SLPC selftests.
>
> Is it then expected to respect the 300MHz max in this case? Or if it 
> can't, should it be reflected in the sysfs readback?

It should respect the 300 Mhz. The only question in my mind is regarding 
efficient frequency. Can we print out what the RP1 is here?

Thanks,

Vinay.

>
>
> Regards,
>
> Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-08  1:31         ` Dixit, Ashutosh
@ 2022-11-08  9:24           ` Tvrtko Ursulin
  2022-11-08 21:02             ` Belgaumkar, Vinay
  2022-11-09  1:49             ` Dixit, Ashutosh
  0 siblings, 2 replies; 34+ messages in thread
From: Tvrtko Ursulin @ 2022-11-08  9:24 UTC (permalink / raw)
  To: Dixit, Ashutosh, Belgaumkar, Vinay; +Cc: igt-dev


On 08/11/2022 01:31, Dixit, Ashutosh wrote:
> On Mon, 07 Nov 2022 16:57:24 -0800, Belgaumkar, Vinay wrote:
>> On 11/7/2022 4:22 PM, Dixit, Ashutosh wrote:
>>> On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
>>> Hi Vinay,
>>>
>>> A question for you below.
>>>
>>>> So I submitted this patch to repro the issue and to print out the requested
>>>> freq from sysfs:
>>>>
>>>> https://patchwork.freedesktop.org/series/110630/
>>>>
>>>> And we can see the output here:
>>>>
>>>> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html
>>>>
>>>> ```
>>>> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux: 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
>>>> Starting subtest: frequency
>>>> Frequency: min=300, max=2050, boost=2050 MHz
>>>> Min frequency: requested 349.7, actual 349.7
>>>> Max frequency: requested 2048.0, actual 2048.0
>>>> Sysfs requested: min 350, max 2050
>>>> Stack trace:
>>>>     #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908 __igt_fail_assert()
>>>>     #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656 __igt_unique____real_main2147()
>>>>     #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147 main()
>>>>     #3 [__libc_start_main+0xf3]
>>>>     #4 [_start+0x2e]
>>>> Subtest frequency: FAIL (2.212s)
>>>> ```
>>>>
>>>> So we clearly see the requested freq from sysfs is indeed 350 MHz so
>>>> SLPC/PCODE is not honoring the set min == max == boost freq (and PMU is
>>>> measuring what sysfs is showing). In general PCODE is the final arbiter in
>>>> such cases and we do occasionally see instances where set freq limits are
>>>> not honored.
>>>>
>>>> I would say if igt@perf_pmu@frequency is testing freq measured by PMU then
>>>> the patch below is correct. Whether SLPC/PCODE is honoring the set freq
>>>> limits should be tested in a SLPC test (which we also have).
>>>
>>> igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we still
>>> see the requested freq to be 350 MHz. Do we have a SLPC test covering this
>>> scenario or should we add one? This is failing on one of the DG2's.
>>
>> Does adding a delay help (around 20 ms for the h2g to go through
>> typically)?
> 
> There is no delay but the test calls gem_quiescent_gpu() after setting 'min
> == max == boost == 300 MHz' and then launches a spinner. We are checking
> the requested freq 500 ms after the spinner is started (so plenty of time
> for the h2g) and still the requested freq is 350 MHz.
> 
>> Also, is there a workload running when we change the min=max=boost to
>> 300?
> 
> No, the workload is started after setting the freq and calling
> gem_quiescent_gpu().

Implication here seems to be that gem_quiescent_gpu() would need to 
cover H2G communication - does it?

>> We already check these things in our SLPC selftests.

Is it then expected to respect the 300MHz max in this case? Or if it 
can't, should it be reflected in the sysfs readback?

Regards,

Tvrtko

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-08  0:57       ` Belgaumkar, Vinay
@ 2022-11-08  1:31         ` Dixit, Ashutosh
  2022-11-08  9:24           ` Tvrtko Ursulin
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-11-08  1:31 UTC (permalink / raw)
  To: Belgaumkar, Vinay; +Cc: igt-dev

On Mon, 07 Nov 2022 16:57:24 -0800, Belgaumkar, Vinay wrote:
>
>
> On 11/7/2022 4:22 PM, Dixit, Ashutosh wrote:
> > On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
> > Hi Vinay,
> >
> > A question for you below.
> >
> >> So I submitted this patch to repro the issue and to print out the requested
> >> freq from sysfs:
> >>
> >> https://patchwork.freedesktop.org/series/110630/
> >>
> >> And we can see the output here:
> >>
> >> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html
> >>
> >> ```
> >> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux: 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
> >> Starting subtest: frequency
> >> Frequency: min=300, max=2050, boost=2050 MHz
> >> Min frequency: requested 349.7, actual 349.7
> >> Max frequency: requested 2048.0, actual 2048.0
> >> Sysfs requested: min 350, max 2050
> >> Stack trace:
> >>    #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908 __igt_fail_assert()
> >>    #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656 __igt_unique____real_main2147()
> >>    #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147 main()
> >>    #3 [__libc_start_main+0xf3]
> >>    #4 [_start+0x2e]
> >> Subtest frequency: FAIL (2.212s)
> >> ```
> >>
> >> So we clearly see the requested freq from sysfs is indeed 350 MHz so
> >> SLPC/PCODE is not honoring the set min == max == boost freq (and PMU is
> >> measuring what sysfs is showing). In general PCODE is the final arbiter in
> >> such cases and we do occasionally see instances where set freq limits are
> >> not honored.
> >>
> >> I would say if igt@perf_pmu@frequency is testing freq measured by PMU then
> >> the patch below is correct. Whether SLPC/PCODE is honoring the set freq
> >> limits should be tested in a SLPC test (which we also have).
> >
> > igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we still
> > see the requested freq to be 350 MHz. Do we have a SLPC test covering this
> > scenario or should we add one? This is failing on one of the DG2's.
>
> Does adding a delay help (around 20 ms for the h2g to go through
> typically)?

There is no delay but the test calls gem_quiescent_gpu() after setting 'min
== max == boost == 300 MHz' and then launches a spinner. We are checking
the requested freq 500 ms after the spinner is started (so plenty of time
for the h2g) and still the requested freq is 350 MHz.

> Also, is there a workload running when we change the min=max=boost to
> 300?

No, the workload is started after setting the freq and calling
gem_quiescent_gpu().

>
> We already check these things in our SLPC selftests.
>
> Thanks,
>
> Vinay.
>
> >
> > Thanks.
> > --
> > Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-08  0:22     ` Dixit, Ashutosh
@ 2022-11-08  0:57       ` Belgaumkar, Vinay
  2022-11-08  1:31         ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Belgaumkar, Vinay @ 2022-11-08  0:57 UTC (permalink / raw)
  To: Dixit, Ashutosh; +Cc: igt-dev


On 11/7/2022 4:22 PM, Dixit, Ashutosh wrote:
> On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
> Hi Vinay,
>
> A question for you below.
>
>> So I submitted this patch to repro the issue and to print out the requested
>> freq from sysfs:
>>
>> https://patchwork.freedesktop.org/series/110630/
>>
>> And we can see the output here:
>>
>> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html
>>
>> ```
>> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux: 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
>> Starting subtest: frequency
>> Frequency: min=300, max=2050, boost=2050 MHz
>> Min frequency: requested 349.7, actual 349.7
>> Max frequency: requested 2048.0, actual 2048.0
>> Sysfs requested: min 350, max 2050
>> Stack trace:
>>    #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908 __igt_fail_assert()
>>    #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656 __igt_unique____real_main2147()
>>    #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147 main()
>>    #3 [__libc_start_main+0xf3]
>>    #4 [_start+0x2e]
>> Subtest frequency: FAIL (2.212s)
>> ```
>>
>> So we clearly see the requested freq from sysfs is indeed 350 MHz so
>> SLPC/PCODE is not honoring the set min == max == boost freq (and PMU is
>> measuring what sysfs is showing). In general PCODE is the final arbiter in
>> such cases and we do occasionally see instances where set freq limits are
>> not honored.
>>
>> I would say if igt@perf_pmu@frequency is testing freq measured by PMU then
>> the patch below is correct. Whether SLPC/PCODE is honoring the set freq
>> limits should be tested in a SLPC test (which we also have).
> igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we still
> see the requested freq to be 350 MHz. Do we have a SLPC test covering this
> scenario or should we add one? This is failing on one of the DG2's.

Does adding a delay help (around 20 ms for the h2g to go through 
typically)? Also, is there a workload running when we change the 
min=max=boost to 300?

We already check these things in our SLPC selftests.

Thanks,

Vinay.

>
> Thanks.
> --
> Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-08  0:18   ` Dixit, Ashutosh
@ 2022-11-08  0:22     ` Dixit, Ashutosh
  2022-11-08  0:57       ` Belgaumkar, Vinay
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-11-08  0:22 UTC (permalink / raw)
  To: Vinay Belgaumkar; +Cc: igt-dev

On Mon, 07 Nov 2022 16:18:31 -0800, Dixit, Ashutosh wrote:
>

Hi Vinay,

A question for you below.

> So I submitted this patch to repro the issue and to print out the requested
> freq from sysfs:
>
> https://patchwork.freedesktop.org/series/110630/
>
> And we can see the output here:
>
> https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html
>
> ```
> IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux: 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
> Starting subtest: frequency
> Frequency: min=300, max=2050, boost=2050 MHz
> Min frequency: requested 349.7, actual 349.7
> Max frequency: requested 2048.0, actual 2048.0
> Sysfs requested: min 350, max 2050
> Stack trace:
>   #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908 __igt_fail_assert()
>   #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656 __igt_unique____real_main2147()
>   #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147 main()
>   #3 [__libc_start_main+0xf3]
>   #4 [_start+0x2e]
> Subtest frequency: FAIL (2.212s)
> ```
>
> So we clearly see the requested freq from sysfs is indeed 350 MHz so
> SLPC/PCODE is not honoring the set min == max == boost freq (and PMU is
> measuring what sysfs is showing). In general PCODE is the final arbiter in
> such cases and we do occasionally see instances where set freq limits are
> not honored.
>
> I would say if igt@perf_pmu@frequency is testing freq measured by PMU then
> the patch below is correct. Whether SLPC/PCODE is honoring the set freq
> limits should be tested in a SLPC test (which we also have).

igt@perf_pmu@frequency sets 'min == max == boost == 300 MHz' but we still
see the requested freq to be 350 MHz. Do we have a SLPC test covering this
scenario or should we add one? This is failing on one of the DG2's.

Thanks.
--
Ashutosh

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-07 10:27 ` Tvrtko Ursulin
@ 2022-11-08  0:18   ` Dixit, Ashutosh
  2022-11-08  0:22     ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Dixit, Ashutosh @ 2022-11-08  0:18 UTC (permalink / raw)
  To: Tvrtko Ursulin; +Cc: igt-dev

On Mon, 07 Nov 2022 02:27:16 -0800, Tvrtko Ursulin wrote:
>

Hi Tvrtko,

> On 07/11/2022 06:23, Ashutosh Dixit wrote:
> > In igt@perf_pmu@frequency subtest, compare the requested freq reported by
> > PMU not against the set freq but against the requested freq reported in
> > sysfs. If requested freq differs from the set freq (even when min == max
> > freq) for whatever reason, the sysfs requested freq should be closer to the
> > PMU measured requested freq which should minimize sporadic failures.
> >
> > Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
>
> So this bug describes the failure looks like this:
>
> Starting subtest: frequency
> (perf_pmu:1798) CRITICAL: Test assertion failure function test_frequency, file ../tests/i915/perf_pmu.c:1636:
> (perf_pmu:1798) CRITICAL: Failed assertion: (double)(min[0]) <= (1.0 + (tolerance)) * (double)(min_freq) && (double)(min[0]) >= (1.0 - (tolerance)) * (double)(min_freq)
> (perf_pmu:1798) CRITICAL: 'min[0]' != 'min_freq' (347.914580 not within +5.0%/-5.0% tolerance of 300.000000)
> Subtest frequency failed.
>
> Which is this in code:
>
>
>	min_freq = igt_sysfs_get_u32(sysfs, "gt_RPn_freq_mhz");
> ...
>	/*
>	 * Set GPU to min frequency and read PMU counters.
>	 */
>	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
>	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
>	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
>	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
>	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
>	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
> ...
>	gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes effect */
>	spin = spin_sync_flags(gem_fd, ahnd, 0, I915_EXEC_DEFAULT);
>
>	slept = pmu_read_multi(fd[0], 2, start);
>	measured_usleep(batch_duration_ns / 1000);
>	slept = pmu_read_multi(fd[0], 2, val) - slept;
>
>	min[0] = 1e9*(val[0] - start[0]) / slept;
>	min[1] = 1e9*(val[1] - start[1]) / slept;
> ...
>	/*
>	 * Restore min/max.
>	 */
>	igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq);
>	if (igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") != min_freq)
>		igt_warn("Unable to restore min frequency to saved value [%u MHz], now %u MHz\n",
>			 min_freq, igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz"));
>	close(fd[0]);
>	close(fd[1]);
>	put_ahnd(ahnd);
>
>	igt_info("Min frequency: requested %.1f, actual %.1f\n",
>		 min[0], min[1]);
>	igt_info("Max frequency: requested %.1f, actual %.1f\n",
>		 max[0], max[1]);
>
>	assert_within_epsilon(min[0], min_freq, tolerance);
>
> Which means sysfs told the test which is the minimum frequency. But it
> really isn't - it turns out 350 MHz is the real minimum? Or setting of
> gt_max_freq_mhz/gt_boost_freq_mhz is not actually respected by SLPC? It
> sounds trivial to respect the minimum frequency so I don't see what is
> the excuse.

So I submitted this patch to repro the issue and to print out the requested
freq from sysfs:

https://patchwork.freedesktop.org/series/110630/

And we can see the output here:

https://intel-gfx-ci.01.org/tree/drm-tip/IGTPW_8061/bat-dg2-11/igt@perf_pmu@frequency.html

```
IGT-Version: 1.26-g1bef4d081 (x86_64) (Linux: 6.1.0-rc4-CI_DRM_12352-gc55ac6a74bd1+ x86_64)
Starting subtest: frequency
Frequency: min=300, max=2050, boost=2050 MHz
Min frequency: requested 349.7, actual 349.7
Max frequency: requested 2048.0, actual 2048.0
Sysfs requested: min 350, max 2050
Stack trace:
  #0 ../../../usr/src/igt-gpu-tools/lib/igt_core.c:1908 __igt_fail_assert()
  #1 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:1656 __igt_unique____real_main2147()
  #2 ../../../usr/src/igt-gpu-tools/tests/i915/perf_pmu.c:2147 main()
  #3 [__libc_start_main+0xf3]
  #4 [_start+0x2e]
Subtest frequency: FAIL (2.212s)
```

So we clearly see the requested freq from sysfs is indeed 350 MHz so
SLPC/PCODE is not honoring the set min == max == boost freq (and PMU is
measuring what sysfs is showing). In general PCODE is the final arbiter in
such cases and we do occasionally see instances where set freq limits are
not honored.

I would say if igt@perf_pmu@frequency is testing freq measured by PMU then
the patch below is correct. Whether SLPC/PCODE is honoring the set freq
limits should be tested in a SLPC test (which we also have).

Thanks.
--
Ashutosh


>
> Regards,
>
> Tvrtko
>
> > Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> > Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> > ---
> >   tests/i915/perf_pmu.c | 8 +++++---
> >   1 file changed, 5 insertions(+), 3 deletions(-)
> >
> > diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> > index f363db2ba13..42b5b7bd6f2 100644
> > --- a/tests/i915/perf_pmu.c
> > +++ b/tests/i915/perf_pmu.c
> > @@ -1546,7 +1546,7 @@ test_interrupts_sync(int gem_fd)
> >   static void
> >   test_frequency(int gem_fd)
> >   {
> > -	uint32_t min_freq, max_freq, boost_freq;
> > +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
> >	uint64_t val[2], start[2], slept;
> >	double min[2], max[2];
> >	igt_spin_t *spin;
> > @@ -1587,6 +1587,7 @@ test_frequency(int gem_fd)
> >		min[0] = 1e9*(val[0] - start[0]) / slept;
> >	min[1] = 1e9*(val[1] - start[1]) / slept;
> > +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> >		igt_spin_free(gem_fd, spin);
> >	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
> > @@ -1613,6 +1614,7 @@ test_frequency(int gem_fd)
> >		max[0] = 1e9*(val[0] - start[0]) / slept;
> >	max[1] = 1e9*(val[1] - start[1]) / slept;
> > +	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
> >		igt_spin_free(gem_fd, spin);
> >	gem_quiescent_gpu(gem_fd);
> > @@ -1633,12 +1635,12 @@ test_frequency(int gem_fd)
> >	igt_info("Max frequency: requested %.1f, actual %.1f\n",
> >		 max[0], max[1]);
> >   -	assert_within_epsilon(min[0], min_freq, tolerance);
> > +	assert_within_epsilon(min[0], min_req, tolerance);
> >	/*
> >	 * On thermally throttled devices we cannot be sure maximum frequency
> >	 * can be reached so use larger tolerance downards.
> >	 */
> > -	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
> > +	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);
> >   }
> >     static void

^ permalink raw reply	[flat|nested] 34+ messages in thread

* Re: [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
  2022-11-07  6:23 Ashutosh Dixit
@ 2022-11-07 10:27 ` Tvrtko Ursulin
  2022-11-08  0:18   ` Dixit, Ashutosh
  0 siblings, 1 reply; 34+ messages in thread
From: Tvrtko Ursulin @ 2022-11-07 10:27 UTC (permalink / raw)
  To: Ashutosh Dixit, igt-dev


On 07/11/2022 06:23, Ashutosh Dixit wrote:
> In igt@perf_pmu@frequency subtest, compare the requested freq reported by
> PMU not against the set freq but against the requested freq reported in
> sysfs. If requested freq differs from the set freq (even when min == max
> freq) for whatever reason, the sysfs requested freq should be closer to the
> PMU measured requested freq which should minimize sporadic failures.
> 
> Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806

So this bug describes the failure looks like this:

Starting subtest: frequency
(perf_pmu:1798) CRITICAL: Test assertion failure function test_frequency, file ../tests/i915/perf_pmu.c:1636:
(perf_pmu:1798) CRITICAL: Failed assertion: (double)(min[0]) <= (1.0 + (tolerance)) * (double)(min_freq) && (double)(min[0]) >= (1.0 - (tolerance)) * (double)(min_freq)
(perf_pmu:1798) CRITICAL: 'min[0]' != 'min_freq' (347.914580 not within +5.0%/-5.0% tolerance of 300.000000)
Subtest frequency failed.

Which is this in code:


	min_freq = igt_sysfs_get_u32(sysfs, "gt_RPn_freq_mhz");
...
	/*
	 * Set GPU to min frequency and read PMU counters.
	 */
	igt_require(igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq));
	igt_require(igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") == min_freq);
	igt_require(igt_sysfs_set_u32(sysfs, "gt_max_freq_mhz", min_freq));
	igt_require(igt_sysfs_get_u32(sysfs, "gt_max_freq_mhz") == min_freq);
	igt_require(igt_sysfs_set_u32(sysfs, "gt_boost_freq_mhz", min_freq));
	igt_require(igt_sysfs_get_u32(sysfs, "gt_boost_freq_mhz") == min_freq);
...
	gem_quiescent_gpu(gem_fd); /* Idle to be sure the change takes effect */
	spin = spin_sync_flags(gem_fd, ahnd, 0, I915_EXEC_DEFAULT);

	slept = pmu_read_multi(fd[0], 2, start);
	measured_usleep(batch_duration_ns / 1000);
	slept = pmu_read_multi(fd[0], 2, val) - slept;

	min[0] = 1e9*(val[0] - start[0]) / slept;
	min[1] = 1e9*(val[1] - start[1]) / slept;
...
	/*
	 * Restore min/max.
	 */
	igt_sysfs_set_u32(sysfs, "gt_min_freq_mhz", min_freq);
	if (igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz") != min_freq)
		igt_warn("Unable to restore min frequency to saved value [%u MHz], now %u MHz\n",
			 min_freq, igt_sysfs_get_u32(sysfs, "gt_min_freq_mhz"));
	close(fd[0]);
	close(fd[1]);
	put_ahnd(ahnd);

	igt_info("Min frequency: requested %.1f, actual %.1f\n",
		 min[0], min[1]);
	igt_info("Max frequency: requested %.1f, actual %.1f\n",
		 max[0], max[1]);

	assert_within_epsilon(min[0], min_freq, tolerance);

Which means sysfs told the test which is the minimum frequency. But it really isn't - it turns out 350 MHz is the real minimum? Or setting of gt_max_freq_mhz/gt_boost_freq_mhz is not actually respected by SLPC? It sounds trivial to respect the minimum frequency so I don't see what is the excuse.

Regards,

Tvrtko

> Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
> Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
> ---
>   tests/i915/perf_pmu.c | 8 +++++---
>   1 file changed, 5 insertions(+), 3 deletions(-)
> 
> diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
> index f363db2ba13..42b5b7bd6f2 100644
> --- a/tests/i915/perf_pmu.c
> +++ b/tests/i915/perf_pmu.c
> @@ -1546,7 +1546,7 @@ test_interrupts_sync(int gem_fd)
>   static void
>   test_frequency(int gem_fd)
>   {
> -	uint32_t min_freq, max_freq, boost_freq;
> +	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
>   	uint64_t val[2], start[2], slept;
>   	double min[2], max[2];
>   	igt_spin_t *spin;
> @@ -1587,6 +1587,7 @@ test_frequency(int gem_fd)
>   
>   	min[0] = 1e9*(val[0] - start[0]) / slept;
>   	min[1] = 1e9*(val[1] - start[1]) / slept;
> +	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>   
>   	igt_spin_free(gem_fd, spin);
>   	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
> @@ -1613,6 +1614,7 @@ test_frequency(int gem_fd)
>   
>   	max[0] = 1e9*(val[0] - start[0]) / slept;
>   	max[1] = 1e9*(val[1] - start[1]) / slept;
> +	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
>   
>   	igt_spin_free(gem_fd, spin);
>   	gem_quiescent_gpu(gem_fd);
> @@ -1633,12 +1635,12 @@ test_frequency(int gem_fd)
>   	igt_info("Max frequency: requested %.1f, actual %.1f\n",
>   		 max[0], max[1]);
>   
> -	assert_within_epsilon(min[0], min_freq, tolerance);
> +	assert_within_epsilon(min[0], min_req, tolerance);
>   	/*
>   	 * On thermally throttled devices we cannot be sure maximum frequency
>   	 * can be reached so use larger tolerance downards.
>   	 */
> -	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
> +	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);
>   }
>   
>   static void

^ permalink raw reply	[flat|nested] 34+ messages in thread

* [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest
@ 2022-11-07  6:23 Ashutosh Dixit
  2022-11-07 10:27 ` Tvrtko Ursulin
  0 siblings, 1 reply; 34+ messages in thread
From: Ashutosh Dixit @ 2022-11-07  6:23 UTC (permalink / raw)
  To: igt-dev

In igt@perf_pmu@frequency subtest, compare the requested freq reported by
PMU not against the set freq but against the requested freq reported in
sysfs. If requested freq differs from the set freq (even when min == max
freq) for whatever reason, the sysfs requested freq should be closer to the
PMU measured requested freq which should minimize sporadic failures.

Bug: https://gitlab.freedesktop.org/drm/intel/-/issues/6806
Cc: Tvrtko Ursulin <tvrtko.ursulin@linux.intel.com>
Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
---
 tests/i915/perf_pmu.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/i915/perf_pmu.c b/tests/i915/perf_pmu.c
index f363db2ba13..42b5b7bd6f2 100644
--- a/tests/i915/perf_pmu.c
+++ b/tests/i915/perf_pmu.c
@@ -1546,7 +1546,7 @@ test_interrupts_sync(int gem_fd)
 static void
 test_frequency(int gem_fd)
 {
-	uint32_t min_freq, max_freq, boost_freq;
+	uint32_t min_freq, max_freq, boost_freq, min_req, max_req;
 	uint64_t val[2], start[2], slept;
 	double min[2], max[2];
 	igt_spin_t *spin;
@@ -1587,6 +1587,7 @@ test_frequency(int gem_fd)
 
 	min[0] = 1e9*(val[0] - start[0]) / slept;
 	min[1] = 1e9*(val[1] - start[1]) / slept;
+	min_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
 
 	igt_spin_free(gem_fd, spin);
 	gem_quiescent_gpu(gem_fd); /* Don't leak busy bo into the next phase */
@@ -1613,6 +1614,7 @@ test_frequency(int gem_fd)
 
 	max[0] = 1e9*(val[0] - start[0]) / slept;
 	max[1] = 1e9*(val[1] - start[1]) / slept;
+	max_req = igt_sysfs_get_u32(sysfs, "gt_cur_freq_mhz");
 
 	igt_spin_free(gem_fd, spin);
 	gem_quiescent_gpu(gem_fd);
@@ -1633,12 +1635,12 @@ test_frequency(int gem_fd)
 	igt_info("Max frequency: requested %.1f, actual %.1f\n",
 		 max[0], max[1]);
 
-	assert_within_epsilon(min[0], min_freq, tolerance);
+	assert_within_epsilon(min[0], min_req, tolerance);
 	/*
 	 * On thermally throttled devices we cannot be sure maximum frequency
 	 * can be reached so use larger tolerance downards.
 	 */
-	__assert_within_epsilon(max[0], max_freq, tolerance, 0.15f);
+	__assert_within_epsilon(max[0], max_req, tolerance, 0.15f);
 }
 
 static void
-- 
2.38.0

^ permalink raw reply related	[flat|nested] 34+ messages in thread

end of thread, other threads:[~2023-01-10 19:49 UTC | newest]

Thread overview: 34+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2022-11-08 19:06 [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest Ashutosh Dixit
2022-11-08 20:36 ` [igt-dev] ✓ Fi.CI.BAT: success for tests/perf_pmu: Compare against requested freq in frequency subtest (rev2) Patchwork
2022-11-09  3:51 ` [igt-dev] ✓ Fi.CI.IGT: " Patchwork
  -- strict thread matches above, loose matches on Subject: below --
2022-11-19  2:00 [igt-dev] [PATCH i-g-t] tests/perf_pmu: Compare against requested freq in frequency subtest Ashutosh Dixit
2022-11-21  9:09 ` Tvrtko Ursulin
2022-11-23  6:03   ` Dixit, Ashutosh
2022-11-24 12:42     ` Tvrtko Ursulin
2022-12-16  6:21       ` Dixit, Ashutosh
2022-12-16  9:37         ` Tvrtko Ursulin
2022-12-16 15:39           ` Rodrigo Vivi
2022-12-17  2:49             ` Dixit, Ashutosh
2022-12-19  8:46               ` Tvrtko Ursulin
2022-12-22 20:28                 ` Rodrigo Vivi
2022-12-23  9:22                   ` Tvrtko Ursulin
2022-12-23 15:39                     ` Rodrigo Vivi
2023-01-05 21:26                       ` Dixit, Ashutosh
2023-01-06 20:12                         ` Rodrigo Vivi
2023-01-06 20:39                           ` Belgaumkar, Vinay
2023-01-06 21:38                             ` Dixit, Ashutosh
2023-01-09 21:01                               ` Rodrigo Vivi
2023-01-10 19:49                                 ` Dixit, Ashutosh
2022-11-07  6:23 Ashutosh Dixit
2022-11-07 10:27 ` Tvrtko Ursulin
2022-11-08  0:18   ` Dixit, Ashutosh
2022-11-08  0:22     ` Dixit, Ashutosh
2022-11-08  0:57       ` Belgaumkar, Vinay
2022-11-08  1:31         ` Dixit, Ashutosh
2022-11-08  9:24           ` Tvrtko Ursulin
2022-11-08 21:02             ` Belgaumkar, Vinay
2022-11-09  1:53               ` Dixit, Ashutosh
2022-11-10  1:37                 ` Belgaumkar, Vinay
2022-11-10  4:20                   ` Dixit, Ashutosh
2022-11-09  1:49             ` Dixit, Ashutosh
2022-11-09  6:03               ` Dixit, Ashutosh

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.