[PATCH v2 0/2] drm/v3d: Expose GPU usage stats

[PATCH v2 0/2] drm/v3d: Expose GPU usage stats

[Maíra Canal]

This patchset exposes GPU usages stats both globally and per-file
descriptor.

The first patch exposes the accumulated amount of active time per client
through the fdinfo infrastructure. The amount of active time is exposed
for each V3D queue. Moreover, it exposes the number of jobs submitted to
each queue.

The second patch exposes the accumulated amount of active time for each
V3D queue, independent of the client. This data is exposed through the
sysfs interface.

With these patches, it is possible to calculate the GPU usage percentage
per queue globally and per-file descriptor.

* Example fdinfo output:

$ cat /proc/1140/fdinfo/4
pos:    0
flags:  02400002
mnt_id: 24
ino:    209
drm-driver:     v3d
drm-client-id:  44
drm-engine-bin:         1661076898 ns
v3d-jobs-bin:   19576 jobs
drm-engine-render:      31469427170 ns
v3d-jobs-render:        19575 jobs
drm-engine-tfu:         5002964 ns
v3d-jobs-tfu:   13 jobs
drm-engine-csd:         188038329691 ns
v3d-jobs-csd:   250393 jobs
drm-engine-cache_clean:         27736024038 ns
v3d-jobs-cache_clean:   250392 job

* Example gputop output:

DRM minor 128
 PID         bin               render               tfu                csd            cache_clean     NAME
1140 |▎                ||██▋               ||                 ||█████████████▍   ||█▋               | computecloth
1158 |▍                ||████████▉         ||                 ||                 ||                 | gears
1002 |▏                ||█▎                ||                 ||                 ||                 | chromium-browse

Best Regards,
- Maíra
---

v1 -> v2: https://lore.kernel.org/dri-devel/20230727142929.1275149-1-mcanal@igalia.com/T/

* Use sysfs to expose global GPU stats (Tvrtko Ursulin)

Maíra Canal (2):
  drm/v3d: Implement show_fdinfo() callback for GPU usage stats
  drm/v3d: Expose the total GPU usage stats on sysfs

 drivers/gpu/drm/v3d/Makefile    |   3 +-
 drivers/gpu/drm/v3d/v3d_drv.c   |  39 +++++++++++-
 drivers/gpu/drm/v3d/v3d_drv.h   |  30 ++++++++++
 drivers/gpu/drm/v3d/v3d_gem.c   |   6 +-
 drivers/gpu/drm/v3d/v3d_irq.c   |  33 +++++++++++
 drivers/gpu/drm/v3d/v3d_sched.c |  35 +++++++++++
 drivers/gpu/drm/v3d/v3d_sysfs.c | 101 ++++++++++++++++++++++++++++++++
 7 files changed, 244 insertions(+), 3 deletions(-)
 create mode 100644 drivers/gpu/drm/v3d/v3d_sysfs.c

--
2.41.0

[PATCH v2 1/2] drm/v3d: Implement show_fdinfo() callback for GPU usage stats

[Maíra Canal]

This patch exposes the accumulated amount of active time per client
through the fdinfo infrastructure. The amount of active time is exposed
for each V3D queue: BIN, RENDER, CSD, TFU and CACHE_CLEAN.

In order to calculate the amount of active time per client, a CPU clock
is used through the function local_clock(). The point where the jobs has
started is marked and is finally compared with the time that the job had
finished.

Moreover, the number of jobs submitted to each queue is also exposed on
fdinfo through the identifier "v3d-jobs-<queue>".

Co-developed-by: Jose Maria Casanova Crespo <jmcasanova@igalia.com>
Signed-off-by: Jose Maria Casanova Crespo <jmcasanova@igalia.com>
Signed-off-by: Maíra Canal <mcanal@igalia.com>
---
 drivers/gpu/drm/v3d/v3d_drv.c   | 30 +++++++++++++++++++++++++++++-
 drivers/gpu/drm/v3d/v3d_drv.h   | 23 +++++++++++++++++++++++
 drivers/gpu/drm/v3d/v3d_gem.c   |  1 +
 drivers/gpu/drm/v3d/v3d_irq.c   | 17 +++++++++++++++++
 drivers/gpu/drm/v3d/v3d_sched.c | 24 ++++++++++++++++++++++++
 5 files changed, 94 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index ffbbe9d527d3..ca65c707da03 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -19,6 +19,7 @@
 #include <linux/module.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/sched/clock.h>
 #include <linux/reset.h>
 
 #include <drm/drm_drv.h>
@@ -111,6 +112,10 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
 	v3d_priv->v3d = v3d;
 
 	for (i = 0; i < V3D_MAX_QUEUES; i++) {
+		v3d_priv->enabled_ns[i] = 0;
+		v3d_priv->start_ns[i] = 0;
+		v3d_priv->jobs_sent[i] = 0;
+
 		sched = &v3d->queue[i].sched;
 		drm_sched_entity_init(&v3d_priv->sched_entity[i],
 				      DRM_SCHED_PRIORITY_NORMAL, &sched,
@@ -136,7 +141,29 @@ v3d_postclose(struct drm_device *dev, struct drm_file *file)
 	kfree(v3d_priv);
 }
 
-DEFINE_DRM_GEM_FOPS(v3d_drm_fops);
+static void v3d_show_fdinfo(struct drm_printer *p, struct drm_file *file)
+{
+	struct v3d_file_priv *file_priv = file->driver_priv;
+	u64 timestamp = local_clock();
+	enum v3d_queue queue;
+
+	for (queue = 0; queue < V3D_MAX_QUEUES; queue++) {
+		drm_printf(p, "drm-engine-%s: \t%llu ns\n",
+			   v3d_queue_to_string(queue),
+			   file_priv->start_ns[queue] ? file_priv->enabled_ns[queue]
+						      + timestamp - file_priv->start_ns[queue]
+						      : file_priv->enabled_ns[queue]);
+
+		drm_printf(p, "v3d-jobs-%s: \t%llu jobs\n",
+			   v3d_queue_to_string(queue), file_priv->jobs_sent[queue]);
+	}
+}
+
+static const struct file_operations v3d_drm_fops = {
+	.owner = THIS_MODULE,
+	DRM_GEM_FOPS,
+	.show_fdinfo = drm_show_fdinfo,
+};
 
 /* DRM_AUTH is required on SUBMIT_CL for now, while we don't have GMP
  * protection between clients.  Note that render nodes would be
@@ -176,6 +203,7 @@ static const struct drm_driver v3d_drm_driver = {
 	.ioctls = v3d_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(v3d_drm_ioctls),
 	.fops = &v3d_drm_fops,
+	.show_fdinfo = v3d_show_fdinfo,
 
 	.name = DRIVER_NAME,
 	.desc = DRIVER_DESC,
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 7f664a4b2a75..7f2897e5b2cb 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -21,6 +21,18 @@ struct reset_control;
 
 #define V3D_MAX_QUEUES (V3D_CACHE_CLEAN + 1)
 
+static inline char *v3d_queue_to_string(enum v3d_queue queue)
+{
+	switch (queue) {
+	case V3D_BIN: return "bin";
+	case V3D_RENDER: return "render";
+	case V3D_TFU: return "tfu";
+	case V3D_CSD: return "csd";
+	case V3D_CACHE_CLEAN: return "cache_clean";
+	}
+	return "UNKNOWN";
+}
+
 struct v3d_queue_state {
 	struct drm_gpu_scheduler sched;
 
@@ -167,6 +179,12 @@ struct v3d_file_priv {
 	} perfmon;
 
 	struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
+
+	u64 start_ns[V3D_MAX_QUEUES];
+
+	u64 enabled_ns[V3D_MAX_QUEUES];
+
+	u64 jobs_sent[V3D_MAX_QUEUES];
 };
 
 struct v3d_bo {
@@ -238,6 +256,11 @@ struct v3d_job {
 	 */
 	struct v3d_perfmon *perfmon;
 
+	/* File descriptor of the process that submitted the job that could be used
+	 * for collecting stats by process of GPU usage.
+	 */
+	struct drm_file *file;
+
 	/* Callback for the freeing of the job on refcount going to 0. */
 	void (*free)(struct kref *ref);
 };
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 2e94ce788c71..40ed0c7c3fad 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -415,6 +415,7 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 	job = *container;
 	job->v3d = v3d;
 	job->free = free;
+	job->file = file_priv;
 
 	ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],
 				 v3d_priv);
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
index e714d5318f30..c898800ae9c2 100644
--- a/drivers/gpu/drm/v3d/v3d_irq.c
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -14,6 +14,7 @@
  */
 
 #include <linux/platform_device.h>
+#include <linux/sched/clock.h>
 
 #include "v3d_drv.h"
 #include "v3d_regs.h"
@@ -100,6 +101,10 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_FLDONE) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->bin_job->base.irq_fence);
+		struct v3d_file_priv *file = v3d->bin_job->base.file->driver_priv;
+
+		file->enabled_ns[V3D_BIN] += local_clock() - file->start_ns[V3D_BIN];
+		file->start_ns[V3D_BIN] = 0;
 
 		trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -109,6 +114,10 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_FRDONE) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->render_job->base.irq_fence);
+		struct v3d_file_priv *file = v3d->render_job->base.file->driver_priv;
+
+		file->enabled_ns[V3D_RENDER] += local_clock() - file->start_ns[V3D_RENDER];
+		file->start_ns[V3D_RENDER] = 0;
 
 		trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -118,6 +127,10 @@ v3d_irq(int irq, void *arg)
 	if (intsts & V3D_INT_CSDDONE) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->csd_job->base.irq_fence);
+		struct v3d_file_priv *file = v3d->csd_job->base.file->driver_priv;
+
+		file->enabled_ns[V3D_CSD] += local_clock() - file->start_ns[V3D_CSD];
+		file->start_ns[V3D_CSD] = 0;
 
 		trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -154,6 +167,10 @@ v3d_hub_irq(int irq, void *arg)
 	if (intsts & V3D_HUB_INT_TFUC) {
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->tfu_job->base.irq_fence);
+		struct v3d_file_priv *file = v3d->tfu_job->base.file->driver_priv;
+
+		file->enabled_ns[V3D_TFU] += local_clock() - file->start_ns[V3D_TFU];
+		file->start_ns[V3D_TFU] = 0;
 
 		trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 06238e6d7f5c..b360709c0765 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -18,6 +18,7 @@
  * semaphores to interlock between them.
  */
 
+#include <linux/sched/clock.h>
 #include <linux/kthread.h>
 
 #include "v3d_drv.h"
@@ -76,6 +77,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_bin_job *job = to_bin_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
+	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 	unsigned long irqflags;
@@ -107,6 +109,9 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
 
+	file->start_ns[V3D_BIN] = local_clock();
+	file->jobs_sent[V3D_BIN]++;
+
 	v3d_switch_perfmon(v3d, &job->base);
 
 	/* Set the current and end address of the control list.
@@ -131,6 +136,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_render_job *job = to_render_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
+	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 
@@ -158,6 +164,9 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
 			    job->start, job->end);
 
+	file->start_ns[V3D_RENDER] = local_clock();
+	file->jobs_sent[V3D_RENDER]++;
+
 	v3d_switch_perfmon(v3d, &job->base);
 
 	/* XXX: Set the QCFG */
@@ -176,6 +185,7 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_tfu_job *job = to_tfu_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
+	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 
@@ -190,6 +200,9 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 
 	trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 
+	file->start_ns[V3D_TFU] = local_clock();
+	file->jobs_sent[V3D_TFU]++;
+
 	V3D_WRITE(V3D_TFU_IIA, job->args.iia);
 	V3D_WRITE(V3D_TFU_IIS, job->args.iis);
 	V3D_WRITE(V3D_TFU_ICA, job->args.ica);
@@ -213,6 +226,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_csd_job *job = to_csd_job(sched_job);
 	struct v3d_dev *v3d = job->base.v3d;
+	struct v3d_file_priv *file = job->base.file->driver_priv;
 	struct drm_device *dev = &v3d->drm;
 	struct dma_fence *fence;
 	int i;
@@ -231,6 +245,9 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 
 	trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 
+	file->start_ns[V3D_CSD] = local_clock();
+	file->jobs_sent[V3D_CSD]++;
+
 	v3d_switch_perfmon(v3d, &job->base);
 
 	for (i = 1; i <= 6; i++)
@@ -246,9 +263,16 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 {
 	struct v3d_job *job = to_v3d_job(sched_job);
 	struct v3d_dev *v3d = job->v3d;
+	struct v3d_file_priv *file = job->file->driver_priv;
+
+	file->start_ns[V3D_CACHE_CLEAN] = local_clock();
+	file->jobs_sent[V3D_CACHE_CLEAN]++;
 
 	v3d_clean_caches(v3d);
 
+	file->enabled_ns[V3D_CACHE_CLEAN] += local_clock() - file->start_ns[V3D_CACHE_CLEAN];
+	file->start_ns[V3D_CACHE_CLEAN] = 0;
+
 	return NULL;
 }
 
-- 
2.41.0

[PATCH v2 2/2] drm/v3d: Expose the total GPU usage stats on sysfs

[Maíra Canal]

The previous patch exposed the accumulated amount of active time per
client for each V3D queue. But this doesn't provide a global notion of
the GPU usage.

Therefore, provide the accumulated amount of active time for each V3D
queue (BIN, RENDER, CSD, TFU and CACHE_CLEAN), considering all the jobs
submitted to the queue, independent of the client.

This data is exposed through the sysfs interface, so that if the
interface is queried at two different points of time the usage percentage
of each of the queues can be calculated.

Co-developed-by: Jose Maria Casanova Crespo <jmcasanova@igalia.com>
Signed-off-by: Jose Maria Casanova Crespo <jmcasanova@igalia.com>
Signed-off-by: Maíra Canal <mcanal@igalia.com>
---
 drivers/gpu/drm/v3d/Makefile    |   3 +-
 drivers/gpu/drm/v3d/v3d_drv.c   |   9 +++
 drivers/gpu/drm/v3d/v3d_drv.h   |   7 +++
 drivers/gpu/drm/v3d/v3d_gem.c   |   5 +-
 drivers/gpu/drm/v3d/v3d_irq.c   |  24 ++++++--
 drivers/gpu/drm/v3d/v3d_sched.c |  13 +++-
 drivers/gpu/drm/v3d/v3d_sysfs.c | 101 ++++++++++++++++++++++++++++++++
 7 files changed, 155 insertions(+), 7 deletions(-)
 create mode 100644 drivers/gpu/drm/v3d/v3d_sysfs.c

diff --git a/drivers/gpu/drm/v3d/Makefile b/drivers/gpu/drm/v3d/Makefile
index e8b314137020..4b21b20e4998 100644
--- a/drivers/gpu/drm/v3d/Makefile
+++ b/drivers/gpu/drm/v3d/Makefile
@@ -11,7 +11,8 @@ v3d-y := \
 	v3d_mmu.o \
 	v3d_perfmon.o \
 	v3d_trace_points.o \
-	v3d_sched.o
+	v3d_sched.o \
+	v3d_sysfs.o
 
 v3d-$(CONFIG_DEBUG_FS) += v3d_debugfs.o
 
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index ca65c707da03..7fc84a2525ca 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -309,8 +309,14 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
 	if (ret)
 		goto irq_disable;
 
+	ret = v3d_sysfs_init(dev);
+	if (ret)
+		goto drm_unregister;
+
 	return 0;
 
+drm_unregister:
+	drm_dev_unregister(drm);
 irq_disable:
 	v3d_irq_disable(v3d);
 gem_destroy:
@@ -324,6 +330,9 @@ static void v3d_platform_drm_remove(struct platform_device *pdev)
 {
 	struct drm_device *drm = platform_get_drvdata(pdev);
 	struct v3d_dev *v3d = to_v3d_dev(drm);
+	struct device *dev = &pdev->dev;
+
+	v3d_sysfs_destroy(dev);
 
 	drm_dev_unregister(drm);
 
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 7f2897e5b2cb..c8f95a91af46 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -38,6 +38,9 @@ struct v3d_queue_state {
 
 	u64 fence_context;
 	u64 emit_seqno;
+
+	u64 start_ns;
+	u64 enabled_ns;
 };
 
 /* Performance monitor object. The perform lifetime is controlled by userspace
@@ -441,3 +444,7 @@ int v3d_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv);
 int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
 				 struct drm_file *file_priv);
+
+/* v3d_sysfs.c */
+int v3d_sysfs_init(struct device *dev);
+void v3d_sysfs_destroy(struct device *dev);
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 40ed0c7c3fad..630ea2db8f8f 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -1014,8 +1014,11 @@ v3d_gem_init(struct drm_device *dev)
 	u32 pt_size = 4096 * 1024;
 	int ret, i;
 
-	for (i = 0; i < V3D_MAX_QUEUES; i++)
+	for (i = 0; i < V3D_MAX_QUEUES; i++) {
 		v3d->queue[i].fence_context = dma_fence_context_alloc(1);
+		v3d->queue[i].start_ns = 0;
+		v3d->queue[i].enabled_ns = 0;
+	}
 
 	spin_lock_init(&v3d->mm_lock);
 	spin_lock_init(&v3d->job_lock);
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
index c898800ae9c2..be4ff7559309 100644
--- a/drivers/gpu/drm/v3d/v3d_irq.c
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -102,9 +102,13 @@ v3d_irq(int irq, void *arg)
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->bin_job->base.irq_fence);
 		struct v3d_file_priv *file = v3d->bin_job->base.file->driver_priv;
+		u64 runtime = local_clock() - file->start_ns[V3D_BIN];
 
-		file->enabled_ns[V3D_BIN] += local_clock() - file->start_ns[V3D_BIN];
 		file->start_ns[V3D_BIN] = 0;
+		v3d->queue[V3D_BIN].start_ns = 0;
+
+		file->enabled_ns[V3D_BIN] += runtime;
+		v3d->queue[V3D_BIN].enabled_ns += runtime;
 
 		trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -115,9 +119,13 @@ v3d_irq(int irq, void *arg)
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->render_job->base.irq_fence);
 		struct v3d_file_priv *file = v3d->render_job->base.file->driver_priv;
+		u64 runtime = local_clock() - file->start_ns[V3D_RENDER];
 
-		file->enabled_ns[V3D_RENDER] += local_clock() - file->start_ns[V3D_RENDER];
 		file->start_ns[V3D_RENDER] = 0;
+		v3d->queue[V3D_RENDER].start_ns = 0;
+
+		file->enabled_ns[V3D_RENDER] += runtime;
+		v3d->queue[V3D_RENDER].enabled_ns += runtime;
 
 		trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -128,9 +136,13 @@ v3d_irq(int irq, void *arg)
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->csd_job->base.irq_fence);
 		struct v3d_file_priv *file = v3d->csd_job->base.file->driver_priv;
+		u64 runtime = local_clock() - file->start_ns[V3D_CSD];
 
-		file->enabled_ns[V3D_CSD] += local_clock() - file->start_ns[V3D_CSD];
 		file->start_ns[V3D_CSD] = 0;
+		v3d->queue[V3D_CSD].start_ns = 0;
+
+		file->enabled_ns[V3D_CSD] += runtime;
+		v3d->queue[V3D_CSD].enabled_ns += runtime;
 
 		trace_v3d_csd_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
@@ -168,9 +180,13 @@ v3d_hub_irq(int irq, void *arg)
 		struct v3d_fence *fence =
 			to_v3d_fence(v3d->tfu_job->base.irq_fence);
 		struct v3d_file_priv *file = v3d->tfu_job->base.file->driver_priv;
+		u64 runtime = local_clock() - file->start_ns[V3D_TFU];
 
-		file->enabled_ns[V3D_TFU] += local_clock() - file->start_ns[V3D_TFU];
 		file->start_ns[V3D_TFU] = 0;
+		v3d->queue[V3D_TFU].start_ns = 0;
+
+		file->enabled_ns[V3D_TFU] += runtime;
+		v3d->queue[V3D_TFU].enabled_ns += runtime;
 
 		trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
 		dma_fence_signal(&fence->base);
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index b360709c0765..1a9c7f395862 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -110,6 +110,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
 			    job->start, job->end);
 
 	file->start_ns[V3D_BIN] = local_clock();
+	v3d->queue[V3D_BIN].start_ns = file->start_ns[V3D_BIN];
 	file->jobs_sent[V3D_BIN]++;
 
 	v3d_switch_perfmon(v3d, &job->base);
@@ -165,6 +166,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
 			    job->start, job->end);
 
 	file->start_ns[V3D_RENDER] = local_clock();
+	v3d->queue[V3D_RENDER].start_ns = file->start_ns[V3D_RENDER];
 	file->jobs_sent[V3D_RENDER]++;
 
 	v3d_switch_perfmon(v3d, &job->base);
@@ -201,6 +203,7 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
 
 	file->start_ns[V3D_TFU] = local_clock();
+	v3d->queue[V3D_TFU].start_ns = file->start_ns[V3D_TFU];
 	file->jobs_sent[V3D_TFU]++;
 
 	V3D_WRITE(V3D_TFU_IIA, job->args.iia);
@@ -246,6 +249,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
 	trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
 
 	file->start_ns[V3D_CSD] = local_clock();
+	v3d->queue[V3D_CSD].start_ns = file->start_ns[V3D_CSD];
 	file->jobs_sent[V3D_CSD]++;
 
 	v3d_switch_perfmon(v3d, &job->base);
@@ -264,14 +268,21 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
 	struct v3d_job *job = to_v3d_job(sched_job);
 	struct v3d_dev *v3d = job->v3d;
 	struct v3d_file_priv *file = job->file->driver_priv;
+	u64 runtime;
 
 	file->start_ns[V3D_CACHE_CLEAN] = local_clock();
+	v3d->queue[V3D_CACHE_CLEAN].start_ns = file->start_ns[V3D_CACHE_CLEAN];
 	file->jobs_sent[V3D_CACHE_CLEAN]++;
 
 	v3d_clean_caches(v3d);
 
-	file->enabled_ns[V3D_CACHE_CLEAN] += local_clock() - file->start_ns[V3D_CACHE_CLEAN];
+	runtime = local_clock() - file->start_ns[V3D_CACHE_CLEAN];
+
+	file->enabled_ns[V3D_CACHE_CLEAN] += runtime;
+	v3d->queue[V3D_CACHE_CLEAN].enabled_ns += runtime;
+
 	file->start_ns[V3D_CACHE_CLEAN] = 0;
+	v3d->queue[V3D_CACHE_CLEAN].start_ns = 0;
 
 	return NULL;
 }
diff --git a/drivers/gpu/drm/v3d/v3d_sysfs.c b/drivers/gpu/drm/v3d/v3d_sysfs.c
new file mode 100644
index 000000000000..19cef3ddb7c6
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_sysfs.c
@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2023 Igalia S.L.
+ */
+
+#include <linux/sched/clock.h>
+#include <linux/sysfs.h>
+
+#include "v3d_drv.h"
+
+static u64
+v3d_sysfs_emit_total_runtime(struct v3d_dev *v3d, enum v3d_queue queue, char *buf)
+{
+	u64 timestamp = local_clock();
+	u64 active_runtime;
+
+	if (v3d->queue[queue].start_ns)
+		active_runtime = timestamp - v3d->queue[queue].start_ns;
+	else
+		active_runtime = 0;
+
+	return sysfs_emit(buf, "timestamp: %llu %s: %llu ns\n",
+			  timestamp,
+			  v3d_queue_to_string(queue),
+			  v3d->queue[queue].enabled_ns + active_runtime);
+}
+
+static ssize_t
+bin_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+
+	return v3d_sysfs_emit_total_runtime(v3d, V3D_BIN, buf);
+}
+static DEVICE_ATTR_RO(bin_queue);
+
+static ssize_t
+render_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+
+	return v3d_sysfs_emit_total_runtime(v3d, V3D_RENDER, buf);
+}
+static DEVICE_ATTR_RO(render_queue);
+
+static ssize_t
+tfu_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+
+	return v3d_sysfs_emit_total_runtime(v3d, V3D_TFU, buf);
+}
+static DEVICE_ATTR_RO(tfu_queue);
+
+static ssize_t
+csd_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+
+	return v3d_sysfs_emit_total_runtime(v3d, V3D_CSD, buf);
+}
+static DEVICE_ATTR_RO(csd_queue);
+
+static ssize_t
+cache_clean_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+
+	return v3d_sysfs_emit_total_runtime(v3d, V3D_CACHE_CLEAN, buf);
+}
+static DEVICE_ATTR_RO(cache_clean_queue);
+
+static struct attribute *v3d_sysfs_entries[] = {
+	&dev_attr_bin_queue.attr,
+	&dev_attr_render_queue.attr,
+	&dev_attr_tfu_queue.attr,
+	&dev_attr_csd_queue.attr,
+	&dev_attr_cache_clean_queue.attr,
+	NULL,
+};
+
+static struct attribute_group v3d_sysfs_attr_group = {
+	.attrs = v3d_sysfs_entries,
+};
+
+int
+v3d_sysfs_init(struct device *dev)
+{
+	return sysfs_create_group(&dev->kobj, &v3d_sysfs_attr_group);
+}
+
+void
+v3d_sysfs_destroy(struct device *dev)
+{
+	return sysfs_remove_group(&dev->kobj, &v3d_sysfs_attr_group);
+}
-- 
2.41.0

Re: [PATCH v2 1/2] drm/v3d: Implement show_fdinfo() callback for GPU usage stats

[Melissa Wen]

[-- Attachment #1.1: Type: text/plain, Size: 11246 bytes --]

On 08/07, Maíra Canal wrote:
> This patch exposes the accumulated amount of active time per client
> through the fdinfo infrastructure. The amount of active time is exposed
> for each V3D queue: BIN, RENDER, CSD, TFU and CACHE_CLEAN.
> 
> In order to calculate the amount of active time per client, a CPU clock
> is used through the function local_clock(). The point where the jobs has
> started is marked and is finally compared with the time that the job had
> finished.
> 
> Moreover, the number of jobs submitted to each queue is also exposed on
> fdinfo through the identifier "v3d-jobs-<queue>".
> 
> Co-developed-by: Jose Maria Casanova Crespo <jmcasanova@igalia.com>
> Signed-off-by: Jose Maria Casanova Crespo <jmcasanova@igalia.com>
> Signed-off-by: Maíra Canal <mcanal@igalia.com>
> ---
>  drivers/gpu/drm/v3d/v3d_drv.c   | 30 +++++++++++++++++++++++++++++-
>  drivers/gpu/drm/v3d/v3d_drv.h   | 23 +++++++++++++++++++++++
>  drivers/gpu/drm/v3d/v3d_gem.c   |  1 +
>  drivers/gpu/drm/v3d/v3d_irq.c   | 17 +++++++++++++++++
>  drivers/gpu/drm/v3d/v3d_sched.c | 24 ++++++++++++++++++++++++
>  5 files changed, 94 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
> index ffbbe9d527d3..ca65c707da03 100644
> --- a/drivers/gpu/drm/v3d/v3d_drv.c
> +++ b/drivers/gpu/drm/v3d/v3d_drv.c
> @@ -19,6 +19,7 @@
>  #include <linux/module.h>
>  #include <linux/of_platform.h>
>  #include <linux/platform_device.h>
> +#include <linux/sched/clock.h>
>  #include <linux/reset.h>
>  
>  #include <drm/drm_drv.h>
> @@ -111,6 +112,10 @@ v3d_open(struct drm_device *dev, struct drm_file *file)
>  	v3d_priv->v3d = v3d;
>  
>  	for (i = 0; i < V3D_MAX_QUEUES; i++) {
> +		v3d_priv->enabled_ns[i] = 0;
> +		v3d_priv->start_ns[i] = 0;
> +		v3d_priv->jobs_sent[i] = 0;
> +
>  		sched = &v3d->queue[i].sched;
>  		drm_sched_entity_init(&v3d_priv->sched_entity[i],
>  				      DRM_SCHED_PRIORITY_NORMAL, &sched,
> @@ -136,7 +141,29 @@ v3d_postclose(struct drm_device *dev, struct drm_file *file)
>  	kfree(v3d_priv);
>  }
>  
> -DEFINE_DRM_GEM_FOPS(v3d_drm_fops);
> +static void v3d_show_fdinfo(struct drm_printer *p, struct drm_file *file)
> +{
> +	struct v3d_file_priv *file_priv = file->driver_priv;
> +	u64 timestamp = local_clock();
> +	enum v3d_queue queue;
> +
> +	for (queue = 0; queue < V3D_MAX_QUEUES; queue++) {
> +		drm_printf(p, "drm-engine-%s: \t%llu ns\n",
> +			   v3d_queue_to_string(queue),
> +			   file_priv->start_ns[queue] ? file_priv->enabled_ns[queue]
> +						      + timestamp - file_priv->start_ns[queue]
> +						      : file_priv->enabled_ns[queue]);
> +
> +		drm_printf(p, "v3d-jobs-%s: \t%llu jobs\n",
> +			   v3d_queue_to_string(queue), file_priv->jobs_sent[queue]);
> +	}
> +}
> +
> +static const struct file_operations v3d_drm_fops = {
> +	.owner = THIS_MODULE,
> +	DRM_GEM_FOPS,
> +	.show_fdinfo = drm_show_fdinfo,
> +};

Dunno where, but could you document somewhere what is the expected
counting behavior in case of a GPU reset?

>  
>  /* DRM_AUTH is required on SUBMIT_CL for now, while we don't have GMP
>   * protection between clients.  Note that render nodes would be
> @@ -176,6 +203,7 @@ static const struct drm_driver v3d_drm_driver = {
>  	.ioctls = v3d_drm_ioctls,
>  	.num_ioctls = ARRAY_SIZE(v3d_drm_ioctls),
>  	.fops = &v3d_drm_fops,
> +	.show_fdinfo = v3d_show_fdinfo,
>  
>  	.name = DRIVER_NAME,
>  	.desc = DRIVER_DESC,
> diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
> index 7f664a4b2a75..7f2897e5b2cb 100644
> --- a/drivers/gpu/drm/v3d/v3d_drv.h
> +++ b/drivers/gpu/drm/v3d/v3d_drv.h
> @@ -21,6 +21,18 @@ struct reset_control;
>  
>  #define V3D_MAX_QUEUES (V3D_CACHE_CLEAN + 1)
>  
> +static inline char *v3d_queue_to_string(enum v3d_queue queue)
> +{
> +	switch (queue) {
> +	case V3D_BIN: return "bin";
> +	case V3D_RENDER: return "render";
> +	case V3D_TFU: return "tfu";
> +	case V3D_CSD: return "csd";
> +	case V3D_CACHE_CLEAN: return "cache_clean";
> +	}
> +	return "UNKNOWN";
> +}
> +
>  struct v3d_queue_state {
>  	struct drm_gpu_scheduler sched;
>  
> @@ -167,6 +179,12 @@ struct v3d_file_priv {
>  	} perfmon;
>  
>  	struct drm_sched_entity sched_entity[V3D_MAX_QUEUES];
> +
> +	u64 start_ns[V3D_MAX_QUEUES];
> +
> +	u64 enabled_ns[V3D_MAX_QUEUES];
> +
> +	u64 jobs_sent[V3D_MAX_QUEUES];
>  };
>  
>  struct v3d_bo {
> @@ -238,6 +256,11 @@ struct v3d_job {
>  	 */
>  	struct v3d_perfmon *perfmon;
>  
> +	/* File descriptor of the process that submitted the job that could be used
> +	 * for collecting stats by process of GPU usage.
> +	 */
> +	struct drm_file *file;
> +
>  	/* Callback for the freeing of the job on refcount going to 0. */
>  	void (*free)(struct kref *ref);
>  };
> diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
> index 2e94ce788c71..40ed0c7c3fad 100644
> --- a/drivers/gpu/drm/v3d/v3d_gem.c
> +++ b/drivers/gpu/drm/v3d/v3d_gem.c
> @@ -415,6 +415,7 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
>  	job = *container;
>  	job->v3d = v3d;
>  	job->free = free;
> +	job->file = file_priv;
>  
>  	ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue],
>  				 v3d_priv);
> diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
> index e714d5318f30..c898800ae9c2 100644
> --- a/drivers/gpu/drm/v3d/v3d_irq.c
> +++ b/drivers/gpu/drm/v3d/v3d_irq.c
> @@ -14,6 +14,7 @@
>   */
>  
>  #include <linux/platform_device.h>
> +#include <linux/sched/clock.h>
>  
>  #include "v3d_drv.h"
>  #include "v3d_regs.h"
> @@ -100,6 +101,10 @@ v3d_irq(int irq, void *arg)
>  	if (intsts & V3D_INT_FLDONE) {
>  		struct v3d_fence *fence =
>  			to_v3d_fence(v3d->bin_job->base.irq_fence);
> +		struct v3d_file_priv *file = v3d->bin_job->base.file->driver_priv;
> +
> +		file->enabled_ns[V3D_BIN] += local_clock() - file->start_ns[V3D_BIN];
> +		file->start_ns[V3D_BIN] = 0;
>  
>  		trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
>  		dma_fence_signal(&fence->base);
> @@ -109,6 +114,10 @@ v3d_irq(int irq, void *arg)
>  	if (intsts & V3D_INT_FRDONE) {
>  		struct v3d_fence *fence =
>  			to_v3d_fence(v3d->render_job->base.irq_fence);
> +		struct v3d_file_priv *file = v3d->render_job->base.file->driver_priv;
> +
> +		file->enabled_ns[V3D_RENDER] += local_clock() - file->start_ns[V3D_RENDER];
> +		file->start_ns[V3D_RENDER] = 0;
>  
>  		trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
>  		dma_fence_signal(&fence->base);
> @@ -118,6 +127,10 @@ v3d_irq(int irq, void *arg)
>  	if (intsts & V3D_INT_CSDDONE) {
>  		struct v3d_fence *fence =
>  			to_v3d_fence(v3d->csd_job->base.irq_fence);
> +		struct v3d_file_priv *file = v3d->csd_job->base.file->driver_priv;
> +
> +		file->enabled_ns[V3D_CSD] += local_clock() - file->start_ns[V3D_CSD];
> +		file->start_ns[V3D_CSD] = 0;
>  
>  		trace_v3d_csd_irq(&v3d->drm, fence->seqno);
>  		dma_fence_signal(&fence->base);
> @@ -154,6 +167,10 @@ v3d_hub_irq(int irq, void *arg)
>  	if (intsts & V3D_HUB_INT_TFUC) {
>  		struct v3d_fence *fence =
>  			to_v3d_fence(v3d->tfu_job->base.irq_fence);
> +		struct v3d_file_priv *file = v3d->tfu_job->base.file->driver_priv;
> +
> +		file->enabled_ns[V3D_TFU] += local_clock() - file->start_ns[V3D_TFU];
> +		file->start_ns[V3D_TFU] = 0;
>  
>  		trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
>  		dma_fence_signal(&fence->base);
> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
> index 06238e6d7f5c..b360709c0765 100644
> --- a/drivers/gpu/drm/v3d/v3d_sched.c
> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
> @@ -18,6 +18,7 @@
>   * semaphores to interlock between them.
>   */
>  
> +#include <linux/sched/clock.h>
>  #include <linux/kthread.h>
>  
>  #include "v3d_drv.h"
> @@ -76,6 +77,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
>  {
>  	struct v3d_bin_job *job = to_bin_job(sched_job);
>  	struct v3d_dev *v3d = job->base.v3d;
> +	struct v3d_file_priv *file = job->base.file->driver_priv;
>  	struct drm_device *dev = &v3d->drm;
>  	struct dma_fence *fence;
>  	unsigned long irqflags;
> @@ -107,6 +109,9 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
>  	trace_v3d_submit_cl(dev, false, to_v3d_fence(fence)->seqno,
>  			    job->start, job->end);
>  
> +	file->start_ns[V3D_BIN] = local_clock();
> +	file->jobs_sent[V3D_BIN]++;
> +
>  	v3d_switch_perfmon(v3d, &job->base);
>  
>  	/* Set the current and end address of the control list.
> @@ -131,6 +136,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
>  {
>  	struct v3d_render_job *job = to_render_job(sched_job);
>  	struct v3d_dev *v3d = job->base.v3d;
> +	struct v3d_file_priv *file = job->base.file->driver_priv;
>  	struct drm_device *dev = &v3d->drm;
>  	struct dma_fence *fence;
>  
> @@ -158,6 +164,9 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
>  	trace_v3d_submit_cl(dev, true, to_v3d_fence(fence)->seqno,
>  			    job->start, job->end);
>  
> +	file->start_ns[V3D_RENDER] = local_clock();
> +	file->jobs_sent[V3D_RENDER]++;
> +
>  	v3d_switch_perfmon(v3d, &job->base);
>  
>  	/* XXX: Set the QCFG */
> @@ -176,6 +185,7 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
>  {
>  	struct v3d_tfu_job *job = to_tfu_job(sched_job);
>  	struct v3d_dev *v3d = job->base.v3d;
> +	struct v3d_file_priv *file = job->base.file->driver_priv;
>  	struct drm_device *dev = &v3d->drm;
>  	struct dma_fence *fence;
>  
> @@ -190,6 +200,9 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
>  
>  	trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
>  
> +	file->start_ns[V3D_TFU] = local_clock();
> +	file->jobs_sent[V3D_TFU]++;
> +
>  	V3D_WRITE(V3D_TFU_IIA, job->args.iia);
>  	V3D_WRITE(V3D_TFU_IIS, job->args.iis);
>  	V3D_WRITE(V3D_TFU_ICA, job->args.ica);
> @@ -213,6 +226,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
>  {
>  	struct v3d_csd_job *job = to_csd_job(sched_job);
>  	struct v3d_dev *v3d = job->base.v3d;
> +	struct v3d_file_priv *file = job->base.file->driver_priv;
>  	struct drm_device *dev = &v3d->drm;
>  	struct dma_fence *fence;
>  	int i;
> @@ -231,6 +245,9 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
>  
>  	trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
>  
> +	file->start_ns[V3D_CSD] = local_clock();
> +	file->jobs_sent[V3D_CSD]++;
> +
>  	v3d_switch_perfmon(v3d, &job->base);
>  
>  	for (i = 1; i <= 6; i++)
> @@ -246,9 +263,16 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
>  {
>  	struct v3d_job *job = to_v3d_job(sched_job);
>  	struct v3d_dev *v3d = job->v3d;
> +	struct v3d_file_priv *file = job->file->driver_priv;
> +
> +	file->start_ns[V3D_CACHE_CLEAN] = local_clock();
> +	file->jobs_sent[V3D_CACHE_CLEAN]++;
>  
>  	v3d_clean_caches(v3d);
>  
> +	file->enabled_ns[V3D_CACHE_CLEAN] += local_clock() - file->start_ns[V3D_CACHE_CLEAN];
> +	file->start_ns[V3D_CACHE_CLEAN] = 0;
> +
>  	return NULL;
>  }
>  
> -- 
> 2.41.0
> 

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v2 2/2] drm/v3d: Expose the total GPU usage stats on sysfs

[Melissa Wen]

[-- Attachment #1: Type: text/plain, Size: 12233 bytes --]

On 08/07, Maíra Canal wrote:
> The previous patch exposed the accumulated amount of active time per
> client for each V3D queue. But this doesn't provide a global notion of
> the GPU usage.
> 
> Therefore, provide the accumulated amount of active time for each V3D
> queue (BIN, RENDER, CSD, TFU and CACHE_CLEAN), considering all the jobs
> submitted to the queue, independent of the client.
> 
> This data is exposed through the sysfs interface, so that if the
> interface is queried at two different points of time the usage percentage
> of each of the queues can be calculated.
> 
> Co-developed-by: Jose Maria Casanova Crespo <jmcasanova@igalia.com>
> Signed-off-by: Jose Maria Casanova Crespo <jmcasanova@igalia.com>
> Signed-off-by: Maíra Canal <mcanal@igalia.com>
> ---
>  drivers/gpu/drm/v3d/Makefile    |   3 +-
>  drivers/gpu/drm/v3d/v3d_drv.c   |   9 +++
>  drivers/gpu/drm/v3d/v3d_drv.h   |   7 +++
>  drivers/gpu/drm/v3d/v3d_gem.c   |   5 +-
>  drivers/gpu/drm/v3d/v3d_irq.c   |  24 ++++++--
>  drivers/gpu/drm/v3d/v3d_sched.c |  13 +++-
>  drivers/gpu/drm/v3d/v3d_sysfs.c | 101 ++++++++++++++++++++++++++++++++
>  7 files changed, 155 insertions(+), 7 deletions(-)
>  create mode 100644 drivers/gpu/drm/v3d/v3d_sysfs.c
> 
> diff --git a/drivers/gpu/drm/v3d/Makefile b/drivers/gpu/drm/v3d/Makefile
> index e8b314137020..4b21b20e4998 100644
> --- a/drivers/gpu/drm/v3d/Makefile
> +++ b/drivers/gpu/drm/v3d/Makefile
> @@ -11,7 +11,8 @@ v3d-y := \
>  	v3d_mmu.o \
>  	v3d_perfmon.o \
>  	v3d_trace_points.o \
> -	v3d_sched.o
> +	v3d_sched.o \
> +	v3d_sysfs.o
>  
>  v3d-$(CONFIG_DEBUG_FS) += v3d_debugfs.o
>  
> diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
> index ca65c707da03..7fc84a2525ca 100644
> --- a/drivers/gpu/drm/v3d/v3d_drv.c
> +++ b/drivers/gpu/drm/v3d/v3d_drv.c
> @@ -309,8 +309,14 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
>  	if (ret)
>  		goto irq_disable;
>  
> +	ret = v3d_sysfs_init(dev);
> +	if (ret)
> +		goto drm_unregister;
> +
>  	return 0;
>  
> +drm_unregister:
> +	drm_dev_unregister(drm);
>  irq_disable:
>  	v3d_irq_disable(v3d);
>  gem_destroy:
> @@ -324,6 +330,9 @@ static void v3d_platform_drm_remove(struct platform_device *pdev)
>  {
>  	struct drm_device *drm = platform_get_drvdata(pdev);
>  	struct v3d_dev *v3d = to_v3d_dev(drm);
> +	struct device *dev = &pdev->dev;
> +
> +	v3d_sysfs_destroy(dev);
>  
>  	drm_dev_unregister(drm);
>  
> diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
> index 7f2897e5b2cb..c8f95a91af46 100644
> --- a/drivers/gpu/drm/v3d/v3d_drv.h
> +++ b/drivers/gpu/drm/v3d/v3d_drv.h
> @@ -38,6 +38,9 @@ struct v3d_queue_state {
>  
>  	u64 fence_context;
>  	u64 emit_seqno;
> +
> +	u64 start_ns;
> +	u64 enabled_ns;
>  };
>  
>  /* Performance monitor object. The perform lifetime is controlled by userspace
> @@ -441,3 +444,7 @@ int v3d_perfmon_destroy_ioctl(struct drm_device *dev, void *data,
>  			      struct drm_file *file_priv);
>  int v3d_perfmon_get_values_ioctl(struct drm_device *dev, void *data,
>  				 struct drm_file *file_priv);
> +
> +/* v3d_sysfs.c */
> +int v3d_sysfs_init(struct device *dev);
> +void v3d_sysfs_destroy(struct device *dev);
> diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
> index 40ed0c7c3fad..630ea2db8f8f 100644
> --- a/drivers/gpu/drm/v3d/v3d_gem.c
> +++ b/drivers/gpu/drm/v3d/v3d_gem.c
> @@ -1014,8 +1014,11 @@ v3d_gem_init(struct drm_device *dev)
>  	u32 pt_size = 4096 * 1024;
>  	int ret, i;
>  
> -	for (i = 0; i < V3D_MAX_QUEUES; i++)
> +	for (i = 0; i < V3D_MAX_QUEUES; i++) {
>  		v3d->queue[i].fence_context = dma_fence_context_alloc(1);
> +		v3d->queue[i].start_ns = 0;
> +		v3d->queue[i].enabled_ns = 0;
> +	}
>  
>  	spin_lock_init(&v3d->mm_lock);
>  	spin_lock_init(&v3d->job_lock);
> diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
> index c898800ae9c2..be4ff7559309 100644
> --- a/drivers/gpu/drm/v3d/v3d_irq.c
> +++ b/drivers/gpu/drm/v3d/v3d_irq.c
> @@ -102,9 +102,13 @@ v3d_irq(int irq, void *arg)
>  		struct v3d_fence *fence =
>  			to_v3d_fence(v3d->bin_job->base.irq_fence);
>  		struct v3d_file_priv *file = v3d->bin_job->base.file->driver_priv;
> +		u64 runtime = local_clock() - file->start_ns[V3D_BIN];
>  
> -		file->enabled_ns[V3D_BIN] += local_clock() - file->start_ns[V3D_BIN];
>  		file->start_ns[V3D_BIN] = 0;
> +		v3d->queue[V3D_BIN].start_ns = 0;
> +
> +		file->enabled_ns[V3D_BIN] += runtime;
> +		v3d->queue[V3D_BIN].enabled_ns += runtime;
>  
>  		trace_v3d_bcl_irq(&v3d->drm, fence->seqno);
>  		dma_fence_signal(&fence->base);
> @@ -115,9 +119,13 @@ v3d_irq(int irq, void *arg)
>  		struct v3d_fence *fence =
>  			to_v3d_fence(v3d->render_job->base.irq_fence);
>  		struct v3d_file_priv *file = v3d->render_job->base.file->driver_priv;
> +		u64 runtime = local_clock() - file->start_ns[V3D_RENDER];
>  
> -		file->enabled_ns[V3D_RENDER] += local_clock() - file->start_ns[V3D_RENDER];
>  		file->start_ns[V3D_RENDER] = 0;
> +		v3d->queue[V3D_RENDER].start_ns = 0;
> +
> +		file->enabled_ns[V3D_RENDER] += runtime;
> +		v3d->queue[V3D_RENDER].enabled_ns += runtime;
>  
>  		trace_v3d_rcl_irq(&v3d->drm, fence->seqno);
>  		dma_fence_signal(&fence->base);
> @@ -128,9 +136,13 @@ v3d_irq(int irq, void *arg)
>  		struct v3d_fence *fence =
>  			to_v3d_fence(v3d->csd_job->base.irq_fence);
>  		struct v3d_file_priv *file = v3d->csd_job->base.file->driver_priv;
> +		u64 runtime = local_clock() - file->start_ns[V3D_CSD];
>  
> -		file->enabled_ns[V3D_CSD] += local_clock() - file->start_ns[V3D_CSD];
>  		file->start_ns[V3D_CSD] = 0;
> +		v3d->queue[V3D_CSD].start_ns = 0;
> +
> +		file->enabled_ns[V3D_CSD] += runtime;
> +		v3d->queue[V3D_CSD].enabled_ns += runtime;
>  
>  		trace_v3d_csd_irq(&v3d->drm, fence->seqno);
>  		dma_fence_signal(&fence->base);
> @@ -168,9 +180,13 @@ v3d_hub_irq(int irq, void *arg)
>  		struct v3d_fence *fence =
>  			to_v3d_fence(v3d->tfu_job->base.irq_fence);
>  		struct v3d_file_priv *file = v3d->tfu_job->base.file->driver_priv;
> +		u64 runtime = local_clock() - file->start_ns[V3D_TFU];
>  
> -		file->enabled_ns[V3D_TFU] += local_clock() - file->start_ns[V3D_TFU];
>  		file->start_ns[V3D_TFU] = 0;
> +		v3d->queue[V3D_TFU].start_ns = 0;
> +
> +		file->enabled_ns[V3D_TFU] += runtime;
> +		v3d->queue[V3D_TFU].enabled_ns += runtime;
>  
>  		trace_v3d_tfu_irq(&v3d->drm, fence->seqno);
>  		dma_fence_signal(&fence->base);
> diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
> index b360709c0765..1a9c7f395862 100644
> --- a/drivers/gpu/drm/v3d/v3d_sched.c
> +++ b/drivers/gpu/drm/v3d/v3d_sched.c
> @@ -110,6 +110,7 @@ static struct dma_fence *v3d_bin_job_run(struct drm_sched_job *sched_job)
>  			    job->start, job->end);
>  
>  	file->start_ns[V3D_BIN] = local_clock();
> +	v3d->queue[V3D_BIN].start_ns = file->start_ns[V3D_BIN];
>  	file->jobs_sent[V3D_BIN]++;
>  
>  	v3d_switch_perfmon(v3d, &job->base);
> @@ -165,6 +166,7 @@ static struct dma_fence *v3d_render_job_run(struct drm_sched_job *sched_job)
>  			    job->start, job->end);
>  
>  	file->start_ns[V3D_RENDER] = local_clock();
> +	v3d->queue[V3D_RENDER].start_ns = file->start_ns[V3D_RENDER];
>  	file->jobs_sent[V3D_RENDER]++;
>  
>  	v3d_switch_perfmon(v3d, &job->base);
> @@ -201,6 +203,7 @@ v3d_tfu_job_run(struct drm_sched_job *sched_job)
>  	trace_v3d_submit_tfu(dev, to_v3d_fence(fence)->seqno);
>  
>  	file->start_ns[V3D_TFU] = local_clock();
> +	v3d->queue[V3D_TFU].start_ns = file->start_ns[V3D_TFU];
>  	file->jobs_sent[V3D_TFU]++;
>  
>  	V3D_WRITE(V3D_TFU_IIA, job->args.iia);
> @@ -246,6 +249,7 @@ v3d_csd_job_run(struct drm_sched_job *sched_job)
>  	trace_v3d_submit_csd(dev, to_v3d_fence(fence)->seqno);
>  
>  	file->start_ns[V3D_CSD] = local_clock();
> +	v3d->queue[V3D_CSD].start_ns = file->start_ns[V3D_CSD];
>  	file->jobs_sent[V3D_CSD]++;
>  
>  	v3d_switch_perfmon(v3d, &job->base);
> @@ -264,14 +268,21 @@ v3d_cache_clean_job_run(struct drm_sched_job *sched_job)
>  	struct v3d_job *job = to_v3d_job(sched_job);
>  	struct v3d_dev *v3d = job->v3d;
>  	struct v3d_file_priv *file = job->file->driver_priv;
> +	u64 runtime;
>  
>  	file->start_ns[V3D_CACHE_CLEAN] = local_clock();
> +	v3d->queue[V3D_CACHE_CLEAN].start_ns = file->start_ns[V3D_CACHE_CLEAN];
>  	file->jobs_sent[V3D_CACHE_CLEAN]++;
>  
>  	v3d_clean_caches(v3d);
>  
> -	file->enabled_ns[V3D_CACHE_CLEAN] += local_clock() - file->start_ns[V3D_CACHE_CLEAN];
> +	runtime = local_clock() - file->start_ns[V3D_CACHE_CLEAN];
> +
> +	file->enabled_ns[V3D_CACHE_CLEAN] += runtime;
> +	v3d->queue[V3D_CACHE_CLEAN].enabled_ns += runtime;
> +
>  	file->start_ns[V3D_CACHE_CLEAN] = 0;
> +	v3d->queue[V3D_CACHE_CLEAN].start_ns = 0;
>  
>  	return NULL;
>  }
> diff --git a/drivers/gpu/drm/v3d/v3d_sysfs.c b/drivers/gpu/drm/v3d/v3d_sysfs.c
> new file mode 100644
> index 000000000000..19cef3ddb7c6
> --- /dev/null
> +++ b/drivers/gpu/drm/v3d/v3d_sysfs.c
> @@ -0,0 +1,101 @@
> +// SPDX-License-Identifier: MIT
> +/*
> + * Copyright © 2023 Igalia S.L.
> + */
> +
> +#include <linux/sched/clock.h>
> +#include <linux/sysfs.h>
> +
> +#include "v3d_drv.h"
> +
> +static u64
> +v3d_sysfs_emit_total_runtime(struct v3d_dev *v3d, enum v3d_queue queue, char *buf)
> +{
> +	u64 timestamp = local_clock();
> +	u64 active_runtime;
> +
> +	if (v3d->queue[queue].start_ns)
> +		active_runtime = timestamp - v3d->queue[queue].start_ns;
> +	else
> +		active_runtime = 0;
> +
> +	return sysfs_emit(buf, "timestamp: %llu %s: %llu ns\n",
> +			  timestamp,
> +			  v3d_queue_to_string(queue),
> +			  v3d->queue[queue].enabled_ns + active_runtime);
> +}

A brief description (code comment) of these outputs would be goo too.

Melissa

> +
> +static ssize_t
> +bin_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
> +{
> +	struct drm_device *drm = dev_get_drvdata(dev);
> +	struct v3d_dev *v3d = to_v3d_dev(drm);
> +
> +	return v3d_sysfs_emit_total_runtime(v3d, V3D_BIN, buf);
> +}
> +static DEVICE_ATTR_RO(bin_queue);
> +
> +static ssize_t
> +render_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
> +{
> +	struct drm_device *drm = dev_get_drvdata(dev);
> +	struct v3d_dev *v3d = to_v3d_dev(drm);
> +
> +	return v3d_sysfs_emit_total_runtime(v3d, V3D_RENDER, buf);
> +}
> +static DEVICE_ATTR_RO(render_queue);
> +
> +static ssize_t
> +tfu_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
> +{
> +	struct drm_device *drm = dev_get_drvdata(dev);
> +	struct v3d_dev *v3d = to_v3d_dev(drm);
> +
> +	return v3d_sysfs_emit_total_runtime(v3d, V3D_TFU, buf);
> +}
> +static DEVICE_ATTR_RO(tfu_queue);
> +
> +static ssize_t
> +csd_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
> +{
> +	struct drm_device *drm = dev_get_drvdata(dev);
> +	struct v3d_dev *v3d = to_v3d_dev(drm);
> +
> +	return v3d_sysfs_emit_total_runtime(v3d, V3D_CSD, buf);
> +}
> +static DEVICE_ATTR_RO(csd_queue);
> +
> +static ssize_t
> +cache_clean_queue_show(struct device *dev, struct device_attribute *attr, char *buf)
> +{
> +	struct drm_device *drm = dev_get_drvdata(dev);
> +	struct v3d_dev *v3d = to_v3d_dev(drm);
> +
> +	return v3d_sysfs_emit_total_runtime(v3d, V3D_CACHE_CLEAN, buf);
> +}
> +static DEVICE_ATTR_RO(cache_clean_queue);
> +
> +static struct attribute *v3d_sysfs_entries[] = {
> +	&dev_attr_bin_queue.attr,
> +	&dev_attr_render_queue.attr,
> +	&dev_attr_tfu_queue.attr,
> +	&dev_attr_csd_queue.attr,
> +	&dev_attr_cache_clean_queue.attr,
> +	NULL,
> +};
> +
> +static struct attribute_group v3d_sysfs_attr_group = {
> +	.attrs = v3d_sysfs_entries,
> +};
> +
> +int
> +v3d_sysfs_init(struct device *dev)
> +{
> +	return sysfs_create_group(&dev->kobj, &v3d_sysfs_attr_group);
> +}
> +
> +void
> +v3d_sysfs_destroy(struct device *dev)
> +{
> +	return sysfs_remove_group(&dev->kobj, &v3d_sysfs_attr_group);
> +}
> -- 
> 2.41.0
> 

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH v2 0/2] drm/v3d: Expose GPU usage stats

[Melissa Wen]

[-- Attachment #1: Type: text/plain, Size: 2945 bytes --]

On 08/07, Maíra Canal wrote:
> This patchset exposes GPU usages stats both globally and per-file
> descriptor.
> 
> The first patch exposes the accumulated amount of active time per client
> through the fdinfo infrastructure. The amount of active time is exposed
> for each V3D queue. Moreover, it exposes the number of jobs submitted to
> each queue.
> 
> The second patch exposes the accumulated amount of active time for each
> V3D queue, independent of the client. This data is exposed through the
> sysfs interface.
> 
> With these patches, it is possible to calculate the GPU usage percentage
> per queue globally and per-file descriptor.
> 
> * Example fdinfo output:
> 
> $ cat /proc/1140/fdinfo/4
> pos:    0
> flags:  02400002
> mnt_id: 24
> ino:    209
> drm-driver:     v3d
> drm-client-id:  44
> drm-engine-bin:         1661076898 ns
> v3d-jobs-bin:   19576 jobs
> drm-engine-render:      31469427170 ns
> v3d-jobs-render:        19575 jobs
> drm-engine-tfu:         5002964 ns
> v3d-jobs-tfu:   13 jobs
> drm-engine-csd:         188038329691 ns
> v3d-jobs-csd:   250393 jobs
> drm-engine-cache_clean:         27736024038 ns
> v3d-jobs-cache_clean:   250392 job
> 
> * Example gputop output:
> 
> DRM minor 128
>  PID         bin               render               tfu                csd            cache_clean     NAME
> 1140 |▎                ||██▋               ||                 ||█████████████▍   ||█▋               | computecloth
> 1158 |▍                ||████████▉         ||                 ||                 ||                 | gears
> 1002 |▏                ||█▎                ||                 ||                 ||                 | chromium-browse
> 

Hi Maíra,

Nice job!

Overall LGTM but I prefer to have an ack from Chema before applying it.

I left some comments regarding documentation for future users.
With this, you can add my r-b in the next version:
Reviewed-by: Melissa Wen <mwen@igalia.com>

Thanks,

Melissa

> Best Regards,
> - Maíra
> ---
> 
> v1 -> v2: https://lore.kernel.org/dri-devel/20230727142929.1275149-1-mcanal@igalia.com/T/
> 
> * Use sysfs to expose global GPU stats (Tvrtko Ursulin)
> 
> Maíra Canal (2):
>   drm/v3d: Implement show_fdinfo() callback for GPU usage stats
>   drm/v3d: Expose the total GPU usage stats on sysfs
> 
>  drivers/gpu/drm/v3d/Makefile    |   3 +-
>  drivers/gpu/drm/v3d/v3d_drv.c   |  39 +++++++++++-
>  drivers/gpu/drm/v3d/v3d_drv.h   |  30 ++++++++++
>  drivers/gpu/drm/v3d/v3d_gem.c   |   6 +-
>  drivers/gpu/drm/v3d/v3d_irq.c   |  33 +++++++++++
>  drivers/gpu/drm/v3d/v3d_sched.c |  35 +++++++++++
>  drivers/gpu/drm/v3d/v3d_sysfs.c | 101 ++++++++++++++++++++++++++++++++
>  7 files changed, 244 insertions(+), 3 deletions(-)
>  create mode 100644 drivers/gpu/drm/v3d/v3d_sysfs.c
> 
> --
> 2.41.0
> 

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]