All of lore.kernel.org
 help / color / mirror / Atom feed
* [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
@ 2020-09-17 17:11 Ramesh Errabolu
  2020-09-19  1:06 ` philip yang
  0 siblings, 1 reply; 4+ messages in thread
From: Ramesh Errabolu @ 2020-09-17 17:11 UTC (permalink / raw)
  To: amd-gfx; +Cc: Ramesh Errabolu

[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.

[How]
Surface files in Sysfs that allow user to determine the number of compute
units that are in use for a given process. One Sysfs file is used per
device.

Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h    |  28 ++++++-
 drivers/gpu/drm/amd/amdkfd/kfd_process.c | 101 +++++++++++++++++++----
 2 files changed, 114 insertions(+), 15 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 023629f28495..2ce03586d0dc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -631,7 +631,7 @@ enum kfd_pdd_bound {
 	PDD_BOUND_SUSPENDED,
 };
 
-#define MAX_SYSFS_FILENAME_LEN 11
+#define MAX_SYSFS_FILENAME_LEN 32
 
 /*
  * SDMA counter runs at 100MHz frequency.
@@ -692,6 +692,32 @@ struct kfd_process_device {
 	uint64_t sdma_past_activity_counter;
 	struct attribute attr_sdma;
 	char sdma_filename[MAX_SYSFS_FILENAME_LEN];
+
+	/*
+	 * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
+	 * that is associated with device encoded by "this" struct instance. The
+	 * value reflects CU usage by all of the waves launched by this process
+	 * on this device. A very important property of occupancy parameter is
+	 * that its value is a a snapshot of current use.
+	 *
+	 * Following is to be noted regarding how this parameter is reported:
+	 *
+	 *  The number of waves that a CU can launch is limited by couple of
+	 *  parameters. These are encoded by struct amdgpu_cu_info instance
+	 *  that is part of every device definition. For GFX9 devices this
+	 *  translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
+	 *  do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
+	 *  when they use. This could change for future devices and therefore
+	 *  this example should be considered as a guide.
+	 *
+	 *  All CU's of a device are available for the process. This may not be true
+	 *  under certain conditions - e.g. CU masking.
+	 *
+	 *  Finally number of CU's that are occupied by a process is affected by both
+	 *  number of CU's a device has along with number of other competing processes
+	 */
+	struct attribute attr_cu_occupancy;
+	char cu_occupancy_filename[MAX_SYSFS_FILENAME_LEN];
 };
 
 #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index a0e12a79ab7d..3baffbc828b2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -249,6 +249,63 @@ static void kfd_sdma_activity_worker(struct work_struct *work)
 	}
 }
 
+/**
+ * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
+ * by current process. Translates acquired wave count into number of compute units
+ * that are occupied.
+ *
+ * @atr: Handle of attribute that allows reporting of wave count. The attribute
+ * handle encapsulates GPU device it is associated with, thereby allowing collection
+ * of waves in flight, etc
+ *
+ * @buffer: Handle of user provided buffer updated with wave count
+ *
+ * Return: Number of bytes written to user buffer or an error value
+ */
+static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
+{
+	int cu_cnt;
+	int wave_cnt;
+	int max_waves_per_cu;
+	struct kfd_dev *dev = NULL;
+	struct kfd_process *proc = NULL;
+	struct amdgpu_device *adev = NULL;
+	struct kfd_process_device *pdd = NULL;
+
+	/* Acquire handle of Process-Device-Data associated with attribute */
+	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
+
+	/*
+	 * Acquire handle of Gpu Device associated with attribute. Determine
+	 * if ability to collect CU occupancy is defined for this device. If
+	 * so acquire the handle of process which encapsulates the PASID of
+	 * process. It is possible that the process might have zero work on
+	 * device. This is determined by checking if process has any queues
+	 * @note: Ability to collect is defined only for Gfx9 devices
+	 */
+	cu_cnt = 0;
+	dev = pdd->dev;
+	proc = pdd->process;
+	if (pdd->qpd.queue_count == 0) {
+		pr_info("%s: Gpu-Id: %d has no active queues for process %d\n",
+			__func__, dev->id, proc->pasid);
+		return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+	}
+
+	/* Collect wave count from device by reading relevant registers */
+	wave_cnt = 0;
+	if (dev->kfd2kgd->get_cu_occupancy != NULL)
+		dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid,
+					       &wave_cnt);
+
+	/* Translate wave count to number of compute units */
+	adev = (struct amdgpu_device *)dev->kgd;
+	max_waves_per_cu = adev->gfx.cu_info.simd_per_cu *
+		adev->gfx.cu_info.max_waves_per_simd;
+	cu_cnt = wave_cnt / max_waves_per_cu;
+	return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+}
+
 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
 			       char *buffer)
 {
@@ -279,6 +336,10 @@ static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
 		return snprintf(buffer, PAGE_SIZE, "%llu\n",
 				(sdma_activity_work_handler.sdma_activity_counter)/
 				 SDMA_ACTIVITY_DIVISOR);
+
+	/* Sysfs handle that gets CU occupancy is per device */
+	} else if (strncmp(attr->name, "cu_occupancy_", 13) == 0) {
+		return kfd_get_cu_occupancy(attr, buffer);
 	} else {
 		pr_err("Invalid attribute");
 		return -EINVAL;
@@ -432,27 +493,37 @@ static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
 	 * Create sysfs files for each GPU:
 	 * - proc/<pid>/vram_<gpuid>
 	 * - proc/<pid>/sdma_<gpuid>
+	 * - proc/<pid>/cu_occupancy_<gpuid>
 	 */
 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 		snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
 			 pdd->dev->id);
-		ret = kfd_sysfs_create_file(p, &pdd->attr_vram, pdd->vram_filename);
+		ret = kfd_sysfs_create_file(p, &pdd->attr_vram,
+					    pdd->vram_filename);
 		if (ret)
 			pr_warn("Creating vram usage for gpu id %d failed",
 				(int)pdd->dev->id);
-
-		snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
-			 pdd->dev->id);
-		ret = kfd_sysfs_create_file(p, &pdd->attr_sdma, pdd->sdma_filename);
-		if (ret)
-			pr_warn("Creating sdma usage for gpu id %d failed",
-				(int)pdd->dev->id);
-	}
-
+		if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
+			snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN,
+				 "sdma_%u", pdd->dev->id);
+			ret = kfd_sysfs_create_file(p, &pdd->attr_sdma,
+						    pdd->sdma_filename);
+			if (ret)
+				pr_warn("Creating sdma usage for gpu id %d failed",
+					(int)pdd->dev->id);
+			snprintf(pdd->cu_occupancy_filename,
+				 MAX_SYSFS_FILENAME_LEN, "cu_occupancy_%u",
+				 pdd->dev->id);
+			ret = kfd_sysfs_create_file(p, &pdd->attr_cu_occupancy,
+						    pdd->cu_occupancy_filename);
+			if (ret)
+				pr_warn("Creating CU occupancy file for gpu id %d failed",
+					(int)pdd->dev->id);
+			}
+		}
 	return ret;
 }
 
-
 void kfd_procfs_del_queue(struct queue *q)
 {
 	if (!q)
@@ -767,8 +838,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
 			amdgpu_amdkfd_gpuvm_release_process_vm(
 					pdd->dev->kgd, pdd->vm);
 			fput(pdd->drm_file);
-		}
-		else if (pdd->vm)
+		} else if (pdd->vm)
 			amdgpu_amdkfd_gpuvm_destroy_process_vm(
 				pdd->dev->kgd, pdd->vm);
 
@@ -815,7 +885,10 @@ static void kfd_process_wq_release(struct work_struct *work)
 
 		list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 			sysfs_remove_file(p->kobj, &pdd->attr_vram);
-			sysfs_remove_file(p->kobj, &pdd->attr_sdma);
+			if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
+				sysfs_remove_file(p->kobj, &pdd->attr_sdma);
+				sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy);
+			}
 		}
 
 		kobject_del(p->kobj);
-- 
2.27.0

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 4+ messages in thread

* Re: [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
  2020-09-17 17:11 [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use Ramesh Errabolu
@ 2020-09-19  1:06 ` philip yang
  0 siblings, 0 replies; 4+ messages in thread
From: philip yang @ 2020-09-19  1:06 UTC (permalink / raw)
  To: Ramesh Errabolu, amd-gfx


On 2020-09-17 1:11 p.m., Ramesh Errabolu wrote:
> +	if (pdd->qpd.queue_count == 0) {
> +		pr_info("%s: Gpu-Id: %d has no active queues for process %d\n",
> +			__func__, dev->id, proc->pasid);

pr_debug can be dynamically enabled in dmesg if needed, pr_info will 
always show in dmesg.

pr_debug("Gpu-Id: %d has no active queues for process %d\n",

		dev->id, proc->pasid);

Philip

> +		return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 4+ messages in thread

* RE: [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
  2020-09-25 22:03 Ramesh Errabolu
@ 2020-09-28 10:54 ` Russell, Kent
  0 siblings, 0 replies; 4+ messages in thread
From: Russell, Kent @ 2020-09-28 10:54 UTC (permalink / raw)
  To: Errabolu, Ramesh, amd-gfx; +Cc: Errabolu, Ramesh

[AMD Public Use]

Some minor things inline

> -----Original Message-----
> From: amd-gfx <amd-gfx-bounces@lists.freedesktop.org> On Behalf Of Ramesh Errabolu
> Sent: Friday, September 25, 2020 6:03 PM
> To: amd-gfx@lists.freedesktop.org
> Cc: Errabolu, Ramesh <Ramesh.Errabolu@amd.com>
> Subject: [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number
> of compute units that are in use.
> 
> [Why]
> Allow user to know how many compute units (CU) are in use at any given
> moment.
> 
> [How]
> Surface files in Sysfs that allow user to determine the number of compute
> units that are in use for a given process. One Sysfs file is used per
> device.
> 
> Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdkfd/kfd_priv.h    | 25 +++++++++
>  drivers/gpu/drm/amd/amdkfd/kfd_process.c | 68 +++++++++++++++++++++++-
>  2 files changed, 92 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> index 56f92cfff591..3df2b9936458 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
> @@ -705,6 +705,31 @@ struct kfd_process_device {
> 
>  	struct kobject *kobj_stats;
>  	unsigned int doorbell_index;
> +
> +	/*
> +	 * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
> +	 * that is associated with device encoded by "this" struct instance. The
> +	 * value reflects CU usage by all of the waves launched by this process
> +	 * on this device. A very important property of occupancy parameter is
> +	 * that its value is a a snapshot of current use.

is a snapshot

> +	 *
> +	 * Following is to be noted regarding how this parameter is reported:
> +	 *
> +	 *  The number of waves that a CU can launch is limited by couple of
> +	 *  parameters. These are encoded by struct amdgpu_cu_info instance
> +	 *  that is part of every device definition. For GFX9 devices this
> +	 *  translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
> +	 *  do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
> +	 *  when they use. This could change for future devices and therefore

when they do use scratch memory

> +	 *  this example should be considered as a guide.
> +	 *
> +	 *  All CU's of a device are available for the process. This may not be true
> +	 *  under certain conditions - e.g. CU masking.
> +	 *
> +	 *  Finally number of CU's that are occupied by a process is affected by both
> +	 *  number of CU's a device has along with number of other competing processes
> +	 */
> +	struct attribute attr_cu_occupancy;
>  };
> 
>  #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> index 17d909c86f50..26b716b5eb23 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> @@ -249,6 +249,52 @@ static void kfd_sdma_activity_worker(struct work_struct *work)
>  	}
>  }
> 
> +/**
> + * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
> + * by current process. Translates acquired wave count into number of compute units
> + * that are occupied.
> + *
> + * @atr: Handle of attribute that allows reporting of wave count. The attribute
> + * handle encapsulates GPU device it is associated with, thereby allowing collection
> + * of waves in flight, etc
> + *
> + * @buffer: Handle of user provided buffer updated with wave count
> + *
> + * Return: Number of bytes written to user buffer or an error value
> + */
> +static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
> +{
> +	int cu_cnt;
> +	int wave_cnt;
> +	int max_waves_per_cu;
> +	struct kfd_dev *dev = NULL;
> +	struct kfd_process *proc = NULL;
> +	struct kfd_process_device *pdd = NULL;
> +
> +	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
> +	dev = pdd->dev;
> +	if (dev->kfd2kgd->get_cu_occupancy == NULL)
> +		return -EINVAL;
> +
> +	cu_cnt = 0;
> +	proc = pdd->process;
> +	if (pdd->qpd.queue_count == 0) {
> +		pr_debug("Gpu-Id: %d has no active queues for process %d\n",
> +			 dev->id, proc->pasid);
> +		return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
> +	}
> +
> +	/* Collect wave count from device if it supports */
> +	wave_cnt = 0;
> +	max_waves_per_cu = 0;
> +	dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
> +			&max_waves_per_cu);
> +
> +	/* Translate wave count to number of compute units */
> +	cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
> +	return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
> +}
> +
>  static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
>  			       char *buffer)
>  {
> @@ -344,6 +390,7 @@ static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
> 
>  	return 0;
>  }
> +
>  static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
>  				     struct attribute *attr, char *buffer)
>  {
> @@ -359,6 +406,10 @@ static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
>  				PAGE_SIZE,
>  				"%llu\n",
>  				jiffies64_to_msecs(evict_jiffies));
> +
> +	/* Sysfs handle that gets CU occupancy is per device */
> +	} else if (strcmp(attr->name, "cu_occupancy") == 0) {
> +		return kfd_get_cu_occupancy(attr, buffer);
>  	} else
You should use } else { here, even for a single line, see https://www.kernel.org/doc/html/v4.10/process/coding-style.html right above section 3.1. Annoyingly, checkpatch doesn't catch this.
>  		pr_err("Invalid attribute");
> 
> @@ -466,6 +517,7 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
>  	 * Create sysfs files for each GPU:
>  	 * - proc/<pid>/stats_<gpuid>/
>  	 * - proc/<pid>/stats_<gpuid>/evicted_ms
> +	 * - proc/<pid>/stats_<gpuid>/cu_occupancy
>  	 */
>  	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
>  		struct kobject *kobj_stats;
> @@ -496,6 +548,19 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
>  		if (ret)
>  			pr_warn("Creating eviction stats for gpuid %d failed",
>  					(int)pdd->dev->id);
> +
> +		/* Add sysfs file to report compute unit occupancy */
> +		if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
> +			pdd->attr_cu_occupancy.name = "cu_occupancy";
> +			pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE;
> +			sysfs_attr_init(&pdd->attr_cu_occupancy);
> +			ret = sysfs_create_file(kobj_stats,
> +						&pdd->attr_cu_occupancy);
> +			if (ret)
> +				pr_warn("Creating %s failed for gpuid: %d",
> +					pdd->attr_cu_occupancy.name,
> +					(int)pdd->dev->id);
> +		}
>  	}
>  err:
>  	return ret;
> @@ -537,7 +602,6 @@ static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
>  	return ret;
>  }
> 
> -
>  void kfd_procfs_del_queue(struct queue *q)
>  {
>  	if (!q)
> @@ -909,6 +973,8 @@ static void kfd_process_wq_release(struct work_struct *work)
>  			sysfs_remove_file(p->kobj, &pdd->attr_vram);
>  			sysfs_remove_file(p->kobj, &pdd->attr_sdma);
>  			sysfs_remove_file(p->kobj, &pdd->attr_evict);
> +			if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL)
> +				sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy);
>  			kobject_del(pdd->kobj_stats);
>  			kobject_put(pdd->kobj_stats);
>  			pdd->kobj_stats = NULL;
> --
> 2.27.0
> 
> _______________________________________________
> amd-gfx mailing list
> amd-gfx@lists.freedesktop.org
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Flists.freedesktop.or
> g%2Fmailman%2Flistinfo%2Famd-
> gfx&amp;data=02%7C01%7Ckent.russell%40amd.com%7C4ed773ec69b04a9d2ee308d8619
> ee024%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637366682305777071&amp
> ;sdata=1NPpCMCB8dfWGwIokNotPm1pJU4AMkeurhcIe0y%2FIaw%3D&amp;reserved=0
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply	[flat|nested] 4+ messages in thread

* [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use.
@ 2020-09-25 22:03 Ramesh Errabolu
  2020-09-28 10:54 ` Russell, Kent
  0 siblings, 1 reply; 4+ messages in thread
From: Ramesh Errabolu @ 2020-09-25 22:03 UTC (permalink / raw)
  To: amd-gfx; +Cc: Ramesh Errabolu

[Why]
Allow user to know how many compute units (CU) are in use at any given
moment.

[How]
Surface files in Sysfs that allow user to determine the number of compute
units that are in use for a given process. One Sysfs file is used per
device.

Signed-off-by: Ramesh Errabolu <Ramesh.Errabolu@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_priv.h    | 25 +++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_process.c | 68 +++++++++++++++++++++++-
 2 files changed, 92 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 56f92cfff591..3df2b9936458 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -705,6 +705,31 @@ struct kfd_process_device {
 
 	struct kobject *kobj_stats;
 	unsigned int doorbell_index;
+
+	/*
+	 * @cu_occupancy: Reports occupancy of Compute Units (CU) of a process
+	 * that is associated with device encoded by "this" struct instance. The
+	 * value reflects CU usage by all of the waves launched by this process
+	 * on this device. A very important property of occupancy parameter is
+	 * that its value is a a snapshot of current use.
+	 *
+	 * Following is to be noted regarding how this parameter is reported:
+	 *
+	 *  The number of waves that a CU can launch is limited by couple of
+	 *  parameters. These are encoded by struct amdgpu_cu_info instance
+	 *  that is part of every device definition. For GFX9 devices this
+	 *  translates to 40 waves (simd_per_cu * max_waves_per_simd) when waves
+	 *  do not use scratch memory and 32 waves (max_scratch_slots_per_cu)
+	 *  when they use. This could change for future devices and therefore
+	 *  this example should be considered as a guide.
+	 *
+	 *  All CU's of a device are available for the process. This may not be true
+	 *  under certain conditions - e.g. CU masking.
+	 *
+	 *  Finally number of CU's that are occupied by a process is affected by both
+	 *  number of CU's a device has along with number of other competing processes
+	 */
+	struct attribute attr_cu_occupancy;
 };
 
 #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 17d909c86f50..26b716b5eb23 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -249,6 +249,52 @@ static void kfd_sdma_activity_worker(struct work_struct *work)
 	}
 }
 
+/**
+ * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
+ * by current process. Translates acquired wave count into number of compute units
+ * that are occupied.
+ *
+ * @atr: Handle of attribute that allows reporting of wave count. The attribute
+ * handle encapsulates GPU device it is associated with, thereby allowing collection
+ * of waves in flight, etc
+ *
+ * @buffer: Handle of user provided buffer updated with wave count
+ *
+ * Return: Number of bytes written to user buffer or an error value
+ */
+static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
+{
+	int cu_cnt;
+	int wave_cnt;
+	int max_waves_per_cu;
+	struct kfd_dev *dev = NULL;
+	struct kfd_process *proc = NULL;
+	struct kfd_process_device *pdd = NULL;
+
+	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
+	dev = pdd->dev;
+	if (dev->kfd2kgd->get_cu_occupancy == NULL)
+		return -EINVAL;
+
+	cu_cnt = 0;
+	proc = pdd->process;
+	if (pdd->qpd.queue_count == 0) {
+		pr_debug("Gpu-Id: %d has no active queues for process %d\n",
+			 dev->id, proc->pasid);
+		return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+	}
+
+	/* Collect wave count from device if it supports */
+	wave_cnt = 0;
+	max_waves_per_cu = 0;
+	dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
+			&max_waves_per_cu);
+
+	/* Translate wave count to number of compute units */
+	cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
+	return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
+}
+
 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
 			       char *buffer)
 {
@@ -344,6 +390,7 @@ static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
 
 	return 0;
 }
+
 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
 				     struct attribute *attr, char *buffer)
 {
@@ -359,6 +406,10 @@ static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
 				PAGE_SIZE,
 				"%llu\n",
 				jiffies64_to_msecs(evict_jiffies));
+
+	/* Sysfs handle that gets CU occupancy is per device */
+	} else if (strcmp(attr->name, "cu_occupancy") == 0) {
+		return kfd_get_cu_occupancy(attr, buffer);
 	} else
 		pr_err("Invalid attribute");
 
@@ -466,6 +517,7 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
 	 * Create sysfs files for each GPU:
 	 * - proc/<pid>/stats_<gpuid>/
 	 * - proc/<pid>/stats_<gpuid>/evicted_ms
+	 * - proc/<pid>/stats_<gpuid>/cu_occupancy
 	 */
 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
 		struct kobject *kobj_stats;
@@ -496,6 +548,19 @@ static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
 		if (ret)
 			pr_warn("Creating eviction stats for gpuid %d failed",
 					(int)pdd->dev->id);
+
+		/* Add sysfs file to report compute unit occupancy */
+		if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
+			pdd->attr_cu_occupancy.name = "cu_occupancy";
+			pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE;
+			sysfs_attr_init(&pdd->attr_cu_occupancy);
+			ret = sysfs_create_file(kobj_stats,
+						&pdd->attr_cu_occupancy);
+			if (ret)
+				pr_warn("Creating %s failed for gpuid: %d",
+					pdd->attr_cu_occupancy.name,
+					(int)pdd->dev->id);
+		}
 	}
 err:
 	return ret;
@@ -537,7 +602,6 @@ static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
 	return ret;
 }
 
-
 void kfd_procfs_del_queue(struct queue *q)
 {
 	if (!q)
@@ -909,6 +973,8 @@ static void kfd_process_wq_release(struct work_struct *work)
 			sysfs_remove_file(p->kobj, &pdd->attr_vram);
 			sysfs_remove_file(p->kobj, &pdd->attr_sdma);
 			sysfs_remove_file(p->kobj, &pdd->attr_evict);
+			if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL)
+				sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy);
 			kobject_del(pdd->kobj_stats);
 			kobject_put(pdd->kobj_stats);
 			pdd->kobj_stats = NULL;
-- 
2.27.0

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

^ permalink raw reply related	[flat|nested] 4+ messages in thread

end of thread, other threads:[~2020-09-28 10:54 UTC | newest]

Thread overview: 4+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2020-09-17 17:11 [PATCH 3/3] drm/amd/amdkfd: Surface files in Sysfs to allow users to get number of compute units that are in use Ramesh Errabolu
2020-09-19  1:06 ` philip yang
2020-09-25 22:03 Ramesh Errabolu
2020-09-28 10:54 ` Russell, Kent

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.