[PATCH 1/2] drm/amdkfd: clarify the origin of cpages returned by migration functions

[PATCH 1/2] drm/amdkfd: clarify the origin of cpages returned by migration functions

[Philip Yang]

cpages is only updated by migrate_vma_setup. So capture its value at
that point to clarify the significance of the number. The next patch
will add counting of actually migrated pages after migrate_vma_pages for
debug purposes.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 43 ++++++++++++------------
 1 file changed, 22 insertions(+), 21 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 147300a161d0..a14d0077e262 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -385,6 +385,7 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
 	struct kfd_process_device *pdd;
 	struct dma_fence *mfence = NULL;
 	struct migrate_vma migrate;
+	unsigned long cpages = 0;
 	dma_addr_t *scratch;
 	size_t size;
 	void *buf;
@@ -414,17 +415,17 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
 		goto out_free;
 	}
 
-	if (migrate.cpages != npages)
-		pr_debug("partial migration, 0x%lx/0x%llx pages migrated\n",
-			 migrate.cpages, npages);
-	else
-		pr_debug("0x%lx pages migrated\n", migrate.cpages);
-
-	if (!migrate.cpages) {
+	cpages = migrate.cpages;
+	if (!cpages) {
 		pr_debug("failed collect migrate sys pages [0x%lx 0x%lx]\n",
 			 prange->start, prange->last);
 		goto out_free;
 	}
+	if (cpages != npages)
+		pr_debug("partial migration, 0x%lx/0x%llx pages migrated\n",
+			 cpages, npages);
+	else
+		pr_debug("0x%lx pages migrated\n", cpages);
 
 	r = svm_migrate_copy_to_vram(adev, prange, &migrate, &mfence, scratch);
 	migrate_vma_pages(&migrate);
@@ -437,12 +438,12 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
 out_free:
 	kvfree(buf);
 out:
-	if (!r && migrate.cpages) {
+	if (!r && cpages) {
 		pdd = svm_range_get_pdd_by_adev(prange, adev);
 		if (pdd)
-			WRITE_ONCE(pdd->page_in, pdd->page_in + migrate.cpages);
+			WRITE_ONCE(pdd->page_in, pdd->page_in + cpages);
 
-		return migrate.cpages;
+		return cpages;
 	}
 	return r;
 }
@@ -619,6 +620,7 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 	struct kfd_process_device *pdd;
 	struct dma_fence *mfence = NULL;
 	struct migrate_vma migrate;
+	unsigned long cpages = 0;
 	dma_addr_t *scratch;
 	size_t size;
 	void *buf;
@@ -648,17 +650,17 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 		goto out_free;
 	}
 
-	if (migrate.cpages != npages)
-		pr_debug("partial migration, 0x%lx/0x%llx pages migrated\n",
-			 migrate.cpages, npages);
-	else
-		pr_debug("0x%lx pages migrated\n", migrate.cpages);
-
-	if (!migrate.cpages) {
+	cpages = migrate.cpages;
+	if (!cpages) {
 		pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",
 			 prange->start, prange->last);
 		goto out_free;
 	}
+	if (cpages != npages)
+		pr_debug("partial migration, 0x%lx/0x%llx pages migrated\n",
+			 cpages, npages);
+	else
+		pr_debug("0x%lx pages migrated\n", cpages);
 
 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,
 				    scratch, npages);
@@ -670,13 +672,12 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 out_free:
 	kvfree(buf);
 out:
-	if (!r && migrate.cpages) {
+	if (!r && cpages) {
 		pdd = svm_range_get_pdd_by_adev(prange, adev);
 		if (pdd)
-			WRITE_ONCE(pdd->page_out,
-				   pdd->page_out + migrate.cpages);
+			WRITE_ONCE(pdd->page_out, pdd->page_out + cpages);
 
-		return migrate.cpages;
+		return cpages;
 	}
 	return r;
 }
-- 
2.17.1

[PATCH 2/2] drm/amdkfd: debug message to count successfully migrated pages

[Philip Yang]

Not all migrate.cpages returned from migrate_vma_setup can be migrated,
for example non anonymous page, or out of device memory. So after
migrate_vma_pages returns, add debug message to count pages are
successfully migrated which has MIGRATE_PFN_VALID and
MIGRATE_PFN_MIGRATE flag set.

Signed-off-by: Philip Yang <Philip.Yang@amd.com>
---
 drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index a14d0077e262..6d8634e40b3b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -268,6 +268,19 @@ static void svm_migrate_put_sys_page(unsigned long addr)
 	put_page(page);
 }
 
+static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)
+{
+	unsigned long cpages = 0;
+	unsigned long i;
+
+	for (i = 0; i < migrate->npages; i++) {
+		if (migrate->src[i] & MIGRATE_PFN_VALID &&
+		    migrate->src[i] & MIGRATE_PFN_MIGRATE)
+			cpages++;
+	}
+	return cpages;
+}
+
 static int
 svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
 			 struct migrate_vma *migrate, struct dma_fence **mfence,
@@ -429,6 +442,10 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
 
 	r = svm_migrate_copy_to_vram(adev, prange, &migrate, &mfence, scratch);
 	migrate_vma_pages(&migrate);
+
+	pr_debug("successful/cpages/npages 0x%lx/0x%lx/0x%lx\n",
+		svm_migrate_successful_pages(&migrate), cpages, migrate.npages);
+
 	svm_migrate_copy_done(adev, mfence);
 	migrate_vma_finalize(&migrate);
 
@@ -665,6 +682,10 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
 	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,
 				    scratch, npages);
 	migrate_vma_pages(&migrate);
+
+	pr_debug("successful/cpages/npages 0x%lx/0x%lx/0x%lx\n",
+		svm_migrate_successful_pages(&migrate), cpages, migrate.npages);
+
 	svm_migrate_copy_done(adev, mfence);
 	migrate_vma_finalize(&migrate);
 	svm_range_dma_unmap(adev->dev, scratch, 0, npages);
-- 
2.17.1

Re: [PATCH 2/2] drm/amdkfd: debug message to count successfully migrated pages

[Felix Kuehling]

Am 2021-10-20 um 8:47 p.m. schrieb Philip Yang:
> Not all migrate.cpages returned from migrate_vma_setup can be migrated,
> for example non anonymous page, or out of device memory. So after
> migrate_vma_pages returns, add debug message to count pages are
> successfully migrated which has MIGRATE_PFN_VALID and
> MIGRATE_PFN_MIGRATE flag set.
>
> Signed-off-by: Philip Yang <Philip.Yang@amd.com>

The series is

Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>


> ---
>  drivers/gpu/drm/amd/amdkfd/kfd_migrate.c | 21 +++++++++++++++++++++
>  1 file changed, 21 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> index a14d0077e262..6d8634e40b3b 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
> @@ -268,6 +268,19 @@ static void svm_migrate_put_sys_page(unsigned long addr)
>  	put_page(page);
>  }
>  
> +static unsigned long svm_migrate_successful_pages(struct migrate_vma *migrate)
> +{
> +	unsigned long cpages = 0;
> +	unsigned long i;
> +
> +	for (i = 0; i < migrate->npages; i++) {
> +		if (migrate->src[i] & MIGRATE_PFN_VALID &&
> +		    migrate->src[i] & MIGRATE_PFN_MIGRATE)
> +			cpages++;
> +	}
> +	return cpages;
> +}
> +
>  static int
>  svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
>  			 struct migrate_vma *migrate, struct dma_fence **mfence,
> @@ -429,6 +442,10 @@ svm_migrate_vma_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
>  
>  	r = svm_migrate_copy_to_vram(adev, prange, &migrate, &mfence, scratch);
>  	migrate_vma_pages(&migrate);
> +
> +	pr_debug("successful/cpages/npages 0x%lx/0x%lx/0x%lx\n",
> +		svm_migrate_successful_pages(&migrate), cpages, migrate.npages);
> +
>  	svm_migrate_copy_done(adev, mfence);
>  	migrate_vma_finalize(&migrate);
>  
> @@ -665,6 +682,10 @@ svm_migrate_vma_to_ram(struct amdgpu_device *adev, struct svm_range *prange,
>  	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,
>  				    scratch, npages);
>  	migrate_vma_pages(&migrate);
> +
> +	pr_debug("successful/cpages/npages 0x%lx/0x%lx/0x%lx\n",
> +		svm_migrate_successful_pages(&migrate), cpages, migrate.npages);
> +
>  	svm_migrate_copy_done(adev, mfence);
>  	migrate_vma_finalize(&migrate);
>  	svm_range_dma_unmap(adev->dev, scratch, 0, npages);