[PATCH] drm/amdkfd: allocate MMIO/DOORBELL BOs with AMDGPU_GEM_CREATE_PREEMPTIBLE

[PATCH] drm/amdkfd: allocate MMIO/DOORBELL BOs with AMDGPU_GEM_CREATE_PREEMPTIBLE

[Lang Yu]

MMIO/DOORBELL BOs' backing resources(bus address resources that are
used to talk to the GPU) are not managed by GTT manager, but they
are counted by GTT manager. That makes no sense.

With AMDGPU_GEM_CREATE_PREEMPTIBLE flag, such BOs will be managed by
PREEMPT manager(for preemptible contexts, e.g., KFD). Then they won't
be evicted and don't need to be pinned as well.

But we still leave these BOs pinned to indicate that the underlying
resource never moves.

Signed-off-by: Lang Yu <Lang.Yu@amd.com>
---
 .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 32 +++++++++----------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 8b14c55a0793..fada3b149361 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1518,26 +1518,26 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
 		domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
 		alloc_flags = 0;
-	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+	} else {
 		domain = AMDGPU_GEM_DOMAIN_GTT;
 		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
 		alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
-		if (!offset || !*offset)
-			return -EINVAL;
-		user_addr = untagged_addr(*offset);
-	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
-			KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
-		domain = AMDGPU_GEM_DOMAIN_GTT;
-		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
-		bo_type = ttm_bo_type_sg;
-		alloc_flags = 0;
-		if (size > UINT_MAX)
+
+		if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
+			if (!offset || !*offset)
+				return -EINVAL;
+			user_addr = untagged_addr(*offset);
+		} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
+				    KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
+			bo_type = ttm_bo_type_sg;
+			if (size > UINT_MAX)
+				return -EINVAL;
+			sg = create_doorbell_sg(*offset, size);
+			if (!sg)
+				return -ENOMEM;
+		} else {
 			return -EINVAL;
-		sg = create_doorbell_sg(*offset, size);
-		if (!sg)
-			return -ENOMEM;
-	} else {
-		return -EINVAL;
+		}
 	}
 
 	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
-- 
2.25.1

Re: [PATCH] drm/amdkfd: allocate MMIO/DOORBELL BOs with AMDGPU_GEM_CREATE_PREEMPTIBLE

[Felix Kuehling]

On 2022-05-11 05:44, Lang Yu wrote:
> MMIO/DOORBELL BOs' backing resources(bus address resources that are
> used to talk to the GPU) are not managed by GTT manager, but they
> are counted by GTT manager. That makes no sense.
>
> With AMDGPU_GEM_CREATE_PREEMPTIBLE flag, such BOs will be managed by
> PREEMPT manager(for preemptible contexts, e.g., KFD). Then they won't
> be evicted and don't need to be pinned as well.
>
> But we still leave these BOs pinned to indicate that the underlying
> resource never moves.
>
> Signed-off-by: Lang Yu <Lang.Yu@amd.com>

Reviewed-by: Felix Kuehling <Felix.Kuehling@amd.com>


> ---
>   .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c  | 32 +++++++++----------
>   1 file changed, 16 insertions(+), 16 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> index 8b14c55a0793..fada3b149361 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
> @@ -1518,26 +1518,26 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
>   	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
>   		domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
>   		alloc_flags = 0;
> -	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
> +	} else {
>   		domain = AMDGPU_GEM_DOMAIN_GTT;
>   		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
>   		alloc_flags = AMDGPU_GEM_CREATE_PREEMPTIBLE;
> -		if (!offset || !*offset)
> -			return -EINVAL;
> -		user_addr = untagged_addr(*offset);
> -	} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
> -			KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
> -		domain = AMDGPU_GEM_DOMAIN_GTT;
> -		alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
> -		bo_type = ttm_bo_type_sg;
> -		alloc_flags = 0;
> -		if (size > UINT_MAX)
> +
> +		if (flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
> +			if (!offset || !*offset)
> +				return -EINVAL;
> +			user_addr = untagged_addr(*offset);
> +		} else if (flags & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL |
> +				    KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)) {
> +			bo_type = ttm_bo_type_sg;
> +			if (size > UINT_MAX)
> +				return -EINVAL;
> +			sg = create_doorbell_sg(*offset, size);
> +			if (!sg)
> +				return -ENOMEM;
> +		} else {
>   			return -EINVAL;
> -		sg = create_doorbell_sg(*offset, size);
> -		if (!sg)
> -			return -ENOMEM;
> -	} else {
> -		return -EINVAL;
> +		}
>   	}
>   
>   	*mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);