[PATCH 2/9] drm/amdgpu: Refine function amdgpu_csa_vaddr

[PATCH 2/9] drm/amdgpu: Refine function amdgpu_csa_vaddr

[Rex Zhu]

on baremetal, driver create csa per ctx.
So add a function argument: ctx_id to
get csa gpu addr.
In Sriov, driver create csa per process,
so ctx id always 1.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c | 5 +++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h | 2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 4 ++--
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c   | 6 +++---
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c   | 4 ++--
 5 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
index 0c590dd..44b046f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
@@ -24,11 +24,12 @@
 
 #include "amdgpu.h"
 
-uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
+uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id)
 {
 	uint64_t addr = adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT;
 
-	addr -= AMDGPU_VA_RESERVED_SIZE;
+	addr -= AMDGPU_VA_RESERVED_SIZE * id;
+
 	addr = amdgpu_gmc_sign_extend(addr);
 
 	return addr;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
index 524b443..aaf1fba 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
@@ -28,7 +28,7 @@
 #define AMDGPU_CSA_SIZE		(128 * 1024)
 
 uint32_t amdgpu_get_total_csa_size(struct amdgpu_device *adev);
-uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev);
+uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id);
 int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct amdgpu_bo **bo,
 				u32 domain, uint32_t size);
 int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 08d04f6..f736bda 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -977,9 +977,9 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 		goto error_vm;
 	}
 
-	if (amdgpu_sriov_vf(adev)) {
-		uint64_t csa_addr = amdgpu_csa_vaddr(adev) & AMDGPU_GMC_HOLE_MASK;
 
+	if (amdgpu_sriov_vf(adev)) {
+		uint64_t csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
 		r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
 						&fpriv->csa_va, csa_addr, AMDGPU_CSA_SIZE);
 		if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index bdae563..d529cef 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -7192,11 +7192,11 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
 	} ce_payload = {};
 
 	if (ring->adev->virt.chained_ib_support) {
-		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
+		ce_payload_addr = amdgpu_csa_vaddr(ring->adev, 1) +
 			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
 		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
 	} else {
-		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
+		ce_payload_addr = amdgpu_csa_vaddr(ring->adev, 1) +
 			offsetof(struct vi_gfx_meta_data, ce_payload);
 		cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
 	}
@@ -7220,7 +7220,7 @@ static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
 		struct vi_de_ib_state_chained_ib chained;
 	} de_payload = {};
 
-	csa_addr = amdgpu_csa_vaddr(ring->adev);
+	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
 	gds_addr = csa_addr + 4096;
 	if (ring->adev->virt.chained_ib_support) {
 		de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 928034c..81c1578 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -4327,7 +4327,7 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
 	int cnt;
 
 	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
-	csa_addr = amdgpu_csa_vaddr(ring->adev);
+	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
@@ -4345,7 +4345,7 @@ static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
 	uint64_t csa_addr, gds_addr;
 	int cnt;
 
-	csa_addr = amdgpu_csa_vaddr(ring->adev);
+	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
 	gds_addr = csa_addr + 4096;
 	de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
 	de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 3/9] drm/amdgpu: Use dynamical reserved vm size

[Rex Zhu]

Use dynamical reserved vm size instand of hardcode.

driver always reserve AMDGPU_VA_RESERVED_SIZE at the
bottom of VM space.

for gfx/sdma mcbp feature,
reserve AMDGPU_VA_RESERVED_SIZ * AMDGPU_VM_MAX_NUM_CTX
at the top of VM space.
For sriov, only need to reserve AMDGPU_VA_RESERVED_SIZE
at the top.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c |  8 ++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 10 ++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c |  3 +--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c  |  1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h  |  1 +
 drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c   |  2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c   |  2 +-
 drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c   |  2 +-
 9 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
index 44b046f..caa71c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
@@ -24,6 +24,14 @@
 
 #include "amdgpu.h"
 
+uint64_t amdgpu_get_reserved_csa_size(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev))
+		return AMDGPU_VA_RESERVED_SIZE;
+	else
+		return AMDGPU_VA_RESERVED_SIZE * AMDGPU_VM_MAX_NUM_CTX;
+}
+
 uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id)
 {
 	uint64_t addr = adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
index aaf1fba..7159d6d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
@@ -27,6 +27,7 @@
 
 #define AMDGPU_CSA_SIZE		(128 * 1024)
 
+uint64_t amdgpu_get_reserved_csa_size(struct amdgpu_device *adev);
 uint32_t amdgpu_get_total_csa_size(struct amdgpu_device *adev);
 uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id);
 int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct amdgpu_bo **bo,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 7b3d1eb..4c12de8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -557,6 +557,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 	struct ww_acquire_ctx ticket;
 	struct list_head list, duplicates;
 	uint64_t va_flags;
+	uint64_t va_reserved, va_top;
 	int r = 0;
 
 	if (args->va_address < AMDGPU_VA_RESERVED_SIZE) {
@@ -565,6 +566,15 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 			args->va_address, AMDGPU_VA_RESERVED_SIZE);
 		return -EINVAL;
 	}
+	va_top = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
+	va_reserved = va_top - adev->vm_manager.reserved_vm_size;
+
+	if (args->va_address > va_reserved && args->va_address < va_top) {
+		dev_dbg(&dev->pdev->dev,
+			"va_address 0x%LX is in reserved area 0x%LX\n",
+			args->va_address, adev->vm_manager.reserved_vm_size);
+		return -EINVAL;
+	}
 
 	if (args->va_address >= AMDGPU_GMC_HOLE_START &&
 	    args->va_address < AMDGPU_GMC_HOLE_END) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index f736bda..52e4e90 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -702,7 +702,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
 			dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION;
 
 		vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
-		vm_size -= AMDGPU_VA_RESERVED_SIZE;
+		vm_size -= adev->vm_manager.reserved_vm_size;
 
 		/* Older VCE FW versions are buggy and can handle only 40bits */
 		if (adev->vce.fw_version &&
@@ -977,7 +977,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 		goto error_vm;
 	}
 
-
 	if (amdgpu_sriov_vf(adev)) {
 		uint64_t csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
 		r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 352b304..2f2b9f4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -2901,6 +2901,7 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
 	}
 
 	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
+	adev->vm_manager.reserved_vm_size = amdgpu_get_reserved_csa_size(adev);
 
 	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
 	if (amdgpu_vm_block_size != -1)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 2a8898d..ec650ac 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -260,6 +260,7 @@ struct amdgpu_vm_manager {
 	unsigned				seqno[AMDGPU_MAX_RINGS];
 
 	uint64_t				max_pfn;
+	uint64_t				reserved_vm_size;
 	uint32_t				num_level;
 	uint32_t				block_size;
 	uint32_t				fragment_size;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index 2821d1d..3351b56 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -471,7 +471,7 @@ static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)
 	if (enable) {
 		uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
 		uint32_t high = adev->vm_manager.max_pfn -
-			(AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT);
+				(adev->vm_manager.reserved_vm_size >> AMDGPU_GPU_PAGE_SHIFT);
 
 		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
 		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 761dcfb..92bd175 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -568,7 +568,7 @@ static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)
 	if (enable) {
 		uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
 		uint32_t high = adev->vm_manager.max_pfn -
-			(AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT);
+				(adev->vm_manager.reserved_vm_size >> AMDGPU_GPU_PAGE_SHIFT);
 
 		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
 		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 531aaf3..d909ddf 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -772,7 +772,7 @@ static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)
 	if (enable) {
 		uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
 		uint32_t high = adev->vm_manager.max_pfn -
-			(AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT);
+				(adev->vm_manager.reserved_vm_size >> AMDGPU_GPU_PAGE_SHIFT);
 
 		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
 		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 3/9] drm/amdgpu: Use dynamical reserved vm size

[Christian König]

Am 06.12.18 um 13:14 schrieb Rex Zhu:
> Use dynamical reserved vm size instand of hardcode.
>
> driver always reserve AMDGPU_VA_RESERVED_SIZE at the
> bottom of VM space.
>
> for gfx/sdma mcbp feature,
> reserve AMDGPU_VA_RESERVED_SIZ * AMDGPU_VM_MAX_NUM_CTX
> at the top of VM space.
> For sriov, only need to reserve AMDGPU_VA_RESERVED_SIZE
> at the top.
>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c |  8 ++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h |  1 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 10 ++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c |  3 +--
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c  |  1 +
>   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h  |  1 +
>   drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c   |  2 +-
>   drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c   |  2 +-
>   drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c   |  2 +-
>   9 files changed, 25 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> index 44b046f..caa71c7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> @@ -24,6 +24,14 @@
>   
>   #include "amdgpu.h"
>   
> +uint64_t amdgpu_get_reserved_csa_size(struct amdgpu_device *adev)
> +{
> +	if (amdgpu_sriov_vf(adev))
> +		return AMDGPU_VA_RESERVED_SIZE;
> +	else
> +		return AMDGPU_VA_RESERVED_SIZE * AMDGPU_VM_MAX_NUM_CTX;

Do we really want AMDGPU_VA_RESERVED_SIZE * AMDGPU_VM_MAX_NUM_CTX here?

AMDGPU_CSA_SIZE * AMDGPU_VM_MAX_NUM_CTX should be sufficient I think.

And we should name the function to amdgpu_csa_get_reserved_vm_space().

> +}
> +
>   uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id)
>   {
>   	uint64_t addr = adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> index aaf1fba..7159d6d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> @@ -27,6 +27,7 @@
>   
>   #define AMDGPU_CSA_SIZE		(128 * 1024)
>   
> +uint64_t amdgpu_get_reserved_csa_size(struct amdgpu_device *adev);
>   uint32_t amdgpu_get_total_csa_size(struct amdgpu_device *adev);
>   uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id);
>   int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct amdgpu_bo **bo,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> index 7b3d1eb..4c12de8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> @@ -557,6 +557,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
>   	struct ww_acquire_ctx ticket;
>   	struct list_head list, duplicates;
>   	uint64_t va_flags;
> +	uint64_t va_reserved, va_top;
>   	int r = 0;
>   
>   	if (args->va_address < AMDGPU_VA_RESERVED_SIZE) {
> @@ -565,6 +566,15 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
>   			args->va_address, AMDGPU_VA_RESERVED_SIZE);
>   		return -EINVAL;
>   	}
> +	va_top = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
> +	va_reserved = va_top - adev->vm_manager.reserved_vm_size;
> +
> +	if (args->va_address > va_reserved && args->va_address < va_top) {
> +		dev_dbg(&dev->pdev->dev,
> +			"va_address 0x%LX is in reserved area 0x%LX\n",
> +			args->va_address, adev->vm_manager.reserved_vm_size);
> +		return -EINVAL;
> +	}
>   
>   	if (args->va_address >= AMDGPU_GMC_HOLE_START &&
>   	    args->va_address < AMDGPU_GMC_HOLE_END) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index f736bda..52e4e90 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -702,7 +702,7 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
>   			dev_info.ids_flags |= AMDGPU_IDS_FLAGS_PREEMPTION;
>   
>   		vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
> -		vm_size -= AMDGPU_VA_RESERVED_SIZE;
> +		vm_size -= adev->vm_manager.reserved_vm_size;
>   
>   		/* Older VCE FW versions are buggy and can handle only 40bits */
>   		if (adev->vce.fw_version &&
> @@ -977,7 +977,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>   		goto error_vm;
>   	}
>   
> -
>   	if (amdgpu_sriov_vf(adev)) {
>   		uint64_t csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
>   		r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index 352b304..2f2b9f4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -2901,6 +2901,7 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
>   	}
>   
>   	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
> +	adev->vm_manager.reserved_vm_size = amdgpu_get_reserved_csa_size(adev);

Maybe calculate that directly as max_pfn - reserved and name it 
something like max_user_pfn.

Regards,
Christian.

>   
>   	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
>   	if (amdgpu_vm_block_size != -1)
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> index 2a8898d..ec650ac 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> @@ -260,6 +260,7 @@ struct amdgpu_vm_manager {
>   	unsigned				seqno[AMDGPU_MAX_RINGS];
>   
>   	uint64_t				max_pfn;
> +	uint64_t				reserved_vm_size;
>   	uint32_t				num_level;
>   	uint32_t				block_size;
>   	uint32_t				fragment_size;
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
> index 2821d1d..3351b56 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
> @@ -471,7 +471,7 @@ static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)
>   	if (enable) {
>   		uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
>   		uint32_t high = adev->vm_manager.max_pfn -
> -			(AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT);
> +				(adev->vm_manager.reserved_vm_size >> AMDGPU_GPU_PAGE_SHIFT);
>   
>   		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
>   		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
> index 761dcfb..92bd175 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
> @@ -568,7 +568,7 @@ static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)
>   	if (enable) {
>   		uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
>   		uint32_t high = adev->vm_manager.max_pfn -
> -			(AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT);
> +				(adev->vm_manager.reserved_vm_size >> AMDGPU_GPU_PAGE_SHIFT);
>   
>   		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
>   		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
> index 531aaf3..d909ddf 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
> @@ -772,7 +772,7 @@ static void gmc_v8_0_set_prt(struct amdgpu_device *adev, bool enable)
>   	if (enable) {
>   		uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
>   		uint32_t high = adev->vm_manager.max_pfn -
> -			(AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT);
> +				(adev->vm_manager.reserved_vm_size >> AMDGPU_GPU_PAGE_SHIFT);
>   
>   		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
>   		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [PATCH 3/9] drm/amdgpu: Use dynamical reserved vm size

[Zhu, Rex]

> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Thursday, December 6, 2018 8:33 PM
> To: Zhu, Rex <Rex.Zhu@amd.com>; amd-gfx@lists.freedesktop.org
> Subject: Re: [PATCH 3/9] drm/amdgpu: Use dynamical reserved vm size
> 
> Am 06.12.18 um 13:14 schrieb Rex Zhu:
> > Use dynamical reserved vm size instand of hardcode.
> >
> > driver always reserve AMDGPU_VA_RESERVED_SIZE at the bottom of VM
> > space.
> >
> > for gfx/sdma mcbp feature,
> > reserve AMDGPU_VA_RESERVED_SIZ * AMDGPU_VM_MAX_NUM_CTX at
> the top of
> > VM space.
> > For sriov, only need to reserve AMDGPU_VA_RESERVED_SIZE at the top.
> >
> > Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c |  8 ++++++++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h |  1 +
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c | 10 ++++++++++
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c |  3 +--
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c  |  1 +
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h  |  1 +
> >   drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c   |  2 +-
> >   drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c   |  2 +-
> >   drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c   |  2 +-
> >   9 files changed, 25 insertions(+), 5 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> > index 44b046f..caa71c7 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> > @@ -24,6 +24,14 @@
> >
> >   #include "amdgpu.h"
> >
> > +uint64_t amdgpu_get_reserved_csa_size(struct amdgpu_device *adev) {
> > +	if (amdgpu_sriov_vf(adev))
> > +		return AMDGPU_VA_RESERVED_SIZE;
> > +	else
> > +		return AMDGPU_VA_RESERVED_SIZE *
> AMDGPU_VM_MAX_NUM_CTX;
> 
> Do we really want AMDGPU_VA_RESERVED_SIZE *
> AMDGPU_VM_MAX_NUM_CTX here?
> 
> AMDGPU_CSA_SIZE * AMDGPU_VM_MAX_NUM_CTX should be sufficient I
> think.

Yes, You are right. I will fix this error.
Thanks.


> And we should name the function to amdgpu_csa_get_reserved_vm_space().
> 
> > +}
> > +
> >   uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id)
> >   {
> >   	uint64_t addr = adev->vm_manager.max_pfn <<
> AMDGPU_GPU_PAGE_SHIFT;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> > index aaf1fba..7159d6d 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> > @@ -27,6 +27,7 @@
> >
> >   #define AMDGPU_CSA_SIZE		(128 * 1024)
> >
> > +uint64_t amdgpu_get_reserved_csa_size(struct amdgpu_device *adev);
> >   uint32_t amdgpu_get_total_csa_size(struct amdgpu_device *adev);
> >   uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id);
> >   int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct
> > amdgpu_bo **bo, diff --git
> a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> > index 7b3d1eb..4c12de8 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> > @@ -557,6 +557,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev,
> void *data,
> >   	struct ww_acquire_ctx ticket;
> >   	struct list_head list, duplicates;
> >   	uint64_t va_flags;
> > +	uint64_t va_reserved, va_top;
> >   	int r = 0;
> >
> >   	if (args->va_address < AMDGPU_VA_RESERVED_SIZE) { @@ -565,6
> +566,15
> > @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
> >   			args->va_address, AMDGPU_VA_RESERVED_SIZE);
> >   		return -EINVAL;
> >   	}
> > +	va_top = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
> > +	va_reserved = va_top - adev->vm_manager.reserved_vm_size;
> > +
> > +	if (args->va_address > va_reserved && args->va_address < va_top) {
> > +		dev_dbg(&dev->pdev->dev,
> > +			"va_address 0x%LX is in reserved area 0x%LX\n",
> > +			args->va_address, adev-
> >vm_manager.reserved_vm_size);
> > +		return -EINVAL;
> > +	}
> >
> >   	if (args->va_address >= AMDGPU_GMC_HOLE_START &&
> >   	    args->va_address < AMDGPU_GMC_HOLE_END) { diff --git
> > a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index f736bda..52e4e90 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -702,7 +702,7 @@ static int amdgpu_info_ioctl(struct drm_device
> *dev, void *data, struct drm_file
> >   			dev_info.ids_flags |=
> AMDGPU_IDS_FLAGS_PREEMPTION;
> >
> >   		vm_size = adev->vm_manager.max_pfn *
> AMDGPU_GPU_PAGE_SIZE;
> > -		vm_size -= AMDGPU_VA_RESERVED_SIZE;
> > +		vm_size -= adev->vm_manager.reserved_vm_size;
> >
> >   		/* Older VCE FW versions are buggy and can handle only
> 40bits */
> >   		if (adev->vce.fw_version &&
> > @@ -977,7 +977,6 @@ int amdgpu_driver_open_kms(struct drm_device
> *dev, struct drm_file *file_priv)
> >   		goto error_vm;
> >   	}
> >
> > -
> >   	if (amdgpu_sriov_vf(adev)) {
> >   		uint64_t csa_addr = amdgpu_csa_vaddr(adev, 1) &
> AMDGPU_GMC_HOLE_MASK;
> >   		r = amdgpu_map_static_csa(adev, &fpriv->vm, adev-
> >virt.csa_obj,
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> > index 352b304..2f2b9f4 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> > @@ -2901,6 +2901,7 @@ void amdgpu_vm_adjust_size(struct
> amdgpu_device *adev, uint32_t min_vm_size,
> >   	}
> >
> >   	adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
> > +	adev->vm_manager.reserved_vm_size =
> > +amdgpu_get_reserved_csa_size(adev);
> 
> Maybe calculate that directly as max_pfn - reserved and name it something
> like max_user_pfn.

Good suggestion.
Thanks

Regards
Rex

> Regards,
> Christian.
> 
> >
> >   	tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
> >   	if (amdgpu_vm_block_size != -1)
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> > index 2a8898d..ec650ac 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
> > @@ -260,6 +260,7 @@ struct amdgpu_vm_manager {
> >   	unsigned
> 	seqno[AMDGPU_MAX_RINGS];
> >
> >   	uint64_t				max_pfn;
> > +	uint64_t				reserved_vm_size;
> >   	uint32_t				num_level;
> >   	uint32_t				block_size;
> >   	uint32_t				fragment_size;
> > diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
> > b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
> > index 2821d1d..3351b56 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
> > @@ -471,7 +471,7 @@ static void gmc_v6_0_set_prt(struct
> amdgpu_device *adev, bool enable)
> >   	if (enable) {
> >   		uint32_t low = AMDGPU_VA_RESERVED_SIZE >>
> AMDGPU_GPU_PAGE_SHIFT;
> >   		uint32_t high = adev->vm_manager.max_pfn -
> > -			(AMDGPU_VA_RESERVED_SIZE >>
> AMDGPU_GPU_PAGE_SHIFT);
> > +				(adev->vm_manager.reserved_vm_size >>
> AMDGPU_GPU_PAGE_SHIFT);
> >
> >   		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
> >   		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low); diff --
> git
> > a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
> > b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
> > index 761dcfb..92bd175 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
> > @@ -568,7 +568,7 @@ static void gmc_v7_0_set_prt(struct
> amdgpu_device *adev, bool enable)
> >   	if (enable) {
> >   		uint32_t low = AMDGPU_VA_RESERVED_SIZE >>
> AMDGPU_GPU_PAGE_SHIFT;
> >   		uint32_t high = adev->vm_manager.max_pfn -
> > -			(AMDGPU_VA_RESERVED_SIZE >>
> AMDGPU_GPU_PAGE_SHIFT);
> > +				(adev->vm_manager.reserved_vm_size >>
> AMDGPU_GPU_PAGE_SHIFT);
> >
> >   		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
> >   		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low); diff --
> git
> > a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
> > b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
> > index 531aaf3..d909ddf 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
> > @@ -772,7 +772,7 @@ static void gmc_v8_0_set_prt(struct
> amdgpu_device *adev, bool enable)
> >   	if (enable) {
> >   		uint32_t low = AMDGPU_VA_RESERVED_SIZE >>
> AMDGPU_GPU_PAGE_SHIFT;
> >   		uint32_t high = adev->vm_manager.max_pfn -
> > -			(AMDGPU_VA_RESERVED_SIZE >>
> AMDGPU_GPU_PAGE_SHIFT);
> > +				(adev->vm_manager.reserved_vm_size >>
> AMDGPU_GPU_PAGE_SHIFT);
> >
> >   		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
> >   		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 4/9] drm/amdgpu: Add a bitmask in amdgpu_ctx_mgr

[Rex Zhu]

used to manager the reserverd vm space.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 4 +++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 6 +++++-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 8edf54b..8802ff2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -529,10 +529,14 @@ int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
 	return 0;
 }
 
-void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
+int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
 {
 	mutex_init(&mgr->lock);
 	idr_init(&mgr->ctx_handles);
+	mgr->resv_vm_bitmap = kzalloc(DIV_ROUND_UP(AMDGPU_VM_MAX_NUM_CTX, BITS_PER_BYTE), GFP_KERNEL);
+	if (unlikely(!mgr->resv_vm_bitmap))
+		return -ENOMEM;
+	return 0;
 }
 
 void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr)
@@ -601,7 +605,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
 		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
 			DRM_ERROR("ctx %p is still alive\n", ctx);
 	}
-
+	kfree(mgr->resv_vm_bitmap);
 	idr_destroy(&mgr->ctx_handles);
 	mutex_destroy(&mgr->lock);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
index b3b012c..94ac951 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
@@ -38,6 +38,7 @@ struct amdgpu_ctx_entity {
 struct amdgpu_ctx {
 	struct kref			refcount;
 	struct amdgpu_device		*adev;
+
 	unsigned			reset_counter;
 	unsigned			reset_counter_query;
 	uint32_t			vram_lost_counter;
@@ -56,6 +57,7 @@ struct amdgpu_ctx_mgr {
 	struct mutex		lock;
 	/* protected by lock */
 	struct idr		ctx_handles;
+	unsigned long		*resv_vm_bitmap;
 };
 
 extern const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM];
@@ -80,7 +82,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
 			       struct drm_sched_entity *entity);
 
-void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
+int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 52e4e90..338a091 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -988,11 +988,15 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	mutex_init(&fpriv->bo_list_lock);
 	idr_init(&fpriv->bo_list_handles);
 
-	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
+	if (amdgpu_ctx_mgr_init(&fpriv->ctx_mgr))
+		goto error_ctx_mgr;
 
 	file_priv->driver_priv = fpriv;
 	goto out_suspend;
 
+error_ctx_mgr:
+	idr_destroy(&fpriv->bo_list_handles);
+	mutex_destroy(&fpriv->bo_list_lock);
 error_vm:
 	amdgpu_vm_fini(adev, &fpriv->vm);
 
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 4/9] drm/amdgpu: Add a bitmask in amdgpu_ctx_mgr

[Christian König]

Am 06.12.18 um 13:14 schrieb Rex Zhu:
> used to manager the reserverd vm space.

Why do we need that?

Christian.

>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 8 ++++++--
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 4 +++-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 6 +++++-
>   3 files changed, 14 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> index 8edf54b..8802ff2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> @@ -529,10 +529,14 @@ int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
>   	return 0;
>   }
>   
> -void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
> +int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
>   {
>   	mutex_init(&mgr->lock);
>   	idr_init(&mgr->ctx_handles);
> +	mgr->resv_vm_bitmap = kzalloc(DIV_ROUND_UP(AMDGPU_VM_MAX_NUM_CTX, BITS_PER_BYTE), GFP_KERNEL);
> +	if (unlikely(!mgr->resv_vm_bitmap))
> +		return -ENOMEM;
> +	return 0;
>   }
>   
>   void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr)
> @@ -601,7 +605,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
>   		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
>   			DRM_ERROR("ctx %p is still alive\n", ctx);
>   	}
> -
> +	kfree(mgr->resv_vm_bitmap);
>   	idr_destroy(&mgr->ctx_handles);
>   	mutex_destroy(&mgr->lock);
>   }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> index b3b012c..94ac951 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> @@ -38,6 +38,7 @@ struct amdgpu_ctx_entity {
>   struct amdgpu_ctx {
>   	struct kref			refcount;
>   	struct amdgpu_device		*adev;
> +
>   	unsigned			reset_counter;
>   	unsigned			reset_counter_query;
>   	uint32_t			vram_lost_counter;
> @@ -56,6 +57,7 @@ struct amdgpu_ctx_mgr {
>   	struct mutex		lock;
>   	/* protected by lock */
>   	struct idr		ctx_handles;
> +	unsigned long		*resv_vm_bitmap;
>   };
>   
>   extern const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM];
> @@ -80,7 +82,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
>   int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
>   			       struct drm_sched_entity *entity);
>   
> -void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
> +int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
>   void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
>   void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr);
>   void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 52e4e90..338a091 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -988,11 +988,15 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>   	mutex_init(&fpriv->bo_list_lock);
>   	idr_init(&fpriv->bo_list_handles);
>   
> -	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
> +	if (amdgpu_ctx_mgr_init(&fpriv->ctx_mgr))
> +		goto error_ctx_mgr;
>   
>   	file_priv->driver_priv = fpriv;
>   	goto out_suspend;
>   
> +error_ctx_mgr:
> +	idr_destroy(&fpriv->bo_list_handles);
> +	mutex_destroy(&fpriv->bo_list_lock);
>   error_vm:
>   	amdgpu_vm_fini(adev, &fpriv->vm);
>   

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

RE: [PATCH 4/9] drm/amdgpu: Add a bitmask in amdgpu_ctx_mgr

[Zhu, Rex]

Hi Christian,

We allocate and map csa per ctx,  need to record the used/free vm space.
So use bitmap to manager the reserved vm space.

Also add resv_space_id in ctx.
When amdgpu_ctx_fini, we can clear the bit in the bitmap.


Best Regards
Rex
> -----Original Message-----
> From: Christian König <ckoenig.leichtzumerken@gmail.com>
> Sent: Thursday, December 6, 2018 8:34 PM
> To: Zhu, Rex <Rex.Zhu@amd.com>; amd-gfx@lists.freedesktop.org
> Subject: Re: [PATCH 4/9] drm/amdgpu: Add a bitmask in amdgpu_ctx_mgr
> 
> Am 06.12.18 um 13:14 schrieb Rex Zhu:
> > used to manager the reserverd vm space.
> 
> Why do we need that?
> 
> Christian.
> 
> >
> > Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 8 ++++++--
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 4 +++-
> >   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 6 +++++-
> >   3 files changed, 14 insertions(+), 4 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> > index 8edf54b..8802ff2 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> > @@ -529,10 +529,14 @@ int amdgpu_ctx_wait_prev_fence(struct
> amdgpu_ctx *ctx,
> >   	return 0;
> >   }
> >
> > -void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
> > +int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
> >   {
> >   	mutex_init(&mgr->lock);
> >   	idr_init(&mgr->ctx_handles);
> > +	mgr->resv_vm_bitmap =
> kzalloc(DIV_ROUND_UP(AMDGPU_VM_MAX_NUM_CTX, BITS_PER_BYTE),
> GFP_KERNEL);
> > +	if (unlikely(!mgr->resv_vm_bitmap))
> > +		return -ENOMEM;
> > +	return 0;
> >   }
> >
> >   void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr) @@
> > -601,7 +605,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr
> *mgr)
> >   		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
> >   			DRM_ERROR("ctx %p is still alive\n", ctx);
> >   	}
> > -
> > +	kfree(mgr->resv_vm_bitmap);
> >   	idr_destroy(&mgr->ctx_handles);
> >   	mutex_destroy(&mgr->lock);
> >   }
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> > index b3b012c..94ac951 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> > @@ -38,6 +38,7 @@ struct amdgpu_ctx_entity {
> >   struct amdgpu_ctx {
> >   	struct kref			refcount;
> >   	struct amdgpu_device		*adev;
> > +
> >   	unsigned			reset_counter;
> >   	unsigned			reset_counter_query;
> >   	uint32_t			vram_lost_counter;
> > @@ -56,6 +57,7 @@ struct amdgpu_ctx_mgr {
> >   	struct mutex		lock;
> >   	/* protected by lock */
> >   	struct idr		ctx_handles;
> > +	unsigned long		*resv_vm_bitmap;
> >   };
> >
> >   extern const unsigned int
> amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM];
> > @@ -80,7 +82,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void
> *data,
> >   int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
> >   			       struct drm_sched_entity *entity);
> >
> > -void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
> > +int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
> >   void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
> >   void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr);
> >   void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr); diff --git
> > a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index 52e4e90..338a091 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -988,11 +988,15 @@ int amdgpu_driver_open_kms(struct drm_device
> *dev, struct drm_file *file_priv)
> >   	mutex_init(&fpriv->bo_list_lock);
> >   	idr_init(&fpriv->bo_list_handles);
> >
> > -	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
> > +	if (amdgpu_ctx_mgr_init(&fpriv->ctx_mgr))
> > +		goto error_ctx_mgr;
> >
> >   	file_priv->driver_priv = fpriv;
> >   	goto out_suspend;
> >
> > +error_ctx_mgr:
> > +	idr_destroy(&fpriv->bo_list_handles);
> > +	mutex_destroy(&fpriv->bo_list_lock);
> >   error_vm:
> >   	amdgpu_vm_fini(adev, &fpriv->vm);
> >

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 4/9] drm/amdgpu: Add a bitmask in amdgpu_ctx_mgr

[Koenig, Christian]

Hi Rex,

ok still doesn't make sense to me. VM updates are pipelined with command 
submissions.

So you just need to create the mapping for the specific ID when the CTX 
is created.

And when the CTX is destroyed you unmap the CSA again.

Or is that for each queue inside the context? Please clarify.

Thanks,
Christian.

Am 06.12.18 um 15:48 schrieb Zhu, Rex:
> Hi Christian,
>
> We allocate and map csa per ctx,  need to record the used/free vm space.
> So use bitmap to manager the reserved vm space.
>
> Also add resv_space_id in ctx.
> When amdgpu_ctx_fini, we can clear the bit in the bitmap.
>
>
> Best Regards
> Rex
>> -----Original Message-----
>> From: Christian König <ckoenig.leichtzumerken@gmail.com>
>> Sent: Thursday, December 6, 2018 8:34 PM
>> To: Zhu, Rex <Rex.Zhu@amd.com>; amd-gfx@lists.freedesktop.org
>> Subject: Re: [PATCH 4/9] drm/amdgpu: Add a bitmask in amdgpu_ctx_mgr
>>
>> Am 06.12.18 um 13:14 schrieb Rex Zhu:
>>> used to manager the reserverd vm space.
>> Why do we need that?
>>
>> Christian.
>>
>>> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 8 ++++++--
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 4 +++-
>>>    drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 6 +++++-
>>>    3 files changed, 14 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
>>> index 8edf54b..8802ff2 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
>>> @@ -529,10 +529,14 @@ int amdgpu_ctx_wait_prev_fence(struct
>> amdgpu_ctx *ctx,
>>>    	return 0;
>>>    }
>>>
>>> -void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
>>> +int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
>>>    {
>>>    	mutex_init(&mgr->lock);
>>>    	idr_init(&mgr->ctx_handles);
>>> +	mgr->resv_vm_bitmap =
>> kzalloc(DIV_ROUND_UP(AMDGPU_VM_MAX_NUM_CTX, BITS_PER_BYTE),
>> GFP_KERNEL);
>>> +	if (unlikely(!mgr->resv_vm_bitmap))
>>> +		return -ENOMEM;
>>> +	return 0;
>>>    }
>>>
>>>    void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr) @@
>>> -601,7 +605,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr
>> *mgr)
>>>    		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
>>>    			DRM_ERROR("ctx %p is still alive\n", ctx);
>>>    	}
>>> -
>>> +	kfree(mgr->resv_vm_bitmap);
>>>    	idr_destroy(&mgr->ctx_handles);
>>>    	mutex_destroy(&mgr->lock);
>>>    }
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
>>> index b3b012c..94ac951 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
>>> @@ -38,6 +38,7 @@ struct amdgpu_ctx_entity {
>>>    struct amdgpu_ctx {
>>>    	struct kref			refcount;
>>>    	struct amdgpu_device		*adev;
>>> +
>>>    	unsigned			reset_counter;
>>>    	unsigned			reset_counter_query;
>>>    	uint32_t			vram_lost_counter;
>>> @@ -56,6 +57,7 @@ struct amdgpu_ctx_mgr {
>>>    	struct mutex		lock;
>>>    	/* protected by lock */
>>>    	struct idr		ctx_handles;
>>> +	unsigned long		*resv_vm_bitmap;
>>>    };
>>>
>>>    extern const unsigned int
>> amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM];
>>> @@ -80,7 +82,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void
>> *data,
>>>    int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
>>>    			       struct drm_sched_entity *entity);
>>>
>>> -void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
>>> +int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
>>>    void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
>>>    void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr);
>>>    void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr); diff --git
>>> a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> index 52e4e90..338a091 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> @@ -988,11 +988,15 @@ int amdgpu_driver_open_kms(struct drm_device
>> *dev, struct drm_file *file_priv)
>>>    	mutex_init(&fpriv->bo_list_lock);
>>>    	idr_init(&fpriv->bo_list_handles);
>>>
>>> -	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
>>> +	if (amdgpu_ctx_mgr_init(&fpriv->ctx_mgr))
>>> +		goto error_ctx_mgr;
>>>
>>>    	file_priv->driver_priv = fpriv;
>>>    	goto out_suspend;
>>>
>>> +error_ctx_mgr:
>>> +	idr_destroy(&fpriv->bo_list_handles);
>>> +	mutex_destroy(&fpriv->bo_list_lock);
>>>    error_vm:
>>>    	amdgpu_vm_fini(adev, &fpriv->vm);
>>>

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 5/9] drm/amdgpu: Delay map sriov csa addr to ctx init

[Rex Zhu]

1. meet kfd request
2. align with baremetal, in baremetal, driver map csa
   per ctx.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 19 ++++++++++++++++++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c |  8 --------
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 8802ff2..3ab7262 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -68,11 +68,13 @@ static int amdgpu_ctx_priority_permit(struct drm_file *filp,
 }
 
 static int amdgpu_ctx_init(struct amdgpu_device *adev,
+			   struct amdgpu_fpriv *fpriv,
 			   enum drm_sched_priority priority,
 			   struct drm_file *filp,
 			   struct amdgpu_ctx *ctx)
 {
 	unsigned num_entities = amdgput_ctx_total_num_entities();
+	uint64_t csa_addr;
 	unsigned i, j;
 	int r;
 
@@ -86,6 +88,21 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 	memset(ctx, 0, sizeof(*ctx));
 	ctx->adev = adev;
 
+	if (amdgpu_sriov_vf(adev)) {
+		if (!fpriv->csa_va) {
+			csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
+			r = amdgpu_map_static_csa(adev, &fpriv->vm,
+						adev->virt.csa_obj,
+						&fpriv->csa_va,
+						csa_addr,
+						AMDGPU_CSA_SIZE);
+			if (r) {
+				amdgpu_free_static_csa(&adev->virt.csa_obj);
+				return -EINVAL;
+			}
+		}
+	}
+
 	ctx->fences = kcalloc(amdgpu_sched_jobs * num_entities,
 			      sizeof(struct dma_fence*), GFP_KERNEL);
 	if (!ctx->fences)
@@ -255,7 +272,7 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 	}
 
 	*id = (uint32_t)r;
-	r = amdgpu_ctx_init(adev, priority, filp, ctx);
+	r = amdgpu_ctx_init(adev, fpriv, priority, filp, ctx);
 	if (r) {
 		idr_remove(&mgr->ctx_handles, *id);
 		*id = 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 338a091..34d000a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -977,14 +977,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 		goto error_vm;
 	}
 
-	if (amdgpu_sriov_vf(adev)) {
-		uint64_t csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
-		r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
-						&fpriv->csa_va, csa_addr, AMDGPU_CSA_SIZE);
-		if (r)
-			goto error_vm;
-	}
-
 	mutex_init(&fpriv->bo_list_lock);
 	idr_init(&fpriv->bo_list_handles);
 
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 5/9] drm/amdgpu: Delay map sriov csa addr to ctx init

[Christian König]

Am 06.12.18 um 13:14 schrieb Rex Zhu:
> 1. meet kfd request
> 2. align with baremetal, in baremetal, driver map csa
>     per ctx.
>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 19 ++++++++++++++++++-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c |  8 --------
>   2 files changed, 18 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> index 8802ff2..3ab7262 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> @@ -68,11 +68,13 @@ static int amdgpu_ctx_priority_permit(struct drm_file *filp,
>   }
>   
>   static int amdgpu_ctx_init(struct amdgpu_device *adev,
> +			   struct amdgpu_fpriv *fpriv,
>   			   enum drm_sched_priority priority,
>   			   struct drm_file *filp,
>   			   struct amdgpu_ctx *ctx)
>   {
>   	unsigned num_entities = amdgput_ctx_total_num_entities();
> +	uint64_t csa_addr;
>   	unsigned i, j;
>   	int r;
>   
> @@ -86,6 +88,21 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
>   	memset(ctx, 0, sizeof(*ctx));
>   	ctx->adev = adev;
>   
> +	if (amdgpu_sriov_vf(adev)) {
> +		if (!fpriv->csa_va) {
> +			csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
> +			r = amdgpu_map_static_csa(adev, &fpriv->vm,
> +						adev->virt.csa_obj,
> +						&fpriv->csa_va,
> +						csa_addr,
> +						AMDGPU_CSA_SIZE);
> +			if (r) {
> +				amdgpu_free_static_csa(&adev->virt.csa_obj);
> +				return -EINVAL;
> +			}

Please double check that this is protected by some lock.

Christian.

> +		}
> +	}
> +
>   	ctx->fences = kcalloc(amdgpu_sched_jobs * num_entities,
>   			      sizeof(struct dma_fence*), GFP_KERNEL);
>   	if (!ctx->fences)
> @@ -255,7 +272,7 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
>   	}
>   
>   	*id = (uint32_t)r;
> -	r = amdgpu_ctx_init(adev, priority, filp, ctx);
> +	r = amdgpu_ctx_init(adev, fpriv, priority, filp, ctx);
>   	if (r) {
>   		idr_remove(&mgr->ctx_handles, *id);
>   		*id = 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 338a091..34d000a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -977,14 +977,6 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>   		goto error_vm;
>   	}
>   
> -	if (amdgpu_sriov_vf(adev)) {
> -		uint64_t csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
> -		r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
> -						&fpriv->csa_va, csa_addr, AMDGPU_CSA_SIZE);
> -		if (r)
> -			goto error_vm;
> -	}
> -
>   	mutex_init(&fpriv->bo_list_lock);
>   	idr_init(&fpriv->bo_list_handles);
>   

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 6/9] drm/amdgpu: Create csa per ctx

[Rex Zhu]

create and map  csa for gfx/sdma engine to save the
middle command buffer when gpu preemption triggered.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  |  9 +++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 55 ++++++++++++++++++++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h |  4 +++
 3 files changed, 56 insertions(+), 12 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 2f189c5c..6f7a2dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -824,8 +824,9 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 	struct amdgpu_device *adev = p->adev;
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_list_entry *e;
-	struct amdgpu_bo_va *bo_va;
+	struct amdgpu_bo_va *bo_va = NULL;
 	struct amdgpu_bo *bo;
+	struct amdgpu_ctx *ctx = p->ctx;
 	int r;
 
 	/* Only for UVD/VCE VM emulation */
@@ -906,11 +907,11 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 	if (r)
 		return r;
 
-	if (amdgpu_sriov_vf(adev)) {
+	bo_va = amdgpu_sriov_vf(adev) ? fpriv->csa_va : ctx->csa_va;
+
+	if (bo_va) {
 		struct dma_fence *f;
 
-		bo_va = fpriv->csa_va;
-		BUG_ON(!bo_va);
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 3ab7262..71831aa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -71,7 +71,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 			   struct amdgpu_fpriv *fpriv,
 			   enum drm_sched_priority priority,
 			   struct drm_file *filp,
-			   struct amdgpu_ctx *ctx)
+			   struct amdgpu_ctx *ctx, uint32_t id)
 {
 	unsigned num_entities = amdgput_ctx_total_num_entities();
 	uint64_t csa_addr;
@@ -87,20 +87,36 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
 
 	memset(ctx, 0, sizeof(*ctx));
 	ctx->adev = adev;
+	csa_addr = amdgpu_csa_vaddr(adev, id) & AMDGPU_GMC_HOLE_MASK;
+	ctx->resv_space_id = id;
 
 	if (amdgpu_sriov_vf(adev)) {
 		if (!fpriv->csa_va) {
-			csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
 			r = amdgpu_map_static_csa(adev, &fpriv->vm,
-						adev->virt.csa_obj,
-						&fpriv->csa_va,
-						csa_addr,
-						AMDGPU_CSA_SIZE);
+							adev->virt.csa_obj,
+							&fpriv->csa_va,
+							csa_addr,
+							AMDGPU_CSA_SIZE);
 			if (r) {
 				amdgpu_free_static_csa(&adev->virt.csa_obj);
 				return -EINVAL;
 			}
 		}
+	} else {
+		r = amdgpu_allocate_static_csa(adev, &ctx->csa_bo,
+						AMDGPU_GEM_DOMAIN_GTT,
+						AMDGPU_CSA_SIZE);
+		if (r) {
+			DRM_ERROR("allocate CSA failed %d\n", r);
+			return r;
+		}
+		r = amdgpu_map_static_csa(adev, &fpriv->vm, ctx->csa_bo,
+						&ctx->csa_va, csa_addr,
+						AMDGPU_CSA_SIZE);
+		if (r) {
+			amdgpu_free_static_csa(&ctx->csa_bo);
+			return -EINVAL;
+		}
 	}
 
 	ctx->fences = kcalloc(amdgpu_sched_jobs * num_entities,
@@ -221,6 +237,16 @@ static void amdgpu_ctx_fini(struct kref *ref)
 	kfree(ctx->fences);
 	kfree(ctx->entities[0]);
 
+	if (!amdgpu_sriov_vf(adev) && ctx->csa_bo) {
+		BUG_ON(amdgpu_bo_reserve(ctx->csa_bo, true));
+		amdgpu_vm_bo_rmv(adev, ctx->csa_va);
+		ctx->csa_va = NULL;
+		amdgpu_bo_unreserve(ctx->csa_bo);
+		amdgpu_free_static_csa(&ctx->csa_bo);
+		if (ctx->ctx_mgr)
+			__clear_bit(ctx->resv_space_id - 1, ctx->ctx_mgr->resv_vm_bitmap);
+	}
+
 	mutex_destroy(&ctx->lock);
 
 	kfree(ctx);
@@ -258,6 +284,7 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
 	struct amdgpu_ctx *ctx;
 	int r;
+	u32 resv_space_id = 1;
 
 	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
@@ -270,14 +297,26 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 		kfree(ctx);
 		return r;
 	}
-
 	*id = (uint32_t)r;
-	r = amdgpu_ctx_init(adev, fpriv, priority, filp, ctx);
+
+	if (!amdgpu_sriov_vf(adev)) {
+		resv_space_id = find_first_zero_bit(mgr->resv_vm_bitmap, AMDGPU_VM_MAX_NUM_CTX);
+		if (resv_space_id < AMDGPU_VM_MAX_NUM_CTX)
+			__set_bit(resv_space_id, mgr->resv_vm_bitmap);
+		else
+			return -ENOMEM;
+		resv_space_id += 1;
+	}
+
+	r = amdgpu_ctx_init(adev, fpriv, priority, filp, ctx, resv_space_id);
 	if (r) {
 		idr_remove(&mgr->ctx_handles, *id);
 		*id = 0;
 		kfree(ctx);
+		mutex_unlock(&mgr->lock);
+		return r;
 	}
+	ctx->ctx_mgr = mgr;
 	mutex_unlock(&mgr->lock);
 	return r;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
index 94ac951..e434a6c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
@@ -38,6 +38,9 @@ struct amdgpu_ctx_entity {
 struct amdgpu_ctx {
 	struct kref			refcount;
 	struct amdgpu_device		*adev;
+	struct amdgpu_ctx_mgr		*ctx_mgr;
+	struct amdgpu_bo		*csa_bo;
+	struct amdgpu_bo_va		*csa_va;
 
 	unsigned			reset_counter;
 	unsigned			reset_counter_query;
@@ -50,6 +53,7 @@ struct amdgpu_ctx {
 	enum drm_sched_priority		override_priority;
 	struct mutex			lock;
 	atomic_t			guilty;
+	u32				resv_space_id;
 };
 
 struct amdgpu_ctx_mgr {
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 6/9] drm/amdgpu: Create csa per ctx

[Christian König]

Am 06.12.18 um 13:14 schrieb Rex Zhu:
> create and map  csa for gfx/sdma engine to save the
> middle command buffer when gpu preemption triggered.
>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  |  9 +++---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 55 ++++++++++++++++++++++++++++-----
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h |  4 +++
>   3 files changed, 56 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 2f189c5c..6f7a2dd 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -824,8 +824,9 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
>   	struct amdgpu_device *adev = p->adev;
>   	struct amdgpu_vm *vm = &fpriv->vm;
>   	struct amdgpu_bo_list_entry *e;
> -	struct amdgpu_bo_va *bo_va;
> +	struct amdgpu_bo_va *bo_va = NULL;
>   	struct amdgpu_bo *bo;
> +	struct amdgpu_ctx *ctx = p->ctx;
>   	int r;
>   
>   	/* Only for UVD/VCE VM emulation */
> @@ -906,11 +907,11 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
>   	if (r)
>   		return r;
>   
> -	if (amdgpu_sriov_vf(adev)) {
> +	bo_va = amdgpu_sriov_vf(adev) ? fpriv->csa_va : ctx->csa_va;
> +
> +	if (bo_va) {
>   		struct dma_fence *f;
>   
> -		bo_va = fpriv->csa_va;
> -		BUG_ON(!bo_va);
>   		r = amdgpu_vm_bo_update(adev, bo_va, false);
>   		if (r)
>   			return r;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> index 3ab7262..71831aa 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> @@ -71,7 +71,7 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
>   			   struct amdgpu_fpriv *fpriv,
>   			   enum drm_sched_priority priority,
>   			   struct drm_file *filp,
> -			   struct amdgpu_ctx *ctx)
> +			   struct amdgpu_ctx *ctx, uint32_t id)
>   {
>   	unsigned num_entities = amdgput_ctx_total_num_entities();
>   	uint64_t csa_addr;
> @@ -87,20 +87,36 @@ static int amdgpu_ctx_init(struct amdgpu_device *adev,
>   
>   	memset(ctx, 0, sizeof(*ctx));
>   	ctx->adev = adev;
> +	csa_addr = amdgpu_csa_vaddr(adev, id) & AMDGPU_GMC_HOLE_MASK;
> +	ctx->resv_space_id = id;

Do we really need resv_space_id? I mean during command submission we 
should have that anyway.

Christian.

>   
>   	if (amdgpu_sriov_vf(adev)) {
>   		if (!fpriv->csa_va) {
> -			csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
>   			r = amdgpu_map_static_csa(adev, &fpriv->vm,
> -						adev->virt.csa_obj,
> -						&fpriv->csa_va,
> -						csa_addr,
> -						AMDGPU_CSA_SIZE);
> +							adev->virt.csa_obj,
> +							&fpriv->csa_va,
> +							csa_addr,
> +							AMDGPU_CSA_SIZE);
>   			if (r) {
>   				amdgpu_free_static_csa(&adev->virt.csa_obj);
>   				return -EINVAL;
>   			}
>   		}
> +	} else {
> +		r = amdgpu_allocate_static_csa(adev, &ctx->csa_bo,
> +						AMDGPU_GEM_DOMAIN_GTT,
> +						AMDGPU_CSA_SIZE);
> +		if (r) {
> +			DRM_ERROR("allocate CSA failed %d\n", r);
> +			return r;
> +		}
> +		r = amdgpu_map_static_csa(adev, &fpriv->vm, ctx->csa_bo,
> +						&ctx->csa_va, csa_addr,
> +						AMDGPU_CSA_SIZE);
> +		if (r) {
> +			amdgpu_free_static_csa(&ctx->csa_bo);
> +			return -EINVAL;
> +		}
>   	}
>   
>   	ctx->fences = kcalloc(amdgpu_sched_jobs * num_entities,
> @@ -221,6 +237,16 @@ static void amdgpu_ctx_fini(struct kref *ref)
>   	kfree(ctx->fences);
>   	kfree(ctx->entities[0]);
>   
> +	if (!amdgpu_sriov_vf(adev) && ctx->csa_bo) {
> +		BUG_ON(amdgpu_bo_reserve(ctx->csa_bo, true));
> +		amdgpu_vm_bo_rmv(adev, ctx->csa_va);
> +		ctx->csa_va = NULL;
> +		amdgpu_bo_unreserve(ctx->csa_bo);
> +		amdgpu_free_static_csa(&ctx->csa_bo);
> +		if (ctx->ctx_mgr)
> +			__clear_bit(ctx->resv_space_id - 1, ctx->ctx_mgr->resv_vm_bitmap);
> +	}
> +
>   	mutex_destroy(&ctx->lock);
>   
>   	kfree(ctx);
> @@ -258,6 +284,7 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
>   	struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr;
>   	struct amdgpu_ctx *ctx;
>   	int r;
> +	u32 resv_space_id = 1;
>   
>   	ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
>   	if (!ctx)
> @@ -270,14 +297,26 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
>   		kfree(ctx);
>   		return r;
>   	}
> -
>   	*id = (uint32_t)r;
> -	r = amdgpu_ctx_init(adev, fpriv, priority, filp, ctx);
> +
> +	if (!amdgpu_sriov_vf(adev)) {
> +		resv_space_id = find_first_zero_bit(mgr->resv_vm_bitmap, AMDGPU_VM_MAX_NUM_CTX);
> +		if (resv_space_id < AMDGPU_VM_MAX_NUM_CTX)
> +			__set_bit(resv_space_id, mgr->resv_vm_bitmap);
> +		else
> +			return -ENOMEM;
> +		resv_space_id += 1;
> +	}
> +
> +	r = amdgpu_ctx_init(adev, fpriv, priority, filp, ctx, resv_space_id);
>   	if (r) {
>   		idr_remove(&mgr->ctx_handles, *id);
>   		*id = 0;
>   		kfree(ctx);
> +		mutex_unlock(&mgr->lock);
> +		return r;
>   	}
> +	ctx->ctx_mgr = mgr;
>   	mutex_unlock(&mgr->lock);
>   	return r;
>   }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> index 94ac951..e434a6c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> @@ -38,6 +38,9 @@ struct amdgpu_ctx_entity {
>   struct amdgpu_ctx {
>   	struct kref			refcount;
>   	struct amdgpu_device		*adev;
> +	struct amdgpu_ctx_mgr		*ctx_mgr;
> +	struct amdgpu_bo		*csa_bo;
> +	struct amdgpu_bo_va		*csa_va;
>   
>   	unsigned			reset_counter;
>   	unsigned			reset_counter_query;
> @@ -50,6 +53,7 @@ struct amdgpu_ctx {
>   	enum drm_sched_priority		override_priority;
>   	struct mutex			lock;
>   	atomic_t			guilty;
> +	u32				resv_space_id;
>   };
>   
>   struct amdgpu_ctx_mgr {

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 7/9] drm/amdgpu: Add csa mc address into job structure

[Rex Zhu]

save csa mc address in the job, so can patch the
address to pm4 when emit_ib even the ctx was freed.

suggested by Christian.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  | 2 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.h | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 6f7a2dd..13f0d7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -230,6 +230,8 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs
 		goto free_all_kdata;
 	}
 
+	p->job->csa_mc_addr = amdgpu_csa_vaddr(p->adev, p->ctx->resv_space_id) & AMDGPU_GMC_HOLE_MASK;
+
 	if (p->uf_entry.tv.bo)
 		p->job->uf_addr = uf_offset;
 	kfree(chunk_array);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index e1b46a6..42c959d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -34,6 +34,7 @@
 		container_of((sched_job), struct amdgpu_job, base)
 
 #define AMDGPU_JOB_GET_VMID(job) ((job) ? (job)->vmid : 0)
+#define AMDGPU_JOB_GET_CSA_MC_ADDR(job) ((job) ? (job)->csa_mc_addr : 0)
 
 struct amdgpu_fence;
 
@@ -56,10 +57,11 @@ struct amdgpu_job {
 	uint32_t		oa_base, oa_size;
 	uint32_t		vram_lost_counter;
 
+	/* csa buffer mc address */
+	uint64_t		csa_mc_addr;
 	/* user fence handling */
 	uint64_t		uf_addr;
 	uint64_t		uf_sequence;
-
 };
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 7/9] drm/amdgpu: Add csa mc address into job structure

[Christian König]

Am 06.12.18 um 13:14 schrieb Rex Zhu:
> save csa mc address in the job, so can patch the
> address to pm4 when emit_ib even the ctx was freed.
>
> suggested by Christian.
>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>

Reviewed-by: Christian König <christian.koenig@amd.com>

> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c  | 2 ++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.h | 4 +++-
>   2 files changed, 5 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 6f7a2dd..13f0d7c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -230,6 +230,8 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs
>   		goto free_all_kdata;
>   	}
>   
> +	p->job->csa_mc_addr = amdgpu_csa_vaddr(p->adev, p->ctx->resv_space_id) & AMDGPU_GMC_HOLE_MASK;
> +
>   	if (p->uf_entry.tv.bo)
>   		p->job->uf_addr = uf_offset;
>   	kfree(chunk_array);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> index e1b46a6..42c959d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
> @@ -34,6 +34,7 @@
>   		container_of((sched_job), struct amdgpu_job, base)
>   
>   #define AMDGPU_JOB_GET_VMID(job) ((job) ? (job)->vmid : 0)
> +#define AMDGPU_JOB_GET_CSA_MC_ADDR(job) ((job) ? (job)->csa_mc_addr : 0)
>   
>   struct amdgpu_fence;
>   
> @@ -56,10 +57,11 @@ struct amdgpu_job {
>   	uint32_t		oa_base, oa_size;
>   	uint32_t		vram_lost_counter;
>   
> +	/* csa buffer mc address */
> +	uint64_t		csa_mc_addr;
>   	/* user fence handling */
>   	uint64_t		uf_addr;
>   	uint64_t		uf_sequence;
> -
>   };
>   
>   int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 8/9] drm/amdgpu: Add a argument in emit_cntxcntl interface

[Rex Zhu]

add a point of struct amdgpu_job in emit_cntxcntl
interface in order to get the csa mc address per ctx
when emit ce metadata in baremetal.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c   |  2 +-
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  4 ++--
 drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c    |  4 +++-
 drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c    |  4 +++-
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c    | 20 ++++++++++++--------
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c    | 16 ++++++++++------
 6 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index c48207b3..5329044 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -208,7 +208,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
 			status |= AMDGPU_HAVE_CTX_SWITCH;
 		status |= job->preamble_status;
 
-		amdgpu_ring_emit_cntxcntl(ring, status);
+		amdgpu_ring_emit_cntxcntl(ring, job, status);
 	}
 
 	for (i = 0; i < num_ibs; ++i) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 0beb01f..7aa46cc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -157,7 +157,7 @@ struct amdgpu_ring_funcs {
 	void (*begin_use)(struct amdgpu_ring *ring);
 	void (*end_use)(struct amdgpu_ring *ring);
 	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
-	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
+	void (*emit_cntxcntl) (struct amdgpu_ring *ring, struct amdgpu_job *job, uint32_t flags);
 	void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
 	void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
 	void (*emit_reg_wait)(struct amdgpu_ring *ring, uint32_t reg,
@@ -236,7 +236,7 @@ struct amdgpu_ring {
 #define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
 #define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
 #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
-#define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
+#define amdgpu_ring_emit_cntxcntl(r, job, d) (r)->funcs->emit_cntxcntl((r), (job), (d))
 #define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
 #define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
 #define amdgpu_ring_emit_reg_wait(r, d, v, m) (r)->funcs->emit_reg_wait((r), (d), (v), (m))
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index 5b25c26..976f94a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -2976,7 +2976,9 @@ static uint64_t gfx_v6_0_get_gpu_clock_counter(struct amdgpu_device *adev)
 	return clock;
 }
 
-static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
+static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring,
+					struct amdgpu_job *job,
+					uint32_t flags)
 {
 	if (flags & AMDGPU_HAVE_CTX_SWITCH)
 		gfx_v6_0_ring_emit_vgt_flush(ring);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 243b8c5..ab62117 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -2275,7 +2275,9 @@ static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
 	amdgpu_ring_write(ring, control);
 }
 
-static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
+static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring,
+					struct amdgpu_job *job,
+					uint32_t flags)
 {
 	uint32_t dw2 = 0;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index d529cef..3ac2d8f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -723,8 +723,8 @@ enum {
 static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
 static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
 static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
-static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);
-static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);
+static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring, struct amdgpu_job *job);
+static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring, struct amdgpu_job *job);
 
 static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
 {
@@ -6127,7 +6127,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
 		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
-			gfx_v8_0_ring_emit_de_meta(ring);
+			gfx_v8_0_ring_emit_de_meta(ring, job);
 	}
 
 	amdgpu_ring_write(ring, header);
@@ -6386,12 +6386,14 @@ static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, 0);
 }
 
-static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
+static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring,
+					struct amdgpu_job *job,
+					uint32_t flags)
 {
 	uint32_t dw2 = 0;
 
 	if (amdgpu_sriov_vf(ring->adev))
-		gfx_v8_0_ring_emit_ce_meta(ring);
+		gfx_v8_0_ring_emit_ce_meta(ring, job);
 
 	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
 	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
@@ -7182,7 +7184,8 @@ static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
 	.funcs = &gfx_v8_0_ip_funcs,
 };
 
-static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
+static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring,
+					struct amdgpu_job *job)
 {
 	uint64_t ce_payload_addr;
 	int cnt_ce;
@@ -7196,7 +7199,7 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
 			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
 		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
 	} else {
-		ce_payload_addr = amdgpu_csa_vaddr(ring->adev, 1) +
+		ce_payload_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job) +
 			offsetof(struct vi_gfx_meta_data, ce_payload);
 		cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
 	}
@@ -7211,7 +7214,8 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
 	amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
 }
 
-static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
+static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring,
+					struct amdgpu_job *job)
 {
 	uint64_t de_payload_addr, gds_addr, csa_addr;
 	int cnt_de;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 81c1578..dd2d535 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -297,7 +297,7 @@ static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
                                  struct amdgpu_cu_info *cu_info);
 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
-static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
+static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, struct amdgpu_job *job);
 
 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
 {
@@ -4067,7 +4067,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
 		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
-			gfx_v9_0_ring_emit_de_meta(ring);
+			gfx_v9_0_ring_emit_de_meta(ring, job);
 	}
 
 	amdgpu_ring_write(ring, header);
@@ -4320,7 +4320,8 @@ static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
 	amdgpu_ring_write(ring, 0);
 }
 
-static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
+static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring,
+					struct amdgpu_job *job)
 {
 	struct v9_ce_ib_state ce_payload = {0};
 	uint64_t csa_addr;
@@ -4339,7 +4340,8 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
 	amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
 }
 
-static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
+static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring,
+					struct amdgpu_job *job)
 {
 	struct v9_de_ib_state de_payload = {0};
 	uint64_t csa_addr, gds_addr;
@@ -4367,12 +4369,14 @@ static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
 	amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
 }
 
-static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
+static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring,
+					struct amdgpu_job *job,
+					uint32_t flags)
 {
 	uint32_t dw2 = 0;
 
 	if (amdgpu_sriov_vf(ring->adev))
-		gfx_v9_0_ring_emit_ce_meta(ring);
+		gfx_v9_0_ring_emit_ce_meta(ring, job);
 
 	gfx_v9_0_ring_emit_tmz(ring, true);
 
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 8/9] drm/amdgpu: Add a argument in emit_cntxcntl interface

[Christian König]

Am 06.12.18 um 13:14 schrieb Rex Zhu:
> add a point of struct amdgpu_job in emit_cntxcntl
> interface in order to get the csa mc address per ctx
> when emit ce metadata in baremetal.
>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>

Reviewed-by: Christian König <christian.koenig@amd.com>

> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c   |  2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  4 ++--
>   drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c    |  4 +++-
>   drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c    |  4 +++-
>   drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c    | 20 ++++++++++++--------
>   drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c    | 16 ++++++++++------
>   6 files changed, 31 insertions(+), 19 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> index c48207b3..5329044 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
> @@ -208,7 +208,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
>   			status |= AMDGPU_HAVE_CTX_SWITCH;
>   		status |= job->preamble_status;
>   
> -		amdgpu_ring_emit_cntxcntl(ring, status);
> +		amdgpu_ring_emit_cntxcntl(ring, job, status);
>   	}
>   
>   	for (i = 0; i < num_ibs; ++i) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index 0beb01f..7aa46cc 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -157,7 +157,7 @@ struct amdgpu_ring_funcs {
>   	void (*begin_use)(struct amdgpu_ring *ring);
>   	void (*end_use)(struct amdgpu_ring *ring);
>   	void (*emit_switch_buffer) (struct amdgpu_ring *ring);
> -	void (*emit_cntxcntl) (struct amdgpu_ring *ring, uint32_t flags);
> +	void (*emit_cntxcntl) (struct amdgpu_ring *ring, struct amdgpu_job *job, uint32_t flags);
>   	void (*emit_rreg)(struct amdgpu_ring *ring, uint32_t reg);
>   	void (*emit_wreg)(struct amdgpu_ring *ring, uint32_t reg, uint32_t val);
>   	void (*emit_reg_wait)(struct amdgpu_ring *ring, uint32_t reg,
> @@ -236,7 +236,7 @@ struct amdgpu_ring {
>   #define amdgpu_ring_emit_gds_switch(r, v, db, ds, wb, ws, ab, as) (r)->funcs->emit_gds_switch((r), (v), (db), (ds), (wb), (ws), (ab), (as))
>   #define amdgpu_ring_emit_hdp_flush(r) (r)->funcs->emit_hdp_flush((r))
>   #define amdgpu_ring_emit_switch_buffer(r) (r)->funcs->emit_switch_buffer((r))
> -#define amdgpu_ring_emit_cntxcntl(r, d) (r)->funcs->emit_cntxcntl((r), (d))
> +#define amdgpu_ring_emit_cntxcntl(r, job, d) (r)->funcs->emit_cntxcntl((r), (job), (d))
>   #define amdgpu_ring_emit_rreg(r, d) (r)->funcs->emit_rreg((r), (d))
>   #define amdgpu_ring_emit_wreg(r, d, v) (r)->funcs->emit_wreg((r), (d), (v))
>   #define amdgpu_ring_emit_reg_wait(r, d, v, m) (r)->funcs->emit_reg_wait((r), (d), (v), (m))
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
> index 5b25c26..976f94a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
> @@ -2976,7 +2976,9 @@ static uint64_t gfx_v6_0_get_gpu_clock_counter(struct amdgpu_device *adev)
>   	return clock;
>   }
>   
> -static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
> +static void gfx_v6_ring_emit_cntxcntl(struct amdgpu_ring *ring,
> +					struct amdgpu_job *job,
> +					uint32_t flags)
>   {
>   	if (flags & AMDGPU_HAVE_CTX_SWITCH)
>   		gfx_v6_0_ring_emit_vgt_flush(ring);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
> index 243b8c5..ab62117 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
> @@ -2275,7 +2275,9 @@ static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
>   	amdgpu_ring_write(ring, control);
>   }
>   
> -static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
> +static void gfx_v7_ring_emit_cntxcntl(struct amdgpu_ring *ring,
> +					struct amdgpu_job *job,
> +					uint32_t flags)
>   {
>   	uint32_t dw2 = 0;
>   
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> index d529cef..3ac2d8f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> @@ -723,8 +723,8 @@ enum {
>   static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev);
>   static u32 gfx_v8_0_get_csb_size(struct amdgpu_device *adev);
>   static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev);
> -static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring);
> -static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring);
> +static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring, struct amdgpu_job *job);
> +static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring, struct amdgpu_job *job);
>   
>   static void gfx_v8_0_init_golden_registers(struct amdgpu_device *adev)
>   {
> @@ -6127,7 +6127,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
>   		control |= INDIRECT_BUFFER_PRE_ENB(1);
>   
>   		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
> -			gfx_v8_0_ring_emit_de_meta(ring);
> +			gfx_v8_0_ring_emit_de_meta(ring, job);
>   	}
>   
>   	amdgpu_ring_write(ring, header);
> @@ -6386,12 +6386,14 @@ static void gfx_v8_ring_emit_sb(struct amdgpu_ring *ring)
>   	amdgpu_ring_write(ring, 0);
>   }
>   
> -static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
> +static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring,
> +					struct amdgpu_job *job,
> +					uint32_t flags)
>   {
>   	uint32_t dw2 = 0;
>   
>   	if (amdgpu_sriov_vf(ring->adev))
> -		gfx_v8_0_ring_emit_ce_meta(ring);
> +		gfx_v8_0_ring_emit_ce_meta(ring, job);
>   
>   	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
>   	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
> @@ -7182,7 +7184,8 @@ static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
>   	.funcs = &gfx_v8_0_ip_funcs,
>   };
>   
> -static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
> +static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring,
> +					struct amdgpu_job *job)
>   {
>   	uint64_t ce_payload_addr;
>   	int cnt_ce;
> @@ -7196,7 +7199,7 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
>   			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
>   		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
>   	} else {
> -		ce_payload_addr = amdgpu_csa_vaddr(ring->adev, 1) +
> +		ce_payload_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job) +
>   			offsetof(struct vi_gfx_meta_data, ce_payload);
>   		cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
>   	}
> @@ -7211,7 +7214,8 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
>   	amdgpu_ring_write_multiple(ring, (void *)&ce_payload, cnt_ce - 2);
>   }
>   
> -static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
> +static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring,
> +					struct amdgpu_job *job)
>   {
>   	uint64_t de_payload_addr, gds_addr, csa_addr;
>   	int cnt_de;
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> index 81c1578..dd2d535 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> @@ -297,7 +297,7 @@ static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
>                                    struct amdgpu_cu_info *cu_info);
>   static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
>   static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
> -static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
> +static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring, struct amdgpu_job *job);
>   
>   static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
>   {
> @@ -4067,7 +4067,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
>   		control |= INDIRECT_BUFFER_PRE_ENB(1);
>   
>   		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
> -			gfx_v9_0_ring_emit_de_meta(ring);
> +			gfx_v9_0_ring_emit_de_meta(ring, job);
>   	}
>   
>   	amdgpu_ring_write(ring, header);
> @@ -4320,7 +4320,8 @@ static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
>   	amdgpu_ring_write(ring, 0);
>   }
>   
> -static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
> +static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring,
> +					struct amdgpu_job *job)
>   {
>   	struct v9_ce_ib_state ce_payload = {0};
>   	uint64_t csa_addr;
> @@ -4339,7 +4340,8 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
>   	amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
>   }
>   
> -static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
> +static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring,
> +					struct amdgpu_job *job)
>   {
>   	struct v9_de_ib_state de_payload = {0};
>   	uint64_t csa_addr, gds_addr;
> @@ -4367,12 +4369,14 @@ static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
>   	amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
>   }
>   
> -static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
> +static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring,
> +					struct amdgpu_job *job,
> +					uint32_t flags)
>   {
>   	uint32_t dw2 = 0;
>   
>   	if (amdgpu_sriov_vf(ring->adev))
> -		gfx_v9_0_ring_emit_ce_meta(ring);
> +		gfx_v9_0_ring_emit_ce_meta(ring, job);
>   
>   	gfx_v9_0_ring_emit_tmz(ring, true);
>   

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 9/9] drm/amdgpu: Remove sriov check when insert ce/de meta_data

[Rex Zhu]

to support cp gfx mid-command buffer preemption in baremetal

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 15 ++++++++++-----
 drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 ++++++++++-----
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index 3ac2d8f..ccc461f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -6123,7 +6123,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 
 	control |= ib->length_dw | (vmid << 24);
 
-	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
+	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
 		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
 		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
@@ -6392,8 +6392,7 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring,
 {
 	uint32_t dw2 = 0;
 
-	if (amdgpu_sriov_vf(ring->adev))
-		gfx_v8_0_ring_emit_ce_meta(ring, job);
+	gfx_v8_0_ring_emit_ce_meta(ring, job);
 
 	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
 	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
@@ -7194,8 +7193,11 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring,
 		struct vi_ce_ib_state_chained_ib chained;
 	} ce_payload = {};
 
+	if (!job)
+		return;
+
 	if (ring->adev->virt.chained_ib_support) {
-		ce_payload_addr = amdgpu_csa_vaddr(ring->adev, 1) +
+		ce_payload_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job) +
 			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
 		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
 	} else {
@@ -7224,7 +7226,10 @@ static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring,
 		struct vi_de_ib_state_chained_ib chained;
 	} de_payload = {};
 
-	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
+	if (!job)
+		return;
+
+	csa_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job);
 	gds_addr = csa_addr + 4096;
 	if (ring->adev->virt.chained_ib_support) {
 		de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index dd2d535..6d38834 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -4063,7 +4063,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
 
 	control |= ib->length_dw | (vmid << 24);
 
-	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
+	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
 		control |= INDIRECT_BUFFER_PRE_ENB(1);
 
 		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
@@ -4327,8 +4327,11 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring,
 	uint64_t csa_addr;
 	int cnt;
 
+	if (!job)
+		return;
+
 	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
-	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
+	csa_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job);
 
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
 	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
@@ -4347,7 +4350,10 @@ static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring,
 	uint64_t csa_addr, gds_addr;
 	int cnt;
 
-	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
+	if (!job)
+		return;
+
+	csa_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job);
 	gds_addr = csa_addr + 4096;
 	de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
 	de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
@@ -4375,8 +4381,7 @@ static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring,
 {
 	uint32_t dw2 = 0;
 
-	if (amdgpu_sriov_vf(ring->adev))
-		gfx_v9_0_ring_emit_ce_meta(ring, job);
+	gfx_v9_0_ring_emit_ce_meta(ring, job);
 
 	gfx_v9_0_ring_emit_tmz(ring, true);
 
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 9/9] drm/amdgpu: Remove sriov check when insert ce/de meta_data

[Christian König]

Am 06.12.18 um 13:14 schrieb Rex Zhu:
> to support cp gfx mid-command buffer preemption in baremetal
>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>

Acked-by: Christian König <christian.koenig@amd.com>

> ---
>   drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c | 15 ++++++++++-----
>   drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c | 15 ++++++++++-----
>   2 files changed, 20 insertions(+), 10 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> index 3ac2d8f..ccc461f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> @@ -6123,7 +6123,7 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
>   
>   	control |= ib->length_dw | (vmid << 24);
>   
> -	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
> +	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
>   		control |= INDIRECT_BUFFER_PRE_ENB(1);
>   
>   		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
> @@ -6392,8 +6392,7 @@ static void gfx_v8_ring_emit_cntxcntl(struct amdgpu_ring *ring,
>   {
>   	uint32_t dw2 = 0;
>   
> -	if (amdgpu_sriov_vf(ring->adev))
> -		gfx_v8_0_ring_emit_ce_meta(ring, job);
> +	gfx_v8_0_ring_emit_ce_meta(ring, job);
>   
>   	dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
>   	if (flags & AMDGPU_HAVE_CTX_SWITCH) {
> @@ -7194,8 +7193,11 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring,
>   		struct vi_ce_ib_state_chained_ib chained;
>   	} ce_payload = {};
>   
> +	if (!job)
> +		return;
> +
>   	if (ring->adev->virt.chained_ib_support) {
> -		ce_payload_addr = amdgpu_csa_vaddr(ring->adev, 1) +
> +		ce_payload_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job) +
>   			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
>   		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
>   	} else {
> @@ -7224,7 +7226,10 @@ static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring,
>   		struct vi_de_ib_state_chained_ib chained;
>   	} de_payload = {};
>   
> -	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
> +	if (!job)
> +		return;
> +
> +	csa_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job);
>   	gds_addr = csa_addr + 4096;
>   	if (ring->adev->virt.chained_ib_support) {
>   		de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> index dd2d535..6d38834 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> @@ -4063,7 +4063,7 @@ static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
>   
>   	control |= ib->length_dw | (vmid << 24);
>   
> -	if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
> +	if (ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
>   		control |= INDIRECT_BUFFER_PRE_ENB(1);
>   
>   		if (!(ib->flags & AMDGPU_IB_FLAG_CE))
> @@ -4327,8 +4327,11 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring,
>   	uint64_t csa_addr;
>   	int cnt;
>   
> +	if (!job)
> +		return;
> +
>   	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
> -	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
> +	csa_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job);
>   
>   	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
>   	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
> @@ -4347,7 +4350,10 @@ static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring,
>   	uint64_t csa_addr, gds_addr;
>   	int cnt;
>   
> -	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
> +	if (!job)
> +		return;
> +
> +	csa_addr = AMDGPU_JOB_GET_CSA_MC_ADDR(job);
>   	gds_addr = csa_addr + 4096;
>   	de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
>   	de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
> @@ -4375,8 +4381,7 @@ static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring,
>   {
>   	uint32_t dw2 = 0;
>   
> -	if (amdgpu_sriov_vf(ring->adev))
> -		gfx_v9_0_ring_emit_ce_meta(ring, job);
> +	gfx_v9_0_ring_emit_ce_meta(ring, job);
>   
>   	gfx_v9_0_ring_emit_tmz(ring, true);
>   

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 2/9] drm/amdgpu: Refine function amdgpu_csa_vaddr

[Christian König]

Am 06.12.18 um 13:14 schrieb Rex Zhu:
> on baremetal, driver create csa per ctx.
> So add a function argument: ctx_id to
> get csa gpu addr.
> In Sriov, driver create csa per process,
> so ctx id always 1.
>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c | 5 +++--
>   drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h | 2 +-
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 4 ++--
>   drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c   | 6 +++---
>   drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c   | 4 ++--
>   5 files changed, 11 insertions(+), 10 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> index 0c590dd..44b046f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c
> @@ -24,11 +24,12 @@
>   
>   #include "amdgpu.h"
>   
> -uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev)
> +uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id)
>   {
>   	uint64_t addr = adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT;
>   
> -	addr -= AMDGPU_VA_RESERVED_SIZE;
> +	addr -= AMDGPU_VA_RESERVED_SIZE * id;
> +

It might be better to add the SRIOV handling here instead. E.g. like the 
following:

if (amdgpu_sriov_vf(adev))
     addr -= AMDGPU_VA_RESERVED_SIZE;
else
     addr -= AMDGPU_CSA_SIZE * id;

Apart from that looks good to me, but might be better to re-order the 
patches.

Regards,
Christian.

>   	addr = amdgpu_gmc_sign_extend(addr);
>   
>   	return addr;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> index 524b443..aaf1fba 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_csa.h
> @@ -28,7 +28,7 @@
>   #define AMDGPU_CSA_SIZE		(128 * 1024)
>   
>   uint32_t amdgpu_get_total_csa_size(struct amdgpu_device *adev);
> -uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev);
> +uint64_t amdgpu_csa_vaddr(struct amdgpu_device *adev, uint32_t id);
>   int amdgpu_allocate_static_csa(struct amdgpu_device *adev, struct amdgpu_bo **bo,
>   				u32 domain, uint32_t size);
>   int amdgpu_map_static_csa(struct amdgpu_device *adev, struct amdgpu_vm *vm,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 08d04f6..f736bda 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -977,9 +977,9 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>   		goto error_vm;
>   	}
>   
> -	if (amdgpu_sriov_vf(adev)) {
> -		uint64_t csa_addr = amdgpu_csa_vaddr(adev) & AMDGPU_GMC_HOLE_MASK;
>   
> +	if (amdgpu_sriov_vf(adev)) {
> +		uint64_t csa_addr = amdgpu_csa_vaddr(adev, 1) & AMDGPU_GMC_HOLE_MASK;
>   		r = amdgpu_map_static_csa(adev, &fpriv->vm, adev->virt.csa_obj,
>   						&fpriv->csa_va, csa_addr, AMDGPU_CSA_SIZE);
>   		if (r)
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> index bdae563..d529cef 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
> @@ -7192,11 +7192,11 @@ static void gfx_v8_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
>   	} ce_payload = {};
>   
>   	if (ring->adev->virt.chained_ib_support) {
> -		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
> +		ce_payload_addr = amdgpu_csa_vaddr(ring->adev, 1) +
>   			offsetof(struct vi_gfx_meta_data_chained_ib, ce_payload);
>   		cnt_ce = (sizeof(ce_payload.chained) >> 2) + 4 - 2;
>   	} else {
> -		ce_payload_addr = amdgpu_csa_vaddr(ring->adev) +
> +		ce_payload_addr = amdgpu_csa_vaddr(ring->adev, 1) +
>   			offsetof(struct vi_gfx_meta_data, ce_payload);
>   		cnt_ce = (sizeof(ce_payload.regular) >> 2) + 4 - 2;
>   	}
> @@ -7220,7 +7220,7 @@ static void gfx_v8_0_ring_emit_de_meta(struct amdgpu_ring *ring)
>   		struct vi_de_ib_state_chained_ib chained;
>   	} de_payload = {};
>   
> -	csa_addr = amdgpu_csa_vaddr(ring->adev);
> +	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
>   	gds_addr = csa_addr + 4096;
>   	if (ring->adev->virt.chained_ib_support) {
>   		de_payload.chained.gds_backup_addrlo = lower_32_bits(gds_addr);
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> index 928034c..81c1578 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
> @@ -4327,7 +4327,7 @@ static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
>   	int cnt;
>   
>   	cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
> -	csa_addr = amdgpu_csa_vaddr(ring->adev);
> +	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
>   
>   	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
>   	amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
> @@ -4345,7 +4345,7 @@ static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
>   	uint64_t csa_addr, gds_addr;
>   	int cnt;
>   
> -	csa_addr = amdgpu_csa_vaddr(ring->adev);
> +	csa_addr = amdgpu_csa_vaddr(ring->adev, 1);
>   	gds_addr = csa_addr + 4096;
>   	de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
>   	de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 1/9] drm/amdgpu: Limit vm max ctx number to 4096

[Rex Zhu]

driver need to reserve resource for each ctx for
some hw features. so add this limitation.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h     | 1 +
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 33f5f2c..3ef5c3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -161,6 +161,7 @@ struct amdgpu_mgpu_info
 extern int amdgpu_cik_support;
 #endif
 
+#define AMDGPU_VM_MAX_NUM_CTX			4096
 #define AMDGPU_SG_THRESHOLD			(256*1024*1024)
 #define AMDGPU_DEFAULT_GTT_SIZE_MB		3072ULL /* 3GB by default */
 #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS	        3000
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index f9b5423..8edf54b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -247,7 +247,7 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev,
 		return -ENOMEM;
 
 	mutex_lock(&mgr->lock);
-	r = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL);
+	r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL);
 	if (r < 0) {
 		mutex_unlock(&mgr->lock);
 		kfree(ctx);
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

[PATCH 4/9] drm/amdgpu: Add a bitmask in amdgpu_ctx_mgr

[Rex Zhu]

used to manager the reserverd vm space.

Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 8 ++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 4 +++-
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 6 +++++-
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 8edf54b..8802ff2 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -529,10 +529,14 @@ int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
 	return 0;
 }
 
-void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
+int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
 {
 	mutex_init(&mgr->lock);
 	idr_init(&mgr->ctx_handles);
+	mgr->resv_vm_bitmap = kzalloc(DIV_ROUND_UP(AMDGPU_VM_MAX_NUM_CTX, BITS_PER_BYTE), GFP_KERNEL);
+	if (unlikely(!mgr->resv_vm_bitmap))
+		return -ENOMEM;
+	return 0;
 }
 
 void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr)
@@ -601,7 +605,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
 		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
 			DRM_ERROR("ctx %p is still alive\n", ctx);
 	}
-
+	kfree(mgr->resv_vm_bitmap);
 	idr_destroy(&mgr->ctx_handles);
 	mutex_destroy(&mgr->lock);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
index b3b012c..94ac951 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
@@ -38,6 +38,7 @@ struct amdgpu_ctx_entity {
 struct amdgpu_ctx {
 	struct kref			refcount;
 	struct amdgpu_device		*adev;
+
 	unsigned			reset_counter;
 	unsigned			reset_counter_query;
 	uint32_t			vram_lost_counter;
@@ -56,6 +57,7 @@ struct amdgpu_ctx_mgr {
 	struct mutex		lock;
 	/* protected by lock */
 	struct idr		ctx_handles;
+	unsigned long		*resv_vm_bitmap;
 };
 
 extern const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM];
@@ -80,7 +82,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
 int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
 			       struct drm_sched_entity *entity);
 
-void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
+int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr);
 void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 52e4e90..338a091 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -988,11 +988,15 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	mutex_init(&fpriv->bo_list_lock);
 	idr_init(&fpriv->bo_list_handles);
 
-	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
+	if (amdgpu_ctx_mgr_init(&fpriv->ctx_mgr))
+		goto error_ctx_mgr;
 
 	file_priv->driver_priv = fpriv;
 	goto out_suspend;
 
+error_ctx_mgr:
+	idr_destroy(&fpriv->bo_list_handles);
+	mutex_destroy(&fpriv->bo_list_lock);
 error_vm:
 	amdgpu_vm_fini(adev, &fpriv->vm);
 
-- 
1.9.1

_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx

Re: [PATCH 4/9] drm/amdgpu: Add a bitmask in amdgpu_ctx_mgr

[Kuehling, Felix]

On 2018-12-06 6:32 a.m., Rex Zhu wrote:
> used to manager the reserverd vm space.
>
> Signed-off-by: Rex Zhu <Rex.Zhu@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 8 ++++++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h | 4 +++-
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 6 +++++-
>  3 files changed, 14 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> index 8edf54b..8802ff2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> @@ -529,10 +529,14 @@ int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
>  	return 0;
>  }
>  
> -void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
> +int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr)
>  {
>  	mutex_init(&mgr->lock);
>  	idr_init(&mgr->ctx_handles);
> +	mgr->resv_vm_bitmap = kzalloc(DIV_ROUND_UP(AMDGPU_VM_MAX_NUM_CTX, BITS_PER_BYTE), GFP_KERNEL);

The bitmap is in (unsigned long) units. So you should round to multiples
of (unsigned long), not multiples of byte. Something like this:

mgr->resv_vm_bitmap = kzalloc(DIV_ROUND_UP(AMDGPU_VM_MAX_NUM_CTX,
BITS_PER_BYTE * sizeof(unsigned long)) * sizeof(unsigned long), GFP_KERNEL);

Regards,
  Felix


> +	if (unlikely(!mgr->resv_vm_bitmap))
> +		return -ENOMEM;
> +	return 0;
>  }
>  
>  void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr)
> @@ -601,7 +605,7 @@ void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr)
>  		if (kref_put(&ctx->refcount, amdgpu_ctx_fini) != 1)
>  			DRM_ERROR("ctx %p is still alive\n", ctx);
>  	}
> -
> +	kfree(mgr->resv_vm_bitmap);
>  	idr_destroy(&mgr->ctx_handles);
>  	mutex_destroy(&mgr->lock);
>  }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> index b3b012c..94ac951 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
> @@ -38,6 +38,7 @@ struct amdgpu_ctx_entity {
>  struct amdgpu_ctx {
>  	struct kref			refcount;
>  	struct amdgpu_device		*adev;
> +
>  	unsigned			reset_counter;
>  	unsigned			reset_counter_query;
>  	uint32_t			vram_lost_counter;
> @@ -56,6 +57,7 @@ struct amdgpu_ctx_mgr {
>  	struct mutex		lock;
>  	/* protected by lock */
>  	struct idr		ctx_handles;
> +	unsigned long		*resv_vm_bitmap;
>  };
>  
>  extern const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM];
> @@ -80,7 +82,7 @@ int amdgpu_ctx_ioctl(struct drm_device *dev, void *data,
>  int amdgpu_ctx_wait_prev_fence(struct amdgpu_ctx *ctx,
>  			       struct drm_sched_entity *entity);
>  
> -void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
> +int amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr);
>  void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr);
>  void amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr);
>  void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 52e4e90..338a091 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -988,11 +988,15 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
>  	mutex_init(&fpriv->bo_list_lock);
>  	idr_init(&fpriv->bo_list_handles);
>  
> -	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr);
> +	if (amdgpu_ctx_mgr_init(&fpriv->ctx_mgr))
> +		goto error_ctx_mgr;
>  
>  	file_priv->driver_priv = fpriv;
>  	goto out_suspend;
>  
> +error_ctx_mgr:
> +	idr_destroy(&fpriv->bo_list_handles);
> +	mutex_destroy(&fpriv->bo_list_lock);
>  error_vm:
>  	amdgpu_vm_fini(adev, &fpriv->vm);
>  
_______________________________________________
amd-gfx mailing list
amd-gfx@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/amd-gfx