[PATCH v4 1/3] drm/amdgpu: Increase GFX6 graphics ring size.

[PATCH v4 1/3] drm/amdgpu: Increase GFX6 graphics ring size.

[Bas Nieuwenhuizen]

To ensure it supports 192 IBs per submission, so we can keep a
simplified IB limit in the follow up patch without having to
look at IP or GPU version.

Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
---
 drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
index c41219e23151..d9ce4d1c50e4 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
@@ -3073,7 +3073,7 @@ static int gfx_v6_0_sw_init(void *handle)
 		ring = &adev->gfx.gfx_ring[i];
 		ring->ring_obj = NULL;
 		sprintf(ring->name, "gfx");
-		r = amdgpu_ring_init(adev, ring, 1024,
+		r = amdgpu_ring_init(adev, ring, 2048,
 				     &adev->gfx.eop_irq,
 				     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
 				     AMDGPU_RING_PRIO_DEFAULT, NULL);
-- 
2.40.0

[PATCH v4 2/3] drm/amdgpu: Add a max ibs per submission limit.

[Bas Nieuwenhuizen]

And ensure each ring supports that many submissions. This makes
sure that we don't get surprises after the submission has been
scheduled where the ring allocation actually gets rejected.

My calculations on the existing limits:
COMPUTE v10: 128
COMPUTE v11: 128
COMPUTE v6: 157
COMPUTE v7: 133
COMPUTE v8: 130
COMPUTE v9: 125
GFX v10: 208
GFX v11: 213
GFX v6: 154 (doubling this in the previous patch)
GFX v7: 226
GFX v8: 213
GFX v9: 208
GFX v9 (SW): 208
SDMA CIK: 87
SDMA SI: 97
SDMA v2.4: 74
SDMA v3.0: 74
SDMA v4.0: 72
SDMA v5.0: 51
SDMA v6.0: 52
UVD ENC v6.0: 98
UVD ENC v7.0: 92
UVD v3.1: 124
UVD v4.2: 124
UVD v5.0: 83
UVD v6.0  (VM): 55
UVD v7.0: 51
VCE v2.0: 126
VCE v3.0 (VM): 98
VCE v4.0: 93
VCN DEC v1.0: 49
VCN DEC v2.0: 51
VCN DEC v3.0: 51
VCN ENC v1.0: 58
VCN ENC v2.0: 93
VCN ENC v3.0: 93
VCN ENC v4.0: 93
VCN JPEG v1.0: 17
VCN JPEG v2.0: 16
VCN JPEG v2.5: 17
VCN JPEG v3.0: 17
VCN JPEG v4.0: 17

Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2498
Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  3 +++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 29 ++++++++++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  1 +
 3 files changed, 33 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 7af3041ccd0e..8362738974c8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -110,6 +110,9 @@ static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,
 	if (r < 0)
 		return r;
 
+	if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type))
+		return -EINVAL;
+
 	++(num_ibs[r]);
 	p->gang_leader_idx = r;
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index dc474b809604..f676c236b657 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -49,6 +49,26 @@
  * them until the pointers are equal again.
  */
 
+/**
+ * amdgpu_ring_max_ibs - Return max IBs that fit in a single submission.
+ *
+ * @type: ring type for which to return the limit.
+ */
+unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type)
+{
+	switch (type) {
+	case AMDGPU_RING_TYPE_GFX:
+		/* Need to keep at least 192 on GFX7+ for old radv. */
+		return 192;
+	case AMDGPU_RING_TYPE_COMPUTE:
+		return 125;
+	case AMDGPU_RING_TYPE_VCN_JPEG:
+		return 16;
+	default:
+		return 49;
+	}
+}
+
 /**
  * amdgpu_ring_alloc - allocate space on the ring buffer
  *
@@ -182,6 +202,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 	int sched_hw_submission = amdgpu_sched_hw_submission;
 	u32 *num_sched;
 	u32 hw_ip;
+	unsigned int max_ibs_dw;
 
 	/* Set the hw submission limit higher for KIQ because
 	 * it's used for a number of gfx/compute tasks by both
@@ -290,6 +311,14 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
 		return r;
 	}
 
+	max_ibs_dw = ring->funcs->emit_frame_size +
+		     amdgpu_ring_max_ibs(ring->funcs->type) * ring->funcs->emit_ib_size;
+	max_ibs_dw = (max_ibs_dw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
+
+	if (WARN_ON(max_ibs_dw > max_dw)) {
+		max_dw = max_ibs_dw;
+	}
+
 	ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission);
 
 	ring->buf_mask = (ring->ring_size / 4) - 1;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 3989e755a5b4..e6e672727529 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -317,6 +317,7 @@ struct amdgpu_ring {
 #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
 #define amdgpu_ring_preempt_ib(r) (r)->funcs->preempt_ib(r)
 
+unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type);
 int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw);
 void amdgpu_ring_ib_begin(struct amdgpu_ring *ring);
 void amdgpu_ring_ib_end(struct amdgpu_ring *ring);
-- 
2.40.0

[PATCH v4 3/3] drm/amdgpu: Add support for querying the max ibs in a submission. (v3)

[Bas Nieuwenhuizen]

This info would be used by radv to figure out when we need to
split a submission into multiple submissions. radv currently has
a limit of 192 which seems to work for most gfx submissions, but
is way too high for e.g. compute or sdma.

Userspace is available at
https://gitlab.freedesktop.org/bnieuwenhuizen/mesa/-/commits/ib-rejection-v3

v3: Completely rewrote based on suggestion of making it a separate query.

Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2498
Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +++++++++
 include/uapi/drm/amdgpu_drm.h           | 2 ++
 2 files changed, 11 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 0efb38539d70..1a2e342af1c0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -1140,6 +1140,15 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		kfree(caps);
 		return r;
 	}
+	case AMDGPU_INFO_MAX_IBS: {
+		uint32_t max_ibs[AMDGPU_HW_IP_NUM];
+
+		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
+			max_ibs[i] = amdgpu_ring_max_ibs(i);
+
+		return copy_to_user(out, max_ibs,
+				    min((size_t)size, sizeof(max_ibs))) ? -EFAULT : 0;
+	}
 	default:
 		DRM_DEBUG_KMS("Invalid request %d\n", info->query);
 		return -EINVAL;
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index b6eb90df5d05..6981e59a9401 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -876,6 +876,8 @@ struct drm_amdgpu_cs_chunk_data {
 	#define AMDGPU_INFO_VIDEO_CAPS_DECODE		0
 	/* Subquery id: Encode */
 	#define AMDGPU_INFO_VIDEO_CAPS_ENCODE		1
+/* Query the max number of IBs per gang per submission */
+#define AMDGPU_INFO_MAX_IBS			0x22
 
 #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT	0
 #define AMDGPU_INFO_MMR_SE_INDEX_MASK	0xff
-- 
2.40.0

Re: [PATCH v4 1/3] drm/amdgpu: Increase GFX6 graphics ring size.

[Christian König]

Am 13.04.23 um 16:22 schrieb Bas Nieuwenhuizen:
> To ensure it supports 192 IBs per submission, so we can keep a
> simplified IB limit in the follow up patch without having to
> look at IP or GPU version.
>
> Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>

I've took the time and double checked the docs and the maximum size of 
the ring buffer on GFX6 is indeed only 1MiB.

So even with this change sched_hw_submission can still go as high as 512 
without problems (the default is 2 or 4 IIRC).

Reviewed-by: Christian König <christian.koenig@amd.com>

> ---
>   drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c | 2 +-
>   1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
> index c41219e23151..d9ce4d1c50e4 100644
> --- a/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/gfx_v6_0.c
> @@ -3073,7 +3073,7 @@ static int gfx_v6_0_sw_init(void *handle)
>   		ring = &adev->gfx.gfx_ring[i];
>   		ring->ring_obj = NULL;
>   		sprintf(ring->name, "gfx");
> -		r = amdgpu_ring_init(adev, ring, 1024,
> +		r = amdgpu_ring_init(adev, ring, 2048,
>   				     &adev->gfx.eop_irq,
>   				     AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP,
>   				     AMDGPU_RING_PRIO_DEFAULT, NULL);

Re: [PATCH v4 2/3] drm/amdgpu: Add a max ibs per submission limit.

[Christian König]

Am 13.04.23 um 16:22 schrieb Bas Nieuwenhuizen:
> And ensure each ring supports that many submissions. This makes
> sure that we don't get surprises after the submission has been
> scheduled where the ring allocation actually gets rejected.
>
> My calculations on the existing limits:
> COMPUTE v10: 128
> COMPUTE v11: 128
> COMPUTE v6: 157
> COMPUTE v7: 133
> COMPUTE v8: 130
> COMPUTE v9: 125
> GFX v10: 208
> GFX v11: 213
> GFX v6: 154 (doubling this in the previous patch)
> GFX v7: 226
> GFX v8: 213
> GFX v9: 208
> GFX v9 (SW): 208
> SDMA CIK: 87
> SDMA SI: 97
> SDMA v2.4: 74
> SDMA v3.0: 74
> SDMA v4.0: 72
> SDMA v5.0: 51
> SDMA v6.0: 52
> UVD ENC v6.0: 98
> UVD ENC v7.0: 92
> UVD v3.1: 124
> UVD v4.2: 124
> UVD v5.0: 83
> UVD v6.0  (VM): 55
> UVD v7.0: 51
> VCE v2.0: 126
> VCE v3.0 (VM): 98
> VCE v4.0: 93
> VCN DEC v1.0: 49
> VCN DEC v2.0: 51
> VCN DEC v3.0: 51
> VCN ENC v1.0: 58
> VCN ENC v2.0: 93
> VCN ENC v3.0: 93
> VCN ENC v4.0: 93
> VCN JPEG v1.0: 17
> VCN JPEG v2.0: 16
> VCN JPEG v2.5: 17
> VCN JPEG v3.0: 17
> VCN JPEG v4.0: 17
>
> Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2498
> Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>

Reviewed-by: Christian König <christian.koenig@amd.com>

> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c   |  3 +++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c | 29 ++++++++++++++++++++++++
>   drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h |  1 +
>   3 files changed, 33 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index 7af3041ccd0e..8362738974c8 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -110,6 +110,9 @@ static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,
>   	if (r < 0)
>   		return r;
>   
> +	if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type))
> +		return -EINVAL;
> +
>   	++(num_ibs[r]);
>   	p->gang_leader_idx = r;
>   	return 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> index dc474b809604..f676c236b657 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
> @@ -49,6 +49,26 @@
>    * them until the pointers are equal again.
>    */
>   
> +/**
> + * amdgpu_ring_max_ibs - Return max IBs that fit in a single submission.
> + *
> + * @type: ring type for which to return the limit.
> + */
> +unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type)
> +{
> +	switch (type) {
> +	case AMDGPU_RING_TYPE_GFX:
> +		/* Need to keep at least 192 on GFX7+ for old radv. */
> +		return 192;
> +	case AMDGPU_RING_TYPE_COMPUTE:
> +		return 125;
> +	case AMDGPU_RING_TYPE_VCN_JPEG:
> +		return 16;
> +	default:
> +		return 49;
> +	}
> +}
> +
>   /**
>    * amdgpu_ring_alloc - allocate space on the ring buffer
>    *
> @@ -182,6 +202,7 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
>   	int sched_hw_submission = amdgpu_sched_hw_submission;
>   	u32 *num_sched;
>   	u32 hw_ip;
> +	unsigned int max_ibs_dw;
>   
>   	/* Set the hw submission limit higher for KIQ because
>   	 * it's used for a number of gfx/compute tasks by both
> @@ -290,6 +311,14 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
>   		return r;
>   	}
>   
> +	max_ibs_dw = ring->funcs->emit_frame_size +
> +		     amdgpu_ring_max_ibs(ring->funcs->type) * ring->funcs->emit_ib_size;
> +	max_ibs_dw = (max_ibs_dw + ring->funcs->align_mask) & ~ring->funcs->align_mask;
> +
> +	if (WARN_ON(max_ibs_dw > max_dw)) {
> +		max_dw = max_ibs_dw;
> +	}
> +
>   	ring->ring_size = roundup_pow_of_two(max_dw * 4 * sched_hw_submission);
>   
>   	ring->buf_mask = (ring->ring_size / 4) - 1;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> index 3989e755a5b4..e6e672727529 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
> @@ -317,6 +317,7 @@ struct amdgpu_ring {
>   #define amdgpu_ring_patch_cond_exec(r,o) (r)->funcs->patch_cond_exec((r),(o))
>   #define amdgpu_ring_preempt_ib(r) (r)->funcs->preempt_ib(r)
>   
> +unsigned int amdgpu_ring_max_ibs(enum amdgpu_ring_type type);
>   int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw);
>   void amdgpu_ring_ib_begin(struct amdgpu_ring *ring);
>   void amdgpu_ring_ib_end(struct amdgpu_ring *ring);

Re: [PATCH v4 3/3] drm/amdgpu: Add support for querying the max ibs in a submission. (v3)

[Christian König]

Am 13.04.23 um 16:22 schrieb Bas Nieuwenhuizen:
> This info would be used by radv to figure out when we need to
> split a submission into multiple submissions. radv currently has
> a limit of 192 which seems to work for most gfx submissions, but
> is way too high for e.g. compute or sdma.
>
> Userspace is available at
> https://gitlab.freedesktop.org/bnieuwenhuizen/mesa/-/commits/ib-rejection-v3
>
> v3: Completely rewrote based on suggestion of making it a separate query.
>
> Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2498
> Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>

Reviewed-by: Christian König <christian.koenig@amd.com>

> ---
>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +++++++++
>   include/uapi/drm/amdgpu_drm.h           | 2 ++
>   2 files changed, 11 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 0efb38539d70..1a2e342af1c0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -1140,6 +1140,15 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>   		kfree(caps);
>   		return r;
>   	}
> +	case AMDGPU_INFO_MAX_IBS: {
> +		uint32_t max_ibs[AMDGPU_HW_IP_NUM];
> +
> +		for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
> +			max_ibs[i] = amdgpu_ring_max_ibs(i);
> +
> +		return copy_to_user(out, max_ibs,
> +				    min((size_t)size, sizeof(max_ibs))) ? -EFAULT : 0;
> +	}
>   	default:
>   		DRM_DEBUG_KMS("Invalid request %d\n", info->query);
>   		return -EINVAL;
> diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
> index b6eb90df5d05..6981e59a9401 100644
> --- a/include/uapi/drm/amdgpu_drm.h
> +++ b/include/uapi/drm/amdgpu_drm.h
> @@ -876,6 +876,8 @@ struct drm_amdgpu_cs_chunk_data {
>   	#define AMDGPU_INFO_VIDEO_CAPS_DECODE		0
>   	/* Subquery id: Encode */
>   	#define AMDGPU_INFO_VIDEO_CAPS_ENCODE		1
> +/* Query the max number of IBs per gang per submission */
> +#define AMDGPU_INFO_MAX_IBS			0x22
>   
>   #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT	0
>   #define AMDGPU_INFO_MMR_SE_INDEX_MASK	0xff

Re: [PATCH v4 3/3] drm/amdgpu: Add support for querying the max ibs in a submission. (v3)

[Alex Deucher]

On Thu, Apr 13, 2023 at 10:25 AM Bas Nieuwenhuizen
<bas@basnieuwenhuizen.nl> wrote:
>
> This info would be used by radv to figure out when we need to
> split a submission into multiple submissions. radv currently has
> a limit of 192 which seems to work for most gfx submissions, but
> is way too high for e.g. compute or sdma.
>
> Userspace is available at
> https://gitlab.freedesktop.org/bnieuwenhuizen/mesa/-/commits/ib-rejection-v3
>
> v3: Completely rewrote based on suggestion of making it a separate query.
>
> Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2498
> Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>

Maybe a 4th patch to bump the driver version?

Alex

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +++++++++
>  include/uapi/drm/amdgpu_drm.h           | 2 ++
>  2 files changed, 11 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 0efb38539d70..1a2e342af1c0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -1140,6 +1140,15 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>                 kfree(caps);
>                 return r;
>         }
> +       case AMDGPU_INFO_MAX_IBS: {
> +               uint32_t max_ibs[AMDGPU_HW_IP_NUM];
> +
> +               for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
> +                       max_ibs[i] = amdgpu_ring_max_ibs(i);
> +
> +               return copy_to_user(out, max_ibs,
> +                                   min((size_t)size, sizeof(max_ibs))) ? -EFAULT : 0;
> +       }
>         default:
>                 DRM_DEBUG_KMS("Invalid request %d\n", info->query);
>                 return -EINVAL;
> diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
> index b6eb90df5d05..6981e59a9401 100644
> --- a/include/uapi/drm/amdgpu_drm.h
> +++ b/include/uapi/drm/amdgpu_drm.h
> @@ -876,6 +876,8 @@ struct drm_amdgpu_cs_chunk_data {
>         #define AMDGPU_INFO_VIDEO_CAPS_DECODE           0
>         /* Subquery id: Encode */
>         #define AMDGPU_INFO_VIDEO_CAPS_ENCODE           1
> +/* Query the max number of IBs per gang per submission */
> +#define AMDGPU_INFO_MAX_IBS                    0x22
>
>  #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT 0
>  #define AMDGPU_INFO_MMR_SE_INDEX_MASK  0xff
> --
> 2.40.0
>

Re: [PATCH v4 3/3] drm/amdgpu: Add support for querying the max ibs in a submission. (v3)

[Bas Nieuwenhuizen]

On Fri, Apr 14, 2023 at 11:18 PM Alex Deucher <alexdeucher@gmail.com> wrote:
>
> On Thu, Apr 13, 2023 at 10:25 AM Bas Nieuwenhuizen
> <bas@basnieuwenhuizen.nl> wrote:
> >
> > This info would be used by radv to figure out when we need to
> > split a submission into multiple submissions. radv currently has
> > a limit of 192 which seems to work for most gfx submissions, but
> > is way too high for e.g. compute or sdma.
> >
> > Userspace is available at
> > https://gitlab.freedesktop.org/bnieuwenhuizen/mesa/-/commits/ib-rejection-v3
> >
> > v3: Completely rewrote based on suggestion of making it a separate query.
> >
> > Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2498
> > Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
>
> Maybe a 4th patch to bump the driver version?

In the userspace I just make the query and put in some guessed values
if it errors out,  which doesn't need the driver version enumeration.

>
> Alex
>
> > ---
> >  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +++++++++
> >  include/uapi/drm/amdgpu_drm.h           | 2 ++
> >  2 files changed, 11 insertions(+)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > index 0efb38539d70..1a2e342af1c0 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > @@ -1140,6 +1140,15 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
> >                 kfree(caps);
> >                 return r;
> >         }
> > +       case AMDGPU_INFO_MAX_IBS: {
> > +               uint32_t max_ibs[AMDGPU_HW_IP_NUM];
> > +
> > +               for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
> > +                       max_ibs[i] = amdgpu_ring_max_ibs(i);
> > +
> > +               return copy_to_user(out, max_ibs,
> > +                                   min((size_t)size, sizeof(max_ibs))) ? -EFAULT : 0;
> > +       }
> >         default:
> >                 DRM_DEBUG_KMS("Invalid request %d\n", info->query);
> >                 return -EINVAL;
> > diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
> > index b6eb90df5d05..6981e59a9401 100644
> > --- a/include/uapi/drm/amdgpu_drm.h
> > +++ b/include/uapi/drm/amdgpu_drm.h
> > @@ -876,6 +876,8 @@ struct drm_amdgpu_cs_chunk_data {
> >         #define AMDGPU_INFO_VIDEO_CAPS_DECODE           0
> >         /* Subquery id: Encode */
> >         #define AMDGPU_INFO_VIDEO_CAPS_ENCODE           1
> > +/* Query the max number of IBs per gang per submission */
> > +#define AMDGPU_INFO_MAX_IBS                    0x22
> >
> >  #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT 0
> >  #define AMDGPU_INFO_MMR_SE_INDEX_MASK  0xff
> > --
> > 2.40.0
> >

Re: [PATCH v4 3/3] drm/amdgpu: Add support for querying the max ibs in a submission. (v3)

[Alex Deucher]

On Fri, Apr 14, 2023 at 5:24 PM Bas Nieuwenhuizen
<bas@basnieuwenhuizen.nl> wrote:
>
> On Fri, Apr 14, 2023 at 11:18 PM Alex Deucher <alexdeucher@gmail.com> wrote:
> >
> > On Thu, Apr 13, 2023 at 10:25 AM Bas Nieuwenhuizen
> > <bas@basnieuwenhuizen.nl> wrote:
> > >
> > > This info would be used by radv to figure out when we need to
> > > split a submission into multiple submissions. radv currently has
> > > a limit of 192 which seems to work for most gfx submissions, but
> > > is way too high for e.g. compute or sdma.
> > >
> > > Userspace is available at
> > > https://gitlab.freedesktop.org/bnieuwenhuizen/mesa/-/commits/ib-rejection-v3
> > >
> > > v3: Completely rewrote based on suggestion of making it a separate query.
> > >
> > > Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2498
> > > Signed-off-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
> >
> > Maybe a 4th patch to bump the driver version?
>
> In the userspace I just make the query and put in some guessed values
> if it errors out,  which doesn't need the driver version enumeration.

Ok.  Applied the series.  Thanks!

Alex


>
> >
> > Alex
> >
> > > ---
> > >  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +++++++++
> > >  include/uapi/drm/amdgpu_drm.h           | 2 ++
> > >  2 files changed, 11 insertions(+)
> > >
> > > diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > > index 0efb38539d70..1a2e342af1c0 100644
> > > --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > > +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> > > @@ -1140,6 +1140,15 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
> > >                 kfree(caps);
> > >                 return r;
> > >         }
> > > +       case AMDGPU_INFO_MAX_IBS: {
> > > +               uint32_t max_ibs[AMDGPU_HW_IP_NUM];
> > > +
> > > +               for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
> > > +                       max_ibs[i] = amdgpu_ring_max_ibs(i);
> > > +
> > > +               return copy_to_user(out, max_ibs,
> > > +                                   min((size_t)size, sizeof(max_ibs))) ? -EFAULT : 0;
> > > +       }
> > >         default:
> > >                 DRM_DEBUG_KMS("Invalid request %d\n", info->query);
> > >                 return -EINVAL;
> > > diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
> > > index b6eb90df5d05..6981e59a9401 100644
> > > --- a/include/uapi/drm/amdgpu_drm.h
> > > +++ b/include/uapi/drm/amdgpu_drm.h
> > > @@ -876,6 +876,8 @@ struct drm_amdgpu_cs_chunk_data {
> > >         #define AMDGPU_INFO_VIDEO_CAPS_DECODE           0
> > >         /* Subquery id: Encode */
> > >         #define AMDGPU_INFO_VIDEO_CAPS_ENCODE           1
> > > +/* Query the max number of IBs per gang per submission */
> > > +#define AMDGPU_INFO_MAX_IBS                    0x22
> > >
> > >  #define AMDGPU_INFO_MMR_SE_INDEX_SHIFT 0
> > >  #define AMDGPU_INFO_MMR_SE_INDEX_MASK  0xff
> > > --
> > > 2.40.0
> > >