[PATCH 1/3] drm/amdgpu: function to read physical xcc_id

[PATCH 1/3] drm/amdgpu: function to read physical xcc_id

[Samir Dhume]

Signed-off-by: Samir Dhume <samir.dhume@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h | 1 +
 drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 8fcf889ddce9..bebda5501cb7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -298,6 +298,7 @@ struct amdgpu_gfx_funcs {
 	int (*switch_partition_mode)(struct amdgpu_device *adev,
 				     int num_xccs_per_xcp);
 	int (*ih_node_to_logical_xcc)(struct amdgpu_device *adev, int ih_node);
+	int (*get_xcc_id)(struct amdgpu_device *adev, int inst);
 };
 
 struct sq_work {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index b53c8fd4e8cf..68508c19a9b3 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -669,6 +669,11 @@ static int gfx_v9_4_3_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
 	return xcc - 1;
 }
 
+static int gfx_v9_4_3_get_xcc_id(struct amdgpu_device *adev, int inst)
+{
+	return RREG32_SOC15(GC, GET_INST(GC, inst), regCP_PSP_XCP_CTL);
+}
+
 static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
 	.get_gpu_clock_counter = &gfx_v9_4_3_get_gpu_clock_counter,
 	.select_se_sh = &gfx_v9_4_3_xcc_select_se_sh,
@@ -678,6 +683,7 @@ static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = {
 	.select_me_pipe_q = &gfx_v9_4_3_select_me_pipe_q,
 	.switch_partition_mode = &gfx_v9_4_3_switch_compute_partition,
 	.ih_node_to_logical_xcc = &gfx_v9_4_3_ih_to_xcc_inst,
+	.get_xcc_id = &gfx_v9_4_3_get_xcc_id,
 };
 
 static int gfx_v9_4_3_aca_bank_generate_report(struct aca_handle *handle,
-- 
2.34.1

[PATCH 2/3] drm/amdgpu: sdma support for sriov cpx mode

[Samir Dhume]

Signed-off-by: Samir Dhume <samir.dhume@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c | 34 +++++++++++++++++++-----
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
index fec5a3d1c4bc..f666ececbe7d 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
@@ -82,17 +82,37 @@ static unsigned sdma_v4_4_2_seq_to_irq_id(int seq_num)
 	}
 }
 
-static int sdma_v4_4_2_irq_id_to_seq(unsigned client_id)
+static int sdma_v4_4_2_irq_id_to_seq(struct amdgpu_device *adev, unsigned client_id)
 {
+
+	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
+	bool sriov_cpx_odd = false;
+	int mode;
+
+	if (amdgpu_sriov_vf(adev)) {
+		mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
+
+		if (mode == AMDGPU_CPX_PARTITION_MODE) {
+			if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
+				sriov_cpx_odd = true;
+		}
+	}
+
 	switch (client_id) {
 	case SOC15_IH_CLIENTID_SDMA0:
 		return 0;
 	case SOC15_IH_CLIENTID_SDMA1:
 		return 1;
 	case SOC15_IH_CLIENTID_SDMA2:
-		return 2;
+		if (sriov_cpx_odd)
+			return 0;
+		else
+			return 2;
 	case SOC15_IH_CLIENTID_SDMA3:
-		return 3;
+		if (sriov_cpx_odd)
+			return 1;
+		else
+			return 3;
 	default:
 		return -EINVAL;
 	}
@@ -1541,7 +1561,7 @@ static int sdma_v4_4_2_process_trap_irq(struct amdgpu_device *adev,
 	uint32_t instance, i;
 
 	DRM_DEBUG("IH: SDMA trap\n");
-	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
+	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
 
 	/* Client id gives the SDMA instance in AID. To know the exact SDMA
 	 * instance, interrupt entry gives the node id which corresponds to the AID instance.
@@ -1584,7 +1604,7 @@ static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
 	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
 		goto out;
 
-	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
+	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
 	if (instance < 0)
 		goto out;
 
@@ -1603,7 +1623,7 @@ static int sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
 
 	DRM_ERROR("Illegal instruction in SDMA command stream\n");
 
-	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
+	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
 	if (instance < 0)
 		return 0;
 
@@ -1647,7 +1667,7 @@ static int sdma_v4_4_2_print_iv_entry(struct amdgpu_device *adev,
 	struct amdgpu_task_info task_info;
 	u64 addr;
 
-	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
+	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
 	if (instance < 0 || instance >= adev->sdma.num_instances) {
 		dev_err(adev->dev, "sdma instance invalid %d\n", instance);
 		return -EINVAL;
-- 
2.34.1

Re: [PATCH 2/3] drm/amdgpu: sdma support for sriov cpx mode

[Felix Kuehling]

On 2024-03-04 10:19, Samir Dhume wrote:
> Signed-off-by: Samir Dhume <samir.dhume@amd.com>

Please add a meaningful commit description to all the patches in the 
series. See one more comment below.


> ---
>   drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c | 34 +++++++++++++++++++-----
>   1 file changed, 27 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
> index fec5a3d1c4bc..f666ececbe7d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
> +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
> @@ -82,17 +82,37 @@ static unsigned sdma_v4_4_2_seq_to_irq_id(int seq_num)
>   	}
>   }
>   
> -static int sdma_v4_4_2_irq_id_to_seq(unsigned client_id)
> +static int sdma_v4_4_2_irq_id_to_seq(struct amdgpu_device *adev, unsigned client_id)
>   {
> +
> +	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
> +	bool sriov_cpx_odd = false;
> +	int mode;
> +
> +	if (amdgpu_sriov_vf(adev)) {
> +		mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);

This queries an MMIO register for the current mode. Is that really 
necessary to do in the interrupt handler? Could we use the partition 
mode stored in xcp_mgr->mode instead?

Regards,
   Felix


> +
> +		if (mode == AMDGPU_CPX_PARTITION_MODE) {
> +			if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
> +				sriov_cpx_odd = true;
> +		}
> +	}
> +
>   	switch (client_id) {
>   	case SOC15_IH_CLIENTID_SDMA0:
>   		return 0;
>   	case SOC15_IH_CLIENTID_SDMA1:
>   		return 1;
>   	case SOC15_IH_CLIENTID_SDMA2:
> -		return 2;
> +		if (sriov_cpx_odd)
> +			return 0;
> +		else
> +			return 2;
>   	case SOC15_IH_CLIENTID_SDMA3:
> -		return 3;
> +		if (sriov_cpx_odd)
> +			return 1;
> +		else
> +			return 3;
>   	default:
>   		return -EINVAL;
>   	}
> @@ -1541,7 +1561,7 @@ static int sdma_v4_4_2_process_trap_irq(struct amdgpu_device *adev,
>   	uint32_t instance, i;
>   
>   	DRM_DEBUG("IH: SDMA trap\n");
> -	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
> +	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
>   
>   	/* Client id gives the SDMA instance in AID. To know the exact SDMA
>   	 * instance, interrupt entry gives the node id which corresponds to the AID instance.
> @@ -1584,7 +1604,7 @@ static int sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
>   	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
>   		goto out;
>   
> -	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
> +	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
>   	if (instance < 0)
>   		goto out;
>   
> @@ -1603,7 +1623,7 @@ static int sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
>   
>   	DRM_ERROR("Illegal instruction in SDMA command stream\n");
>   
> -	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
> +	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
>   	if (instance < 0)
>   		return 0;
>   
> @@ -1647,7 +1667,7 @@ static int sdma_v4_4_2_print_iv_entry(struct amdgpu_device *adev,
>   	struct amdgpu_task_info task_info;
>   	u64 addr;
>   
> -	instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
> +	instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
>   	if (instance < 0 || instance >= adev->sdma.num_instances) {
>   		dev_err(adev->dev, "sdma instance invalid %d\n", instance);
>   		return -EINVAL;

RE: [PATCH 2/3] drm/amdgpu: sdma support for sriov cpx mode

[Dhume, Samir]

[AMD Official Use Only - General]

> -----Original Message-----
> From: Kuehling, Felix <Felix.Kuehling@amd.com>
> Sent: Monday, March 4, 2024 6:47 PM
> To: Dhume, Samir <Samir.Dhume@amd.com>; amd-gfx@lists.freedesktop.org
> Cc: Lazar, Lijo <Lijo.Lazar@amd.com>; Wan, Gavin <Gavin.Wan@amd.com>;
> Liu, Leo <Leo.Liu@amd.com>; Deucher, Alexander
> <Alexander.Deucher@amd.com>
> Subject: Re: [PATCH 2/3] drm/amdgpu: sdma support for sriov cpx mode
>
>
> On 2024-03-04 10:19, Samir Dhume wrote:
> > Signed-off-by: Samir Dhume <samir.dhume@amd.com>
>
> Please add a meaningful commit description to all the patches in the series.
> See one more comment below.


Right!

>
>
> > ---
> >   drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c | 34
> +++++++++++++++++++-----
> >   1 file changed, 27 insertions(+), 7 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
> b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
> > index fec5a3d1c4bc..f666ececbe7d 100644
> > --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
> > +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
> > @@ -82,17 +82,37 @@ static unsigned sdma_v4_4_2_seq_to_irq_id(int
> seq_num)
> >     }
> >   }
> >
> > -static int sdma_v4_4_2_irq_id_to_seq(unsigned client_id)
> > +static int sdma_v4_4_2_irq_id_to_seq(struct amdgpu_device *adev,
> unsigned client_id)
> >   {
> > +
> > +   struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
> > +   bool sriov_cpx_odd = false;
> > +   int mode;
> > +
> > +   if (amdgpu_sriov_vf(adev)) {
> > +           mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
>
> This queries an MMIO register for the current mode. Is that really
> necessary to do in the interrupt handler? Could we use the partition
> mode stored in xcp_mgr->mode instead?

The design appears to be that even when the host sets the mode to DPX/QPX/CPX, each guest sets itself to be in the SPX mode and xcp_mgr->mode is set to SPX.
But I can use a new field in xcp_mgr to reflect the system mode set by the host and remove the MMIO access from the interrupt handler.

Thanks,
samir

>
> Regards,
>    Felix
>
>
> > +
> > +           if (mode == AMDGPU_CPX_PARTITION_MODE) {
> > +                   if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
> > +                           sriov_cpx_odd = true;
> > +           }
> > +   }
> > +
> >     switch (client_id) {
> >     case SOC15_IH_CLIENTID_SDMA0:
> >             return 0;
> >     case SOC15_IH_CLIENTID_SDMA1:
> >             return 1;
> >     case SOC15_IH_CLIENTID_SDMA2:
> > -           return 2;
> > +           if (sriov_cpx_odd)
> > +                   return 0;
> > +           else
> > +                   return 2;
> >     case SOC15_IH_CLIENTID_SDMA3:
> > -           return 3;
> > +           if (sriov_cpx_odd)
> > +                   return 1;
> > +           else
> > +                   return 3;
> >     default:
> >             return -EINVAL;
> >     }
> > @@ -1541,7 +1561,7 @@ static int sdma_v4_4_2_process_trap_irq(struct
> amdgpu_device *adev,
> >     uint32_t instance, i;
> >
> >     DRM_DEBUG("IH: SDMA trap\n");
> > -   instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
> > +   instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
> >
> >     /* Client id gives the SDMA instance in AID. To know the exact SDMA
> >      * instance, interrupt entry gives the node id which corresponds to the
> AID instance.
> > @@ -1584,7 +1604,7 @@ static int
> sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
> >     if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
> >             goto out;
> >
> > -   instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
> > +   instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
> >     if (instance < 0)
> >             goto out;
> >
> > @@ -1603,7 +1623,7 @@ static int
> sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
> >
> >     DRM_ERROR("Illegal instruction in SDMA command stream\n");
> >
> > -   instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
> > +   instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
> >     if (instance < 0)
> >             return 0;
> >
> > @@ -1647,7 +1667,7 @@ static int sdma_v4_4_2_print_iv_entry(struct
> amdgpu_device *adev,
> >     struct amdgpu_task_info task_info;
> >     u64 addr;
> >
> > -   instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
> > +   instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
> >     if (instance < 0 || instance >= adev->sdma.num_instances) {
> >             dev_err(adev->dev, "sdma instance invalid %d\n", instance);
> >             return -EINVAL;

Re: [PATCH 2/3] drm/amdgpu: sdma support for sriov cpx mode

[Felix Kuehling]

On 2024-03-05 14:49, Dhume, Samir wrote:
> [AMD Official Use Only - General]
>
>> -----Original Message-----
>> From: Kuehling, Felix <Felix.Kuehling@amd.com>
>> Sent: Monday, March 4, 2024 6:47 PM
>> To: Dhume, Samir <Samir.Dhume@amd.com>; amd-gfx@lists.freedesktop.org
>> Cc: Lazar, Lijo <Lijo.Lazar@amd.com>; Wan, Gavin <Gavin.Wan@amd.com>;
>> Liu, Leo <Leo.Liu@amd.com>; Deucher, Alexander
>> <Alexander.Deucher@amd.com>
>> Subject: Re: [PATCH 2/3] drm/amdgpu: sdma support for sriov cpx mode
>>
>>
>> On 2024-03-04 10:19, Samir Dhume wrote:
>>> Signed-off-by: Samir Dhume <samir.dhume@amd.com>
>> Please add a meaningful commit description to all the patches in the series.
>> See one more comment below.
>
> Right!
>
>>
>>> ---
>>>    drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c | 34
>> +++++++++++++++++++-----
>>>    1 file changed, 27 insertions(+), 7 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
>> b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
>>> index fec5a3d1c4bc..f666ececbe7d 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
>>> @@ -82,17 +82,37 @@ static unsigned sdma_v4_4_2_seq_to_irq_id(int
>> seq_num)
>>>      }
>>>    }
>>>
>>> -static int sdma_v4_4_2_irq_id_to_seq(unsigned client_id)
>>> +static int sdma_v4_4_2_irq_id_to_seq(struct amdgpu_device *adev,
>> unsigned client_id)
>>>    {
>>> +
>>> +   struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
>>> +   bool sriov_cpx_odd = false;
>>> +   int mode;
>>> +
>>> +   if (amdgpu_sriov_vf(adev)) {
>>> +           mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
>> This queries an MMIO register for the current mode. Is that really
>> necessary to do in the interrupt handler? Could we use the partition
>> mode stored in xcp_mgr->mode instead?
> The design appears to be that even when the host sets the mode to DPX/QPX/CPX, each guest sets itself to be in the SPX mode and xcp_mgr->mode is set to SPX.
> But I can use a new field in xcp_mgr to reflect the system mode set by the host and remove the MMIO access from the interrupt handler.

Can you clarify what it means when the host and guest see a different 
partition mode? Is this the case, where the host partitions the device 
into several VFs, and the guest partitions those VFs further into 
smaller partitions? As far as I know, that finer partitioning in the 
guest is actually controlled by the host as well. If the guest sees SPX 
mode, it means it doesn't partition the VF into smaller pieces.

Instead of looking at the partition mode, would it make more sense to 
just query the number of XCDs in the partition (from the xcc_mask)? That 
should give the right answer regardless of how the host partitioned the GPU.

Regards,
   Felix


>
> Thanks,
> samir
>
>> Regards,
>>     Felix
>>
>>
>>> +
>>> +           if (mode == AMDGPU_CPX_PARTITION_MODE) {
>>> +                   if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
>>> +                           sriov_cpx_odd = true;
>>> +           }
>>> +   }
>>> +
>>>      switch (client_id) {
>>>      case SOC15_IH_CLIENTID_SDMA0:
>>>              return 0;
>>>      case SOC15_IH_CLIENTID_SDMA1:
>>>              return 1;
>>>      case SOC15_IH_CLIENTID_SDMA2:
>>> -           return 2;
>>> +           if (sriov_cpx_odd)
>>> +                   return 0;
>>> +           else
>>> +                   return 2;
>>>      case SOC15_IH_CLIENTID_SDMA3:
>>> -           return 3;
>>> +           if (sriov_cpx_odd)
>>> +                   return 1;
>>> +           else
>>> +                   return 3;
>>>      default:
>>>              return -EINVAL;
>>>      }
>>> @@ -1541,7 +1561,7 @@ static int sdma_v4_4_2_process_trap_irq(struct
>> amdgpu_device *adev,
>>>      uint32_t instance, i;
>>>
>>>      DRM_DEBUG("IH: SDMA trap\n");
>>> -   instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
>>> +   instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
>>>
>>>      /* Client id gives the SDMA instance in AID. To know the exact SDMA
>>>       * instance, interrupt entry gives the node id which corresponds to the
>> AID instance.
>>> @@ -1584,7 +1604,7 @@ static int
>> sdma_v4_4_2_process_ras_data_cb(struct amdgpu_device *adev,
>>>      if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA))
>>>              goto out;
>>>
>>> -   instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
>>> +   instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
>>>      if (instance < 0)
>>>              goto out;
>>>
>>> @@ -1603,7 +1623,7 @@ static int
>> sdma_v4_4_2_process_illegal_inst_irq(struct amdgpu_device *adev,
>>>      DRM_ERROR("Illegal instruction in SDMA command stream\n");
>>>
>>> -   instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
>>> +   instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
>>>      if (instance < 0)
>>>              return 0;
>>>
>>> @@ -1647,7 +1667,7 @@ static int sdma_v4_4_2_print_iv_entry(struct
>> amdgpu_device *adev,
>>>      struct amdgpu_task_info task_info;
>>>      u64 addr;
>>>
>>> -   instance = sdma_v4_4_2_irq_id_to_seq(entry->client_id);
>>> +   instance = sdma_v4_4_2_irq_id_to_seq(adev, entry->client_id);
>>>      if (instance < 0 || instance >= adev->sdma.num_instances) {
>>>              dev_err(adev->dev, "sdma instance invalid %d\n", instance);
>>>              return -EINVAL;

[PATCH 3/3] drm/amdgpu/jpeg: support for sriov cpx mode

[Samir Dhume]

Signed-off-by: Samir Dhume <samir.dhume@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c | 80 +++++++++++++++++++++---
 1 file changed, 73 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
index 32caeb37cef9..4bf087f8ca2b 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
@@ -65,9 +65,15 @@ static int amdgpu_ih_srcid_jpeg[] = {
 static int jpeg_v4_0_3_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
 
 	adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS;
 
+	if (amdgpu_sriov_vf(adev))
+		if (adev->xcp_mgr->funcs->query_partition_mode(xcp_mgr) ==
+				AMDGPU_CPX_PARTITION_MODE)
+			adev->jpeg.num_jpeg_rings = 4;
+
 	jpeg_v4_0_3_set_dec_ring_funcs(adev);
 	jpeg_v4_0_3_set_irq_funcs(adev);
 	jpeg_v4_0_3_set_ras_funcs(adev);
@@ -88,10 +94,28 @@ static int jpeg_v4_0_3_sw_init(void *handle)
 	struct amdgpu_ring *ring;
 	int i, j, r, jpeg_inst;
 
+	bool sriov_cpx_odd = false;
+	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
+	int mode;
+
+	if (amdgpu_sriov_vf(adev)) {
+		mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
+
+		if (mode == AMDGPU_CPX_PARTITION_MODE) {
+			if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
+				sriov_cpx_odd = true;
+		}
+	}
+
 	for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
 		/* JPEG TRAP */
-		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
+		if (!sriov_cpx_odd)
+			r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
 				amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst->irq);
+		else
+			r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
+				amdgpu_ih_srcid_jpeg[j+4], &adev->jpeg.inst->irq);
+
 		if (r)
 			return r;
 	}
@@ -116,10 +140,15 @@ static int jpeg_v4_0_3_sw_init(void *handle)
 					(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 					1 + j + 9 * jpeg_inst;
 			} else {
-				if (j < 4)
+				if ((j < 4) && (!sriov_cpx_odd))
 					ring->doorbell_index =
 						(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
 						4 + j + 32 * jpeg_inst;
+				else if (sriov_cpx_odd)
+					ring->doorbell_index =
+						(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
+						12 + j + 32 * jpeg_inst;
+
 				else
 					ring->doorbell_index =
 						(adev->doorbell_index.vcn.vcn_ring0_1 << 1) +
@@ -186,6 +215,9 @@ static int jpeg_v4_0_3_start_sriov(struct amdgpu_device *adev)
 	uint32_t size, size_dw, item_offset;
 	uint32_t init_status;
 	int i, j, jpeg_inst;
+	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
+	int mode;
+	bool cpx_odd = false;
 
 	struct mmsch_v4_0_cmd_direct_write
 		direct_wt = { {0} };
@@ -197,6 +229,13 @@ static int jpeg_v4_0_3_start_sriov(struct amdgpu_device *adev)
 	end.cmd_header.command_type =
 		MMSCH_COMMAND__END;
 
+	mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
+
+	if (mode == AMDGPU_CPX_PARTITION_MODE) {
+		if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
+			cpx_odd = true;
+	}
+
 	for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {
 		jpeg_inst = GET_INST(JPEG, i);
 
@@ -220,10 +259,14 @@ static int jpeg_v4_0_3_start_sriov(struct amdgpu_device *adev)
 			tmp = SOC15_REG_OFFSET(JPEG, 0, regUVD_JRBC0_UVD_JRBC_RB_SIZE);
 			MMSCH_V4_0_INSERT_DIRECT_WT(tmp, ring->ring_size / 4);
 
-			if (j <= 3) {
+			if ((j <= 3) && (!cpx_odd)) {
 				header.mjpegdec0[j].table_offset = item_offset;
 				header.mjpegdec0[j].init_status = 0;
 				header.mjpegdec0[j].table_size = table_size;
+			} else if (cpx_odd) {
+				header.mjpegdec1[j].table_offset = item_offset;
+				header.mjpegdec1[j].init_status = 0;
+				header.mjpegdec1[j].table_size = table_size;
 			} else {
 				header.mjpegdec1[j - 4].table_offset = item_offset;
 				header.mjpegdec1[j - 4].init_status = 0;
@@ -986,6 +1029,16 @@ static int jpeg_v4_0_3_process_interrupt(struct amdgpu_device *adev,
 				      struct amdgpu_iv_entry *entry)
 {
 	uint32_t i, inst;
+	struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
+	int mode;
+	bool sriov_cpx_odd = false;
+
+	mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
+
+	if (mode == AMDGPU_CPX_PARTITION_MODE) {
+		if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
+			sriov_cpx_odd = true;
+	}
 
 	i = node_id_to_phys_map[entry->node_id];
 	DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n");
@@ -1015,16 +1068,29 @@ static int jpeg_v4_0_3_process_interrupt(struct amdgpu_device *adev,
 		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
 		break;
 	case VCN_4_0__SRCID__JPEG4_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
+		if (!sriov_cpx_odd)
+			amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
+		else
+			amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[0]);
+
 		break;
 	case VCN_4_0__SRCID__JPEG5_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
+		if (!sriov_cpx_odd)
+			amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
+		else
+			amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[1]);
 		break;
 	case VCN_4_0__SRCID__JPEG6_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
+		if (!sriov_cpx_odd)
+			amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
+		else
+			amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[2]);
 		break;
 	case VCN_4_0__SRCID__JPEG7_DECODE:
-		amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
+		if (!sriov_cpx_odd)
+			amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
+		else
+			amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
 		break;
 	default:
 		DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d %d\n",
-- 
2.34.1

RE: [PATCH 3/3] drm/amdgpu/jpeg: support for sriov cpx mode

[Liu, Leo]

[AMD Official Use Only - General]

> -----Original Message-----
> From: Dhume, Samir <Samir.Dhume@amd.com>
> Sent: Monday, March 4, 2024 10:20 AM
> To: amd-gfx@lists.freedesktop.org
> Cc: Dhume, Samir <Samir.Dhume@amd.com>; Lazar, Lijo
> <Lijo.Lazar@amd.com>; Wan, Gavin <Gavin.Wan@amd.com>; Liu, Leo
> <Leo.Liu@amd.com>; Deucher, Alexander <Alexander.Deucher@amd.com>
> Subject: [PATCH 3/3] drm/amdgpu/jpeg: support for sriov cpx mode
>
> Signed-off-by: Samir Dhume <samir.dhume@amd.com>
> ---
>  drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c | 80 +++++++++++++++++++++-
> --
>  1 file changed, 73 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
> b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
> index 32caeb37cef9..4bf087f8ca2b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
> +++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v4_0_3.c
> @@ -65,9 +65,15 @@ static int amdgpu_ih_srcid_jpeg[] = {  static int
> jpeg_v4_0_3_early_init(void *handle)  {
>       struct amdgpu_device *adev = (struct amdgpu_device *)handle;
> +     struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;

In general, to make the path clear, can we define and use the SRIOV specific variables and function within "amdgpu_sriov_vf()" ?

Regards,
Leo

>
>       adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS;
>
> +     if (amdgpu_sriov_vf(adev))
> +             if (adev->xcp_mgr->funcs->query_partition_mode(xcp_mgr) ==
> +                             AMDGPU_CPX_PARTITION_MODE)
> +                     adev->jpeg.num_jpeg_rings = 4;
> +
>       jpeg_v4_0_3_set_dec_ring_funcs(adev);
>       jpeg_v4_0_3_set_irq_funcs(adev);
>       jpeg_v4_0_3_set_ras_funcs(adev);
> @@ -88,10 +94,28 @@ static int jpeg_v4_0_3_sw_init(void *handle)
>       struct amdgpu_ring *ring;
>       int i, j, r, jpeg_inst;
>
> +     bool sriov_cpx_odd = false;
> +     struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
> +     int mode;
> +
> +     if (amdgpu_sriov_vf(adev)) {
> +             mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
> +
> +             if (mode == AMDGPU_CPX_PARTITION_MODE) {
> +                     if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
> +                             sriov_cpx_odd = true;
> +             }
> +     }
> +
>       for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) {
>               /* JPEG TRAP */
> -             r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VCN,
> +             if (!sriov_cpx_odd)
> +                     r = amdgpu_irq_add_id(adev,
> SOC15_IH_CLIENTID_VCN,
>                               amdgpu_ih_srcid_jpeg[j], &adev->jpeg.inst-
> >irq);
> +             else
> +                     r = amdgpu_irq_add_id(adev,
> SOC15_IH_CLIENTID_VCN,
> +                             amdgpu_ih_srcid_jpeg[j+4], &adev->jpeg.inst-
> >irq);
> +
>               if (r)
>                       return r;
>       }
> @@ -116,10 +140,15 @@ static int jpeg_v4_0_3_sw_init(void *handle)
>                                       (adev-
> >doorbell_index.vcn.vcn_ring0_1 << 1) +
>                                       1 + j + 9 * jpeg_inst;
>                       } else {
> -                             if (j < 4)
> +                             if ((j < 4) && (!sriov_cpx_odd))
>                                       ring->doorbell_index =
>                                               (adev-
> >doorbell_index.vcn.vcn_ring0_1 << 1) +
>                                               4 + j + 32 * jpeg_inst;
> +                             else if (sriov_cpx_odd)
> +                                     ring->doorbell_index =
> +                                             (adev-
> >doorbell_index.vcn.vcn_ring0_1 << 1) +
> +                                             12 + j + 32 * jpeg_inst;
> +
>                               else
>                                       ring->doorbell_index =
>                                               (adev-
> >doorbell_index.vcn.vcn_ring0_1 << 1) + @@ -186,6 +215,9 @@ static int
> jpeg_v4_0_3_start_sriov(struct amdgpu_device *adev)
>       uint32_t size, size_dw, item_offset;
>       uint32_t init_status;
>       int i, j, jpeg_inst;
> +     struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
> +     int mode;
> +     bool cpx_odd = false;
>
>       struct mmsch_v4_0_cmd_direct_write
>               direct_wt = { {0} };
> @@ -197,6 +229,13 @@ static int jpeg_v4_0_3_start_sriov(struct
> amdgpu_device *adev)
>       end.cmd_header.command_type =
>               MMSCH_COMMAND__END;
>
> +     mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
> +
> +     if (mode == AMDGPU_CPX_PARTITION_MODE) {
> +             if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
> +                     cpx_odd = true;
> +     }
> +
>       for (i = 0; i < adev->jpeg.num_jpeg_inst; i++) {
>               jpeg_inst = GET_INST(JPEG, i);
>
> @@ -220,10 +259,14 @@ static int jpeg_v4_0_3_start_sriov(struct
> amdgpu_device *adev)
>                       tmp = SOC15_REG_OFFSET(JPEG, 0,
> regUVD_JRBC0_UVD_JRBC_RB_SIZE);
>                       MMSCH_V4_0_INSERT_DIRECT_WT(tmp, ring-
> >ring_size / 4);
>
> -                     if (j <= 3) {
> +                     if ((j <= 3) && (!cpx_odd)) {
>                               header.mjpegdec0[j].table_offset =
> item_offset;
>                               header.mjpegdec0[j].init_status = 0;
>                               header.mjpegdec0[j].table_size = table_size;
> +                     } else if (cpx_odd) {
> +                             header.mjpegdec1[j].table_offset =
> item_offset;
> +                             header.mjpegdec1[j].init_status = 0;
> +                             header.mjpegdec1[j].table_size = table_size;
>                       } else {
>                               header.mjpegdec1[j - 4].table_offset =
> item_offset;
>                               header.mjpegdec1[j - 4].init_status = 0; @@ -
> 986,6 +1029,16 @@ static int jpeg_v4_0_3_process_interrupt(struct
> amdgpu_device *adev,
>                                     struct amdgpu_iv_entry *entry)  {
>       uint32_t i, inst;
> +     struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr;
> +     int mode;
> +     bool sriov_cpx_odd = false;
> +
> +     mode = xcp_mgr->funcs->query_partition_mode(xcp_mgr);
> +
> +     if (mode == AMDGPU_CPX_PARTITION_MODE) {
> +             if (adev->gfx.funcs->get_xcc_id(adev, 0) & 0x1)
> +                     sriov_cpx_odd = true;
> +     }
>
>       i = node_id_to_phys_map[entry->node_id];
>       DRM_DEV_DEBUG(adev->dev, "IH: JPEG TRAP\n"); @@ -1015,16
> +1068,29 @@ static int jpeg_v4_0_3_process_interrupt(struct amdgpu_device
> *adev,
>               amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[3]);
>               break;
>       case VCN_4_0__SRCID__JPEG4_DECODE:
> -             amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[4]);
> +             if (!sriov_cpx_odd)
> +                     amdgpu_fence_process(&adev-
> >jpeg.inst[inst].ring_dec[4]);
> +             else
> +                     amdgpu_fence_process(&adev-
> >jpeg.inst[inst].ring_dec[0]);
> +
>               break;
>       case VCN_4_0__SRCID__JPEG5_DECODE:
> -             amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[5]);
> +             if (!sriov_cpx_odd)
> +                     amdgpu_fence_process(&adev-
> >jpeg.inst[inst].ring_dec[5]);
> +             else
> +                     amdgpu_fence_process(&adev-
> >jpeg.inst[inst].ring_dec[1]);
>               break;
>       case VCN_4_0__SRCID__JPEG6_DECODE:
> -             amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[6]);
> +             if (!sriov_cpx_odd)
> +                     amdgpu_fence_process(&adev-
> >jpeg.inst[inst].ring_dec[6]);
> +             else
> +                     amdgpu_fence_process(&adev-
> >jpeg.inst[inst].ring_dec[2]);
>               break;
>       case VCN_4_0__SRCID__JPEG7_DECODE:
> -             amdgpu_fence_process(&adev->jpeg.inst[inst].ring_dec[7]);
> +             if (!sriov_cpx_odd)
> +                     amdgpu_fence_process(&adev-
> >jpeg.inst[inst].ring_dec[7]);
> +             else
> +                     amdgpu_fence_process(&adev-
> >jpeg.inst[inst].ring_dec[3]);
>               break;
>       default:
>               DRM_DEV_ERROR(adev->dev, "Unhandled interrupt: %d
> %d\n",
> --
> 2.34.1