[RESEND PATCH v0 0/5] wave5 codec driver

[RESEND PATCH v0 0/5] wave5 codec driver

[jackson.lee]

The wave5 codec driver is a stateful encoder/decoder.
The following patches is for supporting yuv422 inpuy format, supporting
runtime suspend/resume feature and extra things.

jackson.lee (5):
  wave5 : Support yuv422 input format for encoder.
  wave5: Support to prepend sps/pps to IDR frame.
  wave5 : Support runtime suspend/resume.
  wave5: Use the bitstream buffer size from host.
  wave5 : Fixed the wrong buffer size formula.

 .../platform/chips-media/wave5/wave5-hw.c     |  11 +-
 .../chips-media/wave5/wave5-vpu-dec.c         |  86 ++++------
 .../chips-media/wave5/wave5-vpu-enc.c         | 159 +++++++++++++++---
 .../platform/chips-media/wave5/wave5-vpu.c    |  68 ++++++++
 .../platform/chips-media/wave5/wave5-vpuapi.c |   7 +
 .../platform/chips-media/wave5/wave5-vpuapi.h |   1 +
 .../media/platform/chips-media/wave5/wave5.h  |   3 +
 7 files changed, 255 insertions(+), 80 deletions(-)

-- 
2.43.0

[RESEND PATCH v0 1/5] wave5 : Support yuv422 input format for encoder.

[jackson.lee]

Encoder supports the following formats.
YUV422P, NV16, NV61, NV16M, NV61M

Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
---
 .../chips-media/wave5/wave5-vpu-enc.c         | 79 ++++++++++++++++++-
 1 file changed, 76 insertions(+), 3 deletions(-)

diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
index f29cfa3af94a..0cb5bfb67258 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
@@ -70,6 +70,41 @@ static const struct vpu_format enc_fmt_list[FMT_TYPES][MAX_FMTS] = {
 			.max_height = W5_MAX_ENC_PIC_HEIGHT,
 			.min_height = W5_MIN_ENC_PIC_HEIGHT,
 		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_YUV422P,
+			.max_width = W5_MAX_ENC_PIC_WIDTH,
+			.min_width = W5_MIN_ENC_PIC_WIDTH,
+			.max_height = W5_MAX_ENC_PIC_HEIGHT,
+			.min_height = W5_MIN_ENC_PIC_HEIGHT,
+		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_NV16,
+			.max_width = W5_MAX_ENC_PIC_WIDTH,
+			.min_width = W5_MIN_ENC_PIC_WIDTH,
+			.max_height = W5_MAX_ENC_PIC_HEIGHT,
+			.min_height = W5_MIN_ENC_PIC_HEIGHT,
+		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61,
+			.max_width = W5_MAX_ENC_PIC_WIDTH,
+			.min_width = W5_MIN_ENC_PIC_WIDTH,
+			.max_height = W5_MAX_ENC_PIC_HEIGHT,
+			.min_height = W5_MIN_ENC_PIC_HEIGHT,
+		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_NV16M,
+			.max_width = W5_MAX_ENC_PIC_WIDTH,
+			.min_width = W5_MIN_ENC_PIC_WIDTH,
+			.max_height = W5_MAX_ENC_PIC_HEIGHT,
+			.min_height = W5_MIN_ENC_PIC_HEIGHT,
+		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61M,
+			.max_width = W5_MAX_ENC_PIC_WIDTH,
+			.min_width = W5_MIN_ENC_PIC_WIDTH,
+			.max_height = W5_MAX_ENC_PIC_HEIGHT,
+			.min_height = W5_MIN_ENC_PIC_HEIGHT,
+		},
 	}
 };
 
@@ -136,6 +171,23 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
 		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
 		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height / 2;
 		break;
+	case V4L2_PIX_FMT_YUV422P:
+	case V4L2_PIX_FMT_NV16:
+	case V4L2_PIX_FMT_NV61:
+		pix_mp->width = width;
+		pix_mp->height = height;
+		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
+		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height * 2;
+		break;
+	case V4L2_PIX_FMT_NV16M:
+	case V4L2_PIX_FMT_NV61M:
+		pix_mp->width = width;
+		pix_mp->height = height;
+		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
+		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
+		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
+		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height;
+		break;
 	default:
 		pix_mp->width = width;
 		pix_mp->height = height;
@@ -155,11 +207,19 @@ static int start_encode(struct vpu_instance *inst, u32 *fail_res)
 	struct enc_param pic_param;
 	u32 stride = ALIGN(inst->dst_fmt.width, 32);
 	u32 luma_size = (stride * inst->dst_fmt.height);
-	u32 chroma_size = ((stride / 2) * (inst->dst_fmt.height / 2));
+	u32 chroma_size;
 
 	memset(&pic_param, 0, sizeof(struct enc_param));
 	memset(&frame_buf, 0, sizeof(struct frame_buffer));
 
+	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV420 ||
+	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV420M)
+		chroma_size = ((stride / 2) * (inst->dst_fmt.height / 2));
+	else if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV422P)
+		chroma_size = ((stride) * (inst->dst_fmt.height / 2));
+	else
+		chroma_size = 0;
+
 	dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
 	if (!dst_buf) {
 		dev_dbg(inst->dev->dev, "%s: No destination buffer found\n", __func__);
@@ -550,11 +610,15 @@ static int wave5_vpu_enc_s_fmt_out(struct file *file, void *fh, struct v4l2_form
 	}
 
 	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12 ||
-	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12M) {
+	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12M ||
+	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16 ||
+	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16M) {
 		inst->cbcr_interleave = true;
 		inst->nv21 = false;
 	} else if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21 ||
-		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21M) {
+		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21M ||
+		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61 ||
+		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61M) {
 		inst->cbcr_interleave = true;
 		inst->nv21 = true;
 	} else {
@@ -1132,6 +1196,15 @@ static void wave5_set_enc_openparam(struct enc_open_param *open_param,
 	u32 num_ctu_row = ALIGN(inst->dst_fmt.height, 64) / 64;
 	u32 num_mb_row = ALIGN(inst->dst_fmt.height, 16) / 16;
 
+	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16 ||
+	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61 ||
+	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV422P ||
+	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16M ||
+	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61M)
+		open_param->src_format = FORMAT_422;
+	else
+		open_param->src_format = FORMAT_420;
+
 	open_param->wave_param.gop_preset_idx = PRESET_IDX_IPP_SINGLE;
 	open_param->wave_param.hvs_qp_scale = 2;
 	open_param->wave_param.hvs_max_delta_qp = 10;
-- 
2.43.0

Re: [RESEND PATCH v0 1/5] wave5 : Support yuv422 input format for encoder.

[Nicolas Dufresne]

Hi Jackson,

Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> Encoder supports the following formats.
> YUV422P, NV16, NV61, NV16M, NV61M
> 
> Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> ---
>  .../chips-media/wave5/wave5-vpu-enc.c         | 79 ++++++++++++++++++-
>  1 file changed, 76 insertions(+), 3 deletions(-)
> 
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> index f29cfa3af94a..0cb5bfb67258 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> @@ -70,6 +70,41 @@ static const struct vpu_format enc_fmt_list[FMT_TYPES][MAX_FMTS] = {
>  			.max_height = W5_MAX_ENC_PIC_HEIGHT,
>  			.min_height = W5_MIN_ENC_PIC_HEIGHT,
>  		},
> +		{
> +			.v4l2_pix_fmt = V4L2_PIX_FMT_YUV422P,
> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> +		},

During upstreaming, we discussed the lack of usage of v4l2-common in this driver
and agreed that future updates such as this one should first port the driver to
use the common helpers instead.

This implies dropping this custom made structure in favour of
v4l2_frmsize_stepwise structure. Unlike yours, you can encoded the needed
padding, allowing to encode this in one place instead of spreading it across
numerous formulas in the code.

With this information, you will be able to use:

  v4l2_apply_frmsize_constraints()
  v4l2_fill_pixfmt_mp()

To adjust your dimensions to padded dimensions and compute your bytesperline
(stride) and sizeimage. You can of course increase the size image after this
call. You can have a look at rkvdec driver as an example.

Please port existing set of pixel formats support, and then add the new pixel
formats. This should remove about 3/4 of this patch and remove that huge risk of
miss-computing a size.

> +		{
> +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV16,
> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> +		},
> +		{
> +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61,
> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> +		},
> +		{
> +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV16M,
> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> +		},
> +		{
> +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61M,
> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> +		},
>  	}
>  };
>  
> @@ -136,6 +171,23 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
>  		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
>  		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height / 2;
>  		break;
> +	case V4L2_PIX_FMT_YUV422P:
> +	case V4L2_PIX_FMT_NV16:
> +	case V4L2_PIX_FMT_NV61:
> +		pix_mp->width = width;
> +		pix_mp->height = height;
> +		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> +		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height * 2;
> +		break;
> +	case V4L2_PIX_FMT_NV16M:
> +	case V4L2_PIX_FMT_NV61M:
> +		pix_mp->width = width;
> +		pix_mp->height = height;
> +		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> +		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
> +		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
> +		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height;
> +		break;
>  	default:
>  		pix_mp->width = width;
>  		pix_mp->height = height;
> @@ -155,11 +207,19 @@ static int start_encode(struct vpu_instance *inst, u32 *fail_res)
>  	struct enc_param pic_param;
>  	u32 stride = ALIGN(inst->dst_fmt.width, 32);
>  	u32 luma_size = (stride * inst->dst_fmt.height);
> -	u32 chroma_size = ((stride / 2) * (inst->dst_fmt.height / 2));
> +	u32 chroma_size;
>  
>  	memset(&pic_param, 0, sizeof(struct enc_param));
>  	memset(&frame_buf, 0, sizeof(struct frame_buffer));
>  
> +	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV420 ||
> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV420M)
> +		chroma_size = ((stride / 2) * (inst->dst_fmt.height / 2));
> +	else if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV422P)
> +		chroma_size = ((stride) * (inst->dst_fmt.height / 2));
> +	else
> +		chroma_size = 0;
> +
>  	dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
>  	if (!dst_buf) {
>  		dev_dbg(inst->dev->dev, "%s: No destination buffer found\n", __func__);
> @@ -550,11 +610,15 @@ static int wave5_vpu_enc_s_fmt_out(struct file *file, void *fh, struct v4l2_form
>  	}
>  
>  	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12 ||
> -	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12M) {
> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12M ||
> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16 ||
> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16M) {
>  		inst->cbcr_interleave = true;
>  		inst->nv21 = false;
>  	} else if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21 ||
> -		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21M) {
> +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21M ||
> +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61 ||
> +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61M) {
>  		inst->cbcr_interleave = true;
>  		inst->nv21 = true;
>  	} else {
> @@ -1132,6 +1196,15 @@ static void wave5_set_enc_openparam(struct enc_open_param *open_param,
>  	u32 num_ctu_row = ALIGN(inst->dst_fmt.height, 64) / 64;
>  	u32 num_mb_row = ALIGN(inst->dst_fmt.height, 16) / 16;
>  
> +	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16 ||
> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61 ||
> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV422P ||
> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16M ||
> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61M)
> +		open_param->src_format = FORMAT_422;
> +	else
> +		open_param->src_format = FORMAT_420;
> +
>  	open_param->wave_param.gop_preset_idx = PRESET_IDX_IPP_SINGLE;
>  	open_param->wave_param.hvs_qp_scale = 2;
>  	open_param->wave_param.hvs_max_delta_qp = 10;

Re: [RESEND PATCH v0 1/5] wave5 : Support yuv422 input format for encoder.

[Sebastian Fricke]

Hey Jackson,

a few additional comments ...

Please extend the patch title a bit, the title has to clearly indicate
where the driver can be found, so the correct title is:

media: chips-media: wave5: Support yuv422 input format for encoder.

and even better might be:

media: chips-media: wave5: Support YUV422 raw pixel-formats on the encoder 

Notice that you do not need punctuation and no space before the ':'.

On 07.02.2024 12:55, Nicolas Dufresne wrote:
>Hi Jackson,
>
>Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
>> Encoder supports the following formats.
>> YUV422P, NV16, NV61, NV16M, NV61M

I would slightly reword this, the way it is written now is a bit
confusing.
First of all, when I look into the encoder, I can see support for the
following formats:
V4L2_PIX_FMT_YUV420
V4L2_PIX_FMT_NV12
V4L2_PIX_FMT_NV21
V4L2_PIX_FMT_YUV420M
V4L2_PIX_FMT_NV12M
V4L2_PIX_FMT_NV21M
V4L2_PIX_FMT_YUV422P
V4L2_PIX_FMT_NV16
V4L2_PIX_FMT_NV61
V4L2_PIX_FMT_NV16M
V4L2_PIX_FMT_NV61M

which is clearly more than would you provide, so your patch adds support
for a couple of new formats but the encoder supports more formats.

Secondly, the commit message should shortly explain what happens in the
patch and the reason for doing it. Stating what the encoder supports
doesn't explain both of these things.

My suggestion for the commit message would be something like:

Add support for the YUV422P, NV16, NV61, NV16M & NV61M raw pixel-formats
to the Wave5 encoder. All these formats have a chroma subsampling ratio
of 4:2:0 and therefore require a new image size calculation as the
driver previously only handled a ratio of 4:2:0.

And when you switch to v4l2_fill_pixfmt_mp please note that in the
commit message as well.

more below ..

>>
>> Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
>> Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
>> ---
>>  .../chips-media/wave5/wave5-vpu-enc.c         | 79 ++++++++++++++++++-
>>  1 file changed, 76 insertions(+), 3 deletions(-)
>>
>> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
>> index f29cfa3af94a..0cb5bfb67258 100644
>> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
>> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
>> @@ -70,6 +70,41 @@ static const struct vpu_format enc_fmt_list[FMT_TYPES][MAX_FMTS] = {
>>  			.max_height = W5_MAX_ENC_PIC_HEIGHT,
>>  			.min_height = W5_MIN_ENC_PIC_HEIGHT,
>>  		},
>> +		{
>> +			.v4l2_pix_fmt = V4L2_PIX_FMT_YUV422P,
>> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
>> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
>> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
>> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
>> +		},
>
>During upstreaming, we discussed the lack of usage of v4l2-common in this driver
>and agreed that future updates such as this one should first port the driver to
>use the common helpers instead.
>
>This implies dropping this custom made structure in favour of
>v4l2_frmsize_stepwise structure. Unlike yours, you can encoded the needed
>padding, allowing to encode this in one place instead of spreading it across
>numerous formulas in the code.
>
>With this information, you will be able to use:
>
>  v4l2_apply_frmsize_constraints()
>  v4l2_fill_pixfmt_mp()
>
>To adjust your dimensions to padded dimensions and compute your bytesperline
>(stride) and sizeimage. You can of course increase the size image after this
>call. You can have a look at rkvdec driver as an example.
>
>Please port existing set of pixel formats support, and then add the new pixel
>formats. This should remove about 3/4 of this patch and remove that huge risk of
>miss-computing a size.

Please have a look at:
https://elixir.bootlin.com/linux/latest/source/drivers/staging/media/rkvdec/rkvdec.c#L257

There you see a nice example of how that can look.

>
>> +		{
>> +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV16,
>> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
>> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
>> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
>> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
>> +		},
>> +		{
>> +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61,
>> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
>> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
>> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
>> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
>> +		},
>> +		{
>> +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV16M,
>> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
>> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
>> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
>> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
>> +		},
>> +		{
>> +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61M,
>> +			.max_width = W5_MAX_ENC_PIC_WIDTH,
>> +			.min_width = W5_MIN_ENC_PIC_WIDTH,
>> +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
>> +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
>> +		},
>>  	}
>>  };
>>
>> @@ -136,6 +171,23 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
>>  		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
>>  		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height / 2;
>>  		break;
>> +	case V4L2_PIX_FMT_YUV422P:
>> +	case V4L2_PIX_FMT_NV16:
>> +	case V4L2_PIX_FMT_NV61:
>> +		pix_mp->width = width;
>> +		pix_mp->height = height;
>> +		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
>> +		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height * 2;
>> +		break;
>> +	case V4L2_PIX_FMT_NV16M:
>> +	case V4L2_PIX_FMT_NV61M:
>> +		pix_mp->width = width;
>> +		pix_mp->height = height;
>> +		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
>> +		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
>> +		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
>> +		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height;
>> +		break;
>>  	default:
>>  		pix_mp->width = width;
>>  		pix_mp->height = height;
>> @@ -155,11 +207,19 @@ static int start_encode(struct vpu_instance *inst, u32 *fail_res)
>>  	struct enc_param pic_param;
>>  	u32 stride = ALIGN(inst->dst_fmt.width, 32);
>>  	u32 luma_size = (stride * inst->dst_fmt.height);
>> -	u32 chroma_size = ((stride / 2) * (inst->dst_fmt.height / 2));
>> +	u32 chroma_size;
>>
>>  	memset(&pic_param, 0, sizeof(struct enc_param));
>>  	memset(&frame_buf, 0, sizeof(struct frame_buffer));
>>
>> +	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV420 ||
>> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV420M)
>> +		chroma_size = ((stride / 2) * (inst->dst_fmt.height / 2));
>> +	else if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV422P)
>> +		chroma_size = ((stride) * (inst->dst_fmt.height / 2));
>> +	else
>> +		chroma_size = 0;

Just making sure, with the previous calculation the chroma size was
unable to ever be 0 and here you say that:
V4L2_PIX_FMT_NV12
V4L2_PIX_FMT_NV21
V4L2_PIX_FMT_NV12M
V4L2_PIX_FMT_NV21M
V4L2_PIX_FMT_NV16
V4L2_PIX_FMT_NV61
V4L2_PIX_FMT_NV16M
V4L2_PIX_FMT_NV61M

All have a chroma size of 0, that seems odd to me as these formats have
a chroma part. Maybe I am misunderstanding something, please provide a
little explanation as a comment on top of that block.

Greetings,
Sebastian

>> +
>>  	dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
>>  	if (!dst_buf) {
>>  		dev_dbg(inst->dev->dev, "%s: No destination buffer found\n", __func__);
>> @@ -550,11 +610,15 @@ static int wave5_vpu_enc_s_fmt_out(struct file *file, void *fh, struct v4l2_form
>>  	}
>>
>>  	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12 ||
>> -	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12M) {
>> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12M ||
>> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16 ||
>> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16M) {
>>  		inst->cbcr_interleave = true;
>>  		inst->nv21 = false;
>>  	} else if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21 ||
>> -		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21M) {
>> +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21M ||
>> +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61 ||
>> +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61M) {
>>  		inst->cbcr_interleave = true;
>>  		inst->nv21 = true;
>>  	} else {
>> @@ -1132,6 +1196,15 @@ static void wave5_set_enc_openparam(struct enc_open_param *open_param,
>>  	u32 num_ctu_row = ALIGN(inst->dst_fmt.height, 64) / 64;
>>  	u32 num_mb_row = ALIGN(inst->dst_fmt.height, 16) / 16;
>>
>> +	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16 ||
>> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61 ||
>> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV422P ||
>> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16M ||
>> +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61M)
>> +		open_param->src_format = FORMAT_422;
>> +	else
>> +		open_param->src_format = FORMAT_420;
>> +
>>  	open_param->wave_param.gop_preset_idx = PRESET_IDX_IPP_SINGLE;
>>  	open_param->wave_param.hvs_qp_scale = 2;
>>  	open_param->wave_param.hvs_max_delta_qp = 10;
>
>

RE: [RESEND PATCH v0 1/5] wave5 : Support yuv422 input format for encoder.

[jackson.lee]

Hello Nicolas

Thanks for your review, we will review them.


> -----Original Message-----
> From: Nicolas Dufresne <nicolas@ndufresne.ca>
> Sent: Thursday, February 8, 2024 2:55 AM
> To: jackson.lee <jackson.lee@chipsnmedia.com>; mchehab@kernel.org; linux-
> media@vger.kernel.org; linux-kernel@vger.kernel.org; Nas Chung
> <nas.chung@chipsnmedia.com>
> Cc: lafley.kim <lafley.kim@chipsnmedia.com>; b-brnich@ti.com
> Subject: Re: [RESEND PATCH v0 1/5] wave5 : Support yuv422 input format for
> encoder.
> 
> Hi Jackson,
> 
> Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> > Encoder supports the following formats.
> > YUV422P, NV16, NV61, NV16M, NV61M
> >
> > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > ---
> >  .../chips-media/wave5/wave5-vpu-enc.c         | 79 ++++++++++++++++++-
> >  1 file changed, 76 insertions(+), 3 deletions(-)
> >
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > index f29cfa3af94a..0cb5bfb67258 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > @@ -70,6 +70,41 @@ static const struct vpu_format
> enc_fmt_list[FMT_TYPES][MAX_FMTS] = {
> >  			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> >  			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> >  		},
> > +		{
> > +			.v4l2_pix_fmt = V4L2_PIX_FMT_YUV422P,
> > +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> > +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> > +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> > +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> > +		},
> 
> During upstreaming, we discussed the lack of usage of v4l2-common in this
> driver and agreed that future updates such as this one should first port
> the driver to use the common helpers instead.
> 
> This implies dropping this custom made structure in favour of
> v4l2_frmsize_stepwise structure. Unlike yours, you can encoded the needed
> padding, allowing to encode this in one place instead of spreading it
> across numerous formulas in the code.
> 
> With this information, you will be able to use:
> 
>   v4l2_apply_frmsize_constraints()
>   v4l2_fill_pixfmt_mp()
> 
> To adjust your dimensions to padded dimensions and compute your
> bytesperline
> (stride) and sizeimage. You can of course increase the size image after
> this call. You can have a look at rkvdec driver as an example.
> 
> Please port existing set of pixel formats support, and then add the new
> pixel formats. This should remove about 3/4 of this patch and remove that
> huge risk of miss-computing a size.
> 
> > +		{
> > +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV16,
> > +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> > +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> > +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> > +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> > +		},
> > +		{
> > +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61,
> > +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> > +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> > +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> > +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> > +		},
> > +		{
> > +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV16M,
> > +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> > +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> > +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> > +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> > +		},
> > +		{
> > +			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61M,
> > +			.max_width = W5_MAX_ENC_PIC_WIDTH,
> > +			.min_width = W5_MIN_ENC_PIC_WIDTH,
> > +			.max_height = W5_MAX_ENC_PIC_HEIGHT,
> > +			.min_height = W5_MIN_ENC_PIC_HEIGHT,
> > +		},
> >  	}
> >  };
> >
> > @@ -136,6 +171,23 @@ static void wave5_update_pix_fmt(struct
> v4l2_pix_format_mplane *pix_mp, unsigned
> >  		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
> >  		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) *
> height / 2;
> >  		break;
> > +	case V4L2_PIX_FMT_YUV422P:
> > +	case V4L2_PIX_FMT_NV16:
> > +	case V4L2_PIX_FMT_NV61:
> > +		pix_mp->width = width;
> > +		pix_mp->height = height;
> > +		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> > +		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) *
> height * 2;
> > +		break;
> > +	case V4L2_PIX_FMT_NV16M:
> > +	case V4L2_PIX_FMT_NV61M:
> > +		pix_mp->width = width;
> > +		pix_mp->height = height;
> > +		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> > +		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) *
> height;
> > +		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
> > +		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) *
> height;
> > +		break;
> >  	default:
> >  		pix_mp->width = width;
> >  		pix_mp->height = height;
> > @@ -155,11 +207,19 @@ static int start_encode(struct vpu_instance *inst,
> u32 *fail_res)
> >  	struct enc_param pic_param;
> >  	u32 stride = ALIGN(inst->dst_fmt.width, 32);
> >  	u32 luma_size = (stride * inst->dst_fmt.height);
> > -	u32 chroma_size = ((stride / 2) * (inst->dst_fmt.height / 2));
> > +	u32 chroma_size;
> >
> >  	memset(&pic_param, 0, sizeof(struct enc_param));
> >  	memset(&frame_buf, 0, sizeof(struct frame_buffer));
> >
> > +	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV420 ||
> > +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV420M)
> > +		chroma_size = ((stride / 2) * (inst->dst_fmt.height / 2));
> > +	else if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV422P)
> > +		chroma_size = ((stride) * (inst->dst_fmt.height / 2));
> > +	else
> > +		chroma_size = 0;
> > +
> >  	dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
> >  	if (!dst_buf) {
> >  		dev_dbg(inst->dev->dev, "%s: No destination buffer found\n",
> > __func__); @@ -550,11 +610,15 @@ static int
> wave5_vpu_enc_s_fmt_out(struct file *file, void *fh, struct v4l2_form
> >  	}
> >
> >  	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12 ||
> > -	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12M) {
> > +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV12M ||
> > +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16 ||
> > +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16M) {
> >  		inst->cbcr_interleave = true;
> >  		inst->nv21 = false;
> >  	} else if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21 ||
> > -		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21M) {
> > +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV21M ||
> > +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61 ||
> > +		   inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61M) {
> >  		inst->cbcr_interleave = true;
> >  		inst->nv21 = true;
> >  	} else {
> > @@ -1132,6 +1196,15 @@ static void wave5_set_enc_openparam(struct
> enc_open_param *open_param,
> >  	u32 num_ctu_row = ALIGN(inst->dst_fmt.height, 64) / 64;
> >  	u32 num_mb_row = ALIGN(inst->dst_fmt.height, 16) / 16;
> >
> > +	if (inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16 ||
> > +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61 ||
> > +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_YUV422P ||
> > +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV16M ||
> > +	    inst->src_fmt.pixelformat == V4L2_PIX_FMT_NV61M)
> > +		open_param->src_format = FORMAT_422;
> > +	else
> > +		open_param->src_format = FORMAT_420;
> > +
> >  	open_param->wave_param.gop_preset_idx = PRESET_IDX_IPP_SINGLE;
> >  	open_param->wave_param.hvs_qp_scale = 2;
> >  	open_param->wave_param.hvs_max_delta_qp = 10;

[RESEND PATCH v0 2/5] wave5: Support to prepend sps/pps to IDR frame.

[jackson.lee]

Indicates whether to generate SPS and PPS at every IDR. Setting it to 0 disables generating SPS and PPS at every IDR.
Setting it to one enables generating SPS and PPS at every IDR.

Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
---
 drivers/media/platform/chips-media/wave5/wave5-hw.c      | 6 ++++--
 drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c | 7 +++++++
 drivers/media/platform/chips-media/wave5/wave5-vpuapi.h  | 1 +
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c
index f1e022fb148e..8ad7f3a28ae1 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
@@ -1602,11 +1602,13 @@ int wave5_vpu_enc_init_seq(struct vpu_instance *inst)
 	if (inst->std == W_AVC_ENC)
 		vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_INTRA_PARAM, p_param->intra_qp |
 				((p_param->intra_period & 0x7ff) << 6) |
-				((p_param->avc_idr_period & 0x7ff) << 17));
+				((p_param->avc_idr_period & 0x7ff) << 17) |
+				(p_param->forced_idr_header_enable << 28));
 	else if (inst->std == W_HEVC_ENC)
 		vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_INTRA_PARAM,
 			      p_param->decoding_refresh_type | (p_param->intra_qp << 3) |
-				(p_param->intra_period << 16));
+			      (p_param->forced_idr_header_enable << 9) |
+			      (p_param->intra_period << 16));
 
 	reg_val = (p_param->rdo_skip << 2) |
 		(p_param->lambda_scaling_enable << 3) |
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
index 0cb5bfb67258..761775216cd4 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
@@ -1125,6 +1125,9 @@ static int wave5_vpu_enc_s_ctrl(struct v4l2_ctrl *ctrl)
 	case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
 		inst->enc_param.entropy_coding_mode = ctrl->val;
 		break;
+	case V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR:
+		inst->enc_param.forced_idr_header_enable = ctrl->val;
+		break;
 	case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT:
 		break;
 	default:
@@ -1292,6 +1295,7 @@ static void wave5_set_enc_openparam(struct enc_open_param *open_param,
 		else
 			open_param->wave_param.intra_refresh_arg = num_ctu_row;
 	}
+	open_param->wave_param.forced_idr_header_enable = input.forced_idr_header_enable;
 }
 
 static int initialize_sequence(struct vpu_instance *inst)
@@ -1775,6 +1779,9 @@ static int wave5_vpu_open_enc(struct file *filp)
 			  0, 1, 1, 0);
 	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
 			  V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, 1);
+	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR,
+			  0, 1, 1, 0);
 
 	if (v4l2_ctrl_hdl->error) {
 		ret = -ENODEV;
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
index 352f6e904e50..3ad6118550ac 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
@@ -566,6 +566,7 @@ struct enc_wave_param {
 	u32 lambda_scaling_enable: 1; /* enable lambda scaling using custom GOP */
 	u32 transform8x8_enable: 1; /* enable 8x8 intra prediction and 8x8 transform */
 	u32 mb_level_rc_enable: 1; /* enable MB-level rate control */
+	u32 forced_idr_header_enable: 1; /* enable header encoding before IDR frame */
 };
 
 struct enc_open_param {
-- 
2.43.0

Re: [RESEND PATCH v0 2/5] wave5: Support to prepend sps/pps to IDR frame.

[Nicolas Dufresne]

Hi Jackson,

Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> Indicates whether to generate SPS and PPS at every IDR. Setting it to 0 disables generating SPS and PPS at every IDR.
> Setting it to one enables generating SPS and PPS at every IDR.
> 
> Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> ---
>  drivers/media/platform/chips-media/wave5/wave5-hw.c      | 6 ++++--
>  drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c | 7 +++++++
>  drivers/media/platform/chips-media/wave5/wave5-vpuapi.h  | 1 +
>  3 files changed, 12 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> index f1e022fb148e..8ad7f3a28ae1 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> @@ -1602,11 +1602,13 @@ int wave5_vpu_enc_init_seq(struct vpu_instance *inst)
>  	if (inst->std == W_AVC_ENC)
>  		vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_INTRA_PARAM, p_param->intra_qp |
>  				((p_param->intra_period & 0x7ff) << 6) |
> -				((p_param->avc_idr_period & 0x7ff) << 17));
> +				((p_param->avc_idr_period & 0x7ff) << 17) |
> +				(p_param->forced_idr_header_enable << 28));

I can spot evident hard-coding of mask and bit shifts in here. In order to
continuously improve this driver code, I would like to see this (and the
following) magic number being defined with well named macros as a preparation
patch to this feature change.

regards,
Nicolas

>  	else if (inst->std == W_HEVC_ENC)
>  		vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_INTRA_PARAM,
>  			      p_param->decoding_refresh_type | (p_param->intra_qp << 3) |
> -				(p_param->intra_period << 16));
> +			      (p_param->forced_idr_header_enable << 9) |
> +			      (p_param->intra_period << 16));
>  
>  	reg_val = (p_param->rdo_skip << 2) |
>  		(p_param->lambda_scaling_enable << 3) |
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> index 0cb5bfb67258..761775216cd4 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> @@ -1125,6 +1125,9 @@ static int wave5_vpu_enc_s_ctrl(struct v4l2_ctrl *ctrl)
>  	case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
>  		inst->enc_param.entropy_coding_mode = ctrl->val;
>  		break;
> +	case V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR:
> +		inst->enc_param.forced_idr_header_enable = ctrl->val;
> +		break;
>  	case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT:
>  		break;
>  	default:
> @@ -1292,6 +1295,7 @@ static void wave5_set_enc_openparam(struct enc_open_param *open_param,
>  		else
>  			open_param->wave_param.intra_refresh_arg = num_ctu_row;
>  	}
> +	open_param->wave_param.forced_idr_header_enable = input.forced_idr_header_enable;
>  }
>  
>  static int initialize_sequence(struct vpu_instance *inst)
> @@ -1775,6 +1779,9 @@ static int wave5_vpu_open_enc(struct file *filp)
>  			  0, 1, 1, 0);
>  	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
>  			  V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, 1);
> +	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
> +			  V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR,
> +			  0, 1, 1, 0);
>  
>  	if (v4l2_ctrl_hdl->error) {
>  		ret = -ENODEV;
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
> index 352f6e904e50..3ad6118550ac 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
> @@ -566,6 +566,7 @@ struct enc_wave_param {
>  	u32 lambda_scaling_enable: 1; /* enable lambda scaling using custom GOP */
>  	u32 transform8x8_enable: 1; /* enable 8x8 intra prediction and 8x8 transform */
>  	u32 mb_level_rc_enable: 1; /* enable MB-level rate control */
> +	u32 forced_idr_header_enable: 1; /* enable header encoding before IDR frame */
>  };
>  
>  struct enc_open_param {

Re: [RESEND PATCH v0 2/5] wave5: Support to prepend sps/pps to IDR frame.

[Sebastian Fricke]

Hey Jackson,

as with the previous review, the title needs to be adjusted 'wave5:' is
not enough.
Also 'Support to prepend sps/pps to IDR' sounds a bit weird and doesn't
quite match what you describe below.
How about:
'Support SPS/PPS generation for each IDR'

On 07.02.2024 13:00, Nicolas Dufresne wrote:
>Hi Jackson,
>
>Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
>> Indicates whether to generate SPS and PPS at every IDR. Setting it to 0 disables generating SPS and PPS at every IDR.
>> Setting it to one enables generating SPS and PPS at every IDR.

My suggestion:

Provide a control to toggle (0 = off / 1 = on), whether the SPS and PPS
are generated for every IDR.

Greetings,
Sebastian

>>
>> Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
>> Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
>> ---
>>  drivers/media/platform/chips-media/wave5/wave5-hw.c      | 6 ++++--
>>  drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c | 7 +++++++
>>  drivers/media/platform/chips-media/wave5/wave5-vpuapi.h  | 1 +
>>  3 files changed, 12 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c
>> index f1e022fb148e..8ad7f3a28ae1 100644
>> --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
>> +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
>> @@ -1602,11 +1602,13 @@ int wave5_vpu_enc_init_seq(struct vpu_instance *inst)
>>  	if (inst->std == W_AVC_ENC)
>>  		vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_INTRA_PARAM, p_param->intra_qp |
>>  				((p_param->intra_period & 0x7ff) << 6) |
>> -				((p_param->avc_idr_period & 0x7ff) << 17));
>> +				((p_param->avc_idr_period & 0x7ff) << 17) |
>> +				(p_param->forced_idr_header_enable << 28));
>
>I can spot evident hard-coding of mask and bit shifts in here. In order to
>continuously improve this driver code, I would like to see this (and the
>following) magic number being defined with well named macros as a preparation
>patch to this feature change.
>
>regards,
>Nicolas
>
>>  	else if (inst->std == W_HEVC_ENC)
>>  		vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_INTRA_PARAM,
>>  			      p_param->decoding_refresh_type | (p_param->intra_qp << 3) |
>> -				(p_param->intra_period << 16));
>> +			      (p_param->forced_idr_header_enable << 9) |
>> +			      (p_param->intra_period << 16));
>>
>>  	reg_val = (p_param->rdo_skip << 2) |
>>  		(p_param->lambda_scaling_enable << 3) |
>> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
>> index 0cb5bfb67258..761775216cd4 100644
>> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
>> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
>> @@ -1125,6 +1125,9 @@ static int wave5_vpu_enc_s_ctrl(struct v4l2_ctrl *ctrl)
>>  	case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
>>  		inst->enc_param.entropy_coding_mode = ctrl->val;
>>  		break;
>> +	case V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR:
>> +		inst->enc_param.forced_idr_header_enable = ctrl->val;
>> +		break;
>>  	case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT:
>>  		break;
>>  	default:
>> @@ -1292,6 +1295,7 @@ static void wave5_set_enc_openparam(struct enc_open_param *open_param,
>>  		else
>>  			open_param->wave_param.intra_refresh_arg = num_ctu_row;
>>  	}
>> +	open_param->wave_param.forced_idr_header_enable = input.forced_idr_header_enable;
>>  }
>>
>>  static int initialize_sequence(struct vpu_instance *inst)
>> @@ -1775,6 +1779,9 @@ static int wave5_vpu_open_enc(struct file *filp)
>>  			  0, 1, 1, 0);
>>  	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
>>  			  V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, 1);
>> +	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
>> +			  V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR,
>> +			  0, 1, 1, 0);
>>
>>  	if (v4l2_ctrl_hdl->error) {
>>  		ret = -ENODEV;
>> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
>> index 352f6e904e50..3ad6118550ac 100644
>> --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
>> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
>> @@ -566,6 +566,7 @@ struct enc_wave_param {
>>  	u32 lambda_scaling_enable: 1; /* enable lambda scaling using custom GOP */
>>  	u32 transform8x8_enable: 1; /* enable 8x8 intra prediction and 8x8 transform */
>>  	u32 mb_level_rc_enable: 1; /* enable MB-level rate control */
>> +	u32 forced_idr_header_enable: 1; /* enable header encoding before IDR frame */
>>  };
>>
>>  struct enc_open_param {
>
>

RE: [RESEND PATCH v0 2/5] wave5: Support to prepend sps/pps to IDR frame.

[jackson.lee]

Hello Sebastian

Thanks your review, we will review your advice.

> -----Original Message-----
> From: Sebastian Fricke <sebastian.fricke@collabora.com>
> Sent: Thursday, February 8, 2024 7:02 PM
> To: Nicolas Dufresne <nicolas@ndufresne.ca>
> Cc: jackson.lee <jackson.lee@chipsnmedia.com>; mchehab@kernel.org; linux-
> media@vger.kernel.org; linux-kernel@vger.kernel.org; Nas Chung
> <nas.chung@chipsnmedia.com>; lafley.kim <lafley.kim@chipsnmedia.com>; b-
> brnich@ti.com
> Subject: Re: [RESEND PATCH v0 2/5] wave5: Support to prepend sps/pps to
> IDR frame.
> 
> Hey Jackson,
> 
> as with the previous review, the title needs to be adjusted 'wave5:' is
> not enough.
> Also 'Support to prepend sps/pps to IDR' sounds a bit weird and doesn't
> quite match what you describe below.
> How about:
> 'Support SPS/PPS generation for each IDR'
> 
> On 07.02.2024 13:00, Nicolas Dufresne wrote:
> >Hi Jackson,
> >
> >Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> >> Indicates whether to generate SPS and PPS at every IDR. Setting it to 0
> disables generating SPS and PPS at every IDR.
> >> Setting it to one enables generating SPS and PPS at every IDR.
> 
> My suggestion:
> 
> Provide a control to toggle (0 = off / 1 = on), whether the SPS and PPS
> are generated for every IDR.
> 
> Greetings,
> Sebastian
> 
> >>
> >> Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> >> Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> >> ---
> >>  drivers/media/platform/chips-media/wave5/wave5-hw.c      | 6 ++++--
> >>  drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c | 7 +++++++
> >> drivers/media/platform/chips-media/wave5/wave5-vpuapi.h  | 1 +
> >>  3 files changed, 12 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> >> b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> >> index f1e022fb148e..8ad7f3a28ae1 100644
> >> --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> >> +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> >> @@ -1602,11 +1602,13 @@ int wave5_vpu_enc_init_seq(struct vpu_instance
> *inst)
> >>  	if (inst->std == W_AVC_ENC)
> >>  		vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_INTRA_PARAM,
> p_param->intra_qp |
> >>  				((p_param->intra_period & 0x7ff) << 6) |
> >> -				((p_param->avc_idr_period & 0x7ff) << 17));
> >> +				((p_param->avc_idr_period & 0x7ff) << 17) |
> >> +				(p_param->forced_idr_header_enable << 28));
> >
> >I can spot evident hard-coding of mask and bit shifts in here. In order
> >to continuously improve this driver code, I would like to see this (and
> >the
> >following) magic number being defined with well named macros as a
> >preparation patch to this feature change.
> >
> >regards,
> >Nicolas
> >
> >>  	else if (inst->std == W_HEVC_ENC)
> >>  		vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_INTRA_PARAM,
> >>  			      p_param->decoding_refresh_type | (p_param-
> >intra_qp << 3) |
> >> -				(p_param->intra_period << 16));
> >> +			      (p_param->forced_idr_header_enable << 9) |
> >> +			      (p_param->intra_period << 16));
> >>
> >>  	reg_val = (p_param->rdo_skip << 2) |
> >>  		(p_param->lambda_scaling_enable << 3) | diff --git
> >> a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> >> b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> >> index 0cb5bfb67258..761775216cd4 100644
> >> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> >> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> >> @@ -1125,6 +1125,9 @@ static int wave5_vpu_enc_s_ctrl(struct v4l2_ctrl
> *ctrl)
> >>  	case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE:
> >>  		inst->enc_param.entropy_coding_mode = ctrl->val;
> >>  		break;
> >> +	case V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR:
> >> +		inst->enc_param.forced_idr_header_enable = ctrl->val;
> >> +		break;
> >>  	case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT:
> >>  		break;
> >>  	default:
> >> @@ -1292,6 +1295,7 @@ static void wave5_set_enc_openparam(struct
> enc_open_param *open_param,
> >>  		else
> >>  			open_param->wave_param.intra_refresh_arg = num_ctu_row;
> >>  	}
> >> +	open_param->wave_param.forced_idr_header_enable =
> >> +input.forced_idr_header_enable;
> >>  }
> >>
> >>  static int initialize_sequence(struct vpu_instance *inst) @@ -1775,6
> >> +1779,9 @@ static int wave5_vpu_open_enc(struct file *filp)
> >>  			  0, 1, 1, 0);
> >>  	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
> >>  			  V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 32, 1, 1);
> >> +	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
> >> +			  V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR,
> >> +			  0, 1, 1, 0);
> >>
> >>  	if (v4l2_ctrl_hdl->error) {
> >>  		ret = -ENODEV;
> >> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
> >> b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
> >> index 352f6e904e50..3ad6118550ac 100644
> >> --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
> >> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
> >> @@ -566,6 +566,7 @@ struct enc_wave_param {
> >>  	u32 lambda_scaling_enable: 1; /* enable lambda scaling using custom
> GOP */
> >>  	u32 transform8x8_enable: 1; /* enable 8x8 intra prediction and 8x8
> transform */
> >>  	u32 mb_level_rc_enable: 1; /* enable MB-level rate control */
> >> +	u32 forced_idr_header_enable: 1; /* enable header encoding before
> >> +IDR frame */
> >>  };
> >>
> >>  struct enc_open_param {
> >
> >

[RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

There are two device run-time PM callbacks defined in 'struct dev_pm_ops'
int (*runtime_suspend)(struct device *dev);
int (*runtime_resume)(struct device *dev);

Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
---
 .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
 .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
 .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
 .../platform/chips-media/wave5/wave5-vpu.c    | 68 +++++++++++++++++++
 .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
 .../media/platform/chips-media/wave5/wave5.h  |  3 +
 6 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c
index 8ad7f3a28ae1..8aade5a38439 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
@@ -503,6 +503,7 @@ int wave5_vpu_build_up_dec_param(struct vpu_instance *inst,
 	/* This register must be reset explicitly */
 	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
 	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1, (COMMAND_QUEUE_DEPTH - 1));
+	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
 
 	ret = send_firmware_command(inst, W5_CREATE_INSTANCE, true, NULL, NULL);
 	if (ret) {
@@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device *dev, u8 *fw, size_t size)
 	return setup_wave5_properties(dev);
 }
 
-static int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
-				size_t size)
+int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
+			 size_t size)
 {
 	u32 reg_val;
 	struct vpu_buf *common_vb;
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
index ef227af72348..328a7a8f26c5 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2021-2023 CHIPS&MEDIA INC
  */
 
+#include <linux/pm_runtime.h>
 #include "wave5-helper.h"
 
 #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
@@ -1387,9 +1388,17 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
 
 	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && inst->state == VPU_INST_STATE_NONE) {
 		struct dec_open_param open_param;
+		int err = 0;
 
 		memset(&open_param, 0, sizeof(struct dec_open_param));
 
+		err = pm_runtime_resume_and_get(inst->dev->dev);
+		if (err) {
+			dev_err(inst->dev->dev, "decoder runtime resume failed %d\n", err);
+			ret = -EINVAL;
+			goto return_buffers;
+		}
+
 		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
 		if (ret)
 			goto return_buffers;
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
index 761775216cd4..ff73d69de41c 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2021-2023 CHIPS&MEDIA INC
  */
 
+#include <linux/pm_runtime.h>
 #include "wave5-helper.h"
 
 #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
@@ -1387,9 +1388,17 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count
 
 	if (inst->state == VPU_INST_STATE_NONE && q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
 		struct enc_open_param open_param;
+		int err = 0;
 
 		memset(&open_param, 0, sizeof(struct enc_open_param));
 
+		err = pm_runtime_resume_and_get(inst->dev->dev);
+		if (err) {
+			dev_err(inst->dev->dev, "encoder runtime resume failed %d\n", err);
+			ret = -EINVAL;
+			goto return_buffers;
+		}
+
 		wave5_set_enc_openparam(&open_param, inst);
 
 		ret = wave5_vpu_enc_open(inst, &open_param);
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu.c b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
index 0d90b5820bef..f81409740a56 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
@@ -10,6 +10,7 @@
 #include <linux/clk.h>
 #include <linux/firmware.h>
 #include <linux/interrupt.h>
+#include <linux/pm_runtime.h>
 #include "wave5-vpu.h"
 #include "wave5-regdefine.h"
 #include "wave5-vpuconfig.h"
@@ -117,6 +118,65 @@ static int wave5_vpu_load_firmware(struct device *dev, const char *fw_name,
 	return 0;
 }
 
+static __maybe_unused int wave5_pm_suspend(struct device *dev)
+{
+	struct vpu_device *vpu = dev_get_drvdata(dev);
+
+	if (pm_runtime_suspended(dev))
+		return 0;
+
+	wave5_vpu_sleep_wake(dev, true, NULL, 0);
+	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
+
+	return 0;
+}
+
+static __maybe_unused int wave5_pm_resume(struct device *dev)
+{
+	struct vpu_device *vpu = dev_get_drvdata(dev);
+	int ret = 0;
+
+	wave5_vpu_sleep_wake(dev, false, NULL, 0);
+	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
+	if (ret) {
+		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
+		return ret;
+	}
+
+	return ret;
+}
+
+static __maybe_unused int wave5_suspend(struct device *dev)
+{
+	struct vpu_device *vpu = dev_get_drvdata(dev);
+	struct vpu_instance *inst;
+
+	list_for_each_entry(inst, &vpu->instances, list)
+		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
+
+	return pm_runtime_force_suspend(dev);
+}
+
+static __maybe_unused int wave5_resume(struct device *dev)
+{
+	struct vpu_device *vpu = dev_get_drvdata(dev);
+	struct vpu_instance *inst;
+	int ret = 0;
+
+	ret = pm_runtime_force_resume(dev);
+	if (ret < 0)
+		return ret;
+
+	list_for_each_entry(inst, &vpu->instances, list)
+		v4l2_m2m_resume(inst->v4l2_m2m_dev);
+
+	return ret;
+}
+
+static const struct dev_pm_ops wave5_pm_ops = {
+	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume, NULL)
+};
+
 static int wave5_vpu_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct platform_device *pdev)
 		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
 	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev->product_code);
 	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
+
+	pm_runtime_enable(&pdev->dev);
+	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
+
 	return 0;
 
 err_enc_unreg:
@@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct platform_device *pdev)
 {
 	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
 
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
 	mutex_destroy(&dev->dev_lock);
 	mutex_destroy(&dev->hw_lock);
 	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
@@ -281,6 +348,7 @@ static struct platform_driver wave5_vpu_driver = {
 	.driver = {
 		.name = VPU_PLATFORM_DEVICE_NAME,
 		.of_match_table = of_match_ptr(wave5_dt_ids),
+		.pm = &wave5_pm_ops,
 		},
 	.probe = wave5_vpu_probe,
 	.remove = wave5_vpu_remove,
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
index 1a3efb638dde..f1f8e4fc8474 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/bug.h>
+#include <linux/pm_runtime.h>
 #include "wave5-vpuapi.h"
 #include "wave5-regdefine.h"
 #include "wave5.h"
@@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res)
 
 	wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task);
 
+	if (!pm_runtime_suspended(inst->dev->dev))
+		pm_runtime_put_sync(inst->dev->dev);
+
 unlock_and_return:
 	mutex_unlock(&vpu_dev->hw_lock);
 
@@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res)
 
 	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
 
+	if (!pm_runtime_suspended(inst->dev->dev))
+		pm_runtime_put_sync(inst->dev->dev);
+
 	mutex_unlock(&vpu_dev->hw_lock);
 
 	return 0;
diff --git a/drivers/media/platform/chips-media/wave5/wave5.h b/drivers/media/platform/chips-media/wave5/wave5.h
index 063028eccd3b..6125eff938a8 100644
--- a/drivers/media/platform/chips-media/wave5/wave5.h
+++ b/drivers/media/platform/chips-media/wave5/wave5.h
@@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct vpu_device *vpu_dev, u32 *revision);
 
 int wave5_vpu_init(struct device *dev, u8 *fw, size_t size);
 
+int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
+			 size_t size);
+
 int wave5_vpu_reset(struct device *dev, enum sw_reset_mode reset_mode);
 
 int wave5_vpu_build_up_dec_param(struct vpu_instance *inst, struct dec_open_param *param);
-- 
2.43.0

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

Hi Jackson,

Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> There are two device run-time PM callbacks defined in 'struct dev_pm_ops'
> int (*runtime_suspend)(struct device *dev);
> int (*runtime_resume)(struct device *dev);

I wonder how useful is it to teach everyone what the generic 'struct dev_pm_ops'
contains. Perhaps you simply wanted that this patch implement both suspend and
resume ops ?

> 
> Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> ---
>  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
>  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
>  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
>  .../platform/chips-media/wave5/wave5-vpu.c    | 68 +++++++++++++++++++
>  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
>  .../media/platform/chips-media/wave5/wave5.h  |  3 +
>  6 files changed, 99 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> index 8ad7f3a28ae1..8aade5a38439 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> @@ -503,6 +503,7 @@ int wave5_vpu_build_up_dec_param(struct vpu_instance *inst,
>  	/* This register must be reset explicitly */
>  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
>  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1, (COMMAND_QUEUE_DEPTH - 1));
> +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);

In some way, the relation between suspend and this register write is not
obvious. If its not related, please do this in its own patch. Otherwise you want
to explain why you needed this (possibly just in the commit message).

>  
>  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE, true, NULL, NULL);
>  	if (ret) {
> @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device *dev, u8 *fw, size_t size)
>  	return setup_wave5_properties(dev);
>  }
>  
> -static int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
> -				size_t size)
> +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
> +			 size_t size)
>  {
>  	u32 reg_val;
>  	struct vpu_buf *common_vb;
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> index ef227af72348..328a7a8f26c5 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> @@ -5,6 +5,7 @@
>   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
>   */
>  
> +#include <linux/pm_runtime.h>
>  #include "wave5-helper.h"
>  
>  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> @@ -1387,9 +1388,17 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
>  
>  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && inst->state == VPU_INST_STATE_NONE) {
>  		struct dec_open_param open_param;
> +		int err = 0;
>  
>  		memset(&open_param, 0, sizeof(struct dec_open_param));
>  
> +		err = pm_runtime_resume_and_get(inst->dev->dev);
> +		if (err) {
> +			dev_err(inst->dev->dev, "decoder runtime resume failed %d\n", err);
> +			ret = -EINVAL;
> +			goto return_buffers;
> +		}
> +
>  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
>  		if (ret)
>  			goto return_buffers;
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> index 761775216cd4..ff73d69de41c 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> @@ -5,6 +5,7 @@
>   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
>   */
>  
> +#include <linux/pm_runtime.h>
>  #include "wave5-helper.h"
>  
>  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> @@ -1387,9 +1388,17 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count
>  
>  	if (inst->state == VPU_INST_STATE_NONE && q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
>  		struct enc_open_param open_param;
> +		int err = 0;
>  
>  		memset(&open_param, 0, sizeof(struct enc_open_param));
>  
> +		err = pm_runtime_resume_and_get(inst->dev->dev);
> +		if (err) {
> +			dev_err(inst->dev->dev, "encoder runtime resume failed %d\n", err);
> +			ret = -EINVAL;
> +			goto return_buffers;
> +		}
> +
>  		wave5_set_enc_openparam(&open_param, inst);
>  
>  		ret = wave5_vpu_enc_open(inst, &open_param);
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu.c b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> index 0d90b5820bef..f81409740a56 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> @@ -10,6 +10,7 @@
>  #include <linux/clk.h>
>  #include <linux/firmware.h>
>  #include <linux/interrupt.h>
> +#include <linux/pm_runtime.h>
>  #include "wave5-vpu.h"
>  #include "wave5-regdefine.h"
>  #include "wave5-vpuconfig.h"
> @@ -117,6 +118,65 @@ static int wave5_vpu_load_firmware(struct device *dev, const char *fw_name,
>  	return 0;
>  }
>  
> +static __maybe_unused int wave5_pm_suspend(struct device *dev)
> +{
> +	struct vpu_device *vpu = dev_get_drvdata(dev);
> +
> +	if (pm_runtime_suspended(dev))
> +		return 0;
> +
> +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> +
> +	return 0;
> +}
> +
> +static __maybe_unused int wave5_pm_resume(struct device *dev)
> +{
> +	struct vpu_device *vpu = dev_get_drvdata(dev);
> +	int ret = 0;
> +
> +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> +	if (ret) {
> +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> +		return ret;
> +	}
> +
> +	return ret;
> +}
> +
> +static __maybe_unused int wave5_suspend(struct device *dev)
> +{
> +	struct vpu_device *vpu = dev_get_drvdata(dev);
> +	struct vpu_instance *inst;
> +
> +	list_for_each_entry(inst, &vpu->instances, list)
> +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> +
> +	return pm_runtime_force_suspend(dev);
> +}
> +
> +static __maybe_unused int wave5_resume(struct device *dev)
> +{
> +	struct vpu_device *vpu = dev_get_drvdata(dev);
> +	struct vpu_instance *inst;
> +	int ret = 0;
> +
> +	ret = pm_runtime_force_resume(dev);
> +	if (ret < 0)
> +		return ret;
> +
> +	list_for_each_entry(inst, &vpu->instances, list)
> +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> +
> +	return ret;
> +}

The functions wave5_suspend() and wave5_resume() are not just "maybe_unsued" but
actually never used. What was the intention ? Considering the usage of
__maybe_unused has been such a bad friend for this one, could you instead
bracket the functions with an explicit ?

#ifdef CONFIG_PM
#endif

This way the compiler will have a word if you implement something that you
forgot to actually use.

> +
> +static const struct dev_pm_ops wave5_pm_ops = {
> +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume, NULL)
> +};
> +
>  static int wave5_vpu_probe(struct platform_device *pdev)
>  {
>  	int ret;
> @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct platform_device *pdev)
>  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
>  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev->product_code);
>  	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
> +
> +	pm_runtime_enable(&pdev->dev);
> +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> +
>  	return 0;
>  
>  err_enc_unreg:
> @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct platform_device *pdev)
>  {
>  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
>  
> +	pm_runtime_put_sync(&pdev->dev);
> +	pm_runtime_disable(&pdev->dev);
> +
>  	mutex_destroy(&dev->dev_lock);
>  	mutex_destroy(&dev->hw_lock);
>  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
> @@ -281,6 +348,7 @@ static struct platform_driver wave5_vpu_driver = {
>  	.driver = {
>  		.name = VPU_PLATFORM_DEVICE_NAME,
>  		.of_match_table = of_match_ptr(wave5_dt_ids),
> +		.pm = &wave5_pm_ops,
>  		},
>  	.probe = wave5_vpu_probe,
>  	.remove = wave5_vpu_remove,
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> index 1a3efb638dde..f1f8e4fc8474 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> @@ -6,6 +6,7 @@
>   */
>  
>  #include <linux/bug.h>
> +#include <linux/pm_runtime.h>
>  #include "wave5-vpuapi.h"
>  #include "wave5-regdefine.h"
>  #include "wave5.h"
> @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res)
>  
>  	wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task);
>  
> +	if (!pm_runtime_suspended(inst->dev->dev))
> +		pm_runtime_put_sync(inst->dev->dev);
> +
>  unlock_and_return:
>  	mutex_unlock(&vpu_dev->hw_lock);
>  
> @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res)
>  
>  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
>  
> +	if (!pm_runtime_suspended(inst->dev->dev))
> +		pm_runtime_put_sync(inst->dev->dev);

This seems very unnatural. We do the get() in "start_streaming()", but the put()
is only done when the device is closed, or when the driver is removed. As this
is not balanced, you seem to have to check the suspended condition all over the
place.

I think we could aim for start_streaming()/stop_streaming() for your get/put
placement. At least they will be bound to an entirely balanced API. But then, a
media player in paused sate will prevent that device from being suspended.

If the HW is capable of preserving enough state, and From the short doc I have
it gives me the impression it can preserve that, I'd suggest to follow what
hantro driver is doing. What is does is that it will do get() in device_run(),
and put() whenever the job completes. This driver has been designed so when
there is no job, it means the firmware is currently idle and looking for more
work. So it seems like the perfect moment to do suspend it.

Nicolas

> +
>  	mutex_unlock(&vpu_dev->hw_lock);
>  
>  	return 0;
> diff --git a/drivers/media/platform/chips-media/wave5/wave5.h b/drivers/media/platform/chips-media/wave5/wave5.h
> index 063028eccd3b..6125eff938a8 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5.h
> +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct vpu_device *vpu_dev, u32 *revision);
>  
>  int wave5_vpu_init(struct device *dev, u8 *fw, size_t size);
>  
> +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
> +			 size_t size);
> +
>  int wave5_vpu_reset(struct device *dev, enum sw_reset_mode reset_mode);
>  
>  int wave5_vpu_build_up_dec_param(struct vpu_instance *inst, struct dec_open_param *param);

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

Hi Nicolas


> This seems very unnatural. We do the get() in "start_streaming()", but the
> put() is only done when the device is closed, or when the driver is
> removed. As this is not balanced, you seem to have to check the suspended
> condition all over the place.
> 
> I think we could aim for start_streaming()/stop_streaming() for your
> get/put placement. At least they will be bound to an entirely balanced API.
> But then, a media player in paused sate will prevent that device from
> being suspended.
> 
> If the HW is capable of preserving enough state, and From the short doc I
> have it gives me the impression it can preserve that, I'd suggest to
> follow what hantro driver is doing. What is does is that it will do get()
> in device_run(), and put() whenever the job completes. This driver has
> been designed so when there is no job, it means the firmware is currently
> idle and looking for more work. So it seems like the perfect moment to do
> suspend it.
>

Thanks your comment,

Currently they are not balanced, 
If we puts "the put functon" into the stop_streaming, our hw is stalled until doing wake-up command, so our v4l2 device become block.
so I'd like to update the below instead of calling get at the start_streaming function.


@@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct file *filp)

        wave5_vdi_allocate_sram(inst->dev);

+       err = pm_runtime_resume_and_get(inst->dev->dev);
+       if (err) {
+               dev_err(inst->dev->dev, "decoder runtime resume failed %d\n", err);
+               ret = -EINVAL;
+               goto cleanup_inst;
+       }
+
        return 0;




> Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> > There are two device run-time PM callbacks defined in 'struct
> dev_pm_ops'
> > int (*runtime_suspend)(struct device *dev); int
> > (*runtime_resume)(struct device *dev);
> 
> I wonder how useful is it to teach everyone what the generic 'struct
> dev_pm_ops'
> contains. Perhaps you simply wanted that this patch implement both suspend
> and resume ops ?
> 
> >
> > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > ---
> >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> >  .../platform/chips-media/wave5/wave5-vpu.c    | 68 +++++++++++++++++++
> >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> >  6 files changed, 99 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > index 8ad7f3a28ae1..8aade5a38439 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > @@ -503,6 +503,7 @@ int wave5_vpu_build_up_dec_param(struct vpu_instance
> *inst,
> >  	/* This register must be reset explicitly */
> >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > (COMMAND_QUEUE_DEPTH - 1));
> > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> 
> In some way, the relation between suspend and this register write is not
> obvious. If its not related, please do this in its own patch. Otherwise
> you want to explain why you needed this (possibly just in the commit
> message).
> 
> >
> >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE, true, NULL,
> NULL);
> >  	if (ret) {
> > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device *dev, u8 *fw,
> size_t size)
> >  	return setup_wave5_properties(dev);
> >  }
> >
> > -static int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake,
> const uint16_t *code,
> > -				size_t size)
> > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const
> uint16_t *code,
> > +			 size_t size)
> >  {
> >  	u32 reg_val;
> >  	struct vpu_buf *common_vb;
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > index ef227af72348..328a7a8f26c5 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > @@ -5,6 +5,7 @@
> >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> >   */
> >
> > +#include <linux/pm_runtime.h>
> >  #include "wave5-helper.h"
> >
> >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > @@ -1387,9 +1388,17 @@ static int wave5_vpu_dec_start_streaming(struct
> > vb2_queue *q, unsigned int count
> >
> >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && inst->state ==
> VPU_INST_STATE_NONE) {
> >  		struct dec_open_param open_param;
> > +		int err = 0;
> >
> >  		memset(&open_param, 0, sizeof(struct dec_open_param));
> >
> > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > +		if (err) {
> > +			dev_err(inst->dev->dev, "decoder runtime resume
> failed %d\n", err);
> > +			ret = -EINVAL;
> > +			goto return_buffers;
> > +		}
> > +
> >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> >  		if (ret)
> >  			goto return_buffers;
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > index 761775216cd4..ff73d69de41c 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > @@ -5,6 +5,7 @@
> >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> >   */
> >
> > +#include <linux/pm_runtime.h>
> >  #include "wave5-helper.h"
> >
> >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > @@ -1387,9 +1388,17 @@ static int wave5_vpu_enc_start_streaming(struct
> > vb2_queue *q, unsigned int count
> >
> >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> >  		struct enc_open_param open_param;
> > +		int err = 0;
> >
> >  		memset(&open_param, 0, sizeof(struct enc_open_param));
> >
> > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > +		if (err) {
> > +			dev_err(inst->dev->dev, "encoder runtime resume
> failed %d\n", err);
> > +			ret = -EINVAL;
> > +			goto return_buffers;
> > +		}
> > +
> >  		wave5_set_enc_openparam(&open_param, inst);
> >
> >  		ret = wave5_vpu_enc_open(inst, &open_param); diff --git
> > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > index 0d90b5820bef..f81409740a56 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > @@ -10,6 +10,7 @@
> >  #include <linux/clk.h>
> >  #include <linux/firmware.h>
> >  #include <linux/interrupt.h>
> > +#include <linux/pm_runtime.h>
> >  #include "wave5-vpu.h"
> >  #include "wave5-regdefine.h"
> >  #include "wave5-vpuconfig.h"
> > @@ -117,6 +118,65 @@ static int wave5_vpu_load_firmware(struct device
> *dev, const char *fw_name,
> >  	return 0;
> >  }
> >
> > +static __maybe_unused int wave5_pm_suspend(struct device *dev) {
> > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > +
> > +	if (pm_runtime_suspended(dev))
> > +		return 0;
> > +
> > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > +
> > +	return 0;
> > +}
> > +
> > +static __maybe_unused int wave5_pm_resume(struct device *dev) {
> > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > +	int ret = 0;
> > +
> > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> > +	if (ret) {
> > +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> > +		return ret;
> > +	}
> > +
> > +	return ret;
> > +}
> > +
> > +static __maybe_unused int wave5_suspend(struct device *dev) {
> > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > +	struct vpu_instance *inst;
> > +
> > +	list_for_each_entry(inst, &vpu->instances, list)
> > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > +
> > +	return pm_runtime_force_suspend(dev); }
> > +
> > +static __maybe_unused int wave5_resume(struct device *dev) {
> > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > +	struct vpu_instance *inst;
> > +	int ret = 0;
> > +
> > +	ret = pm_runtime_force_resume(dev);
> > +	if (ret < 0)
> > +		return ret;
> > +
> > +	list_for_each_entry(inst, &vpu->instances, list)
> > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > +
> > +	return ret;
> > +}
> 
> The functions wave5_suspend() and wave5_resume() are not just
> "maybe_unsued" but actually never used. What was the intention ?
> Considering the usage of __maybe_unused has been such a bad friend for
> this one, could you instead bracket the functions with an explicit ?
> 
> #ifdef CONFIG_PM
> #endif
> 
> This way the compiler will have a word if you implement something that you
> forgot to actually use.
> 
> > +
> > +static const struct dev_pm_ops wave5_pm_ops = {
> > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume, NULL) };
> > +
> >  static int wave5_vpu_probe(struct platform_device *pdev)  {
> >  	int ret;
> > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct platform_device
> *pdev)
> >  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
> >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev->product_code);
> >  	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
> > +
> > +	pm_runtime_enable(&pdev->dev);
> > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > +
> >  	return 0;
> >
> >  err_enc_unreg:
> > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct platform_device
> > *pdev)  {
> >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> >
> > +	pm_runtime_put_sync(&pdev->dev);
> > +	pm_runtime_disable(&pdev->dev);
> > +
> >  	mutex_destroy(&dev->dev_lock);
> >  	mutex_destroy(&dev->hw_lock);
> >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks); @@ -281,6
> > +348,7 @@ static struct platform_driver wave5_vpu_driver = {
> >  	.driver = {
> >  		.name = VPU_PLATFORM_DEVICE_NAME,
> >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > +		.pm = &wave5_pm_ops,
> >  		},
> >  	.probe = wave5_vpu_probe,
> >  	.remove = wave5_vpu_remove,
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > index 1a3efb638dde..f1f8e4fc8474 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > @@ -6,6 +6,7 @@
> >   */
> >
> >  #include <linux/bug.h>
> > +#include <linux/pm_runtime.h>
> >  #include "wave5-vpuapi.h"
> >  #include "wave5-regdefine.h"
> >  #include "wave5.h"
> > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct vpu_instance *inst,
> > u32 *fail_res)
> >
> >  	wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task);
> >
> > +	if (!pm_runtime_suspended(inst->dev->dev))
> > +		pm_runtime_put_sync(inst->dev->dev);
> > +
> >  unlock_and_return:
> >  	mutex_unlock(&vpu_dev->hw_lock);
> >
> > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct vpu_instance *inst,
> > u32 *fail_res)
> >
> >  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
> >
> > +	if (!pm_runtime_suspended(inst->dev->dev))
> > +		pm_runtime_put_sync(inst->dev->dev);
> 
> This seems very unnatural. We do the get() in "start_streaming()", but the
> put() is only done when the device is closed, or when the driver is
> removed. As this is not balanced, you seem to have to check the suspended
> condition all over the place.
> 
> I think we could aim for start_streaming()/stop_streaming() for your
> get/put placement. At least they will be bound to an entirely balanced API.
> But then, a media player in paused sate will prevent that device from
> being suspended.
> 
> If the HW is capable of preserving enough state, and From the short doc I
> have it gives me the impression it can preserve that, I'd suggest to
> follow what hantro driver is doing. What is does is that it will do get()
> in device_run(), and put() whenever the job completes. This driver has
> been designed so when there is no job, it means the firmware is currently
> idle and looking for more work. So it seems like the perfect moment to do
> suspend it.
> 
> Nicolas
> 
> > +
> >  	mutex_unlock(&vpu_dev->hw_lock);
> >
> >  	return 0;
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5.h
> > b/drivers/media/platform/chips-media/wave5/wave5.h
> > index 063028eccd3b..6125eff938a8 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct vpu_device
> > *vpu_dev, u32 *revision);
> >
> >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t size);
> >
> > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const
> uint16_t *code,
> > +			 size_t size);
> > +
> >  int wave5_vpu_reset(struct device *dev, enum sw_reset_mode
> > reset_mode);
> >
> >  int wave5_vpu_build_up_dec_param(struct vpu_instance *inst, struct
> > dec_open_param *param);

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

Hi Jackson,

Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> Hi Nicolas
> 
> 
> > This seems very unnatural. We do the get() in "start_streaming()", but the
> > put() is only done when the device is closed, or when the driver is
> > removed. As this is not balanced, you seem to have to check the suspended
> > condition all over the place.
> > 
> > I think we could aim for start_streaming()/stop_streaming() for your
> > get/put placement. At least they will be bound to an entirely balanced API.
> > But then, a media player in paused sate will prevent that device from
> > being suspended.
> > 
> > If the HW is capable of preserving enough state, and From the short doc I
> > have it gives me the impression it can preserve that, I'd suggest to
> > follow what hantro driver is doing. What is does is that it will do get()
> > in device_run(), and put() whenever the job completes. This driver has
> > been designed so when there is no job, it means the firmware is currently
> > idle and looking for more work. So it seems like the perfect moment to do
> > suspend it.
> > 
> 
> Thanks your comment,
> 
> Currently they are not balanced, 
> If we puts "the put functon" into the stop_streaming, our hw is stalled until doing wake-up command, so our v4l2 device become block.
> so I'd like to update the below instead of calling get at the start_streaming function.
> 
> 
> @@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct file *filp)
> 
>         wave5_vdi_allocate_sram(inst->dev);
> 
> +       err = pm_runtime_resume_and_get(inst->dev->dev);
> +       if (err) {
> +               dev_err(inst->dev->dev, "decoder runtime resume failed %d\n", err);
> +               ret = -EINVAL;
> +               goto cleanup_inst;
> +       }
> +
>         return 0;

I guess we need to discuss the power management strategy for this device. If you
do resume_and_get() in open(), and then put in close(), that seems balanced. But
in term of power saving, it might not be very strong. If you have a media player
that is set to pause and then placed in the background, you still keep the IP
running. This is extremely common, since application cannot close their device
without loosing the reference frames, and thus having to do extra work on resume
to seek back to the previous sync point and drop unneeded frames.

It seems like the whole point of asking the firmware to save the state and
suspend is to be able to do so while there is meaningful sate in the firt place.
If we are to suspend only when there is no meaningful state, we could just free
all resources and power it off completely. (This is just for illustration, its
probably to slow to boot the firmware at runtime)


I understand you suffered lockup with a start_streaming() for resume_and_get(),
and stop_streaming() for put(), this may simply indicate that some hardware
access are needed between these two. Can you write down a power management plan
that would effectively save power in common use cases ? We can certainly help in
refactoring the code to make that happen.

Nicolas

> 
> > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> > > There are two device run-time PM callbacks defined in 'struct
> > dev_pm_ops'
> > > int (*runtime_suspend)(struct device *dev); int
> > > (*runtime_resume)(struct device *dev);
> > 
> > I wonder how useful is it to teach everyone what the generic 'struct
> > dev_pm_ops'
> > contains. Perhaps you simply wanted that this patch implement both suspend
> > and resume ops ?
> > 
> > > 
> > > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > ---
> > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68 +++++++++++++++++++
> > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > 
> > > diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > index 8ad7f3a28ae1..8aade5a38439 100644
> > > --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > @@ -503,6 +503,7 @@ int wave5_vpu_build_up_dec_param(struct vpu_instance
> > *inst,
> > >  	/* This register must be reset explicitly */
> > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > (COMMAND_QUEUE_DEPTH - 1));
> > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > 
> > In some way, the relation between suspend and this register write is not
> > obvious. If its not related, please do this in its own patch. Otherwise
> > you want to explain why you needed this (possibly just in the commit
> > message).
> > 
> > > 
> > >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE, true, NULL,
> > NULL);
> > >  	if (ret) {
> > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device *dev, u8 *fw,
> > size_t size)
> > >  	return setup_wave5_properties(dev);
> > >  }
> > > 
> > > -static int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake,
> > const uint16_t *code,
> > > -				size_t size)
> > > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const
> > uint16_t *code,
> > > +			 size_t size)
> > >  {
> > >  	u32 reg_val;
> > >  	struct vpu_buf *common_vb;
> > > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > index ef227af72348..328a7a8f26c5 100644
> > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > @@ -5,6 +5,7 @@
> > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > >   */
> > > 
> > > +#include <linux/pm_runtime.h>
> > >  #include "wave5-helper.h"
> > > 
> > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > @@ -1387,9 +1388,17 @@ static int wave5_vpu_dec_start_streaming(struct
> > > vb2_queue *q, unsigned int count
> > > 
> > >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && inst->state ==
> > VPU_INST_STATE_NONE) {
> > >  		struct dec_open_param open_param;
> > > +		int err = 0;
> > > 
> > >  		memset(&open_param, 0, sizeof(struct dec_open_param));
> > > 
> > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > +		if (err) {
> > > +			dev_err(inst->dev->dev, "decoder runtime resume
> > failed %d\n", err);
> > > +			ret = -EINVAL;
> > > +			goto return_buffers;
> > > +		}
> > > +
> > >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> > >  		if (ret)
> > >  			goto return_buffers;
> > > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > index 761775216cd4..ff73d69de41c 100644
> > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > @@ -5,6 +5,7 @@
> > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > >   */
> > > 
> > > +#include <linux/pm_runtime.h>
> > >  #include "wave5-helper.h"
> > > 
> > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > @@ -1387,9 +1388,17 @@ static int wave5_vpu_enc_start_streaming(struct
> > > vb2_queue *q, unsigned int count
> > > 
> > >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > >  		struct enc_open_param open_param;
> > > +		int err = 0;
> > > 
> > >  		memset(&open_param, 0, sizeof(struct enc_open_param));
> > > 
> > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > +		if (err) {
> > > +			dev_err(inst->dev->dev, "encoder runtime resume
> > failed %d\n", err);
> > > +			ret = -EINVAL;
> > > +			goto return_buffers;
> > > +		}
> > > +
> > >  		wave5_set_enc_openparam(&open_param, inst);
> > > 
> > >  		ret = wave5_vpu_enc_open(inst, &open_param); diff --git
> > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > index 0d90b5820bef..f81409740a56 100644
> > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > @@ -10,6 +10,7 @@
> > >  #include <linux/clk.h>
> > >  #include <linux/firmware.h>
> > >  #include <linux/interrupt.h>
> > > +#include <linux/pm_runtime.h>
> > >  #include "wave5-vpu.h"
> > >  #include "wave5-regdefine.h"
> > >  #include "wave5-vpuconfig.h"
> > > @@ -117,6 +118,65 @@ static int wave5_vpu_load_firmware(struct device
> > *dev, const char *fw_name,
> > >  	return 0;
> > >  }
> > > 
> > > +static __maybe_unused int wave5_pm_suspend(struct device *dev) {
> > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > +
> > > +	if (pm_runtime_suspended(dev))
> > > +		return 0;
> > > +
> > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > > +
> > > +	return 0;
> > > +}
> > > +
> > > +static __maybe_unused int wave5_pm_resume(struct device *dev) {
> > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > +	int ret = 0;
> > > +
> > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> > > +	if (ret) {
> > > +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> > > +		return ret;
> > > +	}
> > > +
> > > +	return ret;
> > > +}
> > > +
> > > +static __maybe_unused int wave5_suspend(struct device *dev) {
> > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > +	struct vpu_instance *inst;
> > > +
> > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > +
> > > +	return pm_runtime_force_suspend(dev); }
> > > +
> > > +static __maybe_unused int wave5_resume(struct device *dev) {
> > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > +	struct vpu_instance *inst;
> > > +	int ret = 0;
> > > +
> > > +	ret = pm_runtime_force_resume(dev);
> > > +	if (ret < 0)
> > > +		return ret;
> > > +
> > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > +
> > > +	return ret;
> > > +}
> > 
> > The functions wave5_suspend() and wave5_resume() are not just
> > "maybe_unsued" but actually never used. What was the intention ?
> > Considering the usage of __maybe_unused has been such a bad friend for
> > this one, could you instead bracket the functions with an explicit ?
> > 
> > #ifdef CONFIG_PM
> > #endif
> > 
> > This way the compiler will have a word if you implement something that you
> > forgot to actually use.
> > 
> > > +
> > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume, NULL) };
> > > +
> > >  static int wave5_vpu_probe(struct platform_device *pdev)  {
> > >  	int ret;
> > > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct platform_device
> > *pdev)
> > >  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
> > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev->product_code);
> > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
> > > +
> > > +	pm_runtime_enable(&pdev->dev);
> > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > +
> > >  	return 0;
> > > 
> > >  err_enc_unreg:
> > > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct platform_device
> > > *pdev)  {
> > >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> > > 
> > > +	pm_runtime_put_sync(&pdev->dev);
> > > +	pm_runtime_disable(&pdev->dev);
> > > +
> > >  	mutex_destroy(&dev->dev_lock);
> > >  	mutex_destroy(&dev->hw_lock);
> > >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks); @@ -281,6
> > > +348,7 @@ static struct platform_driver wave5_vpu_driver = {
> > >  	.driver = {
> > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > > +		.pm = &wave5_pm_ops,
> > >  		},
> > >  	.probe = wave5_vpu_probe,
> > >  	.remove = wave5_vpu_remove,
> > > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > index 1a3efb638dde..f1f8e4fc8474 100644
> > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > @@ -6,6 +6,7 @@
> > >   */
> > > 
> > >  #include <linux/bug.h>
> > > +#include <linux/pm_runtime.h>
> > >  #include "wave5-vpuapi.h"
> > >  #include "wave5-regdefine.h"
> > >  #include "wave5.h"
> > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct vpu_instance *inst,
> > > u32 *fail_res)
> > > 
> > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task);
> > > 
> > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > +		pm_runtime_put_sync(inst->dev->dev);
> > > +
> > >  unlock_and_return:
> > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > 
> > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct vpu_instance *inst,
> > > u32 *fail_res)
> > > 
> > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
> > > 
> > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > +		pm_runtime_put_sync(inst->dev->dev);
> > 
> > This seems very unnatural. We do the get() in "start_streaming()", but the
> > put() is only done when the device is closed, or when the driver is
> > removed. As this is not balanced, you seem to have to check the suspended
> > condition all over the place.
> > 
> > I think we could aim for start_streaming()/stop_streaming() for your
> > get/put placement. At least they will be bound to an entirely balanced API.
> > But then, a media player in paused sate will prevent that device from
> > being suspended.
> > 
> > If the HW is capable of preserving enough state, and From the short doc I
> > have it gives me the impression it can preserve that, I'd suggest to
> > follow what hantro driver is doing. What is does is that it will do get()
> > in device_run(), and put() whenever the job completes. This driver has
> > been designed so when there is no job, it means the firmware is currently
> > idle and looking for more work. So it seems like the perfect moment to do
> > suspend it.
> > 
> > Nicolas
> > 
> > > +
> > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > 
> > >  	return 0;
> > > diff --git a/drivers/media/platform/chips-media/wave5/wave5.h
> > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > index 063028eccd3b..6125eff938a8 100644
> > > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct vpu_device
> > > *vpu_dev, u32 *revision);
> > > 
> > >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t size);
> > > 
> > > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const
> > uint16_t *code,
> > > +			 size_t size);
> > > +
> > >  int wave5_vpu_reset(struct device *dev, enum sw_reset_mode
> > > reset_mode);
> > > 
> > >  int wave5_vpu_build_up_dec_param(struct vpu_instance *inst, struct
> > > dec_open_param *param);
> 

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

Hello Nicolas

Thanks your comment.


A pause is common state for media player, but our VPU could trigger an interrupt regardless of a player state.
So at a pause state if a power turns off, our driver becomes hang-up.
I think power on/off should be controlled in the open/close function.

Thanks.

> Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > Hi Nicolas
> >
> >
> > > This seems very unnatural. We do the get() in "start_streaming()",
> > > but the
> > > put() is only done when the device is closed, or when the driver is
> > > removed. As this is not balanced, you seem to have to check the
> > > suspended condition all over the place.
> > >
> > > I think we could aim for start_streaming()/stop_streaming() for your
> > > get/put placement. At least they will be bound to an entirely balanced
> API.
> > > But then, a media player in paused sate will prevent that device
> > > from being suspended.
> > >
> > > If the HW is capable of preserving enough state, and From the short
> > > doc I have it gives me the impression it can preserve that, I'd
> > > suggest to follow what hantro driver is doing. What is does is that
> > > it will do get() in device_run(), and put() whenever the job
> > > completes. This driver has been designed so when there is no job, it
> > > means the firmware is currently idle and looking for more work. So
> > > it seems like the perfect moment to do suspend it.
> > >
> >
> > Thanks your comment,
> >
> > Currently they are not balanced,
> > If we puts "the put functon" into the stop_streaming, our hw is stalled
> until doing wake-up command, so our v4l2 device become block.
> > so I'd like to update the below instead of calling get at the
> start_streaming function.
> >
> >
> > @@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct file
> > *filp)
> >
> >         wave5_vdi_allocate_sram(inst->dev);
> >
> > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > +       if (err) {
> > +               dev_err(inst->dev->dev, "decoder runtime resume
> failed %d\n", err);
> > +               ret = -EINVAL;
> > +               goto cleanup_inst;
> > +       }
> > +
> >         return 0;
> 
> I guess we need to discuss the power management strategy for this device.
> If you do resume_and_get() in open(), and then put in close(), that seems
> balanced. But in term of power saving, it might not be very strong. If you
> have a media player that is set to pause and then placed in the background,
> you still keep the IP running. This is extremely common, since application
> cannot close their device without loosing the reference frames, and thus
> having to do extra work on resume to seek back to the previous sync point
> and drop unneeded frames.
> 
> It seems like the whole point of asking the firmware to save the state and
> suspend is to be able to do so while there is meaningful sate in the firt
> place.
> If we are to suspend only when there is no meaningful state, we could just
> free all resources and power it off completely. (This is just for
> illustration, its probably to slow to boot the firmware at runtime)
> 
> 
> I understand you suffered lockup with a start_streaming() for
> resume_and_get(), and stop_streaming() for put(), this may simply indicate
> that some hardware access are needed between these two. Can you write down
> a power management plan that would effectively save power in common use
> cases ? We can certainly help in refactoring the code to make that happen.
> 
> Nicolas
> 
> >
> > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> > > > There are two device run-time PM callbacks defined in 'struct
> > > dev_pm_ops'
> > > > int (*runtime_suspend)(struct device *dev); int
> > > > (*runtime_resume)(struct device *dev);
> > >
> > > I wonder how useful is it to teach everyone what the generic 'struct
> > > dev_pm_ops'
> > > contains. Perhaps you simply wanted that this patch implement both
> > > suspend and resume ops ?
> > >
> > > >
> > > > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > ---
> > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> +++++++++++++++++++
> > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > > > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > >
> > > > diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > index 8ad7f3a28ae1..8aade5a38439 100644
> > > > --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > @@ -503,6 +503,7 @@ int wave5_vpu_build_up_dec_param(struct
> > > > vpu_instance
> > > *inst,
> > > >  	/* This register must be reset explicitly */
> > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > >
> > > In some way, the relation between suspend and this register write is
> > > not obvious. If its not related, please do this in its own patch.
> > > Otherwise you want to explain why you needed this (possibly just in
> > > the commit message).
> > >
> > > >
> > > >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE, true,
> > > > NULL,
> > > NULL);
> > > >  	if (ret) {
> > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device *dev, u8
> > > > *fw,
> > > size_t size)
> > > >  	return setup_wave5_properties(dev);  }
> > > >
> > > > -static int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > i_sleep_wake,
> > > const uint16_t *code,
> > > > -				size_t size)
> > > > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake,
> > > > +const
> > > uint16_t *code,
> > > > +			 size_t size)
> > > >  {
> > > >  	u32 reg_val;
> > > >  	struct vpu_buf *common_vb;
> > > > diff --git
> > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > index ef227af72348..328a7a8f26c5 100644
> > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > @@ -5,6 +5,7 @@
> > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > >   */
> > > >
> > > > +#include <linux/pm_runtime.h>
> > > >  #include "wave5-helper.h"
> > > >
> > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > @@ -1387,9 +1388,17 @@ static int
> > > > wave5_vpu_dec_start_streaming(struct
> > > > vb2_queue *q, unsigned int count
> > > >
> > > >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && inst-
> >state
> > > > ==
> > > VPU_INST_STATE_NONE) {
> > > >  		struct dec_open_param open_param;
> > > > +		int err = 0;
> > > >
> > > >  		memset(&open_param, 0, sizeof(struct dec_open_param));
> > > >
> > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > +		if (err) {
> > > > +			dev_err(inst->dev->dev, "decoder runtime resume
> > > failed %d\n", err);
> > > > +			ret = -EINVAL;
> > > > +			goto return_buffers;
> > > > +		}
> > > > +
> > > >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> > > >  		if (ret)
> > > >  			goto return_buffers;
> > > > diff --git
> > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > index 761775216cd4..ff73d69de41c 100644
> > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > @@ -5,6 +5,7 @@
> > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > >   */
> > > >
> > > > +#include <linux/pm_runtime.h>
> > > >  #include "wave5-helper.h"
> > > >
> > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > @@ -1387,9 +1388,17 @@ static int
> > > > wave5_vpu_enc_start_streaming(struct
> > > > vb2_queue *q, unsigned int count
> > > >
> > > >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > >  		struct enc_open_param open_param;
> > > > +		int err = 0;
> > > >
> > > >  		memset(&open_param, 0, sizeof(struct enc_open_param));
> > > >
> > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > +		if (err) {
> > > > +			dev_err(inst->dev->dev, "encoder runtime resume
> > > failed %d\n", err);
> > > > +			ret = -EINVAL;
> > > > +			goto return_buffers;
> > > > +		}
> > > > +
> > > >  		wave5_set_enc_openparam(&open_param, inst);
> > > >
> > > >  		ret = wave5_vpu_enc_open(inst, &open_param); diff --
> git
> > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > index 0d90b5820bef..f81409740a56 100644
> > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > @@ -10,6 +10,7 @@
> > > >  #include <linux/clk.h>
> > > >  #include <linux/firmware.h>
> > > >  #include <linux/interrupt.h>
> > > > +#include <linux/pm_runtime.h>
> > > >  #include "wave5-vpu.h"
> > > >  #include "wave5-regdefine.h"
> > > >  #include "wave5-vpuconfig.h"
> > > > @@ -117,6 +118,65 @@ static int wave5_vpu_load_firmware(struct
> > > > device
> > > *dev, const char *fw_name,
> > > >  	return 0;
> > > >  }
> > > >
> > > > +static __maybe_unused int wave5_pm_suspend(struct device *dev) {
> > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > +
> > > > +	if (pm_runtime_suspended(dev))
> > > > +		return 0;
> > > > +
> > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > > > +
> > > > +	return 0;
> > > > +}
> > > > +
> > > > +static __maybe_unused int wave5_pm_resume(struct device *dev) {
> > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > +	int ret = 0;
> > > > +
> > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> > > > +	if (ret) {
> > > > +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> > > > +		return ret;
> > > > +	}
> > > > +
> > > > +	return ret;
> > > > +}
> > > > +
> > > > +static __maybe_unused int wave5_suspend(struct device *dev) {
> > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > +	struct vpu_instance *inst;
> > > > +
> > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > +
> > > > +	return pm_runtime_force_suspend(dev); }
> > > > +
> > > > +static __maybe_unused int wave5_resume(struct device *dev) {
> > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > +	struct vpu_instance *inst;
> > > > +	int ret = 0;
> > > > +
> > > > +	ret = pm_runtime_force_resume(dev);
> > > > +	if (ret < 0)
> > > > +		return ret;
> > > > +
> > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > +
> > > > +	return ret;
> > > > +}
> > >
> > > The functions wave5_suspend() and wave5_resume() are not just
> > > "maybe_unsued" but actually never used. What was the intention ?
> > > Considering the usage of __maybe_unused has been such a bad friend
> > > for this one, could you instead bracket the functions with an
> explicit ?
> > >
> > > #ifdef CONFIG_PM
> > > #endif
> > >
> > > This way the compiler will have a word if you implement something
> > > that you forgot to actually use.
> > >
> > > > +
> > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume,
> NULL) };
> > > > +
> > > >  static int wave5_vpu_probe(struct platform_device *pdev)  {
> > > >  	int ret;
> > > > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct
> > > > platform_device
> > > *pdev)
> > > >  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
> > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev-
> >product_code);
> > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
> > > > +
> > > > +	pm_runtime_enable(&pdev->dev);
> > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > +
> > > >  	return 0;
> > > >
> > > >  err_enc_unreg:
> > > > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct
> > > > platform_device
> > > > *pdev)  {
> > > >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> > > >
> > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > +	pm_runtime_disable(&pdev->dev);
> > > > +
> > > >  	mutex_destroy(&dev->dev_lock);
> > > >  	mutex_destroy(&dev->hw_lock);
> > > >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks); @@ -
> 281,6
> > > > +348,7 @@ static struct platform_driver wave5_vpu_driver = {
> > > >  	.driver = {
> > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > > > +		.pm = &wave5_pm_ops,
> > > >  		},
> > > >  	.probe = wave5_vpu_probe,
> > > >  	.remove = wave5_vpu_remove,
> > > > diff --git
> > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > index 1a3efb638dde..f1f8e4fc8474 100644
> > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > @@ -6,6 +6,7 @@
> > > >   */
> > > >
> > > >  #include <linux/bug.h>
> > > > +#include <linux/pm_runtime.h>
> > > >  #include "wave5-vpuapi.h"
> > > >  #include "wave5-regdefine.h"
> > > >  #include "wave5.h"
> > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct vpu_instance
> > > > *inst,
> > > > u32 *fail_res)
> > > >
> > > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task);
> > > >
> > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > +
> > > >  unlock_and_return:
> > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > >
> > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct vpu_instance
> > > > *inst,
> > > > u32 *fail_res)
> > > >
> > > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
> > > >
> > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > +		pm_runtime_put_sync(inst->dev->dev);
> > >
> > > This seems very unnatural. We do the get() in "start_streaming()",
> > > but the
> > > put() is only done when the device is closed, or when the driver is
> > > removed. As this is not balanced, you seem to have to check the
> > > suspended condition all over the place.
> > >
> > > I think we could aim for start_streaming()/stop_streaming() for your
> > > get/put placement. At least they will be bound to an entirely balanced
> API.
> > > But then, a media player in paused sate will prevent that device
> > > from being suspended.
> > >
> > > If the HW is capable of preserving enough state, and From the short
> > > doc I have it gives me the impression it can preserve that, I'd
> > > suggest to follow what hantro driver is doing. What is does is that
> > > it will do get() in device_run(), and put() whenever the job
> > > completes. This driver has been designed so when there is no job, it
> > > means the firmware is currently idle and looking for more work. So
> > > it seems like the perfect moment to do suspend it.
> > >
> > > Nicolas
> > >
> > > > +
> > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > >
> > > >  	return 0;
> > > > diff --git a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > index 063028eccd3b..6125eff938a8 100644
> > > > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct vpu_device
> > > > *vpu_dev, u32 *revision);
> > > >
> > > >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t size);
> > > >
> > > > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake,
> > > > +const
> > > uint16_t *code,
> > > > +			 size_t size);
> > > > +
> > > >  int wave5_vpu_reset(struct device *dev, enum sw_reset_mode
> > > > reset_mode);
> > > >
> > > >  int wave5_vpu_build_up_dec_param(struct vpu_instance *inst,
> > > > struct dec_open_param *param);
> >

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

Le mardi 20 février 2024 à 05:12 +0000, jackson.lee a écrit :
> Hello Nicolas
> 
> Thanks your comment.
> 
> 
> A pause is common state for media player, but our VPU could trigger an interrupt regardless of a player state.
> So at a pause state if a power turns off, our driver becomes hang-up.
> I think power on/off should be controlled in the open/close function.

Please, avoid top posting, this breaks the discussion completely. Walk through
the comments, and reply to them underneath so we can have a proper discussion.

In our experience, and with the rework of the driver we did during up-streaming,
there is no more un-solicited IRQ. The way the driver state machine has been
built, when the m2m framework is idle (no jobs are active or pending), it means
the firmware is stalled, waiting on some application action. Combined with a
timer of course, so we don't actively suspend/resume, this seems adequate place
to do power management. If that application action never occurred passed a
decent delay, we can suspend. Later on, user actions like qbuf, will lead to
device_run() to be called again, and it would be a good place to resume it.
There is of course other places where you'll have to make sure the hardware is
resumed, like on close, as you want to remove the instance. There is also small
queries here and there that need to be surrounded with resume/put, but with the
redesign, most of the HW access now take place inside device_run() only.

Open/Close is not invalid, but it has a lot of issues, as any random application
can endup disabling the kernel ability to save power. Personally, I think we
should at least give it a try, and document valid reason not to do so if we find
hardware issues. Otherwise, this sounds like all we care is ticking the box
"this driver has runtime PM" without actually caring about effective power
saving.

Nicolas

> 
> Thanks.
> 
> > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > Hi Nicolas
> > > 
> > > 
> > > > This seems very unnatural. We do the get() in "start_streaming()",
> > > > but the
> > > > put() is only done when the device is closed, or when the driver is
> > > > removed. As this is not balanced, you seem to have to check the
> > > > suspended condition all over the place.
> > > > 
> > > > I think we could aim for start_streaming()/stop_streaming() for your
> > > > get/put placement. At least they will be bound to an entirely balanced
> > API.
> > > > But then, a media player in paused sate will prevent that device
> > > > from being suspended.
> > > > 
> > > > If the HW is capable of preserving enough state, and From the short
> > > > doc I have it gives me the impression it can preserve that, I'd
> > > > suggest to follow what hantro driver is doing. What is does is that
> > > > it will do get() in device_run(), and put() whenever the job
> > > > completes. This driver has been designed so when there is no job, it
> > > > means the firmware is currently idle and looking for more work. So
> > > > it seems like the perfect moment to do suspend it.
> > > > 
> > > 
> > > Thanks your comment,
> > > 
> > > Currently they are not balanced,
> > > If we puts "the put functon" into the stop_streaming, our hw is stalled
> > until doing wake-up command, so our v4l2 device become block.
> > > so I'd like to update the below instead of calling get at the
> > start_streaming function.
> > > 
> > > 
> > > @@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct file
> > > *filp)
> > > 
> > >         wave5_vdi_allocate_sram(inst->dev);
> > > 
> > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > +       if (err) {
> > > +               dev_err(inst->dev->dev, "decoder runtime resume
> > failed %d\n", err);
> > > +               ret = -EINVAL;
> > > +               goto cleanup_inst;
> > > +       }
> > > +
> > >         return 0;
> > 
> > I guess we need to discuss the power management strategy for this device.
> > If you do resume_and_get() in open(), and then put in close(), that seems
> > balanced. But in term of power saving, it might not be very strong. If you
> > have a media player that is set to pause and then placed in the background,
> > you still keep the IP running. This is extremely common, since application
> > cannot close their device without loosing the reference frames, and thus
> > having to do extra work on resume to seek back to the previous sync point
> > and drop unneeded frames.
> > 
> > It seems like the whole point of asking the firmware to save the state and
> > suspend is to be able to do so while there is meaningful sate in the firt
> > place.
> > If we are to suspend only when there is no meaningful state, we could just
> > free all resources and power it off completely. (This is just for
> > illustration, its probably to slow to boot the firmware at runtime)
> > 
> > 
> > I understand you suffered lockup with a start_streaming() for
> > resume_and_get(), and stop_streaming() for put(), this may simply indicate
> > that some hardware access are needed between these two. Can you write down
> > a power management plan that would effectively save power in common use
> > cases ? We can certainly help in refactoring the code to make that happen.
> > 
> > Nicolas
> > 
> > > 
> > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> > > > > There are two device run-time PM callbacks defined in 'struct
> > > > dev_pm_ops'
> > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > (*runtime_resume)(struct device *dev);
> > > > 
> > > > I wonder how useful is it to teach everyone what the generic 'struct
> > > > dev_pm_ops'
> > > > contains. Perhaps you simply wanted that this patch implement both
> > > > suspend and resume ops ?
> > > > 
> > > > > 
> > > > > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > ---
> > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > +++++++++++++++++++
> > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > > > > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > 
> > > > > diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > @@ -503,6 +503,7 @@ int wave5_vpu_build_up_dec_param(struct
> > > > > vpu_instance
> > > > *inst,
> > > > >  	/* This register must be reset explicitly */
> > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > 
> > > > In some way, the relation between suspend and this register write is
> > > > not obvious. If its not related, please do this in its own patch.
> > > > Otherwise you want to explain why you needed this (possibly just in
> > > > the commit message).
> > > > 
> > > > > 
> > > > >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE, true,
> > > > > NULL,
> > > > NULL);
> > > > >  	if (ret) {
> > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device *dev, u8
> > > > > *fw,
> > > > size_t size)
> > > > >  	return setup_wave5_properties(dev);  }
> > > > > 
> > > > > -static int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > i_sleep_wake,
> > > > const uint16_t *code,
> > > > > -				size_t size)
> > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake,
> > > > > +const
> > > > uint16_t *code,
> > > > > +			 size_t size)
> > > > >  {
> > > > >  	u32 reg_val;
> > > > >  	struct vpu_buf *common_vb;
> > > > > diff --git
> > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > index ef227af72348..328a7a8f26c5 100644
> > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > @@ -5,6 +5,7 @@
> > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > >   */
> > > > > 
> > > > > +#include <linux/pm_runtime.h>
> > > > >  #include "wave5-helper.h"
> > > > > 
> > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > wave5_vpu_dec_start_streaming(struct
> > > > > vb2_queue *q, unsigned int count
> > > > > 
> > > > >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && inst-
> > > state
> > > > > ==
> > > > VPU_INST_STATE_NONE) {
> > > > >  		struct dec_open_param open_param;
> > > > > +		int err = 0;
> > > > > 
> > > > >  		memset(&open_param, 0, sizeof(struct dec_open_param));
> > > > > 
> > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > +		if (err) {
> > > > > +			dev_err(inst->dev->dev, "decoder runtime resume
> > > > failed %d\n", err);
> > > > > +			ret = -EINVAL;
> > > > > +			goto return_buffers;
> > > > > +		}
> > > > > +
> > > > >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > >  		if (ret)
> > > > >  			goto return_buffers;
> > > > > diff --git
> > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > index 761775216cd4..ff73d69de41c 100644
> > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > @@ -5,6 +5,7 @@
> > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > >   */
> > > > > 
> > > > > +#include <linux/pm_runtime.h>
> > > > >  #include "wave5-helper.h"
> > > > > 
> > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > wave5_vpu_enc_start_streaming(struct
> > > > > vb2_queue *q, unsigned int count
> > > > > 
> > > > >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > >  		struct enc_open_param open_param;
> > > > > +		int err = 0;
> > > > > 
> > > > >  		memset(&open_param, 0, sizeof(struct enc_open_param));
> > > > > 
> > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > +		if (err) {
> > > > > +			dev_err(inst->dev->dev, "encoder runtime resume
> > > > failed %d\n", err);
> > > > > +			ret = -EINVAL;
> > > > > +			goto return_buffers;
> > > > > +		}
> > > > > +
> > > > >  		wave5_set_enc_openparam(&open_param, inst);
> > > > > 
> > > > >  		ret = wave5_vpu_enc_open(inst, &open_param); diff --
> > git
> > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > index 0d90b5820bef..f81409740a56 100644
> > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > @@ -10,6 +10,7 @@
> > > > >  #include <linux/clk.h>
> > > > >  #include <linux/firmware.h>
> > > > >  #include <linux/interrupt.h>
> > > > > +#include <linux/pm_runtime.h>
> > > > >  #include "wave5-vpu.h"
> > > > >  #include "wave5-regdefine.h"
> > > > >  #include "wave5-vpuconfig.h"
> > > > > @@ -117,6 +118,65 @@ static int wave5_vpu_load_firmware(struct
> > > > > device
> > > > *dev, const char *fw_name,
> > > > >  	return 0;
> > > > >  }
> > > > > 
> > > > > +static __maybe_unused int wave5_pm_suspend(struct device *dev) {
> > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > +
> > > > > +	if (pm_runtime_suspended(dev))
> > > > > +		return 0;
> > > > > +
> > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > > > > +
> > > > > +	return 0;
> > > > > +}
> > > > > +
> > > > > +static __maybe_unused int wave5_pm_resume(struct device *dev) {
> > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > +	int ret = 0;
> > > > > +
> > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> > > > > +	if (ret) {
> > > > > +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> > > > > +		return ret;
> > > > > +	}
> > > > > +
> > > > > +	return ret;
> > > > > +}
> > > > > +
> > > > > +static __maybe_unused int wave5_suspend(struct device *dev) {
> > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > +	struct vpu_instance *inst;
> > > > > +
> > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > +
> > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > +
> > > > > +static __maybe_unused int wave5_resume(struct device *dev) {
> > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > +	struct vpu_instance *inst;
> > > > > +	int ret = 0;
> > > > > +
> > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > +	if (ret < 0)
> > > > > +		return ret;
> > > > > +
> > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > +
> > > > > +	return ret;
> > > > > +}
> > > > 
> > > > The functions wave5_suspend() and wave5_resume() are not just
> > > > "maybe_unsued" but actually never used. What was the intention ?
> > > > Considering the usage of __maybe_unused has been such a bad friend
> > > > for this one, could you instead bracket the functions with an
> > explicit ?
> > > > 
> > > > #ifdef CONFIG_PM
> > > > #endif
> > > > 
> > > > This way the compiler will have a word if you implement something
> > > > that you forgot to actually use.
> > > > 
> > > > > +
> > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume,
> > NULL) };
> > > > > +
> > > > >  static int wave5_vpu_probe(struct platform_device *pdev)  {
> > > > >  	int ret;
> > > > > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct
> > > > > platform_device
> > > > *pdev)
> > > > >  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
> > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev-
> > > product_code);
> > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
> > > > > +
> > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > +
> > > > >  	return 0;
> > > > > 
> > > > >  err_enc_unreg:
> > > > > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct
> > > > > platform_device
> > > > > *pdev)  {
> > > > >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> > > > > 
> > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > +
> > > > >  	mutex_destroy(&dev->dev_lock);
> > > > >  	mutex_destroy(&dev->hw_lock);
> > > > >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks); @@ -
> > 281,6
> > > > > +348,7 @@ static struct platform_driver wave5_vpu_driver = {
> > > > >  	.driver = {
> > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > > > > +		.pm = &wave5_pm_ops,
> > > > >  		},
> > > > >  	.probe = wave5_vpu_probe,
> > > > >  	.remove = wave5_vpu_remove,
> > > > > diff --git
> > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > @@ -6,6 +6,7 @@
> > > > >   */
> > > > > 
> > > > >  #include <linux/bug.h>
> > > > > +#include <linux/pm_runtime.h>
> > > > >  #include "wave5-vpuapi.h"
> > > > >  #include "wave5-regdefine.h"
> > > > >  #include "wave5.h"
> > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct vpu_instance
> > > > > *inst,
> > > > > u32 *fail_res)
> > > > > 
> > > > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task);
> > > > > 
> > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > +
> > > > >  unlock_and_return:
> > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > 
> > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct vpu_instance
> > > > > *inst,
> > > > > u32 *fail_res)
> > > > > 
> > > > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
> > > > > 
> > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > 
> > > > This seems very unnatural. We do the get() in "start_streaming()",
> > > > but the
> > > > put() is only done when the device is closed, or when the driver is
> > > > removed. As this is not balanced, you seem to have to check the
> > > > suspended condition all over the place.
> > > > 
> > > > I think we could aim for start_streaming()/stop_streaming() for your
> > > > get/put placement. At least they will be bound to an entirely balanced
> > API.
> > > > But then, a media player in paused sate will prevent that device
> > > > from being suspended.
> > > > 
> > > > If the HW is capable of preserving enough state, and From the short
> > > > doc I have it gives me the impression it can preserve that, I'd
> > > > suggest to follow what hantro driver is doing. What is does is that
> > > > it will do get() in device_run(), and put() whenever the job
> > > > completes. This driver has been designed so when there is no job, it
> > > > means the firmware is currently idle and looking for more work. So
> > > > it seems like the perfect moment to do suspend it.
> > > > 
> > > > Nicolas
> > > > 
> > > > > +
> > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > 
> > > > >  	return 0;
> > > > > diff --git a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct vpu_device
> > > > > *vpu_dev, u32 *revision);
> > > > > 
> > > > >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t size);
> > > > > 
> > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake,
> > > > > +const
> > > > uint16_t *code,
> > > > > +			 size_t size);
> > > > > +
> > > > >  int wave5_vpu_reset(struct device *dev, enum sw_reset_mode
> > > > > reset_mode);
> > > > > 
> > > > >  int wave5_vpu_build_up_dec_param(struct vpu_instance *inst,
> > > > > struct dec_open_param *param);
> > > 
> 

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

Hello Nicolas


Thanks for your comment

> Le mardi 20 février 2024 à 05:12 +0000, jackson.lee a écrit :
> > A pause is common state for media player, but our VPU could trigger an
> interrupt regardless of a player state.
> > So at a pause state if a power turns off, our driver becomes hang-up.
> > I think power on/off should be controlled in the open/close function.
> 
> Please, avoid top posting, this breaks the discussion completely. Walk
> through the comments, and reply to them underneath so we can have a proper
> discussion.
> 
> In our experience, and with the rework of the driver we did during up-
> streaming, there is no more un-solicited IRQ. The way the driver state
> machine has been built, when the m2m framework is idle (no jobs are active
> or pending), it means the firmware is stalled, waiting on some application
> action. Combined with a timer of course, so we don't actively
> suspend/resume, this seems adequate place to do power management. If that
> application action never occurred passed a decent delay, we can suspend.
> Later on, user actions like qbuf, will lead to
> device_run() to be called again, and it would be a good place to resume it.

The above comment means like the below ?
The wave5_vpu_dec_finish_decode is called by IRQ if there is a decoded frame.
And at the end of the function, if there is no data queued, v4l2_m2m_job_finish is called.
You means suspend is called in that function and if app feeds new data into v4l2 driver, the device_run then is called,
In the device_run, resume is called.


I have a question, 
If app is paused or resumed , the v4l2's vidioc_decoder_cmd is always called ?

Thanks.


> There is of course other places where you'll have to make sure the
> hardware is resumed, like on close, as you want to remove the instance.
> There is also small queries here and there that need to be surrounded with
> resume/put, but with the redesign, most of the HW access now take place
> inside device_run() only.
> 
> Open/Close is not invalid, but it has a lot of issues, as any random
> application can endup disabling the kernel ability to save power.
> Personally, I think we should at least give it a try, and document valid
> reason not to do so if we find hardware issues. Otherwise, this sounds
> like all we care is ticking the box "this driver has runtime PM" without
> actually caring about effective power saving.
> 
> Nicolas
> 
> >
> > Thanks.
> >
> > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > > Hi Nicolas
> > > >
> > > >
> > > > > This seems very unnatural. We do the get() in
> > > > > "start_streaming()", but the
> > > > > put() is only done when the device is closed, or when the driver
> > > > > is removed. As this is not balanced, you seem to have to check
> > > > > the suspended condition all over the place.
> > > > >
> > > > > I think we could aim for start_streaming()/stop_streaming() for
> > > > > your get/put placement. At least they will be bound to an
> > > > > entirely balanced
> > > API.
> > > > > But then, a media player in paused sate will prevent that device
> > > > > from being suspended.
> > > > >
> > > > > If the HW is capable of preserving enough state, and From the
> > > > > short doc I have it gives me the impression it can preserve
> > > > > that, I'd suggest to follow what hantro driver is doing. What is
> > > > > does is that it will do get() in device_run(), and put()
> > > > > whenever the job completes. This driver has been designed so
> > > > > when there is no job, it means the firmware is currently idle
> > > > > and looking for more work. So it seems like the perfect moment to
> do suspend it.
> > > > >
> > > >
> > > > Thanks your comment,
> > > >
> > > > Currently they are not balanced,
> > > > If we puts "the put functon" into the stop_streaming, our hw is
> > > > stalled
> > > until doing wake-up command, so our v4l2 device become block.
> > > > so I'd like to update the below instead of calling get at the
> > > start_streaming function.
> > > >
> > > >
> > > > @@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct file
> > > > *filp)
> > > >
> > > >         wave5_vdi_allocate_sram(inst->dev);
> > > >
> > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > +       if (err) {
> > > > +               dev_err(inst->dev->dev, "decoder runtime resume
> > > failed %d\n", err);
> > > > +               ret = -EINVAL;
> > > > +               goto cleanup_inst;
> > > > +       }
> > > > +
> > > >         return 0;
> > >
> > > I guess we need to discuss the power management strategy for this
> device.
> > > If you do resume_and_get() in open(), and then put in close(), that
> > > seems balanced. But in term of power saving, it might not be very
> > > strong. If you have a media player that is set to pause and then
> > > placed in the background, you still keep the IP running. This is
> > > extremely common, since application cannot close their device
> > > without loosing the reference frames, and thus having to do extra
> > > work on resume to seek back to the previous sync point and drop
> unneeded frames.
> > >
> > > It seems like the whole point of asking the firmware to save the
> > > state and suspend is to be able to do so while there is meaningful
> > > sate in the firt place.
> > > If we are to suspend only when there is no meaningful state, we
> > > could just free all resources and power it off completely. (This is
> > > just for illustration, its probably to slow to boot the firmware at
> > > runtime)
> > >
> > >
> > > I understand you suffered lockup with a start_streaming() for
> > > resume_and_get(), and stop_streaming() for put(), this may simply
> > > indicate that some hardware access are needed between these two. Can
> > > you write down a power management plan that would effectively save
> > > power in common use cases ? We can certainly help in refactoring the
> code to make that happen.
> > >
> > > Nicolas
> > >
> > > >
> > > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> > > > > > There are two device run-time PM callbacks defined in 'struct
> > > > > dev_pm_ops'
> > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > (*runtime_resume)(struct device *dev);
> > > > >
> > > > > I wonder how useful is it to teach everyone what the generic
> > > > > 'struct dev_pm_ops'
> > > > > contains. Perhaps you simply wanted that this patch implement
> > > > > both suspend and resume ops ?
> > > > >
> > > > > >
> > > > > > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > > ---
> > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > +++++++++++++++++++
> > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > > > > > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> > > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > >
> > > > > > diff --git
> > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > @@ -503,6 +503,7 @@ int wave5_vpu_build_up_dec_param(struct
> > > > > > vpu_instance
> > > > > *inst,
> > > > > >  	/* This register must be reset explicitly */
> > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > >
> > > > > In some way, the relation between suspend and this register
> > > > > write is not obvious. If its not related, please do this in its
> own patch.
> > > > > Otherwise you want to explain why you needed this (possibly just
> > > > > in the commit message).
> > > > >
> > > > > >
> > > > > >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE, true,
> > > > > > NULL,
> > > > > NULL);
> > > > > >  	if (ret) {
> > > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device
> > > > > > *dev, u8 *fw,
> > > > > size_t size)
> > > > > >  	return setup_wave5_properties(dev);  }
> > > > > >
> > > > > > -static int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > i_sleep_wake,
> > > > > const uint16_t *code,
> > > > > > -				size_t size)
> > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > +i_sleep_wake, const
> > > > > uint16_t *code,
> > > > > > +			 size_t size)
> > > > > >  {
> > > > > >  	u32 reg_val;
> > > > > >  	struct vpu_buf *common_vb;
> > > > > > diff --git
> > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > index ef227af72348..328a7a8f26c5 100644
> > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > @@ -5,6 +5,7 @@
> > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > >   */
> > > > > >
> > > > > > +#include <linux/pm_runtime.h>
> > > > > >  #include "wave5-helper.h"
> > > > > >
> > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > vb2_queue *q, unsigned int count
> > > > > >
> > > > > >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && inst-
> > > > state
> > > > > > ==
> > > > > VPU_INST_STATE_NONE) {
> > > > > >  		struct dec_open_param open_param;
> > > > > > +		int err = 0;
> > > > > >
> > > > > >  		memset(&open_param, 0, sizeof(struct dec_open_param));
> > > > > >
> > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > +		if (err) {
> > > > > > +			dev_err(inst->dev->dev, "decoder runtime resume
> > > > > failed %d\n", err);
> > > > > > +			ret = -EINVAL;
> > > > > > +			goto return_buffers;
> > > > > > +		}
> > > > > > +
> > > > > >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > >  		if (ret)
> > > > > >  			goto return_buffers;
> > > > > > diff --git
> > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > index 761775216cd4..ff73d69de41c 100644
> > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > @@ -5,6 +5,7 @@
> > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > >   */
> > > > > >
> > > > > > +#include <linux/pm_runtime.h>
> > > > > >  #include "wave5-helper.h"
> > > > > >
> > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > vb2_queue *q, unsigned int count
> > > > > >
> > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > >  		struct enc_open_param open_param;
> > > > > > +		int err = 0;
> > > > > >
> > > > > >  		memset(&open_param, 0, sizeof(struct enc_open_param));
> > > > > >
> > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > +		if (err) {
> > > > > > +			dev_err(inst->dev->dev, "encoder runtime resume
> > > > > failed %d\n", err);
> > > > > > +			ret = -EINVAL;
> > > > > > +			goto return_buffers;
> > > > > > +		}
> > > > > > +
> > > > > >  		wave5_set_enc_openparam(&open_param, inst);
> > > > > >
> > > > > >  		ret = wave5_vpu_enc_open(inst, &open_param); diff --
> > > git
> > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > index 0d90b5820bef..f81409740a56 100644
> > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > @@ -10,6 +10,7 @@
> > > > > >  #include <linux/clk.h>
> > > > > >  #include <linux/firmware.h>
> > > > > >  #include <linux/interrupt.h>
> > > > > > +#include <linux/pm_runtime.h>
> > > > > >  #include "wave5-vpu.h"
> > > > > >  #include "wave5-regdefine.h"
> > > > > >  #include "wave5-vpuconfig.h"
> > > > > > @@ -117,6 +118,65 @@ static int wave5_vpu_load_firmware(struct
> > > > > > device
> > > > > *dev, const char *fw_name,
> > > > > >  	return 0;
> > > > > >  }
> > > > > >
> > > > > > +static __maybe_unused int wave5_pm_suspend(struct device *dev)
> {
> > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > +
> > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > +		return 0;
> > > > > > +
> > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > > > > > +
> > > > > > +	return 0;
> > > > > > +}
> > > > > > +
> > > > > > +static __maybe_unused int wave5_pm_resume(struct device *dev) {
> > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > +	int ret = 0;
> > > > > > +
> > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> > > > > > +	if (ret) {
> > > > > > +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> > > > > > +		return ret;
> > > > > > +	}
> > > > > > +
> > > > > > +	return ret;
> > > > > > +}
> > > > > > +
> > > > > > +static __maybe_unused int wave5_suspend(struct device *dev) {
> > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > +	struct vpu_instance *inst;
> > > > > > +
> > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > +
> > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > +
> > > > > > +static __maybe_unused int wave5_resume(struct device *dev) {
> > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > +	struct vpu_instance *inst;
> > > > > > +	int ret = 0;
> > > > > > +
> > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > +	if (ret < 0)
> > > > > > +		return ret;
> > > > > > +
> > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > +
> > > > > > +	return ret;
> > > > > > +}
> > > > >
> > > > > The functions wave5_suspend() and wave5_resume() are not just
> > > > > "maybe_unsued" but actually never used. What was the intention ?
> > > > > Considering the usage of __maybe_unused has been such a bad
> > > > > friend for this one, could you instead bracket the functions
> > > > > with an
> > > explicit ?
> > > > >
> > > > > #ifdef CONFIG_PM
> > > > > #endif
> > > > >
> > > > > This way the compiler will have a word if you implement
> > > > > something that you forgot to actually use.
> > > > >
> > > > > > +
> > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume,
> > > NULL) };
> > > > > > +
> > > > > >  static int wave5_vpu_probe(struct platform_device *pdev)  {
> > > > > >  	int ret;
> > > > > > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct
> > > > > > platform_device
> > > > > *pdev)
> > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
> > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev-
> > > > product_code);
> > > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n",
> > > > > > fw_revision);
> > > > > > +
> > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > > +
> > > > > >  	return 0;
> > > > > >
> > > > > >  err_enc_unreg:
> > > > > > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct
> > > > > > platform_device
> > > > > > *pdev)  {
> > > > > >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> > > > > >
> > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > +
> > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks); @@ -
> > > 281,6
> > > > > > +348,7 @@ static struct platform_driver wave5_vpu_driver = {
> > > > > >  	.driver = {
> > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > > > > > +		.pm = &wave5_pm_ops,
> > > > > >  		},
> > > > > >  	.probe = wave5_vpu_probe,
> > > > > >  	.remove = wave5_vpu_remove,
> > > > > > diff --git
> > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > @@ -6,6 +6,7 @@
> > > > > >   */
> > > > > >
> > > > > >  #include <linux/bug.h>
> > > > > > +#include <linux/pm_runtime.h>
> > > > > >  #include "wave5-vpuapi.h"
> > > > > >  #include "wave5-regdefine.h"
> > > > > >  #include "wave5.h"
> > > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct
> > > > > > vpu_instance *inst,
> > > > > > u32 *fail_res)
> > > > > >
> > > > > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task);
> > > > > >
> > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > +
> > > > > >  unlock_and_return:
> > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > >
> > > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct
> > > > > > vpu_instance *inst,
> > > > > > u32 *fail_res)
> > > > > >
> > > > > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
> > > > > >
> > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > >
> > > > > This seems very unnatural. We do the get() in
> > > > > "start_streaming()", but the
> > > > > put() is only done when the device is closed, or when the driver
> > > > > is removed. As this is not balanced, you seem to have to check
> > > > > the suspended condition all over the place.
> > > > >
> > > > > I think we could aim for start_streaming()/stop_streaming() for
> > > > > your get/put placement. At least they will be bound to an
> > > > > entirely balanced
> > > API.
> > > > > But then, a media player in paused sate will prevent that device
> > > > > from being suspended.
> > > > >
> > > > > If the HW is capable of preserving enough state, and From the
> > > > > short doc I have it gives me the impression it can preserve
> > > > > that, I'd suggest to follow what hantro driver is doing. What is
> > > > > does is that it will do get() in device_run(), and put()
> > > > > whenever the job completes. This driver has been designed so
> > > > > when there is no job, it means the firmware is currently idle
> > > > > and looking for more work. So it seems like the perfect moment to
> do suspend it.
> > > > >
> > > > > Nicolas
> > > > >
> > > > > > +
> > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > >
> > > > > >  	return 0;
> > > > > > diff --git a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct vpu_device
> > > > > > *vpu_dev, u32 *revision);
> > > > > >
> > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t size);
> > > > > >
> > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > +i_sleep_wake, const
> > > > > uint16_t *code,
> > > > > > +			 size_t size);
> > > > > > +
> > > > > >  int wave5_vpu_reset(struct device *dev, enum sw_reset_mode
> > > > > > reset_mode);
> > > > > >
> > > > > >  int wave5_vpu_build_up_dec_param(struct vpu_instance *inst,
> > > > > > struct dec_open_param *param);
> > > >
> >

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

Le mercredi 21 février 2024 à 02:27 +0000, jackson.lee a écrit :
> Hello Nicolas
> 
> 
> Thanks for your comment
> 
> > Le mardi 20 février 2024 à 05:12 +0000, jackson.lee a écrit :
> > > A pause is common state for media player, but our VPU could trigger an
> > interrupt regardless of a player state.
> > > So at a pause state if a power turns off, our driver becomes hang-up.
> > > I think power on/off should be controlled in the open/close function.
> > 
> > Please, avoid top posting, this breaks the discussion completely. Walk
> > through the comments, and reply to them underneath so we can have a proper
> > discussion.
> > 
> > In our experience, and with the rework of the driver we did during up-
> > streaming, there is no more un-solicited IRQ. The way the driver state
> > machine has been built, when the m2m framework is idle (no jobs are active
> > or pending), it means the firmware is stalled, waiting on some application
> > action. Combined with a timer of course, so we don't actively
> > suspend/resume, this seems adequate place to do power management. If that
> > application action never occurred passed a decent delay, we can suspend.
> > Later on, user actions like qbuf, will lead to
> > device_run() to be called again, and it would be a good place to resume it.
> 
> The above comment means like the below ?
> The wave5_vpu_dec_finish_decode is called by IRQ if there is a decoded frame.
> And at the end of the function, if there is no data queued, v4l2_m2m_job_finish is called.
> You means suspend is called in that function and if app feeds new data into v4l2 driver, the device_run then is called,
> In the device_run, resume is called.

That is the general idea. (Note that we don't forcefully suspect, for this type
of HW, we should configure a delay. Just saying in case someone miss-interpret
what was written here)

> 
> 
> I have a question, 
> If app is paused or resumed , the v4l2's vidioc_decoder_cmd is always called ?

There is no indication if an app is paused / resumed. Runtime PM in fact should
not depend on application actions in general. In the case of V4L2, most driver
will configure a delay (like 5 seconds). If the driver has been idle (no pending
work, no jobs) withing that delay, it will auto suspend. Then whenever new
activity comes, like a QBUF, we resume.

Nicolas

> 
> Thanks.
> 
> 
> > There is of course other places where you'll have to make sure the
> > hardware is resumed, like on close, as you want to remove the instance.
> > There is also small queries here and there that need to be surrounded with
> > resume/put, but with the redesign, most of the HW access now take place
> > inside device_run() only.
> > 
> > Open/Close is not invalid, but it has a lot of issues, as any random
> > application can endup disabling the kernel ability to save power.
> > Personally, I think we should at least give it a try, and document valid
> > reason not to do so if we find hardware issues. Otherwise, this sounds
> > like all we care is ticking the box "this driver has runtime PM" without
> > actually caring about effective power saving.
> > 
> > Nicolas
> > 
> > > 
> > > Thanks.
> > > 
> > > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > > > Hi Nicolas
> > > > > 
> > > > > 
> > > > > > This seems very unnatural. We do the get() in
> > > > > > "start_streaming()", but the
> > > > > > put() is only done when the device is closed, or when the driver
> > > > > > is removed. As this is not balanced, you seem to have to check
> > > > > > the suspended condition all over the place.
> > > > > > 
> > > > > > I think we could aim for start_streaming()/stop_streaming() for
> > > > > > your get/put placement. At least they will be bound to an
> > > > > > entirely balanced
> > > > API.
> > > > > > But then, a media player in paused sate will prevent that device
> > > > > > from being suspended.
> > > > > > 
> > > > > > If the HW is capable of preserving enough state, and From the
> > > > > > short doc I have it gives me the impression it can preserve
> > > > > > that, I'd suggest to follow what hantro driver is doing. What is
> > > > > > does is that it will do get() in device_run(), and put()
> > > > > > whenever the job completes. This driver has been designed so
> > > > > > when there is no job, it means the firmware is currently idle
> > > > > > and looking for more work. So it seems like the perfect moment to
> > do suspend it.
> > > > > > 
> > > > > 
> > > > > Thanks your comment,
> > > > > 
> > > > > Currently they are not balanced,
> > > > > If we puts "the put functon" into the stop_streaming, our hw is
> > > > > stalled
> > > > until doing wake-up command, so our v4l2 device become block.
> > > > > so I'd like to update the below instead of calling get at the
> > > > start_streaming function.
> > > > > 
> > > > > 
> > > > > @@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct file
> > > > > *filp)
> > > > > 
> > > > >         wave5_vdi_allocate_sram(inst->dev);
> > > > > 
> > > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > +       if (err) {
> > > > > +               dev_err(inst->dev->dev, "decoder runtime resume
> > > > failed %d\n", err);
> > > > > +               ret = -EINVAL;
> > > > > +               goto cleanup_inst;
> > > > > +       }
> > > > > +
> > > > >         return 0;
> > > > 
> > > > I guess we need to discuss the power management strategy for this
> > device.
> > > > If you do resume_and_get() in open(), and then put in close(), that
> > > > seems balanced. But in term of power saving, it might not be very
> > > > strong. If you have a media player that is set to pause and then
> > > > placed in the background, you still keep the IP running. This is
> > > > extremely common, since application cannot close their device
> > > > without loosing the reference frames, and thus having to do extra
> > > > work on resume to seek back to the previous sync point and drop
> > unneeded frames.
> > > > 
> > > > It seems like the whole point of asking the firmware to save the
> > > > state and suspend is to be able to do so while there is meaningful
> > > > sate in the firt place.
> > > > If we are to suspend only when there is no meaningful state, we
> > > > could just free all resources and power it off completely. (This is
> > > > just for illustration, its probably to slow to boot the firmware at
> > > > runtime)
> > > > 
> > > > 
> > > > I understand you suffered lockup with a start_streaming() for
> > > > resume_and_get(), and stop_streaming() for put(), this may simply
> > > > indicate that some hardware access are needed between these two. Can
> > > > you write down a power management plan that would effectively save
> > > > power in common use cases ? We can certainly help in refactoring the
> > code to make that happen.
> > > > 
> > > > Nicolas
> > > > 
> > > > > 
> > > > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> > > > > > > There are two device run-time PM callbacks defined in 'struct
> > > > > > dev_pm_ops'
> > > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > > (*runtime_resume)(struct device *dev);
> > > > > > 
> > > > > > I wonder how useful is it to teach everyone what the generic
> > > > > > 'struct dev_pm_ops'
> > > > > > contains. Perhaps you simply wanted that this patch implement
> > > > > > both suspend and resume ops ?
> > > > > > 
> > > > > > > 
> > > > > > > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > > > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > > > ---
> > > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > > +++++++++++++++++++
> > > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > > > > > > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> > > > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > > > 
> > > > > > > diff --git
> > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > @@ -503,6 +503,7 @@ int wave5_vpu_build_up_dec_param(struct
> > > > > > > vpu_instance
> > > > > > *inst,
> > > > > > >  	/* This register must be reset explicitly */
> > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > > > 
> > > > > > In some way, the relation between suspend and this register
> > > > > > write is not obvious. If its not related, please do this in its
> > own patch.
> > > > > > Otherwise you want to explain why you needed this (possibly just
> > > > > > in the commit message).
> > > > > > 
> > > > > > > 
> > > > > > >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE, true,
> > > > > > > NULL,
> > > > > > NULL);
> > > > > > >  	if (ret) {
> > > > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device
> > > > > > > *dev, u8 *fw,
> > > > > > size_t size)
> > > > > > >  	return setup_wave5_properties(dev);  }
> > > > > > > 
> > > > > > > -static int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > i_sleep_wake,
> > > > > > const uint16_t *code,
> > > > > > > -				size_t size)
> > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > +i_sleep_wake, const
> > > > > > uint16_t *code,
> > > > > > > +			 size_t size)
> > > > > > >  {
> > > > > > >  	u32 reg_val;
> > > > > > >  	struct vpu_buf *common_vb;
> > > > > > > diff --git
> > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > > index ef227af72348..328a7a8f26c5 100644
> > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > > @@ -5,6 +5,7 @@
> > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > >   */
> > > > > > > 
> > > > > > > +#include <linux/pm_runtime.h>
> > > > > > >  #include "wave5-helper.h"
> > > > > > > 
> > > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > > vb2_queue *q, unsigned int count
> > > > > > > 
> > > > > > >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && inst-
> > > > > state
> > > > > > > ==
> > > > > > VPU_INST_STATE_NONE) {
> > > > > > >  		struct dec_open_param open_param;
> > > > > > > +		int err = 0;
> > > > > > > 
> > > > > > >  		memset(&open_param, 0, sizeof(struct dec_open_param));
> > > > > > > 
> > > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > +		if (err) {
> > > > > > > +			dev_err(inst->dev->dev, "decoder runtime resume
> > > > > > failed %d\n", err);
> > > > > > > +			ret = -EINVAL;
> > > > > > > +			goto return_buffers;
> > > > > > > +		}
> > > > > > > +
> > > > > > >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > > >  		if (ret)
> > > > > > >  			goto return_buffers;
> > > > > > > diff --git
> > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > > index 761775216cd4..ff73d69de41c 100644
> > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > > @@ -5,6 +5,7 @@
> > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > >   */
> > > > > > > 
> > > > > > > +#include <linux/pm_runtime.h>
> > > > > > >  #include "wave5-helper.h"
> > > > > > > 
> > > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > > vb2_queue *q, unsigned int count
> > > > > > > 
> > > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > > >  		struct enc_open_param open_param;
> > > > > > > +		int err = 0;
> > > > > > > 
> > > > > > >  		memset(&open_param, 0, sizeof(struct enc_open_param));
> > > > > > > 
> > > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > +		if (err) {
> > > > > > > +			dev_err(inst->dev->dev, "encoder runtime resume
> > > > > > failed %d\n", err);
> > > > > > > +			ret = -EINVAL;
> > > > > > > +			goto return_buffers;
> > > > > > > +		}
> > > > > > > +
> > > > > > >  		wave5_set_enc_openparam(&open_param, inst);
> > > > > > > 
> > > > > > >  		ret = wave5_vpu_enc_open(inst, &open_param); diff --
> > > > git
> > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > index 0d90b5820bef..f81409740a56 100644
> > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > @@ -10,6 +10,7 @@
> > > > > > >  #include <linux/clk.h>
> > > > > > >  #include <linux/firmware.h>
> > > > > > >  #include <linux/interrupt.h>
> > > > > > > +#include <linux/pm_runtime.h>
> > > > > > >  #include "wave5-vpu.h"
> > > > > > >  #include "wave5-regdefine.h"
> > > > > > >  #include "wave5-vpuconfig.h"
> > > > > > > @@ -117,6 +118,65 @@ static int wave5_vpu_load_firmware(struct
> > > > > > > device
> > > > > > *dev, const char *fw_name,
> > > > > > >  	return 0;
> > > > > > >  }
> > > > > > > 
> > > > > > > +static __maybe_unused int wave5_pm_suspend(struct device *dev)
> > {
> > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > +
> > > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > > +		return 0;
> > > > > > > +
> > > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > > > > > > +
> > > > > > > +	return 0;
> > > > > > > +}
> > > > > > > +
> > > > > > > +static __maybe_unused int wave5_pm_resume(struct device *dev) {
> > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > +	int ret = 0;
> > > > > > > +
> > > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> > > > > > > +	if (ret) {
> > > > > > > +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> > > > > > > +		return ret;
> > > > > > > +	}
> > > > > > > +
> > > > > > > +	return ret;
> > > > > > > +}
> > > > > > > +
> > > > > > > +static __maybe_unused int wave5_suspend(struct device *dev) {
> > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > +	struct vpu_instance *inst;
> > > > > > > +
> > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > > +
> > > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > > +
> > > > > > > +static __maybe_unused int wave5_resume(struct device *dev) {
> > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > +	struct vpu_instance *inst;
> > > > > > > +	int ret = 0;
> > > > > > > +
> > > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > > +	if (ret < 0)
> > > > > > > +		return ret;
> > > > > > > +
> > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > > +
> > > > > > > +	return ret;
> > > > > > > +}
> > > > > > 
> > > > > > The functions wave5_suspend() and wave5_resume() are not just
> > > > > > "maybe_unsued" but actually never used. What was the intention ?
> > > > > > Considering the usage of __maybe_unused has been such a bad
> > > > > > friend for this one, could you instead bracket the functions
> > > > > > with an
> > > > explicit ?
> > > > > > 
> > > > > > #ifdef CONFIG_PM
> > > > > > #endif
> > > > > > 
> > > > > > This way the compiler will have a word if you implement
> > > > > > something that you forgot to actually use.
> > > > > > 
> > > > > > > +
> > > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume,
> > > > NULL) };
> > > > > > > +
> > > > > > >  static int wave5_vpu_probe(struct platform_device *pdev)  {
> > > > > > >  	int ret;
> > > > > > > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct
> > > > > > > platform_device
> > > > > > *pdev)
> > > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
> > > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev-
> > > > > product_code);
> > > > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n",
> > > > > > > fw_revision);
> > > > > > > +
> > > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > > > +
> > > > > > >  	return 0;
> > > > > > > 
> > > > > > >  err_enc_unreg:
> > > > > > > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct
> > > > > > > platform_device
> > > > > > > *pdev)  {
> > > > > > >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> > > > > > > 
> > > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > > +
> > > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > > >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks); @@ -
> > > > 281,6
> > > > > > > +348,7 @@ static struct platform_driver wave5_vpu_driver = {
> > > > > > >  	.driver = {
> > > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > > >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > > > > > > +		.pm = &wave5_pm_ops,
> > > > > > >  		},
> > > > > > >  	.probe = wave5_vpu_probe,
> > > > > > >  	.remove = wave5_vpu_remove,
> > > > > > > diff --git
> > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > > index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > > @@ -6,6 +6,7 @@
> > > > > > >   */
> > > > > > > 
> > > > > > >  #include <linux/bug.h>
> > > > > > > +#include <linux/pm_runtime.h>
> > > > > > >  #include "wave5-vpuapi.h"
> > > > > > >  #include "wave5-regdefine.h"
> > > > > > >  #include "wave5.h"
> > > > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct
> > > > > > > vpu_instance *inst,
> > > > > > > u32 *fail_res)
> > > > > > > 
> > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_dec_info->vb_task);
> > > > > > > 
> > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > +
> > > > > > >  unlock_and_return:
> > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > 
> > > > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct
> > > > > > > vpu_instance *inst,
> > > > > > > u32 *fail_res)
> > > > > > > 
> > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
> > > > > > > 
> > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > 
> > > > > > This seems very unnatural. We do the get() in
> > > > > > "start_streaming()", but the
> > > > > > put() is only done when the device is closed, or when the driver
> > > > > > is removed. As this is not balanced, you seem to have to check
> > > > > > the suspended condition all over the place.
> > > > > > 
> > > > > > I think we could aim for start_streaming()/stop_streaming() for
> > > > > > your get/put placement. At least they will be bound to an
> > > > > > entirely balanced
> > > > API.
> > > > > > But then, a media player in paused sate will prevent that device
> > > > > > from being suspended.
> > > > > > 
> > > > > > If the HW is capable of preserving enough state, and From the
> > > > > > short doc I have it gives me the impression it can preserve
> > > > > > that, I'd suggest to follow what hantro driver is doing. What is
> > > > > > does is that it will do get() in device_run(), and put()
> > > > > > whenever the job completes. This driver has been designed so
> > > > > > when there is no job, it means the firmware is currently idle
> > > > > > and looking for more work. So it seems like the perfect moment to
> > do suspend it.
> > > > > > 
> > > > > > Nicolas
> > > > > > 
> > > > > > > +
> > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > 
> > > > > > >  	return 0;
> > > > > > > diff --git a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct vpu_device
> > > > > > > *vpu_dev, u32 *revision);
> > > > > > > 
> > > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t size);
> > > > > > > 
> > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > +i_sleep_wake, const
> > > > > > uint16_t *code,
> > > > > > > +			 size_t size);
> > > > > > > +
> > > > > > >  int wave5_vpu_reset(struct device *dev, enum sw_reset_mode
> > > > > > > reset_mode);
> > > > > > > 
> > > > > > >  int wave5_vpu_build_up_dec_param(struct vpu_instance *inst,
> > > > > > > struct dec_open_param *param);
> > > > > 
> > > 
> 

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

> Le mercredi 21 février 2024 à 02:27 +0000, jackson.lee a écrit :
> > Hello Nicolas
> >
> >
> > Thanks for your comment
> >
> > > Le mardi 20 février 2024 à 05:12 +0000, jackson.lee a écrit :
> > > > A pause is common state for media player, but our VPU could
> > > > trigger an
> > > interrupt regardless of a player state.
> > > > So at a pause state if a power turns off, our driver becomes hang-up.
> > > > I think power on/off should be controlled in the open/close function.
> > >
> > > Please, avoid top posting, this breaks the discussion completely.
> > > Walk through the comments, and reply to them underneath so we can
> > > have a proper discussion.
> > >
> > > In our experience, and with the rework of the driver we did during
> > > up- streaming, there is no more un-solicited IRQ. The way the driver
> > > state machine has been built, when the m2m framework is idle (no
> > > jobs are active or pending), it means the firmware is stalled,
> > > waiting on some application action. Combined with a timer of course,
> > > so we don't actively suspend/resume, this seems adequate place to do
> > > power management. If that application action never occurred passed a
> decent delay, we can suspend.
> > > Later on, user actions like qbuf, will lead to
> > > device_run() to be called again, and it would be a good place to
> resume it.
> >
> > The above comment means like the below ?
> > The wave5_vpu_dec_finish_decode is called by IRQ if there is a decoded
> frame.
> > And at the end of the function, if there is no data queued,
> v4l2_m2m_job_finish is called.
> > You means suspend is called in that function and if app feeds new data
> > into v4l2 driver, the device_run then is called, In the device_run,
> resume is called.
> 
> That is the general idea. (Note that we don't forcefully suspect, for this
> type of HW, we should configure a delay. Just saying in case someone miss-
> interpret what was written here)
> 
> >
> >
> > I have a question,
> > If app is paused or resumed , the v4l2's vidioc_decoder_cmd is always
> called ?
> 
> There is no indication if an app is paused / resumed. Runtime PM in fact
> should not depend on application actions in general. In the case of V4L2,
> most driver will configure a delay (like 5 seconds). If the driver has
> been idle (no pending work, no jobs) withing that delay, it will auto
> suspend. Then whenever new activity comes, like a QBUF, we resume.
> 

The delay means a timer, so there is no input for 5 secs, then timeout callback is called, 
And suspend is set, if new activity comes, the device is resumed again ?
My understanding is correct ?

Thanks.
Jackson



> Nicolas
> 
> >
> > Thanks.
> >
> >
> > > There is of course other places where you'll have to make sure the
> > > hardware is resumed, like on close, as you want to remove the instance.
> > > There is also small queries here and there that need to be
> > > surrounded with resume/put, but with the redesign, most of the HW
> > > access now take place inside device_run() only.
> > >
> > > Open/Close is not invalid, but it has a lot of issues, as any random
> > > application can endup disabling the kernel ability to save power.
> > > Personally, I think we should at least give it a try, and document
> > > valid reason not to do so if we find hardware issues. Otherwise,
> > > this sounds like all we care is ticking the box "this driver has
> > > runtime PM" without actually caring about effective power saving.
> > >
> > > Nicolas
> > >
> > > >
> > > > Thanks.
> > > >
> > > > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > > > > Hi Nicolas
> > > > > >
> > > > > >
> > > > > > > This seems very unnatural. We do the get() in
> > > > > > > "start_streaming()", but the
> > > > > > > put() is only done when the device is closed, or when the
> > > > > > > driver is removed. As this is not balanced, you seem to have
> > > > > > > to check the suspended condition all over the place.
> > > > > > >
> > > > > > > I think we could aim for start_streaming()/stop_streaming()
> > > > > > > for your get/put placement. At least they will be bound to
> > > > > > > an entirely balanced
> > > > > API.
> > > > > > > But then, a media player in paused sate will prevent that
> > > > > > > device from being suspended.
> > > > > > >
> > > > > > > If the HW is capable of preserving enough state, and From
> > > > > > > the short doc I have it gives me the impression it can
> > > > > > > preserve that, I'd suggest to follow what hantro driver is
> > > > > > > doing. What is does is that it will do get() in
> > > > > > > device_run(), and put() whenever the job completes. This
> > > > > > > driver has been designed so when there is no job, it means
> > > > > > > the firmware is currently idle and looking for more work. So
> > > > > > > it seems like the perfect moment to
> > > do suspend it.
> > > > > > >
> > > > > >
> > > > > > Thanks your comment,
> > > > > >
> > > > > > Currently they are not balanced, If we puts "the put functon"
> > > > > > into the stop_streaming, our hw is stalled
> > > > > until doing wake-up command, so our v4l2 device become block.
> > > > > > so I'd like to update the below instead of calling get at the
> > > > > start_streaming function.
> > > > > >
> > > > > >
> > > > > > @@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct
> > > > > > file
> > > > > > *filp)
> > > > > >
> > > > > >         wave5_vdi_allocate_sram(inst->dev);
> > > > > >
> > > > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > +       if (err) {
> > > > > > +               dev_err(inst->dev->dev, "decoder runtime
> > > > > > + resume
> > > > > failed %d\n", err);
> > > > > > +               ret = -EINVAL;
> > > > > > +               goto cleanup_inst;
> > > > > > +       }
> > > > > > +
> > > > > >         return 0;
> > > > >
> > > > > I guess we need to discuss the power management strategy for
> > > > > this
> > > device.
> > > > > If you do resume_and_get() in open(), and then put in close(),
> > > > > that seems balanced. But in term of power saving, it might not
> > > > > be very strong. If you have a media player that is set to pause
> > > > > and then placed in the background, you still keep the IP
> > > > > running. This is extremely common, since application cannot
> > > > > close their device without loosing the reference frames, and
> > > > > thus having to do extra work on resume to seek back to the
> > > > > previous sync point and drop
> > > unneeded frames.
> > > > >
> > > > > It seems like the whole point of asking the firmware to save the
> > > > > state and suspend is to be able to do so while there is
> > > > > meaningful sate in the firt place.
> > > > > If we are to suspend only when there is no meaningful state, we
> > > > > could just free all resources and power it off completely. (This
> > > > > is just for illustration, its probably to slow to boot the
> > > > > firmware at
> > > > > runtime)
> > > > >
> > > > >
> > > > > I understand you suffered lockup with a start_streaming() for
> > > > > resume_and_get(), and stop_streaming() for put(), this may
> > > > > simply indicate that some hardware access are needed between
> > > > > these two. Can you write down a power management plan that would
> > > > > effectively save power in common use cases ? We can certainly
> > > > > help in refactoring the
> > > code to make that happen.
> > > > >
> > > > > Nicolas
> > > > >
> > > > > >
> > > > > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a
> écrit :
> > > > > > > > There are two device run-time PM callbacks defined in
> > > > > > > > 'struct
> > > > > > > dev_pm_ops'
> > > > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > > > (*runtime_resume)(struct device *dev);
> > > > > > >
> > > > > > > I wonder how useful is it to teach everyone what the generic
> > > > > > > 'struct dev_pm_ops'
> > > > > > > contains. Perhaps you simply wanted that this patch
> > > > > > > implement both suspend and resume ops ?
> > > > > > >
> > > > > > > >
> > > > > > > > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > > > > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > > > > ---
> > > > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > > > +++++++++++++++++++
> > > > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > > > > > > > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> > > > > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > > > >
> > > > > > > > diff --git
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > @@ -503,6 +503,7 @@ int
> > > > > > > > wave5_vpu_build_up_dec_param(struct
> > > > > > > > vpu_instance
> > > > > > > *inst,
> > > > > > > >  	/* This register must be reset explicitly */
> > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > > > >
> > > > > > > In some way, the relation between suspend and this register
> > > > > > > write is not obvious. If its not related, please do this in
> > > > > > > its
> > > own patch.
> > > > > > > Otherwise you want to explain why you needed this (possibly
> > > > > > > just in the commit message).
> > > > > > >
> > > > > > > >
> > > > > > > >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE,
> > > > > > > > true, NULL,
> > > > > > > NULL);
> > > > > > > >  	if (ret) {
> > > > > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct device
> > > > > > > > *dev, u8 *fw,
> > > > > > > size_t size)
> > > > > > > >  	return setup_wave5_properties(dev);  }
> > > > > > > >
> > > > > > > > -static int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > i_sleep_wake,
> > > > > > > const uint16_t *code,
> > > > > > > > -				size_t size)
> > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > +i_sleep_wake, const
> > > > > > > uint16_t *code,
> > > > > > > > +			 size_t size)
> > > > > > > >  {
> > > > > > > >  	u32 reg_val;
> > > > > > > >  	struct vpu_buf *common_vb; diff --git
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > > > index ef227af72348..328a7a8f26c5 100644
> > > > > > > > ---
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-d
> > > > > > > > +++ ec.c
> > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > >   */
> > > > > > > >
> > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > >  #include "wave5-helper.h"
> > > > > > > >
> > > > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > >
> > > > > > > >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE &&
> > > > > > > > inst-
> > > > > > state
> > > > > > > > ==
> > > > > > > VPU_INST_STATE_NONE) {
> > > > > > > >  		struct dec_open_param open_param;
> > > > > > > > +		int err = 0;
> > > > > > > >
> > > > > > > >  		memset(&open_param, 0, sizeof(struct
> dec_open_param));
> > > > > > > >
> > > > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > +		if (err) {
> > > > > > > > +			dev_err(inst->dev->dev, "decoder runtime
> resume
> > > > > > > failed %d\n", err);
> > > > > > > > +			ret = -EINVAL;
> > > > > > > > +			goto return_buffers;
> > > > > > > > +		}
> > > > > > > > +
> > > > > > > >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > > > >  		if (ret)
> > > > > > > >  			goto return_buffers;
> > > > > > > > diff --git
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > > > index 761775216cd4..ff73d69de41c 100644
> > > > > > > > ---
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-e
> > > > > > > > +++ nc.c
> > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > >   */
> > > > > > > >
> > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > >  #include "wave5-helper.h"
> > > > > > > >
> > > > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > >
> > > > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > > > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > > > >  		struct enc_open_param open_param;
> > > > > > > > +		int err = 0;
> > > > > > > >
> > > > > > > >  		memset(&open_param, 0, sizeof(struct
> enc_open_param));
> > > > > > > >
> > > > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > +		if (err) {
> > > > > > > > +			dev_err(inst->dev->dev, "encoder runtime
> resume
> > > > > > > failed %d\n", err);
> > > > > > > > +			ret = -EINVAL;
> > > > > > > > +			goto return_buffers;
> > > > > > > > +		}
> > > > > > > > +
> > > > > > > >  		wave5_set_enc_openparam(&open_param, inst);
> > > > > > > >
> > > > > > > >  		ret = wave5_vpu_enc_open(inst, &open_param);
> diff --
> > > > > git
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > index 0d90b5820bef..f81409740a56 100644
> > > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > @@ -10,6 +10,7 @@
> > > > > > > >  #include <linux/clk.h>
> > > > > > > >  #include <linux/firmware.h>  #include <linux/interrupt.h>
> > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > >  #include "wave5-vpu.h"
> > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > >  #include "wave5-vpuconfig.h"
> > > > > > > > @@ -117,6 +118,65 @@ static int
> > > > > > > > wave5_vpu_load_firmware(struct device
> > > > > > > *dev, const char *fw_name,
> > > > > > > >  	return 0;
> > > > > > > >  }
> > > > > > > >
> > > > > > > > +static __maybe_unused int wave5_pm_suspend(struct device
> > > > > > > > +*dev)
> > > {
> > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > +
> > > > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > > > +		return 0;
> > > > > > > > +
> > > > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > > > > > > > +
> > > > > > > > +	return 0;
> > > > > > > > +}
> > > > > > > > +
> > > > > > > > +static __maybe_unused int wave5_pm_resume(struct device
> *dev) {
> > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > +	int ret = 0;
> > > > > > > > +
> > > > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu-
> >clks);
> > > > > > > > +	if (ret) {
> > > > > > > > +		dev_err(dev, "Enabling clocks, fail: %d\n",
> ret);
> > > > > > > > +		return ret;
> > > > > > > > +	}
> > > > > > > > +
> > > > > > > > +	return ret;
> > > > > > > > +}
> > > > > > > > +
> > > > > > > > +static __maybe_unused int wave5_suspend(struct device *dev)
> {
> > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > +
> > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > > > +
> > > > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > > > +
> > > > > > > > +static __maybe_unused int wave5_resume(struct device *dev)
> {
> > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > +	int ret = 0;
> > > > > > > > +
> > > > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > > > +	if (ret < 0)
> > > > > > > > +		return ret;
> > > > > > > > +
> > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > > > +
> > > > > > > > +	return ret;
> > > > > > > > +}
> > > > > > >
> > > > > > > The functions wave5_suspend() and wave5_resume() are not
> > > > > > > just "maybe_unsued" but actually never used. What was the
> intention ?
> > > > > > > Considering the usage of __maybe_unused has been such a bad
> > > > > > > friend for this one, could you instead bracket the functions
> > > > > > > with an
> > > > > explicit ?
> > > > > > >
> > > > > > > #ifdef CONFIG_PM
> > > > > > > #endif
> > > > > > >
> > > > > > > This way the compiler will have a word if you implement
> > > > > > > something that you forgot to actually use.
> > > > > > >
> > > > > > > > +
> > > > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume,
> > > > > NULL) };
> > > > > > > > +
> > > > > > > >  static int wave5_vpu_probe(struct platform_device *pdev)  {
> > > > > > > >  	int ret;
> > > > > > > > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct
> > > > > > > > platform_device
> > > > > > > *pdev)
> > > > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ?
> "'DECODE'" : "");
> > > > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev-
> > > > > > product_code);
> > > > > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n",
> > > > > > > > fw_revision);
> > > > > > > > +
> > > > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > > > > +
> > > > > > > >  	return 0;
> > > > > > > >
> > > > > > > >  err_enc_unreg:
> > > > > > > > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct
> > > > > > > > platform_device
> > > > > > > > *pdev)  {
> > > > > > > >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> > > > > > > >
> > > > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > > > +
> > > > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > > > >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
> @@
> > > > > > > > -
> > > > > 281,6
> > > > > > > > +348,7 @@ static struct platform_driver wave5_vpu_driver =
> > > > > > > > +{
> > > > > > > >  	.driver = {
> > > > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > > > >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > > > > > > > +		.pm = &wave5_pm_ops,
> > > > > > > >  		},
> > > > > > > >  	.probe = wave5_vpu_probe,
> > > > > > > >  	.remove = wave5_vpu_remove, diff --git
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > > > index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > > > ---
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuap
> > > > > > > > +++ i.c
> > > > > > > > @@ -6,6 +6,7 @@
> > > > > > > >   */
> > > > > > > >
> > > > > > > >  #include <linux/bug.h>
> > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > >  #include "wave5-vpuapi.h"
> > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > >  #include "wave5.h"
> > > > > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct
> > > > > > > > vpu_instance *inst,
> > > > > > > > u32 *fail_res)
> > > > > > > >
> > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > &p_dec_info->vb_task);
> > > > > > > >
> > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > +
> > > > > > > >  unlock_and_return:
> > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > >
> > > > > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct
> > > > > > > > vpu_instance *inst,
> > > > > > > > u32 *fail_res)
> > > > > > > >
> > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > &p_enc_info->vb_task);
> > > > > > > >
> > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > >
> > > > > > > This seems very unnatural. We do the get() in
> > > > > > > "start_streaming()", but the
> > > > > > > put() is only done when the device is closed, or when the
> > > > > > > driver is removed. As this is not balanced, you seem to have
> > > > > > > to check the suspended condition all over the place.
> > > > > > >
> > > > > > > I think we could aim for start_streaming()/stop_streaming()
> > > > > > > for your get/put placement. At least they will be bound to
> > > > > > > an entirely balanced
> > > > > API.
> > > > > > > But then, a media player in paused sate will prevent that
> > > > > > > device from being suspended.
> > > > > > >
> > > > > > > If the HW is capable of preserving enough state, and From
> > > > > > > the short doc I have it gives me the impression it can
> > > > > > > preserve that, I'd suggest to follow what hantro driver is
> > > > > > > doing. What is does is that it will do get() in
> > > > > > > device_run(), and put() whenever the job completes. This
> > > > > > > driver has been designed so when there is no job, it means
> > > > > > > the firmware is currently idle and looking for more work. So
> > > > > > > it seems like the perfect moment to
> > > do suspend it.
> > > > > > >
> > > > > > > Nicolas
> > > > > > >
> > > > > > > > +
> > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > >
> > > > > > > >  	return 0;
> > > > > > > > diff --git
> > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct
> > > > > > > > vpu_device *vpu_dev, u32 *revision);
> > > > > > > >
> > > > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t
> > > > > > > > size);
> > > > > > > >
> > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > +i_sleep_wake, const
> > > > > > > uint16_t *code,
> > > > > > > > +			 size_t size);
> > > > > > > > +
> > > > > > > >  int wave5_vpu_reset(struct device *dev, enum
> > > > > > > > sw_reset_mode reset_mode);
> > > > > > > >
> > > > > > > >  int wave5_vpu_build_up_dec_param(struct vpu_instance
> > > > > > > > *inst, struct dec_open_param *param);
> > > > > >
> > > >
> >

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

> 
> > Le mercredi 21 février 2024 à 02:27 +0000, jackson.lee a écrit :
> > > Hello Nicolas
> > >
> > >
> > > Thanks for your comment
> > >
> > > > Le mardi 20 février 2024 à 05:12 +0000, jackson.lee a écrit :
> > > > > A pause is common state for media player, but our VPU could
> > > > > trigger an
> > > > interrupt regardless of a player state.
> > > > > So at a pause state if a power turns off, our driver becomes hang-
> up.
> > > > > I think power on/off should be controlled in the open/close
> function.
> > > >
> > > > Please, avoid top posting, this breaks the discussion completely.
> > > > Walk through the comments, and reply to them underneath so we can
> > > > have a proper discussion.
> > > >
> > > > In our experience, and with the rework of the driver we did during
> > > > up- streaming, there is no more un-solicited IRQ. The way the
> > > > driver state machine has been built, when the m2m framework is
> > > > idle (no jobs are active or pending), it means the firmware is
> > > > stalled, waiting on some application action. Combined with a timer
> > > > of course, so we don't actively suspend/resume, this seems
> > > > adequate place to do power management. If that application action
> > > > never occurred passed a
> > decent delay, we can suspend.
> > > > Later on, user actions like qbuf, will lead to
> > > > device_run() to be called again, and it would be a good place to
> > resume it.
> > >
> > > The above comment means like the below ?
> > > The wave5_vpu_dec_finish_decode is called by IRQ if there is a
> > > decoded
> > frame.
> > > And at the end of the function, if there is no data queued,
> > v4l2_m2m_job_finish is called.
> > > You means suspend is called in that function and if app feeds new
> > > data into v4l2 driver, the device_run then is called, In the
> > > device_run,
> > resume is called.
> >
> > That is the general idea. (Note that we don't forcefully suspect, for
> > this type of HW, we should configure a delay. Just saying in case
> > someone miss- interpret what was written here)
> >
> > >
> > >
> > > I have a question,
> > > If app is paused or resumed , the v4l2's vidioc_decoder_cmd is
> > > always
> > called ?
> >
> > There is no indication if an app is paused / resumed. Runtime PM in
> > fact should not depend on application actions in general. In the case
> > of V4L2, most driver will configure a delay (like 5 seconds). If the
> > driver has been idle (no pending work, no jobs) withing that delay, it
> > will auto suspend. Then whenever new activity comes, like a QBUF, we
> resume.
> >

Below is a pseudo code for configuring a delay for the Run time suspend/resume. 
The logic you are saying means the below ?

Driver_probe 
{
 	pm_runtime_set_autosuspend_delay(vpu->dev, 100);
	pm_runtime_use_autosuspend(vpu->dev);
	pm_runtime_enable(vpu->dev);

}

Device_run 
{

	pm_runtime_resume_and_get(ctx->dev->dev);
}


Finish_job 
{
	pm_runtime_mark_last_busy(vpu->dev);
	pm_runtime_put_autosuspend(vpu->dev);

}



Thanks.
Jackson 

> 
> The delay means a timer, so there is no input for 5 secs, then timeout
> callback is called, And suspend is set, if new activity comes, the device
> is resumed again ?
> My understanding is correct ?
> 


> 
> 
> > Nicolas
> >
> > >
> > > Thanks.
> > >
> > >
> > > > There is of course other places where you'll have to make sure the
> > > > hardware is resumed, like on close, as you want to remove the
> instance.
> > > > There is also small queries here and there that need to be
> > > > surrounded with resume/put, but with the redesign, most of the HW
> > > > access now take place inside device_run() only.
> > > >
> > > > Open/Close is not invalid, but it has a lot of issues, as any
> > > > random application can endup disabling the kernel ability to save
> power.
> > > > Personally, I think we should at least give it a try, and document
> > > > valid reason not to do so if we find hardware issues. Otherwise,
> > > > this sounds like all we care is ticking the box "this driver has
> > > > runtime PM" without actually caring about effective power saving.
> > > >
> > > > Nicolas
> > > >
> > > > >
> > > > > Thanks.
> > > > >
> > > > > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > > > > > Hi Nicolas
> > > > > > >
> > > > > > >
> > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > "start_streaming()", but the
> > > > > > > > put() is only done when the device is closed, or when the
> > > > > > > > driver is removed. As this is not balanced, you seem to
> > > > > > > > have to check the suspended condition all over the place.
> > > > > > > >
> > > > > > > > I think we could aim for
> > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > for your get/put placement. At least they will be bound to
> > > > > > > > an entirely balanced
> > > > > > API.
> > > > > > > > But then, a media player in paused sate will prevent that
> > > > > > > > device from being suspended.
> > > > > > > >
> > > > > > > > If the HW is capable of preserving enough state, and From
> > > > > > > > the short doc I have it gives me the impression it can
> > > > > > > > preserve that, I'd suggest to follow what hantro driver is
> > > > > > > > doing. What is does is that it will do get() in
> > > > > > > > device_run(), and put() whenever the job completes. This
> > > > > > > > driver has been designed so when there is no job, it means
> > > > > > > > the firmware is currently idle and looking for more work.
> > > > > > > > So it seems like the perfect moment to
> > > > do suspend it.
> > > > > > > >
> > > > > > >
> > > > > > > Thanks your comment,
> > > > > > >
> > > > > > > Currently they are not balanced, If we puts "the put functon"
> > > > > > > into the stop_streaming, our hw is stalled
> > > > > > until doing wake-up command, so our v4l2 device become block.
> > > > > > > so I'd like to update the below instead of calling get at
> > > > > > > the
> > > > > > start_streaming function.
> > > > > > >
> > > > > > >
> > > > > > > @@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct
> > > > > > > file
> > > > > > > *filp)
> > > > > > >
> > > > > > >         wave5_vdi_allocate_sram(inst->dev);
> > > > > > >
> > > > > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > +       if (err) {
> > > > > > > +               dev_err(inst->dev->dev, "decoder runtime
> > > > > > > + resume
> > > > > > failed %d\n", err);
> > > > > > > +               ret = -EINVAL;
> > > > > > > +               goto cleanup_inst;
> > > > > > > +       }
> > > > > > > +
> > > > > > >         return 0;
> > > > > >
> > > > > > I guess we need to discuss the power management strategy for
> > > > > > this
> > > > device.
> > > > > > If you do resume_and_get() in open(), and then put in close(),
> > > > > > that seems balanced. But in term of power saving, it might not
> > > > > > be very strong. If you have a media player that is set to
> > > > > > pause and then placed in the background, you still keep the IP
> > > > > > running. This is extremely common, since application cannot
> > > > > > close their device without loosing the reference frames, and
> > > > > > thus having to do extra work on resume to seek back to the
> > > > > > previous sync point and drop
> > > > unneeded frames.
> > > > > >
> > > > > > It seems like the whole point of asking the firmware to save
> > > > > > the state and suspend is to be able to do so while there is
> > > > > > meaningful sate in the firt place.
> > > > > > If we are to suspend only when there is no meaningful state,
> > > > > > we could just free all resources and power it off completely.
> > > > > > (This is just for illustration, its probably to slow to boot
> > > > > > the firmware at
> > > > > > runtime)
> > > > > >
> > > > > >
> > > > > > I understand you suffered lockup with a start_streaming() for
> > > > > > resume_and_get(), and stop_streaming() for put(), this may
> > > > > > simply indicate that some hardware access are needed between
> > > > > > these two. Can you write down a power management plan that
> > > > > > would effectively save power in common use cases ? We can
> > > > > > certainly help in refactoring the
> > > > code to make that happen.
> > > > > >
> > > > > > Nicolas
> > > > > >
> > > > > > >
> > > > > > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a
> > écrit :
> > > > > > > > > There are two device run-time PM callbacks defined in
> > > > > > > > > 'struct
> > > > > > > > dev_pm_ops'
> > > > > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > > > > (*runtime_resume)(struct device *dev);
> > > > > > > >
> > > > > > > > I wonder how useful is it to teach everyone what the
> > > > > > > > generic 'struct dev_pm_ops'
> > > > > > > > contains. Perhaps you simply wanted that this patch
> > > > > > > > implement both suspend and resume ops ?
> > > > > > > >
> > > > > > > > >
> > > > > > > > > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > > > > > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > > > > > ---
> > > > > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > > > > +++++++++++++++++++
> > > > > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > > > > > > > > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> > > > > > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > > > > >
> > > > > > > > > diff --git
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > > index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > > > > ---
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > +++ c
> > > > > > > > > @@ -503,6 +503,7 @@ int
> > > > > > > > > wave5_vpu_build_up_dec_param(struct
> > > > > > > > > vpu_instance
> > > > > > > > *inst,
> > > > > > > > >  	/* This register must be reset explicitly */
> > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > > > > >
> > > > > > > > In some way, the relation between suspend and this
> > > > > > > > register write is not obvious. If its not related, please
> > > > > > > > do this in its
> > > > own patch.
> > > > > > > > Otherwise you want to explain why you needed this
> > > > > > > > (possibly just in the commit message).
> > > > > > > >
> > > > > > > > >
> > > > > > > > >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE,
> > > > > > > > > true, NULL,
> > > > > > > > NULL);
> > > > > > > > >  	if (ret) {
> > > > > > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct
> > > > > > > > > device *dev, u8 *fw,
> > > > > > > > size_t size)
> > > > > > > > >  	return setup_wave5_properties(dev);  }
> > > > > > > > >
> > > > > > > > > -static int wave5_vpu_sleep_wake(struct device *dev,
> > > > > > > > > bool i_sleep_wake,
> > > > > > > > const uint16_t *code,
> > > > > > > > > -				size_t size)
> > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > +i_sleep_wake, const
> > > > > > > > uint16_t *code,
> > > > > > > > > +			 size_t size)
> > > > > > > > >  {
> > > > > > > > >  	u32 reg_val;
> > > > > > > > >  	struct vpu_buf *common_vb; diff --git
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec
> > > > > > > > > .c
> > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec
> > > > > > > > > .c index ef227af72348..328a7a8f26c5 100644
> > > > > > > > > ---
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec
> > > > > > > > > .c
> > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > +++ -d
> > > > > > > > > +++ ec.c
> > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > >   */
> > > > > > > > >
> > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > >
> > > > > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > >
> > > > > > > > >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE &&
> > > > > > > > > inst-
> > > > > > > state
> > > > > > > > > ==
> > > > > > > > VPU_INST_STATE_NONE) {
> > > > > > > > >  		struct dec_open_param open_param;
> > > > > > > > > +		int err = 0;
> > > > > > > > >
> > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > dec_open_param));
> > > > > > > > >
> > > > > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > > +		if (err) {
> > > > > > > > > +			dev_err(inst->dev->dev, "decoder runtime
> > resume
> > > > > > > > failed %d\n", err);
> > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > +			goto return_buffers;
> > > > > > > > > +		}
> > > > > > > > > +
> > > > > > > > >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > > > > >  		if (ret)
> > > > > > > > >  			goto return_buffers; diff --git
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc
> > > > > > > > > .c
> > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc
> > > > > > > > > .c index 761775216cd4..ff73d69de41c 100644
> > > > > > > > > ---
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc
> > > > > > > > > .c
> > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > +++ -e
> > > > > > > > > +++ nc.c
> > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > >   */
> > > > > > > > >
> > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > >
> > > > > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > >
> > > > > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > > > > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > > > > >  		struct enc_open_param open_param;
> > > > > > > > > +		int err = 0;
> > > > > > > > >
> > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > enc_open_param));
> > > > > > > > >
> > > > > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > > +		if (err) {
> > > > > > > > > +			dev_err(inst->dev->dev, "encoder runtime
> > resume
> > > > > > > > failed %d\n", err);
> > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > +			goto return_buffers;
> > > > > > > > > +		}
> > > > > > > > > +
> > > > > > > > >  		wave5_set_enc_openparam(&open_param, inst);
> > > > > > > > >
> > > > > > > > >  		ret = wave5_vpu_enc_open(inst, &open_param);
> > diff --
> > > > > > git
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > > index 0d90b5820bef..f81409740a56 100644
> > > > > > > > > ---
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > +++ .c
> > > > > > > > > @@ -10,6 +10,7 @@
> > > > > > > > >  #include <linux/clk.h>
> > > > > > > > >  #include <linux/firmware.h>  #include
> > > > > > > > > <linux/interrupt.h>
> > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > >  #include "wave5-vpu.h"
> > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > >  #include "wave5-vpuconfig.h"
> > > > > > > > > @@ -117,6 +118,65 @@ static int
> > > > > > > > > wave5_vpu_load_firmware(struct device
> > > > > > > > *dev, const char *fw_name,
> > > > > > > > >  	return 0;
> > > > > > > > >  }
> > > > > > > > >
> > > > > > > > > +static __maybe_unused int wave5_pm_suspend(struct
> > > > > > > > > +device
> > > > > > > > > +*dev)
> > > > {
> > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > +
> > > > > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > > > > +		return 0;
> > > > > > > > > +
> > > > > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > > > > > > > > +
> > > > > > > > > +	return 0;
> > > > > > > > > +}
> > > > > > > > > +
> > > > > > > > > +static __maybe_unused int wave5_pm_resume(struct device
> > *dev) {
> > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > +	int ret = 0;
> > > > > > > > > +
> > > > > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu-
> > >clks);
> > > > > > > > > +	if (ret) {
> > > > > > > > > +		dev_err(dev, "Enabling clocks, fail: %d\n",
> > ret);
> > > > > > > > > +		return ret;
> > > > > > > > > +	}
> > > > > > > > > +
> > > > > > > > > +	return ret;
> > > > > > > > > +}
> > > > > > > > > +
> > > > > > > > > +static __maybe_unused int wave5_suspend(struct device
> > > > > > > > > +*dev)
> > {
> > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > +
> > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > > > > +
> > > > > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > > > > +
> > > > > > > > > +static __maybe_unused int wave5_resume(struct device
> > > > > > > > > +*dev)
> > {
> > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > +	int ret = 0;
> > > > > > > > > +
> > > > > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > > > > +	if (ret < 0)
> > > > > > > > > +		return ret;
> > > > > > > > > +
> > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > > > > +
> > > > > > > > > +	return ret;
> > > > > > > > > +}
> > > > > > > >
> > > > > > > > The functions wave5_suspend() and wave5_resume() are not
> > > > > > > > just "maybe_unsued" but actually never used. What was the
> > intention ?
> > > > > > > > Considering the usage of __maybe_unused has been such a
> > > > > > > > bad friend for this one, could you instead bracket the
> > > > > > > > functions with an
> > > > > > explicit ?
> > > > > > > >
> > > > > > > > #ifdef CONFIG_PM
> > > > > > > > #endif
> > > > > > > >
> > > > > > > > This way the compiler will have a word if you implement
> > > > > > > > something that you forgot to actually use.
> > > > > > > >
> > > > > > > > > +
> > > > > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume,
> > > > > > NULL) };
> > > > > > > > > +
> > > > > > > > >  static int wave5_vpu_probe(struct platform_device *pdev)
> {
> > > > > > > > >  	int ret;
> > > > > > > > > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct
> > > > > > > > > platform_device
> > > > > > > > *pdev)
> > > > > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ?
> > "'DECODE'" : "");
> > > > > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev-
> > > > > > > product_code);
> > > > > > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n",
> > > > > > > > > fw_revision);
> > > > > > > > > +
> > > > > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > > > > > +
> > > > > > > > >  	return 0;
> > > > > > > > >
> > > > > > > > >  err_enc_unreg:
> > > > > > > > > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct
> > > > > > > > > platform_device
> > > > > > > > > *pdev)  {
> > > > > > > > >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> > > > > > > > >
> > > > > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > > > > +
> > > > > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > > > > >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
> > @@
> > > > > > > > > -
> > > > > > 281,6
> > > > > > > > > +348,7 @@ static struct platform_driver wave5_vpu_driver
> > > > > > > > > += {
> > > > > > > > >  	.driver = {
> > > > > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > > > > >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > > > > > > > > +		.pm = &wave5_pm_ops,
> > > > > > > > >  		},
> > > > > > > > >  	.probe = wave5_vpu_probe,
> > > > > > > > >  	.remove = wave5_vpu_remove, diff --git
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.
> > > > > > > > > c
> > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.
> > > > > > > > > c index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > > > > ---
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.
> > > > > > > > > c
> > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > +++ ap
> > > > > > > > > +++ i.c
> > > > > > > > > @@ -6,6 +6,7 @@
> > > > > > > > >   */
> > > > > > > > >
> > > > > > > > >  #include <linux/bug.h>
> > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > >  #include "wave5-vpuapi.h"
> > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > >  #include "wave5.h"
> > > > > > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct
> > > > > > > > > vpu_instance *inst,
> > > > > > > > > u32 *fail_res)
> > > > > > > > >
> > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > &p_dec_info->vb_task);
> > > > > > > > >
> > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > +
> > > > > > > > >  unlock_and_return:
> > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > >
> > > > > > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct
> > > > > > > > > vpu_instance *inst,
> > > > > > > > > u32 *fail_res)
> > > > > > > > >
> > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > &p_enc_info->vb_task);
> > > > > > > > >
> > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > >
> > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > "start_streaming()", but the
> > > > > > > > put() is only done when the device is closed, or when the
> > > > > > > > driver is removed. As this is not balanced, you seem to
> > > > > > > > have to check the suspended condition all over the place.
> > > > > > > >
> > > > > > > > I think we could aim for
> > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > for your get/put placement. At least they will be bound to
> > > > > > > > an entirely balanced
> > > > > > API.
> > > > > > > > But then, a media player in paused sate will prevent that
> > > > > > > > device from being suspended.
> > > > > > > >
> > > > > > > > If the HW is capable of preserving enough state, and From
> > > > > > > > the short doc I have it gives me the impression it can
> > > > > > > > preserve that, I'd suggest to follow what hantro driver is
> > > > > > > > doing. What is does is that it will do get() in
> > > > > > > > device_run(), and put() whenever the job completes. This
> > > > > > > > driver has been designed so when there is no job, it means
> > > > > > > > the firmware is currently idle and looking for more work.
> > > > > > > > So it seems like the perfect moment to
> > > > do suspend it.
> > > > > > > >
> > > > > > > > Nicolas
> > > > > > > >
> > > > > > > > > +
> > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > >
> > > > > > > > >  	return 0;
> > > > > > > > > diff --git
> > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct
> > > > > > > > > vpu_device *vpu_dev, u32 *revision);
> > > > > > > > >
> > > > > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t
> > > > > > > > > size);
> > > > > > > > >
> > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > +i_sleep_wake, const
> > > > > > > > uint16_t *code,
> > > > > > > > > +			 size_t size);
> > > > > > > > > +
> > > > > > > > >  int wave5_vpu_reset(struct device *dev, enum
> > > > > > > > > sw_reset_mode reset_mode);
> > > > > > > > >
> > > > > > > > >  int wave5_vpu_build_up_dec_param(struct vpu_instance
> > > > > > > > > *inst, struct dec_open_param *param);
> > > > > > >
> > > > >
> > >

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

Hello Nicolas

There is nothing in your reply, can you send us your comment again ?


thanks

> -----Original Message-----
> From: Nicolas Dufresne <nicolas@ndufresne.ca>
> Sent: Friday, February 23, 2024 5:56 AM
> To: jackson.lee <jackson.lee@chipsnmedia.com>; mchehab@kernel.org; linux-
> media@vger.kernel.org; linux-kernel@vger.kernel.org; Nas Chung
> <nas.chung@chipsnmedia.com>
> Cc: lafley.kim <lafley.kim@chipsnmedia.com>; b-brnich@ti.com
> Subject: Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.
> 

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

Hi,

Le vendredi 23 février 2024 à 01:06 +0000, jackson.lee a écrit :
> Hello Nicolas
> 
> There is nothing in your reply, can you send us your comment again ?

Thanks for the notification, I've suffered issues with my email client it seems,
I will reply again and hope it will work this time.

Nicolas

> 
> 
> thanks
> 
> > -----Original Message-----
> > From: Nicolas Dufresne <nicolas@ndufresne.ca>
> > Sent: Friday, February 23, 2024 5:56 AM
> > To: jackson.lee <jackson.lee@chipsnmedia.com>; mchehab@kernel.org; linux-
> > media@vger.kernel.org; linux-kernel@vger.kernel.org; Nas Chung
> > <nas.chung@chipsnmedia.com>
> > Cc: lafley.kim <lafley.kim@chipsnmedia.com>; b-brnich@ti.com
> > Subject: Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.
> > 
> 

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

Le jeudi 22 février 2024 à 03:57 +0000, jackson.lee a écrit :
> > 
> > > Le mercredi 21 février 2024 à 02:27 +0000, jackson.lee a écrit :
> > > > Hello Nicolas
> > > > 
> > > > 
> > > > Thanks for your comment
> > > > 
> > > > > Le mardi 20 février 2024 à 05:12 +0000, jackson.lee a écrit :
> > > > > > A pause is common state for media player, but our VPU could
> > > > > > trigger an
> > > > > interrupt regardless of a player state.
> > > > > > So at a pause state if a power turns off, our driver becomes hang-
> > up.
> > > > > > I think power on/off should be controlled in the open/close
> > function.
> > > > > 
> > > > > Please, avoid top posting, this breaks the discussion completely.
> > > > > Walk through the comments, and reply to them underneath so we can
> > > > > have a proper discussion.
> > > > > 
> > > > > In our experience, and with the rework of the driver we did during
> > > > > up- streaming, there is no more un-solicited IRQ. The way the
> > > > > driver state machine has been built, when the m2m framework is
> > > > > idle (no jobs are active or pending), it means the firmware is
> > > > > stalled, waiting on some application action. Combined with a timer
> > > > > of course, so we don't actively suspend/resume, this seems
> > > > > adequate place to do power management. If that application action
> > > > > never occurred passed a
> > > decent delay, we can suspend.
> > > > > Later on, user actions like qbuf, will lead to
> > > > > device_run() to be called again, and it would be a good place to
> > > resume it.
> > > > 
> > > > The above comment means like the below ?
> > > > The wave5_vpu_dec_finish_decode is called by IRQ if there is a
> > > > decoded
> > > frame.
> > > > And at the end of the function, if there is no data queued,
> > > v4l2_m2m_job_finish is called.
> > > > You means suspend is called in that function and if app feeds new
> > > > data into v4l2 driver, the device_run then is called, In the
> > > > device_run,
> > > resume is called.
> > > 
> > > That is the general idea. (Note that we don't forcefully suspect, for
> > > this type of HW, we should configure a delay. Just saying in case
> > > someone miss- interpret what was written here)
> > > 
> > > > 
> > > > 
> > > > I have a question,
> > > > If app is paused or resumed , the v4l2's vidioc_decoder_cmd is
> > > > always
> > > called ?
> > > 
> > > There is no indication if an app is paused / resumed. Runtime PM in
> > > fact should not depend on application actions in general. In the case
> > > of V4L2, most driver will configure a delay (like 5 seconds). If the
> > > driver has been idle (no pending work, no jobs) withing that delay, it
> > > will auto suspend. Then whenever new activity comes, like a QBUF, we
> > resume.
> > > 
> 
> Below is a pseudo code for configuring a delay for the Run time suspend/resume. 
> The logic you are saying means the below ?
> 
> Driver_probe 
> {
>  	pm_runtime_set_autosuspend_delay(vpu->dev, 100);
> 	pm_runtime_use_autosuspend(vpu->dev);
> 	pm_runtime_enable(vpu->dev);
> 
> }
> 
> Device_run 
> {
> 
> 	pm_runtime_resume_and_get(ctx->dev->dev);
> }
> 
> 
> Finish_job 
> {
> 	pm_runtime_mark_last_busy(vpu->dev);
> 	pm_runtime_put_autosuspend(vpu->dev);
> 
> }
> 

This is aligned with what I had in mind. There will be few cases were you'll
need to ensure the hardware is active outside of this (open() and close() are
examples), but otherwise this should in my perception gives the best power
saving. And finally, the delay might need tuning, at first 100ms seems a bit
low, the delay needs to be balanced against the suspend cost.

Nicolas

> 
> 
> Thanks.
> Jackson 
> 
> > 
> > The delay means a timer, so there is no input for 5 secs, then timeout
> > callback is called, And suspend is set, if new activity comes, the device
> > is resumed again ?
> > My understanding is correct ?
> > 
> 
> 
> > 
> > 
> > > Nicolas
> > > 
> > > > 
> > > > Thanks.
> > > > 
> > > > 
> > > > > There is of course other places where you'll have to make sure the
> > > > > hardware is resumed, like on close, as you want to remove the
> > instance.
> > > > > There is also small queries here and there that need to be
> > > > > surrounded with resume/put, but with the redesign, most of the HW
> > > > > access now take place inside device_run() only.
> > > > > 
> > > > > Open/Close is not invalid, but it has a lot of issues, as any
> > > > > random application can endup disabling the kernel ability to save
> > power.
> > > > > Personally, I think we should at least give it a try, and document
> > > > > valid reason not to do so if we find hardware issues. Otherwise,
> > > > > this sounds like all we care is ticking the box "this driver has
> > > > > runtime PM" without actually caring about effective power saving.
> > > > > 
> > > > > Nicolas
> > > > > 
> > > > > > 
> > > > > > Thanks.
> > > > > > 
> > > > > > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > > > > > > Hi Nicolas
> > > > > > > > 
> > > > > > > > 
> > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > "start_streaming()", but the
> > > > > > > > > put() is only done when the device is closed, or when the
> > > > > > > > > driver is removed. As this is not balanced, you seem to
> > > > > > > > > have to check the suspended condition all over the place.
> > > > > > > > > 
> > > > > > > > > I think we could aim for
> > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > for your get/put placement. At least they will be bound to
> > > > > > > > > an entirely balanced
> > > > > > > API.
> > > > > > > > > But then, a media player in paused sate will prevent that
> > > > > > > > > device from being suspended.
> > > > > > > > > 
> > > > > > > > > If the HW is capable of preserving enough state, and From
> > > > > > > > > the short doc I have it gives me the impression it can
> > > > > > > > > preserve that, I'd suggest to follow what hantro driver is
> > > > > > > > > doing. What is does is that it will do get() in
> > > > > > > > > device_run(), and put() whenever the job completes. This
> > > > > > > > > driver has been designed so when there is no job, it means
> > > > > > > > > the firmware is currently idle and looking for more work.
> > > > > > > > > So it seems like the perfect moment to
> > > > > do suspend it.
> > > > > > > > > 
> > > > > > > > 
> > > > > > > > Thanks your comment,
> > > > > > > > 
> > > > > > > > Currently they are not balanced, If we puts "the put functon"
> > > > > > > > into the stop_streaming, our hw is stalled
> > > > > > > until doing wake-up command, so our v4l2 device become block.
> > > > > > > > so I'd like to update the below instead of calling get at
> > > > > > > > the
> > > > > > > start_streaming function.
> > > > > > > > 
> > > > > > > > 
> > > > > > > > @@ -1867,6 +1868,13 @@ static int wave5_vpu_open_dec(struct
> > > > > > > > file
> > > > > > > > *filp)
> > > > > > > > 
> > > > > > > >         wave5_vdi_allocate_sram(inst->dev);
> > > > > > > > 
> > > > > > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > +       if (err) {
> > > > > > > > +               dev_err(inst->dev->dev, "decoder runtime
> > > > > > > > + resume
> > > > > > > failed %d\n", err);
> > > > > > > > +               ret = -EINVAL;
> > > > > > > > +               goto cleanup_inst;
> > > > > > > > +       }
> > > > > > > > +
> > > > > > > >         return 0;
> > > > > > > 
> > > > > > > I guess we need to discuss the power management strategy for
> > > > > > > this
> > > > > device.
> > > > > > > If you do resume_and_get() in open(), and then put in close(),
> > > > > > > that seems balanced. But in term of power saving, it might not
> > > > > > > be very strong. If you have a media player that is set to
> > > > > > > pause and then placed in the background, you still keep the IP
> > > > > > > running. This is extremely common, since application cannot
> > > > > > > close their device without loosing the reference frames, and
> > > > > > > thus having to do extra work on resume to seek back to the
> > > > > > > previous sync point and drop
> > > > > unneeded frames.
> > > > > > > 
> > > > > > > It seems like the whole point of asking the firmware to save
> > > > > > > the state and suspend is to be able to do so while there is
> > > > > > > meaningful sate in the firt place.
> > > > > > > If we are to suspend only when there is no meaningful state,
> > > > > > > we could just free all resources and power it off completely.
> > > > > > > (This is just for illustration, its probably to slow to boot
> > > > > > > the firmware at
> > > > > > > runtime)
> > > > > > > 
> > > > > > > 
> > > > > > > I understand you suffered lockup with a start_streaming() for
> > > > > > > resume_and_get(), and stop_streaming() for put(), this may
> > > > > > > simply indicate that some hardware access are needed between
> > > > > > > these two. Can you write down a power management plan that
> > > > > > > would effectively save power in common use cases ? We can
> > > > > > > certainly help in refactoring the
> > > > > code to make that happen.
> > > > > > > 
> > > > > > > Nicolas
> > > > > > > 
> > > > > > > > 
> > > > > > > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a
> > > écrit :
> > > > > > > > > > There are two device run-time PM callbacks defined in
> > > > > > > > > > 'struct
> > > > > > > > > dev_pm_ops'
> > > > > > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > > > > > (*runtime_resume)(struct device *dev);
> > > > > > > > > 
> > > > > > > > > I wonder how useful is it to teach everyone what the
> > > > > > > > > generic 'struct dev_pm_ops'
> > > > > > > > > contains. Perhaps you simply wanted that this patch
> > > > > > > > > implement both suspend and resume ops ?
> > > > > > > > > 
> > > > > > > > > > 
> > > > > > > > > > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > > > > > > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > > > > > > ---
> > > > > > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > > > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > > > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > > > > > +++++++++++++++++++
> > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7 ++
> > > > > > > > > > .../media/platform/chips-media/wave5/wave5.h  |  3 +
> > > > > > > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > > > > > > 
> > > > > > > > > > diff --git
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > > > index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > > > > > ---
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > +++ c
> > > > > > > > > > @@ -503,6 +503,7 @@ int
> > > > > > > > > > wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > vpu_instance
> > > > > > > > > *inst,
> > > > > > > > > >  	/* This register must be reset explicitly */
> > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > > > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > > > > > > 
> > > > > > > > > In some way, the relation between suspend and this
> > > > > > > > > register write is not obvious. If its not related, please
> > > > > > > > > do this in its
> > > > > own patch.
> > > > > > > > > Otherwise you want to explain why you needed this
> > > > > > > > > (possibly just in the commit message).
> > > > > > > > > 
> > > > > > > > > > 
> > > > > > > > > >  	ret = send_firmware_command(inst, W5_CREATE_INSTANCE,
> > > > > > > > > > true, NULL,
> > > > > > > > > NULL);
> > > > > > > > > >  	if (ret) {
> > > > > > > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct
> > > > > > > > > > device *dev, u8 *fw,
> > > > > > > > > size_t size)
> > > > > > > > > >  	return setup_wave5_properties(dev);  }
> > > > > > > > > > 
> > > > > > > > > > -static int wave5_vpu_sleep_wake(struct device *dev,
> > > > > > > > > > bool i_sleep_wake,
> > > > > > > > > const uint16_t *code,
> > > > > > > > > > -				size_t size)
> > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > > +i_sleep_wake, const
> > > > > > > > > uint16_t *code,
> > > > > > > > > > +			 size_t size)
> > > > > > > > > >  {
> > > > > > > > > >  	u32 reg_val;
> > > > > > > > > >  	struct vpu_buf *common_vb; diff --git
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec
> > > > > > > > > > .c
> > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec
> > > > > > > > > > .c index ef227af72348..328a7a8f26c5 100644
> > > > > > > > > > ---
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec
> > > > > > > > > > .c
> > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > +++ -d
> > > > > > > > > > +++ ec.c
> > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > >   */
> > > > > > > > > > 
> > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > 
> > > > > > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > 
> > > > > > > > > >  	if (q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE &&
> > > > > > > > > > inst-
> > > > > > > > state
> > > > > > > > > > ==
> > > > > > > > > VPU_INST_STATE_NONE) {
> > > > > > > > > >  		struct dec_open_param open_param;
> > > > > > > > > > +		int err = 0;
> > > > > > > > > > 
> > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > dec_open_param));
> > > > > > > > > > 
> > > > > > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > > > +		if (err) {
> > > > > > > > > > +			dev_err(inst->dev->dev, "decoder runtime
> > > resume
> > > > > > > > > failed %d\n", err);
> > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > +		}
> > > > > > > > > > +
> > > > > > > > > >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > > > > > >  		if (ret)
> > > > > > > > > >  			goto return_buffers; diff --git
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc
> > > > > > > > > > .c
> > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc
> > > > > > > > > > .c index 761775216cd4..ff73d69de41c 100644
> > > > > > > > > > ---
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc
> > > > > > > > > > .c
> > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > +++ -e
> > > > > > > > > > +++ nc.c
> > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > >   */
> > > > > > > > > > 
> > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > 
> > > > > > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > 
> > > > > > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > > > > > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > > > > > >  		struct enc_open_param open_param;
> > > > > > > > > > +		int err = 0;
> > > > > > > > > > 
> > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > enc_open_param));
> > > > > > > > > > 
> > > > > > > > > > +		err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > > > +		if (err) {
> > > > > > > > > > +			dev_err(inst->dev->dev, "encoder runtime
> > > resume
> > > > > > > > > failed %d\n", err);
> > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > +		}
> > > > > > > > > > +
> > > > > > > > > >  		wave5_set_enc_openparam(&open_param, inst);
> > > > > > > > > > 
> > > > > > > > > >  		ret = wave5_vpu_enc_open(inst, &open_param);
> > > diff --
> > > > > > > git
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > > > index 0d90b5820bef..f81409740a56 100644
> > > > > > > > > > ---
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > +++ .c
> > > > > > > > > > @@ -10,6 +10,7 @@
> > > > > > > > > >  #include <linux/clk.h>
> > > > > > > > > >  #include <linux/firmware.h>  #include
> > > > > > > > > > <linux/interrupt.h>
> > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > >  #include "wave5-vpu.h"
> > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > >  #include "wave5-vpuconfig.h"
> > > > > > > > > > @@ -117,6 +118,65 @@ static int
> > > > > > > > > > wave5_vpu_load_firmware(struct device
> > > > > > > > > *dev, const char *fw_name,
> > > > > > > > > >  	return 0;
> > > > > > > > > >  }
> > > > > > > > > > 
> > > > > > > > > > +static __maybe_unused int wave5_pm_suspend(struct
> > > > > > > > > > +device
> > > > > > > > > > +*dev)
> > > > > {
> > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > +
> > > > > > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > > > > > +		return 0;
> > > > > > > > > > +
> > > > > > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > > > > > > > > > +
> > > > > > > > > > +	return 0;
> > > > > > > > > > +}
> > > > > > > > > > +
> > > > > > > > > > +static __maybe_unused int wave5_pm_resume(struct device
> > > *dev) {
> > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > +	int ret = 0;
> > > > > > > > > > +
> > > > > > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu-
> > > > clks);
> > > > > > > > > > +	if (ret) {
> > > > > > > > > > +		dev_err(dev, "Enabling clocks, fail: %d\n",
> > > ret);
> > > > > > > > > > +		return ret;
> > > > > > > > > > +	}
> > > > > > > > > > +
> > > > > > > > > > +	return ret;
> > > > > > > > > > +}
> > > > > > > > > > +
> > > > > > > > > > +static __maybe_unused int wave5_suspend(struct device
> > > > > > > > > > +*dev)
> > > {
> > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > +
> > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > > > > > +
> > > > > > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > > > > > +
> > > > > > > > > > +static __maybe_unused int wave5_resume(struct device
> > > > > > > > > > +*dev)
> > > {
> > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > +	int ret = 0;
> > > > > > > > > > +
> > > > > > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > > > > > +	if (ret < 0)
> > > > > > > > > > +		return ret;
> > > > > > > > > > +
> > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > > > > > +
> > > > > > > > > > +	return ret;
> > > > > > > > > > +}
> > > > > > > > > 
> > > > > > > > > The functions wave5_suspend() and wave5_resume() are not
> > > > > > > > > just "maybe_unsued" but actually never used. What was the
> > > intention ?
> > > > > > > > > Considering the usage of __maybe_unused has been such a
> > > > > > > > > bad friend for this one, could you instead bracket the
> > > > > > > > > functions with an
> > > > > > > explicit ?
> > > > > > > > > 
> > > > > > > > > #ifdef CONFIG_PM
> > > > > > > > > #endif
> > > > > > > > > 
> > > > > > > > > This way the compiler will have a word if you implement
> > > > > > > > > something that you forgot to actually use.
> > > > > > > > > 
> > > > > > > > > > +
> > > > > > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume,
> > > > > > > NULL) };
> > > > > > > > > > +
> > > > > > > > > >  static int wave5_vpu_probe(struct platform_device *pdev)
> > {
> > > > > > > > > >  	int ret;
> > > > > > > > > > @@ -232,6 +292,10 @@ static int wave5_vpu_probe(struct
> > > > > > > > > > platform_device
> > > > > > > > > *pdev)
> > > > > > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ?
> > > "'DECODE'" : "");
> > > > > > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev-
> > > > > > > > product_code);
> > > > > > > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n",
> > > > > > > > > > fw_revision);
> > > > > > > > > > +
> > > > > > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > > > > > > +
> > > > > > > > > >  	return 0;
> > > > > > > > > > 
> > > > > > > > > >  err_enc_unreg:
> > > > > > > > > > @@ -254,6 +318,9 @@ static int wave5_vpu_remove(struct
> > > > > > > > > > platform_device
> > > > > > > > > > *pdev)  {
> > > > > > > > > >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> > > > > > > > > > 
> > > > > > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > > > > > +
> > > > > > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > > > > > >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
> > > @@
> > > > > > > > > > -
> > > > > > > 281,6
> > > > > > > > > > +348,7 @@ static struct platform_driver wave5_vpu_driver
> > > > > > > > > > += {
> > > > > > > > > >  	.driver = {
> > > > > > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > > > > > >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > > > > > > > > > +		.pm = &wave5_pm_ops,
> > > > > > > > > >  		},
> > > > > > > > > >  	.probe = wave5_vpu_probe,
> > > > > > > > > >  	.remove = wave5_vpu_remove, diff --git
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.
> > > > > > > > > > c
> > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.
> > > > > > > > > > c index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > > > > > ---
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.
> > > > > > > > > > c
> > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > +++ ap
> > > > > > > > > > +++ i.c
> > > > > > > > > > @@ -6,6 +6,7 @@
> > > > > > > > > >   */
> > > > > > > > > > 
> > > > > > > > > >  #include <linux/bug.h>
> > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > >  #include "wave5-vpuapi.h"
> > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > >  #include "wave5.h"
> > > > > > > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct
> > > > > > > > > > vpu_instance *inst,
> > > > > > > > > > u32 *fail_res)
> > > > > > > > > > 
> > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > &p_dec_info->vb_task);
> > > > > > > > > > 
> > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > > +
> > > > > > > > > >  unlock_and_return:
> > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > 
> > > > > > > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct
> > > > > > > > > > vpu_instance *inst,
> > > > > > > > > > u32 *fail_res)
> > > > > > > > > > 
> > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > &p_enc_info->vb_task);
> > > > > > > > > > 
> > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > 
> > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > "start_streaming()", but the
> > > > > > > > > put() is only done when the device is closed, or when the
> > > > > > > > > driver is removed. As this is not balanced, you seem to
> > > > > > > > > have to check the suspended condition all over the place.
> > > > > > > > > 
> > > > > > > > > I think we could aim for
> > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > for your get/put placement. At least they will be bound to
> > > > > > > > > an entirely balanced
> > > > > > > API.
> > > > > > > > > But then, a media player in paused sate will prevent that
> > > > > > > > > device from being suspended.
> > > > > > > > > 
> > > > > > > > > If the HW is capable of preserving enough state, and From
> > > > > > > > > the short doc I have it gives me the impression it can
> > > > > > > > > preserve that, I'd suggest to follow what hantro driver is
> > > > > > > > > doing. What is does is that it will do get() in
> > > > > > > > > device_run(), and put() whenever the job completes. This
> > > > > > > > > driver has been designed so when there is no job, it means
> > > > > > > > > the firmware is currently idle and looking for more work.
> > > > > > > > > So it seems like the perfect moment to
> > > > > do suspend it.
> > > > > > > > > 
> > > > > > > > > Nicolas
> > > > > > > > > 
> > > > > > > > > > +
> > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > 
> > > > > > > > > >  	return 0;
> > > > > > > > > > diff --git
> > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > > > > > > --- a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct
> > > > > > > > > > vpu_device *vpu_dev, u32 *revision);
> > > > > > > > > > 
> > > > > > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw, size_t
> > > > > > > > > > size);
> > > > > > > > > > 
> > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > > +i_sleep_wake, const
> > > > > > > > > uint16_t *code,
> > > > > > > > > > +			 size_t size);
> > > > > > > > > > +
> > > > > > > > > >  int wave5_vpu_reset(struct device *dev, enum
> > > > > > > > > > sw_reset_mode reset_mode);
> > > > > > > > > > 
> > > > > > > > > >  int wave5_vpu_build_up_dec_param(struct vpu_instance
> > > > > > > > > > *inst, struct dec_open_param *param);
> > > > > > > > 
> > > > > > 
> > > > 
> 

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

Hello Nicolas

> 
> Le jeudi 22 février 2024 à 03:57 +0000, jackson.lee a écrit :
> > >
> > > > Le mercredi 21 février 2024 à 02:27 +0000, jackson.lee a écrit :
> > > > > Hello Nicolas
> > > > >
> > > > >
> > > > > Thanks for your comment
> > > > >
> > > > > > Le mardi 20 février 2024 à 05:12 +0000, jackson.lee a écrit :
> > > > > > > A pause is common state for media player, but our VPU could
> > > > > > > trigger an
> > > > > > interrupt regardless of a player state.
> > > > > > > So at a pause state if a power turns off, our driver becomes
> > > > > > > hang-
> > > up.
> > > > > > > I think power on/off should be controlled in the open/close
> > > function.
> > > > > >
> > > > > > Please, avoid top posting, this breaks the discussion completely.
> > > > > > Walk through the comments, and reply to them underneath so we
> > > > > > can have a proper discussion.
> > > > > >
> > > > > > In our experience, and with the rework of the driver we did
> > > > > > during
> > > > > > up- streaming, there is no more un-solicited IRQ. The way the
> > > > > > driver state machine has been built, when the m2m framework is
> > > > > > idle (no jobs are active or pending), it means the firmware is
> > > > > > stalled, waiting on some application action. Combined with a
> > > > > > timer of course, so we don't actively suspend/resume, this
> > > > > > seems adequate place to do power management. If that
> > > > > > application action never occurred passed a
> > > > decent delay, we can suspend.
> > > > > > Later on, user actions like qbuf, will lead to
> > > > > > device_run() to be called again, and it would be a good place
> > > > > > to
> > > > resume it.
> > > > >
> > > > > The above comment means like the below ?
> > > > > The wave5_vpu_dec_finish_decode is called by IRQ if there is a
> > > > > decoded
> > > > frame.
> > > > > And at the end of the function, if there is no data queued,
> > > > v4l2_m2m_job_finish is called.
> > > > > You means suspend is called in that function and if app feeds
> > > > > new data into v4l2 driver, the device_run then is called, In the
> > > > > device_run,
> > > > resume is called.
> > > >
> > > > That is the general idea. (Note that we don't forcefully suspect,
> > > > for this type of HW, we should configure a delay. Just saying in
> > > > case someone miss- interpret what was written here)
> > > >
> > > > >
> > > > >
> > > > > I have a question,
> > > > > If app is paused or resumed , the v4l2's vidioc_decoder_cmd is
> > > > > always
> > > > called ?
> > > >
> > > > There is no indication if an app is paused / resumed. Runtime PM
> > > > in fact should not depend on application actions in general. In
> > > > the case of V4L2, most driver will configure a delay (like 5
> > > > seconds). If the driver has been idle (no pending work, no jobs)
> > > > withing that delay, it will auto suspend. Then whenever new
> > > > activity comes, like a QBUF, we
> > > resume.
> > > >
> >
> > Below is a pseudo code for configuring a delay for the Run time
> suspend/resume.
> > The logic you are saying means the below ?
> >
> > Driver_probe
> > {
> >  	pm_runtime_set_autosuspend_delay(vpu->dev, 100);
> > 	pm_runtime_use_autosuspend(vpu->dev);
> > 	pm_runtime_enable(vpu->dev);
> >
> > }
> >
> > Device_run
> > {
> >
> > 	pm_runtime_resume_and_get(ctx->dev->dev);
> > }
> >
> >
> > Finish_job
> > {
> > 	pm_runtime_mark_last_busy(vpu->dev);
> > 	pm_runtime_put_autosuspend(vpu->dev);
> >
> > }
> >
> 
> This is aligned with what I had in mind. There will be few cases were
> you'll need to ensure the hardware is active outside of this (open() and
> close() are examples), but otherwise this should in my perception gives
> the best power saving. And finally, the delay might need tuning, at first
> 100ms seems a bit low, the delay needs to be balanced against the suspend
> cost.
> 
> Nicolas
> 

Thanks for your comment.

Thanks
Jackson







> >
> >
> > Thanks.
> > Jackson
> >
> > >
> > > The delay means a timer, so there is no input for 5 secs, then
> > > timeout callback is called, And suspend is set, if new activity
> > > comes, the device is resumed again ?
> > > My understanding is correct ?
> > >
> >
> >
> > >
> > >
> > > > Nicolas
> > > >
> > > > >
> > > > > Thanks.
> > > > >
> > > > >
> > > > > > There is of course other places where you'll have to make sure
> > > > > > the hardware is resumed, like on close, as you want to remove
> > > > > > the
> > > instance.
> > > > > > There is also small queries here and there that need to be
> > > > > > surrounded with resume/put, but with the redesign, most of the
> > > > > > HW access now take place inside device_run() only.
> > > > > >
> > > > > > Open/Close is not invalid, but it has a lot of issues, as any
> > > > > > random application can endup disabling the kernel ability to
> > > > > > save
> > > power.
> > > > > > Personally, I think we should at least give it a try, and
> > > > > > document valid reason not to do so if we find hardware issues.
> > > > > > Otherwise, this sounds like all we care is ticking the box
> > > > > > "this driver has runtime PM" without actually caring about
> effective power saving.
> > > > > >
> > > > > > Nicolas
> > > > > >
> > > > > > >
> > > > > > > Thanks.
> > > > > > >
> > > > > > > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > > > > > > > Hi Nicolas
> > > > > > > > >
> > > > > > > > >
> > > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > > "start_streaming()", but the
> > > > > > > > > > put() is only done when the device is closed, or when
> > > > > > > > > > the driver is removed. As this is not balanced, you
> > > > > > > > > > seem to have to check the suspended condition all over
> the place.
> > > > > > > > > >
> > > > > > > > > > I think we could aim for
> > > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > > for your get/put placement. At least they will be
> > > > > > > > > > bound to an entirely balanced
> > > > > > > > API.
> > > > > > > > > > But then, a media player in paused sate will prevent
> > > > > > > > > > that device from being suspended.
> > > > > > > > > >
> > > > > > > > > > If the HW is capable of preserving enough state, and
> > > > > > > > > > From the short doc I have it gives me the impression
> > > > > > > > > > it can preserve that, I'd suggest to follow what
> > > > > > > > > > hantro driver is doing. What is does is that it will
> > > > > > > > > > do get() in device_run(), and put() whenever the job
> > > > > > > > > > completes. This driver has been designed so when there
> > > > > > > > > > is no job, it means the firmware is currently idle and
> looking for more work.
> > > > > > > > > > So it seems like the perfect moment to
> > > > > > do suspend it.
> > > > > > > > > >
> > > > > > > > >
> > > > > > > > > Thanks your comment,
> > > > > > > > >
> > > > > > > > > Currently they are not balanced, If we puts "the put
> functon"
> > > > > > > > > into the stop_streaming, our hw is stalled
> > > > > > > > until doing wake-up command, so our v4l2 device become block.
> > > > > > > > > so I'd like to update the below instead of calling get
> > > > > > > > > at the
> > > > > > > > start_streaming function.
> > > > > > > > >
> > > > > > > > >
> > > > > > > > > @@ -1867,6 +1868,13 @@ static int
> > > > > > > > > wave5_vpu_open_dec(struct file
> > > > > > > > > *filp)
> > > > > > > > >
> > > > > > > > >         wave5_vdi_allocate_sram(inst->dev);
> > > > > > > > >
> > > > > > > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > > +       if (err) {
> > > > > > > > > +               dev_err(inst->dev->dev, "decoder runtime
> > > > > > > > > + resume
> > > > > > > > failed %d\n", err);
> > > > > > > > > +               ret = -EINVAL;
> > > > > > > > > +               goto cleanup_inst;
> > > > > > > > > +       }
> > > > > > > > > +
> > > > > > > > >         return 0;
> > > > > > > >
> > > > > > > > I guess we need to discuss the power management strategy
> > > > > > > > for this
> > > > > > device.
> > > > > > > > If you do resume_and_get() in open(), and then put in
> > > > > > > > close(), that seems balanced. But in term of power saving,
> > > > > > > > it might not be very strong. If you have a media player
> > > > > > > > that is set to pause and then placed in the background,
> > > > > > > > you still keep the IP running. This is extremely common,
> > > > > > > > since application cannot close their device without
> > > > > > > > loosing the reference frames, and thus having to do extra
> > > > > > > > work on resume to seek back to the previous sync point and
> > > > > > > > drop
> > > > > > unneeded frames.
> > > > > > > >
> > > > > > > > It seems like the whole point of asking the firmware to
> > > > > > > > save the state and suspend is to be able to do so while
> > > > > > > > there is meaningful sate in the firt place.
> > > > > > > > If we are to suspend only when there is no meaningful
> > > > > > > > state, we could just free all resources and power it off
> completely.
> > > > > > > > (This is just for illustration, its probably to slow to
> > > > > > > > boot the firmware at
> > > > > > > > runtime)
> > > > > > > >
> > > > > > > >
> > > > > > > > I understand you suffered lockup with a start_streaming()
> > > > > > > > for resume_and_get(), and stop_streaming() for put(), this
> > > > > > > > may simply indicate that some hardware access are needed
> > > > > > > > between these two. Can you write down a power management
> > > > > > > > plan that would effectively save power in common use cases
> > > > > > > > ? We can certainly help in refactoring the
> > > > > > code to make that happen.
> > > > > > > >
> > > > > > > > Nicolas
> > > > > > > >
> > > > > > > > >
> > > > > > > > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee
> > > > > > > > > > a
> > > > écrit :
> > > > > > > > > > > There are two device run-time PM callbacks defined
> > > > > > > > > > > in 'struct
> > > > > > > > > > dev_pm_ops'
> > > > > > > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > > > > > > (*runtime_resume)(struct device *dev);
> > > > > > > > > >
> > > > > > > > > > I wonder how useful is it to teach everyone what the
> > > > > > > > > > generic 'struct dev_pm_ops'
> > > > > > > > > > contains. Perhaps you simply wanted that this patch
> > > > > > > > > > implement both suspend and resume ops ?
> > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > > Signed-off-by: Jackson Lee
> > > > > > > > > > > <jackson.lee@chipsnmedia.com>
> > > > > > > > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > > > > > > > ---
> > > > > > > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > > > > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > > > > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > > > > > > +++++++++++++++++++
> > > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7
> > > > > > > > > > > ++ .../media/platform/chips-media/wave5/wave5.h  |
> > > > > > > > > > > 3 +
> > > > > > > > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > > > > > > >
> > > > > > > > > > > diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > c index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-
> hw.
> > > > > > > > > > > +++ c
> > > > > > > > > > > @@ -503,6 +503,7 @@ int
> > > > > > > > > > > wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > > vpu_instance
> > > > > > > > > > *inst,
> > > > > > > > > > >  	/* This register must be reset explicitly */
> > > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > > > > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > > > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > > > > > > >
> > > > > > > > > > In some way, the relation between suspend and this
> > > > > > > > > > register write is not obvious. If its not related,
> > > > > > > > > > please do this in its
> > > > > > own patch.
> > > > > > > > > > Otherwise you want to explain why you needed this
> > > > > > > > > > (possibly just in the commit message).
> > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > >  	ret = send_firmware_command(inst,
> > > > > > > > > > > W5_CREATE_INSTANCE, true, NULL,
> > > > > > > > > > NULL);
> > > > > > > > > > >  	if (ret) {
> > > > > > > > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct
> > > > > > > > > > > device *dev, u8 *fw,
> > > > > > > > > > size_t size)
> > > > > > > > > > >  	return setup_wave5_properties(dev);  }
> > > > > > > > > > >
> > > > > > > > > > > -static int wave5_vpu_sleep_wake(struct device *dev,
> > > > > > > > > > > bool i_sleep_wake,
> > > > > > > > > > const uint16_t *code,
> > > > > > > > > > > -				size_t size)
> > > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > > > +i_sleep_wake, const
> > > > > > > > > > uint16_t *code,
> > > > > > > > > > > +			 size_t size)
> > > > > > > > > > >  {
> > > > > > > > > > >  	u32 reg_val;
> > > > > > > > > > >  	struct vpu_buf *common_vb; diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -dec
> > > > > > > > > > > .c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -dec .c index ef227af72348..328a7a8f26c5 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -dec
> > > > > > > > > > > .c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ -vpu
> > > > > > > > > > > +++ -d
> > > > > > > > > > > +++ ec.c
> > > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > > >   */
> > > > > > > > > > >
> > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > >
> > > > > > > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > >
> > > > > > > > > > >  	if (q->type ==
> V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE
> > > > > > > > > > > &&
> > > > > > > > > > > inst-
> > > > > > > > > state
> > > > > > > > > > > ==
> > > > > > > > > > VPU_INST_STATE_NONE) {
> > > > > > > > > > >  		struct dec_open_param open_param;
> > > > > > > > > > > +		int err = 0;
> > > > > > > > > > >
> > > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > > dec_open_param));
> > > > > > > > > > >
> > > > > > > > > > > +		err = pm_runtime_resume_and_get(inst-
> >dev->dev);
> > > > > > > > > > > +		if (err) {
> > > > > > > > > > > +			dev_err(inst->dev->dev, "decoder
> runtime
> > > > resume
> > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > > +		}
> > > > > > > > > > > +
> > > > > > > > > > >  		ret =
> wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > > > > > > >  		if (ret)
> > > > > > > > > > >  			goto return_buffers; diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -enc
> > > > > > > > > > > .c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -enc .c index 761775216cd4..ff73d69de41c 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -enc
> > > > > > > > > > > .c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ -vpu
> > > > > > > > > > > +++ -e
> > > > > > > > > > > +++ nc.c
> > > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > > >   */
> > > > > > > > > > >
> > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > >
> > > > > > > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > >
> > > > > > > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q-
> >type
> > > > > > > > > > > ==
> > > > > > > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > > > > > > >  		struct enc_open_param open_param;
> > > > > > > > > > > +		int err = 0;
> > > > > > > > > > >
> > > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > > enc_open_param));
> > > > > > > > > > >
> > > > > > > > > > > +		err = pm_runtime_resume_and_get(inst-
> >dev->dev);
> > > > > > > > > > > +		if (err) {
> > > > > > > > > > > +			dev_err(inst->dev->dev, "encoder
> runtime
> > > > resume
> > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > > +		}
> > > > > > > > > > > +
> > > > > > > > > > >  		wave5_set_enc_openparam(&open_param,
> inst);
> > > > > > > > > > >
> > > > > > > > > > >  		ret = wave5_vpu_enc_open(inst,
> &open_param);
> > > > diff --
> > > > > > > > git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > .c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > .c index 0d90b5820bef..f81409740a56 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > .c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ -vpu
> > > > > > > > > > > +++ .c
> > > > > > > > > > > @@ -10,6 +10,7 @@
> > > > > > > > > > >  #include <linux/clk.h>
> > > > > > > > > > >  #include <linux/firmware.h>  #include
> > > > > > > > > > > <linux/interrupt.h>
> > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > >  #include "wave5-vpu.h"
> > > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > > >  #include "wave5-vpuconfig.h"
> > > > > > > > > > > @@ -117,6 +118,65 @@ static int
> > > > > > > > > > > wave5_vpu_load_firmware(struct device
> > > > > > > > > > *dev, const char *fw_name,
> > > > > > > > > > >  	return 0;
> > > > > > > > > > >  }
> > > > > > > > > > >
> > > > > > > > > > > +static __maybe_unused int wave5_pm_suspend(struct
> > > > > > > > > > > +device
> > > > > > > > > > > +*dev)
> > > > > > {
> > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > +
> > > > > > > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > > > > > > +		return 0;
> > > > > > > > > > > +
> > > > > > > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks,
> > > > > > > > > > > +vpu->clks);
> > > > > > > > > > > +
> > > > > > > > > > > +	return 0;
> > > > > > > > > > > +}
> > > > > > > > > > > +
> > > > > > > > > > > +static __maybe_unused int wave5_pm_resume(struct
> > > > > > > > > > > +device
> > > > *dev) {
> > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > +	int ret = 0;
> > > > > > > > > > > +
> > > > > > > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks,
> vpu-
> > > > > clks);
> > > > > > > > > > > +	if (ret) {
> > > > > > > > > > > +		dev_err(dev, "Enabling clocks,
> fail: %d\n",
> > > > ret);
> > > > > > > > > > > +		return ret;
> > > > > > > > > > > +	}
> > > > > > > > > > > +
> > > > > > > > > > > +	return ret;
> > > > > > > > > > > +}
> > > > > > > > > > > +
> > > > > > > > > > > +static __maybe_unused int wave5_suspend(struct
> > > > > > > > > > > +device
> > > > > > > > > > > +*dev)
> > > > {
> > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > > +
> > > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > > > > > > +
> > > > > > > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > > > > > > +
> > > > > > > > > > > +static __maybe_unused int wave5_resume(struct
> > > > > > > > > > > +device
> > > > > > > > > > > +*dev)
> > > > {
> > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > > +	int ret = 0;
> > > > > > > > > > > +
> > > > > > > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > > > > > > +	if (ret < 0)
> > > > > > > > > > > +		return ret;
> > > > > > > > > > > +
> > > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > > > > > > +
> > > > > > > > > > > +	return ret;
> > > > > > > > > > > +}
> > > > > > > > > >
> > > > > > > > > > The functions wave5_suspend() and wave5_resume() are
> > > > > > > > > > not just "maybe_unsued" but actually never used. What
> > > > > > > > > > was the
> > > > intention ?
> > > > > > > > > > Considering the usage of __maybe_unused has been such
> > > > > > > > > > a bad friend for this one, could you instead bracket
> > > > > > > > > > the functions with an
> > > > > > > > explicit ?
> > > > > > > > > >
> > > > > > > > > > #ifdef CONFIG_PM
> > > > > > > > > > #endif
> > > > > > > > > >
> > > > > > > > > > This way the compiler will have a word if you
> > > > > > > > > > implement something that you forgot to actually use.
> > > > > > > > > >
> > > > > > > > > > > +
> > > > > > > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend,
> > > > > > > > > > > +wave5_pm_resume,
> > > > > > > > NULL) };
> > > > > > > > > > > +
> > > > > > > > > > >  static int wave5_vpu_probe(struct platform_device
> > > > > > > > > > > *pdev)
> > > {
> > > > > > > > > > >  	int ret;
> > > > > > > > > > > @@ -232,6 +292,10 @@ static int
> > > > > > > > > > > wave5_vpu_probe(struct platform_device
> > > > > > > > > > *pdev)
> > > > > > > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ?
> > > > "'DECODE'" : "");
> > > > > > > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n",
> dev-
> > > > > > > > > product_code);
> > > > > > > > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n",
> > > > > > > > > > > fw_revision);
> > > > > > > > > > > +
> > > > > > > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > > > > > > > +
> > > > > > > > > > >  	return 0;
> > > > > > > > > > >
> > > > > > > > > > >  err_enc_unreg:
> > > > > > > > > > > @@ -254,6 +318,9 @@ static int
> > > > > > > > > > > wave5_vpu_remove(struct platform_device
> > > > > > > > > > > *pdev)  {
> > > > > > > > > > >  	struct vpu_device *dev =
> > > > > > > > > > > dev_get_drvdata(&pdev->dev);
> > > > > > > > > > >
> > > > > > > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > > > > > > +
> > > > > > > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > > > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > > > > > > >  	clk_bulk_disable_unprepare(dev->num_clks,
> > > > > > > > > > > dev->clks);
> > > > @@
> > > > > > > > > > > -
> > > > > > > > 281,6
> > > > > > > > > > > +348,7 @@ static struct platform_driver
> > > > > > > > > > > +wave5_vpu_driver = {
> > > > > > > > > > >  	.driver = {
> > > > > > > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > > > > > > >  		.of_match_table =
> of_match_ptr(wave5_dt_ids),
> > > > > > > > > > > +		.pm = &wave5_pm_ops,
> > > > > > > > > > >  		},
> > > > > > > > > > >  	.probe = wave5_vpu_probe,
> > > > > > > > > > >  	.remove = wave5_vpu_remove, diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-
> vpuapi.
> > > > > > > > > > > c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-
> vpuapi.
> > > > > > > > > > > c index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-
> vpuapi.
> > > > > > > > > > > c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ -vpu
> > > > > > > > > > > +++ ap
> > > > > > > > > > > +++ i.c
> > > > > > > > > > > @@ -6,6 +6,7 @@
> > > > > > > > > > >   */
> > > > > > > > > > >
> > > > > > > > > > >  #include <linux/bug.h>
> > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > >  #include "wave5-vpuapi.h"
> > > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > > >  #include "wave5.h"
> > > > > > > > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct
> > > > > > > > > > > vpu_instance *inst,
> > > > > > > > > > > u32 *fail_res)
> > > > > > > > > > >
> > > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > > &p_dec_info->vb_task);
> > > > > > > > > > >
> > > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > > > +
> > > > > > > > > > >  unlock_and_return:
> > > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > >
> > > > > > > > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct
> > > > > > > > > > > vpu_instance *inst,
> > > > > > > > > > > u32 *fail_res)
> > > > > > > > > > >
> > > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > > &p_enc_info->vb_task);
> > > > > > > > > > >
> > > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > >
> > > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > > "start_streaming()", but the
> > > > > > > > > > put() is only done when the device is closed, or when
> > > > > > > > > > the driver is removed. As this is not balanced, you
> > > > > > > > > > seem to have to check the suspended condition all over
> the place.
> > > > > > > > > >
> > > > > > > > > > I think we could aim for
> > > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > > for your get/put placement. At least they will be
> > > > > > > > > > bound to an entirely balanced
> > > > > > > > API.
> > > > > > > > > > But then, a media player in paused sate will prevent
> > > > > > > > > > that device from being suspended.
> > > > > > > > > >
> > > > > > > > > > If the HW is capable of preserving enough state, and
> > > > > > > > > > From the short doc I have it gives me the impression
> > > > > > > > > > it can preserve that, I'd suggest to follow what
> > > > > > > > > > hantro driver is doing. What is does is that it will
> > > > > > > > > > do get() in device_run(), and put() whenever the job
> > > > > > > > > > completes. This driver has been designed so when there
> > > > > > > > > > is no job, it means the firmware is currently idle and
> looking for more work.
> > > > > > > > > > So it seems like the perfect moment to
> > > > > > do suspend it.
> > > > > > > > > >
> > > > > > > > > > Nicolas
> > > > > > > > > >
> > > > > > > > > > > +
> > > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > >
> > > > > > > > > > >  	return 0;
> > > > > > > > > > > diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ .h
> > > > > > > > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct
> > > > > > > > > > > vpu_device *vpu_dev, u32 *revision);
> > > > > > > > > > >
> > > > > > > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw,
> > > > > > > > > > > size_t size);
> > > > > > > > > > >
> > > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > > > +i_sleep_wake, const
> > > > > > > > > > uint16_t *code,
> > > > > > > > > > > +			 size_t size);
> > > > > > > > > > > +
> > > > > > > > > > >  int wave5_vpu_reset(struct device *dev, enum
> > > > > > > > > > > sw_reset_mode reset_mode);
> > > > > > > > > > >
> > > > > > > > > > >  int wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > > vpu_instance *inst, struct dec_open_param *param);
> > > > > > > > >
> > > > > > >
> > > > >
> >

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

Hello Nicolas

> -----Original Message-----
> From: Nicolas Dufresne <nicolas@ndufresne.ca>
> Sent: Saturday, February 24, 2024 3:10 AM
> To: jackson.lee <jackson.lee@chipsnmedia.com>; mchehab@kernel.org; linux-
> media@vger.kernel.org; linux-kernel@vger.kernel.org; Nas Chung
> <nas.chung@chipsnmedia.com>
> Cc: lafley.kim <lafley.kim@chipsnmedia.com>; b-brnich@ti.com
> Subject: Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.
> 
> Le jeudi 22 février 2024 à 03:57 +0000, jackson.lee a écrit :
> > >
> > > > Le mercredi 21 février 2024 à 02:27 +0000, jackson.lee a écrit :
> > > > > Hello Nicolas
> > > > >
> > > > >
> > > > > Thanks for your comment
> > > > >
> > > > > > Le mardi 20 février 2024 à 05:12 +0000, jackson.lee a écrit :
> > > > > > > A pause is common state for media player, but our VPU could
> > > > > > > trigger an
> > > > > > interrupt regardless of a player state.
> > > > > > > So at a pause state if a power turns off, our driver becomes
> > > > > > > hang-
> > > up.
> > > > > > > I think power on/off should be controlled in the open/close
> > > function.
> > > > > >
> > > > > > Please, avoid top posting, this breaks the discussion completely.
> > > > > > Walk through the comments, and reply to them underneath so we
> > > > > > can have a proper discussion.
> > > > > >
> > > > > > In our experience, and with the rework of the driver we did
> > > > > > during
> > > > > > up- streaming, there is no more un-solicited IRQ. The way the
> > > > > > driver state machine has been built, when the m2m framework is
> > > > > > idle (no jobs are active or pending), it means the firmware is
> > > > > > stalled, waiting on some application action. Combined with a
> > > > > > timer of course, so we don't actively suspend/resume, this
> > > > > > seems adequate place to do power management. If that
> > > > > > application action never occurred passed a
> > > > decent delay, we can suspend.
> > > > > > Later on, user actions like qbuf, will lead to
> > > > > > device_run() to be called again, and it would be a good place
> > > > > > to
> > > > resume it.
> > > > >
> > > > > The above comment means like the below ?
> > > > > The wave5_vpu_dec_finish_decode is called by IRQ if there is a
> > > > > decoded
> > > > frame.
> > > > > And at the end of the function, if there is no data queued,
> > > > v4l2_m2m_job_finish is called.
> > > > > You means suspend is called in that function and if app feeds
> > > > > new data into v4l2 driver, the device_run then is called, In the
> > > > > device_run,
> > > > resume is called.
> > > >
> > > > That is the general idea. (Note that we don't forcefully suspect,
> > > > for this type of HW, we should configure a delay. Just saying in
> > > > case someone miss- interpret what was written here)
> > > >
> > > > >
> > > > >
> > > > > I have a question,
> > > > > If app is paused or resumed , the v4l2's vidioc_decoder_cmd is
> > > > > always
> > > > called ?
> > > >
> > > > There is no indication if an app is paused / resumed. Runtime PM
> > > > in fact should not depend on application actions in general. In
> > > > the case of V4L2, most driver will configure a delay (like 5
> > > > seconds). If the driver has been idle (no pending work, no jobs)
> > > > withing that delay, it will auto suspend. Then whenever new
> > > > activity comes, like a QBUF, we
> > > resume.
> > > >
> >
> > Below is a pseudo code for configuring a delay for the Run time
> suspend/resume.
> > The logic you are saying means the below ?
> >
> > Driver_probe
> > {
> >  	pm_runtime_set_autosuspend_delay(vpu->dev, 100);
> > 	pm_runtime_use_autosuspend(vpu->dev);
> > 	pm_runtime_enable(vpu->dev);
> >
> > }
> >
> > Device_run
> > {
> >
> > 	pm_runtime_resume_and_get(ctx->dev->dev);
> > }
> >
> >
> > Finish_job
> > {
> > 	pm_runtime_mark_last_busy(vpu->dev);
> > 	pm_runtime_put_autosuspend(vpu->dev);
> >
> > }
> >
> 
> This is aligned with what I had in mind. There will be few cases were
> you'll need to ensure the hardware is active outside of this (open() and
> close() are examples), but otherwise this should in my perception gives
> the best power saving. And finally, the delay might need tuning, at first
> 100ms seems a bit low, the delay needs to be balanced against the suspend
> cost.
> 
> Nicolas
> 



I have added the autosuspend feature, can you please review the below code ?
If there is no problem at your side, I will make a patch for upstream.


diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c
index 0b6386f31..2ba9e0f0e 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
@@ -1102,8 +1102,8 @@ int wave5_vpu_re_init(struct device *dev, u8 *fw, size_t size)
 	return setup_wave5_properties(dev);
 }
 
-static int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
-				size_t size)
+int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
+			 size_t size)
 {
 	u32 reg_val;
 	struct vpu_buf *common_vb;
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
index 8b1417ece..4aea66483 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2021-2023 CHIPS&MEDIA INC
  */
 
+#include <linux/pm_runtime.h>
 #include "wave5-helper.h"
 
 #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
@@ -518,6 +519,8 @@ static void wave5_vpu_dec_finish_decode(struct vpu_instance *inst)
 	if (q_status.report_queue_count == 0 &&
 	    (q_status.instance_queue_count == 0 || dec_info.sequence_changed)) {
 		dev_dbg(inst->dev->dev, "%s: finishing job.\n", __func__);
+		pm_runtime_mark_last_busy(inst->dev->dev);
+		pm_runtime_put_autosuspend(inst->dev->dev);
 		v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
 	}
 }
@@ -1382,6 +1385,7 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
 	int ret = 0;
 
 	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
+	pm_runtime_resume_and_get(inst->dev->dev);
 
 	v4l2_m2m_update_start_streaming_state(m2m_ctx, q);
 
@@ -1389,7 +1393,6 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
 		struct dec_open_param open_param;
 
 		memset(&open_param, 0, sizeof(struct dec_open_param));
-
 		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
 		if (ret)
 			goto return_buffers;
@@ -1425,13 +1428,14 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
 			}
 		}
 	}
-
 	return ret;
 
 free_bitstream_vbuf:
 	wave5_vdi_free_dma_memory(inst->dev, &inst->bitstream_vbuf);
 return_buffers:
 	wave5_return_bufs(q, VB2_BUF_STATE_QUEUED);
+	pm_runtime_mark_last_busy(inst->dev->dev);
+	pm_runtime_put_autosuspend(inst->dev->dev);
 	return ret;
 }
 
@@ -1517,6 +1521,7 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q)
 	bool check_cmd = TRUE;
 
 	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
+	pm_runtime_resume_and_get(inst->dev->dev);
 
 	while (check_cmd) {
 		struct queue_status_info q_status;
@@ -1540,6 +1545,9 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q)
 		streamoff_output(q);
 	else
 		streamoff_capture(q);
+
+	pm_runtime_mark_last_busy(inst->dev->dev);
+	pm_runtime_put_autosuspend(inst->dev->dev);
 }
 
 static const struct vb2_ops wave5_vpu_dec_vb2_ops = {
@@ -1626,7 +1634,7 @@ static void wave5_vpu_dec_device_run(void *priv)
 	int ret = 0;
 
 	dev_dbg(inst->dev->dev, "%s: Fill the ring buffer with new bitstream data", __func__);
-
+	pm_runtime_resume_and_get(inst->dev->dev);
 	ret = fill_ringbuffer(inst);
 	if (ret) {
 		dev_warn(inst->dev->dev, "Filling ring buffer failed\n");
@@ -1709,6 +1717,8 @@ static void wave5_vpu_dec_device_run(void *priv)
 
 finish_job_and_return:
 	dev_dbg(inst->dev->dev, "%s: leave and finish job", __func__);
+	pm_runtime_mark_last_busy(inst->dev->dev);
+	pm_runtime_put_autosuspend(inst->dev->dev);
 	v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
 }
 
@@ -1866,7 +1876,6 @@ static int wave5_vpu_open_dec(struct file *filp)
 	}
 
 	wave5_vdi_allocate_sram(inst->dev);
-
 	return 0;
 
 cleanup_inst:
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
index f04baa93a..14fd26204 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2021-2023 CHIPS&MEDIA INC
  */
 
+#include <linux/pm_runtime.h>
 #include "wave5-helper.h"
 
 #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
@@ -1310,6 +1311,7 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count
 	struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx;
 	int ret = 0;
 
+	pm_runtime_resume_and_get(inst->dev->dev);
 	v4l2_m2m_update_start_streaming_state(m2m_ctx, q);
 
 	if (inst->state == VPU_INST_STATE_NONE && q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
@@ -1364,9 +1366,13 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count
 	if (ret)
 		goto return_buffers;
 
+	pm_runtime_mark_last_busy(inst->dev->dev);
+	pm_runtime_put_autosuspend(inst->dev->dev);
 	return 0;
 return_buffers:
 	wave5_return_bufs(q, VB2_BUF_STATE_QUEUED);
+	pm_runtime_mark_last_busy(inst->dev->dev);
+	pm_runtime_put_autosuspend(inst->dev->dev);
 	return ret;
 }
 
@@ -1408,6 +1414,7 @@ static void wave5_vpu_enc_stop_streaming(struct vb2_queue *q)
 	 */
 
 	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
+	pm_runtime_resume_and_get(inst->dev->dev);
 
 	if (wave5_vpu_both_queues_are_streaming(inst))
 		switch_state(inst, VPU_INST_STATE_STOP);
@@ -1432,6 +1439,9 @@ static void wave5_vpu_enc_stop_streaming(struct vb2_queue *q)
 		streamoff_output(inst, q);
 	else
 		streamoff_capture(inst, q);
+
+	pm_runtime_mark_last_busy(inst->dev->dev);
+	pm_runtime_put_autosuspend(inst->dev->dev);
 }
 
 static const struct vb2_ops wave5_vpu_enc_vb2_ops = {
@@ -1478,6 +1488,7 @@ static void wave5_vpu_enc_device_run(void *priv)
 	u32 fail_res = 0;
 	int ret = 0;
 
+	pm_runtime_resume_and_get(inst->dev->dev);
 	switch (inst->state) {
 	case VPU_INST_STATE_PIC_RUN:
 		ret = start_encode(inst, &fail_res);
@@ -1491,6 +1502,8 @@ static void wave5_vpu_enc_device_run(void *priv)
 			break;
 		}
 		dev_dbg(inst->dev->dev, "%s: leave with active job", __func__);
+		pm_runtime_mark_last_busy(inst->dev->dev);
+		pm_runtime_put_autosuspend(inst->dev->dev);
 		return;
 	default:
 		WARN(1, "Execution of a job in state %s is invalid.\n",
@@ -1498,6 +1511,8 @@ static void wave5_vpu_enc_device_run(void *priv)
 		break;
 	}
 	dev_dbg(inst->dev->dev, "%s: leave and finish job", __func__);
+	pm_runtime_mark_last_busy(inst->dev->dev);
+	pm_runtime_put_autosuspend(inst->dev->dev);
 	v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
 }
 
@@ -1735,7 +1750,6 @@ static int wave5_vpu_open_enc(struct file *filp)
 	}
 
 	wave5_vdi_allocate_sram(inst->dev);
-
 	return 0;
 
 cleanup_inst:
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu.c b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
index 2323dba75..6b4794e0e 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
@@ -10,6 +10,7 @@
 #include <linux/clk.h>
 #include <linux/firmware.h>
 #include <linux/interrupt.h>
+#include <linux/pm_runtime.h>
 #include "wave5-vpu.h"
 #include "wave5-regdefine.h"
 #include "wave5-vpuconfig.h"
@@ -117,6 +118,38 @@ static int wave5_vpu_load_firmware(struct device *dev, const char *fw_name,
 	return 0;
 }
 
+static int wave5_pm_suspend(struct device *dev)
+{
+	struct vpu_device *vpu = dev_get_drvdata(dev);
+
+	if (pm_runtime_suspended(dev))
+		return 0;
+
+	wave5_vpu_sleep_wake(dev, true, NULL, 0);
+	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
+
+	return 0;
+}
+
+static int wave5_pm_resume(struct device *dev)
+{
+	struct vpu_device *vpu = dev_get_drvdata(dev);
+	int ret = 0;
+
+	wave5_vpu_sleep_wake(dev, false, NULL, 0);
+	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
+	if (ret) {
+		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
+		return ret;
+	}
+
+	return ret;
+}
+
+static const struct dev_pm_ops wave5_pm_ops = {
+	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume, NULL)
+};
+
 static int wave5_vpu_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -233,6 +266,12 @@ static int wave5_vpu_probe(struct platform_device *pdev)
 		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
 	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev->product_code);
 	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
+
+	pm_runtime_set_autosuspend_delay(&pdev->dev, 5000);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
+
 	return 0;
 
 err_enc_unreg:
@@ -255,6 +294,9 @@ static int wave5_vpu_remove(struct platform_device *pdev)
 {
 	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
 
+	pm_runtime_put_sync(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+
 	mutex_destroy(&dev->dev_lock);
 	mutex_destroy(&dev->hw_lock);
 	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
@@ -282,6 +324,7 @@ static struct platform_driver wave5_vpu_driver = {
 	.driver = {
 		.name = VPU_PLATFORM_DEVICE_NAME,
 		.of_match_table = of_match_ptr(wave5_dt_ids),
+		.pm = &wave5_pm_ops,
 		},
 	.probe = wave5_vpu_probe,
 	.remove = wave5_vpu_remove,
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
index 86b399372..b2357d1b5 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
@@ -6,6 +6,8 @@
  */
 
 #include <linux/bug.h>
+#include <linux/pm_runtime.h>
+#include <linux/delay.h>
 #include "wave5-vpuapi.h"
 #include "wave5-regdefine.h"
 #include "wave5.h"
@@ -200,6 +202,8 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res)
 	if (!inst->codec_info)
 		return -EINVAL;
 
+	pm_runtime_resume_and_get(inst->dev->dev);
+
 	ret = mutex_lock_interruptible(&vpu_dev->hw_lock);
 	if (ret)
 		return ret;
@@ -234,7 +238,7 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res)
 
 unlock_and_return:
 	mutex_unlock(&vpu_dev->hw_lock);
-
+	pm_runtime_put_autosuspend(inst->dev->dev);
 	return ret;
 }
 
@@ -702,6 +706,8 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res)
 	if (!inst->codec_info)
 		return -EINVAL;
 
+	pm_runtime_resume_and_get(inst->dev->dev);
+
 	ret = mutex_lock_interruptible(&vpu_dev->hw_lock);
 	if (ret)
 		return ret;
@@ -733,9 +739,10 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res)
 	}
 
 	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
-
 	mutex_unlock(&vpu_dev->hw_lock);
 
+	pm_runtime_put_autosuspend(inst->dev->dev);
+
 	return 0;
 }
 










> >
> >
> > Thanks.
> > Jackson
> >
> > >
> > > The delay means a timer, so there is no input for 5 secs, then
> > > timeout callback is called, And suspend is set, if new activity
> > > comes, the device is resumed again ?
> > > My understanding is correct ?
> > >
> >
> >
> > >
> > >
> > > > Nicolas
> > > >
> > > > >
> > > > > Thanks.
> > > > >
> > > > >
> > > > > > There is of course other places where you'll have to make sure
> > > > > > the hardware is resumed, like on close, as you want to remove
> > > > > > the
> > > instance.
> > > > > > There is also small queries here and there that need to be
> > > > > > surrounded with resume/put, but with the redesign, most of the
> > > > > > HW access now take place inside device_run() only.
> > > > > >
> > > > > > Open/Close is not invalid, but it has a lot of issues, as any
> > > > > > random application can endup disabling the kernel ability to
> > > > > > save
> > > power.
> > > > > > Personally, I think we should at least give it a try, and
> > > > > > document valid reason not to do so if we find hardware issues.
> > > > > > Otherwise, this sounds like all we care is ticking the box
> > > > > > "this driver has runtime PM" without actually caring about
> effective power saving.
> > > > > >
> > > > > > Nicolas
> > > > > >
> > > > > > >
> > > > > > > Thanks.
> > > > > > >
> > > > > > > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > > > > > > > Hi Nicolas
> > > > > > > > >
> > > > > > > > >
> > > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > > "start_streaming()", but the
> > > > > > > > > > put() is only done when the device is closed, or when
> > > > > > > > > > the driver is removed. As this is not balanced, you
> > > > > > > > > > seem to have to check the suspended condition all over
> the place.
> > > > > > > > > >
> > > > > > > > > > I think we could aim for
> > > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > > for your get/put placement. At least they will be
> > > > > > > > > > bound to an entirely balanced
> > > > > > > > API.
> > > > > > > > > > But then, a media player in paused sate will prevent
> > > > > > > > > > that device from being suspended.
> > > > > > > > > >
> > > > > > > > > > If the HW is capable of preserving enough state, and
> > > > > > > > > > From the short doc I have it gives me the impression
> > > > > > > > > > it can preserve that, I'd suggest to follow what
> > > > > > > > > > hantro driver is doing. What is does is that it will
> > > > > > > > > > do get() in device_run(), and put() whenever the job
> > > > > > > > > > completes. This driver has been designed so when there
> > > > > > > > > > is no job, it means the firmware is currently idle and
> looking for more work.
> > > > > > > > > > So it seems like the perfect moment to
> > > > > > do suspend it.
> > > > > > > > > >
> > > > > > > > >
> > > > > > > > > Thanks your comment,
> > > > > > > > >
> > > > > > > > > Currently they are not balanced, If we puts "the put
> functon"
> > > > > > > > > into the stop_streaming, our hw is stalled
> > > > > > > > until doing wake-up command, so our v4l2 device become block.
> > > > > > > > > so I'd like to update the below instead of calling get
> > > > > > > > > at the
> > > > > > > > start_streaming function.
> > > > > > > > >
> > > > > > > > >
> > > > > > > > > @@ -1867,6 +1868,13 @@ static int
> > > > > > > > > wave5_vpu_open_dec(struct file
> > > > > > > > > *filp)
> > > > > > > > >
> > > > > > > > >         wave5_vdi_allocate_sram(inst->dev);
> > > > > > > > >
> > > > > > > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > > +       if (err) {
> > > > > > > > > +               dev_err(inst->dev->dev, "decoder runtime
> > > > > > > > > + resume
> > > > > > > > failed %d\n", err);
> > > > > > > > > +               ret = -EINVAL;
> > > > > > > > > +               goto cleanup_inst;
> > > > > > > > > +       }
> > > > > > > > > +
> > > > > > > > >         return 0;
> > > > > > > >
> > > > > > > > I guess we need to discuss the power management strategy
> > > > > > > > for this
> > > > > > device.
> > > > > > > > If you do resume_and_get() in open(), and then put in
> > > > > > > > close(), that seems balanced. But in term of power saving,
> > > > > > > > it might not be very strong. If you have a media player
> > > > > > > > that is set to pause and then placed in the background,
> > > > > > > > you still keep the IP running. This is extremely common,
> > > > > > > > since application cannot close their device without
> > > > > > > > loosing the reference frames, and thus having to do extra
> > > > > > > > work on resume to seek back to the previous sync point and
> > > > > > > > drop
> > > > > > unneeded frames.
> > > > > > > >
> > > > > > > > It seems like the whole point of asking the firmware to
> > > > > > > > save the state and suspend is to be able to do so while
> > > > > > > > there is meaningful sate in the firt place.
> > > > > > > > If we are to suspend only when there is no meaningful
> > > > > > > > state, we could just free all resources and power it off
> completely.
> > > > > > > > (This is just for illustration, its probably to slow to
> > > > > > > > boot the firmware at
> > > > > > > > runtime)
> > > > > > > >
> > > > > > > >
> > > > > > > > I understand you suffered lockup with a start_streaming()
> > > > > > > > for resume_and_get(), and stop_streaming() for put(), this
> > > > > > > > may simply indicate that some hardware access are needed
> > > > > > > > between these two. Can you write down a power management
> > > > > > > > plan that would effectively save power in common use cases
> > > > > > > > ? We can certainly help in refactoring the
> > > > > > code to make that happen.
> > > > > > > >
> > > > > > > > Nicolas
> > > > > > > >
> > > > > > > > >
> > > > > > > > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee
> > > > > > > > > > a
> > > > écrit :
> > > > > > > > > > > There are two device run-time PM callbacks defined
> > > > > > > > > > > in 'struct
> > > > > > > > > > dev_pm_ops'
> > > > > > > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > > > > > > (*runtime_resume)(struct device *dev);
> > > > > > > > > >
> > > > > > > > > > I wonder how useful is it to teach everyone what the
> > > > > > > > > > generic 'struct dev_pm_ops'
> > > > > > > > > > contains. Perhaps you simply wanted that this patch
> > > > > > > > > > implement both suspend and resume ops ?
> > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > > Signed-off-by: Jackson Lee
> > > > > > > > > > > <jackson.lee@chipsnmedia.com>
> > > > > > > > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > > > > > > > ---
> > > > > > > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > > > > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > > > > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > > > > > > +++++++++++++++++++
> > > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7
> > > > > > > > > > > ++ .../media/platform/chips-media/wave5/wave5.h  |
> > > > > > > > > > > 3 +
> > > > > > > > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > > > > > > >
> > > > > > > > > > > diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > c index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-
> hw.
> > > > > > > > > > > +++ c
> > > > > > > > > > > @@ -503,6 +503,7 @@ int
> > > > > > > > > > > wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > > vpu_instance
> > > > > > > > > > *inst,
> > > > > > > > > > >  	/* This register must be reset explicitly */
> > > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > > > > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > > > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > > > > > > >
> > > > > > > > > > In some way, the relation between suspend and this
> > > > > > > > > > register write is not obvious. If its not related,
> > > > > > > > > > please do this in its
> > > > > > own patch.
> > > > > > > > > > Otherwise you want to explain why you needed this
> > > > > > > > > > (possibly just in the commit message).
> > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > >  	ret = send_firmware_command(inst,
> > > > > > > > > > > W5_CREATE_INSTANCE, true, NULL,
> > > > > > > > > > NULL);
> > > > > > > > > > >  	if (ret) {
> > > > > > > > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct
> > > > > > > > > > > device *dev, u8 *fw,
> > > > > > > > > > size_t size)
> > > > > > > > > > >  	return setup_wave5_properties(dev);  }
> > > > > > > > > > >
> > > > > > > > > > > -static int wave5_vpu_sleep_wake(struct device *dev,
> > > > > > > > > > > bool i_sleep_wake,
> > > > > > > > > > const uint16_t *code,
> > > > > > > > > > > -				size_t size)
> > > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > > > +i_sleep_wake, const
> > > > > > > > > > uint16_t *code,
> > > > > > > > > > > +			 size_t size)
> > > > > > > > > > >  {
> > > > > > > > > > >  	u32 reg_val;
> > > > > > > > > > >  	struct vpu_buf *common_vb; diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -dec
> > > > > > > > > > > .c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -dec .c index ef227af72348..328a7a8f26c5 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -dec
> > > > > > > > > > > .c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ -vpu
> > > > > > > > > > > +++ -d
> > > > > > > > > > > +++ ec.c
> > > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > > >   */
> > > > > > > > > > >
> > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > >
> > > > > > > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > >
> > > > > > > > > > >  	if (q->type ==
> V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE
> > > > > > > > > > > &&
> > > > > > > > > > > inst-
> > > > > > > > > state
> > > > > > > > > > > ==
> > > > > > > > > > VPU_INST_STATE_NONE) {
> > > > > > > > > > >  		struct dec_open_param open_param;
> > > > > > > > > > > +		int err = 0;
> > > > > > > > > > >
> > > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > > dec_open_param));
> > > > > > > > > > >
> > > > > > > > > > > +		err = pm_runtime_resume_and_get(inst-
> >dev->dev);
> > > > > > > > > > > +		if (err) {
> > > > > > > > > > > +			dev_err(inst->dev->dev, "decoder
> runtime
> > > > resume
> > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > > +		}
> > > > > > > > > > > +
> > > > > > > > > > >  		ret =
> wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > > > > > > >  		if (ret)
> > > > > > > > > > >  			goto return_buffers; diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -enc
> > > > > > > > > > > .c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -enc .c index 761775216cd4..ff73d69de41c 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > -enc
> > > > > > > > > > > .c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ -vpu
> > > > > > > > > > > +++ -e
> > > > > > > > > > > +++ nc.c
> > > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > > >   */
> > > > > > > > > > >
> > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > >
> > > > > > > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > >
> > > > > > > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q-
> >type
> > > > > > > > > > > ==
> > > > > > > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > > > > > > >  		struct enc_open_param open_param;
> > > > > > > > > > > +		int err = 0;
> > > > > > > > > > >
> > > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > > enc_open_param));
> > > > > > > > > > >
> > > > > > > > > > > +		err = pm_runtime_resume_and_get(inst-
> >dev->dev);
> > > > > > > > > > > +		if (err) {
> > > > > > > > > > > +			dev_err(inst->dev->dev, "encoder
> runtime
> > > > resume
> > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > > +		}
> > > > > > > > > > > +
> > > > > > > > > > >  		wave5_set_enc_openparam(&open_param,
> inst);
> > > > > > > > > > >
> > > > > > > > > > >  		ret = wave5_vpu_enc_open(inst,
> &open_param);
> > > > diff --
> > > > > > > > git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > .c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > .c index 0d90b5820bef..f81409740a56 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > .c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ -vpu
> > > > > > > > > > > +++ .c
> > > > > > > > > > > @@ -10,6 +10,7 @@
> > > > > > > > > > >  #include <linux/clk.h>
> > > > > > > > > > >  #include <linux/firmware.h>  #include
> > > > > > > > > > > <linux/interrupt.h>
> > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > >  #include "wave5-vpu.h"
> > > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > > >  #include "wave5-vpuconfig.h"
> > > > > > > > > > > @@ -117,6 +118,65 @@ static int
> > > > > > > > > > > wave5_vpu_load_firmware(struct device
> > > > > > > > > > *dev, const char *fw_name,
> > > > > > > > > > >  	return 0;
> > > > > > > > > > >  }
> > > > > > > > > > >
> > > > > > > > > > > +static __maybe_unused int wave5_pm_suspend(struct
> > > > > > > > > > > +device
> > > > > > > > > > > +*dev)
> > > > > > {
> > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > +
> > > > > > > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > > > > > > +		return 0;
> > > > > > > > > > > +
> > > > > > > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks,
> > > > > > > > > > > +vpu->clks);
> > > > > > > > > > > +
> > > > > > > > > > > +	return 0;
> > > > > > > > > > > +}
> > > > > > > > > > > +
> > > > > > > > > > > +static __maybe_unused int wave5_pm_resume(struct
> > > > > > > > > > > +device
> > > > *dev) {
> > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > +	int ret = 0;
> > > > > > > > > > > +
> > > > > > > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks,
> vpu-
> > > > > clks);
> > > > > > > > > > > +	if (ret) {
> > > > > > > > > > > +		dev_err(dev, "Enabling clocks,
> fail: %d\n",
> > > > ret);
> > > > > > > > > > > +		return ret;
> > > > > > > > > > > +	}
> > > > > > > > > > > +
> > > > > > > > > > > +	return ret;
> > > > > > > > > > > +}
> > > > > > > > > > > +
> > > > > > > > > > > +static __maybe_unused int wave5_suspend(struct
> > > > > > > > > > > +device
> > > > > > > > > > > +*dev)
> > > > {
> > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > > +
> > > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > > > > > > +
> > > > > > > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > > > > > > +
> > > > > > > > > > > +static __maybe_unused int wave5_resume(struct
> > > > > > > > > > > +device
> > > > > > > > > > > +*dev)
> > > > {
> > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > > +	int ret = 0;
> > > > > > > > > > > +
> > > > > > > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > > > > > > +	if (ret < 0)
> > > > > > > > > > > +		return ret;
> > > > > > > > > > > +
> > > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > > > > > > +
> > > > > > > > > > > +	return ret;
> > > > > > > > > > > +}
> > > > > > > > > >
> > > > > > > > > > The functions wave5_suspend() and wave5_resume() are
> > > > > > > > > > not just "maybe_unsued" but actually never used. What
> > > > > > > > > > was the
> > > > intention ?
> > > > > > > > > > Considering the usage of __maybe_unused has been such
> > > > > > > > > > a bad friend for this one, could you instead bracket
> > > > > > > > > > the functions with an
> > > > > > > > explicit ?
> > > > > > > > > >
> > > > > > > > > > #ifdef CONFIG_PM
> > > > > > > > > > #endif
> > > > > > > > > >
> > > > > > > > > > This way the compiler will have a word if you
> > > > > > > > > > implement something that you forgot to actually use.
> > > > > > > > > >
> > > > > > > > > > > +
> > > > > > > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend,
> > > > > > > > > > > +wave5_pm_resume,
> > > > > > > > NULL) };
> > > > > > > > > > > +
> > > > > > > > > > >  static int wave5_vpu_probe(struct platform_device
> > > > > > > > > > > *pdev)
> > > {
> > > > > > > > > > >  	int ret;
> > > > > > > > > > > @@ -232,6 +292,10 @@ static int
> > > > > > > > > > > wave5_vpu_probe(struct platform_device
> > > > > > > > > > *pdev)
> > > > > > > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ?
> > > > "'DECODE'" : "");
> > > > > > > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n",
> dev-
> > > > > > > > > product_code);
> > > > > > > > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n",
> > > > > > > > > > > fw_revision);
> > > > > > > > > > > +
> > > > > > > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > > > > > > > +
> > > > > > > > > > >  	return 0;
> > > > > > > > > > >
> > > > > > > > > > >  err_enc_unreg:
> > > > > > > > > > > @@ -254,6 +318,9 @@ static int
> > > > > > > > > > > wave5_vpu_remove(struct platform_device
> > > > > > > > > > > *pdev)  {
> > > > > > > > > > >  	struct vpu_device *dev =
> > > > > > > > > > > dev_get_drvdata(&pdev->dev);
> > > > > > > > > > >
> > > > > > > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > > > > > > +
> > > > > > > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > > > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > > > > > > >  	clk_bulk_disable_unprepare(dev->num_clks,
> > > > > > > > > > > dev->clks);
> > > > @@
> > > > > > > > > > > -
> > > > > > > > 281,6
> > > > > > > > > > > +348,7 @@ static struct platform_driver
> > > > > > > > > > > +wave5_vpu_driver = {
> > > > > > > > > > >  	.driver = {
> > > > > > > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > > > > > > >  		.of_match_table =
> of_match_ptr(wave5_dt_ids),
> > > > > > > > > > > +		.pm = &wave5_pm_ops,
> > > > > > > > > > >  		},
> > > > > > > > > > >  	.probe = wave5_vpu_probe,
> > > > > > > > > > >  	.remove = wave5_vpu_remove, diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-
> vpuapi.
> > > > > > > > > > > c
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-
> vpuapi.
> > > > > > > > > > > c index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-
> vpuapi.
> > > > > > > > > > > c
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ -vpu
> > > > > > > > > > > +++ ap
> > > > > > > > > > > +++ i.c
> > > > > > > > > > > @@ -6,6 +6,7 @@
> > > > > > > > > > >   */
> > > > > > > > > > >
> > > > > > > > > > >  #include <linux/bug.h>
> > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > >  #include "wave5-vpuapi.h"
> > > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > > >  #include "wave5.h"
> > > > > > > > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct
> > > > > > > > > > > vpu_instance *inst,
> > > > > > > > > > > u32 *fail_res)
> > > > > > > > > > >
> > > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > > &p_dec_info->vb_task);
> > > > > > > > > > >
> > > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > > > +
> > > > > > > > > > >  unlock_and_return:
> > > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > >
> > > > > > > > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct
> > > > > > > > > > > vpu_instance *inst,
> > > > > > > > > > > u32 *fail_res)
> > > > > > > > > > >
> > > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > > &p_enc_info->vb_task);
> > > > > > > > > > >
> > > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > >
> > > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > > "start_streaming()", but the
> > > > > > > > > > put() is only done when the device is closed, or when
> > > > > > > > > > the driver is removed. As this is not balanced, you
> > > > > > > > > > seem to have to check the suspended condition all over
> the place.
> > > > > > > > > >
> > > > > > > > > > I think we could aim for
> > > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > > for your get/put placement. At least they will be
> > > > > > > > > > bound to an entirely balanced
> > > > > > > > API.
> > > > > > > > > > But then, a media player in paused sate will prevent
> > > > > > > > > > that device from being suspended.
> > > > > > > > > >
> > > > > > > > > > If the HW is capable of preserving enough state, and
> > > > > > > > > > From the short doc I have it gives me the impression
> > > > > > > > > > it can preserve that, I'd suggest to follow what
> > > > > > > > > > hantro driver is doing. What is does is that it will
> > > > > > > > > > do get() in device_run(), and put() whenever the job
> > > > > > > > > > completes. This driver has been designed so when there
> > > > > > > > > > is no job, it means the firmware is currently idle and
> looking for more work.
> > > > > > > > > > So it seems like the perfect moment to
> > > > > > do suspend it.
> > > > > > > > > >
> > > > > > > > > > Nicolas
> > > > > > > > > >
> > > > > > > > > > > +
> > > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > >
> > > > > > > > > > >  	return 0;
> > > > > > > > > > > diff --git
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > > > > > > > ---
> > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > +++ .h
> > > > > > > > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct
> > > > > > > > > > > vpu_device *vpu_dev, u32 *revision);
> > > > > > > > > > >
> > > > > > > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw,
> > > > > > > > > > > size_t size);
> > > > > > > > > > >
> > > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > > > +i_sleep_wake, const
> > > > > > > > > > uint16_t *code,
> > > > > > > > > > > +			 size_t size);
> > > > > > > > > > > +
> > > > > > > > > > >  int wave5_vpu_reset(struct device *dev, enum
> > > > > > > > > > > sw_reset_mode reset_mode);
> > > > > > > > > > >
> > > > > > > > > > >  int wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > > vpu_instance *inst, struct dec_open_param *param);
> > > > > > > > >
> > > > > > >
> > > > >
> >

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

Hi Jackson,

sorry for the the delay, my reply below.

Le lundi 26 février 2024 à 01:33 +0000, jackson.lee a écrit :
> Hello Nicolas
> 
[...]
> 
> I have added the autosuspend feature, can you please review the below code ?
> If there is no problem at your side, I will make a patch for upstream.
> 
> 
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> index 0b6386f31..2ba9e0f0e 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> @@ -1102,8 +1102,8 @@ int wave5_vpu_re_init(struct device *dev, u8 *fw, size_t size)
>  	return setup_wave5_properties(dev);
>  }
>  
> -static int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
> -				size_t size)
> +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const uint16_t *code,
> +			 size_t size)
>  {
>  	u32 reg_val;
>  	struct vpu_buf *common_vb;
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> index 8b1417ece..4aea66483 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> @@ -5,6 +5,7 @@
>   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
>   */
>  
> +#include <linux/pm_runtime.h>
>  #include "wave5-helper.h"
>  
>  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> @@ -518,6 +519,8 @@ static void wave5_vpu_dec_finish_decode(struct vpu_instance *inst)
>  	if (q_status.report_queue_count == 0 &&
>  	    (q_status.instance_queue_count == 0 || dec_info.sequence_changed)) {
>  		dev_dbg(inst->dev->dev, "%s: finishing job.\n", __func__);
> +		pm_runtime_mark_last_busy(inst->dev->dev);
> +		pm_runtime_put_autosuspend(inst->dev->dev);
>  		v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
>  	}
>  }
> @@ -1382,6 +1385,7 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
>  	int ret = 0;
>  
>  	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
> +	pm_runtime_resume_and_get(inst->dev->dev);
>  
>  	v4l2_m2m_update_start_streaming_state(m2m_ctx, q);
>  
> @@ -1389,7 +1393,6 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
>  		struct dec_open_param open_param;
>  
>  		memset(&open_param, 0, sizeof(struct dec_open_param));
> -

I'd refrain from mixing style changes with the function changes. Just leave the
empty line, is it harmless. There is other cases in your RFC.

>  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
>  		if (ret)
>  			goto return_buffers;
> @@ -1425,13 +1428,14 @@ static int wave5_vpu_dec_start_streaming(struct vb2_queue *q, unsigned int count
>  			}
>  		}
>  	}
> -

There is a case were userspace may start streaming but never populate the
queues. You should call pm_runtime_put_autosuspend() here for this reason. In
most cases it won't suspend as there device_run() get called very shortly after,
before the timeout expires.

>  	return ret;
>  
>  free_bitstream_vbuf:
>  	wave5_vdi_free_dma_memory(inst->dev, &inst->bitstream_vbuf);
>  return_buffers:
>  	wave5_return_bufs(q, VB2_BUF_STATE_QUEUED);
> +	pm_runtime_mark_last_busy(inst->dev->dev);

For fast operations or synchronous errors, I believe its fine to skip the
"mark_last_busy()" calls. This is resetting the timeout start time, but if the
elapse time is really short, I'm pretty sure there is very little values in
pushing it forward. Removing that call from error case, stream_start/stop should
make the patch a little lighter.

I think overall your RFC is in the right direction, and this is good material
for a V2.

regards,
Nicolas

> +	pm_runtime_put_autosuspend(inst->dev->dev);
>  	return ret;
>  }
>  
> @@ -1517,6 +1521,7 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q)
>  	bool check_cmd = TRUE;
>  
>  	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
> +	pm_runtime_resume_and_get(inst->dev->dev);
>  
>  	while (check_cmd) {
>  		struct queue_status_info q_status;
> @@ -1540,6 +1545,9 @@ static void wave5_vpu_dec_stop_streaming(struct vb2_queue *q)
>  		streamoff_output(q);
>  	else
>  		streamoff_capture(q);
> +
> +	pm_runtime_mark_last_busy(inst->dev->dev);
> +	pm_runtime_put_autosuspend(inst->dev->dev);
>  }
>  
>  static const struct vb2_ops wave5_vpu_dec_vb2_ops = {
> @@ -1626,7 +1634,7 @@ static void wave5_vpu_dec_device_run(void *priv)
>  	int ret = 0;
>  
>  	dev_dbg(inst->dev->dev, "%s: Fill the ring buffer with new bitstream data", __func__);
> -
> +	pm_runtime_resume_and_get(inst->dev->dev);
>  	ret = fill_ringbuffer(inst);
>  	if (ret) {
>  		dev_warn(inst->dev->dev, "Filling ring buffer failed\n");
> @@ -1709,6 +1717,8 @@ static void wave5_vpu_dec_device_run(void *priv)
>  
>  finish_job_and_return:
>  	dev_dbg(inst->dev->dev, "%s: leave and finish job", __func__);
> +	pm_runtime_mark_last_busy(inst->dev->dev);
> +	pm_runtime_put_autosuspend(inst->dev->dev);
>  	v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
>  }
>  
> @@ -1866,7 +1876,6 @@ static int wave5_vpu_open_dec(struct file *filp)
>  	}
>  
>  	wave5_vdi_allocate_sram(inst->dev);
> -
>  	return 0;
>  
>  cleanup_inst:
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> index f04baa93a..14fd26204 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> @@ -5,6 +5,7 @@
>   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
>   */
>  
> +#include <linux/pm_runtime.h>
>  #include "wave5-helper.h"
>  
>  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> @@ -1310,6 +1311,7 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count
>  	struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx;
>  	int ret = 0;
>  
> +	pm_runtime_resume_and_get(inst->dev->dev);
>  	v4l2_m2m_update_start_streaming_state(m2m_ctx, q);
>  
>  	if (inst->state == VPU_INST_STATE_NONE && q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> @@ -1364,9 +1366,13 @@ static int wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count
>  	if (ret)
>  		goto return_buffers;
>  
> +	pm_runtime_mark_last_busy(inst->dev->dev);
> +	pm_runtime_put_autosuspend(inst->dev->dev);
>  	return 0;
>  return_buffers:
>  	wave5_return_bufs(q, VB2_BUF_STATE_QUEUED);
> +	pm_runtime_mark_last_busy(inst->dev->dev);
> +	pm_runtime_put_autosuspend(inst->dev->dev);
>  	return ret;
>  }
>  
> @@ -1408,6 +1414,7 @@ static void wave5_vpu_enc_stop_streaming(struct vb2_queue *q)
>  	 */
>  
>  	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
> +	pm_runtime_resume_and_get(inst->dev->dev);
>  
>  	if (wave5_vpu_both_queues_are_streaming(inst))
>  		switch_state(inst, VPU_INST_STATE_STOP);
> @@ -1432,6 +1439,9 @@ static void wave5_vpu_enc_stop_streaming(struct vb2_queue *q)
>  		streamoff_output(inst, q);
>  	else
>  		streamoff_capture(inst, q);
> +
> +	pm_runtime_mark_last_busy(inst->dev->dev);
> +	pm_runtime_put_autosuspend(inst->dev->dev);
>  }
>  
>  static const struct vb2_ops wave5_vpu_enc_vb2_ops = {
> @@ -1478,6 +1488,7 @@ static void wave5_vpu_enc_device_run(void *priv)
>  	u32 fail_res = 0;
>  	int ret = 0;
>  
> +	pm_runtime_resume_and_get(inst->dev->dev);
>  	switch (inst->state) {
>  	case VPU_INST_STATE_PIC_RUN:
>  		ret = start_encode(inst, &fail_res);
> @@ -1491,6 +1502,8 @@ static void wave5_vpu_enc_device_run(void *priv)
>  			break;
>  		}
>  		dev_dbg(inst->dev->dev, "%s: leave with active job", __func__);
> +		pm_runtime_mark_last_busy(inst->dev->dev);
> +		pm_runtime_put_autosuspend(inst->dev->dev);
>  		return;
>  	default:
>  		WARN(1, "Execution of a job in state %s is invalid.\n",
> @@ -1498,6 +1511,8 @@ static void wave5_vpu_enc_device_run(void *priv)
>  		break;
>  	}
>  	dev_dbg(inst->dev->dev, "%s: leave and finish job", __func__);
> +	pm_runtime_mark_last_busy(inst->dev->dev);
> +	pm_runtime_put_autosuspend(inst->dev->dev);
>  	v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
>  }
>  
> @@ -1735,7 +1750,6 @@ static int wave5_vpu_open_enc(struct file *filp)
>  	}
>  
>  	wave5_vdi_allocate_sram(inst->dev);
> -
>  	return 0;
>  
>  cleanup_inst:
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu.c b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> index 2323dba75..6b4794e0e 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> @@ -10,6 +10,7 @@
>  #include <linux/clk.h>
>  #include <linux/firmware.h>
>  #include <linux/interrupt.h>
> +#include <linux/pm_runtime.h>
>  #include "wave5-vpu.h"
>  #include "wave5-regdefine.h"
>  #include "wave5-vpuconfig.h"
> @@ -117,6 +118,38 @@ static int wave5_vpu_load_firmware(struct device *dev, const char *fw_name,
>  	return 0;
>  }
>  
> +static int wave5_pm_suspend(struct device *dev)
> +{
> +	struct vpu_device *vpu = dev_get_drvdata(dev);
> +
> +	if (pm_runtime_suspended(dev))
> +		return 0;
> +
> +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> +
> +	return 0;
> +}
> +
> +static int wave5_pm_resume(struct device *dev)
> +{
> +	struct vpu_device *vpu = dev_get_drvdata(dev);
> +	int ret = 0;
> +
> +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> +	if (ret) {
> +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> +		return ret;
> +	}
> +
> +	return ret;
> +}
> +
> +static const struct dev_pm_ops wave5_pm_ops = {
> +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume, NULL)
> +};
> +
>  static int wave5_vpu_probe(struct platform_device *pdev)
>  {
>  	int ret;
> @@ -233,6 +266,12 @@ static int wave5_vpu_probe(struct platform_device *pdev)
>  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
>  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev->product_code);
>  	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
> +
> +	pm_runtime_set_autosuspend_delay(&pdev->dev, 5000);
> +	pm_runtime_use_autosuspend(&pdev->dev);
> +	pm_runtime_enable(&pdev->dev);
> +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> +
>  	return 0;
>  
>  err_enc_unreg:
> @@ -255,6 +294,9 @@ static int wave5_vpu_remove(struct platform_device *pdev)
>  {
>  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
>  
> +	pm_runtime_put_sync(&pdev->dev);
> +	pm_runtime_disable(&pdev->dev);
> +
>  	mutex_destroy(&dev->dev_lock);
>  	mutex_destroy(&dev->hw_lock);
>  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks);
> @@ -282,6 +324,7 @@ static struct platform_driver wave5_vpu_driver = {
>  	.driver = {
>  		.name = VPU_PLATFORM_DEVICE_NAME,
>  		.of_match_table = of_match_ptr(wave5_dt_ids),
> +		.pm = &wave5_pm_ops,
>  		},
>  	.probe = wave5_vpu_probe,
>  	.remove = wave5_vpu_remove,
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> index 86b399372..b2357d1b5 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> @@ -6,6 +6,8 @@
>   */
>  
>  #include <linux/bug.h>
> +#include <linux/pm_runtime.h>
> +#include <linux/delay.h>
>  #include "wave5-vpuapi.h"
>  #include "wave5-regdefine.h"
>  #include "wave5.h"
> @@ -200,6 +202,8 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res)
>  	if (!inst->codec_info)
>  		return -EINVAL;
>  
> +	pm_runtime_resume_and_get(inst->dev->dev);
> +
>  	ret = mutex_lock_interruptible(&vpu_dev->hw_lock);
>  	if (ret)
>  		return ret;
> @@ -234,7 +238,7 @@ int wave5_vpu_dec_close(struct vpu_instance *inst, u32 *fail_res)
>  
>  unlock_and_return:
>  	mutex_unlock(&vpu_dev->hw_lock);
> -
> +	pm_runtime_put_autosuspend(inst->dev->dev);
>  	return ret;
>  }
>  
> @@ -702,6 +706,8 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res)
>  	if (!inst->codec_info)
>  		return -EINVAL;
>  
> +	pm_runtime_resume_and_get(inst->dev->dev);
> +
>  	ret = mutex_lock_interruptible(&vpu_dev->hw_lock);
>  	if (ret)
>  		return ret;
> @@ -733,9 +739,10 @@ int wave5_vpu_enc_close(struct vpu_instance *inst, u32 *fail_res)
>  	}
>  
>  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
> -
>  	mutex_unlock(&vpu_dev->hw_lock);
>  
> +	pm_runtime_put_autosuspend(inst->dev->dev);
> +
>  	return 0;
>  }
>  
> 
> 
> 
> 
> 
> 
> 
> 
> 
> 
> > > 
> > > 
> > > Thanks.
> > > Jackson
> > > 
> > > > 
> > > > The delay means a timer, so there is no input for 5 secs, then
> > > > timeout callback is called, And suspend is set, if new activity
> > > > comes, the device is resumed again ?
> > > > My understanding is correct ?
> > > > 
> > > 
> > > 
> > > > 
> > > > 
> > > > > Nicolas
> > > > > 
> > > > > > 
> > > > > > Thanks.
> > > > > > 
> > > > > > 
> > > > > > > There is of course other places where you'll have to make sure
> > > > > > > the hardware is resumed, like on close, as you want to remove
> > > > > > > the
> > > > instance.
> > > > > > > There is also small queries here and there that need to be
> > > > > > > surrounded with resume/put, but with the redesign, most of the
> > > > > > > HW access now take place inside device_run() only.
> > > > > > > 
> > > > > > > Open/Close is not invalid, but it has a lot of issues, as any
> > > > > > > random application can endup disabling the kernel ability to
> > > > > > > save
> > > > power.
> > > > > > > Personally, I think we should at least give it a try, and
> > > > > > > document valid reason not to do so if we find hardware issues.
> > > > > > > Otherwise, this sounds like all we care is ticking the box
> > > > > > > "this driver has runtime PM" without actually caring about
> > effective power saving.
> > > > > > > 
> > > > > > > Nicolas
> > > > > > > 
> > > > > > > > 
> > > > > > > > Thanks.
> > > > > > > > 
> > > > > > > > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a écrit :
> > > > > > > > > > Hi Nicolas
> > > > > > > > > > 
> > > > > > > > > > 
> > > > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > > > "start_streaming()", but the
> > > > > > > > > > > put() is only done when the device is closed, or when
> > > > > > > > > > > the driver is removed. As this is not balanced, you
> > > > > > > > > > > seem to have to check the suspended condition all over
> > the place.
> > > > > > > > > > > 
> > > > > > > > > > > I think we could aim for
> > > > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > > > for your get/put placement. At least they will be
> > > > > > > > > > > bound to an entirely balanced
> > > > > > > > > API.
> > > > > > > > > > > But then, a media player in paused sate will prevent
> > > > > > > > > > > that device from being suspended.
> > > > > > > > > > > 
> > > > > > > > > > > If the HW is capable of preserving enough state, and
> > > > > > > > > > > From the short doc I have it gives me the impression
> > > > > > > > > > > it can preserve that, I'd suggest to follow what
> > > > > > > > > > > hantro driver is doing. What is does is that it will
> > > > > > > > > > > do get() in device_run(), and put() whenever the job
> > > > > > > > > > > completes. This driver has been designed so when there
> > > > > > > > > > > is no job, it means the firmware is currently idle and
> > looking for more work.
> > > > > > > > > > > So it seems like the perfect moment to
> > > > > > > do suspend it.
> > > > > > > > > > > 
> > > > > > > > > > 
> > > > > > > > > > Thanks your comment,
> > > > > > > > > > 
> > > > > > > > > > Currently they are not balanced, If we puts "the put
> > functon"
> > > > > > > > > > into the stop_streaming, our hw is stalled
> > > > > > > > > until doing wake-up command, so our v4l2 device become block.
> > > > > > > > > > so I'd like to update the below instead of calling get
> > > > > > > > > > at the
> > > > > > > > > start_streaming function.
> > > > > > > > > > 
> > > > > > > > > > 
> > > > > > > > > > @@ -1867,6 +1868,13 @@ static int
> > > > > > > > > > wave5_vpu_open_dec(struct file
> > > > > > > > > > *filp)
> > > > > > > > > > 
> > > > > > > > > >         wave5_vdi_allocate_sram(inst->dev);
> > > > > > > > > > 
> > > > > > > > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > > > +       if (err) {
> > > > > > > > > > +               dev_err(inst->dev->dev, "decoder runtime
> > > > > > > > > > + resume
> > > > > > > > > failed %d\n", err);
> > > > > > > > > > +               ret = -EINVAL;
> > > > > > > > > > +               goto cleanup_inst;
> > > > > > > > > > +       }
> > > > > > > > > > +
> > > > > > > > > >         return 0;
> > > > > > > > > 
> > > > > > > > > I guess we need to discuss the power management strategy
> > > > > > > > > for this
> > > > > > > device.
> > > > > > > > > If you do resume_and_get() in open(), and then put in
> > > > > > > > > close(), that seems balanced. But in term of power saving,
> > > > > > > > > it might not be very strong. If you have a media player
> > > > > > > > > that is set to pause and then placed in the background,
> > > > > > > > > you still keep the IP running. This is extremely common,
> > > > > > > > > since application cannot close their device without
> > > > > > > > > loosing the reference frames, and thus having to do extra
> > > > > > > > > work on resume to seek back to the previous sync point and
> > > > > > > > > drop
> > > > > > > unneeded frames.
> > > > > > > > > 
> > > > > > > > > It seems like the whole point of asking the firmware to
> > > > > > > > > save the state and suspend is to be able to do so while
> > > > > > > > > there is meaningful sate in the firt place.
> > > > > > > > > If we are to suspend only when there is no meaningful
> > > > > > > > > state, we could just free all resources and power it off
> > completely.
> > > > > > > > > (This is just for illustration, its probably to slow to
> > > > > > > > > boot the firmware at
> > > > > > > > > runtime)
> > > > > > > > > 
> > > > > > > > > 
> > > > > > > > > I understand you suffered lockup with a start_streaming()
> > > > > > > > > for resume_and_get(), and stop_streaming() for put(), this
> > > > > > > > > may simply indicate that some hardware access are needed
> > > > > > > > > between these two. Can you write down a power management
> > > > > > > > > plan that would effectively save power in common use cases
> > > > > > > > > ? We can certainly help in refactoring the
> > > > > > > code to make that happen.
> > > > > > > > > 
> > > > > > > > > Nicolas
> > > > > > > > > 
> > > > > > > > > > 
> > > > > > > > > > > Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee
> > > > > > > > > > > a
> > > > > écrit :
> > > > > > > > > > > > There are two device run-time PM callbacks defined
> > > > > > > > > > > > in 'struct
> > > > > > > > > > > dev_pm_ops'
> > > > > > > > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > > > > > > > (*runtime_resume)(struct device *dev);
> > > > > > > > > > > 
> > > > > > > > > > > I wonder how useful is it to teach everyone what the
> > > > > > > > > > > generic 'struct dev_pm_ops'
> > > > > > > > > > > contains. Perhaps you simply wanted that this patch
> > > > > > > > > > > implement both suspend and resume ops ?
> > > > > > > > > > > 
> > > > > > > > > > > > 
> > > > > > > > > > > > Signed-off-by: Jackson Lee
> > > > > > > > > > > > <jackson.lee@chipsnmedia.com>
> > > > > > > > > > > > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > > > > > > > > > > > ---
> > > > > > > > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5 +-
> > > > > > > > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9 +++
> > > > > > > > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9 +++
> > > > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > > > > > > > +++++++++++++++++++
> > > > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |  7
> > > > > > > > > > > > ++ .../media/platform/chips-media/wave5/wave5.h  |
> > > > > > > > > > > > 3 +
> > > > > > > > > > > >  6 files changed, 99 insertions(+), 2 deletions(-)
> > > > > > > > > > > > 
> > > > > > > > > > > > diff --git
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > > c
> > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > > c index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > > > > > > > ---
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-hw.
> > > > > > > > > > > > c
> > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5-
> > hw.
> > > > > > > > > > > > +++ c
> > > > > > > > > > > > @@ -503,6 +503,7 @@ int
> > > > > > > > > > > > wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > > > vpu_instance
> > > > > > > > > > > *inst,
> > > > > > > > > > > >  	/* This register must be reset explicitly */
> > > > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_NUM_CQ_DEPTH_M1,
> > > > > > > > > > > > (COMMAND_QUEUE_DEPTH - 1));
> > > > > > > > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL, 0);
> > > > > > > > > > > 
> > > > > > > > > > > In some way, the relation between suspend and this
> > > > > > > > > > > register write is not obvious. If its not related,
> > > > > > > > > > > please do this in its
> > > > > > > own patch.
> > > > > > > > > > > Otherwise you want to explain why you needed this
> > > > > > > > > > > (possibly just in the commit message).
> > > > > > > > > > > 
> > > > > > > > > > > > 
> > > > > > > > > > > >  	ret = send_firmware_command(inst,
> > > > > > > > > > > > W5_CREATE_INSTANCE, true, NULL,
> > > > > > > > > > > NULL);
> > > > > > > > > > > >  	if (ret) {
> > > > > > > > > > > > @@ -1075,8 +1076,8 @@ int wave5_vpu_re_init(struct
> > > > > > > > > > > > device *dev, u8 *fw,
> > > > > > > > > > > size_t size)
> > > > > > > > > > > >  	return setup_wave5_properties(dev);  }
> > > > > > > > > > > > 
> > > > > > > > > > > > -static int wave5_vpu_sleep_wake(struct device *dev,
> > > > > > > > > > > > bool i_sleep_wake,
> > > > > > > > > > > const uint16_t *code,
> > > > > > > > > > > > -				size_t size)
> > > > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > > > > +i_sleep_wake, const
> > > > > > > > > > > uint16_t *code,
> > > > > > > > > > > > +			 size_t size)
> > > > > > > > > > > >  {
> > > > > > > > > > > >  	u32 reg_val;
> > > > > > > > > > > >  	struct vpu_buf *common_vb; diff --git
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > -dec
> > > > > > > > > > > > .c
> > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > -dec .c index ef227af72348..328a7a8f26c5 100644
> > > > > > > > > > > > ---
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > -dec
> > > > > > > > > > > > .c
> > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > +++ -vpu
> > > > > > > > > > > > +++ -d
> > > > > > > > > > > > +++ ec.c
> > > > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > > > >   */
> > > > > > > > > > > > 
> > > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > > > 
> > > > > > > > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > > > 
> > > > > > > > > > > >  	if (q->type ==
> > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE
> > > > > > > > > > > > &&
> > > > > > > > > > > > inst-
> > > > > > > > > > state
> > > > > > > > > > > > ==
> > > > > > > > > > > VPU_INST_STATE_NONE) {
> > > > > > > > > > > >  		struct dec_open_param open_param;
> > > > > > > > > > > > +		int err = 0;
> > > > > > > > > > > > 
> > > > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > > > dec_open_param));
> > > > > > > > > > > > 
> > > > > > > > > > > > +		err = pm_runtime_resume_and_get(inst-
> > > dev->dev);
> > > > > > > > > > > > +		if (err) {
> > > > > > > > > > > > +			dev_err(inst->dev->dev, "decoder
> > runtime
> > > > > resume
> > > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > > > +		}
> > > > > > > > > > > > +
> > > > > > > > > > > >  		ret =
> > wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > > > > > > > >  		if (ret)
> > > > > > > > > > > >  			goto return_buffers; diff --git
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > -enc
> > > > > > > > > > > > .c
> > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > -enc .c index 761775216cd4..ff73d69de41c 100644
> > > > > > > > > > > > ---
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > -enc
> > > > > > > > > > > > .c
> > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > +++ -vpu
> > > > > > > > > > > > +++ -e
> > > > > > > > > > > > +++ nc.c
> > > > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > > > >   */
> > > > > > > > > > > > 
> > > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > > > 
> > > > > > > > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > > > 
> > > > > > > > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q-
> > > type
> > > > > > > > > > > > ==
> > > > > > > > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > > > > > > > >  		struct enc_open_param open_param;
> > > > > > > > > > > > +		int err = 0;
> > > > > > > > > > > > 
> > > > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > > > enc_open_param));
> > > > > > > > > > > > 
> > > > > > > > > > > > +		err = pm_runtime_resume_and_get(inst-
> > > dev->dev);
> > > > > > > > > > > > +		if (err) {
> > > > > > > > > > > > +			dev_err(inst->dev->dev, "encoder
> > runtime
> > > > > resume
> > > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > > > +		}
> > > > > > > > > > > > +
> > > > > > > > > > > >  		wave5_set_enc_openparam(&open_param,
> > inst);
> > > > > > > > > > > > 
> > > > > > > > > > > >  		ret = wave5_vpu_enc_open(inst,
> > &open_param);
> > > > > diff --
> > > > > > > > > git
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > .c
> > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > .c index 0d90b5820bef..f81409740a56 100644
> > > > > > > > > > > > ---
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-vpu
> > > > > > > > > > > > .c
> > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > +++ -vpu
> > > > > > > > > > > > +++ .c
> > > > > > > > > > > > @@ -10,6 +10,7 @@
> > > > > > > > > > > >  #include <linux/clk.h>
> > > > > > > > > > > >  #include <linux/firmware.h>  #include
> > > > > > > > > > > > <linux/interrupt.h>
> > > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > > >  #include "wave5-vpu.h"
> > > > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > > > >  #include "wave5-vpuconfig.h"
> > > > > > > > > > > > @@ -117,6 +118,65 @@ static int
> > > > > > > > > > > > wave5_vpu_load_firmware(struct device
> > > > > > > > > > > *dev, const char *fw_name,
> > > > > > > > > > > >  	return 0;
> > > > > > > > > > > >  }
> > > > > > > > > > > > 
> > > > > > > > > > > > +static __maybe_unused int wave5_pm_suspend(struct
> > > > > > > > > > > > +device
> > > > > > > > > > > > +*dev)
> > > > > > > {
> > > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > > +
> > > > > > > > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > > > > > > > +		return 0;
> > > > > > > > > > > > +
> > > > > > > > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > > > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks,
> > > > > > > > > > > > +vpu->clks);
> > > > > > > > > > > > +
> > > > > > > > > > > > +	return 0;
> > > > > > > > > > > > +}
> > > > > > > > > > > > +
> > > > > > > > > > > > +static __maybe_unused int wave5_pm_resume(struct
> > > > > > > > > > > > +device
> > > > > *dev) {
> > > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > > +	int ret = 0;
> > > > > > > > > > > > +
> > > > > > > > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > > > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks,
> > vpu-
> > > > > > clks);
> > > > > > > > > > > > +	if (ret) {
> > > > > > > > > > > > +		dev_err(dev, "Enabling clocks,
> > fail: %d\n",
> > > > > ret);
> > > > > > > > > > > > +		return ret;
> > > > > > > > > > > > +	}
> > > > > > > > > > > > +
> > > > > > > > > > > > +	return ret;
> > > > > > > > > > > > +}
> > > > > > > > > > > > +
> > > > > > > > > > > > +static __maybe_unused int wave5_suspend(struct
> > > > > > > > > > > > +device
> > > > > > > > > > > > +*dev)
> > > > > {
> > > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > > > +
> > > > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > > > > > > > +
> > > > > > > > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > > > > > > > +
> > > > > > > > > > > > +static __maybe_unused int wave5_resume(struct
> > > > > > > > > > > > +device
> > > > > > > > > > > > +*dev)
> > > > > {
> > > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > > > +	int ret = 0;
> > > > > > > > > > > > +
> > > > > > > > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > > > > > > > +	if (ret < 0)
> > > > > > > > > > > > +		return ret;
> > > > > > > > > > > > +
> > > > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > > > > > > > +
> > > > > > > > > > > > +	return ret;
> > > > > > > > > > > > +}
> > > > > > > > > > > 
> > > > > > > > > > > The functions wave5_suspend() and wave5_resume() are
> > > > > > > > > > > not just "maybe_unsued" but actually never used. What
> > > > > > > > > > > was the
> > > > > intention ?
> > > > > > > > > > > Considering the usage of __maybe_unused has been such
> > > > > > > > > > > a bad friend for this one, could you instead bracket
> > > > > > > > > > > the functions with an
> > > > > > > > > explicit ?
> > > > > > > > > > > 
> > > > > > > > > > > #ifdef CONFIG_PM
> > > > > > > > > > > #endif
> > > > > > > > > > > 
> > > > > > > > > > > This way the compiler will have a word if you
> > > > > > > > > > > implement something that you forgot to actually use.
> > > > > > > > > > > 
> > > > > > > > > > > > +
> > > > > > > > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > > > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend,
> > > > > > > > > > > > +wave5_pm_resume,
> > > > > > > > > NULL) };
> > > > > > > > > > > > +
> > > > > > > > > > > >  static int wave5_vpu_probe(struct platform_device
> > > > > > > > > > > > *pdev)
> > > > {
> > > > > > > > > > > >  	int ret;
> > > > > > > > > > > > @@ -232,6 +292,10 @@ static int
> > > > > > > > > > > > wave5_vpu_probe(struct platform_device
> > > > > > > > > > > *pdev)
> > > > > > > > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ?
> > > > > "'DECODE'" : "");
> > > > > > > > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n",
> > dev-
> > > > > > > > > > product_code);
> > > > > > > > > > > >  	dev_info(&pdev->dev, "Firmware Revision: %u\n",
> > > > > > > > > > > > fw_revision);
> > > > > > > > > > > > +
> > > > > > > > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > > > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > > > > > > > > > > > +
> > > > > > > > > > > >  	return 0;
> > > > > > > > > > > > 
> > > > > > > > > > > >  err_enc_unreg:
> > > > > > > > > > > > @@ -254,6 +318,9 @@ static int
> > > > > > > > > > > > wave5_vpu_remove(struct platform_device
> > > > > > > > > > > > *pdev)  {
> > > > > > > > > > > >  	struct vpu_device *dev =
> > > > > > > > > > > > dev_get_drvdata(&pdev->dev);
> > > > > > > > > > > > 
> > > > > > > > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > > > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > > > > > > > +
> > > > > > > > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > > > > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > > > > > > > >  	clk_bulk_disable_unprepare(dev->num_clks,
> > > > > > > > > > > > dev->clks);
> > > > > @@
> > > > > > > > > > > > -
> > > > > > > > > 281,6
> > > > > > > > > > > > +348,7 @@ static struct platform_driver
> > > > > > > > > > > > +wave5_vpu_driver = {
> > > > > > > > > > > >  	.driver = {
> > > > > > > > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > > > > > > > >  		.of_match_table =
> > of_match_ptr(wave5_dt_ids),
> > > > > > > > > > > > +		.pm = &wave5_pm_ops,
> > > > > > > > > > > >  		},
> > > > > > > > > > > >  	.probe = wave5_vpu_probe,
> > > > > > > > > > > >  	.remove = wave5_vpu_remove, diff --git
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-
> > vpuapi.
> > > > > > > > > > > > c
> > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-
> > vpuapi.
> > > > > > > > > > > > c index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > > > > > > > ---
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-
> > vpuapi.
> > > > > > > > > > > > c
> > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > +++ -vpu
> > > > > > > > > > > > +++ ap
> > > > > > > > > > > > +++ i.c
> > > > > > > > > > > > @@ -6,6 +6,7 @@
> > > > > > > > > > > >   */
> > > > > > > > > > > > 
> > > > > > > > > > > >  #include <linux/bug.h>
> > > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > > >  #include "wave5-vpuapi.h"
> > > > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > > > >  #include "wave5.h"
> > > > > > > > > > > > @@ -232,6 +233,9 @@ int wave5_vpu_dec_close(struct
> > > > > > > > > > > > vpu_instance *inst,
> > > > > > > > > > > > u32 *fail_res)
> > > > > > > > > > > > 
> > > > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > > > &p_dec_info->vb_task);
> > > > > > > > > > > > 
> > > > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > > > > +
> > > > > > > > > > > >  unlock_and_return:
> > > > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > > > 
> > > > > > > > > > > > @@ -734,6 +738,9 @@ int wave5_vpu_enc_close(struct
> > > > > > > > > > > > vpu_instance *inst,
> > > > > > > > > > > > u32 *fail_res)
> > > > > > > > > > > > 
> > > > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > > > &p_enc_info->vb_task);
> > > > > > > > > > > > 
> > > > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > > > 
> > > > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > > > "start_streaming()", but the
> > > > > > > > > > > put() is only done when the device is closed, or when
> > > > > > > > > > > the driver is removed. As this is not balanced, you
> > > > > > > > > > > seem to have to check the suspended condition all over
> > the place.
> > > > > > > > > > > 
> > > > > > > > > > > I think we could aim for
> > > > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > > > for your get/put placement. At least they will be
> > > > > > > > > > > bound to an entirely balanced
> > > > > > > > > API.
> > > > > > > > > > > But then, a media player in paused sate will prevent
> > > > > > > > > > > that device from being suspended.
> > > > > > > > > > > 
> > > > > > > > > > > If the HW is capable of preserving enough state, and
> > > > > > > > > > > From the short doc I have it gives me the impression
> > > > > > > > > > > it can preserve that, I'd suggest to follow what
> > > > > > > > > > > hantro driver is doing. What is does is that it will
> > > > > > > > > > > do get() in device_run(), and put() whenever the job
> > > > > > > > > > > completes. This driver has been designed so when there
> > > > > > > > > > > is no job, it means the firmware is currently idle and
> > looking for more work.
> > > > > > > > > > > So it seems like the perfect moment to
> > > > > > > do suspend it.
> > > > > > > > > > > 
> > > > > > > > > > > Nicolas
> > > > > > > > > > > 
> > > > > > > > > > > > +
> > > > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > > > 
> > > > > > > > > > > >  	return 0;
> > > > > > > > > > > > diff --git
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > > index 063028eccd3b..6125eff938a8 100644
> > > > > > > > > > > > ---
> > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > +++ .h
> > > > > > > > > > > > @@ -56,6 +56,9 @@ int wave5_vpu_get_version(struct
> > > > > > > > > > > > vpu_device *vpu_dev, u32 *revision);
> > > > > > > > > > > > 
> > > > > > > > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw,
> > > > > > > > > > > > size_t size);
> > > > > > > > > > > > 
> > > > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev, bool
> > > > > > > > > > > > +i_sleep_wake, const
> > > > > > > > > > > uint16_t *code,
> > > > > > > > > > > > +			 size_t size);
> > > > > > > > > > > > +
> > > > > > > > > > > >  int wave5_vpu_reset(struct device *dev, enum
> > > > > > > > > > > > sw_reset_mode reset_mode);
> > > > > > > > > > > > 
> > > > > > > > > > > >  int wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > > > vpu_instance *inst, struct dec_open_param *param);
> > > > > > > > > > 
> > > > > > > > 
> > > > > > 
> > > 
> 

RE: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[jackson.lee]

Hello Nicolas

Thanks for your review.

> Hi Jackson,
> 
> sorry for the the delay, my reply below.
> 
> Le lundi 26 février 2024 à 01:33 +0000, jackson.lee a écrit :
> > Hello Nicolas
> >
> [...]
> >
> > I have added the autosuspend feature, can you please review the below
> code ?
> > If there is no problem at your side, I will make a patch for upstream.
> >
> >
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > index 0b6386f31..2ba9e0f0e 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
> > @@ -1102,8 +1102,8 @@ int wave5_vpu_re_init(struct device *dev, u8 *fw,
> size_t size)
> >  	return setup_wave5_properties(dev);
> >  }
> >
> > -static int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake,
> const uint16_t *code,
> > -				size_t size)
> > +int wave5_vpu_sleep_wake(struct device *dev, bool i_sleep_wake, const
> uint16_t *code,
> > +			 size_t size)
> >  {
> >  	u32 reg_val;
> >  	struct vpu_buf *common_vb;
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > index 8b1417ece..4aea66483 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> > @@ -5,6 +5,7 @@
> >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> >   */
> >
> > +#include <linux/pm_runtime.h>
> >  #include "wave5-helper.h"
> >
> >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > @@ -518,6 +519,8 @@ static void wave5_vpu_dec_finish_decode(struct
> vpu_instance *inst)
> >  	if (q_status.report_queue_count == 0 &&
> >  	    (q_status.instance_queue_count == 0 ||
> dec_info.sequence_changed)) {
> >  		dev_dbg(inst->dev->dev, "%s: finishing job.\n", __func__);
> > +		pm_runtime_mark_last_busy(inst->dev->dev);
> > +		pm_runtime_put_autosuspend(inst->dev->dev);
> >  		v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);
> >  	}
> >  }
> > @@ -1382,6 +1385,7 @@ static int wave5_vpu_dec_start_streaming(struct
> vb2_queue *q, unsigned int count
> >  	int ret = 0;
> >
> >  	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
> > +	pm_runtime_resume_and_get(inst->dev->dev);
> >
> >  	v4l2_m2m_update_start_streaming_state(m2m_ctx, q);
> >
> > @@ -1389,7 +1393,6 @@ static int wave5_vpu_dec_start_streaming(struct
> vb2_queue *q, unsigned int count
> >  		struct dec_open_param open_param;
> >
> >  		memset(&open_param, 0, sizeof(struct dec_open_param));
> > -
> 
> I'd refrain from mixing style changes with the function changes. Just
> leave the empty line, is it harmless. There is other cases in your RFC.
> 
> >  		ret = wave5_vpu_dec_allocate_ring_buffer(inst);
> >  		if (ret)
> >  			goto return_buffers;
> > @@ -1425,13 +1428,14 @@ static int wave5_vpu_dec_start_streaming(struct
> vb2_queue *q, unsigned int count
> >  			}
> >  		}
> >  	}
> > -
> 
> There is a case were userspace may start streaming but never populate the
> queues. You should call pm_runtime_put_autosuspend() here for this reason.
> In most cases it won't suspend as there device_run() get called very
> shortly after, before the timeout expires.
> 
> >  	return ret;
> >
> >  free_bitstream_vbuf:
> >  	wave5_vdi_free_dma_memory(inst->dev, &inst->bitstream_vbuf);
> >  return_buffers:
> >  	wave5_return_bufs(q, VB2_BUF_STATE_QUEUED);
> > +	pm_runtime_mark_last_busy(inst->dev->dev);
> 
> For fast operations or synchronous errors, I believe its fine to skip the
> "mark_last_busy()" calls. This is resetting the timeout start time, but if
> the elapse time is really short, I'm pretty sure there is very little
> values in pushing it forward. Removing that call from error case,
> stream_start/stop should make the patch a little lighter.
> 
> I think overall your RFC is in the right direction, and this is good
> material for a V2.
> 
> regards,
> Nicolas
> 
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> >  	return ret;
> >  }
> >
> > @@ -1517,6 +1521,7 @@ static void wave5_vpu_dec_stop_streaming(struct
> vb2_queue *q)
> >  	bool check_cmd = TRUE;
> >
> >  	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
> > +	pm_runtime_resume_and_get(inst->dev->dev);
> >
> >  	while (check_cmd) {
> >  		struct queue_status_info q_status;
> > @@ -1540,6 +1545,9 @@ static void wave5_vpu_dec_stop_streaming(struct
> vb2_queue *q)
> >  		streamoff_output(q);
> >  	else
> >  		streamoff_capture(q);
> > +
> > +	pm_runtime_mark_last_busy(inst->dev->dev);
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> >  }
> >
> >  static const struct vb2_ops wave5_vpu_dec_vb2_ops = { @@ -1626,7
> > +1634,7 @@ static void wave5_vpu_dec_device_run(void *priv)
> >  	int ret = 0;
> >
> >  	dev_dbg(inst->dev->dev, "%s: Fill the ring buffer with new
> bitstream
> > data", __func__);
> > -
> > +	pm_runtime_resume_and_get(inst->dev->dev);
> >  	ret = fill_ringbuffer(inst);
> >  	if (ret) {
> >  		dev_warn(inst->dev->dev, "Filling ring buffer failed\n"); @@
> > -1709,6 +1717,8 @@ static void wave5_vpu_dec_device_run(void *priv)
> >
> >  finish_job_and_return:
> >  	dev_dbg(inst->dev->dev, "%s: leave and finish job", __func__);
> > +	pm_runtime_mark_last_busy(inst->dev->dev);
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> >  	v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);  }
> >
> > @@ -1866,7 +1876,6 @@ static int wave5_vpu_open_dec(struct file *filp)
> >  	}
> >
> >  	wave5_vdi_allocate_sram(inst->dev);
> > -
> >  	return 0;
> >
> >  cleanup_inst:
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > index f04baa93a..14fd26204 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > @@ -5,6 +5,7 @@
> >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> >   */
> >
> > +#include <linux/pm_runtime.h>
> >  #include "wave5-helper.h"
> >
> >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > @@ -1310,6 +1311,7 @@ static int wave5_vpu_enc_start_streaming(struct
> vb2_queue *q, unsigned int count
> >  	struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx;
> >  	int ret = 0;
> >
> > +	pm_runtime_resume_and_get(inst->dev->dev);
> >  	v4l2_m2m_update_start_streaming_state(m2m_ctx, q);
> >
> >  	if (inst->state == VPU_INST_STATE_NONE && q->type ==
> > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { @@ -1364,9 +1366,13 @@ static int
> wave5_vpu_enc_start_streaming(struct vb2_queue *q, unsigned int count
> >  	if (ret)
> >  		goto return_buffers;
> >
> > +	pm_runtime_mark_last_busy(inst->dev->dev);
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> >  	return 0;
> >  return_buffers:
> >  	wave5_return_bufs(q, VB2_BUF_STATE_QUEUED);
> > +	pm_runtime_mark_last_busy(inst->dev->dev);
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> >  	return ret;
> >  }
> >
> > @@ -1408,6 +1414,7 @@ static void wave5_vpu_enc_stop_streaming(struct
> vb2_queue *q)
> >  	 */
> >
> >  	dev_dbg(inst->dev->dev, "%s: type: %u\n", __func__, q->type);
> > +	pm_runtime_resume_and_get(inst->dev->dev);
> >
> >  	if (wave5_vpu_both_queues_are_streaming(inst))
> >  		switch_state(inst, VPU_INST_STATE_STOP); @@ -1432,6 +1439,9
> @@
> > static void wave5_vpu_enc_stop_streaming(struct vb2_queue *q)
> >  		streamoff_output(inst, q);
> >  	else
> >  		streamoff_capture(inst, q);
> > +
> > +	pm_runtime_mark_last_busy(inst->dev->dev);
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> >  }
> >
> >  static const struct vb2_ops wave5_vpu_enc_vb2_ops = { @@ -1478,6
> > +1488,7 @@ static void wave5_vpu_enc_device_run(void *priv)
> >  	u32 fail_res = 0;
> >  	int ret = 0;
> >
> > +	pm_runtime_resume_and_get(inst->dev->dev);
> >  	switch (inst->state) {
> >  	case VPU_INST_STATE_PIC_RUN:
> >  		ret = start_encode(inst, &fail_res); @@ -1491,6 +1502,8 @@
> static
> > void wave5_vpu_enc_device_run(void *priv)
> >  			break;
> >  		}
> >  		dev_dbg(inst->dev->dev, "%s: leave with active job",
> __func__);
> > +		pm_runtime_mark_last_busy(inst->dev->dev);
> > +		pm_runtime_put_autosuspend(inst->dev->dev);
> >  		return;
> >  	default:
> >  		WARN(1, "Execution of a job in state %s is invalid.\n", @@ -
> 1498,6
> > +1511,8 @@ static void wave5_vpu_enc_device_run(void *priv)
> >  		break;
> >  	}
> >  	dev_dbg(inst->dev->dev, "%s: leave and finish job", __func__);
> > +	pm_runtime_mark_last_busy(inst->dev->dev);
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> >  	v4l2_m2m_job_finish(inst->v4l2_m2m_dev, m2m_ctx);  }
> >
> > @@ -1735,7 +1750,6 @@ static int wave5_vpu_open_enc(struct file *filp)
> >  	}
> >
> >  	wave5_vdi_allocate_sram(inst->dev);
> > -
> >  	return 0;
> >
> >  cleanup_inst:
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > index 2323dba75..6b4794e0e 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu.c
> > @@ -10,6 +10,7 @@
> >  #include <linux/clk.h>
> >  #include <linux/firmware.h>
> >  #include <linux/interrupt.h>
> > +#include <linux/pm_runtime.h>
> >  #include "wave5-vpu.h"
> >  #include "wave5-regdefine.h"
> >  #include "wave5-vpuconfig.h"
> > @@ -117,6 +118,38 @@ static int wave5_vpu_load_firmware(struct device
> *dev, const char *fw_name,
> >  	return 0;
> >  }
> >
> > +static int wave5_pm_suspend(struct device *dev) {
> > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > +
> > +	if (pm_runtime_suspended(dev))
> > +		return 0;
> > +
> > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > +	clk_bulk_disable_unprepare(vpu->num_clks, vpu->clks);
> > +
> > +	return 0;
> > +}
> > +
> > +static int wave5_pm_resume(struct device *dev) {
> > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > +	int ret = 0;
> > +
> > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > +	ret = clk_bulk_prepare_enable(vpu->num_clks, vpu->clks);
> > +	if (ret) {
> > +		dev_err(dev, "Enabling clocks, fail: %d\n", ret);
> > +		return ret;
> > +	}
> > +
> > +	return ret;
> > +}
> > +
> > +static const struct dev_pm_ops wave5_pm_ops = {
> > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend, wave5_pm_resume, NULL) };
> > +
> >  static int wave5_vpu_probe(struct platform_device *pdev)  {
> >  	int ret;
> > @@ -233,6 +266,12 @@ static int wave5_vpu_probe(struct platform_device
> *pdev)
> >  		 (match_data->flags & WAVE5_IS_DEC) ? "'DECODE'" : "");
> >  	dev_info(&pdev->dev, "Product Code:      0x%x\n", dev->product_code);
> >  	dev_info(&pdev->dev, "Firmware Revision: %u\n", fw_revision);
> > +
> > +	pm_runtime_set_autosuspend_delay(&pdev->dev, 5000);
> > +	pm_runtime_use_autosuspend(&pdev->dev);
> > +	pm_runtime_enable(&pdev->dev);
> > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL, 0);
> > +
> >  	return 0;
> >
> >  err_enc_unreg:
> > @@ -255,6 +294,9 @@ static int wave5_vpu_remove(struct platform_device
> > *pdev)  {
> >  	struct vpu_device *dev = dev_get_drvdata(&pdev->dev);
> >
> > +	pm_runtime_put_sync(&pdev->dev);
> > +	pm_runtime_disable(&pdev->dev);
> > +
> >  	mutex_destroy(&dev->dev_lock);
> >  	mutex_destroy(&dev->hw_lock);
> >  	clk_bulk_disable_unprepare(dev->num_clks, dev->clks); @@ -282,6
> > +324,7 @@ static struct platform_driver wave5_vpu_driver = {
> >  	.driver = {
> >  		.name = VPU_PLATFORM_DEVICE_NAME,
> >  		.of_match_table = of_match_ptr(wave5_dt_ids),
> > +		.pm = &wave5_pm_ops,
> >  		},
> >  	.probe = wave5_vpu_probe,
> >  	.remove = wave5_vpu_remove,
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > index 86b399372..b2357d1b5 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
> > @@ -6,6 +6,8 @@
> >   */
> >
> >  #include <linux/bug.h>
> > +#include <linux/pm_runtime.h>
> > +#include <linux/delay.h>
> >  #include "wave5-vpuapi.h"
> >  #include "wave5-regdefine.h"
> >  #include "wave5.h"
> > @@ -200,6 +202,8 @@ int wave5_vpu_dec_close(struct vpu_instance *inst,
> u32 *fail_res)
> >  	if (!inst->codec_info)
> >  		return -EINVAL;
> >
> > +	pm_runtime_resume_and_get(inst->dev->dev);
> > +
> >  	ret = mutex_lock_interruptible(&vpu_dev->hw_lock);
> >  	if (ret)
> >  		return ret;
> > @@ -234,7 +238,7 @@ int wave5_vpu_dec_close(struct vpu_instance *inst,
> > u32 *fail_res)
> >
> >  unlock_and_return:
> >  	mutex_unlock(&vpu_dev->hw_lock);
> > -
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> >  	return ret;
> >  }
> >
> > @@ -702,6 +706,8 @@ int wave5_vpu_enc_close(struct vpu_instance *inst,
> u32 *fail_res)
> >  	if (!inst->codec_info)
> >  		return -EINVAL;
> >
> > +	pm_runtime_resume_and_get(inst->dev->dev);
> > +
> >  	ret = mutex_lock_interruptible(&vpu_dev->hw_lock);
> >  	if (ret)
> >  		return ret;
> > @@ -733,9 +739,10 @@ int wave5_vpu_enc_close(struct vpu_instance *inst,
> u32 *fail_res)
> >  	}
> >
> >  	wave5_vdi_free_dma_memory(vpu_dev, &p_enc_info->vb_task);
> > -
> >  	mutex_unlock(&vpu_dev->hw_lock);
> >
> > +	pm_runtime_put_autosuspend(inst->dev->dev);
> > +
> >  	return 0;
> >  }
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> > > >
> > > >
> > > > Thanks.
> > > > Jackson
> > > >
> > > > >
> > > > > The delay means a timer, so there is no input for 5 secs, then
> > > > > timeout callback is called, And suspend is set, if new activity
> > > > > comes, the device is resumed again ?
> > > > > My understanding is correct ?
> > > > >
> > > >
> > > >
> > > > >
> > > > >
> > > > > > Nicolas
> > > > > >
> > > > > > >
> > > > > > > Thanks.
> > > > > > >
> > > > > > >
> > > > > > > > There is of course other places where you'll have to make
> > > > > > > > sure the hardware is resumed, like on close, as you want
> > > > > > > > to remove the
> > > > > instance.
> > > > > > > > There is also small queries here and there that need to be
> > > > > > > > surrounded with resume/put, but with the redesign, most of
> > > > > > > > the HW access now take place inside device_run() only.
> > > > > > > >
> > > > > > > > Open/Close is not invalid, but it has a lot of issues, as
> > > > > > > > any random application can endup disabling the kernel
> > > > > > > > ability to save
> > > > > power.
> > > > > > > > Personally, I think we should at least give it a try, and
> > > > > > > > document valid reason not to do so if we find hardware
> issues.
> > > > > > > > Otherwise, this sounds like all we care is ticking the box
> > > > > > > > "this driver has runtime PM" without actually caring about
> > > effective power saving.
> > > > > > > >
> > > > > > > > Nicolas
> > > > > > > >
> > > > > > > > >
> > > > > > > > > Thanks.
> > > > > > > > >
> > > > > > > > > > Le lundi 19 février 2024 à 04:04 +0000, jackson.lee a
> écrit :
> > > > > > > > > > > Hi Nicolas
> > > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > > > > "start_streaming()", but the
> > > > > > > > > > > > put() is only done when the device is closed, or
> > > > > > > > > > > > when the driver is removed. As this is not
> > > > > > > > > > > > balanced, you seem to have to check the suspended
> > > > > > > > > > > > condition all over
> > > the place.
> > > > > > > > > > > >
> > > > > > > > > > > > I think we could aim for
> > > > > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > > > > for your get/put placement. At least they will be
> > > > > > > > > > > > bound to an entirely balanced
> > > > > > > > > > API.
> > > > > > > > > > > > But then, a media player in paused sate will
> > > > > > > > > > > > prevent that device from being suspended.
> > > > > > > > > > > >
> > > > > > > > > > > > If the HW is capable of preserving enough state,
> > > > > > > > > > > > and From the short doc I have it gives me the
> > > > > > > > > > > > impression it can preserve that, I'd suggest to
> > > > > > > > > > > > follow what hantro driver is doing. What is does
> > > > > > > > > > > > is that it will do get() in device_run(), and
> > > > > > > > > > > > put() whenever the job completes. This driver has
> > > > > > > > > > > > been designed so when there is no job, it means
> > > > > > > > > > > > the firmware is currently idle and
> > > looking for more work.
> > > > > > > > > > > > So it seems like the perfect moment to
> > > > > > > > do suspend it.
> > > > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > > Thanks your comment,
> > > > > > > > > > >
> > > > > > > > > > > Currently they are not balanced, If we puts "the put
> > > functon"
> > > > > > > > > > > into the stop_streaming, our hw is stalled
> > > > > > > > > > until doing wake-up command, so our v4l2 device become
> block.
> > > > > > > > > > > so I'd like to update the below instead of calling
> > > > > > > > > > > get at the
> > > > > > > > > > start_streaming function.
> > > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > > @@ -1867,6 +1868,13 @@ static int
> > > > > > > > > > > wave5_vpu_open_dec(struct file
> > > > > > > > > > > *filp)
> > > > > > > > > > >
> > > > > > > > > > >         wave5_vdi_allocate_sram(inst->dev);
> > > > > > > > > > >
> > > > > > > > > > > +       err = pm_runtime_resume_and_get(inst->dev->dev);
> > > > > > > > > > > +       if (err) {
> > > > > > > > > > > +               dev_err(inst->dev->dev, "decoder
> > > > > > > > > > > + runtime resume
> > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > +               ret = -EINVAL;
> > > > > > > > > > > +               goto cleanup_inst;
> > > > > > > > > > > +       }
> > > > > > > > > > > +
> > > > > > > > > > >         return 0;
> > > > > > > > > >
> > > > > > > > > > I guess we need to discuss the power management
> > > > > > > > > > strategy for this
> > > > > > > > device.
> > > > > > > > > > If you do resume_and_get() in open(), and then put in
> > > > > > > > > > close(), that seems balanced. But in term of power
> > > > > > > > > > saving, it might not be very strong. If you have a
> > > > > > > > > > media player that is set to pause and then placed in
> > > > > > > > > > the background, you still keep the IP running. This is
> > > > > > > > > > extremely common, since application cannot close their
> > > > > > > > > > device without loosing the reference frames, and thus
> > > > > > > > > > having to do extra work on resume to seek back to the
> > > > > > > > > > previous sync point and drop
> > > > > > > > unneeded frames.
> > > > > > > > > >
> > > > > > > > > > It seems like the whole point of asking the firmware
> > > > > > > > > > to save the state and suspend is to be able to do so
> > > > > > > > > > while there is meaningful sate in the firt place.
> > > > > > > > > > If we are to suspend only when there is no meaningful
> > > > > > > > > > state, we could just free all resources and power it
> > > > > > > > > > off
> > > completely.
> > > > > > > > > > (This is just for illustration, its probably to slow
> > > > > > > > > > to boot the firmware at
> > > > > > > > > > runtime)
> > > > > > > > > >
> > > > > > > > > >
> > > > > > > > > > I understand you suffered lockup with a
> > > > > > > > > > start_streaming() for resume_and_get(), and
> > > > > > > > > > stop_streaming() for put(), this may simply indicate
> > > > > > > > > > that some hardware access are needed between these
> > > > > > > > > > two. Can you write down a power management plan that
> > > > > > > > > > would effectively save power in common use cases ? We
> > > > > > > > > > can certainly help in refactoring the
> > > > > > > > code to make that happen.
> > > > > > > > > >
> > > > > > > > > > Nicolas
> > > > > > > > > >
> > > > > > > > > > >
> > > > > > > > > > > > Le mercredi 31 janvier 2024 à 10:30 +0900,
> > > > > > > > > > > > jackson.lee a
> > > > > > écrit :
> > > > > > > > > > > > > There are two device run-time PM callbacks
> > > > > > > > > > > > > defined in 'struct
> > > > > > > > > > > > dev_pm_ops'
> > > > > > > > > > > > > int (*runtime_suspend)(struct device *dev); int
> > > > > > > > > > > > > (*runtime_resume)(struct device *dev);
> > > > > > > > > > > >
> > > > > > > > > > > > I wonder how useful is it to teach everyone what
> > > > > > > > > > > > the generic 'struct dev_pm_ops'
> > > > > > > > > > > > contains. Perhaps you simply wanted that this
> > > > > > > > > > > > patch implement both suspend and resume ops ?
> > > > > > > > > > > >
> > > > > > > > > > > > >
> > > > > > > > > > > > > Signed-off-by: Jackson Lee
> > > > > > > > > > > > > <jackson.lee@chipsnmedia.com>
> > > > > > > > > > > > > Signed-off-by: Nas Chung
> > > > > > > > > > > > > <nas.chung@chipsnmedia.com>
> > > > > > > > > > > > > ---
> > > > > > > > > > > > >  .../platform/chips-media/wave5/wave5-hw.c     |  5
> +-
> > > > > > > > > > > > >  .../chips-media/wave5/wave5-vpu-dec.c         |  9
> +++
> > > > > > > > > > > > >  .../chips-media/wave5/wave5-vpu-enc.c         |  9
> +++
> > > > > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpu.c    | 68
> > > > > > > > > > +++++++++++++++++++
> > > > > > > > > > > > >  .../platform/chips-media/wave5/wave5-vpuapi.c |
> > > > > > > > > > > > > 7
> > > > > > > > > > > > > ++ .../media/platform/chips-media/wave5/wave5.h
> > > > > > > > > > > > > ++ |
> > > > > > > > > > > > > 3 +
> > > > > > > > > > > > >  6 files changed, 99 insertions(+), 2
> > > > > > > > > > > > > deletions(-)
> > > > > > > > > > > > >
> > > > > > > > > > > > > diff --git
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-
> hw.
> > > > > > > > > > > > > c
> > > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5-
> hw.
> > > > > > > > > > > > > c index 8ad7f3a28ae1..8aade5a38439 100644
> > > > > > > > > > > > > ---
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5-
> hw.
> > > > > > > > > > > > > c
> > > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/w
> > > > > > > > > > > > > +++ ave5-
> > > hw.
> > > > > > > > > > > > > +++ c
> > > > > > > > > > > > > @@ -503,6 +503,7 @@ int
> > > > > > > > > > > > > wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > > > > vpu_instance
> > > > > > > > > > > > *inst,
> > > > > > > > > > > > >  	/* This register must be reset explicitly */
> > > > > > > > > > > > >  	vpu_write_reg(inst->dev, W5_CMD_EXT_ADDR, 0);
> > > > > > > > > > > > >  	vpu_write_reg(inst->dev,
> > > > > > > > > > > > > W5_CMD_NUM_CQ_DEPTH_M1, (COMMAND_QUEUE_DEPTH -
> > > > > > > > > > > > > 1));
> > > > > > > > > > > > > +	vpu_write_reg(inst->dev, W5_CMD_ERR_CONCEAL,
> > > > > > > > > > > > > +0);
> > > > > > > > > > > >
> > > > > > > > > > > > In some way, the relation between suspend and this
> > > > > > > > > > > > register write is not obvious. If its not related,
> > > > > > > > > > > > please do this in its
> > > > > > > > own patch.
> > > > > > > > > > > > Otherwise you want to explain why you needed this
> > > > > > > > > > > > (possibly just in the commit message).
> > > > > > > > > > > >
> > > > > > > > > > > > >
> > > > > > > > > > > > >  	ret = send_firmware_command(inst,
> > > > > > > > > > > > > W5_CREATE_INSTANCE, true, NULL,
> > > > > > > > > > > > NULL);
> > > > > > > > > > > > >  	if (ret) {
> > > > > > > > > > > > > @@ -1075,8 +1076,8 @@ int
> > > > > > > > > > > > > wave5_vpu_re_init(struct device *dev, u8 *fw,
> > > > > > > > > > > > size_t size)
> > > > > > > > > > > > >  	return setup_wave5_properties(dev);  }
> > > > > > > > > > > > >
> > > > > > > > > > > > > -static int wave5_vpu_sleep_wake(struct device
> > > > > > > > > > > > > *dev, bool i_sleep_wake,
> > > > > > > > > > > > const uint16_t *code,
> > > > > > > > > > > > > -				size_t size)
> > > > > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev,
> > > > > > > > > > > > > +bool i_sleep_wake, const
> > > > > > > > > > > > uint16_t *code,
> > > > > > > > > > > > > +			 size_t size)
> > > > > > > > > > > > >  {
> > > > > > > > > > > > >  	u32 reg_val;
> > > > > > > > > > > > >  	struct vpu_buf *common_vb; diff --git
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu
> > > > > > > > > > > > > -dec
> > > > > > > > > > > > > .c
> > > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu -dec .c index ef227af72348..328a7a8f26c5
> > > > > > > > > > > > > 100644
> > > > > > > > > > > > > ---
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu
> > > > > > > > > > > > > -dec
> > > > > > > > > > > > > .c
> > > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/w
> > > > > > > > > > > > > +++ ave5
> > > > > > > > > > > > > +++ -vpu
> > > > > > > > > > > > > +++ -d
> > > > > > > > > > > > > +++ ec.c
> > > > > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > > > > >   */
> > > > > > > > > > > > >
> > > > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > > > >
> > > > > > > > > > > > >  #define VPU_DEC_DEV_NAME "C&M Wave5 VPU decoder"
> > > > > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > > > > wave5_vpu_dec_start_streaming(struct
> > > > > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > > > >
> > > > > > > > > > > > >  	if (q->type ==
> > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE
> > > > > > > > > > > > > &&
> > > > > > > > > > > > > inst-
> > > > > > > > > > > state
> > > > > > > > > > > > > ==
> > > > > > > > > > > > VPU_INST_STATE_NONE) {
> > > > > > > > > > > > >  		struct dec_open_param open_param;
> > > > > > > > > > > > > +		int err = 0;
> > > > > > > > > > > > >
> > > > > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > > > > dec_open_param));
> > > > > > > > > > > > >
> > > > > > > > > > > > > +		err = pm_runtime_resume_and_get(inst-
> > > > dev->dev);
> > > > > > > > > > > > > +		if (err) {
> > > > > > > > > > > > > +			dev_err(inst->dev->dev, "decoder
> > > runtime
> > > > > > resume
> > > > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > > > > +		}
> > > > > > > > > > > > > +
> > > > > > > > > > > > >  		ret =
> > > wave5_vpu_dec_allocate_ring_buffer(inst);
> > > > > > > > > > > > >  		if (ret)
> > > > > > > > > > > > >  			goto return_buffers; diff --git
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu
> > > > > > > > > > > > > -enc
> > > > > > > > > > > > > .c
> > > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu -enc .c index 761775216cd4..ff73d69de41c
> > > > > > > > > > > > > 100644
> > > > > > > > > > > > > ---
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu
> > > > > > > > > > > > > -enc
> > > > > > > > > > > > > .c
> > > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/w
> > > > > > > > > > > > > +++ ave5
> > > > > > > > > > > > > +++ -vpu
> > > > > > > > > > > > > +++ -e
> > > > > > > > > > > > > +++ nc.c
> > > > > > > > > > > > > @@ -5,6 +5,7 @@
> > > > > > > > > > > > >   * Copyright (C) 2021-2023 CHIPS&MEDIA INC
> > > > > > > > > > > > >   */
> > > > > > > > > > > > >
> > > > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > > > >  #include "wave5-helper.h"
> > > > > > > > > > > > >
> > > > > > > > > > > > >  #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> > > > > > > > > > > > > @@ -1387,9 +1388,17 @@ static int
> > > > > > > > > > > > > wave5_vpu_enc_start_streaming(struct
> > > > > > > > > > > > > vb2_queue *q, unsigned int count
> > > > > > > > > > > > >
> > > > > > > > > > > > >  	if (inst->state == VPU_INST_STATE_NONE && q-
> > > > type
> > > > > > > > > > > > > ==
> > > > > > > > > > > > V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) {
> > > > > > > > > > > > >  		struct enc_open_param open_param;
> > > > > > > > > > > > > +		int err = 0;
> > > > > > > > > > > > >
> > > > > > > > > > > > >  		memset(&open_param, 0, sizeof(struct
> > > > > > enc_open_param));
> > > > > > > > > > > > >
> > > > > > > > > > > > > +		err = pm_runtime_resume_and_get(inst-
> > > > dev->dev);
> > > > > > > > > > > > > +		if (err) {
> > > > > > > > > > > > > +			dev_err(inst->dev->dev, "encoder
> > > runtime
> > > > > > resume
> > > > > > > > > > > > failed %d\n", err);
> > > > > > > > > > > > > +			ret = -EINVAL;
> > > > > > > > > > > > > +			goto return_buffers;
> > > > > > > > > > > > > +		}
> > > > > > > > > > > > > +
> > > > > > > > > > > > >  		wave5_set_enc_openparam(&open_param,
> > > inst);
> > > > > > > > > > > > >
> > > > > > > > > > > > >  		ret = wave5_vpu_enc_open(inst,
> > > &open_param);
> > > > > > diff --
> > > > > > > > > > git
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu
> > > > > > > > > > > > > .c
> > > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu .c index 0d90b5820bef..f81409740a56 100644
> > > > > > > > > > > > > ---
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -vpu
> > > > > > > > > > > > > .c
> > > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/w
> > > > > > > > > > > > > +++ ave5
> > > > > > > > > > > > > +++ -vpu
> > > > > > > > > > > > > +++ .c
> > > > > > > > > > > > > @@ -10,6 +10,7 @@
> > > > > > > > > > > > >  #include <linux/clk.h>
> > > > > > > > > > > > >  #include <linux/firmware.h>  #include
> > > > > > > > > > > > > <linux/interrupt.h>
> > > > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > > > >  #include "wave5-vpu.h"
> > > > > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > > > > >  #include "wave5-vpuconfig.h"
> > > > > > > > > > > > > @@ -117,6 +118,65 @@ static int
> > > > > > > > > > > > > wave5_vpu_load_firmware(struct device
> > > > > > > > > > > > *dev, const char *fw_name,
> > > > > > > > > > > > >  	return 0;
> > > > > > > > > > > > >  }
> > > > > > > > > > > > >
> > > > > > > > > > > > > +static __maybe_unused int
> > > > > > > > > > > > > +wave5_pm_suspend(struct device
> > > > > > > > > > > > > +*dev)
> > > > > > > > {
> > > > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	if (pm_runtime_suspended(dev))
> > > > > > > > > > > > > +		return 0;
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	wave5_vpu_sleep_wake(dev, true, NULL, 0);
> > > > > > > > > > > > > +	clk_bulk_disable_unprepare(vpu->num_clks,
> > > > > > > > > > > > > +vpu->clks);
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	return 0;
> > > > > > > > > > > > > +}
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +static __maybe_unused int
> > > > > > > > > > > > > +wave5_pm_resume(struct device
> > > > > > *dev) {
> > > > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > > > +	int ret = 0;
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	wave5_vpu_sleep_wake(dev, false, NULL, 0);
> > > > > > > > > > > > > +	ret = clk_bulk_prepare_enable(vpu->num_clks,
> > > vpu-
> > > > > > > clks);
> > > > > > > > > > > > > +	if (ret) {
> > > > > > > > > > > > > +		dev_err(dev, "Enabling clocks,
> > > fail: %d\n",
> > > > > > ret);
> > > > > > > > > > > > > +		return ret;
> > > > > > > > > > > > > +	}
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	return ret;
> > > > > > > > > > > > > +}
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +static __maybe_unused int wave5_suspend(struct
> > > > > > > > > > > > > +device
> > > > > > > > > > > > > +*dev)
> > > > > > {
> > > > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > > > > +		v4l2_m2m_suspend(inst->v4l2_m2m_dev);
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	return pm_runtime_force_suspend(dev); }
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +static __maybe_unused int wave5_resume(struct
> > > > > > > > > > > > > +device
> > > > > > > > > > > > > +*dev)
> > > > > > {
> > > > > > > > > > > > > +	struct vpu_device *vpu = dev_get_drvdata(dev);
> > > > > > > > > > > > > +	struct vpu_instance *inst;
> > > > > > > > > > > > > +	int ret = 0;
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	ret = pm_runtime_force_resume(dev);
> > > > > > > > > > > > > +	if (ret < 0)
> > > > > > > > > > > > > +		return ret;
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	list_for_each_entry(inst, &vpu->instances, list)
> > > > > > > > > > > > > +		v4l2_m2m_resume(inst->v4l2_m2m_dev);
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	return ret;
> > > > > > > > > > > > > +}
> > > > > > > > > > > >
> > > > > > > > > > > > The functions wave5_suspend() and wave5_resume()
> > > > > > > > > > > > are not just "maybe_unsued" but actually never
> > > > > > > > > > > > used. What was the
> > > > > > intention ?
> > > > > > > > > > > > Considering the usage of __maybe_unused has been
> > > > > > > > > > > > such a bad friend for this one, could you instead
> > > > > > > > > > > > bracket the functions with an
> > > > > > > > > > explicit ?
> > > > > > > > > > > >
> > > > > > > > > > > > #ifdef CONFIG_PM
> > > > > > > > > > > > #endif
> > > > > > > > > > > >
> > > > > > > > > > > > This way the compiler will have a word if you
> > > > > > > > > > > > implement something that you forgot to actually use.
> > > > > > > > > > > >
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +static const struct dev_pm_ops wave5_pm_ops = {
> > > > > > > > > > > > > +	SET_RUNTIME_PM_OPS(wave5_pm_suspend,
> > > > > > > > > > > > > +wave5_pm_resume,
> > > > > > > > > > NULL) };
> > > > > > > > > > > > > +
> > > > > > > > > > > > >  static int wave5_vpu_probe(struct
> > > > > > > > > > > > > platform_device
> > > > > > > > > > > > > *pdev)
> > > > > {
> > > > > > > > > > > > >  	int ret;
> > > > > > > > > > > > > @@ -232,6 +292,10 @@ static int
> > > > > > > > > > > > > wave5_vpu_probe(struct platform_device
> > > > > > > > > > > > *pdev)
> > > > > > > > > > > > >  		 (match_data->flags & WAVE5_IS_DEC) ?
> > > > > > "'DECODE'" : "");
> > > > > > > > > > > > >  	dev_info(&pdev->dev, "Product Code:      0x%x\n",
> > > dev-
> > > > > > > > > > > product_code);
> > > > > > > > > > > > >  	dev_info(&pdev->dev, "Firmware Revision:
> > > > > > > > > > > > > %u\n", fw_revision);
> > > > > > > > > > > > > +
> > > > > > > > > > > > > +	pm_runtime_enable(&pdev->dev);
> > > > > > > > > > > > > +	wave5_vpu_sleep_wake(&pdev->dev, true, NULL,
> > > > > > > > > > > > > +0);
> > > > > > > > > > > > > +
> > > > > > > > > > > > >  	return 0;
> > > > > > > > > > > > >
> > > > > > > > > > > > >  err_enc_unreg:
> > > > > > > > > > > > > @@ -254,6 +318,9 @@ static int
> > > > > > > > > > > > > wave5_vpu_remove(struct platform_device
> > > > > > > > > > > > > *pdev)  {
> > > > > > > > > > > > >  	struct vpu_device *dev =
> > > > > > > > > > > > > dev_get_drvdata(&pdev->dev);
> > > > > > > > > > > > >
> > > > > > > > > > > > > +	pm_runtime_put_sync(&pdev->dev);
> > > > > > > > > > > > > +	pm_runtime_disable(&pdev->dev);
> > > > > > > > > > > > > +
> > > > > > > > > > > > >  	mutex_destroy(&dev->dev_lock);
> > > > > > > > > > > > >  	mutex_destroy(&dev->hw_lock);
> > > > > > > > > > > > >  	clk_bulk_disable_unprepare(dev->num_clks,
> > > > > > > > > > > > > dev->clks);
> > > > > > @@
> > > > > > > > > > > > > -
> > > > > > > > > > 281,6
> > > > > > > > > > > > > +348,7 @@ static struct platform_driver
> > > > > > > > > > > > > +wave5_vpu_driver = {
> > > > > > > > > > > > >  	.driver = {
> > > > > > > > > > > > >  		.name = VPU_PLATFORM_DEVICE_NAME,
> > > > > > > > > > > > >  		.of_match_table =
> > > of_match_ptr(wave5_dt_ids),
> > > > > > > > > > > > > +		.pm = &wave5_pm_ops,
> > > > > > > > > > > > >  		},
> > > > > > > > > > > > >  	.probe = wave5_vpu_probe,
> > > > > > > > > > > > >  	.remove = wave5_vpu_remove, diff --git
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -
> > > vpuapi.
> > > > > > > > > > > > > c
> > > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -
> > > vpuapi.
> > > > > > > > > > > > > c index 1a3efb638dde..f1f8e4fc8474 100644
> > > > > > > > > > > > > ---
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > -
> > > vpuapi.
> > > > > > > > > > > > > c
> > > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/w
> > > > > > > > > > > > > +++ ave5
> > > > > > > > > > > > > +++ -vpu
> > > > > > > > > > > > > +++ ap
> > > > > > > > > > > > > +++ i.c
> > > > > > > > > > > > > @@ -6,6 +6,7 @@
> > > > > > > > > > > > >   */
> > > > > > > > > > > > >
> > > > > > > > > > > > >  #include <linux/bug.h>
> > > > > > > > > > > > > +#include <linux/pm_runtime.h>
> > > > > > > > > > > > >  #include "wave5-vpuapi.h"
> > > > > > > > > > > > >  #include "wave5-regdefine.h"
> > > > > > > > > > > > >  #include "wave5.h"
> > > > > > > > > > > > > @@ -232,6 +233,9 @@ int
> > > > > > > > > > > > > wave5_vpu_dec_close(struct vpu_instance *inst,
> > > > > > > > > > > > > u32 *fail_res)
> > > > > > > > > > > > >
> > > > > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > > > > &p_dec_info->vb_task);
> > > > > > > > > > > > >
> > > > > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > > > > > +
> > > > > > > > > > > > >  unlock_and_return:
> > > > > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > > > >
> > > > > > > > > > > > > @@ -734,6 +738,9 @@ int
> > > > > > > > > > > > > wave5_vpu_enc_close(struct vpu_instance *inst,
> > > > > > > > > > > > > u32 *fail_res)
> > > > > > > > > > > > >
> > > > > > > > > > > > >  	wave5_vdi_free_dma_memory(vpu_dev,
> > > > > > > > > > > > > &p_enc_info->vb_task);
> > > > > > > > > > > > >
> > > > > > > > > > > > > +	if (!pm_runtime_suspended(inst->dev->dev))
> > > > > > > > > > > > > +		pm_runtime_put_sync(inst->dev->dev);
> > > > > > > > > > > >
> > > > > > > > > > > > This seems very unnatural. We do the get() in
> > > > > > > > > > > > "start_streaming()", but the
> > > > > > > > > > > > put() is only done when the device is closed, or
> > > > > > > > > > > > when the driver is removed. As this is not
> > > > > > > > > > > > balanced, you seem to have to check the suspended
> > > > > > > > > > > > condition all over
> > > the place.
> > > > > > > > > > > >
> > > > > > > > > > > > I think we could aim for
> > > > > > > > > > > > start_streaming()/stop_streaming()
> > > > > > > > > > > > for your get/put placement. At least they will be
> > > > > > > > > > > > bound to an entirely balanced
> > > > > > > > > > API.
> > > > > > > > > > > > But then, a media player in paused sate will
> > > > > > > > > > > > prevent that device from being suspended.
> > > > > > > > > > > >
> > > > > > > > > > > > If the HW is capable of preserving enough state,
> > > > > > > > > > > > and From the short doc I have it gives me the
> > > > > > > > > > > > impression it can preserve that, I'd suggest to
> > > > > > > > > > > > follow what hantro driver is doing. What is does
> > > > > > > > > > > > is that it will do get() in device_run(), and
> > > > > > > > > > > > put() whenever the job completes. This driver has
> > > > > > > > > > > > been designed so when there is no job, it means
> > > > > > > > > > > > the firmware is currently idle and
> > > looking for more work.
> > > > > > > > > > > > So it seems like the perfect moment to
> > > > > > > > do suspend it.
> > > > > > > > > > > >
> > > > > > > > > > > > Nicolas
> > > > > > > > > > > >
> > > > > > > > > > > > > +
> > > > > > > > > > > > >  	mutex_unlock(&vpu_dev->hw_lock);
> > > > > > > > > > > > >
> > > > > > > > > > > > >  	return 0;
> > > > > > > > > > > > > diff --git
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > .h
> > > > > > > > > > > > > b/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > .h index 063028eccd3b..6125eff938a8 100644
> > > > > > > > > > > > > ---
> > > > > > > > > > > > > a/drivers/media/platform/chips-media/wave5/wave5
> > > > > > > > > > > > > .h
> > > > > > > > > > > > > +++ b/drivers/media/platform/chips-media/wave5/w
> > > > > > > > > > > > > +++ ave5
> > > > > > > > > > > > > +++ .h
> > > > > > > > > > > > > @@ -56,6 +56,9 @@ int
> > > > > > > > > > > > > wave5_vpu_get_version(struct vpu_device
> > > > > > > > > > > > > *vpu_dev, u32 *revision);
> > > > > > > > > > > > >
> > > > > > > > > > > > >  int wave5_vpu_init(struct device *dev, u8 *fw,
> > > > > > > > > > > > > size_t size);
> > > > > > > > > > > > >
> > > > > > > > > > > > > +int wave5_vpu_sleep_wake(struct device *dev,
> > > > > > > > > > > > > +bool i_sleep_wake, const
> > > > > > > > > > > > uint16_t *code,
> > > > > > > > > > > > > +			 size_t size);
> > > > > > > > > > > > > +
> > > > > > > > > > > > >  int wave5_vpu_reset(struct device *dev, enum
> > > > > > > > > > > > > sw_reset_mode reset_mode);
> > > > > > > > > > > > >
> > > > > > > > > > > > >  int wave5_vpu_build_up_dec_param(struct
> > > > > > > > > > > > > vpu_instance *inst, struct dec_open_param
> > > > > > > > > > > > > *param);
> > > > > > > > > > >
> > > > > > > > >
> > > > > > >
> > > >
> >

Re: [RESEND PATCH v0 3/5] wave5 : Support runtime suspend/resume.

[Nicolas Dufresne]

[RESEND PATCH v0 4/5] wave5: Use the bitstream buffer size from host.

[jackson.lee]

In V4L2 spec, Host can set the bitstream buffer size.
Allow the larger size between default size and input size.

Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
---
 drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
index ff73d69de41c..19018ace41b6 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
@@ -11,6 +11,10 @@
 #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
 #define VPU_ENC_DRV_NAME "wave5-enc"
 
+#define DEFAULT_SRC_SIZE(width, height) ({			\
+	(width) * (height) / 8 * 3;					\
+})
+
 static const struct vpu_format enc_fmt_list[FMT_TYPES][MAX_FMTS] = {
 	[VPU_FMT_TYPE_CODEC] = {
 		{
@@ -193,7 +197,8 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
 		pix_mp->width = width;
 		pix_mp->height = height;
 		pix_mp->plane_fmt[0].bytesperline = 0;
-		pix_mp->plane_fmt[0].sizeimage = width * height / 8 * 3;
+		pix_mp->plane_fmt[0].sizeimage = max(DEFAULT_SRC_SIZE(width, height),
+						     pix_mp->plane_fmt[0].sizeimage);
 		break;
 	}
 }
-- 
2.43.0

Re: [RESEND PATCH v0 4/5] wave5: Use the bitstream buffer size from host.

[Andrew Davis]

On 1/30/24 7:30 PM, jackson.lee wrote:
> In V4L2 spec, Host can set the bitstream buffer size.
> Allow the larger size between default size and input size.
> 
> Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> ---
>   drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c | 7 ++++++-
>   1 file changed, 6 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> index ff73d69de41c..19018ace41b6 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> @@ -11,6 +11,10 @@
>   #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
>   #define VPU_ENC_DRV_NAME "wave5-enc"
>   
> +#define DEFAULT_SRC_SIZE(width, height) ({			\
> +	(width) * (height) / 8 * 3;					\

These "\" are tabbed out way to far, and not to the same amount..
Why have this in {;} brackets?
Why multiple lines?
Just:

#define DEFAULT_SRC_SIZE(width, height) ((width) * (height) / 8 * 3)

Andrew

> +})
> +
>   static const struct vpu_format enc_fmt_list[FMT_TYPES][MAX_FMTS] = {
>   	[VPU_FMT_TYPE_CODEC] = {
>   		{
> @@ -193,7 +197,8 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
>   		pix_mp->width = width;
>   		pix_mp->height = height;
>   		pix_mp->plane_fmt[0].bytesperline = 0;
> -		pix_mp->plane_fmt[0].sizeimage = width * height / 8 * 3;
> +		pix_mp->plane_fmt[0].sizeimage = max(DEFAULT_SRC_SIZE(width, height),
> +						     pix_mp->plane_fmt[0].sizeimage);
>   		break;
>   	}
>   }

RE: [RESEND PATCH v0 4/5] wave5: Use the bitstream buffer size from host.

[jackson.lee]

Hello Andrew

I will update it to one line you suggested.

Thanks.

> -----Original Message-----
> From: Andrew Davis <afd@ti.com>
> Sent: Thursday, February 1, 2024 2:37 AM
> To: jackson.lee <jackson.lee@chipsnmedia.com>; mchehab@kernel.org; linux-
> media@vger.kernel.org; linux-kernel@vger.kernel.org; Nas Chung
> <nas.chung@chipsnmedia.com>
> Cc: lafley.kim <lafley.kim@chipsnmedia.com>; b-brnich@ti.com
> Subject: Re: [RESEND PATCH v0 4/5] wave5: Use the bitstream buffer size
> from host.
> 
> On 1/30/24 7:30 PM, jackson.lee wrote:
> > In V4L2 spec, Host can set the bitstream buffer size.
> > Allow the larger size between default size and input size.
> >
> > Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> > Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> > ---
> >   drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c | 7 ++++++-
> >   1 file changed, 6 insertions(+), 1 deletion(-)
> >
> > diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > index ff73d69de41c..19018ace41b6 100644
> > --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> > @@ -11,6 +11,10 @@
> >   #define VPU_ENC_DEV_NAME "C&M Wave5 VPU encoder"
> >   #define VPU_ENC_DRV_NAME "wave5-enc"
> >
> > +#define DEFAULT_SRC_SIZE(width, height) ({			\
> > +	(width) * (height) / 8 * 3;					\
> 
> These "\" are tabbed out way to far, and not to the same amount..
> Why have this in {;} brackets?
> Why multiple lines?
> Just:
> 
> #define DEFAULT_SRC_SIZE(width, height) ((width) * (height) / 8 * 3)
> 
> Andrew
> 
> > +})
> > +
> >   static const struct vpu_format enc_fmt_list[FMT_TYPES][MAX_FMTS] = {
> >   	[VPU_FMT_TYPE_CODEC] = {
> >   		{
> > @@ -193,7 +197,8 @@ static void wave5_update_pix_fmt(struct
> v4l2_pix_format_mplane *pix_mp, unsigned
> >   		pix_mp->width = width;
> >   		pix_mp->height = height;
> >   		pix_mp->plane_fmt[0].bytesperline = 0;
> > -		pix_mp->plane_fmt[0].sizeimage = width * height / 8 * 3;
> > +		pix_mp->plane_fmt[0].sizeimage = max(DEFAULT_SRC_SIZE(width,
> height),
> > +						     pix_mp-
> >plane_fmt[0].sizeimage);
> >   		break;
> >   	}
> >   }

[RESEND PATCH v0 5/5] wave5 : Fixed the wrong buffer size formula.

[jackson.lee]

S_FMT/G_FMT should report the buffer size based on aligned width and height.
And, Host can set the real encoding size through s_selection and g_selection.
So, Driver should use the conf_win information for encoding size instead of size of S_FMT/G_FMT.

Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
---
 .../chips-media/wave5/wave5-vpu-dec.c         | 77 +++++++------------
 .../chips-media/wave5/wave5-vpu-enc.c         | 77 +++++++++++--------
 2 files changed, 72 insertions(+), 82 deletions(-)

diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
index 328a7a8f26c5..fb9449908ebd 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
@@ -243,54 +243,54 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
 	case V4L2_PIX_FMT_NV21:
 		pix_mp->width = round_up(width, 32);
 		pix_mp->height = round_up(height, 16);
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = width * height * 3 / 2;
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height * 3 / 2;
 		break;
 	case V4L2_PIX_FMT_YUV422P:
 	case V4L2_PIX_FMT_NV16:
 	case V4L2_PIX_FMT_NV61:
 		pix_mp->width = round_up(width, 32);
 		pix_mp->height = round_up(height, 16);
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = width * height * 2;
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height * 2;
 		break;
 	case V4L2_PIX_FMT_YUV420M:
 		pix_mp->width = round_up(width, 32);
 		pix_mp->height = round_up(height, 16);
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = width * height;
-		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32) / 2;
-		pix_mp->plane_fmt[1].sizeimage = width * height / 4;
-		pix_mp->plane_fmt[2].bytesperline = round_up(width, 32) / 2;
-		pix_mp->plane_fmt[2].sizeimage = width * height / 4;
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
+		pix_mp->plane_fmt[1].bytesperline = pix_mp->width / 2;
+		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 4;
+		pix_mp->plane_fmt[2].bytesperline = pix_mp->width / 2;
+		pix_mp->plane_fmt[2].sizeimage = pix_mp->width * pix_mp->height / 4;
 		break;
 	case V4L2_PIX_FMT_NV12M:
 	case V4L2_PIX_FMT_NV21M:
 		pix_mp->width = round_up(width, 32);
 		pix_mp->height = round_up(height, 16);
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = width * height;
-		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[1].sizeimage = width * height / 2;
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
+		pix_mp->plane_fmt[1].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 2;
 		break;
 	case V4L2_PIX_FMT_YUV422M:
 		pix_mp->width = round_up(width, 32);
 		pix_mp->height = round_up(height, 16);
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = width * height;
-		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32) / 2;
-		pix_mp->plane_fmt[1].sizeimage = width * height / 2;
-		pix_mp->plane_fmt[2].bytesperline = round_up(width, 32) / 2;
-		pix_mp->plane_fmt[2].sizeimage = width * height / 2;
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
+		pix_mp->plane_fmt[1].bytesperline = pix_mp->width / 2;
+		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 2;
+		pix_mp->plane_fmt[2].bytesperline = pix_mp->width / 2;
+		pix_mp->plane_fmt[2].sizeimage = pix_mp->width * pix_mp->height / 2;
 		break;
 	case V4L2_PIX_FMT_NV16M:
 	case V4L2_PIX_FMT_NV61M:
 		pix_mp->width = round_up(width, 32);
 		pix_mp->height = round_up(height, 16);
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = width * height;
-		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[1].sizeimage = width * height;
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
+		pix_mp->plane_fmt[1].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height;
 		break;
 	default:
 		pix_mp->width = width;
@@ -1003,6 +1003,7 @@ static int wave5_vpu_dec_queue_setup(struct vb2_queue *q, unsigned int *num_buff
 	struct vpu_instance *inst = vb2_get_drv_priv(q);
 	struct v4l2_pix_format_mplane inst_format =
 		(q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) ? inst->src_fmt : inst->dst_fmt;
+	unsigned int i;
 
 	dev_dbg(inst->dev->dev, "%s: num_buffers: %u | num_planes: %u | type: %u\n", __func__,
 		*num_buffers, *num_planes, q->type);
@@ -1016,31 +1017,9 @@ static int wave5_vpu_dec_queue_setup(struct vb2_queue *q, unsigned int *num_buff
 		if (*num_buffers < inst->fbc_buf_count)
 			*num_buffers = inst->fbc_buf_count;
 
-		if (*num_planes == 1) {
-			if (inst->output_format == FORMAT_422)
-				sizes[0] = inst_format.width * inst_format.height * 2;
-			else
-				sizes[0] = inst_format.width * inst_format.height * 3 / 2;
-			dev_dbg(inst->dev->dev, "%s: size[0]: %u\n", __func__, sizes[0]);
-		} else if (*num_planes == 2) {
-			sizes[0] = inst_format.width * inst_format.height;
-			if (inst->output_format == FORMAT_422)
-				sizes[1] = inst_format.width * inst_format.height;
-			else
-				sizes[1] = inst_format.width * inst_format.height / 2;
-			dev_dbg(inst->dev->dev, "%s: size[0]: %u | size[1]: %u\n",
-				__func__, sizes[0], sizes[1]);
-		} else if (*num_planes == 3) {
-			sizes[0] = inst_format.width * inst_format.height;
-			if (inst->output_format == FORMAT_422) {
-				sizes[1] = inst_format.width * inst_format.height / 2;
-				sizes[2] = inst_format.width * inst_format.height / 2;
-			} else {
-				sizes[1] = inst_format.width * inst_format.height / 4;
-				sizes[2] = inst_format.width * inst_format.height / 4;
-			}
-			dev_dbg(inst->dev->dev, "%s: size[0]: %u | size[1]: %u | size[2]: %u\n",
-				__func__, sizes[0], sizes[1], sizes[2]);
+		for (i = 0; i < *num_planes; i++) {
+			sizes[i] = inst_format.plane_fmt[i].sizeimage;
+			dev_dbg(inst->dev->dev, "%s: size[%u]: %u\n", __func__, i, sizes[i]);
 		}
 	}
 
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
index 19018ace41b6..762973d0677b 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
@@ -152,46 +152,46 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
 	case V4L2_PIX_FMT_YUV420:
 	case V4L2_PIX_FMT_NV12:
 	case V4L2_PIX_FMT_NV21:
-		pix_mp->width = width;
-		pix_mp->height = height;
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height * 3 / 2;
+		pix_mp->width = round_up(width, 32);
+		pix_mp->height = round_up(height, 16);
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height * 3 / 2;
 		break;
 	case V4L2_PIX_FMT_YUV420M:
-		pix_mp->width = width;
-		pix_mp->height = height;
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
-		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32) / 2;
-		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height / 4;
-		pix_mp->plane_fmt[2].bytesperline = round_up(width, 32) / 2;
-		pix_mp->plane_fmt[2].sizeimage = round_up(width, 32) * height / 4;
+		pix_mp->width = round_up(width, 32);
+		pix_mp->height = round_up(height, 16);
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
+		pix_mp->plane_fmt[1].bytesperline = pix_mp->width / 2;
+		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 4;
+		pix_mp->plane_fmt[2].bytesperline = pix_mp->width / 2;
+		pix_mp->plane_fmt[2].sizeimage = pix_mp->width * pix_mp->height / 4;
 		break;
 	case V4L2_PIX_FMT_NV12M:
 	case V4L2_PIX_FMT_NV21M:
-		pix_mp->width = width;
-		pix_mp->height = height;
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
-		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height / 2;
+		pix_mp->width = round_up(width, 32);
+		pix_mp->height = round_up(height, 16);
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
+		pix_mp->plane_fmt[1].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 2;
 		break;
 	case V4L2_PIX_FMT_YUV422P:
 	case V4L2_PIX_FMT_NV16:
 	case V4L2_PIX_FMT_NV61:
-		pix_mp->width = width;
-		pix_mp->height = height;
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height * 2;
+		pix_mp->width = round_up(width, 32);
+		pix_mp->height = round_up(height, 16);
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height * 2;
 		break;
 	case V4L2_PIX_FMT_NV16M:
 	case V4L2_PIX_FMT_NV61M:
-		pix_mp->width = width;
-		pix_mp->height = height;
-		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
-		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
-		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height;
+		pix_mp->width = round_up(width, 32);
+		pix_mp->height = round_up(height, 16);
+		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
+		pix_mp->plane_fmt[1].bytesperline = pix_mp->width;
+		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height;
 		break;
 	default:
 		pix_mp->width = width;
@@ -638,6 +638,8 @@ static int wave5_vpu_enc_s_fmt_out(struct file *file, void *fh, struct v4l2_form
 	inst->xfer_func = f->fmt.pix_mp.xfer_func;
 
 	wave5_update_pix_fmt(&inst->dst_fmt, f->fmt.pix_mp.width, f->fmt.pix_mp.height);
+	inst->conf_win.width = inst->dst_fmt.width;
+	inst->conf_win.height = inst->dst_fmt.height;
 
 	return 0;
 }
@@ -653,12 +655,17 @@ static int wave5_vpu_enc_g_selection(struct file *file, void *fh, struct v4l2_se
 	switch (s->target) {
 	case V4L2_SEL_TGT_CROP_DEFAULT:
 	case V4L2_SEL_TGT_CROP_BOUNDS:
-	case V4L2_SEL_TGT_CROP:
 		s->r.left = 0;
 		s->r.top = 0;
 		s->r.width = inst->dst_fmt.width;
 		s->r.height = inst->dst_fmt.height;
 		break;
+	case V4L2_SEL_TGT_CROP:
+		s->r.left = 0;
+		s->r.top = 0;
+		s->r.width = inst->conf_win.width;
+		s->r.height = inst->conf_win.height;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -681,8 +688,10 @@ static int wave5_vpu_enc_s_selection(struct file *file, void *fh, struct v4l2_se
 
 	s->r.left = 0;
 	s->r.top = 0;
-	s->r.width = inst->src_fmt.width;
-	s->r.height = inst->src_fmt.height;
+	s->r.width = min(s->r.width, inst->dst_fmt.width);
+	s->r.height = min(s->r.height, inst->dst_fmt.height);
+
+	inst->conf_win = s->r;
 
 	return 0;
 }
@@ -1229,8 +1238,8 @@ static void wave5_set_enc_openparam(struct enc_open_param *open_param,
 	open_param->wave_param.lambda_scaling_enable = 1;
 
 	open_param->line_buf_int_en = true;
-	open_param->pic_width = inst->dst_fmt.width;
-	open_param->pic_height = inst->dst_fmt.height;
+	open_param->pic_width = inst->conf_win.width;
+	open_param->pic_height = inst->conf_win.height;
 	open_param->frame_rate_info = inst->frame_rate;
 	open_param->rc_enable = inst->rc_enable;
 	if (inst->rc_enable) {
@@ -1806,6 +1815,8 @@ static int wave5_vpu_open_enc(struct file *filp)
 	v4l2_ctrl_handler_setup(v4l2_ctrl_hdl);
 
 	wave5_set_default_format(&inst->src_fmt, &inst->dst_fmt);
+	inst->conf_win.width = inst->dst_fmt.width;
+	inst->conf_win.height = inst->dst_fmt.height;
 	inst->colorspace = V4L2_COLORSPACE_REC709;
 	inst->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
 	inst->quantization = V4L2_QUANTIZATION_DEFAULT;
-- 
2.43.0

Re: [RESEND PATCH v0 5/5] wave5 : Fixed the wrong buffer size formula.

[Nicolas Dufresne]

Hi,

Le mercredi 31 janvier 2024 à 10:30 +0900, jackson.lee a écrit :
> S_FMT/G_FMT should report the buffer size based on aligned width and height.
> And, Host can set the real encoding size through s_selection and g_selection.
> So, Driver should use the conf_win information for encoding size instead of size of S_FMT/G_FMT.

This patch will go away as soon as you have ported to v4l2-common as requested
in patch 1/5. It will also make future addition of pixel formats less tedious.

regards,
Nicolas

> 
> Signed-off-by: Nas Chung <nas.chung@chipsnmedia.com>
> Signed-off-by: Jackson Lee <jackson.lee@chipsnmedia.com>
> ---
>  .../chips-media/wave5/wave5-vpu-dec.c         | 77 +++++++------------
>  .../chips-media/wave5/wave5-vpu-enc.c         | 77 +++++++++++--------
>  2 files changed, 72 insertions(+), 82 deletions(-)
> 
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> index 328a7a8f26c5..fb9449908ebd 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
> @@ -243,54 +243,54 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
>  	case V4L2_PIX_FMT_NV21:
>  		pix_mp->width = round_up(width, 32);
>  		pix_mp->height = round_up(height, 16);
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = width * height * 3 / 2;
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height * 3 / 2;
>  		break;
>  	case V4L2_PIX_FMT_YUV422P:
>  	case V4L2_PIX_FMT_NV16:
>  	case V4L2_PIX_FMT_NV61:
>  		pix_mp->width = round_up(width, 32);
>  		pix_mp->height = round_up(height, 16);
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = width * height * 2;
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height * 2;
>  		break;
>  	case V4L2_PIX_FMT_YUV420M:
>  		pix_mp->width = round_up(width, 32);
>  		pix_mp->height = round_up(height, 16);
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = width * height;
> -		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32) / 2;
> -		pix_mp->plane_fmt[1].sizeimage = width * height / 4;
> -		pix_mp->plane_fmt[2].bytesperline = round_up(width, 32) / 2;
> -		pix_mp->plane_fmt[2].sizeimage = width * height / 4;
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
> +		pix_mp->plane_fmt[1].bytesperline = pix_mp->width / 2;
> +		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 4;
> +		pix_mp->plane_fmt[2].bytesperline = pix_mp->width / 2;
> +		pix_mp->plane_fmt[2].sizeimage = pix_mp->width * pix_mp->height / 4;
>  		break;
>  	case V4L2_PIX_FMT_NV12M:
>  	case V4L2_PIX_FMT_NV21M:
>  		pix_mp->width = round_up(width, 32);
>  		pix_mp->height = round_up(height, 16);
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = width * height;
> -		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[1].sizeimage = width * height / 2;
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
> +		pix_mp->plane_fmt[1].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 2;
>  		break;
>  	case V4L2_PIX_FMT_YUV422M:
>  		pix_mp->width = round_up(width, 32);
>  		pix_mp->height = round_up(height, 16);
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = width * height;
> -		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32) / 2;
> -		pix_mp->plane_fmt[1].sizeimage = width * height / 2;
> -		pix_mp->plane_fmt[2].bytesperline = round_up(width, 32) / 2;
> -		pix_mp->plane_fmt[2].sizeimage = width * height / 2;
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
> +		pix_mp->plane_fmt[1].bytesperline = pix_mp->width / 2;
> +		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 2;
> +		pix_mp->plane_fmt[2].bytesperline = pix_mp->width / 2;
> +		pix_mp->plane_fmt[2].sizeimage = pix_mp->width * pix_mp->height / 2;
>  		break;
>  	case V4L2_PIX_FMT_NV16M:
>  	case V4L2_PIX_FMT_NV61M:
>  		pix_mp->width = round_up(width, 32);
>  		pix_mp->height = round_up(height, 16);
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = width * height;
> -		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[1].sizeimage = width * height;
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
> +		pix_mp->plane_fmt[1].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height;
>  		break;
>  	default:
>  		pix_mp->width = width;
> @@ -1003,6 +1003,7 @@ static int wave5_vpu_dec_queue_setup(struct vb2_queue *q, unsigned int *num_buff
>  	struct vpu_instance *inst = vb2_get_drv_priv(q);
>  	struct v4l2_pix_format_mplane inst_format =
>  		(q->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) ? inst->src_fmt : inst->dst_fmt;
> +	unsigned int i;
>  
>  	dev_dbg(inst->dev->dev, "%s: num_buffers: %u | num_planes: %u | type: %u\n", __func__,
>  		*num_buffers, *num_planes, q->type);
> @@ -1016,31 +1017,9 @@ static int wave5_vpu_dec_queue_setup(struct vb2_queue *q, unsigned int *num_buff
>  		if (*num_buffers < inst->fbc_buf_count)
>  			*num_buffers = inst->fbc_buf_count;
>  
> -		if (*num_planes == 1) {
> -			if (inst->output_format == FORMAT_422)
> -				sizes[0] = inst_format.width * inst_format.height * 2;
> -			else
> -				sizes[0] = inst_format.width * inst_format.height * 3 / 2;
> -			dev_dbg(inst->dev->dev, "%s: size[0]: %u\n", __func__, sizes[0]);
> -		} else if (*num_planes == 2) {
> -			sizes[0] = inst_format.width * inst_format.height;
> -			if (inst->output_format == FORMAT_422)
> -				sizes[1] = inst_format.width * inst_format.height;
> -			else
> -				sizes[1] = inst_format.width * inst_format.height / 2;
> -			dev_dbg(inst->dev->dev, "%s: size[0]: %u | size[1]: %u\n",
> -				__func__, sizes[0], sizes[1]);
> -		} else if (*num_planes == 3) {
> -			sizes[0] = inst_format.width * inst_format.height;
> -			if (inst->output_format == FORMAT_422) {
> -				sizes[1] = inst_format.width * inst_format.height / 2;
> -				sizes[2] = inst_format.width * inst_format.height / 2;
> -			} else {
> -				sizes[1] = inst_format.width * inst_format.height / 4;
> -				sizes[2] = inst_format.width * inst_format.height / 4;
> -			}
> -			dev_dbg(inst->dev->dev, "%s: size[0]: %u | size[1]: %u | size[2]: %u\n",
> -				__func__, sizes[0], sizes[1], sizes[2]);
> +		for (i = 0; i < *num_planes; i++) {
> +			sizes[i] = inst_format.plane_fmt[i].sizeimage;
> +			dev_dbg(inst->dev->dev, "%s: size[%u]: %u\n", __func__, i, sizes[i]);
>  		}
>  	}
>  
> diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> index 19018ace41b6..762973d0677b 100644
> --- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> +++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
> @@ -152,46 +152,46 @@ static void wave5_update_pix_fmt(struct v4l2_pix_format_mplane *pix_mp, unsigned
>  	case V4L2_PIX_FMT_YUV420:
>  	case V4L2_PIX_FMT_NV12:
>  	case V4L2_PIX_FMT_NV21:
> -		pix_mp->width = width;
> -		pix_mp->height = height;
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height * 3 / 2;
> +		pix_mp->width = round_up(width, 32);
> +		pix_mp->height = round_up(height, 16);
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height * 3 / 2;
>  		break;
>  	case V4L2_PIX_FMT_YUV420M:
> -		pix_mp->width = width;
> -		pix_mp->height = height;
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
> -		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32) / 2;
> -		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height / 4;
> -		pix_mp->plane_fmt[2].bytesperline = round_up(width, 32) / 2;
> -		pix_mp->plane_fmt[2].sizeimage = round_up(width, 32) * height / 4;
> +		pix_mp->width = round_up(width, 32);
> +		pix_mp->height = round_up(height, 16);
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
> +		pix_mp->plane_fmt[1].bytesperline = pix_mp->width / 2;
> +		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 4;
> +		pix_mp->plane_fmt[2].bytesperline = pix_mp->width / 2;
> +		pix_mp->plane_fmt[2].sizeimage = pix_mp->width * pix_mp->height / 4;
>  		break;
>  	case V4L2_PIX_FMT_NV12M:
>  	case V4L2_PIX_FMT_NV21M:
> -		pix_mp->width = width;
> -		pix_mp->height = height;
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
> -		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height / 2;
> +		pix_mp->width = round_up(width, 32);
> +		pix_mp->height = round_up(height, 16);
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
> +		pix_mp->plane_fmt[1].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height / 2;
>  		break;
>  	case V4L2_PIX_FMT_YUV422P:
>  	case V4L2_PIX_FMT_NV16:
>  	case V4L2_PIX_FMT_NV61:
> -		pix_mp->width = width;
> -		pix_mp->height = height;
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height * 2;
> +		pix_mp->width = round_up(width, 32);
> +		pix_mp->height = round_up(height, 16);
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height * 2;
>  		break;
>  	case V4L2_PIX_FMT_NV16M:
>  	case V4L2_PIX_FMT_NV61M:
> -		pix_mp->width = width;
> -		pix_mp->height = height;
> -		pix_mp->plane_fmt[0].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[0].sizeimage = round_up(width, 32) * height;
> -		pix_mp->plane_fmt[1].bytesperline = round_up(width, 32);
> -		pix_mp->plane_fmt[1].sizeimage = round_up(width, 32) * height;
> +		pix_mp->width = round_up(width, 32);
> +		pix_mp->height = round_up(height, 16);
> +		pix_mp->plane_fmt[0].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[0].sizeimage = pix_mp->width * pix_mp->height;
> +		pix_mp->plane_fmt[1].bytesperline = pix_mp->width;
> +		pix_mp->plane_fmt[1].sizeimage = pix_mp->width * pix_mp->height;
>  		break;
>  	default:
>  		pix_mp->width = width;
> @@ -638,6 +638,8 @@ static int wave5_vpu_enc_s_fmt_out(struct file *file, void *fh, struct v4l2_form
>  	inst->xfer_func = f->fmt.pix_mp.xfer_func;
>  
>  	wave5_update_pix_fmt(&inst->dst_fmt, f->fmt.pix_mp.width, f->fmt.pix_mp.height);
> +	inst->conf_win.width = inst->dst_fmt.width;
> +	inst->conf_win.height = inst->dst_fmt.height;
>  
>  	return 0;
>  }
> @@ -653,12 +655,17 @@ static int wave5_vpu_enc_g_selection(struct file *file, void *fh, struct v4l2_se
>  	switch (s->target) {
>  	case V4L2_SEL_TGT_CROP_DEFAULT:
>  	case V4L2_SEL_TGT_CROP_BOUNDS:
> -	case V4L2_SEL_TGT_CROP:
>  		s->r.left = 0;
>  		s->r.top = 0;
>  		s->r.width = inst->dst_fmt.width;
>  		s->r.height = inst->dst_fmt.height;
>  		break;
> +	case V4L2_SEL_TGT_CROP:
> +		s->r.left = 0;
> +		s->r.top = 0;
> +		s->r.width = inst->conf_win.width;
> +		s->r.height = inst->conf_win.height;
> +		break;
>  	default:
>  		return -EINVAL;
>  	}
> @@ -681,8 +688,10 @@ static int wave5_vpu_enc_s_selection(struct file *file, void *fh, struct v4l2_se
>  
>  	s->r.left = 0;
>  	s->r.top = 0;
> -	s->r.width = inst->src_fmt.width;
> -	s->r.height = inst->src_fmt.height;
> +	s->r.width = min(s->r.width, inst->dst_fmt.width);
> +	s->r.height = min(s->r.height, inst->dst_fmt.height);
> +
> +	inst->conf_win = s->r;
>  
>  	return 0;
>  }
> @@ -1229,8 +1238,8 @@ static void wave5_set_enc_openparam(struct enc_open_param *open_param,
>  	open_param->wave_param.lambda_scaling_enable = 1;
>  
>  	open_param->line_buf_int_en = true;
> -	open_param->pic_width = inst->dst_fmt.width;
> -	open_param->pic_height = inst->dst_fmt.height;
> +	open_param->pic_width = inst->conf_win.width;
> +	open_param->pic_height = inst->conf_win.height;
>  	open_param->frame_rate_info = inst->frame_rate;
>  	open_param->rc_enable = inst->rc_enable;
>  	if (inst->rc_enable) {
> @@ -1806,6 +1815,8 @@ static int wave5_vpu_open_enc(struct file *filp)
>  	v4l2_ctrl_handler_setup(v4l2_ctrl_hdl);
>  
>  	wave5_set_default_format(&inst->src_fmt, &inst->dst_fmt);
> +	inst->conf_win.width = inst->dst_fmt.width;
> +	inst->conf_win.height = inst->dst_fmt.height;
>  	inst->colorspace = V4L2_COLORSPACE_REC709;
>  	inst->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
>  	inst->quantization = V4L2_QUANTIZATION_DEFAULT;

Re: [RESEND PATCH v0 0/5] wave5 codec driver

[Sebastian Fricke]

Hey Jackson,

On 31.01.2024 10:30, jackson.lee wrote:
>The wave5 codec driver is a stateful encoder/decoder.

You don't have to re introduce the driver here.

>The following patches is for supporting yuv422 inpuy format, supporting
>runtime suspend/resume feature and extra things.

Could you provide some test scores in your next patch version?
Interesting would be your score on V4L2-compliance and also the fluster
score for the decoder.

Greetings,
Sebastian
>
>jackson.lee (5):
>  wave5 : Support yuv422 input format for encoder.
>  wave5: Support to prepend sps/pps to IDR frame.
>  wave5 : Support runtime suspend/resume.
>  wave5: Use the bitstream buffer size from host.
>  wave5 : Fixed the wrong buffer size formula.
>
> .../platform/chips-media/wave5/wave5-hw.c     |  11 +-
> .../chips-media/wave5/wave5-vpu-dec.c         |  86 ++++------
> .../chips-media/wave5/wave5-vpu-enc.c         | 159 +++++++++++++++---
> .../platform/chips-media/wave5/wave5-vpu.c    |  68 ++++++++
> .../platform/chips-media/wave5/wave5-vpuapi.c |   7 +
> .../platform/chips-media/wave5/wave5-vpuapi.h |   1 +
> .../media/platform/chips-media/wave5/wave5.h  |   3 +
> 7 files changed, 255 insertions(+), 80 deletions(-)
>
>-- 
>2.43.0
>
>