[PATCH v2 00/16] media: vicodec: adding support for stateless encoder

[PATCH v2 00/16] media: vicodec: adding support for stateless encoder

[Dafna Hirschfeld]

patches 10-16 are new, patch 16 is experimental.
It is not yet clear how vicodec will update the
request controls so that the userspace can read it.

Dafna Hirschfeld (7):
  media: vicodec: don't test if info is NULL
  media: vicodec: change fwht_encode_frame to get the header separately
  media: vicodec: bugfix: release stateless dec m2m_dev in
    vicodec_remove
  media: vicodec: add field 'dev_inst' to vicodec_ctx
  media: vicodec: Register another node for stateless encoder
  media: vicodec: set pixelformat V4L2_PIX_FMT_FWHT_STATELESS for
    stateless encoder
  media: vicodec: Add support for stateless encoder

Hans Verkuil (9):
  videobuf2-v4l2: set last_buffer_dequeued in dqbuf
  vicodec: move v4l2_ctrl_request_complete after spin_unlock
  vicodec: always return a valid format.
  vicodec: pass on enc output format to capture side
  vicodec: add V4L2_CID_MIN_BUFFERS_FOR_OUTPUT
  vicodec: improve handling of ENC_CMD_STOP/START
  vicodec: restrict decoder format list when src fmt is known
  vicodec: set KEY/PFRAME flag when decoding
  v4l2-ctrls: add v4l2_ctrl_s_ctrl_ptr()

 .../media/common/videobuf2/videobuf2-v4l2.c   |  10 +-
 .../media/platform/vicodec/codec-v4l2-fwht.c  |   9 +-
 .../media/platform/vicodec/codec-v4l2-fwht.h  |   3 +-
 drivers/media/platform/vicodec/vicodec-core.c | 463 +++++++++++++-----
 drivers/media/v4l2-core/v4l2-ctrls.c          |  11 +
 include/media/v4l2-ctrls.h                    |  42 ++
 6 files changed, 392 insertions(+), 146 deletions(-)

-- 
2.17.1

[PATCH v2 01/16] videobuf2-v4l2: set last_buffer_dequeued in dqbuf

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil-cisco@xs4all.nl>

last_buffer_dequeued was set to true in __fill_v4l2_buffer, but this
is called for qbuf as well. Move it to vb2_dqbuf.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/common/videobuf2/videobuf2-v4l2.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c
index b11a779e97b0..379738372651 100644
--- a/drivers/media/common/videobuf2/videobuf2-v4l2.c
+++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c
@@ -562,11 +562,6 @@ static void __fill_v4l2_buffer(struct vb2_buffer *vb, void *pb)
 		b->flags |= V4L2_BUF_FLAG_REQUEST_FD;
 		b->request_fd = vbuf->request_fd;
 	}
-
-	if (!q->is_output &&
-		b->flags & V4L2_BUF_FLAG_DONE &&
-		b->flags & V4L2_BUF_FLAG_LAST)
-		q->last_buffer_dequeued = true;
 }
 
 /*
@@ -785,6 +780,11 @@ int vb2_dqbuf(struct vb2_queue *q, struct v4l2_buffer *b, bool nonblocking)
 
 	ret = vb2_core_dqbuf(q, NULL, b, nonblocking);
 
+	if (!q->is_output &&
+	    b->flags & V4L2_BUF_FLAG_DONE &&
+	    b->flags & V4L2_BUF_FLAG_LAST)
+		q->last_buffer_dequeued = true;
+
 	/*
 	 *  After calling the VIDIOC_DQBUF V4L2_BUF_FLAG_DONE must be
 	 *  cleared.
-- 
2.17.1

[PATCH v2 02/16] vicodec: move v4l2_ctrl_request_complete after spin_unlock

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil-cisco@xs4all.nl>

v4l2_ctrl_request_complete can sleep, so can't be called while
a spinlock is held.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/platform/vicodec/vicodec-core.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index bd01a9206aa6..92647f02fc49 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -442,14 +442,14 @@ static void device_run(void *priv)
 		ctx->comp_has_next_frame = false;
 	}
 	v4l2_m2m_buf_done(dst_buf, state);
-	if (ctx->is_stateless && src_req)
-		v4l2_ctrl_request_complete(src_req, &ctx->hdl);
 
 	ctx->comp_size = 0;
 	ctx->header_size = 0;
 	ctx->comp_magic_cnt = 0;
 	ctx->comp_has_frame = false;
 	spin_unlock(ctx->lock);
+	if (ctx->is_stateless && src_req)
+		v4l2_ctrl_request_complete(src_req, &ctx->hdl);
 
 	if (ctx->is_enc)
 		v4l2_m2m_job_finish(dev->stateful_enc.m2m_dev, ctx->fh.m2m_ctx);
-- 
2.17.1

[PATCH v2 03/16] vicodec: always return a valid format.

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil-cisco@xs4all.nl>

Rather than returning width/height values of 0, just default to
a format. Formats in V4L2 are always supposed to be valid, there
is no concept of an invalid format.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/platform/vicodec/vicodec-core.c | 26 ++++++++++---------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 92647f02fc49..aa3af796929b 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -1771,11 +1771,13 @@ static const struct v4l2_ctrl_config vicodec_ctrl_stateless_state = {
  */
 static int vicodec_open(struct file *file)
 {
+	const struct v4l2_fwht_pixfmt_info *info = v4l2_fwht_get_pixfmt(0);
 	struct video_device *vfd = video_devdata(file);
 	struct vicodec_dev *dev = video_drvdata(file);
 	struct vicodec_ctx *ctx = NULL;
 	struct v4l2_ctrl_handler *hdl;
-	unsigned int size;
+	unsigned int raw_size;
+	unsigned int comp_size;
 	int rc = 0;
 
 	if (mutex_lock_interruptible(vfd->lock))
@@ -1814,7 +1816,7 @@ static int vicodec_open(struct file *file)
 	v4l2_ctrl_handler_setup(hdl);
 
 	if (ctx->is_enc)
-		ctx->q_data[V4L2_M2M_SRC].info = v4l2_fwht_get_pixfmt(0);
+		ctx->q_data[V4L2_M2M_SRC].info = info;
 	else if (ctx->is_stateless)
 		ctx->q_data[V4L2_M2M_SRC].info = &pixfmt_stateless_fwht;
 	else
@@ -1823,22 +1825,22 @@ static int vicodec_open(struct file *file)
 	ctx->q_data[V4L2_M2M_SRC].coded_height = 720;
 	ctx->q_data[V4L2_M2M_SRC].visible_width = 1280;
 	ctx->q_data[V4L2_M2M_SRC].visible_height = 720;
-	size = 1280 * 720 * ctx->q_data[V4L2_M2M_SRC].info->sizeimage_mult /
-		ctx->q_data[V4L2_M2M_SRC].info->sizeimage_div;
+	raw_size = 1280 * 720 * info->sizeimage_mult / info->sizeimage_div;
+	comp_size = 1280 * 720 * pixfmt_fwht.sizeimage_mult /
+				 pixfmt_fwht.sizeimage_div;
 	if (ctx->is_enc || ctx->is_stateless)
-		ctx->q_data[V4L2_M2M_SRC].sizeimage = size;
+		ctx->q_data[V4L2_M2M_SRC].sizeimage = raw_size;
 	else
 		ctx->q_data[V4L2_M2M_SRC].sizeimage =
-			size + sizeof(struct fwht_cframe_hdr);
+			comp_size + sizeof(struct fwht_cframe_hdr);
+	ctx->q_data[V4L2_M2M_DST] = ctx->q_data[V4L2_M2M_SRC];
 	if (ctx->is_enc) {
-		ctx->q_data[V4L2_M2M_DST] = ctx->q_data[V4L2_M2M_SRC];
 		ctx->q_data[V4L2_M2M_DST].info = &pixfmt_fwht;
-		ctx->q_data[V4L2_M2M_DST].sizeimage = 1280 * 720 *
-			ctx->q_data[V4L2_M2M_DST].info->sizeimage_mult /
-			ctx->q_data[V4L2_M2M_DST].info->sizeimage_div +
-			sizeof(struct fwht_cframe_hdr);
+		ctx->q_data[V4L2_M2M_DST].sizeimage =
+			comp_size + sizeof(struct fwht_cframe_hdr);
 	} else {
-		ctx->q_data[V4L2_M2M_DST].info = NULL;
+		ctx->q_data[V4L2_M2M_DST].info = info;
+		ctx->q_data[V4L2_M2M_DST].sizeimage = raw_size;
 	}
 
 	ctx->state.colorspace = V4L2_COLORSPACE_REC709;
-- 
2.17.1

[PATCH v2 04/16] vicodec: pass on enc output format to capture side

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil-cisco@xs4all.nl>

Setting the encoder output format to e.g. 1920x1080 will set the
crop rectangle to 1920x1080, the coded resolution to 1920x1088 and
the capture coded resolution and sizeimage to 1920x1088 as well.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/platform/vicodec/vicodec-core.c | 64 ++++++++++++++++---
 1 file changed, 54 insertions(+), 10 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index aa3af796929b..75b6bc5a6f3b 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -1032,16 +1032,10 @@ static int vidioc_s_fmt(struct vicodec_ctx *ctx, struct v4l2_format *f)
 	default:
 		return -EINVAL;
 	}
-	if (q_data->visible_width > q_data->coded_width)
-		q_data->visible_width = q_data->coded_width;
-	if (q_data->visible_height > q_data->coded_height)
-		q_data->visible_height = q_data->coded_height;
-
 
 	dprintk(ctx->dev,
-		"Setting format for type %d, coded wxh: %dx%d, visible wxh: %dx%d, fourcc: %08x\n",
+		"Setting format for type %d, coded wxh: %dx%d, fourcc: 0x%08x\n",
 		f->type, q_data->coded_width, q_data->coded_height,
-		q_data->visible_width, q_data->visible_height,
 		q_data->info->id);
 
 	return 0;
@@ -1063,18 +1057,69 @@ static int vidioc_s_fmt_vid_out(struct file *file, void *priv,
 				struct v4l2_format *f)
 {
 	struct vicodec_ctx *ctx = file2ctx(file);
-	struct v4l2_pix_format_mplane *pix_mp;
+	struct vicodec_q_data *q_data;
+	struct vicodec_q_data *q_data_cap;
 	struct v4l2_pix_format *pix;
+	struct v4l2_pix_format_mplane *pix_mp;
+	u32 coded_w = 0, coded_h = 0;
+	unsigned int size = 0;
+	u32 w, h;
 	int ret;
 
+	if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) {
+		w = f->fmt.pix.width;
+		h = f->fmt.pix.height;
+	} else {
+		w = f->fmt.pix_mp.width;
+		h = f->fmt.pix_mp.height;
+	}
+	q_data = get_q_data(ctx, f->type);
+	q_data_cap = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE);
+
 	ret = vidioc_try_fmt_vid_out(file, priv, f);
 	if (ret)
 		return ret;
 
+	if (ctx->is_enc) {
+		struct vb2_queue *vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, f->type);
+		struct vb2_queue *vq_cap = v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
+							   V4L2_BUF_TYPE_VIDEO_CAPTURE);
+		const struct v4l2_fwht_pixfmt_info *info = ctx->is_stateless ?
+			&pixfmt_stateless_fwht : &pixfmt_fwht;
+
+		if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) {
+			coded_w = f->fmt.pix.width;
+			coded_h = f->fmt.pix.height;
+		} else {
+			coded_w = f->fmt.pix_mp.width;
+			coded_h = f->fmt.pix_mp.height;
+		}
+		if (vb2_is_busy(vq) && (coded_w != q_data->coded_width ||
+					coded_h != q_data->coded_height))
+			return -EBUSY;
+		size = coded_w * coded_h *
+			info->sizeimage_mult / info->sizeimage_div;
+		if (!ctx->is_stateless)
+			size += sizeof(struct fwht_cframe_hdr);
+
+		if (vb2_is_busy(vq_cap) && size > q_data_cap->sizeimage)
+			return -EBUSY;
+	}
+
 	ret = vidioc_s_fmt(file2ctx(file), f);
 	if (!ret) {
+		if (ctx->is_enc) {
+			q_data->visible_width = w;
+			q_data->visible_height = h;
+			q_data_cap->coded_width = coded_w;
+			q_data_cap->coded_height = coded_h;
+			if (q_data_cap->sizeimage < size)
+				q_data_cap->sizeimage = size;
+			dprintk(ctx->dev,
+				"Setting output crop to %dx%d\n", w, h);
+		}
+
 		switch (f->type) {
-		case V4L2_BUF_TYPE_VIDEO_CAPTURE:
 		case V4L2_BUF_TYPE_VIDEO_OUTPUT:
 			pix = &f->fmt.pix;
 			ctx->state.colorspace = pix->colorspace;
@@ -1082,7 +1127,6 @@ static int vidioc_s_fmt_vid_out(struct file *file, void *priv,
 			ctx->state.ycbcr_enc = pix->ycbcr_enc;
 			ctx->state.quantization = pix->quantization;
 			break;
-		case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE:
 		case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE:
 			pix_mp = &f->fmt.pix_mp;
 			ctx->state.colorspace = pix_mp->colorspace;
-- 
2.17.1

[PATCH v2 05/16] vicodec: add V4L2_CID_MIN_BUFFERS_FOR_OUTPUT

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil-cisco@xs4all.nl>

The stateful encoder requires the presence of this control.
Since a single buffer is sufficient for vicodec, we just
set this control to 1.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/platform/vicodec/vicodec-core.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 75b6bc5a6f3b..b3bf5c4d73cd 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -1841,13 +1841,16 @@ static int vicodec_open(struct file *file)
 	file->private_data = &ctx->fh;
 	ctx->dev = dev;
 	hdl = &ctx->hdl;
-	v4l2_ctrl_handler_init(hdl, 4);
+	v4l2_ctrl_handler_init(hdl, 5);
 	v4l2_ctrl_new_std(hdl, &vicodec_ctrl_ops, V4L2_CID_MPEG_VIDEO_GOP_SIZE,
 			  1, 16, 1, 10);
 	v4l2_ctrl_new_std(hdl, &vicodec_ctrl_ops, V4L2_CID_FWHT_I_FRAME_QP,
 			  1, 31, 1, 20);
 	v4l2_ctrl_new_std(hdl, &vicodec_ctrl_ops, V4L2_CID_FWHT_P_FRAME_QP,
 			  1, 31, 1, 20);
+	if (ctx->is_enc)
+		v4l2_ctrl_new_std(hdl, &vicodec_ctrl_ops,
+				  V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 1, 1, 1);
 	if (ctx->is_stateless)
 		v4l2_ctrl_new_custom(hdl, &vicodec_ctrl_stateless_state, NULL);
 	if (hdl->error) {
-- 
2.17.1

[PATCH v2 06/16] vicodec: improve handling of ENC_CMD_STOP/START

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil-cisco@xs4all.nl>

Correctly handle stopping and restarting the encoder, keeping
track of the stop and drain states.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/platform/vicodec/vicodec-core.c | 121 +++++++++++++++---
 1 file changed, 100 insertions(+), 21 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index b3bf5c4d73cd..a9a0f6a1b69e 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -116,12 +116,14 @@ struct vicodec_ctx {
 	struct vicodec_dev	*dev;
 	bool			is_enc;
 	bool			is_stateless;
+	bool			is_draining;
+	bool			next_is_last;
+	bool			has_stopped;
 	spinlock_t		*lock;
 
 	struct v4l2_ctrl_handler hdl;
 
 	struct vb2_v4l2_buffer *last_src_buf;
-	struct vb2_v4l2_buffer *last_dst_buf;
 
 	/* Source and destination queue data */
 	struct vicodec_q_data   q_data[2];
@@ -138,6 +140,10 @@ struct vicodec_ctx {
 	bool			source_changed;
 };
 
+static const struct v4l2_event vicodec_eos_event = {
+	.type = V4L2_EVENT_EOS
+};
+
 static inline struct vicodec_ctx *file2ctx(struct file *file)
 {
 	return container_of(file->private_data, struct vicodec_ctx, fh);
@@ -397,9 +403,6 @@ static enum vb2_buffer_state get_next_header(struct vicodec_ctx *ctx,
 /* device_run() - prepares and starts the device */
 static void device_run(void *priv)
 {
-	static const struct v4l2_event eos_event = {
-		.type = V4L2_EVENT_EOS
-	};
 	struct vicodec_ctx *ctx = priv;
 	struct vicodec_dev *dev = ctx->dev;
 	struct vb2_v4l2_buffer *src_buf, *dst_buf;
@@ -423,12 +426,12 @@ static void device_run(void *priv)
 	dst_buf->flags &= ~V4L2_BUF_FLAG_LAST;
 	v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, !ctx->is_enc);
 
-	ctx->last_dst_buf = dst_buf;
-
 	spin_lock(ctx->lock);
 	if (!ctx->comp_has_next_frame && src_buf == ctx->last_src_buf) {
 		dst_buf->flags |= V4L2_BUF_FLAG_LAST;
-		v4l2_event_queue_fh(&ctx->fh, &eos_event);
+		v4l2_event_queue_fh(&ctx->fh, &vicodec_eos_event);
+		ctx->is_draining = false;
+		ctx->has_stopped = true;
 	}
 	if (ctx->is_enc || ctx->is_stateless) {
 		src_buf->sequence = q_src->sequence++;
@@ -579,6 +582,8 @@ static int job_ready(void *priv)
 	unsigned int max_to_copy;
 	unsigned int comp_frame_size;
 
+	if (ctx->has_stopped)
+		return 0;
 	if (ctx->source_changed)
 		return 0;
 	if (ctx->is_stateless || ctx->is_enc || ctx->comp_has_frame)
@@ -1217,25 +1222,45 @@ static int vidioc_s_selection(struct file *file, void *priv,
 	return 0;
 }
 
-static void vicodec_mark_last_buf(struct vicodec_ctx *ctx)
+static int vicodec_mark_last_buf(struct vicodec_ctx *ctx)
 {
-	static const struct v4l2_event eos_event = {
-		.type = V4L2_EVENT_EOS
-	};
+	struct vb2_v4l2_buffer *next_dst_buf;
+	int ret = 0;
 
 	spin_lock(ctx->lock);
+	if (ctx->is_draining) {
+		ret = -EBUSY;
+		goto unlock;
+	}
+	if (ctx->has_stopped)
+		goto unlock;
+
 	ctx->last_src_buf = v4l2_m2m_last_src_buf(ctx->fh.m2m_ctx);
-	if (!ctx->last_src_buf && ctx->last_dst_buf) {
-		ctx->last_dst_buf->flags |= V4L2_BUF_FLAG_LAST;
-		v4l2_event_queue_fh(&ctx->fh, &eos_event);
+	ctx->is_draining = true;
+	if (ctx->last_src_buf)
+		goto unlock;
+
+	next_dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+	if (!next_dst_buf) {
+		ctx->next_is_last = true;
+		goto unlock;
 	}
+
+	next_dst_buf->flags |= V4L2_BUF_FLAG_LAST;
+	vb2_buffer_done(&next_dst_buf->vb2_buf, VB2_BUF_STATE_DONE);
+	ctx->is_draining = false;
+	ctx->has_stopped = true;
+	v4l2_event_queue_fh(&ctx->fh, &vicodec_eos_event);
+
+unlock:
 	spin_unlock(ctx->lock);
+	return ret;
 }
 
 static int vicodec_try_encoder_cmd(struct file *file, void *fh,
 				struct v4l2_encoder_cmd *ec)
 {
-	if (ec->cmd != V4L2_ENC_CMD_STOP)
+	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
 		return -EINVAL;
 
 	if (ec->flags & V4L2_ENC_CMD_STOP_AT_GOP_END)
@@ -1254,8 +1279,22 @@ static int vicodec_encoder_cmd(struct file *file, void *fh,
 	if (ret < 0)
 		return ret;
 
-	vicodec_mark_last_buf(ctx);
-	return 0;
+	if (!vb2_is_streaming(&ctx->fh.m2m_ctx->cap_q_ctx.q) ||
+	    !vb2_is_streaming(&ctx->fh.m2m_ctx->out_q_ctx.q))
+		return 0;
+
+	if (ec->cmd == V4L2_ENC_CMD_STOP)
+		return vicodec_mark_last_buf(ctx);
+	ret = 0;
+	spin_lock(ctx->lock);
+	if (ctx->is_draining) {
+		ret = -EBUSY;
+	} else if (ctx->has_stopped) {
+		ctx->has_stopped = false;
+		vb2_clear_last_buffer_dequeued(&ctx->fh.m2m_ctx->cap_q_ctx.q);
+	}
+	spin_unlock(ctx->lock);
+	return ret;
 }
 
 static int vicodec_try_decoder_cmd(struct file *file, void *fh,
@@ -1283,8 +1322,7 @@ static int vicodec_decoder_cmd(struct file *file, void *fh,
 	if (ret < 0)
 		return ret;
 
-	vicodec_mark_last_buf(ctx);
-	return 0;
+	return vicodec_mark_last_buf(ctx);
 }
 
 static int vicodec_enum_framesizes(struct file *file, void *fh,
@@ -1462,6 +1500,23 @@ static void vicodec_buf_queue(struct vb2_buffer *vb)
 		return;
 	}
 
+	if (vb2_is_streaming(vq_cap)) {
+		if (!V4L2_TYPE_IS_OUTPUT(vb->vb2_queue->type) &&
+		    ctx->next_is_last) {
+			unsigned int i;
+
+			for (i = 0; i < vb->num_planes; i++)
+				vb->planes[i].bytesused = 0;
+			vbuf->flags |= V4L2_BUF_FLAG_LAST;
+			vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
+			ctx->is_draining = false;
+			ctx->has_stopped = true;
+			ctx->next_is_last = false;
+			v4l2_event_queue_fh(&ctx->fh, &vicodec_eos_event);
+			return;
+		}
+	}
+
 	/*
 	 * if both queues are streaming, the source change event is
 	 * handled in job_ready
@@ -1571,8 +1626,6 @@ static int vicodec_start_streaming(struct vb2_queue *q,
 
 	if (V4L2_TYPE_IS_OUTPUT(q->type))
 		ctx->last_src_buf = NULL;
-	else
-		ctx->last_dst_buf = NULL;
 
 	state->gop_cnt = 0;
 
@@ -1648,6 +1701,32 @@ static void vicodec_stop_streaming(struct vb2_queue *q)
 
 	vicodec_return_bufs(q, VB2_BUF_STATE_ERROR);
 
+	if (ctx->is_enc) {
+		if (V4L2_TYPE_IS_OUTPUT(q->type)) {
+			if (ctx->is_draining) {
+				struct vb2_v4l2_buffer *next_dst_buf;
+
+				spin_lock(ctx->lock);
+				ctx->last_src_buf = NULL;
+				next_dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
+				if (!next_dst_buf) {
+					ctx->next_is_last = true;
+				} else {
+					next_dst_buf->flags |= V4L2_BUF_FLAG_LAST;
+					vb2_buffer_done(&next_dst_buf->vb2_buf, VB2_BUF_STATE_DONE);
+					ctx->is_draining = false;
+					ctx->has_stopped = true;
+					v4l2_event_queue_fh(&ctx->fh, &vicodec_eos_event);
+				}
+				spin_unlock(ctx->lock);
+			}
+		} else {
+			ctx->is_draining = false;
+			ctx->has_stopped = false;
+			ctx->next_is_last = false;
+		}
+	}
+
 	if ((!V4L2_TYPE_IS_OUTPUT(q->type) && !ctx->is_enc) ||
 	    (V4L2_TYPE_IS_OUTPUT(q->type) && ctx->is_enc)) {
 		if (!ctx->is_stateless)
-- 
2.17.1

[PATCH v2 07/16] vicodec: restrict decoder format list when src fmt is known

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil-cisco@xs4all.nl>

Report the full list of supported decoder formats until the
format encoded in the bitstream is known. After that only
report the formats compatible with that initial format.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/platform/vicodec/vicodec-core.c | 23 +++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index a9a0f6a1b69e..21000cc2f395 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -671,7 +671,6 @@ static int job_ready(void *priv)
 			v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
 
 		update_capture_data_from_header(ctx);
-		ctx->first_source_change_sent = true;
 		v4l2_event_queue_fh(&ctx->fh, &rs_event);
 		set_last_buffer(dst_buf, src_buf, ctx);
 		ctx->source_changed = true;
@@ -718,7 +717,7 @@ static int enum_fmt(struct v4l2_fmtdesc *f, struct vicodec_ctx *ctx,
 		const struct v4l2_fwht_pixfmt_info *info =
 					get_q_data(ctx, f->type)->info;
 
-		if (!info || ctx->is_enc)
+		if (!ctx->first_source_change_sent || ctx->is_enc)
 			info = v4l2_fwht_get_pixfmt(f->index);
 		else
 			info = v4l2_fwht_find_nth_fmt(info->width_div,
@@ -769,9 +768,6 @@ static int vidioc_g_fmt(struct vicodec_ctx *ctx, struct v4l2_format *f)
 	q_data = get_q_data(ctx, f->type);
 	info = q_data->info;
 
-	if (!info)
-		info = v4l2_fwht_get_pixfmt(0);
-
 	switch (f->type) {
 	case V4L2_BUF_TYPE_VIDEO_CAPTURE:
 	case V4L2_BUF_TYPE_VIDEO_OUTPUT:
@@ -1371,6 +1367,16 @@ static int vicodec_subscribe_event(struct v4l2_fh *fh,
 	}
 }
 
+static int vicodec_streamon(struct file *file, void *fh, enum v4l2_buf_type i)
+{
+	struct vicodec_ctx *ctx = container_of(fh, struct vicodec_ctx, fh);
+	int ret = v4l2_m2m_ioctl_streamon(file, fh, i);
+
+	if (!ret && !ctx->is_enc && V4L2_TYPE_IS_OUTPUT(i))
+		ctx->first_source_change_sent = true;
+	return ret;
+}
+
 static const struct v4l2_ioctl_ops vicodec_ioctl_ops = {
 	.vidioc_querycap	= vidioc_querycap,
 
@@ -1402,7 +1408,7 @@ static const struct v4l2_ioctl_ops vicodec_ioctl_ops = {
 	.vidioc_create_bufs	= v4l2_m2m_ioctl_create_bufs,
 	.vidioc_expbuf		= v4l2_m2m_ioctl_expbuf,
 
-	.vidioc_streamon	= v4l2_m2m_ioctl_streamon,
+	.vidioc_streamon	= vicodec_streamon,
 	.vidioc_streamoff	= v4l2_m2m_ioctl_streamoff,
 
 	.vidioc_g_selection	= vidioc_g_selection,
@@ -1618,9 +1624,6 @@ static int vicodec_start_streaming(struct vb2_queue *q,
 	unsigned int total_planes_size;
 	u8 *new_comp_frame = NULL;
 
-	if (!info)
-		return -EINVAL;
-
 	chroma_div = info->width_div * info->height_div;
 	q_data->sequence = 0;
 
@@ -1726,6 +1729,8 @@ static void vicodec_stop_streaming(struct vb2_queue *q)
 			ctx->next_is_last = false;
 		}
 	}
+	if (!ctx->is_enc && V4L2_TYPE_IS_OUTPUT(q->type))
+		ctx->first_source_change_sent = false;
 
 	if ((!V4L2_TYPE_IS_OUTPUT(q->type) && !ctx->is_enc) ||
 	    (V4L2_TYPE_IS_OUTPUT(q->type) && ctx->is_enc)) {
-- 
2.17.1

[PATCH v2 08/16] vicodec: set KEY/PFRAME flag when decoding

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil-cisco@xs4all.nl>

Set V4L2_BUF_FLAG_P/KEYFRAME after decoding a frame.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/platform/vicodec/vicodec-core.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 21000cc2f395..4896f349b4aa 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -335,6 +335,10 @@ static int device_process(struct vicodec_ctx *ctx,
 			copy_cap_to_ref(p_dst, ctx->state.info, &ctx->state);
 
 		vb2_set_plane_payload(&dst_vb->vb2_buf, 0, q_dst->sizeimage);
+		if (ntohl(ctx->state.header.flags) & FWHT_FL_I_FRAME)
+			dst_vb->flags |= V4L2_BUF_FLAG_KEYFRAME;
+		else
+			dst_vb->flags |= V4L2_BUF_FLAG_PFRAME;
 	}
 	return ret;
 }
@@ -410,7 +414,6 @@ static void device_run(void *priv)
 	u32 state;
 	struct media_request *src_req;
 
-
 	src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx);
 	dst_buf = v4l2_m2m_dst_buf_remove(ctx->fh.m2m_ctx);
 	src_req = src_buf->vb2_buf.req_obj.req;
@@ -424,7 +427,7 @@ static void device_run(void *priv)
 	else
 		dst_buf->sequence = q_dst->sequence++;
 	dst_buf->flags &= ~V4L2_BUF_FLAG_LAST;
-	v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, !ctx->is_enc);
+	v4l2_m2m_buf_copy_metadata(src_buf, dst_buf, false);
 
 	spin_lock(ctx->lock);
 	if (!ctx->comp_has_next_frame && src_buf == ctx->last_src_buf) {
-- 
2.17.1

[PATCH v2 09/16] v4l2-ctrls: add v4l2_ctrl_s_ctrl_ptr()

[Dafna Hirschfeld]

From: Hans Verkuil <hverkuil@xs4all.nl>

Add a function to set a compound control by passing a pointer to the
new data.

Functions to set other control types already existed, but not for this.

This is needed for stateless encoders.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/v4l2-core/v4l2-ctrls.c | 11 ++++++++
 include/media/v4l2-ctrls.h           | 42 ++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c
index b1ae2e555c68..c6e8f5838b65 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls.c
@@ -3855,6 +3855,17 @@ int __v4l2_ctrl_s_ctrl_string(struct v4l2_ctrl *ctrl, const char *s)
 }
 EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl_string);
 
+int __v4l2_ctrl_s_ctrl_ptr(struct v4l2_ctrl *ctrl, const void *ptr)
+{
+	lockdep_assert_held(ctrl->handler->lock);
+
+	/* It's a driver bug if this happens. */
+	WARN_ON(!ctrl->is_ptr);
+	memcpy(ctrl->p_new.p, ptr, ctrl->elems * ctrl->elem_size);
+	return set_ctrl(NULL, ctrl, 0);
+}
+EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl_ptr);
+
 void v4l2_ctrl_request_complete(struct media_request *req,
 				struct v4l2_ctrl_handler *main_hdl)
 {
diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h
index bd621cec65a5..87c4ade17e22 100644
--- a/include/media/v4l2-ctrls.h
+++ b/include/media/v4l2-ctrls.h
@@ -1060,6 +1060,48 @@ static inline int v4l2_ctrl_s_ctrl_string(struct v4l2_ctrl *ctrl, const char *s)
 	return rval;
 }
 
+/**
+ * __v4l2_ctrl_s_ctrl_ptr() - Unlocked variant of v4l2_ctrl_s_ctrl_ptr().
+ *
+ * @ctrl:	The control.
+ * @ptr:	Pointer to the new control contents.
+ *
+ * This sets the control's new content safely by going through the control
+ * framework. This function assumes the control's handler is already locked,
+ * allowing it to be used from within the &v4l2_ctrl_ops functions.
+ *
+ * This function is for pointer type controls only and will memcpy()
+ * @ctrl->elems * @ctrl->elem_size bytes from @ptr to the internal
+ * control data.
+ */
+int __v4l2_ctrl_s_ctrl_ptr(struct v4l2_ctrl *ctrl, const void *ptr);
+
+/**
+ * v4l2_ctrl_s_ctrl_ptr() - Helper function to set a control's string value
+ *	 from within a driver.
+ *
+ * @ctrl:	The control.
+ * @ptr:	Pointer to the new control contents.
+ *
+ * This sets the control's new content safely by going through the control
+ * framework. This function will lock the control's handler, so it cannot be
+ * used from within the &v4l2_ctrl_ops functions.
+ *
+ * This function is for pointer type controls only and will memcpy()
+ * @ctrl->elems * @ctrl->elem_size bytes from @ptr to the internal
+ * control data.
+ */
+static inline int v4l2_ctrl_s_ctrl_ptr(struct v4l2_ctrl *ctrl, const void *ptr)
+{
+	int rval;
+
+	v4l2_ctrl_lock(ctrl);
+	rval = __v4l2_ctrl_s_ctrl_ptr(ctrl, ptr);
+	v4l2_ctrl_unlock(ctrl);
+
+	return rval;
+}
+
 /* Internal helper functions that deal with control events. */
 extern const struct v4l2_subscribed_event_ops v4l2_ctrl_sub_ev_ops;
 
-- 
2.17.1

[PATCH v2 10/16] media: vicodec: don't test if info is NULL

[Dafna Hirschfeld]

pixel info of q_data is never set to NULL
so there is no need to test if it is.

Signed-off-by: Dafna Hirschfeld <dafna3@gmail.com>
---
 drivers/media/platform/vicodec/vicodec-core.c | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 4896f349b4aa..52e8128d2d28 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -662,7 +662,6 @@ static int job_ready(void *priv)
 
 	if (ntohl(ctx->state.header.width) != q_dst->visible_width ||
 	    ntohl(ctx->state.header.height) != q_dst->visible_height ||
-	    !q_dst->info ||
 	    hdr_width_div != q_dst->info->width_div ||
 	    hdr_height_div != q_dst->info->height_div) {
 		static const struct v4l2_event rs_event = {
@@ -992,7 +991,6 @@ static int vidioc_s_fmt(struct vicodec_ctx *ctx, struct v4l2_format *f)
 		pix = &f->fmt.pix;
 		if (ctx->is_enc && V4L2_TYPE_IS_OUTPUT(f->type))
 			fmt_changed =
-				!q_data->info ||
 				q_data->info->id != pix->pixelformat ||
 				q_data->coded_width != pix->width ||
 				q_data->coded_height != pix->height;
@@ -1015,7 +1013,6 @@ static int vidioc_s_fmt(struct vicodec_ctx *ctx, struct v4l2_format *f)
 		pix_mp = &f->fmt.pix_mp;
 		if (ctx->is_enc && V4L2_TYPE_IS_OUTPUT(f->type))
 			fmt_changed =
-				!q_data->info ||
 				q_data->info->id != pix_mp->pixelformat ||
 				q_data->coded_width != pix_mp->width ||
 				q_data->coded_height != pix_mp->height;
@@ -1601,10 +1598,6 @@ static unsigned int total_frame_size(struct vicodec_q_data *q_data)
 	unsigned int size;
 	unsigned int chroma_div;
 
-	if (!q_data->info) {
-		WARN_ON(1);
-		return 0;
-	}
 	size = q_data->coded_width * q_data->coded_height;
 	chroma_div = q_data->info->width_div * q_data->info->height_div;
 
@@ -1825,8 +1818,6 @@ static int vicodec_try_ctrl(struct v4l2_ctrl *ctrl)
 
 	switch (ctrl->id) {
 	case V4L2_CID_MPEG_VIDEO_FWHT_PARAMS:
-		if (!q_dst->info)
-			return -EINVAL;
 		params = ctrl->p_new.p_fwht_params;
 		if (params->width > q_dst->coded_width ||
 		    params->width < MIN_WIDTH ||
-- 
2.17.1

[PATCH v2 11/16] media: vicodec: change fwht_encode_frame to get the header separately

[Dafna Hirschfeld]

pass a pointer to the compressed frame header as a
different variable. This is a preparation for the
stateless encoder where the header is not in the same
buffer with the data.

Signed-off-by: Dafna Hirschfeld <dafna3@gmail.com>
---
 drivers/media/platform/vicodec/codec-v4l2-fwht.c | 9 ++++-----
 drivers/media/platform/vicodec/codec-v4l2-fwht.h | 3 ++-
 drivers/media/platform/vicodec/vicodec-core.c    | 9 +++++++--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/drivers/media/platform/vicodec/codec-v4l2-fwht.c b/drivers/media/platform/vicodec/codec-v4l2-fwht.c
index 01e7f09efc4e..6a88c6defd1e 100644
--- a/drivers/media/platform/vicodec/codec-v4l2-fwht.c
+++ b/drivers/media/platform/vicodec/codec-v4l2-fwht.c
@@ -199,12 +199,12 @@ static int prepare_raw_frame(struct fwht_raw_frame *rf,
 	return 0;
 }
 
-int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
+int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out,
+		     struct fwht_cframe_hdr *p_hdr)
 {
 	unsigned int size = state->stride * state->coded_height;
 	unsigned int chroma_stride = state->stride;
 	const struct v4l2_fwht_pixfmt_info *info = state->info;
-	struct fwht_cframe_hdr *p_hdr;
 	struct fwht_cframe cf;
 	struct fwht_raw_frame rf;
 	u32 encoding;
@@ -225,7 +225,7 @@ int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
 
 	cf.i_frame_qp = state->i_frame_qp;
 	cf.p_frame_qp = state->p_frame_qp;
-	cf.rlc_data = (__be16 *)(p_out + sizeof(*p_hdr));
+	cf.rlc_data = (__be16 *)p_out;
 
 	encoding = fwht_encode_frame(&rf, &state->ref_frame, &cf,
 				     !state->gop_cnt,
@@ -238,7 +238,6 @@ int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
 	if (++state->gop_cnt >= state->gop_size)
 		state->gop_cnt = 0;
 
-	p_hdr = (struct fwht_cframe_hdr *)p_out;
 	p_hdr->magic1 = FWHT_MAGIC1;
 	p_hdr->magic2 = FWHT_MAGIC2;
 	p_hdr->version = htonl(FWHT_VERSION);
@@ -266,7 +265,7 @@ int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
 	p_hdr->ycbcr_enc = htonl(state->ycbcr_enc);
 	p_hdr->quantization = htonl(state->quantization);
 	p_hdr->size = htonl(cf.size);
-	return cf.size + sizeof(*p_hdr);
+	return cf.size;
 }
 
 int v4l2_fwht_decode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out)
diff --git a/drivers/media/platform/vicodec/codec-v4l2-fwht.h b/drivers/media/platform/vicodec/codec-v4l2-fwht.h
index 1a0d2a9f931a..e540cd58cf97 100644
--- a/drivers/media/platform/vicodec/codec-v4l2-fwht.h
+++ b/drivers/media/platform/vicodec/codec-v4l2-fwht.h
@@ -58,7 +58,8 @@ const struct v4l2_fwht_pixfmt_info *v4l2_fwht_find_nth_fmt(u32 width_div,
 							  u32 pixenc,
 							  unsigned int start_idx);
 
-int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out);
+int v4l2_fwht_encode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out,
+		     struct fwht_cframe_hdr *hdr);
 int v4l2_fwht_decode(struct v4l2_fwht_state *state, u8 *p_in, u8 *p_out);
 
 #endif
diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 52e8128d2d28..08073f914fd6 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -313,13 +313,18 @@ static int device_process(struct vicodec_ctx *ctx,
 	if (ctx->is_enc) {
 		struct vicodec_q_data *q_src;
 		int comp_sz_or_errcode;
+		struct fwht_cframe_hdr *p_hdr = (struct fwht_cframe_hdr *) p_dst;
+		u8 *comp_buf = p_dst + sizeof(struct fwht_cframe_hdr);
+
 
 		q_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
 		state->info = q_src->info;
-		comp_sz_or_errcode = v4l2_fwht_encode(state, p_src, p_dst);
+		comp_sz_or_errcode = v4l2_fwht_encode(state, p_src, comp_buf,
+						      p_hdr);
 		if (comp_sz_or_errcode < 0)
 			return comp_sz_or_errcode;
-		vb2_set_plane_payload(&dst_vb->vb2_buf, 0, comp_sz_or_errcode);
+		vb2_set_plane_payload(&dst_vb->vb2_buf, 0,
+				      comp_sz_or_errcode + sizeof(*p_hdr));
 	} else {
 		struct vicodec_q_data *q_dst;
 		unsigned int comp_frame_size = ntohl(ctx->state.header.size);
-- 
2.17.1

[PATCH v2 12/16] media: vicodec: bugfix: release stateless dec m2m_dev in vicodec_remove

[Dafna Hirschfeld]

add a call 'v4l2_m2m_release(dev->stateless_dec.m2m_dev)'
in vicodec_remove

Signed-off-by: Dafna Hirschfeld <dafna3@gmail.com>
---
 drivers/media/platform/vicodec/vicodec-core.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 08073f914fd6..3e93e86041f7 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -2254,6 +2254,7 @@ static int vicodec_remove(struct platform_device *pdev)
 
 	v4l2_m2m_release(dev->stateful_enc.m2m_dev);
 	v4l2_m2m_release(dev->stateful_dec.m2m_dev);
+	v4l2_m2m_release(dev->stateless_dec.m2m_dev);
 	video_unregister_device(&dev->stateful_enc.vfd);
 	video_unregister_device(&dev->stateful_dec.vfd);
 	video_unregister_device(&dev->stateless_dec.vfd);
-- 
2.17.1

[PATCH v2 13/16] media: vicodec: add field 'dev_inst' to vicodec_ctx

[Dafna Hirschfeld]

Add the field 'dev_inst' to vicodec_ctx that points to
the 'vicodec_dev_instance'. This saves many if-elses

Signed-off-by: Dafna Hirschfeld <dafna3@gmail.com>
---
 drivers/media/platform/vicodec/vicodec-core.c | 42 ++++++-------------
 1 file changed, 13 insertions(+), 29 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 3e93e86041f7..73661bc96671 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -121,6 +121,7 @@ struct vicodec_ctx {
 	bool			has_stopped;
 	spinlock_t		*lock;
 
+	struct vicodec_dev_instance *dev_inst;
 	struct v4l2_ctrl_handler hdl;
 
 	struct vb2_v4l2_buffer *last_src_buf;
@@ -413,7 +414,6 @@ static enum vb2_buffer_state get_next_header(struct vicodec_ctx *ctx,
 static void device_run(void *priv)
 {
 	struct vicodec_ctx *ctx = priv;
-	struct vicodec_dev *dev = ctx->dev;
 	struct vb2_v4l2_buffer *src_buf, *dst_buf;
 	struct vicodec_q_data *q_src, *q_dst;
 	u32 state;
@@ -462,13 +462,7 @@ static void device_run(void *priv)
 	if (ctx->is_stateless && src_req)
 		v4l2_ctrl_request_complete(src_req, &ctx->hdl);
 
-	if (ctx->is_enc)
-		v4l2_m2m_job_finish(dev->stateful_enc.m2m_dev, ctx->fh.m2m_ctx);
-	else if (ctx->is_stateless)
-		v4l2_m2m_job_finish(dev->stateless_dec.m2m_dev,
-				    ctx->fh.m2m_ctx);
-	else
-		v4l2_m2m_job_finish(dev->stateful_dec.m2m_dev, ctx->fh.m2m_ctx);
+	v4l2_m2m_job_finish(ctx->dev_inst->m2m_dev, ctx->fh.m2m_ctx);
 }
 
 static void job_remove_src_buf(struct vicodec_ctx *ctx, u32 state)
@@ -1787,12 +1781,7 @@ static int queue_init(void *priv, struct vb2_queue *src_vq,
 	src_vq->ops = &vicodec_qops;
 	src_vq->mem_ops = &vb2_vmalloc_memops;
 	src_vq->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_COPY;
-	if (ctx->is_enc)
-		src_vq->lock = &ctx->dev->stateful_enc.mutex;
-	else if (ctx->is_stateless)
-		src_vq->lock = &ctx->dev->stateless_dec.mutex;
-	else
-		src_vq->lock = &ctx->dev->stateful_dec.mutex;
+	src_vq->lock = &ctx->dev_inst->mutex;
 	src_vq->supports_requests = ctx->is_stateless;
 	src_vq->requires_requests = ctx->is_stateless;
 	ret = vb2_queue_init(src_vq);
@@ -1915,10 +1904,15 @@ static int vicodec_open(struct file *file)
 		goto open_unlock;
 	}
 
-	if (vfd == &dev->stateful_enc.vfd)
+	if (vfd == &dev->stateful_enc.vfd) {
 		ctx->is_enc = true;
-	else if (vfd == &dev->stateless_dec.vfd)
+		ctx->dev_inst = &dev->stateful_enc;
+	} else if (vfd == &dev->stateless_dec.vfd) {
 		ctx->is_stateless = true;
+		ctx->dev_inst = &dev->stateless_dec;
+	} else {
+		ctx->dev_inst = &dev->stateful_dec;
+	}
 
 	v4l2_fh_init(&ctx->fh, video_devdata(file));
 	file->private_data = &ctx->fh;
@@ -1975,19 +1969,9 @@ static int vicodec_open(struct file *file)
 
 	ctx->state.colorspace = V4L2_COLORSPACE_REC709;
 
-	if (ctx->is_enc) {
-		ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->stateful_enc.m2m_dev,
-						    ctx, &queue_init);
-		ctx->lock = &dev->stateful_enc.lock;
-	} else if (ctx->is_stateless) {
-		ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->stateless_dec.m2m_dev,
-						    ctx, &queue_init);
-		ctx->lock = &dev->stateless_dec.lock;
-	} else {
-		ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->stateful_dec.m2m_dev,
-						    ctx, &queue_init);
-		ctx->lock = &dev->stateful_dec.lock;
-	}
+	ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(ctx->dev_inst->m2m_dev,
+					    ctx, &queue_init);
+	ctx->lock = &ctx->dev_inst->lock;
 
 	if (IS_ERR(ctx->fh.m2m_ctx)) {
 		rc = PTR_ERR(ctx->fh.m2m_ctx);
-- 
2.17.1

[PATCH v2 14/16] media: vicodec: Register another node for stateless encoder

[Dafna Hirschfeld]

Add stateless encoder instance field to the dev struct and
register another node for the stateless encoder.
The stateless API for the node will be implemented in further
patches.

Signed-off-by: Dafna Hirschfeld <dafna3@gmail.com>
---
 drivers/media/platform/vicodec/vicodec-core.c | 44 +++++++++++++++----
 1 file changed, 35 insertions(+), 9 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 73661bc96671..8780df73a41f 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -104,6 +104,7 @@ struct vicodec_dev {
 	struct v4l2_device	v4l2_dev;
 	struct vicodec_dev_instance stateful_enc;
 	struct vicodec_dev_instance stateful_dec;
+	struct vicodec_dev_instance stateless_enc;
 	struct vicodec_dev_instance stateless_dec;
 #ifdef CONFIG_MEDIA_CONTROLLER
 	struct media_device	mdev;
@@ -1903,16 +1904,20 @@ static int vicodec_open(struct file *file)
 		rc = -ENOMEM;
 		goto open_unlock;
 	}
+	ctx->is_enc = (vfd == &dev->stateful_enc.vfd ||
+		       vfd == &dev->stateless_enc.vfd);
 
-	if (vfd == &dev->stateful_enc.vfd) {
-		ctx->is_enc = true;
+	ctx->is_stateless = (vfd == &dev->stateless_dec.vfd ||
+			     vfd == &dev->stateless_enc.vfd);
+
+	if (vfd == &dev->stateful_enc.vfd)
 		ctx->dev_inst = &dev->stateful_enc;
-	} else if (vfd == &dev->stateless_dec.vfd) {
-		ctx->is_stateless = true;
-		ctx->dev_inst = &dev->stateless_dec;
-	} else {
+	else if (vfd == &dev->stateful_dec.vfd)
 		ctx->dev_inst = &dev->stateful_dec;
-	}
+	else if (vfd == &dev->stateless_enc.vfd)
+		ctx->dev_inst = &dev->stateless_enc;
+	else
+		ctx->dev_inst = &dev->stateless_dec;
 
 	v4l2_fh_init(&ctx->fh, video_devdata(file));
 	file->private_data = &ctx->fh;
@@ -2162,9 +2167,13 @@ static int vicodec_probe(struct platform_device *pdev)
 			      "stateful-decoder", false))
 		goto unreg_sf_enc;
 
+	if (register_instance(dev, &dev->stateless_enc,
+			      "stateless-encoder", false))
+		goto unreg_sf_dec;
+
 	if (register_instance(dev, &dev->stateless_dec,
 			      "stateless-decoder", false))
-		goto unreg_sf_dec;
+		goto unreg_sl_enc;
 
 #ifdef CONFIG_MEDIA_CONTROLLER
 	ret = v4l2_m2m_register_media_controller(dev->stateful_enc.m2m_dev,
@@ -2183,14 +2192,23 @@ static int vicodec_probe(struct platform_device *pdev)
 		goto unreg_m2m_sf_enc_mc;
 	}
 
+	ret = v4l2_m2m_register_media_controller(dev->stateless_enc.m2m_dev,
+						 &dev->stateless_enc.vfd,
+						 MEDIA_ENT_F_PROC_VIDEO_ENCODER);
+	if (ret) {
+		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller for stateless enc\n");
+		goto unreg_m2m_sf_dec_mc;
+	}
+
 	ret = v4l2_m2m_register_media_controller(dev->stateless_dec.m2m_dev,
 						 &dev->stateless_dec.vfd,
 						 MEDIA_ENT_F_PROC_VIDEO_DECODER);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller for stateless dec\n");
-		goto unreg_m2m_sf_dec_mc;
+		goto unreg_m2m_sl_enc_mc;
 	}
 
+
 	ret = media_device_register(&dev->mdev);
 	if (ret) {
 		v4l2_err(&dev->v4l2_dev, "Failed to register mem2mem media device\n");
@@ -2202,6 +2220,8 @@ static int vicodec_probe(struct platform_device *pdev)
 #ifdef CONFIG_MEDIA_CONTROLLER
 unreg_m2m_sl_dec_mc:
 	v4l2_m2m_unregister_media_controller(dev->stateless_dec.m2m_dev);
+unreg_m2m_sl_enc_mc:
+	v4l2_m2m_unregister_media_controller(dev->stateless_enc.m2m_dev);
 unreg_m2m_sf_dec_mc:
 	v4l2_m2m_unregister_media_controller(dev->stateful_dec.m2m_dev);
 unreg_m2m_sf_enc_mc:
@@ -2210,6 +2230,9 @@ static int vicodec_probe(struct platform_device *pdev)
 	video_unregister_device(&dev->stateless_dec.vfd);
 	v4l2_m2m_release(dev->stateless_dec.m2m_dev);
 #endif
+unreg_sl_enc:
+	video_unregister_device(&dev->stateless_enc.vfd);
+	v4l2_m2m_release(dev->stateless_enc.m2m_dev);
 unreg_sf_dec:
 	video_unregister_device(&dev->stateful_dec.vfd);
 	v4l2_m2m_release(dev->stateful_dec.m2m_dev);
@@ -2232,15 +2255,18 @@ static int vicodec_remove(struct platform_device *pdev)
 	media_device_unregister(&dev->mdev);
 	v4l2_m2m_unregister_media_controller(dev->stateful_enc.m2m_dev);
 	v4l2_m2m_unregister_media_controller(dev->stateful_dec.m2m_dev);
+	v4l2_m2m_unregister_media_controller(dev->stateless_enc.m2m_dev);
 	v4l2_m2m_unregister_media_controller(dev->stateless_dec.m2m_dev);
 	media_device_cleanup(&dev->mdev);
 #endif
 
 	v4l2_m2m_release(dev->stateful_enc.m2m_dev);
 	v4l2_m2m_release(dev->stateful_dec.m2m_dev);
+	v4l2_m2m_release(dev->stateless_enc.m2m_dev);
 	v4l2_m2m_release(dev->stateless_dec.m2m_dev);
 	video_unregister_device(&dev->stateful_enc.vfd);
 	video_unregister_device(&dev->stateful_dec.vfd);
+	video_unregister_device(&dev->stateless_enc.vfd);
 	video_unregister_device(&dev->stateless_dec.vfd);
 	v4l2_device_unregister(&dev->v4l2_dev);
 
-- 
2.17.1

[PATCH v2 15/16] media: vicodec: set pixelformat V4L2_PIX_FMT_FWHT_STATELESS for stateless encoder

[Dafna Hirschfeld]

for stateless encoder, set the capture pixelformat
to V4L2_PIX_FMT_FWHT_STATELESS and the pix info to
pixfmt_stateless_fwht

Signed-off-by: Dafna Hirschfeld <dafna3@gmail.com>
---
 drivers/media/platform/vicodec/vicodec-core.c | 32 ++++++++++++-------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index 8780df73a41f..c972c175707c 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -903,8 +903,12 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 		if (multiplanar)
 			return -EINVAL;
 		pix = &f->fmt.pix;
-		pix->pixelformat = ctx->is_enc ? V4L2_PIX_FMT_FWHT :
-				   find_fmt(f->fmt.pix.pixelformat)->id;
+		if (!ctx->is_enc)
+			pix->pixelformat = find_fmt(f->fmt.pix.pixelformat)->id;
+		else if (ctx->is_stateless)
+			pix->pixelformat = V4L2_PIX_FMT_FWHT_STATELESS;
+		else
+			pix->pixelformat = V4L2_PIX_FMT_FWHT;
 		pix->colorspace = ctx->state.colorspace;
 		pix->xfer_func = ctx->state.xfer_func;
 		pix->ycbcr_enc = ctx->state.ycbcr_enc;
@@ -914,8 +918,13 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv,
 		if (!multiplanar)
 			return -EINVAL;
 		pix_mp = &f->fmt.pix_mp;
-		pix_mp->pixelformat = ctx->is_enc ? V4L2_PIX_FMT_FWHT :
-				      find_fmt(pix_mp->pixelformat)->id;
+		if (!ctx->is_enc)
+			pix_mp->pixelformat =
+				find_fmt(f->fmt.pix_mp.pixelformat)->id;
+		else if (ctx->is_stateless)
+			pix_mp->pixelformat = V4L2_PIX_FMT_FWHT_STATELESS;
+		else
+			pix_mp->pixelformat = V4L2_PIX_FMT_FWHT;
 		pix_mp->colorspace = ctx->state.colorspace;
 		pix_mp->xfer_func = ctx->state.xfer_func;
 		pix_mp->ycbcr_enc = ctx->state.ycbcr_enc;
@@ -1647,22 +1656,22 @@ static int vicodec_start_streaming(struct vb2_queue *q,
 	state->stride = q_data->coded_width *
 				info->bytesperline_mult;
 
-	if (ctx->is_stateless) {
+	if (!ctx->is_enc && ctx->is_stateless) {
 		state->ref_stride = state->stride;
 		return 0;
 	}
 	state->ref_stride = q_data->coded_width * info->luma_alpha_step;
 
 	state->ref_frame.buf = kvmalloc(total_planes_size, GFP_KERNEL);
-	state->ref_frame.luma = state->ref_frame.buf;
 	new_comp_frame = kvmalloc(ctx->comp_max_size, GFP_KERNEL);
 
-	if (!state->ref_frame.luma || !new_comp_frame) {
-		kvfree(state->ref_frame.luma);
+	if (!state->ref_frame.buf || !new_comp_frame) {
+		kvfree(state->ref_frame.buf);
 		kvfree(new_comp_frame);
 		vicodec_return_bufs(q, VB2_BUF_STATE_QUEUED);
 		return -ENOMEM;
 	}
+	state->ref_frame.luma = state->ref_frame.buf;
 	/*
 	 * if state->compressed_frame was already allocated then
 	 * it contain data of the first frame of the new resolution
@@ -1964,9 +1973,10 @@ static int vicodec_open(struct file *file)
 			comp_size + sizeof(struct fwht_cframe_hdr);
 	ctx->q_data[V4L2_M2M_DST] = ctx->q_data[V4L2_M2M_SRC];
 	if (ctx->is_enc) {
-		ctx->q_data[V4L2_M2M_DST].info = &pixfmt_fwht;
-		ctx->q_data[V4L2_M2M_DST].sizeimage =
-			comp_size + sizeof(struct fwht_cframe_hdr);
+		ctx->q_data[V4L2_M2M_DST].info = ctx->is_stateless ?
+			&pixfmt_stateless_fwht : &pixfmt_fwht;
+		ctx->q_data[V4L2_M2M_DST].sizeimage = comp_size +
+			sizeof(struct fwht_cframe_hdr);
 	} else {
 		ctx->q_data[V4L2_M2M_DST].info = info;
 		ctx->q_data[V4L2_M2M_DST].sizeimage = raw_size;
-- 
2.17.1

[PATCH v2 16/16] media: vicodec: Add support for stateless encoder

[Dafna Hirschfeld]

Adjust the stateless API code to support
both encoder and decoder.

Signed-off-by: Dafna Hirschfeld <dafna3@gmail.com>
---
 drivers/media/platform/vicodec/vicodec-core.c | 106 ++++++++++++------
 1 file changed, 73 insertions(+), 33 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c b/drivers/media/platform/vicodec/vicodec-core.c
index c972c175707c..e1bfcc480716 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -140,6 +140,7 @@ struct vicodec_ctx {
 	bool			comp_has_next_frame;
 	bool			first_source_change_sent;
 	bool			source_changed;
+	struct v4l2_ctrl	*ctrl_fwht_params;
 };
 
 static const struct v4l2_event vicodec_eos_event = {
@@ -256,6 +257,21 @@ static void update_state_from_header(struct vicodec_ctx *ctx)
 	ctx->state.quantization = ntohl(p_hdr->quantization);
 }
 
+static void update_stateless_params_from_header(const struct vicodec_ctx *ctx,
+						struct v4l2_ctrl_fwht_params *params)
+{
+	const struct fwht_cframe_hdr *p_hdr = &ctx->state.header;
+
+	params->version = ntohl(p_hdr->version);
+	params->width = ntohl(p_hdr->width);
+	params->height = ntohl(p_hdr->height);
+	params->flags =	ntohl(p_hdr->flags);
+	params->colorspace = ntohl(p_hdr->colorspace);
+	params->xfer_func = ntohl(p_hdr->xfer_func);
+	params->ycbcr_enc = ntohl(p_hdr->ycbcr_enc);
+	params->quantization = ntohl(p_hdr->quantization);
+}
+
 static int device_process(struct vicodec_ctx *ctx,
 			  struct vb2_v4l2_buffer *src_vb,
 			  struct vb2_v4l2_buffer *dst_vb)
@@ -276,33 +292,35 @@ static int device_process(struct vicodec_ctx *ctx,
 		ret = v4l2_ctrl_request_setup(src_req, &ctx->hdl);
 		if (ret)
 			return ret;
-		update_state_from_header(ctx);
-
-		ctx->state.header.size =
-			htonl(vb2_get_plane_payload(&src_vb->vb2_buf, 0));
-		/*
-		 * set the reference buffer from the reference timestamp
-		 * only if this is a P-frame
-		 */
-		if (!(ntohl(ctx->state.header.flags) & FWHT_FL_I_FRAME)) {
-			struct vb2_buffer *ref_vb2_buf;
-			int ref_buf_idx;
-			struct vb2_queue *vq_cap =
+		if (!ctx->is_enc) {
+			update_state_from_header(ctx);
+
+			ctx->state.header.size =
+				htonl(vb2_get_plane_payload(&src_vb->vb2_buf, 0));
+			/*
+			 * set the reference buffer from the reference timestamp
+			 * only if this is a P-frame
+			 */
+			if (!(ntohl(ctx->state.header.flags) & FWHT_FL_I_FRAME)) {
+				struct vb2_buffer *ref_vb2_buf;
+				int ref_buf_idx;
+				struct vb2_queue *vq_cap =
 				v4l2_m2m_get_vq(ctx->fh.m2m_ctx,
-						V4L2_BUF_TYPE_VIDEO_CAPTURE);
-
-			ref_buf_idx = vb2_find_timestamp(vq_cap,
-							 ctx->state.ref_frame_ts, 0);
-			if (ref_buf_idx < 0)
-				return -EINVAL;
-
-			ref_vb2_buf = vq_cap->bufs[ref_buf_idx];
-			if (ref_vb2_buf->state == VB2_BUF_STATE_ERROR)
-				ret = -EINVAL;
-			ctx->state.ref_frame.buf =
-				vb2_plane_vaddr(ref_vb2_buf, 0);
-		} else {
-			ctx->state.ref_frame.buf = NULL;
+					V4L2_BUF_TYPE_VIDEO_CAPTURE);
+
+				ref_buf_idx = vb2_find_timestamp(vq_cap,
+					ctx->state.ref_frame_ts, 0);
+				if (ref_buf_idx < 0)
+					return -EINVAL;
+
+				ref_vb2_buf = vq_cap->bufs[ref_buf_idx];
+				if (ref_vb2_buf->state == VB2_BUF_STATE_ERROR)
+					ret = -EINVAL;
+				ctx->state.ref_frame.buf =
+					vb2_plane_vaddr(ref_vb2_buf, 0);
+			} else {
+				ctx->state.ref_frame.buf = NULL;
+			}
 		}
 	}
 	p_dst = vb2_plane_vaddr(&dst_vb->vb2_buf, 0);
@@ -315,18 +333,36 @@ static int device_process(struct vicodec_ctx *ctx,
 	if (ctx->is_enc) {
 		struct vicodec_q_data *q_src;
 		int comp_sz_or_errcode;
-		struct fwht_cframe_hdr *p_hdr = (struct fwht_cframe_hdr *) p_dst;
-		u8 *comp_buf = p_dst + sizeof(struct fwht_cframe_hdr);
-
+		struct fwht_cframe_hdr *p_hdr;
+		u8 *comp_buf;
 
+		if (ctx->is_stateless) {
+			p_hdr = &ctx->state.header;
+			comp_buf = p_dst;
+		} else {
+			p_hdr = (struct fwht_cframe_hdr *) p_dst;
+			comp_buf = p_dst + sizeof(struct fwht_cframe_hdr);
+		}
 		q_src = get_q_data(ctx, V4L2_BUF_TYPE_VIDEO_OUTPUT);
 		state->info = q_src->info;
 		comp_sz_or_errcode = v4l2_fwht_encode(state, p_src, comp_buf,
 						      p_hdr);
 		if (comp_sz_or_errcode < 0)
 			return comp_sz_or_errcode;
-		vb2_set_plane_payload(&dst_vb->vb2_buf, 0,
-				      comp_sz_or_errcode + sizeof(*p_hdr));
+
+		if (!ctx->is_stateless) {
+			comp_sz_or_errcode += sizeof(*p_hdr);
+		} else {
+			int ret;
+			struct v4l2_ctrl_fwht_params params;
+
+			update_stateless_params_from_header(ctx, &params);
+			ret = v4l2_ctrl_s_ctrl_ptr(ctx->ctrl_fwht_params,
+						   &params);
+			if (ret)
+				return ret;
+		}
+		vb2_set_plane_payload(&dst_vb->vb2_buf, 0, comp_sz_or_errcode);
 	} else {
 		struct vicodec_q_data *q_dst;
 		unsigned int comp_frame_size = ntohl(ctx->state.header.size);
@@ -1739,7 +1775,7 @@ static void vicodec_stop_streaming(struct vb2_queue *q)
 
 	if ((!V4L2_TYPE_IS_OUTPUT(q->type) && !ctx->is_enc) ||
 	    (V4L2_TYPE_IS_OUTPUT(q->type) && ctx->is_enc)) {
-		if (!ctx->is_stateless)
+		if (!ctx->is_stateless || ctx->is_enc)
 			kvfree(ctx->state.ref_frame.buf);
 		ctx->state.ref_frame.buf = NULL;
 		ctx->state.ref_frame.luma = NULL;
@@ -1819,6 +1855,8 @@ static int vicodec_try_ctrl(struct v4l2_ctrl *ctrl)
 	const struct v4l2_ctrl_fwht_params *params;
 	struct vicodec_q_data *q_dst = get_q_data(ctx,
 			V4L2_BUF_TYPE_VIDEO_CAPTURE);
+	if (ctx->is_enc)
+		return 0;
 
 	switch (ctrl->id) {
 	case V4L2_CID_MPEG_VIDEO_FWHT_PARAMS:
@@ -1943,7 +1981,9 @@ static int vicodec_open(struct file *file)
 		v4l2_ctrl_new_std(hdl, &vicodec_ctrl_ops,
 				  V4L2_CID_MIN_BUFFERS_FOR_OUTPUT, 1, 1, 1, 1);
 	if (ctx->is_stateless)
-		v4l2_ctrl_new_custom(hdl, &vicodec_ctrl_stateless_state, NULL);
+		ctx->ctrl_fwht_params =
+			v4l2_ctrl_new_custom(hdl, &vicodec_ctrl_stateless_state,
+					     NULL);
 	if (hdl->error) {
 		rc = hdl->error;
 		v4l2_ctrl_handler_free(hdl);
-- 
2.17.1