[RFC PATCH 0/3] Split up v4l2-mem2mem.c into a core and codec part

[RFC PATCH 0/3] Split up v4l2-mem2mem.c into a core and codec part

[Hans Verkuil]

This RFC series splits up v4l2-mem2mem.c into a core and codec part.

This is a first stab at this, I'm not entirely happy with the
fact that v4l2_m2m_qbuf has to call to _v4l2_codec_qbuf helper
for some codec specific handling.

In any case, splitting this up makes it easier to maintain (IMHO)
and probably makes it easier as well to improve the codec part.

Regards,

	Hans

Hans Verkuil (3):
  media: split up v4l2-mem2mem.c into a core and codec part
  media: include v4l2-mem2mem-codec.h in codec drivers
  media: v4l2-mem2mem.h: move codec bits to v4l2-mem2mem-codec.h

 .../media/platform/allegro-dvt/allegro-core.c |   2 +-
 drivers/media/platform/amphion/vdec.c         |   2 +-
 drivers/media/platform/amphion/venc.c         |   2 +-
 .../media/platform/chips-media/coda-common.c  |   2 +-
 .../platform/mediatek/jpeg/mtk_jpeg_core.c    |   2 +-
 .../mediatek/vcodec/decoder/mtk_vcodec_dec.c  |   2 +-
 .../vcodec/decoder/mtk_vcodec_dec_stateless.c |   2 +-
 .../mediatek/vcodec/encoder/mtk_vcodec_enc.c  |   2 +-
 drivers/media/platform/nvidia/tegra-vde/vde.h |   2 +-
 .../media/platform/nxp/imx-jpeg/mxc-jpeg.c    |   2 +-
 .../media/platform/verisilicon/hantro_drv.c   |   2 +-
 .../media/platform/verisilicon/hantro_v4l2.c  |   2 +-
 .../media/test-drivers/vicodec/vicodec-core.c |   2 +-
 drivers/media/test-drivers/visl/visl-dec.c    |   2 +-
 drivers/media/v4l2-core/Makefile              |   2 +
 drivers/media/v4l2-core/v4l2-mem2mem-codec.c  | 359 +++++++++++++++
 .../{v4l2-mem2mem.c => v4l2-mem2mem-core.c}   | 422 +-----------------
 drivers/media/v4l2-core/v4l2-mem2mem-priv.h   | 100 +++++
 drivers/staging/media/meson/vdec/vdec.c       |   2 +-
 drivers/staging/media/rkvdec/rkvdec.c         |   2 +-
 .../staging/media/sunxi/cedrus/cedrus_dec.c   |   2 +-
 .../staging/media/sunxi/cedrus/cedrus_hw.c    |   2 +-
 .../staging/media/sunxi/cedrus/cedrus_video.c |   2 +-
 include/media/v4l2-mem2mem-codec.h            | 187 ++++++++
 include/media/v4l2-mem2mem.h                  | 202 +--------
 25 files changed, 691 insertions(+), 619 deletions(-)
 create mode 100644 drivers/media/v4l2-core/v4l2-mem2mem-codec.c
 rename drivers/media/v4l2-core/{v4l2-mem2mem.c => v4l2-mem2mem-core.c} (72%)
 create mode 100644 drivers/media/v4l2-core/v4l2-mem2mem-priv.h
 create mode 100644 include/media/v4l2-mem2mem-codec.h

-- 
2.40.1

[RFC PATCH 1/3] media: split up v4l2-mem2mem.c into a core and codec part

[Hans Verkuil]

The v4l2-mem2mem.c source contains core functions, valid for any m2m device,
and codec functions that are needed for codec drivers only. Split up the
source into a core source and a codec source to simplify maintenance.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/v4l2-core/Makefile              |   2 +
 drivers/media/v4l2-core/v4l2-mem2mem-codec.c  | 359 +++++++++++++++
 .../{v4l2-mem2mem.c => v4l2-mem2mem-core.c}   | 422 +-----------------
 drivers/media/v4l2-core/v4l2-mem2mem-priv.h   | 100 +++++
 include/media/v4l2-mem2mem-codec.h            |  15 +
 include/media/v4l2-mem2mem.h                  |  31 +-
 6 files changed, 500 insertions(+), 429 deletions(-)
 create mode 100644 drivers/media/v4l2-core/v4l2-mem2mem-codec.c
 rename drivers/media/v4l2-core/{v4l2-mem2mem.c => v4l2-mem2mem-core.c} (72%)
 create mode 100644 drivers/media/v4l2-core/v4l2-mem2mem-priv.h
 create mode 100644 include/media/v4l2-mem2mem-codec.h

diff --git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile
index be2551705755..ae718b8ab46e 100644
--- a/drivers/media/v4l2-core/Makefile
+++ b/drivers/media/v4l2-core/Makefile
@@ -13,6 +13,8 @@ videodev-objs	:=	v4l2-dev.o v4l2-ioctl.o v4l2-device.o v4l2-fh.o \
 			v4l2-ctrls-core.o v4l2-ctrls-api.o \
 			v4l2-ctrls-request.o v4l2-ctrls-defs.o
 
+v4l2-mem2mem-objs	:=	v4l2-mem2mem-core.o v4l2-mem2mem-codec.o
+
 # Please keep it alphabetically sorted by Kconfig name
 # (e. g. LC_ALL=C sort Makefile)
 videodev-$(CONFIG_COMPAT) += v4l2-compat-ioctl32.o
diff --git a/drivers/media/v4l2-core/v4l2-mem2mem-codec.c b/drivers/media/v4l2-core/v4l2-mem2mem-codec.c
new file mode 100644
index 000000000000..6e4b539a3b62
--- /dev/null
+++ b/drivers/media/v4l2-core/v4l2-mem2mem-codec.c
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Memory-to-memory device framework for Video for Linux 2 and vb2.
+ *
+ * Helper functions for devices that use vb2 buffers for both their
+ * source and destination.
+ *
+ * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+#include <linux/module.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+
+#include <media/media-device.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-mem2mem-codec.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-event.h>
+
+#include "v4l2-mem2mem-priv.h"
+
+void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
+				      struct v4l2_m2m_ctx *m2m_ctx,
+				      enum vb2_buffer_state state)
+{
+	struct vb2_v4l2_buffer *src_buf, *dst_buf;
+	bool schedule_next = false;
+	unsigned long flags;
+
+	spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
+	src_buf = v4l2_m2m_src_buf_remove(m2m_ctx);
+	dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
+
+	if (WARN_ON(!src_buf || !dst_buf))
+		goto unlock;
+	dst_buf->is_held = src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
+	if (!dst_buf->is_held) {
+		v4l2_m2m_dst_buf_remove(m2m_ctx);
+		v4l2_m2m_buf_done(dst_buf, state);
+	}
+	/*
+	 * If the request API is being used, returning the OUTPUT
+	 * (src) buffer will wake-up any process waiting on the
+	 * request file descriptor.
+	 *
+	 * Therefore, return the CAPTURE (dst) buffer first,
+	 * to avoid signalling the request file descriptor
+	 * before the CAPTURE buffer is done.
+	 */
+	v4l2_m2m_buf_done(src_buf, state);
+	schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
+unlock:
+	spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
+
+	if (schedule_next)
+		v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx);
+}
+EXPORT_SYMBOL(v4l2_m2m_buf_done_and_job_finish);
+
+/*
+ * This will add the LAST flag and mark the buffer management
+ * state as stopped.
+ * This is called when the last capture buffer must be flagged as LAST
+ * in draining mode from the encoder/decoder driver buf_queue() callback
+ * or from v4l2_update_last_buf_state() when a capture buffer is available.
+ */
+void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
+			       struct vb2_v4l2_buffer *vbuf)
+{
+	vbuf->flags |= V4L2_BUF_FLAG_LAST;
+	vb2_buffer_done(&vbuf->vb2_buf, VB2_BUF_STATE_DONE);
+
+	v4l2_m2m_mark_stopped(m2m_ctx);
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_last_buffer_done);
+
+/* When stop command is issued, update buffer management state */
+static int v4l2_update_last_buf_state(struct v4l2_m2m_ctx *m2m_ctx)
+{
+	struct vb2_v4l2_buffer *next_dst_buf;
+
+	if (m2m_ctx->is_draining)
+		return -EBUSY;
+
+	if (m2m_ctx->has_stopped)
+		return 0;
+
+	m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx);
+	m2m_ctx->is_draining = true;
+
+	/*
+	 * The processing of the last output buffer queued before
+	 * the STOP command is expected to mark the buffer management
+	 * state as stopped with v4l2_m2m_mark_stopped().
+	 */
+	if (m2m_ctx->last_src_buf)
+		return 0;
+
+	/*
+	 * In case the output queue is empty, try to mark the last capture
+	 * buffer as LAST.
+	 */
+	next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
+	if (!next_dst_buf) {
+		/*
+		 * Wait for the next queued one in encoder/decoder driver
+		 * buf_queue() callback using the v4l2_m2m_dst_buf_is_last()
+		 * helper or in v4l2_m2m_qbuf() if encoder/decoder is not yet
+		 * streaming.
+		 */
+		m2m_ctx->next_buf_last = true;
+		return 0;
+	}
+
+	v4l2_m2m_last_buffer_done(m2m_ctx, next_dst_buf);
+
+	return 0;
+}
+
+/*
+ * Updates the encoding/decoding buffer management state, should
+ * be called from encoder/decoder drivers start_streaming()
+ */
+void v4l2_m2m_update_start_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
+					   struct vb2_queue *q)
+{
+	/* If start streaming again, untag the last output buffer */
+	if (V4L2_TYPE_IS_OUTPUT(q->type))
+		m2m_ctx->last_src_buf = NULL;
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_update_start_streaming_state);
+
+/*
+ * Updates the encoding/decoding buffer management state, should
+ * be called from encoder/decoder driver stop_streaming()
+ */
+void v4l2_m2m_update_stop_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
+					  struct vb2_queue *q)
+{
+	if (V4L2_TYPE_IS_OUTPUT(q->type)) {
+		/*
+		 * If in draining state, either mark next dst buffer as
+		 * done or flag next one to be marked as done either
+		 * in encoder/decoder driver buf_queue() callback using
+		 * the v4l2_m2m_dst_buf_is_last() helper or in v4l2_m2m_qbuf()
+		 * if encoder/decoder is not yet streaming
+		 */
+		if (m2m_ctx->is_draining) {
+			struct vb2_v4l2_buffer *next_dst_buf;
+
+			m2m_ctx->last_src_buf = NULL;
+			next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
+			if (!next_dst_buf)
+				m2m_ctx->next_buf_last = true;
+			else
+				v4l2_m2m_last_buffer_done(m2m_ctx,
+							  next_dst_buf);
+		}
+	} else {
+		v4l2_m2m_clear_state(m2m_ctx);
+	}
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_update_stop_streaming_state);
+
+static void v4l2_m2m_force_last_buf_done(struct v4l2_m2m_ctx *m2m_ctx,
+					 struct vb2_queue *q)
+{
+	struct vb2_buffer *vb;
+	struct vb2_v4l2_buffer *vbuf;
+	unsigned int i;
+
+	if (WARN_ON(q->is_output))
+		return;
+	if (list_empty(&q->queued_list))
+		return;
+
+	vb = list_first_entry(&q->queued_list, struct vb2_buffer, queued_entry);
+	for (i = 0; i < vb->num_planes; i++)
+		vb2_set_plane_payload(vb, i, 0);
+
+	/*
+	 * Since the buffer hasn't been queued to the ready queue,
+	 * mark is active and owned before marking it LAST and DONE
+	 */
+	vb->state = VB2_BUF_STATE_ACTIVE;
+	atomic_inc(&q->owned_by_drv_count);
+
+	vbuf = to_vb2_v4l2_buffer(vb);
+	vbuf->field = V4L2_FIELD_NONE;
+
+	v4l2_m2m_last_buffer_done(m2m_ctx, vbuf);
+}
+
+void _v4l2_codec_qbuf(struct v4l2_m2m_ctx *m2m_ctx, struct vb2_queue *vq,
+		      struct v4l2_buffer *buf)
+{
+	/*
+	 * If the capture queue is streaming, but streaming hasn't started
+	 * on the device, but was asked to stop, mark the previously queued
+	 * buffer as DONE with LAST flag since it won't be queued on the
+	 * device.
+	 */
+	if (V4L2_TYPE_IS_CAPTURE(vq->type) &&
+	    vb2_is_streaming(vq) && !vb2_start_streaming_called(vq) &&
+	   (v4l2_m2m_has_stopped(m2m_ctx) || v4l2_m2m_dst_buf_is_last(m2m_ctx)))
+		v4l2_m2m_force_last_buf_done(m2m_ctx, vq);
+	else if (!(buf->flags & V4L2_BUF_FLAG_IN_REQUEST))
+		v4l2_m2m_try_schedule(m2m_ctx);
+}
+
+int v4l2_m2m_ioctl_try_encoder_cmd(struct file *file, void *fh,
+				   struct v4l2_encoder_cmd *ec)
+{
+	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
+		return -EINVAL;
+
+	ec->flags = 0;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_encoder_cmd);
+
+int v4l2_m2m_ioctl_try_decoder_cmd(struct file *file, void *fh,
+				   struct v4l2_decoder_cmd *dc)
+{
+	if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
+		return -EINVAL;
+
+	dc->flags = 0;
+
+	if (dc->cmd == V4L2_DEC_CMD_STOP) {
+		dc->stop.pts = 0;
+	} else if (dc->cmd == V4L2_DEC_CMD_START) {
+		dc->start.speed = 0;
+		dc->start.format = V4L2_DEC_START_FMT_NONE;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_decoder_cmd);
+
+/*
+ * Updates the encoding state on ENC_CMD_STOP/ENC_CMD_START
+ * Should be called from the encoder driver encoder_cmd() callback
+ */
+int v4l2_m2m_encoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+			 struct v4l2_encoder_cmd *ec)
+{
+	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
+		return -EINVAL;
+
+	if (ec->cmd == V4L2_ENC_CMD_STOP)
+		return v4l2_update_last_buf_state(m2m_ctx);
+
+	if (m2m_ctx->is_draining)
+		return -EBUSY;
+
+	if (m2m_ctx->has_stopped)
+		m2m_ctx->has_stopped = false;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_encoder_cmd);
+
+/*
+ * Updates the decoding state on DEC_CMD_STOP/DEC_CMD_START
+ * Should be called from the decoder driver decoder_cmd() callback
+ */
+int v4l2_m2m_decoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+			 struct v4l2_decoder_cmd *dc)
+{
+	if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
+		return -EINVAL;
+
+	if (dc->cmd == V4L2_DEC_CMD_STOP)
+		return v4l2_update_last_buf_state(m2m_ctx);
+
+	if (m2m_ctx->is_draining)
+		return -EBUSY;
+
+	if (m2m_ctx->has_stopped)
+		m2m_ctx->has_stopped = false;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_decoder_cmd);
+
+int v4l2_m2m_ioctl_encoder_cmd(struct file *file, void *priv,
+			       struct v4l2_encoder_cmd *ec)
+{
+	struct v4l2_fh *fh = file->private_data;
+
+	return v4l2_m2m_encoder_cmd(file, fh->m2m_ctx, ec);
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_encoder_cmd);
+
+int v4l2_m2m_ioctl_decoder_cmd(struct file *file, void *priv,
+			       struct v4l2_decoder_cmd *dc)
+{
+	struct v4l2_fh *fh = file->private_data;
+
+	return v4l2_m2m_decoder_cmd(file, fh->m2m_ctx, dc);
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_decoder_cmd);
+
+int v4l2_m2m_ioctl_stateless_try_decoder_cmd(struct file *file, void *fh,
+					     struct v4l2_decoder_cmd *dc)
+{
+	if (dc->cmd != V4L2_DEC_CMD_FLUSH)
+		return -EINVAL;
+
+	dc->flags = 0;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_try_decoder_cmd);
+
+int v4l2_m2m_ioctl_stateless_decoder_cmd(struct file *file, void *priv,
+					 struct v4l2_decoder_cmd *dc)
+{
+	struct v4l2_fh *fh = file->private_data;
+	struct vb2_v4l2_buffer *out_vb, *cap_vb;
+	struct v4l2_m2m_dev *m2m_dev = fh->m2m_ctx->m2m_dev;
+	unsigned long flags;
+	int ret;
+
+	ret = v4l2_m2m_ioctl_stateless_try_decoder_cmd(file, priv, dc);
+	if (ret < 0)
+		return ret;
+
+	spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
+	out_vb = v4l2_m2m_last_src_buf(fh->m2m_ctx);
+	cap_vb = v4l2_m2m_last_dst_buf(fh->m2m_ctx);
+
+	/*
+	 * If there is an out buffer pending, then clear any HOLD flag.
+	 *
+	 * By clearing this flag we ensure that when this output
+	 * buffer is processed any held capture buffer will be released.
+	 */
+	if (out_vb) {
+		out_vb->flags &= ~V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
+	} else if (cap_vb && cap_vb->is_held) {
+		/*
+		 * If there were no output buffers, but there is a
+		 * capture buffer that is held, then release that
+		 * buffer.
+		 */
+		cap_vb->is_held = false;
+		v4l2_m2m_dst_buf_remove(fh->m2m_ctx);
+		v4l2_m2m_buf_done(cap_vb, VB2_BUF_STATE_DONE);
+	}
+	spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_decoder_cmd);
diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem-core.c
similarity index 72%
rename from drivers/media/v4l2-core/v4l2-mem2mem.c
rename to drivers/media/v4l2-core/v4l2-mem2mem-core.c
index 0cc30397fbad..8963a5b29b1d 100644
--- a/drivers/media/v4l2-core/v4l2-mem2mem.c
+++ b/drivers/media/v4l2-core/v4l2-mem2mem-core.c
@@ -21,42 +21,14 @@
 #include <media/v4l2-fh.h>
 #include <media/v4l2-event.h>
 
+#include "v4l2-mem2mem-priv.h"
+
 MODULE_DESCRIPTION("Mem to mem device framework for vb2");
 MODULE_AUTHOR("Pawel Osciak, <pawel@osciak.com>");
 MODULE_LICENSE("GPL");
 
-static bool debug;
-module_param(debug, bool, 0644);
-
-#define dprintk(fmt, arg...)						\
-	do {								\
-		if (debug)						\
-			printk(KERN_DEBUG "%s: " fmt, __func__, ## arg);\
-	} while (0)
-
-
-/* Instance is already queued on the job_queue */
-#define TRANS_QUEUED		(1 << 0)
-/* Instance is currently running in hardware */
-#define TRANS_RUNNING		(1 << 1)
-/* Instance is currently aborting */
-#define TRANS_ABORT		(1 << 2)
-
-
-/* The job queue is not running new jobs */
-#define QUEUE_PAUSED		(1 << 0)
-
-
-/* Offset base for buffers on the destination queue - used to distinguish
- * between source and destination buffers when mmapping - they receive the same
- * offsets but for different queues */
-#define DST_QUEUE_OFF_BASE	(1 << 30)
-
-enum v4l2_m2m_entity_type {
-	MEM2MEM_ENT_TYPE_SOURCE,
-	MEM2MEM_ENT_TYPE_SINK,
-	MEM2MEM_ENT_TYPE_PROC
-};
+bool v4l2_mem2mem_debug;
+module_param_named(debug, v4l2_mem2mem_debug, bool, 0644);
 
 static const char * const m2m_entity_name[] = {
 	"source",
@@ -64,53 +36,6 @@ static const char * const m2m_entity_name[] = {
 	"proc"
 };
 
-/**
- * struct v4l2_m2m_dev - per-device context
- * @source:		&struct media_entity pointer with the source entity
- *			Used only when the M2M device is registered via
- *			v4l2_m2m_register_media_controller().
- * @source_pad:		&struct media_pad with the source pad.
- *			Used only when the M2M device is registered via
- *			v4l2_m2m_register_media_controller().
- * @sink:		&struct media_entity pointer with the sink entity
- *			Used only when the M2M device is registered via
- *			v4l2_m2m_register_media_controller().
- * @sink_pad:		&struct media_pad with the sink pad.
- *			Used only when the M2M device is registered via
- *			v4l2_m2m_register_media_controller().
- * @proc:		&struct media_entity pointer with the M2M device itself.
- * @proc_pads:		&struct media_pad with the @proc pads.
- *			Used only when the M2M device is registered via
- *			v4l2_m2m_unregister_media_controller().
- * @intf_devnode:	&struct media_intf devnode pointer with the interface
- *			with controls the M2M device.
- * @curr_ctx:		currently running instance
- * @job_queue:		instances queued to run
- * @job_spinlock:	protects job_queue
- * @job_work:		worker to run queued jobs.
- * @job_queue_flags:	flags of the queue status, %QUEUE_PAUSED.
- * @m2m_ops:		driver callbacks
- */
-struct v4l2_m2m_dev {
-	struct v4l2_m2m_ctx	*curr_ctx;
-#ifdef CONFIG_MEDIA_CONTROLLER
-	struct media_entity	*source;
-	struct media_pad	source_pad;
-	struct media_entity	sink;
-	struct media_pad	sink_pad;
-	struct media_entity	proc;
-	struct media_pad	proc_pads[2];
-	struct media_intf_devnode *intf_devnode;
-#endif
-
-	struct list_head	job_queue;
-	spinlock_t		job_spinlock;
-	struct work_struct	job_work;
-	unsigned long		job_queue_flags;
-
-	const struct v4l2_m2m_ops *m2m_ops;
-};
-
 static struct v4l2_m2m_queue_ctx *get_queue_ctx(struct v4l2_m2m_ctx *m2m_ctx,
 						enum v4l2_buf_type type)
 {
@@ -445,8 +370,8 @@ static void v4l2_m2m_cancel_job(struct v4l2_m2m_ctx *m2m_ctx)
  * Schedule the next job, called from v4l2_m2m_job_finish() or
  * v4l2_m2m_buf_done_and_job_finish().
  */
-static void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
-				       struct v4l2_m2m_ctx *m2m_ctx)
+void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
+				struct v4l2_m2m_ctx *m2m_ctx)
 {
 	/*
 	 * This instance might have more buffers ready, but since we do not
@@ -466,8 +391,8 @@ static void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
  * Assumes job_spinlock is held, called from v4l2_m2m_job_finish() or
  * v4l2_m2m_buf_done_and_job_finish().
  */
-static bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
-				 struct v4l2_m2m_ctx *m2m_ctx)
+bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
+			  struct v4l2_m2m_ctx *m2m_ctx)
 {
 	if (!m2m_dev->curr_ctx || m2m_dev->curr_ctx != m2m_ctx) {
 		dprintk("Called by an instance not currently running\n");
@@ -503,44 +428,6 @@ void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
 }
 EXPORT_SYMBOL(v4l2_m2m_job_finish);
 
-void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
-				      struct v4l2_m2m_ctx *m2m_ctx,
-				      enum vb2_buffer_state state)
-{
-	struct vb2_v4l2_buffer *src_buf, *dst_buf;
-	bool schedule_next = false;
-	unsigned long flags;
-
-	spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
-	src_buf = v4l2_m2m_src_buf_remove(m2m_ctx);
-	dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
-
-	if (WARN_ON(!src_buf || !dst_buf))
-		goto unlock;
-	dst_buf->is_held = src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
-	if (!dst_buf->is_held) {
-		v4l2_m2m_dst_buf_remove(m2m_ctx);
-		v4l2_m2m_buf_done(dst_buf, state);
-	}
-	/*
-	 * If the request API is being used, returning the OUTPUT
-	 * (src) buffer will wake-up any process waiting on the
-	 * request file descriptor.
-	 *
-	 * Therefore, return the CAPTURE (dst) buffer first,
-	 * to avoid signalling the request file descriptor
-	 * before the CAPTURE buffer is done.
-	 */
-	v4l2_m2m_buf_done(src_buf, state);
-	schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
-unlock:
-	spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
-
-	if (schedule_next)
-		v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx);
-}
-EXPORT_SYMBOL(v4l2_m2m_buf_done_and_job_finish);
-
 void v4l2_m2m_suspend(struct v4l2_m2m_dev *m2m_dev)
 {
 	unsigned long flags;
@@ -621,140 +508,6 @@ int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_querybuf);
 
-/*
- * This will add the LAST flag and mark the buffer management
- * state as stopped.
- * This is called when the last capture buffer must be flagged as LAST
- * in draining mode from the encoder/decoder driver buf_queue() callback
- * or from v4l2_update_last_buf_state() when a capture buffer is available.
- */
-void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
-			       struct vb2_v4l2_buffer *vbuf)
-{
-	vbuf->flags |= V4L2_BUF_FLAG_LAST;
-	vb2_buffer_done(&vbuf->vb2_buf, VB2_BUF_STATE_DONE);
-
-	v4l2_m2m_mark_stopped(m2m_ctx);
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_last_buffer_done);
-
-/* When stop command is issued, update buffer management state */
-static int v4l2_update_last_buf_state(struct v4l2_m2m_ctx *m2m_ctx)
-{
-	struct vb2_v4l2_buffer *next_dst_buf;
-
-	if (m2m_ctx->is_draining)
-		return -EBUSY;
-
-	if (m2m_ctx->has_stopped)
-		return 0;
-
-	m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx);
-	m2m_ctx->is_draining = true;
-
-	/*
-	 * The processing of the last output buffer queued before
-	 * the STOP command is expected to mark the buffer management
-	 * state as stopped with v4l2_m2m_mark_stopped().
-	 */
-	if (m2m_ctx->last_src_buf)
-		return 0;
-
-	/*
-	 * In case the output queue is empty, try to mark the last capture
-	 * buffer as LAST.
-	 */
-	next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
-	if (!next_dst_buf) {
-		/*
-		 * Wait for the next queued one in encoder/decoder driver
-		 * buf_queue() callback using the v4l2_m2m_dst_buf_is_last()
-		 * helper or in v4l2_m2m_qbuf() if encoder/decoder is not yet
-		 * streaming.
-		 */
-		m2m_ctx->next_buf_last = true;
-		return 0;
-	}
-
-	v4l2_m2m_last_buffer_done(m2m_ctx, next_dst_buf);
-
-	return 0;
-}
-
-/*
- * Updates the encoding/decoding buffer management state, should
- * be called from encoder/decoder drivers start_streaming()
- */
-void v4l2_m2m_update_start_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
-					   struct vb2_queue *q)
-{
-	/* If start streaming again, untag the last output buffer */
-	if (V4L2_TYPE_IS_OUTPUT(q->type))
-		m2m_ctx->last_src_buf = NULL;
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_update_start_streaming_state);
-
-/*
- * Updates the encoding/decoding buffer management state, should
- * be called from encoder/decoder driver stop_streaming()
- */
-void v4l2_m2m_update_stop_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
-					  struct vb2_queue *q)
-{
-	if (V4L2_TYPE_IS_OUTPUT(q->type)) {
-		/*
-		 * If in draining state, either mark next dst buffer as
-		 * done or flag next one to be marked as done either
-		 * in encoder/decoder driver buf_queue() callback using
-		 * the v4l2_m2m_dst_buf_is_last() helper or in v4l2_m2m_qbuf()
-		 * if encoder/decoder is not yet streaming
-		 */
-		if (m2m_ctx->is_draining) {
-			struct vb2_v4l2_buffer *next_dst_buf;
-
-			m2m_ctx->last_src_buf = NULL;
-			next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
-			if (!next_dst_buf)
-				m2m_ctx->next_buf_last = true;
-			else
-				v4l2_m2m_last_buffer_done(m2m_ctx,
-							  next_dst_buf);
-		}
-	} else {
-		v4l2_m2m_clear_state(m2m_ctx);
-	}
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_update_stop_streaming_state);
-
-static void v4l2_m2m_force_last_buf_done(struct v4l2_m2m_ctx *m2m_ctx,
-					 struct vb2_queue *q)
-{
-	struct vb2_buffer *vb;
-	struct vb2_v4l2_buffer *vbuf;
-	unsigned int i;
-
-	if (WARN_ON(q->is_output))
-		return;
-	if (list_empty(&q->queued_list))
-		return;
-
-	vb = list_first_entry(&q->queued_list, struct vb2_buffer, queued_entry);
-	for (i = 0; i < vb->num_planes; i++)
-		vb2_set_plane_payload(vb, i, 0);
-
-	/*
-	 * Since the buffer hasn't been queued to the ready queue,
-	 * mark is active and owned before marking it LAST and DONE
-	 */
-	vb->state = VB2_BUF_STATE_ACTIVE;
-	atomic_inc(&q->owned_by_drv_count);
-
-	vbuf = to_vb2_v4l2_buffer(vb);
-	vbuf->field = V4L2_FIELD_NONE;
-
-	v4l2_m2m_last_buffer_done(m2m_ctx, vbuf);
-}
-
 int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		  struct v4l2_buffer *buf)
 {
@@ -777,18 +530,7 @@ int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 	/* Adjust MMAP memory offsets for the CAPTURE queue */
 	v4l2_m2m_adjust_mem_offset(vq, buf);
 
-	/*
-	 * If the capture queue is streaming, but streaming hasn't started
-	 * on the device, but was asked to stop, mark the previously queued
-	 * buffer as DONE with LAST flag since it won't be queued on the
-	 * device.
-	 */
-	if (V4L2_TYPE_IS_CAPTURE(vq->type) &&
-	    vb2_is_streaming(vq) && !vb2_start_streaming_called(vq) &&
-	   (v4l2_m2m_has_stopped(m2m_ctx) || v4l2_m2m_dst_buf_is_last(m2m_ctx)))
-		v4l2_m2m_force_last_buf_done(m2m_ctx, vq);
-	else if (!(buf->flags & V4L2_BUF_FLAG_IN_REQUEST))
-		v4l2_m2m_try_schedule(m2m_ctx);
+	_v4l2_codec_qbuf(m2m_ctx, vq, buf);
 
 	return 0;
 }
@@ -1440,152 +1182,6 @@ int v4l2_m2m_ioctl_streamoff(struct file *file, void *priv,
 }
 EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_streamoff);
 
-int v4l2_m2m_ioctl_try_encoder_cmd(struct file *file, void *fh,
-				   struct v4l2_encoder_cmd *ec)
-{
-	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
-		return -EINVAL;
-
-	ec->flags = 0;
-	return 0;
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_encoder_cmd);
-
-int v4l2_m2m_ioctl_try_decoder_cmd(struct file *file, void *fh,
-				   struct v4l2_decoder_cmd *dc)
-{
-	if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
-		return -EINVAL;
-
-	dc->flags = 0;
-
-	if (dc->cmd == V4L2_DEC_CMD_STOP) {
-		dc->stop.pts = 0;
-	} else if (dc->cmd == V4L2_DEC_CMD_START) {
-		dc->start.speed = 0;
-		dc->start.format = V4L2_DEC_START_FMT_NONE;
-	}
-	return 0;
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_decoder_cmd);
-
-/*
- * Updates the encoding state on ENC_CMD_STOP/ENC_CMD_START
- * Should be called from the encoder driver encoder_cmd() callback
- */
-int v4l2_m2m_encoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
-			 struct v4l2_encoder_cmd *ec)
-{
-	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
-		return -EINVAL;
-
-	if (ec->cmd == V4L2_ENC_CMD_STOP)
-		return v4l2_update_last_buf_state(m2m_ctx);
-
-	if (m2m_ctx->is_draining)
-		return -EBUSY;
-
-	if (m2m_ctx->has_stopped)
-		m2m_ctx->has_stopped = false;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_encoder_cmd);
-
-/*
- * Updates the decoding state on DEC_CMD_STOP/DEC_CMD_START
- * Should be called from the decoder driver decoder_cmd() callback
- */
-int v4l2_m2m_decoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
-			 struct v4l2_decoder_cmd *dc)
-{
-	if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
-		return -EINVAL;
-
-	if (dc->cmd == V4L2_DEC_CMD_STOP)
-		return v4l2_update_last_buf_state(m2m_ctx);
-
-	if (m2m_ctx->is_draining)
-		return -EBUSY;
-
-	if (m2m_ctx->has_stopped)
-		m2m_ctx->has_stopped = false;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_decoder_cmd);
-
-int v4l2_m2m_ioctl_encoder_cmd(struct file *file, void *priv,
-			       struct v4l2_encoder_cmd *ec)
-{
-	struct v4l2_fh *fh = file->private_data;
-
-	return v4l2_m2m_encoder_cmd(file, fh->m2m_ctx, ec);
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_encoder_cmd);
-
-int v4l2_m2m_ioctl_decoder_cmd(struct file *file, void *priv,
-			       struct v4l2_decoder_cmd *dc)
-{
-	struct v4l2_fh *fh = file->private_data;
-
-	return v4l2_m2m_decoder_cmd(file, fh->m2m_ctx, dc);
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_decoder_cmd);
-
-int v4l2_m2m_ioctl_stateless_try_decoder_cmd(struct file *file, void *fh,
-					     struct v4l2_decoder_cmd *dc)
-{
-	if (dc->cmd != V4L2_DEC_CMD_FLUSH)
-		return -EINVAL;
-
-	dc->flags = 0;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_try_decoder_cmd);
-
-int v4l2_m2m_ioctl_stateless_decoder_cmd(struct file *file, void *priv,
-					 struct v4l2_decoder_cmd *dc)
-{
-	struct v4l2_fh *fh = file->private_data;
-	struct vb2_v4l2_buffer *out_vb, *cap_vb;
-	struct v4l2_m2m_dev *m2m_dev = fh->m2m_ctx->m2m_dev;
-	unsigned long flags;
-	int ret;
-
-	ret = v4l2_m2m_ioctl_stateless_try_decoder_cmd(file, priv, dc);
-	if (ret < 0)
-		return ret;
-
-	spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
-	out_vb = v4l2_m2m_last_src_buf(fh->m2m_ctx);
-	cap_vb = v4l2_m2m_last_dst_buf(fh->m2m_ctx);
-
-	/*
-	 * If there is an out buffer pending, then clear any HOLD flag.
-	 *
-	 * By clearing this flag we ensure that when this output
-	 * buffer is processed any held capture buffer will be released.
-	 */
-	if (out_vb) {
-		out_vb->flags &= ~V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
-	} else if (cap_vb && cap_vb->is_held) {
-		/*
-		 * If there were no output buffers, but there is a
-		 * capture buffer that is held, then release that
-		 * buffer.
-		 */
-		cap_vb->is_held = false;
-		v4l2_m2m_dst_buf_remove(fh->m2m_ctx);
-		v4l2_m2m_buf_done(cap_vb, VB2_BUF_STATE_DONE);
-	}
-	spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_decoder_cmd);
-
 /*
  * v4l2_file_operations helpers. It is assumed here same lock is used
  * for the output and the capture buffer queue.
diff --git a/drivers/media/v4l2-core/v4l2-mem2mem-priv.h b/drivers/media/v4l2-core/v4l2-mem2mem-priv.h
new file mode 100644
index 000000000000..65ac006808df
--- /dev/null
+++ b/drivers/media/v4l2-core/v4l2-mem2mem-priv.h
@@ -0,0 +1,100 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Memory-to-memory device framework, private header
+ *
+ * Copyright (c) 2009 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#ifndef _MEDIA_V4L2_MEM2MEM_PRIV_H
+#define _MEDIA_V4L2_MEM2MEM_PRIV_H
+
+extern bool v4l2_mem2mem_debug;
+
+#define dprintk(fmt, arg...)						\
+	do {								\
+		if (v4l2_mem2mem_debug)					\
+			printk(KERN_DEBUG "%s: " fmt, __func__, ## arg);\
+	} while (0)
+
+
+/* Instance is already queued on the job_queue */
+#define TRANS_QUEUED		(1 << 0)
+/* Instance is currently running in hardware */
+#define TRANS_RUNNING		(1 << 1)
+/* Instance is currently aborting */
+#define TRANS_ABORT		(1 << 2)
+
+
+/* The job queue is not running new jobs */
+#define QUEUE_PAUSED		(1 << 0)
+
+
+/* Offset base for buffers on the destination queue - used to distinguish
+ * between source and destination buffers when mmapping - they receive the same
+ * offsets but for different queues */
+#define DST_QUEUE_OFF_BASE	(1 << 30)
+
+enum v4l2_m2m_entity_type {
+	MEM2MEM_ENT_TYPE_SOURCE,
+	MEM2MEM_ENT_TYPE_SINK,
+	MEM2MEM_ENT_TYPE_PROC
+};
+
+/**
+ * struct v4l2_m2m_dev - per-device context
+ * @source:		&struct media_entity pointer with the source entity
+ *			Used only when the M2M device is registered via
+ *			v4l2_m2m_register_media_controller().
+ * @source_pad:		&struct media_pad with the source pad.
+ *			Used only when the M2M device is registered via
+ *			v4l2_m2m_register_media_controller().
+ * @sink:		&struct media_entity pointer with the sink entity
+ *			Used only when the M2M device is registered via
+ *			v4l2_m2m_register_media_controller().
+ * @sink_pad:		&struct media_pad with the sink pad.
+ *			Used only when the M2M device is registered via
+ *			v4l2_m2m_register_media_controller().
+ * @proc:		&struct media_entity pointer with the M2M device itself.
+ * @proc_pads:		&struct media_pad with the @proc pads.
+ *			Used only when the M2M device is registered via
+ *			v4l2_m2m_unregister_media_controller().
+ * @intf_devnode:	&struct media_intf devnode pointer with the interface
+ *			with controls the M2M device.
+ * @curr_ctx:		currently running instance
+ * @job_queue:		instances queued to run
+ * @job_spinlock:	protects job_queue
+ * @job_work:		worker to run queued jobs.
+ * @job_queue_flags:	flags of the queue status, %QUEUE_PAUSED.
+ * @m2m_ops:		driver callbacks
+ */
+struct v4l2_m2m_dev {
+	struct v4l2_m2m_ctx	*curr_ctx;
+#ifdef CONFIG_MEDIA_CONTROLLER
+	struct media_entity	*source;
+	struct media_pad	source_pad;
+	struct media_entity	sink;
+	struct media_pad	sink_pad;
+	struct media_entity	proc;
+	struct media_pad	proc_pads[2];
+	struct media_intf_devnode *intf_devnode;
+#endif
+
+	struct list_head	job_queue;
+	spinlock_t		job_spinlock;
+	struct work_struct	job_work;
+	unsigned long		job_queue_flags;
+
+	const struct v4l2_m2m_ops *m2m_ops;
+};
+
+bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
+			  struct v4l2_m2m_ctx *m2m_ctx);
+void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
+				struct v4l2_m2m_ctx *m2m_ctx);
+
+void _v4l2_codec_qbuf(struct v4l2_m2m_ctx *m2m_ctx, struct vb2_queue *vq,
+		      struct v4l2_buffer *buf);
+
+#endif /* _MEDIA_V4L2_MEM2MEM_PRIV_H */
diff --git a/include/media/v4l2-mem2mem-codec.h b/include/media/v4l2-mem2mem-codec.h
new file mode 100644
index 000000000000..6e3d75fd4017
--- /dev/null
+++ b/include/media/v4l2-mem2mem-codec.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Memory-to-memory device framework, codec specific part.
+ *
+ * Copyright (c) 2009 Samsung Electronics Co., Ltd.
+ * Pawel Osciak, <pawel@osciak.com>
+ * Marek Szyprowski, <m.szyprowski@samsung.com>
+ */
+
+#ifndef _MEDIA_V4L2_MEM2MEM_CODEC_H
+#define _MEDIA_V4L2_MEM2MEM_CODEC_H
+
+#include <media/v4l2-mem2mem.h>
+
+#endif /* _MEDIA_V4L2_MEM2MEM_CODEC_H */
diff --git a/include/media/v4l2-mem2mem.h b/include/media/v4l2-mem2mem.h
index d6c8eb2b5201..cebadec49666 100644
--- a/include/media/v4l2-mem2mem.h
+++ b/include/media/v4l2-mem2mem.h
@@ -75,6 +75,14 @@ struct v4l2_m2m_queue_ctx {
  * struct v4l2_m2m_ctx - Memory to memory context structure
  *
  * @q_lock: struct &mutex lock
+ * @m2m_dev: opaque pointer to the internal data to handle M2M context
+ * @cap_q_ctx: Capture (output to memory) queue context
+ * @out_q_ctx: Output (input from memory) queue context
+ * @queue: List of memory to memory contexts
+ * @job_flags: Job queue flags, used internally by v4l2-mem2mem.c:
+ *		%TRANS_QUEUED, %TRANS_RUNNING and %TRANS_ABORT.
+ * @finished: Wait queue used to signalize when a job queue finished.
+ * @priv: Instance private data
  * @new_frame: valid in the device_run callback: if true, then this
  *		starts a new frame; if false, then this is a new slice
  *		for an existing frame. This is always true unless
@@ -84,14 +92,6 @@ struct v4l2_m2m_queue_ctx {
  * @last_src_buf: indicate the last source buffer for draining
  * @next_buf_last: next capture queud buffer will be tagged as last
  * @has_stopped: indicate the device has been stopped
- * @m2m_dev: opaque pointer to the internal data to handle M2M context
- * @cap_q_ctx: Capture (output to memory) queue context
- * @out_q_ctx: Output (input from memory) queue context
- * @queue: List of memory to memory contexts
- * @job_flags: Job queue flags, used internally by v4l2-mem2mem.c:
- *		%TRANS_QUEUED, %TRANS_RUNNING and %TRANS_ABORT.
- * @finished: Wait queue used to signalize when a job queue finished.
- * @priv: Instance private data
  *
  * The memory to memory context is specific to a file handle, NOT to e.g.
  * a device.
@@ -100,18 +100,10 @@ struct v4l2_m2m_ctx {
 	/* optional cap/out vb2 queues lock */
 	struct mutex			*q_lock;
 
-	bool				new_frame;
-
-	bool				is_draining;
-	struct vb2_v4l2_buffer		*last_src_buf;
-	bool				next_buf_last;
-	bool				has_stopped;
-
 	/* internal use only */
 	struct v4l2_m2m_dev		*m2m_dev;
 
 	struct v4l2_m2m_queue_ctx	cap_q_ctx;
-
 	struct v4l2_m2m_queue_ctx	out_q_ctx;
 
 	/* For device job queue */
@@ -120,6 +112,13 @@ struct v4l2_m2m_ctx {
 	wait_queue_head_t		finished;
 
 	void				*priv;
+
+	/* codec specific fields */
+	bool				new_frame;
+	bool				is_draining;
+	struct vb2_v4l2_buffer		*last_src_buf;
+	bool				next_buf_last;
+	bool				has_stopped;
 };
 
 /**
-- 
2.40.1

[RFC PATCH 2/3] media: include v4l2-mem2mem-codec.h in codec drivers

[Hans Verkuil]

The codec specific parts of v4l2-mem2mem.h will move to
v4l2-mem2mem-codec.h in the next patch, so include this
new header where needed.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 drivers/media/platform/allegro-dvt/allegro-core.c               | 2 +-
 drivers/media/platform/amphion/vdec.c                           | 2 +-
 drivers/media/platform/amphion/venc.c                           | 2 +-
 drivers/media/platform/chips-media/coda-common.c                | 2 +-
 drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c            | 2 +-
 drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec.c | 2 +-
 .../platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c | 2 +-
 drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c | 2 +-
 drivers/media/platform/nvidia/tegra-vde/vde.h                   | 2 +-
 drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c                  | 2 +-
 drivers/media/platform/verisilicon/hantro_drv.c                 | 2 +-
 drivers/media/platform/verisilicon/hantro_v4l2.c                | 2 +-
 drivers/media/test-drivers/vicodec/vicodec-core.c               | 2 +-
 drivers/media/test-drivers/visl/visl-dec.c                      | 2 +-
 drivers/staging/media/meson/vdec/vdec.c                         | 2 +-
 drivers/staging/media/rkvdec/rkvdec.c                           | 2 +-
 drivers/staging/media/sunxi/cedrus/cedrus_dec.c                 | 2 +-
 drivers/staging/media/sunxi/cedrus/cedrus_hw.c                  | 2 +-
 drivers/staging/media/sunxi/cedrus/cedrus_video.c               | 2 +-
 19 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/drivers/media/platform/allegro-dvt/allegro-core.c b/drivers/media/platform/allegro-dvt/allegro-core.c
index da61f9beb6b4..032bec973095 100644
--- a/drivers/media/platform/allegro-dvt/allegro-core.c
+++ b/drivers/media/platform/allegro-dvt/allegro-core.c
@@ -27,7 +27,7 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
 #include <media/v4l2-ioctl.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/videobuf2-dma-contig.h>
 #include <media/videobuf2-v4l2.h>
 
diff --git a/drivers/media/platform/amphion/vdec.c b/drivers/media/platform/amphion/vdec.c
index 133d77d1ea0c..4c4ffbd2967c 100644
--- a/drivers/media/platform/amphion/vdec.c
+++ b/drivers/media/platform/amphion/vdec.c
@@ -13,7 +13,7 @@
 #include <linux/videodev2.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/v4l2-ioctl.h>
 #include <media/videobuf2-v4l2.h>
 #include <media/videobuf2-dma-contig.h>
diff --git a/drivers/media/platform/amphion/venc.c b/drivers/media/platform/amphion/venc.c
index 4eb57d793a9c..ceb3c88e971f 100644
--- a/drivers/media/platform/amphion/venc.c
+++ b/drivers/media/platform/amphion/venc.c
@@ -16,7 +16,7 @@
 #include <linux/vmalloc.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/v4l2-ioctl.h>
 #include <media/videobuf2-v4l2.h>
 #include <media/videobuf2-dma-contig.h>
diff --git a/drivers/media/platform/chips-media/coda-common.c b/drivers/media/platform/chips-media/coda-common.c
index cc4892129aaf..ae01d517e2db 100644
--- a/drivers/media/platform/chips-media/coda-common.c
+++ b/drivers/media/platform/chips-media/coda-common.c
@@ -32,7 +32,7 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
 #include <media/v4l2-ioctl.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/videobuf2-v4l2.h>
 #include <media/videobuf2-dma-contig.h>
 #include <media/videobuf2-vmalloc.h>
diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
index 621038aab116..cde3fa5dc76d 100644
--- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
+++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
@@ -18,7 +18,7 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/v4l2-ioctl.h>
 #include <media/videobuf2-core.h>
 #include <media/videobuf2-dma-contig.h>
diff --git a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec.c b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec.c
index 91ed576d6821..c401fcf662ff 100644
--- a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec.c
+++ b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec.c
@@ -6,7 +6,7 @@
  */
 
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/videobuf2-dma-contig.h>
 
 #include "mtk_vcodec_dec_drv.h"
diff --git a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c
index e29c9c58f3da..85a633f7f22e 100644
--- a/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c
+++ b/drivers/media/platform/mediatek/vcodec/decoder/mtk_vcodec_dec_stateless.c
@@ -3,7 +3,7 @@
 #include <media/videobuf2-v4l2.h>
 #include <media/videobuf2-dma-contig.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <linux/module.h>
 
 #include "mtk_vcodec_dec.h"
diff --git a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c
index 8e44a051edda..a2316eb71201 100644
--- a/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c
+++ b/drivers/media/platform/mediatek/vcodec/encoder/mtk_vcodec_enc.c
@@ -6,7 +6,7 @@
 */
 
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/videobuf2-dma-contig.h>
 #include <linux/pm_runtime.h>
 
diff --git a/drivers/media/platform/nvidia/tegra-vde/vde.h b/drivers/media/platform/nvidia/tegra-vde/vde.h
index 0fbb1f3d2c88..3417594b2661 100644
--- a/drivers/media/platform/nvidia/tegra-vde/vde.h
+++ b/drivers/media/platform/nvidia/tegra-vde/vde.h
@@ -23,7 +23,7 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-event.h>
 #include <media/v4l2-ioctl.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 
 #define ICMDQUE_WR		0x00
 #define CMDQUE_CONTROL		0x08
diff --git a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
index b7a720198ce5..0141b4b45ee2 100644
--- a/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
+++ b/drivers/media/platform/nxp/imx-jpeg/mxc-jpeg.c
@@ -54,7 +54,7 @@
 #include <linux/string.h>
 
 #include <media/v4l2-jpeg.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-common.h>
 #include <media/v4l2-event.h>
diff --git a/drivers/media/platform/verisilicon/hantro_drv.c b/drivers/media/platform/verisilicon/hantro_drv.c
index 423fc85d79ee..c5f408ed14b4 100644
--- a/drivers/media/platform/verisilicon/hantro_drv.c
+++ b/drivers/media/platform/verisilicon/hantro_drv.c
@@ -20,7 +20,7 @@
 #include <linux/videodev2.h>
 #include <linux/workqueue.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/videobuf2-core.h>
 #include <media/videobuf2-vmalloc.h>
 
diff --git a/drivers/media/platform/verisilicon/hantro_v4l2.c b/drivers/media/platform/verisilicon/hantro_v4l2.c
index b3ae037a50f6..879c36b72024 100644
--- a/drivers/media/platform/verisilicon/hantro_v4l2.c
+++ b/drivers/media/platform/verisilicon/hantro_v4l2.c
@@ -22,7 +22,7 @@
 #include <linux/workqueue.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 
 #include "hantro.h"
 #include "hantro_hw.h"
diff --git a/drivers/media/test-drivers/vicodec/vicodec-core.c b/drivers/media/test-drivers/vicodec/vicodec-core.c
index 6f0e20df74e9..e35876efc551 100644
--- a/drivers/media/test-drivers/vicodec/vicodec-core.c
+++ b/drivers/media/test-drivers/vicodec/vicodec-core.c
@@ -16,7 +16,7 @@
 #include <linux/slab.h>
 
 #include <linux/platform_device.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-ctrls.h>
diff --git a/drivers/media/test-drivers/visl/visl-dec.c b/drivers/media/test-drivers/visl/visl-dec.c
index 318d675e5668..a31772f8bef6 100644
--- a/drivers/media/test-drivers/visl/visl-dec.c
+++ b/drivers/media/test-drivers/visl/visl-dec.c
@@ -16,7 +16,7 @@
 
 #include <linux/delay.h>
 #include <linux/workqueue.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/tpg/v4l2-tpg.h>
 
 static void *plane_vaddr(struct tpg_data *tpg, struct vb2_buffer *buf,
diff --git a/drivers/staging/media/meson/vdec/vdec.c b/drivers/staging/media/meson/vdec/vdec.c
index 5ca4b1200831..42177c07e33b 100644
--- a/drivers/staging/media/meson/vdec/vdec.c
+++ b/drivers/staging/media/meson/vdec/vdec.c
@@ -16,7 +16,7 @@
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-event.h>
 #include <media/v4l2-ctrls.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/v4l2-dev.h>
 #include <media/videobuf2-dma-contig.h>
 
diff --git a/drivers/staging/media/rkvdec/rkvdec.c b/drivers/staging/media/rkvdec/rkvdec.c
index 84a41792cb4b..fc9d96824049 100644
--- a/drivers/staging/media/rkvdec/rkvdec.c
+++ b/drivers/staging/media/rkvdec/rkvdec.c
@@ -20,7 +20,7 @@
 #include <linux/videodev2.h>
 #include <linux/workqueue.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 #include <media/videobuf2-core.h>
 #include <media/videobuf2-vmalloc.h>
 
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c
index fbbf9e6f0f50..5f4026693e98 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c
@@ -16,7 +16,7 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 
 #include "cedrus.h"
 #include "cedrus_dec.h"
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_hw.c b/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
index fa86a658fdc6..4e62a2afc920 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_hw.c
@@ -25,7 +25,7 @@
 #include <linux/soc/sunxi/sunxi_sram.h>
 
 #include <media/videobuf2-core.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 
 #include "cedrus.h"
 #include "cedrus_hw.h"
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_video.c b/drivers/staging/media/sunxi/cedrus/cedrus_video.c
index b00feaf4072c..a8ddf5bc7361 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_video.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_video.c
@@ -19,7 +19,7 @@
 #include <media/v4l2-device.h>
 #include <media/v4l2-ioctl.h>
 #include <media/v4l2-event.h>
-#include <media/v4l2-mem2mem.h>
+#include <media/v4l2-mem2mem-codec.h>
 
 #include "cedrus.h"
 #include "cedrus_video.h"
-- 
2.40.1

[RFC PATCH 3/3] media: v4l2-mem2mem.h: move codec bits to v4l2-mem2mem-codec.h

[Hans Verkuil]

Move the codec-specific parts to v4l2-mem2mem-codec.h now
that all codec drivers include this new header.

Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
---
 include/media/v4l2-mem2mem-codec.h | 172 +++++++++++++++++++++++++++++
 include/media/v4l2-mem2mem.h       | 171 ----------------------------
 2 files changed, 172 insertions(+), 171 deletions(-)

diff --git a/include/media/v4l2-mem2mem-codec.h b/include/media/v4l2-mem2mem-codec.h
index 6e3d75fd4017..9508e3a2efaf 100644
--- a/include/media/v4l2-mem2mem-codec.h
+++ b/include/media/v4l2-mem2mem-codec.h
@@ -12,4 +12,176 @@
 
 #include <media/v4l2-mem2mem.h>
 
+/**
+ * v4l2_m2m_buf_done_and_job_finish() - return source/destination buffers with
+ * state and inform the framework that a job has been finished and have it
+ * clean up
+ *
+ * @m2m_dev: opaque pointer to the internal data to handle M2M context
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ * @state: vb2 buffer state passed to v4l2_m2m_buf_done().
+ *
+ * Drivers that set V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF must use this
+ * function instead of job_finish() to take held buffers into account. It is
+ * optional for other drivers.
+ *
+ * This function removes the source buffer from the ready list and returns
+ * it with the given state. The same is done for the destination buffer, unless
+ * it is marked 'held'. In that case the buffer is kept on the ready list.
+ *
+ * After that the job is finished (see job_finish()).
+ *
+ * This allows for multiple output buffers to be used to fill in a single
+ * capture buffer. This is typically used by stateless decoders where
+ * multiple e.g. H.264 slices contribute to a single decoded frame.
+ */
+void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
+				      struct v4l2_m2m_ctx *m2m_ctx,
+				      enum vb2_buffer_state state);
+
+/**
+ * v4l2_m2m_clear_state() - clear encoding/decoding state
+ *
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ */
+static inline void
+v4l2_m2m_clear_state(struct v4l2_m2m_ctx *m2m_ctx)
+{
+	m2m_ctx->next_buf_last = false;
+	m2m_ctx->is_draining = false;
+	m2m_ctx->has_stopped = false;
+}
+
+/**
+ * v4l2_m2m_mark_stopped() - set current encoding/decoding state as stopped
+ *
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ */
+static inline void
+v4l2_m2m_mark_stopped(struct v4l2_m2m_ctx *m2m_ctx)
+{
+	m2m_ctx->next_buf_last = false;
+	m2m_ctx->is_draining = false;
+	m2m_ctx->has_stopped = true;
+}
+
+/**
+ * v4l2_m2m_dst_buf_is_last() - return the current encoding/decoding session
+ * draining management state of next queued capture buffer
+ *
+ * This last capture buffer should be tagged with V4L2_BUF_FLAG_LAST to notify
+ * the end of the capture session.
+ *
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ */
+static inline bool
+v4l2_m2m_dst_buf_is_last(struct v4l2_m2m_ctx *m2m_ctx)
+{
+	return m2m_ctx->is_draining && m2m_ctx->next_buf_last;
+}
+
+/**
+ * v4l2_m2m_has_stopped() - return the current encoding/decoding session
+ * stopped state
+ *
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ */
+static inline bool
+v4l2_m2m_has_stopped(struct v4l2_m2m_ctx *m2m_ctx)
+{
+	return m2m_ctx->has_stopped;
+}
+
+/**
+ * v4l2_m2m_is_last_draining_src_buf() - return the output buffer draining
+ * state in the current encoding/decoding session
+ *
+ * This will identify the last output buffer queued before a session stop
+ * was required, leading to an actual encoding/decoding session stop state
+ * in the encoding/decoding process after being processed.
+ *
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ * @vbuf: pointer to struct &v4l2_buffer
+ */
+static inline bool
+v4l2_m2m_is_last_draining_src_buf(struct v4l2_m2m_ctx *m2m_ctx,
+				  struct vb2_v4l2_buffer *vbuf)
+{
+	return m2m_ctx->is_draining && vbuf == m2m_ctx->last_src_buf;
+}
+
+/**
+ * v4l2_m2m_last_buffer_done() - marks the buffer with LAST flag and DONE
+ *
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ * @vbuf: pointer to struct &v4l2_buffer
+ */
+void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
+			       struct vb2_v4l2_buffer *vbuf);
+
+/**
+ * v4l2_m2m_update_start_streaming_state() - update the encoding/decoding
+ * session state when a start of streaming of a video queue is requested
+ *
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ * @q: queue
+ */
+void v4l2_m2m_update_start_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
+					   struct vb2_queue *q);
+
+/**
+ * v4l2_m2m_update_stop_streaming_state() -  update the encoding/decoding
+ * session state when a stop of streaming of a video queue is requested
+ *
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ * @q: queue
+ */
+void v4l2_m2m_update_stop_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
+					  struct vb2_queue *q);
+
+/**
+ * v4l2_m2m_encoder_cmd() - execute an encoder command
+ *
+ * @file: pointer to struct &file
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ * @ec: pointer to the encoder command
+ */
+int v4l2_m2m_encoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+			 struct v4l2_encoder_cmd *ec);
+
+/**
+ * v4l2_m2m_decoder_cmd() - execute a decoder command
+ *
+ * @file: pointer to struct &file
+ * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
+ * @dc: pointer to the decoder command
+ */
+int v4l2_m2m_decoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
+			 struct v4l2_decoder_cmd *dc);
+
+static inline void v4l2_m2m_set_src_buffered(struct v4l2_m2m_ctx *m2m_ctx,
+					     bool buffered)
+{
+	m2m_ctx->out_q_ctx.buffered = buffered;
+}
+
+static inline void v4l2_m2m_set_dst_buffered(struct v4l2_m2m_ctx *m2m_ctx,
+					     bool buffered)
+{
+	m2m_ctx->cap_q_ctx.buffered = buffered;
+}
+
+int v4l2_m2m_ioctl_encoder_cmd(struct file *file, void *fh,
+			       struct v4l2_encoder_cmd *ec);
+int v4l2_m2m_ioctl_decoder_cmd(struct file *file, void *fh,
+			       struct v4l2_decoder_cmd *dc);
+int v4l2_m2m_ioctl_try_encoder_cmd(struct file *file, void *fh,
+				   struct v4l2_encoder_cmd *ec);
+int v4l2_m2m_ioctl_try_decoder_cmd(struct file *file, void *fh,
+				   struct v4l2_decoder_cmd *dc);
+int v4l2_m2m_ioctl_stateless_try_decoder_cmd(struct file *file, void *fh,
+					     struct v4l2_decoder_cmd *dc);
+int v4l2_m2m_ioctl_stateless_decoder_cmd(struct file *file, void *priv,
+					 struct v4l2_decoder_cmd *dc);
+
 #endif /* _MEDIA_V4L2_MEM2MEM_CODEC_H */
diff --git a/include/media/v4l2-mem2mem.h b/include/media/v4l2-mem2mem.h
index cebadec49666..f99703f4b43f 100644
--- a/include/media/v4l2-mem2mem.h
+++ b/include/media/v4l2-mem2mem.h
@@ -190,119 +190,12 @@ void v4l2_m2m_try_schedule(struct v4l2_m2m_ctx *m2m_ctx);
 void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
 			 struct v4l2_m2m_ctx *m2m_ctx);
 
-/**
- * v4l2_m2m_buf_done_and_job_finish() - return source/destination buffers with
- * state and inform the framework that a job has been finished and have it
- * clean up
- *
- * @m2m_dev: opaque pointer to the internal data to handle M2M context
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- * @state: vb2 buffer state passed to v4l2_m2m_buf_done().
- *
- * Drivers that set V4L2_BUF_CAP_SUPPORTS_M2M_HOLD_CAPTURE_BUF must use this
- * function instead of job_finish() to take held buffers into account. It is
- * optional for other drivers.
- *
- * This function removes the source buffer from the ready list and returns
- * it with the given state. The same is done for the destination buffer, unless
- * it is marked 'held'. In that case the buffer is kept on the ready list.
- *
- * After that the job is finished (see job_finish()).
- *
- * This allows for multiple output buffers to be used to fill in a single
- * capture buffer. This is typically used by stateless decoders where
- * multiple e.g. H.264 slices contribute to a single decoded frame.
- */
-void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
-				      struct v4l2_m2m_ctx *m2m_ctx,
-				      enum vb2_buffer_state state);
-
 static inline void
 v4l2_m2m_buf_done(struct vb2_v4l2_buffer *buf, enum vb2_buffer_state state)
 {
 	vb2_buffer_done(&buf->vb2_buf, state);
 }
 
-/**
- * v4l2_m2m_clear_state() - clear encoding/decoding state
- *
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- */
-static inline void
-v4l2_m2m_clear_state(struct v4l2_m2m_ctx *m2m_ctx)
-{
-	m2m_ctx->next_buf_last = false;
-	m2m_ctx->is_draining = false;
-	m2m_ctx->has_stopped = false;
-}
-
-/**
- * v4l2_m2m_mark_stopped() - set current encoding/decoding state as stopped
- *
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- */
-static inline void
-v4l2_m2m_mark_stopped(struct v4l2_m2m_ctx *m2m_ctx)
-{
-	m2m_ctx->next_buf_last = false;
-	m2m_ctx->is_draining = false;
-	m2m_ctx->has_stopped = true;
-}
-
-/**
- * v4l2_m2m_dst_buf_is_last() - return the current encoding/decoding session
- * draining management state of next queued capture buffer
- *
- * This last capture buffer should be tagged with V4L2_BUF_FLAG_LAST to notify
- * the end of the capture session.
- *
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- */
-static inline bool
-v4l2_m2m_dst_buf_is_last(struct v4l2_m2m_ctx *m2m_ctx)
-{
-	return m2m_ctx->is_draining && m2m_ctx->next_buf_last;
-}
-
-/**
- * v4l2_m2m_has_stopped() - return the current encoding/decoding session
- * stopped state
- *
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- */
-static inline bool
-v4l2_m2m_has_stopped(struct v4l2_m2m_ctx *m2m_ctx)
-{
-	return m2m_ctx->has_stopped;
-}
-
-/**
- * v4l2_m2m_is_last_draining_src_buf() - return the output buffer draining
- * state in the current encoding/decoding session
- *
- * This will identify the last output buffer queued before a session stop
- * was required, leading to an actual encoding/decoding session stop state
- * in the encoding/decoding process after being processed.
- *
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- * @vbuf: pointer to struct &v4l2_buffer
- */
-static inline bool
-v4l2_m2m_is_last_draining_src_buf(struct v4l2_m2m_ctx *m2m_ctx,
-				  struct vb2_v4l2_buffer *vbuf)
-{
-	return m2m_ctx->is_draining && vbuf == m2m_ctx->last_src_buf;
-}
-
-/**
- * v4l2_m2m_last_buffer_done() - marks the buffer with LAST flag and DONE
- *
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- * @vbuf: pointer to struct &v4l2_buffer
- */
-void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
-			       struct vb2_v4l2_buffer *vbuf);
-
 /**
  * v4l2_m2m_suspend() - stop new jobs from being run and wait for current job
  * to finish
@@ -422,46 +315,6 @@ int v4l2_m2m_streamon(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 int v4l2_m2m_streamoff(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
 		       enum v4l2_buf_type type);
 
-/**
- * v4l2_m2m_update_start_streaming_state() - update the encoding/decoding
- * session state when a start of streaming of a video queue is requested
- *
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- * @q: queue
- */
-void v4l2_m2m_update_start_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
-					   struct vb2_queue *q);
-
-/**
- * v4l2_m2m_update_stop_streaming_state() -  update the encoding/decoding
- * session state when a stop of streaming of a video queue is requested
- *
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- * @q: queue
- */
-void v4l2_m2m_update_stop_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
-					  struct vb2_queue *q);
-
-/**
- * v4l2_m2m_encoder_cmd() - execute an encoder command
- *
- * @file: pointer to struct &file
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- * @ec: pointer to the encoder command
- */
-int v4l2_m2m_encoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
-			 struct v4l2_encoder_cmd *ec);
-
-/**
- * v4l2_m2m_decoder_cmd() - execute a decoder command
- *
- * @file: pointer to struct &file
- * @m2m_ctx: m2m context assigned to the instance given by struct &v4l2_m2m_ctx
- * @dc: pointer to the decoder command
- */
-int v4l2_m2m_decoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
-			 struct v4l2_decoder_cmd *dc);
-
 /**
  * v4l2_m2m_poll() - poll replacement, for destination buffers only
  *
@@ -551,18 +404,6 @@ struct v4l2_m2m_ctx *v4l2_m2m_ctx_init(struct v4l2_m2m_dev *m2m_dev,
 		void *drv_priv,
 		int (*queue_init)(void *priv, struct vb2_queue *src_vq, struct vb2_queue *dst_vq));
 
-static inline void v4l2_m2m_set_src_buffered(struct v4l2_m2m_ctx *m2m_ctx,
-					     bool buffered)
-{
-	m2m_ctx->out_q_ctx.buffered = buffered;
-}
-
-static inline void v4l2_m2m_set_dst_buffered(struct v4l2_m2m_ctx *m2m_ctx,
-					     bool buffered)
-{
-	m2m_ctx->cap_q_ctx.buffered = buffered;
-}
-
 /**
  * v4l2_m2m_ctx_release() - release m2m context
  *
@@ -873,18 +714,6 @@ int v4l2_m2m_ioctl_streamon(struct file *file, void *fh,
 				enum v4l2_buf_type type);
 int v4l2_m2m_ioctl_streamoff(struct file *file, void *fh,
 				enum v4l2_buf_type type);
-int v4l2_m2m_ioctl_encoder_cmd(struct file *file, void *fh,
-			       struct v4l2_encoder_cmd *ec);
-int v4l2_m2m_ioctl_decoder_cmd(struct file *file, void *fh,
-			       struct v4l2_decoder_cmd *dc);
-int v4l2_m2m_ioctl_try_encoder_cmd(struct file *file, void *fh,
-				   struct v4l2_encoder_cmd *ec);
-int v4l2_m2m_ioctl_try_decoder_cmd(struct file *file, void *fh,
-				   struct v4l2_decoder_cmd *dc);
-int v4l2_m2m_ioctl_stateless_try_decoder_cmd(struct file *file, void *fh,
-					     struct v4l2_decoder_cmd *dc);
-int v4l2_m2m_ioctl_stateless_decoder_cmd(struct file *file, void *priv,
-					 struct v4l2_decoder_cmd *dc);
 int v4l2_m2m_fop_mmap(struct file *file, struct vm_area_struct *vma);
 __poll_t v4l2_m2m_fop_poll(struct file *file, poll_table *wait);
 
-- 
2.40.1

Re: [RFC PATCH 1/3] media: split up v4l2-mem2mem.c into a core and codec part

[Nicolas Dufresne]

Hi Hans,

thanks for the suggestion, I don't think CODEC is the right classification for
most of the moves here.

Le mercredi 23 août 2023 à 16:23 +0200, Hans Verkuil a écrit :
> The v4l2-mem2mem.c source contains core functions, valid for any m2m device,
> and codec functions that are needed for codec drivers only. Split up the
> source into a core source and a codec source to simplify maintenance.
> 
> Signed-off-by: Hans Verkuil <hverkuil-cisco@xs4all.nl>
> ---
>  drivers/media/v4l2-core/Makefile              |   2 +
>  drivers/media/v4l2-core/v4l2-mem2mem-codec.c  | 359 +++++++++++++++
>  .../{v4l2-mem2mem.c => v4l2-mem2mem-core.c}   | 422 +-----------------
>  drivers/media/v4l2-core/v4l2-mem2mem-priv.h   | 100 +++++
>  include/media/v4l2-mem2mem-codec.h            |  15 +
>  include/media/v4l2-mem2mem.h                  |  31 +-
>  6 files changed, 500 insertions(+), 429 deletions(-)
>  create mode 100644 drivers/media/v4l2-core/v4l2-mem2mem-codec.c
>  rename drivers/media/v4l2-core/{v4l2-mem2mem.c => v4l2-mem2mem-core.c} (72%)
>  create mode 100644 drivers/media/v4l2-core/v4l2-mem2mem-priv.h
>  create mode 100644 include/media/v4l2-mem2mem-codec.h
> 
> diff --git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile
> index be2551705755..ae718b8ab46e 100644
> --- a/drivers/media/v4l2-core/Makefile
> +++ b/drivers/media/v4l2-core/Makefile
> @@ -13,6 +13,8 @@ videodev-objs	:=	v4l2-dev.o v4l2-ioctl.o v4l2-device.o v4l2-fh.o \
>  			v4l2-ctrls-core.o v4l2-ctrls-api.o \
>  			v4l2-ctrls-request.o v4l2-ctrls-defs.o
>  
> +v4l2-mem2mem-objs	:=	v4l2-mem2mem-core.o v4l2-mem2mem-codec.o
> +
>  # Please keep it alphabetically sorted by Kconfig name
>  # (e. g. LC_ALL=C sort Makefile)
>  videodev-$(CONFIG_COMPAT) += v4l2-compat-ioctl32.o
> diff --git a/drivers/media/v4l2-core/v4l2-mem2mem-codec.c b/drivers/media/v4l2-core/v4l2-mem2mem-codec.c
> new file mode 100644
> index 000000000000..6e4b539a3b62
> --- /dev/null
> +++ b/drivers/media/v4l2-core/v4l2-mem2mem-codec.c
> @@ -0,0 +1,359 @@
> +// SPDX-License-Identifier: GPL-2.0-or-later
> +/*
> + * Memory-to-memory device framework for Video for Linux 2 and vb2.
> + *
> + * Helper functions for devices that use vb2 buffers for both their
> + * source and destination.
> + *
> + * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd.
> + * Pawel Osciak, <pawel@osciak.com>
> + * Marek Szyprowski, <m.szyprowski@samsung.com>
> + */
> +#include <linux/module.h>
> +#include <linux/sched.h>
> +#include <linux/slab.h>
> +
> +#include <media/media-device.h>
> +#include <media/videobuf2-v4l2.h>
> +#include <media/v4l2-mem2mem-codec.h>
> +#include <media/v4l2-dev.h>
> +#include <media/v4l2-device.h>
> +#include <media/v4l2-fh.h>
> +#include <media/v4l2-event.h>
> +
> +#include "v4l2-mem2mem-priv.h"
> +
> +void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
> +				      struct v4l2_m2m_ctx *m2m_ctx,
> +				      enum vb2_buffer_state state)

This is not CODEC specific but is a helper for request based drivers. Request
introduces complexity as buffers have to be ready by the time the request is
signalled (so that request can be used when to wait for multiple objects). It is
correct that request by one type of codecs (a quarter of codecs). Though, we
don't intend to prevent usage of requests in the future.

This function also depends on what Tomasz called "simple m2m" behaviour, meaning
that for each src buffer you can exactly one dst buffer with an explicit
DONE/ERROR state. A 1:1 fashion, so this helper would not be usable for stateful
codecs notably.

When I use the m2m, my confusion does not come from CODEC vs the rest of the
world, but between what is only usable for Tomasz "simple m2m".

> +{
> +	struct vb2_v4l2_buffer *src_buf, *dst_buf;
> +	bool schedule_next = false;
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
> +	src_buf = v4l2_m2m_src_buf_remove(m2m_ctx);
> +	dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
> +
> +	if (WARN_ON(!src_buf || !dst_buf))
> +		goto unlock;
> +	dst_buf->is_held = src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
> +	if (!dst_buf->is_held) {
> +		v4l2_m2m_dst_buf_remove(m2m_ctx);
> +		v4l2_m2m_buf_done(dst_buf, state);
> +	}
> +	/*
> +	 * If the request API is being used, returning the OUTPUT
> +	 * (src) buffer will wake-up any process waiting on the
> +	 * request file descriptor.
> +	 *
> +	 * Therefore, return the CAPTURE (dst) buffer first,
> +	 * to avoid signalling the request file descriptor
> +	 * before the CAPTURE buffer is done.
> +	 */
> +	v4l2_m2m_buf_done(src_buf, state);
> +	schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
> +unlock:
> +	spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
> +
> +	if (schedule_next)
> +		v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx);
> +}
> +EXPORT_SYMBOL(v4l2_m2m_buf_done_and_job_finish);
> +
> +/*
> + * This will add the LAST flag and mark the buffer management
> + * state as stopped.
> + * This is called when the last capture buffer must be flagged as LAST
> + * in draining mode from the encoder/decoder driver buf_queue() callback
> + * or from v4l2_update_last_buf_state() when a capture buffer is available.
> + */
> +void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
> +			       struct vb2_v4l2_buffer *vbuf)

As Tomasz stated today, only codec specification mandates CMD_STOP, but it can
be implement by any M2M driver. I don't think its justify to classify it as
CODEC only. Tomasz personal opinion is that it must not be implement unless
strictly needed (and he thinks its wrong for jpeg decoders to implement it,
though if you remove that be aware that you'll break Gst, so we have to keep
it). My opinion is opposite, that its preferable to always offer this API in
order to simplify / unify userspace implementation that actually uses the
performance benefit from vb queues. Personally I think the way is handled /
implement is just unfortunate and too much work for drivers.

It does not justify though classification as CODEC only.

> +{
> +	vbuf->flags |= V4L2_BUF_FLAG_LAST;
> +	vb2_buffer_done(&vbuf->vb2_buf, VB2_BUF_STATE_DONE);
> +
> +	v4l2_m2m_mark_stopped(m2m_ctx);
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_last_buffer_done);
> +
> +/* When stop command is issued, update buffer management state */
> +static int v4l2_update_last_buf_state(struct v4l2_m2m_ctx *m2m_ctx)
> +{
> +	struct vb2_v4l2_buffer *next_dst_buf;
> +
> +	if (m2m_ctx->is_draining)
> +		return -EBUSY;
> +
> +	if (m2m_ctx->has_stopped)
> +		return 0;
> +
> +	m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx);
> +	m2m_ctx->is_draining = true;
> +
> +	/*
> +	 * The processing of the last output buffer queued before
> +	 * the STOP command is expected to mark the buffer management
> +	 * state as stopped with v4l2_m2m_mark_stopped().
> +	 */
> +	if (m2m_ctx->last_src_buf)
> +		return 0;
> +
> +	/*
> +	 * In case the output queue is empty, try to mark the last capture
> +	 * buffer as LAST.
> +	 */
> +	next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
> +	if (!next_dst_buf) {
> +		/*
> +		 * Wait for the next queued one in encoder/decoder driver
> +		 * buf_queue() callback using the v4l2_m2m_dst_buf_is_last()
> +		 * helper or in v4l2_m2m_qbuf() if encoder/decoder is not yet
> +		 * streaming.
> +		 */
> +		m2m_ctx->next_buf_last = true;
> +		return 0;
> +	}
> +
> +	v4l2_m2m_last_buffer_done(m2m_ctx, next_dst_buf);
> +
> +	return 0;
> +}
> +
> +/*
> + * Updates the encoding/decoding buffer management state, should
> + * be called from encoder/decoder drivers start_streaming()
> + */
> +void v4l2_m2m_update_start_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
> +					   struct vb2_queue *q)
> +{
> +	/* If start streaming again, untag the last output buffer */
> +	if (V4L2_TYPE_IS_OUTPUT(q->type))
> +		m2m_ctx->last_src_buf = NULL;
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_update_start_streaming_state);
> +
> +/*
> + * Updates the encoding/decoding buffer management state, should
> + * be called from encoder/decoder driver stop_streaming()
> + */
> +void v4l2_m2m_update_stop_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
> +					  struct vb2_queue *q)
> +{
> +	if (V4L2_TYPE_IS_OUTPUT(q->type)) {
> +		/*
> +		 * If in draining state, either mark next dst buffer as
> +		 * done or flag next one to be marked as done either
> +		 * in encoder/decoder driver buf_queue() callback using
> +		 * the v4l2_m2m_dst_buf_is_last() helper or in v4l2_m2m_qbuf()
> +		 * if encoder/decoder is not yet streaming
> +		 */
> +		if (m2m_ctx->is_draining) {
> +			struct vb2_v4l2_buffer *next_dst_buf;
> +
> +			m2m_ctx->last_src_buf = NULL;
> +			next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
> +			if (!next_dst_buf)
> +				m2m_ctx->next_buf_last = true;
> +			else
> +				v4l2_m2m_last_buffer_done(m2m_ctx,
> +							  next_dst_buf);
> +		}
> +	} else {
> +		v4l2_m2m_clear_state(m2m_ctx);
> +	}
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_update_stop_streaming_state);
> +
> +static void v4l2_m2m_force_last_buf_done(struct v4l2_m2m_ctx *m2m_ctx,
> +					 struct vb2_queue *q)
> +{
> +	struct vb2_buffer *vb;
> +	struct vb2_v4l2_buffer *vbuf;
> +	unsigned int i;
> +
> +	if (WARN_ON(q->is_output))
> +		return;
> +	if (list_empty(&q->queued_list))
> +		return;
> +
> +	vb = list_first_entry(&q->queued_list, struct vb2_buffer, queued_entry);
> +	for (i = 0; i < vb->num_planes; i++)
> +		vb2_set_plane_payload(vb, i, 0);
> +
> +	/*
> +	 * Since the buffer hasn't been queued to the ready queue,
> +	 * mark is active and owned before marking it LAST and DONE
> +	 */
> +	vb->state = VB2_BUF_STATE_ACTIVE;
> +	atomic_inc(&q->owned_by_drv_count);
> +
> +	vbuf = to_vb2_v4l2_buffer(vb);
> +	vbuf->field = V4L2_FIELD_NONE;
> +
> +	v4l2_m2m_last_buffer_done(m2m_ctx, vbuf);
> +}
> +
> +void _v4l2_codec_qbuf(struct v4l2_m2m_ctx *m2m_ctx, struct vb2_queue *vq,
> +		      struct v4l2_buffer *buf)
> +{
> +	/*
> +	 * If the capture queue is streaming, but streaming hasn't started
> +	 * on the device, but was asked to stop, mark the previously queued
> +	 * buffer as DONE with LAST flag since it won't be queued on the
> +	 * device.
> +	 */
> +	if (V4L2_TYPE_IS_CAPTURE(vq->type) &&
> +	    vb2_is_streaming(vq) && !vb2_start_streaming_called(vq) &&
> +	   (v4l2_m2m_has_stopped(m2m_ctx) || v4l2_m2m_dst_buf_is_last(m2m_ctx)))
> +		v4l2_m2m_force_last_buf_done(m2m_ctx, vq);
> +	else if (!(buf->flags & V4L2_BUF_FLAG_IN_REQUEST))
> +		v4l2_m2m_try_schedule(m2m_ctx);
> +}
> +
> +int v4l2_m2m_ioctl_try_encoder_cmd(struct file *file, void *fh,
> +				   struct v4l2_encoder_cmd *ec)
> +{
> +	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
> +		return -EINVAL;
> +
> +	ec->flags = 0;
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_encoder_cmd);
> +
> +int v4l2_m2m_ioctl_try_decoder_cmd(struct file *file, void *fh,
> +				   struct v4l2_decoder_cmd *dc)
> +{
> +	if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
> +		return -EINVAL;
> +
> +	dc->flags = 0;
> +
> +	if (dc->cmd == V4L2_DEC_CMD_STOP) {
> +		dc->stop.pts = 0;
> +	} else if (dc->cmd == V4L2_DEC_CMD_START) {
> +		dc->start.speed = 0;
> +		dc->start.format = V4L2_DEC_START_FMT_NONE;
> +	}
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_decoder_cmd);
> +
> +/*
> + * Updates the encoding state on ENC_CMD_STOP/ENC_CMD_START
> + * Should be called from the encoder driver encoder_cmd() callback
> + */
> +int v4l2_m2m_encoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
> +			 struct v4l2_encoder_cmd *ec)
> +{
> +	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
> +		return -EINVAL;
> +
> +	if (ec->cmd == V4L2_ENC_CMD_STOP)
> +		return v4l2_update_last_buf_state(m2m_ctx);
> +
> +	if (m2m_ctx->is_draining)
> +		return -EBUSY;
> +
> +	if (m2m_ctx->has_stopped)
> +		m2m_ctx->has_stopped = false;
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_encoder_cmd);
> +
> +/*
> + * Updates the decoding state on DEC_CMD_STOP/DEC_CMD_START
> + * Should be called from the decoder driver decoder_cmd() callback
> + */
> +int v4l2_m2m_decoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
> +			 struct v4l2_decoder_cmd *dc)
> +{
> +	if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
> +		return -EINVAL;
> +
> +	if (dc->cmd == V4L2_DEC_CMD_STOP)
> +		return v4l2_update_last_buf_state(m2m_ctx);
> +
> +	if (m2m_ctx->is_draining)
> +		return -EBUSY;
> +
> +	if (m2m_ctx->has_stopped)
> +		m2m_ctx->has_stopped = false;
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_decoder_cmd);
> +
> +int v4l2_m2m_ioctl_encoder_cmd(struct file *file, void *priv,
> +			       struct v4l2_encoder_cmd *ec)
> +{
> +	struct v4l2_fh *fh = file->private_data;
> +
> +	return v4l2_m2m_encoder_cmd(file, fh->m2m_ctx, ec);
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_encoder_cmd);
> +
> +int v4l2_m2m_ioctl_decoder_cmd(struct file *file, void *priv,
> +			       struct v4l2_decoder_cmd *dc)
> +{
> +	struct v4l2_fh *fh = file->private_data;
> +
> +	return v4l2_m2m_decoder_cmd(file, fh->m2m_ctx, dc);
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_decoder_cmd);
> +
> +int v4l2_m2m_ioctl_stateless_try_decoder_cmd(struct file *file, void *fh,
> +					     struct v4l2_decoder_cmd *dc)
> +{
> +	if (dc->cmd != V4L2_DEC_CMD_FLUSH)
> +		return -EINVAL;
> +
> +	dc->flags = 0;
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_try_decoder_cmd);
> +
> +int v4l2_m2m_ioctl_stateless_decoder_cmd(struct file *file, void *priv,
> +					 struct v4l2_decoder_cmd *dc)
> +{
> +	struct v4l2_fh *fh = file->private_data;
> +	struct vb2_v4l2_buffer *out_vb, *cap_vb;
> +	struct v4l2_m2m_dev *m2m_dev = fh->m2m_ctx->m2m_dev;
> +	unsigned long flags;
> +	int ret;
> +
> +	ret = v4l2_m2m_ioctl_stateless_try_decoder_cmd(file, priv, dc);
> +	if (ret < 0)
> +		return ret;
> +
> +	spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
> +	out_vb = v4l2_m2m_last_src_buf(fh->m2m_ctx);
> +	cap_vb = v4l2_m2m_last_dst_buf(fh->m2m_ctx);
> +
> +	/*
> +	 * If there is an out buffer pending, then clear any HOLD flag.
> +	 *
> +	 * By clearing this flag we ensure that when this output
> +	 * buffer is processed any held capture buffer will be released.
> +	 */
> +	if (out_vb) {
> +		out_vb->flags &= ~V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
> +	} else if (cap_vb && cap_vb->is_held) {
> +		/*
> +		 * If there were no output buffers, but there is a
> +		 * capture buffer that is held, then release that
> +		 * buffer.
> +		 */
> +		cap_vb->is_held = false;
> +		v4l2_m2m_dst_buf_remove(fh->m2m_ctx);
> +		v4l2_m2m_buf_done(cap_vb, VB2_BUF_STATE_DONE);
> +	}
> +	spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
> +
> +	return 0;
> +}
> +EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_decoder_cmd);
> diff --git a/drivers/media/v4l2-core/v4l2-mem2mem.c b/drivers/media/v4l2-core/v4l2-mem2mem-core.c
> similarity index 72%
> rename from drivers/media/v4l2-core/v4l2-mem2mem.c
> rename to drivers/media/v4l2-core/v4l2-mem2mem-core.c
> index 0cc30397fbad..8963a5b29b1d 100644
> --- a/drivers/media/v4l2-core/v4l2-mem2mem.c
> +++ b/drivers/media/v4l2-core/v4l2-mem2mem-core.c
> @@ -21,42 +21,14 @@
>  #include <media/v4l2-fh.h>
>  #include <media/v4l2-event.h>
>  
> +#include "v4l2-mem2mem-priv.h"
> +
>  MODULE_DESCRIPTION("Mem to mem device framework for vb2");
>  MODULE_AUTHOR("Pawel Osciak, <pawel@osciak.com>");
>  MODULE_LICENSE("GPL");
>  
> -static bool debug;
> -module_param(debug, bool, 0644);
> -
> -#define dprintk(fmt, arg...)						\
> -	do {								\
> -		if (debug)						\
> -			printk(KERN_DEBUG "%s: " fmt, __func__, ## arg);\
> -	} while (0)
> -
> -
> -/* Instance is already queued on the job_queue */
> -#define TRANS_QUEUED		(1 << 0)
> -/* Instance is currently running in hardware */
> -#define TRANS_RUNNING		(1 << 1)
> -/* Instance is currently aborting */
> -#define TRANS_ABORT		(1 << 2)
> -
> -
> -/* The job queue is not running new jobs */
> -#define QUEUE_PAUSED		(1 << 0)
> -
> -
> -/* Offset base for buffers on the destination queue - used to distinguish
> - * between source and destination buffers when mmapping - they receive the same
> - * offsets but for different queues */
> -#define DST_QUEUE_OFF_BASE	(1 << 30)
> -
> -enum v4l2_m2m_entity_type {
> -	MEM2MEM_ENT_TYPE_SOURCE,
> -	MEM2MEM_ENT_TYPE_SINK,
> -	MEM2MEM_ENT_TYPE_PROC
> -};
> +bool v4l2_mem2mem_debug;
> +module_param_named(debug, v4l2_mem2mem_debug, bool, 0644);
>  
>  static const char * const m2m_entity_name[] = {
>  	"source",
> @@ -64,53 +36,6 @@ static const char * const m2m_entity_name[] = {
>  	"proc"
>  };
>  
> -/**
> - * struct v4l2_m2m_dev - per-device context
> - * @source:		&struct media_entity pointer with the source entity
> - *			Used only when the M2M device is registered via
> - *			v4l2_m2m_register_media_controller().
> - * @source_pad:		&struct media_pad with the source pad.
> - *			Used only when the M2M device is registered via
> - *			v4l2_m2m_register_media_controller().
> - * @sink:		&struct media_entity pointer with the sink entity
> - *			Used only when the M2M device is registered via
> - *			v4l2_m2m_register_media_controller().
> - * @sink_pad:		&struct media_pad with the sink pad.
> - *			Used only when the M2M device is registered via
> - *			v4l2_m2m_register_media_controller().
> - * @proc:		&struct media_entity pointer with the M2M device itself.
> - * @proc_pads:		&struct media_pad with the @proc pads.
> - *			Used only when the M2M device is registered via
> - *			v4l2_m2m_unregister_media_controller().
> - * @intf_devnode:	&struct media_intf devnode pointer with the interface
> - *			with controls the M2M device.
> - * @curr_ctx:		currently running instance
> - * @job_queue:		instances queued to run
> - * @job_spinlock:	protects job_queue
> - * @job_work:		worker to run queued jobs.
> - * @job_queue_flags:	flags of the queue status, %QUEUE_PAUSED.
> - * @m2m_ops:		driver callbacks
> - */
> -struct v4l2_m2m_dev {
> -	struct v4l2_m2m_ctx	*curr_ctx;
> -#ifdef CONFIG_MEDIA_CONTROLLER
> -	struct media_entity	*source;
> -	struct media_pad	source_pad;
> -	struct media_entity	sink;
> -	struct media_pad	sink_pad;
> -	struct media_entity	proc;
> -	struct media_pad	proc_pads[2];
> -	struct media_intf_devnode *intf_devnode;
> -#endif
> -
> -	struct list_head	job_queue;
> -	spinlock_t		job_spinlock;
> -	struct work_struct	job_work;
> -	unsigned long		job_queue_flags;
> -
> -	const struct v4l2_m2m_ops *m2m_ops;
> -};
> -
>  static struct v4l2_m2m_queue_ctx *get_queue_ctx(struct v4l2_m2m_ctx *m2m_ctx,
>  						enum v4l2_buf_type type)
>  {
> @@ -445,8 +370,8 @@ static void v4l2_m2m_cancel_job(struct v4l2_m2m_ctx *m2m_ctx)
>   * Schedule the next job, called from v4l2_m2m_job_finish() or
>   * v4l2_m2m_buf_done_and_job_finish().
>   */
> -static void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
> -				       struct v4l2_m2m_ctx *m2m_ctx)
> +void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
> +				struct v4l2_m2m_ctx *m2m_ctx)
>  {
>  	/*
>  	 * This instance might have more buffers ready, but since we do not
> @@ -466,8 +391,8 @@ static void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
>   * Assumes job_spinlock is held, called from v4l2_m2m_job_finish() or
>   * v4l2_m2m_buf_done_and_job_finish().
>   */
> -static bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
> -				 struct v4l2_m2m_ctx *m2m_ctx)
> +bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
> +			  struct v4l2_m2m_ctx *m2m_ctx)
>  {
>  	if (!m2m_dev->curr_ctx || m2m_dev->curr_ctx != m2m_ctx) {
>  		dprintk("Called by an instance not currently running\n");
> @@ -503,44 +428,6 @@ void v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
>  }
>  EXPORT_SYMBOL(v4l2_m2m_job_finish);
>  
> -void v4l2_m2m_buf_done_and_job_finish(struct v4l2_m2m_dev *m2m_dev,
> -				      struct v4l2_m2m_ctx *m2m_ctx,
> -				      enum vb2_buffer_state state)
> -{
> -	struct vb2_v4l2_buffer *src_buf, *dst_buf;
> -	bool schedule_next = false;
> -	unsigned long flags;
> -
> -	spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
> -	src_buf = v4l2_m2m_src_buf_remove(m2m_ctx);
> -	dst_buf = v4l2_m2m_next_dst_buf(m2m_ctx);
> -
> -	if (WARN_ON(!src_buf || !dst_buf))
> -		goto unlock;
> -	dst_buf->is_held = src_buf->flags & V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
> -	if (!dst_buf->is_held) {
> -		v4l2_m2m_dst_buf_remove(m2m_ctx);
> -		v4l2_m2m_buf_done(dst_buf, state);
> -	}
> -	/*
> -	 * If the request API is being used, returning the OUTPUT
> -	 * (src) buffer will wake-up any process waiting on the
> -	 * request file descriptor.
> -	 *
> -	 * Therefore, return the CAPTURE (dst) buffer first,
> -	 * to avoid signalling the request file descriptor
> -	 * before the CAPTURE buffer is done.
> -	 */
> -	v4l2_m2m_buf_done(src_buf, state);
> -	schedule_next = _v4l2_m2m_job_finish(m2m_dev, m2m_ctx);
> -unlock:
> -	spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
> -
> -	if (schedule_next)
> -		v4l2_m2m_schedule_next_job(m2m_dev, m2m_ctx);
> -}
> -EXPORT_SYMBOL(v4l2_m2m_buf_done_and_job_finish);
> -
>  void v4l2_m2m_suspend(struct v4l2_m2m_dev *m2m_dev)
>  {
>  	unsigned long flags;
> @@ -621,140 +508,6 @@ int v4l2_m2m_querybuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
>  }
>  EXPORT_SYMBOL_GPL(v4l2_m2m_querybuf);
>  
> -/*
> - * This will add the LAST flag and mark the buffer management
> - * state as stopped.
> - * This is called when the last capture buffer must be flagged as LAST
> - * in draining mode from the encoder/decoder driver buf_queue() callback
> - * or from v4l2_update_last_buf_state() when a capture buffer is available.
> - */
> -void v4l2_m2m_last_buffer_done(struct v4l2_m2m_ctx *m2m_ctx,
> -			       struct vb2_v4l2_buffer *vbuf)
> -{
> -	vbuf->flags |= V4L2_BUF_FLAG_LAST;
> -	vb2_buffer_done(&vbuf->vb2_buf, VB2_BUF_STATE_DONE);
> -
> -	v4l2_m2m_mark_stopped(m2m_ctx);
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_last_buffer_done);
> -
> -/* When stop command is issued, update buffer management state */
> -static int v4l2_update_last_buf_state(struct v4l2_m2m_ctx *m2m_ctx)
> -{
> -	struct vb2_v4l2_buffer *next_dst_buf;
> -
> -	if (m2m_ctx->is_draining)
> -		return -EBUSY;
> -
> -	if (m2m_ctx->has_stopped)
> -		return 0;
> -
> -	m2m_ctx->last_src_buf = v4l2_m2m_last_src_buf(m2m_ctx);
> -	m2m_ctx->is_draining = true;
> -
> -	/*
> -	 * The processing of the last output buffer queued before
> -	 * the STOP command is expected to mark the buffer management
> -	 * state as stopped with v4l2_m2m_mark_stopped().
> -	 */
> -	if (m2m_ctx->last_src_buf)
> -		return 0;
> -
> -	/*
> -	 * In case the output queue is empty, try to mark the last capture
> -	 * buffer as LAST.
> -	 */
> -	next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
> -	if (!next_dst_buf) {
> -		/*
> -		 * Wait for the next queued one in encoder/decoder driver
> -		 * buf_queue() callback using the v4l2_m2m_dst_buf_is_last()
> -		 * helper or in v4l2_m2m_qbuf() if encoder/decoder is not yet
> -		 * streaming.
> -		 */
> -		m2m_ctx->next_buf_last = true;
> -		return 0;
> -	}
> -
> -	v4l2_m2m_last_buffer_done(m2m_ctx, next_dst_buf);
> -
> -	return 0;
> -}
> -
> -/*
> - * Updates the encoding/decoding buffer management state, should
> - * be called from encoder/decoder drivers start_streaming()
> - */
> -void v4l2_m2m_update_start_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
> -					   struct vb2_queue *q)
> -{
> -	/* If start streaming again, untag the last output buffer */
> -	if (V4L2_TYPE_IS_OUTPUT(q->type))
> -		m2m_ctx->last_src_buf = NULL;
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_update_start_streaming_state);
> -
> -/*
> - * Updates the encoding/decoding buffer management state, should
> - * be called from encoder/decoder driver stop_streaming()
> - */
> -void v4l2_m2m_update_stop_streaming_state(struct v4l2_m2m_ctx *m2m_ctx,
> -					  struct vb2_queue *q)
> -{
> -	if (V4L2_TYPE_IS_OUTPUT(q->type)) {
> -		/*
> -		 * If in draining state, either mark next dst buffer as
> -		 * done or flag next one to be marked as done either
> -		 * in encoder/decoder driver buf_queue() callback using
> -		 * the v4l2_m2m_dst_buf_is_last() helper or in v4l2_m2m_qbuf()
> -		 * if encoder/decoder is not yet streaming
> -		 */
> -		if (m2m_ctx->is_draining) {
> -			struct vb2_v4l2_buffer *next_dst_buf;
> -
> -			m2m_ctx->last_src_buf = NULL;
> -			next_dst_buf = v4l2_m2m_dst_buf_remove(m2m_ctx);
> -			if (!next_dst_buf)
> -				m2m_ctx->next_buf_last = true;
> -			else
> -				v4l2_m2m_last_buffer_done(m2m_ctx,
> -							  next_dst_buf);
> -		}
> -	} else {
> -		v4l2_m2m_clear_state(m2m_ctx);
> -	}
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_update_stop_streaming_state);
> -
> -static void v4l2_m2m_force_last_buf_done(struct v4l2_m2m_ctx *m2m_ctx,
> -					 struct vb2_queue *q)
> -{
> -	struct vb2_buffer *vb;
> -	struct vb2_v4l2_buffer *vbuf;
> -	unsigned int i;
> -
> -	if (WARN_ON(q->is_output))
> -		return;
> -	if (list_empty(&q->queued_list))
> -		return;
> -
> -	vb = list_first_entry(&q->queued_list, struct vb2_buffer, queued_entry);
> -	for (i = 0; i < vb->num_planes; i++)
> -		vb2_set_plane_payload(vb, i, 0);
> -
> -	/*
> -	 * Since the buffer hasn't been queued to the ready queue,
> -	 * mark is active and owned before marking it LAST and DONE
> -	 */
> -	vb->state = VB2_BUF_STATE_ACTIVE;
> -	atomic_inc(&q->owned_by_drv_count);
> -
> -	vbuf = to_vb2_v4l2_buffer(vb);
> -	vbuf->field = V4L2_FIELD_NONE;
> -
> -	v4l2_m2m_last_buffer_done(m2m_ctx, vbuf);
> -}
> -
>  int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
>  		  struct v4l2_buffer *buf)
>  {
> @@ -777,18 +530,7 @@ int v4l2_m2m_qbuf(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
>  	/* Adjust MMAP memory offsets for the CAPTURE queue */
>  	v4l2_m2m_adjust_mem_offset(vq, buf);
>  
> -	/*
> -	 * If the capture queue is streaming, but streaming hasn't started
> -	 * on the device, but was asked to stop, mark the previously queued
> -	 * buffer as DONE with LAST flag since it won't be queued on the
> -	 * device.
> -	 */
> -	if (V4L2_TYPE_IS_CAPTURE(vq->type) &&
> -	    vb2_is_streaming(vq) && !vb2_start_streaming_called(vq) &&
> -	   (v4l2_m2m_has_stopped(m2m_ctx) || v4l2_m2m_dst_buf_is_last(m2m_ctx)))
> -		v4l2_m2m_force_last_buf_done(m2m_ctx, vq);
> -	else if (!(buf->flags & V4L2_BUF_FLAG_IN_REQUEST))
> -		v4l2_m2m_try_schedule(m2m_ctx);
> +	_v4l2_codec_qbuf(m2m_ctx, vq, buf);
>  
>  	return 0;
>  }
> @@ -1440,152 +1182,6 @@ int v4l2_m2m_ioctl_streamoff(struct file *file, void *priv,
>  }
>  EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_streamoff);
>  
> -int v4l2_m2m_ioctl_try_encoder_cmd(struct file *file, void *fh,
> -				   struct v4l2_encoder_cmd *ec)
> -{
> -	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
> -		return -EINVAL;
> -
> -	ec->flags = 0;
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_encoder_cmd);
> -
> -int v4l2_m2m_ioctl_try_decoder_cmd(struct file *file, void *fh,
> -				   struct v4l2_decoder_cmd *dc)
> -{
> -	if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
> -		return -EINVAL;
> -
> -	dc->flags = 0;
> -
> -	if (dc->cmd == V4L2_DEC_CMD_STOP) {
> -		dc->stop.pts = 0;
> -	} else if (dc->cmd == V4L2_DEC_CMD_START) {
> -		dc->start.speed = 0;
> -		dc->start.format = V4L2_DEC_START_FMT_NONE;
> -	}
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_try_decoder_cmd);
> -
> -/*
> - * Updates the encoding state on ENC_CMD_STOP/ENC_CMD_START
> - * Should be called from the encoder driver encoder_cmd() callback
> - */
> -int v4l2_m2m_encoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
> -			 struct v4l2_encoder_cmd *ec)
> -{
> -	if (ec->cmd != V4L2_ENC_CMD_STOP && ec->cmd != V4L2_ENC_CMD_START)
> -		return -EINVAL;
> -
> -	if (ec->cmd == V4L2_ENC_CMD_STOP)
> -		return v4l2_update_last_buf_state(m2m_ctx);
> -
> -	if (m2m_ctx->is_draining)
> -		return -EBUSY;
> -
> -	if (m2m_ctx->has_stopped)
> -		m2m_ctx->has_stopped = false;
> -
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_encoder_cmd);
> -
> -/*
> - * Updates the decoding state on DEC_CMD_STOP/DEC_CMD_START
> - * Should be called from the decoder driver decoder_cmd() callback
> - */
> -int v4l2_m2m_decoder_cmd(struct file *file, struct v4l2_m2m_ctx *m2m_ctx,
> -			 struct v4l2_decoder_cmd *dc)
> -{
> -	if (dc->cmd != V4L2_DEC_CMD_STOP && dc->cmd != V4L2_DEC_CMD_START)
> -		return -EINVAL;
> -
> -	if (dc->cmd == V4L2_DEC_CMD_STOP)
> -		return v4l2_update_last_buf_state(m2m_ctx);
> -
> -	if (m2m_ctx->is_draining)
> -		return -EBUSY;
> -
> -	if (m2m_ctx->has_stopped)
> -		m2m_ctx->has_stopped = false;
> -
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_decoder_cmd);
> -
> -int v4l2_m2m_ioctl_encoder_cmd(struct file *file, void *priv,
> -			       struct v4l2_encoder_cmd *ec)
> -{
> -	struct v4l2_fh *fh = file->private_data;
> -
> -	return v4l2_m2m_encoder_cmd(file, fh->m2m_ctx, ec);
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_encoder_cmd);
> -
> -int v4l2_m2m_ioctl_decoder_cmd(struct file *file, void *priv,
> -			       struct v4l2_decoder_cmd *dc)
> -{
> -	struct v4l2_fh *fh = file->private_data;
> -
> -	return v4l2_m2m_decoder_cmd(file, fh->m2m_ctx, dc);
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_decoder_cmd);
> -
> -int v4l2_m2m_ioctl_stateless_try_decoder_cmd(struct file *file, void *fh,
> -					     struct v4l2_decoder_cmd *dc)
> -{
> -	if (dc->cmd != V4L2_DEC_CMD_FLUSH)
> -		return -EINVAL;
> -
> -	dc->flags = 0;
> -
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_try_decoder_cmd);
> -
> -int v4l2_m2m_ioctl_stateless_decoder_cmd(struct file *file, void *priv,
> -					 struct v4l2_decoder_cmd *dc)
> -{
> -	struct v4l2_fh *fh = file->private_data;
> -	struct vb2_v4l2_buffer *out_vb, *cap_vb;
> -	struct v4l2_m2m_dev *m2m_dev = fh->m2m_ctx->m2m_dev;
> -	unsigned long flags;
> -	int ret;
> -
> -	ret = v4l2_m2m_ioctl_stateless_try_decoder_cmd(file, priv, dc);
> -	if (ret < 0)
> -		return ret;
> -
> -	spin_lock_irqsave(&m2m_dev->job_spinlock, flags);
> -	out_vb = v4l2_m2m_last_src_buf(fh->m2m_ctx);
> -	cap_vb = v4l2_m2m_last_dst_buf(fh->m2m_ctx);
> -
> -	/*
> -	 * If there is an out buffer pending, then clear any HOLD flag.
> -	 *
> -	 * By clearing this flag we ensure that when this output
> -	 * buffer is processed any held capture buffer will be released.
> -	 */
> -	if (out_vb) {
> -		out_vb->flags &= ~V4L2_BUF_FLAG_M2M_HOLD_CAPTURE_BUF;
> -	} else if (cap_vb && cap_vb->is_held) {
> -		/*
> -		 * If there were no output buffers, but there is a
> -		 * capture buffer that is held, then release that
> -		 * buffer.
> -		 */
> -		cap_vb->is_held = false;
> -		v4l2_m2m_dst_buf_remove(fh->m2m_ctx);
> -		v4l2_m2m_buf_done(cap_vb, VB2_BUF_STATE_DONE);
> -	}
> -	spin_unlock_irqrestore(&m2m_dev->job_spinlock, flags);
> -
> -	return 0;
> -}
> -EXPORT_SYMBOL_GPL(v4l2_m2m_ioctl_stateless_decoder_cmd);
> -
>  /*
>   * v4l2_file_operations helpers. It is assumed here same lock is used
>   * for the output and the capture buffer queue.
> diff --git a/drivers/media/v4l2-core/v4l2-mem2mem-priv.h b/drivers/media/v4l2-core/v4l2-mem2mem-priv.h
> new file mode 100644
> index 000000000000..65ac006808df
> --- /dev/null
> +++ b/drivers/media/v4l2-core/v4l2-mem2mem-priv.h
> @@ -0,0 +1,100 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * Memory-to-memory device framework, private header
> + *
> + * Copyright (c) 2009 Samsung Electronics Co., Ltd.
> + * Pawel Osciak, <pawel@osciak.com>
> + * Marek Szyprowski, <m.szyprowski@samsung.com>
> + */
> +
> +#ifndef _MEDIA_V4L2_MEM2MEM_PRIV_H
> +#define _MEDIA_V4L2_MEM2MEM_PRIV_H
> +
> +extern bool v4l2_mem2mem_debug;
> +
> +#define dprintk(fmt, arg...)						\
> +	do {								\
> +		if (v4l2_mem2mem_debug)					\
> +			printk(KERN_DEBUG "%s: " fmt, __func__, ## arg);\
> +	} while (0)
> +
> +
> +/* Instance is already queued on the job_queue */
> +#define TRANS_QUEUED		(1 << 0)
> +/* Instance is currently running in hardware */
> +#define TRANS_RUNNING		(1 << 1)
> +/* Instance is currently aborting */
> +#define TRANS_ABORT		(1 << 2)
> +
> +
> +/* The job queue is not running new jobs */
> +#define QUEUE_PAUSED		(1 << 0)
> +
> +
> +/* Offset base for buffers on the destination queue - used to distinguish
> + * between source and destination buffers when mmapping - they receive the same
> + * offsets but for different queues */
> +#define DST_QUEUE_OFF_BASE	(1 << 30)
> +
> +enum v4l2_m2m_entity_type {
> +	MEM2MEM_ENT_TYPE_SOURCE,
> +	MEM2MEM_ENT_TYPE_SINK,
> +	MEM2MEM_ENT_TYPE_PROC
> +};
> +
> +/**
> + * struct v4l2_m2m_dev - per-device context
> + * @source:		&struct media_entity pointer with the source entity
> + *			Used only when the M2M device is registered via
> + *			v4l2_m2m_register_media_controller().
> + * @source_pad:		&struct media_pad with the source pad.
> + *			Used only when the M2M device is registered via
> + *			v4l2_m2m_register_media_controller().
> + * @sink:		&struct media_entity pointer with the sink entity
> + *			Used only when the M2M device is registered via
> + *			v4l2_m2m_register_media_controller().
> + * @sink_pad:		&struct media_pad with the sink pad.
> + *			Used only when the M2M device is registered via
> + *			v4l2_m2m_register_media_controller().
> + * @proc:		&struct media_entity pointer with the M2M device itself.
> + * @proc_pads:		&struct media_pad with the @proc pads.
> + *			Used only when the M2M device is registered via
> + *			v4l2_m2m_unregister_media_controller().
> + * @intf_devnode:	&struct media_intf devnode pointer with the interface
> + *			with controls the M2M device.
> + * @curr_ctx:		currently running instance
> + * @job_queue:		instances queued to run
> + * @job_spinlock:	protects job_queue
> + * @job_work:		worker to run queued jobs.
> + * @job_queue_flags:	flags of the queue status, %QUEUE_PAUSED.
> + * @m2m_ops:		driver callbacks
> + */
> +struct v4l2_m2m_dev {
> +	struct v4l2_m2m_ctx	*curr_ctx;
> +#ifdef CONFIG_MEDIA_CONTROLLER
> +	struct media_entity	*source;
> +	struct media_pad	source_pad;
> +	struct media_entity	sink;
> +	struct media_pad	sink_pad;
> +	struct media_entity	proc;
> +	struct media_pad	proc_pads[2];
> +	struct media_intf_devnode *intf_devnode;
> +#endif
> +
> +	struct list_head	job_queue;
> +	spinlock_t		job_spinlock;
> +	struct work_struct	job_work;
> +	unsigned long		job_queue_flags;
> +
> +	const struct v4l2_m2m_ops *m2m_ops;
> +};
> +
> +bool _v4l2_m2m_job_finish(struct v4l2_m2m_dev *m2m_dev,
> +			  struct v4l2_m2m_ctx *m2m_ctx);
> +void v4l2_m2m_schedule_next_job(struct v4l2_m2m_dev *m2m_dev,
> +				struct v4l2_m2m_ctx *m2m_ctx);
> +
> +void _v4l2_codec_qbuf(struct v4l2_m2m_ctx *m2m_ctx, struct vb2_queue *vq,
> +		      struct v4l2_buffer *buf);
> +
> +#endif /* _MEDIA_V4L2_MEM2MEM_PRIV_H */
> diff --git a/include/media/v4l2-mem2mem-codec.h b/include/media/v4l2-mem2mem-codec.h
> new file mode 100644
> index 000000000000..6e3d75fd4017
> --- /dev/null
> +++ b/include/media/v4l2-mem2mem-codec.h
> @@ -0,0 +1,15 @@
> +/* SPDX-License-Identifier: GPL-2.0-or-later */
> +/*
> + * Memory-to-memory device framework, codec specific part.
> + *
> + * Copyright (c) 2009 Samsung Electronics Co., Ltd.
> + * Pawel Osciak, <pawel@osciak.com>
> + * Marek Szyprowski, <m.szyprowski@samsung.com>
> + */
> +
> +#ifndef _MEDIA_V4L2_MEM2MEM_CODEC_H
> +#define _MEDIA_V4L2_MEM2MEM_CODEC_H
> +
> +#include <media/v4l2-mem2mem.h>
> +
> +#endif /* _MEDIA_V4L2_MEM2MEM_CODEC_H */
> diff --git a/include/media/v4l2-mem2mem.h b/include/media/v4l2-mem2mem.h
> index d6c8eb2b5201..cebadec49666 100644
> --- a/include/media/v4l2-mem2mem.h
> +++ b/include/media/v4l2-mem2mem.h
> @@ -75,6 +75,14 @@ struct v4l2_m2m_queue_ctx {
>   * struct v4l2_m2m_ctx - Memory to memory context structure
>   *
>   * @q_lock: struct &mutex lock
> + * @m2m_dev: opaque pointer to the internal data to handle M2M context
> + * @cap_q_ctx: Capture (output to memory) queue context
> + * @out_q_ctx: Output (input from memory) queue context
> + * @queue: List of memory to memory contexts
> + * @job_flags: Job queue flags, used internally by v4l2-mem2mem.c:
> + *		%TRANS_QUEUED, %TRANS_RUNNING and %TRANS_ABORT.
> + * @finished: Wait queue used to signalize when a job queue finished.
> + * @priv: Instance private data
>   * @new_frame: valid in the device_run callback: if true, then this
>   *		starts a new frame; if false, then this is a new slice
>   *		for an existing frame. This is always true unless
> @@ -84,14 +92,6 @@ struct v4l2_m2m_queue_ctx {
>   * @last_src_buf: indicate the last source buffer for draining
>   * @next_buf_last: next capture queud buffer will be tagged as last
>   * @has_stopped: indicate the device has been stopped
> - * @m2m_dev: opaque pointer to the internal data to handle M2M context
> - * @cap_q_ctx: Capture (output to memory) queue context
> - * @out_q_ctx: Output (input from memory) queue context
> - * @queue: List of memory to memory contexts
> - * @job_flags: Job queue flags, used internally by v4l2-mem2mem.c:
> - *		%TRANS_QUEUED, %TRANS_RUNNING and %TRANS_ABORT.
> - * @finished: Wait queue used to signalize when a job queue finished.
> - * @priv: Instance private data
>   *
>   * The memory to memory context is specific to a file handle, NOT to e.g.
>   * a device.
> @@ -100,18 +100,10 @@ struct v4l2_m2m_ctx {
>  	/* optional cap/out vb2 queues lock */
>  	struct mutex			*q_lock;
>  
> -	bool				new_frame;
> -
> -	bool				is_draining;
> -	struct vb2_v4l2_buffer		*last_src_buf;
> -	bool				next_buf_last;
> -	bool				has_stopped;
> -
>  	/* internal use only */
>  	struct v4l2_m2m_dev		*m2m_dev;
>  
>  	struct v4l2_m2m_queue_ctx	cap_q_ctx;
> -
>  	struct v4l2_m2m_queue_ctx	out_q_ctx;
>  
>  	/* For device job queue */
> @@ -120,6 +112,13 @@ struct v4l2_m2m_ctx {
>  	wait_queue_head_t		finished;
>  
>  	void				*priv;
> +
> +	/* codec specific fields */
> +	bool				new_frame;
> +	bool				is_draining;
> +	struct vb2_v4l2_buffer		*last_src_buf;
> +	bool				next_buf_last;
> +	bool				has_stopped;
>  };
>  
>  /**