[RFC V3 0/2] arm64: imx8mm: Enable Hantro VPUs

[RFC V3 0/2] arm64: imx8mm: Enable Hantro VPUs

[Adam Ford]

The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
to be related to the video decoders used on the i.MX8MQ, but because of
how the Mini handles the power domains, the VPU driver does not need to
handle all the functions, so a new compatible flag is required.

V3 is rebased from git://linuxtv.org/hverkuil/media_tree.git for-v5.17c
This branch has support for VP9.

I set cma=512M, but this may not be enough memory as some tests appeard to run out of memory

V3 of this series has several changes:

Update imx8m_vpu_hw to add missing 'reg' reference names for G2 and include references to VP9
Update device tree to remove IMX8MQ_VPU_RESET, remove some duplicate vpu clock parenting
Fix missing reg-names from vpu_g2 node.
Apply patch [1] to manage the power domains powering down.
[1] - https://lore.kernel.org/linux-arm-kernel/20211016210547.171717-1-marex@denx.de/t/

With the above, the following Fluster scores are produced:

G1:
./fluster.py run -dGStreamer-H.264-V4L2SL-Gst1.0
Ran 90/135 tests successfully               in 74.406 secs

./fluster.py run -d GStreamer-VP8-V4L2SL-Gst1.0
Ran 55/61 tests successfully               in 8.080 secs

G2:
./fluster.py run -d GStreamer-VP9-V4L2SL-Gst1.0
Ran 127/303 tests successfully               in 203.873 secs

Fluster and G-Streamer were both built from their respective git repos using their respective master/main branches.


Adam Ford (2):
  media: hantro: Add support for i.MX8M Mini
  arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2

 arch/arm64/boot/dts/freescale/imx8mm.dtsi   | 41 +++++++++++++
 drivers/staging/media/hantro/hantro_drv.c   |  2 +
 drivers/staging/media/hantro/hantro_hw.h    |  2 +
 drivers/staging/media/hantro/imx8m_vpu_hw.c | 65 +++++++++++++++++++++
 4 files changed, 110 insertions(+)

-- 
2.32.0

[RFC V3 1/2] media: hantro: Add support for i.MX8M Mini

[Adam Ford]

The i.MX8M Mini has a similar implementation of the Hantro G1 and
G2 decoders, but the Mini uses the vpu-blk-ctrl for handling the
VPU resets through the power domain system.  As such, there are
functions present in the 8MQ that are not applicable to the Mini
which requires the driver to have a different compatible flags.

Signed-off-by: Adam Ford <aford173@gmail.com>

diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c
index ab2467998d29..44b604d25a33 100644
--- a/drivers/staging/media/hantro/hantro_drv.c
+++ b/drivers/staging/media/hantro/hantro_drv.c
@@ -608,6 +608,8 @@ static const struct of_device_id of_hantro_match[] = {
 	{ .compatible = "rockchip,rk3399-vpu", .data = &rk3399_vpu_variant, },
 #endif
 #ifdef CONFIG_VIDEO_HANTRO_IMX8M
+	{ .compatible = "nxp,imx8mm-vpu", .data = &imx8mm_vpu_variant, },
+	{ .compatible = "nxp,imx8mm-vpu-g2", .data = &imx8mm_vpu_g2_variant },
 	{ .compatible = "nxp,imx8mq-vpu", .data = &imx8mq_vpu_variant, },
 	{ .compatible = "nxp,imx8mq-vpu-g2", .data = &imx8mq_vpu_g2_variant },
 #endif
diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h
index dbe51303724b..05cfafd9f1d7 100644
--- a/drivers/staging/media/hantro/hantro_hw.h
+++ b/drivers/staging/media/hantro/hantro_hw.h
@@ -299,6 +299,8 @@ enum hantro_enc_fmt {
 	ROCKCHIP_VPU_ENC_FMT_UYVY422 = 3,
 };
 
+extern const struct hantro_variant imx8mm_vpu_g2_variant;
+extern const struct hantro_variant imx8mm_vpu_variant;
 extern const struct hantro_variant imx8mq_vpu_g2_variant;
 extern const struct hantro_variant imx8mq_vpu_variant;
 extern const struct hantro_variant px30_vpu_variant;
diff --git a/drivers/staging/media/hantro/imx8m_vpu_hw.c b/drivers/staging/media/hantro/imx8m_vpu_hw.c
index 1a43f6fceef9..871df29dcfc7 100644
--- a/drivers/staging/media/hantro/imx8m_vpu_hw.c
+++ b/drivers/staging/media/hantro/imx8m_vpu_hw.c
@@ -271,6 +271,38 @@ static const struct hantro_codec_ops imx8mq_vpu_g2_codec_ops[] = {
 	},
 };
 
+static const struct hantro_codec_ops imx8mm_vpu_codec_ops[] = {
+	[HANTRO_MODE_MPEG2_DEC] = {
+		.run = hantro_g1_mpeg2_dec_run,
+		.init = hantro_mpeg2_dec_init,
+		.exit = hantro_mpeg2_dec_exit,
+	},
+	[HANTRO_MODE_VP8_DEC] = {
+		.run = hantro_g1_vp8_dec_run,
+		.init = hantro_vp8_dec_init,
+		.exit = hantro_vp8_dec_exit,
+	},
+	[HANTRO_MODE_H264_DEC] = {
+		.run = hantro_g1_h264_dec_run,
+		.init = hantro_h264_dec_init,
+		.exit = hantro_h264_dec_exit,
+	},
+};
+
+static const struct hantro_codec_ops imx8mm_vpu_g2_codec_ops[] = {
+	[HANTRO_MODE_HEVC_DEC] = {
+		.run = hantro_g2_hevc_dec_run,
+		.init = hantro_hevc_dec_init,
+		.exit = hantro_hevc_dec_exit,
+	},
+	[HANTRO_MODE_VP9_DEC] = {
+		.run = hantro_g2_vp9_dec_run,
+		.done = hantro_g2_vp9_dec_done,
+		.init = hantro_vp9_dec_init,
+		.exit = hantro_vp9_dec_exit,
+	},
+};
+
 /*
  * VPU variants.
  */
@@ -286,6 +318,11 @@ static const struct hantro_irq imx8mq_g2_irqs[] = {
 static const char * const imx8mq_clk_names[] = { "g1", "g2", "bus" };
 static const char * const imx8mq_reg_names[] = { "g1", "g2", "ctrl" };
 
+static const char * const imx8mm_g1_clk_names[] = { "g1", "bus" };
+static const char * const imx8mm_g1_reg_names[] = { "g1" };
+static const char * const imx8mm_g2_clk_names[] = { "g2", "bus" };
+static const char * const imx8mm_g2_reg_names[] = { "g2" };
+
 const struct hantro_variant imx8mq_vpu_variant = {
 	.dec_fmts = imx8m_vpu_dec_fmts,
 	.num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts),
@@ -321,3 +358,31 @@ const struct hantro_variant imx8mq_vpu_g2_variant = {
 	.clk_names = imx8mq_clk_names,
 	.num_clocks = ARRAY_SIZE(imx8mq_clk_names),
 };
+
+const struct hantro_variant imx8mm_vpu_variant = {
+	.dec_fmts = imx8m_vpu_dec_fmts,
+	.num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts),
+	.codec = HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER |
+		 HANTRO_H264_DECODER,
+	.codec_ops = imx8mm_vpu_codec_ops,
+	.irqs = imx8mq_irqs,
+	.num_irqs = ARRAY_SIZE(imx8mq_irqs),
+	.clk_names = imx8mm_g1_clk_names,
+	.num_clocks = ARRAY_SIZE(imx8mm_g1_clk_names),
+	.reg_names = imx8mm_g1_reg_names,
+	.num_regs = ARRAY_SIZE(imx8mm_g1_reg_names),
+};
+
+const struct hantro_variant imx8mm_vpu_g2_variant = {
+	.dec_offset = 0x0,
+	.dec_fmts = imx8m_vpu_g2_dec_fmts,
+	.num_dec_fmts = ARRAY_SIZE(imx8m_vpu_g2_dec_fmts),
+	.codec = HANTRO_HEVC_DECODER | HANTRO_VP9_DECODER,
+	.codec_ops = imx8mq_vpu_g2_codec_ops,
+	.irqs = imx8mq_g2_irqs,
+	.num_irqs = ARRAY_SIZE(imx8mq_g2_irqs),
+	.clk_names = imx8mm_g2_clk_names,
+	.num_clocks = ARRAY_SIZE(imx8mm_g2_clk_names),
+	.reg_names = imx8mm_g2_reg_names,
+	.num_regs = ARRAY_SIZE(imx8mm_g2_reg_names),
+};
-- 
2.32.0

[RFC V3 2/2] arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2

[Adam Ford]

Enable two hardware Hantro decoders called G1 and G2.

Signed-off-by: Adam Ford <aford173@gmail.com>

diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
index c2f3f118f82e..eb9dcd9d1a31 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
@@ -1197,6 +1197,47 @@ gpu_2d: gpu@38008000 {
 			power-domains = <&pgc_gpu>;
 		};
 
+		vpu_g1: video-codec@38300000 {
+			compatible = "nxp,imx8mm-vpu";
+			reg = <0x38300000 0x10000>;
+			reg-names = "g1";
+			interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "g1";
+			clocks = <&clk IMX8MM_CLK_VPU_G1_ROOT>,
+				 <&clk IMX8MM_CLK_VPU_DEC_ROOT>;
+			clock-names = "g1", "bus";
+			assigned-clocks = <&clk IMX8MM_CLK_VPU_G1>,
+					  <&clk IMX8MM_CLK_VPU_BUS>,
+					  <&clk IMX8MM_VPU_PLL_BYPASS>;
+			assigned-clock-parents = <&clk IMX8MM_VPU_PLL_OUT>,
+						 <&clk IMX8MM_SYS_PLL1_800M>,
+						 <&clk IMX8MM_VPU_PLL>;
+			assigned-clock-rates = <600000000>,
+					       <800000000>,
+					       <0>;
+			power-domains = <&vpu_blk_ctrl IMX8MM_VPUBLK_PD_G1>;
+		};
+
+		vpu_g2: video-codec@38310000 {
+			compatible = "nxp,imx8mm-vpu-g2";
+			reg = <0x38310000 0x10000>;
+			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "g2";
+			clocks = <&clk IMX8MM_CLK_VPU_G2_ROOT>,
+				 <&clk IMX8MM_CLK_VPU_DEC_ROOT>;
+			clock-names = "g2",  "bus";
+			assigned-clocks = <&clk IMX8MM_CLK_VPU_G2>,
+					 <&clk IMX8MM_CLK_VPU_BUS>,
+					 <&clk IMX8MM_VPU_PLL_BYPASS>;
+			assigned-clock-parents = <&clk IMX8MM_VPU_PLL_OUT>,
+						<&clk IMX8MM_SYS_PLL1_800M>,
+						<&clk IMX8MM_VPU_PLL>;
+			assigned-clock-rates = <600000000>,
+					       <800000000>,
+					       <0>;
+			power-domains = <&vpu_blk_ctrl IMX8MM_VPUBLK_PD_G2>;
+		};
+
 		vpu_blk_ctrl: blk-ctrl@38330000 {
 			compatible = "fsl,imx8mm-vpu-blk-ctrl", "syscon";
 			reg = <0x38330000 0x100>;
-- 
2.32.0

Re: [RFC V3 0/2] arm64: imx8mm: Enable Hantro VPUs

[Adam Ford]

On Wed, Dec 1, 2021 at 10:17 PM Adam Ford <aford173@gmail.com> wrote:
>
> The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> to be related to the video decoders used on the i.MX8MQ, but because of
> how the Mini handles the power domains, the VPU driver does not need to
> handle all the functions, so a new compatible flag is required.
>
> V3 is rebased from git://linuxtv.org/hverkuil/media_tree.git for-v5.17c
> This branch has support for VP9.
>
> I set cma=512M, but this may not be enough memory as some tests appeard to run out of memory
>
> V3 of this series has several changes:
>
> Update imx8m_vpu_hw to add missing 'reg' reference names for G2 and include references to VP9
> Update device tree to remove IMX8MQ_VPU_RESET, remove some duplicate vpu clock parenting
> Fix missing reg-names from vpu_g2 node.
> Apply patch [1] to manage the power domains powering down.
> [1] - https://lore.kernel.org/linux-arm-kernel/20211016210547.171717-1-marex@denx.de/t/
>
> With the above, the following Fluster scores are produced:
>
> G1:
> ./fluster.py run -dGStreamer-H.264-V4L2SL-Gst1.0
> Ran 90/135 tests successfully               in 74.406 secs
>
> ./fluster.py run -d GStreamer-VP8-V4L2SL-Gst1.0
> Ran 55/61 tests successfully               in 8.080 secs
>
> G2:
> ./fluster.py run -d GStreamer-VP9-V4L2SL-Gst1.0
> Ran 127/303 tests successfully               in 203.873 secs
>
> Fluster and G-Streamer were both built from their respective git repos using their respective master/main branches.
>

I should note, that both interrupts appear to be triggering.

# cat /proc/interrupts |grep codec
 57:      13442          0          0          0     GICv3  39 Level
  38300000.video-codec
 58:       7815          0          0          0     GICv3  40 Level
  38310000.video-codec




>
> Adam Ford (2):
>   media: hantro: Add support for i.MX8M Mini
>   arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
>
>  arch/arm64/boot/dts/freescale/imx8mm.dtsi   | 41 +++++++++++++
>  drivers/staging/media/hantro/hantro_drv.c   |  2 +
>  drivers/staging/media/hantro/hantro_hw.h    |  2 +
>  drivers/staging/media/hantro/imx8m_vpu_hw.c | 65 +++++++++++++++++++++
>  4 files changed, 110 insertions(+)
>
> --
> 2.32.0
>

Re: [RFC V3 0/2] arm64: imx8mm: Enable Hantro VPUs

[Tim Harvey]

On Thu, Dec 2, 2021 at 4:29 AM Adam Ford <aford173@gmail.com> wrote:
>
> On Wed, Dec 1, 2021 at 10:17 PM Adam Ford <aford173@gmail.com> wrote:
> >
> > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > to be related to the video decoders used on the i.MX8MQ, but because of
> > how the Mini handles the power domains, the VPU driver does not need to
> > handle all the functions, so a new compatible flag is required.
> >
> > V3 is rebased from git://linuxtv.org/hverkuil/media_tree.git for-v5.17c
> > This branch has support for VP9.
> >
> > I set cma=512M, but this may not be enough memory as some tests appeard to run out of memory
> >
> > V3 of this series has several changes:
> >
> > Update imx8m_vpu_hw to add missing 'reg' reference names for G2 and include references to VP9
> > Update device tree to remove IMX8MQ_VPU_RESET, remove some duplicate vpu clock parenting
> > Fix missing reg-names from vpu_g2 node.
> > Apply patch [1] to manage the power domains powering down.
> > [1] - https://lore.kernel.org/linux-arm-kernel/20211016210547.171717-1-marex@denx.de/t/
> >
> > With the above, the following Fluster scores are produced:
> >
> > G1:
> > ./fluster.py run -dGStreamer-H.264-V4L2SL-Gst1.0
> > Ran 90/135 tests successfully               in 74.406 secs
> >
> > ./fluster.py run -d GStreamer-VP8-V4L2SL-Gst1.0
> > Ran 55/61 tests successfully               in 8.080 secs
> >
> > G2:
> > ./fluster.py run -d GStreamer-VP9-V4L2SL-Gst1.0
> > Ran 127/303 tests successfully               in 203.873 secs
> >
> > Fluster and G-Streamer were both built from their respective git repos using their respective master/main branches.
> >
>
> I should note, that both interrupts appear to be triggering.
>
> # cat /proc/interrupts |grep codec
>  57:      13442          0          0          0     GICv3  39 Level
>   38300000.video-codec
>  58:       7815          0          0          0     GICv3  40 Level
>   38310000.video-codec
>

Adam,

On another thread you had let me know that you also removed the reset
from the pgc_vpumix power domain which does appear to resolve the
hang:

diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
index eb9dcd9d1a31..31710af544dc 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
@@ -681,7 +681,6 @@
                                                clocks = <&clk
IMX8MM_CLK_VPU_DEC_ROOT>;
                                                assigned-clocks =
<&clk IMX8MM_CLK_VPU_BUS>;
                                                assigned-clock-parents
= <&clk IMX8MM_SYS_PLL1_800M>;
-                                               resets = <&src
IMX8MQ_RESET_VPU_RESET>;
                                        };

                                        pgc_vpu_g1: power-domain@7 {

That would make such a patch have a 'Fixes commit d39d4bb15310
("arm64: dts: imx8mm: add GPC node")' but of course that vpu domain
isn't active until your series so I'm not sure if we should send this
separate or squash it with "arm64: dts: imx8mm: Enable VPU-G1 and
VPU-G2". I'm also not clear if removing the reset requires some
further discussion with Lucas.

Best regards,

Tim

Re: [RFC V3 0/2] arm64: imx8mm: Enable Hantro VPUs

[Adam Ford]

On Thu, Dec 2, 2021 at 12:54 PM Tim Harvey <tharvey@gateworks.com> wrote:
>
> On Thu, Dec 2, 2021 at 4:29 AM Adam Ford <aford173@gmail.com> wrote:
> >
> > On Wed, Dec 1, 2021 at 10:17 PM Adam Ford <aford173@gmail.com> wrote:
> > >
> > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > how the Mini handles the power domains, the VPU driver does not need to
> > > handle all the functions, so a new compatible flag is required.
> > >
> > > V3 is rebased from git://linuxtv.org/hverkuil/media_tree.git for-v5.17c
> > > This branch has support for VP9.
> > >
> > > I set cma=512M, but this may not be enough memory as some tests appeard to run out of memory
> > >
> > > V3 of this series has several changes:
> > >
> > > Update imx8m_vpu_hw to add missing 'reg' reference names for G2 and include references to VP9
> > > Update device tree to remove IMX8MQ_VPU_RESET, remove some duplicate vpu clock parenting
> > > Fix missing reg-names from vpu_g2 node.
> > > Apply patch [1] to manage the power domains powering down.
> > > [1] - https://lore.kernel.org/linux-arm-kernel/20211016210547.171717-1-marex@denx.de/t/
> > >
> > > With the above, the following Fluster scores are produced:
> > >
> > > G1:
> > > ./fluster.py run -dGStreamer-H.264-V4L2SL-Gst1.0
> > > Ran 90/135 tests successfully               in 74.406 secs
> > >
> > > ./fluster.py run -d GStreamer-VP8-V4L2SL-Gst1.0
> > > Ran 55/61 tests successfully               in 8.080 secs
> > >
> > > G2:
> > > ./fluster.py run -d GStreamer-VP9-V4L2SL-Gst1.0
> > > Ran 127/303 tests successfully               in 203.873 secs
> > >
> > > Fluster and G-Streamer were both built from their respective git repos using their respective master/main branches.
> > >
> >
> > I should note, that both interrupts appear to be triggering.
> >
> > # cat /proc/interrupts |grep codec
> >  57:      13442          0          0          0     GICv3  39 Level
> >   38300000.video-codec
> >  58:       7815          0          0          0     GICv3  40 Level
> >   38310000.video-codec
> >
>
> Adam,
>
> On another thread you had let me know that you also removed the reset
> from the pgc_vpumix power domain which does appear to resolve the
> hang:
>
> diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> index eb9dcd9d1a31..31710af544dc 100644
> --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> @@ -681,7 +681,6 @@
>                                                 clocks = <&clk
> IMX8MM_CLK_VPU_DEC_ROOT>;
>                                                 assigned-clocks =
> <&clk IMX8MM_CLK_VPU_BUS>;
>                                                 assigned-clock-parents
> = <&clk IMX8MM_SYS_PLL1_800M>;
> -                                               resets = <&src
> IMX8MQ_RESET_VPU_RESET>;
>                                         };
>
>                                         pgc_vpu_g1: power-domain@7 {
>
> That would make such a patch have a 'Fixes commit d39d4bb15310
> ("arm64: dts: imx8mm: add GPC node")' but of course that vpu domain
> isn't active until your series so I'm not sure if we should send this
> separate or squash it with "arm64: dts: imx8mm: Enable VPU-G1 and
> VPU-G2". I'm also not clear if removing the reset requires some
> further discussion with Lucas.

Unless there is objection from Lucas, I'll likely make it the first
patch in the series marking it with a fixes tag so it gets backported,
then the rest of the series would be adding the bindings, update the
driver and adding the G1 and G2 nodes.

adam
>
> Best regards,
>
> Tim

Re: [RFC V3 0/2] arm64: imx8mm: Enable Hantro VPUs

[Tim Harvey]

On Thu, Dec 2, 2021 at 1:00 PM Adam Ford <aford173@gmail.com> wrote:
>
> On Thu, Dec 2, 2021 at 12:54 PM Tim Harvey <tharvey@gateworks.com> wrote:
> >
> > On Thu, Dec 2, 2021 at 4:29 AM Adam Ford <aford173@gmail.com> wrote:
> > >
> > > On Wed, Dec 1, 2021 at 10:17 PM Adam Ford <aford173@gmail.com> wrote:
> > > >
> > > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > > how the Mini handles the power domains, the VPU driver does not need to
> > > > handle all the functions, so a new compatible flag is required.
> > > >
> > > > V3 is rebased from git://linuxtv.org/hverkuil/media_tree.git for-v5.17c
> > > > This branch has support for VP9.
> > > >
> > > > I set cma=512M, but this may not be enough memory as some tests appeard to run out of memory
> > > >
> > > > V3 of this series has several changes:
> > > >
> > > > Update imx8m_vpu_hw to add missing 'reg' reference names for G2 and include references to VP9
> > > > Update device tree to remove IMX8MQ_VPU_RESET, remove some duplicate vpu clock parenting
> > > > Fix missing reg-names from vpu_g2 node.
> > > > Apply patch [1] to manage the power domains powering down.
> > > > [1] - https://lore.kernel.org/linux-arm-kernel/20211016210547.171717-1-marex@denx.de/t/
> > > >
> > > > With the above, the following Fluster scores are produced:
> > > >
> > > > G1:
> > > > ./fluster.py run -dGStreamer-H.264-V4L2SL-Gst1.0
> > > > Ran 90/135 tests successfully               in 74.406 secs
> > > >
> > > > ./fluster.py run -d GStreamer-VP8-V4L2SL-Gst1.0
> > > > Ran 55/61 tests successfully               in 8.080 secs
> > > >
> > > > G2:
> > > > ./fluster.py run -d GStreamer-VP9-V4L2SL-Gst1.0
> > > > Ran 127/303 tests successfully               in 203.873 secs
> > > >
> > > > Fluster and G-Streamer were both built from their respective git repos using their respective master/main branches.
> > > >
> > >
> > > I should note, that both interrupts appear to be triggering.
> > >
> > > # cat /proc/interrupts |grep codec
> > >  57:      13442          0          0          0     GICv3  39 Level
> > >   38300000.video-codec
> > >  58:       7815          0          0          0     GICv3  40 Level
> > >   38310000.video-codec
> > >
> >
> > Adam,
> >
> > On another thread you had let me know that you also removed the reset
> > from the pgc_vpumix power domain which does appear to resolve the
> > hang:
> >
> > diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> > b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> > index eb9dcd9d1a31..31710af544dc 100644
> > --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> > +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> > @@ -681,7 +681,6 @@
> >                                                 clocks = <&clk
> > IMX8MM_CLK_VPU_DEC_ROOT>;
> >                                                 assigned-clocks =
> > <&clk IMX8MM_CLK_VPU_BUS>;
> >                                                 assigned-clock-parents
> > = <&clk IMX8MM_SYS_PLL1_800M>;
> > -                                               resets = <&src
> > IMX8MQ_RESET_VPU_RESET>;
> >                                         };
> >
> >                                         pgc_vpu_g1: power-domain@7 {
> >
> > That would make such a patch have a 'Fixes commit d39d4bb15310
> > ("arm64: dts: imx8mm: add GPC node")' but of course that vpu domain
> > isn't active until your series so I'm not sure if we should send this
> > separate or squash it with "arm64: dts: imx8mm: Enable VPU-G1 and
> > VPU-G2". I'm also not clear if removing the reset requires some
> > further discussion with Lucas.
>
> Unless there is objection from Lucas, I'll likely make it the first
> patch in the series marking it with a fixes tag so it gets backported,
> then the rest of the series would be adding the bindings, update the
> driver and adding the G1 and G2 nodes.
>

Adam,

I've also gotten decode+display working for vp8/h264 using this series
and gstreamer-1.19.3 (although I have to use software colorspace
conversion)

# source: vp8 software encode on x86
gst-launch-1.0 -v videotestsrc ! vp8enc ! rtpvp8pay ! udpsink
host=172.24.33.15 port=9001
# sink: vp8 hardware decode on imx8mm
gst-launch-1.0 udpsrc port=9001 caps = 'application/x-rtp,
media=(string)video, clock-rate=(int)90000, encoding-name=(string)VP8,
payload=(int)96, ssrc=(uint)3363940374,
timestamp-offset=(uint)3739685909, seqnum-offset=(uint)28161,
a-framerate=(string)30' ! rtpvp8depay ! v4l2slvp8dec ! videoconvert !
kmssink

# source: h264 software encode on x86
gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
x264enc ! rtph264pay ! udpsink host=172.24.33.15 port=9001
# sink: h264 hardware decode on imx8mm
gst-launch-1.0 udpsrc port=9001 caps = 'application/x-rtp,
media=(string)video, clock-rate=(int)90000,
encoding-name=(string)H264, packetization-mode=(string)1,
profile-level-id=(string)64001f,
sprop-parameter-sets=(string)"Z2QAH6zZQMg9sBagwCC0oAAAAwAgAAAHkeMGMsA\=\,aOvssiw\=",
payload=(int)96, ssrc=(uint)2753453329,
timestamp-offset=(uint)3593065282, seqnum-offset=(uint)12297,
a-framerate=(string)30' ! rtph264depay ! v4l2slh264dec ! videoconvert
! kmssink

# source: vp9 software encode on x86
gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
vp9enc ! rtpvp9pay ! udpsink host=172.24.33.15 port=9001
# sink: vp9 hardware decode on imx8mm
gst-launch-1.0 udpsrc port=9001 caps = 'application/x-rtp,
media=(string)video, clock-rate=(int)90000, encoding-name=(string)VP9,
payload=(int)96, ssrc=(uint)2246741422,
timestamp-offset=(uint)3441735424, seqnum-offset=(uint)30250,
a-framerate=(string)30' ! rtpvp9depay ! v4l2slvp9dec ! fakesink
^^^ this fails with no-negotiated

Note I have to use videoconvert because v4l2slvp8dev src is
NV12/YUY2/NV12_32L32 and from testing only BGRx appears compatible
with kmssink (even though gst-inspect kmssink says it can sink
NV12/YUY2). With the 800x480 resolution of my display the CPU overhead
of software colorspcae conversion with videoconvert only about 9%

I haven't yet gotten vp9 decode+display working yet as 'rtpvp9depay !
v4l2slvp9dec ! fakesink' does not negotiate and it might be because my
vp9enc source is on an old gstreamer 1.16.

When you post the next series please add:
Tested-By: Tim Harvey <tharvey@gateworks.com>

Re: [RFC V3 0/2] arm64: imx8mm: Enable Hantro VPUs

[Nicolas Dufresne]

Le mercredi 01 décembre 2021 à 22:16 -0600, Adam Ford a écrit :
> The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> to be related to the video decoders used on the i.MX8MQ, but because of
> how the Mini handles the power domains, the VPU driver does not need to
> handle all the functions, so a new compatible flag is required.
> 
> V3 is rebased from git://linuxtv.org/hverkuil/media_tree.git for-v5.17c
> This branch has support for VP9.
> 
> I set cma=512M, but this may not be enough memory as some tests appeard to run out of memory

Consider passing -j1 to fluster and using patience, this will spare some memory.
You can also increase the timeout (like -t 60), that often helps when CMA is
involved (due to software conversion happening). Thanks for the serious testing,
this is really appreciated that your submission really reflects the state.

> 
> V3 of this series has several changes:
> 
> Update imx8m_vpu_hw to add missing 'reg' reference names for G2 and include references to VP9
> Update device tree to remove IMX8MQ_VPU_RESET, remove some duplicate vpu clock parenting
> Fix missing reg-names from vpu_g2 node.
> Apply patch [1] to manage the power domains powering down.
> [1] - https://lore.kernel.org/linux-arm-kernel/20211016210547.171717-1-marex@denx.de/t/
> 
> With the above, the following Fluster scores are produced:
> 
> G1:
> ./fluster.py run -dGStreamer-H.264-V4L2SL-Gst1.0
> Ran 90/135 tests successfully               in 74.406 secs

This matches my results on imx8mq.

> 
> ./fluster.py run -d GStreamer-VP8-V4L2SL-Gst1.0
> Ran 55/61 tests successfully               in 8.080 secs

Idem.

> 
> G2:
> ./fluster.py run -d GStreamer-VP9-V4L2SL-Gst1.0
> Ran 127/303 tests successfully               in 203.873 secs

Benjamin reports 129, it could be flux. This is still a bit early, hence the low
score. But most streams do playback normally.

> 
> Fluster and G-Streamer were both built from their respective git repos using their respective master/main branches.

That gave me an idea, I'll try and print the GStreamer version/commit inside
fluster along with fluster own commit (ideally in the short summary).

> 
> 
> Adam Ford (2):
>   media: hantro: Add support for i.MX8M Mini
>   arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> 
>  arch/arm64/boot/dts/freescale/imx8mm.dtsi   | 41 +++++++++++++
>  drivers/staging/media/hantro/hantro_drv.c   |  2 +
>  drivers/staging/media/hantro/hantro_hw.h    |  2 +
>  drivers/staging/media/hantro/imx8m_vpu_hw.c | 65 +++++++++++++++++++++
>  4 files changed, 110 insertions(+)
> 

Re: [RFC V3 0/2] arm64: imx8mm: Enable Hantro VPUs

[Lucas Stach]

Am Donnerstag, dem 02.12.2021 um 10:54 -0800 schrieb Tim Harvey:
> On Thu, Dec 2, 2021 at 4:29 AM Adam Ford <aford173@gmail.com> wrote:
> > 
> > On Wed, Dec 1, 2021 at 10:17 PM Adam Ford <aford173@gmail.com> wrote:
> > > 
> > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > how the Mini handles the power domains, the VPU driver does not need to
> > > handle all the functions, so a new compatible flag is required.
> > > 
> > > V3 is rebased from git://linuxtv.org/hverkuil/media_tree.git for-v5.17c
> > > This branch has support for VP9.
> > > 
> > > I set cma=512M, but this may not be enough memory as some tests appeard to run out of memory
> > > 
> > > V3 of this series has several changes:
> > > 
> > > Update imx8m_vpu_hw to add missing 'reg' reference names for G2 and include references to VP9
> > > Update device tree to remove IMX8MQ_VPU_RESET, remove some duplicate vpu clock parenting
> > > Fix missing reg-names from vpu_g2 node.
> > > Apply patch [1] to manage the power domains powering down.
> > > [1] - https://lore.kernel.org/linux-arm-kernel/20211016210547.171717-1-marex@denx.de/t/
> > > 
> > > With the above, the following Fluster scores are produced:
> > > 
> > > G1:
> > > ./fluster.py run -dGStreamer-H.264-V4L2SL-Gst1.0
> > > Ran 90/135 tests successfully               in 74.406 secs
> > > 
> > > ./fluster.py run -d GStreamer-VP8-V4L2SL-Gst1.0
> > > Ran 55/61 tests successfully               in 8.080 secs
> > > 
> > > G2:
> > > ./fluster.py run -d GStreamer-VP9-V4L2SL-Gst1.0
> > > Ran 127/303 tests successfully               in 203.873 secs
> > > 
> > > Fluster and G-Streamer were both built from their respective git repos using their respective master/main branches.
> > > 
> > 
> > I should note, that both interrupts appear to be triggering.
> > 
> > # cat /proc/interrupts |grep codec
> >  57:      13442          0          0          0     GICv3  39 Level
> >   38300000.video-codec
> >  58:       7815          0          0          0     GICv3  40 Level
> >   38310000.video-codec
> > 
> 
> Adam,
> 
> On another thread you had let me know that you also removed the reset
> from the pgc_vpumix power domain which does appear to resolve the
> hang:
> 
> diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> index eb9dcd9d1a31..31710af544dc 100644
> --- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> +++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
> @@ -681,7 +681,6 @@
>                                                 clocks = <&clk
> IMX8MM_CLK_VPU_DEC_ROOT>;
>                                                 assigned-clocks =
> <&clk IMX8MM_CLK_VPU_BUS>;
>                                                 assigned-clock-parents
> = <&clk IMX8MM_SYS_PLL1_800M>;
> -                                               resets = <&src
> IMX8MQ_RESET_VPU_RESET>;
>                                         };
> 
>                                         pgc_vpu_g1: power-domain@7 {
> 
> That would make such a patch have a 'Fixes commit d39d4bb15310
> ("arm64: dts: imx8mm: add GPC node")' but of course that vpu domain
> isn't active until your series so I'm not sure if we should send this
> separate or squash it with "arm64: dts: imx8mm: Enable VPU-G1 and
> VPU-G2". I'm also not clear if removing the reset requires some
> further discussion with Lucas.
> 
I'm fine with removing the reset when it fixes things. In normal
operation the reset should already be triggered by the GPC itself via a
hardware mechanism. We know that this doesn't work for the GPU reset on
the i.MX8MM, so we have the ability for the driver to handle it by
poking the SRC explicitly.

Adding the reset to the VPU DT description wasn't done because I know
that we need it, but more of a "I know that things are broken with the
GPU domain, so better be safe than sorry with the VPU domain". My line
of thought clearly was that it may not be needed, but it may prevent
some issues in the long run. If it is _causing_ issues however, there
is no need to discuss anything, just get rid of it.

Regards,
Lucas