* [RFC 1/5] media: hantro: Add support for i.MX8M Mini
2021-11-06 18:37 [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Adam Ford
@ 2021-11-06 18:37 ` Adam Ford
2021-11-20 16:03 ` Adam Ford
2021-11-06 18:37 ` [RFC 2/5] arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2 Adam Ford
` (4 subsequent siblings)
5 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-06 18:37 UTC (permalink / raw)
To: linux-media
Cc: tharvey, frieder.schrempf, marek.vasut, jagan, aford, cstevens,
Adam Ford, Ezequiel Garcia, Philipp Zabel, Mauro Carvalho Chehab,
Rob Herring, Shawn Guo, Sascha Hauer, Pengutronix Kernel Team,
Fabio Estevam, NXP Linux Team, Greg Kroah-Hartman,
Heiko Stuebner, Lucas Stach, Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
The i.MX8M Mini has a similar implementation of the Hantro G1 and
G2 decoders, but the Mini uses the vpu-blk-ctrl for handling the
VPU resets through the power domain system. As such, there are
functions present in the 8MQ that are not applicable to the Mini
which requires the driver to have a different compatible flags.
Signed-off-by: Adam Ford <aford173@gmail.com>
---
drivers/staging/media/hantro/hantro_drv.c | 2 +
drivers/staging/media/hantro/hantro_hw.h | 2 +
drivers/staging/media/hantro/imx8m_vpu_hw.c | 60 +++++++++++++++++++++
3 files changed, 64 insertions(+)
diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c
index fb82b9297a2b..2aa1c520be50 100644
--- a/drivers/staging/media/hantro/hantro_drv.c
+++ b/drivers/staging/media/hantro/hantro_drv.c
@@ -592,6 +592,8 @@ static const struct of_device_id of_hantro_match[] = {
{ .compatible = "rockchip,rk3399-vpu", .data = &rk3399_vpu_variant, },
#endif
#ifdef CONFIG_VIDEO_HANTRO_IMX8M
+ { .compatible = "nxp,imx8mm-vpu", .data = &imx8mm_vpu_variant, },
+ { .compatible = "nxp,imx8mm-vpu-g2", .data = &imx8mm_vpu_g2_variant },
{ .compatible = "nxp,imx8mq-vpu", .data = &imx8mq_vpu_variant, },
{ .compatible = "nxp,imx8mq-vpu-g2", .data = &imx8mq_vpu_g2_variant },
#endif
diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h
index 267a6d33a47b..ae7c3fff760c 100644
--- a/drivers/staging/media/hantro/hantro_hw.h
+++ b/drivers/staging/media/hantro/hantro_hw.h
@@ -211,6 +211,8 @@ enum hantro_enc_fmt {
ROCKCHIP_VPU_ENC_FMT_UYVY422 = 3,
};
+extern const struct hantro_variant imx8mm_vpu_g2_variant;
+extern const struct hantro_variant imx8mm_vpu_variant;
extern const struct hantro_variant imx8mq_vpu_g2_variant;
extern const struct hantro_variant imx8mq_vpu_variant;
extern const struct hantro_variant px30_vpu_variant;
diff --git a/drivers/staging/media/hantro/imx8m_vpu_hw.c b/drivers/staging/media/hantro/imx8m_vpu_hw.c
index ea919bfb9891..c819609d14d1 100644
--- a/drivers/staging/media/hantro/imx8m_vpu_hw.c
+++ b/drivers/staging/media/hantro/imx8m_vpu_hw.c
@@ -242,6 +242,32 @@ static const struct hantro_codec_ops imx8mq_vpu_g2_codec_ops[] = {
},
};
+static const struct hantro_codec_ops imx8mm_vpu_codec_ops[] = {
+ [HANTRO_MODE_MPEG2_DEC] = {
+ .run = hantro_g1_mpeg2_dec_run,
+ .init = hantro_mpeg2_dec_init,
+ .exit = hantro_mpeg2_dec_exit,
+ },
+ [HANTRO_MODE_VP8_DEC] = {
+ .run = hantro_g1_vp8_dec_run,
+ .init = hantro_vp8_dec_init,
+ .exit = hantro_vp8_dec_exit,
+ },
+ [HANTRO_MODE_H264_DEC] = {
+ .run = hantro_g1_h264_dec_run,
+ .init = hantro_h264_dec_init,
+ .exit = hantro_h264_dec_exit,
+ },
+};
+
+static const struct hantro_codec_ops imx8mm_vpu_g2_codec_ops[] = {
+ [HANTRO_MODE_HEVC_DEC] = {
+ .run = hantro_g2_hevc_dec_run,
+ .init = hantro_hevc_dec_init,
+ .exit = hantro_hevc_dec_exit,
+ },
+};
+
/*
* VPU variants.
*/
@@ -257,6 +283,11 @@ static const struct hantro_irq imx8mq_g2_irqs[] = {
static const char * const imx8mq_clk_names[] = { "g1", "g2", "bus" };
static const char * const imx8mq_reg_names[] = { "g1", "g2", "ctrl" };
+static const char * const imx8mm_g1_clk_names[] = { "g1", "bus" };
+static const char * const imx8mm_g1_reg_names[] = { "g1" };
+static const char * const imx8mm_g2_clk_names[] = { "g2", "bus" };
+static const char * const imx8mm_g2_reg_names[] = { "g2" };
+
const struct hantro_variant imx8mq_vpu_variant = {
.dec_fmts = imx8m_vpu_dec_fmts,
.num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts),
@@ -289,3 +320,32 @@ const struct hantro_variant imx8mq_vpu_g2_variant = {
.clk_names = imx8mq_clk_names,
.num_clocks = ARRAY_SIZE(imx8mq_clk_names),
};
+
+const struct hantro_variant imx8mm_vpu_variant = {
+ .dec_fmts = imx8m_vpu_dec_fmts,
+ .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts),
+ .postproc_fmts = imx8m_vpu_postproc_fmts,
+ .num_postproc_fmts = ARRAY_SIZE(imx8m_vpu_postproc_fmts),
+ .postproc_regs = &hantro_g1_postproc_regs,
+ .codec = HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER |
+ HANTRO_H264_DECODER,
+ .codec_ops = imx8mm_vpu_codec_ops,
+ .irqs = imx8mq_irqs,
+ .num_irqs = ARRAY_SIZE(imx8mq_irqs),
+ .clk_names = imx8mm_g1_clk_names,
+ .num_clocks = ARRAY_SIZE(imx8mm_g1_clk_names),
+ .reg_names = imx8mm_g1_reg_names,
+ .num_regs = ARRAY_SIZE(imx8mm_g1_reg_names)
+};
+
+const struct hantro_variant imx8mm_vpu_g2_variant = {
+ .dec_offset = 0x0,
+ .dec_fmts = imx8m_vpu_g2_dec_fmts,
+ .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_g2_dec_fmts),
+ .codec = HANTRO_HEVC_DECODER,
+ .codec_ops = imx8mm_vpu_g2_codec_ops,
+ .irqs = imx8mq_g2_irqs,
+ .num_irqs = ARRAY_SIZE(imx8mq_g2_irqs),
+ .clk_names = imx8mm_g2_clk_names,
+ .num_clocks = ARRAY_SIZE(imx8mm_g2_reg_names),
+};
--
2.32.0
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply related [flat|nested] 42+ messages in thread* Re: [RFC 1/5] media: hantro: Add support for i.MX8M Mini
2021-11-06 18:37 ` [RFC 1/5] media: hantro: Add support for i.MX8M Mini Adam Ford
@ 2021-11-20 16:03 ` Adam Ford
2021-11-25 15:35 ` Hans Verkuil
0 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-20 16:03 UTC (permalink / raw)
To: linux-media
Cc: Tim Harvey, Schrempf Frieder, Marek Vasut, Jagan Teki,
Adam Ford-BE, cstevens, Ezequiel Garcia, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM, Nicolas Dufresne
On Sat, Nov 6, 2021 at 1:38 PM Adam Ford <aford173@gmail.com> wrote:
>
> The i.MX8M Mini has a similar implementation of the Hantro G1 and
> G2 decoders, but the Mini uses the vpu-blk-ctrl for handling the
> VPU resets through the power domain system. As such, there are
> functions present in the 8MQ that are not applicable to the Mini
> which requires the driver to have a different compatible flags.
>
+ Nicolas Dufresne
From what I can tell, the implementation between 8MQ and 8MM for the
G2 is basically the same with the only be difference being the
power-domain controller handling the vpu-blk-ctrl stuff in the Mini.
It doesn't hang, so I don't think the power-domain is a problem, but
the G2 throws some splat when v4l2-compliance is invoked.
# v4l2-compliance -d1
v4l2-compliance SHA: not available
, 64 bits, 64-bit time_t
Compliance test for hantro-vpu device /dev/video1:
Driver Info:
Driver name : hantro-vpu
Card type : nxp,imx8mm-vpu-g2-dec
Bus info : platform: hantro-vpu
Driver v[ 30.136983] ------------[ cut here ]------------
[ 30.142038] WARNING: CPU: 3 PID: 534 at mm/page_alloc.c:5344
__alloc_pages+0x5b4/0xbe0
[ 30.149971] Modules linked in: 8021q garp mrp stp llc af_alg
caam_jr caamhash_desc caamalg_desc crypto_engine rng_core authenc
libdes crct10dif_ce snd_soc_fsl_asoc_card snd_soc_imx_audmux
snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc hantro_vpu(C)
v4l2_h264 brcmfmac v4l2_mem2mem videobuf2_vmalloc brcmutil
videobuf2_dma_contig cfg80211 videobuf2_memops videobuf2_v4l2
videobuf2_common snd_soc_wm8962 videodev hci_uart mc etnaviv gpu_sched
btqca btbcm caam rtc_pcf85363 clk_bd718x7 at24 spi_imx spi_bitbang
rtc_snvs error snvs_pwrkey snd_soc_fsl_sai imx_pcm_dma imx8mm_thermal
imx_cpufreq_dt bluetooth ecdh_generic ecc rfkill fuse drm ipv6
[ 30.206845] CPU: 3 PID: 534 Comm: v4l2-compliance Tainted: G
C 5.16.0-rc1-00292-gb166e5dc4755-dirty #19
[ 30.217544] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
Development Kit (DT)
[ 30.224940] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 30.231904] pc : __alloc_pages+0x5b4/0xbe0
[ 30.236007] lr : __dma_direct_alloc_pages+0x17c/0x1e0
[ 30.241065] sp : ffff80001265b810
[ 30.244379] x29: ffff80001265b810 x28: 0000000000000034 x27: ffff00000b9ec220
[ 30.251523] x26: ffff00000038d810 x25: 0000000000000000 x24: ffff00000038d810
[ 30.258668] x23: ffff80001010c52c x22: ffff00000038d810 x21: 00000000ffffffff
[ 30.265813] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
[ 30.272960] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
[ 30.280101] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
ersion : 5.16.0
Capabilities [ 30.287247] x11: ffff80001231d000 x10:
ffff00000b9ec33a x9 : ffff00000b9ec32a
[ 30.297242] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
[ 30.304385] x5 : 00000000bfffffff x4 : ffff00000a72f000 x3 : 0000000000000000
[ 30.311529] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
[ 30.318676] Call trace:
[ 30.321124] __alloc_pages+0x5b4/0xbe0
[ 30.324880] __dma_direct_alloc_pages+0x17c/0x1e0
[ 30.329588] dma_direct_alloc+0x70/0x310
[ 30.333514] dma_alloc_attrs+0x7c/0xe4
[ 30.337266] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
[ 30.343038] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
[ 30.348805] device_run+0xac/0x110 [hantro_vpu]
[ 30.353354] v4l2_m2m_try_run+0x84/0x134 [v4l2_mem2mem]
[ 30.358603] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
[ 30.364368] v4l_streamon+0x28/0x34 [videodev]
[ 30.368874] __video_do_ioctl+0x178/0x3dc [videodev]
[ 30.373889] video_usercopy+0x368/0x6dc [videodev]
[ 30.378734] video_ioctl2+0x1c/0x30 [videodev]
[ 30.383230] v4l2_ioctl+0x44/0x64 [videodev]
[ 30.387554] __arm64_sys_ioctl+0xac/0xf0
[ 30.391482] invoke_syscall+0x48/0x114
[ 30.395238] el0_svc_common.constprop.0+0xd4/0xfc
[ 30.399947] do_el0_svc+0x2c/0x94
[ 30.403264] el0_svc+0x28/0x80
[ 30.406327] el0t_64_sync_handler+0xa8/0x130
[ 30.410603] el0t_64_sync+0x1a0/0x1a4
[ 30.414272] ---[ end trace 878112336a378214 ]---
I was hoping someone might have some suggestions on something to try..
thanks,
adam
> Signed-off-by: Adam Ford <aford173@gmail.com>
> ---
> drivers/staging/media/hantro/hantro_drv.c | 2 +
> drivers/staging/media/hantro/hantro_hw.h | 2 +
> drivers/staging/media/hantro/imx8m_vpu_hw.c | 60 +++++++++++++++++++++
> 3 files changed, 64 insertions(+)
>
> diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c
> index fb82b9297a2b..2aa1c520be50 100644
> --- a/drivers/staging/media/hantro/hantro_drv.c
> +++ b/drivers/staging/media/hantro/hantro_drv.c
> @@ -592,6 +592,8 @@ static const struct of_device_id of_hantro_match[] = {
> { .compatible = "rockchip,rk3399-vpu", .data = &rk3399_vpu_variant, },
> #endif
> #ifdef CONFIG_VIDEO_HANTRO_IMX8M
> + { .compatible = "nxp,imx8mm-vpu", .data = &imx8mm_vpu_variant, },
> + { .compatible = "nxp,imx8mm-vpu-g2", .data = &imx8mm_vpu_g2_variant },
> { .compatible = "nxp,imx8mq-vpu", .data = &imx8mq_vpu_variant, },
> { .compatible = "nxp,imx8mq-vpu-g2", .data = &imx8mq_vpu_g2_variant },
> #endif
> diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h
> index 267a6d33a47b..ae7c3fff760c 100644
> --- a/drivers/staging/media/hantro/hantro_hw.h
> +++ b/drivers/staging/media/hantro/hantro_hw.h
> @@ -211,6 +211,8 @@ enum hantro_enc_fmt {
> ROCKCHIP_VPU_ENC_FMT_UYVY422 = 3,
> };
>
> +extern const struct hantro_variant imx8mm_vpu_g2_variant;
> +extern const struct hantro_variant imx8mm_vpu_variant;
> extern const struct hantro_variant imx8mq_vpu_g2_variant;
> extern const struct hantro_variant imx8mq_vpu_variant;
> extern const struct hantro_variant px30_vpu_variant;
> diff --git a/drivers/staging/media/hantro/imx8m_vpu_hw.c b/drivers/staging/media/hantro/imx8m_vpu_hw.c
> index ea919bfb9891..c819609d14d1 100644
> --- a/drivers/staging/media/hantro/imx8m_vpu_hw.c
> +++ b/drivers/staging/media/hantro/imx8m_vpu_hw.c
> @@ -242,6 +242,32 @@ static const struct hantro_codec_ops imx8mq_vpu_g2_codec_ops[] = {
> },
> };
>
> +static const struct hantro_codec_ops imx8mm_vpu_codec_ops[] = {
> + [HANTRO_MODE_MPEG2_DEC] = {
> + .run = hantro_g1_mpeg2_dec_run,
> + .init = hantro_mpeg2_dec_init,
> + .exit = hantro_mpeg2_dec_exit,
> + },
> + [HANTRO_MODE_VP8_DEC] = {
> + .run = hantro_g1_vp8_dec_run,
> + .init = hantro_vp8_dec_init,
> + .exit = hantro_vp8_dec_exit,
> + },
> + [HANTRO_MODE_H264_DEC] = {
> + .run = hantro_g1_h264_dec_run,
> + .init = hantro_h264_dec_init,
> + .exit = hantro_h264_dec_exit,
> + },
> +};
> +
> +static const struct hantro_codec_ops imx8mm_vpu_g2_codec_ops[] = {
> + [HANTRO_MODE_HEVC_DEC] = {
> + .run = hantro_g2_hevc_dec_run,
> + .init = hantro_hevc_dec_init,
> + .exit = hantro_hevc_dec_exit,
> + },
> +};
> +
> /*
> * VPU variants.
> */
> @@ -257,6 +283,11 @@ static const struct hantro_irq imx8mq_g2_irqs[] = {
> static const char * const imx8mq_clk_names[] = { "g1", "g2", "bus" };
> static const char * const imx8mq_reg_names[] = { "g1", "g2", "ctrl" };
>
> +static const char * const imx8mm_g1_clk_names[] = { "g1", "bus" };
> +static const char * const imx8mm_g1_reg_names[] = { "g1" };
> +static const char * const imx8mm_g2_clk_names[] = { "g2", "bus" };
> +static const char * const imx8mm_g2_reg_names[] = { "g2" };
> +
> const struct hantro_variant imx8mq_vpu_variant = {
> .dec_fmts = imx8m_vpu_dec_fmts,
> .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts),
> @@ -289,3 +320,32 @@ const struct hantro_variant imx8mq_vpu_g2_variant = {
> .clk_names = imx8mq_clk_names,
> .num_clocks = ARRAY_SIZE(imx8mq_clk_names),
> };
> +
> +const struct hantro_variant imx8mm_vpu_variant = {
> + .dec_fmts = imx8m_vpu_dec_fmts,
> + .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts),
> + .postproc_fmts = imx8m_vpu_postproc_fmts,
> + .num_postproc_fmts = ARRAY_SIZE(imx8m_vpu_postproc_fmts),
> + .postproc_regs = &hantro_g1_postproc_regs,
> + .codec = HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER |
> + HANTRO_H264_DECODER,
> + .codec_ops = imx8mm_vpu_codec_ops,
> + .irqs = imx8mq_irqs,
> + .num_irqs = ARRAY_SIZE(imx8mq_irqs),
> + .clk_names = imx8mm_g1_clk_names,
> + .num_clocks = ARRAY_SIZE(imx8mm_g1_clk_names),
> + .reg_names = imx8mm_g1_reg_names,
> + .num_regs = ARRAY_SIZE(imx8mm_g1_reg_names)
> +};
> +
> +const struct hantro_variant imx8mm_vpu_g2_variant = {
> + .dec_offset = 0x0,
> + .dec_fmts = imx8m_vpu_g2_dec_fmts,
> + .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_g2_dec_fmts),
> + .codec = HANTRO_HEVC_DECODER,
> + .codec_ops = imx8mm_vpu_g2_codec_ops,
> + .irqs = imx8mq_g2_irqs,
> + .num_irqs = ARRAY_SIZE(imx8mq_g2_irqs),
> + .clk_names = imx8mm_g2_clk_names,
> + .num_clocks = ARRAY_SIZE(imx8mm_g2_reg_names),
> +};
> --
> 2.32.0
>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 1/5] media: hantro: Add support for i.MX8M Mini
2021-11-20 16:03 ` Adam Ford
@ 2021-11-25 15:35 ` Hans Verkuil
0 siblings, 0 replies; 42+ messages in thread
From: Hans Verkuil @ 2021-11-25 15:35 UTC (permalink / raw)
To: Adam Ford, linux-media
Cc: Tim Harvey, Schrempf Frieder, Marek Vasut, Jagan Teki,
Adam Ford-BE, cstevens, Ezequiel Garcia, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM, Nicolas Dufresne
On 20/11/2021 17:03, Adam Ford wrote:
> On Sat, Nov 6, 2021 at 1:38 PM Adam Ford <aford173@gmail.com> wrote:
>>
>> The i.MX8M Mini has a similar implementation of the Hantro G1 and
>> G2 decoders, but the Mini uses the vpu-blk-ctrl for handling the
>> VPU resets through the power domain system. As such, there are
>> functions present in the 8MQ that are not applicable to the Mini
>> which requires the driver to have a different compatible flags.
>>
> + Nicolas Dufresne
>
> From what I can tell, the implementation between 8MQ and 8MM for the
> G2 is basically the same with the only be difference being the
> power-domain controller handling the vpu-blk-ctrl stuff in the Mini.
> It doesn't hang, so I don't think the power-domain is a problem, but
> the G2 throws some splat when v4l2-compliance is invoked.
>
> # v4l2-compliance -d1
> v4l2-compliance SHA: not available
That indicates that you did not compile v4l2-compliance from the git repo.
Please do that, that way I know you are using the latest compliance test.
git clone git://linuxtv.org/v4l-utils.git
cd v4l-utils
./bootstrap.sh
./configure
make
sudo make install
> , 64 bits, 64-bit time_t
>
> Compliance test for hantro-vpu device /dev/video1:
>
> Driver Info:
> Driver name : hantro-vpu
> Card type : nxp,imx8mm-vpu-g2-dec
> Bus info : platform: hantro-vpu
>
> Driver v[ 30.136983] ------------[ cut here ]------------
> [ 30.142038] WARNING: CPU: 3 PID: 534 at mm/page_alloc.c:5344
> __alloc_pages+0x5b4/0xbe0
Hmm:
if (unlikely(order >= MAX_ORDER)) {
WARN_ON_ONCE(!(gfp & __GFP_NOWARN));
return NULL;
}
video needs loads of memory, so if MAX_ORDER is too low it might hit this.
It's tied to kernel config option FORCE_MAX_ZONEORDER.
The hantro driver uses vb2_dma_contig_memops, so the memory has to be physically
contiguous. That does indeed typically require larger MAX_ORDER values.
Regards,
Hans
> [ 30.149971] Modules linked in: 8021q garp mrp stp llc af_alg
> caam_jr caamhash_desc caamalg_desc crypto_engine rng_core authenc
> libdes crct10dif_ce snd_soc_fsl_asoc_card snd_soc_imx_audmux
> snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc hantro_vpu(C)
> v4l2_h264 brcmfmac v4l2_mem2mem videobuf2_vmalloc brcmutil
> videobuf2_dma_contig cfg80211 videobuf2_memops videobuf2_v4l2
> videobuf2_common snd_soc_wm8962 videodev hci_uart mc etnaviv gpu_sched
> btqca btbcm caam rtc_pcf85363 clk_bd718x7 at24 spi_imx spi_bitbang
> rtc_snvs error snvs_pwrkey snd_soc_fsl_sai imx_pcm_dma imx8mm_thermal
> imx_cpufreq_dt bluetooth ecdh_generic ecc rfkill fuse drm ipv6
> [ 30.206845] CPU: 3 PID: 534 Comm: v4l2-compliance Tainted: G
> C 5.16.0-rc1-00292-gb166e5dc4755-dirty #19
> [ 30.217544] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
> Development Kit (DT)
> [ 30.224940] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> [ 30.231904] pc : __alloc_pages+0x5b4/0xbe0
> [ 30.236007] lr : __dma_direct_alloc_pages+0x17c/0x1e0
> [ 30.241065] sp : ffff80001265b810
> [ 30.244379] x29: ffff80001265b810 x28: 0000000000000034 x27: ffff00000b9ec220
> [ 30.251523] x26: ffff00000038d810 x25: 0000000000000000 x24: ffff00000038d810
> [ 30.258668] x23: ffff80001010c52c x22: ffff00000038d810 x21: 00000000ffffffff
> [ 30.265813] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
> [ 30.272960] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
> [ 30.280101] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
> ersion : 5.16.0
> Capabilities [ 30.287247] x11: ffff80001231d000 x10:
> ffff00000b9ec33a x9 : ffff00000b9ec32a
> [ 30.297242] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
> [ 30.304385] x5 : 00000000bfffffff x4 : ffff00000a72f000 x3 : 0000000000000000
> [ 30.311529] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
> [ 30.318676] Call trace:
> [ 30.321124] __alloc_pages+0x5b4/0xbe0
> [ 30.324880] __dma_direct_alloc_pages+0x17c/0x1e0
> [ 30.329588] dma_direct_alloc+0x70/0x310
> [ 30.333514] dma_alloc_attrs+0x7c/0xe4
> [ 30.337266] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
> [ 30.343038] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
> [ 30.348805] device_run+0xac/0x110 [hantro_vpu]
> [ 30.353354] v4l2_m2m_try_run+0x84/0x134 [v4l2_mem2mem]
> [ 30.358603] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
> [ 30.364368] v4l_streamon+0x28/0x34 [videodev]
> [ 30.368874] __video_do_ioctl+0x178/0x3dc [videodev]
> [ 30.373889] video_usercopy+0x368/0x6dc [videodev]
> [ 30.378734] video_ioctl2+0x1c/0x30 [videodev]
> [ 30.383230] v4l2_ioctl+0x44/0x64 [videodev]
> [ 30.387554] __arm64_sys_ioctl+0xac/0xf0
> [ 30.391482] invoke_syscall+0x48/0x114
> [ 30.395238] el0_svc_common.constprop.0+0xd4/0xfc
> [ 30.399947] do_el0_svc+0x2c/0x94
> [ 30.403264] el0_svc+0x28/0x80
> [ 30.406327] el0t_64_sync_handler+0xa8/0x130
> [ 30.410603] el0t_64_sync+0x1a0/0x1a4
> [ 30.414272] ---[ end trace 878112336a378214 ]---
>
> I was hoping someone might have some suggestions on something to try..
>
> thanks,
>
> adam
>> Signed-off-by: Adam Ford <aford173@gmail.com>
>> ---
>> drivers/staging/media/hantro/hantro_drv.c | 2 +
>> drivers/staging/media/hantro/hantro_hw.h | 2 +
>> drivers/staging/media/hantro/imx8m_vpu_hw.c | 60 +++++++++++++++++++++
>> 3 files changed, 64 insertions(+)
>>
>> diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c
>> index fb82b9297a2b..2aa1c520be50 100644
>> --- a/drivers/staging/media/hantro/hantro_drv.c
>> +++ b/drivers/staging/media/hantro/hantro_drv.c
>> @@ -592,6 +592,8 @@ static const struct of_device_id of_hantro_match[] = {
>> { .compatible = "rockchip,rk3399-vpu", .data = &rk3399_vpu_variant, },
>> #endif
>> #ifdef CONFIG_VIDEO_HANTRO_IMX8M
>> + { .compatible = "nxp,imx8mm-vpu", .data = &imx8mm_vpu_variant, },
>> + { .compatible = "nxp,imx8mm-vpu-g2", .data = &imx8mm_vpu_g2_variant },
>> { .compatible = "nxp,imx8mq-vpu", .data = &imx8mq_vpu_variant, },
>> { .compatible = "nxp,imx8mq-vpu-g2", .data = &imx8mq_vpu_g2_variant },
>> #endif
>> diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h
>> index 267a6d33a47b..ae7c3fff760c 100644
>> --- a/drivers/staging/media/hantro/hantro_hw.h
>> +++ b/drivers/staging/media/hantro/hantro_hw.h
>> @@ -211,6 +211,8 @@ enum hantro_enc_fmt {
>> ROCKCHIP_VPU_ENC_FMT_UYVY422 = 3,
>> };
>>
>> +extern const struct hantro_variant imx8mm_vpu_g2_variant;
>> +extern const struct hantro_variant imx8mm_vpu_variant;
>> extern const struct hantro_variant imx8mq_vpu_g2_variant;
>> extern const struct hantro_variant imx8mq_vpu_variant;
>> extern const struct hantro_variant px30_vpu_variant;
>> diff --git a/drivers/staging/media/hantro/imx8m_vpu_hw.c b/drivers/staging/media/hantro/imx8m_vpu_hw.c
>> index ea919bfb9891..c819609d14d1 100644
>> --- a/drivers/staging/media/hantro/imx8m_vpu_hw.c
>> +++ b/drivers/staging/media/hantro/imx8m_vpu_hw.c
>> @@ -242,6 +242,32 @@ static const struct hantro_codec_ops imx8mq_vpu_g2_codec_ops[] = {
>> },
>> };
>>
>> +static const struct hantro_codec_ops imx8mm_vpu_codec_ops[] = {
>> + [HANTRO_MODE_MPEG2_DEC] = {
>> + .run = hantro_g1_mpeg2_dec_run,
>> + .init = hantro_mpeg2_dec_init,
>> + .exit = hantro_mpeg2_dec_exit,
>> + },
>> + [HANTRO_MODE_VP8_DEC] = {
>> + .run = hantro_g1_vp8_dec_run,
>> + .init = hantro_vp8_dec_init,
>> + .exit = hantro_vp8_dec_exit,
>> + },
>> + [HANTRO_MODE_H264_DEC] = {
>> + .run = hantro_g1_h264_dec_run,
>> + .init = hantro_h264_dec_init,
>> + .exit = hantro_h264_dec_exit,
>> + },
>> +};
>> +
>> +static const struct hantro_codec_ops imx8mm_vpu_g2_codec_ops[] = {
>> + [HANTRO_MODE_HEVC_DEC] = {
>> + .run = hantro_g2_hevc_dec_run,
>> + .init = hantro_hevc_dec_init,
>> + .exit = hantro_hevc_dec_exit,
>> + },
>> +};
>> +
>> /*
>> * VPU variants.
>> */
>> @@ -257,6 +283,11 @@ static const struct hantro_irq imx8mq_g2_irqs[] = {
>> static const char * const imx8mq_clk_names[] = { "g1", "g2", "bus" };
>> static const char * const imx8mq_reg_names[] = { "g1", "g2", "ctrl" };
>>
>> +static const char * const imx8mm_g1_clk_names[] = { "g1", "bus" };
>> +static const char * const imx8mm_g1_reg_names[] = { "g1" };
>> +static const char * const imx8mm_g2_clk_names[] = { "g2", "bus" };
>> +static const char * const imx8mm_g2_reg_names[] = { "g2" };
>> +
>> const struct hantro_variant imx8mq_vpu_variant = {
>> .dec_fmts = imx8m_vpu_dec_fmts,
>> .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts),
>> @@ -289,3 +320,32 @@ const struct hantro_variant imx8mq_vpu_g2_variant = {
>> .clk_names = imx8mq_clk_names,
>> .num_clocks = ARRAY_SIZE(imx8mq_clk_names),
>> };
>> +
>> +const struct hantro_variant imx8mm_vpu_variant = {
>> + .dec_fmts = imx8m_vpu_dec_fmts,
>> + .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_dec_fmts),
>> + .postproc_fmts = imx8m_vpu_postproc_fmts,
>> + .num_postproc_fmts = ARRAY_SIZE(imx8m_vpu_postproc_fmts),
>> + .postproc_regs = &hantro_g1_postproc_regs,
>> + .codec = HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER |
>> + HANTRO_H264_DECODER,
>> + .codec_ops = imx8mm_vpu_codec_ops,
>> + .irqs = imx8mq_irqs,
>> + .num_irqs = ARRAY_SIZE(imx8mq_irqs),
>> + .clk_names = imx8mm_g1_clk_names,
>> + .num_clocks = ARRAY_SIZE(imx8mm_g1_clk_names),
>> + .reg_names = imx8mm_g1_reg_names,
>> + .num_regs = ARRAY_SIZE(imx8mm_g1_reg_names)
>> +};
>> +
>> +const struct hantro_variant imx8mm_vpu_g2_variant = {
>> + .dec_offset = 0x0,
>> + .dec_fmts = imx8m_vpu_g2_dec_fmts,
>> + .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_g2_dec_fmts),
>> + .codec = HANTRO_HEVC_DECODER,
>> + .codec_ops = imx8mm_vpu_g2_codec_ops,
>> + .irqs = imx8mq_g2_irqs,
>> + .num_irqs = ARRAY_SIZE(imx8mq_g2_irqs),
>> + .clk_names = imx8mm_g2_clk_names,
>> + .num_clocks = ARRAY_SIZE(imx8mm_g2_reg_names),
>> +};
>> --
>> 2.32.0
>>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* [RFC 2/5] arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
2021-11-06 18:37 [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Adam Ford
2021-11-06 18:37 ` [RFC 1/5] media: hantro: Add support for i.MX8M Mini Adam Ford
@ 2021-11-06 18:37 ` Adam Ford
2021-11-06 18:37 ` [RFC 3/5] media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT Adam Ford
` (3 subsequent siblings)
5 siblings, 0 replies; 42+ messages in thread
From: Adam Ford @ 2021-11-06 18:37 UTC (permalink / raw)
To: linux-media
Cc: tharvey, frieder.schrempf, marek.vasut, jagan, aford, cstevens,
Adam Ford, Ezequiel Garcia, Philipp Zabel, Mauro Carvalho Chehab,
Rob Herring, Shawn Guo, Sascha Hauer, Pengutronix Kernel Team,
Fabio Estevam, NXP Linux Team, Greg Kroah-Hartman,
Heiko Stuebner, Lucas Stach, Joakim Zhang, Krzysztof Kozlowski,
Peng Fan, Alice Guo, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Enable two hardware Hantro decoders called G1 and G2.
Signed-off-by: Adam Ford <aford173@gmail.com>
---
arch/arm64/boot/dts/freescale/imx8mm.dtsi | 41 +++++++++++++++++++++++
1 file changed, 41 insertions(+)
diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
index 1f69c14d953f..725c3113831e 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
@@ -1248,6 +1248,47 @@ gpu_2d: gpu@38008000 {
power-domains = <&pgc_gpu>;
};
+ vpu_g1: video-codec@38300000 {
+ compatible = "nxp,imx8mm-vpu";
+ reg = <0x38300000 0x10000>;
+ reg-names = "g1";
+ interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "g1";
+ clocks = <&clk IMX8MM_CLK_VPU_G1_ROOT>,
+ <&clk IMX8MM_CLK_VPU_DEC_ROOT>;
+ clock-names = "g1", "bus";
+ assigned-clocks = <&clk IMX8MM_CLK_VPU_G1>,
+ <&clk IMX8MM_CLK_VPU_BUS>,
+ <&clk IMX8MM_VPU_PLL_BYPASS>;
+ assigned-clock-parents = <&clk IMX8MM_VPU_PLL_OUT>,
+ <&clk IMX8MM_SYS_PLL1_800M>,
+ <&clk IMX8MM_VPU_PLL>;
+ assigned-clock-rates = <600000000>,
+ <800000000>,
+ <0>;
+ power-domains = <&vpu_blk_ctrl IMX8MM_VPUBLK_PD_G1>;
+ };
+
+ vpu_g2: video-codec@38310000 {
+ compatible = "nxp,imx8mm-vpu-g2";
+ reg = <0x38310000 0x10000>;
+ interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "g2";
+ clocks = <&clk IMX8MM_CLK_VPU_G2_ROOT>,
+ <&clk IMX8MM_CLK_VPU_DEC_ROOT>;
+ clock-names = "g2", "bus";
+ assigned-clocks = <&clk IMX8MM_CLK_VPU_G2>,
+ <&clk IMX8MM_CLK_VPU_BUS>,
+ <&clk IMX8MM_VPU_PLL_BYPASS>;
+ assigned-clock-parents = <&clk IMX8MM_VPU_PLL_OUT>,
+ <&clk IMX8MM_SYS_PLL1_800M>,
+ <&clk IMX8MM_VPU_PLL>;
+ assigned-clock-rates = <600000000>,
+ <800000000>,
+ <0>;
+ power-domains = <&vpu_blk_ctrl IMX8MM_VPUBLK_PD_G2>;
+ };
+
vpu_blk_ctrl: blk-ctrl@38330000 {
compatible = "fsl,imx8mm-vpu-blk-ctrl", "syscon";
reg = <0x38330000 0x100>;
--
2.32.0
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply related [flat|nested] 42+ messages in thread* [RFC 3/5] media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
2021-11-06 18:37 [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Adam Ford
2021-11-06 18:37 ` [RFC 1/5] media: hantro: Add support for i.MX8M Mini Adam Ford
2021-11-06 18:37 ` [RFC 2/5] arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2 Adam Ford
@ 2021-11-06 18:37 ` Adam Ford
2021-11-06 18:38 ` [RFC 4/5] media: hantro: Add H1 encoder support on i.MX8M Mini Adam Ford
` (2 subsequent siblings)
5 siblings, 0 replies; 42+ messages in thread
From: Adam Ford @ 2021-11-06 18:37 UTC (permalink / raw)
To: linux-media
Cc: tharvey, frieder.schrempf, marek.vasut, jagan, aford, cstevens,
Adam Ford, Ezequiel Garcia, Philipp Zabel, Mauro Carvalho Chehab,
Rob Herring, Shawn Guo, Sascha Hauer, Pengutronix Kernel Team,
Fabio Estevam, NXP Linux Team, Greg Kroah-Hartman,
Heiko Stuebner, Lucas Stach, Joakim Zhang, Peng Fan, Alice Guo,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
The H1 encoder used on some Rockchip devices appears to be the
same or similar H1 encoder used on the i.MX8M Mini, so let's
rename the supported formats to a more generic term like
HANTRO_VPU_ENC_FMT.
There are no functional changes.
Signed-off-by: Adam Ford <aford173@gmail.com>
---
drivers/staging/media/hantro/hantro_hw.h | 16 ++++++------
.../staging/media/hantro/rockchip_vpu_hw.c | 26 +++++++++----------
2 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h
index ae7c3fff760c..c276ecd57066 100644
--- a/drivers/staging/media/hantro/hantro_hw.h
+++ b/drivers/staging/media/hantro/hantro_hw.h
@@ -199,16 +199,16 @@ struct hantro_codec_ops {
/**
* enum hantro_enc_fmt - source format ID for hardware registers.
*
- * @ROCKCHIP_VPU_ENC_FMT_YUV420P: Y/CbCr 4:2:0 planar format
- * @ROCKCHIP_VPU_ENC_FMT_YUV420SP: Y/CbCr 4:2:0 semi-planar format
- * @ROCKCHIP_VPU_ENC_FMT_YUYV422: YUV 4:2:2 packed format (YUYV)
- * @ROCKCHIP_VPU_ENC_FMT_UYVY422: YUV 4:2:2 packed format (UYVY)
+ * @HANTRO_VPU_ENC_FMT_YUV420P: Y/CbCr 4:2:0 planar format
+ * @HANTRO_VPU_ENC_FMT_YUV420SP: Y/CbCr 4:2:0 semi-planar format
+ * @HANTRO_VPU_ENC_FMT_YUYV422: YUV 4:2:2 packed format (YUYV)
+ * @HANTRO_VPU_ENC_FMT_UYVY422: YUV 4:2:2 packed format (UYVY)
*/
enum hantro_enc_fmt {
- ROCKCHIP_VPU_ENC_FMT_YUV420P = 0,
- ROCKCHIP_VPU_ENC_FMT_YUV420SP = 1,
- ROCKCHIP_VPU_ENC_FMT_YUYV422 = 2,
- ROCKCHIP_VPU_ENC_FMT_UYVY422 = 3,
+ HANTRO_VPU_ENC_FMT_YUV420P = 0,
+ HANTRO_VPU_ENC_FMT_YUV420SP = 1,
+ HANTRO_VPU_ENC_FMT_YUYV422 = 2,
+ HANTRO_VPU_ENC_FMT_UYVY422 = 3,
};
extern const struct hantro_variant imx8mm_vpu_g2_variant;
diff --git a/drivers/staging/media/hantro/rockchip_vpu_hw.c b/drivers/staging/media/hantro/rockchip_vpu_hw.c
index d4f52957cc53..7c8dc211dbc8 100644
--- a/drivers/staging/media/hantro/rockchip_vpu_hw.c
+++ b/drivers/staging/media/hantro/rockchip_vpu_hw.c
@@ -21,26 +21,26 @@
* Supported formats.
*/
-static const struct hantro_fmt rockchip_vpu_enc_fmts[] = {
+static const struct hantro_fmt HANTRO_VPU_ENC_FMTS[] = {
{
.fourcc = V4L2_PIX_FMT_YUV420M,
.codec_mode = HANTRO_MODE_NONE,
- .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUV420P,
+ .enc_fmt = HANTRO_VPU_ENC_FMT_YUV420P,
},
{
.fourcc = V4L2_PIX_FMT_NV12M,
.codec_mode = HANTRO_MODE_NONE,
- .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUV420SP,
+ .enc_fmt = HANTRO_VPU_ENC_FMT_YUV420SP,
},
{
.fourcc = V4L2_PIX_FMT_YUYV,
.codec_mode = HANTRO_MODE_NONE,
- .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUYV422,
+ .enc_fmt = HANTRO_VPU_ENC_FMT_YUYV422,
},
{
.fourcc = V4L2_PIX_FMT_UYVY,
.codec_mode = HANTRO_MODE_NONE,
- .enc_fmt = ROCKCHIP_VPU_ENC_FMT_UYVY422,
+ .enc_fmt = HANTRO_VPU_ENC_FMT_UYVY422,
},
{
.fourcc = V4L2_PIX_FMT_JPEG,
@@ -478,8 +478,8 @@ const struct hantro_variant rk3036_vpu_variant = {
*/
const struct hantro_variant rk3066_vpu_variant = {
.enc_offset = 0x0,
- .enc_fmts = rockchip_vpu_enc_fmts,
- .num_enc_fmts = ARRAY_SIZE(rockchip_vpu_enc_fmts),
+ .enc_fmts = HANTRO_VPU_ENC_FMTS,
+ .num_enc_fmts = ARRAY_SIZE(HANTRO_VPU_ENC_FMTS),
.dec_offset = 0x400,
.dec_fmts = rk3066_vpu_dec_fmts,
.num_dec_fmts = ARRAY_SIZE(rk3066_vpu_dec_fmts),
@@ -498,8 +498,8 @@ const struct hantro_variant rk3066_vpu_variant = {
const struct hantro_variant rk3288_vpu_variant = {
.enc_offset = 0x0,
- .enc_fmts = rockchip_vpu_enc_fmts,
- .num_enc_fmts = ARRAY_SIZE(rockchip_vpu_enc_fmts),
+ .enc_fmts = HANTRO_VPU_ENC_FMTS,
+ .num_enc_fmts = ARRAY_SIZE(HANTRO_VPU_ENC_FMTS),
.dec_offset = 0x400,
.dec_fmts = rk3288_vpu_dec_fmts,
.num_dec_fmts = ARRAY_SIZE(rk3288_vpu_dec_fmts),
@@ -534,8 +534,8 @@ const struct hantro_variant rk3328_vpu_variant = {
const struct hantro_variant rk3399_vpu_variant = {
.enc_offset = 0x0,
- .enc_fmts = rockchip_vpu_enc_fmts,
- .num_enc_fmts = ARRAY_SIZE(rockchip_vpu_enc_fmts),
+ .enc_fmts = HANTRO_VPU_ENC_FMTS,
+ .num_enc_fmts = ARRAY_SIZE(HANTRO_VPU_ENC_FMTS),
.dec_offset = 0x400,
.dec_fmts = rk3399_vpu_dec_fmts,
.num_dec_fmts = ARRAY_SIZE(rk3399_vpu_dec_fmts),
@@ -551,8 +551,8 @@ const struct hantro_variant rk3399_vpu_variant = {
const struct hantro_variant px30_vpu_variant = {
.enc_offset = 0x0,
- .enc_fmts = rockchip_vpu_enc_fmts,
- .num_enc_fmts = ARRAY_SIZE(rockchip_vpu_enc_fmts),
+ .enc_fmts = HANTRO_VPU_ENC_FMTS,
+ .num_enc_fmts = ARRAY_SIZE(HANTRO_VPU_ENC_FMTS),
.dec_offset = 0x400,
.dec_fmts = rk3399_vpu_dec_fmts,
.num_dec_fmts = ARRAY_SIZE(rk3399_vpu_dec_fmts),
--
2.32.0
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply related [flat|nested] 42+ messages in thread* [RFC 4/5] media: hantro: Add H1 encoder support on i.MX8M Mini
2021-11-06 18:37 [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Adam Ford
` (2 preceding siblings ...)
2021-11-06 18:37 ` [RFC 3/5] media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT Adam Ford
@ 2021-11-06 18:38 ` Adam Ford
2021-11-07 23:21 ` Adam Ford
2021-11-06 18:38 ` [RFC 5/5] arm64: dts: imx8mm: Enable Hantro H1 Encoder Adam Ford
2021-11-08 13:59 ` [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Nicolas Dufresne
5 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-06 18:38 UTC (permalink / raw)
To: linux-media
Cc: tharvey, frieder.schrempf, marek.vasut, jagan, aford, cstevens,
Adam Ford, Ezequiel Garcia, Philipp Zabel, Mauro Carvalho Chehab,
Rob Herring, Shawn Guo, Sascha Hauer, Pengutronix Kernel Team,
Fabio Estevam, NXP Linux Team, Greg Kroah-Hartman,
Heiko Stuebner, Lucas Stach, Joakim Zhang, Krzysztof Kozlowski,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
The i.MX8M Mini has supports the Hantro H1 encoder, so
enable it using the same supported formats as other devices
using the H1 encoder.
Signed-off-by: Adam Ford <aford173@gmail.com>
---
drivers/staging/media/hantro/hantro_drv.c | 1 +
drivers/staging/media/hantro/hantro_hw.h | 1 +
drivers/staging/media/hantro/imx8m_vpu_hw.c | 83 +++++++++++++++++++++
3 files changed, 85 insertions(+)
diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c
index 2aa1c520be50..29e8dc52c2e4 100644
--- a/drivers/staging/media/hantro/hantro_drv.c
+++ b/drivers/staging/media/hantro/hantro_drv.c
@@ -594,6 +594,7 @@ static const struct of_device_id of_hantro_match[] = {
#ifdef CONFIG_VIDEO_HANTRO_IMX8M
{ .compatible = "nxp,imx8mm-vpu", .data = &imx8mm_vpu_variant, },
{ .compatible = "nxp,imx8mm-vpu-g2", .data = &imx8mm_vpu_g2_variant },
+ { .compatible = "nxp,imx8mm-vpu-h1", .data = &imx8mm_vpu_h1_variant },
{ .compatible = "nxp,imx8mq-vpu", .data = &imx8mq_vpu_variant, },
{ .compatible = "nxp,imx8mq-vpu-g2", .data = &imx8mq_vpu_g2_variant },
#endif
diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h
index c276ecd57066..9f6ae5bf13ee 100644
--- a/drivers/staging/media/hantro/hantro_hw.h
+++ b/drivers/staging/media/hantro/hantro_hw.h
@@ -212,6 +212,7 @@ enum hantro_enc_fmt {
};
extern const struct hantro_variant imx8mm_vpu_g2_variant;
+extern const struct hantro_variant imx8mm_vpu_h1_variant;
extern const struct hantro_variant imx8mm_vpu_variant;
extern const struct hantro_variant imx8mq_vpu_g2_variant;
extern const struct hantro_variant imx8mq_vpu_variant;
diff --git a/drivers/staging/media/hantro/imx8m_vpu_hw.c b/drivers/staging/media/hantro/imx8m_vpu_hw.c
index c819609d14d1..69760f88efa5 100644
--- a/drivers/staging/media/hantro/imx8m_vpu_hw.c
+++ b/drivers/staging/media/hantro/imx8m_vpu_hw.c
@@ -12,6 +12,7 @@
#include "hantro_jpeg.h"
#include "hantro_g1_regs.h"
#include "hantro_g2_regs.h"
+#include "hantro_h1_regs.h"
#define CTRL_SOFT_RESET 0x00
#define RESET_G1 BIT(1)
@@ -151,6 +152,43 @@ static const struct hantro_fmt imx8m_vpu_g2_dec_fmts[] = {
},
};
+static const struct hantro_fmt imx8m_vpu_h1_enc_fmts[] = {
+ {
+ .fourcc = V4L2_PIX_FMT_YUV420M,
+ .codec_mode = HANTRO_MODE_NONE,
+ .enc_fmt = HANTRO_VPU_ENC_FMT_YUV420P,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_NV12M,
+ .codec_mode = HANTRO_MODE_NONE,
+ .enc_fmt = HANTRO_VPU_ENC_FMT_YUV420SP,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_YUYV,
+ .codec_mode = HANTRO_MODE_NONE,
+ .enc_fmt = HANTRO_VPU_ENC_FMT_YUYV422,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_UYVY,
+ .codec_mode = HANTRO_MODE_NONE,
+ .enc_fmt = HANTRO_VPU_ENC_FMT_UYVY422,
+ },
+ {
+ .fourcc = V4L2_PIX_FMT_JPEG,
+ .codec_mode = HANTRO_MODE_JPEG_ENC,
+ .max_depth = 2,
+ .header_size = JPEG_HEADER_SIZE,
+ .frmsize = {
+ .min_width = 96,
+ .max_width = 8192,
+ .step_width = MB_DIM,
+ .min_height = 32,
+ .max_height = 8192,
+ .step_height = MB_DIM,
+ },
+ },
+};
+
static irqreturn_t imx8m_vpu_g1_irq(int irq, void *dev_id)
{
struct hantro_dev *vpu = dev_id;
@@ -187,6 +225,24 @@ static irqreturn_t imx8m_vpu_g2_irq(int irq, void *dev_id)
return IRQ_HANDLED;
}
+static irqreturn_t imx8m_vpu_h1_irq(int irq, void *dev_id)
+{
+ struct hantro_dev *vpu = dev_id;
+ enum vb2_buffer_state state;
+ u32 status;
+
+ status = vepu_read(vpu, H1_REG_INTERRUPT);
+ state = (status & H1_REG_INTERRUPT_FRAME_RDY) ?
+ VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR;
+
+ vepu_write(vpu, 0, H1_REG_INTERRUPT);
+ vepu_write(vpu, 0, H1_REG_AXI_CTRL);
+
+ hantro_irq_done(vpu, state);
+
+ return IRQ_HANDLED;
+}
+
static int imx8mq_vpu_hw_init(struct hantro_dev *vpu)
{
vpu->ctrl_base = vpu->reg_bases[vpu->variant->num_regs - 1];
@@ -268,6 +324,15 @@ static const struct hantro_codec_ops imx8mm_vpu_g2_codec_ops[] = {
},
};
+static const struct hantro_codec_ops imx8mm_vpu_h1_codec_ops[] = {
+ [HANTRO_MODE_JPEG_ENC] = {
+ .run = hantro_h1_jpeg_enc_run,
+ .init = hantro_jpeg_enc_init,
+ .done = hantro_jpeg_enc_done,
+ .exit = hantro_jpeg_enc_exit,
+ },
+};
+
/*
* VPU variants.
*/
@@ -280,6 +345,10 @@ static const struct hantro_irq imx8mq_g2_irqs[] = {
{ "g2", imx8m_vpu_g2_irq },
};
+static const struct hantro_irq imx8mq_h1_irqs[] = {
+ { "h1", imx8m_vpu_h1_irq },
+};
+
static const char * const imx8mq_clk_names[] = { "g1", "g2", "bus" };
static const char * const imx8mq_reg_names[] = { "g1", "g2", "ctrl" };
@@ -287,6 +356,8 @@ static const char * const imx8mm_g1_clk_names[] = { "g1", "bus" };
static const char * const imx8mm_g1_reg_names[] = { "g1" };
static const char * const imx8mm_g2_clk_names[] = { "g2", "bus" };
static const char * const imx8mm_g2_reg_names[] = { "g2" };
+static const char * const imx8mm_h1_clk_names[] = { "h1", "bus" };
+static const char * const imx8mm_h1_reg_names[] = { "h1" };
const struct hantro_variant imx8mq_vpu_variant = {
.dec_fmts = imx8m_vpu_dec_fmts,
@@ -349,3 +420,15 @@ const struct hantro_variant imx8mm_vpu_g2_variant = {
.clk_names = imx8mm_g2_clk_names,
.num_clocks = ARRAY_SIZE(imx8mm_g2_reg_names),
};
+
+const struct hantro_variant imx8mm_vpu_h1_variant = {
+ .enc_offset = 0x0,
+ .enc_fmts = imx8m_vpu_h1_enc_fmts,
+ .num_enc_fmts = ARRAY_SIZE(imx8m_vpu_h1_enc_fmts),
+ .codec = HANTRO_JPEG_ENCODER,
+ .codec_ops = imx8mm_vpu_h1_codec_ops,
+ .irqs = imx8mq_h1_irqs,
+ .num_irqs = ARRAY_SIZE(imx8mq_h1_irqs),
+ .clk_names = imx8mm_h1_clk_names,
+ .num_clocks = ARRAY_SIZE(imx8mm_h1_clk_names)
+};
--
2.32.0
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply related [flat|nested] 42+ messages in thread* Re: [RFC 4/5] media: hantro: Add H1 encoder support on i.MX8M Mini
2021-11-06 18:38 ` [RFC 4/5] media: hantro: Add H1 encoder support on i.MX8M Mini Adam Ford
@ 2021-11-07 23:21 ` Adam Ford
0 siblings, 0 replies; 42+ messages in thread
From: Adam Ford @ 2021-11-07 23:21 UTC (permalink / raw)
To: linux-media
Cc: Tim Harvey, Schrempf Frieder, Marek Vasut, Jagan Teki,
Adam Ford-BE, cstevens, Ezequiel Garcia, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Krzysztof Kozlowski, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Sat, Nov 6, 2021 at 1:38 PM Adam Ford <aford173@gmail.com> wrote:
>
> The i.MX8M Mini has supports the Hantro H1 encoder, so
> enable it using the same supported formats as other devices
> using the H1 encoder.
>
> Signed-off-by: Adam Ford <aford173@gmail.com>
> ---
I randomly checked one or two registers when I first attempted this
patch, but it does appear that the H1 is likely not to work yet due to
too many differences. It appears that patches 4 and 5 will likely
have to be dropped until they can be investigated and reworked.
I am still running into some power domain issues for the rest of the
series where I cannot read the regmaps for the vpu blk-ctrl register
even when H1 is disabled.
adam
> drivers/staging/media/hantro/hantro_drv.c | 1 +
> drivers/staging/media/hantro/hantro_hw.h | 1 +
> drivers/staging/media/hantro/imx8m_vpu_hw.c | 83 +++++++++++++++++++++
> 3 files changed, 85 insertions(+)
>
> diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c
> index 2aa1c520be50..29e8dc52c2e4 100644
> --- a/drivers/staging/media/hantro/hantro_drv.c
> +++ b/drivers/staging/media/hantro/hantro_drv.c
> @@ -594,6 +594,7 @@ static const struct of_device_id of_hantro_match[] = {
> #ifdef CONFIG_VIDEO_HANTRO_IMX8M
> { .compatible = "nxp,imx8mm-vpu", .data = &imx8mm_vpu_variant, },
> { .compatible = "nxp,imx8mm-vpu-g2", .data = &imx8mm_vpu_g2_variant },
> + { .compatible = "nxp,imx8mm-vpu-h1", .data = &imx8mm_vpu_h1_variant },
> { .compatible = "nxp,imx8mq-vpu", .data = &imx8mq_vpu_variant, },
> { .compatible = "nxp,imx8mq-vpu-g2", .data = &imx8mq_vpu_g2_variant },
> #endif
> diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h
> index c276ecd57066..9f6ae5bf13ee 100644
> --- a/drivers/staging/media/hantro/hantro_hw.h
> +++ b/drivers/staging/media/hantro/hantro_hw.h
> @@ -212,6 +212,7 @@ enum hantro_enc_fmt {
> };
>
> extern const struct hantro_variant imx8mm_vpu_g2_variant;
> +extern const struct hantro_variant imx8mm_vpu_h1_variant;
> extern const struct hantro_variant imx8mm_vpu_variant;
> extern const struct hantro_variant imx8mq_vpu_g2_variant;
> extern const struct hantro_variant imx8mq_vpu_variant;
> diff --git a/drivers/staging/media/hantro/imx8m_vpu_hw.c b/drivers/staging/media/hantro/imx8m_vpu_hw.c
> index c819609d14d1..69760f88efa5 100644
> --- a/drivers/staging/media/hantro/imx8m_vpu_hw.c
> +++ b/drivers/staging/media/hantro/imx8m_vpu_hw.c
> @@ -12,6 +12,7 @@
> #include "hantro_jpeg.h"
> #include "hantro_g1_regs.h"
> #include "hantro_g2_regs.h"
> +#include "hantro_h1_regs.h"
>
> #define CTRL_SOFT_RESET 0x00
> #define RESET_G1 BIT(1)
> @@ -151,6 +152,43 @@ static const struct hantro_fmt imx8m_vpu_g2_dec_fmts[] = {
> },
> };
>
> +static const struct hantro_fmt imx8m_vpu_h1_enc_fmts[] = {
> + {
> + .fourcc = V4L2_PIX_FMT_YUV420M,
> + .codec_mode = HANTRO_MODE_NONE,
> + .enc_fmt = HANTRO_VPU_ENC_FMT_YUV420P,
> + },
> + {
> + .fourcc = V4L2_PIX_FMT_NV12M,
> + .codec_mode = HANTRO_MODE_NONE,
> + .enc_fmt = HANTRO_VPU_ENC_FMT_YUV420SP,
> + },
> + {
> + .fourcc = V4L2_PIX_FMT_YUYV,
> + .codec_mode = HANTRO_MODE_NONE,
> + .enc_fmt = HANTRO_VPU_ENC_FMT_YUYV422,
> + },
> + {
> + .fourcc = V4L2_PIX_FMT_UYVY,
> + .codec_mode = HANTRO_MODE_NONE,
> + .enc_fmt = HANTRO_VPU_ENC_FMT_UYVY422,
> + },
> + {
> + .fourcc = V4L2_PIX_FMT_JPEG,
> + .codec_mode = HANTRO_MODE_JPEG_ENC,
> + .max_depth = 2,
> + .header_size = JPEG_HEADER_SIZE,
> + .frmsize = {
> + .min_width = 96,
> + .max_width = 8192,
> + .step_width = MB_DIM,
> + .min_height = 32,
> + .max_height = 8192,
> + .step_height = MB_DIM,
> + },
> + },
> +};
> +
> static irqreturn_t imx8m_vpu_g1_irq(int irq, void *dev_id)
> {
> struct hantro_dev *vpu = dev_id;
> @@ -187,6 +225,24 @@ static irqreturn_t imx8m_vpu_g2_irq(int irq, void *dev_id)
> return IRQ_HANDLED;
> }
>
> +static irqreturn_t imx8m_vpu_h1_irq(int irq, void *dev_id)
> +{
> + struct hantro_dev *vpu = dev_id;
> + enum vb2_buffer_state state;
> + u32 status;
> +
> + status = vepu_read(vpu, H1_REG_INTERRUPT);
> + state = (status & H1_REG_INTERRUPT_FRAME_RDY) ?
> + VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR;
> +
> + vepu_write(vpu, 0, H1_REG_INTERRUPT);
> + vepu_write(vpu, 0, H1_REG_AXI_CTRL);
> +
> + hantro_irq_done(vpu, state);
> +
> + return IRQ_HANDLED;
> +}
> +
> static int imx8mq_vpu_hw_init(struct hantro_dev *vpu)
> {
> vpu->ctrl_base = vpu->reg_bases[vpu->variant->num_regs - 1];
> @@ -268,6 +324,15 @@ static const struct hantro_codec_ops imx8mm_vpu_g2_codec_ops[] = {
> },
> };
>
> +static const struct hantro_codec_ops imx8mm_vpu_h1_codec_ops[] = {
> + [HANTRO_MODE_JPEG_ENC] = {
> + .run = hantro_h1_jpeg_enc_run,
> + .init = hantro_jpeg_enc_init,
> + .done = hantro_jpeg_enc_done,
> + .exit = hantro_jpeg_enc_exit,
> + },
> +};
> +
> /*
> * VPU variants.
> */
> @@ -280,6 +345,10 @@ static const struct hantro_irq imx8mq_g2_irqs[] = {
> { "g2", imx8m_vpu_g2_irq },
> };
>
> +static const struct hantro_irq imx8mq_h1_irqs[] = {
> + { "h1", imx8m_vpu_h1_irq },
> +};
> +
> static const char * const imx8mq_clk_names[] = { "g1", "g2", "bus" };
> static const char * const imx8mq_reg_names[] = { "g1", "g2", "ctrl" };
>
> @@ -287,6 +356,8 @@ static const char * const imx8mm_g1_clk_names[] = { "g1", "bus" };
> static const char * const imx8mm_g1_reg_names[] = { "g1" };
> static const char * const imx8mm_g2_clk_names[] = { "g2", "bus" };
> static const char * const imx8mm_g2_reg_names[] = { "g2" };
> +static const char * const imx8mm_h1_clk_names[] = { "h1", "bus" };
> +static const char * const imx8mm_h1_reg_names[] = { "h1" };
>
> const struct hantro_variant imx8mq_vpu_variant = {
> .dec_fmts = imx8m_vpu_dec_fmts,
> @@ -349,3 +420,15 @@ const struct hantro_variant imx8mm_vpu_g2_variant = {
> .clk_names = imx8mm_g2_clk_names,
> .num_clocks = ARRAY_SIZE(imx8mm_g2_reg_names),
> };
> +
> +const struct hantro_variant imx8mm_vpu_h1_variant = {
> + .enc_offset = 0x0,
> + .enc_fmts = imx8m_vpu_h1_enc_fmts,
> + .num_enc_fmts = ARRAY_SIZE(imx8m_vpu_h1_enc_fmts),
> + .codec = HANTRO_JPEG_ENCODER,
> + .codec_ops = imx8mm_vpu_h1_codec_ops,
> + .irqs = imx8mq_h1_irqs,
> + .num_irqs = ARRAY_SIZE(imx8mq_h1_irqs),
> + .clk_names = imx8mm_h1_clk_names,
> + .num_clocks = ARRAY_SIZE(imx8mm_h1_clk_names)
> +};
> --
> 2.32.0
>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* [RFC 5/5] arm64: dts: imx8mm: Enable Hantro H1 Encoder
2021-11-06 18:37 [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Adam Ford
` (3 preceding siblings ...)
2021-11-06 18:38 ` [RFC 4/5] media: hantro: Add H1 encoder support on i.MX8M Mini Adam Ford
@ 2021-11-06 18:38 ` Adam Ford
2021-11-08 13:59 ` [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Nicolas Dufresne
5 siblings, 0 replies; 42+ messages in thread
From: Adam Ford @ 2021-11-06 18:38 UTC (permalink / raw)
To: linux-media
Cc: tharvey, frieder.schrempf, marek.vasut, jagan, aford, cstevens,
Adam Ford, Ezequiel Garcia, Philipp Zabel, Mauro Carvalho Chehab,
Rob Herring, Shawn Guo, Sascha Hauer, Pengutronix Kernel Team,
Fabio Estevam, NXP Linux Team, Greg Kroah-Hartman,
Heiko Stuebner, Lucas Stach, Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
The i.MX8M Mini has a Hantro H1 video encoder which appears
as a media device.
Media device information
------------------------
driver hantro-vpu
model hantro-vpu
serial
bus info platform: hantro-vpu
hw revision 0x0
driver version 5.15.0
Device topology
- entity 1: nxp,imx8mm-vpu-h1-enc-source (1 pad, 1 link)
type Node subtype V4L flags 0
device node name /dev/video3
pad0: Source
-> "nxp,imx8mm-vpu-h1-enc-proc":0 [ENABLED,IMMUTABLE]
- entity 3: nxp,imx8mm-vpu-h1-enc-proc (2 pads, 2 links)
type Node subtype Unknown flags 0
pad0: Sink
<- "nxp,imx8mm-vpu-h1-enc-source":0 [ENABLED,IMMUTABLE]
pad1: Source
-> "nxp,imx8mm-vpu-h1-enc-sink":0 [ENABLED,IMMUTABLE]
- entity 6: nxp,imx8mm-vpu-h1-enc-sink (1 pad, 1 link)
type Node subtype V4L flags 0
device node name /dev/video3
pad0: Sink
<- "nxp,imx8mm-vpu-h1-enc-proc":1 [ENABLED,IMMUTABLE]
Signed-off-by: Adam Ford <aford173@gmail.com>
---
arch/arm64/boot/dts/freescale/imx8mm.dtsi | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/arch/arm64/boot/dts/freescale/imx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
index 725c3113831e..b4c204cbced8 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm.dtsi
@@ -1289,6 +1289,26 @@ vpu_g2: video-codec@38310000 {
power-domains = <&vpu_blk_ctrl IMX8MM_VPUBLK_PD_G2>;
};
+ vpu_h1: video-codec@38320000 {
+ compatible = "nxp,imx8mm-vpu-h1";
+ reg = <0x38320000 0x10000>;
+ interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+ interrupt-names = "h1";
+ clocks = <&clk IMX8MM_CLK_VPU_H1_ROOT>,
+ <&clk IMX8MM_CLK_VPU_DEC_ROOT>;
+ clock-names = "h1", "bus";
+ assigned-clocks = <&clk IMX8MM_CLK_VPU_H1>,
+ <&clk IMX8MM_CLK_VPU_BUS>,
+ <&clk IMX8MM_VPU_PLL_BYPASS>;
+ assigned-clock-parents = <&clk IMX8MM_VPU_PLL_OUT>,
+ <&clk IMX8MM_SYS_PLL1_800M>,
+ <&clk IMX8MM_VPU_PLL>;
+ assigned-clock-rates = <600000000>,
+ <800000000>,
+ <0>;
+ power-domains = <&vpu_blk_ctrl IMX8MM_VPUBLK_PD_H1>;
+ };
+
vpu_blk_ctrl: blk-ctrl@38330000 {
compatible = "fsl,imx8mm-vpu-blk-ctrl", "syscon";
reg = <0x38330000 0x100>;
--
2.32.0
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply related [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-06 18:37 [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Adam Ford
` (4 preceding siblings ...)
2021-11-06 18:38 ` [RFC 5/5] arm64: dts: imx8mm: Enable Hantro H1 Encoder Adam Ford
@ 2021-11-08 13:59 ` Nicolas Dufresne
2021-11-08 16:33 ` Adam Ford
5 siblings, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-11-08 13:59 UTC (permalink / raw)
To: Adam Ford, linux-media
Cc: tharvey, frieder.schrempf, marek.vasut, jagan, aford, cstevens,
Ezequiel Garcia, Philipp Zabel, Mauro Carvalho Chehab,
Rob Herring, Shawn Guo, Sascha Hauer, Pengutronix Kernel Team,
Fabio Estevam, NXP Linux Team, Greg Kroah-Hartman,
Heiko Stuebner, Lucas Stach, Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Hi Adam,
thanks for you work, I'll try and reply about the GStreamer questions below, if
you have further question feel free to ask.
Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> to be related to the video decoders used on the i.MX8MQ, but because of
> how the Mini handles the power domains, the VPU driver does not need to
> handle all the functions, so a new compatible flag is required.
>
> This is an RFC because I don't have functional video on my system yet,
> and I'm hoping there might be people who do and can help test this.
> I have only tested this far enough to see they enumerate and appear
> as /dev/videoX and /dev/mediaX devices.
I will check the patchset, but you need in the mini-variant to disable the G1
post processor, because this block was fused out. We didn't make it optional
from the start as according to the V1 of the TRM it was there, but that error
was corrected in V3.
>
> I am also curious to know if/what gstreamer plugins are necessary. In
> NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> would be more generic plugins that I can use to test. I am hoping some
> of the linux-media experts might chime in on how to best test.
I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
H264 and VP8. Remember that the related plugin depends on libgudev (a glib
binding of udev).
For the encoder, I believe only JPEG maybe supported, since this is all there is
mainline for RK3288 (and perhaps other RK). But this will need testing and
debugging as the G1 version is slightly newer on NXP SoC.
>
> Lastly, I didn't update any device tree binding YAML files, because
> I know there have been some discussions about the power domains on the
> imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> or if the existing one for te imx8mq should just be modified.
>
> This will likely require the following series in order to apply correctly:
> https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
>
> Adam Ford (5):
> media: hantro: Add support for i.MX8M Mini
> arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> media: hantro: Add H1 encoder support on i.MX8M Mini
> arm64: dts: imx8mm: Enable Hantro H1 Encoder
>
> arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> drivers/staging/media/hantro/hantro_drv.c | 3 +
> drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> 5 files changed, 231 insertions(+), 21 deletions(-)
>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-08 13:59 ` [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs Nicolas Dufresne
@ 2021-11-08 16:33 ` Adam Ford
2021-11-09 15:57 ` Nicolas Dufresne
0 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-08 16:33 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: linux-media, Tim Harvey, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Mon, Nov 8, 2021 at 7:59 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
>
> Hi Adam,
>
> thanks for you work, I'll try and reply about the GStreamer questions below, if
> you have further question feel free to ask.
>
> Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > to be related to the video decoders used on the i.MX8MQ, but because of
> > how the Mini handles the power domains, the VPU driver does not need to
> > handle all the functions, so a new compatible flag is required.
> >
> > This is an RFC because I don't have functional video on my system yet,
> > and I'm hoping there might be people who do and can help test this.
> > I have only tested this far enough to see they enumerate and appear
> > as /dev/videoX and /dev/mediaX devices.
>
> I will check the patchset, but you need in the mini-variant to disable the G1
> post processor, because this block was fused out. We didn't make it optional
Thanks for being willing to review this.
> from the start as according to the V1 of the TRM it was there, but that error
> was corrected in V3.
Thanks for the clarification. It wasn't obvious to me, because in
some instances the PP looked like it was there and sometimes not
there. I'll remove the postproc stuff.
>
> >
> > I am also curious to know if/what gstreamer plugins are necessary. In
> > NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> > would be more generic plugins that I can use to test. I am hoping some
> > of the linux-media experts might chime in on how to best test.
>
> I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
> single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
> H264 and VP8. Remember that the related plugin depends on libgudev (a glib
> binding of udev).
Thanks for the tip.
>
> For the encoder, I believe only JPEG maybe supported, since this is all there is
> mainline for RK3288 (and perhaps other RK). But this will need testing and
> debugging as the G1 version is slightly newer on NXP SoC.
For what it's worth the G1 seems to repond cleanly to the inquiries
from v42-compliance.
The G2 throws some splat when I run v4l2-compliance, but I am still
investigating that.
[ 405.456979] ------------[ cut here ]------------
[ 405.464173] WARNING: CPU: 0 PID: 563 at mm/page_alloc.c:5344
__alloc_pages+0x5a4/0xbe0
[ 405.472104] Modules linked in: 8021q garp mrp stp llc caam_jr
caamhash_desc caamalg_desc crypto_engine rng_core authenc libdes
imx7_media_csi(C) crct10dif_ce imx_media_common(C)
snd_soc_fsl_asoc_card imx7_mipi_csis(C) snd_soc_imx_audmux
snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc brcmfmac
brcmutil hantro_vpu(C) v4l2_h264 v4l2_mem2mem videobuf2_vmalloc
videobuf2_dma_contig videobuf2_memops cfg80211 ov5640 videobuf2_v4l2
v4l2_fwnode v4l2_async videobuf2_common videodev etnaviv gpu_sched
hci_uart mc btqca btbcm snd_soc_wm8962 at24 spi_imx rtc_pcf85363
rtc_snvs clk_bd718x7 spi_bitbang snvs_pwrkey snd_soc_fsl_sai
imx_pcm_dma caam error bluetooth imx8mm_thermal ecdh_generic
imx_cpufreq_dt ecc rfkill fuse drm ipv6
[ 405.535835] CPU: 0 PID: 563 Comm: v4l2-compliance Tainted: G D
C 5.15.0-next-20211105-00010-g4bb8e8a25d3c-dirty #28
[ 405.547401] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
Development Kit (DT)
[ 405.554797] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 405.561762] pc : __alloc_pages+0x5a4/0xbe0
[ 405.565861] lr : __dma_direct_alloc_pages+0x17c/0x1e0
[ 405.570917] sp : ffff800012443810
[ 405.574232] x29: ffff800012443810 x28: 0000000000000000 x27: ffff000005288220
[ 405.581375] x26: 0000000000000034 x25: 0000000000000000 x24: ffff000000259010
[ 405.588517] x23: ffff80001011ab7c x22: ffff000000259010 x21: 00000000ffffffff
[ 405.595659] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
[ 405.602803] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
[ 405.609947] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
[ 405.617090] x11: ffff80001241d000 x10: ffff00000528833a x9 : ffff00000528832a
[ 405.624232] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
[ 405.631378] x5 : 00000000bfffffff x4 : ffff000009e30dc0 x3 : 0000000000000000
[ 405.638520] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
[ 405.645666] Call trace:
[ 405.648113] __alloc_pages+0x5a4/0xbe0
[ 405.651862] __dma_direct_alloc_pages+0x17c/0x1e0
[ 405.656569] dma_direct_alloc+0x70/0x310
[ 405.660494] dma_alloc_attrs+0x7c/0xe4
[ 405.664246] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
[ 405.670021] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
[ 405.675791] device_run+0xac/0x110 [hantro_vpu]
[ 405.680345] v4l2_m2m_try_run+0xac/0x1b0 [v4l2_mem2mem]
[ 405.685598] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
[ 405.691366] v4l_streamon+0x28/0x34 [videodev]
[ 405.695877] __video_do_ioctl+0x178/0x3dc [videodev]
[ 405.700897] video_usercopy+0x368/0x6dc [videodev]
[ 405.705745] video_ioctl2+0x1c/0x30 [videodev]
[ 405.710246] v4l2_ioctl+0x44/0x64 [videodev]
[ 405.714574] __arm64_sys_ioctl+0xac/0xf0
[ 405.718502] invoke_syscall+0x48/0x114
[ 405.722258] el0_svc_common.constprop.0+0xd4/0xfc
[ 405.726969] do_el0_svc+0x2c/0x94
[ 405.730286] el0_svc+0x28/0x80
[ 405.733348] el0t_64_sync_handler+0xa8/0x130
[ 405.737619] el0t_64_sync+0x1a0/0x1a4
[ 405.741287] ---[ end trace 270ed4a899803006 ]---
The H1 encoder seems to hang the system when I run v4l2-compliance on
it when H1 is set up as I submitted the patch. I tried dropping all
the encoder formats except the JPEG format, and it doesn't hang any
more, but it also doesn't really do anything.
The datasheet only references VPU_H1 as supporting VP8 and H.264, so I
am not sure JPEG is even supported.
The log from v4l2-compliance on the H1 with everything except the JPEG
removed looks like:
root@beacon-imx8mm-kit:~# v4l2-compliance -d2
v4l2-compliance SHA: not available
, 64 bits, 64-bit time_t
Segmentation fault
root@beacon-imx8mm-kit:~#
Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
: Internal error: Oops: 96000004 [#2] SMP
Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
: Code: 52800001 aa1403e0 d2801802 95c31ab9 (b9400aa1)
I want to install Gstreamer, but I don't have functioning DSI video,
so I am not entirely sure how I will go about testing the decoders
except by using fakesink
If the G1 ends up working with some of the newer Gstreamer stuff, I
might just submit a formal patch to just enable the G1 for now.
adam
>
> >
> > Lastly, I didn't update any device tree binding YAML files, because
> > I know there have been some discussions about the power domains on the
> > imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> > or if the existing one for te imx8mq should just be modified.
> >
> > This will likely require the following series in order to apply correctly:
> > https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
> >
> > Adam Ford (5):
> > media: hantro: Add support for i.MX8M Mini
> > arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> > media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> > media: hantro: Add H1 encoder support on i.MX8M Mini
> > arm64: dts: imx8mm: Enable Hantro H1 Encoder
> >
> > arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> > drivers/staging/media/hantro/hantro_drv.c | 3 +
> > drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> > drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> > .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> > 5 files changed, 231 insertions(+), 21 deletions(-)
> >
>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-08 16:33 ` Adam Ford
@ 2021-11-09 15:57 ` Nicolas Dufresne
2021-11-16 23:23 ` Tim Harvey
0 siblings, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-11-09 15:57 UTC (permalink / raw)
To: Adam Ford
Cc: linux-media, Tim Harvey, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le lundi 08 novembre 2021 à 10:33 -0600, Adam Ford a écrit :
> On Mon, Nov 8, 2021 at 7:59 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> >
> > Hi Adam,
> >
> > thanks for you work, I'll try and reply about the GStreamer questions below, if
> > you have further question feel free to ask.
> >
> > Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > how the Mini handles the power domains, the VPU driver does not need to
> > > handle all the functions, so a new compatible flag is required.
> > >
> > > This is an RFC because I don't have functional video on my system yet,
> > > and I'm hoping there might be people who do and can help test this.
> > > I have only tested this far enough to see they enumerate and appear
> > > as /dev/videoX and /dev/mediaX devices.
> >
> > I will check the patchset, but you need in the mini-variant to disable the G1
> > post processor, because this block was fused out. We didn't make it optional
>
> Thanks for being willing to review this.
>
> > from the start as according to the V1 of the TRM it was there, but that error
> > was corrected in V3.
>
> Thanks for the clarification. It wasn't obvious to me, because in
> some instances the PP looked like it was there and sometimes not
> there. I'll remove the postproc stuff.
>
> >
> > >
> > > I am also curious to know if/what gstreamer plugins are necessary. In
> > > NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> > > would be more generic plugins that I can use to test. I am hoping some
> > > of the linux-media experts might chime in on how to best test.
> >
> > I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
> > single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
> > H264 and VP8. Remember that the related plugin depends on libgudev (a glib
> > binding of udev).
>
> Thanks for the tip.
>
> >
> > For the encoder, I believe only JPEG maybe supported, since this is all there is
> > mainline for RK3288 (and perhaps other RK). But this will need testing and
> > debugging as the G1 version is slightly newer on NXP SoC.
>
> For what it's worth the G1 seems to repond cleanly to the inquiries
> from v42-compliance.
> The G2 throws some splat when I run v4l2-compliance, but I am still
> investigating that.
>
> [ 405.456979] ------------[ cut here ]------------
> [ 405.464173] WARNING: CPU: 0 PID: 563 at mm/page_alloc.c:5344
> __alloc_pages+0x5a4/0xbe0
> [ 405.472104] Modules linked in: 8021q garp mrp stp llc caam_jr
> caamhash_desc caamalg_desc crypto_engine rng_core authenc libdes
> imx7_media_csi(C) crct10dif_ce imx_media_common(C)
> snd_soc_fsl_asoc_card imx7_mipi_csis(C) snd_soc_imx_audmux
> snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc brcmfmac
> brcmutil hantro_vpu(C) v4l2_h264 v4l2_mem2mem videobuf2_vmalloc
> videobuf2_dma_contig videobuf2_memops cfg80211 ov5640 videobuf2_v4l2
> v4l2_fwnode v4l2_async videobuf2_common videodev etnaviv gpu_sched
> hci_uart mc btqca btbcm snd_soc_wm8962 at24 spi_imx rtc_pcf85363
> rtc_snvs clk_bd718x7 spi_bitbang snvs_pwrkey snd_soc_fsl_sai
> imx_pcm_dma caam error bluetooth imx8mm_thermal ecdh_generic
> imx_cpufreq_dt ecc rfkill fuse drm ipv6
> [ 405.535835] CPU: 0 PID: 563 Comm: v4l2-compliance Tainted: G D
> C 5.15.0-next-20211105-00010-g4bb8e8a25d3c-dirty #28
> [ 405.547401] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
> Development Kit (DT)
> [ 405.554797] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> [ 405.561762] pc : __alloc_pages+0x5a4/0xbe0
> [ 405.565861] lr : __dma_direct_alloc_pages+0x17c/0x1e0
> [ 405.570917] sp : ffff800012443810
> [ 405.574232] x29: ffff800012443810 x28: 0000000000000000 x27: ffff000005288220
> [ 405.581375] x26: 0000000000000034 x25: 0000000000000000 x24: ffff000000259010
> [ 405.588517] x23: ffff80001011ab7c x22: ffff000000259010 x21: 00000000ffffffff
> [ 405.595659] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
> [ 405.602803] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
> [ 405.609947] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
> [ 405.617090] x11: ffff80001241d000 x10: ffff00000528833a x9 : ffff00000528832a
> [ 405.624232] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
> [ 405.631378] x5 : 00000000bfffffff x4 : ffff000009e30dc0 x3 : 0000000000000000
> [ 405.638520] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
> [ 405.645666] Call trace:
> [ 405.648113] __alloc_pages+0x5a4/0xbe0
> [ 405.651862] __dma_direct_alloc_pages+0x17c/0x1e0
> [ 405.656569] dma_direct_alloc+0x70/0x310
> [ 405.660494] dma_alloc_attrs+0x7c/0xe4
> [ 405.664246] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
> [ 405.670021] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
> [ 405.675791] device_run+0xac/0x110 [hantro_vpu]
> [ 405.680345] v4l2_m2m_try_run+0xac/0x1b0 [v4l2_mem2mem]
> [ 405.685598] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
> [ 405.691366] v4l_streamon+0x28/0x34 [videodev]
> [ 405.695877] __video_do_ioctl+0x178/0x3dc [videodev]
> [ 405.700897] video_usercopy+0x368/0x6dc [videodev]
> [ 405.705745] video_ioctl2+0x1c/0x30 [videodev]
> [ 405.710246] v4l2_ioctl+0x44/0x64 [videodev]
> [ 405.714574] __arm64_sys_ioctl+0xac/0xf0
> [ 405.718502] invoke_syscall+0x48/0x114
> [ 405.722258] el0_svc_common.constprop.0+0xd4/0xfc
> [ 405.726969] do_el0_svc+0x2c/0x94
> [ 405.730286] el0_svc+0x28/0x80
> [ 405.733348] el0t_64_sync_handler+0xa8/0x130
> [ 405.737619] el0t_64_sync+0x1a0/0x1a4
> [ 405.741287] ---[ end trace 270ed4a899803006 ]---
>
> The H1 encoder seems to hang the system when I run v4l2-compliance on
> it when H1 is set up as I submitted the patch. I tried dropping all
> the encoder formats except the JPEG format, and it doesn't hang any
> more, but it also doesn't really do anything.
> The datasheet only references VPU_H1 as supporting VP8 and H.264, so I
> am not sure JPEG is even supported.
If JPEG is not supported, then there is nothing left for mainline in this
regard. The kernel control interface and encoding flow needs to be designed and
specified for encoders like VP8 and H264. Some prototypes and prior-art exist
though, but nothing ever got formalized in the form of a specification.
>
> The log from v4l2-compliance on the H1 with everything except the JPEG
> removed looks like:
>
> root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> v4l2-compliance SHA: not available
> , 64 bits, 64-bit time_t
>
> Segmentation fault
> root@beacon-imx8mm-kit:~#
> Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> : Internal error: Oops: 96000004 [#2] SMP
>
> Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> : Code: 52800001 aa1403e0 d2801802 95c31ab9 (b9400aa1)
>
> I want to install Gstreamer, but I don't have functioning DSI video,
> so I am not entirely sure how I will go about testing the decoders
> except by using fakesink
We too don't have an mainline DSI to test the CODECs on recent NXP SoC. For
decoders we use fluster, a tool that runs publicly available conformance test.
It will simply decode to disk and compare a checksum of the decoded image
against the compliant checksum (produced by the reference decoders). For you
interested, it uses the new videocodectestsink, which is specialized for
producing or calculating conformance image/checksum.
https://github.com/fluendo/fluster
We have added support for GStreamer stateless decoders already.
>
> If the G1 ends up working with some of the newer Gstreamer stuff, I
> might just submit a formal patch to just enable the G1 for now.
This looks like a good idea indeed.
>
> adam
> >
> > >
> > > Lastly, I didn't update any device tree binding YAML files, because
> > > I know there have been some discussions about the power domains on the
> > > imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> > > or if the existing one for te imx8mq should just be modified.
> > >
> > > This will likely require the following series in order to apply correctly:
> > > https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
> > >
> > > Adam Ford (5):
> > > media: hantro: Add support for i.MX8M Mini
> > > arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> > > media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> > > media: hantro: Add H1 encoder support on i.MX8M Mini
> > > arm64: dts: imx8mm: Enable Hantro H1 Encoder
> > >
> > > arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> > > drivers/staging/media/hantro/hantro_drv.c | 3 +
> > > drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> > > drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> > > .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> > > 5 files changed, 231 insertions(+), 21 deletions(-)
> > >
> >
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-09 15:57 ` Nicolas Dufresne
@ 2021-11-16 23:23 ` Tim Harvey
2021-11-18 14:30 ` Nicolas Dufresne
0 siblings, 1 reply; 42+ messages in thread
From: Tim Harvey @ 2021-11-16 23:23 UTC (permalink / raw)
To: Nicolas Dufresne, Adam Ford
Cc: linux-media, Schrempf Frieder, Marek Vasut, Jagan Teki,
Adam Ford-BE, cstevens, Ezequiel Garcia, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Tue, Nov 9, 2021 at 7:57 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
>
> Le lundi 08 novembre 2021 à 10:33 -0600, Adam Ford a écrit :
> > On Mon, Nov 8, 2021 at 7:59 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > >
> > > Hi Adam,
> > >
> > > thanks for you work, I'll try and reply about the GStreamer questions below, if
> > > you have further question feel free to ask.
> > >
> > > Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> > > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > > how the Mini handles the power domains, the VPU driver does not need to
> > > > handle all the functions, so a new compatible flag is required.
> > > >
> > > > This is an RFC because I don't have functional video on my system yet,
> > > > and I'm hoping there might be people who do and can help test this.
> > > > I have only tested this far enough to see they enumerate and appear
> > > > as /dev/videoX and /dev/mediaX devices.
> > >
> > > I will check the patchset, but you need in the mini-variant to disable the G1
> > > post processor, because this block was fused out. We didn't make it optional
> >
> > Thanks for being willing to review this.
> >
> > > from the start as according to the V1 of the TRM it was there, but that error
> > > was corrected in V3.
> >
> > Thanks for the clarification. It wasn't obvious to me, because in
> > some instances the PP looked like it was there and sometimes not
> > there. I'll remove the postproc stuff.
> >
> > >
> > > >
> > > > I am also curious to know if/what gstreamer plugins are necessary. In
> > > > NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> > > > would be more generic plugins that I can use to test. I am hoping some
> > > > of the linux-media experts might chime in on how to best test.
> > >
> > > I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
> > > single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
> > > H264 and VP8. Remember that the related plugin depends on libgudev (a glib
> > > binding of udev).
> >
> > Thanks for the tip.
> >
> > >
> > > For the encoder, I believe only JPEG maybe supported, since this is all there is
> > > mainline for RK3288 (and perhaps other RK). But this will need testing and
> > > debugging as the G1 version is slightly newer on NXP SoC.
> >
> > For what it's worth the G1 seems to repond cleanly to the inquiries
> > from v42-compliance.
> > The G2 throws some splat when I run v4l2-compliance, but I am still
> > investigating that.
> >
> > [ 405.456979] ------------[ cut here ]------------
> > [ 405.464173] WARNING: CPU: 0 PID: 563 at mm/page_alloc.c:5344
> > __alloc_pages+0x5a4/0xbe0
> > [ 405.472104] Modules linked in: 8021q garp mrp stp llc caam_jr
> > caamhash_desc caamalg_desc crypto_engine rng_core authenc libdes
> > imx7_media_csi(C) crct10dif_ce imx_media_common(C)
> > snd_soc_fsl_asoc_card imx7_mipi_csis(C) snd_soc_imx_audmux
> > snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc brcmfmac
> > brcmutil hantro_vpu(C) v4l2_h264 v4l2_mem2mem videobuf2_vmalloc
> > videobuf2_dma_contig videobuf2_memops cfg80211 ov5640 videobuf2_v4l2
> > v4l2_fwnode v4l2_async videobuf2_common videodev etnaviv gpu_sched
> > hci_uart mc btqca btbcm snd_soc_wm8962 at24 spi_imx rtc_pcf85363
> > rtc_snvs clk_bd718x7 spi_bitbang snvs_pwrkey snd_soc_fsl_sai
> > imx_pcm_dma caam error bluetooth imx8mm_thermal ecdh_generic
> > imx_cpufreq_dt ecc rfkill fuse drm ipv6
> > [ 405.535835] CPU: 0 PID: 563 Comm: v4l2-compliance Tainted: G D
> > C 5.15.0-next-20211105-00010-g4bb8e8a25d3c-dirty #28
> > [ 405.547401] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
> > Development Kit (DT)
> > [ 405.554797] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> > [ 405.561762] pc : __alloc_pages+0x5a4/0xbe0
> > [ 405.565861] lr : __dma_direct_alloc_pages+0x17c/0x1e0
> > [ 405.570917] sp : ffff800012443810
> > [ 405.574232] x29: ffff800012443810 x28: 0000000000000000 x27: ffff000005288220
> > [ 405.581375] x26: 0000000000000034 x25: 0000000000000000 x24: ffff000000259010
> > [ 405.588517] x23: ffff80001011ab7c x22: ffff000000259010 x21: 00000000ffffffff
> > [ 405.595659] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
> > [ 405.602803] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
> > [ 405.609947] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
> > [ 405.617090] x11: ffff80001241d000 x10: ffff00000528833a x9 : ffff00000528832a
> > [ 405.624232] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
> > [ 405.631378] x5 : 00000000bfffffff x4 : ffff000009e30dc0 x3 : 0000000000000000
> > [ 405.638520] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
> > [ 405.645666] Call trace:
> > [ 405.648113] __alloc_pages+0x5a4/0xbe0
> > [ 405.651862] __dma_direct_alloc_pages+0x17c/0x1e0
> > [ 405.656569] dma_direct_alloc+0x70/0x310
> > [ 405.660494] dma_alloc_attrs+0x7c/0xe4
> > [ 405.664246] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
> > [ 405.670021] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
> > [ 405.675791] device_run+0xac/0x110 [hantro_vpu]
> > [ 405.680345] v4l2_m2m_try_run+0xac/0x1b0 [v4l2_mem2mem]
> > [ 405.685598] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
> > [ 405.691366] v4l_streamon+0x28/0x34 [videodev]
> > [ 405.695877] __video_do_ioctl+0x178/0x3dc [videodev]
> > [ 405.700897] video_usercopy+0x368/0x6dc [videodev]
> > [ 405.705745] video_ioctl2+0x1c/0x30 [videodev]
> > [ 405.710246] v4l2_ioctl+0x44/0x64 [videodev]
> > [ 405.714574] __arm64_sys_ioctl+0xac/0xf0
> > [ 405.718502] invoke_syscall+0x48/0x114
> > [ 405.722258] el0_svc_common.constprop.0+0xd4/0xfc
> > [ 405.726969] do_el0_svc+0x2c/0x94
> > [ 405.730286] el0_svc+0x28/0x80
> > [ 405.733348] el0t_64_sync_handler+0xa8/0x130
> > [ 405.737619] el0t_64_sync+0x1a0/0x1a4
> > [ 405.741287] ---[ end trace 270ed4a899803006 ]---
> >
> > The H1 encoder seems to hang the system when I run v4l2-compliance on
> > it when H1 is set up as I submitted the patch. I tried dropping all
> > the encoder formats except the JPEG format, and it doesn't hang any
> > more, but it also doesn't really do anything.
> > The datasheet only references VPU_H1 as supporting VP8 and H.264, so I
> > am not sure JPEG is even supported.
>
> If JPEG is not supported, then there is nothing left for mainline in this
> regard. The kernel control interface and encoding flow needs to be designed and
> specified for encoders like VP8 and H264. Some prototypes and prior-art exist
> though, but nothing ever got formalized in the form of a specification.
>
> >
> > The log from v4l2-compliance on the H1 with everything except the JPEG
> > removed looks like:
> >
> > root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> > v4l2-compliance SHA: not available
> > , 64 bits, 64-bit time_t
> >
> > Segmentation fault
> > root@beacon-imx8mm-kit:~#
> > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > : Internal error: Oops: 96000004 [#2] SMP
> >
> > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > : Code: 52800001 aa1403e0 d2801802 95c31ab9 (b9400aa1)
> >
> > I want to install Gstreamer, but I don't have functioning DSI video,
> > so I am not entirely sure how I will go about testing the decoders
> > except by using fakesink
>
> We too don't have an mainline DSI to test the CODECs on recent NXP SoC. For
> decoders we use fluster, a tool that runs publicly available conformance test.
> It will simply decode to disk and compare a checksum of the decoded image
> against the compliant checksum (produced by the reference decoders). For you
> interested, it uses the new videocodectestsink, which is specialized for
> producing or calculating conformance image/checksum.
>
> https://github.com/fluendo/fluster
>
> We have added support for GStreamer stateless decoders already.
>
> >
> > If the G1 ends up working with some of the newer Gstreamer stuff, I
> > might just submit a formal patch to just enable the G1 for now.
>
> This looks like a good idea indeed.
>
> >
> > adam
> > >
> > > >
> > > > Lastly, I didn't update any device tree binding YAML files, because
> > > > I know there have been some discussions about the power domains on the
> > > > imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> > > > or if the existing one for te imx8mq should just be modified.
> > > >
> > > > This will likely require the following series in order to apply correctly:
> > > > https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
> > > >
> > > > Adam Ford (5):
> > > > media: hantro: Add support for i.MX8M Mini
> > > > arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> > > > media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> > > > media: hantro: Add H1 encoder support on i.MX8M Mini
> > > > arm64: dts: imx8mm: Enable Hantro H1 Encoder
> > > >
> > > > arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> > > > drivers/staging/media/hantro/hantro_drv.c | 3 +
> > > > drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> > > > drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> > > > .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> > > > 5 files changed, 231 insertions(+), 21 deletions(-)
> > > >
> > >
>
Nicolas and Adam,
For the H1 patches in this series: I've been able to test the IMX8MM
H1 JPEG encode using GStreamer 1.18.5:
$ gst-inspect-1.0 | grep -e "v4l2.*enc"
video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
$ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
host=192.168.1.146 port=5000
viewed on client@192.168.1.146 via:
$ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
rtpjpegdepay ! jpegdec ! autovideosink
For the G1/G2 patches in the series I don't see any Gstreamer
'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
I have CSI capture and DSI display currently working on
imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
can't efficiently convert to something the JPEG encoder likes without
bayer2rgbneon (a libneon version).
I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
wide range of data formats but I'm not sure how to tap into this as
that hardware is managed by the vivante driver. On the IMX6QDL there
is a separate IPU block that Philipp Zabel wrote a nice mem2mem
csc/scaler driver for but I don't see any equivalent currently for
IMX8MM.
Best regards,
Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-16 23:23 ` Tim Harvey
@ 2021-11-18 14:30 ` Nicolas Dufresne
2021-11-18 16:20 ` Tim Harvey
0 siblings, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-11-18 14:30 UTC (permalink / raw)
To: Tim Harvey, Adam Ford
Cc: linux-media, Schrempf Frieder, Marek Vasut, Jagan Teki,
Adam Ford-BE, cstevens, Ezequiel Garcia, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le mardi 16 novembre 2021 à 15:23 -0800, Tim Harvey a écrit :
> On Tue, Nov 9, 2021 at 7:57 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> >
> > Le lundi 08 novembre 2021 à 10:33 -0600, Adam Ford a écrit :
> > > On Mon, Nov 8, 2021 at 7:59 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > >
> > > > Hi Adam,
> > > >
> > > > thanks for you work, I'll try and reply about the GStreamer questions below, if
> > > > you have further question feel free to ask.
> > > >
> > > > Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> > > > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > > > how the Mini handles the power domains, the VPU driver does not need to
> > > > > handle all the functions, so a new compatible flag is required.
> > > > >
> > > > > This is an RFC because I don't have functional video on my system yet,
> > > > > and I'm hoping there might be people who do and can help test this.
> > > > > I have only tested this far enough to see they enumerate and appear
> > > > > as /dev/videoX and /dev/mediaX devices.
> > > >
> > > > I will check the patchset, but you need in the mini-variant to disable the G1
> > > > post processor, because this block was fused out. We didn't make it optional
> > >
> > > Thanks for being willing to review this.
> > >
> > > > from the start as according to the V1 of the TRM it was there, but that error
> > > > was corrected in V3.
> > >
> > > Thanks for the clarification. It wasn't obvious to me, because in
> > > some instances the PP looked like it was there and sometimes not
> > > there. I'll remove the postproc stuff.
> > >
> > > >
> > > > >
> > > > > I am also curious to know if/what gstreamer plugins are necessary. In
> > > > > NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> > > > > would be more generic plugins that I can use to test. I am hoping some
> > > > > of the linux-media experts might chime in on how to best test.
> > > >
> > > > I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
> > > > single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
> > > > H264 and VP8. Remember that the related plugin depends on libgudev (a glib
> > > > binding of udev).
> > >
> > > Thanks for the tip.
> > >
> > > >
> > > > For the encoder, I believe only JPEG maybe supported, since this is all there is
> > > > mainline for RK3288 (and perhaps other RK). But this will need testing and
> > > > debugging as the G1 version is slightly newer on NXP SoC.
> > >
> > > For what it's worth the G1 seems to repond cleanly to the inquiries
> > > from v42-compliance.
> > > The G2 throws some splat when I run v4l2-compliance, but I am still
> > > investigating that.
> > >
> > > [ 405.456979] ------------[ cut here ]------------
> > > [ 405.464173] WARNING: CPU: 0 PID: 563 at mm/page_alloc.c:5344
> > > __alloc_pages+0x5a4/0xbe0
> > > [ 405.472104] Modules linked in: 8021q garp mrp stp llc caam_jr
> > > caamhash_desc caamalg_desc crypto_engine rng_core authenc libdes
> > > imx7_media_csi(C) crct10dif_ce imx_media_common(C)
> > > snd_soc_fsl_asoc_card imx7_mipi_csis(C) snd_soc_imx_audmux
> > > snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc brcmfmac
> > > brcmutil hantro_vpu(C) v4l2_h264 v4l2_mem2mem videobuf2_vmalloc
> > > videobuf2_dma_contig videobuf2_memops cfg80211 ov5640 videobuf2_v4l2
> > > v4l2_fwnode v4l2_async videobuf2_common videodev etnaviv gpu_sched
> > > hci_uart mc btqca btbcm snd_soc_wm8962 at24 spi_imx rtc_pcf85363
> > > rtc_snvs clk_bd718x7 spi_bitbang snvs_pwrkey snd_soc_fsl_sai
> > > imx_pcm_dma caam error bluetooth imx8mm_thermal ecdh_generic
> > > imx_cpufreq_dt ecc rfkill fuse drm ipv6
> > > [ 405.535835] CPU: 0 PID: 563 Comm: v4l2-compliance Tainted: G D
> > > C 5.15.0-next-20211105-00010-g4bb8e8a25d3c-dirty #28
> > > [ 405.547401] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
> > > Development Kit (DT)
> > > [ 405.554797] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> > > [ 405.561762] pc : __alloc_pages+0x5a4/0xbe0
> > > [ 405.565861] lr : __dma_direct_alloc_pages+0x17c/0x1e0
> > > [ 405.570917] sp : ffff800012443810
> > > [ 405.574232] x29: ffff800012443810 x28: 0000000000000000 x27: ffff000005288220
> > > [ 405.581375] x26: 0000000000000034 x25: 0000000000000000 x24: ffff000000259010
> > > [ 405.588517] x23: ffff80001011ab7c x22: ffff000000259010 x21: 00000000ffffffff
> > > [ 405.595659] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
> > > [ 405.602803] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
> > > [ 405.609947] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
> > > [ 405.617090] x11: ffff80001241d000 x10: ffff00000528833a x9 : ffff00000528832a
> > > [ 405.624232] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
> > > [ 405.631378] x5 : 00000000bfffffff x4 : ffff000009e30dc0 x3 : 0000000000000000
> > > [ 405.638520] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
> > > [ 405.645666] Call trace:
> > > [ 405.648113] __alloc_pages+0x5a4/0xbe0
> > > [ 405.651862] __dma_direct_alloc_pages+0x17c/0x1e0
> > > [ 405.656569] dma_direct_alloc+0x70/0x310
> > > [ 405.660494] dma_alloc_attrs+0x7c/0xe4
> > > [ 405.664246] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
> > > [ 405.670021] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
> > > [ 405.675791] device_run+0xac/0x110 [hantro_vpu]
> > > [ 405.680345] v4l2_m2m_try_run+0xac/0x1b0 [v4l2_mem2mem]
> > > [ 405.685598] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
> > > [ 405.691366] v4l_streamon+0x28/0x34 [videodev]
> > > [ 405.695877] __video_do_ioctl+0x178/0x3dc [videodev]
> > > [ 405.700897] video_usercopy+0x368/0x6dc [videodev]
> > > [ 405.705745] video_ioctl2+0x1c/0x30 [videodev]
> > > [ 405.710246] v4l2_ioctl+0x44/0x64 [videodev]
> > > [ 405.714574] __arm64_sys_ioctl+0xac/0xf0
> > > [ 405.718502] invoke_syscall+0x48/0x114
> > > [ 405.722258] el0_svc_common.constprop.0+0xd4/0xfc
> > > [ 405.726969] do_el0_svc+0x2c/0x94
> > > [ 405.730286] el0_svc+0x28/0x80
> > > [ 405.733348] el0t_64_sync_handler+0xa8/0x130
> > > [ 405.737619] el0t_64_sync+0x1a0/0x1a4
> > > [ 405.741287] ---[ end trace 270ed4a899803006 ]---
> > >
> > > The H1 encoder seems to hang the system when I run v4l2-compliance on
> > > it when H1 is set up as I submitted the patch. I tried dropping all
> > > the encoder formats except the JPEG format, and it doesn't hang any
> > > more, but it also doesn't really do anything.
> > > The datasheet only references VPU_H1 as supporting VP8 and H.264, so I
> > > am not sure JPEG is even supported.
> >
> > If JPEG is not supported, then there is nothing left for mainline in this
> > regard. The kernel control interface and encoding flow needs to be designed and
> > specified for encoders like VP8 and H264. Some prototypes and prior-art exist
> > though, but nothing ever got formalized in the form of a specification.
> >
> > >
> > > The log from v4l2-compliance on the H1 with everything except the JPEG
> > > removed looks like:
> > >
> > > root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> > > v4l2-compliance SHA: not available
> > > , 64 bits, 64-bit time_t
> > >
> > > Segmentation fault
> > > root@beacon-imx8mm-kit:~#
> > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > : Internal error: Oops: 96000004 [#2] SMP
> > >
> > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > : Code: 52800001 aa1403e0 d2801802 95c31ab9 (b9400aa1)
> > >
> > > I want to install Gstreamer, but I don't have functioning DSI video,
> > > so I am not entirely sure how I will go about testing the decoders
> > > except by using fakesink
> >
> > We too don't have an mainline DSI to test the CODECs on recent NXP SoC. For
> > decoders we use fluster, a tool that runs publicly available conformance test.
> > It will simply decode to disk and compare a checksum of the decoded image
> > against the compliant checksum (produced by the reference decoders). For you
> > interested, it uses the new videocodectestsink, which is specialized for
> > producing or calculating conformance image/checksum.
> >
> > https://github.com/fluendo/fluster
> >
> > We have added support for GStreamer stateless decoders already.
> >
> > >
> > > If the G1 ends up working with some of the newer Gstreamer stuff, I
> > > might just submit a formal patch to just enable the G1 for now.
> >
> > This looks like a good idea indeed.
> >
> > >
> > > adam
> > > >
> > > > >
> > > > > Lastly, I didn't update any device tree binding YAML files, because
> > > > > I know there have been some discussions about the power domains on the
> > > > > imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> > > > > or if the existing one for te imx8mq should just be modified.
> > > > >
> > > > > This will likely require the following series in order to apply correctly:
> > > > > https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
> > > > >
> > > > > Adam Ford (5):
> > > > > media: hantro: Add support for i.MX8M Mini
> > > > > arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> > > > > media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> > > > > media: hantro: Add H1 encoder support on i.MX8M Mini
> > > > > arm64: dts: imx8mm: Enable Hantro H1 Encoder
> > > > >
> > > > > arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> > > > > drivers/staging/media/hantro/hantro_drv.c | 3 +
> > > > > drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> > > > > drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> > > > > .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> > > > > 5 files changed, 231 insertions(+), 21 deletions(-)
> > > > >
> > > >
> >
>
> Nicolas and Adam,
>
> For the H1 patches in this series: I've been able to test the IMX8MM
> H1 JPEG encode using GStreamer 1.18.5:
> $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
^ v4l2jpegenc
This is just a transcript error ?
> host=192.168.1.146 port=5000
> viewed on client@192.168.1.146 via:
> $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> rtpjpegdepay ! jpegdec ! autovideosink
>
> For the G1/G2 patches in the series I don't see any Gstreamer
> 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
a single repository. We are very close to 1.20, which will include stable API
support of H264, MPEG2 and VP8 decoding.
>
> I have CSI capture and DSI display currently working on
> imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> can't efficiently convert to something the JPEG encoder likes without
> bayer2rgbneon (a libneon version).
>
> I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> wide range of data formats but I'm not sure how to tap into this as
> that hardware is managed by the vivante driver. On the IMX6QDL there
> is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> csc/scaler driver for but I don't see any equivalent currently for
> IMX8MM.
>
> Best regards,
>
> Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-18 14:30 ` Nicolas Dufresne
@ 2021-11-18 16:20 ` Tim Harvey
2021-11-18 18:16 ` Adam Ford
2021-11-23 0:06 ` Tim Harvey
0 siblings, 2 replies; 42+ messages in thread
From: Tim Harvey @ 2021-11-18 16:20 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: Adam Ford, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Thu, Nov 18, 2021 at 6:30 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
>
> Le mardi 16 novembre 2021 à 15:23 -0800, Tim Harvey a écrit :
> > On Tue, Nov 9, 2021 at 7:57 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > >
> > > Le lundi 08 novembre 2021 à 10:33 -0600, Adam Ford a écrit :
> > > > On Mon, Nov 8, 2021 at 7:59 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > >
> > > > > Hi Adam,
> > > > >
> > > > > thanks for you work, I'll try and reply about the GStreamer questions below, if
> > > > > you have further question feel free to ask.
> > > > >
> > > > > Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> > > > > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > > > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > > > > how the Mini handles the power domains, the VPU driver does not need to
> > > > > > handle all the functions, so a new compatible flag is required.
> > > > > >
> > > > > > This is an RFC because I don't have functional video on my system yet,
> > > > > > and I'm hoping there might be people who do and can help test this.
> > > > > > I have only tested this far enough to see they enumerate and appear
> > > > > > as /dev/videoX and /dev/mediaX devices.
> > > > >
> > > > > I will check the patchset, but you need in the mini-variant to disable the G1
> > > > > post processor, because this block was fused out. We didn't make it optional
> > > >
> > > > Thanks for being willing to review this.
> > > >
> > > > > from the start as according to the V1 of the TRM it was there, but that error
> > > > > was corrected in V3.
> > > >
> > > > Thanks for the clarification. It wasn't obvious to me, because in
> > > > some instances the PP looked like it was there and sometimes not
> > > > there. I'll remove the postproc stuff.
> > > >
> > > > >
> > > > > >
> > > > > > I am also curious to know if/what gstreamer plugins are necessary. In
> > > > > > NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> > > > > > would be more generic plugins that I can use to test. I am hoping some
> > > > > > of the linux-media experts might chime in on how to best test.
> > > > >
> > > > > I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
> > > > > single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
> > > > > H264 and VP8. Remember that the related plugin depends on libgudev (a glib
> > > > > binding of udev).
> > > >
> > > > Thanks for the tip.
> > > >
> > > > >
> > > > > For the encoder, I believe only JPEG maybe supported, since this is all there is
> > > > > mainline for RK3288 (and perhaps other RK). But this will need testing and
> > > > > debugging as the G1 version is slightly newer on NXP SoC.
> > > >
> > > > For what it's worth the G1 seems to repond cleanly to the inquiries
> > > > from v42-compliance.
> > > > The G2 throws some splat when I run v4l2-compliance, but I am still
> > > > investigating that.
> > > >
> > > > [ 405.456979] ------------[ cut here ]------------
> > > > [ 405.464173] WARNING: CPU: 0 PID: 563 at mm/page_alloc.c:5344
> > > > __alloc_pages+0x5a4/0xbe0
> > > > [ 405.472104] Modules linked in: 8021q garp mrp stp llc caam_jr
> > > > caamhash_desc caamalg_desc crypto_engine rng_core authenc libdes
> > > > imx7_media_csi(C) crct10dif_ce imx_media_common(C)
> > > > snd_soc_fsl_asoc_card imx7_mipi_csis(C) snd_soc_imx_audmux
> > > > snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc brcmfmac
> > > > brcmutil hantro_vpu(C) v4l2_h264 v4l2_mem2mem videobuf2_vmalloc
> > > > videobuf2_dma_contig videobuf2_memops cfg80211 ov5640 videobuf2_v4l2
> > > > v4l2_fwnode v4l2_async videobuf2_common videodev etnaviv gpu_sched
> > > > hci_uart mc btqca btbcm snd_soc_wm8962 at24 spi_imx rtc_pcf85363
> > > > rtc_snvs clk_bd718x7 spi_bitbang snvs_pwrkey snd_soc_fsl_sai
> > > > imx_pcm_dma caam error bluetooth imx8mm_thermal ecdh_generic
> > > > imx_cpufreq_dt ecc rfkill fuse drm ipv6
> > > > [ 405.535835] CPU: 0 PID: 563 Comm: v4l2-compliance Tainted: G D
> > > > C 5.15.0-next-20211105-00010-g4bb8e8a25d3c-dirty #28
> > > > [ 405.547401] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
> > > > Development Kit (DT)
> > > > [ 405.554797] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> > > > [ 405.561762] pc : __alloc_pages+0x5a4/0xbe0
> > > > [ 405.565861] lr : __dma_direct_alloc_pages+0x17c/0x1e0
> > > > [ 405.570917] sp : ffff800012443810
> > > > [ 405.574232] x29: ffff800012443810 x28: 0000000000000000 x27: ffff000005288220
> > > > [ 405.581375] x26: 0000000000000034 x25: 0000000000000000 x24: ffff000000259010
> > > > [ 405.588517] x23: ffff80001011ab7c x22: ffff000000259010 x21: 00000000ffffffff
> > > > [ 405.595659] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
> > > > [ 405.602803] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
> > > > [ 405.609947] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
> > > > [ 405.617090] x11: ffff80001241d000 x10: ffff00000528833a x9 : ffff00000528832a
> > > > [ 405.624232] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
> > > > [ 405.631378] x5 : 00000000bfffffff x4 : ffff000009e30dc0 x3 : 0000000000000000
> > > > [ 405.638520] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
> > > > [ 405.645666] Call trace:
> > > > [ 405.648113] __alloc_pages+0x5a4/0xbe0
> > > > [ 405.651862] __dma_direct_alloc_pages+0x17c/0x1e0
> > > > [ 405.656569] dma_direct_alloc+0x70/0x310
> > > > [ 405.660494] dma_alloc_attrs+0x7c/0xe4
> > > > [ 405.664246] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
> > > > [ 405.670021] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
> > > > [ 405.675791] device_run+0xac/0x110 [hantro_vpu]
> > > > [ 405.680345] v4l2_m2m_try_run+0xac/0x1b0 [v4l2_mem2mem]
> > > > [ 405.685598] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
> > > > [ 405.691366] v4l_streamon+0x28/0x34 [videodev]
> > > > [ 405.695877] __video_do_ioctl+0x178/0x3dc [videodev]
> > > > [ 405.700897] video_usercopy+0x368/0x6dc [videodev]
> > > > [ 405.705745] video_ioctl2+0x1c/0x30 [videodev]
> > > > [ 405.710246] v4l2_ioctl+0x44/0x64 [videodev]
> > > > [ 405.714574] __arm64_sys_ioctl+0xac/0xf0
> > > > [ 405.718502] invoke_syscall+0x48/0x114
> > > > [ 405.722258] el0_svc_common.constprop.0+0xd4/0xfc
> > > > [ 405.726969] do_el0_svc+0x2c/0x94
> > > > [ 405.730286] el0_svc+0x28/0x80
> > > > [ 405.733348] el0t_64_sync_handler+0xa8/0x130
> > > > [ 405.737619] el0t_64_sync+0x1a0/0x1a4
> > > > [ 405.741287] ---[ end trace 270ed4a899803006 ]---
> > > >
> > > > The H1 encoder seems to hang the system when I run v4l2-compliance on
> > > > it when H1 is set up as I submitted the patch. I tried dropping all
> > > > the encoder formats except the JPEG format, and it doesn't hang any
> > > > more, but it also doesn't really do anything.
> > > > The datasheet only references VPU_H1 as supporting VP8 and H.264, so I
> > > > am not sure JPEG is even supported.
> > >
> > > If JPEG is not supported, then there is nothing left for mainline in this
> > > regard. The kernel control interface and encoding flow needs to be designed and
> > > specified for encoders like VP8 and H264. Some prototypes and prior-art exist
> > > though, but nothing ever got formalized in the form of a specification.
> > >
> > > >
> > > > The log from v4l2-compliance on the H1 with everything except the JPEG
> > > > removed looks like:
> > > >
> > > > root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> > > > v4l2-compliance SHA: not available
> > > > , 64 bits, 64-bit time_t
> > > >
> > > > Segmentation fault
> > > > root@beacon-imx8mm-kit:~#
> > > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > > : Internal error: Oops: 96000004 [#2] SMP
> > > >
> > > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > > : Code: 52800001 aa1403e0 d2801802 95c31ab9 (b9400aa1)
> > > >
> > > > I want to install Gstreamer, but I don't have functioning DSI video,
> > > > so I am not entirely sure how I will go about testing the decoders
> > > > except by using fakesink
> > >
> > > We too don't have an mainline DSI to test the CODECs on recent NXP SoC. For
> > > decoders we use fluster, a tool that runs publicly available conformance test.
> > > It will simply decode to disk and compare a checksum of the decoded image
> > > against the compliant checksum (produced by the reference decoders). For you
> > > interested, it uses the new videocodectestsink, which is specialized for
> > > producing or calculating conformance image/checksum.
> > >
> > > https://github.com/fluendo/fluster
> > >
> > > We have added support for GStreamer stateless decoders already.
> > >
> > > >
> > > > If the G1 ends up working with some of the newer Gstreamer stuff, I
> > > > might just submit a formal patch to just enable the G1 for now.
> > >
> > > This looks like a good idea indeed.
> > >
> > > >
> > > > adam
> > > > >
> > > > > >
> > > > > > Lastly, I didn't update any device tree binding YAML files, because
> > > > > > I know there have been some discussions about the power domains on the
> > > > > > imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> > > > > > or if the existing one for te imx8mq should just be modified.
> > > > > >
> > > > > > This will likely require the following series in order to apply correctly:
> > > > > > https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
> > > > > >
> > > > > > Adam Ford (5):
> > > > > > media: hantro: Add support for i.MX8M Mini
> > > > > > arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> > > > > > media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> > > > > > media: hantro: Add H1 encoder support on i.MX8M Mini
> > > > > > arm64: dts: imx8mm: Enable Hantro H1 Encoder
> > > > > >
> > > > > > arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> > > > > > drivers/staging/media/hantro/hantro_drv.c | 3 +
> > > > > > drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> > > > > > drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> > > > > > .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> > > > > > 5 files changed, 231 insertions(+), 21 deletions(-)
> > > > > >
> > > > >
> > >
> >
> > Nicolas and Adam,
> >
> > For the H1 patches in this series: I've been able to test the IMX8MM
> > H1 JPEG encode using GStreamer 1.18.5:
> > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> ^ v4l2jpegenc
>
> This is just a transcript error ?
Nicolas,
No! Thanks for catching my mistake. I was testing with software encode... ooops!
'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
the board so likely a power-domain issue there?
>
> > host=192.168.1.146 port=5000
> > viewed on client@192.168.1.146 via:
> > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > rtpjpegdepay ! jpegdec ! autovideosink
> >
> > For the G1/G2 patches in the series I don't see any Gstreamer
> > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
>
> Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> a single repository. We are very close to 1.20, which will include stable API
> support of H264, MPEG2 and VP8 decoding.
>
Ok, let me see if I can navigate through the build process and I'll
get back to you.
Thanks,
Tim
> >
> > I have CSI capture and DSI display currently working on
> > imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> > supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> > can't efficiently convert to something the JPEG encoder likes without
> > bayer2rgbneon (a libneon version).
> >
> > I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> > wide range of data formats but I'm not sure how to tap into this as
> > that hardware is managed by the vivante driver. On the IMX6QDL there
> > is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> > csc/scaler driver for but I don't see any equivalent currently for
> > IMX8MM.
> >
> > Best regards,
> >
> > Tim
>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-18 16:20 ` Tim Harvey
@ 2021-11-18 18:16 ` Adam Ford
2021-11-19 16:29 ` Nicolas Dufresne
2021-11-23 0:06 ` Tim Harvey
1 sibling, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-18 18:16 UTC (permalink / raw)
To: Tim Harvey
Cc: Nicolas Dufresne, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Thu, Nov 18, 2021 at 10:20 AM Tim Harvey <tharvey@gateworks.com> wrote:
>
> On Thu, Nov 18, 2021 at 6:30 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> >
> > Le mardi 16 novembre 2021 à 15:23 -0800, Tim Harvey a écrit :
> > > On Tue, Nov 9, 2021 at 7:57 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > >
> > > > Le lundi 08 novembre 2021 à 10:33 -0600, Adam Ford a écrit :
> > > > > On Mon, Nov 8, 2021 at 7:59 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > > >
> > > > > > Hi Adam,
> > > > > >
> > > > > > thanks for you work, I'll try and reply about the GStreamer questions below, if
> > > > > > you have further question feel free to ask.
> > > > > >
> > > > > > Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> > > > > > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > > > > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > > > > > how the Mini handles the power domains, the VPU driver does not need to
> > > > > > > handle all the functions, so a new compatible flag is required.
> > > > > > >
> > > > > > > This is an RFC because I don't have functional video on my system yet,
> > > > > > > and I'm hoping there might be people who do and can help test this.
> > > > > > > I have only tested this far enough to see they enumerate and appear
> > > > > > > as /dev/videoX and /dev/mediaX devices.
> > > > > >
> > > > > > I will check the patchset, but you need in the mini-variant to disable the G1
> > > > > > post processor, because this block was fused out. We didn't make it optional
> > > > >
> > > > > Thanks for being willing to review this.
> > > > >
> > > > > > from the start as according to the V1 of the TRM it was there, but that error
> > > > > > was corrected in V3.
> > > > >
> > > > > Thanks for the clarification. It wasn't obvious to me, because in
> > > > > some instances the PP looked like it was there and sometimes not
> > > > > there. I'll remove the postproc stuff.
> > > > >
> > > > > >
> > > > > > >
> > > > > > > I am also curious to know if/what gstreamer plugins are necessary. In
> > > > > > > NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> > > > > > > would be more generic plugins that I can use to test. I am hoping some
> > > > > > > of the linux-media experts might chime in on how to best test.
> > > > > >
> > > > > > I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
> > > > > > single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
> > > > > > H264 and VP8. Remember that the related plugin depends on libgudev (a glib
> > > > > > binding of udev).
> > > > >
> > > > > Thanks for the tip.
> > > > >
> > > > > >
> > > > > > For the encoder, I believe only JPEG maybe supported, since this is all there is
> > > > > > mainline for RK3288 (and perhaps other RK). But this will need testing and
> > > > > > debugging as the G1 version is slightly newer on NXP SoC.
> > > > >
> > > > > For what it's worth the G1 seems to repond cleanly to the inquiries
> > > > > from v42-compliance.
> > > > > The G2 throws some splat when I run v4l2-compliance, but I am still
> > > > > investigating that.
> > > > >
> > > > > [ 405.456979] ------------[ cut here ]------------
> > > > > [ 405.464173] WARNING: CPU: 0 PID: 563 at mm/page_alloc.c:5344
> > > > > __alloc_pages+0x5a4/0xbe0
> > > > > [ 405.472104] Modules linked in: 8021q garp mrp stp llc caam_jr
> > > > > caamhash_desc caamalg_desc crypto_engine rng_core authenc libdes
> > > > > imx7_media_csi(C) crct10dif_ce imx_media_common(C)
> > > > > snd_soc_fsl_asoc_card imx7_mipi_csis(C) snd_soc_imx_audmux
> > > > > snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc brcmfmac
> > > > > brcmutil hantro_vpu(C) v4l2_h264 v4l2_mem2mem videobuf2_vmalloc
> > > > > videobuf2_dma_contig videobuf2_memops cfg80211 ov5640 videobuf2_v4l2
> > > > > v4l2_fwnode v4l2_async videobuf2_common videodev etnaviv gpu_sched
> > > > > hci_uart mc btqca btbcm snd_soc_wm8962 at24 spi_imx rtc_pcf85363
> > > > > rtc_snvs clk_bd718x7 spi_bitbang snvs_pwrkey snd_soc_fsl_sai
> > > > > imx_pcm_dma caam error bluetooth imx8mm_thermal ecdh_generic
> > > > > imx_cpufreq_dt ecc rfkill fuse drm ipv6
> > > > > [ 405.535835] CPU: 0 PID: 563 Comm: v4l2-compliance Tainted: G D
> > > > > C 5.15.0-next-20211105-00010-g4bb8e8a25d3c-dirty #28
> > > > > [ 405.547401] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
> > > > > Development Kit (DT)
> > > > > [ 405.554797] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> > > > > [ 405.561762] pc : __alloc_pages+0x5a4/0xbe0
> > > > > [ 405.565861] lr : __dma_direct_alloc_pages+0x17c/0x1e0
> > > > > [ 405.570917] sp : ffff800012443810
> > > > > [ 405.574232] x29: ffff800012443810 x28: 0000000000000000 x27: ffff000005288220
> > > > > [ 405.581375] x26: 0000000000000034 x25: 0000000000000000 x24: ffff000000259010
> > > > > [ 405.588517] x23: ffff80001011ab7c x22: ffff000000259010 x21: 00000000ffffffff
> > > > > [ 405.595659] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
> > > > > [ 405.602803] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
> > > > > [ 405.609947] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
> > > > > [ 405.617090] x11: ffff80001241d000 x10: ffff00000528833a x9 : ffff00000528832a
> > > > > [ 405.624232] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
> > > > > [ 405.631378] x5 : 00000000bfffffff x4 : ffff000009e30dc0 x3 : 0000000000000000
> > > > > [ 405.638520] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
> > > > > [ 405.645666] Call trace:
> > > > > [ 405.648113] __alloc_pages+0x5a4/0xbe0
> > > > > [ 405.651862] __dma_direct_alloc_pages+0x17c/0x1e0
> > > > > [ 405.656569] dma_direct_alloc+0x70/0x310
> > > > > [ 405.660494] dma_alloc_attrs+0x7c/0xe4
> > > > > [ 405.664246] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
> > > > > [ 405.670021] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
> > > > > [ 405.675791] device_run+0xac/0x110 [hantro_vpu]
> > > > > [ 405.680345] v4l2_m2m_try_run+0xac/0x1b0 [v4l2_mem2mem]
> > > > > [ 405.685598] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
> > > > > [ 405.691366] v4l_streamon+0x28/0x34 [videodev]
> > > > > [ 405.695877] __video_do_ioctl+0x178/0x3dc [videodev]
> > > > > [ 405.700897] video_usercopy+0x368/0x6dc [videodev]
> > > > > [ 405.705745] video_ioctl2+0x1c/0x30 [videodev]
> > > > > [ 405.710246] v4l2_ioctl+0x44/0x64 [videodev]
> > > > > [ 405.714574] __arm64_sys_ioctl+0xac/0xf0
> > > > > [ 405.718502] invoke_syscall+0x48/0x114
> > > > > [ 405.722258] el0_svc_common.constprop.0+0xd4/0xfc
> > > > > [ 405.726969] do_el0_svc+0x2c/0x94
> > > > > [ 405.730286] el0_svc+0x28/0x80
> > > > > [ 405.733348] el0t_64_sync_handler+0xa8/0x130
> > > > > [ 405.737619] el0t_64_sync+0x1a0/0x1a4
> > > > > [ 405.741287] ---[ end trace 270ed4a899803006 ]---
> > > > >
> > > > > The H1 encoder seems to hang the system when I run v4l2-compliance on
> > > > > it when H1 is set up as I submitted the patch. I tried dropping all
> > > > > the encoder formats except the JPEG format, and it doesn't hang any
> > > > > more, but it also doesn't really do anything.
> > > > > The datasheet only references VPU_H1 as supporting VP8 and H.264, so I
> > > > > am not sure JPEG is even supported.
> > > >
> > > > If JPEG is not supported, then there is nothing left for mainline in this
> > > > regard. The kernel control interface and encoding flow needs to be designed and
> > > > specified for encoders like VP8 and H264. Some prototypes and prior-art exist
> > > > though, but nothing ever got formalized in the form of a specification.
> > > >
> > > > >
> > > > > The log from v4l2-compliance on the H1 with everything except the JPEG
> > > > > removed looks like:
> > > > >
> > > > > root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> > > > > v4l2-compliance SHA: not available
> > > > > , 64 bits, 64-bit time_t
> > > > >
> > > > > Segmentation fault
> > > > > root@beacon-imx8mm-kit:~#
> > > > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > > > : Internal error: Oops: 96000004 [#2] SMP
> > > > >
> > > > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > > > : Code: 52800001 aa1403e0 d2801802 95c31ab9 (b9400aa1)
> > > > >
> > > > > I want to install Gstreamer, but I don't have functioning DSI video,
> > > > > so I am not entirely sure how I will go about testing the decoders
> > > > > except by using fakesink
> > > >
> > > > We too don't have an mainline DSI to test the CODECs on recent NXP SoC. For
> > > > decoders we use fluster, a tool that runs publicly available conformance test.
> > > > It will simply decode to disk and compare a checksum of the decoded image
> > > > against the compliant checksum (produced by the reference decoders). For you
> > > > interested, it uses the new videocodectestsink, which is specialized for
> > > > producing or calculating conformance image/checksum.
> > > >
> > > > https://github.com/fluendo/fluster
> > > >
> > > > We have added support for GStreamer stateless decoders already.
> > > >
> > > > >
> > > > > If the G1 ends up working with some of the newer Gstreamer stuff, I
> > > > > might just submit a formal patch to just enable the G1 for now.
> > > >
> > > > This looks like a good idea indeed.
> > > >
> > > > >
> > > > > adam
> > > > > >
> > > > > > >
> > > > > > > Lastly, I didn't update any device tree binding YAML files, because
> > > > > > > I know there have been some discussions about the power domains on the
> > > > > > > imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> > > > > > > or if the existing one for te imx8mq should just be modified.
> > > > > > >
> > > > > > > This will likely require the following series in order to apply correctly:
> > > > > > > https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
> > > > > > >
> > > > > > > Adam Ford (5):
> > > > > > > media: hantro: Add support for i.MX8M Mini
> > > > > > > arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> > > > > > > media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> > > > > > > media: hantro: Add H1 encoder support on i.MX8M Mini
> > > > > > > arm64: dts: imx8mm: Enable Hantro H1 Encoder
> > > > > > >
> > > > > > > arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> > > > > > > drivers/staging/media/hantro/hantro_drv.c | 3 +
> > > > > > > drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> > > > > > > drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> > > > > > > .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> > > > > > > 5 files changed, 231 insertions(+), 21 deletions(-)
> > > > > > >
> > > > > >
> > > >
> > >
> > > Nicolas and Adam,
> > >
> > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > H1 JPEG encode using GStreamer 1.18.5:
> > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > ^ v4l2jpegenc
> >
> > This is just a transcript error ?
>
> Nicolas,
>
> No! Thanks for catching my mistake. I was testing with software encode... ooops!
>
> 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> the board so likely a power-domain issue there?
The v4l2-compliance tests fail on the h1 decoder with a hang, but I
think we're writing to registers which are not documented in the Mini
TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
feature, but some of the registers state JPEG, but because some of the
registers written for the H1 are not documented in the TRM. If those
registers are restricted or not in this SoC, I am concerned that it
might be related. I'll try to run some more tests this weekend to
check on the status of the power-domain stuff.
>
> >
> > > host=192.168.1.146 port=5000
> > > viewed on client@192.168.1.146 via:
> > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > rtpjpegdepay ! jpegdec ! autovideosink
> > >
> > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> >
> > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > a single repository. We are very close to 1.20, which will include stable API
> > support of H264, MPEG2 and VP8 decoding.
> >
>
> Ok, let me see if I can navigate through the build process and I'll
> get back to you.
>
> Thanks,
>
> Tim
>
> > >
> > > I have CSI capture and DSI display currently working on
> > > imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> > > supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> > > can't efficiently convert to something the JPEG encoder likes without
> > > bayer2rgbneon (a libneon version).
> > >
> > > I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> > > wide range of data formats but I'm not sure how to tap into this as
> > > that hardware is managed by the vivante driver. On the IMX6QDL there
> > > is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> > > csc/scaler driver for but I don't see any equivalent currently for
> > > IMX8MM.
> > >
> > > Best regards,
> > >
> > > Tim
> >
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-18 18:16 ` Adam Ford
@ 2021-11-19 16:29 ` Nicolas Dufresne
2021-11-19 23:37 ` Adam Ford
0 siblings, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-11-19 16:29 UTC (permalink / raw)
To: Adam Ford, Tim Harvey
Cc: linux-media, Schrempf Frieder, Marek Vasut, Jagan Teki,
Adam Ford-BE, cstevens, Ezequiel Garcia, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Hi Adam, Tim,
[...]
> > > > Nicolas and Adam,
> > > >
> > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > ^ v4l2jpegenc
> > >
> > > This is just a transcript error ?
> >
> > Nicolas,
> >
> > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> >
> > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > the board so likely a power-domain issue there?
>
> The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> think we're writing to registers which are not documented in the Mini
> TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> feature, but some of the registers state JPEG, but because some of the
> registers written for the H1 are not documented in the TRM. If those
> registers are restricted or not in this SoC, I am concerned that it
> might be related. I'll try to run some more tests this weekend to
> check on the status of the power-domain stuff.
To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
PROF/profile is on or off). If your board hang while reading this, you likely
didn't get the power bit right.
IMX8 has an undocumented control block thing that we have been fighting with in
imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
on NXP solution to be submitted (they asked us to wait, still waiting =)).
> >
> > >
> > > > host=192.168.1.146 port=5000
> > > > viewed on client@192.168.1.146 via:
> > > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > > rtpjpegdepay ! jpegdec ! autovideosink
> > > >
> > > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> > >
> > > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > > a single repository. We are very close to 1.20, which will include stable API
> > > support of H264, MPEG2 and VP8 decoding.
> > >
> >
> > Ok, let me see if I can navigate through the build process and I'll
> > get back to you.
> >
> > Thanks,
> >
> > Tim
> >
> > > >
> > > > I have CSI capture and DSI display currently working on
> > > > imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> > > > supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> > > > can't efficiently convert to something the JPEG encoder likes without
> > > > bayer2rgbneon (a libneon version).
> > > >
> > > > I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> > > > wide range of data formats but I'm not sure how to tap into this as
> > > > that hardware is managed by the vivante driver. On the IMX6QDL there
> > > > is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> > > > csc/scaler driver for but I don't see any equivalent currently for
> > > > IMX8MM.
> > > >
> > > > Best regards,
> > > >
> > > > Tim
> > >
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-19 16:29 ` Nicolas Dufresne
@ 2021-11-19 23:37 ` Adam Ford
2021-11-20 15:36 ` Adam Ford
0 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-19 23:37 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: Tim Harvey, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
>
> Hi Adam, Tim,
>
> [...]
> > > > > Nicolas and Adam,
> > > > >
> > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > ^ v4l2jpegenc
> > > >
> > > > This is just a transcript error ?
> > >
> > > Nicolas,
> > >
> > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > >
> > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > the board so likely a power-domain issue there?
> >
> > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > think we're writing to registers which are not documented in the Mini
> > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > feature, but some of the registers state JPEG, but because some of the
> > registers written for the H1 are not documented in the TRM. If those
> > registers are restricted or not in this SoC, I am concerned that it
> > might be related. I'll try to run some more tests this weekend to
> > check on the status of the power-domain stuff.
>
> To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> PROF/profile is on or off). If your board hang while reading this, you likely
> didn't get the power bit right.
>
> IMX8 has an undocumented control block thing that we have been fighting with in
> imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> on NXP solution to be submitted (they asked us to wait, still waiting =)).
Nicolas,
Thanks for the suggestion to read offset FC. There was an attempt
made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
power-domains with the GPC driver. Unfortunately, it does appear to
hang, so it might not be operating correctly.
Lucas,
Do you have any idea of stuff I can try to see if the power domain is
coming online correctly?
[ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
[ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
[ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
[ 10.728927] vpu: set fuse bits to enable
[ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
[ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
[ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
[ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
[ 11.004726] hantro-vpu 38300000.video-codec: registered
nxp,imx8mm-vpu-dec as /dev/video0
[ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
[ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
[ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
[ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
[ 11.139634] hantro-vpu 38310000.video-codec: registered
nxp,imx8mm-vpu-g2-dec as /dev/video1
[ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
[ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
[ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
[ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
[ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
<hang>
adam
>
> > >
> > > >
> > > > > host=192.168.1.146 port=5000
> > > > > viewed on client@192.168.1.146 via:
> > > > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > > > rtpjpegdepay ! jpegdec ! autovideosink
> > > > >
> > > > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> > > >
> > > > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > > > a single repository. We are very close to 1.20, which will include stable API
> > > > support of H264, MPEG2 and VP8 decoding.
> > > >
> > >
> > > Ok, let me see if I can navigate through the build process and I'll
> > > get back to you.
> > >
> > > Thanks,
> > >
> > > Tim
> > >
> > > > >
> > > > > I have CSI capture and DSI display currently working on
> > > > > imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> > > > > supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> > > > > can't efficiently convert to something the JPEG encoder likes without
> > > > > bayer2rgbneon (a libneon version).
> > > > >
> > > > > I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> > > > > wide range of data formats but I'm not sure how to tap into this as
> > > > > that hardware is managed by the vivante driver. On the IMX6QDL there
> > > > > is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> > > > > csc/scaler driver for but I don't see any equivalent currently for
> > > > > IMX8MM.
> > > > >
> > > > > Best regards,
> > > > >
> > > > > Tim
> > > >
>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-19 23:37 ` Adam Ford
@ 2021-11-20 15:36 ` Adam Ford
2021-11-22 17:25 ` Tim Harvey
0 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-20 15:36 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: Tim Harvey, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Fri, Nov 19, 2021 at 5:37 PM Adam Ford <aford173@gmail.com> wrote:
>
> On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> >
> > Hi Adam, Tim,
> >
> > [...]
> > > > > > Nicolas and Adam,
> > > > > >
> > > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > > ^ v4l2jpegenc
> > > > >
> > > > > This is just a transcript error ?
> > > >
> > > > Nicolas,
> > > >
> > > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > > >
> > > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > > the board so likely a power-domain issue there?
> > >
> > > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > > think we're writing to registers which are not documented in the Mini
> > > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > > feature, but some of the registers state JPEG, but because some of the
> > > registers written for the H1 are not documented in the TRM. If those
> > > registers are restricted or not in this SoC, I am concerned that it
> > > might be related. I'll try to run some more tests this weekend to
> > > check on the status of the power-domain stuff.
> >
> > To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> > in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> > synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> > PROF/profile is on or off). If your board hang while reading this, you likely
> > didn't get the power bit right.
> >
> > IMX8 has an undocumented control block thing that we have been fighting with in
> > imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> > on NXP solution to be submitted (they asked us to wait, still waiting =)).
>
> Nicolas,
>
> Thanks for the suggestion to read offset FC. There was an attempt
> made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
> power-domains with the GPC driver. Unfortunately, it does appear to
> hang, so it might not be operating correctly.
>
> Lucas,
>
> Do you have any idea of stuff I can try to see if the power domain is
> coming online correctly?
>
> [ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
> [ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
> [ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
> [ 10.728927] vpu: set fuse bits to enable
> [ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
> [ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
> [ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
> [ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
> [ 11.004726] hantro-vpu 38300000.video-codec: registered
> nxp,imx8mm-vpu-dec as /dev/video0
> [ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
> [ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
> [ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
> [ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
> [ 11.139634] hantro-vpu 38310000.video-codec: registered
> nxp,imx8mm-vpu-g2-dec as /dev/video1
> [ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
> [ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
> [ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
> [ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
> [ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
>
> <hang>
>
> adam
>
Nicolas, Tim, and Lucas,
I think I have the hanging resolved in the power domains, and I'll be
pushing the fix to the GPCv2.
For the H1 Encoder, I added some debugging code to read the offset
0xfc and print some data based on the findings of that VPU-h1 offset.
I basically check the various bits per the TRM to see if they are set
and print some splat to indicate whether or not the function is
supported.
[ 8.861865] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
[ 8.870594] hantro-vpu 38320000.video-codec: Stabilization supported by HW
[ 8.889341] hantro-vpu 38320000.video-codec: VP8 encoding supported by HW
[ 8.899386] hantro-vpu 38320000.video-codec: H.264 encoding supported by HW
[ 8.918171] hantro-vpu 38320000.video-codec: RGB to YUV conversion
supported by HW
[ 8.934067] hantro-vpu 38320000.video-codec: registered
nxp,imx8mm-vpu-h1-enc as /dev/video2
Unfortunately, JPEG is not listed as supported. :-(
However, the hanging stops occurring, so I'll be posting a patch to
update the GPCv2 code. I can reduce sone device tree duplication, and
the G2 throws some splat, but that will be a separate discussion.
I can also run v4l2-compliance on the H1 node, and it responds without hanging.
root@beacon-imx8mm-kit:~# v4l2-compliance -d2
v4l2-compliance SHA: not available
, 64 bits, 64-bit time_t
Compliance test for hantro-vpu device /dev/video2:
Driver Info:
Driver name : hantro-vpu
Card type : nxp,imx8mm-vpu-h1-enc
Bus info : platform: hantro-vpu
Driver version : 5.16.0
Capabilities : 0x84204000
Video Memory-to-Memory Multiplanar
Streaming
Extended Pix Format
Device Capabilities
Device Caps : 0x04204000
Video Memory-to-Memory Multiplanar
< snip>
Total for hantro-vpu device /dev/video2: 46, Succeeded: 46, Failed: 0,
Warnings: 0
I'll do an RFCv2 on the Hantro G1 and G2 with the H1 removed based on
the updated GPCv2 code I'll be pushing shortly, but at least the
system doesn't hang, so I'm fairly confident the power domains are
working better now even if we cannot support the JPEG.
adam
> >
> > > >
> > > > >
> > > > > > host=192.168.1.146 port=5000
> > > > > > viewed on client@192.168.1.146 via:
> > > > > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > > > > rtpjpegdepay ! jpegdec ! autovideosink
> > > > > >
> > > > > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > > > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> > > > >
> > > > > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > > > > a single repository. We are very close to 1.20, which will include stable API
> > > > > support of H264, MPEG2 and VP8 decoding.
> > > > >
> > > >
> > > > Ok, let me see if I can navigate through the build process and I'll
> > > > get back to you.
> > > >
> > > > Thanks,
> > > >
> > > > Tim
> > > >
> > > > > >
> > > > > > I have CSI capture and DSI display currently working on
> > > > > > imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> > > > > > supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> > > > > > can't efficiently convert to something the JPEG encoder likes without
> > > > > > bayer2rgbneon (a libneon version).
> > > > > >
> > > > > > I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> > > > > > wide range of data formats but I'm not sure how to tap into this as
> > > > > > that hardware is managed by the vivante driver. On the IMX6QDL there
> > > > > > is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> > > > > > csc/scaler driver for but I don't see any equivalent currently for
> > > > > > IMX8MM.
> > > > > >
> > > > > > Best regards,
> > > > > >
> > > > > > Tim
> > > > >
> >
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-20 15:36 ` Adam Ford
@ 2021-11-22 17:25 ` Tim Harvey
2021-11-23 20:07 ` Nicolas Dufresne
0 siblings, 1 reply; 42+ messages in thread
From: Tim Harvey @ 2021-11-22 17:25 UTC (permalink / raw)
To: Adam Ford
Cc: Nicolas Dufresne, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Sat, Nov 20, 2021 at 7:36 AM Adam Ford <aford173@gmail.com> wrote:
>
> On Fri, Nov 19, 2021 at 5:37 PM Adam Ford <aford173@gmail.com> wrote:
> >
> > On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > >
> > > Hi Adam, Tim,
> > >
> > > [...]
> > > > > > > Nicolas and Adam,
> > > > > > >
> > > > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > > > ^ v4l2jpegenc
> > > > > >
> > > > > > This is just a transcript error ?
> > > > >
> > > > > Nicolas,
> > > > >
> > > > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > > > >
> > > > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > > > the board so likely a power-domain issue there?
> > > >
> > > > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > > > think we're writing to registers which are not documented in the Mini
> > > > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > > > feature, but some of the registers state JPEG, but because some of the
> > > > registers written for the H1 are not documented in the TRM. If those
> > > > registers are restricted or not in this SoC, I am concerned that it
> > > > might be related. I'll try to run some more tests this weekend to
> > > > check on the status of the power-domain stuff.
> > >
> > > To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> > > in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> > > synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> > > PROF/profile is on or off). If your board hang while reading this, you likely
> > > didn't get the power bit right.
> > >
> > > IMX8 has an undocumented control block thing that we have been fighting with in
> > > imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> > > on NXP solution to be submitted (they asked us to wait, still waiting =)).
> >
> > Nicolas,
> >
> > Thanks for the suggestion to read offset FC. There was an attempt
> > made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
> > power-domains with the GPC driver. Unfortunately, it does appear to
> > hang, so it might not be operating correctly.
> >
> > Lucas,
> >
> > Do you have any idea of stuff I can try to see if the power domain is
> > coming online correctly?
> >
> > [ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
> > [ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
> > [ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
> > [ 10.728927] vpu: set fuse bits to enable
> > [ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
> > [ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
> > [ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
> > [ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
> > [ 11.004726] hantro-vpu 38300000.video-codec: registered
> > nxp,imx8mm-vpu-dec as /dev/video0
> > [ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
> > [ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
> > [ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
> > [ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
> > [ 11.139634] hantro-vpu 38310000.video-codec: registered
> > nxp,imx8mm-vpu-g2-dec as /dev/video1
> > [ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
> > [ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
> > [ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
> > [ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
> > [ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> >
> > <hang>
> >
> > adam
> >
>
> Nicolas, Tim, and Lucas,
>
> I think I have the hanging resolved in the power domains, and I'll be
> pushing the fix to the GPCv2.
>
> For the H1 Encoder, I added some debugging code to read the offset
> 0xfc and print some data based on the findings of that VPU-h1 offset.
> I basically check the various bits per the TRM to see if they are set
> and print some splat to indicate whether or not the function is
> supported.
>
> [ 8.861865] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> [ 8.870594] hantro-vpu 38320000.video-codec: Stabilization supported by HW
> [ 8.889341] hantro-vpu 38320000.video-codec: VP8 encoding supported by HW
> [ 8.899386] hantro-vpu 38320000.video-codec: H.264 encoding supported by HW
> [ 8.918171] hantro-vpu 38320000.video-codec: RGB to YUV conversion
> supported by HW
> [ 8.934067] hantro-vpu 38320000.video-codec: registered
> nxp,imx8mm-vpu-h1-enc as /dev/video2
>
> Unfortunately, JPEG is not listed as supported. :-(
Adam,
Well not having JPEG encode support is unfortunate, and unexpected. Do
we not have hantro support yet for VP8/H264 encode?
I haven't quite figured out how to build a modern mono-repo gstreamer
on the ubuntu 20.04 rootfs I'm using so I haven't been able to test
VPU encode/decode properly. I'll keep working on it when I'm back in
the office the following week.
Best regards,
Tim
>
> However, the hanging stops occurring, so I'll be posting a patch to
> update the GPCv2 code. I can reduce sone device tree duplication, and
> the G2 throws some splat, but that will be a separate discussion.
>
> I can also run v4l2-compliance on the H1 node, and it responds without hanging.
>
> root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> v4l2-compliance SHA: not available
> , 64 bits, 64-bit time_t
>
> Compliance test for hantro-vpu device /dev/video2:
>
> Driver Info:
> Driver name : hantro-vpu
> Card type : nxp,imx8mm-vpu-h1-enc
> Bus info : platform: hantro-vpu
> Driver version : 5.16.0
> Capabilities : 0x84204000
> Video Memory-to-Memory Multiplanar
> Streaming
> Extended Pix Format
> Device Capabilities
> Device Caps : 0x04204000
> Video Memory-to-Memory Multiplanar
>
> < snip>
>
> Total for hantro-vpu device /dev/video2: 46, Succeeded: 46, Failed: 0,
> Warnings: 0
>
> I'll do an RFCv2 on the Hantro G1 and G2 with the H1 removed based on
> the updated GPCv2 code I'll be pushing shortly, but at least the
> system doesn't hang, so I'm fairly confident the power domains are
> working better now even if we cannot support the JPEG.
>
> adam
>
> > >
> > > > >
> > > > > >
> > > > > > > host=192.168.1.146 port=5000
> > > > > > > viewed on client@192.168.1.146 via:
> > > > > > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > > > > > rtpjpegdepay ! jpegdec ! autovideosink
> > > > > > >
> > > > > > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > > > > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> > > > > >
> > > > > > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > > > > > a single repository. We are very close to 1.20, which will include stable API
> > > > > > support of H264, MPEG2 and VP8 decoding.
> > > > > >
> > > > >
> > > > > Ok, let me see if I can navigate through the build process and I'll
> > > > > get back to you.
> > > > >
> > > > > Thanks,
> > > > >
> > > > > Tim
> > > > >
> > > > > > >
> > > > > > > I have CSI capture and DSI display currently working on
> > > > > > > imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> > > > > > > supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> > > > > > > can't efficiently convert to something the JPEG encoder likes without
> > > > > > > bayer2rgbneon (a libneon version).
> > > > > > >
> > > > > > > I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> > > > > > > wide range of data formats but I'm not sure how to tap into this as
> > > > > > > that hardware is managed by the vivante driver. On the IMX6QDL there
> > > > > > > is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> > > > > > > csc/scaler driver for but I don't see any equivalent currently for
> > > > > > > IMX8MM.
> > > > > > >
> > > > > > > Best regards,
> > > > > > >
> > > > > > > Tim
> > > > > >
> > >
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-22 17:25 ` Tim Harvey
@ 2021-11-23 20:07 ` Nicolas Dufresne
2021-11-29 16:48 ` Adam Ford
0 siblings, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-11-23 20:07 UTC (permalink / raw)
To: Tim Harvey, Adam Ford
Cc: linux-media, Schrempf Frieder, Marek Vasut, Jagan Teki,
Adam Ford-BE, cstevens, Ezequiel Garcia, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le lundi 22 novembre 2021 à 09:25 -0800, Tim Harvey a écrit :
> On Sat, Nov 20, 2021 at 7:36 AM Adam Ford <aford173@gmail.com> wrote:
> >
> > On Fri, Nov 19, 2021 at 5:37 PM Adam Ford <aford173@gmail.com> wrote:
> > >
> > > On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > >
> > > > Hi Adam, Tim,
> > > >
> > > > [...]
> > > > > > > > Nicolas and Adam,
> > > > > > > >
> > > > > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > > > > ^ v4l2jpegenc
> > > > > > >
> > > > > > > This is just a transcript error ?
> > > > > >
> > > > > > Nicolas,
> > > > > >
> > > > > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > > > > >
> > > > > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > > > > the board so likely a power-domain issue there?
> > > > >
> > > > > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > > > > think we're writing to registers which are not documented in the Mini
> > > > > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > > > > feature, but some of the registers state JPEG, but because some of the
> > > > > registers written for the H1 are not documented in the TRM. If those
> > > > > registers are restricted or not in this SoC, I am concerned that it
> > > > > might be related. I'll try to run some more tests this weekend to
> > > > > check on the status of the power-domain stuff.
> > > >
> > > > To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> > > > in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> > > > synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> > > > PROF/profile is on or off). If your board hang while reading this, you likely
> > > > didn't get the power bit right.
> > > >
> > > > IMX8 has an undocumented control block thing that we have been fighting with in
> > > > imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> > > > on NXP solution to be submitted (they asked us to wait, still waiting =)).
> > >
> > > Nicolas,
> > >
> > > Thanks for the suggestion to read offset FC. There was an attempt
> > > made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
> > > power-domains with the GPC driver. Unfortunately, it does appear to
> > > hang, so it might not be operating correctly.
> > >
> > > Lucas,
> > >
> > > Do you have any idea of stuff I can try to see if the power domain is
> > > coming online correctly?
> > >
> > > [ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
> > > [ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
> > > [ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
> > > [ 10.728927] vpu: set fuse bits to enable
> > > [ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
> > > [ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
> > > [ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
> > > [ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
> > > [ 11.004726] hantro-vpu 38300000.video-codec: registered
> > > nxp,imx8mm-vpu-dec as /dev/video0
> > > [ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
> > > [ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
> > > [ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
> > > [ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
> > > [ 11.139634] hantro-vpu 38310000.video-codec: registered
> > > nxp,imx8mm-vpu-g2-dec as /dev/video1
> > > [ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
> > > [ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
> > > [ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
> > > [ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
> > > [ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > >
> > > <hang>
> > >
> > > adam
> > >
> >
> > Nicolas, Tim, and Lucas,
> >
> > I think I have the hanging resolved in the power domains, and I'll be
> > pushing the fix to the GPCv2.
> >
> > For the H1 Encoder, I added some debugging code to read the offset
> > 0xfc and print some data based on the findings of that VPU-h1 offset.
> > I basically check the various bits per the TRM to see if they are set
> > and print some splat to indicate whether or not the function is
> > supported.
> >
> > [ 8.861865] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > [ 8.870594] hantro-vpu 38320000.video-codec: Stabilization supported by HW
> > [ 8.889341] hantro-vpu 38320000.video-codec: VP8 encoding supported by HW
> > [ 8.899386] hantro-vpu 38320000.video-codec: H.264 encoding supported by HW
> > [ 8.918171] hantro-vpu 38320000.video-codec: RGB to YUV conversion
> > supported by HW
> > [ 8.934067] hantro-vpu 38320000.video-codec: registered
> > nxp,imx8mm-vpu-h1-enc as /dev/video2
> >
> > Unfortunately, JPEG is not listed as supported. :-(
>
> Adam,
>
> Well not having JPEG encode support is unfortunate, and unexpected. Do
> we not have hantro support yet for VP8/H264 encode?
There is no mainline support yet. You can derive from RK3288 support using Google ChromeOS method (a v4l2 plugins that simulate in userspace a stateful encoder):
- libv4l plugins / https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/refs/heads/master
- Kernel Driver / https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu/
>
> I haven't quite figured out how to build a modern mono-repo gstreamer
> on the ubuntu 20.04 rootfs I'm using so I haven't been able to test
> VPU encode/decode properly. I'll keep working on it when I'm back in
> the office the following week.
Did a quick test to make sure there isn't any ubuntu specific blockers, here's a
dirty script that produce a minimal GStreamer, there was really nothing special
compare to other meson projects. Note that I use --wrap-mode=nofallback to avoid
letting GStreamer complete it's feature-set by downloading the planet. This
already build quite a lot and could likely be made smaller by avoid plugins-good
build-dep call, but then you need to check for v4l2odecs and video4linux devs
(mostly gudev a glib udev binding).
# Install ubuntu
podman run -it --rm ubuntu:20.04
sed -i "s/# deb-src/deb-src/" /etc/apt/sources.list
apt update
apt build-dep gstreamer1.0-plugins-good
apt install git python3-pip flex bison
# Need a newer meson
pip3 install --user meson
export PATH=$PATH:~/.local/bin
# Build GStreamer
git clone https://gitlab.freedesktop.org/gstreamer/gstreamer.git
cd gstreamer
meson setup build --wrap-mode=nofallback
ninja -C build
# Run in-place
./gst-env.py
gst-inspect-1.0 v4l2codecs
gst-inspect 1.0 video4linux2
>
> Best regards,
>
> Tim
>
> >
> > However, the hanging stops occurring, so I'll be posting a patch to
> > update the GPCv2 code. I can reduce sone device tree duplication, and
> > the G2 throws some splat, but that will be a separate discussion.
> >
> > I can also run v4l2-compliance on the H1 node, and it responds without hanging.
> >
> > root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> > v4l2-compliance SHA: not available
> > , 64 bits, 64-bit time_t
> >
> > Compliance test for hantro-vpu device /dev/video2:
> >
> > Driver Info:
> > Driver name : hantro-vpu
> > Card type : nxp,imx8mm-vpu-h1-enc
> > Bus info : platform: hantro-vpu
> > Driver version : 5.16.0
> > Capabilities : 0x84204000
> > Video Memory-to-Memory Multiplanar
> > Streaming
> > Extended Pix Format
> > Device Capabilities
> > Device Caps : 0x04204000
> > Video Memory-to-Memory Multiplanar
> >
> > < snip>
> >
> > Total for hantro-vpu device /dev/video2: 46, Succeeded: 46, Failed: 0,
> > Warnings: 0
> >
> > I'll do an RFCv2 on the Hantro G1 and G2 with the H1 removed based on
> > the updated GPCv2 code I'll be pushing shortly, but at least the
> > system doesn't hang, so I'm fairly confident the power domains are
> > working better now even if we cannot support the JPEG.
> >
> > adam
> >
> > > >
> > > > > >
> > > > > > >
> > > > > > > > host=192.168.1.146 port=5000
> > > > > > > > viewed on client@192.168.1.146 via:
> > > > > > > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > > > > > > rtpjpegdepay ! jpegdec ! autovideosink
> > > > > > > >
> > > > > > > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > > > > > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> > > > > > >
> > > > > > > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > > > > > > a single repository. We are very close to 1.20, which will include stable API
> > > > > > > support of H264, MPEG2 and VP8 decoding.
> > > > > > >
> > > > > >
> > > > > > Ok, let me see if I can navigate through the build process and I'll
> > > > > > get back to you.
> > > > > >
> > > > > > Thanks,
> > > > > >
> > > > > > Tim
> > > > > >
> > > > > > > >
> > > > > > > > I have CSI capture and DSI display currently working on
> > > > > > > > imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> > > > > > > > supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> > > > > > > > can't efficiently convert to something the JPEG encoder likes without
> > > > > > > > bayer2rgbneon (a libneon version).
> > > > > > > >
> > > > > > > > I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> > > > > > > > wide range of data formats but I'm not sure how to tap into this as
> > > > > > > > that hardware is managed by the vivante driver. On the IMX6QDL there
> > > > > > > > is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> > > > > > > > csc/scaler driver for but I don't see any equivalent currently for
> > > > > > > > IMX8MM.
> > > > > > > >
> > > > > > > > Best regards,
> > > > > > > >
> > > > > > > > Tim
> > > > > > >
> > > >
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-23 20:07 ` Nicolas Dufresne
@ 2021-11-29 16:48 ` Adam Ford
2021-11-29 16:54 ` Ezequiel Garcia
0 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-29 16:48 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: Tim Harvey, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Tue, Nov 23, 2021 at 2:07 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
>
> Le lundi 22 novembre 2021 à 09:25 -0800, Tim Harvey a écrit :
> > On Sat, Nov 20, 2021 at 7:36 AM Adam Ford <aford173@gmail.com> wrote:
> > >
> > > On Fri, Nov 19, 2021 at 5:37 PM Adam Ford <aford173@gmail.com> wrote:
> > > >
> > > > On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > >
> > > > > Hi Adam, Tim,
> > > > >
> > > > > [...]
> > > > > > > > > Nicolas and Adam,
> > > > > > > > >
> > > > > > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > > > > > ^ v4l2jpegenc
> > > > > > > >
> > > > > > > > This is just a transcript error ?
> > > > > > >
> > > > > > > Nicolas,
> > > > > > >
> > > > > > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > > > > > >
> > > > > > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > > > > > the board so likely a power-domain issue there?
> > > > > >
> > > > > > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > > > > > think we're writing to registers which are not documented in the Mini
> > > > > > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > > > > > feature, but some of the registers state JPEG, but because some of the
> > > > > > registers written for the H1 are not documented in the TRM. If those
> > > > > > registers are restricted or not in this SoC, I am concerned that it
> > > > > > might be related. I'll try to run some more tests this weekend to
> > > > > > check on the status of the power-domain stuff.
> > > > >
> > > > > To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> > > > > in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> > > > > synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> > > > > PROF/profile is on or off). If your board hang while reading this, you likely
> > > > > didn't get the power bit right.
> > > > >
> > > > > IMX8 has an undocumented control block thing that we have been fighting with in
> > > > > imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> > > > > on NXP solution to be submitted (they asked us to wait, still waiting =)).
> > > >
> > > > Nicolas,
> > > >
> > > > Thanks for the suggestion to read offset FC. There was an attempt
> > > > made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
> > > > power-domains with the GPC driver. Unfortunately, it does appear to
> > > > hang, so it might not be operating correctly.
> > > >
> > > > Lucas,
> > > >
> > > > Do you have any idea of stuff I can try to see if the power domain is
> > > > coming online correctly?
> > > >
> > > > [ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
> > > > [ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
> > > > [ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
> > > > [ 10.728927] vpu: set fuse bits to enable
> > > > [ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
> > > > [ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
> > > > [ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
> > > > [ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
> > > > [ 11.004726] hantro-vpu 38300000.video-codec: registered
> > > > nxp,imx8mm-vpu-dec as /dev/video0
> > > > [ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
> > > > [ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
> > > > [ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
> > > > [ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
> > > > [ 11.139634] hantro-vpu 38310000.video-codec: registered
> > > > nxp,imx8mm-vpu-g2-dec as /dev/video1
> > > > [ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
> > > > [ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
> > > > [ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
> > > > [ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
> > > > [ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > >
> > > > <hang>
> > > >
> > > > adam
> > > >
> > >
> > > Nicolas, Tim, and Lucas,
> > >
> > > I think I have the hanging resolved in the power domains, and I'll be
> > > pushing the fix to the GPCv2.
> > >
> > > For the H1 Encoder, I added some debugging code to read the offset
> > > 0xfc and print some data based on the findings of that VPU-h1 offset.
> > > I basically check the various bits per the TRM to see if they are set
> > > and print some splat to indicate whether or not the function is
> > > supported.
> > >
> > > [ 8.861865] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > [ 8.870594] hantro-vpu 38320000.video-codec: Stabilization supported by HW
> > > [ 8.889341] hantro-vpu 38320000.video-codec: VP8 encoding supported by HW
> > > [ 8.899386] hantro-vpu 38320000.video-codec: H.264 encoding supported by HW
> > > [ 8.918171] hantro-vpu 38320000.video-codec: RGB to YUV conversion
> > > supported by HW
> > > [ 8.934067] hantro-vpu 38320000.video-codec: registered
> > > nxp,imx8mm-vpu-h1-enc as /dev/video2
> > >
> > > Unfortunately, JPEG is not listed as supported. :-(
> >
> > Adam,
> >
> > Well not having JPEG encode support is unfortunate, and unexpected. Do
> > we not have hantro support yet for VP8/H264 encode?
>
> There is no mainline support yet. You can derive from RK3288 support using Google ChromeOS method (a v4l2 plugins that simulate in userspace a stateful encoder):
>
> - libv4l plugins / https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/refs/heads/master
> - Kernel Driver / https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu/
>
> >
> > I haven't quite figured out how to build a modern mono-repo gstreamer
> > on the ubuntu 20.04 rootfs I'm using so I haven't been able to test
> > VPU encode/decode properly. I'll keep working on it when I'm back in
> > the office the following week.
>
> Did a quick test to make sure there isn't any ubuntu specific blockers, here's a
> dirty script that produce a minimal GStreamer, there was really nothing special
> compare to other meson projects. Note that I use --wrap-mode=nofallback to avoid
> letting GStreamer complete it's feature-set by downloading the planet. This
> already build quite a lot and could likely be made smaller by avoid plugins-good
> build-dep call, but then you need to check for v4l2odecs and video4linux devs
> (mostly gudev a glib udev binding).
>
> # Install ubuntu
> podman run -it --rm ubuntu:20.04
> sed -i "s/# deb-src/deb-src/" /etc/apt/sources.list
> apt update
> apt build-dep gstreamer1.0-plugins-good
> apt install git python3-pip flex bison
>
> # Need a newer meson
> pip3 install --user meson
> export PATH=$PATH:~/.local/bin
>
> # Build GStreamer
> git clone https://gitlab.freedesktop.org/gstreamer/gstreamer.git
> cd gstreamer
> meson setup build --wrap-mode=nofallback
> ninja -C build
>
> # Run in-place
> ./gst-env.py
> gst-inspect-1.0 v4l2codecs
> gst-inspect 1.0 video4linux2
>
Thanks for the suggestions.
I downloaded what's in the master repo:
[gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 v4l2codecs
** (gst-plugin-scanner:7317): CRITICAL **: 10:29:51.847: can't find
gi.repository.Gst
Plugin Details:
Name v4l2codecs
Description V4L2 CODEC Accelerators plugin
Filename
/root/gstreamer/builddir/subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so
Version 1.19.3.1
License LGPL
Source module gst-plugins-bad
Binary package GStreamer Bad Plug-ins git
Origin URL Unknown package origin
v4l2slh264dec: V4L2 Stateless H.264 Video Decoder
v4l2slmpeg2dec: V4L2 Stateless Mpeg2 Video Decoder
v4l2slvp8alphadecodebin: VP8 Alpha Decoder
v4l2slvp8dec: V4L2 Stateless VP8 Video Decoder
4 features:
+-- 4 elements
[gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 video4linux2
Plugin Details:
Name video4linux2
Description elements for Video 4 Linux
Filename
/root/gstreamer/builddir/subprojects/gst-plugins-good/sys/v4l2/libgstvideo4linux2.so
Version 1.19.3.1
License LGPL
Source module gst-plugins-good
Binary package GStreamer Good Plug-ins git
Origin URL Unknown package origin
v4l2deviceprovider: Video (video4linux2) Device Provider
v4l2jpegenc: V4L2 JPEG Encoder
v4l2radio: Radio (video4linux2) Tuner
v4l2sink: Video (video4linux2) Sink
v4l2src: Video (video4linux2) Source
5 features:
+-- 4 elements
+-- 1 device providers
I still have the H1 encoder enabled, but I know JPEG isn't supported,
so I'm going to attempt to do some decoding and pipe to fakesink since
I don't have a functional display yet.
gst-launch-1.0 -ev filesrc location=trailer_1080p_h264_mp3.avi !
decodebin3 ! fakesink
Redistribute latency...
/GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:src:
caps = video/x-raw, format=(string)NV12, width=(int)1920,
height=(int)1080, interlace-mode=(string)progressive,
multiview-mode=(string)mono,
multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
/GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0:
caps = video/x-raw, format=(string)NV12, width=(int)1920,
height=(int)1080, interlace-mode=(string)progressive,
multiview-mode=(string)mono,
multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
/GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0.GstProxyPad:proxypad6:
caps = video/x-raw, format=(string)NV12, width=(int)1920,
height=(int)1080, interlace-mode=(string)progressive,
multiview-mode=(string)mono,
multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
/GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/GstMultiQueue:multiqueue0:
min-interleave-time = 300000000
Redistribute latency...
/GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:sink:
caps = video/x-h264, variant=(string)itu, framerate=(fraction)25/1,
width=(int)1920, height=(int)1080, chroma-format=(string)4:2:0,
bit-depth-luma=(uint)8, bit-depth-chroma=(uint)8,
parsed=(boolean)true, stream-format=(string)avc, alignment=(string)au,
profile=(string)high, level=(string)4,
codec_data=(buffer)01640028ffe1001a67640028acd940780227e584000003000400000300c83c60c65801000668ebe3cb22c0
New clock: GstSystemClock
And it appears to stream, because the counter increases. I haven't
checked the CPU utilization, but the fact that it shows v4l2slh264dec
is good.
Is there a way to know if/how the decoder is using the proper VPU? I
assume if it wasn't using the proper one, it would fail, but was just
curious.
I think I'll redo the patch without the RFC and without the H1 encoder
unless anyone has any objections. I know I need to rebase on
linux-next anyway because the patches don't apply cleanly. Is there a
specific branch I should use? I don't know if this goes through
Shawn's IMX tree or the media tree (or a combination)
adam
adam
> >
> > Best regards,
> >
> > Tim
> >
> > >
> > > However, the hanging stops occurring, so I'll be posting a patch to
> > > update the GPCv2 code. I can reduce sone device tree duplication, and
> > > the G2 throws some splat, but that will be a separate discussion.
> > >
> > > I can also run v4l2-compliance on the H1 node, and it responds without hanging.
> > >
> > > root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> > > v4l2-compliance SHA: not available
> > > , 64 bits, 64-bit time_t
> > >
> > > Compliance test for hantro-vpu device /dev/video2:
> > >
> > > Driver Info:
> > > Driver name : hantro-vpu
> > > Card type : nxp,imx8mm-vpu-h1-enc
> > > Bus info : platform: hantro-vpu
> > > Driver version : 5.16.0
> > > Capabilities : 0x84204000
> > > Video Memory-to-Memory Multiplanar
> > > Streaming
> > > Extended Pix Format
> > > Device Capabilities
> > > Device Caps : 0x04204000
> > > Video Memory-to-Memory Multiplanar
> > >
> > > < snip>
> > >
> > > Total for hantro-vpu device /dev/video2: 46, Succeeded: 46, Failed: 0,
> > > Warnings: 0
> > >
> > > I'll do an RFCv2 on the Hantro G1 and G2 with the H1 removed based on
> > > the updated GPCv2 code I'll be pushing shortly, but at least the
> > > system doesn't hang, so I'm fairly confident the power domains are
> > > working better now even if we cannot support the JPEG.
> > >
> > > adam
> > >
> > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > > host=192.168.1.146 port=5000
> > > > > > > > > viewed on client@192.168.1.146 via:
> > > > > > > > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > > > > > > > rtpjpegdepay ! jpegdec ! autovideosink
> > > > > > > > >
> > > > > > > > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > > > > > > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> > > > > > > >
> > > > > > > > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > > > > > > > a single repository. We are very close to 1.20, which will include stable API
> > > > > > > > support of H264, MPEG2 and VP8 decoding.
> > > > > > > >
> > > > > > >
> > > > > > > Ok, let me see if I can navigate through the build process and I'll
> > > > > > > get back to you.
> > > > > > >
> > > > > > > Thanks,
> > > > > > >
> > > > > > > Tim
> > > > > > >
> > > > > > > > >
> > > > > > > > > I have CSI capture and DSI display currently working on
> > > > > > > > > imx8mm-venice-gw73xx-0x that I can play with. The CSI sensor only
> > > > > > > > > supports RAW8/RAW10 (and gstreamer currently only supports RAW8) and I
> > > > > > > > > can't efficiently convert to something the JPEG encoder likes without
> > > > > > > > > bayer2rgbneon (a libneon version).
> > > > > > > > >
> > > > > > > > > I see from the IMX8MMRM that the 2D GPU supports scaling etc with a
> > > > > > > > > wide range of data formats but I'm not sure how to tap into this as
> > > > > > > > > that hardware is managed by the vivante driver. On the IMX6QDL there
> > > > > > > > > is a separate IPU block that Philipp Zabel wrote a nice mem2mem
> > > > > > > > > csc/scaler driver for but I don't see any equivalent currently for
> > > > > > > > > IMX8MM.
> > > > > > > > >
> > > > > > > > > Best regards,
> > > > > > > > >
> > > > > > > > > Tim
> > > > > > > >
> > > > >
>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-29 16:48 ` Adam Ford
@ 2021-11-29 16:54 ` Ezequiel Garcia
2021-11-29 18:59 ` Adam Ford
0 siblings, 1 reply; 42+ messages in thread
From: Ezequiel Garcia @ 2021-11-29 16:54 UTC (permalink / raw)
To: Adam Ford
Cc: Nicolas Dufresne, Tim Harvey, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Mon, 29 Nov 2021 at 13:48, Adam Ford <aford173@gmail.com> wrote:
>
> On Tue, Nov 23, 2021 at 2:07 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> >
> > Le lundi 22 novembre 2021 à 09:25 -0800, Tim Harvey a écrit :
> > > On Sat, Nov 20, 2021 at 7:36 AM Adam Ford <aford173@gmail.com> wrote:
> > > >
> > > > On Fri, Nov 19, 2021 at 5:37 PM Adam Ford <aford173@gmail.com> wrote:
> > > > >
> > > > > On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > > >
> > > > > > Hi Adam, Tim,
> > > > > >
> > > > > > [...]
> > > > > > > > > > Nicolas and Adam,
> > > > > > > > > >
> > > > > > > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > > > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > > > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > > > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > > > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > > > > > > ^ v4l2jpegenc
> > > > > > > > >
> > > > > > > > > This is just a transcript error ?
> > > > > > > >
> > > > > > > > Nicolas,
> > > > > > > >
> > > > > > > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > > > > > > >
> > > > > > > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > > > > > > the board so likely a power-domain issue there?
> > > > > > >
> > > > > > > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > > > > > > think we're writing to registers which are not documented in the Mini
> > > > > > > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > > > > > > feature, but some of the registers state JPEG, but because some of the
> > > > > > > registers written for the H1 are not documented in the TRM. If those
> > > > > > > registers are restricted or not in this SoC, I am concerned that it
> > > > > > > might be related. I'll try to run some more tests this weekend to
> > > > > > > check on the status of the power-domain stuff.
> > > > > >
> > > > > > To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> > > > > > in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> > > > > > synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> > > > > > PROF/profile is on or off). If your board hang while reading this, you likely
> > > > > > didn't get the power bit right.
> > > > > >
> > > > > > IMX8 has an undocumented control block thing that we have been fighting with in
> > > > > > imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> > > > > > on NXP solution to be submitted (they asked us to wait, still waiting =)).
> > > > >
> > > > > Nicolas,
> > > > >
> > > > > Thanks for the suggestion to read offset FC. There was an attempt
> > > > > made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
> > > > > power-domains with the GPC driver. Unfortunately, it does appear to
> > > > > hang, so it might not be operating correctly.
> > > > >
> > > > > Lucas,
> > > > >
> > > > > Do you have any idea of stuff I can try to see if the power domain is
> > > > > coming online correctly?
> > > > >
> > > > > [ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
> > > > > [ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
> > > > > [ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
> > > > > [ 10.728927] vpu: set fuse bits to enable
> > > > > [ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
> > > > > [ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
> > > > > [ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
> > > > > [ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
> > > > > [ 11.004726] hantro-vpu 38300000.video-codec: registered
> > > > > nxp,imx8mm-vpu-dec as /dev/video0
> > > > > [ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
> > > > > [ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
> > > > > [ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
> > > > > [ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
> > > > > [ 11.139634] hantro-vpu 38310000.video-codec: registered
> > > > > nxp,imx8mm-vpu-g2-dec as /dev/video1
> > > > > [ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
> > > > > [ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
> > > > > [ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
> > > > > [ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
> > > > > [ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > > >
> > > > > <hang>
> > > > >
> > > > > adam
> > > > >
> > > >
> > > > Nicolas, Tim, and Lucas,
> > > >
> > > > I think I have the hanging resolved in the power domains, and I'll be
> > > > pushing the fix to the GPCv2.
> > > >
> > > > For the H1 Encoder, I added some debugging code to read the offset
> > > > 0xfc and print some data based on the findings of that VPU-h1 offset.
> > > > I basically check the various bits per the TRM to see if they are set
> > > > and print some splat to indicate whether or not the function is
> > > > supported.
> > > >
> > > > [ 8.861865] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > > [ 8.870594] hantro-vpu 38320000.video-codec: Stabilization supported by HW
> > > > [ 8.889341] hantro-vpu 38320000.video-codec: VP8 encoding supported by HW
> > > > [ 8.899386] hantro-vpu 38320000.video-codec: H.264 encoding supported by HW
> > > > [ 8.918171] hantro-vpu 38320000.video-codec: RGB to YUV conversion
> > > > supported by HW
> > > > [ 8.934067] hantro-vpu 38320000.video-codec: registered
> > > > nxp,imx8mm-vpu-h1-enc as /dev/video2
> > > >
> > > > Unfortunately, JPEG is not listed as supported. :-(
> > >
> > > Adam,
> > >
> > > Well not having JPEG encode support is unfortunate, and unexpected. Do
> > > we not have hantro support yet for VP8/H264 encode?
> >
> > There is no mainline support yet. You can derive from RK3288 support using Google ChromeOS method (a v4l2 plugins that simulate in userspace a stateful encoder):
> >
> > - libv4l plugins / https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/refs/heads/master
> > - Kernel Driver / https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu/
> >
> > >
> > > I haven't quite figured out how to build a modern mono-repo gstreamer
> > > on the ubuntu 20.04 rootfs I'm using so I haven't been able to test
> > > VPU encode/decode properly. I'll keep working on it when I'm back in
> > > the office the following week.
> >
> > Did a quick test to make sure there isn't any ubuntu specific blockers, here's a
> > dirty script that produce a minimal GStreamer, there was really nothing special
> > compare to other meson projects. Note that I use --wrap-mode=nofallback to avoid
> > letting GStreamer complete it's feature-set by downloading the planet. This
> > already build quite a lot and could likely be made smaller by avoid plugins-good
> > build-dep call, but then you need to check for v4l2odecs and video4linux devs
> > (mostly gudev a glib udev binding).
> >
> > # Install ubuntu
> > podman run -it --rm ubuntu:20.04
> > sed -i "s/# deb-src/deb-src/" /etc/apt/sources.list
> > apt update
> > apt build-dep gstreamer1.0-plugins-good
> > apt install git python3-pip flex bison
> >
> > # Need a newer meson
> > pip3 install --user meson
> > export PATH=$PATH:~/.local/bin
> >
> > # Build GStreamer
> > git clone https://gitlab.freedesktop.org/gstreamer/gstreamer.git
> > cd gstreamer
> > meson setup build --wrap-mode=nofallback
> > ninja -C build
> >
> > # Run in-place
> > ./gst-env.py
> > gst-inspect-1.0 v4l2codecs
> > gst-inspect 1.0 video4linux2
> >
> Thanks for the suggestions.
>
> I downloaded what's in the master repo:
>
> [gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 v4l2codecs
>
> ** (gst-plugin-scanner:7317): CRITICAL **: 10:29:51.847: can't find
> gi.repository.Gst
> Plugin Details:
> Name v4l2codecs
> Description V4L2 CODEC Accelerators plugin
> Filename
> /root/gstreamer/builddir/subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so
> Version 1.19.3.1
> License LGPL
> Source module gst-plugins-bad
> Binary package GStreamer Bad Plug-ins git
> Origin URL Unknown package origin
>
> v4l2slh264dec: V4L2 Stateless H.264 Video Decoder
> v4l2slmpeg2dec: V4L2 Stateless Mpeg2 Video Decoder
> v4l2slvp8alphadecodebin: VP8 Alpha Decoder
> v4l2slvp8dec: V4L2 Stateless VP8 Video Decoder
>
> 4 features:
> +-- 4 elements
>
> [gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 video4linux2
> Plugin Details:
> Name video4linux2
> Description elements for Video 4 Linux
> Filename
> /root/gstreamer/builddir/subprojects/gst-plugins-good/sys/v4l2/libgstvideo4linux2.so
> Version 1.19.3.1
> License LGPL
> Source module gst-plugins-good
> Binary package GStreamer Good Plug-ins git
> Origin URL Unknown package origin
>
> v4l2deviceprovider: Video (video4linux2) Device Provider
> v4l2jpegenc: V4L2 JPEG Encoder
> v4l2radio: Radio (video4linux2) Tuner
> v4l2sink: Video (video4linux2) Sink
> v4l2src: Video (video4linux2) Source
>
> 5 features:
> +-- 4 elements
> +-- 1 device providers
>
> I still have the H1 encoder enabled, but I know JPEG isn't supported,
> so I'm going to attempt to do some decoding and pipe to fakesink since
> I don't have a functional display yet.
>
> gst-launch-1.0 -ev filesrc location=trailer_1080p_h264_mp3.avi !
> decodebin3 ! fakesink
>
> Redistribute latency...
> /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:src:
> caps = video/x-raw, format=(string)NV12, width=(int)1920,
> height=(int)1080, interlace-mode=(string)progressive,
> multiview-mode=(string)mono,
> multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0:
> caps = video/x-raw, format=(string)NV12, width=(int)1920,
> height=(int)1080, interlace-mode=(string)progressive,
> multiview-mode=(string)mono,
> multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0.GstProxyPad:proxypad6:
> caps = video/x-raw, format=(string)NV12, width=(int)1920,
> height=(int)1080, interlace-mode=(string)progressive,
> multiview-mode=(string)mono,
> multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/GstMultiQueue:multiqueue0:
> min-interleave-time = 300000000
> Redistribute latency...
> /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:sink:
> caps = video/x-h264, variant=(string)itu, framerate=(fraction)25/1,
> width=(int)1920, height=(int)1080, chroma-format=(string)4:2:0,
> bit-depth-luma=(uint)8, bit-depth-chroma=(uint)8,
> parsed=(boolean)true, stream-format=(string)avc, alignment=(string)au,
> profile=(string)high, level=(string)4,
> codec_data=(buffer)01640028ffe1001a67640028acd940780227e584000003000400000300c83c60c65801000668ebe3cb22c0
> New clock: GstSystemClock
>
> And it appears to stream, because the counter increases. I haven't
> checked the CPU utilization, but the fact that it shows v4l2slh264dec
> is good.
>
> Is there a way to know if/how the decoder is using the proper VPU? I
> assume if it wasn't using the proper one, it would fail, but was just
> curious.
>
A few ways. You can check /proc/interrupts, which should have
VPU activity.
Or enable debug messages for the module,
using the debug hantro parameter. V4L2 has debug messages
that you can enable, see /sys/class/video4linux/video0/dev_debug.
Instead of fakesink you can output to pngenc/jpegenc and check the output
is visually correct. If at all possible, the proper way is to use Fluster,
and report the score you get:
https://github.com/fluendo/fluster
It should be easy to use.
> I think I'll redo the patch without the RFC and without the H1 encoder
> unless anyone has any objections. I know I need to rebase on
> linux-next anyway because the patches don't apply cleanly. Is there a
> specific branch I should use? I don't know if this goes through
> Shawn's IMX tree or the media tree (or a combination)
>
You should rebase on media's master branch:
https://git.linuxtv.org/media_tree.git/log/
Thanks,
Ezequiel
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-29 16:54 ` Ezequiel Garcia
@ 2021-11-29 18:59 ` Adam Ford
2021-11-29 19:35 ` Tim Harvey
0 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-29 18:59 UTC (permalink / raw)
To: Ezequiel Garcia
Cc: Nicolas Dufresne, Tim Harvey, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Mon, Nov 29, 2021 at 10:54 AM Ezequiel Garcia
<ezequiel@vanguardiasur.com.ar> wrote:
>
> On Mon, 29 Nov 2021 at 13:48, Adam Ford <aford173@gmail.com> wrote:
> >
> > On Tue, Nov 23, 2021 at 2:07 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > >
> > > Le lundi 22 novembre 2021 à 09:25 -0800, Tim Harvey a écrit :
> > > > On Sat, Nov 20, 2021 at 7:36 AM Adam Ford <aford173@gmail.com> wrote:
> > > > >
> > > > > On Fri, Nov 19, 2021 at 5:37 PM Adam Ford <aford173@gmail.com> wrote:
> > > > > >
> > > > > > On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > > > >
> > > > > > > Hi Adam, Tim,
> > > > > > >
> > > > > > > [...]
> > > > > > > > > > > Nicolas and Adam,
> > > > > > > > > > >
> > > > > > > > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > > > > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > > > > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > > > > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > > > > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > > > > > > > ^ v4l2jpegenc
> > > > > > > > > >
> > > > > > > > > > This is just a transcript error ?
> > > > > > > > >
> > > > > > > > > Nicolas,
> > > > > > > > >
> > > > > > > > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > > > > > > > >
> > > > > > > > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > > > > > > > the board so likely a power-domain issue there?
> > > > > > > >
> > > > > > > > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > > > > > > > think we're writing to registers which are not documented in the Mini
> > > > > > > > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > > > > > > > feature, but some of the registers state JPEG, but because some of the
> > > > > > > > registers written for the H1 are not documented in the TRM. If those
> > > > > > > > registers are restricted or not in this SoC, I am concerned that it
> > > > > > > > might be related. I'll try to run some more tests this weekend to
> > > > > > > > check on the status of the power-domain stuff.
> > > > > > >
> > > > > > > To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> > > > > > > in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> > > > > > > synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> > > > > > > PROF/profile is on or off). If your board hang while reading this, you likely
> > > > > > > didn't get the power bit right.
> > > > > > >
> > > > > > > IMX8 has an undocumented control block thing that we have been fighting with in
> > > > > > > imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> > > > > > > on NXP solution to be submitted (they asked us to wait, still waiting =)).
> > > > > >
> > > > > > Nicolas,
> > > > > >
> > > > > > Thanks for the suggestion to read offset FC. There was an attempt
> > > > > > made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
> > > > > > power-domains with the GPC driver. Unfortunately, it does appear to
> > > > > > hang, so it might not be operating correctly.
> > > > > >
> > > > > > Lucas,
> > > > > >
> > > > > > Do you have any idea of stuff I can try to see if the power domain is
> > > > > > coming online correctly?
> > > > > >
> > > > > > [ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
> > > > > > [ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
> > > > > > [ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
> > > > > > [ 10.728927] vpu: set fuse bits to enable
> > > > > > [ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
> > > > > > [ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
> > > > > > [ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
> > > > > > [ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
> > > > > > [ 11.004726] hantro-vpu 38300000.video-codec: registered
> > > > > > nxp,imx8mm-vpu-dec as /dev/video0
> > > > > > [ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
> > > > > > [ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
> > > > > > [ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
> > > > > > [ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
> > > > > > [ 11.139634] hantro-vpu 38310000.video-codec: registered
> > > > > > nxp,imx8mm-vpu-g2-dec as /dev/video1
> > > > > > [ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
> > > > > > [ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
> > > > > > [ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
> > > > > > [ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
> > > > > > [ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > > > >
> > > > > > <hang>
> > > > > >
> > > > > > adam
> > > > > >
> > > > >
> > > > > Nicolas, Tim, and Lucas,
> > > > >
> > > > > I think I have the hanging resolved in the power domains, and I'll be
> > > > > pushing the fix to the GPCv2.
> > > > >
> > > > > For the H1 Encoder, I added some debugging code to read the offset
> > > > > 0xfc and print some data based on the findings of that VPU-h1 offset.
> > > > > I basically check the various bits per the TRM to see if they are set
> > > > > and print some splat to indicate whether or not the function is
> > > > > supported.
> > > > >
> > > > > [ 8.861865] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > > > [ 8.870594] hantro-vpu 38320000.video-codec: Stabilization supported by HW
> > > > > [ 8.889341] hantro-vpu 38320000.video-codec: VP8 encoding supported by HW
> > > > > [ 8.899386] hantro-vpu 38320000.video-codec: H.264 encoding supported by HW
> > > > > [ 8.918171] hantro-vpu 38320000.video-codec: RGB to YUV conversion
> > > > > supported by HW
> > > > > [ 8.934067] hantro-vpu 38320000.video-codec: registered
> > > > > nxp,imx8mm-vpu-h1-enc as /dev/video2
> > > > >
> > > > > Unfortunately, JPEG is not listed as supported. :-(
> > > >
> > > > Adam,
> > > >
> > > > Well not having JPEG encode support is unfortunate, and unexpected. Do
> > > > we not have hantro support yet for VP8/H264 encode?
> > >
> > > There is no mainline support yet. You can derive from RK3288 support using Google ChromeOS method (a v4l2 plugins that simulate in userspace a stateful encoder):
> > >
> > > - libv4l plugins / https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/refs/heads/master
> > > - Kernel Driver / https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu/
> > >
> > > >
> > > > I haven't quite figured out how to build a modern mono-repo gstreamer
> > > > on the ubuntu 20.04 rootfs I'm using so I haven't been able to test
> > > > VPU encode/decode properly. I'll keep working on it when I'm back in
> > > > the office the following week.
> > >
> > > Did a quick test to make sure there isn't any ubuntu specific blockers, here's a
> > > dirty script that produce a minimal GStreamer, there was really nothing special
> > > compare to other meson projects. Note that I use --wrap-mode=nofallback to avoid
> > > letting GStreamer complete it's feature-set by downloading the planet. This
> > > already build quite a lot and could likely be made smaller by avoid plugins-good
> > > build-dep call, but then you need to check for v4l2odecs and video4linux devs
> > > (mostly gudev a glib udev binding).
> > >
> > > # Install ubuntu
> > > podman run -it --rm ubuntu:20.04
> > > sed -i "s/# deb-src/deb-src/" /etc/apt/sources.list
> > > apt update
> > > apt build-dep gstreamer1.0-plugins-good
> > > apt install git python3-pip flex bison
> > >
> > > # Need a newer meson
> > > pip3 install --user meson
> > > export PATH=$PATH:~/.local/bin
> > >
> > > # Build GStreamer
> > > git clone https://gitlab.freedesktop.org/gstreamer/gstreamer.git
> > > cd gstreamer
> > > meson setup build --wrap-mode=nofallback
> > > ninja -C build
> > >
> > > # Run in-place
> > > ./gst-env.py
> > > gst-inspect-1.0 v4l2codecs
> > > gst-inspect 1.0 video4linux2
> > >
> > Thanks for the suggestions.
> >
> > I downloaded what's in the master repo:
> >
> > [gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 v4l2codecs
> >
> > ** (gst-plugin-scanner:7317): CRITICAL **: 10:29:51.847: can't find
> > gi.repository.Gst
> > Plugin Details:
> > Name v4l2codecs
> > Description V4L2 CODEC Accelerators plugin
> > Filename
> > /root/gstreamer/builddir/subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so
> > Version 1.19.3.1
> > License LGPL
> > Source module gst-plugins-bad
> > Binary package GStreamer Bad Plug-ins git
> > Origin URL Unknown package origin
> >
> > v4l2slh264dec: V4L2 Stateless H.264 Video Decoder
> > v4l2slmpeg2dec: V4L2 Stateless Mpeg2 Video Decoder
> > v4l2slvp8alphadecodebin: VP8 Alpha Decoder
> > v4l2slvp8dec: V4L2 Stateless VP8 Video Decoder
> >
> > 4 features:
> > +-- 4 elements
> >
> > [gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 video4linux2
> > Plugin Details:
> > Name video4linux2
> > Description elements for Video 4 Linux
> > Filename
> > /root/gstreamer/builddir/subprojects/gst-plugins-good/sys/v4l2/libgstvideo4linux2.so
> > Version 1.19.3.1
> > License LGPL
> > Source module gst-plugins-good
> > Binary package GStreamer Good Plug-ins git
> > Origin URL Unknown package origin
> >
> > v4l2deviceprovider: Video (video4linux2) Device Provider
> > v4l2jpegenc: V4L2 JPEG Encoder
> > v4l2radio: Radio (video4linux2) Tuner
> > v4l2sink: Video (video4linux2) Sink
> > v4l2src: Video (video4linux2) Source
> >
> > 5 features:
> > +-- 4 elements
> > +-- 1 device providers
> >
> > I still have the H1 encoder enabled, but I know JPEG isn't supported,
> > so I'm going to attempt to do some decoding and pipe to fakesink since
> > I don't have a functional display yet.
> >
> > gst-launch-1.0 -ev filesrc location=trailer_1080p_h264_mp3.avi !
> > decodebin3 ! fakesink
> >
> > Redistribute latency...
> > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:src:
> > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > height=(int)1080, interlace-mode=(string)progressive,
> > multiview-mode=(string)mono,
> > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0:
> > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > height=(int)1080, interlace-mode=(string)progressive,
> > multiview-mode=(string)mono,
> > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0.GstProxyPad:proxypad6:
> > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > height=(int)1080, interlace-mode=(string)progressive,
> > multiview-mode=(string)mono,
> > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/GstMultiQueue:multiqueue0:
> > min-interleave-time = 300000000
> > Redistribute latency...
> > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:sink:
> > caps = video/x-h264, variant=(string)itu, framerate=(fraction)25/1,
> > width=(int)1920, height=(int)1080, chroma-format=(string)4:2:0,
> > bit-depth-luma=(uint)8, bit-depth-chroma=(uint)8,
> > parsed=(boolean)true, stream-format=(string)avc, alignment=(string)au,
> > profile=(string)high, level=(string)4,
> > codec_data=(buffer)01640028ffe1001a67640028acd940780227e584000003000400000300c83c60c65801000668ebe3cb22c0
> > New clock: GstSystemClock
> >
> > And it appears to stream, because the counter increases. I haven't
> > checked the CPU utilization, but the fact that it shows v4l2slh264dec
> > is good.
> >
> > Is there a way to know if/how the decoder is using the proper VPU? I
> > assume if it wasn't using the proper one, it would fail, but was just
> > curious.
> >
>
> A few ways. You can check /proc/interrupts, which should have
> VPU activity.
>
> Or enable debug messages for the module,
> using the debug hantro parameter. V4L2 has debug messages
> that you can enable, see /sys/class/video4linux/video0/dev_debug.
>
> Instead of fakesink you can output to pngenc/jpegenc and check the output
> is visually correct. If at all possible, the proper way is to use Fluster,
> and report the score you get:
>
> https://github.com/fluendo/fluster
>
I ran fluster on the VP8 decoder, but only 55/61 passed.
****************************************************************************************************
Running test suite VP8-TEST-VECTORS with decoder GStreamer-VP8-V4L2SL-Gst1.0
Using 4 parallel job(s)
****************************************************************************************************
[TEST SUITE ] (DECODER ) TEST VECTOR
... RESULT
----------------------------------------------------------------------
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-004 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-001 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-002 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-003 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-005 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-006 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-007 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-008 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-011 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-009 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-012 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-013 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-014 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-010 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-016 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-017 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-018 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1400
... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1416
... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1417
... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1411
... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1402
... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1412
... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1424
... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-01 ... Fail
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-02 ... Fail
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-03 ... Fail
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-04 ... Fail
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1401 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1418
... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1403 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1407 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1408 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1409 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1413 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1415 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1425 ... Fail
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1426 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1427 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1432 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1435 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1436 ... Fail
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-00-comprehensive-015 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1441 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1437 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-04-partitions-1404 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1442 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-04-partitions-1405 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-04-partitions-1406 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1428 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1429 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1431 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1410 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-03-segmentation-1414 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1430 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1433 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1438 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1434 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1439 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1440 ... Success
[VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
vp80-05-sharpness-1443 ... Success
=======================================================================
FAIL: vp80-03-segmentation-01 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
self.assertEqual(
AssertionError: 'db954c077b7a3f34a448ceaacf8f525c' !=
'8bbb396a9bdf8afa250d3b2e45e6b367'
- db954c077b7a3f34a448ceaacf8f525c
+ 8bbb396a9bdf8afa250d3b2e45e6b367
: vp80-03-segmentation-01
=======================================================================
FAIL: vp80-03-segmentation-02 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
self.assertEqual(
AssertionError: '4d2d65efeee1c83772c33a13446bd1a4' !=
'1b2061d4a74549228769f8e292bcb15f'
- 4d2d65efeee1c83772c33a13446bd1a4
+ 1b2061d4a74549228769f8e292bcb15f
: vp80-03-segmentation-02
=======================================================================
FAIL: vp80-03-segmentation-03 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
self.assertEqual(
AssertionError: '73d864433691f8db43257b88495ac8c3' !=
'fd1eb6ebd7100995bad11042a9bea048'
- 73d864433691f8db43257b88495ac8c3
+ fd1eb6ebd7100995bad11042a9bea048
: vp80-03-segmentation-03
=======================================================================
FAIL: vp80-03-segmentation-04 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
self.assertEqual(
AssertionError: '7f846c8bd7cdfe61f8542f382f9d8eeb' !=
'0c27a47c4fd8bbfce173d005bef8be6a'
- 7f846c8bd7cdfe61f8542f382f9d8eeb
+ 0c27a47c4fd8bbfce173d005bef8be6a
: vp80-03-segmentation-04
=======================================================================
FAIL: vp80-03-segmentation-1425 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
self.assertEqual(
AssertionError: '96ffacf0c3eae59b58252be24a60e9b2' !=
'83e8a322e8ab23e60ba16430aacad827'
- 96ffacf0c3eae59b58252be24a60e9b2
+ 83e8a322e8ab23e60ba16430aacad827
: vp80-03-segmentation-1425
=======================================================================
FAIL: vp80-03-segmentation-1436 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
self.assertEqual(
AssertionError: 'bfd17a557ee1ba347c755a18ce5a64a6' !=
'5bca61a733c1936205f82de1492a1b2b'
- bfd17a557ee1ba347c755a18ce5a64a6
+ 5bca61a733c1936205f82de1492a1b2b
: vp80-03-segmentation-1436
Ran 55/61 tests successfully in 12.104 secs
I am not that familiar with this tool, but I assume failures are bad.
However these look like Python errors and not gst errors.
The H264 decoder resulted in:
Ran 85/135 tests successfully in 57.821 secs
I can provide the splat if you want. Those looked like gst errors,
because most of the error messages state the gst-launch-1.0 returned
non-zero exit status 1.
> It should be easy to use.
It was.
>
> > I think I'll redo the patch without the RFC and without the H1 encoder
> > unless anyone has any objections. I know I need to rebase on
> > linux-next anyway because the patches don't apply cleanly. Is there a
> > specific branch I should use? I don't know if this goes through
> > Shawn's IMX tree or the media tree (or a combination)
> >
>
> You should rebase on media's master branch:
>
> https://git.linuxtv.org/media_tree.git/log/
I'll submit the patch with a cover letter with the results of the VP8
and H264 fluster test in the cover letter. Is there a stateless
decoder for the VP9 decoder? gst-inspect only shows the following
v4l2codecs.
v4l2slh264dec: V4L2 Stateless H.264 Video Decoder
v4l2slmpeg2dec: V4L2 Stateless Mpeg2 Video Decoder
v4l2slvp8alphadecodebin: VP8 Alpha Decoder
v4l2slvp8dec: V4L2 Stateless VP8 Video Decoder
thanks for all your help. Hopefully we can get this integrated soon.
adam
>
> Thanks,
> Ezequiel
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-29 18:59 ` Adam Ford
@ 2021-11-29 19:35 ` Tim Harvey
2021-11-29 19:42 ` Adam Ford
2021-11-30 14:00 ` Ezequiel Garcia
0 siblings, 2 replies; 42+ messages in thread
From: Tim Harvey @ 2021-11-29 19:35 UTC (permalink / raw)
To: Adam Ford
Cc: Ezequiel Garcia, Nicolas Dufresne, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
>
> On Mon, Nov 29, 2021 at 10:54 AM Ezequiel Garcia
> <ezequiel@vanguardiasur.com.ar> wrote:
> >
> > On Mon, 29 Nov 2021 at 13:48, Adam Ford <aford173@gmail.com> wrote:
> > >
> > > On Tue, Nov 23, 2021 at 2:07 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > >
> > > > Le lundi 22 novembre 2021 à 09:25 -0800, Tim Harvey a écrit :
> > > > > On Sat, Nov 20, 2021 at 7:36 AM Adam Ford <aford173@gmail.com> wrote:
> > > > > >
> > > > > > On Fri, Nov 19, 2021 at 5:37 PM Adam Ford <aford173@gmail.com> wrote:
> > > > > > >
> > > > > > > On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > > > > >
> > > > > > > > Hi Adam, Tim,
> > > > > > > >
> > > > > > > > [...]
> > > > > > > > > > > > Nicolas and Adam,
> > > > > > > > > > > >
> > > > > > > > > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > > > > > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > > > > > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > > > > > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > > > > > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > > > > > > > > ^ v4l2jpegenc
> > > > > > > > > > >
> > > > > > > > > > > This is just a transcript error ?
> > > > > > > > > >
> > > > > > > > > > Nicolas,
> > > > > > > > > >
> > > > > > > > > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > > > > > > > > >
> > > > > > > > > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > > > > > > > > the board so likely a power-domain issue there?
> > > > > > > > >
> > > > > > > > > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > > > > > > > > think we're writing to registers which are not documented in the Mini
> > > > > > > > > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > > > > > > > > feature, but some of the registers state JPEG, but because some of the
> > > > > > > > > registers written for the H1 are not documented in the TRM. If those
> > > > > > > > > registers are restricted or not in this SoC, I am concerned that it
> > > > > > > > > might be related. I'll try to run some more tests this weekend to
> > > > > > > > > check on the status of the power-domain stuff.
> > > > > > > >
> > > > > > > > To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> > > > > > > > in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> > > > > > > > synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> > > > > > > > PROF/profile is on or off). If your board hang while reading this, you likely
> > > > > > > > didn't get the power bit right.
> > > > > > > >
> > > > > > > > IMX8 has an undocumented control block thing that we have been fighting with in
> > > > > > > > imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> > > > > > > > on NXP solution to be submitted (they asked us to wait, still waiting =)).
> > > > > > >
> > > > > > > Nicolas,
> > > > > > >
> > > > > > > Thanks for the suggestion to read offset FC. There was an attempt
> > > > > > > made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
> > > > > > > power-domains with the GPC driver. Unfortunately, it does appear to
> > > > > > > hang, so it might not be operating correctly.
> > > > > > >
> > > > > > > Lucas,
> > > > > > >
> > > > > > > Do you have any idea of stuff I can try to see if the power domain is
> > > > > > > coming online correctly?
> > > > > > >
> > > > > > > [ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
> > > > > > > [ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
> > > > > > > [ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
> > > > > > > [ 10.728927] vpu: set fuse bits to enable
> > > > > > > [ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
> > > > > > > [ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
> > > > > > > [ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
> > > > > > > [ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
> > > > > > > [ 11.004726] hantro-vpu 38300000.video-codec: registered
> > > > > > > nxp,imx8mm-vpu-dec as /dev/video0
> > > > > > > [ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
> > > > > > > [ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
> > > > > > > [ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
> > > > > > > [ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
> > > > > > > [ 11.139634] hantro-vpu 38310000.video-codec: registered
> > > > > > > nxp,imx8mm-vpu-g2-dec as /dev/video1
> > > > > > > [ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
> > > > > > > [ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
> > > > > > > [ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
> > > > > > > [ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
> > > > > > > [ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > > > > >
> > > > > > > <hang>
> > > > > > >
> > > > > > > adam
> > > > > > >
> > > > > >
> > > > > > Nicolas, Tim, and Lucas,
> > > > > >
> > > > > > I think I have the hanging resolved in the power domains, and I'll be
> > > > > > pushing the fix to the GPCv2.
> > > > > >
> > > > > > For the H1 Encoder, I added some debugging code to read the offset
> > > > > > 0xfc and print some data based on the findings of that VPU-h1 offset.
> > > > > > I basically check the various bits per the TRM to see if they are set
> > > > > > and print some splat to indicate whether or not the function is
> > > > > > supported.
> > > > > >
> > > > > > [ 8.861865] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > > > > [ 8.870594] hantro-vpu 38320000.video-codec: Stabilization supported by HW
> > > > > > [ 8.889341] hantro-vpu 38320000.video-codec: VP8 encoding supported by HW
> > > > > > [ 8.899386] hantro-vpu 38320000.video-codec: H.264 encoding supported by HW
> > > > > > [ 8.918171] hantro-vpu 38320000.video-codec: RGB to YUV conversion
> > > > > > supported by HW
> > > > > > [ 8.934067] hantro-vpu 38320000.video-codec: registered
> > > > > > nxp,imx8mm-vpu-h1-enc as /dev/video2
> > > > > >
> > > > > > Unfortunately, JPEG is not listed as supported. :-(
> > > > >
> > > > > Adam,
> > > > >
> > > > > Well not having JPEG encode support is unfortunate, and unexpected. Do
> > > > > we not have hantro support yet for VP8/H264 encode?
> > > >
> > > > There is no mainline support yet. You can derive from RK3288 support using Google ChromeOS method (a v4l2 plugins that simulate in userspace a stateful encoder):
> > > >
> > > > - libv4l plugins / https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/refs/heads/master
> > > > - Kernel Driver / https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu/
> > > >
> > > > >
> > > > > I haven't quite figured out how to build a modern mono-repo gstreamer
> > > > > on the ubuntu 20.04 rootfs I'm using so I haven't been able to test
> > > > > VPU encode/decode properly. I'll keep working on it when I'm back in
> > > > > the office the following week.
> > > >
> > > > Did a quick test to make sure there isn't any ubuntu specific blockers, here's a
> > > > dirty script that produce a minimal GStreamer, there was really nothing special
> > > > compare to other meson projects. Note that I use --wrap-mode=nofallback to avoid
> > > > letting GStreamer complete it's feature-set by downloading the planet. This
> > > > already build quite a lot and could likely be made smaller by avoid plugins-good
> > > > build-dep call, but then you need to check for v4l2odecs and video4linux devs
> > > > (mostly gudev a glib udev binding).
> > > >
> > > > # Install ubuntu
> > > > podman run -it --rm ubuntu:20.04
> > > > sed -i "s/# deb-src/deb-src/" /etc/apt/sources.list
> > > > apt update
> > > > apt build-dep gstreamer1.0-plugins-good
> > > > apt install git python3-pip flex bison
> > > >
> > > > # Need a newer meson
> > > > pip3 install --user meson
> > > > export PATH=$PATH:~/.local/bin
> > > >
> > > > # Build GStreamer
> > > > git clone https://gitlab.freedesktop.org/gstreamer/gstreamer.git
> > > > cd gstreamer
> > > > meson setup build --wrap-mode=nofallback
> > > > ninja -C build
> > > >
> > > > # Run in-place
> > > > ./gst-env.py
> > > > gst-inspect-1.0 v4l2codecs
> > > > gst-inspect 1.0 video4linux2
> > > >
> > > Thanks for the suggestions.
> > >
> > > I downloaded what's in the master repo:
> > >
> > > [gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 v4l2codecs
> > >
> > > ** (gst-plugin-scanner:7317): CRITICAL **: 10:29:51.847: can't find
> > > gi.repository.Gst
> > > Plugin Details:
> > > Name v4l2codecs
> > > Description V4L2 CODEC Accelerators plugin
> > > Filename
> > > /root/gstreamer/builddir/subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so
> > > Version 1.19.3.1
> > > License LGPL
> > > Source module gst-plugins-bad
> > > Binary package GStreamer Bad Plug-ins git
> > > Origin URL Unknown package origin
> > >
> > > v4l2slh264dec: V4L2 Stateless H.264 Video Decoder
> > > v4l2slmpeg2dec: V4L2 Stateless Mpeg2 Video Decoder
> > > v4l2slvp8alphadecodebin: VP8 Alpha Decoder
> > > v4l2slvp8dec: V4L2 Stateless VP8 Video Decoder
> > >
> > > 4 features:
> > > +-- 4 elements
> > >
> > > [gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 video4linux2
> > > Plugin Details:
> > > Name video4linux2
> > > Description elements for Video 4 Linux
> > > Filename
> > > /root/gstreamer/builddir/subprojects/gst-plugins-good/sys/v4l2/libgstvideo4linux2.so
> > > Version 1.19.3.1
> > > License LGPL
> > > Source module gst-plugins-good
> > > Binary package GStreamer Good Plug-ins git
> > > Origin URL Unknown package origin
> > >
> > > v4l2deviceprovider: Video (video4linux2) Device Provider
> > > v4l2jpegenc: V4L2 JPEG Encoder
> > > v4l2radio: Radio (video4linux2) Tuner
> > > v4l2sink: Video (video4linux2) Sink
> > > v4l2src: Video (video4linux2) Source
> > >
> > > 5 features:
> > > +-- 4 elements
> > > +-- 1 device providers
> > >
> > > I still have the H1 encoder enabled, but I know JPEG isn't supported,
> > > so I'm going to attempt to do some decoding and pipe to fakesink since
> > > I don't have a functional display yet.
> > >
> > > gst-launch-1.0 -ev filesrc location=trailer_1080p_h264_mp3.avi !
> > > decodebin3 ! fakesink
> > >
> > > Redistribute latency...
> > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:src:
> > > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > > height=(int)1080, interlace-mode=(string)progressive,
> > > multiview-mode=(string)mono,
> > > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0:
> > > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > > height=(int)1080, interlace-mode=(string)progressive,
> > > multiview-mode=(string)mono,
> > > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0.GstProxyPad:proxypad6:
> > > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > > height=(int)1080, interlace-mode=(string)progressive,
> > > multiview-mode=(string)mono,
> > > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/GstMultiQueue:multiqueue0:
> > > min-interleave-time = 300000000
> > > Redistribute latency...
> > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:sink:
> > > caps = video/x-h264, variant=(string)itu, framerate=(fraction)25/1,
> > > width=(int)1920, height=(int)1080, chroma-format=(string)4:2:0,
> > > bit-depth-luma=(uint)8, bit-depth-chroma=(uint)8,
> > > parsed=(boolean)true, stream-format=(string)avc, alignment=(string)au,
> > > profile=(string)high, level=(string)4,
> > > codec_data=(buffer)01640028ffe1001a67640028acd940780227e584000003000400000300c83c60c65801000668ebe3cb22c0
> > > New clock: GstSystemClock
> > >
> > > And it appears to stream, because the counter increases. I haven't
> > > checked the CPU utilization, but the fact that it shows v4l2slh264dec
> > > is good.
> > >
> > > Is there a way to know if/how the decoder is using the proper VPU? I
> > > assume if it wasn't using the proper one, it would fail, but was just
> > > curious.
> > >
> >
> > A few ways. You can check /proc/interrupts, which should have
> > VPU activity.
> >
> > Or enable debug messages for the module,
> > using the debug hantro parameter. V4L2 has debug messages
> > that you can enable, see /sys/class/video4linux/video0/dev_debug.
> >
> > Instead of fakesink you can output to pngenc/jpegenc and check the output
> > is visually correct. If at all possible, the proper way is to use Fluster,
> > and report the score you get:
> >
> > https://github.com/fluendo/fluster
> >
>
> I ran fluster on the VP8 decoder, but only 55/61 passed.
>
> ****************************************************************************************************
> Running test suite VP8-TEST-VECTORS with decoder GStreamer-VP8-V4L2SL-Gst1.0
> Using 4 parallel job(s)
> ****************************************************************************************************
>
> [TEST SUITE ] (DECODER ) TEST VECTOR
> ... RESULT
> ----------------------------------------------------------------------
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-004 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-001 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-002 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-003 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-005 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-006 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-007 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-008 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-011 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-009 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-012 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-013 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-014 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-010 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-016 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-017 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-018 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1400
> ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1416
> ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1417
> ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1411
> ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1402
> ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1412
> ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1424
> ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-01 ... Fail
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-02 ... Fail
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-03 ... Fail
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-04 ... Fail
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1401 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1418
> ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1403 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1407 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1408 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1409 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1413 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1415 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1425 ... Fail
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1426 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1427 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1432 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1435 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1436 ... Fail
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-00-comprehensive-015 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1441 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1437 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-04-partitions-1404 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1442 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-04-partitions-1405 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-04-partitions-1406 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1428 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1429 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1431 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1410 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-03-segmentation-1414 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1430 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1433 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1438 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1434 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1439 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1440 ... Success
> [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> vp80-05-sharpness-1443 ... Success
>
>
> =======================================================================
> FAIL: vp80-03-segmentation-01 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> self.assertEqual(
> AssertionError: 'db954c077b7a3f34a448ceaacf8f525c' !=
> '8bbb396a9bdf8afa250d3b2e45e6b367'
> - db954c077b7a3f34a448ceaacf8f525c
> + 8bbb396a9bdf8afa250d3b2e45e6b367
> : vp80-03-segmentation-01
>
> =======================================================================
> FAIL: vp80-03-segmentation-02 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> self.assertEqual(
> AssertionError: '4d2d65efeee1c83772c33a13446bd1a4' !=
> '1b2061d4a74549228769f8e292bcb15f'
> - 4d2d65efeee1c83772c33a13446bd1a4
> + 1b2061d4a74549228769f8e292bcb15f
> : vp80-03-segmentation-02
>
> =======================================================================
> FAIL: vp80-03-segmentation-03 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> self.assertEqual(
> AssertionError: '73d864433691f8db43257b88495ac8c3' !=
> 'fd1eb6ebd7100995bad11042a9bea048'
> - 73d864433691f8db43257b88495ac8c3
> + fd1eb6ebd7100995bad11042a9bea048
> : vp80-03-segmentation-03
>
> =======================================================================
> FAIL: vp80-03-segmentation-04 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> self.assertEqual(
> AssertionError: '7f846c8bd7cdfe61f8542f382f9d8eeb' !=
> '0c27a47c4fd8bbfce173d005bef8be6a'
> - 7f846c8bd7cdfe61f8542f382f9d8eeb
> + 0c27a47c4fd8bbfce173d005bef8be6a
> : vp80-03-segmentation-04
>
> =======================================================================
> FAIL: vp80-03-segmentation-1425 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> self.assertEqual(
> AssertionError: '96ffacf0c3eae59b58252be24a60e9b2' !=
> '83e8a322e8ab23e60ba16430aacad827'
> - 96ffacf0c3eae59b58252be24a60e9b2
> + 83e8a322e8ab23e60ba16430aacad827
> : vp80-03-segmentation-1425
>
> =======================================================================
> FAIL: vp80-03-segmentation-1436 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> self.assertEqual(
> AssertionError: 'bfd17a557ee1ba347c755a18ce5a64a6' !=
> '5bca61a733c1936205f82de1492a1b2b'
> - bfd17a557ee1ba347c755a18ce5a64a6
> + 5bca61a733c1936205f82de1492a1b2b
> : vp80-03-segmentation-1436
>
> Ran 55/61 tests successfully in 12.104 secs
>
> I am not that familiar with this tool, but I assume failures are bad.
> However these look like Python errors and not gst errors.
>
> The H264 decoder resulted in:
>
> Ran 85/135 tests successfully in 57.821 secs
>
> I can provide the splat if you want. Those looked like gst errors,
> because most of the error messages state the gst-launch-1.0 returned
> non-zero exit status 1.
>
>
> > It should be easy to use.
>
> It was.
> >
> > > I think I'll redo the patch without the RFC and without the H1 encoder
> > > unless anyone has any objections. I know I need to rebase on
> > > linux-next anyway because the patches don't apply cleanly. Is there a
> > > specific branch I should use? I don't know if this goes through
> > > Shawn's IMX tree or the media tree (or a combination)
> > >
> >
> > You should rebase on media's master branch:
> >
> > https://git.linuxtv.org/media_tree.git/log/
>
> I'll submit the patch with a cover letter with the results of the VP8
> and H264 fluster test in the cover letter. Is there a stateless
> decoder for the VP9 decoder? gst-inspect only shows the following
> v4l2codecs.
>
> v4l2slh264dec: V4L2 Stateless H.264 Video Decoder
> v4l2slmpeg2dec: V4L2 Stateless Mpeg2 Video Decoder
> v4l2slvp8alphadecodebin: VP8 Alpha Decoder
> v4l2slvp8dec: V4L2 Stateless VP8 Video Decoder
>
> thanks for all your help. Hopefully we can get this integrated soon.
>
Adam,
What deps did you install in order to get v4l2codecs building? I
installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
(not sure if I needed to re-configure somehow) but there is still
nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
how to find out what dependencies are needed or what may be missing.
Best regards,
Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-29 19:35 ` Tim Harvey
@ 2021-11-29 19:42 ` Adam Ford
2021-11-30 14:00 ` Ezequiel Garcia
1 sibling, 0 replies; 42+ messages in thread
From: Adam Ford @ 2021-11-29 19:42 UTC (permalink / raw)
To: Tim Harvey
Cc: Ezequiel Garcia, Nicolas Dufresne, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Mon, Nov 29, 2021 at 1:36 PM Tim Harvey <tharvey@gateworks.com> wrote:
>
> On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
> >
> > On Mon, Nov 29, 2021 at 10:54 AM Ezequiel Garcia
> > <ezequiel@vanguardiasur.com.ar> wrote:
> > >
> > > On Mon, 29 Nov 2021 at 13:48, Adam Ford <aford173@gmail.com> wrote:
> > > >
> > > > On Tue, Nov 23, 2021 at 2:07 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > >
> > > > > Le lundi 22 novembre 2021 à 09:25 -0800, Tim Harvey a écrit :
> > > > > > On Sat, Nov 20, 2021 at 7:36 AM Adam Ford <aford173@gmail.com> wrote:
> > > > > > >
> > > > > > > On Fri, Nov 19, 2021 at 5:37 PM Adam Ford <aford173@gmail.com> wrote:
> > > > > > > >
> > > > > > > > On Fri, Nov 19, 2021 at 10:29 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > > > > > >
> > > > > > > > > Hi Adam, Tim,
> > > > > > > > >
> > > > > > > > > [...]
> > > > > > > > > > > > > Nicolas and Adam,
> > > > > > > > > > > > >
> > > > > > > > > > > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > > > > > > > > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > > > > > > > > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > > > > > > > > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > > > > > > > > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > > > > > > > > > > ^ v4l2jpegenc
> > > > > > > > > > > >
> > > > > > > > > > > > This is just a transcript error ?
> > > > > > > > > > >
> > > > > > > > > > > Nicolas,
> > > > > > > > > > >
> > > > > > > > > > > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> > > > > > > > > > >
> > > > > > > > > > > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > > > > > > > > > > the board so likely a power-domain issue there?
> > > > > > > > > >
> > > > > > > > > > The v4l2-compliance tests fail on the h1 decoder with a hang, but I
> > > > > > > > > > think we're writing to registers which are not documented in the Mini
> > > > > > > > > > TRM. The Mini TRM doesn't explicitly show the JPEG encoding as a
> > > > > > > > > > feature, but some of the registers state JPEG, but because some of the
> > > > > > > > > > registers written for the H1 are not documented in the TRM. If those
> > > > > > > > > > registers are restricted or not in this SoC, I am concerned that it
> > > > > > > > > > might be related. I'll try to run some more tests this weekend to
> > > > > > > > > > check on the status of the power-domain stuff.
> > > > > > > > >
> > > > > > > > > To verify if the HW support JPEG encoding you can read SWREG63 bit 25. This is
> > > > > > > > > in the TRM, just not labelled properly. To mimic the decoding side, would be "HW
> > > > > > > > > synthesis config register X" with the bit labelled SW_ENC_JPEG_PROF (but
> > > > > > > > > PROF/profile is on or off). If your board hang while reading this, you likely
> > > > > > > > > didn't get the power bit right.
> > > > > > > > >
> > > > > > > > > IMX8 has an undocumented control block thing that we have been fighting with in
> > > > > > > > > imx8q, perhaps that's your issue. Few driver was proposed, we are still pending
> > > > > > > > > on NXP solution to be submitted (they asked us to wait, still waiting =)).
> > > > > > > >
> > > > > > > > Nicolas,
> > > > > > > >
> > > > > > > > Thanks for the suggestion to read offset FC. There was an attempt
> > > > > > > > made by Lucas Stach to develop a VPU blk-ctrl driver to coordinate the
> > > > > > > > power-domains with the GPC driver. Unfortunately, it does appear to
> > > > > > > > hang, so it might not be operating correctly.
> > > > > > > >
> > > > > > > > Lucas,
> > > > > > > >
> > > > > > > > Do you have any idea of stuff I can try to see if the power domain is
> > > > > > > > coming online correctly?
> > > > > > > >
> > > > > > > > [ 10.434727] imx-pgc imx-pgc-domain.6: request the vpumix domain to power up
> > > > > > > > [ 10.463647] imx-pgc imx-pgc-domain.6: request the vpumix ADB400 to power up
> > > > > > > > [ 10.517155] imx-pgc imx-pgc-domain.6: genpd vpumix success
> > > > > > > > [ 10.728927] vpu: set fuse bits to enable
> > > > > > > > [ 10.825500] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g1 GPC domain
> > > > > > > > [ 10.878986] imx-pgc imx-pgc-domain.7: request the vpu-g1 domain to power up
> > > > > > > > [ 10.932429] imx-pgc imx-pgc-domain.7: genpd vpu-g1 success
> > > > > > > > [ 10.971988] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g1 success
> > > > > > > > [ 11.004726] hantro-vpu 38300000.video-codec: registered
> > > > > > > > nxp,imx8mm-vpu-dec as /dev/video0
> > > > > > > > [ 11.040760] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-g2 GPC domain
> > > > > > > > [ 11.066181] imx-pgc imx-pgc-domain.8: request the vpu-g2 domain to power up
> > > > > > > > [ 11.087887] imx-pgc imx-pgc-domain.8: genpd vpu-g2 success
> > > > > > > > [ 11.113808] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-g2 success
> > > > > > > > [ 11.139634] hantro-vpu 38310000.video-codec: registered
> > > > > > > > nxp,imx8mm-vpu-g2-dec as /dev/video1
> > > > > > > > [ 11.156463] imx8m-blk-ctrl 38330000.blk-ctrl: power vpublk-h1 GPC domain
> > > > > > > > [ 11.170817] imx-pgc imx-pgc-domain.9: request the vpu-h1 domain to power up
> > > > > > > > [ 11.232990] imx-pgc imx-pgc-domain.9: genpd vpu-h1 success
> > > > > > > > [ 11.252546] imx8m-blk-ctrl 38330000.blk-ctrl: genpd vpublk-h1 success
> > > > > > > > [ 11.266152] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > > > > > >
> > > > > > > > <hang>
> > > > > > > >
> > > > > > > > adam
> > > > > > > >
> > > > > > >
> > > > > > > Nicolas, Tim, and Lucas,
> > > > > > >
> > > > > > > I think I have the hanging resolved in the power domains, and I'll be
> > > > > > > pushing the fix to the GPCv2.
> > > > > > >
> > > > > > > For the H1 Encoder, I added some debugging code to read the offset
> > > > > > > 0xfc and print some data based on the findings of that VPU-h1 offset.
> > > > > > > I basically check the various bits per the TRM to see if they are set
> > > > > > > and print some splat to indicate whether or not the function is
> > > > > > > supported.
> > > > > > >
> > > > > > > [ 8.861865] hantro-vpu 38320000.video-codec: Checking vpu->enc_base + 0xfc
> > > > > > > [ 8.870594] hantro-vpu 38320000.video-codec: Stabilization supported by HW
> > > > > > > [ 8.889341] hantro-vpu 38320000.video-codec: VP8 encoding supported by HW
> > > > > > > [ 8.899386] hantro-vpu 38320000.video-codec: H.264 encoding supported by HW
> > > > > > > [ 8.918171] hantro-vpu 38320000.video-codec: RGB to YUV conversion
> > > > > > > supported by HW
> > > > > > > [ 8.934067] hantro-vpu 38320000.video-codec: registered
> > > > > > > nxp,imx8mm-vpu-h1-enc as /dev/video2
> > > > > > >
> > > > > > > Unfortunately, JPEG is not listed as supported. :-(
> > > > > >
> > > > > > Adam,
> > > > > >
> > > > > > Well not having JPEG encode support is unfortunate, and unexpected. Do
> > > > > > we not have hantro support yet for VP8/H264 encode?
> > > > >
> > > > > There is no mainline support yet. You can derive from RK3288 support using Google ChromeOS method (a v4l2 plugins that simulate in userspace a stateful encoder):
> > > > >
> > > > > - libv4l plugins / https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/refs/heads/master
> > > > > - Kernel Driver / https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu/
> > > > >
> > > > > >
> > > > > > I haven't quite figured out how to build a modern mono-repo gstreamer
> > > > > > on the ubuntu 20.04 rootfs I'm using so I haven't been able to test
> > > > > > VPU encode/decode properly. I'll keep working on it when I'm back in
> > > > > > the office the following week.
> > > > >
> > > > > Did a quick test to make sure there isn't any ubuntu specific blockers, here's a
> > > > > dirty script that produce a minimal GStreamer, there was really nothing special
> > > > > compare to other meson projects. Note that I use --wrap-mode=nofallback to avoid
> > > > > letting GStreamer complete it's feature-set by downloading the planet. This
> > > > > already build quite a lot and could likely be made smaller by avoid plugins-good
> > > > > build-dep call, but then you need to check for v4l2odecs and video4linux devs
> > > > > (mostly gudev a glib udev binding).
> > > > >
> > > > > # Install ubuntu
> > > > > podman run -it --rm ubuntu:20.04
> > > > > sed -i "s/# deb-src/deb-src/" /etc/apt/sources.list
> > > > > apt update
> > > > > apt build-dep gstreamer1.0-plugins-good
> > > > > apt install git python3-pip flex bison
> > > > >
> > > > > # Need a newer meson
> > > > > pip3 install --user meson
> > > > > export PATH=$PATH:~/.local/bin
> > > > >
> > > > > # Build GStreamer
> > > > > git clone https://gitlab.freedesktop.org/gstreamer/gstreamer.git
> > > > > cd gstreamer
> > > > > meson setup build --wrap-mode=nofallback
> > > > > ninja -C build
> > > > >
> > > > > # Run in-place
> > > > > ./gst-env.py
> > > > > gst-inspect-1.0 v4l2codecs
> > > > > gst-inspect 1.0 video4linux2
> > > > >
> > > > Thanks for the suggestions.
> > > >
> > > > I downloaded what's in the master repo:
> > > >
> > > > [gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 v4l2codecs
> > > >
> > > > ** (gst-plugin-scanner:7317): CRITICAL **: 10:29:51.847: can't find
> > > > gi.repository.Gst
> > > > Plugin Details:
> > > > Name v4l2codecs
> > > > Description V4L2 CODEC Accelerators plugin
> > > > Filename
> > > > /root/gstreamer/builddir/subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so
> > > > Version 1.19.3.1
> > > > License LGPL
> > > > Source module gst-plugins-bad
> > > > Binary package GStreamer Bad Plug-ins git
> > > > Origin URL Unknown package origin
> > > >
> > > > v4l2slh264dec: V4L2 Stateless H.264 Video Decoder
> > > > v4l2slmpeg2dec: V4L2 Stateless Mpeg2 Video Decoder
> > > > v4l2slvp8alphadecodebin: VP8 Alpha Decoder
> > > > v4l2slvp8dec: V4L2 Stateless VP8 Video Decoder
> > > >
> > > > 4 features:
> > > > +-- 4 elements
> > > >
> > > > [gst-main] root@localhost:~/gstreamer# gst-inspect-1.0 video4linux2
> > > > Plugin Details:
> > > > Name video4linux2
> > > > Description elements for Video 4 Linux
> > > > Filename
> > > > /root/gstreamer/builddir/subprojects/gst-plugins-good/sys/v4l2/libgstvideo4linux2.so
> > > > Version 1.19.3.1
> > > > License LGPL
> > > > Source module gst-plugins-good
> > > > Binary package GStreamer Good Plug-ins git
> > > > Origin URL Unknown package origin
> > > >
> > > > v4l2deviceprovider: Video (video4linux2) Device Provider
> > > > v4l2jpegenc: V4L2 JPEG Encoder
> > > > v4l2radio: Radio (video4linux2) Tuner
> > > > v4l2sink: Video (video4linux2) Sink
> > > > v4l2src: Video (video4linux2) Source
> > > >
> > > > 5 features:
> > > > +-- 4 elements
> > > > +-- 1 device providers
> > > >
> > > > I still have the H1 encoder enabled, but I know JPEG isn't supported,
> > > > so I'm going to attempt to do some decoding and pipe to fakesink since
> > > > I don't have a functional display yet.
> > > >
> > > > gst-launch-1.0 -ev filesrc location=trailer_1080p_h264_mp3.avi !
> > > > decodebin3 ! fakesink
> > > >
> > > > Redistribute latency...
> > > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:src:
> > > > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > > > height=(int)1080, interlace-mode=(string)progressive,
> > > > multiview-mode=(string)mono,
> > > > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > > > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0:
> > > > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > > > height=(int)1080, interlace-mode=(string)progressive,
> > > > multiview-mode=(string)mono,
> > > > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > > > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0.GstGhostPad:video_0.GstProxyPad:proxypad6:
> > > > caps = video/x-raw, format=(string)NV12, width=(int)1920,
> > > > height=(int)1080, interlace-mode=(string)progressive,
> > > > multiview-mode=(string)mono,
> > > > multiview-flags=(GstVideoMultiviewFlagsSet)0:ffffffff:/right-view-first/left-flipped/left-flopped/right-flipped/right-flopped/half-aspect/mixed-mono,
> > > > pixel-aspect-ratio=(fraction)1/1, framerate=(fraction)25/1
> > > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/GstMultiQueue:multiqueue0:
> > > > min-interleave-time = 300000000
> > > > Redistribute latency...
> > > > /GstPipeline:pipeline0/GstDecodebin3:decodebin3-0/v4l2slh264dec:v4l2slh264dec0.GstPad:sink:
> > > > caps = video/x-h264, variant=(string)itu, framerate=(fraction)25/1,
> > > > width=(int)1920, height=(int)1080, chroma-format=(string)4:2:0,
> > > > bit-depth-luma=(uint)8, bit-depth-chroma=(uint)8,
> > > > parsed=(boolean)true, stream-format=(string)avc, alignment=(string)au,
> > > > profile=(string)high, level=(string)4,
> > > > codec_data=(buffer)01640028ffe1001a67640028acd940780227e584000003000400000300c83c60c65801000668ebe3cb22c0
> > > > New clock: GstSystemClock
> > > >
> > > > And it appears to stream, because the counter increases. I haven't
> > > > checked the CPU utilization, but the fact that it shows v4l2slh264dec
> > > > is good.
> > > >
> > > > Is there a way to know if/how the decoder is using the proper VPU? I
> > > > assume if it wasn't using the proper one, it would fail, but was just
> > > > curious.
> > > >
> > >
> > > A few ways. You can check /proc/interrupts, which should have
> > > VPU activity.
> > >
> > > Or enable debug messages for the module,
> > > using the debug hantro parameter. V4L2 has debug messages
> > > that you can enable, see /sys/class/video4linux/video0/dev_debug.
> > >
> > > Instead of fakesink you can output to pngenc/jpegenc and check the output
> > > is visually correct. If at all possible, the proper way is to use Fluster,
> > > and report the score you get:
> > >
> > > https://github.com/fluendo/fluster
> > >
> >
> > I ran fluster on the VP8 decoder, but only 55/61 passed.
> >
> > ****************************************************************************************************
> > Running test suite VP8-TEST-VECTORS with decoder GStreamer-VP8-V4L2SL-Gst1.0
> > Using 4 parallel job(s)
> > ****************************************************************************************************
> >
> > [TEST SUITE ] (DECODER ) TEST VECTOR
> > ... RESULT
> > ----------------------------------------------------------------------
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-004 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-001 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-002 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-003 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-005 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-006 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-007 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-008 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-011 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-009 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-012 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-013 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-014 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-010 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-016 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-017 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-018 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1400
> > ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1416
> > ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1417
> > ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-01-intra-1411
> > ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1402
> > ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1412
> > ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1424
> > ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-01 ... Fail
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-02 ... Fail
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-03 ... Fail
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-04 ... Fail
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1401 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0) vp80-02-inter-1418
> > ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1403 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1407 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1408 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1409 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1413 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1415 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1425 ... Fail
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1426 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1427 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1432 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1435 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1436 ... Fail
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-00-comprehensive-015 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1441 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1437 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-04-partitions-1404 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1442 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-04-partitions-1405 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-04-partitions-1406 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1428 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1429 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1431 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1410 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-03-segmentation-1414 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1430 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1433 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1438 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1434 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1439 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1440 ... Success
> > [VP8-TEST-VECTORS] (GStreamer-VP8-V4L2SL-Gst1.0)
> > vp80-05-sharpness-1443 ... Success
> >
> >
> > =======================================================================
> > FAIL: vp80-03-segmentation-01 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> > ----------------------------------------------------------------------
> > Traceback (most recent call last):
> > File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> > self.assertEqual(
> > AssertionError: 'db954c077b7a3f34a448ceaacf8f525c' !=
> > '8bbb396a9bdf8afa250d3b2e45e6b367'
> > - db954c077b7a3f34a448ceaacf8f525c
> > + 8bbb396a9bdf8afa250d3b2e45e6b367
> > : vp80-03-segmentation-01
> >
> > =======================================================================
> > FAIL: vp80-03-segmentation-02 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> > ----------------------------------------------------------------------
> > Traceback (most recent call last):
> > File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> > self.assertEqual(
> > AssertionError: '4d2d65efeee1c83772c33a13446bd1a4' !=
> > '1b2061d4a74549228769f8e292bcb15f'
> > - 4d2d65efeee1c83772c33a13446bd1a4
> > + 1b2061d4a74549228769f8e292bcb15f
> > : vp80-03-segmentation-02
> >
> > =======================================================================
> > FAIL: vp80-03-segmentation-03 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> > ----------------------------------------------------------------------
> > Traceback (most recent call last):
> > File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> > self.assertEqual(
> > AssertionError: '73d864433691f8db43257b88495ac8c3' !=
> > 'fd1eb6ebd7100995bad11042a9bea048'
> > - 73d864433691f8db43257b88495ac8c3
> > + fd1eb6ebd7100995bad11042a9bea048
> > : vp80-03-segmentation-03
> >
> > =======================================================================
> > FAIL: vp80-03-segmentation-04 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> > ----------------------------------------------------------------------
> > Traceback (most recent call last):
> > File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> > self.assertEqual(
> > AssertionError: '7f846c8bd7cdfe61f8542f382f9d8eeb' !=
> > '0c27a47c4fd8bbfce173d005bef8be6a'
> > - 7f846c8bd7cdfe61f8542f382f9d8eeb
> > + 0c27a47c4fd8bbfce173d005bef8be6a
> > : vp80-03-segmentation-04
> >
> > =======================================================================
> > FAIL: vp80-03-segmentation-1425 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> > ----------------------------------------------------------------------
> > Traceback (most recent call last):
> > File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> > self.assertEqual(
> > AssertionError: '96ffacf0c3eae59b58252be24a60e9b2' !=
> > '83e8a322e8ab23e60ba16430aacad827'
> > - 96ffacf0c3eae59b58252be24a60e9b2
> > + 83e8a322e8ab23e60ba16430aacad827
> > : vp80-03-segmentation-1425
> >
> > =======================================================================
> > FAIL: vp80-03-segmentation-1436 (GStreamer-VP8-V4L2SL-Gst1.0.VP8-TEST-VECTORS)
> > ----------------------------------------------------------------------
> > Traceback (most recent call last):
> > File "/root/gstreamer/fluster/fluster/test.py", line 104, in _test
> > self.assertEqual(
> > AssertionError: 'bfd17a557ee1ba347c755a18ce5a64a6' !=
> > '5bca61a733c1936205f82de1492a1b2b'
> > - bfd17a557ee1ba347c755a18ce5a64a6
> > + 5bca61a733c1936205f82de1492a1b2b
> > : vp80-03-segmentation-1436
> >
> > Ran 55/61 tests successfully in 12.104 secs
> >
> > I am not that familiar with this tool, but I assume failures are bad.
> > However these look like Python errors and not gst errors.
> >
> > The H264 decoder resulted in:
> >
> > Ran 85/135 tests successfully in 57.821 secs
> >
> > I can provide the splat if you want. Those looked like gst errors,
> > because most of the error messages state the gst-launch-1.0 returned
> > non-zero exit status 1.
> >
> >
> > > It should be easy to use.
> >
> > It was.
> > >
> > > > I think I'll redo the patch without the RFC and without the H1 encoder
> > > > unless anyone has any objections. I know I need to rebase on
> > > > linux-next anyway because the patches don't apply cleanly. Is there a
> > > > specific branch I should use? I don't know if this goes through
> > > > Shawn's IMX tree or the media tree (or a combination)
> > > >
> > >
> > > You should rebase on media's master branch:
> > >
> > > https://git.linuxtv.org/media_tree.git/log/
> >
> > I'll submit the patch with a cover letter with the results of the VP8
> > and H264 fluster test in the cover letter. Is there a stateless
> > decoder for the VP9 decoder? gst-inspect only shows the following
> > v4l2codecs.
> >
> > v4l2slh264dec: V4L2 Stateless H.264 Video Decoder
> > v4l2slmpeg2dec: V4L2 Stateless Mpeg2 Video Decoder
> > v4l2slvp8alphadecodebin: VP8 Alpha Decoder
> > v4l2slvp8dec: V4L2 Stateless VP8 Video Decoder
> >
> > thanks for all your help. Hopefully we can get this integrated soon.
> >
>
> Adam,
>
> What deps did you install in order to get v4l2codecs building? I
> installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> (not sure if I needed to re-configure somehow) but there is still
> nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> how to find out what dependencies are needed or what may be missing.
Instead of building just a limited release of gstreamer, I had to
build the whole thing.
meson builddir
ninja -C builddir -j2
running with -j4 ran the imx8mm out of memory and slowed to a crawl to
the point where it didn't finish even after building overnight.
With that, I was able to execute gst-env.py to run streamer without
installing it. In that environment, I was able to run fluster to get
the results I published.
adam
>
> Best regards,
>
> Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-29 19:35 ` Tim Harvey
2021-11-29 19:42 ` Adam Ford
@ 2021-11-30 14:00 ` Ezequiel Garcia
2021-11-30 19:28 ` Tim Harvey
1 sibling, 1 reply; 42+ messages in thread
From: Ezequiel Garcia @ 2021-11-30 14:00 UTC (permalink / raw)
To: Tim Harvey
Cc: Adam Ford, Nicolas Dufresne, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Hi Tim,
On Mon, 29 Nov 2021 at 16:36, Tim Harvey <tharvey@gateworks.com> wrote:
>
> On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
..
> >
>
> Adam,
>
> What deps did you install in order to get v4l2codecs building? I
> installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> (not sure if I needed to re-configure somehow) but there is still
> nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> how to find out what dependencies are needed or what may be missing.
>
At least in my case (Centps-derivative), this is what I've done:
...
gst-plugins-bad| Run-time dependency gudev-1.0 found: NO (tried
pkgconfig and cmake)
Installed gudev ... and then:
...
gst-plugins-bad| Dependency gudev-1.0 found: YES 232 (cached)
...
gst-plugins-bad 1.19.3.1
Plugins : accurip, adpcmdec, adpcmenc, aiff, asfmux,
audiobuffersplit, audiofxbad, audiomixmatrix, audiolatency,
audiovisualizers, autoconvert, bayer,
camerabin, codecalpha, coloreffects,
debugutilsbad, dvbsubenc, dvbsuboverlay, dvdspu, faceoverlay,
festival, fieldanalysis, freeverb, frei0r,
gaudieffects, gdp, geometrictransform,
id3tag, inter, interlace, ivfparse, ivtc, jp2kdecimator, jpegformat,
rfbsrc, midi, mpegpsdemux,
mpegpsmux, mpegtsdemux, mpegtsmux, mxf,
netsim, rtponvif, pcapparse, pnm, proxy, legacyrawparse,
removesilence, rist, rtmp2, rtpmanagerbad,
sdpelem, segmentclip, siren, smooth,
speed, subenc, switchbin, timecode, transcode, videofiltersbad,
videoframe_audiolevel, videoparsersbad,
videosignal, vmnc, y4mdec, decklink, dvb,
fbdevsink, ipcpipeline, nvcodec, shm, v4l2codecs, hls, sctp
GStreamer current master build fails. It's a known issue which will be
fixed today:
[...]
[8/9] Compiling C object
subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
FAILED: subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
cc -Isubprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p
-Isubprojects/gst-plugins-bad/sys/v4l2codecs
-I../subprojects/gst-plugins-bad/sys/v4l2codecs
-Isubprojects/gst-plugins-bad -I../subprojects/gst-plugins-bad
-Isubprojects/gstreamer/libs -I../subprojects/gstreamer/libs
-Isubprojects/gstreamer -I../subprojects/gstreamer
-Isubprojects/gst-plugins-bad/gst-libs
-I../subprojects/gst-plugins-bad/gst-libs
-Isubprojects/gst-plugins-base/gst-libs
-I../subprojects/gst-plugins-base/gst-libs -Isubprojects/orc
-I../subprojects/orc -Isubprojects/gstreamer/gst
-Isubprojects/gst-plugins-base/gst-libs/gst/video
-Isubprojects/gst-plugins-base/gst-libs/gst/pbutils
-Isubprojects/gst-plugins-base/gst-libs/gst/audio
-Isubprojects/gst-plugins-base/gst-libs/gst/tag
-I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
-I/usr/include/gudev-1.0 -fdiagnostics-color=always
-D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -O2 -g -fvisibility=hidden
-fno-strict-aliasing -DG_DISABLE_DEPRECATED -Wmissing-prototypes
-Wdeclaration-after-statement -Wold-style-definition
-Wmissing-declarations -Wredundant-decls -Wwrite-strings -Wformat
-Wformat-security -Winit-self -Wmissing-include-dirs -Waddress
-Wno-multichar -Wvla -Wpointer-arith -fPIC -pthread -DHAVE_CONFIG_H
-MD -MQ subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
-MF subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o.d
-o subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
-c ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c
../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:92:3:
error: unknown type name ‘grefcount’
grefcount ref_count;
^~~~~~~~~
../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
function ‘gst_v4l2_codec_vp9_dec_picture_data_new’:
../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:106:3:
warning: implicit declaration of function ‘g_ref_count_init’; did you
mean ‘g_cond_init’? [-Wimplicit-function-declaration]
g_ref_count_init (&pic_data->ref_count);
^~~~~~~~~~~~~~~~
g_cond_init
../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
function ‘gst_v4l2_codec_vp9_dec_picture_data_ref’:
../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:118:3:
warning: implicit declaration of function ‘g_ref_count_inc’; did you
mean ‘g_strv_contains’? [-Wimplicit-function-declaration]
g_ref_count_inc (&data->ref_count);
^~~~~~~~~~~~~~~
g_strv_contains
../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
function ‘gst_v4l2_codec_vp9_dec_picture_data_unref’:
../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:125:7:
warning: implicit declaration of function ‘g_ref_count_dec’
[-Wimplicit-function-declaration]
if (g_ref_count_dec (&data->ref_count)) {
^~~~~~~~~~~~~~~
ninja: build stopped: subcommand failed.
Hope this helps get you started!
Ezequiel
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-30 14:00 ` Ezequiel Garcia
@ 2021-11-30 19:28 ` Tim Harvey
2021-11-30 20:33 ` Adam Ford
2021-12-03 4:34 ` Nicolas Dufresne
0 siblings, 2 replies; 42+ messages in thread
From: Tim Harvey @ 2021-11-30 19:28 UTC (permalink / raw)
To: Ezequiel Garcia, Nicolas Dufresne
Cc: Adam Ford, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Tue, Nov 30, 2021 at 6:00 AM Ezequiel Garcia
<ezequiel@vanguardiasur.com.ar> wrote:
>
> Hi Tim,
>
> On Mon, 29 Nov 2021 at 16:36, Tim Harvey <tharvey@gateworks.com> wrote:
> >
> > On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
> ..
> > >
> >
> > Adam,
> >
> > What deps did you install in order to get v4l2codecs building? I
> > installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> > (not sure if I needed to re-configure somehow) but there is still
> > nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> > configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> > how to find out what dependencies are needed or what may be missing.
> >
>
> At least in my case (Centps-derivative), this is what I've done:
>
> ...
> gst-plugins-bad| Run-time dependency gudev-1.0 found: NO (tried
> pkgconfig and cmake)
>
> Installed gudev ... and then:
>
> ...
> gst-plugins-bad| Dependency gudev-1.0 found: YES 232 (cached)
> ...
> gst-plugins-bad 1.19.3.1
>
> Plugins : accurip, adpcmdec, adpcmenc, aiff, asfmux,
> audiobuffersplit, audiofxbad, audiomixmatrix, audiolatency,
> audiovisualizers, autoconvert, bayer,
> camerabin, codecalpha, coloreffects,
> debugutilsbad, dvbsubenc, dvbsuboverlay, dvdspu, faceoverlay,
> festival, fieldanalysis, freeverb, frei0r,
> gaudieffects, gdp, geometrictransform,
> id3tag, inter, interlace, ivfparse, ivtc, jp2kdecimator, jpegformat,
> rfbsrc, midi, mpegpsdemux,
> mpegpsmux, mpegtsdemux, mpegtsmux, mxf,
> netsim, rtponvif, pcapparse, pnm, proxy, legacyrawparse,
> removesilence, rist, rtmp2, rtpmanagerbad,
> sdpelem, segmentclip, siren, smooth,
> speed, subenc, switchbin, timecode, transcode, videofiltersbad,
> videoframe_audiolevel, videoparsersbad,
> videosignal, vmnc, y4mdec, decklink, dvb,
> fbdevsink, ipcpipeline, nvcodec, shm, v4l2codecs, hls, sctp
>
> GStreamer current master build fails. It's a known issue which will be
> fixed today:
>
> [...]
> [8/9] Compiling C object
> subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> FAILED: subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> cc -Isubprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p
> -Isubprojects/gst-plugins-bad/sys/v4l2codecs
> -I../subprojects/gst-plugins-bad/sys/v4l2codecs
> -Isubprojects/gst-plugins-bad -I../subprojects/gst-plugins-bad
> -Isubprojects/gstreamer/libs -I../subprojects/gstreamer/libs
> -Isubprojects/gstreamer -I../subprojects/gstreamer
> -Isubprojects/gst-plugins-bad/gst-libs
> -I../subprojects/gst-plugins-bad/gst-libs
> -Isubprojects/gst-plugins-base/gst-libs
> -I../subprojects/gst-plugins-base/gst-libs -Isubprojects/orc
> -I../subprojects/orc -Isubprojects/gstreamer/gst
> -Isubprojects/gst-plugins-base/gst-libs/gst/video
> -Isubprojects/gst-plugins-base/gst-libs/gst/pbutils
> -Isubprojects/gst-plugins-base/gst-libs/gst/audio
> -Isubprojects/gst-plugins-base/gst-libs/gst/tag
> -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
> -I/usr/include/gudev-1.0 -fdiagnostics-color=always
> -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -O2 -g -fvisibility=hidden
> -fno-strict-aliasing -DG_DISABLE_DEPRECATED -Wmissing-prototypes
> -Wdeclaration-after-statement -Wold-style-definition
> -Wmissing-declarations -Wredundant-decls -Wwrite-strings -Wformat
> -Wformat-security -Winit-self -Wmissing-include-dirs -Waddress
> -Wno-multichar -Wvla -Wpointer-arith -fPIC -pthread -DHAVE_CONFIG_H
> -MD -MQ subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> -MF subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o.d
> -o subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> -c ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c
> ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:92:3:
> error: unknown type name ‘grefcount’
> grefcount ref_count;
> ^~~~~~~~~
> ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> function ‘gst_v4l2_codec_vp9_dec_picture_data_new’:
> ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:106:3:
> warning: implicit declaration of function ‘g_ref_count_init’; did you
> mean ‘g_cond_init’? [-Wimplicit-function-declaration]
> g_ref_count_init (&pic_data->ref_count);
> ^~~~~~~~~~~~~~~~
> g_cond_init
> ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> function ‘gst_v4l2_codec_vp9_dec_picture_data_ref’:
> ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:118:3:
> warning: implicit declaration of function ‘g_ref_count_inc’; did you
> mean ‘g_strv_contains’? [-Wimplicit-function-declaration]
> g_ref_count_inc (&data->ref_count);
> ^~~~~~~~~~~~~~~
> g_strv_contains
> ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> function ‘gst_v4l2_codec_vp9_dec_picture_data_unref’:
> ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:125:7:
> warning: implicit declaration of function ‘g_ref_count_dec’
> [-Wimplicit-function-declaration]
> if (g_ref_count_dec (&data->ref_count)) {
> ^~~~~~~~~~~~~~~
> ninja: build stopped: subcommand failed.
>
> Hope this helps get you started!
> Ezequiel
Ezequiel and Nicolas,
Thanks - I did manage to get gstreamer 1.19.3 built successfully with
v4l2codecs finally by getting the correct dependencies. I've attempted
to software encode from another system and decode/display on the IMX8M
Mini but thus far have not been successful.
I see that v4l2codecs plugin v4l2slh264dec/v4l2slmpeg2dec/v4l2slvp8dec
and these all can output video/x-raw NV12/YUY2 which kmssink should
accept so I'm attempting the following :
# vp8 encode from x86
gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
vp8enc ! rtpvp8pay ! udpsink host=172.24.33.15 port=9001
# vp8 decode on imx8mm@172.24.33.15 which has a 800x480 display
[gst-main] root@focal-venice:~/gstreamer/build# gst-launch-1.0 -v
udpsrc port=9001 caps = "application/x-rtp, media=(string)video,
clock-rate=(int)90000, encoding-name=(string)VP8, payload=(int)96,
ssrc=(uint)2745262155, timestamp-offset=(uint)2515032683,
seqnum-offset=(uint)19579, a-framerate=(string)30" ! rtpvp8depay !
v4l2slvp8dec ! kmssink
Setting pipeline to PAUSED ...
Pipeline is live and does not need PREROLL ...
/GstPipeline:pipeline0/GstKMSSink:kmssink0: display-width = 800
/GstPipeline:pipeline0/GstKMSSink:kmssink0: display-height = 480
Pipeline is PREROLLED ...
Setting pipeline to PLAYING ...
/GstPipeline:pipeline0/GstUDPSrc:udpsrc0.GstPad:src: caps =
application/x-rtp, media=(string)video, clock-rate=(int)90000,
encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
a-framerate=(string)30
New clock: GstSystemClock
/GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:sink: caps =
application/x-rtp, media=(string)video, clock-rate=(int)90000,
encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
a-framerate=(string)30
/GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:src: caps =
video/x-vp8, framerate=(fraction)0/1, height=(int)480, width=(int)800,
profile=(string)0
ERROR: from element /GstPipeline:pipeline0/GstUDPSrc:udpsrc0: Internal
data stream error.
Additional debug info:
../subprojects/gstreamer/libs/gst/base/gstbasesrc.c(3127):
gst_base_src_loop (): /GstPipeline:pipeline0/GstUDPSrc:udpsrc0:
streaming stopped, reason not-negotiated (-4)
Execution ended after 0:00:02.076839644
Setting pipeline to NULL ...
Freeing pipeline ...
I'm getting the same thing when trying to use h264.
I've never quite been able to grasp how to debug GStreamer's
negotiation issues. If I end with fakesink it appears to decode so it
must be the v4l2slvp8dec to kmssink. I tried forcing the pixel format
using 'v4l2slvp8dec ! "video/x-raw,format=(string)NV12" ! kmssink' but
I still get the negotiation error.
What interrupts should I be seeing in /proc/interrupts? I don't see
anything vpu/hantro related there.
I also want to make sure I have a basic understanding of the vpu
drivers and usersapce on the IMX8M Mini. The IMX6Q/DL that I'm more
familiar with has a vpu that is supported by the GStreamer video4linux
plugin which shows the following (on GStreamer 1.16.2):
v4l2jpegenc: V4L2 JPEG Encoder
v4l2jpegdec: V4L2 JPEG Decoder
v4l2h264enc: V4L2 H.264 Encoder
v4l2mpeg4enc: V4L2 MPEG4 Encoder
v4l2mpeg4dec: V4L2 MPEG4 Decoder
v4l2mpeg2dec: V4L2 MPEG2 Decoder
v4l2h264dec: V4L2 H264 Decoder
The IMX6Q/DL also has an IPU that has an M2M driver that provides the
following for scaling/colorspace conversion:
v4l2convert: V4L2 Video Converter
I believe what I'm reading is that the IMX8M Mini Hantro codecs are
'stateful' where more software is required to drive them and is
supported by the newer v4l2codecs plugin. I haven't been able to
understand what kernel version/requirements the v4l2codecs plugin
users/requires.
I'm also trying to understand how we can get scaling/colorspace
conversion on the IMX8M Mini. The IMX8M lacks an IPU... is there some
way to utilize scaling/colorspace conversion from the 2D GPU bound to
the etnaviv driver?
Best regards,
Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-30 19:28 ` Tim Harvey
@ 2021-11-30 20:33 ` Adam Ford
2021-12-17 4:48 ` Ezequiel Garcia
2021-12-03 4:34 ` Nicolas Dufresne
1 sibling, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-11-30 20:33 UTC (permalink / raw)
To: Tim Harvey
Cc: Ezequiel Garcia, Nicolas Dufresne, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Tue, Nov 30, 2021 at 1:28 PM Tim Harvey <tharvey@gateworks.com> wrote:
>
> On Tue, Nov 30, 2021 at 6:00 AM Ezequiel Garcia
> <ezequiel@vanguardiasur.com.ar> wrote:
> >
> > Hi Tim,
> >
> > On Mon, 29 Nov 2021 at 16:36, Tim Harvey <tharvey@gateworks.com> wrote:
> > >
> > > On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
> > ..
> > > >
> > >
> > > Adam,
> > >
> > > What deps did you install in order to get v4l2codecs building? I
> > > installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> > > (not sure if I needed to re-configure somehow) but there is still
> > > nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> > > configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> > > how to find out what dependencies are needed or what may be missing.
> > >
> >
> > At least in my case (Centps-derivative), this is what I've done:
> >
> > ...
> > gst-plugins-bad| Run-time dependency gudev-1.0 found: NO (tried
> > pkgconfig and cmake)
> >
> > Installed gudev ... and then:
> >
> > ...
> > gst-plugins-bad| Dependency gudev-1.0 found: YES 232 (cached)
> > ...
> > gst-plugins-bad 1.19.3.1
> >
> > Plugins : accurip, adpcmdec, adpcmenc, aiff, asfmux,
> > audiobuffersplit, audiofxbad, audiomixmatrix, audiolatency,
> > audiovisualizers, autoconvert, bayer,
> > camerabin, codecalpha, coloreffects,
> > debugutilsbad, dvbsubenc, dvbsuboverlay, dvdspu, faceoverlay,
> > festival, fieldanalysis, freeverb, frei0r,
> > gaudieffects, gdp, geometrictransform,
> > id3tag, inter, interlace, ivfparse, ivtc, jp2kdecimator, jpegformat,
> > rfbsrc, midi, mpegpsdemux,
> > mpegpsmux, mpegtsdemux, mpegtsmux, mxf,
> > netsim, rtponvif, pcapparse, pnm, proxy, legacyrawparse,
> > removesilence, rist, rtmp2, rtpmanagerbad,
> > sdpelem, segmentclip, siren, smooth,
> > speed, subenc, switchbin, timecode, transcode, videofiltersbad,
> > videoframe_audiolevel, videoparsersbad,
> > videosignal, vmnc, y4mdec, decklink, dvb,
> > fbdevsink, ipcpipeline, nvcodec, shm, v4l2codecs, hls, sctp
> >
> > GStreamer current master build fails. It's a known issue which will be
> > fixed today:
> >
> > [...]
> > [8/9] Compiling C object
> > subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > FAILED: subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > cc -Isubprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p
> > -Isubprojects/gst-plugins-bad/sys/v4l2codecs
> > -I../subprojects/gst-plugins-bad/sys/v4l2codecs
> > -Isubprojects/gst-plugins-bad -I../subprojects/gst-plugins-bad
> > -Isubprojects/gstreamer/libs -I../subprojects/gstreamer/libs
> > -Isubprojects/gstreamer -I../subprojects/gstreamer
> > -Isubprojects/gst-plugins-bad/gst-libs
> > -I../subprojects/gst-plugins-bad/gst-libs
> > -Isubprojects/gst-plugins-base/gst-libs
> > -I../subprojects/gst-plugins-base/gst-libs -Isubprojects/orc
> > -I../subprojects/orc -Isubprojects/gstreamer/gst
> > -Isubprojects/gst-plugins-base/gst-libs/gst/video
> > -Isubprojects/gst-plugins-base/gst-libs/gst/pbutils
> > -Isubprojects/gst-plugins-base/gst-libs/gst/audio
> > -Isubprojects/gst-plugins-base/gst-libs/gst/tag
> > -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
> > -I/usr/include/gudev-1.0 -fdiagnostics-color=always
> > -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -O2 -g -fvisibility=hidden
> > -fno-strict-aliasing -DG_DISABLE_DEPRECATED -Wmissing-prototypes
> > -Wdeclaration-after-statement -Wold-style-definition
> > -Wmissing-declarations -Wredundant-decls -Wwrite-strings -Wformat
> > -Wformat-security -Winit-self -Wmissing-include-dirs -Waddress
> > -Wno-multichar -Wvla -Wpointer-arith -fPIC -pthread -DHAVE_CONFIG_H
> > -MD -MQ subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > -MF subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o.d
> > -o subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > -c ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:92:3:
> > error: unknown type name ‘grefcount’
> > grefcount ref_count;
> > ^~~~~~~~~
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > function ‘gst_v4l2_codec_vp9_dec_picture_data_new’:
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:106:3:
> > warning: implicit declaration of function ‘g_ref_count_init’; did you
> > mean ‘g_cond_init’? [-Wimplicit-function-declaration]
> > g_ref_count_init (&pic_data->ref_count);
> > ^~~~~~~~~~~~~~~~
> > g_cond_init
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > function ‘gst_v4l2_codec_vp9_dec_picture_data_ref’:
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:118:3:
> > warning: implicit declaration of function ‘g_ref_count_inc’; did you
> > mean ‘g_strv_contains’? [-Wimplicit-function-declaration]
> > g_ref_count_inc (&data->ref_count);
> > ^~~~~~~~~~~~~~~
> > g_strv_contains
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > function ‘gst_v4l2_codec_vp9_dec_picture_data_unref’:
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:125:7:
> > warning: implicit declaration of function ‘g_ref_count_dec’
> > [-Wimplicit-function-declaration]
> > if (g_ref_count_dec (&data->ref_count)) {
> > ^~~~~~~~~~~~~~~
> > ninja: build stopped: subcommand failed.
> >
> > Hope this helps get you started!
> > Ezequiel
>
> Ezequiel and Nicolas,
>
> Thanks - I did manage to get gstreamer 1.19.3 built successfully with
> v4l2codecs finally by getting the correct dependencies. I've attempted
> to software encode from another system and decode/display on the IMX8M
> Mini but thus far have not been successful.
I will post a V2 last today with the Mini's post-processing removed.
Someone, I apologize that I forget who, mentioned it was fused out of
the Mini, so the testing I've been doing was with that removed and I
removed the H1 encoder since the Mini doesn't support JPEG encoding.
I don't understand the YAML very well, and I'm struggling with the
dt-bindings which is the main reason I havne't submitted a formal
patch yet.
To date, I have only tested with v4l2-compliance and with the fluster
app that was recommended.
>
> I see that v4l2codecs plugin v4l2slh264dec/v4l2slmpeg2dec/v4l2slvp8dec
> and these all can output video/x-raw NV12/YUY2 which kmssink should
> accept so I'm attempting the following :
>
> # vp8 encode from x86
> gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
> vp8enc ! rtpvp8pay ! udpsink host=172.24.33.15 port=9001
> # vp8 decode on imx8mm@172.24.33.15 which has a 800x480 display
> [gst-main] root@focal-venice:~/gstreamer/build# gst-launch-1.0 -v
> udpsrc port=9001 caps = "application/x-rtp, media=(string)video,
> clock-rate=(int)90000, encoding-name=(string)VP8, payload=(int)96,
> ssrc=(uint)2745262155, timestamp-offset=(uint)2515032683,
> seqnum-offset=(uint)19579, a-framerate=(string)30" ! rtpvp8depay !
> v4l2slvp8dec ! kmssink
> Setting pipeline to PAUSED ...
> Pipeline is live and does not need PREROLL ...
> /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-width = 800
> /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-height = 480
> Pipeline is PREROLLED ...
> Setting pipeline to PLAYING ...
> /GstPipeline:pipeline0/GstUDPSrc:udpsrc0.GstPad:src: caps =
> application/x-rtp, media=(string)video, clock-rate=(int)90000,
> encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> a-framerate=(string)30
> New clock: GstSystemClock
> /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:sink: caps =
> application/x-rtp, media=(string)video, clock-rate=(int)90000,
> encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> a-framerate=(string)30
> /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:src: caps =
> video/x-vp8, framerate=(fraction)0/1, height=(int)480, width=(int)800,
> profile=(string)0
> ERROR: from element /GstPipeline:pipeline0/GstUDPSrc:udpsrc0: Internal
> data stream error.
> Additional debug info:
> ../subprojects/gstreamer/libs/gst/base/gstbasesrc.c(3127):
> gst_base_src_loop (): /GstPipeline:pipeline0/GstUDPSrc:udpsrc0:
> streaming stopped, reason not-negotiated (-4)
> Execution ended after 0:00:02.076839644
> Setting pipeline to NULL ...
> Freeing pipeline ...
>
> I'm getting the same thing when trying to use h264.
>
> I've never quite been able to grasp how to debug GStreamer's
> negotiation issues. If I end with fakesink it appears to decode so it
> must be the v4l2slvp8dec to kmssink. I tried forcing the pixel format
> using 'v4l2slvp8dec ! "video/x-raw,format=(string)NV12" ! kmssink' but
> I still get the negotiation error.
>
> What interrupts should I be seeing in /proc/interrupts? I don't see
> anything vpu/hantro related there.
I don't have my Mini with me right now, but with my current patch set
that I mentioned above, I am able to see the interrupts for the
video-codec@38300000 increase each time I run a test with fluster.
>
> I also want to make sure I have a basic understanding of the vpu
> drivers and usersapce on the IMX8M Mini. The IMX6Q/DL that I'm more
> familiar with has a vpu that is supported by the GStreamer video4linux
> plugin which shows the following (on GStreamer 1.16.2):
> v4l2jpegenc: V4L2 JPEG Encoder
> v4l2jpegdec: V4L2 JPEG Decoder
> v4l2h264enc: V4L2 H.264 Encoder
> v4l2mpeg4enc: V4L2 MPEG4 Encoder
> v4l2mpeg4dec: V4L2 MPEG4 Decoder
> v4l2mpeg2dec: V4L2 MPEG2 Decoder
> v4l2h264dec: V4L2 H264 Decoder
> The IMX6Q/DL also has an IPU that has an M2M driver that provides the
> following for scaling/colorspace conversion:
> v4l2convert: V4L2 Video Converter
>
> I believe what I'm reading is that the IMX8M Mini Hantro codecs are
> 'stateful' where more software is required to drive them and is
> supported by the newer v4l2codecs plugin. I haven't been able to
> understand what kernel version/requirements the v4l2codecs plugin
> users/requires.
>
> I'm also trying to understand how we can get scaling/colorspace
> conversion on the IMX8M Mini. The IMX8M lacks an IPU... is there some
> way to utilize scaling/colorspace conversion from the 2D GPU bound to
> the etnaviv driver?
>
> Best regards,
>
> Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-30 20:33 ` Adam Ford
@ 2021-12-17 4:48 ` Ezequiel Garcia
2021-12-17 13:15 ` Adam Ford
0 siblings, 1 reply; 42+ messages in thread
From: Ezequiel Garcia @ 2021-12-17 4:48 UTC (permalink / raw)
To: Adam Ford
Cc: Tim Harvey, Nicolas Dufresne, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Hi Adam,
>
> I will post a V2 last today with the Mini's post-processing removed.
> Someone, I apologize that I forget who, mentioned it was fused out of
> the Mini, so the testing I've been doing was with that removed and I
> removed the H1 encoder since the Mini doesn't support JPEG encoding.
>
[...]
Resurrecting this thread here. IMX8MMRM Rev. 0, 02/2019 mentions
post-processor features for G1 and G2.
Have you checked the fuse and synth registers to see if they throw
any useful information about the hardware? For instance,
comparing PP fuse register (SWREG99) and
Synthesis configuration register post-processor (SWREG100)
in both 8MQ and 8MM could be useful.
As I mentioned on my previous mail, even if G1 PP is disabled
on the Mini, I would imagine the G2 can do linear NV12 (aka raster-scan)
which in our hantro driver jargon is a "post-processed" format :-)
Thanks,
Ezequiel
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-17 4:48 ` Ezequiel Garcia
@ 2021-12-17 13:15 ` Adam Ford
2021-12-17 17:13 ` Nicolas Dufresne
0 siblings, 1 reply; 42+ messages in thread
From: Adam Ford @ 2021-12-17 13:15 UTC (permalink / raw)
To: Ezequiel Garcia
Cc: Tim Harvey, Nicolas Dufresne, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Thu, Dec 16, 2021 at 10:49 PM Ezequiel Garcia
<ezequiel@vanguardiasur.com.ar> wrote:
>
> Hi Adam,
>
> >
> > I will post a V2 last today with the Mini's post-processing removed.
> > Someone, I apologize that I forget who, mentioned it was fused out of
> > the Mini, so the testing I've been doing was with that removed and I
> > removed the H1 encoder since the Mini doesn't support JPEG encoding.
> >
> [...]
>
> Resurrecting this thread here. IMX8MMRM Rev. 0, 02/2019 mentions
> post-processor features for G1 and G2.
>
> Have you checked the fuse and synth registers to see if they throw
> any useful information about the hardware? For instance,
> comparing PP fuse register (SWREG99) and
> Synthesis configuration register post-processor (SWREG100)
> in both 8MQ and 8MM could be useful.
>
> As I mentioned on my previous mail, even if G1 PP is disabled
> on the Mini, I would imagine the G2 can do linear NV12 (aka raster-scan)
> which in our hantro driver jargon is a "post-processed" format :-)
You're likely right. I was going on memory from an e-mail from
Nicloas Defresne who wrote:
"I will check the patchset, but you need in the mini-variant to disable the G1
post processor, because this block was fused out. We didn't make it optional
from the start as according to the V1 of the TRM it was there, but that error
was corrected in V3."
In my head I assumed the G2 was affected as well, but when I double
checked his email, and based on the above statement, the G2
post-processing is probably there, so I'll run some tests with the G2
post-processing enabled. I'll also double check those registers on
both to confirm what they read. I am not sure when I'll have time
because I leave for London next week, and I won't return until early
January, but I'll do what I can.
adam
>
> Thanks,
> Ezequiel
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-17 13:15 ` Adam Ford
@ 2021-12-17 17:13 ` Nicolas Dufresne
2021-12-17 17:26 ` Tim Harvey
0 siblings, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-12-17 17:13 UTC (permalink / raw)
To: Adam Ford, Ezequiel Garcia
Cc: Tim Harvey, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le vendredi 17 décembre 2021 à 07:15 -0600, Adam Ford a écrit :
> On Thu, Dec 16, 2021 at 10:49 PM Ezequiel Garcia
> <ezequiel@vanguardiasur.com.ar> wrote:
> >
> > Hi Adam,
> >
> > >
> > > I will post a V2 last today with the Mini's post-processing removed.
> > > Someone, I apologize that I forget who, mentioned it was fused out of
> > > the Mini, so the testing I've been doing was with that removed and I
> > > removed the H1 encoder since the Mini doesn't support JPEG encoding.
> > >
> > [...]
> >
> > Resurrecting this thread here. IMX8MMRM Rev. 0, 02/2019 mentions
> > post-processor features for G1 and G2.
> >
> > Have you checked the fuse and synth registers to see if they throw
> > any useful information about the hardware? For instance,
> > comparing PP fuse register (SWREG99) and
> > Synthesis configuration register post-processor (SWREG100)
> > in both 8MQ and 8MM could be useful.
> >
> > As I mentioned on my previous mail, even if G1 PP is disabled
> > on the Mini, I would imagine the G2 can do linear NV12 (aka raster-scan)
> > which in our hantro driver jargon is a "post-processed" format :-)
>
> You're likely right. I was going on memory from an e-mail from
> Nicloas Defresne who wrote:
>
> "I will check the patchset, but you need in the mini-variant to disable the G1
> post processor, because this block was fused out. We didn't make it optional
> from the start as according to the V1 of the TRM it was there, but that error
> was corrected in V3."
>
> In my head I assumed the G2 was affected as well, but when I double
> checked his email, and based on the above statement, the G2
> post-processing is probably there, so I'll run some tests with the G2
> post-processing enabled. I'll also double check those registers on
> both to confirm what they read. I am not sure when I'll have time
> because I leave for London next week, and I won't return until early
> January, but I'll do what I can.
Sorry if this was a bit ambiguous, indeed I meant the G1 only. I've learned
later that the design of the Mini is that there is a good pre-processor in the
H1 block (encoder), so for the targeted use-cases this shall be sufficient for
most users (the output of the G1 is suitable for GPU and Display already, so the
post processor is not strictly needed).
regards,
Nicolas
>
> adam
> >
> > Thanks,
> > Ezequiel
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-17 17:13 ` Nicolas Dufresne
@ 2021-12-17 17:26 ` Tim Harvey
2021-12-17 17:52 ` Nicolas Dufresne
0 siblings, 1 reply; 42+ messages in thread
From: Tim Harvey @ 2021-12-17 17:26 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: Adam Ford, Ezequiel Garcia, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Fri, Dec 17, 2021 at 9:13 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
>
> Le vendredi 17 décembre 2021 à 07:15 -0600, Adam Ford a écrit :
> > On Thu, Dec 16, 2021 at 10:49 PM Ezequiel Garcia
> > <ezequiel@vanguardiasur.com.ar> wrote:
> > >
> > > Hi Adam,
> > >
> > > >
> > > > I will post a V2 last today with the Mini's post-processing removed.
> > > > Someone, I apologize that I forget who, mentioned it was fused out of
> > > > the Mini, so the testing I've been doing was with that removed and I
> > > > removed the H1 encoder since the Mini doesn't support JPEG encoding.
> > > >
> > > [...]
> > >
> > > Resurrecting this thread here. IMX8MMRM Rev. 0, 02/2019 mentions
> > > post-processor features for G1 and G2.
> > >
> > > Have you checked the fuse and synth registers to see if they throw
> > > any useful information about the hardware? For instance,
> > > comparing PP fuse register (SWREG99) and
> > > Synthesis configuration register post-processor (SWREG100)
> > > in both 8MQ and 8MM could be useful.
> > >
> > > As I mentioned on my previous mail, even if G1 PP is disabled
> > > on the Mini, I would imagine the G2 can do linear NV12 (aka raster-scan)
> > > which in our hantro driver jargon is a "post-processed" format :-)
> >
> > You're likely right. I was going on memory from an e-mail from
> > Nicloas Defresne who wrote:
> >
> > "I will check the patchset, but you need in the mini-variant to disable the G1
> > post processor, because this block was fused out. We didn't make it optional
> > from the start as according to the V1 of the TRM it was there, but that error
> > was corrected in V3."
> >
> > In my head I assumed the G2 was affected as well, but when I double
> > checked his email, and based on the above statement, the G2
> > post-processing is probably there, so I'll run some tests with the G2
> > post-processing enabled. I'll also double check those registers on
> > both to confirm what they read. I am not sure when I'll have time
> > because I leave for London next week, and I won't return until early
> > January, but I'll do what I can.
>
> Sorry if this was a bit ambiguous, indeed I meant the G1 only. I've learned
> later that the design of the Mini is that there is a good pre-processor in the
> H1 block (encoder), so for the targeted use-cases this shall be sufficient for
> most users (the output of the G1 is suitable for GPU and Display already, so the
> post processor is not strictly needed).
>
Nicolas,
Does this mean that if the IMX8MM G2 may be able to output a wider
array of pixel formats and that the H1 encoder may be able to accept a
wider array of pixel formats? Is this code already in place in the
hantro driver and it just needs to be enabled if the IMX8MM can handle
it or is there code to be written?
I'm not clear if anyone is working on IMX8MM VPU H1 support. You had
mentioned that some support [1] and [2] can be derived from the RK3288
using the Google ChromeOS method (a v4l2 plugin that simulates in
userspace a stateful encoder). I'm not sure if this is worth pursuing
if others are working on stateless encode support in kernel and
gstreamer.
Best Regards,
Tim
[1] libv4l plugins /
https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/refs/heads/master
[2] Kernel Driver /
https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu/
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-17 17:26 ` Tim Harvey
@ 2021-12-17 17:52 ` Nicolas Dufresne
2021-12-20 3:13 ` Chen-Yu Tsai
0 siblings, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-12-17 17:52 UTC (permalink / raw)
To: Tim Harvey
Cc: Adam Ford, Ezequiel Garcia, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le vendredi 17 décembre 2021 à 09:26 -0800, Tim Harvey a écrit :
> On Fri, Dec 17, 2021 at 9:13 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> >
> > Le vendredi 17 décembre 2021 à 07:15 -0600, Adam Ford a écrit :
> > > On Thu, Dec 16, 2021 at 10:49 PM Ezequiel Garcia
> > > <ezequiel@vanguardiasur.com.ar> wrote:
> > > >
> > > > Hi Adam,
> > > >
> > > > >
> > > > > I will post a V2 last today with the Mini's post-processing removed.
> > > > > Someone, I apologize that I forget who, mentioned it was fused out of
> > > > > the Mini, so the testing I've been doing was with that removed and I
> > > > > removed the H1 encoder since the Mini doesn't support JPEG encoding.
> > > > >
> > > > [...]
> > > >
> > > > Resurrecting this thread here. IMX8MMRM Rev. 0, 02/2019 mentions
> > > > post-processor features for G1 and G2.
> > > >
> > > > Have you checked the fuse and synth registers to see if they throw
> > > > any useful information about the hardware? For instance,
> > > > comparing PP fuse register (SWREG99) and
> > > > Synthesis configuration register post-processor (SWREG100)
> > > > in both 8MQ and 8MM could be useful.
> > > >
> > > > As I mentioned on my previous mail, even if G1 PP is disabled
> > > > on the Mini, I would imagine the G2 can do linear NV12 (aka raster-scan)
> > > > which in our hantro driver jargon is a "post-processed" format :-)
> > >
> > > You're likely right. I was going on memory from an e-mail from
> > > Nicloas Defresne who wrote:
> > >
> > > "I will check the patchset, but you need in the mini-variant to disable the G1
> > > post processor, because this block was fused out. We didn't make it optional
> > > from the start as according to the V1 of the TRM it was there, but that error
> > > was corrected in V3."
> > >
> > > In my head I assumed the G2 was affected as well, but when I double
> > > checked his email, and based on the above statement, the G2
> > > post-processing is probably there, so I'll run some tests with the G2
> > > post-processing enabled. I'll also double check those registers on
> > > both to confirm what they read. I am not sure when I'll have time
> > > because I leave for London next week, and I won't return until early
> > > January, but I'll do what I can.
> >
> > Sorry if this was a bit ambiguous, indeed I meant the G1 only. I've learned
> > later that the design of the Mini is that there is a good pre-processor in the
> > H1 block (encoder), so for the targeted use-cases this shall be sufficient for
> > most users (the output of the G1 is suitable for GPU and Display already, so the
> > post processor is not strictly needed).
> >
>
> Nicolas,
>
> Does this mean that if the IMX8MM G2 may be able to output a wider
> array of pixel formats and that the H1 encoder may be able to accept a
> wider array of pixel formats? Is this code already in place in the
No since the G2 post processor does not have a color converter (it is very
limited). In term of format, this is pretty much identical, produces linear or
tiled. The difference is that G1 supports the two layout natively, not the G2.
> hantro driver and it just needs to be enabled if the IMX8MM can handle
> it or is there code to be written?
>
> I'm not clear if anyone is working on IMX8MM VPU H1 support. You had
> mentioned that some support [1] and [2] can be derived from the RK3288
> using the Google ChromeOS method (a v4l2 plugin that simulates in
> userspace a stateful encoder). I'm not sure if this is worth pursuing
> if others are working on stateless encode support in kernel and
> gstreamer.
My colleagues started last week the project of crafting mainline stateless
encoder uAPI. This is too early. In older project, we have had good success with
the emulated stateful encoder. It is of course quite limited, but works in
gstreamer, ffmpeg and chromium. It is also likely safer compared to the vendor
provided driver.
p.s. From my knowledge, there is virtually no difference between the H1 on
RK3288 and IMX8MM/P, but we've learn from G1 that there could effectively have
more of less features.
>
> Best Regards,
>
> Tim
> [1] libv4l plugins /
> https://chromium.googlesource.com/chromiumos/third_party/libv4lplugins/+/refs/heads/master
> [2] Kernel Driver /
> https://chromium.googlesource.com/chromiumos/third_party/kernel/+/chromeos-4.4/drivers/media/platform/rockchip-vpu/
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-17 17:52 ` Nicolas Dufresne
@ 2021-12-20 3:13 ` Chen-Yu Tsai
0 siblings, 0 replies; 42+ messages in thread
From: Chen-Yu Tsai @ 2021-12-20 3:13 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: Tim Harvey, Adam Ford, Ezequiel Garcia, linux-media,
Schrempf Frieder, Marek Vasut, Jagan Teki, Adam Ford-BE,
cstevens, Philipp Zabel, Mauro Carvalho Chehab, Rob Herring,
Shawn Guo, Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Sat, Dec 18, 2021 at 1:52 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> Le vendredi 17 décembre 2021 à 09:26 -0800, Tim Harvey a écrit :
> > I'm not clear if anyone is working on IMX8MM VPU H1 support. You had
> > mentioned that some support [1] and [2] can be derived from the RK3288
> > using the Google ChromeOS method (a v4l2 plugin that simulates in
> > userspace a stateful encoder). I'm not sure if this is worth pursuing
> > if others are working on stateless encode support in kernel and
> > gstreamer.
>
> My colleagues started last week the project of crafting mainline stateless
> encoder uAPI. This is too early. In older project, we have had good success with
> the emulated stateful encoder. It is of course quite limited, but works in
> gstreamer, ffmpeg and chromium. It is also likely safer compared to the vendor
> provided driver.
If people still want to play with the old emulated stateful encoder, there
is a forward-ported version in the ChromeOS v5.10 kernel now. Note that
RK3288 (H1) support is untested. Also, RK3288 and RK3399 require different
versions of the userspace plugin. And the RK3288 version might require some
updates to add SELECTION API support.
ChenYu
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-30 19:28 ` Tim Harvey
2021-11-30 20:33 ` Adam Ford
@ 2021-12-03 4:34 ` Nicolas Dufresne
2021-12-03 16:46 ` Tim Harvey
1 sibling, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-12-03 4:34 UTC (permalink / raw)
To: Tim Harvey, Ezequiel Garcia
Cc: Adam Ford, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le mardi 30 novembre 2021 à 11:28 -0800, Tim Harvey a écrit :
> On Tue, Nov 30, 2021 at 6:00 AM Ezequiel Garcia
> <ezequiel@vanguardiasur.com.ar> wrote:
> >
> > Hi Tim,
> >
> > On Mon, 29 Nov 2021 at 16:36, Tim Harvey <tharvey@gateworks.com> wrote:
> > >
> > > On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
> > ..
> > > >
> > >
> > > Adam,
> > >
> > > What deps did you install in order to get v4l2codecs building? I
> > > installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> > > (not sure if I needed to re-configure somehow) but there is still
> > > nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> > > configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> > > how to find out what dependencies are needed or what may be missing.
> > >
> >
> > At least in my case (Centps-derivative), this is what I've done:
> >
> > ...
> > gst-plugins-bad| Run-time dependency gudev-1.0 found: NO (tried
> > pkgconfig and cmake)
> >
> > Installed gudev ... and then:
> >
> > ...
> > gst-plugins-bad| Dependency gudev-1.0 found: YES 232 (cached)
> > ...
> > gst-plugins-bad 1.19.3.1
> >
> > Plugins : accurip, adpcmdec, adpcmenc, aiff, asfmux,
> > audiobuffersplit, audiofxbad, audiomixmatrix, audiolatency,
> > audiovisualizers, autoconvert, bayer,
> > camerabin, codecalpha, coloreffects,
> > debugutilsbad, dvbsubenc, dvbsuboverlay, dvdspu, faceoverlay,
> > festival, fieldanalysis, freeverb, frei0r,
> > gaudieffects, gdp, geometrictransform,
> > id3tag, inter, interlace, ivfparse, ivtc, jp2kdecimator, jpegformat,
> > rfbsrc, midi, mpegpsdemux,
> > mpegpsmux, mpegtsdemux, mpegtsmux, mxf,
> > netsim, rtponvif, pcapparse, pnm, proxy, legacyrawparse,
> > removesilence, rist, rtmp2, rtpmanagerbad,
> > sdpelem, segmentclip, siren, smooth,
> > speed, subenc, switchbin, timecode, transcode, videofiltersbad,
> > videoframe_audiolevel, videoparsersbad,
> > videosignal, vmnc, y4mdec, decklink, dvb,
> > fbdevsink, ipcpipeline, nvcodec, shm, v4l2codecs, hls, sctp
> >
> > GStreamer current master build fails. It's a known issue which will be
> > fixed today:
> >
> > [...]
> > [8/9] Compiling C object
> > subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > FAILED: subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > cc -Isubprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p
> > -Isubprojects/gst-plugins-bad/sys/v4l2codecs
> > -I../subprojects/gst-plugins-bad/sys/v4l2codecs
> > -Isubprojects/gst-plugins-bad -I../subprojects/gst-plugins-bad
> > -Isubprojects/gstreamer/libs -I../subprojects/gstreamer/libs
> > -Isubprojects/gstreamer -I../subprojects/gstreamer
> > -Isubprojects/gst-plugins-bad/gst-libs
> > -I../subprojects/gst-plugins-bad/gst-libs
> > -Isubprojects/gst-plugins-base/gst-libs
> > -I../subprojects/gst-plugins-base/gst-libs -Isubprojects/orc
> > -I../subprojects/orc -Isubprojects/gstreamer/gst
> > -Isubprojects/gst-plugins-base/gst-libs/gst/video
> > -Isubprojects/gst-plugins-base/gst-libs/gst/pbutils
> > -Isubprojects/gst-plugins-base/gst-libs/gst/audio
> > -Isubprojects/gst-plugins-base/gst-libs/gst/tag
> > -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
> > -I/usr/include/gudev-1.0 -fdiagnostics-color=always
> > -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -O2 -g -fvisibility=hidden
> > -fno-strict-aliasing -DG_DISABLE_DEPRECATED -Wmissing-prototypes
> > -Wdeclaration-after-statement -Wold-style-definition
> > -Wmissing-declarations -Wredundant-decls -Wwrite-strings -Wformat
> > -Wformat-security -Winit-self -Wmissing-include-dirs -Waddress
> > -Wno-multichar -Wvla -Wpointer-arith -fPIC -pthread -DHAVE_CONFIG_H
> > -MD -MQ subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > -MF subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o.d
> > -o subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > -c ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:92:3:
> > error: unknown type name ‘grefcount’
> > grefcount ref_count;
> > ^~~~~~~~~
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > function ‘gst_v4l2_codec_vp9_dec_picture_data_new’:
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:106:3:
> > warning: implicit declaration of function ‘g_ref_count_init’; did you
> > mean ‘g_cond_init’? [-Wimplicit-function-declaration]
> > g_ref_count_init (&pic_data->ref_count);
> > ^~~~~~~~~~~~~~~~
> > g_cond_init
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > function ‘gst_v4l2_codec_vp9_dec_picture_data_ref’:
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:118:3:
> > warning: implicit declaration of function ‘g_ref_count_inc’; did you
> > mean ‘g_strv_contains’? [-Wimplicit-function-declaration]
> > g_ref_count_inc (&data->ref_count);
> > ^~~~~~~~~~~~~~~
> > g_strv_contains
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > function ‘gst_v4l2_codec_vp9_dec_picture_data_unref’:
> > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:125:7:
> > warning: implicit declaration of function ‘g_ref_count_dec’
> > [-Wimplicit-function-declaration]
> > if (g_ref_count_dec (&data->ref_count)) {
> > ^~~~~~~~~~~~~~~
> > ninja: build stopped: subcommand failed.
> >
> > Hope this helps get you started!
> > Ezequiel
>
> Ezequiel and Nicolas,
>
> Thanks - I did manage to get gstreamer 1.19.3 built successfully with
> v4l2codecs finally by getting the correct dependencies. I've attempted
> to software encode from another system and decode/display on the IMX8M
> Mini but thus far have not been successful.
>
> I see that v4l2codecs plugin v4l2slh264dec/v4l2slmpeg2dec/v4l2slvp8dec
> and these all can output video/x-raw NV12/YUY2 which kmssink should
> accept so I'm attempting the following :
>
> # vp8 encode from x86
> gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
> vp8enc ! rtpvp8pay ! udpsink host=172.24.33.15 port=9001
> # vp8 decode on imx8mm@172.24.33.15 which has a 800x480 display
> [gst-main] root@focal-venice:~/gstreamer/build# gst-launch-1.0 -v
> udpsrc port=9001 caps = "application/x-rtp, media=(string)video,
> clock-rate=(int)90000, encoding-name=(string)VP8, payload=(int)96,
> ssrc=(uint)2745262155, timestamp-offset=(uint)2515032683,
> seqnum-offset=(uint)19579, a-framerate=(string)30" ! rtpvp8depay !
> v4l2slvp8dec ! kmssink
> Setting pipeline to PAUSED ...
> Pipeline is live and does not need PREROLL ...
> /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-width = 800
> /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-height = 480
> Pipeline is PREROLLED ...
> Setting pipeline to PLAYING ...
> /GstPipeline:pipeline0/GstUDPSrc:udpsrc0.GstPad:src: caps =
> application/x-rtp, media=(string)video, clock-rate=(int)90000,
> encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> a-framerate=(string)30
> New clock: GstSystemClock
> /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:sink: caps =
> application/x-rtp, media=(string)video, clock-rate=(int)90000,
> encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> a-framerate=(string)30
> /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:src: caps =
> video/x-vp8, framerate=(fraction)0/1, height=(int)480, width=(int)800,
> profile=(string)0
> ERROR: from element /GstPipeline:pipeline0/GstUDPSrc:udpsrc0: Internal
> data stream error.
> Additional debug info:
> ../subprojects/gstreamer/libs/gst/base/gstbasesrc.c(3127):
> gst_base_src_loop (): /GstPipeline:pipeline0/GstUDPSrc:udpsrc0:
> streaming stopped, reason not-negotiated (-4)
> Execution ended after 0:00:02.076839644
> Setting pipeline to NULL ...
> Freeing pipeline ...
>
> I'm getting the same thing when trying to use h264.
>
> I've never quite been able to grasp how to debug GStreamer's
> negotiation issues. If I end with fakesink it appears to decode so it
> must be the v4l2slvp8dec to kmssink. I tried forcing the pixel format
> using 'v4l2slvp8dec ! "video/x-raw,format=(string)NV12" ! kmssink' but
> I still get the negotiation error.
>
> What interrupts should I be seeing in /proc/interrupts? I don't see
> anything vpu/hantro related there.
>
> I also want to make sure I have a basic understanding of the vpu
> drivers and usersapce on the IMX8M Mini. The IMX6Q/DL that I'm more
> familiar with has a vpu that is supported by the GStreamer video4linux
> plugin which shows the following (on GStreamer 1.16.2):
> v4l2jpegenc: V4L2 JPEG Encoder
> v4l2jpegdec: V4L2 JPEG Decoder
> v4l2h264enc: V4L2 H.264 Encoder
> v4l2mpeg4enc: V4L2 MPEG4 Encoder
> v4l2mpeg4dec: V4L2 MPEG4 Decoder
> v4l2mpeg2dec: V4L2 MPEG2 Decoder
> v4l2h264dec: V4L2 H264 Decoder
> The IMX6Q/DL also has an IPU that has an M2M driver that provides the
> following for scaling/colorspace conversion:
> v4l2convert: V4L2 Video Converter
>
> I believe what I'm reading is that the IMX8M Mini Hantro codecs are
> 'stateful' where more software is required to drive them and is
'stateless'. the rest is right.
> supported by the newer v4l2codecs plugin. I haven't been able to
> understand what kernel version/requirements the v4l2codecs plugin
> users/requires.
After H264 debacle with 5.9, 5.10 and 5.11 API break and GStreamer not getting
enough release to support all of these we started merging support for CODECs
only when the stable uAPI land. I made an exception for VP9 as it is already
applied in the media tree and didn't want to miss 1.20 release.
So to answer you question, it depends on when the CODEC uAPI landed.
>
> I'm also trying to understand how we can get scaling/colorspace
> conversion on the IMX8M Mini. The IMX8M lacks an IPU... is there some
> way to utilize scaling/colorspace conversion from the 2D GPU bound to
> the etnaviv driver?
The concept of the mini, is that you would be using th encoder for anything that
isn't going to the display. So they only integrated the Hantro PP on the
encoder. Unfortunately, you'll have to be patient for mainline stateless encoder
support, we barely scratch the surface of this subject, but its being worked on.
Unlike the IMX8MQ, you don't have the option to output YUYV (packed yuv 4:2:2)
which would satisfy the 2D GPU support hoold to the DMABuf import path.
When the display driver gets ready and upstream (it's been only 2-3 years now),
you'll get linear NV12 support along with G2 compression support (this one is
not supported by the GPU, so it will be tricky to expose in userland). I don't
think the display support 4L4 tiles, but you GPU most likely can do with the
right shared and texelFetch() or vulkan equivalent if that exist on that target.
>
> Best regards,
>
> Tim
regards,
Nicolas
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-03 4:34 ` Nicolas Dufresne
@ 2021-12-03 16:46 ` Tim Harvey
2021-12-03 19:37 ` Nicolas Dufresne
0 siblings, 1 reply; 42+ messages in thread
From: Tim Harvey @ 2021-12-03 16:46 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: Ezequiel Garcia, Adam Ford, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Thu, Dec 2, 2021 at 8:34 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
>
> Le mardi 30 novembre 2021 à 11:28 -0800, Tim Harvey a écrit :
> > On Tue, Nov 30, 2021 at 6:00 AM Ezequiel Garcia
> > <ezequiel@vanguardiasur.com.ar> wrote:
> > >
> > > Hi Tim,
> > >
> > > On Mon, 29 Nov 2021 at 16:36, Tim Harvey <tharvey@gateworks.com> wrote:
> > > >
> > > > On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
> > > ..
> > > > >
> > > >
> > > > Adam,
> > > >
> > > > What deps did you install in order to get v4l2codecs building? I
> > > > installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> > > > (not sure if I needed to re-configure somehow) but there is still
> > > > nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> > > > configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> > > > how to find out what dependencies are needed or what may be missing.
> > > >
> > >
> > > At least in my case (Centps-derivative), this is what I've done:
> > >
> > > ...
> > > gst-plugins-bad| Run-time dependency gudev-1.0 found: NO (tried
> > > pkgconfig and cmake)
> > >
> > > Installed gudev ... and then:
> > >
> > > ...
> > > gst-plugins-bad| Dependency gudev-1.0 found: YES 232 (cached)
> > > ...
> > > gst-plugins-bad 1.19.3.1
> > >
> > > Plugins : accurip, adpcmdec, adpcmenc, aiff, asfmux,
> > > audiobuffersplit, audiofxbad, audiomixmatrix, audiolatency,
> > > audiovisualizers, autoconvert, bayer,
> > > camerabin, codecalpha, coloreffects,
> > > debugutilsbad, dvbsubenc, dvbsuboverlay, dvdspu, faceoverlay,
> > > festival, fieldanalysis, freeverb, frei0r,
> > > gaudieffects, gdp, geometrictransform,
> > > id3tag, inter, interlace, ivfparse, ivtc, jp2kdecimator, jpegformat,
> > > rfbsrc, midi, mpegpsdemux,
> > > mpegpsmux, mpegtsdemux, mpegtsmux, mxf,
> > > netsim, rtponvif, pcapparse, pnm, proxy, legacyrawparse,
> > > removesilence, rist, rtmp2, rtpmanagerbad,
> > > sdpelem, segmentclip, siren, smooth,
> > > speed, subenc, switchbin, timecode, transcode, videofiltersbad,
> > > videoframe_audiolevel, videoparsersbad,
> > > videosignal, vmnc, y4mdec, decklink, dvb,
> > > fbdevsink, ipcpipeline, nvcodec, shm, v4l2codecs, hls, sctp
> > >
> > > GStreamer current master build fails. It's a known issue which will be
> > > fixed today:
> > >
> > > [...]
> > > [8/9] Compiling C object
> > > subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > FAILED: subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > cc -Isubprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p
> > > -Isubprojects/gst-plugins-bad/sys/v4l2codecs
> > > -I../subprojects/gst-plugins-bad/sys/v4l2codecs
> > > -Isubprojects/gst-plugins-bad -I../subprojects/gst-plugins-bad
> > > -Isubprojects/gstreamer/libs -I../subprojects/gstreamer/libs
> > > -Isubprojects/gstreamer -I../subprojects/gstreamer
> > > -Isubprojects/gst-plugins-bad/gst-libs
> > > -I../subprojects/gst-plugins-bad/gst-libs
> > > -Isubprojects/gst-plugins-base/gst-libs
> > > -I../subprojects/gst-plugins-base/gst-libs -Isubprojects/orc
> > > -I../subprojects/orc -Isubprojects/gstreamer/gst
> > > -Isubprojects/gst-plugins-base/gst-libs/gst/video
> > > -Isubprojects/gst-plugins-base/gst-libs/gst/pbutils
> > > -Isubprojects/gst-plugins-base/gst-libs/gst/audio
> > > -Isubprojects/gst-plugins-base/gst-libs/gst/tag
> > > -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
> > > -I/usr/include/gudev-1.0 -fdiagnostics-color=always
> > > -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -O2 -g -fvisibility=hidden
> > > -fno-strict-aliasing -DG_DISABLE_DEPRECATED -Wmissing-prototypes
> > > -Wdeclaration-after-statement -Wold-style-definition
> > > -Wmissing-declarations -Wredundant-decls -Wwrite-strings -Wformat
> > > -Wformat-security -Winit-self -Wmissing-include-dirs -Waddress
> > > -Wno-multichar -Wvla -Wpointer-arith -fPIC -pthread -DHAVE_CONFIG_H
> > > -MD -MQ subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > -MF subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o.d
> > > -o subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > -c ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c
> > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:92:3:
> > > error: unknown type name ‘grefcount’
> > > grefcount ref_count;
> > > ^~~~~~~~~
> > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > function ‘gst_v4l2_codec_vp9_dec_picture_data_new’:
> > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:106:3:
> > > warning: implicit declaration of function ‘g_ref_count_init’; did you
> > > mean ‘g_cond_init’? [-Wimplicit-function-declaration]
> > > g_ref_count_init (&pic_data->ref_count);
> > > ^~~~~~~~~~~~~~~~
> > > g_cond_init
> > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > function ‘gst_v4l2_codec_vp9_dec_picture_data_ref’:
> > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:118:3:
> > > warning: implicit declaration of function ‘g_ref_count_inc’; did you
> > > mean ‘g_strv_contains’? [-Wimplicit-function-declaration]
> > > g_ref_count_inc (&data->ref_count);
> > > ^~~~~~~~~~~~~~~
> > > g_strv_contains
> > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > function ‘gst_v4l2_codec_vp9_dec_picture_data_unref’:
> > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:125:7:
> > > warning: implicit declaration of function ‘g_ref_count_dec’
> > > [-Wimplicit-function-declaration]
> > > if (g_ref_count_dec (&data->ref_count)) {
> > > ^~~~~~~~~~~~~~~
> > > ninja: build stopped: subcommand failed.
> > >
> > > Hope this helps get you started!
> > > Ezequiel
> >
> > Ezequiel and Nicolas,
> >
> > Thanks - I did manage to get gstreamer 1.19.3 built successfully with
> > v4l2codecs finally by getting the correct dependencies. I've attempted
> > to software encode from another system and decode/display on the IMX8M
> > Mini but thus far have not been successful.
> >
> > I see that v4l2codecs plugin v4l2slh264dec/v4l2slmpeg2dec/v4l2slvp8dec
> > and these all can output video/x-raw NV12/YUY2 which kmssink should
> > accept so I'm attempting the following :
> >
> > # vp8 encode from x86
> > gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
> > vp8enc ! rtpvp8pay ! udpsink host=172.24.33.15 port=9001
> > # vp8 decode on imx8mm@172.24.33.15 which has a 800x480 display
> > [gst-main] root@focal-venice:~/gstreamer/build# gst-launch-1.0 -v
> > udpsrc port=9001 caps = "application/x-rtp, media=(string)video,
> > clock-rate=(int)90000, encoding-name=(string)VP8, payload=(int)96,
> > ssrc=(uint)2745262155, timestamp-offset=(uint)2515032683,
> > seqnum-offset=(uint)19579, a-framerate=(string)30" ! rtpvp8depay !
> > v4l2slvp8dec ! kmssink
> > Setting pipeline to PAUSED ...
> > Pipeline is live and does not need PREROLL ...
> > /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-width = 800
> > /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-height = 480
> > Pipeline is PREROLLED ...
> > Setting pipeline to PLAYING ...
> > /GstPipeline:pipeline0/GstUDPSrc:udpsrc0.GstPad:src: caps =
> > application/x-rtp, media=(string)video, clock-rate=(int)90000,
> > encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> > timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> > a-framerate=(string)30
> > New clock: GstSystemClock
> > /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:sink: caps =
> > application/x-rtp, media=(string)video, clock-rate=(int)90000,
> > encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> > timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> > a-framerate=(string)30
> > /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:src: caps =
> > video/x-vp8, framerate=(fraction)0/1, height=(int)480, width=(int)800,
> > profile=(string)0
> > ERROR: from element /GstPipeline:pipeline0/GstUDPSrc:udpsrc0: Internal
> > data stream error.
> > Additional debug info:
> > ../subprojects/gstreamer/libs/gst/base/gstbasesrc.c(3127):
> > gst_base_src_loop (): /GstPipeline:pipeline0/GstUDPSrc:udpsrc0:
> > streaming stopped, reason not-negotiated (-4)
> > Execution ended after 0:00:02.076839644
> > Setting pipeline to NULL ...
> > Freeing pipeline ...
> >
> > I'm getting the same thing when trying to use h264.
> >
> > I've never quite been able to grasp how to debug GStreamer's
> > negotiation issues. If I end with fakesink it appears to decode so it
> > must be the v4l2slvp8dec to kmssink. I tried forcing the pixel format
> > using 'v4l2slvp8dec ! "video/x-raw,format=(string)NV12" ! kmssink' but
> > I still get the negotiation error.
> >
> > What interrupts should I be seeing in /proc/interrupts? I don't see
> > anything vpu/hantro related there.
> >
> > I also want to make sure I have a basic understanding of the vpu
> > drivers and usersapce on the IMX8M Mini. The IMX6Q/DL that I'm more
> > familiar with has a vpu that is supported by the GStreamer video4linux
> > plugin which shows the following (on GStreamer 1.16.2):
> > v4l2jpegenc: V4L2 JPEG Encoder
> > v4l2jpegdec: V4L2 JPEG Decoder
> > v4l2h264enc: V4L2 H.264 Encoder
> > v4l2mpeg4enc: V4L2 MPEG4 Encoder
> > v4l2mpeg4dec: V4L2 MPEG4 Decoder
> > v4l2mpeg2dec: V4L2 MPEG2 Decoder
> > v4l2h264dec: V4L2 H264 Decoder
> > The IMX6Q/DL also has an IPU that has an M2M driver that provides the
> > following for scaling/colorspace conversion:
> > v4l2convert: V4L2 Video Converter
> >
> > I believe what I'm reading is that the IMX8M Mini Hantro codecs are
> > 'stateful' where more software is required to drive them and is
>
> 'stateless'. the rest is right.
>
> > supported by the newer v4l2codecs plugin. I haven't been able to
> > understand what kernel version/requirements the v4l2codecs plugin
> > users/requires.
>
> After H264 debacle with 5.9, 5.10 and 5.11 API break and GStreamer not getting
> enough release to support all of these we started merging support for CODECs
> only when the stable uAPI land. I made an exception for VP9 as it is already
> applied in the media tree and didn't want to miss 1.20 release.
>
> So to answer you question, it depends on when the CODEC uAPI landed.
>
Ok, thanks for the explanation.
> >
> > I'm also trying to understand how we can get scaling/colorspace
> > conversion on the IMX8M Mini. The IMX8M lacks an IPU... is there some
> > way to utilize scaling/colorspace conversion from the 2D GPU bound to
> > the etnaviv driver?
>
> The concept of the mini, is that you would be using th encoder for anything that
> isn't going to the display. So they only integrated the Hantro PP on the
> encoder. Unfortunately, you'll have to be patient for mainline stateless encoder
> support, we barely scratch the surface of this subject, but its being worked on.
After some searching for Hantro PP I see that the IMXMQ (IMX8M
Dual/QuadLite/Quad) mentions Hantro PP. From the IMX8MDQLQRM section
14.1.2.1 Decoder Features:
<quote>
Video post-processing features
- Frame rotation 90 degrees left/right
- Frame mirroring horizontally/vertically
- Frame cropping
- Frame conversion from YCbCr formats to 16-bit or 32-bit RGB formats
- Frame scaling with maximum up-scaling factor of 3
- Two rectangular or alpha blending masks for output frame
The post-processing features can be used in pipeline with the
decoder. The postprocessing features can also be used as stand-alone,
without performing any decoding
</quote>
The above is under the VPU_G1 section and the same is not mentioned
for VPU_G2 and the IMX8MQ doesn't have encode support. Where do you
see that the IMX8MM has the Hantro PP on the H1? I know the TRM's lack
a lot of info so perhaps you know more about the internals than what
the TRM states.
I also found on a forum
(https://community.nxp.com/t5/i-MX-Processors/imx8mq-Hantro-G1-scaling/m-p/1285343)
that NXP's BSP doesn't use the Hantro for scaling (and likely not csc
either) and they use the GPU instead. I'm still unclear if/how you
could tap into the 2D GPU to use its scaling/conversion if it's bound
to the etnaviv driver.
> Unlike the IMX8MQ, you don't have the option to output YUYV (packed yuv 4:2:2)
> which would satisfy the 2D GPU support hoold to the DMABuf import path.
>
> When the display driver gets ready and upstream (it's been only 2-3 years now),
> you'll get linear NV12 support along with G2 compression support (this one is
> not supported by the GPU, so it will be tricky to expose in userland). I don't
> think the display support 4L4 tiles, but you GPU most likely can do with the
> right shared and texelFetch() or vulkan equivalent if that exist on that target.
Do you mean the Samsung Exynos DRM driver (which doesn't yet have
support for IMX8MM) or drivers/gpu/drm/mxsfb?
I'm currently using a pretty old patchset that adds IMX8MM support to
the drm/exynos driver. I'm way out of my realm when talking about
GPU/VPU and display drivers.
Best regards,
Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-03 16:46 ` Tim Harvey
@ 2021-12-03 19:37 ` Nicolas Dufresne
2021-12-06 9:20 ` Lucas Stach
0 siblings, 1 reply; 42+ messages in thread
From: Nicolas Dufresne @ 2021-12-03 19:37 UTC (permalink / raw)
To: Tim Harvey
Cc: Ezequiel Garcia, Adam Ford, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach, Joakim Zhang,
Alice Guo, Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le vendredi 03 décembre 2021 à 08:46 -0800, Tim Harvey a écrit :
> On Thu, Dec 2, 2021 at 8:34 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> >
> > Le mardi 30 novembre 2021 à 11:28 -0800, Tim Harvey a écrit :
> > > On Tue, Nov 30, 2021 at 6:00 AM Ezequiel Garcia
> > > <ezequiel@vanguardiasur.com.ar> wrote:
> > > >
> > > > Hi Tim,
> > > >
> > > > On Mon, 29 Nov 2021 at 16:36, Tim Harvey <tharvey@gateworks.com> wrote:
> > > > >
> > > > > On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
> > > > ..
> > > > > >
> > > > >
> > > > > Adam,
> > > > >
> > > > > What deps did you install in order to get v4l2codecs building? I
> > > > > installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> > > > > (not sure if I needed to re-configure somehow) but there is still
> > > > > nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> > > > > configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> > > > > how to find out what dependencies are needed or what may be missing.
> > > > >
> > > >
> > > > At least in my case (Centps-derivative), this is what I've done:
> > > >
> > > > ...
> > > > gst-plugins-bad| Run-time dependency gudev-1.0 found: NO (tried
> > > > pkgconfig and cmake)
> > > >
> > > > Installed gudev ... and then:
> > > >
> > > > ...
> > > > gst-plugins-bad| Dependency gudev-1.0 found: YES 232 (cached)
> > > > ...
> > > > gst-plugins-bad 1.19.3.1
> > > >
> > > > Plugins : accurip, adpcmdec, adpcmenc, aiff, asfmux,
> > > > audiobuffersplit, audiofxbad, audiomixmatrix, audiolatency,
> > > > audiovisualizers, autoconvert, bayer,
> > > > camerabin, codecalpha, coloreffects,
> > > > debugutilsbad, dvbsubenc, dvbsuboverlay, dvdspu, faceoverlay,
> > > > festival, fieldanalysis, freeverb, frei0r,
> > > > gaudieffects, gdp, geometrictransform,
> > > > id3tag, inter, interlace, ivfparse, ivtc, jp2kdecimator, jpegformat,
> > > > rfbsrc, midi, mpegpsdemux,
> > > > mpegpsmux, mpegtsdemux, mpegtsmux, mxf,
> > > > netsim, rtponvif, pcapparse, pnm, proxy, legacyrawparse,
> > > > removesilence, rist, rtmp2, rtpmanagerbad,
> > > > sdpelem, segmentclip, siren, smooth,
> > > > speed, subenc, switchbin, timecode, transcode, videofiltersbad,
> > > > videoframe_audiolevel, videoparsersbad,
> > > > videosignal, vmnc, y4mdec, decklink, dvb,
> > > > fbdevsink, ipcpipeline, nvcodec, shm, v4l2codecs, hls, sctp
> > > >
> > > > GStreamer current master build fails. It's a known issue which will be
> > > > fixed today:
> > > >
> > > > [...]
> > > > [8/9] Compiling C object
> > > > subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > FAILED: subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > cc -Isubprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p
> > > > -Isubprojects/gst-plugins-bad/sys/v4l2codecs
> > > > -I../subprojects/gst-plugins-bad/sys/v4l2codecs
> > > > -Isubprojects/gst-plugins-bad -I../subprojects/gst-plugins-bad
> > > > -Isubprojects/gstreamer/libs -I../subprojects/gstreamer/libs
> > > > -Isubprojects/gstreamer -I../subprojects/gstreamer
> > > > -Isubprojects/gst-plugins-bad/gst-libs
> > > > -I../subprojects/gst-plugins-bad/gst-libs
> > > > -Isubprojects/gst-plugins-base/gst-libs
> > > > -I../subprojects/gst-plugins-base/gst-libs -Isubprojects/orc
> > > > -I../subprojects/orc -Isubprojects/gstreamer/gst
> > > > -Isubprojects/gst-plugins-base/gst-libs/gst/video
> > > > -Isubprojects/gst-plugins-base/gst-libs/gst/pbutils
> > > > -Isubprojects/gst-plugins-base/gst-libs/gst/audio
> > > > -Isubprojects/gst-plugins-base/gst-libs/gst/tag
> > > > -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
> > > > -I/usr/include/gudev-1.0 -fdiagnostics-color=always
> > > > -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -O2 -g -fvisibility=hidden
> > > > -fno-strict-aliasing -DG_DISABLE_DEPRECATED -Wmissing-prototypes
> > > > -Wdeclaration-after-statement -Wold-style-definition
> > > > -Wmissing-declarations -Wredundant-decls -Wwrite-strings -Wformat
> > > > -Wformat-security -Winit-self -Wmissing-include-dirs -Waddress
> > > > -Wno-multichar -Wvla -Wpointer-arith -fPIC -pthread -DHAVE_CONFIG_H
> > > > -MD -MQ subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > -MF subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o.d
> > > > -o subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > -c ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c
> > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:92:3:
> > > > error: unknown type name ‘grefcount’
> > > > grefcount ref_count;
> > > > ^~~~~~~~~
> > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_new’:
> > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:106:3:
> > > > warning: implicit declaration of function ‘g_ref_count_init’; did you
> > > > mean ‘g_cond_init’? [-Wimplicit-function-declaration]
> > > > g_ref_count_init (&pic_data->ref_count);
> > > > ^~~~~~~~~~~~~~~~
> > > > g_cond_init
> > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_ref’:
> > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:118:3:
> > > > warning: implicit declaration of function ‘g_ref_count_inc’; did you
> > > > mean ‘g_strv_contains’? [-Wimplicit-function-declaration]
> > > > g_ref_count_inc (&data->ref_count);
> > > > ^~~~~~~~~~~~~~~
> > > > g_strv_contains
> > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_unref’:
> > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:125:7:
> > > > warning: implicit declaration of function ‘g_ref_count_dec’
> > > > [-Wimplicit-function-declaration]
> > > > if (g_ref_count_dec (&data->ref_count)) {
> > > > ^~~~~~~~~~~~~~~
> > > > ninja: build stopped: subcommand failed.
> > > >
> > > > Hope this helps get you started!
> > > > Ezequiel
> > >
> > > Ezequiel and Nicolas,
> > >
> > > Thanks - I did manage to get gstreamer 1.19.3 built successfully with
> > > v4l2codecs finally by getting the correct dependencies. I've attempted
> > > to software encode from another system and decode/display on the IMX8M
> > > Mini but thus far have not been successful.
> > >
> > > I see that v4l2codecs plugin v4l2slh264dec/v4l2slmpeg2dec/v4l2slvp8dec
> > > and these all can output video/x-raw NV12/YUY2 which kmssink should
> > > accept so I'm attempting the following :
> > >
> > > # vp8 encode from x86
> > > gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
> > > vp8enc ! rtpvp8pay ! udpsink host=172.24.33.15 port=9001
> > > # vp8 decode on imx8mm@172.24.33.15 which has a 800x480 display
> > > [gst-main] root@focal-venice:~/gstreamer/build# gst-launch-1.0 -v
> > > udpsrc port=9001 caps = "application/x-rtp, media=(string)video,
> > > clock-rate=(int)90000, encoding-name=(string)VP8, payload=(int)96,
> > > ssrc=(uint)2745262155, timestamp-offset=(uint)2515032683,
> > > seqnum-offset=(uint)19579, a-framerate=(string)30" ! rtpvp8depay !
> > > v4l2slvp8dec ! kmssink
> > > Setting pipeline to PAUSED ...
> > > Pipeline is live and does not need PREROLL ...
> > > /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-width = 800
> > > /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-height = 480
> > > Pipeline is PREROLLED ...
> > > Setting pipeline to PLAYING ...
> > > /GstPipeline:pipeline0/GstUDPSrc:udpsrc0.GstPad:src: caps =
> > > application/x-rtp, media=(string)video, clock-rate=(int)90000,
> > > encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> > > timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> > > a-framerate=(string)30
> > > New clock: GstSystemClock
> > > /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:sink: caps =
> > > application/x-rtp, media=(string)video, clock-rate=(int)90000,
> > > encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> > > timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> > > a-framerate=(string)30
> > > /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:src: caps =
> > > video/x-vp8, framerate=(fraction)0/1, height=(int)480, width=(int)800,
> > > profile=(string)0
> > > ERROR: from element /GstPipeline:pipeline0/GstUDPSrc:udpsrc0: Internal
> > > data stream error.
> > > Additional debug info:
> > > ../subprojects/gstreamer/libs/gst/base/gstbasesrc.c(3127):
> > > gst_base_src_loop (): /GstPipeline:pipeline0/GstUDPSrc:udpsrc0:
> > > streaming stopped, reason not-negotiated (-4)
> > > Execution ended after 0:00:02.076839644
> > > Setting pipeline to NULL ...
> > > Freeing pipeline ...
> > >
> > > I'm getting the same thing when trying to use h264.
> > >
> > > I've never quite been able to grasp how to debug GStreamer's
> > > negotiation issues. If I end with fakesink it appears to decode so it
> > > must be the v4l2slvp8dec to kmssink. I tried forcing the pixel format
> > > using 'v4l2slvp8dec ! "video/x-raw,format=(string)NV12" ! kmssink' but
> > > I still get the negotiation error.
> > >
> > > What interrupts should I be seeing in /proc/interrupts? I don't see
> > > anything vpu/hantro related there.
> > >
> > > I also want to make sure I have a basic understanding of the vpu
> > > drivers and usersapce on the IMX8M Mini. The IMX6Q/DL that I'm more
> > > familiar with has a vpu that is supported by the GStreamer video4linux
> > > plugin which shows the following (on GStreamer 1.16.2):
> > > v4l2jpegenc: V4L2 JPEG Encoder
> > > v4l2jpegdec: V4L2 JPEG Decoder
> > > v4l2h264enc: V4L2 H.264 Encoder
> > > v4l2mpeg4enc: V4L2 MPEG4 Encoder
> > > v4l2mpeg4dec: V4L2 MPEG4 Decoder
> > > v4l2mpeg2dec: V4L2 MPEG2 Decoder
> > > v4l2h264dec: V4L2 H264 Decoder
> > > The IMX6Q/DL also has an IPU that has an M2M driver that provides the
> > > following for scaling/colorspace conversion:
> > > v4l2convert: V4L2 Video Converter
> > >
> > > I believe what I'm reading is that the IMX8M Mini Hantro codecs are
> > > 'stateful' where more software is required to drive them and is
> >
> > 'stateless'. the rest is right.
> >
> > > supported by the newer v4l2codecs plugin. I haven't been able to
> > > understand what kernel version/requirements the v4l2codecs plugin
> > > users/requires.
> >
> > After H264 debacle with 5.9, 5.10 and 5.11 API break and GStreamer not getting
> > enough release to support all of these we started merging support for CODECs
> > only when the stable uAPI land. I made an exception for VP9 as it is already
> > applied in the media tree and didn't want to miss 1.20 release.
> >
> > So to answer you question, it depends on when the CODEC uAPI landed.
> >
>
> Ok, thanks for the explanation.
>
> > >
> > > I'm also trying to understand how we can get scaling/colorspace
> > > conversion on the IMX8M Mini. The IMX8M lacks an IPU... is there some
> > > way to utilize scaling/colorspace conversion from the 2D GPU bound to
> > > the etnaviv driver?
> >
> > The concept of the mini, is that you would be using th encoder for anything that
> > isn't going to the display. So they only integrated the Hantro PP on the
> > encoder. Unfortunately, you'll have to be patient for mainline stateless encoder
> > support, we barely scratch the surface of this subject, but its being worked on.
>
> After some searching for Hantro PP I see that the IMXMQ (IMX8M
> Dual/QuadLite/Quad) mentions Hantro PP. From the IMX8MDQLQRM section
> 14.1.2.1 Decoder Features:
> <quote>
> Video post-processing features
> - Frame rotation 90 degrees left/right
> - Frame mirroring horizontally/vertically
> - Frame cropping
> - Frame conversion from YCbCr formats to 16-bit or 32-bit RGB formats
> - Frame scaling with maximum up-scaling factor of 3
> - Two rectangular or alpha blending masks for output frame
>
> The post-processing features can be used in pipeline with the
> decoder. The postprocessing features can also be used as stand-alone,
> without performing any decoding
> </quote>
>
> The above is under the VPU_G1 section and the same is not mentioned
> for VPU_G2 and the IMX8MQ doesn't have encode support. Where do you
> see that the IMX8MM has the Hantro PP on the H1? I know the TRM's lack
> a lot of info so perhaps you know more about the internals than what
> the TRM states.
I've got told that by someone with contacts at NXP recently (in IRC). I haven't
verified it though, it just made sense for the targeted use of th mini. Hantro
G1 driver does not yet expose an M2M for the standalone mode of the PP, but
shall be possible. Decode an PP cannot run concurrently though, so concurrent PP
and decode will have big impact on performance.
The G2 PP is different, but I *think* its always there. It's not doing much,
linear NV12 (detiling from 4x4 linear tiles), and can scale down by factor of 2,
4 an 8. If there is more feature I'm not aware.
>
> I also found on a forum
> (https://community.nxp.com/t5/i-MX-Processors/imx8mq-Hantro-G1-scaling/m-p/1285343)
> that NXP's BSP doesn't use the Hantro for scaling (and likely not csc
> either) and they use the GPU instead. I'm still unclear if/how you
> could tap into the 2D GPU to use its scaling/conversion if it's bound
> to the etnaviv driver.
>
> > Unlike the IMX8MQ, you don't have the option to output YUYV (packed yuv 4:2:2)
> > which would satisfy the 2D GPU support hoold to the DMABuf import path.
As I'm saying above, you can't, there is no NV12 support in that 2D GPU from
what I was old by Etnaviv folks, only YUYV (4:2:2 packed). Shaders is the only
option.
> >
> > When the display driver gets ready and upstream (it's been only 2-3 years now),
> > you'll get linear NV12 support along with G2 compression support (this one is
> > not supported by the GPU, so it will be tricky to expose in userland). I don't
> > think the display support 4L4 tiles, but you GPU most likely can do with the
> > right shared and texelFetch() or vulkan equivalent if that exist on that target.
>
> Do you mean the Samsung Exynos DRM driver (which doesn't yet have
> support for IMX8MM) or drivers/gpu/drm/mxsfb?
>
> I'm currently using a pretty old patchset that adds IMX8MM support to
> the drm/exynos driver. I'm way out of my realm when talking about
> GPU/VPU and display drivers.
Didn't know the mini was using Samsung display controller. Didn't even know that
chip could exist outside of Exynos chips. On imx8mq, they have a NXP display
chip and it is new. Downstream driver exist, and upstream driver is being worked
on.
>
> Best regards,
>
> Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-03 19:37 ` Nicolas Dufresne
@ 2021-12-06 9:20 ` Lucas Stach
2021-12-06 20:46 ` Nicolas Dufresne
0 siblings, 1 reply; 42+ messages in thread
From: Lucas Stach @ 2021-12-06 9:20 UTC (permalink / raw)
To: Nicolas Dufresne, Tim Harvey
Cc: Ezequiel Garcia, Adam Ford, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Joakim Zhang, Alice Guo,
Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Am Freitag, dem 03.12.2021 um 14:37 -0500 schrieb Nicolas Dufresne:
> Le vendredi 03 décembre 2021 à 08:46 -0800, Tim Harvey a écrit :
> > On Thu, Dec 2, 2021 at 8:34 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > >
> > > Le mardi 30 novembre 2021 à 11:28 -0800, Tim Harvey a écrit :
> > > > On Tue, Nov 30, 2021 at 6:00 AM Ezequiel Garcia
> > > > <ezequiel@vanguardiasur.com.ar> wrote:
> > > > >
> > > > > Hi Tim,
> > > > >
> > > > > On Mon, 29 Nov 2021 at 16:36, Tim Harvey <tharvey@gateworks.com> wrote:
> > > > > >
> > > > > > On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
> > > > > ..
> > > > > > >
> > > > > >
> > > > > > Adam,
> > > > > >
> > > > > > What deps did you install in order to get v4l2codecs building? I
> > > > > > installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> > > > > > (not sure if I needed to re-configure somehow) but there is still
> > > > > > nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> > > > > > configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> > > > > > how to find out what dependencies are needed or what may be missing.
> > > > > >
> > > > >
> > > > > At least in my case (Centps-derivative), this is what I've done:
> > > > >
> > > > > ...
> > > > > gst-plugins-bad| Run-time dependency gudev-1.0 found: NO (tried
> > > > > pkgconfig and cmake)
> > > > >
> > > > > Installed gudev ... and then:
> > > > >
> > > > > ...
> > > > > gst-plugins-bad| Dependency gudev-1.0 found: YES 232 (cached)
> > > > > ...
> > > > > gst-plugins-bad 1.19.3.1
> > > > >
> > > > > Plugins : accurip, adpcmdec, adpcmenc, aiff, asfmux,
> > > > > audiobuffersplit, audiofxbad, audiomixmatrix, audiolatency,
> > > > > audiovisualizers, autoconvert, bayer,
> > > > > camerabin, codecalpha, coloreffects,
> > > > > debugutilsbad, dvbsubenc, dvbsuboverlay, dvdspu, faceoverlay,
> > > > > festival, fieldanalysis, freeverb, frei0r,
> > > > > gaudieffects, gdp, geometrictransform,
> > > > > id3tag, inter, interlace, ivfparse, ivtc, jp2kdecimator, jpegformat,
> > > > > rfbsrc, midi, mpegpsdemux,
> > > > > mpegpsmux, mpegtsdemux, mpegtsmux, mxf,
> > > > > netsim, rtponvif, pcapparse, pnm, proxy, legacyrawparse,
> > > > > removesilence, rist, rtmp2, rtpmanagerbad,
> > > > > sdpelem, segmentclip, siren, smooth,
> > > > > speed, subenc, switchbin, timecode, transcode, videofiltersbad,
> > > > > videoframe_audiolevel, videoparsersbad,
> > > > > videosignal, vmnc, y4mdec, decklink, dvb,
> > > > > fbdevsink, ipcpipeline, nvcodec, shm, v4l2codecs, hls, sctp
> > > > >
> > > > > GStreamer current master build fails. It's a known issue which will be
> > > > > fixed today:
> > > > >
> > > > > [...]
> > > > > [8/9] Compiling C object
> > > > > subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > > FAILED: subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > > cc -Isubprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p
> > > > > -Isubprojects/gst-plugins-bad/sys/v4l2codecs
> > > > > -I../subprojects/gst-plugins-bad/sys/v4l2codecs
> > > > > -Isubprojects/gst-plugins-bad -I../subprojects/gst-plugins-bad
> > > > > -Isubprojects/gstreamer/libs -I../subprojects/gstreamer/libs
> > > > > -Isubprojects/gstreamer -I../subprojects/gstreamer
> > > > > -Isubprojects/gst-plugins-bad/gst-libs
> > > > > -I../subprojects/gst-plugins-bad/gst-libs
> > > > > -Isubprojects/gst-plugins-base/gst-libs
> > > > > -I../subprojects/gst-plugins-base/gst-libs -Isubprojects/orc
> > > > > -I../subprojects/orc -Isubprojects/gstreamer/gst
> > > > > -Isubprojects/gst-plugins-base/gst-libs/gst/video
> > > > > -Isubprojects/gst-plugins-base/gst-libs/gst/pbutils
> > > > > -Isubprojects/gst-plugins-base/gst-libs/gst/audio
> > > > > -Isubprojects/gst-plugins-base/gst-libs/gst/tag
> > > > > -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
> > > > > -I/usr/include/gudev-1.0 -fdiagnostics-color=always
> > > > > -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -O2 -g -fvisibility=hidden
> > > > > -fno-strict-aliasing -DG_DISABLE_DEPRECATED -Wmissing-prototypes
> > > > > -Wdeclaration-after-statement -Wold-style-definition
> > > > > -Wmissing-declarations -Wredundant-decls -Wwrite-strings -Wformat
> > > > > -Wformat-security -Winit-self -Wmissing-include-dirs -Waddress
> > > > > -Wno-multichar -Wvla -Wpointer-arith -fPIC -pthread -DHAVE_CONFIG_H
> > > > > -MD -MQ subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > > -MF subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o.d
> > > > > -o subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > > -c ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c
> > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:92:3:
> > > > > error: unknown type name ‘grefcount’
> > > > > grefcount ref_count;
> > > > > ^~~~~~~~~
> > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_new’:
> > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:106:3:
> > > > > warning: implicit declaration of function ‘g_ref_count_init’; did you
> > > > > mean ‘g_cond_init’? [-Wimplicit-function-declaration]
> > > > > g_ref_count_init (&pic_data->ref_count);
> > > > > ^~~~~~~~~~~~~~~~
> > > > > g_cond_init
> > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_ref’:
> > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:118:3:
> > > > > warning: implicit declaration of function ‘g_ref_count_inc’; did you
> > > > > mean ‘g_strv_contains’? [-Wimplicit-function-declaration]
> > > > > g_ref_count_inc (&data->ref_count);
> > > > > ^~~~~~~~~~~~~~~
> > > > > g_strv_contains
> > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_unref’:
> > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:125:7:
> > > > > warning: implicit declaration of function ‘g_ref_count_dec’
> > > > > [-Wimplicit-function-declaration]
> > > > > if (g_ref_count_dec (&data->ref_count)) {
> > > > > ^~~~~~~~~~~~~~~
> > > > > ninja: build stopped: subcommand failed.
> > > > >
> > > > > Hope this helps get you started!
> > > > > Ezequiel
> > > >
> > > > Ezequiel and Nicolas,
> > > >
> > > > Thanks - I did manage to get gstreamer 1.19.3 built successfully with
> > > > v4l2codecs finally by getting the correct dependencies. I've attempted
> > > > to software encode from another system and decode/display on the IMX8M
> > > > Mini but thus far have not been successful.
> > > >
> > > > I see that v4l2codecs plugin v4l2slh264dec/v4l2slmpeg2dec/v4l2slvp8dec
> > > > and these all can output video/x-raw NV12/YUY2 which kmssink should
> > > > accept so I'm attempting the following :
> > > >
> > > > # vp8 encode from x86
> > > > gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
> > > > vp8enc ! rtpvp8pay ! udpsink host=172.24.33.15 port=9001
> > > > # vp8 decode on imx8mm@172.24.33.15 which has a 800x480 display
> > > > [gst-main] root@focal-venice:~/gstreamer/build# gst-launch-1.0 -v
> > > > udpsrc port=9001 caps = "application/x-rtp, media=(string)video,
> > > > clock-rate=(int)90000, encoding-name=(string)VP8, payload=(int)96,
> > > > ssrc=(uint)2745262155, timestamp-offset=(uint)2515032683,
> > > > seqnum-offset=(uint)19579, a-framerate=(string)30" ! rtpvp8depay !
> > > > v4l2slvp8dec ! kmssink
> > > > Setting pipeline to PAUSED ...
> > > > Pipeline is live and does not need PREROLL ...
> > > > /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-width = 800
> > > > /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-height = 480
> > > > Pipeline is PREROLLED ...
> > > > Setting pipeline to PLAYING ...
> > > > /GstPipeline:pipeline0/GstUDPSrc:udpsrc0.GstPad:src: caps =
> > > > application/x-rtp, media=(string)video, clock-rate=(int)90000,
> > > > encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> > > > timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> > > > a-framerate=(string)30
> > > > New clock: GstSystemClock
> > > > /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:sink: caps =
> > > > application/x-rtp, media=(string)video, clock-rate=(int)90000,
> > > > encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> > > > timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> > > > a-framerate=(string)30
> > > > /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:src: caps =
> > > > video/x-vp8, framerate=(fraction)0/1, height=(int)480, width=(int)800,
> > > > profile=(string)0
> > > > ERROR: from element /GstPipeline:pipeline0/GstUDPSrc:udpsrc0: Internal
> > > > data stream error.
> > > > Additional debug info:
> > > > ../subprojects/gstreamer/libs/gst/base/gstbasesrc.c(3127):
> > > > gst_base_src_loop (): /GstPipeline:pipeline0/GstUDPSrc:udpsrc0:
> > > > streaming stopped, reason not-negotiated (-4)
> > > > Execution ended after 0:00:02.076839644
> > > > Setting pipeline to NULL ...
> > > > Freeing pipeline ...
> > > >
> > > > I'm getting the same thing when trying to use h264.
> > > >
> > > > I've never quite been able to grasp how to debug GStreamer's
> > > > negotiation issues. If I end with fakesink it appears to decode so it
> > > > must be the v4l2slvp8dec to kmssink. I tried forcing the pixel format
> > > > using 'v4l2slvp8dec ! "video/x-raw,format=(string)NV12" ! kmssink' but
> > > > I still get the negotiation error.
> > > >
> > > > What interrupts should I be seeing in /proc/interrupts? I don't see
> > > > anything vpu/hantro related there.
> > > >
> > > > I also want to make sure I have a basic understanding of the vpu
> > > > drivers and usersapce on the IMX8M Mini. The IMX6Q/DL that I'm more
> > > > familiar with has a vpu that is supported by the GStreamer video4linux
> > > > plugin which shows the following (on GStreamer 1.16.2):
> > > > v4l2jpegenc: V4L2 JPEG Encoder
> > > > v4l2jpegdec: V4L2 JPEG Decoder
> > > > v4l2h264enc: V4L2 H.264 Encoder
> > > > v4l2mpeg4enc: V4L2 MPEG4 Encoder
> > > > v4l2mpeg4dec: V4L2 MPEG4 Decoder
> > > > v4l2mpeg2dec: V4L2 MPEG2 Decoder
> > > > v4l2h264dec: V4L2 H264 Decoder
> > > > The IMX6Q/DL also has an IPU that has an M2M driver that provides the
> > > > following for scaling/colorspace conversion:
> > > > v4l2convert: V4L2 Video Converter
> > > >
> > > > I believe what I'm reading is that the IMX8M Mini Hantro codecs are
> > > > 'stateful' where more software is required to drive them and is
> > >
> > > 'stateless'. the rest is right.
> > >
> > > > supported by the newer v4l2codecs plugin. I haven't been able to
> > > > understand what kernel version/requirements the v4l2codecs plugin
> > > > users/requires.
> > >
> > > After H264 debacle with 5.9, 5.10 and 5.11 API break and GStreamer not getting
> > > enough release to support all of these we started merging support for CODECs
> > > only when the stable uAPI land. I made an exception for VP9 as it is already
> > > applied in the media tree and didn't want to miss 1.20 release.
> > >
> > > So to answer you question, it depends on when the CODEC uAPI landed.
> > >
> >
> > Ok, thanks for the explanation.
> >
> > > >
> > > > I'm also trying to understand how we can get scaling/colorspace
> > > > conversion on the IMX8M Mini. The IMX8M lacks an IPU... is there some
> > > > way to utilize scaling/colorspace conversion from the 2D GPU bound to
> > > > the etnaviv driver?
> > >
> > > The concept of the mini, is that you would be using th encoder for anything that
> > > isn't going to the display. So they only integrated the Hantro PP on the
> > > encoder. Unfortunately, you'll have to be patient for mainline stateless encoder
> > > support, we barely scratch the surface of this subject, but its being worked on.
> >
> > After some searching for Hantro PP I see that the IMXMQ (IMX8M
> > Dual/QuadLite/Quad) mentions Hantro PP. From the IMX8MDQLQRM section
> > 14.1.2.1 Decoder Features:
> > <quote>
> > Video post-processing features
> > - Frame rotation 90 degrees left/right
> > - Frame mirroring horizontally/vertically
> > - Frame cropping
> > - Frame conversion from YCbCr formats to 16-bit or 32-bit RGB formats
> > - Frame scaling with maximum up-scaling factor of 3
> > - Two rectangular or alpha blending masks for output frame
> >
> > The post-processing features can be used in pipeline with the
> > decoder. The postprocessing features can also be used as stand-alone,
> > without performing any decoding
> > </quote>
> >
> > The above is under the VPU_G1 section and the same is not mentioned
> > for VPU_G2 and the IMX8MQ doesn't have encode support. Where do you
> > see that the IMX8MM has the Hantro PP on the H1? I know the TRM's lack
> > a lot of info so perhaps you know more about the internals than what
> > the TRM states.
>
> I've got told that by someone with contacts at NXP recently (in IRC). I haven't
> verified it though, it just made sense for the targeted use of th mini. Hantro
> G1 driver does not yet expose an M2M for the standalone mode of the PP, but
> shall be possible. Decode an PP cannot run concurrently though, so concurrent PP
> and decode will have big impact on performance.
>
> The G2 PP is different, but I *think* its always there. It's not doing much,
> linear NV12 (detiling from 4x4 linear tiles), and can scale down by factor of 2,
> 4 an 8. If there is more feature I'm not aware.
>
> >
> > I also found on a forum
> > (https://community.nxp.com/t5/i-MX-Processors/imx8mq-Hantro-G1-scaling/m-p/1285343)
> > that NXP's BSP doesn't use the Hantro for scaling (and likely not csc
> > either) and they use the GPU instead. I'm still unclear if/how you
> > could tap into the 2D GPU to use its scaling/conversion if it's bound
> > to the etnaviv driver.
> >
> > > Unlike the IMX8MQ, you don't have the option to output YUYV (packed yuv 4:2:2)
> > > which would satisfy the 2D GPU support hoold to the DMABuf import path.
>
> As I'm saying above, you can't, there is no NV12 support in that 2D GPU from
> what I was old by Etnaviv folks, only YUYV (4:2:2 packed). Shaders is the only
> option.
>
That's not correct. The 3D GPU can sample directly from YUYV, so it's
the optimal zero-copy format if you are going to use the video with the
3D GPU. Most of the 2D GPU cores (and I think the one of the 8MM is no
exception) can handle planar formats, some of them only at reduced rate
of 1 Pixel/clock, instead of the usual 2 Pixels/clock.
> > >
> > > When the display driver gets ready and upstream (it's been only 2-3 years now),
> > > you'll get linear NV12 support along with G2 compression support (this one is
> > > not supported by the GPU, so it will be tricky to expose in userland). I don't
> > > think the display support 4L4 tiles, but you GPU most likely can do with the
> > > right shared and texelFetch() or vulkan equivalent if that exist on that target.
> >
> > Do you mean the Samsung Exynos DRM driver (which doesn't yet have
> > support for IMX8MM) or drivers/gpu/drm/mxsfb?
> >
> > I'm currently using a pretty old patchset that adds IMX8MM support to
> > the drm/exynos driver. I'm way out of my realm when talking about
> > GPU/VPU and display drivers.
>
> Didn't know the mini was using Samsung display controller. Didn't even know that
> chip could exist outside of Exynos chips. On imx8mq, they have a NXP display
> chip and it is new. Downstream driver exist, and upstream driver is being worked
> on.
The display controller on the 8MM is derived from the existing simple
eLCDIF controllers found on earlier i.MX SoCs. 8MQ is the only one with
the very capable DCSS display controller. In fact the 8MQ DCSS is
upstream, what is missing is the Cadence eDP/HDMI bridge driver. The
MIPI-DSI controller and PHY on the 8MM is Samsung IP.
Regards,
Lucas
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-12-06 9:20 ` Lucas Stach
@ 2021-12-06 20:46 ` Nicolas Dufresne
0 siblings, 0 replies; 42+ messages in thread
From: Nicolas Dufresne @ 2021-12-06 20:46 UTC (permalink / raw)
To: Lucas Stach, Tim Harvey
Cc: Ezequiel Garcia, Adam Ford, linux-media, Schrempf Frieder,
Marek Vasut, Jagan Teki, Adam Ford-BE, cstevens, Philipp Zabel,
Mauro Carvalho Chehab, Rob Herring, Shawn Guo, Sascha Hauer,
Pengutronix Kernel Team, Fabio Estevam, NXP Linux Team,
Greg Kroah-Hartman, Heiko Stuebner, Joakim Zhang, Alice Guo,
Peng Fan, open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le lundi 06 décembre 2021 à 10:20 +0100, Lucas Stach a écrit :
> Am Freitag, dem 03.12.2021 um 14:37 -0500 schrieb Nicolas Dufresne:
> > Le vendredi 03 décembre 2021 à 08:46 -0800, Tim Harvey a écrit :
> > > On Thu, Dec 2, 2021 at 8:34 PM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > >
> > > > Le mardi 30 novembre 2021 à 11:28 -0800, Tim Harvey a écrit :
> > > > > On Tue, Nov 30, 2021 at 6:00 AM Ezequiel Garcia
> > > > > <ezequiel@vanguardiasur.com.ar> wrote:
> > > > > >
> > > > > > Hi Tim,
> > > > > >
> > > > > > On Mon, 29 Nov 2021 at 16:36, Tim Harvey <tharvey@gateworks.com> wrote:
> > > > > > >
> > > > > > > On Mon, Nov 29, 2021 at 10:59 AM Adam Ford <aford173@gmail.com> wrote:
> > > > > > ..
> > > > > > > >
> > > > > > >
> > > > > > > Adam,
> > > > > > >
> > > > > > > What deps did you install in order to get v4l2codecs building? I
> > > > > > > installed libgudev-1.0-dev based on Nicolas' suggestion and rebuilt
> > > > > > > (not sure if I needed to re-configure somehow) but there is still
> > > > > > > nothing in build/subprojects/gst-plugins-bad/sys/v4l2codecs/. A 'meson
> > > > > > > configure' tells me that v4l2codecs is set to 'auto' but I'm not sure
> > > > > > > how to find out what dependencies are needed or what may be missing.
> > > > > > >
> > > > > >
> > > > > > At least in my case (Centps-derivative), this is what I've done:
> > > > > >
> > > > > > ...
> > > > > > gst-plugins-bad| Run-time dependency gudev-1.0 found: NO (tried
> > > > > > pkgconfig and cmake)
> > > > > >
> > > > > > Installed gudev ... and then:
> > > > > >
> > > > > > ...
> > > > > > gst-plugins-bad| Dependency gudev-1.0 found: YES 232 (cached)
> > > > > > ...
> > > > > > gst-plugins-bad 1.19.3.1
> > > > > >
> > > > > > Plugins : accurip, adpcmdec, adpcmenc, aiff, asfmux,
> > > > > > audiobuffersplit, audiofxbad, audiomixmatrix, audiolatency,
> > > > > > audiovisualizers, autoconvert, bayer,
> > > > > > camerabin, codecalpha, coloreffects,
> > > > > > debugutilsbad, dvbsubenc, dvbsuboverlay, dvdspu, faceoverlay,
> > > > > > festival, fieldanalysis, freeverb, frei0r,
> > > > > > gaudieffects, gdp, geometrictransform,
> > > > > > id3tag, inter, interlace, ivfparse, ivtc, jp2kdecimator, jpegformat,
> > > > > > rfbsrc, midi, mpegpsdemux,
> > > > > > mpegpsmux, mpegtsdemux, mpegtsmux, mxf,
> > > > > > netsim, rtponvif, pcapparse, pnm, proxy, legacyrawparse,
> > > > > > removesilence, rist, rtmp2, rtpmanagerbad,
> > > > > > sdpelem, segmentclip, siren, smooth,
> > > > > > speed, subenc, switchbin, timecode, transcode, videofiltersbad,
> > > > > > videoframe_audiolevel, videoparsersbad,
> > > > > > videosignal, vmnc, y4mdec, decklink, dvb,
> > > > > > fbdevsink, ipcpipeline, nvcodec, shm, v4l2codecs, hls, sctp
> > > > > >
> > > > > > GStreamer current master build fails. It's a known issue which will be
> > > > > > fixed today:
> > > > > >
> > > > > > [...]
> > > > > > [8/9] Compiling C object
> > > > > > subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > > > FAILED: subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > > > cc -Isubprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p
> > > > > > -Isubprojects/gst-plugins-bad/sys/v4l2codecs
> > > > > > -I../subprojects/gst-plugins-bad/sys/v4l2codecs
> > > > > > -Isubprojects/gst-plugins-bad -I../subprojects/gst-plugins-bad
> > > > > > -Isubprojects/gstreamer/libs -I../subprojects/gstreamer/libs
> > > > > > -Isubprojects/gstreamer -I../subprojects/gstreamer
> > > > > > -Isubprojects/gst-plugins-bad/gst-libs
> > > > > > -I../subprojects/gst-plugins-bad/gst-libs
> > > > > > -Isubprojects/gst-plugins-base/gst-libs
> > > > > > -I../subprojects/gst-plugins-base/gst-libs -Isubprojects/orc
> > > > > > -I../subprojects/orc -Isubprojects/gstreamer/gst
> > > > > > -Isubprojects/gst-plugins-base/gst-libs/gst/video
> > > > > > -Isubprojects/gst-plugins-base/gst-libs/gst/pbutils
> > > > > > -Isubprojects/gst-plugins-base/gst-libs/gst/audio
> > > > > > -Isubprojects/gst-plugins-base/gst-libs/gst/tag
> > > > > > -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include
> > > > > > -I/usr/include/gudev-1.0 -fdiagnostics-color=always
> > > > > > -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -O2 -g -fvisibility=hidden
> > > > > > -fno-strict-aliasing -DG_DISABLE_DEPRECATED -Wmissing-prototypes
> > > > > > -Wdeclaration-after-statement -Wold-style-definition
> > > > > > -Wmissing-declarations -Wredundant-decls -Wwrite-strings -Wformat
> > > > > > -Wformat-security -Winit-self -Wmissing-include-dirs -Waddress
> > > > > > -Wno-multichar -Wvla -Wpointer-arith -fPIC -pthread -DHAVE_CONFIG_H
> > > > > > -MD -MQ subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > > > -MF subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o.d
> > > > > > -o subprojects/gst-plugins-bad/sys/v4l2codecs/libgstv4l2codecs.so.p/gstv4l2codecvp9dec.c.o
> > > > > > -c ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c
> > > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:92:3:
> > > > > > error: unknown type name ‘grefcount’
> > > > > > grefcount ref_count;
> > > > > > ^~~~~~~~~
> > > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_new’:
> > > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:106:3:
> > > > > > warning: implicit declaration of function ‘g_ref_count_init’; did you
> > > > > > mean ‘g_cond_init’? [-Wimplicit-function-declaration]
> > > > > > g_ref_count_init (&pic_data->ref_count);
> > > > > > ^~~~~~~~~~~~~~~~
> > > > > > g_cond_init
> > > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_ref’:
> > > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:118:3:
> > > > > > warning: implicit declaration of function ‘g_ref_count_inc’; did you
> > > > > > mean ‘g_strv_contains’? [-Wimplicit-function-declaration]
> > > > > > g_ref_count_inc (&data->ref_count);
> > > > > > ^~~~~~~~~~~~~~~
> > > > > > g_strv_contains
> > > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c: In
> > > > > > function ‘gst_v4l2_codec_vp9_dec_picture_data_unref’:
> > > > > > ../subprojects/gst-plugins-bad/sys/v4l2codecs/gstv4l2codecvp9dec.c:125:7:
> > > > > > warning: implicit declaration of function ‘g_ref_count_dec’
> > > > > > [-Wimplicit-function-declaration]
> > > > > > if (g_ref_count_dec (&data->ref_count)) {
> > > > > > ^~~~~~~~~~~~~~~
> > > > > > ninja: build stopped: subcommand failed.
> > > > > >
> > > > > > Hope this helps get you started!
> > > > > > Ezequiel
> > > > >
> > > > > Ezequiel and Nicolas,
> > > > >
> > > > > Thanks - I did manage to get gstreamer 1.19.3 built successfully with
> > > > > v4l2codecs finally by getting the correct dependencies. I've attempted
> > > > > to software encode from another system and decode/display on the IMX8M
> > > > > Mini but thus far have not been successful.
> > > > >
> > > > > I see that v4l2codecs plugin v4l2slh264dec/v4l2slmpeg2dec/v4l2slvp8dec
> > > > > and these all can output video/x-raw NV12/YUY2 which kmssink should
> > > > > accept so I'm attempting the following :
> > > > >
> > > > > # vp8 encode from x86
> > > > > gst-launch-1.0 -v videotestsrc ! video/x-raw,width=800,height=480 !
> > > > > vp8enc ! rtpvp8pay ! udpsink host=172.24.33.15 port=9001
> > > > > # vp8 decode on imx8mm@172.24.33.15 which has a 800x480 display
> > > > > [gst-main] root@focal-venice:~/gstreamer/build# gst-launch-1.0 -v
> > > > > udpsrc port=9001 caps = "application/x-rtp, media=(string)video,
> > > > > clock-rate=(int)90000, encoding-name=(string)VP8, payload=(int)96,
> > > > > ssrc=(uint)2745262155, timestamp-offset=(uint)2515032683,
> > > > > seqnum-offset=(uint)19579, a-framerate=(string)30" ! rtpvp8depay !
> > > > > v4l2slvp8dec ! kmssink
> > > > > Setting pipeline to PAUSED ...
> > > > > Pipeline is live and does not need PREROLL ...
> > > > > /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-width = 800
> > > > > /GstPipeline:pipeline0/GstKMSSink:kmssink0: display-height = 480
> > > > > Pipeline is PREROLLED ...
> > > > > Setting pipeline to PLAYING ...
> > > > > /GstPipeline:pipeline0/GstUDPSrc:udpsrc0.GstPad:src: caps =
> > > > > application/x-rtp, media=(string)video, clock-rate=(int)90000,
> > > > > encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> > > > > timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> > > > > a-framerate=(string)30
> > > > > New clock: GstSystemClock
> > > > > /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:sink: caps =
> > > > > application/x-rtp, media=(string)video, clock-rate=(int)90000,
> > > > > encoding-name=(string)VP8, payload=(int)96, ssrc=(uint)2745262155,
> > > > > timestamp-offset=(uint)2515032683, seqnum-offset=(uint)19579,
> > > > > a-framerate=(string)30
> > > > > /GstPipeline:pipeline0/GstRtpVP8Depay:rtpvp8depay0.GstPad:src: caps =
> > > > > video/x-vp8, framerate=(fraction)0/1, height=(int)480, width=(int)800,
> > > > > profile=(string)0
> > > > > ERROR: from element /GstPipeline:pipeline0/GstUDPSrc:udpsrc0: Internal
> > > > > data stream error.
> > > > > Additional debug info:
> > > > > ../subprojects/gstreamer/libs/gst/base/gstbasesrc.c(3127):
> > > > > gst_base_src_loop (): /GstPipeline:pipeline0/GstUDPSrc:udpsrc0:
> > > > > streaming stopped, reason not-negotiated (-4)
> > > > > Execution ended after 0:00:02.076839644
> > > > > Setting pipeline to NULL ...
> > > > > Freeing pipeline ...
> > > > >
> > > > > I'm getting the same thing when trying to use h264.
> > > > >
> > > > > I've never quite been able to grasp how to debug GStreamer's
> > > > > negotiation issues. If I end with fakesink it appears to decode so it
> > > > > must be the v4l2slvp8dec to kmssink. I tried forcing the pixel format
> > > > > using 'v4l2slvp8dec ! "video/x-raw,format=(string)NV12" ! kmssink' but
> > > > > I still get the negotiation error.
> > > > >
> > > > > What interrupts should I be seeing in /proc/interrupts? I don't see
> > > > > anything vpu/hantro related there.
> > > > >
> > > > > I also want to make sure I have a basic understanding of the vpu
> > > > > drivers and usersapce on the IMX8M Mini. The IMX6Q/DL that I'm more
> > > > > familiar with has a vpu that is supported by the GStreamer video4linux
> > > > > plugin which shows the following (on GStreamer 1.16.2):
> > > > > v4l2jpegenc: V4L2 JPEG Encoder
> > > > > v4l2jpegdec: V4L2 JPEG Decoder
> > > > > v4l2h264enc: V4L2 H.264 Encoder
> > > > > v4l2mpeg4enc: V4L2 MPEG4 Encoder
> > > > > v4l2mpeg4dec: V4L2 MPEG4 Decoder
> > > > > v4l2mpeg2dec: V4L2 MPEG2 Decoder
> > > > > v4l2h264dec: V4L2 H264 Decoder
> > > > > The IMX6Q/DL also has an IPU that has an M2M driver that provides the
> > > > > following for scaling/colorspace conversion:
> > > > > v4l2convert: V4L2 Video Converter
> > > > >
> > > > > I believe what I'm reading is that the IMX8M Mini Hantro codecs are
> > > > > 'stateful' where more software is required to drive them and is
> > > >
> > > > 'stateless'. the rest is right.
> > > >
> > > > > supported by the newer v4l2codecs plugin. I haven't been able to
> > > > > understand what kernel version/requirements the v4l2codecs plugin
> > > > > users/requires.
> > > >
> > > > After H264 debacle with 5.9, 5.10 and 5.11 API break and GStreamer not getting
> > > > enough release to support all of these we started merging support for CODECs
> > > > only when the stable uAPI land. I made an exception for VP9 as it is already
> > > > applied in the media tree and didn't want to miss 1.20 release.
> > > >
> > > > So to answer you question, it depends on when the CODEC uAPI landed.
> > > >
> > >
> > > Ok, thanks for the explanation.
> > >
> > > > >
> > > > > I'm also trying to understand how we can get scaling/colorspace
> > > > > conversion on the IMX8M Mini. The IMX8M lacks an IPU... is there some
> > > > > way to utilize scaling/colorspace conversion from the 2D GPU bound to
> > > > > the etnaviv driver?
> > > >
> > > > The concept of the mini, is that you would be using th encoder for anything that
> > > > isn't going to the display. So they only integrated the Hantro PP on the
> > > > encoder. Unfortunately, you'll have to be patient for mainline stateless encoder
> > > > support, we barely scratch the surface of this subject, but its being worked on.
> > >
> > > After some searching for Hantro PP I see that the IMXMQ (IMX8M
> > > Dual/QuadLite/Quad) mentions Hantro PP. From the IMX8MDQLQRM section
> > > 14.1.2.1 Decoder Features:
> > > <quote>
> > > Video post-processing features
> > > - Frame rotation 90 degrees left/right
> > > - Frame mirroring horizontally/vertically
> > > - Frame cropping
> > > - Frame conversion from YCbCr formats to 16-bit or 32-bit RGB formats
> > > - Frame scaling with maximum up-scaling factor of 3
> > > - Two rectangular or alpha blending masks for output frame
> > >
> > > The post-processing features can be used in pipeline with the
> > > decoder. The postprocessing features can also be used as stand-alone,
> > > without performing any decoding
> > > </quote>
> > >
> > > The above is under the VPU_G1 section and the same is not mentioned
> > > for VPU_G2 and the IMX8MQ doesn't have encode support. Where do you
> > > see that the IMX8MM has the Hantro PP on the H1? I know the TRM's lack
> > > a lot of info so perhaps you know more about the internals than what
> > > the TRM states.
> >
> > I've got told that by someone with contacts at NXP recently (in IRC). I haven't
> > verified it though, it just made sense for the targeted use of th mini. Hantro
> > G1 driver does not yet expose an M2M for the standalone mode of the PP, but
> > shall be possible. Decode an PP cannot run concurrently though, so concurrent PP
> > and decode will have big impact on performance.
> >
> > The G2 PP is different, but I *think* its always there. It's not doing much,
> > linear NV12 (detiling from 4x4 linear tiles), and can scale down by factor of 2,
> > 4 an 8. If there is more feature I'm not aware.
> >
> > >
> > > I also found on a forum
> > > (https://community.nxp.com/t5/i-MX-Processors/imx8mq-Hantro-G1-scaling/m-p/1285343)
> > > that NXP's BSP doesn't use the Hantro for scaling (and likely not csc
> > > either) and they use the GPU instead. I'm still unclear if/how you
> > > could tap into the 2D GPU to use its scaling/conversion if it's bound
> > > to the etnaviv driver.
> > >
> > > > Unlike the IMX8MQ, you don't have the option to output YUYV (packed yuv 4:2:2)
> > > > which would satisfy the 2D GPU support hoold to the DMABuf import path.
> >
> > As I'm saying above, you can't, there is no NV12 support in that 2D GPU from
> > what I was old by Etnaviv folks, only YUYV (4:2:2 packed). Shaders is the only
> > option.
> >
> That's not correct. The 3D GPU can sample directly from YUYV, so it's
> the optimal zero-copy format if you are going to use the video with the
> 3D GPU. Most of the 2D GPU cores (and I think the one of the 8MM is no
> exception) can handle planar formats, some of them only at reduced rate
> of 1 Pixel/clock, instead of the usual 2 Pixels/clock.
Isn't that enough to disqualify that as "optimal" ?
> > > >
> > > > When the display driver gets ready and upstream (it's been only 2-3 years now),
> > > > you'll get linear NV12 support along with G2 compression support (this one is
> > > > not supported by the GPU, so it will be tricky to expose in userland). I don't
> > > > think the display support 4L4 tiles, but you GPU most likely can do with the
> > > > right shared and texelFetch() or vulkan equivalent if that exist on that target.
> > >
> > > Do you mean the Samsung Exynos DRM driver (which doesn't yet have
> > > support for IMX8MM) or drivers/gpu/drm/mxsfb?
> > >
> > > I'm currently using a pretty old patchset that adds IMX8MM support to
> > > the drm/exynos driver. I'm way out of my realm when talking about
> > > GPU/VPU and display drivers.
> >
> > Didn't know the mini was using Samsung display controller. Didn't even know that
> > chip could exist outside of Exynos chips. On imx8mq, they have a NXP display
> > chip and it is new. Downstream driver exist, and upstream driver is being worked
> > on.
>
> The display controller on the 8MM is derived from the existing simple
> eLCDIF controllers found on earlier i.MX SoCs. 8MQ is the only one with
> the very capable DCSS display controller. In fact the 8MQ DCSS is
> upstream, what is missing is the Cadence eDP/HDMI bridge driver. The
> MIPI-DSI controller and PHY on the 8MM is Samsung IP.
Thanks for clarification.
>
> Regards,
> Lucas
>
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-18 16:20 ` Tim Harvey
2021-11-18 18:16 ` Adam Ford
@ 2021-11-23 0:06 ` Tim Harvey
2021-11-23 20:10 ` Nicolas Dufresne
1 sibling, 1 reply; 42+ messages in thread
From: Tim Harvey @ 2021-11-23 0:06 UTC (permalink / raw)
To: Nicolas Dufresne
Cc: Adam Ford, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
On Thu, Nov 18, 2021 at 8:20 AM Tim Harvey <tharvey@gateworks.com> wrote:
>
> On Thu, Nov 18, 2021 at 6:30 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> >
> > Le mardi 16 novembre 2021 à 15:23 -0800, Tim Harvey a écrit :
> > > On Tue, Nov 9, 2021 at 7:57 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > >
> > > > Le lundi 08 novembre 2021 à 10:33 -0600, Adam Ford a écrit :
> > > > > On Mon, Nov 8, 2021 at 7:59 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > > >
> > > > > > Hi Adam,
> > > > > >
> > > > > > thanks for you work, I'll try and reply about the GStreamer questions below, if
> > > > > > you have further question feel free to ask.
> > > > > >
> > > > > > Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> > > > > > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > > > > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > > > > > how the Mini handles the power domains, the VPU driver does not need to
> > > > > > > handle all the functions, so a new compatible flag is required.
> > > > > > >
> > > > > > > This is an RFC because I don't have functional video on my system yet,
> > > > > > > and I'm hoping there might be people who do and can help test this.
> > > > > > > I have only tested this far enough to see they enumerate and appear
> > > > > > > as /dev/videoX and /dev/mediaX devices.
> > > > > >
> > > > > > I will check the patchset, but you need in the mini-variant to disable the G1
> > > > > > post processor, because this block was fused out. We didn't make it optional
> > > > >
> > > > > Thanks for being willing to review this.
> > > > >
> > > > > > from the start as according to the V1 of the TRM it was there, but that error
> > > > > > was corrected in V3.
> > > > >
> > > > > Thanks for the clarification. It wasn't obvious to me, because in
> > > > > some instances the PP looked like it was there and sometimes not
> > > > > there. I'll remove the postproc stuff.
> > > > >
> > > > > >
> > > > > > >
> > > > > > > I am also curious to know if/what gstreamer plugins are necessary. In
> > > > > > > NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> > > > > > > would be more generic plugins that I can use to test. I am hoping some
> > > > > > > of the linux-media experts might chime in on how to best test.
> > > > > >
> > > > > > I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
> > > > > > single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
> > > > > > H264 and VP8. Remember that the related plugin depends on libgudev (a glib
> > > > > > binding of udev).
> > > > >
> > > > > Thanks for the tip.
> > > > >
> > > > > >
> > > > > > For the encoder, I believe only JPEG maybe supported, since this is all there is
> > > > > > mainline for RK3288 (and perhaps other RK). But this will need testing and
> > > > > > debugging as the G1 version is slightly newer on NXP SoC.
> > > > >
> > > > > For what it's worth the G1 seems to repond cleanly to the inquiries
> > > > > from v42-compliance.
> > > > > The G2 throws some splat when I run v4l2-compliance, but I am still
> > > > > investigating that.
> > > > >
> > > > > [ 405.456979] ------------[ cut here ]------------
> > > > > [ 405.464173] WARNING: CPU: 0 PID: 563 at mm/page_alloc.c:5344
> > > > > __alloc_pages+0x5a4/0xbe0
> > > > > [ 405.472104] Modules linked in: 8021q garp mrp stp llc caam_jr
> > > > > caamhash_desc caamalg_desc crypto_engine rng_core authenc libdes
> > > > > imx7_media_csi(C) crct10dif_ce imx_media_common(C)
> > > > > snd_soc_fsl_asoc_card imx7_mipi_csis(C) snd_soc_imx_audmux
> > > > > snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc brcmfmac
> > > > > brcmutil hantro_vpu(C) v4l2_h264 v4l2_mem2mem videobuf2_vmalloc
> > > > > videobuf2_dma_contig videobuf2_memops cfg80211 ov5640 videobuf2_v4l2
> > > > > v4l2_fwnode v4l2_async videobuf2_common videodev etnaviv gpu_sched
> > > > > hci_uart mc btqca btbcm snd_soc_wm8962 at24 spi_imx rtc_pcf85363
> > > > > rtc_snvs clk_bd718x7 spi_bitbang snvs_pwrkey snd_soc_fsl_sai
> > > > > imx_pcm_dma caam error bluetooth imx8mm_thermal ecdh_generic
> > > > > imx_cpufreq_dt ecc rfkill fuse drm ipv6
> > > > > [ 405.535835] CPU: 0 PID: 563 Comm: v4l2-compliance Tainted: G D
> > > > > C 5.15.0-next-20211105-00010-g4bb8e8a25d3c-dirty #28
> > > > > [ 405.547401] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
> > > > > Development Kit (DT)
> > > > > [ 405.554797] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> > > > > [ 405.561762] pc : __alloc_pages+0x5a4/0xbe0
> > > > > [ 405.565861] lr : __dma_direct_alloc_pages+0x17c/0x1e0
> > > > > [ 405.570917] sp : ffff800012443810
> > > > > [ 405.574232] x29: ffff800012443810 x28: 0000000000000000 x27: ffff000005288220
> > > > > [ 405.581375] x26: 0000000000000034 x25: 0000000000000000 x24: ffff000000259010
> > > > > [ 405.588517] x23: ffff80001011ab7c x22: ffff000000259010 x21: 00000000ffffffff
> > > > > [ 405.595659] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
> > > > > [ 405.602803] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
> > > > > [ 405.609947] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
> > > > > [ 405.617090] x11: ffff80001241d000 x10: ffff00000528833a x9 : ffff00000528832a
> > > > > [ 405.624232] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
> > > > > [ 405.631378] x5 : 00000000bfffffff x4 : ffff000009e30dc0 x3 : 0000000000000000
> > > > > [ 405.638520] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
> > > > > [ 405.645666] Call trace:
> > > > > [ 405.648113] __alloc_pages+0x5a4/0xbe0
> > > > > [ 405.651862] __dma_direct_alloc_pages+0x17c/0x1e0
> > > > > [ 405.656569] dma_direct_alloc+0x70/0x310
> > > > > [ 405.660494] dma_alloc_attrs+0x7c/0xe4
> > > > > [ 405.664246] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
> > > > > [ 405.670021] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
> > > > > [ 405.675791] device_run+0xac/0x110 [hantro_vpu]
> > > > > [ 405.680345] v4l2_m2m_try_run+0xac/0x1b0 [v4l2_mem2mem]
> > > > > [ 405.685598] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
> > > > > [ 405.691366] v4l_streamon+0x28/0x34 [videodev]
> > > > > [ 405.695877] __video_do_ioctl+0x178/0x3dc [videodev]
> > > > > [ 405.700897] video_usercopy+0x368/0x6dc [videodev]
> > > > > [ 405.705745] video_ioctl2+0x1c/0x30 [videodev]
> > > > > [ 405.710246] v4l2_ioctl+0x44/0x64 [videodev]
> > > > > [ 405.714574] __arm64_sys_ioctl+0xac/0xf0
> > > > > [ 405.718502] invoke_syscall+0x48/0x114
> > > > > [ 405.722258] el0_svc_common.constprop.0+0xd4/0xfc
> > > > > [ 405.726969] do_el0_svc+0x2c/0x94
> > > > > [ 405.730286] el0_svc+0x28/0x80
> > > > > [ 405.733348] el0t_64_sync_handler+0xa8/0x130
> > > > > [ 405.737619] el0t_64_sync+0x1a0/0x1a4
> > > > > [ 405.741287] ---[ end trace 270ed4a899803006 ]---
> > > > >
> > > > > The H1 encoder seems to hang the system when I run v4l2-compliance on
> > > > > it when H1 is set up as I submitted the patch. I tried dropping all
> > > > > the encoder formats except the JPEG format, and it doesn't hang any
> > > > > more, but it also doesn't really do anything.
> > > > > The datasheet only references VPU_H1 as supporting VP8 and H.264, so I
> > > > > am not sure JPEG is even supported.
> > > >
> > > > If JPEG is not supported, then there is nothing left for mainline in this
> > > > regard. The kernel control interface and encoding flow needs to be designed and
> > > > specified for encoders like VP8 and H264. Some prototypes and prior-art exist
> > > > though, but nothing ever got formalized in the form of a specification.
> > > >
> > > > >
> > > > > The log from v4l2-compliance on the H1 with everything except the JPEG
> > > > > removed looks like:
> > > > >
> > > > > root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> > > > > v4l2-compliance SHA: not available
> > > > > , 64 bits, 64-bit time_t
> > > > >
> > > > > Segmentation fault
> > > > > root@beacon-imx8mm-kit:~#
> > > > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > > > : Internal error: Oops: 96000004 [#2] SMP
> > > > >
> > > > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > > > : Code: 52800001 aa1403e0 d2801802 95c31ab9 (b9400aa1)
> > > > >
> > > > > I want to install Gstreamer, but I don't have functioning DSI video,
> > > > > so I am not entirely sure how I will go about testing the decoders
> > > > > except by using fakesink
> > > >
> > > > We too don't have an mainline DSI to test the CODECs on recent NXP SoC. For
> > > > decoders we use fluster, a tool that runs publicly available conformance test.
> > > > It will simply decode to disk and compare a checksum of the decoded image
> > > > against the compliant checksum (produced by the reference decoders). For you
> > > > interested, it uses the new videocodectestsink, which is specialized for
> > > > producing or calculating conformance image/checksum.
> > > >
> > > > https://github.com/fluendo/fluster
> > > >
> > > > We have added support for GStreamer stateless decoders already.
> > > >
> > > > >
> > > > > If the G1 ends up working with some of the newer Gstreamer stuff, I
> > > > > might just submit a formal patch to just enable the G1 for now.
> > > >
> > > > This looks like a good idea indeed.
> > > >
> > > > >
> > > > > adam
> > > > > >
> > > > > > >
> > > > > > > Lastly, I didn't update any device tree binding YAML files, because
> > > > > > > I know there have been some discussions about the power domains on the
> > > > > > > imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> > > > > > > or if the existing one for te imx8mq should just be modified.
> > > > > > >
> > > > > > > This will likely require the following series in order to apply correctly:
> > > > > > > https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
> > > > > > >
> > > > > > > Adam Ford (5):
> > > > > > > media: hantro: Add support for i.MX8M Mini
> > > > > > > arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> > > > > > > media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> > > > > > > media: hantro: Add H1 encoder support on i.MX8M Mini
> > > > > > > arm64: dts: imx8mm: Enable Hantro H1 Encoder
> > > > > > >
> > > > > > > arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> > > > > > > drivers/staging/media/hantro/hantro_drv.c | 3 +
> > > > > > > drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> > > > > > > drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> > > > > > > .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> > > > > > > 5 files changed, 231 insertions(+), 21 deletions(-)
> > > > > > >
> > > > > >
> > > >
> > >
> > > Nicolas and Adam,
> > >
> > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > H1 JPEG encode using GStreamer 1.18.5:
> > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > ^ v4l2jpegenc
> >
> > This is just a transcript error ?
>
> Nicolas,
>
> No! Thanks for catching my mistake. I was testing with software encode... ooops!
>
> 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> the board so likely a power-domain issue there?
>
> >
> > > host=192.168.1.146 port=5000
> > > viewed on client@192.168.1.146 via:
> > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > rtpjpegdepay ! jpegdec ! autovideosink
> > >
> > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> >
> > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > a single repository. We are very close to 1.20, which will include stable API
> > support of H264, MPEG2 and VP8 decoding.
> >
>
> Ok, let me see if I can navigate through the build process and I'll
> get back to you.
Nicolas,
I've built gstreamer 1.19.3 from git and still don't see any decoder elements:
[gst-main] root@focal-venice:~# dmesg | grep -i hantro
[ 8.048311] hantro_vpu: module is from the staging directory, the
quality is unknown, you have been warned.
[ 8.069029] hantro-vpu 38300000.video-codec: registered
nxp,imx8mm-vpu-dec as /dev/video0
[ 8.070164] hantro-vpu 38310000.video-codec: registered
nxp,imx8mm-vpu-g2-dec as /dev/video1
[ 8.072553] hantro_vpu: module is from the staging directory, the
quality is unknown, you have been warned.
[ 8.074205] hantro-vpu 38320000.video-codec: registered
nxp,imx8mm-vpu-h1-enc as /dev/video2
[ 8.088548] hantro_vpu: module is from the staging directory, the
quality is unknown, you have been warned.
[gst-main] root@focal-venice:~# ls -l /lib/firmware/vpu
total 604
-rwxr-xr-x 1 root root 522024 Nov 23 00:02 vpu_fw_imx8_dec.bin
-rwxr-xr-x 1 root root 91920 Nov 23 00:02 vpu_fw_imx8_enc.bin
[gst-main] root@focal-venice:~# cat /sys/class/video4linux/video*/name
nxp,imx8mm-vpu-dec
nxp,imx8mm-vpu-g2-dec
nxp,imx8mm-vpu-h1-enc
[gst-main] root@focal-venice:~# gst-inspect-1.0 --version
gst-inspect-1.0 version 1.19.3
GStreamer 1.19.3 (GIT)
Unknown package origin
[gst-main] root@focal-venice:~# gst-inspect-1.0 | grep v4l2
video4linux2: v4l2deviceprovider (GstDeviceProviderFactory)
video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
video4linux2: v4l2radio: Radio (video4linux2) Tuner
video4linux2: v4l2sink: Video (video4linux2) Sink
video4linux2: v4l2src: Video (video4linux2) Source
The v4l2jpegenc encoder element is there because I haven't disabled it
from Adam's patch yet.
Is there some debugging you want me to add to gstreamer?
Thanks,
Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread
* Re: [RFC 0/5] arm64: imx8mm: Enable Hantro VPUs
2021-11-23 0:06 ` Tim Harvey
@ 2021-11-23 20:10 ` Nicolas Dufresne
0 siblings, 0 replies; 42+ messages in thread
From: Nicolas Dufresne @ 2021-11-23 20:10 UTC (permalink / raw)
To: Tim Harvey
Cc: Adam Ford, linux-media, Schrempf Frieder, Marek Vasut,
Jagan Teki, Adam Ford-BE, cstevens, Ezequiel Garcia,
Philipp Zabel, Mauro Carvalho Chehab, Rob Herring, Shawn Guo,
Sascha Hauer, Pengutronix Kernel Team, Fabio Estevam,
NXP Linux Team, Greg Kroah-Hartman, Heiko Stuebner, Lucas Stach,
Joakim Zhang, Alice Guo, Peng Fan,
open list:HANTRO VPU CODEC DRIVER,
open list:OPEN FIRMWARE AND FLATTENED DEVICE TREE BINDINGS,
moderated list:ARM/FREESCALE IMX / MXC ARM ARCHITECTURE,
open list, open list:STAGING SUBSYSTEM
Le lundi 22 novembre 2021 à 16:06 -0800, Tim Harvey a écrit :
> On Thu, Nov 18, 2021 at 8:20 AM Tim Harvey <tharvey@gateworks.com> wrote:
> >
> > On Thu, Nov 18, 2021 at 6:30 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > >
> > > Le mardi 16 novembre 2021 à 15:23 -0800, Tim Harvey a écrit :
> > > > On Tue, Nov 9, 2021 at 7:57 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > >
> > > > > Le lundi 08 novembre 2021 à 10:33 -0600, Adam Ford a écrit :
> > > > > > On Mon, Nov 8, 2021 at 7:59 AM Nicolas Dufresne <nicolas@ndufresne.ca> wrote:
> > > > > > >
> > > > > > > Hi Adam,
> > > > > > >
> > > > > > > thanks for you work, I'll try and reply about the GStreamer questions below, if
> > > > > > > you have further question feel free to ask.
> > > > > > >
> > > > > > > Le samedi 06 novembre 2021 à 13:37 -0500, Adam Ford a écrit :
> > > > > > > > The i.MX8M has two Hantro video decoders, called G1 and G2 which appear
> > > > > > > > to be related to the video decoders used on the i.MX8MQ, but because of
> > > > > > > > how the Mini handles the power domains, the VPU driver does not need to
> > > > > > > > handle all the functions, so a new compatible flag is required.
> > > > > > > >
> > > > > > > > This is an RFC because I don't have functional video on my system yet,
> > > > > > > > and I'm hoping there might be people who do and can help test this.
> > > > > > > > I have only tested this far enough to see they enumerate and appear
> > > > > > > > as /dev/videoX and /dev/mediaX devices.
> > > > > > >
> > > > > > > I will check the patchset, but you need in the mini-variant to disable the G1
> > > > > > > post processor, because this block was fused out. We didn't make it optional
> > > > > >
> > > > > > Thanks for being willing to review this.
> > > > > >
> > > > > > > from the start as according to the V1 of the TRM it was there, but that error
> > > > > > > was corrected in V3.
> > > > > >
> > > > > > Thanks for the clarification. It wasn't obvious to me, because in
> > > > > > some instances the PP looked like it was there and sometimes not
> > > > > > there. I'll remove the postproc stuff.
> > > > > >
> > > > > > >
> > > > > > > >
> > > > > > > > I am also curious to know if/what gstreamer plugins are necessary. In
> > > > > > > > NXP's custom kernel, there are IMX-specific plugins, and I was hoping there
> > > > > > > > would be more generic plugins that I can use to test. I am hoping some
> > > > > > > > of the linux-media experts might chime in on how to best test.
> > > > > > >
> > > > > > > I will recommend using GStreamer 1.19.3 or main branch (GStreamer is now a
> > > > > > > single git repo). You will then be able to test Hantro G1 decoding of MPEG2,
> > > > > > > H264 and VP8. Remember that the related plugin depends on libgudev (a glib
> > > > > > > binding of udev).
> > > > > >
> > > > > > Thanks for the tip.
> > > > > >
> > > > > > >
> > > > > > > For the encoder, I believe only JPEG maybe supported, since this is all there is
> > > > > > > mainline for RK3288 (and perhaps other RK). But this will need testing and
> > > > > > > debugging as the G1 version is slightly newer on NXP SoC.
> > > > > >
> > > > > > For what it's worth the G1 seems to repond cleanly to the inquiries
> > > > > > from v42-compliance.
> > > > > > The G2 throws some splat when I run v4l2-compliance, but I am still
> > > > > > investigating that.
> > > > > >
> > > > > > [ 405.456979] ------------[ cut here ]------------
> > > > > > [ 405.464173] WARNING: CPU: 0 PID: 563 at mm/page_alloc.c:5344
> > > > > > __alloc_pages+0x5a4/0xbe0
> > > > > > [ 405.472104] Modules linked in: 8021q garp mrp stp llc caam_jr
> > > > > > caamhash_desc caamalg_desc crypto_engine rng_core authenc libdes
> > > > > > imx7_media_csi(C) crct10dif_ce imx_media_common(C)
> > > > > > snd_soc_fsl_asoc_card imx7_mipi_csis(C) snd_soc_imx_audmux
> > > > > > snd_soc_simple_card_utils fsl_imx8_ddr_perf imx8m_ddrc brcmfmac
> > > > > > brcmutil hantro_vpu(C) v4l2_h264 v4l2_mem2mem videobuf2_vmalloc
> > > > > > videobuf2_dma_contig videobuf2_memops cfg80211 ov5640 videobuf2_v4l2
> > > > > > v4l2_fwnode v4l2_async videobuf2_common videodev etnaviv gpu_sched
> > > > > > hci_uart mc btqca btbcm snd_soc_wm8962 at24 spi_imx rtc_pcf85363
> > > > > > rtc_snvs clk_bd718x7 spi_bitbang snvs_pwrkey snd_soc_fsl_sai
> > > > > > imx_pcm_dma caam error bluetooth imx8mm_thermal ecdh_generic
> > > > > > imx_cpufreq_dt ecc rfkill fuse drm ipv6
> > > > > > [ 405.535835] CPU: 0 PID: 563 Comm: v4l2-compliance Tainted: G D
> > > > > > C 5.15.0-next-20211105-00010-g4bb8e8a25d3c-dirty #28
> > > > > > [ 405.547401] Hardware name: Beacon EmbeddedWorks i.MX8M Mini
> > > > > > Development Kit (DT)
> > > > > > [ 405.554797] pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> > > > > > [ 405.561762] pc : __alloc_pages+0x5a4/0xbe0
> > > > > > [ 405.565861] lr : __dma_direct_alloc_pages+0x17c/0x1e0
> > > > > > [ 405.570917] sp : ffff800012443810
> > > > > > [ 405.574232] x29: ffff800012443810 x28: 0000000000000000 x27: ffff000005288220
> > > > > > [ 405.581375] x26: 0000000000000034 x25: 0000000000000000 x24: ffff000000259010
> > > > > > [ 405.588517] x23: ffff80001011ab7c x22: ffff000000259010 x21: 00000000ffffffff
> > > > > > [ 405.595659] x20: 0000000000000cc1 x19: 0000000000000000 x18: 0000000000000000
> > > > > > [ 405.602803] x17: 0000000000000000 x16: 0000000000000000 x15: 0000000000000000
> > > > > > [ 405.609947] x14: 0000000000000001 x13: 0000000000000000 x12: 0000000000000000
> > > > > > [ 405.617090] x11: ffff80001241d000 x10: ffff00000528833a x9 : ffff00000528832a
> > > > > > [ 405.624232] x8 : 0000000000000000 x7 : 0000000000000000 x6 : 0000000000000cc0
> > > > > > [ 405.631378] x5 : 00000000bfffffff x4 : ffff000009e30dc0 x3 : 0000000000000000
> > > > > > [ 405.638520] x2 : 0000000000000000 x1 : 0000000000000001 x0 : 0000000000000cc1
> > > > > > [ 405.645666] Call trace:
> > > > > > [ 405.648113] __alloc_pages+0x5a4/0xbe0
> > > > > > [ 405.651862] __dma_direct_alloc_pages+0x17c/0x1e0
> > > > > > [ 405.656569] dma_direct_alloc+0x70/0x310
> > > > > > [ 405.660494] dma_alloc_attrs+0x7c/0xe4
> > > > > > [ 405.664246] hantro_hevc_get_ref_buf+0x15c/0x184 [hantro_vpu]
> > > > > > [ 405.670021] hantro_g2_hevc_dec_run+0x3b8/0x1910 [hantro_vpu]
> > > > > > [ 405.675791] device_run+0xac/0x110 [hantro_vpu]
> > > > > > [ 405.680345] v4l2_m2m_try_run+0xac/0x1b0 [v4l2_mem2mem]
> > > > > > [ 405.685598] v4l2_m2m_ioctl_streamon+0x84/0xa0 [v4l2_mem2mem]
> > > > > > [ 405.691366] v4l_streamon+0x28/0x34 [videodev]
> > > > > > [ 405.695877] __video_do_ioctl+0x178/0x3dc [videodev]
> > > > > > [ 405.700897] video_usercopy+0x368/0x6dc [videodev]
> > > > > > [ 405.705745] video_ioctl2+0x1c/0x30 [videodev]
> > > > > > [ 405.710246] v4l2_ioctl+0x44/0x64 [videodev]
> > > > > > [ 405.714574] __arm64_sys_ioctl+0xac/0xf0
> > > > > > [ 405.718502] invoke_syscall+0x48/0x114
> > > > > > [ 405.722258] el0_svc_common.constprop.0+0xd4/0xfc
> > > > > > [ 405.726969] do_el0_svc+0x2c/0x94
> > > > > > [ 405.730286] el0_svc+0x28/0x80
> > > > > > [ 405.733348] el0t_64_sync_handler+0xa8/0x130
> > > > > > [ 405.737619] el0t_64_sync+0x1a0/0x1a4
> > > > > > [ 405.741287] ---[ end trace 270ed4a899803006 ]---
> > > > > >
> > > > > > The H1 encoder seems to hang the system when I run v4l2-compliance on
> > > > > > it when H1 is set up as I submitted the patch. I tried dropping all
> > > > > > the encoder formats except the JPEG format, and it doesn't hang any
> > > > > > more, but it also doesn't really do anything.
> > > > > > The datasheet only references VPU_H1 as supporting VP8 and H.264, so I
> > > > > > am not sure JPEG is even supported.
> > > > >
> > > > > If JPEG is not supported, then there is nothing left for mainline in this
> > > > > regard. The kernel control interface and encoding flow needs to be designed and
> > > > > specified for encoders like VP8 and H264. Some prototypes and prior-art exist
> > > > > though, but nothing ever got formalized in the form of a specification.
> > > > >
> > > > > >
> > > > > > The log from v4l2-compliance on the H1 with everything except the JPEG
> > > > > > removed looks like:
> > > > > >
> > > > > > root@beacon-imx8mm-kit:~# v4l2-compliance -d2
> > > > > > v4l2-compliance SHA: not available
> > > > > > , 64 bits, 64-bit time_t
> > > > > >
> > > > > > Segmentation fault
> > > > > > root@beacon-imx8mm-kit:~#
> > > > > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > > > > : Internal error: Oops: 96000004 [#2] SMP
> > > > > >
> > > > > > Message from syslogd@ at Thu Jan 1 00:05:07 1970 ...
> > > > > > : Code: 52800001 aa1403e0 d2801802 95c31ab9 (b9400aa1)
> > > > > >
> > > > > > I want to install Gstreamer, but I don't have functioning DSI video,
> > > > > > so I am not entirely sure how I will go about testing the decoders
> > > > > > except by using fakesink
> > > > >
> > > > > We too don't have an mainline DSI to test the CODECs on recent NXP SoC. For
> > > > > decoders we use fluster, a tool that runs publicly available conformance test.
> > > > > It will simply decode to disk and compare a checksum of the decoded image
> > > > > against the compliant checksum (produced by the reference decoders). For you
> > > > > interested, it uses the new videocodectestsink, which is specialized for
> > > > > producing or calculating conformance image/checksum.
> > > > >
> > > > > https://github.com/fluendo/fluster
> > > > >
> > > > > We have added support for GStreamer stateless decoders already.
> > > > >
> > > > > >
> > > > > > If the G1 ends up working with some of the newer Gstreamer stuff, I
> > > > > > might just submit a formal patch to just enable the G1 for now.
> > > > >
> > > > > This looks like a good idea indeed.
> > > > >
> > > > > >
> > > > > > adam
> > > > > > >
> > > > > > > >
> > > > > > > > Lastly, I didn't update any device tree binding YAML files, because
> > > > > > > > I know there have been some discussions about the power domains on the
> > > > > > > > imx8mq, and I wasn't sure if the imx8mm should get a separate YAML file
> > > > > > > > or if the existing one for te imx8mq should just be modified.
> > > > > > > >
> > > > > > > > This will likely require the following series in order to apply correctly:
> > > > > > > > https://patchwork.kernel.org/project/linux-arm-kernel/list/?series=576407
> > > > > > > >
> > > > > > > > Adam Ford (5):
> > > > > > > > media: hantro: Add support for i.MX8M Mini
> > > > > > > > arm64: dts: imx8mm: Enable VPU-G1 and VPU-G2
> > > > > > > > media: hantro: Rename ROCKCHIP_VPU_ENC_FMT to HANTRO_VPU_ENC_FMT
> > > > > > > > media: hantro: Add H1 encoder support on i.MX8M Mini
> > > > > > > > arm64: dts: imx8mm: Enable Hantro H1 Encoder
> > > > > > > >
> > > > > > > > arch/arm64/boot/dts/freescale/imx8mm.dtsi | 61 ++++++++
> > > > > > > > drivers/staging/media/hantro/hantro_drv.c | 3 +
> > > > > > > > drivers/staging/media/hantro/hantro_hw.h | 19 ++-
> > > > > > > > drivers/staging/media/hantro/imx8m_vpu_hw.c | 143 ++++++++++++++++++
> > > > > > > > .../staging/media/hantro/rockchip_vpu_hw.c | 26 ++--
> > > > > > > > 5 files changed, 231 insertions(+), 21 deletions(-)
> > > > > > > >
> > > > > > >
> > > > >
> > > >
> > > > Nicolas and Adam,
> > > >
> > > > For the H1 patches in this series: I've been able to test the IMX8MM
> > > > H1 JPEG encode using GStreamer 1.18.5:
> > > > $ gst-inspect-1.0 | grep -e "v4l2.*enc"
> > > > video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> > > > $ gst-launch-1.0 videotestsrc ! jpegenc ! rtpjpegpay ! udpsink
> > > ^ v4l2jpegenc
> > >
> > > This is just a transcript error ?
> >
> > Nicolas,
> >
> > No! Thanks for catching my mistake. I was testing with software encode... ooops!
> >
> > 'gst-launch-1.0 videotestsrc ! v4l2jpegenc ! fakesink' actually hangs
> > the board so likely a power-domain issue there?
> >
> > >
> > > > host=192.168.1.146 port=5000
> > > > viewed on client@192.168.1.146 via:
> > > > $ gst-launch-1.0 udpsrc port=5000 ! application/x-rtp,payload=96 !
> > > > rtpjpegdepay ! jpegdec ! autovideosink
> > > >
> > > > For the G1/G2 patches in the series I don't see any Gstreamer
> > > > 'v4l2.*dec' elements. Perhaps I need a newer version of Gstreamer.
> > >
> > > Most likely yes, I suggest building gstreamer/ branch "main", GStreamer has now
> > > a single repository. We are very close to 1.20, which will include stable API
> > > support of H264, MPEG2 and VP8 decoding.
> > >
> >
> > Ok, let me see if I can navigate through the build process and I'll
> > get back to you.
>
> Nicolas,
>
> I've built gstreamer 1.19.3 from git and still don't see any decoder elements:
>
> [gst-main] root@focal-venice:~# dmesg | grep -i hantro
> [ 8.048311] hantro_vpu: module is from the staging directory, the
> quality is unknown, you have been warned.
> [ 8.069029] hantro-vpu 38300000.video-codec: registered
> nxp,imx8mm-vpu-dec as /dev/video0
> [ 8.070164] hantro-vpu 38310000.video-codec: registered
> nxp,imx8mm-vpu-g2-dec as /dev/video1
> [ 8.072553] hantro_vpu: module is from the staging directory, the
> quality is unknown, you have been warned.
> [ 8.074205] hantro-vpu 38320000.video-codec: registered
> nxp,imx8mm-vpu-h1-enc as /dev/video2
> [ 8.088548] hantro_vpu: module is from the staging directory, the
> quality is unknown, you have been warned.
> [gst-main] root@focal-venice:~# ls -l /lib/firmware/vpu
> total 604
> -rwxr-xr-x 1 root root 522024 Nov 23 00:02 vpu_fw_imx8_dec.bin
> -rwxr-xr-x 1 root root 91920 Nov 23 00:02 vpu_fw_imx8_enc.bin
> [gst-main] root@focal-venice:~# cat /sys/class/video4linux/video*/name
> nxp,imx8mm-vpu-dec
> nxp,imx8mm-vpu-g2-dec
> nxp,imx8mm-vpu-h1-enc
> [gst-main] root@focal-venice:~# gst-inspect-1.0 --version
> gst-inspect-1.0 version 1.19.3
> GStreamer 1.19.3 (GIT)
> Unknown package origin
> [gst-main] root@focal-venice:~# gst-inspect-1.0 | grep v4l2
> video4linux2: v4l2deviceprovider (GstDeviceProviderFactory)
> video4linux2: v4l2jpegenc: V4L2 JPEG Encoder
> video4linux2: v4l2radio: Radio (video4linux2) Tuner
> video4linux2: v4l2sink: Video (video4linux2) Sink
> video4linux2: v4l2src: Video (video4linux2) Source
>
> The v4l2jpegenc encoder element is there because I haven't disabled it
> from Adam's patch yet.
>
> Is there some debugging you want me to add to gstreamer?
See my instructions on your previous comment giving instructions to build
gstreamer on ubuntu 20.04 (and verify that the two plugins needed are there),
you likely are missing v4l2codecs plugin deps (like gudev).
>
> Thanks,
>
> Tim
_______________________________________________
Linux-rockchip mailing list
Linux-rockchip@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-rockchip
^ permalink raw reply [flat|nested] 42+ messages in thread