[RFC PATCH v1 0/7] media&omap4: introduce face detection(FD) driver

All of lore.kernel.org
 help / color / mirror / Atom feed

* [RFC PATCH v1 0/7] media&omap4: introduce face detection(FD) driver
@ 2011-12-02 15:02 ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, linux-media

Hi,

These v1 patches(against -next tree) introduce v4l2 based face
detection(FD) device driver, and enable FD hardware[1] on omap4 SoC..
The idea of implementing it on v4l2 is from from Alan Cox, Sylwester
and Greg-Kh.

For verification purpose, I write one user space utility[2] to
test the module and driver, follows its basic functions:

	- detect faces in input grayscal picture(PGM raw, 320 by 240)
	- detect faces in input y8 format video stream
	- plot a rectangle to mark the detected faces, and save it as 
	another same type grayscal picture

Looks the performance of the module is not bad, see some detection
results on the link[3][4].

Face detection can be used to implement some interesting applications
(camera, face unlock, baby monitor, ...).

TODO:
	- implement FD setting interfaces with v4l2 controls or
	ext controls

thanks,
--
Ming Lei

[1], Ch9 of OMAP4 Technical Reference Manual
[2], http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif
[3], http://kernel.ubuntu.com/~ming/dev/fdif/output
[4], All pictures are taken from http://www.google.com/imghp
and converted to pnm from jpeg format, only for test purpose.

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 0/7] media&omap4: introduce face detection(FD) driver
@ 2011-12-02 15:02 ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

These v1 patches(against -next tree) introduce v4l2 based face
detection(FD) device driver, and enable FD hardware[1] on omap4 SoC..
The idea of implementing it on v4l2 is from from Alan Cox, Sylwester
and Greg-Kh.

For verification purpose, I write one user space utility[2] to
test the module and driver, follows its basic functions:

	- detect faces in input grayscal picture(PGM raw, 320 by 240)
	- detect faces in input y8 format video stream
	- plot a rectangle to mark the detected faces, and save it as 
	another same type grayscal picture

Looks the performance of the module is not bad, see some detection
results on the link[3][4].

Face detection can be used to implement some interesting applications
(camera, face unlock, baby monitor, ...).

TODO:
	- implement FD setting interfaces with v4l2 controls or
	ext controls

thanks,
--
Ming Lei

[1], Ch9 of OMAP4 Technical Reference Manual
[2], http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif
[3], http://kernel.ubuntu.com/~ming/dev/fdif/output
[4], All pictures are taken from http://www.google.com/imghp
and converted to pnm from jpeg format, only for test purpose.

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 1/7] omap4: introduce fdif(face detect module) hwmod
  2011-12-02 15:02 ` Ming Lei
@ 2011-12-02 15:02   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, linux-media, Ming Lei

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 arch/arm/mach-omap2/omap_hwmod_44xx_data.c |   81 ++++++++++++++++++++++++++++
 1 files changed, 81 insertions(+), 0 deletions(-)

diff --git a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
index 6cf21ee..30db754 100644
--- a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
@@ -53,6 +53,7 @@ static struct omap_hwmod omap44xx_dmm_hwmod;
 static struct omap_hwmod omap44xx_dsp_hwmod;
 static struct omap_hwmod omap44xx_dss_hwmod;
 static struct omap_hwmod omap44xx_emif_fw_hwmod;
+static struct omap_hwmod omap44xx_fdif_hwmod;
 static struct omap_hwmod omap44xx_hsi_hwmod;
 static struct omap_hwmod omap44xx_ipu_hwmod;
 static struct omap_hwmod omap44xx_iss_hwmod;
@@ -354,6 +355,14 @@ static struct omap_hwmod_ocp_if omap44xx_dma_system__l3_main_2 = {
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* fdif -> l3_main_2 */
+static struct omap_hwmod_ocp_if omap44xx_fdif__l3_main_2 = {
+	.master		= &omap44xx_fdif_hwmod,
+	.slave		= &omap44xx_l3_main_2_hwmod,
+	.clk		= "l3_div_ck",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* hsi -> l3_main_2 */
 static struct omap_hwmod_ocp_if omap44xx_hsi__l3_main_2 = {
 	.master		= &omap44xx_hsi_hwmod,
@@ -5444,6 +5453,75 @@ static struct omap_hwmod omap44xx_wd_timer3_hwmod = {
 	.slaves_cnt	= ARRAY_SIZE(omap44xx_wd_timer3_slaves),
 };
 
+/* 'fdif' class */
+static struct omap_hwmod_class_sysconfig omap44xx_fdif_sysc = {
+	.rev_offs	= 0x0000,
+	.sysc_offs	= 0x0010,
+	.sysc_flags	= (SYSC_HAS_MIDLEMODE | SYSC_HAS_RESET_STATUS |
+			   SYSC_HAS_SIDLEMODE | SYSC_HAS_SOFTRESET),
+	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
+			   MSTANDBY_FORCE | MSTANDBY_NO |
+			   MSTANDBY_SMART),
+	.sysc_fields	= &omap_hwmod_sysc_type2,
+};
+
+static struct omap_hwmod_class omap44xx_fdif_hwmod_class = {
+	.name	= "fdif",
+	.sysc	= &omap44xx_fdif_sysc,
+};
+
+/*fdif*/
+static struct omap_hwmod_addr_space omap44xx_fdif_addrs[] = {
+	{
+		.pa_start	= 0x4a10a000,
+		.pa_end		= 0x4a10afff,
+		.flags		= ADDR_TYPE_RT
+	},
+	{ }
+};
+
+/* l4_cfg -> fdif */
+static struct omap_hwmod_ocp_if omap44xx_l4_cfg__fdif = {
+	.master		= &omap44xx_l4_cfg_hwmod,
+	.slave		= &omap44xx_fdif_hwmod,
+	.clk		= "l4_div_ck",
+	.addr		= omap44xx_fdif_addrs,
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* fdif slave ports */
+static struct omap_hwmod_ocp_if *omap44xx_fdif_slaves[] = {
+	&omap44xx_l4_cfg__fdif,
+};
+static struct omap_hwmod_irq_info omap44xx_fdif_irqs[] = {
+	{ .irq = 69 + OMAP44XX_IRQ_GIC_START },
+	{ .irq = -1 }
+};
+
+/* fdif master ports */
+static struct omap_hwmod_ocp_if *omap44xx_fdif_masters[] = {
+	&omap44xx_fdif__l3_main_2,
+};
+
+static struct omap_hwmod omap44xx_fdif_hwmod = {
+	.name		= "fdif",
+	.class		= &omap44xx_fdif_hwmod_class,
+	.clkdm_name	= "iss_clkdm",
+	.mpu_irqs	= omap44xx_fdif_irqs,
+	.main_clk	= "fdif_fck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = OMAP4_CM_CAM_FDIF_CLKCTRL_OFFSET,
+			.context_offs = OMAP4_RM_CAM_FDIF_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_SWCTRL,
+		},
+	},
+	.slaves		= omap44xx_fdif_slaves,
+	.slaves_cnt	= ARRAY_SIZE(omap44xx_fdif_slaves),
+	.masters	= omap44xx_fdif_masters,
+	.masters_cnt	= ARRAY_SIZE(omap44xx_fdif_masters),
+};
+
 static __initdata struct omap_hwmod *omap44xx_hwmods[] = {
 
 	/* dmm class */
@@ -5593,6 +5671,9 @@ static __initdata struct omap_hwmod *omap44xx_hwmods[] = {
 	&omap44xx_wd_timer2_hwmod,
 	&omap44xx_wd_timer3_hwmod,
 
+	/* fdif class */
+	&omap44xx_fdif_hwmod,
+
 	NULL,
 };
 
-- 
1.7.5.4


^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 1/7] omap4: introduce fdif(face detect module) hwmod
@ 2011-12-02 15:02   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: linux-arm-kernel

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 arch/arm/mach-omap2/omap_hwmod_44xx_data.c |   81 ++++++++++++++++++++++++++++
 1 files changed, 81 insertions(+), 0 deletions(-)

diff --git a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
index 6cf21ee..30db754 100644
--- a/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
+++ b/arch/arm/mach-omap2/omap_hwmod_44xx_data.c
@@ -53,6 +53,7 @@ static struct omap_hwmod omap44xx_dmm_hwmod;
 static struct omap_hwmod omap44xx_dsp_hwmod;
 static struct omap_hwmod omap44xx_dss_hwmod;
 static struct omap_hwmod omap44xx_emif_fw_hwmod;
+static struct omap_hwmod omap44xx_fdif_hwmod;
 static struct omap_hwmod omap44xx_hsi_hwmod;
 static struct omap_hwmod omap44xx_ipu_hwmod;
 static struct omap_hwmod omap44xx_iss_hwmod;
@@ -354,6 +355,14 @@ static struct omap_hwmod_ocp_if omap44xx_dma_system__l3_main_2 = {
 	.user		= OCP_USER_MPU | OCP_USER_SDMA,
 };
 
+/* fdif -> l3_main_2 */
+static struct omap_hwmod_ocp_if omap44xx_fdif__l3_main_2 = {
+	.master		= &omap44xx_fdif_hwmod,
+	.slave		= &omap44xx_l3_main_2_hwmod,
+	.clk		= "l3_div_ck",
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
 /* hsi -> l3_main_2 */
 static struct omap_hwmod_ocp_if omap44xx_hsi__l3_main_2 = {
 	.master		= &omap44xx_hsi_hwmod,
@@ -5444,6 +5453,75 @@ static struct omap_hwmod omap44xx_wd_timer3_hwmod = {
 	.slaves_cnt	= ARRAY_SIZE(omap44xx_wd_timer3_slaves),
 };
 
+/* 'fdif' class */
+static struct omap_hwmod_class_sysconfig omap44xx_fdif_sysc = {
+	.rev_offs	= 0x0000,
+	.sysc_offs	= 0x0010,
+	.sysc_flags	= (SYSC_HAS_MIDLEMODE | SYSC_HAS_RESET_STATUS |
+			   SYSC_HAS_SIDLEMODE | SYSC_HAS_SOFTRESET),
+	.idlemodes	= (SIDLE_FORCE | SIDLE_NO | SIDLE_SMART |
+			   MSTANDBY_FORCE | MSTANDBY_NO |
+			   MSTANDBY_SMART),
+	.sysc_fields	= &omap_hwmod_sysc_type2,
+};
+
+static struct omap_hwmod_class omap44xx_fdif_hwmod_class = {
+	.name	= "fdif",
+	.sysc	= &omap44xx_fdif_sysc,
+};
+
+/*fdif*/
+static struct omap_hwmod_addr_space omap44xx_fdif_addrs[] = {
+	{
+		.pa_start	= 0x4a10a000,
+		.pa_end		= 0x4a10afff,
+		.flags		= ADDR_TYPE_RT
+	},
+	{ }
+};
+
+/* l4_cfg -> fdif */
+static struct omap_hwmod_ocp_if omap44xx_l4_cfg__fdif = {
+	.master		= &omap44xx_l4_cfg_hwmod,
+	.slave		= &omap44xx_fdif_hwmod,
+	.clk		= "l4_div_ck",
+	.addr		= omap44xx_fdif_addrs,
+	.user		= OCP_USER_MPU | OCP_USER_SDMA,
+};
+
+/* fdif slave ports */
+static struct omap_hwmod_ocp_if *omap44xx_fdif_slaves[] = {
+	&omap44xx_l4_cfg__fdif,
+};
+static struct omap_hwmod_irq_info omap44xx_fdif_irqs[] = {
+	{ .irq = 69 + OMAP44XX_IRQ_GIC_START },
+	{ .irq = -1 }
+};
+
+/* fdif master ports */
+static struct omap_hwmod_ocp_if *omap44xx_fdif_masters[] = {
+	&omap44xx_fdif__l3_main_2,
+};
+
+static struct omap_hwmod omap44xx_fdif_hwmod = {
+	.name		= "fdif",
+	.class		= &omap44xx_fdif_hwmod_class,
+	.clkdm_name	= "iss_clkdm",
+	.mpu_irqs	= omap44xx_fdif_irqs,
+	.main_clk	= "fdif_fck",
+	.prcm = {
+		.omap4 = {
+			.clkctrl_offs = OMAP4_CM_CAM_FDIF_CLKCTRL_OFFSET,
+			.context_offs = OMAP4_RM_CAM_FDIF_CONTEXT_OFFSET,
+			.modulemode   = MODULEMODE_SWCTRL,
+		},
+	},
+	.slaves		= omap44xx_fdif_slaves,
+	.slaves_cnt	= ARRAY_SIZE(omap44xx_fdif_slaves),
+	.masters	= omap44xx_fdif_masters,
+	.masters_cnt	= ARRAY_SIZE(omap44xx_fdif_masters),
+};
+
 static __initdata struct omap_hwmod *omap44xx_hwmods[] = {
 
 	/* dmm class */
@@ -5593,6 +5671,9 @@ static __initdata struct omap_hwmod *omap44xx_hwmods[] = {
 	&omap44xx_wd_timer2_hwmod,
 	&omap44xx_wd_timer3_hwmod,
 
+	/* fdif class */
+	&omap44xx_fdif_hwmod,
+
 	NULL,
 };
 
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 2/7] omap4: build fdif omap device from hwmod
  2011-12-02 15:02 ` Ming Lei
@ 2011-12-02 15:02   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, linux-media, Ming Lei

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 arch/arm/mach-omap2/devices.c |   33 +++++++++++++++++++++++++++++++++
 1 files changed, 33 insertions(+), 0 deletions(-)

diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
index 1166bdc..a392af5 100644
--- a/arch/arm/mach-omap2/devices.c
+++ b/arch/arm/mach-omap2/devices.c
@@ -728,6 +728,38 @@ void __init omap242x_init_mmc(struct omap_mmc_platform_data **mmc_data)
 
 #endif
 
+static struct platform_device* __init omap4_init_fdif(void)
+{
+	int id = -1;
+	struct platform_device *pd;
+	struct omap_hwmod *oh;
+	const char *dev_name = "fdif";
+
+	oh = omap_hwmod_lookup("fdif");
+	if (!oh) {
+		pr_err("Could not look up fdif hwmod\n");
+		return NULL;
+	}
+
+	pd = omap_device_build(dev_name, id, oh, NULL, 0, NULL, 0, 0);
+	WARN(IS_ERR(pd), "Can't build omap_device for %s.\n",
+				dev_name);
+	return pd;
+}
+
+static void __init omap_init_fdif(void)
+{
+	if (cpu_is_omap44xx()) {
+		struct platform_device *pd;
+
+		pd = omap4_init_fdif();
+		if (!pd)
+			return;
+
+		pm_runtime_enable(&pd->dev);
+	}
+}
+
 /*-------------------------------------------------------------------------*/
 
 #if defined(CONFIG_HDQ_MASTER_OMAP) || defined(CONFIG_HDQ_MASTER_OMAP_MODULE)
@@ -808,6 +840,7 @@ static int __init omap2_init_devices(void)
 	omap_init_sham();
 	omap_init_aes();
 	omap_init_vout();
+	omap_init_fdif();
 
 	return 0;
 }
-- 
1.7.5.4


^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 2/7] omap4: build fdif omap device from hwmod
@ 2011-12-02 15:02   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: linux-arm-kernel

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 arch/arm/mach-omap2/devices.c |   33 +++++++++++++++++++++++++++++++++
 1 files changed, 33 insertions(+), 0 deletions(-)

diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
index 1166bdc..a392af5 100644
--- a/arch/arm/mach-omap2/devices.c
+++ b/arch/arm/mach-omap2/devices.c
@@ -728,6 +728,38 @@ void __init omap242x_init_mmc(struct omap_mmc_platform_data **mmc_data)
 
 #endif
 
+static struct platform_device* __init omap4_init_fdif(void)
+{
+	int id = -1;
+	struct platform_device *pd;
+	struct omap_hwmod *oh;
+	const char *dev_name = "fdif";
+
+	oh = omap_hwmod_lookup("fdif");
+	if (!oh) {
+		pr_err("Could not look up fdif hwmod\n");
+		return NULL;
+	}
+
+	pd = omap_device_build(dev_name, id, oh, NULL, 0, NULL, 0, 0);
+	WARN(IS_ERR(pd), "Can't build omap_device for %s.\n",
+				dev_name);
+	return pd;
+}
+
+static void __init omap_init_fdif(void)
+{
+	if (cpu_is_omap44xx()) {
+		struct platform_device *pd;
+
+		pd = omap4_init_fdif();
+		if (!pd)
+			return;
+
+		pm_runtime_enable(&pd->dev);
+	}
+}
+
 /*-------------------------------------------------------------------------*/
 
 #if defined(CONFIG_HDQ_MASTER_OMAP) || defined(CONFIG_HDQ_MASTER_OMAP_MODULE)
@@ -808,6 +840,7 @@ static int __init omap2_init_devices(void)
 	omap_init_sham();
 	omap_init_aes();
 	omap_init_vout();
+	omap_init_fdif();
 
 	return 0;
 }
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 3/7] media: videobuf2: move out of setting pgprot_noncached from vb2_mmap_pfn_range
  2011-12-02 15:02 ` Ming Lei
  (?)
@ 2011-12-02 15:02   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, linux-media, Ming Lei

So that we can reuse vb2_mmap_pfn_range for the coming videobuf2_page
memops.

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/videobuf2-dma-contig.c |    1 +
 drivers/media/video/videobuf2-memops.c     |    1 -
 2 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/drivers/media/video/videobuf2-dma-contig.c b/drivers/media/video/videobuf2-dma-contig.c
index f17ad98..0ea8866 100644
--- a/drivers/media/video/videobuf2-dma-contig.c
+++ b/drivers/media/video/videobuf2-dma-contig.c
@@ -106,6 +106,7 @@ static int vb2_dma_contig_mmap(void *buf_priv, struct vm_area_struct *vma)
 		return -EINVAL;
 	}
 
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	return vb2_mmap_pfn_range(vma, buf->dma_addr, buf->size,
 				  &vb2_common_vm_ops, &buf->handler);
 }
diff --git a/drivers/media/video/videobuf2-memops.c b/drivers/media/video/videobuf2-memops.c
index 71a7a78..77e0def 100644
--- a/drivers/media/video/videobuf2-memops.c
+++ b/drivers/media/video/videobuf2-memops.c
@@ -162,7 +162,6 @@ int vb2_mmap_pfn_range(struct vm_area_struct *vma, unsigned long paddr,
 
 	size = min_t(unsigned long, vma->vm_end - vma->vm_start, size);
 
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	ret = remap_pfn_range(vma, vma->vm_start, paddr >> PAGE_SHIFT,
 				size, vma->vm_page_prot);
 	if (ret) {
-- 
1.7.5.4


^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 3/7] media: videobuf2: move out of setting pgprot_noncached from vb2_mmap_pfn_range
@ 2011-12-02 15:02   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Greg KH, Ming Lei, linux-kernel, Sylwester Nawrocki, Alan Cox,
	linux-omap, linux-arm-kernel, linux-media

So that we can reuse vb2_mmap_pfn_range for the coming videobuf2_page
memops.

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/videobuf2-dma-contig.c |    1 +
 drivers/media/video/videobuf2-memops.c     |    1 -
 2 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/drivers/media/video/videobuf2-dma-contig.c b/drivers/media/video/videobuf2-dma-contig.c
index f17ad98..0ea8866 100644
--- a/drivers/media/video/videobuf2-dma-contig.c
+++ b/drivers/media/video/videobuf2-dma-contig.c
@@ -106,6 +106,7 @@ static int vb2_dma_contig_mmap(void *buf_priv, struct vm_area_struct *vma)
 		return -EINVAL;
 	}
 
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	return vb2_mmap_pfn_range(vma, buf->dma_addr, buf->size,
 				  &vb2_common_vm_ops, &buf->handler);
 }
diff --git a/drivers/media/video/videobuf2-memops.c b/drivers/media/video/videobuf2-memops.c
index 71a7a78..77e0def 100644
--- a/drivers/media/video/videobuf2-memops.c
+++ b/drivers/media/video/videobuf2-memops.c
@@ -162,7 +162,6 @@ int vb2_mmap_pfn_range(struct vm_area_struct *vma, unsigned long paddr,
 
 	size = min_t(unsigned long, vma->vm_end - vma->vm_start, size);
 
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	ret = remap_pfn_range(vma, vma->vm_start, paddr >> PAGE_SHIFT,
 				size, vma->vm_page_prot);
 	if (ret) {
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 3/7] media: videobuf2: move out of setting pgprot_noncached from vb2_mmap_pfn_range
@ 2011-12-02 15:02   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: linux-arm-kernel

So that we can reuse vb2_mmap_pfn_range for the coming videobuf2_page
memops.

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/videobuf2-dma-contig.c |    1 +
 drivers/media/video/videobuf2-memops.c     |    1 -
 2 files changed, 1 insertions(+), 1 deletions(-)

diff --git a/drivers/media/video/videobuf2-dma-contig.c b/drivers/media/video/videobuf2-dma-contig.c
index f17ad98..0ea8866 100644
--- a/drivers/media/video/videobuf2-dma-contig.c
+++ b/drivers/media/video/videobuf2-dma-contig.c
@@ -106,6 +106,7 @@ static int vb2_dma_contig_mmap(void *buf_priv, struct vm_area_struct *vma)
 		return -EINVAL;
 	}
 
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	return vb2_mmap_pfn_range(vma, buf->dma_addr, buf->size,
 				  &vb2_common_vm_ops, &buf->handler);
 }
diff --git a/drivers/media/video/videobuf2-memops.c b/drivers/media/video/videobuf2-memops.c
index 71a7a78..77e0def 100644
--- a/drivers/media/video/videobuf2-memops.c
+++ b/drivers/media/video/videobuf2-memops.c
@@ -162,7 +162,6 @@ int vb2_mmap_pfn_range(struct vm_area_struct *vma, unsigned long paddr,
 
 	size = min_t(unsigned long, vma->vm_end - vma->vm_start, size);
 
-	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 	ret = remap_pfn_range(vma, vma->vm_start, paddr >> PAGE_SHIFT,
 				size, vma->vm_page_prot);
 	if (ret) {
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 4/7] media: videobuf2: introduce VIDEOBUF2_PAGE memops
  2011-12-02 15:02 ` Ming Lei
@ 2011-12-02 15:02   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, linux-media, Ming Lei

DMA contig memory resource is very limited and precious, also
accessing to it from CPU is very slow on some platform.

For some cases(such as the comming face detection driver), DMA Streaming
buffer is enough, so introduce VIDEOBUF2_PAGE to allocate continuous
physical memory but letting video device driver to handle DMA buffer mapping
and unmapping things.

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/Kconfig          |    4 +
 drivers/media/video/Makefile         |    1 +
 drivers/media/video/videobuf2-page.c |  115 ++++++++++++++++++++++++++++++++++
 include/media/videobuf2-page.h       |   20 ++++++
 4 files changed, 140 insertions(+), 0 deletions(-)
 create mode 100644 drivers/media/video/videobuf2-page.c
 create mode 100644 include/media/videobuf2-page.h

diff --git a/drivers/media/video/Kconfig b/drivers/media/video/Kconfig
index 4e8a0c4..5684a00 100644
--- a/drivers/media/video/Kconfig
+++ b/drivers/media/video/Kconfig
@@ -60,6 +60,10 @@ config VIDEOBUF2_VMALLOC
 	select VIDEOBUF2_MEMOPS
 	tristate
 
+config VIDEOBUF2_PAGE
+	select VIDEOBUF2_CORE
+	select VIDEOBUF2_MEMOPS
+	tristate
 
 config VIDEOBUF2_DMA_SG
 	#depends on HAS_DMA
diff --git a/drivers/media/video/Makefile b/drivers/media/video/Makefile
index ddeaa6c..bc797f2 100644
--- a/drivers/media/video/Makefile
+++ b/drivers/media/video/Makefile
@@ -125,6 +125,7 @@ obj-$(CONFIG_VIDEO_BTCX)  += btcx-risc.o
 obj-$(CONFIG_VIDEOBUF2_CORE)		+= videobuf2-core.o
 obj-$(CONFIG_VIDEOBUF2_MEMOPS)		+= videobuf2-memops.o
 obj-$(CONFIG_VIDEOBUF2_VMALLOC)		+= videobuf2-vmalloc.o
+obj-$(CONFIG_VIDEOBUF2_PAGE)		+= videobuf2-page.o
 obj-$(CONFIG_VIDEOBUF2_DMA_CONTIG)	+= videobuf2-dma-contig.o
 obj-$(CONFIG_VIDEOBUF2_DMA_SG)		+= videobuf2-dma-sg.o
 
diff --git a/drivers/media/video/videobuf2-page.c b/drivers/media/video/videobuf2-page.c
new file mode 100644
index 0000000..b3f003a
--- /dev/null
+++ b/drivers/media/video/videobuf2-page.c
@@ -0,0 +1,115 @@
+/*
+ * videobuf2-page.c - page memory allocator for videobuf2
+ *
+ * Copyright (C) 2011 Canonical Ltd.
+ *
+ * Author: Ming Lei <ming.lei@canonical.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-memops.h>
+
+struct vb2_page_buf {
+	void				*vaddr;
+	unsigned long			size;
+	atomic_t			refcount;
+	struct vb2_vmarea_handler	handler;
+};
+
+static void vb2_page_put(void *buf_priv);
+
+static void *vb2_page_alloc(void *alloc_ctx, unsigned long size)
+{
+	struct vb2_page_buf *buf;
+
+	buf = kzalloc(sizeof *buf, GFP_KERNEL);
+	if (!buf)
+		return NULL;
+
+	buf->size = size;
+	buf->vaddr = (void *)__get_free_pages(GFP_KERNEL,
+			get_order(buf->size));
+	buf->handler.refcount = &buf->refcount;
+	buf->handler.put = vb2_page_put;
+	buf->handler.arg = buf;
+
+	if (!buf->vaddr) {
+		printk(KERN_ERR "page of size %ld failed\n", buf->size);
+		kfree(buf);
+		return NULL;
+	}
+
+	atomic_inc(&buf->refcount);
+	printk(KERN_DEBUG "Allocated page buffer of size %ld at vaddr=%p\n",
+			buf->size, buf->vaddr);
+
+	return buf;
+}
+
+static void vb2_page_put(void *buf_priv)
+{
+	struct vb2_page_buf *buf = buf_priv;
+
+	if (atomic_dec_and_test(&buf->refcount)) {
+		printk(KERN_DEBUG "%s: Freeing page mem at vaddr=%p\n",
+			__func__, buf->vaddr);
+		free_pages((unsigned long)buf->vaddr, get_order(buf->size));
+		kfree(buf);
+	}
+}
+
+static void *vb2_page_vaddr(void *buf_priv)
+{
+	struct vb2_page_buf *buf = buf_priv;
+
+	BUG_ON(!buf);
+
+	if (!buf->vaddr) {
+		printk(KERN_ERR "Address of an unallocated plane requested\n");
+		return NULL;
+	}
+
+	return buf->vaddr;
+}
+
+static unsigned int vb2_page_num_users(void *buf_priv)
+{
+	struct vb2_page_buf *buf = buf_priv;
+	return atomic_read(&buf->refcount);
+}
+
+static int vb2_page_mmap(void *buf_priv, struct vm_area_struct *vma)
+{
+	struct vb2_page_buf *buf = buf_priv;
+
+	if (!buf) {
+		printk(KERN_ERR "No memory to map\n");
+		return -EINVAL;
+	}
+
+	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	return vb2_mmap_pfn_range(vma, virt_to_phys(buf->vaddr),
+			buf->size, &vb2_common_vm_ops,
+			&buf->handler);
+}
+
+const struct vb2_mem_ops vb2_page_memops = {
+	.alloc		= vb2_page_alloc,
+	.put		= vb2_page_put,
+	.vaddr		= vb2_page_vaddr,
+	.mmap		= vb2_page_mmap,
+	.num_users	= vb2_page_num_users,
+};
+EXPORT_SYMBOL_GPL(vb2_page_memops);
+
+MODULE_DESCRIPTION("page memory handling routines for videobuf2");
+MODULE_AUTHOR("Ming Lei");
+MODULE_LICENSE("GPL");
diff --git a/include/media/videobuf2-page.h b/include/media/videobuf2-page.h
new file mode 100644
index 0000000..29b3e20
--- /dev/null
+++ b/include/media/videobuf2-page.h
@@ -0,0 +1,20 @@
+/*
+ * videobuf2-vmalloc.h - vmalloc memory allocator for videobuf2
+ *
+ * Copyright (C) 2010 Samsung Electronics
+ *
+ * Author: Pawel Osciak <pawel@osciak.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#ifndef _MEDIA_VIDEOBUF2_VMALLOC_H
+#define _MEDIA_VIDEOBUF2_VMALLOC_H
+
+#include <media/videobuf2-core.h>
+
+extern const struct vb2_mem_ops vb2_page_memops;
+
+#endif
-- 
1.7.5.4


^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 4/7] media: videobuf2: introduce VIDEOBUF2_PAGE memops
@ 2011-12-02 15:02   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: linux-arm-kernel

DMA contig memory resource is very limited and precious, also
accessing to it from CPU is very slow on some platform.

For some cases(such as the comming face detection driver), DMA Streaming
buffer is enough, so introduce VIDEOBUF2_PAGE to allocate continuous
physical memory but letting video device driver to handle DMA buffer mapping
and unmapping things.

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/Kconfig          |    4 +
 drivers/media/video/Makefile         |    1 +
 drivers/media/video/videobuf2-page.c |  115 ++++++++++++++++++++++++++++++++++
 include/media/videobuf2-page.h       |   20 ++++++
 4 files changed, 140 insertions(+), 0 deletions(-)
 create mode 100644 drivers/media/video/videobuf2-page.c
 create mode 100644 include/media/videobuf2-page.h

diff --git a/drivers/media/video/Kconfig b/drivers/media/video/Kconfig
index 4e8a0c4..5684a00 100644
--- a/drivers/media/video/Kconfig
+++ b/drivers/media/video/Kconfig
@@ -60,6 +60,10 @@ config VIDEOBUF2_VMALLOC
 	select VIDEOBUF2_MEMOPS
 	tristate
 
+config VIDEOBUF2_PAGE
+	select VIDEOBUF2_CORE
+	select VIDEOBUF2_MEMOPS
+	tristate
 
 config VIDEOBUF2_DMA_SG
 	#depends on HAS_DMA
diff --git a/drivers/media/video/Makefile b/drivers/media/video/Makefile
index ddeaa6c..bc797f2 100644
--- a/drivers/media/video/Makefile
+++ b/drivers/media/video/Makefile
@@ -125,6 +125,7 @@ obj-$(CONFIG_VIDEO_BTCX)  += btcx-risc.o
 obj-$(CONFIG_VIDEOBUF2_CORE)		+= videobuf2-core.o
 obj-$(CONFIG_VIDEOBUF2_MEMOPS)		+= videobuf2-memops.o
 obj-$(CONFIG_VIDEOBUF2_VMALLOC)		+= videobuf2-vmalloc.o
+obj-$(CONFIG_VIDEOBUF2_PAGE)		+= videobuf2-page.o
 obj-$(CONFIG_VIDEOBUF2_DMA_CONTIG)	+= videobuf2-dma-contig.o
 obj-$(CONFIG_VIDEOBUF2_DMA_SG)		+= videobuf2-dma-sg.o
 
diff --git a/drivers/media/video/videobuf2-page.c b/drivers/media/video/videobuf2-page.c
new file mode 100644
index 0000000..b3f003a
--- /dev/null
+++ b/drivers/media/video/videobuf2-page.c
@@ -0,0 +1,115 @@
+/*
+ * videobuf2-page.c - page memory allocator for videobuf2
+ *
+ * Copyright (C) 2011 Canonical Ltd.
+ *
+ * Author: Ming Lei <ming.lei@canonical.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-memops.h>
+
+struct vb2_page_buf {
+	void				*vaddr;
+	unsigned long			size;
+	atomic_t			refcount;
+	struct vb2_vmarea_handler	handler;
+};
+
+static void vb2_page_put(void *buf_priv);
+
+static void *vb2_page_alloc(void *alloc_ctx, unsigned long size)
+{
+	struct vb2_page_buf *buf;
+
+	buf = kzalloc(sizeof *buf, GFP_KERNEL);
+	if (!buf)
+		return NULL;
+
+	buf->size = size;
+	buf->vaddr = (void *)__get_free_pages(GFP_KERNEL,
+			get_order(buf->size));
+	buf->handler.refcount = &buf->refcount;
+	buf->handler.put = vb2_page_put;
+	buf->handler.arg = buf;
+
+	if (!buf->vaddr) {
+		printk(KERN_ERR "page of size %ld failed\n", buf->size);
+		kfree(buf);
+		return NULL;
+	}
+
+	atomic_inc(&buf->refcount);
+	printk(KERN_DEBUG "Allocated page buffer of size %ld at vaddr=%p\n",
+			buf->size, buf->vaddr);
+
+	return buf;
+}
+
+static void vb2_page_put(void *buf_priv)
+{
+	struct vb2_page_buf *buf = buf_priv;
+
+	if (atomic_dec_and_test(&buf->refcount)) {
+		printk(KERN_DEBUG "%s: Freeing page mem at vaddr=%p\n",
+			__func__, buf->vaddr);
+		free_pages((unsigned long)buf->vaddr, get_order(buf->size));
+		kfree(buf);
+	}
+}
+
+static void *vb2_page_vaddr(void *buf_priv)
+{
+	struct vb2_page_buf *buf = buf_priv;
+
+	BUG_ON(!buf);
+
+	if (!buf->vaddr) {
+		printk(KERN_ERR "Address of an unallocated plane requested\n");
+		return NULL;
+	}
+
+	return buf->vaddr;
+}
+
+static unsigned int vb2_page_num_users(void *buf_priv)
+{
+	struct vb2_page_buf *buf = buf_priv;
+	return atomic_read(&buf->refcount);
+}
+
+static int vb2_page_mmap(void *buf_priv, struct vm_area_struct *vma)
+{
+	struct vb2_page_buf *buf = buf_priv;
+
+	if (!buf) {
+		printk(KERN_ERR "No memory to map\n");
+		return -EINVAL;
+	}
+
+	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	return vb2_mmap_pfn_range(vma, virt_to_phys(buf->vaddr),
+			buf->size, &vb2_common_vm_ops,
+			&buf->handler);
+}
+
+const struct vb2_mem_ops vb2_page_memops = {
+	.alloc		= vb2_page_alloc,
+	.put		= vb2_page_put,
+	.vaddr		= vb2_page_vaddr,
+	.mmap		= vb2_page_mmap,
+	.num_users	= vb2_page_num_users,
+};
+EXPORT_SYMBOL_GPL(vb2_page_memops);
+
+MODULE_DESCRIPTION("page memory handling routines for videobuf2");
+MODULE_AUTHOR("Ming Lei");
+MODULE_LICENSE("GPL");
diff --git a/include/media/videobuf2-page.h b/include/media/videobuf2-page.h
new file mode 100644
index 0000000..29b3e20
--- /dev/null
+++ b/include/media/videobuf2-page.h
@@ -0,0 +1,20 @@
+/*
+ * videobuf2-vmalloc.h - vmalloc memory allocator for videobuf2
+ *
+ * Copyright (C) 2010 Samsung Electronics
+ *
+ * Author: Pawel Osciak <pawel@osciak.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation.
+ */
+
+#ifndef _MEDIA_VIDEOBUF2_VMALLOC_H
+#define _MEDIA_VIDEOBUF2_VMALLOC_H
+
+#include <media/videobuf2-core.h>
+
+extern const struct vb2_mem_ops vb2_page_memops;
+
+#endif
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-02 15:02 ` Ming Lei
@ 2011-12-02 15:02   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, linux-media, Ming Lei

This patch introduces two new IOCTLs and related data
structure defination which will be used by the coming
face detection video device.

The two IOCTLs and related data structure are used by
user space application to retrieve the results of face
detection. They can be called after one v4l2_buffer
has been ioctl(VIDIOC_DQBUF) and before it will be
ioctl(VIDIOC_QBUF).

The utility fdif[1] is useing the two IOCTLs to find
faces deteced in raw images or video streams.

[1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/v4l2-ioctl.c |   38 ++++++++++++++++++++
 include/linux/videodev2.h        |   70 ++++++++++++++++++++++++++++++++++++++
 include/media/v4l2-ioctl.h       |    6 +++
 3 files changed, 114 insertions(+), 0 deletions(-)

diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
index e1da8fc..fc6266f 100644
--- a/drivers/media/video/v4l2-ioctl.c
+++ b/drivers/media/video/v4l2-ioctl.c
@@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
 		dbgarg(cmd, "index=%d", b->index);
 		break;
 	}
+	case VIDIOC_G_FD_RESULT:
+	{
+		struct v4l2_fd_result *fr = arg;
+
+		if (!ops->vidioc_g_fd_result)
+			break;
+
+		ret = ops->vidioc_g_fd_result(file, fh, fr);
+
+		dbgarg(cmd, "index=%d", fr->buf_index);
+		break;
+	}
+	case VIDIOC_G_FD_COUNT:
+	{
+		struct v4l2_fd_count *fc = arg;
+
+		if (!ops->vidioc_g_fd_count)
+			break;
+
+		ret = ops->vidioc_g_fd_count(file, fh, fc);
+
+		dbgarg(cmd, "index=%d", fc->buf_index);
+		break;
+	}
 	default:
 		if (!ops->vidioc_default)
 			break;
@@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
 		}
 		break;
 	}
+
+	case VIDIOC_G_FD_RESULT: {
+		struct v4l2_fd_result *fr = parg;
+
+		if (fr->face_cnt != 0) {
+			*user_ptr = (void __user *)fr->fd;
+			*kernel_ptr = (void *)&fr->fd;
+			*array_size = sizeof(struct v4l2_fd_detection)
+				    * fr->face_cnt;
+			ret = 1;
+		}
+		break;
+
+	}
 	}
 
 	return ret;
diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
index 4b752d5..073eb4d 100644
--- a/include/linux/videodev2.h
+++ b/include/linux/videodev2.h
@@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
 	__u32			reserved[8];
 };
 
+/**
+ * struct v4l2_obj_detection
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @centerx:	return, position in x direction of detected object
+ * @centery:	return, position in y direction of detected object
+ * @angle:	return, angle of detected object
+ * 		0 deg ~ 359 deg, vertical is 0 deg, clockwise
+ * @sizex:	return, size in x direction of detected object
+ * @sizey:	return, size in y direction of detected object
+ * @confidence:	return, confidence level of detection result
+ * 		0: the heighest level, 9: the lowest level
+ * @reserved:	future extensions
+ */
+struct v4l2_obj_detection {
+	__u16		centerx;
+	__u16		centery;
+	__u16		angle;
+	__u16		sizex;
+	__u16		sizey;
+	__u16		confidence;
+	__u32		reserved[4];
+};
+
+#define V4L2_FD_HAS_LEFT_EYE	0x1
+#define V4L2_FD_HAS_RIGHT_EYE	0x2
+#define V4L2_FD_HAS_MOUTH	0x4
+#define V4L2_FD_HAS_FACE	0x8
+
+/**
+ * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
+ * @flag:	return, describe which objects are detected
+ * @left_eye:	return, left_eye position if detected
+ * @right_eye:	return, right_eye position if detected
+ * @mouth_eye:	return, mouth_eye position if detected
+ * @face:	return, face position if detected
+ */
+struct v4l2_fd_detection {
+	__u32	flag;
+	struct v4l2_obj_detection	left_eye;
+	struct v4l2_obj_detection	right_eye;
+	struct v4l2_obj_detection	mouth;
+	struct v4l2_obj_detection	face;
+};
+
+/**
+ * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @face_cnt:	return, how many faces detected from the @buf_index
+ * @fd:		return, result of faces' detection
+ */
+struct v4l2_fd_result {
+	__u32	buf_index;
+	__u32	face_cnt;
+	__u32	reserved[6];
+	struct v4l2_fd_detection *fd;
+};
+
+/**
+ * struct v4l2_fd_count - VIDIOC_G_FD_COUNT argument
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @face_cnt:	return, how many faces detected from the @buf_index
+ */
+struct v4l2_fd_count {
+	__u32	buf_index;
+	__u32	face_cnt;
+	__u32	reserved[6];
+};
+
 /*
  *	I O C T L   C O D E S   F O R   V I D E O   D E V I C E S
  *
@@ -2254,6 +2322,8 @@ struct v4l2_create_buffers {
    versions */
 #define VIDIOC_CREATE_BUFS	_IOWR('V', 92, struct v4l2_create_buffers)
 #define VIDIOC_PREPARE_BUF	_IOWR('V', 93, struct v4l2_buffer)
+#define VIDIOC_G_FD_COUNT	_IOWR('V', 94, struct v4l2_fd_count)
+#define VIDIOC_G_FD_RESULT	_IOWR('V', 95, struct v4l2_fd_result)
 
 /* Reminder: when adding new ioctls please add support for them to
    drivers/media/video/v4l2-compat-ioctl32.c as well! */
diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h
index 4d1c74a..19f03b0 100644
--- a/include/media/v4l2-ioctl.h
+++ b/include/media/v4l2-ioctl.h
@@ -270,6 +270,12 @@ struct v4l2_ioctl_ops {
 	int (*vidioc_unsubscribe_event)(struct v4l2_fh *fh,
 					struct v4l2_event_subscription *sub);
 
+	/* Face detect IOCTLs */
+	int (*vidioc_g_fd_count) (struct file *file, void *fh,
+					struct v4l2_fd_count *arg);
+	int (*vidioc_g_fd_result) (struct file *file, void *fh,
+					struct v4l2_fd_result *arg);
+
 	/* For other private ioctls */
 	long (*vidioc_default)	       (struct file *file, void *fh,
 					bool valid_prio, int cmd, void *arg);
-- 
1.7.5.4


^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-02 15:02   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: linux-arm-kernel

This patch introduces two new IOCTLs and related data
structure defination which will be used by the coming
face detection video device.

The two IOCTLs and related data structure are used by
user space application to retrieve the results of face
detection. They can be called after one v4l2_buffer
has been ioctl(VIDIOC_DQBUF) and before it will be
ioctl(VIDIOC_QBUF).

The utility fdif[1] is useing the two IOCTLs to find
faces deteced in raw images or video streams.

[1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/v4l2-ioctl.c |   38 ++++++++++++++++++++
 include/linux/videodev2.h        |   70 ++++++++++++++++++++++++++++++++++++++
 include/media/v4l2-ioctl.h       |    6 +++
 3 files changed, 114 insertions(+), 0 deletions(-)

diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
index e1da8fc..fc6266f 100644
--- a/drivers/media/video/v4l2-ioctl.c
+++ b/drivers/media/video/v4l2-ioctl.c
@@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
 		dbgarg(cmd, "index=%d", b->index);
 		break;
 	}
+	case VIDIOC_G_FD_RESULT:
+	{
+		struct v4l2_fd_result *fr = arg;
+
+		if (!ops->vidioc_g_fd_result)
+			break;
+
+		ret = ops->vidioc_g_fd_result(file, fh, fr);
+
+		dbgarg(cmd, "index=%d", fr->buf_index);
+		break;
+	}
+	case VIDIOC_G_FD_COUNT:
+	{
+		struct v4l2_fd_count *fc = arg;
+
+		if (!ops->vidioc_g_fd_count)
+			break;
+
+		ret = ops->vidioc_g_fd_count(file, fh, fc);
+
+		dbgarg(cmd, "index=%d", fc->buf_index);
+		break;
+	}
 	default:
 		if (!ops->vidioc_default)
 			break;
@@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
 		}
 		break;
 	}
+
+	case VIDIOC_G_FD_RESULT: {
+		struct v4l2_fd_result *fr = parg;
+
+		if (fr->face_cnt != 0) {
+			*user_ptr = (void __user *)fr->fd;
+			*kernel_ptr = (void *)&fr->fd;
+			*array_size = sizeof(struct v4l2_fd_detection)
+				    * fr->face_cnt;
+			ret = 1;
+		}
+		break;
+
+	}
 	}
 
 	return ret;
diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
index 4b752d5..073eb4d 100644
--- a/include/linux/videodev2.h
+++ b/include/linux/videodev2.h
@@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
 	__u32			reserved[8];
 };
 
+/**
+ * struct v4l2_obj_detection
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @centerx:	return, position in x direction of detected object
+ * @centery:	return, position in y direction of detected object
+ * @angle:	return, angle of detected object
+ * 		0 deg ~ 359 deg, vertical is 0 deg, clockwise
+ * @sizex:	return, size in x direction of detected object
+ * @sizey:	return, size in y direction of detected object
+ * @confidence:	return, confidence level of detection result
+ * 		0: the heighest level, 9: the lowest level
+ * @reserved:	future extensions
+ */
+struct v4l2_obj_detection {
+	__u16		centerx;
+	__u16		centery;
+	__u16		angle;
+	__u16		sizex;
+	__u16		sizey;
+	__u16		confidence;
+	__u32		reserved[4];
+};
+
+#define V4L2_FD_HAS_LEFT_EYE	0x1
+#define V4L2_FD_HAS_RIGHT_EYE	0x2
+#define V4L2_FD_HAS_MOUTH	0x4
+#define V4L2_FD_HAS_FACE	0x8
+
+/**
+ * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
+ * @flag:	return, describe which objects are detected
+ * @left_eye:	return, left_eye position if detected
+ * @right_eye:	return, right_eye position if detected
+ * @mouth_eye:	return, mouth_eye position if detected
+ * @face:	return, face position if detected
+ */
+struct v4l2_fd_detection {
+	__u32	flag;
+	struct v4l2_obj_detection	left_eye;
+	struct v4l2_obj_detection	right_eye;
+	struct v4l2_obj_detection	mouth;
+	struct v4l2_obj_detection	face;
+};
+
+/**
+ * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @face_cnt:	return, how many faces detected from the @buf_index
+ * @fd:		return, result of faces' detection
+ */
+struct v4l2_fd_result {
+	__u32	buf_index;
+	__u32	face_cnt;
+	__u32	reserved[6];
+	struct v4l2_fd_detection *fd;
+};
+
+/**
+ * struct v4l2_fd_count - VIDIOC_G_FD_COUNT argument
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @face_cnt:	return, how many faces detected from the @buf_index
+ */
+struct v4l2_fd_count {
+	__u32	buf_index;
+	__u32	face_cnt;
+	__u32	reserved[6];
+};
+
 /*
  *	I O C T L   C O D E S   F O R   V I D E O   D E V I C E S
  *
@@ -2254,6 +2322,8 @@ struct v4l2_create_buffers {
    versions */
 #define VIDIOC_CREATE_BUFS	_IOWR('V', 92, struct v4l2_create_buffers)
 #define VIDIOC_PREPARE_BUF	_IOWR('V', 93, struct v4l2_buffer)
+#define VIDIOC_G_FD_COUNT	_IOWR('V', 94, struct v4l2_fd_count)
+#define VIDIOC_G_FD_RESULT	_IOWR('V', 95, struct v4l2_fd_result)
 
 /* Reminder: when adding new ioctls please add support for them to
    drivers/media/video/v4l2-compat-ioctl32.c as well! */
diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h
index 4d1c74a..19f03b0 100644
--- a/include/media/v4l2-ioctl.h
+++ b/include/media/v4l2-ioctl.h
@@ -270,6 +270,12 @@ struct v4l2_ioctl_ops {
 	int (*vidioc_unsubscribe_event)(struct v4l2_fh *fh,
 					struct v4l2_event_subscription *sub);
 
+	/* Face detect IOCTLs */
+	int (*vidioc_g_fd_count) (struct file *file, void *fh,
+					struct v4l2_fd_count *arg);
+	int (*vidioc_g_fd_result) (struct file *file, void *fh,
+					struct v4l2_fd_result *arg);
+
 	/* For other private ioctls */
 	long (*vidioc_default)	       (struct file *file, void *fh,
 					bool valid_prio, int cmd, void *arg);
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-02 15:02 ` Ming Lei
@ 2011-12-02 15:02   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, linux-media, Ming Lei

This patch introduces one driver for face detection purpose.

The driver is responsible for all v4l2 stuff, buffer management
and other general things, and doesn't touch face detection hardware
directly. Several interfaces are exported to low level drivers
(such as the coming omap4 FD driver)which will communicate with
face detection hw module.

So the driver will make driving face detection hw modules more
easy.

TODO:
	- implement FD setting interfaces with v4l2 controls or
	ext controls

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/Kconfig       |    2 +
 drivers/media/video/Makefile      |    1 +
 drivers/media/video/fdif/Kconfig  |    7 +
 drivers/media/video/fdif/Makefile |    1 +
 drivers/media/video/fdif/fdif.c   |  645 +++++++++++++++++++++++++++++++++++++
 drivers/media/video/fdif/fdif.h   |  114 +++++++
 6 files changed, 770 insertions(+), 0 deletions(-)
 create mode 100644 drivers/media/video/fdif/Kconfig
 create mode 100644 drivers/media/video/fdif/Makefile
 create mode 100644 drivers/media/video/fdif/fdif.c
 create mode 100644 drivers/media/video/fdif/fdif.h

diff --git a/drivers/media/video/Kconfig b/drivers/media/video/Kconfig
index 5684a00..2b01402 100644
--- a/drivers/media/video/Kconfig
+++ b/drivers/media/video/Kconfig
@@ -1166,3 +1166,5 @@ config VIDEO_SAMSUNG_S5P_MFC
 	    MFC 5.1 driver for V4L2.
 
 endif # V4L_MEM2MEM_DRIVERS
+
+source "drivers/media/video/fdif/Kconfig"
diff --git a/drivers/media/video/Makefile b/drivers/media/video/Makefile
index bc797f2..fdf6b1a 100644
--- a/drivers/media/video/Makefile
+++ b/drivers/media/video/Makefile
@@ -197,6 +197,7 @@ obj-$(CONFIG_VIDEO_IR_I2C)  += ir-kbd-i2c.o
 obj-y	+= davinci/
 
 obj-$(CONFIG_ARCH_OMAP)	+= omap/
+obj-$(CONFIG_FDIF)	+= fdif/
 
 ccflags-y += -Idrivers/media/dvb/dvb-core
 ccflags-y += -Idrivers/media/dvb/frontends
diff --git a/drivers/media/video/fdif/Kconfig b/drivers/media/video/fdif/Kconfig
new file mode 100644
index 0000000..e214cb4
--- /dev/null
+++ b/drivers/media/video/fdif/Kconfig
@@ -0,0 +1,7 @@
+config FDIF
+	depends on VIDEO_DEV && VIDEO_V4L2
+	select VIDEOBUF2_PAGE
+	tristate "Face Detection module"
+	help
+	  The FDIF is a face detection module, which can be integrated into
+	  some SoCs to detect the location of faces in one image or video.
diff --git a/drivers/media/video/fdif/Makefile b/drivers/media/video/fdif/Makefile
new file mode 100644
index 0000000..ba1e4c8
--- /dev/null
+++ b/drivers/media/video/fdif/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_FDIF)		+= fdif.o
diff --git a/drivers/media/video/fdif/fdif.c b/drivers/media/video/fdif/fdif.c
new file mode 100644
index 0000000..84522d6
--- /dev/null
+++ b/drivers/media/video/fdif/fdif.c
@@ -0,0 +1,645 @@
+/*
+ *      fdif.c  --  face detection module driver
+ *
+ *      Copyright (C) 2011  Ming Lei (ming.lei@canonical.com)
+ *
+ *      This program is free software; you can redistribute it and/or modify
+ *      it under the terms of the GNU General Public License as published by
+ *      the Free Software Foundation; either version 2 of the License, or
+ *      (at your option) any later version.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU General Public License for more details.
+ *
+ *      You should have received a copy of the GNU General Public License
+ *      along with this program; if not, write to the Free Software
+ *      Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+/*****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/signal.h>
+#include <linux/wait.h>
+#include <linux/poll.h>
+#include <linux/mman.h>
+#include <linux/pm_runtime.h>
+#include <linux/delay.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <asm/uaccess.h>
+#include <asm/byteorder.h>
+#include <asm/io.h>
+#include "fdif.h"
+
+static unsigned debug = 0;
+module_param(debug, uint, 0644);
+MODULE_PARM_DESC(debug, "activates debug info");
+
+static LIST_HEAD(fdif_devlist);
+static unsigned video_nr = -1;
+
+int fdif_open(struct file *file)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+
+	kref_get(&dev->ref);
+	return v4l2_fh_open(file);
+}
+
+static unsigned int
+fdif_poll(struct file *file, struct poll_table_struct *wait)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	unsigned int mask = 0;
+	unsigned long flags;
+
+	poll_wait(file, &dev->fdif_dq.wq, wait);
+
+	spin_lock_irqsave(&dev->lock, flags);
+	if ((file->f_mode & FMODE_READ) &&
+		!list_empty(&dev->fdif_dq.complete))
+		mask |= POLLIN | POLLWRNORM;
+	spin_unlock_irqrestore(&dev->lock, flags);
+	return mask;
+}
+
+static int fdif_close(struct file *file)
+{
+	struct video_device  *vdev = video_devdata(file);
+	struct fdif_dev *dev = video_drvdata(file);
+	int ret;
+
+	dprintk(dev, 1, "close called (dev=%s), file %p\n",
+		video_device_node_name(vdev), file);
+
+	if (v4l2_fh_is_singular_file(file))
+		vb2_queue_release(&dev->vbq);
+
+	ret = v4l2_fh_release(file);
+	kref_put(&dev->ref, fdif_release);
+
+	return ret;
+}
+
+static int fdif_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	int ret;
+
+	dprintk(dev, 1, "mmap called, vma=0x%08lx\n", (unsigned long)vma);
+
+	ret = vb2_mmap(&dev->vbq, vma);
+	dprintk(dev, 1, "vma start=0x%08lx, size=%ld, ret=%d\n",
+		(unsigned long)vma->vm_start,
+		(unsigned long)vma->vm_end - (unsigned long)vma->vm_start,
+		ret);
+	return ret;
+}
+
+static const struct v4l2_file_operations fdif_fops = {
+	.owner		= THIS_MODULE,
+	.open           = fdif_open,
+	.release        = fdif_close,
+	.poll		= fdif_poll,
+	.unlocked_ioctl = video_ioctl2, /* V4L2 ioctl handler */
+	.mmap           = fdif_mmap,
+};
+
+/* ------------------------------------------------------------------
+	IOCTL vidioc handling
+   ------------------------------------------------------------------*/
+static int vidioc_querycap(struct file *file, void  *priv,
+					struct v4l2_capability *cap)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+
+	strcpy(cap->driver, "fdif");
+	strcpy(cap->card, "fdif");
+	strlcpy(cap->bus_info, dev->v4l2_dev.name, sizeof(cap->bus_info));
+	cap->capabilities = V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_STREAMING;
+
+	return 0;
+}
+
+static int vidioc_enum_fmt_vid_out(struct file *file, void  *priv,
+					struct v4l2_fmtdesc *f)
+{
+	struct fdif_fmt *fmt;
+	struct fdif_dev *dev = video_drvdata(file);
+
+	if (f->index >= dev->ops->fmt_cnt) {
+		return -EINVAL;
+	}
+
+	fmt = &dev->ops->table[f->index];
+
+	strlcpy(f->description, fmt->name, sizeof(f->description));
+	f->pixelformat = fmt->fourcc;
+	return 0;
+}
+
+static int vidioc_g_fmt_vid_out(struct file *file, void *priv,
+					struct v4l2_format *f)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+
+	f->fmt.pix.width        = dev->s.width;
+	f->fmt.pix.height       = dev->s.height;
+	f->fmt.pix.field        = dev->s.field;
+	f->fmt.pix.pixelformat  = dev->s.fmt->fourcc;
+	f->fmt.pix.bytesperline =
+		(f->fmt.pix.width * dev->s.fmt->depth) >> 3;
+	f->fmt.pix.sizeimage =
+		f->fmt.pix.height * f->fmt.pix.bytesperline;
+	return 0;
+}
+
+static int vidioc_reqbufs(struct file *file, void *priv,
+			  struct v4l2_requestbuffers *p)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_reqbufs(&dev->vbq, p);
+}
+
+static int vidioc_querybuf(struct file *file, void *priv, struct v4l2_buffer *p)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_querybuf(&dev->vbq, p);
+}
+
+static int vidioc_qbuf(struct file *file, void *priv, struct v4l2_buffer *p)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_qbuf(&dev->vbq, p);
+}
+
+static int vidioc_dqbuf(struct file *file, void *priv, struct v4l2_buffer *p)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_dqbuf(&dev->vbq, p, file->f_flags & O_NONBLOCK);
+}
+
+static int vidioc_streamon(struct file *file, void *priv, enum v4l2_buf_type i)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_streamon(&dev->vbq, i);
+}
+
+static int vidioc_streamoff(struct file *file, void *priv, enum v4l2_buf_type i)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_streamoff(&dev->vbq, i);
+}
+
+static int vidioc_g_fd_count(struct file *file, void *priv,
+					struct v4l2_fd_count *f)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	unsigned long flags;
+	struct v4l2_fdif_result *tmp;
+	int ret = -EINVAL;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_for_each_entry(tmp, &dev->fdif_dq.complete, list)
+		if (tmp->index == f->buf_index) {
+			f->face_cnt = tmp->face_cnt;
+			ret = 0;
+			break;
+		}
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	return ret;
+}
+
+static int vidioc_g_fd_result(struct file *file, void *priv,
+					struct v4l2_fd_result *f)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	unsigned long flags;
+	struct v4l2_fdif_result *tmp;
+	struct v4l2_fdif_result *fr = NULL;
+	unsigned int cnt = 0;
+	int ret = -EINVAL;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_for_each_entry(tmp, &dev->fdif_dq.complete, list)
+		if (tmp->index == f->buf_index) {
+			fr = tmp;
+			list_del(&tmp->list);
+			break;
+		}
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	if (fr) {
+		ret = 0;
+		cnt = min(f->face_cnt, fr->face_cnt);
+		if (cnt)
+			memcpy(f->fd, fr->faces,
+				sizeof(struct v4l2_fd_detection) * cnt);
+		f->face_cnt = cnt;
+		kfree(fr->faces);
+		kfree(fr);
+	}
+	return ret;
+}
+
+static const struct v4l2_ioctl_ops fdif_ioctl_ops = {
+	.vidioc_querycap      = vidioc_querycap,
+	.vidioc_enum_fmt_vid_out  = vidioc_enum_fmt_vid_out,
+	.vidioc_g_fmt_vid_out     = vidioc_g_fmt_vid_out,
+	.vidioc_reqbufs       = vidioc_reqbufs,
+	.vidioc_querybuf      = vidioc_querybuf,
+	.vidioc_qbuf          = vidioc_qbuf,
+	.vidioc_dqbuf         = vidioc_dqbuf,
+	.vidioc_streamon      = vidioc_streamon,
+	.vidioc_streamoff     = vidioc_streamoff,
+	.vidioc_g_fd_count    = vidioc_g_fd_count,
+	.vidioc_g_fd_result   = vidioc_g_fd_result,
+};
+
+static void fdif_vdev_release(struct video_device *vdev)
+{
+	kfree(vdev->lock);
+	video_device_release(vdev);
+}
+
+static struct video_device fdif_template = {
+	.name		= "fdif",
+	.fops           = &fdif_fops,
+	.ioctl_ops 	= &fdif_ioctl_ops,
+	.release	= fdif_vdev_release,
+};
+
+static int fdif_start_detect(struct fdif_dev *dev)
+{
+	int ret;
+
+	dprintk(dev, 1, "%s\n", __func__);
+
+	ret = dev->ops->start_detect(dev);
+
+	dprintk(dev, 1, "returning from %s, ret is %d\n",
+			__func__, ret);
+	return ret;
+}
+
+static void fdif_stop_detect(struct fdif_dev *dev)
+{
+	struct fdif_dmaqueue *dma_q = &dev->fdif_dq;
+	unsigned long flags;
+
+	dprintk(dev, 1, "%s\n", __func__);
+
+	/*stop hardware first*/
+	dev->ops->stop_detect(dev);
+
+	spin_lock_irqsave(&dev->lock, flags);
+	/* Release all active buffers */
+	while (!list_empty(&dma_q->active)) {
+		struct fdif_buffer *buf;
+		buf = list_entry(dma_q->active.next, struct fdif_buffer, list);
+		list_del(&buf->list);
+		spin_unlock_irqrestore(&dev->lock, flags);
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_ERROR);
+		dprintk(dev, 2, "[%p/%d] done\n", buf, buf->vb.v4l2_buf.index);
+		spin_lock_irqsave(&dev->lock, flags);
+	}
+
+	/* Release all complete detect result, so user space __must__ read
+	 * the results before stream off*/
+	while (!list_empty(&dma_q->complete)) {
+		struct v4l2_fdif_result *result;
+		result = list_entry(dma_q->complete.next, struct v4l2_fdif_result, list);
+		list_del(&result->list);
+		spin_unlock_irqrestore(&dev->lock, flags);
+		kfree(result->faces);
+		kfree(result);
+		dprintk(dev, 2, "[buf->index:%d] result removed\n", result->index);
+		spin_lock_irqsave(&dev->lock, flags);
+	}
+	spin_unlock_irqrestore(&dev->lock, flags);
+}
+
+/* ------------------------------------------------------------------
+	Videobuf operations
+   ------------------------------------------------------------------*/
+static int queue_setup(struct vb2_queue *vq, const struct v4l2_format *fmt,
+				unsigned int *nbuffers, unsigned int *nplanes,
+				unsigned int sizes[], void *alloc_ctxs[])
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+	unsigned long size;
+
+	dprintk(dev, 1, "%s\n", __func__);
+
+	BUG_ON(!dev->s.fmt);
+	size = (dev->s.width * dev->s.height * dev->s.fmt->depth) >> 3;
+
+	if (0 == *nbuffers)
+		*nbuffers = 2;
+	*nplanes = 1;
+	sizes[0] = size;
+
+	dprintk(dev, 1, "%s, count=%d, size=%ld\n", __func__,
+		*nbuffers, size);
+
+	return 0;
+}
+
+static int buffer_init(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+
+	/*
+	 * This callback is called once per buffer, after its allocation.
+	 *
+	 * Vivi does not allow changing format during streaming, but it is
+	 * possible to do so when streaming is paused (i.e. in streamoff state).
+	 * Buffers however are not freed when going into streamoff and so
+	 * buffer size verification has to be done in buffer_prepare, on each
+	 * qbuf.
+	 * It would be best to move verification code here to buf_init and
+	 * s_fmt though.
+	 */
+	dprintk(dev, 1, "%s vaddr=%p\n", __func__,
+			vb2_plane_vaddr(vb, 0));
+
+	return 0;
+}
+
+static int buffer_prepare(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+	struct fdif_buffer *buf = container_of(vb, struct fdif_buffer, vb);
+	unsigned long size;
+
+	dprintk(dev, 1, "%s, field=%d\n", __func__, vb->v4l2_buf.field);
+
+	BUG_ON(!dev->s.fmt);
+	size = (dev->s.width * dev->s.height * dev->s.fmt->depth) >> 3;
+	if (vb2_plane_size(vb, 0) < size) {
+		dprintk(dev, 1, "%s data will not fit into plane (%lu < %lu)\n",
+				__func__, vb2_plane_size(vb, 0), size);
+		return -EINVAL;
+	}
+
+	vb2_set_plane_payload(&buf->vb, 0, size);
+
+	return 0;
+}
+
+static int buffer_finish(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+	dprintk(dev, 1, "%s\n", __func__);
+	return 0;
+}
+
+static void buffer_cleanup(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+	dprintk(dev, 1, "%s\n", __func__);
+}
+
+static void buffer_queue(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+	struct fdif_buffer *buf = container_of(vb, struct fdif_buffer, vb);
+	struct fdif_dmaqueue *dq = &dev->fdif_dq;
+	unsigned long flags = 0;
+
+	dprintk(dev, 1, "%s vaddr:%p\n", __func__,
+			vb2_plane_vaddr(vb, 0));
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_add_tail(&buf->list, &dq->active);
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	if (vb->vb2_queue->streaming)
+		dev->ops->submit_detect(dev);
+}
+
+static int start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+	dprintk(dev, 1, "%s\n", __func__);
+	return fdif_start_detect(dev);
+}
+
+/* abort streaming and wait for last buffer */
+static int stop_streaming(struct vb2_queue *vq)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+	dprintk(dev, 1, "%s\n", __func__);
+	fdif_stop_detect(dev);
+	return 0;
+}
+
+static void fdif_lock(struct vb2_queue *vq)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+
+	mutex_lock(&dev->mutex);
+}
+
+static void fdif_unlock(struct vb2_queue *vq)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+	mutex_unlock(&dev->mutex);
+}
+static struct vb2_ops fdif_video_qops = {
+	.queue_setup		= queue_setup,
+	.buf_init		= buffer_init,
+	.buf_prepare		= buffer_prepare,
+	.buf_finish		= buffer_finish,
+	.buf_cleanup		= buffer_cleanup,
+	.buf_queue		= buffer_queue,
+	.start_streaming	= start_streaming,
+	.stop_streaming		= stop_streaming,
+	.wait_prepare		= fdif_unlock,
+	.wait_finish		= fdif_lock,
+};
+
+/*only store one detection result for one buf*/
+void fdif_add_detection(struct fdif_dev *dev,
+		struct v4l2_fdif_result *v4l2_fr)
+{
+	unsigned long flags;
+	struct v4l2_fdif_result *old = NULL;
+	struct v4l2_fdif_result *tmp;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_for_each_entry(tmp, &dev->fdif_dq.complete, list)
+		if (tmp->index == v4l2_fr->index) {
+			old = tmp;
+			list_del(&tmp->list);
+			break;
+		}
+	list_add_tail(&v4l2_fr->list, &dev->fdif_dq.complete);
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	if (old) {
+		kfree(old->faces);
+		kfree(old);
+	}
+}
+EXPORT_SYMBOL_GPL(fdif_add_detection);
+
+struct fdif_buffer *fdif_get_buffer(struct fdif_dev *dev)
+{
+	struct fdif_buffer *buf = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	if (list_empty(&dev->fdif_dq.active))
+		goto out;
+	buf = list_entry(dev->fdif_dq.active.next,
+				struct fdif_buffer, list);
+	list_del(&buf->list);
+out:
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	return buf;
+}
+EXPORT_SYMBOL_GPL(fdif_get_buffer);
+
+void fdif_release(struct kref *ref)
+{
+	struct fdif_dev *dev = container_of(ref, struct fdif_dev, ref);
+
+	v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
+		video_device_node_name(dev->vfd));
+
+	list_del(&dev->fdif_devlist);
+	video_unregister_device(dev->vfd);
+	v4l2_device_unregister(&dev->v4l2_dev);
+	kfree(dev);
+}
+EXPORT_SYMBOL_GPL(fdif_release);
+
+int fdif_create_instance(struct device *parent, int priv_size,
+		struct fdif_ops *ops, struct fdif_dev **fdif_dev)
+{
+	struct fdif_dev *dev;
+	struct video_device *vfd;
+	struct vb2_queue *q;
+	int ret, len;
+	struct mutex	*vmutex;
+
+	dev = kzalloc(sizeof(*dev) + priv_size, GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	kref_init(&dev->ref);
+	dev->ops = ops;
+	dev->dev = parent;
+
+	len = snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name),
+			"%s", "fdif");
+	dev->v4l2_dev.name[len] = '\0';
+
+	ret = v4l2_device_register(dev->dev, &dev->v4l2_dev);
+	if (ret)
+		goto free_dev;
+
+	/* initialize locks */
+	spin_lock_init(&dev->lock);
+
+	/* initialize queue */
+	q = &dev->vbq;
+	memset(q, 0, sizeof(dev->vbq));
+	q->type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+	q->io_modes = VB2_MMAP;
+	q->drv_priv = dev;
+	q->buf_struct_size = sizeof(struct fdif_buffer);
+	q->ops = &fdif_video_qops;
+	q->mem_ops = &vb2_page_memops;
+
+	vb2_queue_init(q);
+
+	mutex_init(&dev->mutex);
+
+	/* init video dma queues */
+	INIT_LIST_HEAD(&dev->fdif_dq.active);
+	INIT_LIST_HEAD(&dev->fdif_dq.complete);
+	init_waitqueue_head(&dev->fdif_dq.wq);
+
+	ret = -ENOMEM;
+	vfd = video_device_alloc();
+	if (!vfd)
+		goto unreg_dev;
+
+	*vfd = fdif_template;
+	vfd->debug = debug;
+	vfd->v4l2_dev = &dev->v4l2_dev;
+	set_bit(V4L2_FL_USE_FH_PRIO, &vfd->flags);
+
+	vmutex = kzalloc(sizeof(struct mutex), GFP_KERNEL);
+	if (!vmutex)
+		goto err_alloc_mutex;
+
+	mutex_init(vmutex);
+	/*
+	 * Provide a mutex to v4l2 core. It will be used to protect
+	 * all fops and v4l2 ioctls.
+	 */
+	vfd->lock = vmutex;
+
+	ret = video_register_device(vfd, VFL_TYPE_GRABBER, video_nr);
+	if (ret < 0)
+		goto rel_vdev;
+
+	if (video_nr != -1)
+		video_nr++;
+
+	dev->vfd = vfd;
+	video_set_drvdata(vfd, dev);
+
+	/* Now that everything is fine, let's add it to device list */
+	list_add_tail(&dev->fdif_devlist, &fdif_devlist);
+
+	v4l2_info(&dev->v4l2_dev, "V4L2 device registered as %s\n",
+		  video_device_node_name(vfd));
+
+	*fdif_dev = dev;
+	return 0;
+
+rel_vdev:
+	kfree(vmutex);
+err_alloc_mutex:
+	video_device_release(vfd);
+unreg_dev:
+	v4l2_device_unregister(&dev->v4l2_dev);
+free_dev:
+	kfree(dev);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(fdif_create_instance);
+
+static int __init fdif_init(void)
+{
+	return 0;
+}
+
+static void __exit fdif_exit(void)
+{
+}
+
+module_init(fdif_init);
+module_exit(fdif_exit);
+
+MODULE_DESCRIPTION("face detection module");
+MODULE_AUTHOR("Ming Lei");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/video/fdif/fdif.h b/drivers/media/video/fdif/fdif.h
new file mode 100644
index 0000000..ae37ab8
--- /dev/null
+++ b/drivers/media/video/fdif/fdif.h
@@ -0,0 +1,114 @@
+#ifndef _LINUX_FDIF_H
+#define _LINUX_FDIF_H
+
+#include <linux/types.h>
+#include <linux/magic.h>
+#include <linux/errno.h>
+#include <linux/kref.h>
+#include <linux/kernel.h>
+#include <linux/videodev2.h>
+#include <media/videobuf2-page.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-common.h>
+
+#define MAX_FACE_COUNT		40
+
+#define	FACE_SIZE_20_PIXELS	0
+#define	FACE_SIZE_25_PIXELS	1
+#define	FACE_SIZE_32_PIXELS	2
+#define	FACE_SIZE_40_PIXELS	3
+
+#define FACE_DIR_UP		0
+#define FACE_DIR_RIGHT		1
+#define FACE_DIR_LIFT		2
+
+struct fdif_fmt {
+	char  *name;
+	u32   fourcc;          /* v4l2 format id */
+	int   depth;
+	int   width, height;
+};
+
+struct fdif_setting {
+	struct fdif_fmt            *fmt;
+	enum v4l2_field            field;
+
+	int 			min_face_size;
+	int			face_dir;
+
+	int			startx, starty;
+	int			sizex, sizey;
+	int			lhit;
+
+	int			width, height;
+};
+
+/* buffer for one video frame */
+struct fdif_buffer {
+	/* common v4l buffer stuff -- must be first */
+	struct vb2_buffer	vb;
+	struct list_head	list;
+};
+
+
+struct v4l2_fdif_result {
+	struct list_head		list;
+	unsigned int			face_cnt;
+	struct v4l2_fd_detection	*faces;
+
+	/*v4l2 buffer index*/
+	__u32				index;
+};
+
+struct fdif_dmaqueue {
+	struct list_head	complete;
+	struct list_head	active;
+	wait_queue_head_t	wq;
+};
+
+
+struct fdif_dev {
+	struct kref		ref;
+	struct device		*dev;
+
+	struct list_head        fdif_devlist;
+	struct v4l2_device	v4l2_dev;
+	struct vb2_queue        vbq;
+	struct mutex            mutex;
+	spinlock_t		lock;
+
+	struct video_device        *vfd;
+	struct fdif_dmaqueue	fdif_dq;
+
+	/*setting*/
+	struct fdif_setting	s;
+
+	struct fdif_ops	*ops;
+
+	unsigned long	priv[0];
+};
+
+struct fdif_ops {
+	struct fdif_fmt *table;
+	int fmt_cnt;
+	int (*start_detect)(struct fdif_dev *fdif);
+	int (*stop_detect)(struct fdif_dev *fdif);
+	int (*submit_detect)(struct fdif_dev *fdif);
+};
+
+#define dprintk(dev, level, fmt, arg...) \
+	v4l2_dbg(level, debug, &dev->v4l2_dev, fmt, ## arg)
+
+
+extern int fdif_create_instance(struct device *parent, int priv_size,
+		struct fdif_ops *ops, struct fdif_dev **dev);
+extern void fdif_release(struct kref *ref);
+extern void fdif_add_detection(struct fdif_dev *dev,
+		struct v4l2_fdif_result *v4l2_fr);
+extern struct fdif_buffer *fdif_get_buffer(struct fdif_dev *dev);
+
+#endif /* _LINUX_FDIF_H */
-- 
1.7.5.4


^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-02 15:02   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: linux-arm-kernel

This patch introduces one driver for face detection purpose.

The driver is responsible for all v4l2 stuff, buffer management
and other general things, and doesn't touch face detection hardware
directly. Several interfaces are exported to low level drivers
(such as the coming omap4 FD driver)which will communicate with
face detection hw module.

So the driver will make driving face detection hw modules more
easy.

TODO:
	- implement FD setting interfaces with v4l2 controls or
	ext controls

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/Kconfig       |    2 +
 drivers/media/video/Makefile      |    1 +
 drivers/media/video/fdif/Kconfig  |    7 +
 drivers/media/video/fdif/Makefile |    1 +
 drivers/media/video/fdif/fdif.c   |  645 +++++++++++++++++++++++++++++++++++++
 drivers/media/video/fdif/fdif.h   |  114 +++++++
 6 files changed, 770 insertions(+), 0 deletions(-)
 create mode 100644 drivers/media/video/fdif/Kconfig
 create mode 100644 drivers/media/video/fdif/Makefile
 create mode 100644 drivers/media/video/fdif/fdif.c
 create mode 100644 drivers/media/video/fdif/fdif.h

diff --git a/drivers/media/video/Kconfig b/drivers/media/video/Kconfig
index 5684a00..2b01402 100644
--- a/drivers/media/video/Kconfig
+++ b/drivers/media/video/Kconfig
@@ -1166,3 +1166,5 @@ config VIDEO_SAMSUNG_S5P_MFC
 	    MFC 5.1 driver for V4L2.
 
 endif # V4L_MEM2MEM_DRIVERS
+
+source "drivers/media/video/fdif/Kconfig"
diff --git a/drivers/media/video/Makefile b/drivers/media/video/Makefile
index bc797f2..fdf6b1a 100644
--- a/drivers/media/video/Makefile
+++ b/drivers/media/video/Makefile
@@ -197,6 +197,7 @@ obj-$(CONFIG_VIDEO_IR_I2C)  += ir-kbd-i2c.o
 obj-y	+= davinci/
 
 obj-$(CONFIG_ARCH_OMAP)	+= omap/
+obj-$(CONFIG_FDIF)	+= fdif/
 
 ccflags-y += -Idrivers/media/dvb/dvb-core
 ccflags-y += -Idrivers/media/dvb/frontends
diff --git a/drivers/media/video/fdif/Kconfig b/drivers/media/video/fdif/Kconfig
new file mode 100644
index 0000000..e214cb4
--- /dev/null
+++ b/drivers/media/video/fdif/Kconfig
@@ -0,0 +1,7 @@
+config FDIF
+	depends on VIDEO_DEV && VIDEO_V4L2
+	select VIDEOBUF2_PAGE
+	tristate "Face Detection module"
+	help
+	  The FDIF is a face detection module, which can be integrated into
+	  some SoCs to detect the location of faces in one image or video.
diff --git a/drivers/media/video/fdif/Makefile b/drivers/media/video/fdif/Makefile
new file mode 100644
index 0000000..ba1e4c8
--- /dev/null
+++ b/drivers/media/video/fdif/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_FDIF)		+= fdif.o
diff --git a/drivers/media/video/fdif/fdif.c b/drivers/media/video/fdif/fdif.c
new file mode 100644
index 0000000..84522d6
--- /dev/null
+++ b/drivers/media/video/fdif/fdif.c
@@ -0,0 +1,645 @@
+/*
+ *      fdif.c  --  face detection module driver
+ *
+ *      Copyright (C) 2011  Ming Lei (ming.lei at canonical.com)
+ *
+ *      This program is free software; you can redistribute it and/or modify
+ *      it under the terms of the GNU General Public License as published by
+ *      the Free Software Foundation; either version 2 of the License, or
+ *      (at your option) any later version.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU General Public License for more details.
+ *
+ *      You should have received a copy of the GNU General Public License
+ *      along with this program; if not, write to the Free Software
+ *      Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+/*****************************************************************************/
+
+#include <linux/module.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/signal.h>
+#include <linux/wait.h>
+#include <linux/poll.h>
+#include <linux/mman.h>
+#include <linux/pm_runtime.h>
+#include <linux/delay.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <asm/uaccess.h>
+#include <asm/byteorder.h>
+#include <asm/io.h>
+#include "fdif.h"
+
+static unsigned debug = 0;
+module_param(debug, uint, 0644);
+MODULE_PARM_DESC(debug, "activates debug info");
+
+static LIST_HEAD(fdif_devlist);
+static unsigned video_nr = -1;
+
+int fdif_open(struct file *file)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+
+	kref_get(&dev->ref);
+	return v4l2_fh_open(file);
+}
+
+static unsigned int
+fdif_poll(struct file *file, struct poll_table_struct *wait)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	unsigned int mask = 0;
+	unsigned long flags;
+
+	poll_wait(file, &dev->fdif_dq.wq, wait);
+
+	spin_lock_irqsave(&dev->lock, flags);
+	if ((file->f_mode & FMODE_READ) &&
+		!list_empty(&dev->fdif_dq.complete))
+		mask |= POLLIN | POLLWRNORM;
+	spin_unlock_irqrestore(&dev->lock, flags);
+	return mask;
+}
+
+static int fdif_close(struct file *file)
+{
+	struct video_device  *vdev = video_devdata(file);
+	struct fdif_dev *dev = video_drvdata(file);
+	int ret;
+
+	dprintk(dev, 1, "close called (dev=%s), file %p\n",
+		video_device_node_name(vdev), file);
+
+	if (v4l2_fh_is_singular_file(file))
+		vb2_queue_release(&dev->vbq);
+
+	ret = v4l2_fh_release(file);
+	kref_put(&dev->ref, fdif_release);
+
+	return ret;
+}
+
+static int fdif_mmap(struct file *file, struct vm_area_struct *vma)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	int ret;
+
+	dprintk(dev, 1, "mmap called, vma=0x%08lx\n", (unsigned long)vma);
+
+	ret = vb2_mmap(&dev->vbq, vma);
+	dprintk(dev, 1, "vma start=0x%08lx, size=%ld, ret=%d\n",
+		(unsigned long)vma->vm_start,
+		(unsigned long)vma->vm_end - (unsigned long)vma->vm_start,
+		ret);
+	return ret;
+}
+
+static const struct v4l2_file_operations fdif_fops = {
+	.owner		= THIS_MODULE,
+	.open           = fdif_open,
+	.release        = fdif_close,
+	.poll		= fdif_poll,
+	.unlocked_ioctl = video_ioctl2, /* V4L2 ioctl handler */
+	.mmap           = fdif_mmap,
+};
+
+/* ------------------------------------------------------------------
+	IOCTL vidioc handling
+   ------------------------------------------------------------------*/
+static int vidioc_querycap(struct file *file, void  *priv,
+					struct v4l2_capability *cap)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+
+	strcpy(cap->driver, "fdif");
+	strcpy(cap->card, "fdif");
+	strlcpy(cap->bus_info, dev->v4l2_dev.name, sizeof(cap->bus_info));
+	cap->capabilities = V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_STREAMING;
+
+	return 0;
+}
+
+static int vidioc_enum_fmt_vid_out(struct file *file, void  *priv,
+					struct v4l2_fmtdesc *f)
+{
+	struct fdif_fmt *fmt;
+	struct fdif_dev *dev = video_drvdata(file);
+
+	if (f->index >= dev->ops->fmt_cnt) {
+		return -EINVAL;
+	}
+
+	fmt = &dev->ops->table[f->index];
+
+	strlcpy(f->description, fmt->name, sizeof(f->description));
+	f->pixelformat = fmt->fourcc;
+	return 0;
+}
+
+static int vidioc_g_fmt_vid_out(struct file *file, void *priv,
+					struct v4l2_format *f)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+
+	f->fmt.pix.width        = dev->s.width;
+	f->fmt.pix.height       = dev->s.height;
+	f->fmt.pix.field        = dev->s.field;
+	f->fmt.pix.pixelformat  = dev->s.fmt->fourcc;
+	f->fmt.pix.bytesperline =
+		(f->fmt.pix.width * dev->s.fmt->depth) >> 3;
+	f->fmt.pix.sizeimage =
+		f->fmt.pix.height * f->fmt.pix.bytesperline;
+	return 0;
+}
+
+static int vidioc_reqbufs(struct file *file, void *priv,
+			  struct v4l2_requestbuffers *p)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_reqbufs(&dev->vbq, p);
+}
+
+static int vidioc_querybuf(struct file *file, void *priv, struct v4l2_buffer *p)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_querybuf(&dev->vbq, p);
+}
+
+static int vidioc_qbuf(struct file *file, void *priv, struct v4l2_buffer *p)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_qbuf(&dev->vbq, p);
+}
+
+static int vidioc_dqbuf(struct file *file, void *priv, struct v4l2_buffer *p)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_dqbuf(&dev->vbq, p, file->f_flags & O_NONBLOCK);
+}
+
+static int vidioc_streamon(struct file *file, void *priv, enum v4l2_buf_type i)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_streamon(&dev->vbq, i);
+}
+
+static int vidioc_streamoff(struct file *file, void *priv, enum v4l2_buf_type i)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	dprintk(dev, 1, "%s\n", __func__);
+	return vb2_streamoff(&dev->vbq, i);
+}
+
+static int vidioc_g_fd_count(struct file *file, void *priv,
+					struct v4l2_fd_count *f)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	unsigned long flags;
+	struct v4l2_fdif_result *tmp;
+	int ret = -EINVAL;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_for_each_entry(tmp, &dev->fdif_dq.complete, list)
+		if (tmp->index == f->buf_index) {
+			f->face_cnt = tmp->face_cnt;
+			ret = 0;
+			break;
+		}
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	return ret;
+}
+
+static int vidioc_g_fd_result(struct file *file, void *priv,
+					struct v4l2_fd_result *f)
+{
+	struct fdif_dev *dev = video_drvdata(file);
+	unsigned long flags;
+	struct v4l2_fdif_result *tmp;
+	struct v4l2_fdif_result *fr = NULL;
+	unsigned int cnt = 0;
+	int ret = -EINVAL;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_for_each_entry(tmp, &dev->fdif_dq.complete, list)
+		if (tmp->index == f->buf_index) {
+			fr = tmp;
+			list_del(&tmp->list);
+			break;
+		}
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	if (fr) {
+		ret = 0;
+		cnt = min(f->face_cnt, fr->face_cnt);
+		if (cnt)
+			memcpy(f->fd, fr->faces,
+				sizeof(struct v4l2_fd_detection) * cnt);
+		f->face_cnt = cnt;
+		kfree(fr->faces);
+		kfree(fr);
+	}
+	return ret;
+}
+
+static const struct v4l2_ioctl_ops fdif_ioctl_ops = {
+	.vidioc_querycap      = vidioc_querycap,
+	.vidioc_enum_fmt_vid_out  = vidioc_enum_fmt_vid_out,
+	.vidioc_g_fmt_vid_out     = vidioc_g_fmt_vid_out,
+	.vidioc_reqbufs       = vidioc_reqbufs,
+	.vidioc_querybuf      = vidioc_querybuf,
+	.vidioc_qbuf          = vidioc_qbuf,
+	.vidioc_dqbuf         = vidioc_dqbuf,
+	.vidioc_streamon      = vidioc_streamon,
+	.vidioc_streamoff     = vidioc_streamoff,
+	.vidioc_g_fd_count    = vidioc_g_fd_count,
+	.vidioc_g_fd_result   = vidioc_g_fd_result,
+};
+
+static void fdif_vdev_release(struct video_device *vdev)
+{
+	kfree(vdev->lock);
+	video_device_release(vdev);
+}
+
+static struct video_device fdif_template = {
+	.name		= "fdif",
+	.fops           = &fdif_fops,
+	.ioctl_ops 	= &fdif_ioctl_ops,
+	.release	= fdif_vdev_release,
+};
+
+static int fdif_start_detect(struct fdif_dev *dev)
+{
+	int ret;
+
+	dprintk(dev, 1, "%s\n", __func__);
+
+	ret = dev->ops->start_detect(dev);
+
+	dprintk(dev, 1, "returning from %s, ret is %d\n",
+			__func__, ret);
+	return ret;
+}
+
+static void fdif_stop_detect(struct fdif_dev *dev)
+{
+	struct fdif_dmaqueue *dma_q = &dev->fdif_dq;
+	unsigned long flags;
+
+	dprintk(dev, 1, "%s\n", __func__);
+
+	/*stop hardware first*/
+	dev->ops->stop_detect(dev);
+
+	spin_lock_irqsave(&dev->lock, flags);
+	/* Release all active buffers */
+	while (!list_empty(&dma_q->active)) {
+		struct fdif_buffer *buf;
+		buf = list_entry(dma_q->active.next, struct fdif_buffer, list);
+		list_del(&buf->list);
+		spin_unlock_irqrestore(&dev->lock, flags);
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_ERROR);
+		dprintk(dev, 2, "[%p/%d] done\n", buf, buf->vb.v4l2_buf.index);
+		spin_lock_irqsave(&dev->lock, flags);
+	}
+
+	/* Release all complete detect result, so user space __must__ read
+	 * the results before stream off*/
+	while (!list_empty(&dma_q->complete)) {
+		struct v4l2_fdif_result *result;
+		result = list_entry(dma_q->complete.next, struct v4l2_fdif_result, list);
+		list_del(&result->list);
+		spin_unlock_irqrestore(&dev->lock, flags);
+		kfree(result->faces);
+		kfree(result);
+		dprintk(dev, 2, "[buf->index:%d] result removed\n", result->index);
+		spin_lock_irqsave(&dev->lock, flags);
+	}
+	spin_unlock_irqrestore(&dev->lock, flags);
+}
+
+/* ------------------------------------------------------------------
+	Videobuf operations
+   ------------------------------------------------------------------*/
+static int queue_setup(struct vb2_queue *vq, const struct v4l2_format *fmt,
+				unsigned int *nbuffers, unsigned int *nplanes,
+				unsigned int sizes[], void *alloc_ctxs[])
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+	unsigned long size;
+
+	dprintk(dev, 1, "%s\n", __func__);
+
+	BUG_ON(!dev->s.fmt);
+	size = (dev->s.width * dev->s.height * dev->s.fmt->depth) >> 3;
+
+	if (0 == *nbuffers)
+		*nbuffers = 2;
+	*nplanes = 1;
+	sizes[0] = size;
+
+	dprintk(dev, 1, "%s, count=%d, size=%ld\n", __func__,
+		*nbuffers, size);
+
+	return 0;
+}
+
+static int buffer_init(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+
+	/*
+	 * This callback is called once per buffer, after its allocation.
+	 *
+	 * Vivi does not allow changing format during streaming, but it is
+	 * possible to do so when streaming is paused (i.e. in streamoff state).
+	 * Buffers however are not freed when going into streamoff and so
+	 * buffer size verification has to be done in buffer_prepare, on each
+	 * qbuf.
+	 * It would be best to move verification code here to buf_init and
+	 * s_fmt though.
+	 */
+	dprintk(dev, 1, "%s vaddr=%p\n", __func__,
+			vb2_plane_vaddr(vb, 0));
+
+	return 0;
+}
+
+static int buffer_prepare(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+	struct fdif_buffer *buf = container_of(vb, struct fdif_buffer, vb);
+	unsigned long size;
+
+	dprintk(dev, 1, "%s, field=%d\n", __func__, vb->v4l2_buf.field);
+
+	BUG_ON(!dev->s.fmt);
+	size = (dev->s.width * dev->s.height * dev->s.fmt->depth) >> 3;
+	if (vb2_plane_size(vb, 0) < size) {
+		dprintk(dev, 1, "%s data will not fit into plane (%lu < %lu)\n",
+				__func__, vb2_plane_size(vb, 0), size);
+		return -EINVAL;
+	}
+
+	vb2_set_plane_payload(&buf->vb, 0, size);
+
+	return 0;
+}
+
+static int buffer_finish(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+	dprintk(dev, 1, "%s\n", __func__);
+	return 0;
+}
+
+static void buffer_cleanup(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+	dprintk(dev, 1, "%s\n", __func__);
+}
+
+static void buffer_queue(struct vb2_buffer *vb)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
+	struct fdif_buffer *buf = container_of(vb, struct fdif_buffer, vb);
+	struct fdif_dmaqueue *dq = &dev->fdif_dq;
+	unsigned long flags = 0;
+
+	dprintk(dev, 1, "%s vaddr:%p\n", __func__,
+			vb2_plane_vaddr(vb, 0));
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_add_tail(&buf->list, &dq->active);
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	if (vb->vb2_queue->streaming)
+		dev->ops->submit_detect(dev);
+}
+
+static int start_streaming(struct vb2_queue *vq, unsigned int count)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+	dprintk(dev, 1, "%s\n", __func__);
+	return fdif_start_detect(dev);
+}
+
+/* abort streaming and wait for last buffer */
+static int stop_streaming(struct vb2_queue *vq)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+	dprintk(dev, 1, "%s\n", __func__);
+	fdif_stop_detect(dev);
+	return 0;
+}
+
+static void fdif_lock(struct vb2_queue *vq)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+
+	mutex_lock(&dev->mutex);
+}
+
+static void fdif_unlock(struct vb2_queue *vq)
+{
+	struct fdif_dev *dev = vb2_get_drv_priv(vq);
+	mutex_unlock(&dev->mutex);
+}
+static struct vb2_ops fdif_video_qops = {
+	.queue_setup		= queue_setup,
+	.buf_init		= buffer_init,
+	.buf_prepare		= buffer_prepare,
+	.buf_finish		= buffer_finish,
+	.buf_cleanup		= buffer_cleanup,
+	.buf_queue		= buffer_queue,
+	.start_streaming	= start_streaming,
+	.stop_streaming		= stop_streaming,
+	.wait_prepare		= fdif_unlock,
+	.wait_finish		= fdif_lock,
+};
+
+/*only store one detection result for one buf*/
+void fdif_add_detection(struct fdif_dev *dev,
+		struct v4l2_fdif_result *v4l2_fr)
+{
+	unsigned long flags;
+	struct v4l2_fdif_result *old = NULL;
+	struct v4l2_fdif_result *tmp;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	list_for_each_entry(tmp, &dev->fdif_dq.complete, list)
+		if (tmp->index == v4l2_fr->index) {
+			old = tmp;
+			list_del(&tmp->list);
+			break;
+		}
+	list_add_tail(&v4l2_fr->list, &dev->fdif_dq.complete);
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	if (old) {
+		kfree(old->faces);
+		kfree(old);
+	}
+}
+EXPORT_SYMBOL_GPL(fdif_add_detection);
+
+struct fdif_buffer *fdif_get_buffer(struct fdif_dev *dev)
+{
+	struct fdif_buffer *buf = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dev->lock, flags);
+	if (list_empty(&dev->fdif_dq.active))
+		goto out;
+	buf = list_entry(dev->fdif_dq.active.next,
+				struct fdif_buffer, list);
+	list_del(&buf->list);
+out:
+	spin_unlock_irqrestore(&dev->lock, flags);
+
+	return buf;
+}
+EXPORT_SYMBOL_GPL(fdif_get_buffer);
+
+void fdif_release(struct kref *ref)
+{
+	struct fdif_dev *dev = container_of(ref, struct fdif_dev, ref);
+
+	v4l2_info(&dev->v4l2_dev, "unregistering %s\n",
+		video_device_node_name(dev->vfd));
+
+	list_del(&dev->fdif_devlist);
+	video_unregister_device(dev->vfd);
+	v4l2_device_unregister(&dev->v4l2_dev);
+	kfree(dev);
+}
+EXPORT_SYMBOL_GPL(fdif_release);
+
+int fdif_create_instance(struct device *parent, int priv_size,
+		struct fdif_ops *ops, struct fdif_dev **fdif_dev)
+{
+	struct fdif_dev *dev;
+	struct video_device *vfd;
+	struct vb2_queue *q;
+	int ret, len;
+	struct mutex	*vmutex;
+
+	dev = kzalloc(sizeof(*dev) + priv_size, GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	kref_init(&dev->ref);
+	dev->ops = ops;
+	dev->dev = parent;
+
+	len = snprintf(dev->v4l2_dev.name, sizeof(dev->v4l2_dev.name),
+			"%s", "fdif");
+	dev->v4l2_dev.name[len] = '\0';
+
+	ret = v4l2_device_register(dev->dev, &dev->v4l2_dev);
+	if (ret)
+		goto free_dev;
+
+	/* initialize locks */
+	spin_lock_init(&dev->lock);
+
+	/* initialize queue */
+	q = &dev->vbq;
+	memset(q, 0, sizeof(dev->vbq));
+	q->type = V4L2_BUF_TYPE_VIDEO_OUTPUT;
+	q->io_modes = VB2_MMAP;
+	q->drv_priv = dev;
+	q->buf_struct_size = sizeof(struct fdif_buffer);
+	q->ops = &fdif_video_qops;
+	q->mem_ops = &vb2_page_memops;
+
+	vb2_queue_init(q);
+
+	mutex_init(&dev->mutex);
+
+	/* init video dma queues */
+	INIT_LIST_HEAD(&dev->fdif_dq.active);
+	INIT_LIST_HEAD(&dev->fdif_dq.complete);
+	init_waitqueue_head(&dev->fdif_dq.wq);
+
+	ret = -ENOMEM;
+	vfd = video_device_alloc();
+	if (!vfd)
+		goto unreg_dev;
+
+	*vfd = fdif_template;
+	vfd->debug = debug;
+	vfd->v4l2_dev = &dev->v4l2_dev;
+	set_bit(V4L2_FL_USE_FH_PRIO, &vfd->flags);
+
+	vmutex = kzalloc(sizeof(struct mutex), GFP_KERNEL);
+	if (!vmutex)
+		goto err_alloc_mutex;
+
+	mutex_init(vmutex);
+	/*
+	 * Provide a mutex to v4l2 core. It will be used to protect
+	 * all fops and v4l2 ioctls.
+	 */
+	vfd->lock = vmutex;
+
+	ret = video_register_device(vfd, VFL_TYPE_GRABBER, video_nr);
+	if (ret < 0)
+		goto rel_vdev;
+
+	if (video_nr != -1)
+		video_nr++;
+
+	dev->vfd = vfd;
+	video_set_drvdata(vfd, dev);
+
+	/* Now that everything is fine, let's add it to device list */
+	list_add_tail(&dev->fdif_devlist, &fdif_devlist);
+
+	v4l2_info(&dev->v4l2_dev, "V4L2 device registered as %s\n",
+		  video_device_node_name(vfd));
+
+	*fdif_dev = dev;
+	return 0;
+
+rel_vdev:
+	kfree(vmutex);
+err_alloc_mutex:
+	video_device_release(vfd);
+unreg_dev:
+	v4l2_device_unregister(&dev->v4l2_dev);
+free_dev:
+	kfree(dev);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(fdif_create_instance);
+
+static int __init fdif_init(void)
+{
+	return 0;
+}
+
+static void __exit fdif_exit(void)
+{
+}
+
+module_init(fdif_init);
+module_exit(fdif_exit);
+
+MODULE_DESCRIPTION("face detection module");
+MODULE_AUTHOR("Ming Lei");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/video/fdif/fdif.h b/drivers/media/video/fdif/fdif.h
new file mode 100644
index 0000000..ae37ab8
--- /dev/null
+++ b/drivers/media/video/fdif/fdif.h
@@ -0,0 +1,114 @@
+#ifndef _LINUX_FDIF_H
+#define _LINUX_FDIF_H
+
+#include <linux/types.h>
+#include <linux/magic.h>
+#include <linux/errno.h>
+#include <linux/kref.h>
+#include <linux/kernel.h>
+#include <linux/videodev2.h>
+#include <media/videobuf2-page.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-fh.h>
+#include <media/v4l2-event.h>
+#include <media/v4l2-common.h>
+
+#define MAX_FACE_COUNT		40
+
+#define	FACE_SIZE_20_PIXELS	0
+#define	FACE_SIZE_25_PIXELS	1
+#define	FACE_SIZE_32_PIXELS	2
+#define	FACE_SIZE_40_PIXELS	3
+
+#define FACE_DIR_UP		0
+#define FACE_DIR_RIGHT		1
+#define FACE_DIR_LIFT		2
+
+struct fdif_fmt {
+	char  *name;
+	u32   fourcc;          /* v4l2 format id */
+	int   depth;
+	int   width, height;
+};
+
+struct fdif_setting {
+	struct fdif_fmt            *fmt;
+	enum v4l2_field            field;
+
+	int 			min_face_size;
+	int			face_dir;
+
+	int			startx, starty;
+	int			sizex, sizey;
+	int			lhit;
+
+	int			width, height;
+};
+
+/* buffer for one video frame */
+struct fdif_buffer {
+	/* common v4l buffer stuff -- must be first */
+	struct vb2_buffer	vb;
+	struct list_head	list;
+};
+
+
+struct v4l2_fdif_result {
+	struct list_head		list;
+	unsigned int			face_cnt;
+	struct v4l2_fd_detection	*faces;
+
+	/*v4l2 buffer index*/
+	__u32				index;
+};
+
+struct fdif_dmaqueue {
+	struct list_head	complete;
+	struct list_head	active;
+	wait_queue_head_t	wq;
+};
+
+
+struct fdif_dev {
+	struct kref		ref;
+	struct device		*dev;
+
+	struct list_head        fdif_devlist;
+	struct v4l2_device	v4l2_dev;
+	struct vb2_queue        vbq;
+	struct mutex            mutex;
+	spinlock_t		lock;
+
+	struct video_device        *vfd;
+	struct fdif_dmaqueue	fdif_dq;
+
+	/*setting*/
+	struct fdif_setting	s;
+
+	struct fdif_ops	*ops;
+
+	unsigned long	priv[0];
+};
+
+struct fdif_ops {
+	struct fdif_fmt *table;
+	int fmt_cnt;
+	int (*start_detect)(struct fdif_dev *fdif);
+	int (*stop_detect)(struct fdif_dev *fdif);
+	int (*submit_detect)(struct fdif_dev *fdif);
+};
+
+#define dprintk(dev, level, fmt, arg...) \
+	v4l2_dbg(level, debug, &dev->v4l2_dev, fmt, ## arg)
+
+
+extern int fdif_create_instance(struct device *parent, int priv_size,
+		struct fdif_ops *ops, struct fdif_dev **dev);
+extern void fdif_release(struct kref *ref);
+extern void fdif_add_detection(struct fdif_dev *dev,
+		struct v4l2_fdif_result *v4l2_fr);
+extern struct fdif_buffer *fdif_get_buffer(struct fdif_dev *dev);
+
+#endif /* _LINUX_FDIF_H */
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 7/7] media: video: introduce omap4 face detection module driver
  2011-12-02 15:02 ` Ming Lei
@ 2011-12-02 15:02   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, linux-media, Ming Lei

The patch introduces one face detection device driver for
driving face detection hardware on omap4[1].

Most things of the driver are dealing with omap4 face detection
hardware.

This driver is platform independent, so in theory it can
be used to drive same IP module on other platforms.

[1], Ch9 of OMAP4 Technical Reference Manual

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/fdif/Kconfig      |    6 +
 drivers/media/video/fdif/Makefile     |    1 +
 drivers/media/video/fdif/fdif_omap4.c |  663 +++++++++++++++++++++++++++++++++
 3 files changed, 670 insertions(+), 0 deletions(-)
 create mode 100644 drivers/media/video/fdif/fdif_omap4.c

diff --git a/drivers/media/video/fdif/Kconfig b/drivers/media/video/fdif/Kconfig
index e214cb4..0482a83 100644
--- a/drivers/media/video/fdif/Kconfig
+++ b/drivers/media/video/fdif/Kconfig
@@ -5,3 +5,9 @@ config FDIF
 	help
 	  The FDIF is a face detection module, which can be integrated into
 	  some SoCs to detect the location of faces in one image or video.
+
+config FDIF_OMAP4
+	depends on FDIF
+	tristate "OMAP4 Face Detection module"
+	help
+	  OMAP4 face detection support
diff --git a/drivers/media/video/fdif/Makefile b/drivers/media/video/fdif/Makefile
index ba1e4c8..3744ced 100644
--- a/drivers/media/video/fdif/Makefile
+++ b/drivers/media/video/fdif/Makefile
@@ -1 +1,2 @@
 obj-$(CONFIG_FDIF)		+= fdif.o
+obj-$(CONFIG_FDIF_OMAP4)	+= fdif_omap4.o
diff --git a/drivers/media/video/fdif/fdif_omap4.c b/drivers/media/video/fdif/fdif_omap4.c
new file mode 100644
index 0000000..956ec51
--- /dev/null
+++ b/drivers/media/video/fdif/fdif_omap4.c
@@ -0,0 +1,663 @@
+/*
+ *      fdif_omap4.c  --  face detection module driver
+ *
+ *      Copyright (C) 2011  Ming Lei (ming.lei@canonical.com)
+ *
+ *      This program is free software; you can redistribute it and/or modify
+ *      it under the terms of the GNU General Public License as published by
+ *      the Free Software Foundation; either version 2 of the License, or
+ *      (at your option) any later version.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU General Public License for more details.
+ *
+ *      You should have received a copy of the GNU General Public License
+ *      along with this program; if not, write to the Free Software
+ *      Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+/*****************************************************************************/
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/signal.h>
+#include <linux/wait.h>
+#include <linux/poll.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/delay.h>
+#include <linux/user_namespace.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include "fdif.h"
+#include <asm/uaccess.h>
+#include <asm/byteorder.h>
+#include <asm/io.h>
+
+#undef DEBUG
+
+#define PICT_SIZE_X     320
+#define PICT_SIZE_Y     240
+
+#define	WORK_MEM_SIZE	(52*1024)
+
+/* 9.5 FDIF Register Manua of TI OMAP4 TRM */
+#define FDIF_REVISION		0x0
+#define FDIF_HWINFO		0x4
+#define FDIF_SYSCONFIG		0x10
+#define SOFTRESET		(1 << 0)
+
+#define FDIF_IRQSTATUS_RAW_j	(0x24 + 2*0x10)
+#define FDIF_IRQSTATUS_j	(0x28 + 2*0x10)
+#define FDIF_IRQENABLE_SET_j	(0x2c + 2*0x10)
+#define FDIF_IRQENABLE_CLR_j	(0x30 + 2*0x10)
+#define FINISH_IRQ		(1 << 8)
+#define ERR_IRQ			(1 << 0)
+
+#define FDIF_PICADDR		0x60
+#define FDIF_CTRL		0x64
+#define CTRL_MAX_TAGS		0x0A
+
+#define FDIF_WKADDR		0x68
+#define FD_CTRL			0x80
+#define CTRL_FINISH		(1 << 2)
+#define CTRL_RUN		(1 << 1)
+#define CTRL_SRST		(1 << 0)
+
+
+#define FD_DNUM			0x84
+#define FD_DCOND		0x88
+#define FD_STARTX		0x8c
+#define FD_STARTY		0x90
+#define FD_SIZEX		0x94
+#define FD_SIZEY		0x98
+#define FD_LHIT			0x9c
+#define FD_CENTERX_i		0x160
+#define FD_CENTERY_i		0x164
+#define FD_CONFSIZE_i		0x168
+#define FD_ANGLE_i		0x16c
+
+static inline void fd_writel(void __iomem *base, u32 reg, u32 val)
+{
+	__raw_writel(val, base + reg);
+}
+
+static inline u32 fd_readl(void __iomem *base, u32 reg)
+{
+	return __raw_readl(base + reg);
+}
+
+struct fdif_qvga {
+	struct fdif_dev *dev;
+
+	/*should be removed*/
+	struct platform_device	*pdev;
+	int			irq;
+	void __iomem		*base;
+
+	void			*work_mem_addr;
+	dma_addr_t 		work_dma;
+	dma_addr_t 		pict_dma;
+	unsigned long 		pict_mem_len;
+
+	struct fdif_buffer	*pending;
+	spinlock_t		lock;
+};
+
+struct fdif_fmt qvga_fmt[] = {
+	{
+		.name		= "8  Greyscale",
+		.fourcc		= V4L2_PIX_FMT_GREY,
+		.depth		= 8,
+		.width		= PICT_SIZE_X,
+		.height		= PICT_SIZE_Y,
+	},
+};
+
+
+#ifdef DEBUG
+static void dump_fdif_setting(struct fdif_qvga *fdif, const char *func)
+{
+	printk("%s: %s\n", func, __func__);
+	printk("work mem addr:%8p\n", fdif->work_mem_addr);
+	printk("face size=%2d, face dir=%2d, lhit=%d\n",
+			fdif->dev->s.min_face_size, fdif->dev->s.face_dir,
+			fdif->dev->s.lhit);
+	printk("startx =%4d starty=%4d sizex=%4d sizey=%4d\n",
+			fdif->dev->s.startx, fdif->dev->s.starty,
+			fdif->dev->s.sizex, fdif->dev->s.sizey);
+}
+
+static void dump_fdif_results(struct v4l2_fdif_result *fdif, const char *func)
+{
+	int idx;
+
+	printk("%s: %s\n", func, __func__);
+
+	printk("found %d faces, but index:%d\n", fdif->face_cnt,
+			fdif->index);
+	for(idx=0; idx < fdif->face_cnt; idx++) {
+		struct v4l2_fd_detection *fr = &fdif->faces[idx];
+		printk("	No.%d x=%3d y=%2d sz=%2d ang=%3d conf=%2d\n",
+				idx, fr->face.centerx, fr->face.centery,
+				fr->face.sizex, fr->face.angle,
+				fr->face.confidence);
+	}
+}
+
+static void dump_fdif_regs(struct fdif_qvga *fdif, const char *func)
+{
+	printk("%s:%s\n", __func__, func);
+	printk("FDIF_CTRL=%08x FDIF_SYSCONFIG=%08x\n",
+			fd_readl(fdif->base, FDIF_CTRL),
+			fd_readl(fdif->base, FDIF_SYSCONFIG));
+	printk("FDIF_IRQSTATUS_RAW_j=%08x FDIF_IRQSTATUS_j=%08x\n",
+			fd_readl(fdif->base, FDIF_IRQSTATUS_RAW_j),
+			fd_readl(fdif->base, FDIF_IRQSTATUS_j));
+	printk("FDIF_PICADDR=%08x FDIF_WKADDR=%08x\n",
+			fd_readl(fdif->base, FDIF_PICADDR),
+			fd_readl(fdif->base, FDIF_WKADDR));
+	printk("FD_CTRL=%04x, FDIF_IRQENABLE_SET_j=%04x\n",
+			fd_readl(fdif->base, FD_CTRL),
+			fd_readl(fdif->base, FDIF_IRQENABLE_SET_j));
+}
+
+#else
+static inline void dump_fdif_setting(struct fdif_qvga *fdif, const char *func)
+{
+}
+static inline void dump_fdif_results(struct v4l2_fdif_result *fdif, const char *func)
+{
+}
+static inline void dump_fdif_regs(struct fdif_qvga *fdif, const char *func)
+{
+}
+#endif
+
+static void install_default_setting(struct fdif_qvga *fdif)
+{
+	fdif->dev->s.fmt		= &qvga_fmt[0];
+	fdif->dev->s.field		= V4L2_FIELD_NONE;
+
+	fdif->dev->s.min_face_size	= FACE_SIZE_25_PIXELS;
+	fdif->dev->s.face_dir		= FACE_DIR_UP;
+	fdif->dev->s.startx		= 0;
+	fdif->dev->s.starty		= 0;
+	fdif->dev->s.sizex		= 0x140;
+	fdif->dev->s.sizey		= 0xf0;
+	fdif->dev->s.lhit		= 0x5;
+
+	fdif->dev->s.width		= PICT_SIZE_X;
+	fdif->dev->s.height		= PICT_SIZE_Y;
+}
+
+static void commit_image_setting(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+	struct vb2_buffer *vb = &fdif->pending->vb;
+	void *pict_vaddr = vb2_plane_vaddr(vb, 0);
+
+	fdif->pict_mem_len = vb2_plane_size(vb, 0);
+	fdif->pict_dma = dma_map_single(&fdif->pdev->dev,
+				pict_vaddr,
+				fdif->pict_mem_len,
+				DMA_TO_DEVICE);
+
+	fd_writel(fdif->base, FDIF_PICADDR, fdif->pict_dma);
+
+	conf = (fdif->dev->s.min_face_size & 0x3) ||
+		((fdif->dev->s.face_dir & 0x3) << 2);
+	fd_writel(fdif->base, FD_DCOND, conf);
+
+	fd_writel(fdif->base, FD_STARTX, fdif->dev->s.startx);
+	fd_writel(fdif->base, FD_STARTY, fdif->dev->s.starty);
+	fd_writel(fdif->base, FD_SIZEX, fdif->dev->s.sizex);
+	fd_writel(fdif->base, FD_SIZEY, fdif->dev->s.sizey);
+	fd_writel(fdif->base, FD_LHIT, fdif->dev->s.lhit);
+}
+
+
+/*softreset fdif*/
+static int softreset_fdif(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+	int to = 0;
+
+	conf = fd_readl(fdif->base, FDIF_SYSCONFIG);
+	conf |= SOFTRESET;
+	fd_writel(fdif->base, FDIF_SYSCONFIG, conf);
+
+	while ((conf & SOFTRESET) && to++ < 2000) {
+		conf = fd_readl(fdif->base, FDIF_SYSCONFIG);
+		udelay(2);
+	}
+
+	if (to == 2000)
+		dev_err(&fdif->pdev->dev, "%s: reset failed\n", __func__);
+
+	return to == 2000;
+}
+
+static void __start_detect(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+
+	dump_fdif_setting(fdif, __func__);
+
+	commit_image_setting(fdif);
+
+	/*enable finish irq*/
+	conf = FINISH_IRQ;
+	fd_writel(fdif->base, FDIF_IRQENABLE_SET_j, conf);
+
+	/*set RUN flag*/
+	conf = CTRL_RUN;
+	fd_writel(fdif->base, FD_CTRL, conf);
+
+	dump_fdif_regs(fdif, __func__);
+}
+
+static void __stop_detect(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+
+	dump_fdif_regs(fdif, __func__);
+
+	dma_unmap_single(&fdif->pdev->dev, fdif->pict_dma,
+				fdif->pict_mem_len,
+				DMA_TO_DEVICE);
+	/*disable finish irq*/
+	conf = FINISH_IRQ;
+	fd_writel(fdif->base, FDIF_IRQENABLE_CLR_j, conf);
+
+	/*mark FINISH flag*/
+	conf = CTRL_FINISH;
+	fd_writel(fdif->base, FD_CTRL, conf);
+}
+
+static int read_faces(struct fdif_qvga *fdif, int is_err)
+{
+	int cnt, idx = 0;
+	struct v4l2_fdif_result *v4l2_fr;
+	struct fdif_dev *dev = fdif->dev;
+	struct fdif_buffer *buf;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fdif->lock, flags);
+
+	buf = fdif->pending;
+	if (!buf) {
+		WARN_ON(1);
+		cnt = -EIO;
+		goto out;
+	}
+
+	buf->vb.v4l2_buf.sequence++;
+
+	if (!is_err)
+		cnt = fd_readl(fdif->base, FD_DNUM) & 0x3f;
+	else
+		cnt = 0;
+
+	v4l2_fr = kzalloc(sizeof(*v4l2_fr), GFP_ATOMIC);
+	if (!v4l2_fr) {
+		cnt = -ENOMEM;
+		goto out;
+	}
+	if (cnt)
+		v4l2_fr->faces =
+			kmalloc(sizeof(struct v4l2_fd_detection) * cnt,
+				GFP_ATOMIC);
+
+	if (cnt && !v4l2_fr->faces) {
+		cnt = -ENOMEM;
+		goto out_err;
+	}
+
+	v4l2_fr->face_cnt = cnt;
+	v4l2_fr->index = buf->vb.v4l2_buf.index;
+
+	while(idx < cnt) {
+		struct v4l2_fd_detection *fr = &v4l2_fr->faces[idx];
+
+		fr->flag = V4L2_FD_HAS_FACE;
+
+		fr->face.centerx = fd_readl(fdif->base,
+				FD_CENTERX_i + idx * 0x10) & 0x1ff;
+		fr->face.centery = fd_readl(fdif->base,
+				FD_CENTERY_i + idx * 0x10) & 0xff;
+		fr->face.angle = fd_readl(fdif->base,
+				FD_ANGLE_i + idx * 0x10) & 0x1ff;
+		fr->face.sizex = fd_readl(fdif->base,
+				FD_CONFSIZE_i + idx * 0x10);
+		fr->face.confidence = (fr->face.sizex >> 8) & 0xf;
+		fr->face.sizey = fr->face.sizex = fr->face.sizex & 0xff;
+
+		idx++;
+	}
+
+	__stop_detect(fdif);
+	fdif->pending = NULL;
+	spin_unlock_irqrestore(&fdif->lock, flags);
+
+	dump_fdif_results(v4l2_fr, __func__);
+
+	/*queue the detection result to complete queue*/
+	fdif_add_detection(dev, v4l2_fr);
+
+	if (is_err)
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_ERROR);
+	else
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_DONE);
+
+	wake_up(&dev->fdif_dq.wq);
+
+	return cnt;
+
+out_err:
+	kfree(v4l2_fr);
+out:
+	spin_unlock_irqrestore(&fdif->lock, flags);
+	return cnt;
+}
+
+static int __submit_detection(struct fdif_qvga *fdif)
+{
+	struct fdif_dev *dev = fdif->dev;
+	struct fdif_buffer *buf;
+	unsigned long flags;
+	unsigned int ret = 0;
+
+	buf = fdif_get_buffer(dev);
+	if (!buf)
+		goto out;
+
+	spin_lock_irqsave(&fdif->lock, flags);
+	if (fdif->pending) {
+		spin_unlock_irqrestore(&fdif->lock, flags);
+		ret = -EBUSY;
+		goto out;
+	}
+	fdif->pending = buf;
+	__start_detect(fdif);
+	spin_unlock_irqrestore(&fdif->lock, flags);
+
+	return 0;
+
+out:
+	return ret;
+}
+
+static irqreturn_t handle_detection(int irq, void *__fdif)
+{
+	unsigned long irqsts;
+	struct fdif_qvga *fdif = __fdif;
+	irqreturn_t ret = IRQ_HANDLED;
+
+	/*clear irq status*/
+	irqsts = fd_readl(fdif->base, FDIF_IRQSTATUS_j);
+
+	if (irqsts & (FINISH_IRQ | ERR_IRQ)) {
+		int is_err = irqsts & ERR_IRQ;
+
+		fd_writel(fdif->base, FDIF_IRQSTATUS_j, irqsts);
+
+		read_faces(fdif, is_err);
+		if (is_err)
+			softreset_fdif(fdif);
+
+		__submit_detection(fdif);
+	} else {
+		ret = IRQ_NONE;
+	}
+
+	return ret;
+}
+
+static void fdif_global_init(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+	struct device *dev = &fdif->pdev->dev;
+
+	/*softreset fdif*/
+	softreset_fdif(fdif);
+
+	/*set max tags*/
+	conf = fd_readl(fdif->base, FDIF_CTRL);
+	conf &= ~0x1e;
+	conf |= (CTRL_MAX_TAGS << 1);
+	fd_writel(fdif->base, FDIF_CTRL, conf);
+
+	/*enable error irq*/
+	conf = ERR_IRQ;
+	fd_writel(fdif->base, FDIF_IRQENABLE_SET_j, conf);
+
+	fdif->work_dma = dma_map_single(dev,
+				fdif->work_mem_addr,
+		                WORK_MEM_SIZE,
+				DMA_TO_DEVICE);
+	fd_writel(fdif->base, FDIF_WKADDR, fdif->work_dma);
+}
+
+static void fdif_global_deinit(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+	struct device *dev = &fdif->pdev->dev;
+
+	/*enable error irq*/
+	conf = ERR_IRQ;
+	fd_writel(fdif->base, FDIF_IRQENABLE_CLR_j, conf);
+
+	dma_unmap_single(dev, fdif->work_dma,
+			WORK_MEM_SIZE, DMA_TO_DEVICE);
+}
+
+
+static int start_detect(struct fdif_dev *dev)
+{
+	struct fdif_qvga *fdif = dev_get_drvdata(dev->dev);
+
+	pm_runtime_get_sync(dev->dev);
+	fdif_global_init(fdif);
+
+	return  __submit_detection(fdif);
+}
+
+static int stop_detect(struct fdif_dev *dev)
+{
+	struct fdif_qvga *fdif = dev_get_drvdata(dev->dev);
+	unsigned long flags, irqsts;
+	struct fdif_buffer	*buf;
+
+	spin_lock_irqsave(&fdif->lock, flags);
+
+	/*stop current transfer first*/
+	__stop_detect(fdif);
+
+	buf = fdif->pending;
+	fdif->pending = NULL;
+
+	/*clear irq status in case that it is set*/
+	irqsts = fd_readl(fdif->base, FDIF_IRQSTATUS_j);
+	fd_writel(fdif->base, FDIF_IRQSTATUS_j, irqsts);
+
+	spin_unlock_irqrestore(&fdif->lock, flags);
+
+	if (buf)
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_ERROR);
+
+	fdif_global_deinit(fdif);
+	pm_runtime_put(dev->dev);
+	return 0;
+}
+
+static int submit_detect(struct fdif_dev *dev)
+{
+	struct fdif_qvga *fdif = dev_get_drvdata(dev->dev);
+
+	__submit_detection(fdif);
+
+	return 0;
+}
+
+static struct fdif_ops qvga_ops = {
+	.table	= qvga_fmt,
+	.fmt_cnt = 1,
+	.start_detect = start_detect,
+	.stop_detect = stop_detect,
+	.submit_detect = submit_detect,
+};
+
+static int fdif_probe(struct platform_device *pdev)
+{
+	struct device	*dev = &pdev->dev;
+	struct fdif_qvga *fdif;
+	struct fdif_dev *fdev;
+	struct resource *res;
+	int ret, order;
+
+	ret = fdif_create_instance(dev, sizeof(struct fdif_qvga),
+			&qvga_ops, &fdev);
+	if (ret) {
+		dev_err(dev, "fdif_create_instance failed:%d\n", ret);
+		goto end_probe;
+	}
+
+	fdif = (struct fdif_qvga *)fdev->priv;
+	fdif->dev = fdev;
+
+	spin_lock_init(&fdif->lock);
+	fdif->pdev = pdev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(dev, "fdif get resource failed\n");
+		ret = -ENODEV;
+		goto err_iomap;
+	}
+
+	fdif->base = ioremap(res->start, resource_size(res));
+	if (!fdif->base) {
+		dev_err(dev, "fdif ioremap failed\n");
+		ret = -ENOMEM;
+		goto err_iomap;
+	}
+
+	fdif->irq = platform_get_irq(pdev, 0);
+	if (fdif->irq < 0) {
+		dev_err(dev, "fdif get irq failed\n");
+		ret = -ENODEV;
+		goto err_get_irq;
+	}
+
+	ret = request_irq(fdif->irq, handle_detection, 0, "fdif-qvga",
+			fdif);
+	if (ret) {
+		dev_err(dev, "request_irq failed:%d\n", ret);
+		goto err_get_irq;
+	}
+
+	order = get_order(WORK_MEM_SIZE);
+	fdif->work_mem_addr = (void *)__get_free_pages(GFP_KERNEL, order);
+	if (!fdif->work_mem_addr) {
+		dev_err(dev, "fdif buffer allocation(%d) failed\n", order);
+		ret = -ENOMEM;
+		goto err_work_mem;
+	}
+
+	install_default_setting(fdif);
+
+	platform_set_drvdata(pdev, fdif);
+
+	pm_suspend_ignore_children(dev, true);
+	pm_runtime_get_sync(dev);
+	dev_info(dev, "fdif version=%8x hwinfo=%08x\n",
+			fd_readl(fdif->base, FDIF_REVISION),
+			fd_readl(fdif->base, FDIF_HWINFO));
+	pm_runtime_put(dev);
+
+	return 0;
+
+err_work_mem:
+	free_irq(fdif->irq, fdif);
+err_get_irq:
+	iounmap(fdif->base);
+err_iomap:
+	kref_put(&fdif->dev->ref, fdif_release);
+end_probe:
+	return ret;
+}
+
+static int fdif_remove(struct platform_device *pdev)
+{
+	struct fdif_qvga *fdif = platform_get_drvdata(pdev);
+	int order;
+
+	platform_set_drvdata(pdev, NULL);
+
+	free_irq(fdif->irq, fdif);
+
+	order = get_order(WORK_MEM_SIZE);
+	free_pages((unsigned long)fdif->work_mem_addr, order);
+
+	iounmap(fdif->base);
+
+	kref_put(&fdif->dev->ref, fdif_release);
+
+	return 0;
+}
+
+static int fdif_suspend(struct platform_device *pdev, pm_message_t msg)
+{
+	return 0;
+}
+
+static int fdif_resume(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static struct platform_device_id fdif_device_ids[] = {
+	{.name = "fdif"},
+	{},
+};
+
+struct platform_driver fdif_driver = {
+	.probe =	fdif_probe,
+	.remove =	fdif_remove,
+	.suspend =	fdif_suspend,
+	.resume =	fdif_resume,
+	.driver = {
+		.name  =	"fdif",
+		.owner =	THIS_MODULE,
+	},
+	.id_table = 	fdif_device_ids,
+};
+
+static int __init omap4_fdif_init(void)
+{
+	int retval;
+	retval = platform_driver_register(&fdif_driver);
+	if (retval) {
+		printk(KERN_ERR "Unable to register fdif driver\n");
+		return retval;
+	}
+	return 0;
+}
+
+static void omap4_fdif_cleanup(void)
+{
+	platform_driver_unregister(&fdif_driver);
+}
+
+module_init(omap4_fdif_init);
+module_exit(omap4_fdif_cleanup);
+
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:fdif");
+MODULE_AUTHOR("Ming Lei");
-- 
1.7.5.4


^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 7/7] media: video: introduce omap4 face detection module driver
@ 2011-12-02 15:02   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02 15:02 UTC (permalink / raw)
  To: linux-arm-kernel

The patch introduces one face detection device driver for
driving face detection hardware on omap4[1].

Most things of the driver are dealing with omap4 face detection
hardware.

This driver is platform independent, so in theory it can
be used to drive same IP module on other platforms.

[1], Ch9 of OMAP4 Technical Reference Manual

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/fdif/Kconfig      |    6 +
 drivers/media/video/fdif/Makefile     |    1 +
 drivers/media/video/fdif/fdif_omap4.c |  663 +++++++++++++++++++++++++++++++++
 3 files changed, 670 insertions(+), 0 deletions(-)
 create mode 100644 drivers/media/video/fdif/fdif_omap4.c

diff --git a/drivers/media/video/fdif/Kconfig b/drivers/media/video/fdif/Kconfig
index e214cb4..0482a83 100644
--- a/drivers/media/video/fdif/Kconfig
+++ b/drivers/media/video/fdif/Kconfig
@@ -5,3 +5,9 @@ config FDIF
 	help
 	  The FDIF is a face detection module, which can be integrated into
 	  some SoCs to detect the location of faces in one image or video.
+
+config FDIF_OMAP4
+	depends on FDIF
+	tristate "OMAP4 Face Detection module"
+	help
+	  OMAP4 face detection support
diff --git a/drivers/media/video/fdif/Makefile b/drivers/media/video/fdif/Makefile
index ba1e4c8..3744ced 100644
--- a/drivers/media/video/fdif/Makefile
+++ b/drivers/media/video/fdif/Makefile
@@ -1 +1,2 @@
 obj-$(CONFIG_FDIF)		+= fdif.o
+obj-$(CONFIG_FDIF_OMAP4)	+= fdif_omap4.o
diff --git a/drivers/media/video/fdif/fdif_omap4.c b/drivers/media/video/fdif/fdif_omap4.c
new file mode 100644
index 0000000..956ec51
--- /dev/null
+++ b/drivers/media/video/fdif/fdif_omap4.c
@@ -0,0 +1,663 @@
+/*
+ *      fdif_omap4.c  --  face detection module driver
+ *
+ *      Copyright (C) 2011  Ming Lei (ming.lei at canonical.com)
+ *
+ *      This program is free software; you can redistribute it and/or modify
+ *      it under the terms of the GNU General Public License as published by
+ *      the Free Software Foundation; either version 2 of the License, or
+ *      (at your option) any later version.
+ *
+ *      This program is distributed in the hope that it will be useful,
+ *      but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *      GNU General Public License for more details.
+ *
+ *      You should have received a copy of the GNU General Public License
+ *      along with this program; if not, write to the Free Software
+ *      Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+/*****************************************************************************/
+#include <linux/init.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/slab.h>
+#include <linux/signal.h>
+#include <linux/wait.h>
+#include <linux/poll.h>
+#include <linux/module.h>
+#include <linux/pm_runtime.h>
+#include <linux/delay.h>
+#include <linux/user_namespace.h>
+#include <linux/platform_device.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include "fdif.h"
+#include <asm/uaccess.h>
+#include <asm/byteorder.h>
+#include <asm/io.h>
+
+#undef DEBUG
+
+#define PICT_SIZE_X     320
+#define PICT_SIZE_Y     240
+
+#define	WORK_MEM_SIZE	(52*1024)
+
+/* 9.5 FDIF Register Manua of TI OMAP4 TRM */
+#define FDIF_REVISION		0x0
+#define FDIF_HWINFO		0x4
+#define FDIF_SYSCONFIG		0x10
+#define SOFTRESET		(1 << 0)
+
+#define FDIF_IRQSTATUS_RAW_j	(0x24 + 2*0x10)
+#define FDIF_IRQSTATUS_j	(0x28 + 2*0x10)
+#define FDIF_IRQENABLE_SET_j	(0x2c + 2*0x10)
+#define FDIF_IRQENABLE_CLR_j	(0x30 + 2*0x10)
+#define FINISH_IRQ		(1 << 8)
+#define ERR_IRQ			(1 << 0)
+
+#define FDIF_PICADDR		0x60
+#define FDIF_CTRL		0x64
+#define CTRL_MAX_TAGS		0x0A
+
+#define FDIF_WKADDR		0x68
+#define FD_CTRL			0x80
+#define CTRL_FINISH		(1 << 2)
+#define CTRL_RUN		(1 << 1)
+#define CTRL_SRST		(1 << 0)
+
+
+#define FD_DNUM			0x84
+#define FD_DCOND		0x88
+#define FD_STARTX		0x8c
+#define FD_STARTY		0x90
+#define FD_SIZEX		0x94
+#define FD_SIZEY		0x98
+#define FD_LHIT			0x9c
+#define FD_CENTERX_i		0x160
+#define FD_CENTERY_i		0x164
+#define FD_CONFSIZE_i		0x168
+#define FD_ANGLE_i		0x16c
+
+static inline void fd_writel(void __iomem *base, u32 reg, u32 val)
+{
+	__raw_writel(val, base + reg);
+}
+
+static inline u32 fd_readl(void __iomem *base, u32 reg)
+{
+	return __raw_readl(base + reg);
+}
+
+struct fdif_qvga {
+	struct fdif_dev *dev;
+
+	/*should be removed*/
+	struct platform_device	*pdev;
+	int			irq;
+	void __iomem		*base;
+
+	void			*work_mem_addr;
+	dma_addr_t 		work_dma;
+	dma_addr_t 		pict_dma;
+	unsigned long 		pict_mem_len;
+
+	struct fdif_buffer	*pending;
+	spinlock_t		lock;
+};
+
+struct fdif_fmt qvga_fmt[] = {
+	{
+		.name		= "8  Greyscale",
+		.fourcc		= V4L2_PIX_FMT_GREY,
+		.depth		= 8,
+		.width		= PICT_SIZE_X,
+		.height		= PICT_SIZE_Y,
+	},
+};
+
+
+#ifdef DEBUG
+static void dump_fdif_setting(struct fdif_qvga *fdif, const char *func)
+{
+	printk("%s: %s\n", func, __func__);
+	printk("work mem addr:%8p\n", fdif->work_mem_addr);
+	printk("face size=%2d, face dir=%2d, lhit=%d\n",
+			fdif->dev->s.min_face_size, fdif->dev->s.face_dir,
+			fdif->dev->s.lhit);
+	printk("startx =%4d starty=%4d sizex=%4d sizey=%4d\n",
+			fdif->dev->s.startx, fdif->dev->s.starty,
+			fdif->dev->s.sizex, fdif->dev->s.sizey);
+}
+
+static void dump_fdif_results(struct v4l2_fdif_result *fdif, const char *func)
+{
+	int idx;
+
+	printk("%s: %s\n", func, __func__);
+
+	printk("found %d faces, but index:%d\n", fdif->face_cnt,
+			fdif->index);
+	for(idx=0; idx < fdif->face_cnt; idx++) {
+		struct v4l2_fd_detection *fr = &fdif->faces[idx];
+		printk("	No.%d x=%3d y=%2d sz=%2d ang=%3d conf=%2d\n",
+				idx, fr->face.centerx, fr->face.centery,
+				fr->face.sizex, fr->face.angle,
+				fr->face.confidence);
+	}
+}
+
+static void dump_fdif_regs(struct fdif_qvga *fdif, const char *func)
+{
+	printk("%s:%s\n", __func__, func);
+	printk("FDIF_CTRL=%08x FDIF_SYSCONFIG=%08x\n",
+			fd_readl(fdif->base, FDIF_CTRL),
+			fd_readl(fdif->base, FDIF_SYSCONFIG));
+	printk("FDIF_IRQSTATUS_RAW_j=%08x FDIF_IRQSTATUS_j=%08x\n",
+			fd_readl(fdif->base, FDIF_IRQSTATUS_RAW_j),
+			fd_readl(fdif->base, FDIF_IRQSTATUS_j));
+	printk("FDIF_PICADDR=%08x FDIF_WKADDR=%08x\n",
+			fd_readl(fdif->base, FDIF_PICADDR),
+			fd_readl(fdif->base, FDIF_WKADDR));
+	printk("FD_CTRL=%04x, FDIF_IRQENABLE_SET_j=%04x\n",
+			fd_readl(fdif->base, FD_CTRL),
+			fd_readl(fdif->base, FDIF_IRQENABLE_SET_j));
+}
+
+#else
+static inline void dump_fdif_setting(struct fdif_qvga *fdif, const char *func)
+{
+}
+static inline void dump_fdif_results(struct v4l2_fdif_result *fdif, const char *func)
+{
+}
+static inline void dump_fdif_regs(struct fdif_qvga *fdif, const char *func)
+{
+}
+#endif
+
+static void install_default_setting(struct fdif_qvga *fdif)
+{
+	fdif->dev->s.fmt		= &qvga_fmt[0];
+	fdif->dev->s.field		= V4L2_FIELD_NONE;
+
+	fdif->dev->s.min_face_size	= FACE_SIZE_25_PIXELS;
+	fdif->dev->s.face_dir		= FACE_DIR_UP;
+	fdif->dev->s.startx		= 0;
+	fdif->dev->s.starty		= 0;
+	fdif->dev->s.sizex		= 0x140;
+	fdif->dev->s.sizey		= 0xf0;
+	fdif->dev->s.lhit		= 0x5;
+
+	fdif->dev->s.width		= PICT_SIZE_X;
+	fdif->dev->s.height		= PICT_SIZE_Y;
+}
+
+static void commit_image_setting(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+	struct vb2_buffer *vb = &fdif->pending->vb;
+	void *pict_vaddr = vb2_plane_vaddr(vb, 0);
+
+	fdif->pict_mem_len = vb2_plane_size(vb, 0);
+	fdif->pict_dma = dma_map_single(&fdif->pdev->dev,
+				pict_vaddr,
+				fdif->pict_mem_len,
+				DMA_TO_DEVICE);
+
+	fd_writel(fdif->base, FDIF_PICADDR, fdif->pict_dma);
+
+	conf = (fdif->dev->s.min_face_size & 0x3) ||
+		((fdif->dev->s.face_dir & 0x3) << 2);
+	fd_writel(fdif->base, FD_DCOND, conf);
+
+	fd_writel(fdif->base, FD_STARTX, fdif->dev->s.startx);
+	fd_writel(fdif->base, FD_STARTY, fdif->dev->s.starty);
+	fd_writel(fdif->base, FD_SIZEX, fdif->dev->s.sizex);
+	fd_writel(fdif->base, FD_SIZEY, fdif->dev->s.sizey);
+	fd_writel(fdif->base, FD_LHIT, fdif->dev->s.lhit);
+}
+
+
+/*softreset fdif*/
+static int softreset_fdif(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+	int to = 0;
+
+	conf = fd_readl(fdif->base, FDIF_SYSCONFIG);
+	conf |= SOFTRESET;
+	fd_writel(fdif->base, FDIF_SYSCONFIG, conf);
+
+	while ((conf & SOFTRESET) && to++ < 2000) {
+		conf = fd_readl(fdif->base, FDIF_SYSCONFIG);
+		udelay(2);
+	}
+
+	if (to == 2000)
+		dev_err(&fdif->pdev->dev, "%s: reset failed\n", __func__);
+
+	return to == 2000;
+}
+
+static void __start_detect(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+
+	dump_fdif_setting(fdif, __func__);
+
+	commit_image_setting(fdif);
+
+	/*enable finish irq*/
+	conf = FINISH_IRQ;
+	fd_writel(fdif->base, FDIF_IRQENABLE_SET_j, conf);
+
+	/*set RUN flag*/
+	conf = CTRL_RUN;
+	fd_writel(fdif->base, FD_CTRL, conf);
+
+	dump_fdif_regs(fdif, __func__);
+}
+
+static void __stop_detect(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+
+	dump_fdif_regs(fdif, __func__);
+
+	dma_unmap_single(&fdif->pdev->dev, fdif->pict_dma,
+				fdif->pict_mem_len,
+				DMA_TO_DEVICE);
+	/*disable finish irq*/
+	conf = FINISH_IRQ;
+	fd_writel(fdif->base, FDIF_IRQENABLE_CLR_j, conf);
+
+	/*mark FINISH flag*/
+	conf = CTRL_FINISH;
+	fd_writel(fdif->base, FD_CTRL, conf);
+}
+
+static int read_faces(struct fdif_qvga *fdif, int is_err)
+{
+	int cnt, idx = 0;
+	struct v4l2_fdif_result *v4l2_fr;
+	struct fdif_dev *dev = fdif->dev;
+	struct fdif_buffer *buf;
+	unsigned long flags;
+
+	spin_lock_irqsave(&fdif->lock, flags);
+
+	buf = fdif->pending;
+	if (!buf) {
+		WARN_ON(1);
+		cnt = -EIO;
+		goto out;
+	}
+
+	buf->vb.v4l2_buf.sequence++;
+
+	if (!is_err)
+		cnt = fd_readl(fdif->base, FD_DNUM) & 0x3f;
+	else
+		cnt = 0;
+
+	v4l2_fr = kzalloc(sizeof(*v4l2_fr), GFP_ATOMIC);
+	if (!v4l2_fr) {
+		cnt = -ENOMEM;
+		goto out;
+	}
+	if (cnt)
+		v4l2_fr->faces =
+			kmalloc(sizeof(struct v4l2_fd_detection) * cnt,
+				GFP_ATOMIC);
+
+	if (cnt && !v4l2_fr->faces) {
+		cnt = -ENOMEM;
+		goto out_err;
+	}
+
+	v4l2_fr->face_cnt = cnt;
+	v4l2_fr->index = buf->vb.v4l2_buf.index;
+
+	while(idx < cnt) {
+		struct v4l2_fd_detection *fr = &v4l2_fr->faces[idx];
+
+		fr->flag = V4L2_FD_HAS_FACE;
+
+		fr->face.centerx = fd_readl(fdif->base,
+				FD_CENTERX_i + idx * 0x10) & 0x1ff;
+		fr->face.centery = fd_readl(fdif->base,
+				FD_CENTERY_i + idx * 0x10) & 0xff;
+		fr->face.angle = fd_readl(fdif->base,
+				FD_ANGLE_i + idx * 0x10) & 0x1ff;
+		fr->face.sizex = fd_readl(fdif->base,
+				FD_CONFSIZE_i + idx * 0x10);
+		fr->face.confidence = (fr->face.sizex >> 8) & 0xf;
+		fr->face.sizey = fr->face.sizex = fr->face.sizex & 0xff;
+
+		idx++;
+	}
+
+	__stop_detect(fdif);
+	fdif->pending = NULL;
+	spin_unlock_irqrestore(&fdif->lock, flags);
+
+	dump_fdif_results(v4l2_fr, __func__);
+
+	/*queue the detection result to complete queue*/
+	fdif_add_detection(dev, v4l2_fr);
+
+	if (is_err)
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_ERROR);
+	else
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_DONE);
+
+	wake_up(&dev->fdif_dq.wq);
+
+	return cnt;
+
+out_err:
+	kfree(v4l2_fr);
+out:
+	spin_unlock_irqrestore(&fdif->lock, flags);
+	return cnt;
+}
+
+static int __submit_detection(struct fdif_qvga *fdif)
+{
+	struct fdif_dev *dev = fdif->dev;
+	struct fdif_buffer *buf;
+	unsigned long flags;
+	unsigned int ret = 0;
+
+	buf = fdif_get_buffer(dev);
+	if (!buf)
+		goto out;
+
+	spin_lock_irqsave(&fdif->lock, flags);
+	if (fdif->pending) {
+		spin_unlock_irqrestore(&fdif->lock, flags);
+		ret = -EBUSY;
+		goto out;
+	}
+	fdif->pending = buf;
+	__start_detect(fdif);
+	spin_unlock_irqrestore(&fdif->lock, flags);
+
+	return 0;
+
+out:
+	return ret;
+}
+
+static irqreturn_t handle_detection(int irq, void *__fdif)
+{
+	unsigned long irqsts;
+	struct fdif_qvga *fdif = __fdif;
+	irqreturn_t ret = IRQ_HANDLED;
+
+	/*clear irq status*/
+	irqsts = fd_readl(fdif->base, FDIF_IRQSTATUS_j);
+
+	if (irqsts & (FINISH_IRQ | ERR_IRQ)) {
+		int is_err = irqsts & ERR_IRQ;
+
+		fd_writel(fdif->base, FDIF_IRQSTATUS_j, irqsts);
+
+		read_faces(fdif, is_err);
+		if (is_err)
+			softreset_fdif(fdif);
+
+		__submit_detection(fdif);
+	} else {
+		ret = IRQ_NONE;
+	}
+
+	return ret;
+}
+
+static void fdif_global_init(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+	struct device *dev = &fdif->pdev->dev;
+
+	/*softreset fdif*/
+	softreset_fdif(fdif);
+
+	/*set max tags*/
+	conf = fd_readl(fdif->base, FDIF_CTRL);
+	conf &= ~0x1e;
+	conf |= (CTRL_MAX_TAGS << 1);
+	fd_writel(fdif->base, FDIF_CTRL, conf);
+
+	/*enable error irq*/
+	conf = ERR_IRQ;
+	fd_writel(fdif->base, FDIF_IRQENABLE_SET_j, conf);
+
+	fdif->work_dma = dma_map_single(dev,
+				fdif->work_mem_addr,
+		                WORK_MEM_SIZE,
+				DMA_TO_DEVICE);
+	fd_writel(fdif->base, FDIF_WKADDR, fdif->work_dma);
+}
+
+static void fdif_global_deinit(struct fdif_qvga *fdif)
+{
+	unsigned long conf;
+	struct device *dev = &fdif->pdev->dev;
+
+	/*enable error irq*/
+	conf = ERR_IRQ;
+	fd_writel(fdif->base, FDIF_IRQENABLE_CLR_j, conf);
+
+	dma_unmap_single(dev, fdif->work_dma,
+			WORK_MEM_SIZE, DMA_TO_DEVICE);
+}
+
+
+static int start_detect(struct fdif_dev *dev)
+{
+	struct fdif_qvga *fdif = dev_get_drvdata(dev->dev);
+
+	pm_runtime_get_sync(dev->dev);
+	fdif_global_init(fdif);
+
+	return  __submit_detection(fdif);
+}
+
+static int stop_detect(struct fdif_dev *dev)
+{
+	struct fdif_qvga *fdif = dev_get_drvdata(dev->dev);
+	unsigned long flags, irqsts;
+	struct fdif_buffer	*buf;
+
+	spin_lock_irqsave(&fdif->lock, flags);
+
+	/*stop current transfer first*/
+	__stop_detect(fdif);
+
+	buf = fdif->pending;
+	fdif->pending = NULL;
+
+	/*clear irq status in case that it is set*/
+	irqsts = fd_readl(fdif->base, FDIF_IRQSTATUS_j);
+	fd_writel(fdif->base, FDIF_IRQSTATUS_j, irqsts);
+
+	spin_unlock_irqrestore(&fdif->lock, flags);
+
+	if (buf)
+		vb2_buffer_done(&buf->vb, VB2_BUF_STATE_ERROR);
+
+	fdif_global_deinit(fdif);
+	pm_runtime_put(dev->dev);
+	return 0;
+}
+
+static int submit_detect(struct fdif_dev *dev)
+{
+	struct fdif_qvga *fdif = dev_get_drvdata(dev->dev);
+
+	__submit_detection(fdif);
+
+	return 0;
+}
+
+static struct fdif_ops qvga_ops = {
+	.table	= qvga_fmt,
+	.fmt_cnt = 1,
+	.start_detect = start_detect,
+	.stop_detect = stop_detect,
+	.submit_detect = submit_detect,
+};
+
+static int fdif_probe(struct platform_device *pdev)
+{
+	struct device	*dev = &pdev->dev;
+	struct fdif_qvga *fdif;
+	struct fdif_dev *fdev;
+	struct resource *res;
+	int ret, order;
+
+	ret = fdif_create_instance(dev, sizeof(struct fdif_qvga),
+			&qvga_ops, &fdev);
+	if (ret) {
+		dev_err(dev, "fdif_create_instance failed:%d\n", ret);
+		goto end_probe;
+	}
+
+	fdif = (struct fdif_qvga *)fdev->priv;
+	fdif->dev = fdev;
+
+	spin_lock_init(&fdif->lock);
+	fdif->pdev = pdev;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(dev, "fdif get resource failed\n");
+		ret = -ENODEV;
+		goto err_iomap;
+	}
+
+	fdif->base = ioremap(res->start, resource_size(res));
+	if (!fdif->base) {
+		dev_err(dev, "fdif ioremap failed\n");
+		ret = -ENOMEM;
+		goto err_iomap;
+	}
+
+	fdif->irq = platform_get_irq(pdev, 0);
+	if (fdif->irq < 0) {
+		dev_err(dev, "fdif get irq failed\n");
+		ret = -ENODEV;
+		goto err_get_irq;
+	}
+
+	ret = request_irq(fdif->irq, handle_detection, 0, "fdif-qvga",
+			fdif);
+	if (ret) {
+		dev_err(dev, "request_irq failed:%d\n", ret);
+		goto err_get_irq;
+	}
+
+	order = get_order(WORK_MEM_SIZE);
+	fdif->work_mem_addr = (void *)__get_free_pages(GFP_KERNEL, order);
+	if (!fdif->work_mem_addr) {
+		dev_err(dev, "fdif buffer allocation(%d) failed\n", order);
+		ret = -ENOMEM;
+		goto err_work_mem;
+	}
+
+	install_default_setting(fdif);
+
+	platform_set_drvdata(pdev, fdif);
+
+	pm_suspend_ignore_children(dev, true);
+	pm_runtime_get_sync(dev);
+	dev_info(dev, "fdif version=%8x hwinfo=%08x\n",
+			fd_readl(fdif->base, FDIF_REVISION),
+			fd_readl(fdif->base, FDIF_HWINFO));
+	pm_runtime_put(dev);
+
+	return 0;
+
+err_work_mem:
+	free_irq(fdif->irq, fdif);
+err_get_irq:
+	iounmap(fdif->base);
+err_iomap:
+	kref_put(&fdif->dev->ref, fdif_release);
+end_probe:
+	return ret;
+}
+
+static int fdif_remove(struct platform_device *pdev)
+{
+	struct fdif_qvga *fdif = platform_get_drvdata(pdev);
+	int order;
+
+	platform_set_drvdata(pdev, NULL);
+
+	free_irq(fdif->irq, fdif);
+
+	order = get_order(WORK_MEM_SIZE);
+	free_pages((unsigned long)fdif->work_mem_addr, order);
+
+	iounmap(fdif->base);
+
+	kref_put(&fdif->dev->ref, fdif_release);
+
+	return 0;
+}
+
+static int fdif_suspend(struct platform_device *pdev, pm_message_t msg)
+{
+	return 0;
+}
+
+static int fdif_resume(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static struct platform_device_id fdif_device_ids[] = {
+	{.name = "fdif"},
+	{},
+};
+
+struct platform_driver fdif_driver = {
+	.probe =	fdif_probe,
+	.remove =	fdif_remove,
+	.suspend =	fdif_suspend,
+	.resume =	fdif_resume,
+	.driver = {
+		.name  =	"fdif",
+		.owner =	THIS_MODULE,
+	},
+	.id_table = 	fdif_device_ids,
+};
+
+static int __init omap4_fdif_init(void)
+{
+	int retval;
+	retval = platform_driver_register(&fdif_driver);
+	if (retval) {
+		printk(KERN_ERR "Unable to register fdif driver\n");
+		return retval;
+	}
+	return 0;
+}
+
+static void omap4_fdif_cleanup(void)
+{
+	platform_driver_unregister(&fdif_driver);
+}
+
+module_init(omap4_fdif_init);
+module_exit(omap4_fdif_cleanup);
+
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:fdif");
+MODULE_AUTHOR("Ming Lei");
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 2/7] omap4: build fdif omap device from hwmod
  2011-12-02 15:02   ` Ming Lei
@ 2011-12-02 16:28     ` Aguirre, Sergio
  -1 siblings, 0 replies; 91+ messages in thread
From: Aguirre, Sergio @ 2011-12-02 16:28 UTC (permalink / raw)
  To: Ming Lei
  Cc: Mauro Carvalho Chehab, Tony Lindgren, Sylwester Nawrocki,
	Greg KH, Alan Cox, linux-omap, linux-arm-kernel, linux-kernel,
	linux-media

Hi Ming,

Thanks for the patches.

On Fri, Dec 2, 2011 at 9:02 AM, Ming Lei <ming.lei@canonical.com> wrote:
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
>  arch/arm/mach-omap2/devices.c |   33 +++++++++++++++++++++++++++++++++
>  1 files changed, 33 insertions(+), 0 deletions(-)
>
> diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
> index 1166bdc..a392af5 100644
> --- a/arch/arm/mach-omap2/devices.c
> +++ b/arch/arm/mach-omap2/devices.c
> @@ -728,6 +728,38 @@ void __init omap242x_init_mmc(struct omap_mmc_platform_data **mmc_data)
>
>  #endif
>
> +static struct platform_device* __init omap4_init_fdif(void)
> +{
> +       int id = -1;

You could remove this , as it is being used only once, and never changed.

> +       struct platform_device *pd;
> +       struct omap_hwmod *oh;
> +       const char *dev_name = "fdif";
> +
> +       oh = omap_hwmod_lookup("fdif");
> +       if (!oh) {
> +               pr_err("Could not look up fdif hwmod\n");
> +               return NULL;
> +       }
> +
> +       pd = omap_device_build(dev_name, id, oh, NULL, 0, NULL, 0, 0);

Just do:

pd = omap_device_build(dev_name, -1, oh, NULL, 0, NULL, 0, 0);

> +       WARN(IS_ERR(pd), "Can't build omap_device for %s.\n",
> +                               dev_name);
> +       return pd;
> +}
> +
> +static void __init omap_init_fdif(void)
> +{
> +       if (cpu_is_omap44xx()) {
> +               struct platform_device *pd;
> +
> +               pd = omap4_init_fdif();
> +               if (!pd)
> +                       return;
> +
> +               pm_runtime_enable(&pd->dev);
> +       }
> +}

IMHO, you could reduce 1 level of indentation here, like this:

static void __init omap_init_fdif(void)
{
	struct platform_device *pd;

	if (!cpu_is_omap44xx())
		return;

	pd = omap4_init_fdif();
	if (!pd)
		return;

	pm_runtime_enable(&pd->dev);
}

Regards,
Sergio

> +
>  /*-------------------------------------------------------------------------*/
>
>  #if defined(CONFIG_HDQ_MASTER_OMAP) || defined(CONFIG_HDQ_MASTER_OMAP_MODULE)
> @@ -808,6 +840,7 @@ static int __init omap2_init_devices(void)
>        omap_init_sham();
>        omap_init_aes();
>        omap_init_vout();
> +       omap_init_fdif();
>
>        return 0;
>  }
> --
> 1.7.5.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 2/7] omap4: build fdif omap device from hwmod
@ 2011-12-02 16:28     ` Aguirre, Sergio
  0 siblings, 0 replies; 91+ messages in thread
From: Aguirre, Sergio @ 2011-12-02 16:28 UTC (permalink / raw)
  To: linux-arm-kernel

Hi Ming,

Thanks for the patches.

On Fri, Dec 2, 2011 at 9:02 AM, Ming Lei <ming.lei@canonical.com> wrote:
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
> ?arch/arm/mach-omap2/devices.c | ? 33 +++++++++++++++++++++++++++++++++
> ?1 files changed, 33 insertions(+), 0 deletions(-)
>
> diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
> index 1166bdc..a392af5 100644
> --- a/arch/arm/mach-omap2/devices.c
> +++ b/arch/arm/mach-omap2/devices.c
> @@ -728,6 +728,38 @@ void __init omap242x_init_mmc(struct omap_mmc_platform_data **mmc_data)
>
> ?#endif
>
> +static struct platform_device* __init omap4_init_fdif(void)
> +{
> + ? ? ? int id = -1;

You could remove this , as it is being used only once, and never changed.

> + ? ? ? struct platform_device *pd;
> + ? ? ? struct omap_hwmod *oh;
> + ? ? ? const char *dev_name = "fdif";
> +
> + ? ? ? oh = omap_hwmod_lookup("fdif");
> + ? ? ? if (!oh) {
> + ? ? ? ? ? ? ? pr_err("Could not look up fdif hwmod\n");
> + ? ? ? ? ? ? ? return NULL;
> + ? ? ? }
> +
> + ? ? ? pd = omap_device_build(dev_name, id, oh, NULL, 0, NULL, 0, 0);

Just do:

pd = omap_device_build(dev_name, -1, oh, NULL, 0, NULL, 0, 0);

> + ? ? ? WARN(IS_ERR(pd), "Can't build omap_device for %s.\n",
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? dev_name);
> + ? ? ? return pd;
> +}
> +
> +static void __init omap_init_fdif(void)
> +{
> + ? ? ? if (cpu_is_omap44xx()) {
> + ? ? ? ? ? ? ? struct platform_device *pd;
> +
> + ? ? ? ? ? ? ? pd = omap4_init_fdif();
> + ? ? ? ? ? ? ? if (!pd)
> + ? ? ? ? ? ? ? ? ? ? ? return;
> +
> + ? ? ? ? ? ? ? pm_runtime_enable(&pd->dev);
> + ? ? ? }
> +}

IMHO, you could reduce 1 level of indentation here, like this:

static void __init omap_init_fdif(void)
{
	struct platform_device *pd;

	if (!cpu_is_omap44xx())
		return;

	pd = omap4_init_fdif();
	if (!pd)
		return;

	pm_runtime_enable(&pd->dev);
}

Regards,
Sergio

> +
> ?/*-------------------------------------------------------------------------*/
>
> ?#if defined(CONFIG_HDQ_MASTER_OMAP) || defined(CONFIG_HDQ_MASTER_OMAP_MODULE)
> @@ -808,6 +840,7 @@ static int __init omap2_init_devices(void)
> ? ? ? ?omap_init_sham();
> ? ? ? ?omap_init_aes();
> ? ? ? ?omap_init_vout();
> + ? ? ? omap_init_fdif();
>
> ? ? ? ?return 0;
> ?}
> --
> 1.7.5.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at ?http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 2/7] omap4: build fdif omap device from hwmod
  2011-12-02 16:28     ` Aguirre, Sergio
@ 2011-12-05  4:27       ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-05  4:27 UTC (permalink / raw)
  To: Aguirre, Sergio
  Cc: Mauro Carvalho Chehab, Tony Lindgren, Sylwester Nawrocki,
	Greg KH, Alan Cox, linux-omap, linux-arm-kernel, linux-kernel,
	linux-media

Hi,

On Sat, Dec 3, 2011 at 12:28 AM, Aguirre, Sergio <saaguirre@ti.com> wrote:
> Hi Ming,
>
> Thanks for the patches.

Thanks for your review.

> On Fri, Dec 2, 2011 at 9:02 AM, Ming Lei <ming.lei@canonical.com> wrote:
>> Signed-off-by: Ming Lei <ming.lei@canonical.com>
>> ---
>>  arch/arm/mach-omap2/devices.c |   33 +++++++++++++++++++++++++++++++++
>>  1 files changed, 33 insertions(+), 0 deletions(-)
>>
>> diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
>> index 1166bdc..a392af5 100644
>> --- a/arch/arm/mach-omap2/devices.c
>> +++ b/arch/arm/mach-omap2/devices.c
>> @@ -728,6 +728,38 @@ void __init omap242x_init_mmc(struct omap_mmc_platform_data **mmc_data)
>>
>>  #endif
>>
>> +static struct platform_device* __init omap4_init_fdif(void)
>> +{
>> +       int id = -1;
>
> You could remove this , as it is being used only once, and never changed.

Yes.

>
>> +       struct platform_device *pd;
>> +       struct omap_hwmod *oh;
>> +       const char *dev_name = "fdif";
>> +
>> +       oh = omap_hwmod_lookup("fdif");
>> +       if (!oh) {
>> +               pr_err("Could not look up fdif hwmod\n");
>> +               return NULL;
>> +       }
>> +
>> +       pd = omap_device_build(dev_name, id, oh, NULL, 0, NULL, 0, 0);
>
> Just do:
>
> pd = omap_device_build(dev_name, -1, oh, NULL, 0, NULL, 0, 0);
>
>> +       WARN(IS_ERR(pd), "Can't build omap_device for %s.\n",
>> +                               dev_name);
>> +       return pd;
>> +}
>> +
>> +static void __init omap_init_fdif(void)
>> +{
>> +       if (cpu_is_omap44xx()) {
>> +               struct platform_device *pd;
>> +
>> +               pd = omap4_init_fdif();
>> +               if (!pd)
>> +                       return;
>> +
>> +               pm_runtime_enable(&pd->dev);
>> +       }
>> +}
>
> IMHO, you could reduce 1 level of indentation here, like this:
>
> static void __init omap_init_fdif(void)
> {
>        struct platform_device *pd;
>
>        if (!cpu_is_omap44xx())
>                return;
>
>        pd = omap4_init_fdif();
>        if (!pd)
>                return;
>
>        pm_runtime_enable(&pd->dev);
> }

OK, will take this.

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 2/7] omap4: build fdif omap device from hwmod
@ 2011-12-05  4:27       ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-05  4:27 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Sat, Dec 3, 2011 at 12:28 AM, Aguirre, Sergio <saaguirre@ti.com> wrote:
> Hi Ming,
>
> Thanks for the patches.

Thanks for your review.

> On Fri, Dec 2, 2011 at 9:02 AM, Ming Lei <ming.lei@canonical.com> wrote:
>> Signed-off-by: Ming Lei <ming.lei@canonical.com>
>> ---
>> ?arch/arm/mach-omap2/devices.c | ? 33 +++++++++++++++++++++++++++++++++
>> ?1 files changed, 33 insertions(+), 0 deletions(-)
>>
>> diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c
>> index 1166bdc..a392af5 100644
>> --- a/arch/arm/mach-omap2/devices.c
>> +++ b/arch/arm/mach-omap2/devices.c
>> @@ -728,6 +728,38 @@ void __init omap242x_init_mmc(struct omap_mmc_platform_data **mmc_data)
>>
>> ?#endif
>>
>> +static struct platform_device* __init omap4_init_fdif(void)
>> +{
>> + ? ? ? int id = -1;
>
> You could remove this , as it is being used only once, and never changed.

Yes.

>
>> + ? ? ? struct platform_device *pd;
>> + ? ? ? struct omap_hwmod *oh;
>> + ? ? ? const char *dev_name = "fdif";
>> +
>> + ? ? ? oh = omap_hwmod_lookup("fdif");
>> + ? ? ? if (!oh) {
>> + ? ? ? ? ? ? ? pr_err("Could not look up fdif hwmod\n");
>> + ? ? ? ? ? ? ? return NULL;
>> + ? ? ? }
>> +
>> + ? ? ? pd = omap_device_build(dev_name, id, oh, NULL, 0, NULL, 0, 0);
>
> Just do:
>
> pd = omap_device_build(dev_name, -1, oh, NULL, 0, NULL, 0, 0);
>
>> + ? ? ? WARN(IS_ERR(pd), "Can't build omap_device for %s.\n",
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? dev_name);
>> + ? ? ? return pd;
>> +}
>> +
>> +static void __init omap_init_fdif(void)
>> +{
>> + ? ? ? if (cpu_is_omap44xx()) {
>> + ? ? ? ? ? ? ? struct platform_device *pd;
>> +
>> + ? ? ? ? ? ? ? pd = omap4_init_fdif();
>> + ? ? ? ? ? ? ? if (!pd)
>> + ? ? ? ? ? ? ? ? ? ? ? return;
>> +
>> + ? ? ? ? ? ? ? pm_runtime_enable(&pd->dev);
>> + ? ? ? }
>> +}
>
> IMHO, you could reduce 1 level of indentation here, like this:
>
> static void __init omap_init_fdif(void)
> {
> ? ? ? ?struct platform_device *pd;
>
> ? ? ? ?if (!cpu_is_omap44xx())
> ? ? ? ? ? ? ? ?return;
>
> ? ? ? ?pd = omap4_init_fdif();
> ? ? ? ?if (!pd)
> ? ? ? ? ? ? ? ?return;
>
> ? ? ? ?pm_runtime_enable(&pd->dev);
> }

OK, will take this.

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-02 15:02   ` Ming Lei
@ 2011-12-05 21:55     ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-05 21:55 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi Ming,

(I've pruned the Cc list, leaving just the mailing lists)

On 12/02/2011 04:02 PM, Ming Lei wrote:
> This patch introduces one driver for face detection purpose.
> 
> The driver is responsible for all v4l2 stuff, buffer management
> and other general things, and doesn't touch face detection hardware
> directly. Several interfaces are exported to low level drivers
> (such as the coming omap4 FD driver)which will communicate with
> face detection hw module.
> 
> So the driver will make driving face detection hw modules more
> easy.


I would hold on for a moment on implementing generic face detection
module which is based on the V4L2 video device interface. We need to
first define an API that would be also usable at sub-device interface
level (http://linuxtv.org/downloads/v4l-dvb-apis/subdev.html).
AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
reasonable to use the videodev interface for passing data to the kernel
from user space.

But there might be face detection devices that accept data from other
H/W modules, e.g. transferred through SoC internal data buses between
image processing pipeline blocks. Thus any new interfaces need to be
designed with such devices in mind.

Also the face detection hardware block might now have an input DMA
engine in it, the data could be fed from memory through some other
subsystem (e.g. resize/colour converter). Then the driver for that
subsystem would implement a video node.

I'm for leaving the buffer handling details for individual drivers
and focusing on a standard interface for applications, i.e. new
ioctl(s) and controls.

> 
> TODO:
> 	- implement FD setting interfaces with v4l2 controls or
> 	ext controls
> 
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
>  drivers/media/video/Kconfig       |    2 +
>  drivers/media/video/Makefile      |    1 +
>  drivers/media/video/fdif/Kconfig  |    7 +
>  drivers/media/video/fdif/Makefile |    1 +
>  drivers/media/video/fdif/fdif.c   |  645 +++++++++++++++++++++++++++++++++++++
>  drivers/media/video/fdif/fdif.h   |  114 +++++++
>  6 files changed, 770 insertions(+), 0 deletions(-)
>  create mode 100644 drivers/media/video/fdif/Kconfig
>  create mode 100644 drivers/media/video/fdif/Makefile
>  create mode 100644 drivers/media/video/fdif/fdif.c
>  create mode 100644 drivers/media/video/fdif/fdif.h

[...]

> diff --git a/drivers/media/video/fdif/fdif.h b/drivers/media/video/fdif/fdif.h
> new file mode 100644
> index 0000000..ae37ab8
> --- /dev/null
> +++ b/drivers/media/video/fdif/fdif.h
> @@ -0,0 +1,114 @@
> +#ifndef _LINUX_FDIF_H
> +#define _LINUX_FDIF_H
> +
> +#include <linux/types.h>
> +#include <linux/magic.h>
> +#include <linux/errno.h>
> +#include <linux/kref.h>
> +#include <linux/kernel.h>
> +#include <linux/videodev2.h>
> +#include <media/videobuf2-page.h>
> +#include <media/v4l2-device.h>
> +#include <media/v4l2-ioctl.h>
> +#include <media/v4l2-ctrls.h>
> +#include <media/v4l2-fh.h>
> +#include <media/v4l2-event.h>
> +#include <media/v4l2-common.h>
> +
> +#define MAX_FACE_COUNT		40
> +
> +#define	FACE_SIZE_20_PIXELS	0
> +#define	FACE_SIZE_25_PIXELS	1
> +#define	FACE_SIZE_32_PIXELS	2
> +#define	FACE_SIZE_40_PIXELS	3

This is still OMAP4 FDIF specific, we need to think about v4l2 controls
for this. An ideal would be a menu control type that supports pixel size
(width/height), but unfortunately something like this isn't available
in v4l2 yet.

> +
> +#define FACE_DIR_UP		0
> +#define FACE_DIR_RIGHT		1
> +#define FACE_DIR_LIFT		2
> +
> +struct fdif_fmt {
> +	char  *name;
> +	u32   fourcc;          /* v4l2 format id */
> +	int   depth;
> +	int   width, height;

Could width/height be negative ? I don't think it's the case for pixel
resolution. The more proper data type would be u32.

Please refer to struct v4l2_pix_format or struct v4l2_rect.

> +};
> +
> +struct fdif_setting {
> +	struct fdif_fmt            *fmt;
> +	enum v4l2_field            field;
> +
> +	int 			min_face_size;
> +	int			face_dir;
> +
> +	int			startx, starty;

s32

> +	int			sizex, sizey;

u32

> +	int			lhit;
> +
> +	int			width, height;

u32

> +};
> +
> +/* buffer for one video frame */
> +struct fdif_buffer {
> +	/* common v4l buffer stuff -- must be first */
> +	struct vb2_buffer	vb;
> +	struct list_head	list;
> +};
> +
> +
> +struct v4l2_fdif_result {
> +	struct list_head		list;
> +	unsigned int			face_cnt;
> +	struct v4l2_fd_detection	*faces;
> +
> +	/*v4l2 buffer index*/
> +	__u32				index;
> +};
> +
> +struct fdif_dmaqueue {
> +	struct list_head	complete;
> +	struct list_head	active;
> +	wait_queue_head_t	wq;
> +};
> +
> +
> +struct fdif_dev {
> +	struct kref		ref;
> +	struct device		*dev;
> +
> +	struct list_head        fdif_devlist;
> +	struct v4l2_device	v4l2_dev;
> +	struct vb2_queue        vbq;
> +	struct mutex            mutex;
> +	spinlock_t		lock;
> +
> +	struct video_device        *vfd;
> +	struct fdif_dmaqueue	fdif_dq;
> +
> +	/*setting*/
> +	struct fdif_setting	s;

yy, please make it more descriptive. e.g.

	struct fdif_config	config;

> +
> +	struct fdif_ops	*ops;
> +
> +	unsigned long	priv[0];
> +};
> +
[...]

--

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-05 21:55     ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-05 21:55 UTC (permalink / raw)
  To: linux-arm-kernel

Hi Ming,

(I've pruned the Cc list, leaving just the mailing lists)

On 12/02/2011 04:02 PM, Ming Lei wrote:
> This patch introduces one driver for face detection purpose.
> 
> The driver is responsible for all v4l2 stuff, buffer management
> and other general things, and doesn't touch face detection hardware
> directly. Several interfaces are exported to low level drivers
> (such as the coming omap4 FD driver)which will communicate with
> face detection hw module.
> 
> So the driver will make driving face detection hw modules more
> easy.


I would hold on for a moment on implementing generic face detection
module which is based on the V4L2 video device interface. We need to
first define an API that would be also usable at sub-device interface
level (http://linuxtv.org/downloads/v4l-dvb-apis/subdev.html).
AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
reasonable to use the videodev interface for passing data to the kernel
from user space.

But there might be face detection devices that accept data from other
H/W modules, e.g. transferred through SoC internal data buses between
image processing pipeline blocks. Thus any new interfaces need to be
designed with such devices in mind.

Also the face detection hardware block might now have an input DMA
engine in it, the data could be fed from memory through some other
subsystem (e.g. resize/colour converter). Then the driver for that
subsystem would implement a video node.

I'm for leaving the buffer handling details for individual drivers
and focusing on a standard interface for applications, i.e. new
ioctl(s) and controls.

> 
> TODO:
> 	- implement FD setting interfaces with v4l2 controls or
> 	ext controls
> 
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
>  drivers/media/video/Kconfig       |    2 +
>  drivers/media/video/Makefile      |    1 +
>  drivers/media/video/fdif/Kconfig  |    7 +
>  drivers/media/video/fdif/Makefile |    1 +
>  drivers/media/video/fdif/fdif.c   |  645 +++++++++++++++++++++++++++++++++++++
>  drivers/media/video/fdif/fdif.h   |  114 +++++++
>  6 files changed, 770 insertions(+), 0 deletions(-)
>  create mode 100644 drivers/media/video/fdif/Kconfig
>  create mode 100644 drivers/media/video/fdif/Makefile
>  create mode 100644 drivers/media/video/fdif/fdif.c
>  create mode 100644 drivers/media/video/fdif/fdif.h

[...]

> diff --git a/drivers/media/video/fdif/fdif.h b/drivers/media/video/fdif/fdif.h
> new file mode 100644
> index 0000000..ae37ab8
> --- /dev/null
> +++ b/drivers/media/video/fdif/fdif.h
> @@ -0,0 +1,114 @@
> +#ifndef _LINUX_FDIF_H
> +#define _LINUX_FDIF_H
> +
> +#include <linux/types.h>
> +#include <linux/magic.h>
> +#include <linux/errno.h>
> +#include <linux/kref.h>
> +#include <linux/kernel.h>
> +#include <linux/videodev2.h>
> +#include <media/videobuf2-page.h>
> +#include <media/v4l2-device.h>
> +#include <media/v4l2-ioctl.h>
> +#include <media/v4l2-ctrls.h>
> +#include <media/v4l2-fh.h>
> +#include <media/v4l2-event.h>
> +#include <media/v4l2-common.h>
> +
> +#define MAX_FACE_COUNT		40
> +
> +#define	FACE_SIZE_20_PIXELS	0
> +#define	FACE_SIZE_25_PIXELS	1
> +#define	FACE_SIZE_32_PIXELS	2
> +#define	FACE_SIZE_40_PIXELS	3

This is still OMAP4 FDIF specific, we need to think about v4l2 controls
for this. An ideal would be a menu control type that supports pixel size
(width/height), but unfortunately something like this isn't available
in v4l2 yet.

> +
> +#define FACE_DIR_UP		0
> +#define FACE_DIR_RIGHT		1
> +#define FACE_DIR_LIFT		2
> +
> +struct fdif_fmt {
> +	char  *name;
> +	u32   fourcc;          /* v4l2 format id */
> +	int   depth;
> +	int   width, height;

Could width/height be negative ? I don't think it's the case for pixel
resolution. The more proper data type would be u32.

Please refer to struct v4l2_pix_format or struct v4l2_rect.

> +};
> +
> +struct fdif_setting {
> +	struct fdif_fmt            *fmt;
> +	enum v4l2_field            field;
> +
> +	int 			min_face_size;
> +	int			face_dir;
> +
> +	int			startx, starty;

s32

> +	int			sizex, sizey;

u32

> +	int			lhit;
> +
> +	int			width, height;

u32

> +};
> +
> +/* buffer for one video frame */
> +struct fdif_buffer {
> +	/* common v4l buffer stuff -- must be first */
> +	struct vb2_buffer	vb;
> +	struct list_head	list;
> +};
> +
> +
> +struct v4l2_fdif_result {
> +	struct list_head		list;
> +	unsigned int			face_cnt;
> +	struct v4l2_fd_detection	*faces;
> +
> +	/*v4l2 buffer index*/
> +	__u32				index;
> +};
> +
> +struct fdif_dmaqueue {
> +	struct list_head	complete;
> +	struct list_head	active;
> +	wait_queue_head_t	wq;
> +};
> +
> +
> +struct fdif_dev {
> +	struct kref		ref;
> +	struct device		*dev;
> +
> +	struct list_head        fdif_devlist;
> +	struct v4l2_device	v4l2_dev;
> +	struct vb2_queue        vbq;
> +	struct mutex            mutex;
> +	spinlock_t		lock;
> +
> +	struct video_device        *vfd;
> +	struct fdif_dmaqueue	fdif_dq;
> +
> +	/*setting*/
> +	struct fdif_setting	s;

yy, please make it more descriptive. e.g.

	struct fdif_config	config;

> +
> +	struct fdif_ops	*ops;
> +
> +	unsigned long	priv[0];
> +};
> +
[...]

--

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-02 15:02   ` Ming Lei
@ 2011-12-05 22:15     ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-05 22:15 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

On 12/02/2011 04:02 PM, Ming Lei wrote:
> This patch introduces two new IOCTLs and related data
> structure defination which will be used by the coming
> face detection video device.
> 
> The two IOCTLs and related data structure are used by
> user space application to retrieve the results of face
> detection. They can be called after one v4l2_buffer
> has been ioctl(VIDIOC_DQBUF) and before it will be
> ioctl(VIDIOC_QBUF).
> 
> The utility fdif[1] is useing the two IOCTLs to find
> faces deteced in raw images or video streams.
> 
> [1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif
> 
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
>  drivers/media/video/v4l2-ioctl.c |   38 ++++++++++++++++++++
>  include/linux/videodev2.h        |   70 ++++++++++++++++++++++++++++++++++++++
>  include/media/v4l2-ioctl.h       |    6 +++
>  3 files changed, 114 insertions(+), 0 deletions(-)
> 
> diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
> index e1da8fc..fc6266f 100644
> --- a/drivers/media/video/v4l2-ioctl.c
> +++ b/drivers/media/video/v4l2-ioctl.c
> @@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
>  		dbgarg(cmd, "index=%d", b->index);
>  		break;
>  	}
> +	case VIDIOC_G_FD_RESULT:
> +	{
> +		struct v4l2_fd_result *fr = arg;
> +
> +		if (!ops->vidioc_g_fd_result)
> +			break;
> +
> +		ret = ops->vidioc_g_fd_result(file, fh, fr);
> +
> +		dbgarg(cmd, "index=%d", fr->buf_index);
> +		break;
> +	}
> +	case VIDIOC_G_FD_COUNT:
> +	{
> +		struct v4l2_fd_count *fc = arg;
> +
> +		if (!ops->vidioc_g_fd_count)
> +			break;
> +
> +		ret = ops->vidioc_g_fd_count(file, fh, fc);
> +
> +		dbgarg(cmd, "index=%d", fc->buf_index);
> +		break;
> +	}
>  	default:
>  		if (!ops->vidioc_default)
>  			break;
> @@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
>  		}
>  		break;
>  	}
> +
> +	case VIDIOC_G_FD_RESULT: {
> +		struct v4l2_fd_result *fr = parg;
> +
> +		if (fr->face_cnt != 0) {
> +			*user_ptr = (void __user *)fr->fd;
> +			*kernel_ptr = (void *)&fr->fd;
> +			*array_size = sizeof(struct v4l2_fd_detection)
> +				    * fr->face_cnt;
> +			ret = 1;
> +		}
> +		break;
> +
> +	}
>  	}
>  
>  	return ret;
> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
> index 4b752d5..073eb4d 100644
> --- a/include/linux/videodev2.h
> +++ b/include/linux/videodev2.h
> @@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
>  	__u32			reserved[8];
>  };
>  
> +/**
> + * struct v4l2_obj_detection
> + * @buf_index:	entry, index of v4l2_buffer for face detection
> + * @centerx:	return, position in x direction of detected object
> + * @centery:	return, position in y direction of detected object
> + * @angle:	return, angle of detected object
> + * 		0 deg ~ 359 deg, vertical is 0 deg, clockwise
> + * @sizex:	return, size in x direction of detected object
> + * @sizey:	return, size in y direction of detected object
> + * @confidence:	return, confidence level of detection result
> + * 		0: the heighest level, 9: the lowest level

Hmm, not a good idea to align a public interface to the capabilities
of a single hardware implementation. min/max confidence could be queried with
relevant controls and here we could remove the line implying range.

> + * @reserved:	future extensions
> + */
> +struct v4l2_obj_detection {
> +	__u16		centerx;
> +	__u16		centery;
> +	__u16		angle;
> +	__u16		sizex;
> +	__u16		sizey;

How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
for size but there seems to be missing en equivalent for position, e.g.

struct v4l2_position {
	__s32 x;
	__s32 y;
};

> +	__u16		confidence;
> +	__u32		reserved[4];
> +};
> +
> +#define V4L2_FD_HAS_LEFT_EYE	0x1
> +#define V4L2_FD_HAS_RIGHT_EYE	0x2
> +#define V4L2_FD_HAS_MOUTH	0x4
> +#define V4L2_FD_HAS_FACE	0x8
> +
> +/**
> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
> + * @flag:	return, describe which objects are detected
> + * @left_eye:	return, left_eye position if detected
> + * @right_eye:	return, right_eye position if detected
> + * @mouth_eye:	return, mouth_eye position if detected

mouth_eye ? ;)

> + * @face:	return, face position if detected
> + */
> +struct v4l2_fd_detection {
> +	__u32	flag;
> +	struct v4l2_obj_detection	left_eye;
> +	struct v4l2_obj_detection	right_eye;
> +	struct v4l2_obj_detection	mouth;
> +	struct v4l2_obj_detection	face;

I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
and then struct v4l2_fd_detection would be simply an array of
struct v4l2_obj_detection, i.e.

struct v4l2_fd_detection {
	unsigned int count;
	struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
};

This might be more flexible, e.g. if in the future some hardware supports
detecting wrinkles, we could easily add that by just defining a new flag:
V4L2_FD_HAS_WRINKLES, etc.


--

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-05 22:15     ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-05 22:15 UTC (permalink / raw)
  To: linux-arm-kernel

On 12/02/2011 04:02 PM, Ming Lei wrote:
> This patch introduces two new IOCTLs and related data
> structure defination which will be used by the coming
> face detection video device.
> 
> The two IOCTLs and related data structure are used by
> user space application to retrieve the results of face
> detection. They can be called after one v4l2_buffer
> has been ioctl(VIDIOC_DQBUF) and before it will be
> ioctl(VIDIOC_QBUF).
> 
> The utility fdif[1] is useing the two IOCTLs to find
> faces deteced in raw images or video streams.
> 
> [1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif
> 
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
>  drivers/media/video/v4l2-ioctl.c |   38 ++++++++++++++++++++
>  include/linux/videodev2.h        |   70 ++++++++++++++++++++++++++++++++++++++
>  include/media/v4l2-ioctl.h       |    6 +++
>  3 files changed, 114 insertions(+), 0 deletions(-)
> 
> diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
> index e1da8fc..fc6266f 100644
> --- a/drivers/media/video/v4l2-ioctl.c
> +++ b/drivers/media/video/v4l2-ioctl.c
> @@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
>  		dbgarg(cmd, "index=%d", b->index);
>  		break;
>  	}
> +	case VIDIOC_G_FD_RESULT:
> +	{
> +		struct v4l2_fd_result *fr = arg;
> +
> +		if (!ops->vidioc_g_fd_result)
> +			break;
> +
> +		ret = ops->vidioc_g_fd_result(file, fh, fr);
> +
> +		dbgarg(cmd, "index=%d", fr->buf_index);
> +		break;
> +	}
> +	case VIDIOC_G_FD_COUNT:
> +	{
> +		struct v4l2_fd_count *fc = arg;
> +
> +		if (!ops->vidioc_g_fd_count)
> +			break;
> +
> +		ret = ops->vidioc_g_fd_count(file, fh, fc);
> +
> +		dbgarg(cmd, "index=%d", fc->buf_index);
> +		break;
> +	}
>  	default:
>  		if (!ops->vidioc_default)
>  			break;
> @@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
>  		}
>  		break;
>  	}
> +
> +	case VIDIOC_G_FD_RESULT: {
> +		struct v4l2_fd_result *fr = parg;
> +
> +		if (fr->face_cnt != 0) {
> +			*user_ptr = (void __user *)fr->fd;
> +			*kernel_ptr = (void *)&fr->fd;
> +			*array_size = sizeof(struct v4l2_fd_detection)
> +				    * fr->face_cnt;
> +			ret = 1;
> +		}
> +		break;
> +
> +	}
>  	}
>  
>  	return ret;
> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
> index 4b752d5..073eb4d 100644
> --- a/include/linux/videodev2.h
> +++ b/include/linux/videodev2.h
> @@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
>  	__u32			reserved[8];
>  };
>  
> +/**
> + * struct v4l2_obj_detection
> + * @buf_index:	entry, index of v4l2_buffer for face detection
> + * @centerx:	return, position in x direction of detected object
> + * @centery:	return, position in y direction of detected object
> + * @angle:	return, angle of detected object
> + * 		0 deg ~ 359 deg, vertical is 0 deg, clockwise
> + * @sizex:	return, size in x direction of detected object
> + * @sizey:	return, size in y direction of detected object
> + * @confidence:	return, confidence level of detection result
> + * 		0: the heighest level, 9: the lowest level

Hmm, not a good idea to align a public interface to the capabilities
of a single hardware implementation. min/max confidence could be queried with
relevant controls and here we could remove the line implying range.

> + * @reserved:	future extensions
> + */
> +struct v4l2_obj_detection {
> +	__u16		centerx;
> +	__u16		centery;
> +	__u16		angle;
> +	__u16		sizex;
> +	__u16		sizey;

How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
for size but there seems to be missing en equivalent for position, e.g.

struct v4l2_position {
	__s32 x;
	__s32 y;
};

> +	__u16		confidence;
> +	__u32		reserved[4];
> +};
> +
> +#define V4L2_FD_HAS_LEFT_EYE	0x1
> +#define V4L2_FD_HAS_RIGHT_EYE	0x2
> +#define V4L2_FD_HAS_MOUTH	0x4
> +#define V4L2_FD_HAS_FACE	0x8
> +
> +/**
> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
> + * @flag:	return, describe which objects are detected
> + * @left_eye:	return, left_eye position if detected
> + * @right_eye:	return, right_eye position if detected
> + * @mouth_eye:	return, mouth_eye position if detected

mouth_eye ? ;)

> + * @face:	return, face position if detected
> + */
> +struct v4l2_fd_detection {
> +	__u32	flag;
> +	struct v4l2_obj_detection	left_eye;
> +	struct v4l2_obj_detection	right_eye;
> +	struct v4l2_obj_detection	mouth;
> +	struct v4l2_obj_detection	face;

I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
and then struct v4l2_fd_detection would be simply an array of
struct v4l2_obj_detection, i.e.

struct v4l2_fd_detection {
	unsigned int count;
	struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
};

This might be more flexible, e.g. if in the future some hardware supports
detecting wrinkles, we could easily add that by just defining a new flag:
V4L2_FD_HAS_WRINKLES, etc.


--

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-05 21:55     ` Sylwester Nawrocki
@ 2011-12-06 14:07       ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06 14:07 UTC (permalink / raw)
  To: Sylwester Nawrocki
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi,

Thanks for your review.

On Tue, Dec 6, 2011 at 5:55 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> Hi Ming,
>
> (I've pruned the Cc list, leaving just the mailing lists)
>
> On 12/02/2011 04:02 PM, Ming Lei wrote:
>> This patch introduces one driver for face detection purpose.
>>
>> The driver is responsible for all v4l2 stuff, buffer management
>> and other general things, and doesn't touch face detection hardware
>> directly. Several interfaces are exported to low level drivers
>> (such as the coming omap4 FD driver)which will communicate with
>> face detection hw module.
>>
>> So the driver will make driving face detection hw modules more
>> easy.
>
>
> I would hold on for a moment on implementing generic face detection
> module which is based on the V4L2 video device interface. We need to
> first define an API that would be also usable at sub-device interface
> level (http://linuxtv.org/downloads/v4l-dvb-apis/subdev.html).

If we can define a good/stable enough APIs between kernel and user space,
I think the patches can be merged first. For internal kernel APIs, we should
allow it to evolve as new hardware comes or new features are to be introduced.

I understand the API you mentioned here should belong to kernel internal
API, correct me if it is wrong.

> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
> reasonable to use the videodev interface for passing data to the kernel
> from user space.
>
> But there might be face detection devices that accept data from other
> H/W modules, e.g. transferred through SoC internal data buses between
> image processing pipeline blocks. Thus any new interfaces need to be
> designed with such devices in mind.
>
> Also the face detection hardware block might now have an input DMA
> engine in it, the data could be fed from memory through some other
> subsystem (e.g. resize/colour converter). Then the driver for that
> subsystem would implement a video node.

I think the direct input image or frame data to FD should be from memory
no matter the actual data is from external H/W modules or input DMA because
FD will take lot of time to detect faces in one image or frame and FD can't
have so much memory to cache several images or frames data.

If you have seen this kind of FD hardware design, please let me know.

> I'm for leaving the buffer handling details for individual drivers
> and focusing on a standard interface for applications, i.e. new

I think leaving buffer handling details in generic FD driver or
individual drivers
doesn't matter now, since it don't have effect on interfaces between kernel
and user space.

> ioctl(s) and controls.
>
>>
>> TODO:
>>       - implement FD setting interfaces with v4l2 controls or
>>       ext controls
>>
>> Signed-off-by: Ming Lei <ming.lei@canonical.com>
>> ---
>>  drivers/media/video/Kconfig       |    2 +
>>  drivers/media/video/Makefile      |    1 +
>>  drivers/media/video/fdif/Kconfig  |    7 +
>>  drivers/media/video/fdif/Makefile |    1 +
>>  drivers/media/video/fdif/fdif.c   |  645 +++++++++++++++++++++++++++++++++++++
>>  drivers/media/video/fdif/fdif.h   |  114 +++++++
>>  6 files changed, 770 insertions(+), 0 deletions(-)
>>  create mode 100644 drivers/media/video/fdif/Kconfig
>>  create mode 100644 drivers/media/video/fdif/Makefile
>>  create mode 100644 drivers/media/video/fdif/fdif.c
>>  create mode 100644 drivers/media/video/fdif/fdif.h
>
> [...]
>
>> diff --git a/drivers/media/video/fdif/fdif.h b/drivers/media/video/fdif/fdif.h
>> new file mode 100644
>> index 0000000..ae37ab8
>> --- /dev/null
>> +++ b/drivers/media/video/fdif/fdif.h
>> @@ -0,0 +1,114 @@
>> +#ifndef _LINUX_FDIF_H
>> +#define _LINUX_FDIF_H
>> +
>> +#include <linux/types.h>
>> +#include <linux/magic.h>
>> +#include <linux/errno.h>
>> +#include <linux/kref.h>
>> +#include <linux/kernel.h>
>> +#include <linux/videodev2.h>
>> +#include <media/videobuf2-page.h>
>> +#include <media/v4l2-device.h>
>> +#include <media/v4l2-ioctl.h>
>> +#include <media/v4l2-ctrls.h>
>> +#include <media/v4l2-fh.h>
>> +#include <media/v4l2-event.h>
>> +#include <media/v4l2-common.h>
>> +
>> +#define MAX_FACE_COUNT               40
>> +
>> +#define      FACE_SIZE_20_PIXELS     0
>> +#define      FACE_SIZE_25_PIXELS     1
>> +#define      FACE_SIZE_32_PIXELS     2
>> +#define      FACE_SIZE_40_PIXELS     3
>
> This is still OMAP4 FDIF specific, we need to think about v4l2 controls
> for this. An ideal would be a menu control type that supports pixel size
> (width/height), but unfortunately something like this isn't available
> in v4l2 yet.

Yes, it is on TODO list, :-)

>> +
>> +#define FACE_DIR_UP          0
>> +#define FACE_DIR_RIGHT               1
>> +#define FACE_DIR_LIFT                2
>> +
>> +struct fdif_fmt {
>> +     char  *name;
>> +     u32   fourcc;          /* v4l2 format id */
>> +     int   depth;
>> +     int   width, height;
>
> Could width/height be negative ? I don't think it's the case for pixel
> resolution. The more proper data type would be u32.

Yes, they should be, will fix it in next version.

> Please refer to struct v4l2_pix_format or struct v4l2_rect.
>
>> +};
>> +
>> +struct fdif_setting {
>> +     struct fdif_fmt            *fmt;
>> +     enum v4l2_field            field;
>> +
>> +     int                     min_face_size;
>> +     int                     face_dir;
>> +
>> +     int                     startx, starty;
>
> s32
>
>> +     int                     sizex, sizey;
>
> u32
>
>> +     int                     lhit;
>> +
>> +     int                     width, height;
>
> u32
>
>> +};
>> +
>> +/* buffer for one video frame */
>> +struct fdif_buffer {
>> +     /* common v4l buffer stuff -- must be first */
>> +     struct vb2_buffer       vb;
>> +     struct list_head        list;
>> +};
>> +
>> +
>> +struct v4l2_fdif_result {
>> +     struct list_head                list;
>> +     unsigned int                    face_cnt;
>> +     struct v4l2_fd_detection        *faces;
>> +
>> +     /*v4l2 buffer index*/
>> +     __u32                           index;
>> +};
>> +
>> +struct fdif_dmaqueue {
>> +     struct list_head        complete;
>> +     struct list_head        active;
>> +     wait_queue_head_t       wq;
>> +};
>> +
>> +
>> +struct fdif_dev {
>> +     struct kref             ref;
>> +     struct device           *dev;
>> +
>> +     struct list_head        fdif_devlist;
>> +     struct v4l2_device      v4l2_dev;
>> +     struct vb2_queue        vbq;
>> +     struct mutex            mutex;
>> +     spinlock_t              lock;
>> +
>> +     struct video_device        *vfd;
>> +     struct fdif_dmaqueue    fdif_dq;
>> +
>> +     /*setting*/
>> +     struct fdif_setting     s;
>
> yy, please make it more descriptive. e.g.

Sounds reasonable, :-)

>
>        struct fdif_config      config;
>
>> +
>> +     struct fdif_ops *ops;
>> +
>> +     unsigned long   priv[0];
>> +};
>> +
> [...]
>
> --
>
> Regards,
> Sylwester
> --
> To unsubscribe from this list: send the line "unsubscribe linux-omap" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-06 14:07       ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06 14:07 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

Thanks for your review.

On Tue, Dec 6, 2011 at 5:55 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> Hi Ming,
>
> (I've pruned the Cc list, leaving just the mailing lists)
>
> On 12/02/2011 04:02 PM, Ming Lei wrote:
>> This patch introduces one driver for face detection purpose.
>>
>> The driver is responsible for all v4l2 stuff, buffer management
>> and other general things, and doesn't touch face detection hardware
>> directly. Several interfaces are exported to low level drivers
>> (such as the coming omap4 FD driver)which will communicate with
>> face detection hw module.
>>
>> So the driver will make driving face detection hw modules more
>> easy.
>
>
> I would hold on for a moment on implementing generic face detection
> module which is based on the V4L2 video device interface. We need to
> first define an API that would be also usable at sub-device interface
> level (http://linuxtv.org/downloads/v4l-dvb-apis/subdev.html).

If we can define a good/stable enough APIs between kernel and user space,
I think the patches can be merged first. For internal kernel APIs, we should
allow it to evolve as new hardware comes or new features are to be introduced.

I understand the API you mentioned here should belong to kernel internal
API, correct me if it is wrong.

> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
> reasonable to use the videodev interface for passing data to the kernel
> from user space.
>
> But there might be face detection devices that accept data from other
> H/W modules, e.g. transferred through SoC internal data buses between
> image processing pipeline blocks. Thus any new interfaces need to be
> designed with such devices in mind.
>
> Also the face detection hardware block might now have an input DMA
> engine in it, the data could be fed from memory through some other
> subsystem (e.g. resize/colour converter). Then the driver for that
> subsystem would implement a video node.

I think the direct input image or frame data to FD should be from memory
no matter the actual data is from external H/W modules or input DMA because
FD will take lot of time to detect faces in one image or frame and FD can't
have so much memory to cache several images or frames data.

If you have seen this kind of FD hardware design, please let me know.

> I'm for leaving the buffer handling details for individual drivers
> and focusing on a standard interface for applications, i.e. new

I think leaving buffer handling details in generic FD driver or
individual drivers
doesn't matter now, since it don't have effect on interfaces between kernel
and user space.

> ioctl(s) and controls.
>
>>
>> TODO:
>> ? ? ? - implement FD setting interfaces with v4l2 controls or
>> ? ? ? ext controls
>>
>> Signed-off-by: Ming Lei <ming.lei@canonical.com>
>> ---
>> ?drivers/media/video/Kconfig ? ? ? | ? ?2 +
>> ?drivers/media/video/Makefile ? ? ?| ? ?1 +
>> ?drivers/media/video/fdif/Kconfig ?| ? ?7 +
>> ?drivers/media/video/fdif/Makefile | ? ?1 +
>> ?drivers/media/video/fdif/fdif.c ? | ?645 +++++++++++++++++++++++++++++++++++++
>> ?drivers/media/video/fdif/fdif.h ? | ?114 +++++++
>> ?6 files changed, 770 insertions(+), 0 deletions(-)
>> ?create mode 100644 drivers/media/video/fdif/Kconfig
>> ?create mode 100644 drivers/media/video/fdif/Makefile
>> ?create mode 100644 drivers/media/video/fdif/fdif.c
>> ?create mode 100644 drivers/media/video/fdif/fdif.h
>
> [...]
>
>> diff --git a/drivers/media/video/fdif/fdif.h b/drivers/media/video/fdif/fdif.h
>> new file mode 100644
>> index 0000000..ae37ab8
>> --- /dev/null
>> +++ b/drivers/media/video/fdif/fdif.h
>> @@ -0,0 +1,114 @@
>> +#ifndef _LINUX_FDIF_H
>> +#define _LINUX_FDIF_H
>> +
>> +#include <linux/types.h>
>> +#include <linux/magic.h>
>> +#include <linux/errno.h>
>> +#include <linux/kref.h>
>> +#include <linux/kernel.h>
>> +#include <linux/videodev2.h>
>> +#include <media/videobuf2-page.h>
>> +#include <media/v4l2-device.h>
>> +#include <media/v4l2-ioctl.h>
>> +#include <media/v4l2-ctrls.h>
>> +#include <media/v4l2-fh.h>
>> +#include <media/v4l2-event.h>
>> +#include <media/v4l2-common.h>
>> +
>> +#define MAX_FACE_COUNT ? ? ? ? ? ? ? 40
>> +
>> +#define ? ? ?FACE_SIZE_20_PIXELS ? ? 0
>> +#define ? ? ?FACE_SIZE_25_PIXELS ? ? 1
>> +#define ? ? ?FACE_SIZE_32_PIXELS ? ? 2
>> +#define ? ? ?FACE_SIZE_40_PIXELS ? ? 3
>
> This is still OMAP4 FDIF specific, we need to think about v4l2 controls
> for this. An ideal would be a menu control type that supports pixel size
> (width/height), but unfortunately something like this isn't available
> in v4l2 yet.

Yes, it is on TODO list, :-)

>> +
>> +#define FACE_DIR_UP ? ? ? ? ?0
>> +#define FACE_DIR_RIGHT ? ? ? ? ? ? ? 1
>> +#define FACE_DIR_LIFT ? ? ? ? ? ? ? ?2
>> +
>> +struct fdif_fmt {
>> + ? ? char ?*name;
>> + ? ? u32 ? fourcc; ? ? ? ? ?/* v4l2 format id */
>> + ? ? int ? depth;
>> + ? ? int ? width, height;
>
> Could width/height be negative ? I don't think it's the case for pixel
> resolution. The more proper data type would be u32.

Yes, they should be, will fix it in next version.

> Please refer to struct v4l2_pix_format or struct v4l2_rect.
>
>> +};
>> +
>> +struct fdif_setting {
>> + ? ? struct fdif_fmt ? ? ? ? ? ?*fmt;
>> + ? ? enum v4l2_field ? ? ? ? ? ?field;
>> +
>> + ? ? int ? ? ? ? ? ? ? ? ? ? min_face_size;
>> + ? ? int ? ? ? ? ? ? ? ? ? ? face_dir;
>> +
>> + ? ? int ? ? ? ? ? ? ? ? ? ? startx, starty;
>
> s32
>
>> + ? ? int ? ? ? ? ? ? ? ? ? ? sizex, sizey;
>
> u32
>
>> + ? ? int ? ? ? ? ? ? ? ? ? ? lhit;
>> +
>> + ? ? int ? ? ? ? ? ? ? ? ? ? width, height;
>
> u32
>
>> +};
>> +
>> +/* buffer for one video frame */
>> +struct fdif_buffer {
>> + ? ? /* common v4l buffer stuff -- must be first */
>> + ? ? struct vb2_buffer ? ? ? vb;
>> + ? ? struct list_head ? ? ? ?list;
>> +};
>> +
>> +
>> +struct v4l2_fdif_result {
>> + ? ? struct list_head ? ? ? ? ? ? ? ?list;
>> + ? ? unsigned int ? ? ? ? ? ? ? ? ? ?face_cnt;
>> + ? ? struct v4l2_fd_detection ? ? ? ?*faces;
>> +
>> + ? ? /*v4l2 buffer index*/
>> + ? ? __u32 ? ? ? ? ? ? ? ? ? ? ? ? ? index;
>> +};
>> +
>> +struct fdif_dmaqueue {
>> + ? ? struct list_head ? ? ? ?complete;
>> + ? ? struct list_head ? ? ? ?active;
>> + ? ? wait_queue_head_t ? ? ? wq;
>> +};
>> +
>> +
>> +struct fdif_dev {
>> + ? ? struct kref ? ? ? ? ? ? ref;
>> + ? ? struct device ? ? ? ? ? *dev;
>> +
>> + ? ? struct list_head ? ? ? ?fdif_devlist;
>> + ? ? struct v4l2_device ? ? ?v4l2_dev;
>> + ? ? struct vb2_queue ? ? ? ?vbq;
>> + ? ? struct mutex ? ? ? ? ? ?mutex;
>> + ? ? spinlock_t ? ? ? ? ? ? ?lock;
>> +
>> + ? ? struct video_device ? ? ? ?*vfd;
>> + ? ? struct fdif_dmaqueue ? ?fdif_dq;
>> +
>> + ? ? /*setting*/
>> + ? ? struct fdif_setting ? ? s;
>
> yy, please make it more descriptive. e.g.

Sounds reasonable, :-)

>
> ? ? ? ?struct fdif_config ? ? ?config;
>
>> +
>> + ? ? struct fdif_ops *ops;
>> +
>> + ? ? unsigned long ? priv[0];
>> +};
>> +
> [...]
>
> --
>
> Regards,
> Sylwester
> --
> To unsubscribe from this list: send the line "unsubscribe linux-omap" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at ?http://vger.kernel.org/majordomo-info.html

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-06 14:07       ` Ming Lei
@ 2011-12-06 22:01         ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-06 22:01 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

On 12/06/2011 03:07 PM, Ming Lei wrote:
> Hi,
> 
> Thanks for your review.
> 
> On Tue, Dec 6, 2011 at 5:55 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
>> Hi Ming,
>>
>> (I've pruned the Cc list, leaving just the mailing lists)
>>
>> On 12/02/2011 04:02 PM, Ming Lei wrote:
>>> This patch introduces one driver for face detection purpose.
>>>
>>> The driver is responsible for all v4l2 stuff, buffer management
>>> and other general things, and doesn't touch face detection hardware
>>> directly. Several interfaces are exported to low level drivers
>>> (such as the coming omap4 FD driver)which will communicate with
>>> face detection hw module.
>>>
>>> So the driver will make driving face detection hw modules more
>>> easy.
>>
>>
>> I would hold on for a moment on implementing generic face detection
>> module which is based on the V4L2 video device interface. We need to
>> first define an API that would be also usable at sub-device interface
>> level (http://linuxtv.org/downloads/v4l-dvb-apis/subdev.html).
> 
> If we can define a good/stable enough APIs between kernel and user space,
> I think the patches can be merged first. For internal kernel APIs, we should
> allow it to evolve as new hardware comes or new features are to be introduced.

I also don't see a problem in discussing it a bit more;)

> 
> I understand the API you mentioned here should belong to kernel internal
> API, correct me if it is wrong.

Yes, I meant the in kernel design, i.e. generic face detection kernel module
and an OMAP4 FDIF driver. It makes lots of sense to separate common code
in this way, maybe even when there would be only OMAP devices using it.

I'm sure now the Samsung devices won't fit in video output node based driver
design. They read image data in different ways and also the FD result format
is totally different.

> 
>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>> reasonable to use the videodev interface for passing data to the kernel
>> from user space.
>>
>> But there might be face detection devices that accept data from other
>> H/W modules, e.g. transferred through SoC internal data buses between
>> image processing pipeline blocks. Thus any new interfaces need to be
>> designed with such devices in mind.
>>
>> Also the face detection hardware block might now have an input DMA
>> engine in it, the data could be fed from memory through some other
>> subsystem (e.g. resize/colour converter). Then the driver for that
>> subsystem would implement a video node.
> 
> I think the direct input image or frame data to FD should be from memory
> no matter the actual data is from external H/W modules or input DMA because
> FD will take lot of time to detect faces in one image or frame and FD can't
> have so much memory to cache several images or frames data.

Sorry, I cannot provide much details at the moment, but there exists hardware
that reads data from internal SoC buses and even if it uses some sort of
cache memory it doesn't necessarily have to be available for the user.

Still the FD result is associated with an image frame for such H/W, but not
necessarily with a memory buffer queued by a user application.

How long it approximately takes to process single image for OMAP4 FDIF ?

> 
> If you have seen this kind of FD hardware design, please let me know.
> 
>> I'm for leaving the buffer handling details for individual drivers
>> and focusing on a standard interface for applications, i.e. new
> 
> I think leaving buffer handling details in generic FD driver or
> individual drivers
> doesn't matter now, since it don't have effect on interfaces between kernel
> and user space.

I think you misunderstood me. I wasn't talking about core/driver module split,
I meant we should not be making the user interface video node centric.

I think for Samsung devices I'll need a capture video node for passing
the result to the user. So instead of associating FD result with a buffer index
we could try to use the frame sequence number (struct v4l2_buffer.sequence,
http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).

It might be much better as the v4l2 events are associated with the frame
sequence. And if we use controls then you get control events for free,
and each event carries a frame sequence number int it
(http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-dqevent.html).

-- 

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-06 22:01         ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-06 22:01 UTC (permalink / raw)
  To: linux-arm-kernel

On 12/06/2011 03:07 PM, Ming Lei wrote:
> Hi,
> 
> Thanks for your review.
> 
> On Tue, Dec 6, 2011 at 5:55 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
>> Hi Ming,
>>
>> (I've pruned the Cc list, leaving just the mailing lists)
>>
>> On 12/02/2011 04:02 PM, Ming Lei wrote:
>>> This patch introduces one driver for face detection purpose.
>>>
>>> The driver is responsible for all v4l2 stuff, buffer management
>>> and other general things, and doesn't touch face detection hardware
>>> directly. Several interfaces are exported to low level drivers
>>> (such as the coming omap4 FD driver)which will communicate with
>>> face detection hw module.
>>>
>>> So the driver will make driving face detection hw modules more
>>> easy.
>>
>>
>> I would hold on for a moment on implementing generic face detection
>> module which is based on the V4L2 video device interface. We need to
>> first define an API that would be also usable at sub-device interface
>> level (http://linuxtv.org/downloads/v4l-dvb-apis/subdev.html).
> 
> If we can define a good/stable enough APIs between kernel and user space,
> I think the patches can be merged first. For internal kernel APIs, we should
> allow it to evolve as new hardware comes or new features are to be introduced.

I also don't see a problem in discussing it a bit more;)

> 
> I understand the API you mentioned here should belong to kernel internal
> API, correct me if it is wrong.

Yes, I meant the in kernel design, i.e. generic face detection kernel module
and an OMAP4 FDIF driver. It makes lots of sense to separate common code
in this way, maybe even when there would be only OMAP devices using it.

I'm sure now the Samsung devices won't fit in video output node based driver
design. They read image data in different ways and also the FD result format
is totally different.

> 
>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>> reasonable to use the videodev interface for passing data to the kernel
>> from user space.
>>
>> But there might be face detection devices that accept data from other
>> H/W modules, e.g. transferred through SoC internal data buses between
>> image processing pipeline blocks. Thus any new interfaces need to be
>> designed with such devices in mind.
>>
>> Also the face detection hardware block might now have an input DMA
>> engine in it, the data could be fed from memory through some other
>> subsystem (e.g. resize/colour converter). Then the driver for that
>> subsystem would implement a video node.
> 
> I think the direct input image or frame data to FD should be from memory
> no matter the actual data is from external H/W modules or input DMA because
> FD will take lot of time to detect faces in one image or frame and FD can't
> have so much memory to cache several images or frames data.

Sorry, I cannot provide much details at the moment, but there exists hardware
that reads data from internal SoC buses and even if it uses some sort of
cache memory it doesn't necessarily have to be available for the user.

Still the FD result is associated with an image frame for such H/W, but not
necessarily with a memory buffer queued by a user application.

How long it approximately takes to process single image for OMAP4 FDIF ?

> 
> If you have seen this kind of FD hardware design, please let me know.
> 
>> I'm for leaving the buffer handling details for individual drivers
>> and focusing on a standard interface for applications, i.e. new
> 
> I think leaving buffer handling details in generic FD driver or
> individual drivers
> doesn't matter now, since it don't have effect on interfaces between kernel
> and user space.

I think you misunderstood me. I wasn't talking about core/driver module split,
I meant we should not be making the user interface video node centric.

I think for Samsung devices I'll need a capture video node for passing
the result to the user. So instead of associating FD result with a buffer index
we could try to use the frame sequence number (struct v4l2_buffer.sequence,
http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).

It might be much better as the v4l2 events are associated with the frame
sequence. And if we use controls then you get control events for free,
and each event carries a frame sequence number int it
(http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-dqevent.html).

-- 

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-06 22:01         ` Sylwester Nawrocki
@ 2011-12-06 22:39           ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-06 22:39 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

On 12/06/2011 11:01 PM, Sylwester Nawrocki wrote:
[...]
>
> It might be much better as the v4l2 events are associated with the frame
> sequence. And if we use controls then you get control events for free,
> and each event carries a frame sequence number int it
> (http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-dqevent.html).

Oops, just ignore that. The frame sequence number is only available for
V4L2_EVENT_FRAME_SYNC events.



^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-06 22:39           ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-06 22:39 UTC (permalink / raw)
  To: linux-arm-kernel

On 12/06/2011 11:01 PM, Sylwester Nawrocki wrote:
[...]
>
> It might be much better as the v4l2 events are associated with the frame
> sequence. And if we use controls then you get control events for free,
> and each event carries a frame sequence number int it
> (http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-dqevent.html).

Oops, just ignore that. The frame sequence number is only available for
V4L2_EVENT_FRAME_SYNC events.

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-06 22:01         ` Sylwester Nawrocki
@ 2011-12-07 13:40           ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-07 13:40 UTC (permalink / raw)
  To: Sylwester Nawrocki
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi,

On Wed, Dec 7, 2011 at 6:01 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/06/2011 03:07 PM, Ming Lei wrote:
>> Hi,
>>
>> Thanks for your review.
>>
>> On Tue, Dec 6, 2011 at 5:55 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
>>> Hi Ming,
>>>
>>> (I've pruned the Cc list, leaving just the mailing lists)
>>>
>>> On 12/02/2011 04:02 PM, Ming Lei wrote:
>>>> This patch introduces one driver for face detection purpose.
>>>>
>>>> The driver is responsible for all v4l2 stuff, buffer management
>>>> and other general things, and doesn't touch face detection hardware
>>>> directly. Several interfaces are exported to low level drivers
>>>> (such as the coming omap4 FD driver)which will communicate with
>>>> face detection hw module.
>>>>
>>>> So the driver will make driving face detection hw modules more
>>>> easy.
>>>
>>>
>>> I would hold on for a moment on implementing generic face detection
>>> module which is based on the V4L2 video device interface. We need to
>>> first define an API that would be also usable at sub-device interface
>>> level (http://linuxtv.org/downloads/v4l-dvb-apis/subdev.html).
>>
>> If we can define a good/stable enough APIs between kernel and user space,
>> I think the patches can be merged first. For internal kernel APIs, we should
>> allow it to evolve as new hardware comes or new features are to be introduced.
>
> I also don't see a problem in discussing it a bit more;)

OK, fair enough, let's discuss it, :-)

>
>>
>> I understand the API you mentioned here should belong to kernel internal
>> API, correct me if it is wrong.
>
> Yes, I meant the in kernel design, i.e. generic face detection kernel module
> and an OMAP4 FDIF driver. It makes lots of sense to separate common code
> in this way, maybe even when there would be only OMAP devices using it.

Yes, that is the motivation of the generic FD module. I think we can focus on
two use cases for the generic FD now:

- one is to detect faces from user space image data

- another one is to detect faces in image data generated from HW(SoC
internal bus, resize hardware)

For OMAP4 hardware, input data is always from physically continuous
memory directly, so it is very easy to support the two cases. For the
use case 2,
if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
to pass the image buffer produced by other HW to FD hw directly.

For other FD hardware, if it supports to detect faces in image data from
physically continuous memory, I think the patch is OK to support it.

If the FD hw doesn't support to detect faces from physically continuous
memory, I have some questions: how does user space app to parse the
FD result if application can't get the input image data? If user space can
get image data, how does it connect the image data with FD result? and
what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
image data?

> I'm sure now the Samsung devices won't fit in video output node based driver
> design. They read image data in different ways and also the FD result format
> is totally different.

I think user space will need the FD result, so it is very important to define
API to describe the FD result format to user space. And the input about your
FD HW result format is certainly helpful to define the API.

>>
>>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>>> reasonable to use the videodev interface for passing data to the kernel
>>> from user space.
>>>
>>> But there might be face detection devices that accept data from other
>>> H/W modules, e.g. transferred through SoC internal data buses between
>>> image processing pipeline blocks. Thus any new interfaces need to be
>>> designed with such devices in mind.
>>>
>>> Also the face detection hardware block might now have an input DMA
>>> engine in it, the data could be fed from memory through some other
>>> subsystem (e.g. resize/colour converter). Then the driver for that
>>> subsystem would implement a video node.
>>
>> I think the direct input image or frame data to FD should be from memory
>> no matter the actual data is from external H/W modules or input DMA because
>> FD will take lot of time to detect faces in one image or frame and FD can't
>> have so much memory to cache several images or frames data.
>
> Sorry, I cannot provide much details at the moment, but there exists hardware
> that reads data from internal SoC buses and even if it uses some sort of
> cache memory it doesn't necessarily have to be available for the user.

Without some hardware background, it is not easy to give a generic FD module
design.

> Still the FD result is associated with an image frame for such H/W, but not
> necessarily with a memory buffer queued by a user application.

For user space application, it doesn't make sense to handle FD results
only without image data.  Even though there are other ways of input
image data to FD, user space still need to know the image data, so it makes
sense to associate FD result with a memory buffer.

> How long it approximately takes to process single image for OMAP4 FDIF ?

See the link[2], and my test result is basically consistent with the data.

>>
>> If you have seen this kind of FD hardware design, please let me know.
>>
>>> I'm for leaving the buffer handling details for individual drivers
>>> and focusing on a standard interface for applications, i.e. new
>>
>> I think leaving buffer handling details in generic FD driver or
>> individual drivers
>> doesn't matter now, since it don't have effect on interfaces between kernel
>> and user space.
>
> I think you misunderstood me. I wasn't talking about core/driver module split,
> I meant we should not be making the user interface video node centric.
>
> I think for Samsung devices I'll need a capture video node for passing

Why is it a capture video node instead of OUTPUT v4l2 device? I think the
device name should be decided from the view of face detection function:
FD need input image data and produce detection result.

> the result to the user. So instead of associating FD result with a buffer index

See the explanation above.

> we could try to use the frame sequence number (struct v4l2_buffer.sequence,
> http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).
>
> It might be much better as the v4l2 events are associated with the frame
> sequence. And if we use controls then you get control events for free,
> and each event carries a frame sequence number int it
> (http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-dqevent.html).
>
> --
>
> Regards,
> Sylwester
> --
> To unsubscribe from this list: send the line "unsubscribe linux-omap" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

thanks,
--
Ming Lei

[1], http://marc.info/?t=132281644700005&r=1&w=2
[2], http://e2e.ti.com/support/embedded/linux/f/354/t/128938.aspx#462740

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-07 13:40           ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-07 13:40 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Wed, Dec 7, 2011 at 6:01 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/06/2011 03:07 PM, Ming Lei wrote:
>> Hi,
>>
>> Thanks for your review.
>>
>> On Tue, Dec 6, 2011 at 5:55 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
>>> Hi Ming,
>>>
>>> (I've pruned the Cc list, leaving just the mailing lists)
>>>
>>> On 12/02/2011 04:02 PM, Ming Lei wrote:
>>>> This patch introduces one driver for face detection purpose.
>>>>
>>>> The driver is responsible for all v4l2 stuff, buffer management
>>>> and other general things, and doesn't touch face detection hardware
>>>> directly. Several interfaces are exported to low level drivers
>>>> (such as the coming omap4 FD driver)which will communicate with
>>>> face detection hw module.
>>>>
>>>> So the driver will make driving face detection hw modules more
>>>> easy.
>>>
>>>
>>> I would hold on for a moment on implementing generic face detection
>>> module which is based on the V4L2 video device interface. We need to
>>> first define an API that would be also usable at sub-device interface
>>> level (http://linuxtv.org/downloads/v4l-dvb-apis/subdev.html).
>>
>> If we can define a good/stable enough APIs between kernel and user space,
>> I think the patches can be merged first. For internal kernel APIs, we should
>> allow it to evolve as new hardware comes or new features are to be introduced.
>
> I also don't see a problem in discussing it a bit more;)

OK, fair enough, let's discuss it, :-)

>
>>
>> I understand the API you mentioned here should belong to kernel internal
>> API, correct me if it is wrong.
>
> Yes, I meant the in kernel design, i.e. generic face detection kernel module
> and an OMAP4 FDIF driver. It makes lots of sense to separate common code
> in this way, maybe even when there would be only OMAP devices using it.

Yes, that is the motivation of the generic FD module. I think we can focus on
two use cases for the generic FD now:

- one is to detect faces from user space image data

- another one is to detect faces in image data generated from HW(SoC
internal bus, resize hardware)

For OMAP4 hardware, input data is always from physically continuous
memory directly, so it is very easy to support the two cases. For the
use case 2,
if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
to pass the image buffer produced by other HW to FD hw directly.

For other FD hardware, if it supports to detect faces in image data from
physically continuous memory, I think the patch is OK to support it.

If the FD hw doesn't support to detect faces from physically continuous
memory, I have some questions: how does user space app to parse the
FD result if application can't get the input image data? If user space can
get image data, how does it connect the image data with FD result? and
what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
image data?

> I'm sure now the Samsung devices won't fit in video output node based driver
> design. They read image data in different ways and also the FD result format
> is totally different.

I think user space will need the FD result, so it is very important to define
API to describe the FD result format to user space. And the input about your
FD HW result format is certainly helpful to define the API.

>>
>>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>>> reasonable to use the videodev interface for passing data to the kernel
>>> from user space.
>>>
>>> But there might be face detection devices that accept data from other
>>> H/W modules, e.g. transferred through SoC internal data buses between
>>> image processing pipeline blocks. Thus any new interfaces need to be
>>> designed with such devices in mind.
>>>
>>> Also the face detection hardware block might now have an input DMA
>>> engine in it, the data could be fed from memory through some other
>>> subsystem (e.g. resize/colour converter). Then the driver for that
>>> subsystem would implement a video node.
>>
>> I think the direct input image or frame data to FD should be from memory
>> no matter the actual data is from external H/W modules or input DMA because
>> FD will take lot of time to detect faces in one image or frame and FD can't
>> have so much memory to cache several images or frames data.
>
> Sorry, I cannot provide much details at the moment, but there exists hardware
> that reads data from internal SoC buses and even if it uses some sort of
> cache memory it doesn't necessarily have to be available for the user.

Without some hardware background, it is not easy to give a generic FD module
design.

> Still the FD result is associated with an image frame for such H/W, but not
> necessarily with a memory buffer queued by a user application.

For user space application, it doesn't make sense to handle FD results
only without image data.  Even though there are other ways of input
image data to FD, user space still need to know the image data, so it makes
sense to associate FD result with a memory buffer.

> How long it approximately takes to process single image for OMAP4 FDIF ?

See the link[2], and my test result is basically consistent with the data.

>>
>> If you have seen this kind of FD hardware design, please let me know.
>>
>>> I'm for leaving the buffer handling details for individual drivers
>>> and focusing on a standard interface for applications, i.e. new
>>
>> I think leaving buffer handling details in generic FD driver or
>> individual drivers
>> doesn't matter now, since it don't have effect on interfaces between kernel
>> and user space.
>
> I think you misunderstood me. I wasn't talking about core/driver module split,
> I meant we should not be making the user interface video node centric.
>
> I think for Samsung devices I'll need a capture video node for passing

Why is it a capture video node instead of OUTPUT v4l2 device? I think the
device name should be decided from the view of face detection function:
FD need input image data and produce detection result.

> the result to the user. So instead of associating FD result with a buffer index

See the explanation above.

> we could try to use the frame sequence number (struct v4l2_buffer.sequence,
> http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).
>
> It might be much better as the v4l2 events are associated with the frame
> sequence. And if we use controls then you get control events for free,
> and each event carries a frame sequence number int it
> (http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-dqevent.html).
>
> --
>
> Regards,
> Sylwester
> --
> To unsubscribe from this list: send the line "unsubscribe linux-omap" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at ?http://vger.kernel.org/majordomo-info.html

thanks,
--
Ming Lei

[1], http://marc.info/?t=132281644700005&r=1&w=2
[2], http://e2e.ti.com/support/embedded/linux/f/354/t/128938.aspx#462740

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-05 22:15     ` Sylwester Nawrocki
@ 2011-12-08  3:42       ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-08  3:42 UTC (permalink / raw)
  To: Sylwester Nawrocki
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi,

On Tue, Dec 6, 2011 at 6:15 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/02/2011 04:02 PM, Ming Lei wrote:
>> This patch introduces two new IOCTLs and related data
>> structure defination which will be used by the coming
>> face detection video device.
>>
>> The two IOCTLs and related data structure are used by
>> user space application to retrieve the results of face
>> detection. They can be called after one v4l2_buffer
>> has been ioctl(VIDIOC_DQBUF) and before it will be
>> ioctl(VIDIOC_QBUF).
>>
>> The utility fdif[1] is useing the two IOCTLs to find
>> faces deteced in raw images or video streams.
>>
>> [1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif
>>
>> Signed-off-by: Ming Lei <ming.lei@canonical.com>
>> ---
>>  drivers/media/video/v4l2-ioctl.c |   38 ++++++++++++++++++++
>>  include/linux/videodev2.h        |   70 ++++++++++++++++++++++++++++++++++++++
>>  include/media/v4l2-ioctl.h       |    6 +++
>>  3 files changed, 114 insertions(+), 0 deletions(-)
>>
>> diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
>> index e1da8fc..fc6266f 100644
>> --- a/drivers/media/video/v4l2-ioctl.c
>> +++ b/drivers/media/video/v4l2-ioctl.c
>> @@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
>>               dbgarg(cmd, "index=%d", b->index);
>>               break;
>>       }
>> +     case VIDIOC_G_FD_RESULT:
>> +     {
>> +             struct v4l2_fd_result *fr = arg;
>> +
>> +             if (!ops->vidioc_g_fd_result)
>> +                     break;
>> +
>> +             ret = ops->vidioc_g_fd_result(file, fh, fr);
>> +
>> +             dbgarg(cmd, "index=%d", fr->buf_index);
>> +             break;
>> +     }
>> +     case VIDIOC_G_FD_COUNT:
>> +     {
>> +             struct v4l2_fd_count *fc = arg;
>> +
>> +             if (!ops->vidioc_g_fd_count)
>> +                     break;
>> +
>> +             ret = ops->vidioc_g_fd_count(file, fh, fc);
>> +
>> +             dbgarg(cmd, "index=%d", fc->buf_index);
>> +             break;
>> +     }
>>       default:
>>               if (!ops->vidioc_default)
>>                       break;
>> @@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
>>               }
>>               break;
>>       }
>> +
>> +     case VIDIOC_G_FD_RESULT: {
>> +             struct v4l2_fd_result *fr = parg;
>> +
>> +             if (fr->face_cnt != 0) {
>> +                     *user_ptr = (void __user *)fr->fd;
>> +                     *kernel_ptr = (void *)&fr->fd;
>> +                     *array_size = sizeof(struct v4l2_fd_detection)
>> +                                 * fr->face_cnt;
>> +                     ret = 1;
>> +             }
>> +             break;
>> +
>> +     }
>>       }
>>
>>       return ret;
>> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
>> index 4b752d5..073eb4d 100644
>> --- a/include/linux/videodev2.h
>> +++ b/include/linux/videodev2.h
>> @@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
>>       __u32                   reserved[8];
>>  };
>>
>> +/**
>> + * struct v4l2_obj_detection
>> + * @buf_index:       entry, index of v4l2_buffer for face detection
>> + * @centerx: return, position in x direction of detected object
>> + * @centery: return, position in y direction of detected object
>> + * @angle:   return, angle of detected object
>> + *           0 deg ~ 359 deg, vertical is 0 deg, clockwise
>> + * @sizex:   return, size in x direction of detected object
>> + * @sizey:   return, size in y direction of detected object
>> + * @confidence:      return, confidence level of detection result
>> + *           0: the heighest level, 9: the lowest level
>
> Hmm, not a good idea to align a public interface to the capabilities
> of a single hardware implementation.

I think that the current omap interface is general enough, so why can't
we use it as public interface?

> min/max confidence could be queried with
> relevant controls and here we could remove the line implying range.

No, the confidence is used to describe the probability about
the correctness of the current detection result. Anyway, no FD can
make sure that it is 100% correct.  Other HW can normalize its
confidence level to 0~9 so that application can handle it easily, IMO.

>> + * @reserved:        future extensions
>> + */
>> +struct v4l2_obj_detection {
>> +     __u16           centerx;
>> +     __u16           centery;
>> +     __u16           angle;
>> +     __u16           sizex;
>> +     __u16           sizey;
>
> How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
> After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
> for size but there seems to be missing en equivalent for position, e.g.

Maybe user space would like to plot a circle or ellipse over the detected
objection, and I am sure that I have seen this kind of plot over detected
face before.

> struct v4l2_position {
>        __s32 x;
>        __s32 y;
> };
>
>> +     __u16           confidence;
>> +     __u32           reserved[4];
>> +};
>> +
>> +#define V4L2_FD_HAS_LEFT_EYE 0x1
>> +#define V4L2_FD_HAS_RIGHT_EYE        0x2
>> +#define V4L2_FD_HAS_MOUTH    0x4
>> +#define V4L2_FD_HAS_FACE     0x8
>> +
>> +/**
>> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
>> + * @flag:    return, describe which objects are detected
>> + * @left_eye:        return, left_eye position if detected
>> + * @right_eye:       return, right_eye position if detected
>> + * @mouth_eye:       return, mouth_eye position if detected
>
> mouth_eye ? ;)

Sorry, it should be mouth, :-)

>
>> + * @face:    return, face position if detected
>> + */
>> +struct v4l2_fd_detection {
>> +     __u32   flag;
>> +     struct v4l2_obj_detection       left_eye;
>> +     struct v4l2_obj_detection       right_eye;
>> +     struct v4l2_obj_detection       mouth;
>> +     struct v4l2_obj_detection       face;
>
> I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
> and then struct v4l2_fd_detection would be simply an array of
> struct v4l2_obj_detection, i.e.
>
> struct v4l2_fd_detection {
>        unsigned int count;
>        struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
> };
>
> This might be more flexible, e.g. if in the future some hardware supports
> detecting wrinkles, we could easily add that by just defining a new flag:
> V4L2_FD_HAS_WRINKLES, etc.

This is a bit flexible, but not explicit enough for describing
interface, how about
reserving these as below for future usage?

	struct v4l2_fd_detection {
		__u32   flag;
		Struct v4l2_obj_detection       left_eye;
		Struct v4l2_obj_detection       right_eye;
		Struct v4l2_obj_detection       mouth;
		Struct v4l2_obj_detection       face;
		Struct v4l2_obj_detection       reserved[4];
	};


thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-08  3:42       ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-08  3:42 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Tue, Dec 6, 2011 at 6:15 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/02/2011 04:02 PM, Ming Lei wrote:
>> This patch introduces two new IOCTLs and related data
>> structure defination which will be used by the coming
>> face detection video device.
>>
>> The two IOCTLs and related data structure are used by
>> user space application to retrieve the results of face
>> detection. They can be called after one v4l2_buffer
>> has been ioctl(VIDIOC_DQBUF) and before it will be
>> ioctl(VIDIOC_QBUF).
>>
>> The utility fdif[1] is useing the two IOCTLs to find
>> faces deteced in raw images or video streams.
>>
>> [1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif
>>
>> Signed-off-by: Ming Lei <ming.lei@canonical.com>
>> ---
>> ?drivers/media/video/v4l2-ioctl.c | ? 38 ++++++++++++++++++++
>> ?include/linux/videodev2.h ? ? ? ?| ? 70 ++++++++++++++++++++++++++++++++++++++
>> ?include/media/v4l2-ioctl.h ? ? ? | ? ?6 +++
>> ?3 files changed, 114 insertions(+), 0 deletions(-)
>>
>> diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
>> index e1da8fc..fc6266f 100644
>> --- a/drivers/media/video/v4l2-ioctl.c
>> +++ b/drivers/media/video/v4l2-ioctl.c
>> @@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
>> ? ? ? ? ? ? ? dbgarg(cmd, "index=%d", b->index);
>> ? ? ? ? ? ? ? break;
>> ? ? ? }
>> + ? ? case VIDIOC_G_FD_RESULT:
>> + ? ? {
>> + ? ? ? ? ? ? struct v4l2_fd_result *fr = arg;
>> +
>> + ? ? ? ? ? ? if (!ops->vidioc_g_fd_result)
>> + ? ? ? ? ? ? ? ? ? ? break;
>> +
>> + ? ? ? ? ? ? ret = ops->vidioc_g_fd_result(file, fh, fr);
>> +
>> + ? ? ? ? ? ? dbgarg(cmd, "index=%d", fr->buf_index);
>> + ? ? ? ? ? ? break;
>> + ? ? }
>> + ? ? case VIDIOC_G_FD_COUNT:
>> + ? ? {
>> + ? ? ? ? ? ? struct v4l2_fd_count *fc = arg;
>> +
>> + ? ? ? ? ? ? if (!ops->vidioc_g_fd_count)
>> + ? ? ? ? ? ? ? ? ? ? break;
>> +
>> + ? ? ? ? ? ? ret = ops->vidioc_g_fd_count(file, fh, fc);
>> +
>> + ? ? ? ? ? ? dbgarg(cmd, "index=%d", fc->buf_index);
>> + ? ? ? ? ? ? break;
>> + ? ? }
>> ? ? ? default:
>> ? ? ? ? ? ? ? if (!ops->vidioc_default)
>> ? ? ? ? ? ? ? ? ? ? ? break;
>> @@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
>> ? ? ? ? ? ? ? }
>> ? ? ? ? ? ? ? break;
>> ? ? ? }
>> +
>> + ? ? case VIDIOC_G_FD_RESULT: {
>> + ? ? ? ? ? ? struct v4l2_fd_result *fr = parg;
>> +
>> + ? ? ? ? ? ? if (fr->face_cnt != 0) {
>> + ? ? ? ? ? ? ? ? ? ? *user_ptr = (void __user *)fr->fd;
>> + ? ? ? ? ? ? ? ? ? ? *kernel_ptr = (void *)&fr->fd;
>> + ? ? ? ? ? ? ? ? ? ? *array_size = sizeof(struct v4l2_fd_detection)
>> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? * fr->face_cnt;
>> + ? ? ? ? ? ? ? ? ? ? ret = 1;
>> + ? ? ? ? ? ? }
>> + ? ? ? ? ? ? break;
>> +
>> + ? ? }
>> ? ? ? }
>>
>> ? ? ? return ret;
>> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
>> index 4b752d5..073eb4d 100644
>> --- a/include/linux/videodev2.h
>> +++ b/include/linux/videodev2.h
>> @@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
>> ? ? ? __u32 ? ? ? ? ? ? ? ? ? reserved[8];
>> ?};
>>
>> +/**
>> + * struct v4l2_obj_detection
>> + * @buf_index: ? ? ? entry, index of v4l2_buffer for face detection
>> + * @centerx: return, position in x direction of detected object
>> + * @centery: return, position in y direction of detected object
>> + * @angle: ? return, angle of detected object
>> + * ? ? ? ? ? 0 deg ~ 359 deg, vertical is 0 deg, clockwise
>> + * @sizex: ? return, size in x direction of detected object
>> + * @sizey: ? return, size in y direction of detected object
>> + * @confidence: ? ? ?return, confidence level of detection result
>> + * ? ? ? ? ? 0: the heighest level, 9: the lowest level
>
> Hmm, not a good idea to align a public interface to the capabilities
> of a single hardware implementation.

I think that the current omap interface is general enough, so why can't
we use it as public interface?

> min/max confidence could be queried with
> relevant controls and here we could remove the line implying range.

No, the confidence is used to describe the probability about
the correctness of the current detection result. Anyway, no FD can
make sure that it is 100% correct.  Other HW can normalize its
confidence level to 0~9 so that application can handle it easily, IMO.

>> + * @reserved: ? ? ? ?future extensions
>> + */
>> +struct v4l2_obj_detection {
>> + ? ? __u16 ? ? ? ? ? centerx;
>> + ? ? __u16 ? ? ? ? ? centery;
>> + ? ? __u16 ? ? ? ? ? angle;
>> + ? ? __u16 ? ? ? ? ? sizex;
>> + ? ? __u16 ? ? ? ? ? sizey;
>
> How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
> After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
> for size but there seems to be missing en equivalent for position, e.g.

Maybe user space would like to plot a circle or ellipse over the detected
objection, and I am sure that I have seen this kind of plot over detected
face before.

> struct v4l2_position {
> ? ? ? ?__s32 x;
> ? ? ? ?__s32 y;
> };
>
>> + ? ? __u16 ? ? ? ? ? confidence;
>> + ? ? __u32 ? ? ? ? ? reserved[4];
>> +};
>> +
>> +#define V4L2_FD_HAS_LEFT_EYE 0x1
>> +#define V4L2_FD_HAS_RIGHT_EYE ? ? ? ?0x2
>> +#define V4L2_FD_HAS_MOUTH ? ?0x4
>> +#define V4L2_FD_HAS_FACE ? ? 0x8
>> +
>> +/**
>> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
>> + * @flag: ? ?return, describe which objects are detected
>> + * @left_eye: ? ? ? ?return, left_eye position if detected
>> + * @right_eye: ? ? ? return, right_eye position if detected
>> + * @mouth_eye: ? ? ? return, mouth_eye position if detected
>
> mouth_eye ? ;)

Sorry, it should be mouth, :-)

>
>> + * @face: ? ?return, face position if detected
>> + */
>> +struct v4l2_fd_detection {
>> + ? ? __u32 ? flag;
>> + ? ? struct v4l2_obj_detection ? ? ? left_eye;
>> + ? ? struct v4l2_obj_detection ? ? ? right_eye;
>> + ? ? struct v4l2_obj_detection ? ? ? mouth;
>> + ? ? struct v4l2_obj_detection ? ? ? face;
>
> I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
> and then struct v4l2_fd_detection would be simply an array of
> struct v4l2_obj_detection, i.e.
>
> struct v4l2_fd_detection {
> ? ? ? ?unsigned int count;
> ? ? ? ?struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
> };
>
> This might be more flexible, e.g. if in the future some hardware supports
> detecting wrinkles, we could easily add that by just defining a new flag:
> V4L2_FD_HAS_WRINKLES, etc.

This is a bit flexible, but not explicit enough for describing
interface, how about
reserving these as below for future usage?

	struct v4l2_fd_detection {
		__u32   flag;
		Struct v4l2_obj_detection       left_eye;
		Struct v4l2_obj_detection       right_eye;
		Struct v4l2_obj_detection       mouth;
		Struct v4l2_obj_detection       face;
		Struct v4l2_obj_detection       reserved[4];
	};


thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-08  3:42       ` Ming Lei
@ 2011-12-08 22:27         ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-08 22:27 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

On 12/08/2011 04:42 AM, Ming Lei wrote:
>>> +/**
>>> + * struct v4l2_obj_detection
>>> + * @buf_index:       entry, index of v4l2_buffer for face detection

I would prefer having the frame sequence number here. It will be more
future proof IMHO. If for instance we decide to use such an ioctl on
a v4l2 sub-device, without dequeuing buffers, there will be no problem
with that. And still in your specific use case it's not big deal to
look up the buffer index given it's sequence number in the application.

>>> + * @centerx: return, position in x direction of detected object
>>> + * @centery: return, position in y direction of detected object
>>> + * @angle:   return, angle of detected object
>>> + *           0 deg ~ 359 deg, vertical is 0 deg, clockwise
>>> + * @sizex:   return, size in x direction of detected object
>>> + * @sizey:   return, size in y direction of detected object
>>> + * @confidence:      return, confidence level of detection result
>>> + *           0: the heighest level, 9: the lowest level
>>
>> Hmm, not a good idea to align a public interface to the capabilities
>> of a single hardware implementation.
> 
> I think that the current omap interface is general enough, so why can't
> we use it as public interface?

I meant exactly the line implying the range. What if for some hardware
it's 0..11 ?

> 
>> min/max confidence could be queried with
>> relevant controls and here we could remove the line implying range.
> 
> No, the confidence is used to describe the probability about
> the correctness of the current detection result. Anyway, no FD can
> make sure that it is 100% correct.  Other HW can normalize its
> confidence level to 0~9 so that application can handle it easily, IMO.

1..100 might be better, to minimize rounding errors. Nevertheless IMO if we
can export an exact range supported by FD device we should do it, and let
upper layers do the normalization. And the bigger numbers should mean higher
confidence, consistently for all devices.

Do you think we could assume that the FD threshold range (FD_LHIT register
in case of OMAP4) is always same as the result confidence level ?

If so then the confidence level range could possibly be queried with the
detection threshold control. We could name it V4L2_CID_FD_CONFIDENCE_THRESHOLD
for example.
I could take care of preparing the control class draft and the documentation
for it.

> 
>>> + * @reserved:        future extensions
>>> + */
>>> +struct v4l2_obj_detection {

How about changing name of this structure to v4l2_fd_primitive or v4l2_fd_shape ?

>>> +     __u16           centerx;
>>> +     __u16           centery;
>>> +     __u16           angle;
>>> +     __u16           sizex;
>>> +     __u16           sizey;
>>
>> How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
>> After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
>> for size but there seems to be missing en equivalent for position, e.g.
> 
> Maybe user space would like to plot a circle or ellipse over the detected
> objection, and I am sure that I have seen this kind of plot over detected
> face before.

OK, in any way I suggest to replace all __u16 with __u32, to minimize performance
issues and be consistent with the data type specifying pixel values elsewhere in
V4L.
It makes sense to make 'confidence' __u32 as well and add a flags attribute to
indicate the shape.

>>
>>> +     __u16           confidence;
>>> +     __u32           reserved[4];

And then
	  __u32           reserved[10];

or
	  __u32           reserved[2];

>>> +};
>>> +
>>> +#define V4L2_FD_HAS_LEFT_EYE 0x1
>>> +#define V4L2_FD_HAS_RIGHT_EYE        0x2
>>> +#define V4L2_FD_HAS_MOUTH    0x4
>>> +#define V4L2_FD_HAS_FACE     0x8

Do you think we could change it to:

#define V4L2_FD_FL_LEFT_EYE	(1 << 0)
#define V4L2_FD_FL_RIGHT_EYE	(1 << 1)
#define V4L2_FD_FL_MOUTH	(1 << 2)
#define V4L2_FD_FL_FACE		(1 << 3)

and add:

#define V4L2_FD_FL_SMILE	(1 << 4)
#define V4L2_FD_FL_BLINK	(1 << 5)

?
>>> +
>>> +/**
>>> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
>>> + * @flag:    return, describe which objects are detected
>>> + * @left_eye:        return, left_eye position if detected
>>> + * @right_eye:       return, right_eye position if detected
>>> + * @mouth_eye:       return, mouth_eye position if detected
>>
>> mouth_eye ? ;)
> 
> Sorry, it should be mouth, :-)

:) also the word return could be omitted.

> 
>>
>>> + * @face:    return, face position if detected
>>> + */
>>> +struct v4l2_fd_detection {

How about changing the name to v4l2_fd_object ?

>>> +     __u32   flag;
>>> +     struct v4l2_obj_detection       left_eye;
>>> +     struct v4l2_obj_detection       right_eye;
>>> +     struct v4l2_obj_detection       mouth;
>>> +     struct v4l2_obj_detection       face;
>>
>> I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
>> and then struct v4l2_fd_detection would be simply an array of
>> struct v4l2_obj_detection, i.e.
>>
>> struct v4l2_fd_detection {
>>        unsigned int count;
>>        struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
>> };
>>
>> This might be more flexible, e.g. if in the future some hardware supports
>> detecting wrinkles, we could easily add that by just defining a new flag:
>> V4L2_FD_HAS_WRINKLES, etc.
> 
> This is a bit flexible, but not explicit enough for describing
> interface, how about reserving these as below for future usage?
> 
> 	struct v4l2_fd_detection {
> 		__u32   flag;
> 		Struct v4l2_obj_detection       left_eye;
> 		Struct v4l2_obj_detection       right_eye;
> 		Struct v4l2_obj_detection       mouth;
> 		Struct v4l2_obj_detection       face;
> 		Struct v4l2_obj_detection       reserved[4];
> 	};

OK, and how about this:

 	struct v4l2_fd_object {
 		struct v4l2_fd_shape	left_eye;
 		struct v4l2_fd_shape	right_eye;
 		struct v4l2_fd_shape	mouth;
 		struct v4l2_fd_shape	face;
		__u32			reserved[33];	
 		__u32   		flags;
	} __packed;

?

-- 
Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-08 22:27         ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-08 22:27 UTC (permalink / raw)
  To: linux-arm-kernel

On 12/08/2011 04:42 AM, Ming Lei wrote:
>>> +/**
>>> + * struct v4l2_obj_detection
>>> + * @buf_index:       entry, index of v4l2_buffer for face detection

I would prefer having the frame sequence number here. It will be more
future proof IMHO. If for instance we decide to use such an ioctl on
a v4l2 sub-device, without dequeuing buffers, there will be no problem
with that. And still in your specific use case it's not big deal to
look up the buffer index given it's sequence number in the application.

>>> + * @centerx: return, position in x direction of detected object
>>> + * @centery: return, position in y direction of detected object
>>> + * @angle:   return, angle of detected object
>>> + *           0 deg ~ 359 deg, vertical is 0 deg, clockwise
>>> + * @sizex:   return, size in x direction of detected object
>>> + * @sizey:   return, size in y direction of detected object
>>> + * @confidence:      return, confidence level of detection result
>>> + *           0: the heighest level, 9: the lowest level
>>
>> Hmm, not a good idea to align a public interface to the capabilities
>> of a single hardware implementation.
> 
> I think that the current omap interface is general enough, so why can't
> we use it as public interface?

I meant exactly the line implying the range. What if for some hardware
it's 0..11 ?

> 
>> min/max confidence could be queried with
>> relevant controls and here we could remove the line implying range.
> 
> No, the confidence is used to describe the probability about
> the correctness of the current detection result. Anyway, no FD can
> make sure that it is 100% correct.  Other HW can normalize its
> confidence level to 0~9 so that application can handle it easily, IMO.

1..100 might be better, to minimize rounding errors. Nevertheless IMO if we
can export an exact range supported by FD device we should do it, and let
upper layers do the normalization. And the bigger numbers should mean higher
confidence, consistently for all devices.

Do you think we could assume that the FD threshold range (FD_LHIT register
in case of OMAP4) is always same as the result confidence level ?

If so then the confidence level range could possibly be queried with the
detection threshold control. We could name it V4L2_CID_FD_CONFIDENCE_THRESHOLD
for example.
I could take care of preparing the control class draft and the documentation
for it.

> 
>>> + * @reserved:        future extensions
>>> + */
>>> +struct v4l2_obj_detection {

How about changing name of this structure to v4l2_fd_primitive or v4l2_fd_shape ?

>>> +     __u16           centerx;
>>> +     __u16           centery;
>>> +     __u16           angle;
>>> +     __u16           sizex;
>>> +     __u16           sizey;
>>
>> How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
>> After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
>> for size but there seems to be missing en equivalent for position, e.g.
> 
> Maybe user space would like to plot a circle or ellipse over the detected
> objection, and I am sure that I have seen this kind of plot over detected
> face before.

OK, in any way I suggest to replace all __u16 with __u32, to minimize performance
issues and be consistent with the data type specifying pixel values elsewhere in
V4L.
It makes sense to make 'confidence' __u32 as well and add a flags attribute to
indicate the shape.

>>
>>> +     __u16           confidence;
>>> +     __u32           reserved[4];

And then
	  __u32           reserved[10];

or
	  __u32           reserved[2];

>>> +};
>>> +
>>> +#define V4L2_FD_HAS_LEFT_EYE 0x1
>>> +#define V4L2_FD_HAS_RIGHT_EYE        0x2
>>> +#define V4L2_FD_HAS_MOUTH    0x4
>>> +#define V4L2_FD_HAS_FACE     0x8

Do you think we could change it to:

#define V4L2_FD_FL_LEFT_EYE	(1 << 0)
#define V4L2_FD_FL_RIGHT_EYE	(1 << 1)
#define V4L2_FD_FL_MOUTH	(1 << 2)
#define V4L2_FD_FL_FACE		(1 << 3)

and add:

#define V4L2_FD_FL_SMILE	(1 << 4)
#define V4L2_FD_FL_BLINK	(1 << 5)

?
>>> +
>>> +/**
>>> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
>>> + * @flag:    return, describe which objects are detected
>>> + * @left_eye:        return, left_eye position if detected
>>> + * @right_eye:       return, right_eye position if detected
>>> + * @mouth_eye:       return, mouth_eye position if detected
>>
>> mouth_eye ? ;)
> 
> Sorry, it should be mouth, :-)

:) also the word return could be omitted.

> 
>>
>>> + * @face:    return, face position if detected
>>> + */
>>> +struct v4l2_fd_detection {

How about changing the name to v4l2_fd_object ?

>>> +     __u32   flag;
>>> +     struct v4l2_obj_detection       left_eye;
>>> +     struct v4l2_obj_detection       right_eye;
>>> +     struct v4l2_obj_detection       mouth;
>>> +     struct v4l2_obj_detection       face;
>>
>> I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
>> and then struct v4l2_fd_detection would be simply an array of
>> struct v4l2_obj_detection, i.e.
>>
>> struct v4l2_fd_detection {
>>        unsigned int count;
>>        struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
>> };
>>
>> This might be more flexible, e.g. if in the future some hardware supports
>> detecting wrinkles, we could easily add that by just defining a new flag:
>> V4L2_FD_HAS_WRINKLES, etc.
> 
> This is a bit flexible, but not explicit enough for describing
> interface, how about reserving these as below for future usage?
> 
> 	struct v4l2_fd_detection {
> 		__u32   flag;
> 		Struct v4l2_obj_detection       left_eye;
> 		Struct v4l2_obj_detection       right_eye;
> 		Struct v4l2_obj_detection       mouth;
> 		Struct v4l2_obj_detection       face;
> 		Struct v4l2_obj_detection       reserved[4];
> 	};

OK, and how about this:

 	struct v4l2_fd_object {
 		struct v4l2_fd_shape	left_eye;
 		struct v4l2_fd_shape	right_eye;
 		struct v4l2_fd_shape	mouth;
 		struct v4l2_fd_shape	face;
		__u32			reserved[33];	
 		__u32   		flags;
	} __packed;

?

-- 
Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-07 13:40           ` Ming Lei
@ 2011-12-08 23:25             ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-08 23:25 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

On 12/07/2011 02:40 PM, Ming Lei wrote:
>>> I understand the API you mentioned here should belong to kernel internal
>>> API, correct me if it is wrong.
>>
>> Yes, I meant the in kernel design, i.e. generic face detection kernel module
>> and an OMAP4 FDIF driver. It makes lots of sense to separate common code
>> in this way, maybe even when there would be only OMAP devices using it.
> 
> Yes, that is the motivation of the generic FD module. I think we can focus on
> two use cases for the generic FD now:
> 
> - one is to detect faces from user space image data
> 
> - another one is to detect faces in image data generated from HW(SoC
> internal bus, resize hardware)
> 
> For OMAP4 hardware, input data is always from physically continuous
> memory directly, so it is very easy to support the two cases. For the
> use case 2,
> if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
> to pass the image buffer produced by other HW to FD hw directly.

Some H/W uses direct data buses to exchange data between processing blocks,
and there is no need for additional memory. We only need to manage the data
links, for which MC has been designed.

> 
> For other FD hardware, if it supports to detect faces in image data from
> physically continuous memory, I think the patch is OK to support it.
> 
> If the FD hw doesn't support to detect faces from physically continuous
> memory, I have some questions: how does user space app to parse the
> FD result if application can't get the input image data? If user space can

Do we need the map of detected objects on a input image in all cases ?
If an application needs only coordinates of detected object on a video
signal to for example, focus on it, trigger some action, or just count
detected faces, etc. Perhaps there are more practical similar use cases.

> get image data, how does it connect the image data with FD result? and

If hardware provides frame sequence numbers the FD result can be associated
with a frame, whether it's passing through H/W interconnect or is located
in memory.

> what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
> image data?

We have USERPTR and MMAP memeory buffer for streaming IO, those use
v4l2_buffer 1). read()/write() is also used 2), mostly for compressed formats.
Except that there are works on shared buffers.

> 
>> I'm sure now the Samsung devices won't fit in video output node based driver
>> design. They read image data in different ways and also the FD result format
>> is totally different.
> 
> I think user space will need the FD result, so it is very important to define
> API to describe the FD result format to user space. And the input about your
> FD HW result format is certainly helpful to define the API.

I'll post exact attributes generated by our FD detection H/W soon.

> 
>>>
>>>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>>>> reasonable to use the videodev interface for passing data to the kernel
>>>> from user space.
>>>>
>>>> But there might be face detection devices that accept data from other
>>>> H/W modules, e.g. transferred through SoC internal data buses between
>>>> image processing pipeline blocks. Thus any new interfaces need to be
>>>> designed with such devices in mind.
>>>>
>>>> Also the face detection hardware block might now have an input DMA
>>>> engine in it, the data could be fed from memory through some other
>>>> subsystem (e.g. resize/colour converter). Then the driver for that
>>>> subsystem would implement a video node.
>>>
>>> I think the direct input image or frame data to FD should be from memory
>>> no matter the actual data is from external H/W modules or input DMA because
>>> FD will take lot of time to detect faces in one image or frame and FD can't
>>> have so much memory to cache several images or frames data.
>>
>> Sorry, I cannot provide much details at the moment, but there exists hardware
>> that reads data from internal SoC buses and even if it uses some sort of
>> cache memory it doesn't necessarily have to be available for the user.
> 
> Without some hardware background, it is not easy to give a generic FD module
> design.

Yes, please give me some time so I can prepare the list of requirements.

> 
>> Still the FD result is associated with an image frame for such H/W, but not
>> necessarily with a memory buffer queued by a user application.
> 
> For user space application, it doesn't make sense to handle FD results
> only without image data.  Even though there are other ways of input
> image data to FD, user space still need to know the image data, so it makes
> sense to associate FD result with a memory buffer.
> 
>> How long it approximately takes to process single image for OMAP4 FDIF ?
> 
> See the link[2], and my test result is basically consistent with the data.

Thanks. The processing time is rather low, looks like we could easily detect
objects in each frame with 30..50 fps.

> 
>>>
>>> If you have seen this kind of FD hardware design, please let me know.
>>>
>>>> I'm for leaving the buffer handling details for individual drivers
>>>> and focusing on a standard interface for applications, i.e. new
>>>
>>> I think leaving buffer handling details in generic FD driver or
>>> individual drivers
>>> doesn't matter now, since it don't have effect on interfaces between kernel
>>> and user space.
>>
>> I think you misunderstood me. I wasn't talking about core/driver module split,
>> I meant we should not be making the user interface video node centric.
>>
>> I think for Samsung devices I'll need a capture video node for passing
> 
> Why is it a capture video node instead of OUTPUT v4l2 device? I think the

Let's forget about this capture device, I'm just in progress of learning
our devices internals, that's quite huge IPs..

> device name should be decided from the view of face detection function:
> FD need input image data and produce detection result.

No, we should be flexible about where the data comes from to a FD subsystem
and how it is then processed - if it is passed to some other processing block
or it is transferred to memory with DMA and returned to the user. And we
should use Media Controller API to route the data according to application
requirements.
OMAP4 FDIF is pretty simple, we need to think more about integrating FD with
existing data pipelines.

> 
>> the result to the user. So instead of associating FD result with a buffer index
> 
> See the explanation above.

No, I still insist on using frame sequence number rather than buffer index :-)

> 
>> we could try to use the frame sequence number (struct v4l2_buffer.sequence,
>> http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).
> 
> [1], http://marc.info/?t=132281644700005&r=1&w=2
> [2], http://e2e.ti.com/support/embedded/linux/f/354/t/128938.aspx#462740

1) http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-qbuf.html
2) http://linuxtv.org/downloads/v4l-dvb-apis/func-write.html


--

Thanks,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-08 23:25             ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-08 23:25 UTC (permalink / raw)
  To: linux-arm-kernel

On 12/07/2011 02:40 PM, Ming Lei wrote:
>>> I understand the API you mentioned here should belong to kernel internal
>>> API, correct me if it is wrong.
>>
>> Yes, I meant the in kernel design, i.e. generic face detection kernel module
>> and an OMAP4 FDIF driver. It makes lots of sense to separate common code
>> in this way, maybe even when there would be only OMAP devices using it.
> 
> Yes, that is the motivation of the generic FD module. I think we can focus on
> two use cases for the generic FD now:
> 
> - one is to detect faces from user space image data
> 
> - another one is to detect faces in image data generated from HW(SoC
> internal bus, resize hardware)
> 
> For OMAP4 hardware, input data is always from physically continuous
> memory directly, so it is very easy to support the two cases. For the
> use case 2,
> if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
> to pass the image buffer produced by other HW to FD hw directly.

Some H/W uses direct data buses to exchange data between processing blocks,
and there is no need for additional memory. We only need to manage the data
links, for which MC has been designed.

> 
> For other FD hardware, if it supports to detect faces in image data from
> physically continuous memory, I think the patch is OK to support it.
> 
> If the FD hw doesn't support to detect faces from physically continuous
> memory, I have some questions: how does user space app to parse the
> FD result if application can't get the input image data? If user space can

Do we need the map of detected objects on a input image in all cases ?
If an application needs only coordinates of detected object on a video
signal to for example, focus on it, trigger some action, or just count
detected faces, etc. Perhaps there are more practical similar use cases.

> get image data, how does it connect the image data with FD result? and

If hardware provides frame sequence numbers the FD result can be associated
with a frame, whether it's passing through H/W interconnect or is located
in memory.

> what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
> image data?

We have USERPTR and MMAP memeory buffer for streaming IO, those use
v4l2_buffer 1). read()/write() is also used 2), mostly for compressed formats.
Except that there are works on shared buffers.

> 
>> I'm sure now the Samsung devices won't fit in video output node based driver
>> design. They read image data in different ways and also the FD result format
>> is totally different.
> 
> I think user space will need the FD result, so it is very important to define
> API to describe the FD result format to user space. And the input about your
> FD HW result format is certainly helpful to define the API.

I'll post exact attributes generated by our FD detection H/W soon.

> 
>>>
>>>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>>>> reasonable to use the videodev interface for passing data to the kernel
>>>> from user space.
>>>>
>>>> But there might be face detection devices that accept data from other
>>>> H/W modules, e.g. transferred through SoC internal data buses between
>>>> image processing pipeline blocks. Thus any new interfaces need to be
>>>> designed with such devices in mind.
>>>>
>>>> Also the face detection hardware block might now have an input DMA
>>>> engine in it, the data could be fed from memory through some other
>>>> subsystem (e.g. resize/colour converter). Then the driver for that
>>>> subsystem would implement a video node.
>>>
>>> I think the direct input image or frame data to FD should be from memory
>>> no matter the actual data is from external H/W modules or input DMA because
>>> FD will take lot of time to detect faces in one image or frame and FD can't
>>> have so much memory to cache several images or frames data.
>>
>> Sorry, I cannot provide much details at the moment, but there exists hardware
>> that reads data from internal SoC buses and even if it uses some sort of
>> cache memory it doesn't necessarily have to be available for the user.
> 
> Without some hardware background, it is not easy to give a generic FD module
> design.

Yes, please give me some time so I can prepare the list of requirements.

> 
>> Still the FD result is associated with an image frame for such H/W, but not
>> necessarily with a memory buffer queued by a user application.
> 
> For user space application, it doesn't make sense to handle FD results
> only without image data.  Even though there are other ways of input
> image data to FD, user space still need to know the image data, so it makes
> sense to associate FD result with a memory buffer.
> 
>> How long it approximately takes to process single image for OMAP4 FDIF ?
> 
> See the link[2], and my test result is basically consistent with the data.

Thanks. The processing time is rather low, looks like we could easily detect
objects in each frame with 30..50 fps.

> 
>>>
>>> If you have seen this kind of FD hardware design, please let me know.
>>>
>>>> I'm for leaving the buffer handling details for individual drivers
>>>> and focusing on a standard interface for applications, i.e. new
>>>
>>> I think leaving buffer handling details in generic FD driver or
>>> individual drivers
>>> doesn't matter now, since it don't have effect on interfaces between kernel
>>> and user space.
>>
>> I think you misunderstood me. I wasn't talking about core/driver module split,
>> I meant we should not be making the user interface video node centric.
>>
>> I think for Samsung devices I'll need a capture video node for passing
> 
> Why is it a capture video node instead of OUTPUT v4l2 device? I think the

Let's forget about this capture device, I'm just in progress of learning
our devices internals, that's quite huge IPs..

> device name should be decided from the view of face detection function:
> FD need input image data and produce detection result.

No, we should be flexible about where the data comes from to a FD subsystem
and how it is then processed - if it is passed to some other processing block
or it is transferred to memory with DMA and returned to the user. And we
should use Media Controller API to route the data according to application
requirements.
OMAP4 FDIF is pretty simple, we need to think more about integrating FD with
existing data pipelines.

> 
>> the result to the user. So instead of associating FD result with a buffer index
> 
> See the explanation above.

No, I still insist on using frame sequence number rather than buffer index :-)

> 
>> we could try to use the frame sequence number (struct v4l2_buffer.sequence,
>> http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).
> 
> [1], http://marc.info/?t=132281644700005&r=1&w=2
> [2], http://e2e.ti.com/support/embedded/linux/f/354/t/128938.aspx#462740

1) http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-qbuf.html
2) http://linuxtv.org/downloads/v4l-dvb-apis/func-write.html


--

Thanks,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-08 22:27         ` Sylwester Nawrocki
@ 2011-12-09  4:34           ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-09  4:34 UTC (permalink / raw)
  To: Sylwester Nawrocki
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi,

On Fri, Dec 9, 2011 at 6:27 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/08/2011 04:42 AM, Ming Lei wrote:
>>>> +/**
>>>> + * struct v4l2_obj_detection
>>>> + * @buf_index:       entry, index of v4l2_buffer for face detection
>
> I would prefer having the frame sequence number here. It will be more
> future proof IMHO. If for instance we decide to use such an ioctl on
> a v4l2 sub-device, without dequeuing buffers, there will be no problem
> with that. And still in your specific use case it's not big deal to
> look up the buffer index given it's sequence number in the application.

OK, take your suggestion to use frame index, but I still have question
about it, see my question in another thread.

>
>>>> + * @centerx: return, position in x direction of detected object
>>>> + * @centery: return, position in y direction of detected object
>>>> + * @angle:   return, angle of detected object
>>>> + *           0 deg ~ 359 deg, vertical is 0 deg, clockwise
>>>> + * @sizex:   return, size in x direction of detected object
>>>> + * @sizey:   return, size in y direction of detected object
>>>> + * @confidence:      return, confidence level of detection result
>>>> + *           0: the heighest level, 9: the lowest level
>>>
>>> Hmm, not a good idea to align a public interface to the capabilities
>>> of a single hardware implementation.
>>
>> I think that the current omap interface is general enough, so why can't
>> we use it as public interface?
>
> I meant exactly the line implying the range. What if for some hardware
> it's 0..11 ?

We can let driver to normalize it to user which doesn't care if the range
is 0~11 or 10~21, a uniform range should always make user happy,
shouldn't it?

>
>>
>>> min/max confidence could be queried with
>>> relevant controls and here we could remove the line implying range.
>>
>> No, the confidence is used to describe the probability about
>> the correctness of the current detection result. Anyway, no FD can
>> make sure that it is 100% correct.  Other HW can normalize its
>> confidence level to 0~9 so that application can handle it easily, IMO.
>
> 1..100 might be better, to minimize rounding errors. Nevertheless IMO if we
> can export an exact range supported by FD device we should do it, and let
> upper layers do the normalization. And the bigger numbers should mean higher
> confidence, consistently for all devices.

Looks 1..100 is better, and I will change it to 1..100.

>
> Do you think we could assume that the FD threshold range (FD_LHIT register
> in case of OMAP4) is always same as the result confidence level ?

No, they are different. FD_LHIT is used to guild FD HW to detect more
faces but more false positives __or__ less faces but less false positives.

A control class is needed to be introduced for adjusting this value of FD
HW, and I think a normalized range is better too.

>
> If so then the confidence level range could possibly be queried with the
> detection threshold control. We could name it V4L2_CID_FD_CONFIDENCE_THRESHOLD

As I said above, there is no advantage to export the range to user, and a
uniform range will make user happy.

> for example.
> I could take care of preparing the control class draft and the documentation
> for it.

It is great to hear it, :-)

>
>>
>>>> + * @reserved:        future extensions
>>>> + */
>>>> +struct v4l2_obj_detection {
>
> How about changing name of this structure to v4l2_fd_primitive or v4l2_fd_shape ?
>

I think v4l2_obj_detection is better because it can be reused to describe
some other kind of object detection from video in the future.

>>>> +     __u16           centerx;
>>>> +     __u16           centery;
>>>> +     __u16           angle;
>>>> +     __u16           sizex;
>>>> +     __u16           sizey;
>>>
>>> How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
>>> After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
>>> for size but there seems to be missing en equivalent for position, e.g.
>>
>> Maybe user space would like to plot a circle or ellipse over the detected
>> objection, and I am sure that I have seen this kind of plot over detected
>> face before.
>
> OK, in any way I suggest to replace all __u16 with __u32, to minimize performance
> issues and be consistent with the data type specifying pixel values elsewhere in
> V4L.

OK, but may introduce more memory footprint for the fd result.

> It makes sense to make 'confidence' __u32 as well and add a flags attribute to
> indicate the shape.

Sounds good.

>
>>>
>>>> +     __u16           confidence;
>>>> +     __u32           reserved[4];
>
> And then
>          __u32           reserved[10];
>
> or
>          __u32           reserved[2];
>
>>>> +};
>>>> +
>>>> +#define V4L2_FD_HAS_LEFT_EYE 0x1
>>>> +#define V4L2_FD_HAS_RIGHT_EYE        0x2
>>>> +#define V4L2_FD_HAS_MOUTH    0x4
>>>> +#define V4L2_FD_HAS_FACE     0x8
>
> Do you think we could change it to:
>
> #define V4L2_FD_FL_LEFT_EYE     (1 << 0)
> #define V4L2_FD_FL_RIGHT_EYE    (1 << 1)
> #define V4L2_FD_FL_MOUTH        (1 << 2)
> #define V4L2_FD_FL_FACE         (1 << 3)

OK

> and add:
>
> #define V4L2_FD_FL_SMILE        (1 << 4)
> #define V4L2_FD_FL_BLINK        (1 << 5)

Do you have any suggestion about how to describe this kind of
detection?

>>>> +
>>>> +/**
>>>> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
>>>> + * @flag:    return, describe which objects are detected
>>>> + * @left_eye:        return, left_eye position if detected
>>>> + * @right_eye:       return, right_eye position if detected
>>>> + * @mouth_eye:       return, mouth_eye position if detected
>>>
>>> mouth_eye ? ;)
>>
>> Sorry, it should be mouth, :-)
>
> :) also the word return could be omitted.
>
>>
>>>
>>>> + * @face:    return, face position if detected
>>>> + */
>>>> +struct v4l2_fd_detection {
>
> How about changing the name to v4l2_fd_object ?

I think the structure is used to describe one single detection result which
may include several kind of objects detected, so sounds
v4l2_fd_detection is better than v4l2_fd_object(s?).

>
>>>> +     __u32   flag;
>>>> +     struct v4l2_obj_detection       left_eye;
>>>> +     struct v4l2_obj_detection       right_eye;
>>>> +     struct v4l2_obj_detection       mouth;
>>>> +     struct v4l2_obj_detection       face;
>>>
>>> I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
>>> and then struct v4l2_fd_detection would be simply an array of
>>> struct v4l2_obj_detection, i.e.
>>>
>>> struct v4l2_fd_detection {
>>>        unsigned int count;
>>>        struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
>>> };
>>>
>>> This might be more flexible, e.g. if in the future some hardware supports
>>> detecting wrinkles, we could easily add that by just defining a new flag:
>>> V4L2_FD_HAS_WRINKLES, etc.
>>
>> This is a bit flexible, but not explicit enough for describing
>> interface, how about reserving these as below for future usage?
>>
>>       struct v4l2_fd_detection {
>>               __u32   flag;
>>               Struct v4l2_obj_detection       left_eye;
>>               Struct v4l2_obj_detection       right_eye;
>>               Struct v4l2_obj_detection       mouth;
>>               Struct v4l2_obj_detection       face;
>>               Struct v4l2_obj_detection       reserved[4];
>>       };
>
> OK, and how about this:
>
>        struct v4l2_fd_object {
>                struct v4l2_fd_shape    left_eye;
>                struct v4l2_fd_shape    right_eye;
>                struct v4l2_fd_shape    mouth;
>                struct v4l2_fd_shape    face;
>                __u32                   reserved[33];

Why is struct 'v4l2_fd_shape    reserved[4]' removed?

>                __u32                   flags;
>        } __packed;

Why is '__packed' needed? It will introduce performance loss if we have
not good reason to do it.


thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-09  4:34           ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-09  4:34 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Fri, Dec 9, 2011 at 6:27 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/08/2011 04:42 AM, Ming Lei wrote:
>>>> +/**
>>>> + * struct v4l2_obj_detection
>>>> + * @buf_index: ? ? ? entry, index of v4l2_buffer for face detection
>
> I would prefer having the frame sequence number here. It will be more
> future proof IMHO. If for instance we decide to use such an ioctl on
> a v4l2 sub-device, without dequeuing buffers, there will be no problem
> with that. And still in your specific use case it's not big deal to
> look up the buffer index given it's sequence number in the application.

OK, take your suggestion to use frame index, but I still have question
about it, see my question in another thread.

>
>>>> + * @centerx: return, position in x direction of detected object
>>>> + * @centery: return, position in y direction of detected object
>>>> + * @angle: ? return, angle of detected object
>>>> + * ? ? ? ? ? 0 deg ~ 359 deg, vertical is 0 deg, clockwise
>>>> + * @sizex: ? return, size in x direction of detected object
>>>> + * @sizey: ? return, size in y direction of detected object
>>>> + * @confidence: ? ? ?return, confidence level of detection result
>>>> + * ? ? ? ? ? 0: the heighest level, 9: the lowest level
>>>
>>> Hmm, not a good idea to align a public interface to the capabilities
>>> of a single hardware implementation.
>>
>> I think that the current omap interface is general enough, so why can't
>> we use it as public interface?
>
> I meant exactly the line implying the range. What if for some hardware
> it's 0..11 ?

We can let driver to normalize it to user which doesn't care if the range
is 0~11 or 10~21, a uniform range should always make user happy,
shouldn't it?

>
>>
>>> min/max confidence could be queried with
>>> relevant controls and here we could remove the line implying range.
>>
>> No, the confidence is used to describe the probability about
>> the correctness of the current detection result. Anyway, no FD can
>> make sure that it is 100% correct. ?Other HW can normalize its
>> confidence level to 0~9 so that application can handle it easily, IMO.
>
> 1..100 might be better, to minimize rounding errors. Nevertheless IMO if we
> can export an exact range supported by FD device we should do it, and let
> upper layers do the normalization. And the bigger numbers should mean higher
> confidence, consistently for all devices.

Looks 1..100 is better, and I will change it to 1..100.

>
> Do you think we could assume that the FD threshold range (FD_LHIT register
> in case of OMAP4) is always same as the result confidence level ?

No, they are different. FD_LHIT is used to guild FD HW to detect more
faces but more false positives __or__ less faces but less false positives.

A control class is needed to be introduced for adjusting this value of FD
HW, and I think a normalized range is better too.

>
> If so then the confidence level range could possibly be queried with the
> detection threshold control. We could name it V4L2_CID_FD_CONFIDENCE_THRESHOLD

As I said above, there is no advantage to export the range to user, and a
uniform range will make user happy.

> for example.
> I could take care of preparing the control class draft and the documentation
> for it.

It is great to hear it, :-)

>
>>
>>>> + * @reserved: ? ? ? ?future extensions
>>>> + */
>>>> +struct v4l2_obj_detection {
>
> How about changing name of this structure to v4l2_fd_primitive or v4l2_fd_shape ?
>

I think v4l2_obj_detection is better because it can be reused to describe
some other kind of object detection from video in the future.

>>>> + ? ? __u16 ? ? ? ? ? centerx;
>>>> + ? ? __u16 ? ? ? ? ? centery;
>>>> + ? ? __u16 ? ? ? ? ? angle;
>>>> + ? ? __u16 ? ? ? ? ? sizex;
>>>> + ? ? __u16 ? ? ? ? ? sizey;
>>>
>>> How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
>>> After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
>>> for size but there seems to be missing en equivalent for position, e.g.
>>
>> Maybe user space would like to plot a circle or ellipse over the detected
>> objection, and I am sure that I have seen this kind of plot over detected
>> face before.
>
> OK, in any way I suggest to replace all __u16 with __u32, to minimize performance
> issues and be consistent with the data type specifying pixel values elsewhere in
> V4L.

OK, but may introduce more memory footprint for the fd result.

> It makes sense to make 'confidence' __u32 as well and add a flags attribute to
> indicate the shape.

Sounds good.

>
>>>
>>>> + ? ? __u16 ? ? ? ? ? confidence;
>>>> + ? ? __u32 ? ? ? ? ? reserved[4];
>
> And then
> ? ? ? ? ?__u32 ? ? ? ? ? reserved[10];
>
> or
> ? ? ? ? ?__u32 ? ? ? ? ? reserved[2];
>
>>>> +};
>>>> +
>>>> +#define V4L2_FD_HAS_LEFT_EYE 0x1
>>>> +#define V4L2_FD_HAS_RIGHT_EYE ? ? ? ?0x2
>>>> +#define V4L2_FD_HAS_MOUTH ? ?0x4
>>>> +#define V4L2_FD_HAS_FACE ? ? 0x8
>
> Do you think we could change it to:
>
> #define V4L2_FD_FL_LEFT_EYE ? ? (1 << 0)
> #define V4L2_FD_FL_RIGHT_EYE ? ?(1 << 1)
> #define V4L2_FD_FL_MOUTH ? ? ? ?(1 << 2)
> #define V4L2_FD_FL_FACE ? ? ? ? (1 << 3)

OK

> and add:
>
> #define V4L2_FD_FL_SMILE ? ? ? ?(1 << 4)
> #define V4L2_FD_FL_BLINK ? ? ? ?(1 << 5)

Do you have any suggestion about how to describe this kind of
detection?

>>>> +
>>>> +/**
>>>> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
>>>> + * @flag: ? ?return, describe which objects are detected
>>>> + * @left_eye: ? ? ? ?return, left_eye position if detected
>>>> + * @right_eye: ? ? ? return, right_eye position if detected
>>>> + * @mouth_eye: ? ? ? return, mouth_eye position if detected
>>>
>>> mouth_eye ? ;)
>>
>> Sorry, it should be mouth, :-)
>
> :) also the word return could be omitted.
>
>>
>>>
>>>> + * @face: ? ?return, face position if detected
>>>> + */
>>>> +struct v4l2_fd_detection {
>
> How about changing the name to v4l2_fd_object ?

I think the structure is used to describe one single detection result which
may include several kind of objects detected, so sounds
v4l2_fd_detection is better than v4l2_fd_object(s?).

>
>>>> + ? ? __u32 ? flag;
>>>> + ? ? struct v4l2_obj_detection ? ? ? left_eye;
>>>> + ? ? struct v4l2_obj_detection ? ? ? right_eye;
>>>> + ? ? struct v4l2_obj_detection ? ? ? mouth;
>>>> + ? ? struct v4l2_obj_detection ? ? ? face;
>>>
>>> I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
>>> and then struct v4l2_fd_detection would be simply an array of
>>> struct v4l2_obj_detection, i.e.
>>>
>>> struct v4l2_fd_detection {
>>> ? ? ? ?unsigned int count;
>>> ? ? ? ?struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
>>> };
>>>
>>> This might be more flexible, e.g. if in the future some hardware supports
>>> detecting wrinkles, we could easily add that by just defining a new flag:
>>> V4L2_FD_HAS_WRINKLES, etc.
>>
>> This is a bit flexible, but not explicit enough for describing
>> interface, how about reserving these as below for future usage?
>>
>> ? ? ? struct v4l2_fd_detection {
>> ? ? ? ? ? ? ? __u32 ? flag;
>> ? ? ? ? ? ? ? Struct v4l2_obj_detection ? ? ? left_eye;
>> ? ? ? ? ? ? ? Struct v4l2_obj_detection ? ? ? right_eye;
>> ? ? ? ? ? ? ? Struct v4l2_obj_detection ? ? ? mouth;
>> ? ? ? ? ? ? ? Struct v4l2_obj_detection ? ? ? face;
>> ? ? ? ? ? ? ? Struct v4l2_obj_detection ? ? ? reserved[4];
>> ? ? ? };
>
> OK, and how about this:
>
> ? ? ? ?struct v4l2_fd_object {
> ? ? ? ? ? ? ? ?struct v4l2_fd_shape ? ?left_eye;
> ? ? ? ? ? ? ? ?struct v4l2_fd_shape ? ?right_eye;
> ? ? ? ? ? ? ? ?struct v4l2_fd_shape ? ?mouth;
> ? ? ? ? ? ? ? ?struct v4l2_fd_shape ? ?face;
> ? ? ? ? ? ? ? ?__u32 ? ? ? ? ? ? ? ? ? reserved[33];

Why is struct 'v4l2_fd_shape ? ?reserved[4]' removed?

> ? ? ? ? ? ? ? ?__u32 ? ? ? ? ? ? ? ? ? flags;
> ? ? ? ?} __packed;

Why is '__packed' needed? It will introduce performance loss if we have
not good reason to do it.


thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-08 23:25             ` Sylwester Nawrocki
  (?)
@ 2011-12-09 15:10               ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-09 15:10 UTC (permalink / raw)
  To: Sylwester Nawrocki
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

On Fri, Dec 9, 2011 at 7:25 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/07/2011 02:40 PM, Ming Lei wrote:
>>>> I understand the API you mentioned here should belong to kernel internal
>>>> API, correct me if it is wrong.
>>>
>>> Yes, I meant the in kernel design, i.e. generic face detection kernel module
>>> and an OMAP4 FDIF driver. It makes lots of sense to separate common code
>>> in this way, maybe even when there would be only OMAP devices using it.
>>
>> Yes, that is the motivation of the generic FD module. I think we can focus on
>> two use cases for the generic FD now:
>>
>> - one is to detect faces from user space image data
>>
>> - another one is to detect faces in image data generated from HW(SoC
>> internal bus, resize hardware)
>>
>> For OMAP4 hardware, input data is always from physically continuous
>> memory directly, so it is very easy to support the two cases. For the
>> use case 2,
>> if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
>> to pass the image buffer produced by other HW to FD hw directly.
>
> Some H/W uses direct data buses to exchange data between processing blocks,
> and there is no need for additional memory. We only need to manage the data
> links, for which MC has been designed.

For OMAP4 FD, it is not needed to include FD into MC framework since a
intermediate buffer is always required. If your HW doesn't belong to this
case, what is the output of your HW FD in the link? Also sounds FD results
may not be needed at all for use space application in the case.

>
>>
>> For other FD hardware, if it supports to detect faces in image data from
>> physically continuous memory, I think the patch is OK to support it.
>>
>> If the FD hw doesn't support to detect faces from physically continuous
>> memory, I have some questions: how does user space app to parse the
>> FD result if application can't get the input image data? If user space can
>
> Do we need the map of detected objects on a input image in all cases ?

For normal cases, I think we need, :-)

> If an application needs only coordinates of detected object on a video
> signal to for example, focus on it, trigger some action, or just count
> detected faces, etc. Perhaps there are more practical similar use cases.

Could you provide some practical use cases about these?

>> get image data, how does it connect the image data with FD result? and
>
> If hardware provides frame sequence numbers the FD result can be associated
> with a frame, whether it's passing through H/W interconnect or is located
> in memory.

If FD result is associated with a frame, how can user space get the frame seq
if no v4l2 buffer is involved? Without a frame sequence, it is a bit
difficult to
retrieve FD results from user space.

>> what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
>> image data?
>
> We have USERPTR and MMAP memeory buffer for streaming IO, those use
> v4l2_buffer 1). read()/write() is also used 2), mostly for compressed formats.
> Except that there are works on shared buffers.

If the input image data is from other HW(SoC bus, resizer HW, ...), is the
v4l2_buffer needed for the FD HW driver or application?

>
>>
>>> I'm sure now the Samsung devices won't fit in video output node based driver
>>> design. They read image data in different ways and also the FD result format
>>> is totally different.
>>
>> I think user space will need the FD result, so it is very important to define
>> API to describe the FD result format to user space. And the input about your
>> FD HW result format is certainly helpful to define the API.
>
> I'll post exact attributes generated by our FD detection H/W soon.

Good news, :-)

>>
>>>>
>>>>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>>>>> reasonable to use the videodev interface for passing data to the kernel
>>>>> from user space.
>>>>>
>>>>> But there might be face detection devices that accept data from other
>>>>> H/W modules, e.g. transferred through SoC internal data buses between
>>>>> image processing pipeline blocks. Thus any new interfaces need to be
>>>>> designed with such devices in mind.
>>>>>
>>>>> Also the face detection hardware block might now have an input DMA
>>>>> engine in it, the data could be fed from memory through some other
>>>>> subsystem (e.g. resize/colour converter). Then the driver for that
>>>>> subsystem would implement a video node.
>>>>
>>>> I think the direct input image or frame data to FD should be from memory
>>>> no matter the actual data is from external H/W modules or input DMA because
>>>> FD will take lot of time to detect faces in one image or frame and FD can't
>>>> have so much memory to cache several images or frames data.
>>>
>>> Sorry, I cannot provide much details at the moment, but there exists hardware
>>> that reads data from internal SoC buses and even if it uses some sort of
>>> cache memory it doesn't necessarily have to be available for the user.
>>
>> Without some hardware background, it is not easy to give a generic FD module
>> design.
>
> Yes, please give me some time so I can prepare the list of requirements.
>
>>
>>> Still the FD result is associated with an image frame for such H/W, but not
>>> necessarily with a memory buffer queued by a user application.
>>
>> For user space application, it doesn't make sense to handle FD results
>> only without image data.  Even though there are other ways of input
>> image data to FD, user space still need to know the image data, so it makes
>> sense to associate FD result with a memory buffer.
>>
>>> How long it approximately takes to process single image for OMAP4 FDIF ?
>>
>> See the link[2], and my test result is basically consistent with the data.
>
> Thanks. The processing time is rather low, looks like we could easily detect
> objects in each frame with 30..50 fps.

The detection time on image or frame data may be changed a lot, are
you sure that your FD HW can handle the data flow correctly? I understand
you FD HW has to integrate at least two buffers for coping with the issue, so it
should introduce some extra HW cost.

>>
>>>>
>>>> If you have seen this kind of FD hardware design, please let me know.
>>>>
>>>>> I'm for leaving the buffer handling details for individual drivers
>>>>> and focusing on a standard interface for applications, i.e. new
>>>>
>>>> I think leaving buffer handling details in generic FD driver or
>>>> individual drivers
>>>> doesn't matter now, since it don't have effect on interfaces between kernel
>>>> and user space.
>>>
>>> I think you misunderstood me. I wasn't talking about core/driver module split,
>>> I meant we should not be making the user interface video node centric.
>>>
>>> I think for Samsung devices I'll need a capture video node for passing
>>
>> Why is it a capture video node instead of OUTPUT v4l2 device? I think the
>
> Let's forget about this capture device, I'm just in progress of learning
> our devices internals, that's quite huge IPs..
>
>> device name should be decided from the view of face detection function:
>> FD need input image data and produce detection result.
>
> No, we should be flexible about where the data comes from to a FD subsystem
> and how it is then processed - if it is passed to some other processing block
> or it is transferred to memory with DMA and returned to the user. And we
> should use Media Controller API to route the data according to application
> requirements.
> OMAP4 FDIF is pretty simple, we need to think more about integrating FD with
> existing data pipelines.

We are discussing about it, :-)

>
>>
>>> the result to the user. So instead of associating FD result with a buffer index
>>
>> See the explanation above.
>
> No, I still insist on using frame sequence number rather than buffer index :-)

As I mentioned above, how does user space get frame sequence number
for retrieving FD results if no v4l2_buffer is involved for FD driver
and application?

>>
>>> we could try to use the frame sequence number (struct v4l2_buffer.sequence,
>>> http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).
>>
>> [1], http://marc.info/?t=132281644700005&r=1&w=2
>> [2], http://e2e.ti.com/support/embedded/linux/f/354/t/128938.aspx#462740
>
> 1) http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-qbuf.html
> 2) http://linuxtv.org/downloads/v4l-dvb-apis/func-write.html
>
>
> --
>
> Thanks,
> Sylwester
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-09 15:10               ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-09 15:10 UTC (permalink / raw)
  To: Sylwester Nawrocki
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

On Fri, Dec 9, 2011 at 7:25 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/07/2011 02:40 PM, Ming Lei wrote:
>>>> I understand the API you mentioned here should belong to kernel internal
>>>> API, correct me if it is wrong.
>>>
>>> Yes, I meant the in kernel design, i.e. generic face detection kernel module
>>> and an OMAP4 FDIF driver. It makes lots of sense to separate common code
>>> in this way, maybe even when there would be only OMAP devices using it.
>>
>> Yes, that is the motivation of the generic FD module. I think we can focus on
>> two use cases for the generic FD now:
>>
>> - one is to detect faces from user space image data
>>
>> - another one is to detect faces in image data generated from HW(SoC
>> internal bus, resize hardware)
>>
>> For OMAP4 hardware, input data is always from physically continuous
>> memory directly, so it is very easy to support the two cases. For the
>> use case 2,
>> if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
>> to pass the image buffer produced by other HW to FD hw directly.
>
> Some H/W uses direct data buses to exchange data between processing blocks,
> and there is no need for additional memory. We only need to manage the data
> links, for which MC has been designed.

For OMAP4 FD, it is not needed to include FD into MC framework since a
intermediate buffer is always required. If your HW doesn't belong to this
case, what is the output of your HW FD in the link? Also sounds FD results
may not be needed at all for use space application in the case.

>
>>
>> For other FD hardware, if it supports to detect faces in image data from
>> physically continuous memory, I think the patch is OK to support it.
>>
>> If the FD hw doesn't support to detect faces from physically continuous
>> memory, I have some questions: how does user space app to parse the
>> FD result if application can't get the input image data? If user space can
>
> Do we need the map of detected objects on a input image in all cases ?

For normal cases, I think we need, :-)

> If an application needs only coordinates of detected object on a video
> signal to for example, focus on it, trigger some action, or just count
> detected faces, etc. Perhaps there are more practical similar use cases.

Could you provide some practical use cases about these?

>> get image data, how does it connect the image data with FD result? and
>
> If hardware provides frame sequence numbers the FD result can be associated
> with a frame, whether it's passing through H/W interconnect or is located
> in memory.

If FD result is associated with a frame, how can user space get the frame seq
if no v4l2 buffer is involved? Without a frame sequence, it is a bit
difficult to
retrieve FD results from user space.

>> what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
>> image data?
>
> We have USERPTR and MMAP memeory buffer for streaming IO, those use
> v4l2_buffer 1). read()/write() is also used 2), mostly for compressed formats.
> Except that there are works on shared buffers.

If the input image data is from other HW(SoC bus, resizer HW, ...), is the
v4l2_buffer needed for the FD HW driver or application?

>
>>
>>> I'm sure now the Samsung devices won't fit in video output node based driver
>>> design. They read image data in different ways and also the FD result format
>>> is totally different.
>>
>> I think user space will need the FD result, so it is very important to define
>> API to describe the FD result format to user space. And the input about your
>> FD HW result format is certainly helpful to define the API.
>
> I'll post exact attributes generated by our FD detection H/W soon.

Good news, :-)

>>
>>>>
>>>>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>>>>> reasonable to use the videodev interface for passing data to the kernel
>>>>> from user space.
>>>>>
>>>>> But there might be face detection devices that accept data from other
>>>>> H/W modules, e.g. transferred through SoC internal data buses between
>>>>> image processing pipeline blocks. Thus any new interfaces need to be
>>>>> designed with such devices in mind.
>>>>>
>>>>> Also the face detection hardware block might now have an input DMA
>>>>> engine in it, the data could be fed from memory through some other
>>>>> subsystem (e.g. resize/colour converter). Then the driver for that
>>>>> subsystem would implement a video node.
>>>>
>>>> I think the direct input image or frame data to FD should be from memory
>>>> no matter the actual data is from external H/W modules or input DMA because
>>>> FD will take lot of time to detect faces in one image or frame and FD can't
>>>> have so much memory to cache several images or frames data.
>>>
>>> Sorry, I cannot provide much details at the moment, but there exists hardware
>>> that reads data from internal SoC buses and even if it uses some sort of
>>> cache memory it doesn't necessarily have to be available for the user.
>>
>> Without some hardware background, it is not easy to give a generic FD module
>> design.
>
> Yes, please give me some time so I can prepare the list of requirements.
>
>>
>>> Still the FD result is associated with an image frame for such H/W, but not
>>> necessarily with a memory buffer queued by a user application.
>>
>> For user space application, it doesn't make sense to handle FD results
>> only without image data.  Even though there are other ways of input
>> image data to FD, user space still need to know the image data, so it makes
>> sense to associate FD result with a memory buffer.
>>
>>> How long it approximately takes to process single image for OMAP4 FDIF ?
>>
>> See the link[2], and my test result is basically consistent with the data.
>
> Thanks. The processing time is rather low, looks like we could easily detect
> objects in each frame with 30..50 fps.

The detection time on image or frame data may be changed a lot, are
you sure that your FD HW can handle the data flow correctly? I understand
you FD HW has to integrate at least two buffers for coping with the issue, so it
should introduce some extra HW cost.

>>
>>>>
>>>> If you have seen this kind of FD hardware design, please let me know.
>>>>
>>>>> I'm for leaving the buffer handling details for individual drivers
>>>>> and focusing on a standard interface for applications, i.e. new
>>>>
>>>> I think leaving buffer handling details in generic FD driver or
>>>> individual drivers
>>>> doesn't matter now, since it don't have effect on interfaces between kernel
>>>> and user space.
>>>
>>> I think you misunderstood me. I wasn't talking about core/driver module split,
>>> I meant we should not be making the user interface video node centric.
>>>
>>> I think for Samsung devices I'll need a capture video node for passing
>>
>> Why is it a capture video node instead of OUTPUT v4l2 device? I think the
>
> Let's forget about this capture device, I'm just in progress of learning
> our devices internals, that's quite huge IPs..
>
>> device name should be decided from the view of face detection function:
>> FD need input image data and produce detection result.
>
> No, we should be flexible about where the data comes from to a FD subsystem
> and how it is then processed - if it is passed to some other processing block
> or it is transferred to memory with DMA and returned to the user. And we
> should use Media Controller API to route the data according to application
> requirements.
> OMAP4 FDIF is pretty simple, we need to think more about integrating FD with
> existing data pipelines.

We are discussing about it, :-)

>
>>
>>> the result to the user. So instead of associating FD result with a buffer index
>>
>> See the explanation above.
>
> No, I still insist on using frame sequence number rather than buffer index :-)

As I mentioned above, how does user space get frame sequence number
for retrieving FD results if no v4l2_buffer is involved for FD driver
and application?

>>
>>> we could try to use the frame sequence number (struct v4l2_buffer.sequence,
>>> http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).
>>
>> [1], http://marc.info/?t=132281644700005&r=1&w=2
>> [2], http://e2e.ti.com/support/embedded/linux/f/354/t/128938.aspx#462740
>
> 1) http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-qbuf.html
> 2) http://linuxtv.org/downloads/v4l-dvb-apis/func-write.html
>
>
> --
>
> Thanks,
> Sylwester
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

thanks,
--
Ming Lei
--
To unsubscribe from this list: send the line "unsubscribe linux-omap" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-09 15:10               ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-09 15:10 UTC (permalink / raw)
  To: linux-arm-kernel

On Fri, Dec 9, 2011 at 7:25 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
> On 12/07/2011 02:40 PM, Ming Lei wrote:
>>>> I understand the API you mentioned here should belong to kernel internal
>>>> API, correct me if it is wrong.
>>>
>>> Yes, I meant the in kernel design, i.e. generic face detection kernel module
>>> and an OMAP4 FDIF driver. It makes lots of sense to separate common code
>>> in this way, maybe even when there would be only OMAP devices using it.
>>
>> Yes, that is the motivation of the generic FD module. I think we can focus on
>> two use cases for the generic FD now:
>>
>> - one is to detect faces from user space image data
>>
>> - another one is to detect faces in image data generated from HW(SoC
>> internal bus, resize hardware)
>>
>> For OMAP4 hardware, input data is always from physically continuous
>> memory directly, so it is very easy to support the two cases. For the
>> use case 2,
>> if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
>> to pass the image buffer produced by other HW to FD hw directly.
>
> Some H/W uses direct data buses to exchange data between processing blocks,
> and there is no need for additional memory. We only need to manage the data
> links, for which MC has been designed.

For OMAP4 FD, it is not needed to include FD into MC framework since a
intermediate buffer is always required. If your HW doesn't belong to this
case, what is the output of your HW FD in the link? Also sounds FD results
may not be needed at all for use space application in the case.

>
>>
>> For other FD hardware, if it supports to detect faces in image data from
>> physically continuous memory, I think the patch is OK to support it.
>>
>> If the FD hw doesn't support to detect faces from physically continuous
>> memory, I have some questions: how does user space app to parse the
>> FD result if application can't get the input image data? If user space can
>
> Do we need the map of detected objects on a input image in all cases ?

For normal cases, I think we need, :-)

> If an application needs only coordinates of detected object on a video
> signal to for example, focus on it, trigger some action, or just count
> detected faces, etc. Perhaps there are more practical similar use cases.

Could you provide some practical use cases about these?

>> get image data, how does it connect the image data with FD result? and
>
> If hardware provides frame sequence numbers the FD result can be associated
> with a frame, whether it's passing through H/W interconnect or is located
> in memory.

If FD result is associated with a frame, how can user space get the frame seq
if no v4l2 buffer is involved? Without a frame sequence, it is a bit
difficult to
retrieve FD results from user space.

>> what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
>> image data?
>
> We have USERPTR and MMAP memeory buffer for streaming IO, those use
> v4l2_buffer 1). read()/write() is also used 2), mostly for compressed formats.
> Except that there are works on shared buffers.

If the input image data is from other HW(SoC bus, resizer HW, ...), is the
v4l2_buffer needed for the FD HW driver or application?

>
>>
>>> I'm sure now the Samsung devices won't fit in video output node based driver
>>> design. They read image data in different ways and also the FD result format
>>> is totally different.
>>
>> I think user space will need the FD result, so it is very important to define
>> API to describe the FD result format to user space. And the input about your
>> FD HW result format is certainly helpful to define the API.
>
> I'll post exact attributes generated by our FD detection H/W soon.

Good news, :-)

>>
>>>>
>>>>> AFAICS OMAP4 FDIF processes only data stored in memory, thus it seems
>>>>> reasonable to use the videodev interface for passing data to the kernel
>>>>> from user space.
>>>>>
>>>>> But there might be face detection devices that accept data from other
>>>>> H/W modules, e.g. transferred through SoC internal data buses between
>>>>> image processing pipeline blocks. Thus any new interfaces need to be
>>>>> designed with such devices in mind.
>>>>>
>>>>> Also the face detection hardware block might now have an input DMA
>>>>> engine in it, the data could be fed from memory through some other
>>>>> subsystem (e.g. resize/colour converter). Then the driver for that
>>>>> subsystem would implement a video node.
>>>>
>>>> I think the direct input image or frame data to FD should be from memory
>>>> no matter the actual data is from external H/W modules or input DMA because
>>>> FD will take lot of time to detect faces in one image or frame and FD can't
>>>> have so much memory to cache several images or frames data.
>>>
>>> Sorry, I cannot provide much details at the moment, but there exists hardware
>>> that reads data from internal SoC buses and even if it uses some sort of
>>> cache memory it doesn't necessarily have to be available for the user.
>>
>> Without some hardware background, it is not easy to give a generic FD module
>> design.
>
> Yes, please give me some time so I can prepare the list of requirements.
>
>>
>>> Still the FD result is associated with an image frame for such H/W, but not
>>> necessarily with a memory buffer queued by a user application.
>>
>> For user space application, it doesn't make sense to handle FD results
>> only without image data. ?Even though there are other ways of input
>> image data to FD, user space still need to know the image data, so it makes
>> sense to associate FD result with a memory buffer.
>>
>>> How long it approximately takes to process single image for OMAP4 FDIF ?
>>
>> See the link[2], and my test result is basically consistent with the data.
>
> Thanks. The processing time is rather low, looks like we could easily detect
> objects in each frame with 30..50 fps.

The detection time on image or frame data may be changed a lot, are
you sure that your FD HW can handle the data flow correctly? I understand
you FD HW has to integrate at least two buffers for coping with the issue, so it
should introduce some extra HW cost.

>>
>>>>
>>>> If you have seen this kind of FD hardware design, please let me know.
>>>>
>>>>> I'm for leaving the buffer handling details for individual drivers
>>>>> and focusing on a standard interface for applications, i.e. new
>>>>
>>>> I think leaving buffer handling details in generic FD driver or
>>>> individual drivers
>>>> doesn't matter now, since it don't have effect on interfaces between kernel
>>>> and user space.
>>>
>>> I think you misunderstood me. I wasn't talking about core/driver module split,
>>> I meant we should not be making the user interface video node centric.
>>>
>>> I think for Samsung devices I'll need a capture video node for passing
>>
>> Why is it a capture video node instead of OUTPUT v4l2 device? I think the
>
> Let's forget about this capture device, I'm just in progress of learning
> our devices internals, that's quite huge IPs..
>
>> device name should be decided from the view of face detection function:
>> FD need input image data and produce detection result.
>
> No, we should be flexible about where the data comes from to a FD subsystem
> and how it is then processed - if it is passed to some other processing block
> or it is transferred to memory with DMA and returned to the user. And we
> should use Media Controller API to route the data according to application
> requirements.
> OMAP4 FDIF is pretty simple, we need to think more about integrating FD with
> existing data pipelines.

We are discussing about it, :-)

>
>>
>>> the result to the user. So instead of associating FD result with a buffer index
>>
>> See the explanation above.
>
> No, I still insist on using frame sequence number rather than buffer index :-)

As I mentioned above, how does user space get frame sequence number
for retrieving FD results if no v4l2_buffer is involved for FD driver
and application?

>>
>>> we could try to use the frame sequence number (struct v4l2_buffer.sequence,
>>> http://linuxtv.org/downloads/v4l-dvb-apis/buffer.html#v4l2-buffer).
>>
>> [1], http://marc.info/?t=132281644700005&r=1&w=2
>> [2], http://e2e.ti.com/support/embedded/linux/f/354/t/128938.aspx#462740
>
> 1) http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-qbuf.html
> 2) http://linuxtv.org/downloads/v4l-dvb-apis/func-write.html
>
>
> --
>
> Thanks,
> Sylwester
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at ?http://vger.kernel.org/majordomo-info.html

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-09  4:34           ` Ming Lei
@ 2011-12-11 17:27             ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-11 17:27 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi,

On 12/09/2011 05:34 AM, Ming Lei wrote:
>>>>> + * struct v4l2_obj_detection
>>>>> + * @buf_index:       entry, index of v4l2_buffer for face detection
>>
>> I would prefer having the frame sequence number here. It will be more
>> future proof IMHO. If for instance we decide to use such an ioctl on
>> a v4l2 sub-device, without dequeuing buffers, there will be no problem
>> with that. And still in your specific use case it's not big deal to
>> look up the buffer index given it's sequence number in the application.
> 
> OK, take your suggestion to use frame index, but I still have question
> about it, see my question in another thread.

Buffer index is not enough as it's local to a DMA engine and applications,
while frame sequence is unique throughout whole pipeline.

>>>>> + * @centerx: return, position in x direction of detected object
>>>>> + * @centery: return, position in y direction of detected object
>>>>> + * @angle:   return, angle of detected object
>>>>> + *           0 deg ~ 359 deg, vertical is 0 deg, clockwise
>>>>> + * @sizex:   return, size in x direction of detected object
>>>>> + * @sizey:   return, size in y direction of detected object
>>>>> + * @confidence:      return, confidence level of detection result
>>>>> + *           0: the heighest level, 9: the lowest level
>>>>
>>>> Hmm, not a good idea to align a public interface to the capabilities
>>>> of a single hardware implementation.
>>>
>>> I think that the current omap interface is general enough, so why can't
>>> we use it as public interface?
>>
>> I meant exactly the line implying the range. What if for some hardware
>> it's 0..11 ?
> 
> We can let driver to normalize it to user which doesn't care if the range
> is 0~11 or 10~21, a uniform range should always make user happy,
> shouldn't it?

Perhaps yes, I don't have strong opinion on how much knowing an exact range
is important.

>>>> min/max confidence could be queried with
>>>> relevant controls and here we could remove the line implying range.
>>>
>>> No, the confidence is used to describe the probability about
>>> the correctness of the current detection result. Anyway, no FD can
>>> make sure that it is 100% correct.  Other HW can normalize its
>>> confidence level to 0~9 so that application can handle it easily, IMO.
>>
>> 1..100 might be better, to minimize rounding errors. Nevertheless IMO if we
>> can export an exact range supported by FD device we should do it, and let
>> upper layers do the normalization. And the bigger numbers should mean higher
>> confidence, consistently for all devices.
> 
> Looks 1..100 is better, and I will change it to 1..100.

Or do we make it 0..100 and assume 0 is impossible (insignificant) value ?

>> Do you think we could assume that the FD threshold range (FD_LHIT register
>> in case of OMAP4) is always same as the result confidence level ?
> 
> No, they are different. FD_LHIT is used to guild FD HW to detect more
> faces but more false positives __or__ less faces but less false positives.

OK. But still value range for them is 0..9, isn't it ?

> A control class is needed to be introduced for adjusting this value of FD
> HW, and I think a normalized range is better too.

We don't necessarily need to think about v4l2 application as an end user
program.

>> If so then the confidence level range could possibly be queried with the
>> detection threshold control. We could name it V4L2_CID_FD_CONFIDENCE_THRESHOLD
> 
> As I said above, there is no advantage to export the range to user, and a
> uniform range will make user happy.

We don't need to export anything, i.e. no additional effort is needed.
As you may know, each control has min/max/step value associated with it and
each driver must initialize it to sane values. Control properties can be
queried by applications
(http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-queryctrl.html).

This is the only overhead for applications, to query the controls. What
they usually normally do, in order to initialize control panels, etc.

[...]
>>>>> +struct v4l2_obj_detection {
>>
>> How about changing name of this structure to v4l2_fd_primitive or v4l2_fd_shape ?
> 
> I think v4l2_obj_detection is better because it can be reused to describe
> some other kind of object detection from video in the future.

Perhaps name "v4l2_fd_object" would fit better for this type of data.


>>>>> +     __u16           centerx;
>>>>> +     __u16           centery;
>>>>> +     __u16           angle;
>>>>> +     __u16           sizex;
>>>>> +     __u16           sizey;
>>>>
>>>> How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
>>>> After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
>>>> for size but there seems to be missing en equivalent for position, e.g.
>>>
>>> Maybe user space would like to plot a circle or ellipse over the detected
>>> objection, and I am sure that I have seen this kind of plot over detected
>>> face before.

More important is what is returned from face detector. In all cases I'm aware those
are rectangles. The application may do anything with the data

>>
>> OK, in any way I suggest to replace all __u16 with __u32, to minimize performance
>> issues and be consistent with the data type specifying pixel values elsewhere in
>> V4L.
> 
> OK, but may introduce more memory footprint for the fd result.

So maybe:

struct v4l2_point {
	__s32	x;
	__s32	y;
};

struct v4l2_size {
	__s32	w;
	__s32	h;
};

struct v4l2_fd_object {
	__u32			flags;
	__s16           	confidence;
	__s16           	angle;
	struct v4l2_size	size;
	struct v4l2_point	center;
	__u16			reserved[4];
} __packed;

For flags we might need something like:
#define V4L2_FD_OBJ_RECTANGLE  (1 << 0)
...

OR

struct v4l2_fd_object {
	union {
		struct v4l2_rect rect;
		__u32	data[4];
	}u;
} __packed;

And I'm not sure if we need parameters like confidence or angle in v4l2_fd_object
data structure. Isn't it enough to have such properties per struct
v4l2_fd_detection only ? In your driver you don't even use other struct
v4l2_obj_detection instances than 'face'. It's rather significant waste of memory.

>> It makes sense to make 'confidence' __u32 as well and add a flags attribute to
>> indicate the shape.
> 
> Sounds good.
...
>> #define V4L2_FD_FL_LEFT_EYE     (1 << 0)
>> #define V4L2_FD_FL_RIGHT_EYE    (1 << 1)
>> #define V4L2_FD_FL_MOUTH        (1 << 2)
>> #define V4L2_FD_FL_FACE         (1 << 3)
> 
> OK
> 
>> and add:
>>
>> #define V4L2_FD_FL_SMILE        (1 << 4)
>> #define V4L2_FD_FL_BLINK        (1 << 5)
> 
> Do you have any suggestion about how to describe this kind of
> detection?

We could add smile_level and blink_level to struct v4l2_fd_result.
And then keep the two above flags or not.

>>>>> +/**
>>>>> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
>>>>> + * @flag:    return, describe which objects are detected
>>>>> + * @left_eye:        return, left_eye position if detected
>>>>> + * @right_eye:       return, right_eye position if detected
>>>>> + * @mouth_eye:       return, mouth_eye position if detected
...
>>>>> + * @face:    return, face position if detected
>>>>> + */
>>>>> +struct v4l2_fd_detection {
>>
>> How about changing the name to v4l2_fd_object ?
> 
> I think the structure is used to describe one single detection result which
> may include several kind of objects detected, so sounds
> v4l2_fd_detection is better than v4l2_fd_object(s?).

"detection" doesn't really say if it is face detection process' configuration
or its result. I believe we can do better than
v4l2_fd_detection/v4l2_obj_detection pair.


>>>>> +     __u32   flag;
>>>>> +     struct v4l2_obj_detection       left_eye;
>>>>> +     struct v4l2_obj_detection       right_eye;
>>>>> +     struct v4l2_obj_detection       mouth;
>>>>> +     struct v4l2_obj_detection       face;
>>>>
>>>> I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
>>>> and then struct v4l2_fd_detection would be simply an array of
>>>> struct v4l2_obj_detection, i.e.
>>>>
>>>> struct v4l2_fd_detection {
>>>>        unsigned int count;
>>>>        struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
>>>> };
>>>>
>>>> This might be more flexible, e.g. if in the future some hardware supports
>>>> detecting wrinkles, we could easily add that by just defining a new flag:
>>>> V4L2_FD_HAS_WRINKLES, etc.
>>>
>>> This is a bit flexible, but not explicit enough for describing
>>> interface, how about reserving these as below for future usage?
>>>
>>>       struct v4l2_fd_detection {
>>>               __u32   flag;
>>>               Struct v4l2_obj_detection       left_eye;
>>>               Struct v4l2_obj_detection       right_eye;
>>>               Struct v4l2_obj_detection       mouth;
>>>               Struct v4l2_obj_detection       face;
>>>               Struct v4l2_obj_detection       reserved[4];
>>>       };
>>
>> OK, and how about this:
>>
>>        struct v4l2_fd_object {
>>                struct v4l2_fd_shape    left_eye;
>>                struct v4l2_fd_shape    right_eye;
>>                struct v4l2_fd_shape    mouth;
>>                struct v4l2_fd_shape    face;
>>                __u32                   reserved[33];
> 
> Why is struct 'v4l2_fd_shape    reserved[4]' removed?

We can still add struct v4l2_fd_shape in place of the reserved[]
array in future if needed. But something else than v4l2_fd_shape may be
needed, so plain u32 array looked more reasonable to me.

>>                __u32                   flags;
>>        } __packed;
> 
> Why is '__packed' needed? It will introduce performance loss if we have
> not good reason to do it.

IMHO we can use attribute packed to make sure the structure layout is same
on all architectures. And design the data structure so performance loss
is minimal, i.e. no weird padding is necessary.

The face detector output structure might then be:

struct v4l2_fd_detection {	
	struct v4l2_fd_object	left_eye;
	struct v4l2_fd_object	right_eye;
	struct v4l2_fd_object	mouth;
	struct v4l2_fd_object	face;
	__s32           	confidence;
	__s32           	angle;
	__u32			flags;
	__u32			smile_level;
	__u32			blink_level;
	__u32                   reserved[37];
} __packed;


But I can't stop myself from thinking it's better to use the v4l2 event API
for passing FD output data between drivers and applications. You basically
reimplemented it in patches 6/7, 7/7. The "only" issue is that event type
specific payload size in struct v4l2_event is only 64 bytes and maximum count
of detected faces per frame isn't constant API wide and hardware dependant...


-- 

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-11 17:27             ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-11 17:27 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On 12/09/2011 05:34 AM, Ming Lei wrote:
>>>>> + * struct v4l2_obj_detection
>>>>> + * @buf_index:       entry, index of v4l2_buffer for face detection
>>
>> I would prefer having the frame sequence number here. It will be more
>> future proof IMHO. If for instance we decide to use such an ioctl on
>> a v4l2 sub-device, without dequeuing buffers, there will be no problem
>> with that. And still in your specific use case it's not big deal to
>> look up the buffer index given it's sequence number in the application.
> 
> OK, take your suggestion to use frame index, but I still have question
> about it, see my question in another thread.

Buffer index is not enough as it's local to a DMA engine and applications,
while frame sequence is unique throughout whole pipeline.

>>>>> + * @centerx: return, position in x direction of detected object
>>>>> + * @centery: return, position in y direction of detected object
>>>>> + * @angle:   return, angle of detected object
>>>>> + *           0 deg ~ 359 deg, vertical is 0 deg, clockwise
>>>>> + * @sizex:   return, size in x direction of detected object
>>>>> + * @sizey:   return, size in y direction of detected object
>>>>> + * @confidence:      return, confidence level of detection result
>>>>> + *           0: the heighest level, 9: the lowest level
>>>>
>>>> Hmm, not a good idea to align a public interface to the capabilities
>>>> of a single hardware implementation.
>>>
>>> I think that the current omap interface is general enough, so why can't
>>> we use it as public interface?
>>
>> I meant exactly the line implying the range. What if for some hardware
>> it's 0..11 ?
> 
> We can let driver to normalize it to user which doesn't care if the range
> is 0~11 or 10~21, a uniform range should always make user happy,
> shouldn't it?

Perhaps yes, I don't have strong opinion on how much knowing an exact range
is important.

>>>> min/max confidence could be queried with
>>>> relevant controls and here we could remove the line implying range.
>>>
>>> No, the confidence is used to describe the probability about
>>> the correctness of the current detection result. Anyway, no FD can
>>> make sure that it is 100% correct.  Other HW can normalize its
>>> confidence level to 0~9 so that application can handle it easily, IMO.
>>
>> 1..100 might be better, to minimize rounding errors. Nevertheless IMO if we
>> can export an exact range supported by FD device we should do it, and let
>> upper layers do the normalization. And the bigger numbers should mean higher
>> confidence, consistently for all devices.
> 
> Looks 1..100 is better, and I will change it to 1..100.

Or do we make it 0..100 and assume 0 is impossible (insignificant) value ?

>> Do you think we could assume that the FD threshold range (FD_LHIT register
>> in case of OMAP4) is always same as the result confidence level ?
> 
> No, they are different. FD_LHIT is used to guild FD HW to detect more
> faces but more false positives __or__ less faces but less false positives.

OK. But still value range for them is 0..9, isn't it ?

> A control class is needed to be introduced for adjusting this value of FD
> HW, and I think a normalized range is better too.

We don't necessarily need to think about v4l2 application as an end user
program.

>> If so then the confidence level range could possibly be queried with the
>> detection threshold control. We could name it V4L2_CID_FD_CONFIDENCE_THRESHOLD
> 
> As I said above, there is no advantage to export the range to user, and a
> uniform range will make user happy.

We don't need to export anything, i.e. no additional effort is needed.
As you may know, each control has min/max/step value associated with it and
each driver must initialize it to sane values. Control properties can be
queried by applications
(http://linuxtv.org/downloads/v4l-dvb-apis/vidioc-queryctrl.html).

This is the only overhead for applications, to query the controls. What
they usually normally do, in order to initialize control panels, etc.

[...]
>>>>> +struct v4l2_obj_detection {
>>
>> How about changing name of this structure to v4l2_fd_primitive or v4l2_fd_shape ?
> 
> I think v4l2_obj_detection is better because it can be reused to describe
> some other kind of object detection from video in the future.

Perhaps name "v4l2_fd_object" would fit better for this type of data.


>>>>> +     __u16           centerx;
>>>>> +     __u16           centery;
>>>>> +     __u16           angle;
>>>>> +     __u16           sizex;
>>>>> +     __u16           sizey;
>>>>
>>>> How about using struct v4l2_rect in place of centerx/centery, sizex/sizey ?
>>>> After all it describes a rectangle. We could also use struct v4l2_frmsize_discrete
>>>> for size but there seems to be missing en equivalent for position, e.g.
>>>
>>> Maybe user space would like to plot a circle or ellipse over the detected
>>> objection, and I am sure that I have seen this kind of plot over detected
>>> face before.

More important is what is returned from face detector. In all cases I'm aware those
are rectangles. The application may do anything with the data

>>
>> OK, in any way I suggest to replace all __u16 with __u32, to minimize performance
>> issues and be consistent with the data type specifying pixel values elsewhere in
>> V4L.
> 
> OK, but may introduce more memory footprint for the fd result.

So maybe:

struct v4l2_point {
	__s32	x;
	__s32	y;
};

struct v4l2_size {
	__s32	w;
	__s32	h;
};

struct v4l2_fd_object {
	__u32			flags;
	__s16           	confidence;
	__s16           	angle;
	struct v4l2_size	size;
	struct v4l2_point	center;
	__u16			reserved[4];
} __packed;

For flags we might need something like:
#define V4L2_FD_OBJ_RECTANGLE  (1 << 0)
...

OR

struct v4l2_fd_object {
	union {
		struct v4l2_rect rect;
		__u32	data[4];
	}u;
} __packed;

And I'm not sure if we need parameters like confidence or angle in v4l2_fd_object
data structure. Isn't it enough to have such properties per struct
v4l2_fd_detection only ? In your driver you don't even use other struct
v4l2_obj_detection instances than 'face'. It's rather significant waste of memory.

>> It makes sense to make 'confidence' __u32 as well and add a flags attribute to
>> indicate the shape.
> 
> Sounds good.
...
>> #define V4L2_FD_FL_LEFT_EYE     (1 << 0)
>> #define V4L2_FD_FL_RIGHT_EYE    (1 << 1)
>> #define V4L2_FD_FL_MOUTH        (1 << 2)
>> #define V4L2_FD_FL_FACE         (1 << 3)
> 
> OK
> 
>> and add:
>>
>> #define V4L2_FD_FL_SMILE        (1 << 4)
>> #define V4L2_FD_FL_BLINK        (1 << 5)
> 
> Do you have any suggestion about how to describe this kind of
> detection?

We could add smile_level and blink_level to struct v4l2_fd_result.
And then keep the two above flags or not.

>>>>> +/**
>>>>> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
>>>>> + * @flag:    return, describe which objects are detected
>>>>> + * @left_eye:        return, left_eye position if detected
>>>>> + * @right_eye:       return, right_eye position if detected
>>>>> + * @mouth_eye:       return, mouth_eye position if detected
...
>>>>> + * @face:    return, face position if detected
>>>>> + */
>>>>> +struct v4l2_fd_detection {
>>
>> How about changing the name to v4l2_fd_object ?
> 
> I think the structure is used to describe one single detection result which
> may include several kind of objects detected, so sounds
> v4l2_fd_detection is better than v4l2_fd_object(s?).

"detection" doesn't really say if it is face detection process' configuration
or its result. I believe we can do better than
v4l2_fd_detection/v4l2_obj_detection pair.


>>>>> +     __u32   flag;
>>>>> +     struct v4l2_obj_detection       left_eye;
>>>>> +     struct v4l2_obj_detection       right_eye;
>>>>> +     struct v4l2_obj_detection       mouth;
>>>>> +     struct v4l2_obj_detection       face;
>>>>
>>>> I would do this differently, i.e. put "flag" inside struct v4l2_obj_detection
>>>> and then struct v4l2_fd_detection would be simply an array of
>>>> struct v4l2_obj_detection, i.e.
>>>>
>>>> struct v4l2_fd_detection {
>>>>        unsigned int count;
>>>>        struct v4l2_obj_detection [V4L2_MAX_FD_OBJECT_NUM];
>>>> };
>>>>
>>>> This might be more flexible, e.g. if in the future some hardware supports
>>>> detecting wrinkles, we could easily add that by just defining a new flag:
>>>> V4L2_FD_HAS_WRINKLES, etc.
>>>
>>> This is a bit flexible, but not explicit enough for describing
>>> interface, how about reserving these as below for future usage?
>>>
>>>       struct v4l2_fd_detection {
>>>               __u32   flag;
>>>               Struct v4l2_obj_detection       left_eye;
>>>               Struct v4l2_obj_detection       right_eye;
>>>               Struct v4l2_obj_detection       mouth;
>>>               Struct v4l2_obj_detection       face;
>>>               Struct v4l2_obj_detection       reserved[4];
>>>       };
>>
>> OK, and how about this:
>>
>>        struct v4l2_fd_object {
>>                struct v4l2_fd_shape    left_eye;
>>                struct v4l2_fd_shape    right_eye;
>>                struct v4l2_fd_shape    mouth;
>>                struct v4l2_fd_shape    face;
>>                __u32                   reserved[33];
> 
> Why is struct 'v4l2_fd_shape    reserved[4]' removed?

We can still add struct v4l2_fd_shape in place of the reserved[]
array in future if needed. But something else than v4l2_fd_shape may be
needed, so plain u32 array looked more reasonable to me.

>>                __u32                   flags;
>>        } __packed;
> 
> Why is '__packed' needed? It will introduce performance loss if we have
> not good reason to do it.

IMHO we can use attribute packed to make sure the structure layout is same
on all architectures. And design the data structure so performance loss
is minimal, i.e. no weird padding is necessary.

The face detector output structure might then be:

struct v4l2_fd_detection {	
	struct v4l2_fd_object	left_eye;
	struct v4l2_fd_object	right_eye;
	struct v4l2_fd_object	mouth;
	struct v4l2_fd_object	face;
	__s32           	confidence;
	__s32           	angle;
	__u32			flags;
	__u32			smile_level;
	__u32			blink_level;
	__u32                   reserved[37];
} __packed;


But I can't stop myself from thinking it's better to use the v4l2 event API
for passing FD output data between drivers and applications. You basically
reimplemented it in patches 6/7, 7/7. The "only" issue is that event type
specific payload size in struct v4l2_event is only 64 bytes and maximum count
of detected faces per frame isn't constant API wide and hardware dependant...


-- 

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-09 15:10               ` Ming Lei
  (?)
@ 2011-12-11 17:43                 ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-11 17:43 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

On 12/09/2011 04:10 PM, Ming Lei wrote:
> On Fri, Dec 9, 2011 at 7:25 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
>> On 12/07/2011 02:40 PM, Ming Lei wrote:
>>> Yes, that is the motivation of the generic FD module. I think we can focus on
>>> two use cases for the generic FD now:
>>>
>>> - one is to detect faces from user space image data
>>>
>>> - another one is to detect faces in image data generated from HW(SoC
>>> internal bus, resize hardware)
>>>
>>> For OMAP4 hardware, input data is always from physically continuous
>>> memory directly, so it is very easy to support the two cases. For the
>>> use case 2,
>>> if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
>>> to pass the image buffer produced by other HW to FD hw directly.
>>
>> Some H/W uses direct data buses to exchange data between processing blocks,
>> and there is no need for additional memory. We only need to manage the data
>> links, for which MC has been designed.
> 
> For OMAP4 FD, it is not needed to include FD into MC framework since a
> intermediate buffer is always required. If your HW doesn't belong to this
> case, what is the output of your HW FD in the link? Also sounds FD results
> may not be needed at all for use space application in the case.

The result data is similar to OMAP4 one, plus a few other attributes.
User buffers may be filled by other than FD device driver.

>>> For other FD hardware, if it supports to detect faces in image data from
>>> physically continuous memory, I think the patch is OK to support it.
>>>
>>> If the FD hw doesn't support to detect faces from physically continuous
>>> memory, I have some questions: how does user space app to parse the
>>> FD result if application can't get the input image data? If user space can
>>
>> Do we need the map of detected objects on a input image in all cases ?
> 
> For normal cases, I think we need, :-)
> 
>> If an application needs only coordinates of detected object on a video
>> signal to for example, focus on it, trigger some action, or just count
>> detected faces, etc. Perhaps there are more practical similar use cases.
> 
> Could you provide some practical use cases about these?

As above, and any device with a camera that controls something and makes
decision according to presence of human face in his view.

>>> get image data, how does it connect the image data with FD result? and
>>
>> If hardware provides frame sequence numbers the FD result can be associated
>> with a frame, whether it's passing through H/W interconnect or is located
>> in memory.
> 
> If FD result is associated with a frame, how can user space get the frame seq
> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
> difficult to retrieve FD results from user space.

If you pass image data in memory buffers from user space, yes, it could be
impossible. If there is no buffers you don't need to associate FD result
with particular image data. There will be just ascending frame identifiers
in reported fd result data...

>>> what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
>>> image data?
>>
>> We have USERPTR and MMAP memeory buffer for streaming IO, those use
>> v4l2_buffer 1). read()/write() is also used 2), mostly for compressed formats.
>> Except that there are works on shared buffers.
> 
> If the input image data is from other HW(SoC bus, resizer HW, ...), is the
> v4l2_buffer needed for the FD HW driver or application?

Not really, still v4l2_buffer may be used by other (sub)driver within same video
processing pipeline.

>>>> How long it approximately takes to process single image for OMAP4 FDIF ?
>>>
>>> See the link[2], and my test result is basically consistent with the data.
>>
>> Thanks. The processing time is rather low, looks like we could easily detect
>> objects in each frame with 30..50 fps.
> 
> The detection time on image or frame data may be changed a lot, are
> you sure that your FD HW can handle the data flow correctly? I understand
> you FD HW has to integrate at least two buffers for coping with the issue, so it
> should introduce some extra HW cost.
> 

I'm not absolutely sure, untill I write the driver and test it myself.. ;)

[...]
>> No, I still insist on using frame sequence number rather than buffer index :-)
> 
> As I mentioned above, how does user space get frame sequence number
> for retrieving FD results if no v4l2_buffer is involved for FD driver
> and application?

It will be included in the FD result... or in a dedicated v4l2 event data structure.
More important, at the end of the day, we'll be getting buffers with image data
at some stage of a video pipeline, which would contain same frame identifier
(I think we can ignore v4l2_buffer.field for FD purpose).

-- 

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-11 17:43                 ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-11 17:43 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-kernel, linux-arm-kernel, linux-media

On 12/09/2011 04:10 PM, Ming Lei wrote:
> On Fri, Dec 9, 2011 at 7:25 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
>> On 12/07/2011 02:40 PM, Ming Lei wrote:
>>> Yes, that is the motivation of the generic FD module. I think we can focus on
>>> two use cases for the generic FD now:
>>>
>>> - one is to detect faces from user space image data
>>>
>>> - another one is to detect faces in image data generated from HW(SoC
>>> internal bus, resize hardware)
>>>
>>> For OMAP4 hardware, input data is always from physically continuous
>>> memory directly, so it is very easy to support the two cases. For the
>>> use case 2,
>>> if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
>>> to pass the image buffer produced by other HW to FD hw directly.
>>
>> Some H/W uses direct data buses to exchange data between processing blocks,
>> and there is no need for additional memory. We only need to manage the data
>> links, for which MC has been designed.
> 
> For OMAP4 FD, it is not needed to include FD into MC framework since a
> intermediate buffer is always required. If your HW doesn't belong to this
> case, what is the output of your HW FD in the link? Also sounds FD results
> may not be needed at all for use space application in the case.

The result data is similar to OMAP4 one, plus a few other attributes.
User buffers may be filled by other than FD device driver.

>>> For other FD hardware, if it supports to detect faces in image data from
>>> physically continuous memory, I think the patch is OK to support it.
>>>
>>> If the FD hw doesn't support to detect faces from physically continuous
>>> memory, I have some questions: how does user space app to parse the
>>> FD result if application can't get the input image data? If user space can
>>
>> Do we need the map of detected objects on a input image in all cases ?
> 
> For normal cases, I think we need, :-)
> 
>> If an application needs only coordinates of detected object on a video
>> signal to for example, focus on it, trigger some action, or just count
>> detected faces, etc. Perhaps there are more practical similar use cases.
> 
> Could you provide some practical use cases about these?

As above, and any device with a camera that controls something and makes
decision according to presence of human face in his view.

>>> get image data, how does it connect the image data with FD result? and
>>
>> If hardware provides frame sequence numbers the FD result can be associated
>> with a frame, whether it's passing through H/W interconnect or is located
>> in memory.
> 
> If FD result is associated with a frame, how can user space get the frame seq
> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
> difficult to retrieve FD results from user space.

If you pass image data in memory buffers from user space, yes, it could be
impossible. If there is no buffers you don't need to associate FD result
with particular image data. There will be just ascending frame identifiers
in reported fd result data...

>>> what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
>>> image data?
>>
>> We have USERPTR and MMAP memeory buffer for streaming IO, those use
>> v4l2_buffer 1). read()/write() is also used 2), mostly for compressed formats.
>> Except that there are works on shared buffers.
> 
> If the input image data is from other HW(SoC bus, resizer HW, ...), is the
> v4l2_buffer needed for the FD HW driver or application?

Not really, still v4l2_buffer may be used by other (sub)driver within same video
processing pipeline.

>>>> How long it approximately takes to process single image for OMAP4 FDIF ?
>>>
>>> See the link[2], and my test result is basically consistent with the data.
>>
>> Thanks. The processing time is rather low, looks like we could easily detect
>> objects in each frame with 30..50 fps.
> 
> The detection time on image or frame data may be changed a lot, are
> you sure that your FD HW can handle the data flow correctly? I understand
> you FD HW has to integrate at least two buffers for coping with the issue, so it
> should introduce some extra HW cost.
> 

I'm not absolutely sure, untill I write the driver and test it myself.. ;)

[...]
>> No, I still insist on using frame sequence number rather than buffer index :-)
> 
> As I mentioned above, how does user space get frame sequence number
> for retrieving FD results if no v4l2_buffer is involved for FD driver
> and application?

It will be included in the FD result... or in a dedicated v4l2 event data structure.
More important, at the end of the day, we'll be getting buffers with image data
at some stage of a video pipeline, which would contain same frame identifier
(I think we can ignore v4l2_buffer.field for FD purpose).

-- 

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-11 17:43                 ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-11 17:43 UTC (permalink / raw)
  To: linux-arm-kernel

On 12/09/2011 04:10 PM, Ming Lei wrote:
> On Fri, Dec 9, 2011 at 7:25 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
>> On 12/07/2011 02:40 PM, Ming Lei wrote:
>>> Yes, that is the motivation of the generic FD module. I think we can focus on
>>> two use cases for the generic FD now:
>>>
>>> - one is to detect faces from user space image data
>>>
>>> - another one is to detect faces in image data generated from HW(SoC
>>> internal bus, resize hardware)
>>>
>>> For OMAP4 hardware, input data is always from physically continuous
>>> memory directly, so it is very easy to support the two cases. For the
>>> use case 2,
>>> if buffer copy is to be avoided, we can use the coming shared dma-buf[1]
>>> to pass the image buffer produced by other HW to FD hw directly.
>>
>> Some H/W uses direct data buses to exchange data between processing blocks,
>> and there is no need for additional memory. We only need to manage the data
>> links, for which MC has been designed.
> 
> For OMAP4 FD, it is not needed to include FD into MC framework since a
> intermediate buffer is always required. If your HW doesn't belong to this
> case, what is the output of your HW FD in the link? Also sounds FD results
> may not be needed at all for use space application in the case.

The result data is similar to OMAP4 one, plus a few other attributes.
User buffers may be filled by other than FD device driver.

>>> For other FD hardware, if it supports to detect faces in image data from
>>> physically continuous memory, I think the patch is OK to support it.
>>>
>>> If the FD hw doesn't support to detect faces from physically continuous
>>> memory, I have some questions: how does user space app to parse the
>>> FD result if application can't get the input image data? If user space can
>>
>> Do we need the map of detected objects on a input image in all cases ?
> 
> For normal cases, I think we need, :-)
> 
>> If an application needs only coordinates of detected object on a video
>> signal to for example, focus on it, trigger some action, or just count
>> detected faces, etc. Perhaps there are more practical similar use cases.
> 
> Could you provide some practical use cases about these?

As above, and any device with a camera that controls something and makes
decision according to presence of human face in his view.

>>> get image data, how does it connect the image data with FD result? and
>>
>> If hardware provides frame sequence numbers the FD result can be associated
>> with a frame, whether it's passing through H/W interconnect or is located
>> in memory.
> 
> If FD result is associated with a frame, how can user space get the frame seq
> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
> difficult to retrieve FD results from user space.

If you pass image data in memory buffers from user space, yes, it could be
impossible. If there is no buffers you don't need to associate FD result
with particular image data. There will be just ascending frame identifiers
in reported fd result data...

>>> what standard v4l2 ways(v4l2_buffer?) can the app use to describe the
>>> image data?
>>
>> We have USERPTR and MMAP memeory buffer for streaming IO, those use
>> v4l2_buffer 1). read()/write() is also used 2), mostly for compressed formats.
>> Except that there are works on shared buffers.
> 
> If the input image data is from other HW(SoC bus, resizer HW, ...), is the
> v4l2_buffer needed for the FD HW driver or application?

Not really, still v4l2_buffer may be used by other (sub)driver within same video
processing pipeline.

>>>> How long it approximately takes to process single image for OMAP4 FDIF ?
>>>
>>> See the link[2], and my test result is basically consistent with the data.
>>
>> Thanks. The processing time is rather low, looks like we could easily detect
>> objects in each frame with 30..50 fps.
> 
> The detection time on image or frame data may be changed a lot, are
> you sure that your FD HW can handle the data flow correctly? I understand
> you FD HW has to integrate at least two buffers for coping with the issue, so it
> should introduce some extra HW cost.
> 

I'm not absolutely sure, untill I write the driver and test it myself.. ;)

[...]
>> No, I still insist on using frame sequence number rather than buffer index :-)
> 
> As I mentioned above, how does user space get frame sequence number
> for retrieving FD results if no v4l2_buffer is involved for FD driver
> and application?

It will be included in the FD result... or in a dedicated v4l2 event data structure.
More important, at the end of the day, we'll be getting buffers with image data
at some stage of a video pipeline, which would contain same frame identifier
(I think we can ignore v4l2_buffer.field for FD purpose).

-- 

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-02 15:02   ` Ming Lei
@ 2011-12-11 18:38     ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-11 18:38 UTC (permalink / raw)
  To: Ming Lei; +Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi Ming,

On 12/02/2011 04:02 PM, Ming Lei wrote:
> This patch introduces one driver for face detection purpose.
> 
> The driver is responsible for all v4l2 stuff, buffer management
> and other general things, and doesn't touch face detection hardware
> directly. Several interfaces are exported to low level drivers
> (such as the coming omap4 FD driver)which will communicate with
> face detection hw module.
> 
> So the driver will make driving face detection hw modules more
> easy.
> 
> TODO:
> 	- implement FD setting interfaces with v4l2 controls or
> 	ext controls
> 
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
...
> +static int vidioc_g_fd_result(struct file *file, void *priv,
> +					struct v4l2_fd_result *f)
> +{
> +	struct fdif_dev *dev = video_drvdata(file);
> +	unsigned long flags;
> +	struct v4l2_fdif_result *tmp;
> +	struct v4l2_fdif_result *fr = NULL;
> +	unsigned int cnt = 0;
> +	int ret = -EINVAL;
> +
> +	spin_lock_irqsave(&dev->lock, flags);
> +	list_for_each_entry(tmp, &dev->fdif_dq.complete, list)
> +		if (tmp->index == f->buf_index) {
> +			fr = tmp;
> +			list_del(&tmp->list);
> +			break;
> +		}
> +	spin_unlock_irqrestore(&dev->lock, flags);
> +
> +	if (fr) {
> +		ret = 0;
> +		cnt = min(f->face_cnt, fr->face_cnt);
> +		if (cnt)
> +			memcpy(f->fd, fr->faces,
> +				sizeof(struct v4l2_fd_detection) * cnt);
> +		f->face_cnt = cnt;
> +		kfree(fr->faces);
> +		kfree(fr);

struct v4l2_fdif_result is allocated in HW driver at the time when FD result
is read out and it is freed in generic module here. Not sure if it is a good
practice to split memory management like this. Also IMHO it would be much
better to pre-allocate memory for FD output data, according to maximum number
of detected faces per frame. It could be more reliable than allocating memory
in interrupt context per each frame.


> +	}
> +	return ret;
> +}

--
Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-11 18:38     ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-11 18:38 UTC (permalink / raw)
  To: linux-arm-kernel

Hi Ming,

On 12/02/2011 04:02 PM, Ming Lei wrote:
> This patch introduces one driver for face detection purpose.
> 
> The driver is responsible for all v4l2 stuff, buffer management
> and other general things, and doesn't touch face detection hardware
> directly. Several interfaces are exported to low level drivers
> (such as the coming omap4 FD driver)which will communicate with
> face detection hw module.
> 
> So the driver will make driving face detection hw modules more
> easy.
> 
> TODO:
> 	- implement FD setting interfaces with v4l2 controls or
> 	ext controls
> 
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
...
> +static int vidioc_g_fd_result(struct file *file, void *priv,
> +					struct v4l2_fd_result *f)
> +{
> +	struct fdif_dev *dev = video_drvdata(file);
> +	unsigned long flags;
> +	struct v4l2_fdif_result *tmp;
> +	struct v4l2_fdif_result *fr = NULL;
> +	unsigned int cnt = 0;
> +	int ret = -EINVAL;
> +
> +	spin_lock_irqsave(&dev->lock, flags);
> +	list_for_each_entry(tmp, &dev->fdif_dq.complete, list)
> +		if (tmp->index == f->buf_index) {
> +			fr = tmp;
> +			list_del(&tmp->list);
> +			break;
> +		}
> +	spin_unlock_irqrestore(&dev->lock, flags);
> +
> +	if (fr) {
> +		ret = 0;
> +		cnt = min(f->face_cnt, fr->face_cnt);
> +		if (cnt)
> +			memcpy(f->fd, fr->faces,
> +				sizeof(struct v4l2_fd_detection) * cnt);
> +		f->face_cnt = cnt;
> +		kfree(fr->faces);
> +		kfree(fr);

struct v4l2_fdif_result is allocated in HW driver at the time when FD result
is read out and it is freed in generic module here. Not sure if it is a good
practice to split memory management like this. Also IMHO it would be much
better to pre-allocate memory for FD output data, according to maximum number
of detected faces per frame. It could be more reliable than allocating memory
in interrupt context per each frame.


> +	}
> +	return ret;
> +}

--
Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-11 17:43                 ` Sylwester Nawrocki
@ 2011-12-12  9:49                   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-12  9:49 UTC (permalink / raw)
  To: Sylwester Nawrocki
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi,

On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>

>> For OMAP4 FD, it is not needed to include FD into MC framework since a
>> intermediate buffer is always required. If your HW doesn't belong to this
>> case, what is the output of your HW FD in the link? Also sounds FD results
>> may not be needed at all for use space application in the case.
>
> The result data is similar to OMAP4 one, plus a few other attributes.
> User buffers may be filled by other than FD device driver.

OK.

>> Could you provide some practical use cases about these?
>
> As above, and any device with a camera that controls something and makes
> decision according to presence of human face in his view.

Sounds a reasonable case, :-)

>> If FD result is associated with a frame, how can user space get the frame seq
>> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
>> difficult to retrieve FD results from user space.
>
> If you pass image data in memory buffers from user space, yes, it could be
> impossible.

It is easy to get the frame sequence from v4l2_buffer for the case too, :-)

>
> Not really, still v4l2_buffer may be used by other (sub)driver within same video
> processing pipeline.

OK.

A related question: how can we make one application to support the two kinds of
devices(input from user space data as OMAP4, input from SoC bus as Samsung)
at the same time? Maybe some capability info is to be exported to user space?
or other suggestions?

And will your Samsung FD HW support to detect faces from memory? or just only
detect from SoC bus?

> It will be included in the FD result... or in a dedicated v4l2 event data structure.
> More important, at the end of the day, we'll be getting buffers with image data
> at some stage of a video pipeline, which would contain same frame identifier
> (I think we can ignore v4l2_buffer.field for FD purpose).

OK, I will associate FD result with frame identifier, and not invent a
dedicated v4l2 event for query frame seq now until a specific requirement
for it is proposed.

I will convert/integrate recent discussions into patches of v2 for further
review, and sub device support will be provided. But before starting to do it,
I am still not clear how to integrate FD into MC framework. I understand FD
sub device is only a media entity, so how can FD sub device find the media
device(struct media_device)?  or just needn't to care about it now?

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-12  9:49                   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-12  9:49 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>

>> For OMAP4 FD, it is not needed to include FD into MC framework since a
>> intermediate buffer is always required. If your HW doesn't belong to this
>> case, what is the output of your HW FD in the link? Also sounds FD results
>> may not be needed at all for use space application in the case.
>
> The result data is similar to OMAP4 one, plus a few other attributes.
> User buffers may be filled by other than FD device driver.

OK.

>> Could you provide some practical use cases about these?
>
> As above, and any device with a camera that controls something and makes
> decision according to presence of human face in his view.

Sounds a reasonable case, :-)

>> If FD result is associated with a frame, how can user space get the frame seq
>> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
>> difficult to retrieve FD results from user space.
>
> If you pass image data in memory buffers from user space, yes, it could be
> impossible.

It is easy to get the frame sequence from v4l2_buffer for the case too, :-)

>
> Not really, still v4l2_buffer may be used by other (sub)driver within same video
> processing pipeline.

OK.

A related question: how can we make one application to support the two kinds of
devices(input from user space data as OMAP4, input from SoC bus as Samsung)
at the same time? Maybe some capability info is to be exported to user space?
or other suggestions?

And will your Samsung FD HW support to detect faces from memory? or just only
detect from SoC bus?

> It will be included in the FD result... or in a dedicated v4l2 event data structure.
> More important, at the end of the day, we'll be getting buffers with image data
> at some stage of a video pipeline, which would contain same frame identifier
> (I think we can ignore v4l2_buffer.field for FD purpose).

OK, I will associate FD result with frame identifier, and not invent a
dedicated v4l2 event for query frame seq now until a specific requirement
for it is proposed.

I will convert/integrate recent discussions into patches of v2 for further
review, and sub device support will be provided. But before starting to do it,
I am still not clear how to integrate FD into MC framework. I understand FD
sub device is only a media entity, so how can FD sub device find the media
device(struct media_device)?  or just needn't to care about it now?

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* RE: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-12  9:49                   ` Ming Lei
@ 2011-12-12 12:08                     ` HeungJun, Kim
  -1 siblings, 0 replies; 91+ messages in thread
From: HeungJun, Kim @ 2011-12-12 12:08 UTC (permalink / raw)
  To: 'Ming Lei', 'Sylwester Nawrocki'
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi Ming,

It's maybe late, but I want to suggest one thing about FD API.

This OMAP FD block looks detection ability of just face.
But, It's possible to occur another device which can detect
specific "object" or "patterns". Moreover, this API can expand
"object recognition" area. So, I think it's good to change the API name
like "v4l2_recog".

Actually, I'm preparing similar control class for mainline with m5mols
camera sensor driver. The m5mols camera sensor has the function about
"face detection". But, I has experienced about Robot Recognition, and I
remember the image processing chip which can detect spefic "pattern".
So, I hesitated naming the API(control or ioctl whatever) with "face".
It can be possible to provide just "object" or "pattern", not face.
Even user library on windows, there is famous "OpenCV". And this is also
support not only "face", but also "object".

The function of OMAP FDIF looks like m5mols ISP's one.
please understand I don't have experience about OMAP AP. But, I can tell
you it's better to use the name "object recognition", not the "face detection",
for any other device or driver.

In a few days, I'll share the CIDs I have thought for m5mols driver.
And, I hope to discuss about this with OMAP FDIF.

Thank you.

Regards,
Heungjun Kim


> -----Original Message-----
> From: linux-media-owner@vger.kernel.org [mailto:linux-media-
> owner@vger.kernel.org] On Behalf Of Ming Lei
> Sent: Monday, December 12, 2011 6:50 PM
> To: Sylwester Nawrocki
> Cc: linux-omap@vger.kernel.org; linux-arm-kernel@lists.infradead.org; linux-
> kernel@vger.kernel.org; linux-media@vger.kernel.org
> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver
> module
> 
> Hi,
> 
> On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>
> 
> >> For OMAP4 FD, it is not needed to include FD into MC framework since a
> >> intermediate buffer is always required. If your HW doesn't belong to this
> >> case, what is the output of your HW FD in the link? Also sounds FD results
> >> may not be needed at all for use space application in the case.
> >
> > The result data is similar to OMAP4 one, plus a few other attributes.
> > User buffers may be filled by other than FD device driver.
> 
> OK.
> 
> 
> >> Could you provide some practical use cases about these?
> >
> > As above, and any device with a camera that controls something and makes
> > decision according to presence of human face in his view.
> 
> Sounds a reasonable case, :-)
> 
> 
> >> If FD result is associated with a frame, how can user space get the frame
> seq
> >> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
> >> difficult to retrieve FD results from user space.
> >
> > If you pass image data in memory buffers from user space, yes, it could be
> > impossible.
> 
> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)
> 
> >
> > Not really, still v4l2_buffer may be used by other (sub)driver within same
> video
> > processing pipeline.
> 
> OK.
> 
> A related question: how can we make one application to support the two kinds
of
> devices(input from user space data as OMAP4, input from SoC bus as Samsung)
> at the same time? Maybe some capability info is to be exported to user space?
> or other suggestions?
> 
> And will your Samsung FD HW support to detect faces from memory? or just only
> detect from SoC bus?
> 
> 
> > It will be included in the FD result... or in a dedicated v4l2 event data
> structure.
> > More important, at the end of the day, we'll be getting buffers with image
> data
> > at some stage of a video pipeline, which would contain same frame identifier
> > (I think we can ignore v4l2_buffer.field for FD purpose).
> 
> OK, I will associate FD result with frame identifier, and not invent a
> dedicated v4l2 event for query frame seq now until a specific requirement
> for it is proposed.
> 
> I will convert/integrate recent discussions into patches of v2 for further
> review, and sub device support will be provided. But before starting to do it,
> I am still not clear how to integrate FD into MC framework. I understand FD
> sub device is only a media entity, so how can FD sub device find the media
> device(struct media_device)?  or just needn't to care about it now?
> 
> 
> thanks,
> --
> Ming Lei
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html


^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-12 12:08                     ` HeungJun, Kim
  0 siblings, 0 replies; 91+ messages in thread
From: HeungJun, Kim @ 2011-12-12 12:08 UTC (permalink / raw)
  To: linux-arm-kernel

Hi Ming,

It's maybe late, but I want to suggest one thing about FD API.

This OMAP FD block looks detection ability of just face.
But, It's possible to occur another device which can detect
specific "object" or "patterns". Moreover, this API can expand
"object recognition" area. So, I think it's good to change the API name
like "v4l2_recog".

Actually, I'm preparing similar control class for mainline with m5mols
camera sensor driver. The m5mols camera sensor has the function about
"face detection". But, I has experienced about Robot Recognition, and I
remember the image processing chip which can detect spefic "pattern".
So, I hesitated naming the API(control or ioctl whatever) with "face".
It can be possible to provide just "object" or "pattern", not face.
Even user library on windows, there is famous "OpenCV". And this is also
support not only "face", but also "object".

The function of OMAP FDIF looks like m5mols ISP's one.
please understand I don't have experience about OMAP AP. But, I can tell
you it's better to use the name "object recognition", not the "face detection",
for any other device or driver.

In a few days, I'll share the CIDs I have thought for m5mols driver.
And, I hope to discuss about this with OMAP FDIF.

Thank you.

Regards,
Heungjun Kim


> -----Original Message-----
> From: linux-media-owner at vger.kernel.org [mailto:linux-media-
> owner at vger.kernel.org] On Behalf Of Ming Lei
> Sent: Monday, December 12, 2011 6:50 PM
> To: Sylwester Nawrocki
> Cc: linux-omap at vger.kernel.org; linux-arm-kernel at lists.infradead.org; linux-
> kernel at vger.kernel.org; linux-media at vger.kernel.org
> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver
> module
> 
> Hi,
> 
> On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>
> 
> >> For OMAP4 FD, it is not needed to include FD into MC framework since a
> >> intermediate buffer is always required. If your HW doesn't belong to this
> >> case, what is the output of your HW FD in the link? Also sounds FD results
> >> may not be needed at all for use space application in the case.
> >
> > The result data is similar to OMAP4 one, plus a few other attributes.
> > User buffers may be filled by other than FD device driver.
> 
> OK.
> 
> 
> >> Could you provide some practical use cases about these?
> >
> > As above, and any device with a camera that controls something and makes
> > decision according to presence of human face in his view.
> 
> Sounds a reasonable case, :-)
> 
> 
> >> If FD result is associated with a frame, how can user space get the frame
> seq
> >> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
> >> difficult to retrieve FD results from user space.
> >
> > If you pass image data in memory buffers from user space, yes, it could be
> > impossible.
> 
> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)
> 
> >
> > Not really, still v4l2_buffer may be used by other (sub)driver within same
> video
> > processing pipeline.
> 
> OK.
> 
> A related question: how can we make one application to support the two kinds
of
> devices(input from user space data as OMAP4, input from SoC bus as Samsung)
> at the same time? Maybe some capability info is to be exported to user space?
> or other suggestions?
> 
> And will your Samsung FD HW support to detect faces from memory? or just only
> detect from SoC bus?
> 
> 
> > It will be included in the FD result... or in a dedicated v4l2 event data
> structure.
> > More important, at the end of the day, we'll be getting buffers with image
> data
> > at some stage of a video pipeline, which would contain same frame identifier
> > (I think we can ignore v4l2_buffer.field for FD purpose).
> 
> OK, I will associate FD result with frame identifier, and not invent a
> dedicated v4l2 event for query frame seq now until a specific requirement
> for it is proposed.
> 
> I will convert/integrate recent discussions into patches of v2 for further
> review, and sub device support will be provided. But before starting to do it,
> I am still not clear how to integrate FD into MC framework. I understand FD
> sub device is only a media entity, so how can FD sub device find the media
> device(struct media_device)?  or just needn't to care about it now?
> 
> 
> thanks,
> --
> Ming Lei
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-12  9:49                   ` Ming Lei
@ 2011-12-12 21:57                     ` Sylwester Nawrocki
  -1 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-12 21:57 UTC (permalink / raw)
  To: Ming Lei
  Cc: linux-omap, linux-arm-kernel, linux-kernel, linux-media, Kyungmin Park

Hi,

On 12/12/2011 10:49 AM, Ming Lei wrote:
>>> If FD result is associated with a frame, how can user space get the frame
>>> seq if no v4l2 buffer is involved? Without a frame sequence, it is a bit
>>> difficult to retrieve FD results from user space.
>>
>> If you pass image data in memory buffers from user space, yes, it could be
>> impossible.
> 
> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)

Oops, have mixed something up ;)

>> Not really, still v4l2_buffer may be used by other (sub)driver within same
>> video processing pipeline.
> 
> OK.
> 
> A related question: how can we make one application to support the two kinds 
> of devices(input from user space data as OMAP4, input from SoC bus as Samsung)
> at the same time? Maybe some capability info is to be exported to user space?
> or other suggestions?

Good question. To let applications know that a video device is not just
an ordinary video output device I suppose we'll need a new object
detection/recognition capability flag.
V4L2_CAPS_OBJECT_DETECTION, V4L2_CAP_OBJECT_RECOGNITION or something similar.

It's probably safe to assume the SoC will support either input method at time,
not both simultaneously. Then it could be, for example, modelled with a video
node and a subdev:


	     user image data                   video capture
             for FD                            stream
             +-------------+                  +-------------+
             | /dev/video0 |                  | /dev/video0 |
             |   OUTPUT    |                  |  CAPTURE    |
             +------+------+                  +------+------+
                    |                                |
                    v                                ^
..------+        +--+--+----------+-----+            |
image   | link0  | pad | face     | pad |            |
sensor  +-->-----+  0  | detector |  1  |            |
sub-dev +-->-+   |     | sub-dev  |     |            |
..------+    |   +-----+----------+-----+            |
             |                                       |
             |   +--+--+------------+-----+          |
             |   | pad | image      | pad |          |
             +---+  0  | processing |  1  +----------+
          link1  |     | sub-dev    |     |
                 +-----+------------+-----+

User space can control state of link0. If the link is active (streaming) then
access to /dev/video0 would be blocked by the driver, e.g. with EBUSY errno.
This means that only one data source can be attached to an input pad (pad0).
These are intrinsic properties of Media Controller/v4l2 subdev API.


> And will your Samsung FD HW support to detect faces from memory? or just only
> detect from SoC bus?

I think we should be prepared for both configurations, as on a diagram above.

[...]
> OK, I will associate FD result with frame identifier, and not invent a
> dedicated v4l2 event for query frame seq now until a specific requirement
> for it is proposed.
> 
> I will convert/integrate recent discussions into patches of v2 for further

Sure, sounds like a good idea.

> review, and sub device support will be provided. But before starting to do it,
> I am still not clear how to integrate FD into MC framework. I understand FD
> sub device is only a media entity, so how can FD sub device find the media
> device(struct media_device)?  or just needn't to care about it now?

The media device driver will register all entities that belong to it and will
create relevant links between entities' pads, which then can be activated by
applications. How the entities are registered is another topic, that we don't
need to be concerned about at the moment. If you're curious see
drivers/media/video/omap3isp or driver/media/video/s5p-fimc for example media
device drivers.

-- 
Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-12 21:57                     ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-12 21:57 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On 12/12/2011 10:49 AM, Ming Lei wrote:
>>> If FD result is associated with a frame, how can user space get the frame
>>> seq if no v4l2 buffer is involved? Without a frame sequence, it is a bit
>>> difficult to retrieve FD results from user space.
>>
>> If you pass image data in memory buffers from user space, yes, it could be
>> impossible.
> 
> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)

Oops, have mixed something up ;)

>> Not really, still v4l2_buffer may be used by other (sub)driver within same
>> video processing pipeline.
> 
> OK.
> 
> A related question: how can we make one application to support the two kinds 
> of devices(input from user space data as OMAP4, input from SoC bus as Samsung)
> at the same time? Maybe some capability info is to be exported to user space?
> or other suggestions?

Good question. To let applications know that a video device is not just
an ordinary video output device I suppose we'll need a new object
detection/recognition capability flag.
V4L2_CAPS_OBJECT_DETECTION, V4L2_CAP_OBJECT_RECOGNITION or something similar.

It's probably safe to assume the SoC will support either input method at time,
not both simultaneously. Then it could be, for example, modelled with a video
node and a subdev:


	     user image data                   video capture
             for FD                            stream
             +-------------+                  +-------------+
             | /dev/video0 |                  | /dev/video0 |
             |   OUTPUT    |                  |  CAPTURE    |
             +------+------+                  +------+------+
                    |                                |
                    v                                ^
..------+        +--+--+----------+-----+            |
image   | link0  | pad | face     | pad |            |
sensor  +-->-----+  0  | detector |  1  |            |
sub-dev +-->-+   |     | sub-dev  |     |            |
..------+    |   +-----+----------+-----+            |
             |                                       |
             |   +--+--+------------+-----+          |
             |   | pad | image      | pad |          |
             +---+  0  | processing |  1  +----------+
          link1  |     | sub-dev    |     |
                 +-----+------------+-----+

User space can control state of link0. If the link is active (streaming) then
access to /dev/video0 would be blocked by the driver, e.g. with EBUSY errno.
This means that only one data source can be attached to an input pad (pad0).
These are intrinsic properties of Media Controller/v4l2 subdev API.


> And will your Samsung FD HW support to detect faces from memory? or just only
> detect from SoC bus?

I think we should be prepared for both configurations, as on a diagram above.

[...]
> OK, I will associate FD result with frame identifier, and not invent a
> dedicated v4l2 event for query frame seq now until a specific requirement
> for it is proposed.
> 
> I will convert/integrate recent discussions into patches of v2 for further

Sure, sounds like a good idea.

> review, and sub device support will be provided. But before starting to do it,
> I am still not clear how to integrate FD into MC framework. I understand FD
> sub device is only a media entity, so how can FD sub device find the media
> device(struct media_device)?  or just needn't to care about it now?

The media device driver will register all entities that belong to it and will
create relevant links between entities' pads, which then can be activated by
applications. How the entities are registered is another topic, that we don't
need to be concerned about at the moment. If you're curious see
drivers/media/video/omap3isp or driver/media/video/s5p-fimc for example media
device drivers.

-- 
Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-12 12:08                     ` HeungJun, Kim
@ 2011-12-13  4:01                       ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-13  4:01 UTC (permalink / raw)
  To: HeungJun, Kim
  Cc: Sylwester Nawrocki, linux-omap, linux-arm-kernel, linux-kernel,
	linux-media

Hi,

On Mon, Dec 12, 2011 at 8:08 PM, HeungJun, Kim <riverful.kim@samsung.com> wrote:
> Hi Ming,
>
> It's maybe late, but I want to suggest one thing about FD API.
>
> This OMAP FD block looks detection ability of just face.
> But, It's possible to occur another device which can detect
> specific "object" or "patterns". Moreover, this API can expand
> "object recognition" area. So, I think it's good to change the API name
> like "v4l2_recog".

IMO, object detection is better,  at least now OMAP4 and samsung has
face detection IP module, and face recognition is often done on results
of face detection and more complicated interfaces will be involved.

>
> Actually, I'm preparing similar control class for mainline with m5mols
> camera sensor driver. The m5mols camera sensor has the function about
> "face detection". But, I has experienced about Robot Recognition, and I
> remember the image processing chip which can detect spefic "pattern".
> So, I hesitated naming the API(control or ioctl whatever) with "face".
> It can be possible to provide just "object" or "pattern", not face.
> Even user library on windows, there is famous "OpenCV". And this is also
> support not only "face", but also "object".

Yes, object is better than face, and we can use enum flag to describe that
the objects detected are which kind of objects. In fact, I plan to rename the
face detection generic driver as object detection generic driver and let
hardware driver to handle the object detection details.

>
> The function of OMAP FDIF looks like m5mols ISP's one.
> please understand I don't have experience about OMAP AP. But, I can tell
> you it's better to use the name "object recognition", not the "face detection",
> for any other device or driver.
>
> In a few days, I'll share the CIDs I have thought for m5mols driver.
> And, I hope to discuss about this with OMAP FDIF.

You have been doing it already, :-)

thanks,
--
Ming Lei

>
> Thank you.
>
> Regards,
> Heungjun Kim
>
>
>> -----Original Message-----
>> From: linux-media-owner@vger.kernel.org [mailto:linux-media-
>> owner@vger.kernel.org] On Behalf Of Ming Lei
>> Sent: Monday, December 12, 2011 6:50 PM
>> To: Sylwester Nawrocki
>> Cc: linux-omap@vger.kernel.org; linux-arm-kernel@lists.infradead.org; linux-
>> kernel@vger.kernel.org; linux-media@vger.kernel.org
>> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver
>> module
>>
>> Hi,
>>
>> On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>
>>
>> >> For OMAP4 FD, it is not needed to include FD into MC framework since a
>> >> intermediate buffer is always required. If your HW doesn't belong to this
>> >> case, what is the output of your HW FD in the link? Also sounds FD results
>> >> may not be needed at all for use space application in the case.
>> >
>> > The result data is similar to OMAP4 one, plus a few other attributes.
>> > User buffers may be filled by other than FD device driver.
>>
>> OK.
>>
>>
>> >> Could you provide some practical use cases about these?
>> >
>> > As above, and any device with a camera that controls something and makes
>> > decision according to presence of human face in his view.
>>
>> Sounds a reasonable case, :-)
>>
>>
>> >> If FD result is associated with a frame, how can user space get the frame
>> seq
>> >> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
>> >> difficult to retrieve FD results from user space.
>> >
>> > If you pass image data in memory buffers from user space, yes, it could be
>> > impossible.
>>
>> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)
>>
>> >
>> > Not really, still v4l2_buffer may be used by other (sub)driver within same
>> video
>> > processing pipeline.
>>
>> OK.
>>
>> A related question: how can we make one application to support the two kinds
> of
>> devices(input from user space data as OMAP4, input from SoC bus as Samsung)
>> at the same time? Maybe some capability info is to be exported to user space?
>> or other suggestions?
>>
>> And will your Samsung FD HW support to detect faces from memory? or just only
>> detect from SoC bus?
>>
>>
>> > It will be included in the FD result... or in a dedicated v4l2 event data
>> structure.
>> > More important, at the end of the day, we'll be getting buffers with image
>> data
>> > at some stage of a video pipeline, which would contain same frame identifier
>> > (I think we can ignore v4l2_buffer.field for FD purpose).
>>
>> OK, I will associate FD result with frame identifier, and not invent a
>> dedicated v4l2 event for query frame seq now until a specific requirement
>> for it is proposed.
>>
>> I will convert/integrate recent discussions into patches of v2 for further
>> review, and sub device support will be provided. But before starting to do it,
>> I am still not clear how to integrate FD into MC framework. I understand FD
>> sub device is only a media entity, so how can FD sub device find the media
>> device(struct media_device)?  or just needn't to care about it now?
>>
>>
>> thanks,
>> --
>> Ming Lei
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-media" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-omap" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-13  4:01                       ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-13  4:01 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Mon, Dec 12, 2011 at 8:08 PM, HeungJun, Kim <riverful.kim@samsung.com> wrote:
> Hi Ming,
>
> It's maybe late, but I want to suggest one thing about FD API.
>
> This OMAP FD block looks detection ability of just face.
> But, It's possible to occur another device which can detect
> specific "object" or "patterns". Moreover, this API can expand
> "object recognition" area. So, I think it's good to change the API name
> like "v4l2_recog".

IMO, object detection is better,  at least now OMAP4 and samsung has
face detection IP module, and face recognition is often done on results
of face detection and more complicated interfaces will be involved.

>
> Actually, I'm preparing similar control class for mainline with m5mols
> camera sensor driver. The m5mols camera sensor has the function about
> "face detection". But, I has experienced about Robot Recognition, and I
> remember the image processing chip which can detect spefic "pattern".
> So, I hesitated naming the API(control or ioctl whatever) with "face".
> It can be possible to provide just "object" or "pattern", not face.
> Even user library on windows, there is famous "OpenCV". And this is also
> support not only "face", but also "object".

Yes, object is better than face, and we can use enum flag to describe that
the objects detected are which kind of objects. In fact, I plan to rename the
face detection generic driver as object detection generic driver and let
hardware driver to handle the object detection details.

>
> The function of OMAP FDIF looks like m5mols ISP's one.
> please understand I don't have experience about OMAP AP. But, I can tell
> you it's better to use the name "object recognition", not the "face detection",
> for any other device or driver.
>
> In a few days, I'll share the CIDs I have thought for m5mols driver.
> And, I hope to discuss about this with OMAP FDIF.

You have been doing it already, :-)

thanks,
--
Ming Lei

>
> Thank you.
>
> Regards,
> Heungjun Kim
>
>
>> -----Original Message-----
>> From: linux-media-owner at vger.kernel.org [mailto:linux-media-
>> owner at vger.kernel.org] On Behalf Of Ming Lei
>> Sent: Monday, December 12, 2011 6:50 PM
>> To: Sylwester Nawrocki
>> Cc: linux-omap at vger.kernel.org; linux-arm-kernel at lists.infradead.org; linux-
>> kernel at vger.kernel.org; linux-media at vger.kernel.org
>> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver
>> module
>>
>> Hi,
>>
>> On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>
>>
>> >> For OMAP4 FD, it is not needed to include FD into MC framework since a
>> >> intermediate buffer is always required. If your HW doesn't belong to this
>> >> case, what is the output of your HW FD in the link? Also sounds FD results
>> >> may not be needed at all for use space application in the case.
>> >
>> > The result data is similar to OMAP4 one, plus a few other attributes.
>> > User buffers may be filled by other than FD device driver.
>>
>> OK.
>>
>>
>> >> Could you provide some practical use cases about these?
>> >
>> > As above, and any device with a camera that controls something and makes
>> > decision according to presence of human face in his view.
>>
>> Sounds a reasonable case, :-)
>>
>>
>> >> If FD result is associated with a frame, how can user space get the frame
>> seq
>> >> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
>> >> difficult to retrieve FD results from user space.
>> >
>> > If you pass image data in memory buffers from user space, yes, it could be
>> > impossible.
>>
>> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)
>>
>> >
>> > Not really, still v4l2_buffer may be used by other (sub)driver within same
>> video
>> > processing pipeline.
>>
>> OK.
>>
>> A related question: how can we make one application to support the two kinds
> of
>> devices(input from user space data as OMAP4, input from SoC bus as Samsung)
>> at the same time? Maybe some capability info is to be exported to user space?
>> or other suggestions?
>>
>> And will your Samsung FD HW support to detect faces from memory? or just only
>> detect from SoC bus?
>>
>>
>> > It will be included in the FD result... or in a dedicated v4l2 event data
>> structure.
>> > More important, at the end of the day, we'll be getting buffers with image
>> data
>> > at some stage of a video pipeline, which would contain same frame identifier
>> > (I think we can ignore v4l2_buffer.field for FD purpose).
>>
>> OK, I will associate FD result with frame identifier, and not invent a
>> dedicated v4l2 event for query frame seq now until a specific requirement
>> for it is proposed.
>>
>> I will convert/integrate recent discussions into patches of v2 for further
>> review, and sub device support will be provided. But before starting to do it,
>> I am still not clear how to integrate FD into MC framework. I understand FD
>> sub device is only a media entity, so how can FD sub device find the media
>> device(struct media_device)? ?or just needn't to care about it now?
>>
>>
>> thanks,
>> --
>> Ming Lei
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-media" in
>> the body of a message to majordomo at vger.kernel.org
>> More majordomo info at ?http://vger.kernel.org/majordomo-info.html
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-omap" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at ?http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* RE: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-13  4:01                       ` Ming Lei
@ 2011-12-13  5:59                         ` HeungJun, Kim
  -1 siblings, 0 replies; 91+ messages in thread
From: HeungJun, Kim @ 2011-12-13  5:59 UTC (permalink / raw)
  To: 'Ming Lei'
  Cc: 'Sylwester Nawrocki',
	linux-omap, linux-arm-kernel, linux-kernel, linux-media

Hi Ming and Sylwester,

Thanks for the reply.

> -----Original Message-----
> From: Ming Lei [mailto:ming.lei@canonical.com]
> Sent: Tuesday, December 13, 2011 1:02 PM
> To: HeungJun, Kim
> Cc: Sylwester Nawrocki; linux-omap@vger.kernel.org; linux-arm-
> kernel@lists.infradead.org; linux-kernel@vger.kernel.org; linux-
> media@vger.kernel.org
> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver
> module
> 
> Hi,
> 
> On Mon, Dec 12, 2011 at 8:08 PM, HeungJun, Kim <riverful.kim@samsung.com>
wrote:
> > Hi Ming,
> >
> > It's maybe late, but I want to suggest one thing about FD API.
> >
> > This OMAP FD block looks detection ability of just face.
> > But, It's possible to occur another device which can detect
> > specific "object" or "patterns". Moreover, this API can expand
> > "object recognition" area. So, I think it's good to change the API name
> > like "v4l2_recog".
> 
> IMO, object detection is better,  at least now OMAP4 and samsung has
> face detection IP module, and face recognition is often done on results
> of face detection and more complicated interfaces will be involved.
Actually, the detection point is different, I guess.
The OMAP has the detection block separately, named FDIF. But, Samsung
Exynos doing detection process with externel sensor - m5mols, in our case.
This sensor(m5mols) has ISP function, and these ISP can process detection.
The expert of FIMC is Sylwester. Probably, he can tell the differences
between both in more details. :)

> 
> >
> > Actually, I'm preparing similar control class for mainline with m5mols
> > camera sensor driver. The m5mols camera sensor has the function about
> > "face detection". But, I has experienced about Robot Recognition, and I
> > remember the image processing chip which can detect spefic "pattern".
> > So, I hesitated naming the API(control or ioctl whatever) with "face".
> > It can be possible to provide just "object" or "pattern", not face.
> > Even user library on windows, there is famous "OpenCV". And this is also
> > support not only "face", but also "object".
> 
> Yes, object is better than face, and we can use enum flag to describe that
> the objects detected are which kind of objects. In fact, I plan to rename the
> face detection generic driver as object detection generic driver and let
> hardware driver to handle the object detection details.
> 
> >
> > The function of OMAP FDIF looks like m5mols ISP's one.
> > please understand I don't have experience about OMAP AP. But, I can tell
> > you it's better to use the name "object recognition", not the "face
> detection",
> > for any other device or driver.
> >
> > In a few days, I'll share the CIDs I have thought for m5mols driver.
> > And, I hope to discuss about this with OMAP FDIF.
> 
> You have been doing it already, :-)
Yeah, actually I did. :)
But, until I see OMAP FDIF case, I did not recognize AP(like OMAP) can
have detection capability. :) So, although I did not think about at that time,
I also think we should re-consider this case for now.

As I look around your patch quickly, the functions is very similar with ours.
(even detection of left/right eye)
So, the problem is there are two ways to proceed "recognition".
- Does it handle at the level of IOCTLs? or CIDs?

If the detection algorithm is proceeded at the level of HW block,
it's right the platform driver provide APIs as IOCTLs(as you're FDIF patches).
On the other hand, if it is proceeded at the sensor(subdevice) level,
I think it's more right to control using CIDs.

We need the solution including those two cases.
And the name - object detection? object recognition?

So, do you have any good ideas?

ps. There can be another not-matched HW block level issues.
But, the problem which I can check is just above issue for now.


Thanks,
Heungjun Kim


> 
> thanks,
> --
> Ming Lei
> 
> >
> > Thank you.
> >
> > Regards,
> > Heungjun Kim
> >
> >
> >> -----Original Message-----
> >> From: linux-media-owner@vger.kernel.org [mailto:linux-media-
> >> owner@vger.kernel.org] On Behalf Of Ming Lei
> >> Sent: Monday, December 12, 2011 6:50 PM
> >> To: Sylwester Nawrocki
> >> Cc: linux-omap@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
linux-
> >> kernel@vger.kernel.org; linux-media@vger.kernel.org
> >> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection
> driver
> >> module
> >>
> >> Hi,
> >>
> >> On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>
> >>
> >> >> For OMAP4 FD, it is not needed to include FD into MC framework since a
> >> >> intermediate buffer is always required. If your HW doesn't belong to
this
> >> >> case, what is the output of your HW FD in the link? Also sounds FD
> results
> >> >> may not be needed at all for use space application in the case.
> >> >
> >> > The result data is similar to OMAP4 one, plus a few other attributes.
> >> > User buffers may be filled by other than FD device driver.
> >>
> >> OK.
> >>
> >>
> >> >> Could you provide some practical use cases about these?
> >> >
> >> > As above, and any device with a camera that controls something and makes
> >> > decision according to presence of human face in his view.
> >>
> >> Sounds a reasonable case, :-)
> >>
> >>
> >> >> If FD result is associated with a frame, how can user space get the
frame
> >> seq
> >> >> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
> >> >> difficult to retrieve FD results from user space.
> >> >
> >> > If you pass image data in memory buffers from user space, yes, it could
be
> >> > impossible.
> >>
> >> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)
> >>
> >> >
> >> > Not really, still v4l2_buffer may be used by other (sub)driver within
same
> >> video
> >> > processing pipeline.
> >>
> >> OK.
> >>
> >> A related question: how can we make one application to support the two
kinds
> > of
> >> devices(input from user space data as OMAP4, input from SoC bus as Samsung)
> >> at the same time? Maybe some capability info is to be exported to user
space?
> >> or other suggestions?
> >>
> >> And will your Samsung FD HW support to detect faces from memory? or just
> only
> >> detect from SoC bus?
> >>
> >>
> >> > It will be included in the FD result... or in a dedicated v4l2 event data
> >> structure.
> >> > More important, at the end of the day, we'll be getting buffers with
image
> >> data
> >> > at some stage of a video pipeline, which would contain same frame
> identifier
> >> > (I think we can ignore v4l2_buffer.field for FD purpose).
> >>
> >> OK, I will associate FD result with frame identifier, and not invent a
> >> dedicated v4l2 event for query frame seq now until a specific requirement
> >> for it is proposed.
> >>
> >> I will convert/integrate recent discussions into patches of v2 for further
> >> review, and sub device support will be provided. But before starting to do
> it,
> >> I am still not clear how to integrate FD into MC framework. I understand FD
> >> sub device is only a media entity, so how can FD sub device find the media
> >> device(struct media_device)?  or just needn't to care about it now?
> >>
> >>
> >> thanks,
> >> --
> >> Ming Lei
> >> --
> >> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> >> the body of a message to majordomo@vger.kernel.org
> >> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> >
> > --
> > To unsubscribe from this list: send the line "unsubscribe linux-omap" in
> > the body of a message to majordomo@vger.kernel.org
> > More majordomo info at  http://vger.kernel.org/majordomo-info.html


^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-13  5:59                         ` HeungJun, Kim
  0 siblings, 0 replies; 91+ messages in thread
From: HeungJun, Kim @ 2011-12-13  5:59 UTC (permalink / raw)
  To: linux-arm-kernel

Hi Ming and Sylwester,

Thanks for the reply.

> -----Original Message-----
> From: Ming Lei [mailto:ming.lei at canonical.com]
> Sent: Tuesday, December 13, 2011 1:02 PM
> To: HeungJun, Kim
> Cc: Sylwester Nawrocki; linux-omap at vger.kernel.org; linux-arm-
> kernel at lists.infradead.org; linux-kernel at vger.kernel.org; linux-
> media at vger.kernel.org
> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver
> module
> 
> Hi,
> 
> On Mon, Dec 12, 2011 at 8:08 PM, HeungJun, Kim <riverful.kim@samsung.com>
wrote:
> > Hi Ming,
> >
> > It's maybe late, but I want to suggest one thing about FD API.
> >
> > This OMAP FD block looks detection ability of just face.
> > But, It's possible to occur another device which can detect
> > specific "object" or "patterns". Moreover, this API can expand
> > "object recognition" area. So, I think it's good to change the API name
> > like "v4l2_recog".
> 
> IMO, object detection is better,  at least now OMAP4 and samsung has
> face detection IP module, and face recognition is often done on results
> of face detection and more complicated interfaces will be involved.
Actually, the detection point is different, I guess.
The OMAP has the detection block separately, named FDIF. But, Samsung
Exynos doing detection process with externel sensor - m5mols, in our case.
This sensor(m5mols) has ISP function, and these ISP can process detection.
The expert of FIMC is Sylwester. Probably, he can tell the differences
between both in more details. :)

> 
> >
> > Actually, I'm preparing similar control class for mainline with m5mols
> > camera sensor driver. The m5mols camera sensor has the function about
> > "face detection". But, I has experienced about Robot Recognition, and I
> > remember the image processing chip which can detect spefic "pattern".
> > So, I hesitated naming the API(control or ioctl whatever) with "face".
> > It can be possible to provide just "object" or "pattern", not face.
> > Even user library on windows, there is famous "OpenCV". And this is also
> > support not only "face", but also "object".
> 
> Yes, object is better than face, and we can use enum flag to describe that
> the objects detected are which kind of objects. In fact, I plan to rename the
> face detection generic driver as object detection generic driver and let
> hardware driver to handle the object detection details.
> 
> >
> > The function of OMAP FDIF looks like m5mols ISP's one.
> > please understand I don't have experience about OMAP AP. But, I can tell
> > you it's better to use the name "object recognition", not the "face
> detection",
> > for any other device or driver.
> >
> > In a few days, I'll share the CIDs I have thought for m5mols driver.
> > And, I hope to discuss about this with OMAP FDIF.
> 
> You have been doing it already, :-)
Yeah, actually I did. :)
But, until I see OMAP FDIF case, I did not recognize AP(like OMAP) can
have detection capability. :) So, although I did not think about at that time,
I also think we should re-consider this case for now.

As I look around your patch quickly, the functions is very similar with ours.
(even detection of left/right eye)
So, the problem is there are two ways to proceed "recognition".
- Does it handle at the level of IOCTLs? or CIDs?

If the detection algorithm is proceeded at the level of HW block,
it's right the platform driver provide APIs as IOCTLs(as you're FDIF patches).
On the other hand, if it is proceeded at the sensor(subdevice) level,
I think it's more right to control using CIDs.

We need the solution including those two cases.
And the name - object detection? object recognition?

So, do you have any good ideas?

ps. There can be another not-matched HW block level issues.
But, the problem which I can check is just above issue for now.


Thanks,
Heungjun Kim


> 
> thanks,
> --
> Ming Lei
> 
> >
> > Thank you.
> >
> > Regards,
> > Heungjun Kim
> >
> >
> >> -----Original Message-----
> >> From: linux-media-owner at vger.kernel.org [mailto:linux-media-
> >> owner at vger.kernel.org] On Behalf Of Ming Lei
> >> Sent: Monday, December 12, 2011 6:50 PM
> >> To: Sylwester Nawrocki
> >> Cc: linux-omap at vger.kernel.org; linux-arm-kernel at lists.infradead.org;
linux-
> >> kernel at vger.kernel.org; linux-media at vger.kernel.org
> >> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection
> driver
> >> module
> >>
> >> Hi,
> >>
> >> On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>
> >>
> >> >> For OMAP4 FD, it is not needed to include FD into MC framework since a
> >> >> intermediate buffer is always required. If your HW doesn't belong to
this
> >> >> case, what is the output of your HW FD in the link? Also sounds FD
> results
> >> >> may not be needed at all for use space application in the case.
> >> >
> >> > The result data is similar to OMAP4 one, plus a few other attributes.
> >> > User buffers may be filled by other than FD device driver.
> >>
> >> OK.
> >>
> >>
> >> >> Could you provide some practical use cases about these?
> >> >
> >> > As above, and any device with a camera that controls something and makes
> >> > decision according to presence of human face in his view.
> >>
> >> Sounds a reasonable case, :-)
> >>
> >>
> >> >> If FD result is associated with a frame, how can user space get the
frame
> >> seq
> >> >> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
> >> >> difficult to retrieve FD results from user space.
> >> >
> >> > If you pass image data in memory buffers from user space, yes, it could
be
> >> > impossible.
> >>
> >> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)
> >>
> >> >
> >> > Not really, still v4l2_buffer may be used by other (sub)driver within
same
> >> video
> >> > processing pipeline.
> >>
> >> OK.
> >>
> >> A related question: how can we make one application to support the two
kinds
> > of
> >> devices(input from user space data as OMAP4, input from SoC bus as Samsung)
> >> at the same time? Maybe some capability info is to be exported to user
space?
> >> or other suggestions?
> >>
> >> And will your Samsung FD HW support to detect faces from memory? or just
> only
> >> detect from SoC bus?
> >>
> >>
> >> > It will be included in the FD result... or in a dedicated v4l2 event data
> >> structure.
> >> > More important, at the end of the day, we'll be getting buffers with
image
> >> data
> >> > at some stage of a video pipeline, which would contain same frame
> identifier
> >> > (I think we can ignore v4l2_buffer.field for FD purpose).
> >>
> >> OK, I will associate FD result with frame identifier, and not invent a
> >> dedicated v4l2 event for query frame seq now until a specific requirement
> >> for it is proposed.
> >>
> >> I will convert/integrate recent discussions into patches of v2 for further
> >> review, and sub device support will be provided. But before starting to do
> it,
> >> I am still not clear how to integrate FD into MC framework. I understand FD
> >> sub device is only a media entity, so how can FD sub device find the media
> >> device(struct media_device)? ?or just needn't to care about it now?
> >>
> >>
> >> thanks,
> >> --
> >> Ming Lei
> >> --
> >> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> >> the body of a message to majordomo at vger.kernel.org
> >> More majordomo info at ?http://vger.kernel.org/majordomo-info.html
> >
> > --
> > To unsubscribe from this list: send the line "unsubscribe linux-omap" in
> > the body of a message to majordomo at vger.kernel.org
> > More majordomo info at ?http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver module
  2011-12-13  5:59                         ` HeungJun, Kim
@ 2011-12-13  6:29                           ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-13  6:29 UTC (permalink / raw)
  To: HeungJun, Kim
  Cc: Sylwester Nawrocki, linux-omap, linux-arm-kernel, linux-kernel,
	linux-media

On Tue, Dec 13, 2011 at 1:59 PM, HeungJun, Kim <riverful.kim@samsung.com> wrote:
> Hi Ming and Sylwester,
>
> Thanks for the reply.
>
>> -----Original Message-----
>> From: Ming Lei [mailto:ming.lei@canonical.com]
>> Sent: Tuesday, December 13, 2011 1:02 PM
>> To: HeungJun, Kim
>> Cc: Sylwester Nawrocki; linux-omap@vger.kernel.org; linux-arm-
>> kernel@lists.infradead.org; linux-kernel@vger.kernel.org; linux-
>> media@vger.kernel.org
>> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver
>> module
>>
>> Hi,
>>
>> On Mon, Dec 12, 2011 at 8:08 PM, HeungJun, Kim <riverful.kim@samsung.com>
> wrote:
>> > Hi Ming,
>> >
>> > It's maybe late, but I want to suggest one thing about FD API.
>> >
>> > This OMAP FD block looks detection ability of just face.
>> > But, It's possible to occur another device which can detect
>> > specific "object" or "patterns". Moreover, this API can expand
>> > "object recognition" area. So, I think it's good to change the API name
>> > like "v4l2_recog".
>>
>> IMO, object detection is better,  at least now OMAP4 and samsung has
>> face detection IP module, and face recognition is often done on results
>> of face detection and more complicated interfaces will be involved.
> Actually, the detection point is different, I guess.
> The OMAP has the detection block separately, named FDIF. But, Samsung
> Exynos doing detection process with externel sensor - m5mols, in our case.
> This sensor(m5mols) has ISP function, and these ISP can process detection.
> The expert of FIMC is Sylwester. Probably, he can tell the differences
> between both in more details. :)
>
>>
>> >
>> > Actually, I'm preparing similar control class for mainline with m5mols
>> > camera sensor driver. The m5mols camera sensor has the function about
>> > "face detection". But, I has experienced about Robot Recognition, and I
>> > remember the image processing chip which can detect spefic "pattern".
>> > So, I hesitated naming the API(control or ioctl whatever) with "face".
>> > It can be possible to provide just "object" or "pattern", not face.
>> > Even user library on windows, there is famous "OpenCV". And this is also
>> > support not only "face", but also "object".
>>
>> Yes, object is better than face, and we can use enum flag to describe that
>> the objects detected are which kind of objects. In fact, I plan to rename the
>> face detection generic driver as object detection generic driver and let
>> hardware driver to handle the object detection details.
>>
>> >
>> > The function of OMAP FDIF looks like m5mols ISP's one.
>> > please understand I don't have experience about OMAP AP. But, I can tell
>> > you it's better to use the name "object recognition", not the "face
>> detection",
>> > for any other device or driver.
>> >
>> > In a few days, I'll share the CIDs I have thought for m5mols driver.
>> > And, I hope to discuss about this with OMAP FDIF.
>>
>> You have been doing it already, :-)
> Yeah, actually I did. :)
> But, until I see OMAP FDIF case, I did not recognize AP(like OMAP) can
> have detection capability. :) So, although I did not think about at that time,
> I also think we should re-consider this case for now.
>
> As I look around your patch quickly, the functions is very similar with ours.
> (even detection of left/right eye)
> So, the problem is there are two ways to proceed "recognition".
> - Does it handle at the level of IOCTLs? or CIDs?

The patch introduces two IOCTL to retrieve object detection result.
I think it should be handled by IOCTL. The interface is a little complex,
so it is not easy to handle it by CIDs, IMO.

>
> If the detection algorithm is proceeded at the level of HW block,
> it's right the platform driver provide APIs as IOCTLs(as you're FDIF patches).
> On the other hand, if it is proceeded at the sensor(subdevice) level,
> I think it's more right to control using CIDs.

Certainly, some generic CIDs for object detection will be introduced later and
will be handled in the object detection(the current fdif generic
driver, patch 6/7)
generic driver.

> We need the solution including those two cases.
> And the name - object detection? object recognition?

I think object detection is better. For example, face detection is very
different with face recognition, isn't it?

thanks,
--
Ming Lei

>
> So, do you have any good ideas?
>
> ps. There can be another not-matched HW block level issues.
> But, the problem which I can check is just above issue for now.
>
>
> Thanks,
> Heungjun Kim
>
>
>>
>> thanks,
>> --
>> Ming Lei
>>
>> >
>> > Thank you.
>> >
>> > Regards,
>> > Heungjun Kim
>> >
>> >
>> >> -----Original Message-----
>> >> From: linux-media-owner@vger.kernel.org [mailto:linux-media-
>> >> owner@vger.kernel.org] On Behalf Of Ming Lei
>> >> Sent: Monday, December 12, 2011 6:50 PM
>> >> To: Sylwester Nawrocki
>> >> Cc: linux-omap@vger.kernel.org; linux-arm-kernel@lists.infradead.org;
> linux-
>> >> kernel@vger.kernel.org; linux-media@vger.kernel.org
>> >> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection
>> driver
>> >> module
>> >>
>> >> Hi,
>> >>
>> >> On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>
>> >>
>> >> >> For OMAP4 FD, it is not needed to include FD into MC framework since a
>> >> >> intermediate buffer is always required. If your HW doesn't belong to
> this
>> >> >> case, what is the output of your HW FD in the link? Also sounds FD
>> results
>> >> >> may not be needed at all for use space application in the case.
>> >> >
>> >> > The result data is similar to OMAP4 one, plus a few other attributes.
>> >> > User buffers may be filled by other than FD device driver.
>> >>
>> >> OK.
>> >>
>> >>
>> >> >> Could you provide some practical use cases about these?
>> >> >
>> >> > As above, and any device with a camera that controls something and makes
>> >> > decision according to presence of human face in his view.
>> >>
>> >> Sounds a reasonable case, :-)
>> >>
>> >>
>> >> >> If FD result is associated with a frame, how can user space get the
> frame
>> >> seq
>> >> >> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
>> >> >> difficult to retrieve FD results from user space.
>> >> >
>> >> > If you pass image data in memory buffers from user space, yes, it could
> be
>> >> > impossible.
>> >>
>> >> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)
>> >>
>> >> >
>> >> > Not really, still v4l2_buffer may be used by other (sub)driver within
> same
>> >> video
>> >> > processing pipeline.
>> >>
>> >> OK.
>> >>
>> >> A related question: how can we make one application to support the two
> kinds
>> > of
>> >> devices(input from user space data as OMAP4, input from SoC bus as Samsung)
>> >> at the same time? Maybe some capability info is to be exported to user
> space?
>> >> or other suggestions?
>> >>
>> >> And will your Samsung FD HW support to detect faces from memory? or just
>> only
>> >> detect from SoC bus?
>> >>
>> >>
>> >> > It will be included in the FD result... or in a dedicated v4l2 event data
>> >> structure.
>> >> > More important, at the end of the day, we'll be getting buffers with
> image
>> >> data
>> >> > at some stage of a video pipeline, which would contain same frame
>> identifier
>> >> > (I think we can ignore v4l2_buffer.field for FD purpose).
>> >>
>> >> OK, I will associate FD result with frame identifier, and not invent a
>> >> dedicated v4l2 event for query frame seq now until a specific requirement
>> >> for it is proposed.
>> >>
>> >> I will convert/integrate recent discussions into patches of v2 for further
>> >> review, and sub device support will be provided. But before starting to do
>> it,
>> >> I am still not clear how to integrate FD into MC framework. I understand FD
>> >> sub device is only a media entity, so how can FD sub device find the media
>> >> device(struct media_device)?  or just needn't to care about it now?
>> >>
>> >>
>> >> thanks,
>> >> --
>> >> Ming Lei
>> >> --
>> >> To unsubscribe from this list: send the line "unsubscribe linux-media" in
>> >> the body of a message to majordomo@vger.kernel.org
>> >> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>> >
>> > --
>> > To unsubscribe from this list: send the line "unsubscribe linux-omap" in
>> > the body of a message to majordomo@vger.kernel.org
>> > More majordomo info at  http://vger.kernel.org/majordomo-info.html
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 6/7] media: video: introduce face detection driver module
@ 2011-12-13  6:29                           ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-13  6:29 UTC (permalink / raw)
  To: linux-arm-kernel

On Tue, Dec 13, 2011 at 1:59 PM, HeungJun, Kim <riverful.kim@samsung.com> wrote:
> Hi Ming and Sylwester,
>
> Thanks for the reply.
>
>> -----Original Message-----
>> From: Ming Lei [mailto:ming.lei at canonical.com]
>> Sent: Tuesday, December 13, 2011 1:02 PM
>> To: HeungJun, Kim
>> Cc: Sylwester Nawrocki; linux-omap at vger.kernel.org; linux-arm-
>> kernel at lists.infradead.org; linux-kernel at vger.kernel.org; linux-
>> media at vger.kernel.org
>> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection driver
>> module
>>
>> Hi,
>>
>> On Mon, Dec 12, 2011 at 8:08 PM, HeungJun, Kim <riverful.kim@samsung.com>
> wrote:
>> > Hi Ming,
>> >
>> > It's maybe late, but I want to suggest one thing about FD API.
>> >
>> > This OMAP FD block looks detection ability of just face.
>> > But, It's possible to occur another device which can detect
>> > specific "object" or "patterns". Moreover, this API can expand
>> > "object recognition" area. So, I think it's good to change the API name
>> > like "v4l2_recog".
>>
>> IMO, object detection is better, ?at least now OMAP4 and samsung has
>> face detection IP module, and face recognition is often done on results
>> of face detection and more complicated interfaces will be involved.
> Actually, the detection point is different, I guess.
> The OMAP has the detection block separately, named FDIF. But, Samsung
> Exynos doing detection process with externel sensor - m5mols, in our case.
> This sensor(m5mols) has ISP function, and these ISP can process detection.
> The expert of FIMC is Sylwester. Probably, he can tell the differences
> between both in more details. :)
>
>>
>> >
>> > Actually, I'm preparing similar control class for mainline with m5mols
>> > camera sensor driver. The m5mols camera sensor has the function about
>> > "face detection". But, I has experienced about Robot Recognition, and I
>> > remember the image processing chip which can detect spefic "pattern".
>> > So, I hesitated naming the API(control or ioctl whatever) with "face".
>> > It can be possible to provide just "object" or "pattern", not face.
>> > Even user library on windows, there is famous "OpenCV". And this is also
>> > support not only "face", but also "object".
>>
>> Yes, object is better than face, and we can use enum flag to describe that
>> the objects detected are which kind of objects. In fact, I plan to rename the
>> face detection generic driver as object detection generic driver and let
>> hardware driver to handle the object detection details.
>>
>> >
>> > The function of OMAP FDIF looks like m5mols ISP's one.
>> > please understand I don't have experience about OMAP AP. But, I can tell
>> > you it's better to use the name "object recognition", not the "face
>> detection",
>> > for any other device or driver.
>> >
>> > In a few days, I'll share the CIDs I have thought for m5mols driver.
>> > And, I hope to discuss about this with OMAP FDIF.
>>
>> You have been doing it already, :-)
> Yeah, actually I did. :)
> But, until I see OMAP FDIF case, I did not recognize AP(like OMAP) can
> have detection capability. :) So, although I did not think about at that time,
> I also think we should re-consider this case for now.
>
> As I look around your patch quickly, the functions is very similar with ours.
> (even detection of left/right eye)
> So, the problem is there are two ways to proceed "recognition".
> - Does it handle at the level of IOCTLs? or CIDs?

The patch introduces two IOCTL to retrieve object detection result.
I think it should be handled by IOCTL. The interface is a little complex,
so it is not easy to handle it by CIDs, IMO.

>
> If the detection algorithm is proceeded at the level of HW block,
> it's right the platform driver provide APIs as IOCTLs(as you're FDIF patches).
> On the other hand, if it is proceeded at the sensor(subdevice) level,
> I think it's more right to control using CIDs.

Certainly, some generic CIDs for object detection will be introduced later and
will be handled in the object detection(the current fdif generic
driver, patch 6/7)
generic driver.

> We need the solution including those two cases.
> And the name - object detection? object recognition?

I think object detection is better. For example, face detection is very
different with face recognition, isn't it?

thanks,
--
Ming Lei

>
> So, do you have any good ideas?
>
> ps. There can be another not-matched HW block level issues.
> But, the problem which I can check is just above issue for now.
>
>
> Thanks,
> Heungjun Kim
>
>
>>
>> thanks,
>> --
>> Ming Lei
>>
>> >
>> > Thank you.
>> >
>> > Regards,
>> > Heungjun Kim
>> >
>> >
>> >> -----Original Message-----
>> >> From: linux-media-owner at vger.kernel.org [mailto:linux-media-
>> >> owner at vger.kernel.org] On Behalf Of Ming Lei
>> >> Sent: Monday, December 12, 2011 6:50 PM
>> >> To: Sylwester Nawrocki
>> >> Cc: linux-omap at vger.kernel.org; linux-arm-kernel at lists.infradead.org;
> linux-
>> >> kernel at vger.kernel.org; linux-media at vger.kernel.org
>> >> Subject: Re: [RFC PATCH v1 6/7] media: video: introduce face detection
>> driver
>> >> module
>> >>
>> >> Hi,
>> >>
>> >> On Mon, Dec 12, 2011 at 1:43 AM, Sylwester Nawrocki <snjw23@gmail.com>
>> >>
>> >> >> For OMAP4 FD, it is not needed to include FD into MC framework since a
>> >> >> intermediate buffer is always required. If your HW doesn't belong to
> this
>> >> >> case, what is the output of your HW FD in the link? Also sounds FD
>> results
>> >> >> may not be needed at all for use space application in the case.
>> >> >
>> >> > The result data is similar to OMAP4 one, plus a few other attributes.
>> >> > User buffers may be filled by other than FD device driver.
>> >>
>> >> OK.
>> >>
>> >>
>> >> >> Could you provide some practical use cases about these?
>> >> >
>> >> > As above, and any device with a camera that controls something and makes
>> >> > decision according to presence of human face in his view.
>> >>
>> >> Sounds a reasonable case, :-)
>> >>
>> >>
>> >> >> If FD result is associated with a frame, how can user space get the
> frame
>> >> seq
>> >> >> if no v4l2 buffer is involved? Without a frame sequence, it is a bit
>> >> >> difficult to retrieve FD results from user space.
>> >> >
>> >> > If you pass image data in memory buffers from user space, yes, it could
> be
>> >> > impossible.
>> >>
>> >> It is easy to get the frame sequence from v4l2_buffer for the case too, :-)
>> >>
>> >> >
>> >> > Not really, still v4l2_buffer may be used by other (sub)driver within
> same
>> >> video
>> >> > processing pipeline.
>> >>
>> >> OK.
>> >>
>> >> A related question: how can we make one application to support the two
> kinds
>> > of
>> >> devices(input from user space data as OMAP4, input from SoC bus as Samsung)
>> >> at the same time? Maybe some capability info is to be exported to user
> space?
>> >> or other suggestions?
>> >>
>> >> And will your Samsung FD HW support to detect faces from memory? or just
>> only
>> >> detect from SoC bus?
>> >>
>> >>
>> >> > It will be included in the FD result... or in a dedicated v4l2 event data
>> >> structure.
>> >> > More important, at the end of the day, we'll be getting buffers with
> image
>> >> data
>> >> > at some stage of a video pipeline, which would contain same frame
>> identifier
>> >> > (I think we can ignore v4l2_buffer.field for FD purpose).
>> >>
>> >> OK, I will associate FD result with frame identifier, and not invent a
>> >> dedicated v4l2 event for query frame seq now until a specific requirement
>> >> for it is proposed.
>> >>
>> >> I will convert/integrate recent discussions into patches of v2 for further
>> >> review, and sub device support will be provided. But before starting to do
>> it,
>> >> I am still not clear how to integrate FD into MC framework. I understand FD
>> >> sub device is only a media entity, so how can FD sub device find the media
>> >> device(struct media_device)? ?or just needn't to care about it now?
>> >>
>> >>
>> >> thanks,
>> >> --
>> >> Ming Lei
>> >> --
>> >> To unsubscribe from this list: send the line "unsubscribe linux-media" in
>> >> the body of a message to majordomo at vger.kernel.org
>> >> More majordomo info at ?http://vger.kernel.org/majordomo-info.html
>> >
>> > --
>> > To unsubscribe from this list: send the line "unsubscribe linux-omap" in
>> > the body of a message to majordomo at vger.kernel.org
>> > More majordomo info at ?http://vger.kernel.org/majordomo-info.html
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo at vger.kernel.org
> More majordomo info at ?http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-02 15:02   ` Ming Lei
  (?)
  (?)
@ 2011-12-14 15:34   ` Sakari Ailus
  2011-12-14 15:57     ` Ming Lei
  -1 siblings, 1 reply; 91+ messages in thread
From: Sakari Ailus @ 2011-12-14 15:34 UTC (permalink / raw)
  To: Ming Lei; +Cc: Sylwester Nawrocki, linux-media

Hi Ming,

Thanks for the patchset.

(Dropped a lot of folks from cc --- I doubt Alan Cox or Greg
Kroah-Hartman have immediate interest towards this.)

On Fri, Dec 02, 2011 at 11:02:50PM +0800, Ming Lei wrote:
> This patch introduces two new IOCTLs and related data
> structure defination which will be used by the coming
> face detection video device.
> 
> The two IOCTLs and related data structure are used by
> user space application to retrieve the results of face
> detection. They can be called after one v4l2_buffer
> has been ioctl(VIDIOC_DQBUF) and before it will be
> ioctl(VIDIOC_QBUF).
> 
> The utility fdif[1] is useing the two IOCTLs to find
> faces deteced in raw images or video streams.
> 
> [1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif
> 
> Signed-off-by: Ming Lei <ming.lei@canonical.com>
> ---
>  drivers/media/video/v4l2-ioctl.c |   38 ++++++++++++++++++++
>  include/linux/videodev2.h        |   70 ++++++++++++++++++++++++++++++++++++++
>  include/media/v4l2-ioctl.h       |    6 +++
>  3 files changed, 114 insertions(+), 0 deletions(-)
> 
> diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
> index e1da8fc..fc6266f 100644
> --- a/drivers/media/video/v4l2-ioctl.c
> +++ b/drivers/media/video/v4l2-ioctl.c
> @@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
>  		dbgarg(cmd, "index=%d", b->index);
>  		break;
>  	}
> +	case VIDIOC_G_FD_RESULT:
> +	{
> +		struct v4l2_fd_result *fr = arg;
> +
> +		if (!ops->vidioc_g_fd_result)
> +			break;
> +
> +		ret = ops->vidioc_g_fd_result(file, fh, fr);
> +
> +		dbgarg(cmd, "index=%d", fr->buf_index);
> +		break;
> +	}
> +	case VIDIOC_G_FD_COUNT:
> +	{
> +		struct v4l2_fd_count *fc = arg;
> +
> +		if (!ops->vidioc_g_fd_count)
> +			break;
> +
> +		ret = ops->vidioc_g_fd_count(file, fh, fc);
> +
> +		dbgarg(cmd, "index=%d", fc->buf_index);
> +		break;
> +	}

The patch description tells these ioctls may be called between... what? I'd
think such information could be better provided as events.

How is face detection enabled or disabled?

Could you detect other objects than faces?

Would events be large enough to deliver you the necessary data? We could
also consider delivering the information as a data structure on a separate
plane.

>  	default:
>  		if (!ops->vidioc_default)
>  			break;
> @@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
>  		}
>  		break;
>  	}
> +
> +	case VIDIOC_G_FD_RESULT: {
> +		struct v4l2_fd_result *fr = parg;
> +
> +		if (fr->face_cnt != 0) {
> +			*user_ptr = (void __user *)fr->fd;
> +			*kernel_ptr = (void *)&fr->fd;
> +			*array_size = sizeof(struct v4l2_fd_detection)
> +				    * fr->face_cnt;
> +			ret = 1;
> +		}
> +		break;
> +
> +	}
>  	}
>  
>  	return ret;
> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
> index 4b752d5..073eb4d 100644
> --- a/include/linux/videodev2.h
> +++ b/include/linux/videodev2.h
> @@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
>  	__u32			reserved[8];
>  };
>  
> +/**
> + * struct v4l2_obj_detection
> + * @buf_index:	entry, index of v4l2_buffer for face detection
> + * @centerx:	return, position in x direction of detected object
> + * @centery:	return, position in y direction of detected object
> + * @angle:	return, angle of detected object
> + * 		0 deg ~ 359 deg, vertical is 0 deg, clockwise
> + * @sizex:	return, size in x direction of detected object
> + * @sizey:	return, size in y direction of detected object
> + * @confidence:	return, confidence level of detection result
> + * 		0: the heighest level, 9: the lowest level
> + * @reserved:	future extensions
> + */
> +struct v4l2_obj_detection {
> +	__u16		centerx;
> +	__u16		centery;
> +	__u16		angle;
> +	__u16		sizex;
> +	__u16		sizey;
> +	__u16		confidence;
> +	__u32		reserved[4];
> +};
> +
> +#define V4L2_FD_HAS_LEFT_EYE	0x1
> +#define V4L2_FD_HAS_RIGHT_EYE	0x2
> +#define V4L2_FD_HAS_MOUTH	0x4
> +#define V4L2_FD_HAS_FACE	0x8
> +
> +/**
> + * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
> + * @flag:	return, describe which objects are detected
> + * @left_eye:	return, left_eye position if detected
> + * @right_eye:	return, right_eye position if detected
> + * @mouth_eye:	return, mouth_eye position if detected
> + * @face:	return, face position if detected
> + */
> +struct v4l2_fd_detection {
> +	__u32	flag;
> +	struct v4l2_obj_detection	left_eye;
> +	struct v4l2_obj_detection	right_eye;
> +	struct v4l2_obj_detection	mouth;
> +	struct v4l2_obj_detection	face;
> +};
> +
> +/**
> + * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
> + * @buf_index:	entry, index of v4l2_buffer for face detection
> + * @face_cnt:	return, how many faces detected from the @buf_index
> + * @fd:		return, result of faces' detection
> + */
> +struct v4l2_fd_result {
> +	__u32	buf_index;
> +	__u32	face_cnt;
> +	__u32	reserved[6];
> +	struct v4l2_fd_detection *fd;

Aligning structure sizes to a power of two is considered to be a good
practice.

> +};
> +
> +/**
> + * struct v4l2_fd_count - VIDIOC_G_FD_COUNT argument
> + * @buf_index:	entry, index of v4l2_buffer for face detection
> + * @face_cnt:	return, how many faces detected from the @buf_index
> + */
> +struct v4l2_fd_count {
> +	__u32	buf_index;
> +	__u32	face_cnt;
> +	__u32	reserved[6];
> +};
> +
>  /*
>   *	I O C T L   C O D E S   F O R   V I D E O   D E V I C E S
>   *
> @@ -2254,6 +2322,8 @@ struct v4l2_create_buffers {
>     versions */
>  #define VIDIOC_CREATE_BUFS	_IOWR('V', 92, struct v4l2_create_buffers)
>  #define VIDIOC_PREPARE_BUF	_IOWR('V', 93, struct v4l2_buffer)
> +#define VIDIOC_G_FD_COUNT	_IOWR('V', 94, struct v4l2_fd_count)
> +#define VIDIOC_G_FD_RESULT	_IOWR('V', 95, struct v4l2_fd_result)
>  
>  /* Reminder: when adding new ioctls please add support for them to
>     drivers/media/video/v4l2-compat-ioctl32.c as well! */
> diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h
> index 4d1c74a..19f03b0 100644
> --- a/include/media/v4l2-ioctl.h
> +++ b/include/media/v4l2-ioctl.h
> @@ -270,6 +270,12 @@ struct v4l2_ioctl_ops {
>  	int (*vidioc_unsubscribe_event)(struct v4l2_fh *fh,
>  					struct v4l2_event_subscription *sub);
>  
> +	/* Face detect IOCTLs */
> +	int (*vidioc_g_fd_count) (struct file *file, void *fh,
> +					struct v4l2_fd_count *arg);
> +	int (*vidioc_g_fd_result) (struct file *file, void *fh,
> +					struct v4l2_fd_result *arg);
> +
>  	/* For other private ioctls */
>  	long (*vidioc_default)	       (struct file *file, void *fh,
>  					bool valid_prio, int cmd, void *arg);
> -- 
> 1.7.5.4
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-media" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

-- 
Sakari Ailus
e-mail: sakari.ailus@iki.fi	jabber/XMPP/Gmail: sailus@retiisi.org.uk

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-14 15:34   ` Sakari Ailus
@ 2011-12-14 15:57     ` Ming Lei
  2011-12-21 19:32       ` Sylwester Nawrocki
  0 siblings, 1 reply; 91+ messages in thread
From: Ming Lei @ 2011-12-14 15:57 UTC (permalink / raw)
  To: Sakari Ailus; +Cc: Sylwester Nawrocki, linux-media

Hi,

On Wed, Dec 14, 2011 at 11:34 PM, Sakari Ailus <sakari.ailus@iki.fi> wrote:

>> +     case VIDIOC_G_FD_RESULT:
>> +     {
>> +             struct v4l2_fd_result *fr = arg;
>> +
>> +             if (!ops->vidioc_g_fd_result)
>> +                     break;
>> +
>> +             ret = ops->vidioc_g_fd_result(file, fh, fr);
>> +
>> +             dbgarg(cmd, "index=%d", fr->buf_index);
>> +             break;
>> +     }
>> +     case VIDIOC_G_FD_COUNT:
>> +     {
>> +             struct v4l2_fd_count *fc = arg;
>> +
>> +             if (!ops->vidioc_g_fd_count)
>> +                     break;
>> +
>> +             ret = ops->vidioc_g_fd_count(file, fh, fc);
>> +
>> +             dbgarg(cmd, "index=%d", fc->buf_index);
>> +             break;
>> +     }
>
> The patch description tells these ioctls may be called between... what? I'd

In fact, these ioctls should be called after return from poll.

> think such information could be better provided as events.

Yes, I still think so, but the length of returned data is variant, also
event has the 64 byte length's limitation.

> How is face detection enabled or disabled?

Currently, streaming on will trigger detection enabling, and streaming off
will trigger detection disabling.

>
> Could you detect other objects than faces?

The -v2 has been extended to detect objects, not limited to faces.

> Would events be large enough to deliver you the necessary data? We could

Looks like event(64 bytes) is not large enough to deliver the data.

> also consider delivering the information as a data structure on a separate
> plane.

Could you let me know how to do it?

>> +/**
>> + * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
>> + * @buf_index:       entry, index of v4l2_buffer for face detection
>> + * @face_cnt:        return, how many faces detected from the @buf_index
>> + * @fd:              return, result of faces' detection
>> + */
>> +struct v4l2_fd_result {
>> +     __u32   buf_index;
>> +     __u32   face_cnt;
>> +     __u32   reserved[6];
>> +     struct v4l2_fd_detection *fd;
>
> Aligning structure sizes to a power of two is considered to be a good
> practice.

There are many changes on these structure in -v2.


thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-14 15:57     ` Ming Lei
@ 2011-12-21 19:32       ` Sylwester Nawrocki
  2011-12-26  2:00         ` Ming Lei
  0 siblings, 1 reply; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-21 19:32 UTC (permalink / raw)
  To: Ming Lei; +Cc: Sakari Ailus, linux-media

Hi Ming,

On 12/14/2011 04:57 PM, Ming Lei wrote:
> Hi,
> 
> On Wed, Dec 14, 2011 at 11:34 PM, Sakari Ailus <sakari.ailus@iki.fi> wrote:
> 
>>> +     case VIDIOC_G_FD_RESULT:
>>> +     {
>>> +             struct v4l2_fd_result *fr = arg;
>>> +
>>> +             if (!ops->vidioc_g_fd_result)
>>> +                     break;
>>> +
>>> +             ret = ops->vidioc_g_fd_result(file, fh, fr);
>>> +
>>> +             dbgarg(cmd, "index=%d", fr->buf_index);
>>> +             break;
>>> +     }
>>> +     case VIDIOC_G_FD_COUNT:
>>> +     {
>>> +             struct v4l2_fd_count *fc = arg;
>>> +
>>> +             if (!ops->vidioc_g_fd_count)
>>> +                     break;
>>> +
>>> +             ret = ops->vidioc_g_fd_count(file, fh, fc);
>>> +
>>> +             dbgarg(cmd, "index=%d", fc->buf_index);
>>> +             break;
>>> +     }
>>
>> The patch description tells these ioctls may be called between... what? I'd
> 
> In fact, these ioctls should be called after return from poll.
> 
>> think such information could be better provided as events.
> 
> Yes, I still think so, but the length of returned data is variant, also
> event has the 64 byte length's limitation.

Right, I'm afraid it's not even enough for single object description.

I have been thinking about adding variable payload support to v4l2 event API.

>> How is face detection enabled or disabled?
> 
> Currently, streaming on will trigger detection enabling, and streaming off
> will trigger detection disabling.

We would need to develop a boolean control for this I think, this seems one of
the basic features for the configuration interface.

>> Could you detect other objects than faces?
> 
> The -v2 has been extended to detect objects, not limited to faces.
> 
>> Would events be large enough to deliver you the necessary data? We could
> 
> Looks like event(64 bytes) is not large enough to deliver the data.
> 
>> also consider delivering the information as a data structure on a separate
>> plane.
> 
> Could you let me know how to do it?

You would have to use multi-planar interface for that, which would introduce
additional complexity at user interface. Moreover variable plane count is not
supported in vb2. Relatively significant effort is required to add this IMHO.

-- 

Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-21 19:32       ` Sylwester Nawrocki
@ 2011-12-26  2:00         ` Ming Lei
  2011-12-27 20:53           ` Sylwester Nawrocki
  0 siblings, 1 reply; 91+ messages in thread
From: Ming Lei @ 2011-12-26  2:00 UTC (permalink / raw)
  To: Sylwester Nawrocki; +Cc: Sakari Ailus, linux-media

Hi,

On Thu, Dec 22, 2011 at 3:32 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:

>>> How is face detection enabled or disabled?
>>
>> Currently, streaming on will trigger detection enabling, and streaming off
>> will trigger detection disabling.
>
> We would need to develop a boolean control for this I think, this seems one of
> the basic features for the configuration interface.

Yes, it is another way to do it, but considered that for the current two use
cases(detect objects on user space image or video, detect objects on
video stream from internal SoC bus), it is implicit that the video device
should have stream capability, so I think it is still OK to do it via
streaming on
and streaming off interface.

>>
>> Could you let me know how to do it?
>
> You would have to use multi-planar interface for that, which would introduce
> additional complexity at user interface. Moreover variable plane count is not
> supported in vb2. Relatively significant effort is required to add this IMHO.

So the the introduced two IOCTLs are good to do it, right?

Sylwester, could you help to review the v2 patches if you are available?

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-26  2:00         ` Ming Lei
@ 2011-12-27 20:53           ` Sylwester Nawrocki
  0 siblings, 0 replies; 91+ messages in thread
From: Sylwester Nawrocki @ 2011-12-27 20:53 UTC (permalink / raw)
  To: Ming Lei; +Cc: Sakari Ailus, linux-media

Hi Ming,

On 12/26/2011 03:00 AM, Ming Lei wrote:
> On Thu, Dec 22, 2011 at 3:32 AM, Sylwester Nawrocki <snjw23@gmail.com> wrote:
>>>> How is face detection enabled or disabled?
>>>
>>> Currently, streaming on will trigger detection enabling, and streaming
>>> off will trigger detection disabling.
>>
>> We would need to develop a boolean control for this I think, this seems 
>> one of the basic features for the configuration interface.
> 
> Yes, it is another way to do it, but considered that for the current two
> use cases(detect objects on user space image or video, detect objects on
> video stream from internal SoC bus), it is implicit that the video device
> should have stream capability, so I think it is still OK to do it via
> streaming on and streaming off interface.

IMHO for drivers that support FD and don't expose an FD start control
applications could assume that VIDIOC_STREAMON/VIDIOC_STREAMOFF also
enables/disables face detection.
In other words if an FD enable control is present face detection would
be disabled until an FD start control is set.

We will also need a FD status control.

>>> Could you let me know how to do it?
>>
>> You would have to use multi-planar interface for that, which would 
>> introduce additional complexity at user interface. Moreover variable plane
>> count is not supported in vb2. Relatively significant effort is required 
>> to add this IMHO.
> 
> So the the introduced two IOCTLs are good to do it, right?

Yes, and no :-)

Since we have a third case to consider:
 1) camera sensor with an embedded ISP that supports face detection.

Another two are:
 2) image input from memory only (like OMAP4 FDIF),
 3) image input from memory OR from external image sensor.

In case 1) we will eventually have a DMA that captures frames to memory.
It's different from 2) and 3) that registers containing result are accessed
through I2C. Then it may be desirable to retrieve detection result data
separately for each object, due to bus access latencies, rather reading all
data in single ioctl.

> Sylwester, could you help to review the v2 patches if you are available?

Yes, I'll try to find some time for it tomorrow.

I would have a general note - I don't really think we should bother with
generic FD kernel module now. We have already generic building blocks in V4L
and hardware is so different that it's seems inappropriate to try to generalize
the implementation before we have clear user interface semantics settled.

-- 
Regards,
Sylwester

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-06 15:45                     ` Ming Lei
@ 2011-12-06 20:59                       ` Arnd Bergmann
  -1 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-06 20:59 UTC (permalink / raw)
  To: linux-arm-kernel
  Cc: Ming Lei, Tony Lindgren, Greg KH, linux-kernel,
	Mauro Carvalho Chehab, Sylwester Nawrocki, Alan Cox, linux-omap

On Tuesday 06 December 2011 23:45:27 Ming Lei wrote:
> >
> > and it is not difficult to implement it in a generic way so that new
> > array parameters can be supported as 64/32 compatible.
> 
> How about the below patch to support 64/32 compatible array parameter?

Looks technically correct to me, if you replace the 'return -ENOMEM' with
'ret = -ENOMEM;\n goto out;'.

The video_usercopy function already has multiple micro-optimizations
(on-stack buffers, cmd_input_size) that make it more complex than
I'd hope, but your addition seems ok to me.

Let's see what Mauro thinks.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-06 20:59                       ` Arnd Bergmann
  0 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-06 20:59 UTC (permalink / raw)
  To: linux-arm-kernel

On Tuesday 06 December 2011 23:45:27 Ming Lei wrote:
> >
> > and it is not difficult to implement it in a generic way so that new
> > array parameters can be supported as 64/32 compatible.
> 
> How about the below patch to support 64/32 compatible array parameter?

Looks technically correct to me, if you replace the 'return -ENOMEM' with
'ret = -ENOMEM;\n goto out;'.

The video_usercopy function already has multiple micro-optimizations
(on-stack buffers, cmd_input_size) that make it more complex than
I'd hope, but your addition seems ok to me.

Let's see what Mauro thinks.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-06 14:52                   ` Ming Lei
@ 2011-12-06 15:45                     ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06 15:45 UTC (permalink / raw)
  To: Arnd Bergmann
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

Hi,

On Tue, Dec 6, 2011 at 10:52 PM, Ming Lei <ming.lei@canonical.com> wrote:
> On Tue, Dec 6, 2011 at 10:41 PM, Arnd Bergmann <arnd@arndb.de> wrote:
>> 3. extending video_usercopy in some way to make this work, preferably
>>   in a generic way.
>
> Maybe this one is a good choice, and I think that it is worthy to
> support the below kind of array parameter:
>
> struct v4l2_fd_result {
>        __u32   buf_index;
>        __u32   face_cnt;
>        __u32   reserved[6];
>        struct v4l2_fd_detection fd[];
> };
>
> and it is not difficult to implement it in a generic way so that new
> array parameters can be supported as 64/32 compatible.

How about the blow patch to support 64/32 compatible array parameter?

diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
index e1da8fc..72c81f7 100644
--- a/drivers/media/video/v4l2-ioctl.c
+++ b/drivers/media/video/v4l2-ioctl.c
@@ -2239,6 +2239,26 @@ static int check_array_args(unsigned int cmd,
void *parg, size_t *array_size,
 	return ret;
 }

+static int is_64_32_array_args(unsigned int cmd, void *parg, int *extra_len)
+{
+	int ret = 0;
+
+	switch (cmd) {
+	case VIDIOC_G_FD_RESULT: {
+		struct v4l2_fd_result *fr = parg;
+
+		*extra_len = fr->faces_cnt *
+			sizeof(struct v4l2_fd_detection);
+		ret = 1;
+		break;
+	}
+	default:
+		break;
+	}
+
+	return ret;
+}
+
 long
 video_usercopy(struct file *file, unsigned int cmd, unsigned long arg,
 	       v4l2_kioctl func)
@@ -2251,6 +2271,7 @@ video_usercopy(struct file *file, unsigned int
cmd, unsigned long arg,
 	size_t  array_size = 0;
 	void __user *user_ptr = NULL;
 	void	**kernel_ptr = NULL;
+	int	extra = 0;

 	/*  Copy arguments into temp kernel buffer  */
 	if (_IOC_DIR(cmd) != _IOC_NONE) {
@@ -2280,9 +2301,29 @@ video_usercopy(struct file *file, unsigned int
cmd, unsigned long arg,
 		}
 	}

-	err = check_array_args(cmd, parg, &array_size, &user_ptr, &kernel_ptr);
+	if (is_64_32_array_args(cmd, parg, &extra)) {
+		int size;
+		void *old_mbuf;
+
+		err = 0;
+		if (!extra)
+			goto out_array_args;
+		old_mbuf = mbuf;
+		size = extra + _IOC_SIZE(cmd);
+		mbuf = kmalloc(size, GFP_KERNEL);
+		if (NULL == mbuf)
+			return -ENOMEM;
+		memcpy(mbuf, parg, _IOC_SIZE(cmd));
+		parg = mbuf;
+		kfree(old_mbuf);
+	} else {
+		err = check_array_args(cmd, parg, &array_size,
+				&user_ptr, &kernel_ptr);
+	}
+
 	if (err < 0)
 		goto out;
+out_array_args:
 	has_array_args = err;

 	if (has_array_args) {
@@ -2321,7 +2362,7 @@ out_array_args:
 	switch (_IOC_DIR(cmd)) {
 	case _IOC_READ:
 	case (_IOC_WRITE | _IOC_READ):
-		if (copy_to_user((void __user *)arg, parg, _IOC_SIZE(cmd)))
+		if (copy_to_user((void __user *)arg, parg, _IOC_SIZE(cmd) + extra))
 			err = -EFAULT;
 		break;
 	}



thanks,
--
Ming Lei

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-06 15:45                     ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06 15:45 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Tue, Dec 6, 2011 at 10:52 PM, Ming Lei <ming.lei@canonical.com> wrote:
> On Tue, Dec 6, 2011 at 10:41 PM, Arnd Bergmann <arnd@arndb.de> wrote:
>> 3. extending video_usercopy in some way to make this work, preferably
>> ? in a generic way.
>
> Maybe this one is a good choice, and I think that it is worthy to
> support the below kind of array parameter:
>
> struct v4l2_fd_result {
> ? ? ? ?__u32 ? buf_index;
> ? ? ? ?__u32 ? face_cnt;
> ? ? ? ?__u32 ? reserved[6];
> ? ? ? ?struct v4l2_fd_detection fd[];
> };
>
> and it is not difficult to implement it in a generic way so that new
> array parameters can be supported as 64/32 compatible.

How about the blow patch to support 64/32 compatible array parameter?

diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
index e1da8fc..72c81f7 100644
--- a/drivers/media/video/v4l2-ioctl.c
+++ b/drivers/media/video/v4l2-ioctl.c
@@ -2239,6 +2239,26 @@ static int check_array_args(unsigned int cmd,
void *parg, size_t *array_size,
 	return ret;
 }

+static int is_64_32_array_args(unsigned int cmd, void *parg, int *extra_len)
+{
+	int ret = 0;
+
+	switch (cmd) {
+	case VIDIOC_G_FD_RESULT: {
+		struct v4l2_fd_result *fr = parg;
+
+		*extra_len = fr->faces_cnt *
+			sizeof(struct v4l2_fd_detection);
+		ret = 1;
+		break;
+	}
+	default:
+		break;
+	}
+
+	return ret;
+}
+
 long
 video_usercopy(struct file *file, unsigned int cmd, unsigned long arg,
 	       v4l2_kioctl func)
@@ -2251,6 +2271,7 @@ video_usercopy(struct file *file, unsigned int
cmd, unsigned long arg,
 	size_t  array_size = 0;
 	void __user *user_ptr = NULL;
 	void	**kernel_ptr = NULL;
+	int	extra = 0;

 	/*  Copy arguments into temp kernel buffer  */
 	if (_IOC_DIR(cmd) != _IOC_NONE) {
@@ -2280,9 +2301,29 @@ video_usercopy(struct file *file, unsigned int
cmd, unsigned long arg,
 		}
 	}

-	err = check_array_args(cmd, parg, &array_size, &user_ptr, &kernel_ptr);
+	if (is_64_32_array_args(cmd, parg, &extra)) {
+		int size;
+		void *old_mbuf;
+
+		err = 0;
+		if (!extra)
+			goto out_array_args;
+		old_mbuf = mbuf;
+		size = extra + _IOC_SIZE(cmd);
+		mbuf = kmalloc(size, GFP_KERNEL);
+		if (NULL == mbuf)
+			return -ENOMEM;
+		memcpy(mbuf, parg, _IOC_SIZE(cmd));
+		parg = mbuf;
+		kfree(old_mbuf);
+	} else {
+		err = check_array_args(cmd, parg, &array_size,
+				&user_ptr, &kernel_ptr);
+	}
+
 	if (err < 0)
 		goto out;
+out_array_args:
 	has_array_args = err;

 	if (has_array_args) {
@@ -2321,7 +2362,7 @@ out_array_args:
 	switch (_IOC_DIR(cmd)) {
 	case _IOC_READ:
 	case (_IOC_WRITE | _IOC_READ):
-		if (copy_to_user((void __user *)arg, parg, _IOC_SIZE(cmd)))
+		if (copy_to_user((void __user *)arg, parg, _IOC_SIZE(cmd) + extra))
 			err = -EFAULT;
 		break;
 	}



thanks,
--
Ming Lei

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-06 14:41                 ` Arnd Bergmann
@ 2011-12-06 14:52                   ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06 14:52 UTC (permalink / raw)
  To: Arnd Bergmann
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

On Tue, Dec 6, 2011 at 10:41 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Tuesday 06 December 2011, Ming Lei wrote:
>> > Using an array added to the end of the v4l2_fd_result structure
>> > rather than a pointer would really make this easier IMHO.
>>
>> I have tried to do this, but video_usercopy needs a few changes
>> to handle array args if no indirect pointer is passed to kernel.
>
> Ah, I see. Or you would have to encode the array size into the
> ioctl command, which is also ugly in a different way.
>
>> I am not sure if media guys are happy to accept the changes, :-)
>
> Maybe Mauro can comment on which solution he prefers then, given
> the choice between:
>
> 1. adding another handler in drivers/media/video/v4l2-compat-ioctl32.c
>
> 2. passing a pointer that is casted to __u64 in user space an back
>   in the kernel
>
> 3. extending video_usercopy in some way to make this work, preferably
>   in a generic way.

Maybe this one is a good choice, and I think that it is worthy to
support the below kind of array parameter:

struct v4l2_fd_result {
	__u32   buf_index;
	__u32   face_cnt;
	__u32   reserved[6];
	struct v4l2_fd_detection fd[];
};

and it is not difficult to implement it in a generic way so that new
array parameters can be supported as 64/32 compatible.

> 4. using a variable command number like
>   #define VIDIOC_G_FD_RESULT(num)      _IOC(_IOC_READ|_IOC_WRITE,'V', 95, \
>                sizeof(struct v4l2_fd_result) + (num) * sizeof(struct v4l2_fd_detection)
>
> 5. requiring the interface to be simplified to return only a single
>   struct v4l2_fd_detection at a time

Maybe this one is not user friendly since other v4l2 interfaces provide
array parameters way, :-)

> I agree that none of these are nice. My preferred option would be last one,
> but I don't know how performance critical the interface is or if it would
> cause any races that you want to avoid.


thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-06 14:52                   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06 14:52 UTC (permalink / raw)
  To: linux-arm-kernel

On Tue, Dec 6, 2011 at 10:41 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Tuesday 06 December 2011, Ming Lei wrote:
>> > Using an array added to the end of the v4l2_fd_result structure
>> > rather than a pointer would really make this easier IMHO.
>>
>> I have tried to do this, but video_usercopy needs a few changes
>> to handle array args if no indirect pointer is passed to kernel.
>
> Ah, I see. Or you would have to encode the array size into the
> ioctl command, which is also ugly in a different way.
>
>> I am not sure if media guys are happy to accept the changes, :-)
>
> Maybe Mauro can comment on which solution he prefers then, given
> the choice between:
>
> 1. adding another handler in drivers/media/video/v4l2-compat-ioctl32.c
>
> 2. passing a pointer that is casted to __u64 in user space an back
> ? in the kernel
>
> 3. extending video_usercopy in some way to make this work, preferably
> ? in a generic way.

Maybe this one is a good choice, and I think that it is worthy to
support the below kind of array parameter:

struct v4l2_fd_result {
	__u32   buf_index;
	__u32   face_cnt;
	__u32   reserved[6];
	struct v4l2_fd_detection fd[];
};

and it is not difficult to implement it in a generic way so that new
array parameters can be supported as 64/32 compatible.

> 4. using a variable command number like
> ? #define VIDIOC_G_FD_RESULT(num) ? ? ?_IOC(_IOC_READ|_IOC_WRITE,'V', 95, \
> ? ? ? ? ? ? ? ?sizeof(struct v4l2_fd_result) + (num) * sizeof(struct v4l2_fd_detection)
>
> 5. requiring the interface to be simplified to return only a single
> ? struct v4l2_fd_detection at a time

Maybe this one is not user friendly since other v4l2 interfaces provide
array parameters way, :-)

> I agree that none of these are nice. My preferred option would be last one,
> but I don't know how performance critical the interface is or if it would
> cause any races that you want to avoid.


thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-06 13:11               ` Ming Lei
@ 2011-12-06 14:41                 ` Arnd Bergmann
  -1 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-06 14:41 UTC (permalink / raw)
  To: Ming Lei
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

On Tuesday 06 December 2011, Ming Lei wrote:
> > Using an array added to the end of the v4l2_fd_result structure
> > rather than a pointer would really make this easier IMHO.
> 
> I have tried to do this, but video_usercopy needs a few changes
> to handle array args if no indirect pointer is passed to kernel.

Ah, I see. Or you would have to encode the array size into the
ioctl command, which is also ugly in a different way.

> I am not sure if media guys are happy to accept the changes, :-)

Maybe Mauro can comment on which solution he prefers then, given
the choice between:

1. adding another handler in drivers/media/video/v4l2-compat-ioctl32.c

2. passing a pointer that is casted to __u64 in user space an back
   in the kernel

3. extending video_usercopy in some way to make this work, preferably
   in a generic way.

4. using a variable command number like
   #define VIDIOC_G_FD_RESULT(num)	_IOC(_IOC_READ|_IOC_WRITE,'V', 95, \
		sizeof(struct v4l2_fd_result) + (num) * sizeof(struct v4l2_fd_detection)

5. requiring the interface to be simplified to return only a single
   struct v4l2_fd_detection at a time

I agree that none of these are nice. My preferred option would be last one,
but I don't know how performance critical the interface is or if it would
cause any races that you want to avoid.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-06 14:41                 ` Arnd Bergmann
  0 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-06 14:41 UTC (permalink / raw)
  To: linux-arm-kernel

On Tuesday 06 December 2011, Ming Lei wrote:
> > Using an array added to the end of the v4l2_fd_result structure
> > rather than a pointer would really make this easier IMHO.
> 
> I have tried to do this, but video_usercopy needs a few changes
> to handle array args if no indirect pointer is passed to kernel.

Ah, I see. Or you would have to encode the array size into the
ioctl command, which is also ugly in a different way.

> I am not sure if media guys are happy to accept the changes, :-)

Maybe Mauro can comment on which solution he prefers then, given
the choice between:

1. adding another handler in drivers/media/video/v4l2-compat-ioctl32.c

2. passing a pointer that is casted to __u64 in user space an back
   in the kernel

3. extending video_usercopy in some way to make this work, preferably
   in a generic way.

4. using a variable command number like
   #define VIDIOC_G_FD_RESULT(num)	_IOC(_IOC_READ|_IOC_WRITE,'V', 95, \
		sizeof(struct v4l2_fd_result) + (num) * sizeof(struct v4l2_fd_detection)

5. requiring the interface to be simplified to return only a single
   struct v4l2_fd_detection at a time

I agree that none of these are nice. My preferred option would be last one,
but I don't know how performance critical the interface is or if it would
cause any races that you want to avoid.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-06 12:55             ` Arnd Bergmann
@ 2011-12-06 13:11               ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06 13:11 UTC (permalink / raw)
  To: Arnd Bergmann
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

On Tue, Dec 6, 2011 at 8:55 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Tuesday 06 December 2011, Ming Lei wrote:
>> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
>> index 073eb4d..8aeaa1e 100644
>> --- a/include/linux/videodev2.h
>> +++ b/include/linux/videodev2.h
>> @@ -2214,7 +2214,12 @@ struct v4l2_fd_result {
>>         __u32   buf_index;
>>         __u32   face_cnt;
>>         __u32   reserved[6];
>> -       struct v4l2_fd_detection *fd;
>> +
>> +       /*make 64/32 compatible*/
>> +       union {
>> +               struct v4l2_fd_detection *fd;
>> +               __u64   dummy;
>> +       };
>>  };
>>
>
> That's not compatible, at least not on any big-endian architecture.
> If you want to have an indirect pointer, you have to cast it to the
> __u64 member in user space and back in kernel space.

Looks like it is a bit ugly.

>
> Using an array added to the end of the v4l2_fd_result structure
> rather than a pointer would really make this easier IMHO.

I have tried to do this, but video_usercopy needs a few changes
to handle array args if no indirect pointer is passed to kernel.

I am not sure if media guys are happy to accept the changes, :-)

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-06 13:11               ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06 13:11 UTC (permalink / raw)
  To: linux-arm-kernel

On Tue, Dec 6, 2011 at 8:55 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Tuesday 06 December 2011, Ming Lei wrote:
>> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
>> index 073eb4d..8aeaa1e 100644
>> --- a/include/linux/videodev2.h
>> +++ b/include/linux/videodev2.h
>> @@ -2214,7 +2214,12 @@ struct v4l2_fd_result {
>> ? ? ? ? __u32 ? buf_index;
>> ? ? ? ? __u32 ? face_cnt;
>> ? ? ? ? __u32 ? reserved[6];
>> - ? ? ? struct v4l2_fd_detection *fd;
>> +
>> + ? ? ? /*make 64/32 compatible*/
>> + ? ? ? union {
>> + ? ? ? ? ? ? ? struct v4l2_fd_detection *fd;
>> + ? ? ? ? ? ? ? __u64 ? dummy;
>> + ? ? ? };
>> ?};
>>
>
> That's not compatible, at least not on any big-endian architecture.
> If you want to have an indirect pointer, you have to cast it to the
> __u64 member in user space and back in kernel space.

Looks like it is a bit ugly.

>
> Using an array added to the end of the v4l2_fd_result structure
> rather than a pointer would really make this easier IMHO.

I have tried to do this, but video_usercopy needs a few changes
to handle array args if no indirect pointer is passed to kernel.

I am not sure if media guys are happy to accept the changes, :-)

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-06  6:30           ` Ming Lei
@ 2011-12-06 12:55             ` Arnd Bergmann
  -1 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-06 12:55 UTC (permalink / raw)
  To: Ming Lei
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

On Tuesday 06 December 2011, Ming Lei wrote:
> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
> index 073eb4d..8aeaa1e 100644
> --- a/include/linux/videodev2.h
> +++ b/include/linux/videodev2.h
> @@ -2214,7 +2214,12 @@ struct v4l2_fd_result {
>         __u32   buf_index;
>         __u32   face_cnt;
>         __u32   reserved[6];
> -       struct v4l2_fd_detection *fd;
> +
> +       /*make 64/32 compatible*/
> +       union {
> +               struct v4l2_fd_detection *fd;
> +               __u64   dummy;
> +       };
>  };
> 

That's not compatible, at least not on any big-endian architecture.
If you want to have an indirect pointer, you have to cast it to the
__u64 member in user space and back in kernel space.

Using an array added to the end of the v4l2_fd_result structure
rather than a pointer would really make this easier IMHO.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-06 12:55             ` Arnd Bergmann
  0 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-06 12:55 UTC (permalink / raw)
  To: linux-arm-kernel

On Tuesday 06 December 2011, Ming Lei wrote:
> diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
> index 073eb4d..8aeaa1e 100644
> --- a/include/linux/videodev2.h
> +++ b/include/linux/videodev2.h
> @@ -2214,7 +2214,12 @@ struct v4l2_fd_result {
>         __u32   buf_index;
>         __u32   face_cnt;
>         __u32   reserved[6];
> -       struct v4l2_fd_detection *fd;
> +
> +       /*make 64/32 compatible*/
> +       union {
> +               struct v4l2_fd_detection *fd;
> +               __u64   dummy;
> +       };
>  };
> 

That's not compatible, at least not on any big-endian architecture.
If you want to have an indirect pointer, you have to cast it to the
__u64 member in user space and back in kernel space.

Using an array added to the end of the v4l2_fd_result structure
rather than a pointer would really make this easier IMHO.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-05 14:37         ` Arnd Bergmann
@ 2011-12-06  6:30           ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06  6:30 UTC (permalink / raw)
  To: Arnd Bergmann
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

Hi,

On Mon, Dec 5, 2011 at 10:37 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Sunday 04 December 2011, Ming Lei wrote:
>> >
>> > This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
>> > user space will see an incompatible layout.
>>
>> I agree that this is not 32/64 bit safe, but I understand lib32 can handle
>> this correctly, otherwise many 32bit applications can't run on current
>> 64bit kernel
>> since many kernel structures used by user space contained pointer,
>> such as struct v4l2_buffer, struct v4l2_ext_controls in v4l2 ABI.
>
> The other ones you mentioned are handled in the kernel in
> drivers/media/video/v4l2-compat-ioctl32.c. For new ioctl commands,
> it's better to define the data structure in a compatible way
> so you do not need a wrapper like that.

OK, I opt to use __u64 to pass the user pointer, so how about the blow
changes on this patch?

diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
index 073eb4d..8aeaa1e 100644
--- a/include/linux/videodev2.h
+++ b/include/linux/videodev2.h
@@ -2214,7 +2214,12 @@ struct v4l2_fd_result {
 	__u32	buf_index;
 	__u32	face_cnt;
 	__u32	reserved[6];
-	struct v4l2_fd_detection *fd;
+
+	/*make 64/32 compatible*/
+	union {
+		struct v4l2_fd_detection *fd;
+		__u64	dummy;
+	};
 };

 /**

thanks,
--
Ming Lei

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-06  6:30           ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-06  6:30 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Mon, Dec 5, 2011 at 10:37 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Sunday 04 December 2011, Ming Lei wrote:
>> >
>> > This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
>> > user space will see an incompatible layout.
>>
>> I agree that this is not 32/64 bit safe, but I understand lib32 can handle
>> this correctly, otherwise many 32bit applications can't run on current
>> 64bit kernel
>> since many kernel structures used by user space contained pointer,
>> such as struct v4l2_buffer, struct v4l2_ext_controls in v4l2 ABI.
>
> The other ones you mentioned are handled in the kernel in
> drivers/media/video/v4l2-compat-ioctl32.c. For new ioctl commands,
> it's better to define the data structure in a compatible way
> so you do not need a wrapper like that.

OK, I opt to use __u64 to pass the user pointer, so how about the blow
changes on this patch?

diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
index 073eb4d..8aeaa1e 100644
--- a/include/linux/videodev2.h
+++ b/include/linux/videodev2.h
@@ -2214,7 +2214,12 @@ struct v4l2_fd_result {
 	__u32	buf_index;
 	__u32	face_cnt;
 	__u32	reserved[6];
-	struct v4l2_fd_detection *fd;
+
+	/*make 64/32 compatible*/
+	union {
+		struct v4l2_fd_detection *fd;
+		__u64	dummy;
+	};
 };

 /**

thanks,
--
Ming Lei

^ permalink raw reply related	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-04 11:18       ` Ming Lei
@ 2011-12-05 14:37         ` Arnd Bergmann
  -1 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-05 14:37 UTC (permalink / raw)
  To: Ming Lei
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

On Sunday 04 December 2011, Ming Lei wrote:
> >
> > This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
> > user space will see an incompatible layout.
> 
> I agree that this is not 32/64 bit safe, but I understand lib32 can handle
> this correctly, otherwise many 32bit applications can't run on current
> 64bit kernel
> since many kernel structures used by user space contained pointer,
> such as struct v4l2_buffer, struct v4l2_ext_controls in v4l2 ABI.

The other ones you mentioned are handled in the kernel in
drivers/media/video/v4l2-compat-ioctl32.c. For new ioctl commands,
it's better to define the data structure in a compatible way
so you do not need a wrapper like that.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-05 14:37         ` Arnd Bergmann
  0 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-05 14:37 UTC (permalink / raw)
  To: linux-arm-kernel

On Sunday 04 December 2011, Ming Lei wrote:
> >
> > This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
> > user space will see an incompatible layout.
> 
> I agree that this is not 32/64 bit safe, but I understand lib32 can handle
> this correctly, otherwise many 32bit applications can't run on current
> 64bit kernel
> since many kernel structures used by user space contained pointer,
> such as struct v4l2_buffer, struct v4l2_ext_controls in v4l2 ABI.

The other ones you mentioned are handled in the kernel in
drivers/media/video/v4l2-compat-ioctl32.c. For new ioctl commands,
it's better to define the data structure in a compatible way
so you do not need a wrapper like that.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-02 12:33     ` Arnd Bergmann
  (?)
@ 2011-12-04 11:18       ` Ming Lei
  -1 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-04 11:18 UTC (permalink / raw)
  To: Arnd Bergmann
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

Hi,

On Fri, Dec 2, 2011 at 8:33 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Friday 02 December 2011 17:12:56 Ming Lei wrote:
>> +/**
>> + * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
>> + * @buf_index: entry, index of v4l2_buffer for face detection
>> + * @face_cnt:  return, how many faces detected from the @buf_index
>> + * @fd:                return, result of faces' detection
>> + */
>> +struct v4l2_fd_result {
>> +       __u32   buf_index;
>> +       __u32   face_cnt;
>> +       __u32   reserved[6];
>> +       struct v4l2_fd_detection *fd;
>> +};
>
>
> This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
> user space will see an incompatible layout.

I agree that this is not 32/64 bit safe, but I understand lib32 can handle
this correctly, otherwise many 32bit applications can't run on current
64bit kernel
since many kernel structures used by user space contained pointer,
such as struct v4l2_buffer, struct v4l2_ext_controls in v4l2 ABI.

> One way to solve this is to remove the pointer and just start the array
> directly after the __u32 members. Alternatively, you can use a __u64
> to pass the pointer in an integer representation.

So I think this need not to be solved.

>
> A nicer interface from the data structure perspective would be to
> get rid of the array altogether and always return exactly one face.

I choose to return array to user space since v4l2 ioctl has provided
this kind of support, see video_usercopy().

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-04 11:18       ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-04 11:18 UTC (permalink / raw)
  To: Arnd Bergmann
  Cc: linux-arm-kernel, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

Hi,

On Fri, Dec 2, 2011 at 8:33 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Friday 02 December 2011 17:12:56 Ming Lei wrote:
>> +/**
>> + * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
>> + * @buf_index: entry, index of v4l2_buffer for face detection
>> + * @face_cnt:  return, how many faces detected from the @buf_index
>> + * @fd:                return, result of faces' detection
>> + */
>> +struct v4l2_fd_result {
>> +       __u32   buf_index;
>> +       __u32   face_cnt;
>> +       __u32   reserved[6];
>> +       struct v4l2_fd_detection *fd;
>> +};
>
>
> This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
> user space will see an incompatible layout.

I agree that this is not 32/64 bit safe, but I understand lib32 can handle
this correctly, otherwise many 32bit applications can't run on current
64bit kernel
since many kernel structures used by user space contained pointer,
such as struct v4l2_buffer, struct v4l2_ext_controls in v4l2 ABI.

> One way to solve this is to remove the pointer and just start the array
> directly after the __u32 members. Alternatively, you can use a __u64
> to pass the pointer in an integer representation.

So I think this need not to be solved.

>
> A nicer interface from the data structure perspective would be to
> get rid of the array altogether and always return exactly one face.

I choose to return array to user space since v4l2 ioctl has provided
this kind of support, see video_usercopy().

thanks,
--
Ming Lei
--
To unsubscribe from this list: send the line "unsubscribe linux-omap" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-04 11:18       ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-04 11:18 UTC (permalink / raw)
  To: linux-arm-kernel

Hi,

On Fri, Dec 2, 2011 at 8:33 PM, Arnd Bergmann <arnd@arndb.de> wrote:
> On Friday 02 December 2011 17:12:56 Ming Lei wrote:
>> +/**
>> + * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
>> + * @buf_index: entry, index of v4l2_buffer for face detection
>> + * @face_cnt: ?return, how many faces detected from the @buf_index
>> + * @fd: ? ? ? ? ? ? ? ?return, result of faces' detection
>> + */
>> +struct v4l2_fd_result {
>> + ? ? ? __u32 ? buf_index;
>> + ? ? ? __u32 ? face_cnt;
>> + ? ? ? __u32 ? reserved[6];
>> + ? ? ? struct v4l2_fd_detection *fd;
>> +};
>
>
> This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
> user space will see an incompatible layout.

I agree that this is not 32/64 bit safe, but I understand lib32 can handle
this correctly, otherwise many 32bit applications can't run on current
64bit kernel
since many kernel structures used by user space contained pointer,
such as struct v4l2_buffer, struct v4l2_ext_controls in v4l2 ABI.

> One way to solve this is to remove the pointer and just start the array
> directly after the __u32 members. Alternatively, you can use a __u64
> to pass the pointer in an integer representation.

So I think this need not to be solved.

>
> A nicer interface from the data structure perspective would be to
> get rid of the array altogether and always return exactly one face.

I choose to return array to user space since v4l2 ioctl has provided
this kind of support, see video_usercopy().

thanks,
--
Ming Lei

^ permalink raw reply	[flat|nested] 91+ messages in thread

* Re: [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-02  9:12   ` Ming Lei
@ 2011-12-02 12:33     ` Arnd Bergmann
  -1 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-02 12:33 UTC (permalink / raw)
  To: linux-arm-kernel
  Cc: Ming Lei, Mauro Carvalho Chehab, Tony Lindgren, Greg KH,
	linux-kernel, Sylwester Nawrocki, Alan Cox, linux-omap

On Friday 02 December 2011 17:12:56 Ming Lei wrote:
> +/**
> + * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
> + * @buf_index: entry, index of v4l2_buffer for face detection
> + * @face_cnt:  return, how many faces detected from the @buf_index
> + * @fd:                return, result of faces' detection
> + */
> +struct v4l2_fd_result {
> +       __u32   buf_index;
> +       __u32   face_cnt;
> +       __u32   reserved[6];
> +       struct v4l2_fd_detection *fd;
> +};

This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
user space will see an incompatible layout.

One way to solve this is to remove the pointer and just start the array
directly after the __u32 members. Alternatively, you can use a __u64
to pass the pointer in an integer representation.

A nicer interface from the data structure perspective would be to
get rid of the array altogether and always return exactly one face.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-02 12:33     ` Arnd Bergmann
  0 siblings, 0 replies; 91+ messages in thread
From: Arnd Bergmann @ 2011-12-02 12:33 UTC (permalink / raw)
  To: linux-arm-kernel

On Friday 02 December 2011 17:12:56 Ming Lei wrote:
> +/**
> + * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
> + * @buf_index: entry, index of v4l2_buffer for face detection
> + * @face_cnt:  return, how many faces detected from the @buf_index
> + * @fd:                return, result of faces' detection
> + */
> +struct v4l2_fd_result {
> +       __u32   buf_index;
> +       __u32   face_cnt;
> +       __u32   reserved[6];
> +       struct v4l2_fd_detection *fd;
> +};

This data structure is not 32/64 bit safe: running a 64 bit kernel with 32 bit
user space will see an incompatible layout.

One way to solve this is to remove the pointer and just start the array
directly after the __u32 members. Alternatively, you can use a __u64
to pass the pointer in an integer representation.

A nicer interface from the data structure perspective would be to
get rid of the array altogether and always return exactly one face.

	Arnd

^ permalink raw reply	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
  2011-12-02  9:12 [RFC PATCH v1 0/7] media&omap4: introduce face detection(FD) driver Ming Lei
@ 2011-12-02  9:12   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02  9:12 UTC (permalink / raw)
  To: Mauro Carvalho Chehab, Tony Lindgren
  Cc: Sylwester Nawrocki, Greg KH, Alan Cox, linux-omap,
	linux-arm-kernel, linux-kernel, Ming Lei

This patch introduces two new IOCTLs and related data
structure defination which will be used by the coming
face detection video device.

The two IOCTLs and related data structure are used by
user space application to retrieve the results of face
detection. They can be called after one v4l2_buffer
has been ioctl(VIDIOC_DQBUF) and before it will be
ioctl(VIDIOC_QBUF).

The utility fdif[1] is useing the two IOCTLs to find
faces deteced in raw images or video streams.

[1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/v4l2-ioctl.c |   38 ++++++++++++++++++++
 include/linux/videodev2.h        |   70 ++++++++++++++++++++++++++++++++++++++
 include/media/v4l2-ioctl.h       |    6 +++
 3 files changed, 114 insertions(+), 0 deletions(-)

diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
index e1da8fc..fc6266f 100644
--- a/drivers/media/video/v4l2-ioctl.c
+++ b/drivers/media/video/v4l2-ioctl.c
@@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
 		dbgarg(cmd, "index=%d", b->index);
 		break;
 	}
+	case VIDIOC_G_FD_RESULT:
+	{
+		struct v4l2_fd_result *fr = arg;
+
+		if (!ops->vidioc_g_fd_result)
+			break;
+
+		ret = ops->vidioc_g_fd_result(file, fh, fr);
+
+		dbgarg(cmd, "index=%d", fr->buf_index);
+		break;
+	}
+	case VIDIOC_G_FD_COUNT:
+	{
+		struct v4l2_fd_count *fc = arg;
+
+		if (!ops->vidioc_g_fd_count)
+			break;
+
+		ret = ops->vidioc_g_fd_count(file, fh, fc);
+
+		dbgarg(cmd, "index=%d", fc->buf_index);
+		break;
+	}
 	default:
 		if (!ops->vidioc_default)
 			break;
@@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
 		}
 		break;
 	}
+
+	case VIDIOC_G_FD_RESULT: {
+		struct v4l2_fd_result *fr = parg;
+
+		if (fr->face_cnt != 0) {
+			*user_ptr = (void __user *)fr->fd;
+			*kernel_ptr = (void *)&fr->fd;
+			*array_size = sizeof(struct v4l2_fd_detection)
+				    * fr->face_cnt;
+			ret = 1;
+		}
+		break;
+
+	}
 	}
 
 	return ret;
diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
index 4b752d5..073eb4d 100644
--- a/include/linux/videodev2.h
+++ b/include/linux/videodev2.h
@@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
 	__u32			reserved[8];
 };
 
+/**
+ * struct v4l2_obj_detection
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @centerx:	return, position in x direction of detected object
+ * @centery:	return, position in y direction of detected object
+ * @angle:	return, angle of detected object
+ * 		0 deg ~ 359 deg, vertical is 0 deg, clockwise
+ * @sizex:	return, size in x direction of detected object
+ * @sizey:	return, size in y direction of detected object
+ * @confidence:	return, confidence level of detection result
+ * 		0: the heighest level, 9: the lowest level
+ * @reserved:	future extensions
+ */
+struct v4l2_obj_detection {
+	__u16		centerx;
+	__u16		centery;
+	__u16		angle;
+	__u16		sizex;
+	__u16		sizey;
+	__u16		confidence;
+	__u32		reserved[4];
+};
+
+#define V4L2_FD_HAS_LEFT_EYE	0x1
+#define V4L2_FD_HAS_RIGHT_EYE	0x2
+#define V4L2_FD_HAS_MOUTH	0x4
+#define V4L2_FD_HAS_FACE	0x8
+
+/**
+ * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
+ * @flag:	return, describe which objects are detected
+ * @left_eye:	return, left_eye position if detected
+ * @right_eye:	return, right_eye position if detected
+ * @mouth_eye:	return, mouth_eye position if detected
+ * @face:	return, face position if detected
+ */
+struct v4l2_fd_detection {
+	__u32	flag;
+	struct v4l2_obj_detection	left_eye;
+	struct v4l2_obj_detection	right_eye;
+	struct v4l2_obj_detection	mouth;
+	struct v4l2_obj_detection	face;
+};
+
+/**
+ * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @face_cnt:	return, how many faces detected from the @buf_index
+ * @fd:		return, result of faces' detection
+ */
+struct v4l2_fd_result {
+	__u32	buf_index;
+	__u32	face_cnt;
+	__u32	reserved[6];
+	struct v4l2_fd_detection *fd;
+};
+
+/**
+ * struct v4l2_fd_count - VIDIOC_G_FD_COUNT argument
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @face_cnt:	return, how many faces detected from the @buf_index
+ */
+struct v4l2_fd_count {
+	__u32	buf_index;
+	__u32	face_cnt;
+	__u32	reserved[6];
+};
+
 /*
  *	I O C T L   C O D E S   F O R   V I D E O   D E V I C E S
  *
@@ -2254,6 +2322,8 @@ struct v4l2_create_buffers {
    versions */
 #define VIDIOC_CREATE_BUFS	_IOWR('V', 92, struct v4l2_create_buffers)
 #define VIDIOC_PREPARE_BUF	_IOWR('V', 93, struct v4l2_buffer)
+#define VIDIOC_G_FD_COUNT	_IOWR('V', 94, struct v4l2_fd_count)
+#define VIDIOC_G_FD_RESULT	_IOWR('V', 95, struct v4l2_fd_result)
 
 /* Reminder: when adding new ioctls please add support for them to
    drivers/media/video/v4l2-compat-ioctl32.c as well! */
diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h
index 4d1c74a..19f03b0 100644
--- a/include/media/v4l2-ioctl.h
+++ b/include/media/v4l2-ioctl.h
@@ -270,6 +270,12 @@ struct v4l2_ioctl_ops {
 	int (*vidioc_unsubscribe_event)(struct v4l2_fh *fh,
 					struct v4l2_event_subscription *sub);
 
+	/* Face detect IOCTLs */
+	int (*vidioc_g_fd_count) (struct file *file, void *fh,
+					struct v4l2_fd_count *arg);
+	int (*vidioc_g_fd_result) (struct file *file, void *fh,
+					struct v4l2_fd_result *arg);
+
 	/* For other private ioctls */
 	long (*vidioc_default)	       (struct file *file, void *fh,
 					bool valid_prio, int cmd, void *arg);
-- 
1.7.5.4


^ permalink raw reply related	[flat|nested] 91+ messages in thread

* [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection
@ 2011-12-02  9:12   ` Ming Lei
  0 siblings, 0 replies; 91+ messages in thread
From: Ming Lei @ 2011-12-02  9:12 UTC (permalink / raw)
  To: linux-arm-kernel

This patch introduces two new IOCTLs and related data
structure defination which will be used by the coming
face detection video device.

The two IOCTLs and related data structure are used by
user space application to retrieve the results of face
detection. They can be called after one v4l2_buffer
has been ioctl(VIDIOC_DQBUF) and before it will be
ioctl(VIDIOC_QBUF).

The utility fdif[1] is useing the two IOCTLs to find
faces deteced in raw images or video streams.

[1],http://kernel.ubuntu.com/git?p=ming/fdif.git;a=shortlog;h=refs/heads/v4l2-fdif

Signed-off-by: Ming Lei <ming.lei@canonical.com>
---
 drivers/media/video/v4l2-ioctl.c |   38 ++++++++++++++++++++
 include/linux/videodev2.h        |   70 ++++++++++++++++++++++++++++++++++++++
 include/media/v4l2-ioctl.h       |    6 +++
 3 files changed, 114 insertions(+), 0 deletions(-)

diff --git a/drivers/media/video/v4l2-ioctl.c b/drivers/media/video/v4l2-ioctl.c
index e1da8fc..fc6266f 100644
--- a/drivers/media/video/v4l2-ioctl.c
+++ b/drivers/media/video/v4l2-ioctl.c
@@ -2140,6 +2140,30 @@ static long __video_do_ioctl(struct file *file,
 		dbgarg(cmd, "index=%d", b->index);
 		break;
 	}
+	case VIDIOC_G_FD_RESULT:
+	{
+		struct v4l2_fd_result *fr = arg;
+
+		if (!ops->vidioc_g_fd_result)
+			break;
+
+		ret = ops->vidioc_g_fd_result(file, fh, fr);
+
+		dbgarg(cmd, "index=%d", fr->buf_index);
+		break;
+	}
+	case VIDIOC_G_FD_COUNT:
+	{
+		struct v4l2_fd_count *fc = arg;
+
+		if (!ops->vidioc_g_fd_count)
+			break;
+
+		ret = ops->vidioc_g_fd_count(file, fh, fc);
+
+		dbgarg(cmd, "index=%d", fc->buf_index);
+		break;
+	}
 	default:
 		if (!ops->vidioc_default)
 			break;
@@ -2234,6 +2258,20 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size,
 		}
 		break;
 	}
+
+	case VIDIOC_G_FD_RESULT: {
+		struct v4l2_fd_result *fr = parg;
+
+		if (fr->face_cnt != 0) {
+			*user_ptr = (void __user *)fr->fd;
+			*kernel_ptr = (void *)&fr->fd;
+			*array_size = sizeof(struct v4l2_fd_detection)
+				    * fr->face_cnt;
+			ret = 1;
+		}
+		break;
+
+	}
 	}
 
 	return ret;
diff --git a/include/linux/videodev2.h b/include/linux/videodev2.h
index 4b752d5..073eb4d 100644
--- a/include/linux/videodev2.h
+++ b/include/linux/videodev2.h
@@ -2160,6 +2160,74 @@ struct v4l2_create_buffers {
 	__u32			reserved[8];
 };
 
+/**
+ * struct v4l2_obj_detection
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @centerx:	return, position in x direction of detected object
+ * @centery:	return, position in y direction of detected object
+ * @angle:	return, angle of detected object
+ * 		0 deg ~ 359 deg, vertical is 0 deg, clockwise
+ * @sizex:	return, size in x direction of detected object
+ * @sizey:	return, size in y direction of detected object
+ * @confidence:	return, confidence level of detection result
+ * 		0: the heighest level, 9: the lowest level
+ * @reserved:	future extensions
+ */
+struct v4l2_obj_detection {
+	__u16		centerx;
+	__u16		centery;
+	__u16		angle;
+	__u16		sizex;
+	__u16		sizey;
+	__u16		confidence;
+	__u32		reserved[4];
+};
+
+#define V4L2_FD_HAS_LEFT_EYE	0x1
+#define V4L2_FD_HAS_RIGHT_EYE	0x2
+#define V4L2_FD_HAS_MOUTH	0x4
+#define V4L2_FD_HAS_FACE	0x8
+
+/**
+ * struct v4l2_fd_detection - VIDIOC_G_FD_RESULT argument
+ * @flag:	return, describe which objects are detected
+ * @left_eye:	return, left_eye position if detected
+ * @right_eye:	return, right_eye position if detected
+ * @mouth_eye:	return, mouth_eye position if detected
+ * @face:	return, face position if detected
+ */
+struct v4l2_fd_detection {
+	__u32	flag;
+	struct v4l2_obj_detection	left_eye;
+	struct v4l2_obj_detection	right_eye;
+	struct v4l2_obj_detection	mouth;
+	struct v4l2_obj_detection	face;
+};
+
+/**
+ * struct v4l2_fd_result - VIDIOC_G_FD_RESULT argument
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @face_cnt:	return, how many faces detected from the @buf_index
+ * @fd:		return, result of faces' detection
+ */
+struct v4l2_fd_result {
+	__u32	buf_index;
+	__u32	face_cnt;
+	__u32	reserved[6];
+	struct v4l2_fd_detection *fd;
+};
+
+/**
+ * struct v4l2_fd_count - VIDIOC_G_FD_COUNT argument
+ * @buf_index:	entry, index of v4l2_buffer for face detection
+ * @face_cnt:	return, how many faces detected from the @buf_index
+ */
+struct v4l2_fd_count {
+	__u32	buf_index;
+	__u32	face_cnt;
+	__u32	reserved[6];
+};
+
 /*
  *	I O C T L   C O D E S   F O R   V I D E O   D E V I C E S
  *
@@ -2254,6 +2322,8 @@ struct v4l2_create_buffers {
    versions */
 #define VIDIOC_CREATE_BUFS	_IOWR('V', 92, struct v4l2_create_buffers)
 #define VIDIOC_PREPARE_BUF	_IOWR('V', 93, struct v4l2_buffer)
+#define VIDIOC_G_FD_COUNT	_IOWR('V', 94, struct v4l2_fd_count)
+#define VIDIOC_G_FD_RESULT	_IOWR('V', 95, struct v4l2_fd_result)
 
 /* Reminder: when adding new ioctls please add support for them to
    drivers/media/video/v4l2-compat-ioctl32.c as well! */
diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h
index 4d1c74a..19f03b0 100644
--- a/include/media/v4l2-ioctl.h
+++ b/include/media/v4l2-ioctl.h
@@ -270,6 +270,12 @@ struct v4l2_ioctl_ops {
 	int (*vidioc_unsubscribe_event)(struct v4l2_fh *fh,
 					struct v4l2_event_subscription *sub);
 
+	/* Face detect IOCTLs */
+	int (*vidioc_g_fd_count) (struct file *file, void *fh,
+					struct v4l2_fd_count *arg);
+	int (*vidioc_g_fd_result) (struct file *file, void *fh,
+					struct v4l2_fd_result *arg);
+
 	/* For other private ioctls */
 	long (*vidioc_default)	       (struct file *file, void *fh,
 					bool valid_prio, int cmd, void *arg);
-- 
1.7.5.4

^ permalink raw reply related	[flat|nested] 91+ messages in thread

end of thread, other threads:[~2011-12-27 20:53 UTC | newest]

Thread overview: 91+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2011-12-02 15:02 [RFC PATCH v1 0/7] media&omap4: introduce face detection(FD) driver Ming Lei
2011-12-02 15:02 ` Ming Lei
2011-12-02 15:02 ` [RFC PATCH v1 1/7] omap4: introduce fdif(face detect module) hwmod Ming Lei
2011-12-02 15:02   ` Ming Lei
2011-12-02 15:02 ` [RFC PATCH v1 2/7] omap4: build fdif omap device from hwmod Ming Lei
2011-12-02 15:02   ` Ming Lei
2011-12-02 16:28   ` Aguirre, Sergio
2011-12-02 16:28     ` Aguirre, Sergio
2011-12-05  4:27     ` Ming Lei
2011-12-05  4:27       ` Ming Lei
2011-12-02 15:02 ` [RFC PATCH v1 3/7] media: videobuf2: move out of setting pgprot_noncached from vb2_mmap_pfn_range Ming Lei
2011-12-02 15:02   ` Ming Lei
2011-12-02 15:02   ` Ming Lei
2011-12-02 15:02 ` [RFC PATCH v1 4/7] media: videobuf2: introduce VIDEOBUF2_PAGE memops Ming Lei
2011-12-02 15:02   ` Ming Lei
2011-12-02 15:02 ` [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection Ming Lei
2011-12-02 15:02   ` Ming Lei
2011-12-05 22:15   ` Sylwester Nawrocki
2011-12-05 22:15     ` Sylwester Nawrocki
2011-12-08  3:42     ` Ming Lei
2011-12-08  3:42       ` Ming Lei
2011-12-08 22:27       ` Sylwester Nawrocki
2011-12-08 22:27         ` Sylwester Nawrocki
2011-12-09  4:34         ` Ming Lei
2011-12-09  4:34           ` Ming Lei
2011-12-11 17:27           ` Sylwester Nawrocki
2011-12-11 17:27             ` Sylwester Nawrocki
2011-12-14 15:34   ` Sakari Ailus
2011-12-14 15:57     ` Ming Lei
2011-12-21 19:32       ` Sylwester Nawrocki
2011-12-26  2:00         ` Ming Lei
2011-12-27 20:53           ` Sylwester Nawrocki
2011-12-02 15:02 ` [RFC PATCH v1 6/7] media: video: introduce face detection driver module Ming Lei
2011-12-02 15:02   ` Ming Lei
2011-12-05 21:55   ` Sylwester Nawrocki
2011-12-05 21:55     ` Sylwester Nawrocki
2011-12-06 14:07     ` Ming Lei
2011-12-06 14:07       ` Ming Lei
2011-12-06 22:01       ` Sylwester Nawrocki
2011-12-06 22:01         ` Sylwester Nawrocki
2011-12-06 22:39         ` Sylwester Nawrocki
2011-12-06 22:39           ` Sylwester Nawrocki
2011-12-07 13:40         ` Ming Lei
2011-12-07 13:40           ` Ming Lei
2011-12-08 23:25           ` Sylwester Nawrocki
2011-12-08 23:25             ` Sylwester Nawrocki
2011-12-09 15:10             ` Ming Lei
2011-12-09 15:10               ` Ming Lei
2011-12-09 15:10               ` Ming Lei
2011-12-11 17:43               ` Sylwester Nawrocki
2011-12-11 17:43                 ` Sylwester Nawrocki
2011-12-11 17:43                 ` Sylwester Nawrocki
2011-12-12  9:49                 ` Ming Lei
2011-12-12  9:49                   ` Ming Lei
2011-12-12 12:08                   ` HeungJun, Kim
2011-12-12 12:08                     ` HeungJun, Kim
2011-12-13  4:01                     ` Ming Lei
2011-12-13  4:01                       ` Ming Lei
2011-12-13  5:59                       ` HeungJun, Kim
2011-12-13  5:59                         ` HeungJun, Kim
2011-12-13  6:29                         ` Ming Lei
2011-12-13  6:29                           ` Ming Lei
2011-12-12 21:57                   ` Sylwester Nawrocki
2011-12-12 21:57                     ` Sylwester Nawrocki
2011-12-11 18:38   ` Sylwester Nawrocki
2011-12-11 18:38     ` Sylwester Nawrocki
2011-12-02 15:02 ` [RFC PATCH v1 7/7] media: video: introduce omap4 face detection module driver Ming Lei
2011-12-02 15:02   ` Ming Lei
  -- strict thread matches above, loose matches on Subject: below --
2011-12-02  9:12 [RFC PATCH v1 0/7] media&omap4: introduce face detection(FD) driver Ming Lei
2011-12-02  9:12 ` [RFC PATCH v1 5/7] media: v4l2: introduce two IOCTLs for face detection Ming Lei
2011-12-02  9:12   ` Ming Lei
2011-12-02 12:33   ` Arnd Bergmann
2011-12-02 12:33     ` Arnd Bergmann
2011-12-04 11:18     ` Ming Lei
2011-12-04 11:18       ` Ming Lei
2011-12-04 11:18       ` Ming Lei
2011-12-05 14:37       ` Arnd Bergmann
2011-12-05 14:37         ` Arnd Bergmann
2011-12-06  6:30         ` Ming Lei
2011-12-06  6:30           ` Ming Lei
2011-12-06 12:55           ` Arnd Bergmann
2011-12-06 12:55             ` Arnd Bergmann
2011-12-06 13:11             ` Ming Lei
2011-12-06 13:11               ` Ming Lei
2011-12-06 14:41               ` Arnd Bergmann
2011-12-06 14:41                 ` Arnd Bergmann
2011-12-06 14:52                 ` Ming Lei
2011-12-06 14:52                   ` Ming Lei
2011-12-06 15:45                   ` Ming Lei
2011-12-06 15:45                     ` Ming Lei
2011-12-06 20:59                     ` Arnd Bergmann
2011-12-06 20:59                       ` Arnd Bergmann

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.