i.MX Processors中的主题 Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Tue, 09 Jun 2020 12:46:47 GMT

Hi,

Currenty I am working on i.MX8M Mini platform with Camera module OV4689.

OmniVision | CMOS Image Sensor Manufacturer | Imaging Solutions

To add support for this driver, I used the driver from below opensource

linux-rk3328-box/ov4689.c at master · hanwckf/linux-rk3328-box · GitHub

In our Linux 4.14.98 MIPI driver added support for MEDIA_BUS_FMT_SBGGR10_1X10 like this:

drivers/media/platform/mxc/capture/mx6s_capture.c

static struct mx6s_fmt formats[] = {
{
.name = "UYVY-16",
.fourcc = V4L2_PIX_FMT_UYVY,
.pixelformat = V4L2_PIX_FMT_UYVY,
.mbus_code = MEDIA_BUS_FMT_UYVY8_2X8,
.bpp = 2,
}, {
.name = "YUYV-16",
.fourcc = V4L2_PIX_FMT_YUYV,
.pixelformat = V4L2_PIX_FMT_YUYV,
.mbus_code = MEDIA_BUS_FMT_YUYV8_2X8,
.bpp = 2,
}, {
.name = "YUV32 (X-Y-U-V)",
.fourcc = V4L2_PIX_FMT_YUV32,
.pixelformat = V4L2_PIX_FMT_YUV32,
.mbus_code = MEDIA_BUS_FMT_AYUV8_1X32,
.bpp = 4,
}, {
.name = "RAWRGB8 (SBGGR8)",
.fourcc = V4L2_PIX_FMT_SBGGR8,
.pixelformat = V4L2_PIX_FMT_SBGGR8,
.mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8,
.bpp = 1,
}, {
.name = "RAWRGB10 (SBGGR10)",
.fourcc = V4L2_PIX_FMT_SBGGR10,
.pixelformat = V4L2_PIX_FMT_SBGGR10,
.mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10,
.bpp = 2,
}

};

drivers/media/platform/mxc/capture/mxc_mipi_csi.c

static const struct csis_pix_format mipi_csis_formats[] = {
{
.code = MEDIA_BUS_FMT_YUYV8_2X8,
.fmt_reg = MIPI_CSIS_ISPCFG_FMT_YCBCR422_8BIT,
.data_alignment = 16,
}, {
.code = MEDIA_BUS_FMT_VYUY8_2X8,
.fmt_reg = MIPI_CSIS_ISPCFG_FMT_YCBCR422_8BIT,
.data_alignment = 16,
}, {
.code = MEDIA_BUS_FMT_SBGGR8_1X8,
.fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW8,
.data_alignment = 8,
}, {
.code = MEDIA_BUS_FMT_SBGGR10_1X10,
.fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW10,
.data_alignment = 16,
}

};

And Inside the ov4689.c driver file modified two function to support to higher 4.14.98 version

./drivers/media/i2c/ov4689.c

- sd->entity.type = MEDIA_ENT_T_V4L2_SUBDEV_SENSOR;
- ret = media_entity_init(&sd->entity, 1, &ov4689->pad, 0);

+ sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
+ ret = media_entity_pads_init(&sd->entity, 1, &ov4689->pad);

During boot, driver is initialized and OV4689 Camera IC is detecting.

/dev/video0 node also created and able to query the information.

root@iWave-G34M:~#
root@iWave-G34M:~# v4l2-ctl --list-formats
ioctl: VIDIOC_ENUM_FMT
Index : 0
Type : Video Capture
Pixel Format: 'BG10'
Name : 10-bit Bayer BGBG/GRGR

root@iWave-G34M:~# v4l2-ctl --device /dev/video0 --list-formats-ext
ioctl: VIDIOC_ENUM_FMT
Index : 0
Type : Video Capture
Pixel Format: 'BG10'
Name : 10-bit Bayer BGBG/GRGR
Size: Discrete 2688x1520
Size: Discrete 1920x1080

root@iWave-G34M:~#
root@iWave-G34M:~#
root@iWave-G34M:~# v4l2-ctl --device /dev/video0 --list-framesizes=BG10
ioctl: VIDIOC_ENUM_FRAMESIZES
Size: Discrete 2688x1520
Size: Discrete 1920x1080
root@iWave-G34M:~#

I run below command to capture one frame and its creating frame.raw file

v4l2-ctl --device /dev/video0 --set-fmt-video=width=2688,height=1520,pixelformat=BG10
v4l2-ctl --device /dev/video0 --stream-mmap --stream-to=frame.raw --stream-count=1

Once done I am able to convert that frame.raw file to png and image with black and white is there.

Below is the content of dts file:

arch/arm64/boot/dts/freescale/fsl-imx8mm-iwg34m.dts

&csi1_bridge {
fsl,mipi-mode;
status = "okay";
port {
csi1_ep: endpoint {
remote-endpoint = <&csi1_mipi_ep>;
};
};
};

&mipi_csi_1 { /* MIPI CSI: OV5640 MIPI Camera */
#address-cells = <1>;
#size-cells = <0>;
status = "okay";
port {
mipi1_sensor_ep: endpoint1 {
remote-endpoint = <&ov4689_mipi1_ep>;
data-lanes = <2>;
csis-hs-settle = <13>;
csis-clk-settle = <2>;
csis-wclk;
};

csi1_mipi_ep: endpoint2 {
remote-endpoint = <&csi1_ep>;
};
};
};

&i2c3 { /* I2C: I2C3 Bus */

....

ov4689: ov4689@36 { /* MIPI CSI: OV5640 Camera */
compatible = "ovti,ov4689";
status = "okay";
reg = <0x36>;
clocks = <&clk IMX8MM_CLK_CLKO1_DIV>;
clock-names = "xvclk";
/* avdd-supply = <>; */
/* dvdd-supply = <>; */
/* dovdd-supply = <>; */
/* reset-gpios = <>; */
avdd-supply = <&reg_2p8v>;
dovdd-supply = <&reg_1p8v>;
dvdd-supply = <&reg_1p2v>;

pinctrl-names = "iwg,camera_default";
pinctrl-0 = <&pinctrl_csi_rst_pwdn>;
assigned-clocks = <&clk IMX8MM_CLK_CLKO1_SRC>,
<&clk IMX8MM_CLK_CLKO1_DIV>;
assigned-clock-parents = <&clk IMX8MM_CLK_24M>;
assigned-clock-rates = <0>, <24000000>;
csi_id = <0>;
reset-gpios = <&gpio5 24 GPIO_ACTIVE_LOW>;
pwdn-gpios = <&gpio5 23 GPIO_ACTIVE_LOW>;
mclk = <24000000>;
mclk_source = <0>;
port {
ov4689_mipi1_ep: endpoint {
remote-endpoint = <&mipi1_sensor_ep>;
};
};
};

}

arch/arm64/boot/dts/freescale/fsl-imx8mm.dtsi

csi1_bridge: csi1_bridge@32e20000 {
compatible = "fsl,imx8mm-csi", "fsl,imx8mq-csi", "fsl,imx6s-csi";
reg = <0x0 0x32e20000 0x0 0x10000>;
interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
clocks = <&clk IMX8MM_CLK_DISP_AXI_ROOT>,
<&clk IMX8MM_CLK_CSI1_ROOT>,
<&clk IMX8MM_CLK_DISP_APB_ROOT>;
clock-names = "disp-axi", "csi_mclk", "disp_dcic";
power-domains = <&dispmix_pd>;
status = "disabled";
};

mipi_csi_1: mipi_csi@32e30000 {
compatible = "fsl,imx8mm-mipi-csi";
reg = <0x0 0x32e30000 0x0 0x1000>;
interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
clock-frequency = <333000000>;
clocks = <&clk IMX8MM_CLK_CSI1_CORE_DIV>,
<&clk IMX8MM_CLK_CSI1_PHY_REF_DIV>,
<&clk IMX8MM_CLK_DISP_AXI_ROOT>,
<&clk IMX8MM_CLK_DISP_APB_ROOT>;
clock-names = "mipi_clk", "phy_clk", "disp_axi", "disp_apb";
bus-width = <4>;
csi-gpr = <&dispmix_gpr>;
power-domains = <&mipi_pd>;
status = "disabled";
};

From the datasheet I came to know that "CFA (Chroma): RGB Bayer"

When I tried to capture the video using below command I am getting error:

root@iWave-G34M:~#

gst-launch-1.0 v4l2src device=/dev/video0 num-buffers=100 ! video/x-raw,width=1920,height=1080 ! vpuenc_h264 ! avimux ! filesink location=test.mp4

Setting pipeline to PAUSED ...
====== VPUENC: 4.4.5 build on Mar 20 2020 10:11:59. ======
wrapper: 3.0.0 (VPUWRAPPER_ARM64_LINUX Build on Feb 20 2020 11:05:40)
vpulib: 1.1.1
firmware: 1.1.1.65535
Pipeline is live and does not need PREROLL ...
Setting pipeline to PLAYING ...
New clock: GstSystemClock
ERROR: from element /GstPipeline:pipeline0/GstV4l2Src:v4l2src0: Internal data stream error.
Additional debug info:
../../../../git/libs/gst/base/gstbasesrc.c(3055): gst_base_src_loop (): /GstPipeline:pipeline0/GstV4l2Src:v4l2src0:
streaming stopped, reason not-negotiated (-4)
Execution ended after 0:00:00.001057125
Setting pipeline to PAUSED ...
Setting pipeline to READY ...
Setting pipeline to NULL ...
Freeing pipeline ...
root@iWave-G34M:~#

root@iWave-G34M:~#gst-launch-1.0 v4l2src device=/dev/video0 num-buffers=100 ! video/x-raw,width=2688,height=1520 ! vpuenc_h264 ! avimux ! filesink location=test.mp4

WARNING: erroneous pipeline: could not link v4l2src0 to vpuenc_h264-0, vpuenc_h264-0 can't handle caps video/x-raw, width=(int)2688, height=(int)1520
root@iWave-G34M:~#

Please provide the proper "gst-launch" command to capture the video or image for RGB Bayer format?

Is there any application is there to capture the image or video in linux ?

Anything I missed or configured wrong? Can you suggest me?

Thanks and Regards,

Devendra

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

igorpadykov — Tue, 09 Jun 2020 23:32:35 GMT

Hi Devendra

this input data format is not supported by i.MX8M Mini vpu, as seen

from log " vpuenc_h264-0 can't handle caps video/x-raw" and description

i.MX8M Mini Reference Manual for vpu encoder capabilities:

and one can follow suggestions provided on

OV5647 raw camera : lower half image is black (blanking ?)

Best regards
igor
-----------------------------------------------------------------------------------------------------------------------
Note: If this post answers your question, please click the Correct Answer button. Thank you!
-----------------------------------------------------------------------------------------------------------------------

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Fri, 12 Jun 2020 11:29:51 GMT

Dear Igor,

Thank you for your reply.

From the link you shared above I came to know that we can use the Software application

software ISP - https://www.nxp.com/design/designs/i-mx8-software-image-signal-processing:SOFTISP-I.MX8

gtec-demo-framework/DemoApps/OpenCL/SoftISP at master · NXPmicro/gtec-demo-framework · GitHub

gtec-demo-framework/DemoApps/OpenVX/SoftISP at master · NXPmicro/gtec-demo-framework · GitHub

Can we run these applications on our i.MX8M Mini target?

There mentioned like:

Supported operating systems

Android NDK
Linux with various windowing systems (Yocto).
Ubuntu 18.04
Windows 7+

Is there any guide which explains how we can add these to youcto and build ?

Thanks and Regards,

Devendra

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Fri, 12 Jun 2020 11:38:54 GMT

With this application (http://www.easy2convert.com/dlnaaak/raw2jpg.exe ), when I converted (frame.raw) file I captured through camera to JPG its coming like this:

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

igorpadykov — Fri, 12 Jun 2020 15:00:10 GMT

Hi Devendra

https://www.nxp.com/design/designs/i-mx8-software-image-signal-processing:SOFTISP-I.MX8#Design-Resources

Best regards
igor

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Sat, 13 Jun 2020 07:54:03 GMT

Dear Igor,

Thank you for your reply.

Is that SoftISP tool is paid one?

Is it free or we need to pay?

Is this possible to get built demo image of below two SoftISP for i.MX8M Mini?

gtec-demo-framework/DemoApps/OpenVX/SoftISP at master · NXPmicro/gtec-demo-framework · GitHub

gtec-demo-framework/DemoApps/OpenCL/SoftISP at master · NXPmicro/gtec-demo-framework · GitHub

If possible can you share the document how we can run that demo?

I am able to download the gtec-demo-framework source code. For building I tried below methods:

I followed the methods mentioned in below link:

Building using a prebuild Yocto SDK

gtec-demo-framework/Setup_guide_yocto.md at master · NXPmicro/gtec-demo-framework · GitHub

For i.MX8M Mini I build the SDK with bitbake command

$bitbake meta-toolchain

$cd tmp/deploy/sdk

$./fsl-imx-xwayland-glibc-x86_64-meta-toolchain-aarch64-toolchain-L4.14.98-2.0.0_GA.sh

And installed in "/opt/fsl-imx-xwayland/L4.14.98-2.0.0_GA".

Then follow the method mentioned in above link for environment setup and followed the "Using the demo framework" steps:

cd ~/gtec-demo-framework-master
. /opt/fsl-imx-xwayland/L4.14.98-2.0.0_GA/environment-setup-aarch64-poky-linux
export FSL_PLATFORM_NAME=Yocto
export ROOTFS=/opt/fsl-imx-xwayland/L4.14.98-2.0.0_GA/sysroots/aarch64-poky-linux
source prepare.sh
FslBuild.py --Variants [WindowSystem=FB] -t sdk --BuildThreads 2

Then I am getting error like below:

Anything I missed here? Is there any proper document is there to build ?

Thanks and Regards,

Devendra

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

igorpadykov — Sat, 13 Jun 2020 10:57:58 GMT

Hi Devendra

one can look at app note

Software ISP Application Note

Best regards
igor

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Sat, 13 Jun 2020 11:07:38 GMT

Dear Igor,

In app note there is no steps to build the SoftISP module. Please can

you share proper document?

Thanks and Regards,

Devendra

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

igorpadykov — Sat, 13 Jun 2020 12:37:22 GMT

gtec-demo-framework/DemoApps/OpenVX/SoftISP at master · NXPmicro/gtec-demo-framework · GitHub

gtec-demo-framework/Setup_guide_yocto.md at master · NXPmicro/gtec-demo-framework · GitHub

Best regards
igor

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Sat, 13 Jun 2020 12:44:50 GMT

Dear Igor,

I followed the steps mentioned in that link.

But I got the build error.

Please can you share the proper document to build?

Thanks and Regards,

Devendra

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Sat, 13 Jun 2020 13:39:29 GMT

Dear Igor,

I followed the "Building using a full Yocto build" also.

Below is the steps I followed:

DISTRO=fsl-imx-xwayland MACHINE=imx8mm_iwg34m source fsl-setup-release.sh -b build_imx8mm_gtec -e wayland

In conf/local.conf I added as below

CORE_IMAGE_EXTRA_INSTALL += " assimp devil gtest fmt"

bitbake fsl-image-gui

bitbake meta-toolchain

bitbake meta-ide-support

mkdir ~/unpacked-rootfs/build-wayland_test

runqemu-extract-sdk tmp/deploy/images/imx8mm_iwg34m/fsl-image-gui-imx8mm_iwg34m.tar.bz2 ~/unpacked-rootfs/build-wayland_test

cd ~/gtec-demo-framework-master
pushd /home/devendra/AndroidPie/i.MX8MMini_Android_Patch_Draft/LINUX_BSP/iwg34-release-bsp/build_imx8mm/tmp
source environment-setup-aarch64-poky-linux
export ROOTFS=/home/devendra/unpacked-rootfs/build-wayland_test
export FSL_PLATFORM_NAME=Yocto
popd
source prepare.sh
FslBuild.py --Variants [WindowSystem=Wayland] -t sdk --BuildThreads 2

Now also same error I am getting as in previous.

Thanks and Regards,

Devendra

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Tue, 16 Jun 2020 10:48:35 GMT

Dear Igor,

In below file there is some external dependecy is added like this

gtec-demo-framework-master/ThirdParty/fmt/Fsl.gen

gtec-demo-framework/Fsl.gen at master · NXPmicro/gtec-demo-framework · GitHub

<?xml version="1.0" encoding="UTF-8"?>
<FslBuildGen xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../FslBuildGen.xsd">
<ExternalLibrary Name="fmt" CreationYear="2019">
<Default.Platform.Supported Value="false"/>
<Define Name="FSL_ENABLE_FMT" Access="Public"/>
<Platform Name="Android" Supported="true">
<Dependency Name="Recipe.fmt_6_1_2"/>
</Platform>
<Platform Name="Yocto" Supported="true">
<ExternalDependency Name="fmt" Access="Public" Type="StaticLib"/>
</Platform>
<Platform Name="Ubuntu" Supported="true">
<Dependency Name="Recipe.fmt_6_1_2"/>
</Platform>
<Platform Name="Windows" Supported="true">
<Dependency Name="Recipe.fmt_6_1_2"/>
</Platform>
</ExternalLibrary>
</FslBuildGen>

What is the difference between below these two lines?

<ExternalDependency Name="fmt" Access="Public" Type="StaticLib"/> and <Dependency Name="Recipe.fmt_6_1_2"/>
I changed like below and and now able to build

<Platform Name="Yocto" Supported="true">
<Dependency Name="Recipe.fmt_6_1_2"/>

</Platform>

But its only builded "FslUnitTest". SoftISP is not builded.

If I enabled ExternalDependency Name="fmt" What configuration I need to do in host PC?

Thanks and Regards,

Devendra

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadi1 — Wed, 17 Jun 2020 12:24:58 GMT

Dear Igor,

From the build guide I came to know that we can build SOftISP in the following DISTRO's

Wayland, FB and X11

In the Linux user guide mentioned like following distros are supported:

fsl-imx-x11 - X11 graphics are not supported on i.MX 8.
fsl-imx-wayland - Wayland weston graphics.
fsl-imx-xwayland - Wayland graphics and X11. X11 applications using EGL are not supported.
fsl-imx-fb - Frame Buffer graphics - no X11 or Wayland. Frame Buffer is not supported on i.MX 8.

But I am using fsl-imx-xwayland .

Using this can we build that application?

Thanks and Regards,

Devendra

Re: Command to capture video or image for 10 bit RGB layer with OV4689 IC

devendradevadig — Fri, 19 Jun 2020 12:56:18 GMT

Dear Igor,

I build the OpenCL.SoftISP and OpenVX.SoftISP by building fsl-imx-wayland for imx8m mini by following standalone method. During compilation of application, it ask for these dependencies: "gtest fmt rapidopencl rapidopenvx stb gli glm assimp devil libpng openvx"

I added all these by manually copying to sysroot and able to build the application. And added those to filesystem on target also.

When I am running th eapplication I am getting below error:

root@iWave-G34M:~#
root@iWave-G34M:~#
root@iWave-G34M:~# OpenCL.SoftISP -h
terminate called after throwing a[ 1114.860124] audit: type=1701 audit(1582024697.916:18): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=3810 comm="OpenCL.SoftISP" exe="/usr/bin/OpenCL.SoftISP" sig=6 res=1
n instance of 'std::runtime_error'
what(): failed to create the full path
Aborted (core dumped)
root@iWave-G34M:~#
root@iWave-G34M:~#

root@iWave-G34M:~#
root@iWave-G34M:~# OpenVX.SoftISP
terminate called after throwing a[ 1502.711961] audit: type=1701 audit(1582025085.768:20): auid=4294967295 uid=0 gid=0 ses=4294967295 pid=3815 comm="OpenVX.SoftISP" exe="/usr/bin/OpenVX.SoftISP" sig=6 res=1
n instance of 'std::runtime_error'
what(): failed to create the full path
Aborted (core dumped)
root@iWave-G34M:~#
root@iWave-G34M:~#

Even when I try to run with "-h" option for help, that time also getting same error.

Can you explain how we need to use this application? What are the arguments we need to give?

https://community.nxp.com/external-link.jspa?url=https%3A%2F%2Fgithub.com%2FNXPmicro%2Fgtec-demo-framework%2Ftree%2Fmast…

This link contains only list of arguments, but if I used that above error I am getting.

In this link, comparision sheet, for imx8m mini Hardware 2D/3D Graphics Acceleration feature its mentioned like :

1 x GCNanoUltra 3D (1 shader) OpenGL® ES 2.0, 1 x GC328 2D

https://www.nxp.com/docs/en/supporting-information/IMX8LAYERCMPR.pdf

This means thes OpenVX and OpenCL applications will not run on imx8m mini target ?

Thanks and Regards,

Devendra