Custom camera driver adaptation on iMX8M Mini

[khang_letruong]

Dear experts,

I'm in the process of porting a custom camera to iMX8M Mini from its original driver dedicated to Nvidia Jetson platforms.

I've finished the control part via I2C bus. However, I've got following errors from the V4L2 compliance test :

# v4l2-compliance -s
[  409.499911] mxc_mipi-csi 32e30000.mipi_csi: format not match
[  409.505648] unknown pixelformat:'    '
[  409.509419] mx6s-csi 32e20000.csi1_bridge: Fourcc format (0x00000000) invalid.
[  409.516674] unknown pixelformat:'����'
[  409.520493] mx6s-csi 32e20000.csi1_bridge: Fourcc format (0xffffffff) invalid.
[  409.527756] unknown pixelformat:'    '
[  409.531517] mx6s-csi 32e20000.csi1_bridge: Fourcc format (0x00000000) invalid.
[  409.538806] unknown pixelformat:'    '
[  409.542568] mx6s-csi 32e20000.csi1_bridge: Fourcc format (0x00000000) invalid.
[  409.549811] unknown pixelformat:'    '
[  409.553574] mx6s-csi 32e20000.csi1_bridge: Fourcc format (0x00000000) invalid.
[  409.560929] mx6s-csi 32e20000.csi1_bridge: dma_alloc_coherent of size 0 failed
[  409.568389] unknown pixelformat:'    '
[  409.572162] mx6s-csi 32e20000.csi1_bridge: Fourcc format (0x00000000) invalid.
[  409.581812]  mcu_stream_config(1839) ISP Status = 0x0000 , Ret code = 0x02
[  409.588794]  ISP is Unintialized or Busy STATUS = 0x0000 Errcode = 0x02 !!
[  409.595780] ov2311 2-004a: ov2311_s_parm: Failed stream_config

By digging into the source code of original driver, I found  :

...

struct ecam_ov2311_colorfmt {
    unsigned int            code;
    enum v4l2_colorspace        colorspace;
    int                pix_fmt;
    enum v4l2_xfer_func        xfer_func;
    enum v4l2_ycbcr_encoding    ycbcr_enc;
    enum v4l2_quantization        quantization;
};

static const struct ecam_ov2311_colorfmt ecam_ov23111_color_fmts[] = {
    {
        MEDIA_BUS_FMT_SRGGB12_1X12,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_SRGGB12,
    },
    {
        MEDIA_BUS_FMT_SGRBG12_1X12,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_SGRBG12,
    },
    {
        MEDIA_BUS_FMT_SRGGB10_1X10,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_SRGGB10,
    },
    {
        MEDIA_BUS_FMT_SGRBG10_1X10,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_SGRBG10,
    },
    {
        MEDIA_BUS_FMT_SBGGR10_1X10,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_SBGGR10,
    },
    {
        MEDIA_BUS_FMT_SRGGB8_1X8,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_SRGGB8,
    },
    {
        MEDIA_BUS_FMT_YUYV8_1X16,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_YUYV,
    },
    {
        MEDIA_BUS_FMT_YVYU8_1X16,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_YVYU,
    },
    {
        MEDIA_BUS_FMT_UYVY8_1X16,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_UYVY,
    },
    {
        MEDIA_BUS_FMT_VYUY8_1X16,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_VYUY,
    },
    {
        MEDIA_BUS_FMT_YUYV8_2X8,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_YUYV,
    },
    {
        MEDIA_BUS_FMT_YVYU8_2X8,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_YVYU,
    },
    {
        MEDIA_BUS_FMT_UYVY8_2X8,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_UYVY,
    },
    {
        MEDIA_BUS_FMT_VYUY8_2X8,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_VYUY,
    },
    /*
     * The below two formats are not supported by VI4,
     * keep them at the last to ensure they get discarded
     */
    {
        MEDIA_BUS_FMT_XRGGB10P_3X10,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_XRGGB10P,
    },
    {
        MEDIA_BUS_FMT_XBGGR10P_3X10,
        V4L2_COLORSPACE_SRGB,
        V4L2_PIX_FMT_XRGGB10P,
    },
#if defined(CONFIG_MXC_CAMERA_ECAM_OV2311_MIPI_V2)
    /* e-con: extending support for GREY format */
    {
               MEDIA_BUS_FMT_Y8_1X8,
               V4L2_COLORSPACE_SRGB,
               V4L2_PIX_FMT_GREY,
    },
#endif
};

However, in the current BSP of iMX8M Mini (imx-yocto-L4.9.88_2.0.0), and based on the error messages, I also found :

====  drivers/media/platform/mxc/capture/mx6s_capture.c ====

...

static struct mx6s_fmt formats[] = {
    {
        .name        = "UYVY-16",
        .fourcc        = V4L2_PIX_FMT_UYVY,
        .pixelformat    = V4L2_PIX_FMT_UYVY,
        .mbus_code    = MEDIA_BUS_FMT_UYVY8_2X8,
        .bpp        = 2,
    }, {
        .name        = "YUYV-16",
        .fourcc        = V4L2_PIX_FMT_YUYV,
        .pixelformat    = V4L2_PIX_FMT_YUYV,
        .mbus_code    = MEDIA_BUS_FMT_YUYV8_2X8,
        .bpp        = 2,
    }, {
        .name        = "YUV32 (X-Y-U-V)",
        .fourcc        = V4L2_PIX_FMT_YUV32,
        .pixelformat    = V4L2_PIX_FMT_YUV32,
        .mbus_code    = MEDIA_BUS_FMT_AYUV8_1X32,
        .bpp        = 4,
    }, {
        .name        = "RAWRGB8 (SBGGR8)",
        .fourcc        = V4L2_PIX_FMT_SBGGR8,
        .pixelformat    = V4L2_PIX_FMT_SBGGR8,
        .mbus_code    = MEDIA_BUS_FMT_SBGGR8_1X8,
        .bpp        = 1,
    }
};

====  drivers/media/platform/mxc/capture/mxc_mipi_csi.c ====

static const struct csis_pix_format mipi_csis_formats[] = {
    {
        .code = MEDIA_BUS_FMT_YUYV8_2X8,
        .fmt_reg = MIPI_CSIS_ISPCFG_FMT_YCBCR422_8BIT,
        .data_alignment = 16,
    }, {
        .code = MEDIA_BUS_FMT_VYUY8_2X8,
        .fmt_reg = MIPI_CSIS_ISPCFG_FMT_YCBCR422_8BIT,
        .data_alignment = 16,
    }, {
        .code = MEDIA_BUS_FMT_SBGGR8_1X8,
        .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW8,
        .data_alignment = 8,
    }
};

Could someone give me an idea how to extend the mx6s_fmt as well as mipi_csis_formats tables in order to take into account the custom camera's formats, please ?

Thanks very much in advance,

Khang

[suzannomer258]

The NXP i.MX8M-Mini SoM is the first i.MX 8 device to feature advanced 14nm FinFET technology walmartone. Efficient in every respect, the board measures just 28mm x 38mm x 4mm and consumes from as little as 0.5W to just 3W depending on configuration.

[khang_letruong]

Dear suzannomer258@gmail.com‌,

Sorry but I wonder how it relates to the subject of custom camera porting?

Best,

Khang

[igorpadykov]

Hi Khang

may be useful try to adapt camera format for already supported in driver and

one can look at examples below

GStreamer Based Image Signal Processor - IMX8 Pipelines 

Best regards
igor
-----------------------------------------------------------------------------------------------------------------------
Note: If this post answers your question, please click the Correct Answer button. Thank you!
-----------------------------------------------------------------------------------------------------------------------

[khang_letruong]

Dear igorpadykov‌,

First of all, thanks for your reply to my question that was quite generic. After asking, I was digging into the source as well as other questions/discussions related to the subject and there's been some progress.  Specifically, I made following modifications :

====  drivers/media/platform/mxc/capture/mx6s_capture.c ====

static int mx6s_configure_csi(struct mx6s_csi_dev *csi_dev)
{

   ...
    switch (csi_dev->fmt->pixelformat) {
    case V4L2_PIX_FMT_YUV32:
    case V4L2_PIX_FMT_SBGGR8:
        width = pix->width;
        break;
    case V4L2_PIX_FMT_UYVY:
    case V4L2_PIX_FMT_YUYV:
#if defined (CONFIG_MXC_CAMERA_ECAM_OV2311_MIPI_V2)
    case V4L2_PIX_FMT_GREY:
#endif
        if (csi_dev->csi_mipi_mode == true)
            width = pix->width;
        else
            /* For parallel 8-bit sensor input */
            width = pix->width * 2;
        break;

    default:
        pr_debug("   case not supported\n");
        return -EINVAL;
    }
    csi_set_imagpara(csi_dev, width, pix->height);

    if (csi_dev->csi_mipi_mode == true) {

       ...

        switch (csi_dev->fmt->pixelformat) {
        case V4L2_PIX_FMT_UYVY:
        case V4L2_PIX_FMT_YUYV:
            cr18 |= BIT_MIPI_DATA_FORMAT_YUV422_8B;
            break;
        case V4L2_PIX_FMT_SBGGR8:
            cr18 |= BIT_MIPI_DATA_FORMAT_RAW8;
            break;
#if defined (CONFIG_MXC_CAMERA_ECAM_OV2311_MIPI_V2)
        case V4L2_PIX_FMT_GREY:
            cr18 |= BIT_MIPI_DATA_FORMAT_RAW10;
            break;
#endif
        default:
            pr_debug("   fmt not supported\n");
            return -EINVAL;
        }

        csi_write(csi_dev, cr18, CSI_CSICR18);
    }
    return 0;
}

====  drivers/media/platform/mxc/capture/mxc_mipi_csi.c ====

static const struct csis_pix_format mipi_csis_formats[] = {

    ...

    }, {
        .code = MEDIA_BUS_FMT_SBGGR8_1X8,
        .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW8,
        .data_alignment = 8,
    }
#if defined (CONFIG_MXC_CAMERA_ECAM_OV2311_MIPI_V2)
    , {
        .code = MEDIA_BUS_FMT_Y8_1X8,
        .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW10,
        .data_alignment = 8,
    }
#endif

};

==== arch/arm64/boot/dts/freescale/fsl-imx8mm-evk.dts ====

&mipi_csi_1 {
        #address-cells = <1>;
        #size-cells = <0>;
        status = "okay";
        port {
                mipi1_sensor_ep: endpoint1 {
                        remote-endpoint = <&ecam_ov2311_mipi1_ep>;
                        data-lanes = <2>;
                        csis-hs-settle = <13>; // ?
                        csis-clk-settle = <2>;  // ?
                        csis-wclk;
                };

                csi1_mipi_ep: endpoint2 {
                        remote-endpoint = <&csi1_ep>;
                };
        };
};

...

       ecam_ov2311_mipi: ecam_ov2311_mipi@4a {

                compatible = "dynimlabs,ecam_ov2311_mipi";
                reg = <0x4a>;
                status = "okay";
                pinctrl-names = "default";
                pinctrl-0 = <&pinctrl_csi_pwn>, <&pinctrl_csi_rst>;
                clocks = <&clk IMX8MM_CLK_CLKO1_DIV>;
                clock-names = "csi_mclk";
                assigned-clocks = <&clk IMX8MM_CLK_CLKO1_SRC>,
                                  <&clk IMX8MM_CLK_CLKO1_DIV>;
                assigned-clock-parents = <&clk IMX8MM_CLK_24M>;
                assigned-clock-rates = <266000000>, <333000000>, <66000000>; // ?
                csi_id = <0>;
                pwn-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>;
                rst-gpios = <&gpio1 6 GPIO_ACTIVE_HIGH>;
                mclk = <24000000>;
                mclk_source = <0>;
                // Sensor's specific properties which are needed in the module driver
                position = "CAMA";

                mode0 { /* OV2311_MODE_3200x1300_30FPS */
                active_w = "3200";
                active_h = "1300";
                pixel_t = "grey";
                };

         };

...

The camera is actually a proprietary stereo camera that combines imaging data from 2 ov2311 sensors and outputs 10-bit raw data. The control is done via an internal MCU.

With above modification :

root@imx8mmevk:~# v4l2-ctl -d /dev/video0 --list-formats-ext

ioctl: VIDIOC_ENUM_FMT
        Index       : 0
        Type        : Video Capture
        Pixel Format: 'GREY'
        Name        : 8-bit Greyscale
                Size: Discrete 3200x1300
                        Interval: Discrete 0.033s (30.000 fps)

And I was able to stream data using following command :

$ v4l2-ctl --set-fmt-video=width=1600,height=1300,pixelformat=GREY --stream-mmap --stream-count=1 --stream-to=grab-grey1600x1300.raw

-rw-r--r-- 1 root root 2080000 Jul 17 09:03 grab-grey1600x1300.raw

However, if I try to set the correct resolution provided by the camera which is 3200x1300 :

$ v4l2-ctl --set-fmt-video=width=3200,height=1300,pixelformat=GREY --stream-mmap --stream-count=1 --stream-to=grab-grey3200x1300.raw

Then I got series of "base address switching Change Err"

I understand that there's been also questions / discussions about above error and the investigation is in progress but I would really appreciate I you could point out what is inappropriate among the above info and/or give me more advises to move forward.

I attach the patches (debug messages included) for my modification and the log files (with as much as possible traces) for each v4l2-ctl stream testing. I'm able provide the module driver's source in case necessary.

Thank you in advance and best regards,

Khang

[santhosh2]

Hi,

I am using an HD camera that outputs on HDMI, we have used the HDMI to MIPI Bridge for converting HDMI output to MIPI CSI2. and the camera generates the clock. How do i customize the fsl-imx8mm-evk.dts file to set as external clock and also help in understanding to customize the ov5640 code to be compatible with our camera

Regards

Santhosh

[khang_letruong]

Dear santhoshkumar‌,

Sorry for late reply, but I found that you already got the answer, right ?

https://community.nxp.com/message/1357231?commentID=1357231#comment-1357231 

[santhosh2]

Dear Khang,

No not yet... My issue is still open

Regards

Santhosh kumar

[igorpadykov]

Hi Khang

seems such cameras are already supported by e-con systems

http://linuxgizmos.com/13mp-mipi-csi2-cam-plugs-into-variscite-i-mx8m-board/ 

NVIDIA Jetson Cameras 

Best regards
igor

[khang_letruong]

Dear igorpadykov‌,

Thanks for the quick reply. Obviously the do have competences in developing/porting their camera modules to another platform such as iMX8M but unluckily that is not the case of  the mentioned one that we really need for our project.

Best regards,

Khang

[khang_letruong]

Hi all,

I finally got the problem  "base address switching Change Err" fixed by applying the solution from this link : Fixing 'base address switching Change Err' which occurs randomly and was able to stream the data with desired resolution 3200 x 1300.

Thanks anyway for the support.

BR,

Khang