custom image sensor driver

[zohargolan]

Hello NXP community,

I am trying to write a driver for our custom MIPI camera for IMX8MP device. The camera is acting as the master on the MIPI bus, and is generating the MIPI clock by itself. I am using Yocto Mickledore. I could not find anywhere information if this mode of operation is supported. I did see that all the drivers I was looking at have the clock generated as part of the driver. Is this mode supported and what do I need to do in the driver to disable the host MIPI clock and only use the sensor clock

Any suggestion will be appreciated

Thank you,

Zohar

[jimmychan]

Could you show me how you connect the camera to i.MX8MP?

[zohargolan]

Hi Jimmy,

I connected the camera on the MIPI bus using 4 lanes with a clock of 504MHz.

I also connected GPIOs and I2C, but the camera at this moment does not use any of those signals and start streaming MIPI data on power on

[jimmychan]

What is the camera?

Do you mean the camera driver cannot control the camera? Is there any error message when you use the camera to capture?

[zohargolan]

Hi Jimmy,

Thank you for coming back to me. Here is what I have. I am working on a custom driver for my MIPI CSI2 camera. The camera (at current configuration) does not have any power control, reset control, and I2C control. It just start sending data on the 4 lanes MIPI port at power on.

This is the strip down simplistic driver I am trying

#include <asm/unaligned.h>

#include <linux/acpi.h>

#include <linux/delay.h>

#include <linux/i2c.h>

#include <linux/module.h>

#include <media/v4l2-ctrls.h>

#include <media/v4l2-device.h>

#include <media/v4l2-fwnode.h>

#define xxxxxx_LINK_FREQ 504000000ULL

#define xxxxxx_VIDEO_WIDTH 1280

#define xxxxxx_VIDEO_HEIGHT 960

#define xxxxxx_DATA_LANES 4

#define xxxxxx_RGB_DEPTH 8

#define xxxxxx_CSI_PIXEL_RATE (xxxxxx_LINK_FREQ * 2 * xxxxxx_DATA_LANES / \

xxxxxx_RGB_DEPTH)

#define xxxxxx_FLL_90FPS 0x0500

#define xxxxxx_FLL_90FPS_MIN 0x0500

#define xxxxxx_FLL_MAX 0x7fff

/* Exposure controls from sensor */

#define xxxxxx_REG_EXPOSURE 0x0074

#define xxxxxx_EXPOSURE_MIN 6

#define xxxxxx_EXPOSURE_MAX_MARGIN 2

#define xxxxxx_EXPOSURE_STEP 1

/* Analog gain controls from sensor */

#define xxxxxx_REG_ANALOG_GAIN 0x0077

#define xxxxxx_ANAL_GAIN_MIN 0

#define xxxxxx_ANAL_GAIN_MAX 240

#define xxxxxx_ANAL_GAIN_STEP 1

/* Digital gain controls from sensor */

#define xxxxxx_DGTL_GAIN_MIN 0

#define xxxxxx_DGTL_GAIN_MAX 2048

#define xxxxxx_DGTL_GAIN_STEP 1

#define xxxxxx_DGTL_GAIN_DEFAULT 256

enum {

xxxxxx_LINK_FREQ_INDEX,

};

struct xxxxxx_reg {

u16 address;

u16 val;

};

struct xxxxxx_reg_list {

u32 num_of_regs;

const struct xxxxxx_reg *regs;

};

struct xxxxxx_link_freq_config {

const struct xxxxxx_reg_list reg_list;

};

struct xxxxxx_mode {

/* Frame width in pixels */

u32 width;

/* Frame height in pixels */

u32 height;

/* Horizontal timining size */

u32 llp;

/* Default vertical timining size */

u32 fll_def;

/* Min vertical timining size */

u32 fll_min;

/* Link frequency needed for this resolution */

u32 link_freq_index;

/* Sensor register settings for this resolution */

const struct xxxxxx_reg_list reg_list;

};

#define to_xxxxxx(_sd) container_of(_sd, struct xxxxxx, sd)

static const char * const xxxxxx_test_pattern_menu[] = {

"Disabled",

"Solid Colour",

"100% Colour Bars",

"Fade To Grey Colour Bars",

"PN9",

"Gradient Horizontal",

"Gradient Vertical",

"Check Board",

"Slant Pattern",

};

static const s64 link_freq_menu_items[] = {

xxxxxx_LINK_FREQ,

};

static const struct xxxxxx_mode supported_modes[] = {

{

.width = xxxxxx_VIDEO_WIDTH,

.height = xxxxxx_VIDEO_HEIGHT,

.fll_def = xxxxxx_FLL_90FPS,

.fll_min = xxxxxx_FLL_90FPS_MIN,

.llp = 0x0b00,

.link_freq_index = xxxxxx_LINK_FREQ_INDEX,

}

};

struct xxxxxx {

struct v4l2_subdev sd;

struct media_pad pad;

struct v4l2_ctrl_handler ctrl_handler;

/* V4L2 Controls */

struct v4l2_ctrl *link_freq;

struct v4l2_ctrl *pixel_rate;

struct v4l2_ctrl *vblank;

struct v4l2_ctrl *hblank;

struct v4l2_ctrl *exposure;

/* Current mode */

const struct xxxxxx_mode *cur_mode;

/* To serialize asynchronus callbacks */

struct mutex mutex;

/* Streaming on/off */

bool streaming;

/* True if the device has been identified */

bool identified;

};

static int xxxxxx_s_power(struct v4l2_subdev *sd, int on)

{

struct xxxxxx *sensor = to_xxxxxx(sd);

int ret = 0;

mutex_lock(&sensor->mutex);

mutex_unlock(&sensor->mutex);

return ret;

}

static int xxxxxx_set_ctrl(struct v4l2_ctrl *ctrl)

{

struct xxxxxx *xxxxxx = container_of(ctrl->handler,

     struct xxxxxx, ctrl_handler);

struct i2c_client *client = v4l2_get_subdevdata(&xxxxxx->sd);

s64 exposure_max;

int ret = 0;

/* Propagate change of current control to all related controls */

if (ctrl->id == V4L2_CID_VBLANK) {

/* Update max exposure while meeting expected vblanking */

exposure_max = xxxxxx->cur_mode->height + ctrl->val -

       xxxxxx_EXPOSURE_MAX_MARGIN;

__v4l2_ctrl_modify_range(xxxxxx->exposure,

xxxxxx->exposure->minimum,

exposure_max, xxxxxx->exposure->step,

exposure_max);

}

switch (ctrl->id) {

case V4L2_CID_ANALOGUE_GAIN:

break;

case V4L2_CID_DIGITAL_GAIN:

break;

case V4L2_CID_EXPOSURE:

break;

case V4L2_CID_VBLANK:

/* Update FLL that meets expected vertical blanking */

break;

case V4L2_CID_TEST_PATTERN:

break;

default:

ret = -EINVAL;

break;

}

return ret;

}

static const struct v4l2_ctrl_ops xxxxxx_ctrl_ops = {

.s_ctrl = xxxxxx_set_ctrl,

};

static int xxxxxx_init_controls(struct xxxxxx *xxxxxx)

{

struct v4l2_ctrl_handler *ctrl_hdlr;

s64 exposure_max, h_blank;

int ret;

ctrl_hdlr = &xxxxxx->ctrl_handler;

ret = v4l2_ctrl_handler_init(ctrl_hdlr, 4);

if (ret)

return ret;

ctrl_hdlr->lock = &xxxxxx->mutex;

xxxxxx->link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr, &xxxxxx_ctrl_ops,

V4L2_CID_LINK_FREQ, ARRAY_SIZE(link_freq_menu_items) - 1, 0,

link_freq_menu_items);

if (xxxxxx->link_freq)

xxxxxx->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;

xxxxxx->pixel_rate = v4l2_ctrl_new_std

    (ctrl_hdlr, &xxxxxx_ctrl_ops,

     V4L2_CID_PIXEL_RATE, 0,

     xxxxxx_CSI_PIXEL_RATE,

     1,

xxxxxx_CSI_PIXEL_RATE);

xxxxxx->vblank = v4l2_ctrl_new_std(ctrl_hdlr, &xxxxxx_ctrl_ops,

  V4L2_CID_VBLANK,

  xxxxxx_FLL_90FPS_MIN -

  xxxxxx->cur_mode->height,

  xxxxxx_FLL_MAX -

  xxxxxx->cur_mode->height, 1,

  xxxxxx->cur_mode->fll_def -

  xxxxxx->cur_mode->height);

h_blank = xxxxxx->cur_mode->llp - xxxxxx->cur_mode->width;

xxxxxx->hblank = v4l2_ctrl_new_std(ctrl_hdlr, &xxxxxx_ctrl_ops,

  V4L2_CID_HBLANK, h_blank, h_blank, 1,

  h_blank);

if (xxxxxx->hblank)

xxxxxx->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY;

xxxxxx->sd.ctrl_handler = ctrl_hdlr;

return 0;

}

static void xxxxxx_assign_pad_format(const struct xxxxxx_mode *mode,

    struct v4l2_mbus_framefmt *fmt)

{

fmt->width = mode->width;

fmt->height = mode->height;

fmt->code = MEDIA_BUS_FMT_Y8_1X8;

fmt->field = V4L2_FIELD_NONE;

}

static int xxxxxx_set_stream(struct v4l2_subdev *sd, int enable)

{

/*

* Don't need to do anything here, just assume the source is streaming

* already.

*/

return 0;

}

static int xxxxxx_set_format(struct v4l2_subdev *sd,

    struct v4l2_subdev_state *sd_state,

    struct v4l2_subdev_format *fmt)

{

struct xxxxxx *xxxxxx = to_xxxxxx(sd);

const struct xxxxxx_mode *mode;

s32 vblank_def, h_blank;

mode = v4l2_find_nearest_size(supported_modes,

      ARRAY_SIZE(supported_modes), width,

      height, fmt->format.width,

      fmt->format.height);

mutex_lock(&xxxxxx->mutex);

xxxxxx_assign_pad_format(mode, &fmt->format);

if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) {

*v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format;

} else {

xxxxxx->cur_mode = mode;

__v4l2_ctrl_s_ctrl(xxxxxx->link_freq, mode->link_freq_index);

__v4l2_ctrl_s_ctrl_int64(xxxxxx->pixel_rate, xxxxxx_CSI_PIXEL_RATE);

/* Update limits and set FPS to default */

vblank_def = mode->fll_def - mode->height;

__v4l2_ctrl_modify_range(xxxxxx->vblank,

mode->fll_min - mode->height,

xxxxxx_FLL_MAX - mode->height, 1,

vblank_def);

__v4l2_ctrl_s_ctrl(xxxxxx->vblank, vblank_def);

h_blank = xxxxxx->cur_mode->llp - xxxxxx->cur_mode->width;

__v4l2_ctrl_modify_range(xxxxxx->hblank, h_blank, h_blank, 1,

h_blank);

}

mutex_unlock(&xxxxxx->mutex);

return 0;

}

static int xxxxxx_get_format(struct v4l2_subdev *sd,

    struct v4l2_subdev_state *sd_state,

    struct v4l2_subdev_format *fmt)

{

struct xxxxxx *xxxxxx = to_xxxxxx(sd);

mutex_lock(&xxxxxx->mutex);

if (fmt->which == V4L2_SUBDEV_FORMAT_TRY)

fmt->format = *v4l2_subdev_get_try_format(&xxxxxx->sd,

  sd_state,

  fmt->pad);

else

xxxxxx_assign_pad_format(xxxxxx->cur_mode, &fmt->format);

mutex_unlock(&xxxxxx->mutex);

return 0;

}

static int xxxxxx_enum_mbus_code(struct v4l2_subdev *sd,

struct v4l2_subdev_state *sd_state,

struct v4l2_subdev_mbus_code_enum *code)

{

if (code->index > 0)

return -EINVAL;

code->code = MEDIA_BUS_FMT_Y8_1X8;

return 0;

}

static int xxxxxx_enum_frame_size(struct v4l2_subdev *sd,

struct v4l2_subdev_state *sd_state,

struct v4l2_subdev_frame_size_enum *fse)

{

if (fse->index >= ARRAY_SIZE(supported_modes))

return -EINVAL;

if (fse->code != MEDIA_BUS_FMT_Y8_1X8)

return -EINVAL;

fse->min_width = supported_modes[fse->index].width;

fse->max_width = supported_modes[fse->index].width;

fse->min_height = supported_modes[fse->index].height;

fse->max_height = supported_modes[fse->index].height;

return 0;

}

static const struct v4l2_subdev_core_ops sensor_core_ops = {

.subscribe_event = v4l2_ctrl_subdev_subscribe_event,

.s_power = xxxxxx_s_power,

};

static int xxxxxx_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh)

{

struct xxxxxx *xxxxxx = to_xxxxxx(sd);

mutex_lock(&xxxxxx->mutex);

xxxxxx_assign_pad_format(&supported_modes[0],

v4l2_subdev_get_try_format(sd, fh->state, 0));

mutex_unlock(&xxxxxx->mutex);

return 0;

}

static const struct v4l2_subdev_video_ops xxxxxx_video_ops = {

.s_stream = xxxxxx_set_stream,

};

static const struct v4l2_subdev_pad_ops xxxxxx_pad_ops = {

.set_fmt = xxxxxx_set_format,

.get_fmt = xxxxxx_get_format,

.enum_mbus_code = xxxxxx_enum_mbus_code,

.enum_frame_size = xxxxxx_enum_frame_size,

};

static const struct v4l2_subdev_ops xxxxxx_subdev_ops = {

.video = &xxxxxx_video_ops,

.core = &sensor_core_ops,

.pad = &xxxxxx_pad_ops,

};

static int xxxxxx_link_setup(struct media_entity *entity,

   const struct media_pad *local,

   const struct media_pad *remote, u32 flags)

{

printk("xxxxxx - link setup");

return 0;

}

static const struct media_entity_operations xxxxxx_sd_entity_ops = {

.link_setup = xxxxxx_link_setup,

};

static const struct v4l2_subdev_internal_ops xxxxxx_internal_ops = {

.open = xxxxxx_open,

};

static void xxxxxx_remove(struct i2c_client *client)

{

struct v4l2_subdev *sd = i2c_get_clientdata(client);

struct xxxxxx *xxxxxx = to_xxxxxx(sd);

media_entity_cleanup(&sd->entity);

v4l2_ctrl_handler_free(sd->ctrl_handler);

mutex_destroy(&xxxxxx->mutex);

}

static int xxxxxx_probe(struct i2c_client *client)

{

struct xxxxxx *xxxxxx;

int ret;

xxxxxx = devm_kzalloc(&client->dev, sizeof(*xxxxxx), GFP_KERNEL);

printk("xxxxxx - kzalloc %d", xxxxxx);

if (!xxxxxx)

return -ENOMEM;

v4l2_i2c_subdev_init(&xxxxxx->sd, client, &xxxxxx_subdev_ops);

mutex_init(&xxxxxx->mutex);

xxxxxx->cur_mode = &supported_modes[0];

ret = xxxxxx_init_controls(xxxxxx);

printk("xxxxxx - xxxxxx_init_controls %d", ret);

if (ret) {

dev_err(&client->dev, "failed to init controls: %d", ret);

goto probe_error_v4l2_ctrl_handler_free;

}

xxxxxx->sd.internal_ops = &xxxxxx_internal_ops;

xxxxxx->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;

xxxxxx->sd.entity.ops = &xxxxxx_sd_entity_ops;

xxxxxx->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR;

xxxxxx->pad.flags = MEDIA_PAD_FL_SOURCE;

ret = media_entity_pads_init(&xxxxxx->sd.entity, 1, &xxxxxx->pad);

printk("xxxxxx - media_entity_pads_init %d", (int)ret);

if (ret) {

dev_err(&client->dev, "failed to init entity pads: %d", ret);

goto probe_error_v4l2_ctrl_handler_free;

}

ret = v4l2_async_register_subdev_sensor(&xxxxxx->sd);

printk("xxxxxx - v4l2_async_register_subdev_sensor %d", ret);

if (ret < 0) {

dev_err(&client->dev, "failed to register V4L2 subdev: %d",

ret);

goto probe_error_media_entity_cleanup;

}

printk("xxxxxx - finish probing");

return 0;

probe_error_media_entity_cleanup:

media_entity_cleanup(&xxxxxx->sd.entity);

probe_error_v4l2_ctrl_handler_free:

v4l2_ctrl_handler_free(xxxxxx->sd.ctrl_handler);

mutex_destroy(&xxxxxx->mutex);

return ret;

}

static const struct of_device_id xxxxxx_dt_ids[] = {

{ .compatible = "yyyyyy,xxxxxx" },

{},

};

MODULE_DEVICE_TABLE(of, xxxxxx_dt_ids);

static struct i2c_driver xxxxxx_i2c_driver = {

.driver = {

.name = "xxxxxx",

.of_match_table = xxxxxx_dt_ids,

},

.probe_new = xxxxxx_probe,

.remove = xxxxxx_remove,

};

module_i2c_driver(xxxxxx_i2c_driver);

MODULE_AUTHOR("Zohar Golan");

MODULE_DESCRIPTION("yyyyyy xxxxxx sensor driver");

MODULE_LICENSE("GPL v2");

at boot time, I can see in dmesg that the driver is loading with no errors (I have some printk that I removed from the code above).

I can see there are some new entries in v4l2-ctl

root@imx8mp-var-dart:~# v4l2-ctl --list-devices
[ 102.334469] mxc_isi.1: is_entity_link_setup, No remote pad found!
():
/dev/v4l-subdev0

FSL Capture Media Device (platform:32c00000.bus:camera):
/dev/media0

mxc-isi-cap (platform:32e00000.isi:cap_devic):
/dev/video3

mxc-isi-m2m (platform:32e00000.isi:m2m_devic):
/dev/video2

mxc-isi-cap (platform:32e02000.isi:cap_devic):
/dev/video4

vsi_v4l2dec (platform:vsi_v4l2dec):
/dev/video1

vsi_v4l2enc (platform:vsi_v4l2enc):
/dev/video0

however when I am checking the formats of the different drivers, I don't see what I am expecting

root@imx8mp-var-dart:~# v4l2-ctl -d /dev/video3 --list-formats
ioctl: VIDIOC_ENUM_FMT
Type: Video Capture Multiplanar

[0]: 'RGBP' (16-bit RGB 5-6-5)
[1]: 'RGB3' (24-bit RGB 8-8-8)
[2]: 'BGR3' (24-bit BGR 8-8-8)
[3]: 'YUYV' (YUYV 4:2:2)
[4]: 'YUV4' (32-bit A/XYUV 8-8-8-8)
[5]: 'NV12' (Y/UV 4:2:0)
[6]: 'NM12' (Y/UV 4:2:0 (N-C))
[7]: 'YM24' (Planar YUV 4:4:4 (N-C))
[8]: 'XR24' (32-bit BGRX 8-8-8-8)
[9]: 'AR24' (32-bit BGRA 8-8-8-8)

I expected to see GRAY8 or Y8

Also when I am trying the following gstreamer command

root@imx8mp-var-dart:~# gst-launch-1.0 v4l2src device=/dev/video3 ! waylandsink
[ 359.543456] mxc_isi.1: is_entity_link_setup, No remote pad found!
[ 359.948288] mxc_isi.1: is_entity_link_setup, No remote pad found!
Setting pipeline to PAUSED ...
Pipeline is live and does not need PREROLL ...
Pipeline is PREROLLED ...
Setting pipeline to PLAYING ...
New clock: GstSystemClock
[ 359.989811] isi-capture 32e00000.isi:cap_device: mxc_isi_source_fmt_init: src:(1280,960), dst:(3840,2160) Not support upscale
ERROR: from element /GstPipeline:pipeline0/GstV4l2Src:v4l2src0: Failed to allocate required memory.
Additional debug info:
/usr/src/debug/gstreamer1.0-plugins-good/1.22.0.imx-r0/sys/v4l2/gstv4l2src.c(977): gst_v4l2src_decide_allocation (): /GstPipeline:pipeline0/GstV4l2Src:v4l2src0:
Buffer pool activation failed
Execution ended after 0:00:00.027122983
Setting pipeline to NULL ...
ERROR: from element /GstPipeline:pipeline0/GstV4l2Src:v4l2src0: Internal data stream error.
Additional debug info:
/usr/src/debug/gstreamer1.0/1.22.0.imx-r0/libs/gst/base/gstbasesrc.c(3132): gst_base_src_loop (): /GstPipeline:pipeline0/GstV4l2Src:v4l2src0:
streaming stopped, reason not-negotiated (-4)
Freeing pipeline ...

Zohar

[zohargolan]

also if I am sending the following command GST_DEBUG=3 gst-launch-1.0 -v v4l2src device=/dev/video3 ! video/x-raw,width=1280,height=960,framerate=90/1 ! waylandsink

I don't get an error, and the GStreamer is not stopping, but no video is shown

root@imx8mp-var-dart:~# GST_DEBUG=3 gst-launch-1.0 -v v4l2src device=/dev/video3 ! video/x-raw,width=1280,height=960,framerate=90/1 ! waylandsink
Setting pipeline to PAUSED ...
Pipeline is live and does not need PREROLL ...
0:00:00.086316000 1809 0xaaaad3a1de40 WARN v4l2 gstv4l2object.c:4726:gst_v4l2_object_get_crop_rect:<v4l2src0:src> Failed to get default crop rectangle with VIDIOC_G_SELECTION: Invalid argument
0:00:00.086430500 1809 0xaaaad3a1de40 WARN v4l2 gstv4l2object.c:4927:gst_v4l2_object_probe_caps:<v4l2src0:src> Failed to probe pixel aspect ratio with VIDIOC_CROPCAP: Invalid argument
Pipeline is PREROLLED ...
Setting pipeline to PLAYING ...
New clock: GstSystemClock
/GstPipeline:pipeline0/GstV4l2Src:v4l2src0.GstPad:src: caps = video/x-raw, width=(int)1280, height=(int)960, framerate=(fraction)90/1, format=(string)YUY2, interlace-mode=(string)progressive, colorimetry=(string)1:4:5:1
/GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:src: caps = video/x-raw, width=(int)1280, height=(int)960, framerate=(fraction)90/1, format=(string)YUY2, interlace-mode=(string)progressive, colorimetry=(string)1:4:5:1
/GstPipeline:pipeline0/GstWaylandSink:waylandsink0.GstPad:sink: caps = video/x-raw, width=(int)1280, height=(int)960, framerate=(fraction)90/1, format=(string)YUY2, interlace-mode=(string)progressive, colorimetry=(string)1:4:5:1
0:00:00.088664375 1809 0xaaaad3a1de40 WARN v4l2 gstv4l2object.c:4726:gst_v4l2_object_get_crop_rect:<v4l2src0:src> Failed to get default crop rectangle with VIDIOC_G_SELECTION: Invalid argument
/GstPipeline:pipeline0/GstCapsFilter:capsfilter0.GstPad:sink: caps = video/x-raw, width=(int)1280, height=(int)960, framerate=(fraction)90/1, format=(string)YUY2, interlace-mode=(string)progressive, colorimetry=(string)1:4:5:1
0:00:00.088779000 1809 0xaaaad3a1de40 WARN v4l2 gstv4l2object.c:3440:gst_v4l2_object_reset_compose_region:<v4l2src0:src> Failed to get default compose rectangle with VIDIOC_G_SELECTION: Invalid argument