First thing that you need in order to work with mipi camera on software level is kernel driver. Linux uses v4l2 layer in order to provide API for userspace applications. RockChip extends base driver requirements with couple vendor-specific requirements.

- private ioctl calls RKMODULE_GET_MODULE_INFO, RKMODULE_SET_QUICK_STREAM

- bunch of specific v4l2 callbacks, such as enum_bus_code, enum_frame_size, s_stream

Unfortunately, rockchip did not include imx477 driver with their linux 5.10 kernel,but luckily RPi has open source driver used on their boards.

Let's add couple missing parts.

V4L2 Driver

ioctl's

static void imx477_get_module_inf(struct imx477 *imx477,
                  struct rkmodule_inf *inf)
{
    printk(KERN_INFO "imx477: imx477_get_module_inf\n");

    memset(inf, 0, sizeof(*inf));
    strlcpy(inf->base.sensor, IMX477_NAME, sizeof(inf->base.sensor));
    strlcpy(inf->base.module, "IMX477", sizeof(inf->base.module));
    strlcpy(inf->base.lens, "Leica", sizeof(inf->base.lens));
}


static long imx477_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg)
{
    struct imx477 *imx477 = to_imx477(sd);
    u32 stream;
    long ret = 0;
    struct rkmodule_hdr_cfg *hdr;
    
    printk(KERN_INFO "imx477: imx477_ioctl cmd=%d\n", cmd);

    switch (cmd) {
    case RKMODULE_GET_MODULE_INFO:
        imx477_get_module_inf(imx477, (struct rkmodule_inf *)arg);
        break;
    case RKMODULE_SET_QUICK_STREAM:

        stream = *((u32 *)arg);
        if (stream)
            ret = imx477_write_reg(imx477, IMX477_REG_MODE_SELECT,
                IMX477_REG_VALUE_08BIT, IMX477_MODE_STREAMING);
        else
            ret = imx477_write_reg(imx477, IMX477_REG_MODE_SELECT,
                IMX477_REG_VALUE_08BIT, IMX477_MODE_STANDBY);
        break;
    default:
        ret = -ENOIOCTLCMD;
        break;
    }

    printk(KERN_INFO "imx477: imx477_ioctl => %ld\n", ret);

    return ret;
}

frame information callbacks

static int imx477_enum_frame_size(struct v4l2_subdev *sd,
                  struct v4l2_subdev_pad_config *cfg,
                  struct v4l2_subdev_frame_size_enum *fse)
{
    if (fse->index >= ARRAY_SIZE(supported_modes))
        return -EINVAL;
    fse->min_width  = supported_modes[fse->index].width;
    fse->max_width  = supported_modes[fse->index].width;
    fse->max_height = supported_modes[fse->index].height;
    fse->min_height = supported_modes[fse->index].height;
    return 0;
}

static int imx477_g_frame_interval(struct v4l2_subdev *sd,
                   struct v4l2_subdev_frame_interval *fi)
{
    struct imx477 *imx477 = to_imx477(sd);
    const struct imx477_mode *mode = imx477->mode;

    mutex_lock(&imx477->mutex);
    fi->interval = mode->timeperframe_min;
    mutex_unlock(&imx477->mutex);
    return 0;
}

In imx477_enum_frame_size you can see that when trying to access index out of bounds, driver returns EINVAL. That is made in order for the tool to keep calling imx477_enum_frame_size and incrementing index until it finally found boundary by EINVAL return code. The is no way for userspace to get all frame sizes at once.

That's pretty much it. ideally, the only thing left after patching driver is to build and "register" new device in linux kernel configuration using dts file (I'll describe this part later), but RPi driver implementation has couple things that I assume to be bugs and those bugs affect RKISP making it complain about his life.

First, driver defines crop section of 4056x3040 mode as following:

#define IMX477_NATIVE_WIDTH        4072U
#define IMX477_NATIVE_HEIGHT        3176U
#define IMX477_PIXEL_ARRAY_LEFT        8U
#define IMX477_PIXEL_ARRAY_TOP        16U


.crop = {
    .left = IMX477_PIXEL_ARRAY_LEFT,
    .top = IMX477_PIXEL_ARRAY_TOP,
    .width = 4056,
    .height = 3040,
}

That means, when being in full size mode, we are trying to crop 4056 + 8 pixels out of 4056 image size. IMX477_PIXEL_ARRAY_LEFT could be meaningful if we would use native width and height, but driver does not do that and instead requests only 4056 pixel width.

#define IMX477_NATIVE_WIDTH        4072U
#define IMX477_NATIVE_HEIGHT        3176U

Also the isp input requires 16 pixels width alignments, and the 8 pixels height alignments. Again, luckily, we can get appropriate implementation from imx577 driver code.

#define CROP_START(SRC, DST) (((SRC) - (DST)) / 2 / 4 * 4)
#define DST_WIDTH_4048 4048

static int imx477_get_selection(struct v4l2_subdev *sd,
                struct v4l2_subdev_pad_config *cfg,
                struct v4l2_subdev_selection *sel)
{
    struct imx477 *imx477 = to_imx477(sd);

    printk(KERN_INFO "imx477: imx477_get_selection\n");

    if (sel->target == V4L2_SEL_TGT_CROP_BOUNDS) {
        if (imx477->mode->width == 4056) {
            sel->r.left = CROP_START(imx477->mode->width, DST_WIDTH_4048);
            sel->r.width = DST_WIDTH_4048;
            sel->r.top = CROP_START(imx477->mode->height, imx477->mode->height);
            sel->r.height = imx477->mode->height;
        } else {
            sel->r.left = CROP_START(imx477->mode->width,
                            imx477->mode->width);
            sel->r.width = imx477->mode->width;
            sel->r.top = CROP_START(imx477->mode->height,
                            imx477->mode->height);
            sel->r.height = imx477->mode->height;
        }
        return 0;
    }

    return -EINVAL;
}

Complete driver code is available. here.

Device Tree Overlay

In order for camera driver to be loaded, we need to tell kernel when it should load it and what i2c parameters to use. In order to do that, DTS configuration exists.

Taking ov13850 configuration as reference, let's add configutation for imx477.

    imx477_2: imx477-2@1a {
        compatible = "sony,imx477";
        status = "disabled";
        reg = <0x1a>;
        clocks = <&cru CLK_MIPI_CAMARAOUT_M4>;
        clock-names = "xvclk";
                power-domains = <&power RK3588_PD_VI>;
        pinctrl-names = "default";
                pinctrl-0 = <&mipim0_camera4_clk>;
        rockchip,grf = <&sys_grf>;
        reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>;
        pwdn-gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_HIGH>;
        rockchip,camera-module-index = <0>;
        rockchip,camera-module-facing = "back";
        rockchip,camera-module-name = "CMK-CT0116";
        rockchip,camera-module-lens-name = "default";
        port {
            imx477_out: endpoint {
                remote-endpoint = <&mipi_in_cam2>;
                data-lanes = <1 2>;
            };
        };
    };

The only parameters that we should change are reg and *-gpios. reg is basically the i2c address of the module, it is usually defined by vendor in specification. gpios are just references to the io pins located on the board that we would like to use later in driver.

    /* Request optional enable pin */
    imx477->power_gpio = devm_gpiod_get_optional(dev, "reset",
                             GPIOD_OUT_HIGH);

    imx477->gpo_gpio = devm_gpiod_get_optional(dev, "pwdn",
                             GPIOD_OUT_HIGH);

The rest of the parameters either references to other devices or not so important in terms of initial driver configuration.

Both device tree file and device tree overlay are open-sourced too.

Testing

With all that done, we can finally get some images from the camera. At this point, we won't get any automatic exposure and gain control, same as control of white balance.

That is because we did not calibrate camera yet and rkISP does not "know" how to process the image and give feedback to the driver. Calibration process and tools will be described in the next articles.

Resources

- Rockchip_Driver_Guide_VI_EN_v1.1.1 - updated version of rockchip's guide with included paragraphs about rk3588 exclusive features.