"""
Camera Module
=============
This module provides comprehensive data structures for representing camera sensors,
including intrinsic parameters, distortion models, rolling shutter characteristics,
and captured image data.
The camera models are aligned with COLMAP conventions for compatibility with
3D reconstruction pipelines.
Classes
-------
DistortionParameters
Lens distortion coefficients for various camera models.
CameraIntrinsics
Internal camera parameters (focal length, principal point, image size).
RollingShutterParameters
Rolling shutter timing and direction configuration.
ImageData
Metadata and optional pixel data for a single captured image.
Camera
Complete camera sensor representation with calibration and captured data.
Camera Models
-------------
Supported camera models (aligned with COLMAP):
- ``simple_pinhole``: fx, cx, cy (fx = fy)
- ``pinhole``: fx, fy, cx, cy
- ``simple_radial``: fx, cx, cy, k1
- ``radial``: fx, cx, cy, k1, k2
- ``opencv``: fx, fy, cx, cy, k1, k2, p1, p2
- ``opencv_fisheye``: fx, fy, cx, cy, k1, k2, k3, k4
- ``equirectangular``: For 360° cameras
- ``omnidirectionalstereo_l/r``: ODS stereo projection
- ``vr180_l/r``: VR180 format
- ``orthophoto``: Orthographic projection
- ``fisheye624``: 6+2+4 parameter fisheye model
See Also
--------
- COLMAP camera models: https://colmap.github.io/cameras.html
- OpenCV camera calibration: https://docs.opencv.org/4.x/dc/dbb/tutorial_py_calibration.html
Example
-------
Creating a camera sensor::
import numpy as np
from mimic_data_api.camera import (
Camera, CameraIntrinsics, DistortionParameters, ImageData
)
from mimic_data_api.trajectory_numpy import WorldTrajectory
camera = Camera(
id=1,
name="front_camera",
camera_model="opencv",
intrinsics=CameraIntrinsics(
fx=1000.0, fy=1000.0,
cx=640.0, cy=360.0,
skew=0.0,
width=1280.0, height=720.0
),
distortion_parameters=DistortionParameters(k1=0.1, k2=0.01),
extrinsics=np.eye(4),
trajectory=trajectory,
images_data=[ImageData(timestamp=0.0, filepath="img_0.jpg", undistorded=False)]
)
"""
from __future__ import annotations
from typing import Literal
from pydantic import BaseModel, Field, field_validator, model_validator
import numpy as np
from mimic_data_api.extrinsic import SensorExtrinsic
from mimic_data_api.trajectory_numpy import WorldTrajectory
from mimic_data_api.unstructured.images import (
RGBImage,
DepthImage,
NormalImage,
SegmentationImage,
)
# Camera Models aligned with colmap
# https://colmap.github.io/cameras.html
# https://github.com/colmap/colmap/blob/main/src/colmap/sensor/models.h
CameraModels = Literal[
"simple_pinhole",
"pinhole",
"simple_radial",
"radial",
"opencv",
"opencv_fisheye",
"equirectangular",
"omnidirectionalstereo_l",
"omnidirectionalstereo_r",
"vr180_l",
"vr180_r",
"orthophoto",
"fisheye624",
"mei",
]
[docs]
class DistortionParameters(BaseModel):
"""Lens distortion coefficients for camera calibration.
This class supports various distortion models including standard OpenCV
radial-tangential distortion, fisheye models, and extended models with
thin prism distortion.
The distortion is typically applied to normalized image coordinates before
projection through the intrinsic matrix.
Attributes
----------
p1 : float
First tangential distortion coefficient. Default: 0.0
p2 : float
Second tangential distortion coefficient. Default: 0.0
k1 : float
First radial distortion coefficient. Default: 0.0
k2 : float
Second radial distortion coefficient. Default: 0.0
k3 : float
Third radial distortion coefficient. Default: 0.0
k4 : float
Fourth radial distortion coefficient (fisheye models). Default: 0.0
k5 : float or None
Fifth radial distortion coefficient (FullOpenCV model). Optional.
k6 : float or None
Sixth radial distortion coefficient (FullOpenCV model). Optional.
k0 : float or None
Zeroth radial coefficient (RadTanThinPrismFisheye model). Optional.
p0 : float or None
Zeroth tangential coefficient (RadTanThinPrismFisheye model). Optional.
s0 : float or None
Zeroth thin prism distortion coefficient. Optional.
s1 : float or None
First thin prism distortion coefficient. Optional.
s2 : float or None
Second thin prism distortion coefficient. Optional.
s3 : float or None
Third thin prism distortion coefficient. Optional.
sx1 : float or None
First thin prism x-coefficient (ThinPrismFisheye model). Optional.
sy1 : float or None
First thin prism y-coefficient (ThinPrismFisheye model). Optional.
xi : float or None
Projection parameter (Mei / Omnidirectional models). Optional.
Note
----
The standard OpenCV distortion model uses:
.. math::
x_{distorted} = x(1 + k_1 r^2 + k_2 r^4 + k_3 r^6) + 2p_1 xy + p_2(r^2 + 2x^2)
y_{distorted} = y(1 + k_1 r^2 + k_2 r^4 + k_3 r^6) + p_1(r^2 + 2y^2) + 2p_2 xy
where :math:`r^2 = x^2 + y^2`.
Example
-------
>>> distortion = DistortionParameters(k1=-0.1, k2=0.01, p1=0.001, p2=-0.001)
"""
p1: float = 0.0
p2: float = 0.0
k1: float = 0.0
k2: float = 0.0
k3: float = 0.0
k4: float = Field(
default=0.0,
description="k4 is depending on camera model, used in some models like FullOpenCV or OpencvFisheye",
)
k5: float | None = Field(
default=None, description="k5 is optional, used in some models like FullOpenCV"
)
k6: float | None = Field(
default=None, description="k6 is optional, used in some models like FullOpenCV"
)
k0: float | None = Field(
default=None,
description="k0 is optional, used in some models like RadTanThinPrismFisheye",
)
p0: float | None = Field(
default=None,
description="p0 is optional, used in some models like RadTanThinPrismFisheye",
)
s0: float | None = Field(
default=None,
description="s0 is optional, used in some models like RadTanThinPrismFisheye",
)
s1: float | None = Field(
default=None,
description="s1 is optional, used in some models like RadTanThinPrismFisheye",
)
s2: float | None = Field(
default=None,
description="s2 is optional, used in some models like RadTanThinPrismFisheye",
)
s3: float | None = Field(
default=None,
description="s3 is optional, used in some models like RadTanThinPrismFisheye",
)
sx1: float | None = Field(
default=None,
description="sx1 is optional, used in some models like ThinPrismFisheye",
)
sy1: float | None = Field(
default=None,
description="sx1 is optional, used in some models like ThinPrismFisheye",
)
xi: float | None = Field(
default=None,
description="xi is optional, used in some models like Mei",
)
[docs]
class CameraIntrinsics(BaseModel):
"""Internal camera calibration parameters.
This class encapsulates the intrinsic camera matrix parameters that define
the projection from 3D camera coordinates to 2D image coordinates.
Attributes
----------
fx : float
Focal length in pixels along the x-axis.
fy : float
Focal length in pixels along the y-axis.
cx : float
Principal point x-coordinate in pixels (typically image center).
cy : float
Principal point y-coordinate in pixels (typically image center).
skew : float
Skew coefficient (typically 0 for modern cameras).
height : float
Image height in pixels.
width : float
Image width in pixels.
Note
----
The intrinsic matrix K is structured as:
.. math::
K = \\begin{bmatrix}
f_x & skew & c_x \\\\
0 & f_y & c_y \\\\
0 & 0 & 1
\\end{bmatrix}
Example
-------
>>> intrinsics = CameraIntrinsics(
... fx=1000.0, fy=1000.0,
... cx=640.0, cy=360.0,
... skew=0.0,
... width=1280.0, height=720.0
... )
>>> K = intrinsics.as_K()
>>> print(K.shape)
(3, 3)
"""
fx: float
fy: float
cx: float
cy: float
skew: float
height: float
width: float
[docs]
def as_K(self, *, dtype=np.float32) -> np.ndarray:
"""Return the 3x3 intrinsic matrix for pinhole-like models.
Constructs the camera calibration matrix K that projects 3D points
in camera coordinates to 2D homogeneous image coordinates.
Parameters
----------
dtype : numpy.dtype, optional
Data type for the output array. Default: ``np.float32``
Returns
-------
numpy.ndarray
A 3x3 intrinsic calibration matrix K.
Example
-------
>>> intrinsics = CameraIntrinsics(
... fx=500.0, fy=500.0, cx=320.0, cy=240.0,
... skew=0.0, width=640.0, height=480.0
... )
>>> K = intrinsics.as_K()
>>> # Project a 3D point at (1, 0, 5) in camera coordinates
>>> point_3d = np.array([1.0, 0.0, 5.0])
>>> point_2d_homogeneous = K @ point_3d
>>> point_2d = point_2d_homogeneous[:2] / point_2d_homogeneous[2]
"""
K = np.array(
[[self.fx, self.skew, self.cx], [0.0, self.fy, self.cy], [0.0, 0.0, 1.0]],
dtype=dtype,
)
return K
[docs]
class RollingShutterParameters(BaseModel):
"""Rolling shutter timing and direction parameters.
Defines the rolling shutter behavior for cameras that don't capture
all pixels simultaneously. Essential for accurate motion compensation
in dynamic scenes.
Attributes
----------
readout_time : float
Total time to read out the entire sensor in seconds.
A value of 0.0 indicates a global shutter. Default: 0.0
readout_direction : {"up_to_down", "down_to_up", "left_to_right", "right_to_left", "global"}
Direction in which sensor rows/columns are read out.
``"global"`` indicates simultaneous readout (global shutter). Default: ``"global"``
Note
----
For rolling shutter cameras, different rows are captured at slightly
different times. The time offset for row ``r`` is:
.. math::
t_{row} = t_{frame} + \\frac{r}{H} \\cdot t_{readout}
where ``H`` is the image height and the direction is top-to-bottom.
Example
-------
>>> rs = RollingShutterParameters(
... readout_time=0.033, # 33ms readout
... readout_direction="up_to_down"
... )
"""
readout_time: float = Field(
default=0.0,
description="Readout time in seconds. If 0, the camera is assumed to have a global shutter.",
)
readout_direction: Literal[
"up_to_down", "down_to_up", "left_to_right", "right_to_left", "global"
] = "global"
[docs]
class ImageData(BaseModel):
"""Metadata and optional pixel data for a captured camera image.
This class stores information about individual images captured by a camera,
including the file location, timestamp, and processing state.
Attributes
----------
filepath : str
Path to the image file. Must be PNG or JPEG format.
timestamp : float
Capture timestamp in seconds (must be >= 0).
undistorded : bool
``True`` if the image has been undistorted (lens distortion removed),
``False`` if it contains original lens distortion.
rolling_shutter : RollingShutterParameters
Rolling shutter parameters for this specific image.
Default: global shutter (no rolling shutter effect).
image : RGBImage or None
Optional in-memory image data. Not serialized to JSON.
Raises
------
ValueError
If filepath doesn't end with .png, .jpg, or .jpeg.
Example
-------
>>> img_data = ImageData(
... filepath="data/images/frame_000.png",
... timestamp=0.0,
... undistorded=False
... )
"""
filepath: str = Field(description="Filepath to the image, expecting png or jpg.")
timestamp: float = Field(ge=0, description="Timestamp of the image in seconds.")
undistorded: bool = Field(description="Whether the image is undistorted or not.")
rolling_shutter: RollingShutterParameters = Field(
default=RollingShutterParameters()
)
image: RGBImage | None = Field(
default=None,
exclude=True,
description="Image data, will not be serialized in json.",
)
semantic_segmentation_filepath: str | None = Field(default=None)
semantic_segmentation_image: SegmentationImage | None = Field(
default=None, exclude=True
)
instance_segmentation_filepath: str | None = Field(default=None)
instance_segmentation_image: SegmentationImage | None = Field(
default=None, exclude=True
)
lidar_depth_filepath: str | None = Field(default=None)
lidar_depth_image: DepthImage | None = Field(default=None, exclude=True)
monodepth_filepath: str | None = Field(default=None)
monodepth_image: DepthImage | None = Field(default=None, exclude=True)
normal_filepath: str | None = Field(default=None)
normal_segmentation_image: NormalImage | None = Field(default=None, exclude=True)
wb_params: list[float] | None = Field(default=None)
brightness_control: float | None = Field(default=None)
dynamic_mask_filepath: str | None = Field(default=None)
dynamic_mask_image: SegmentationImage | None = Field(
default=None, exclude=True
)
mask_filepath: str | None = Field(default=None)
mask_image: SegmentationImage | None = Field(default=None, exclude=True)
vignetting_mask_filepath: str | None = Field(default=None)
vignetting_mask_image: SegmentationImage | None = Field(
default=None, exclude=True
)
[docs]
@field_validator("filepath")
def validate_filepath(cls, v: str) -> str:
"""Validate that the filepath has a supported image extension.
Parameters
----------
v : str
The filepath to validate.
Returns
-------
str
The validated filepath.
Raises
------
ValueError
If the filepath doesn't end with a supported extension.
"""
if not v.lower().endswith((".png", ".jpg", ".jpeg")):
raise ValueError("Filepath must end with .png, .jpg, or .jpeg")
return v
[docs]
class Camera(BaseModel):
"""Complete camera sensor representation.
This class encapsulates all information about a camera sensor including
its calibration (intrinsic and extrinsic), projection model, trajectory,
and captured image data.
Attributes
----------
id : int
Unique identifier for the camera within the sensor suite.
name : str
Human-readable name for the camera (e.g., "front_left", "rear").
camera_model : CameraModels
Projection model type (e.g., "pinhole", "opencv", "opencv_fisheye").
distortion_parameters : DistortionParameters
Lens distortion coefficients for the camera model.
extrinsics : SensorExtrinsic
4x4 transformation matrix from sensor frame to vehicle frame.
trajectory : WorldTrajectory
Time-series of camera poses in world coordinates.
intrinsics : CameraIntrinsics
Internal camera parameters (focal length, principal point, image size).
images_data : list of ImageData
List of captured images with their metadata.
Raises
------
ValueError
If image timestamps don't match trajectory timestamps.
Note
----
The ``images_data`` timestamps must exactly match the ``trajectory.timestamps``
array. This ensures consistent indexing between poses and images.
Example
-------
>>> camera = Camera(
... id=0,
... name="front_camera",
... camera_model="pinhole",
... distortion_parameters=DistortionParameters(),
... extrinsics=np.eye(4),
... trajectory=trajectory,
... intrinsics=CameraIntrinsics(
... fx=500, fy=500, cx=320, cy=240,
... skew=0, width=640, height=480
... ),
... images_data=[ImageData(timestamp=0.0, filepath="img.png", undistorded=True)]
... )
"""
id: int
name: str
camera_model: CameraModels
distortion_parameters: DistortionParameters
extrinsics: SensorExtrinsic = Field(
description="Extrinsic calibration from sensor to vehicle frame"
)
trajectory: WorldTrajectory
intrinsics: CameraIntrinsics = Field(
description="Camera intrinsics including focal lengths, principal point, and image size."
)
images_data: list[ImageData] = Field(
description="List of images data captured by the camera."
)
[docs]
@model_validator(mode="after")
def validate_images_trajectory_timestamps(self) -> Camera:
"""Validate that image timestamps match trajectory timestamps.
Returns
-------
Camera
The validated camera instance.
Raises
------
ValueError
If image timestamps don't match trajectory timestamps.
"""
image_timestamps = [image.timestamp for image in self.images_data]
if image_timestamps != self.trajectory.timestamps.tolist():
raise ValueError("Image timestamps should match trajectory timestamps.")
return self
[docs]
def __eq__(self, other: object) -> bool:
"""Check equality with another Camera.
Performs deep comparison of all camera attributes including
calibration parameters, trajectory, and image data.
Parameters
----------
other : object
The object to compare against.
Returns
-------
bool
``True`` if cameras are equal, ``False`` otherwise.
"""
if not isinstance(other, Camera):
return False
if self.id != other.id:
return False
if self.name != other.name:
return False
if self.camera_model != other.camera_model:
return False
if self.distortion_parameters != other.distortion_parameters:
return False
if not np.allclose(self.extrinsics, other.extrinsics):
return False
if self.trajectory != other.trajectory:
return False
if self.images_data != other.images_data:
return False
if self.intrinsics != other.intrinsics:
return False
return True
if __name__ == "__main__":
print(Camera.model_json_schema())
import numpy as np
cam = Camera(
id=1,
name="Test Camera",
camera_model="pinhole",
distortion_parameters=DistortionParameters(p1=0.1, p2=0.1, k1=0.01, k2=0.01),
# rolling_shutter=False,
intrinsics=CameraIntrinsics(
fx=1000.0,
fy=1000.0,
cx=640.0,
cy=360.0,
skew=0.0,
height=720.0,
width=1280.0,
),
trajectory=WorldTrajectory(
timestamps=np.arange(0, 10, dtype=float),
object2world=np.array(10 * np.eye(4).tolist()).reshape(10, 4, 4),
),
images_data=[
ImageData(timestamp=0, filepath="path/to/image1.jpg", undistorded=True),
ImageData(timestamp=1, filepath="path/to/image2.jpg", undistorded=True),
ImageData(timestamp=2, filepath="path/to/image3.jpg", undistorded=True),
ImageData(timestamp=3, filepath="path/to/image4.jpg", undistorded=True),
ImageData(timestamp=4, filepath="path/to/image5.jpg", undistorded=True),
ImageData(timestamp=5, filepath="path/to/image6.jpg", undistorded=True),
ImageData(timestamp=6, filepath="path/to/image7.jpg", undistorded=True),
ImageData(timestamp=7, filepath="path/to/image8.jpg", undistorded=True),
ImageData(timestamp=8, filepath="path/to/image9.jpg", undistorded=True),
ImageData(timestamp=9, filepath="path/to/image10.jpg", undistorded=True),
],
extrinsics=np.eye(4),
)
print(cam)
json_dump = cam.model_dump_json(indent=2)
cam_round_trip = Camera.model_validate_json(json_dump)
assert cam_round_trip == cam, "Round trip validation failed"
print("Camera model validation successful")