chore(cleanup): clean up the T4Dataset 1.0 stack

docs/schema/v1/index.md

@@ -162,7 +162,7 @@ Provenance {
 
 Every 0.x frame becomes a `sample`: the files of all channels with the same frame number (e.g. `00042.jpg` and `00042.pcd.bin`, as the 0.x converter names them) form one frame. 0.x samples, which are annotated frames only, keep their tokens; 0.x samples of the same frame (one per camera image in camera-only datasets) are merged into the earliest one. A frame with a 2D label is a key frame.
 
-Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. The map is the dataset's own `map/` folder: `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`, and map element ids (e.g. lamp ways) refer to it. An empty `log.data_captured` becomes the log's first sample time (ISO 8601, UTC).
+Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 4: `equidistant`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. The map is the dataset's own `map/` folder: `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`, and map element ids (e.g. lamp ways) refer to it. An empty `log.data_captured` becomes the log's first sample time (ISO 8601, UTC).
 
 Labels are `manual` unless 0.x marks them as autolabel (`provenance` = model names). `instance.nbr_annotations` and its first / last annotation are computed over its 3D boxes and 2D labels; 0.x tools count them differently (some write 0 for 2D labels). Categories without `index` get their position in `category.json`, as the 0.x devkit reads them (all set or none). `surface_ann` rows become `object_ann` rows without an instance, with `bbox` computed from the mask. 0.x visibility `unavailable` (or any unknown level) becomes `visibility_token: null`, and the aliases `v0-40`, `v40-60`, `v60-80`, `v80-100` map to `none`, `partial`, `most`, `full`.
 

docs/schema/v1/table.md

@@ -68,7 +68,7 @@ sample_data {
   "sample_token":     <str> -- Foreign key to the `Sample` (frame) this observation belongs to.
   "ego_pose_token":   <str> -- Foreign key to the `EgoPose` table.
   "sensor_token":     <str> -- Foreign key to the `Sensor` table. Replaces `calibrated_sensor_token`.
-  "filename":         <option[str]> -- Path to the channel file (video or point cloud container), or to a per-frame file for datasets that keep 0.x files.
+  "filename":         <str> -- Path to the channel file (video or point cloud container), or to a per-frame file for datasets that keep 0.x files.
   "frame_index":      <option[int]> -- Index of the frame inside the channel file, starting at 0. `null` for per-frame files.
   "fileformat":       <enum["jpg", "png", "pcd", "bin", "pcd.bin", "av1_mp4", "parquet"]> -- Format of the file.
   "num_points":       <option[int]> -- Number of points in the frame. LiDAR only.

t4_devkit/cli/version.py

@@ -6,11 +6,10 @@
 import typer
 from rich.console import Console
 
-console = Console()
-
 
 def version_callback(ctx: typer.Context, value: bool) -> None:
     if value:
         version = importlib.metadata.version("t4-devkit")
+        console = Console()
         console.print(f"[bold green]{ctx.info_name}[/bold green]: [cyan]{version}[/cyan]")
         raise typer.Exit()

t4_devkit/common/io.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import threading
@@ -25,11 +26,12 @@
     "save_pointcloud",
     "load_image",
     "save_video",
+    "file_sha256",
     "V0_BIN_FIELDS",
 )
 
 # 0.x `.bin` layout: this list cut to `num_pts_feats`, all float32 (as written by the converter).
-# `.bin` stores no names; these are the 1.0 names (0.x calls `channel` "ring"), all float32.
+# `.bin` stores no names; these are the 1.0 names (0.x calls `channel` "ring").
 V0_BIN_FIELDS = ("x", "y", "z", "intensity", "channel", "return_type", "time_stamp")
 
 
@@ -58,6 +60,22 @@ def save_json(data: Any, filename: PathLike) -> None:
         json.dump(data, f, ensure_ascii=False, indent=4)
 
 
+def file_sha256(filepath: PathLike) -> str:
+    """Return the SHA-256 of a file, read in 1 MiB chunks (videos and rosbags can be large).
+
+    Args:
+        filepath (PathLike): File to hash.
+
+    Returns:
+        The hex digest.
+    """
+    digest = hashlib.sha256()
+    with open(os.fspath(filepath), "rb") as f:
+        for chunk in iter(lambda: f.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
 def load_pointcloud(
     filepath: PathLike,
     *,
@@ -293,7 +311,7 @@ def _add_video_stream(
     preset: int,
     crf: int,
 ) -> av.video.stream.VideoStream:
-    """The AV1 stream of a camera video, sized by its first image."""
+    """Return the AV1 stream of a camera video, sized by its first image."""
     stream = container.add_stream("libsvtav1", rate=frame_rate)
     stream.height, stream.width = image.shape[:2]
     stream.pix_fmt = pixel_format

t4_devkit/dataclass/pointcloud.py

@@ -28,9 +28,6 @@
     "Stamp",
 ]
 
-# Point fields of `LidarPointCloud.points`, in row order.
-XYZI = ("x", "y", "z", "intensity")
-
 
 @define
 class PointCloudSourceInfo:
@@ -274,9 +271,10 @@ def from_file(
                 raise ValueError(f"Unknown 0.x point layout with {num_pts_feats} fields")
             fields = [(name, "float32") for name in V0_BIN_FIELDS[:num_pts_feats]]
 
-        columns = load_pointcloud(filepath, frame_index=frame_index, fields=fields, columns=XYZI)
-        points = np.empty((len(XYZI), len(columns["x"])), dtype=np.float32)
-        for i, name in enumerate(XYZI):
+        xyzi = ("x", "y", "z", "intensity")  # rows of `points`
+        columns = load_pointcloud(filepath, frame_index=frame_index, fields=fields, columns=xyzi)
+        points = np.empty((len(xyzi), len(columns["x"])), dtype=np.float32)
+        for i, name in enumerate(xyzi):
             points[i] = columns[name]
         return cls(points, metainfo=metainfo)
 
@@ -409,8 +407,8 @@ class SegmentationPointCloud(PointCloud):
 
     Deprecated:
         Kept only for 0.x datasets, whose labels are a separate `.bin` file. In 1.0, labels are
-        columns of the LiDAR frame and come with `load_pointcloud()`. To be removed once
-        `T4Devkit` moves to 1.0.
+        columns of the LiDAR frame and come with `load_pointcloud()`; `T4Devkit` no longer
+        uses this class.
 
     Attributes:
         points (NDArrayFloat): Points matrix in the shape of (4, N).

t4_devkit/helper/rendering.py

@@ -65,7 +65,7 @@ def __init__(self, t4: T4Devkit) -> None:
         self._executor = concurrent.futures.ThreadPoolExecutor()
 
     def _first_sample_data_tokens(self, modality: SensorModality) -> list[str]:
-        """The first frame of each channel of a modality, in sensor order.
+        """Return the first frame of each channel of a modality, in sensor order.
 
         A recalibration adds a `sensor` row with the same channel, and source LiDARs of a merged
         channel have no frames, so channels are taken once and sensors without frames skipped.
@@ -581,7 +581,7 @@ def _render_annotation2ds(
             current_sample_token = sample.next
 
     def _annotation_sample(self, annotation_token: str | None) -> Sample | None:
-        """The sample of an instance's first or last annotation, or None if there is none.
+        """Return the sample of an instance's first or last annotation, or None if there is none.
 
         The annotation is a `sample_annotation`, or an `object_ann` for instances labeled only in
         2D, whose sample is that of its frame.

t4_devkit/sanity/checker.py

@@ -5,6 +5,7 @@
 from typing import TYPE_CHECKING, NewType
 
 from returns.maybe import Maybe, Nothing, Some
+from returns.pipeline import is_successful
 
 from t4_devkit.schema.v1 import SCHEMAS as V1_SCHEMAS
 
@@ -141,10 +142,12 @@ class RecordsChecker(ManifestChecker):
     tables: tuple[V1SchemaName, ...]
 
     def can_skip(self, context: SanityContext) -> Maybe[Reason]:
-        skip = super().can_skip(context)
-        if skip != Nothing:
-            return skip
-        invalid = [table.value for table in self.tables if context.load_records(table) == Nothing]
+        match super().can_skip(context):
+            case Some(reason):
+                return Some(reason)
+        invalid = [
+            table.value for table in self.tables if not is_successful(context.load_records(table))
+        ]
         if invalid:
             return Maybe.from_value(Reason(f"Tables not listed or not valid: {', '.join(invalid)}"))
         return Nothing

t4_devkit/sanity/context.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import hashlib
 from functools import cached_property
 from pathlib import Path
 
@@ -12,6 +11,7 @@
 from typing_extensions import TYPE_CHECKING, Self
 
 from t4_devkit import DBMetadata
+from t4_devkit.common.io import file_sha256
 from t4_devkit.lanelet import LaneletParser
 from t4_devkit.schema import SchemaName
 from t4_devkit.schema.v1 import MANIFEST_FILENAME, SCHEMAS, Manifest
@@ -104,8 +104,7 @@ def raw_manifest(self) -> Maybe[dict]:
         """Return `manifest.json` as read, or Nothing if it cannot be read."""
         match self.manifest_json:
             case Some(x) if x.exists():
-                result = load_json_safe(x)
-                return Some(result.unwrap()) if is_successful(result) else Nothing
+                return result_to_maybe(load_json_safe(x))
         return Nothing
 
     @cached_property
@@ -161,9 +160,7 @@ def save_table(self, table: V1SchemaName, records: list[dict]) -> bool:
         if filepath is None or not is_successful(save_json_safe(records, filepath)):
             return False
         manifest = self.raw_manifest.unwrap()
-        manifest["tables"][table.value]["sha256"] = hashlib.sha256(
-            filepath.read_bytes()
-        ).hexdigest()
+        manifest["tables"][table.value]["sha256"] = file_sha256(filepath)
         saved = is_successful(save_json_safe(manifest, self.manifest_json.unwrap()))
         # read again on next use
         self._tables.pop(table, None)

t4_devkit/sanity/format/base.py

@@ -69,9 +69,9 @@ class TableFieldChecker(ManifestChecker):
     table: V1SchemaName
 
     def can_skip(self, context: SanityContext) -> Maybe[Reason]:
-        skip = super().can_skip(context)
-        if skip != Nothing:
-            return skip
+        match super().can_skip(context):
+            case Some(reason):
+                return Some(reason)
         if (
             not self.table.is_optional()
             and self.table.value not in context.manifest.unwrap().tables

t4_devkit/sanity/record/rec012.py

@@ -31,7 +31,7 @@ class REC012(RecordsChecker):
     )
 
     def check(self, context: SanityContext) -> list[Reason] | None:
-        counts = _sample_counts(context)
+        counts = self._sample_counts(context)
         return [
             Reason(
                 f"Scene {scene.token}: 'nbr_samples' {scene.nbr_samples}, has {counts[scene.token]}"
@@ -41,14 +41,15 @@ def check(self, context: SanityContext) -> list[Reason] | None:
         ] or None
 
     def fix(self, context: SanityContext) -> bool:
-        counts = _sample_counts(context)
+        counts = self._sample_counts(context)
         records = context.load_table(SchemaName.SCENE).unwrap()
         for record in records:
             record["nbr_samples"] = counts[record["token"]]
         return context.save_table(SchemaName.SCENE, records)
 
-
-def _sample_counts(context: SanityContext) -> Counter[str]:
-    return Counter(
-        sample.scene_token for sample in context.load_records(SchemaName.SAMPLE).unwrap()
-    )
+    @staticmethod
+    def _sample_counts(context: SanityContext) -> Counter[str]:
+        """Return the number of samples of each scene."""
+        return Counter(
+            sample.scene_token for sample in context.load_records(SchemaName.SAMPLE).unwrap()
+        )

t4_devkit/sanity/record/rec013.py

@@ -2,9 +2,7 @@
 
 from typing import TYPE_CHECKING
 
-from collections import defaultdict
-
-from t4_devkit.schema.v1 import SchemaName
+from t4_devkit.schema.v1 import Instance, SchemaName
 
 from ..checker import RecordsChecker, RuleID, RuleName, Severity
 from ..registry import CHECKERS
@@ -37,7 +35,7 @@ class REC013(RecordsChecker):
     )
 
     def check(self, context: SanityContext) -> list[Reason] | None:
-        expected = _expected_annotations(context)
+        expected = self._expected(context)
         failures = []
         for instance in context.load_records(SchemaName.INSTANCE).unwrap():
             count, first, last = expected.get(instance.token, (0, None, None))
@@ -58,7 +56,7 @@ def check(self, context: SanityContext) -> list[Reason] | None:
         return failures or None
 
     def fix(self, context: SanityContext) -> bool:
-        expected = _expected_annotations(context)
+        expected = self._expected(context)
         records = context.load_table(SchemaName.INSTANCE).unwrap()
         for record in records:
             count, first, last = expected.get(record["token"], (0, None, None))
@@ -67,21 +65,19 @@ def fix(self, context: SanityContext) -> bool:
             record["last_annotation_token"] = last
         return context.save_table(SchemaName.INSTANCE, records)
 
-
-def _expected_annotations(context: SanityContext) -> dict[str, tuple[int, str, str]]:
-    """Return the count, first and last annotation token in time of each instance with annotations.
-
-    A `SampleAnnotation` is at its sample's time, an `ObjectAnn` at its frame's time.
-    """
-    samples = {sample.token: sample for sample in context.load_records(SchemaName.SAMPLE).unwrap()}
-    frames = {frame.token: frame for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap()}
-    times: dict[str, list[tuple[int, str]]] = defaultdict(list)
-    for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap():
-        if ann.sample_token in samples:
-            times[ann.instance_token].append((samples[ann.sample_token].timestamp, ann.token))
-    for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap():
-        if ann.instance_token is not None and ann.sample_data_token in frames:
-            times[ann.instance_token].append((frames[ann.sample_data_token].timestamp, ann.token))
-    return {
-        token: (len(entries), min(entries)[1], max(entries)[1]) for token, entries in times.items()
-    }
+    @staticmethod
+    def _expected(context: SanityContext) -> dict[str, tuple[int, str, str]]:
+        """Return the count and first / last label token of each instance with labels."""
+        samples = {s.token: s for s in context.load_records(SchemaName.SAMPLE).unwrap()}
+        frames = {f.token: f for f in context.load_records(SchemaName.SAMPLE_DATA).unwrap()}
+        boxes = [
+            (ann.instance_token, samples[ann.sample_token].timestamp, ann.token)
+            for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap()
+            if ann.sample_token in samples
+        ]
+        labels_2d = [
+            (ann.instance_token, frames[ann.sample_data_token].timestamp, ann.token)
+            for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap()
+            if ann.instance_token is not None and ann.sample_data_token in frames
+        ]
+        return Instance.annotation_spans(boxes + labels_2d)

t4_devkit/sanity/record/rec019.py

@@ -37,20 +37,14 @@ class REC019(RecordsChecker):
 
     def check(self, context: SanityContext) -> list[Reason] | None:
         sensors = context.load_records(SchemaName.SENSOR).unwrap()
-        # source LiDARs of a merged channel have no frames of their own
-        sources = {
-            channel
-            for sensor in sensors
-            if sensor.lidar is not None
-            for channel in sensor.lidar.source_channels
-        }
         indices: dict[str, list[int | None]] = defaultdict(list)
         for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap():
             indices[frame.sensor_token].append(frame.frame_index)
         failures = []
         for sensor in sensors:
             stored = channel_file(sensor)
-            if stored is None or sensor.channel in sources:
+            # per-frame files, and source LiDARs of a merged channel (no file of their own)
+            if stored is None:
                 continue
             _, frame_count = stored
             if sorted(indices[sensor.token], key=lambda i: -1 if i is None else i) != list(

t4_devkit/sanity/record/rec022.py

@@ -38,29 +38,25 @@ class REC022(RecordsChecker):
     )
 
     def check(self, context: SanityContext) -> list[Reason] | None:
+        # 2000-01-01 and 2100-01-01 in Unix microseconds
+        low, high = 946_684_800_000_000, 4_102_444_800_000_000
+        fields_of = {
+            SchemaName.SAMPLE: ("timestamp",),
+            SchemaName.SAMPLE_DATA: ("timestamp",),
+            SchemaName.EGO_POSE: ("timestamp",),
+            SchemaName.VEHICLE_STATE: ("timestamp",),
+            SchemaName.TRAFFIC_LIGHT: ("timestamp",),
+            SchemaName.ROUTE: ("valid_from", "valid_until"),
+            SchemaName.SCENARIO: ("start_timestamp", "end_timestamp", "key_timestamp"),
+            SchemaName.CAPTION: ("start_timestamp", "end_timestamp"),
+        }
         failures = []
-        for table, fields in TIMESTAMP_FIELDS.items():
+        for table in self.tables:
             for record in context.load_records(table).unwrap():
-                for name in fields:
+                for name in fields_of[table]:
                     value = getattr(record, name)
-                    if value is not None and not MIN_TIMESTAMP <= value < MAX_TIMESTAMP:
+                    if value is not None and not low <= value < high:
                         failures.append(
                             Reason(f"{record_name(table)} {record.token}: '{name}' {value}")
                         )
         return failures or None
-
-
-# 2000-01-01 and 2100-01-01 in Unix microseconds.
-MIN_TIMESTAMP = 946_684_800_000_000
-MAX_TIMESTAMP = 4_102_444_800_000_000
-
-TIMESTAMP_FIELDS = {
-    SchemaName.SAMPLE: ("timestamp",),
-    SchemaName.SAMPLE_DATA: ("timestamp",),
-    SchemaName.EGO_POSE: ("timestamp",),
-    SchemaName.VEHICLE_STATE: ("timestamp",),
-    SchemaName.TRAFFIC_LIGHT: ("timestamp",),
-    SchemaName.ROUTE: ("valid_from", "valid_until"),
-    SchemaName.SCENARIO: ("start_timestamp", "end_timestamp", "key_timestamp"),
-    SchemaName.CAPTION: ("start_timestamp", "end_timestamp"),
-}

t4_devkit/sanity/record/rec027.py

@@ -42,11 +42,9 @@ def check(self, context: SanityContext) -> list[Reason] | None:
             for route in context.load_records(SchemaName.ROUTE).unwrap()
             for pose in (route.start_pose, route.goal_pose)
         ]
+        tolerance = 1e-3
         return [
             Reason(f"{table} {token}: rotation norm {rotation.norm:.6f}")
             for table, token, rotation in rotations
-            if abs(rotation.norm - 1.0) > NORM_TOLERANCE
+            if abs(rotation.norm - 1.0) > tolerance
         ] or None
-
-
-NORM_TOLERANCE = 1e-3