From a1ece120809a992f0e56a4de3dfebc1e1c1ce502 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 16:50:48 +0900 Subject: [PATCH 01/10] chore: fix stale notes and remove dead code --- docs/schema/v1/index.md | 2 +- t4_devkit/common/io.py | 2 +- t4_devkit/dataclass/pointcloud.py | 4 ++-- t4_devkit/sanity/record/rec019.py | 10 ++-------- t4_devkit/sanity/reference/files.py | 7 ++++++- t4_devkit/schema/v1/convert_v0.py | 4 +--- 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/docs/schema/v1/index.md b/docs/schema/v1/index.md index 3a1a1bb6..713d2b5f 100644 --- a/docs/schema/v1/index.md +++ b/docs/schema/v1/index.md @@ -162,7 +162,7 @@ Provenance { Every 0.x frame becomes a `sample`: the files of all channels with the same frame number (e.g. `00042.jpg` and `00042.pcd.bin`, as the 0.x converter names them) form one frame. 0.x samples, which are annotated frames only, keep their tokens; 0.x samples of the same frame (one per camera image in camera-only datasets) are merged into the earliest one. A frame with a 2D label is a key frame. -Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. The map is the dataset's own `map/` folder: `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`, and map element ids (e.g. lamp ways) refer to it. An empty `log.data_captured` becomes the log's first sample time (ISO 8601, UTC). +Each 0.x `calibrated_sensor` row becomes a `sensor` row, keeping its token. The camera block takes its intrinsics and distortion from it, its size from `sample_data`, and the distortion model from the number of coefficients (0: `none`, 4: `equidistant`, 5: `plumb_bob`, 8: `rational_polynomial`). The LiDAR block is described from the first frame. The map is the dataset's own `map/` folder: `sha256` and `projector` are read from `map/lanelet2_map.osm` and `map/map_projector_info.yaml`, and map element ids (e.g. lamp ways) refer to it. An empty `log.data_captured` becomes the log's first sample time (ISO 8601, UTC). Labels are `manual` unless 0.x marks them as autolabel (`provenance` = model names). `instance.nbr_annotations` and its first / last annotation are computed over its 3D boxes and 2D labels; 0.x tools count them differently (some write 0 for 2D labels). Categories without `index` get their position in `category.json`, as the 0.x devkit reads them (all set or none). `surface_ann` rows become `object_ann` rows without an instance, with `bbox` computed from the mask. 0.x visibility `unavailable` (or any unknown level) becomes `visibility_token: null`, and the aliases `v0-40`, `v40-60`, `v60-80`, `v80-100` map to `none`, `partial`, `most`, `full`. diff --git a/t4_devkit/common/io.py b/t4_devkit/common/io.py index 866b4b06..413f3772 100644 --- a/t4_devkit/common/io.py +++ b/t4_devkit/common/io.py @@ -29,7 +29,7 @@ ) # 0.x `.bin` layout: this list cut to `num_pts_feats`, all float32 (as written by the converter). -# `.bin` stores no names; these are the 1.0 names (0.x calls `channel` "ring"), all float32. +# `.bin` stores no names; these are the 1.0 names (0.x calls `channel` "ring"). V0_BIN_FIELDS = ("x", "y", "z", "intensity", "channel", "return_type", "time_stamp") diff --git a/t4_devkit/dataclass/pointcloud.py b/t4_devkit/dataclass/pointcloud.py index f503382a..436a0a52 100644 --- a/t4_devkit/dataclass/pointcloud.py +++ b/t4_devkit/dataclass/pointcloud.py @@ -409,8 +409,8 @@ class SegmentationPointCloud(PointCloud): Deprecated: Kept only for 0.x datasets, whose labels are a separate `.bin` file. In 1.0, labels are - columns of the LiDAR frame and come with `load_pointcloud()`. To be removed once - `T4Devkit` moves to 1.0. + columns of the LiDAR frame and come with `load_pointcloud()`; `T4Devkit` no longer + uses this class. Attributes: points (NDArrayFloat): Points matrix in the shape of (4, N). diff --git a/t4_devkit/sanity/record/rec019.py b/t4_devkit/sanity/record/rec019.py index c59ac5b3..90e35df7 100644 --- a/t4_devkit/sanity/record/rec019.py +++ b/t4_devkit/sanity/record/rec019.py @@ -37,20 +37,14 @@ class REC019(RecordsChecker): def check(self, context: SanityContext) -> list[Reason] | None: sensors = context.load_records(SchemaName.SENSOR).unwrap() - # source LiDARs of a merged channel have no frames of their own - sources = { - channel - for sensor in sensors - if sensor.lidar is not None - for channel in sensor.lidar.source_channels - } indices: dict[str, list[int | None]] = defaultdict(list) for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap(): indices[frame.sensor_token].append(frame.frame_index) failures = [] for sensor in sensors: stored = channel_file(sensor) - if stored is None or sensor.channel in sources: + # per-frame files, and source LiDARs of a merged channel (no file of their own) + if stored is None: continue _, frame_count = stored if sorted(indices[sensor.token], key=lambda i: -1 if i is None else i) != list( diff --git a/t4_devkit/sanity/reference/files.py b/t4_devkit/sanity/reference/files.py index dd66cc8e..d0c97c95 100644 --- a/t4_devkit/sanity/reference/files.py +++ b/t4_devkit/sanity/reference/files.py @@ -16,7 +16,12 @@ from t4_devkit.schema.v1 import Sensor -__all__ = ["describe_lidar_file", "describe_video_file", "lidar_column_values"] +__all__ = [ + "describe_lidar_file", + "describe_video_file", + "lidar_column_values", + "lidar_partly_null_frames", +] def describe_lidar_file(sensor: Sensor, path: Path) -> list[Reason]: diff --git a/t4_devkit/schema/v1/convert_v0.py b/t4_devkit/schema/v1/convert_v0.py index a0937bcb..402721bb 100644 --- a/t4_devkit/schema/v1/convert_v0.py +++ b/t4_devkit/schema/v1/convert_v0.py @@ -502,7 +502,6 @@ def _frame_samples(self) -> tuple[list[Record], dict[str, str], dict[str, str]]: `sample_data` token, and the token kept for each merged 0.x sample. """ v0_samples = {record["token"]: record for record in self.v0["sample"]} - annotated = {record["sample_token"] for record in self.v0["sample_annotation"]} lidars = { calibration["token"] for calibration in self.v0["calibrated_sensor"] @@ -540,8 +539,7 @@ def _frame_samples(self) -> tuple[list[Record], dict[str, str], dict[str, str]]: "token": token, "timestamp": min(frame["timestamp"] for frame in lidar_frames or frames), "scene_token": scene_token, - "has_key_frame": any(self._is_key_frame(frame) for frame in frames) - or bool(annotated & set(key_samples)), + "has_key_frame": any(self._is_key_frame(frame) for frame in frames), "next": None, "prev": None, } From 56960c3fa84e1e13a63071c31855de56aee8063e Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 16:59:31 +0900 Subject: [PATCH 02/10] refactor: one implementation for file hashes, instance label counts and asset folders --- t4_devkit/common/io.py | 18 ++++++++++++ t4_devkit/sanity/context.py | 6 ++-- t4_devkit/sanity/record/rec013.py | 42 +++++++++++++--------------- t4_devkit/sanity/reference/ref207.py | 5 ++-- t4_devkit/sanity/structure/str012.py | 14 ++-------- t4_devkit/sanity/structure/str013.py | 7 ++--- t4_devkit/schema/v1/convert_v0.py | 37 ++++++++++-------------- t4_devkit/schema/v1/instance.py | 27 ++++++++++++++++++ t4_devkit/schema/v1/manifest.py | 5 +++- t4_devkit/schema/v1/migrate_v0.py | 14 ++-------- t4_devkit/schema/v1/writer.py | 15 ++-------- 11 files changed, 98 insertions(+), 92 deletions(-) diff --git a/t4_devkit/common/io.py b/t4_devkit/common/io.py index 413f3772..4372be3d 100644 --- a/t4_devkit/common/io.py +++ b/t4_devkit/common/io.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import os import threading @@ -25,6 +26,7 @@ "save_pointcloud", "load_image", "save_video", + "file_sha256", "V0_BIN_FIELDS", ) @@ -58,6 +60,22 @@ def save_json(data: Any, filename: PathLike) -> None: json.dump(data, f, ensure_ascii=False, indent=4) +def file_sha256(filepath: PathLike) -> str: + """Return the SHA-256 of a file, read in 1 MiB chunks (videos and rosbags can be large). + + Args: + filepath (PathLike): File to hash. + + Returns: + The hex digest. + """ + digest = hashlib.sha256() + with open(os.fspath(filepath), "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + def load_pointcloud( filepath: PathLike, *, diff --git a/t4_devkit/sanity/context.py b/t4_devkit/sanity/context.py index 485195a1..bcb866ed 100644 --- a/t4_devkit/sanity/context.py +++ b/t4_devkit/sanity/context.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib from functools import cached_property from pathlib import Path @@ -12,6 +11,7 @@ from typing_extensions import TYPE_CHECKING, Self from t4_devkit import DBMetadata +from t4_devkit.common.io import file_sha256 from t4_devkit.lanelet import LaneletParser from t4_devkit.schema import SchemaName from t4_devkit.schema.v1 import MANIFEST_FILENAME, SCHEMAS, Manifest @@ -161,9 +161,7 @@ def save_table(self, table: V1SchemaName, records: list[dict]) -> bool: if filepath is None or not is_successful(save_json_safe(records, filepath)): return False manifest = self.raw_manifest.unwrap() - manifest["tables"][table.value]["sha256"] = hashlib.sha256( - filepath.read_bytes() - ).hexdigest() + manifest["tables"][table.value]["sha256"] = file_sha256(filepath) saved = is_successful(save_json_safe(manifest, self.manifest_json.unwrap())) # read again on next use self._tables.pop(table, None) diff --git a/t4_devkit/sanity/record/rec013.py b/t4_devkit/sanity/record/rec013.py index 8568f388..034b57b5 100644 --- a/t4_devkit/sanity/record/rec013.py +++ b/t4_devkit/sanity/record/rec013.py @@ -2,9 +2,7 @@ from typing import TYPE_CHECKING -from collections import defaultdict - -from t4_devkit.schema.v1 import SchemaName +from t4_devkit.schema.v1 import Instance, SchemaName from ..checker import RecordsChecker, RuleID, RuleName, Severity from ..registry import CHECKERS @@ -37,7 +35,7 @@ class REC013(RecordsChecker): ) def check(self, context: SanityContext) -> list[Reason] | None: - expected = _expected_annotations(context) + expected = self._expected(context) failures = [] for instance in context.load_records(SchemaName.INSTANCE).unwrap(): count, first, last = expected.get(instance.token, (0, None, None)) @@ -58,7 +56,7 @@ def check(self, context: SanityContext) -> list[Reason] | None: return failures or None def fix(self, context: SanityContext) -> bool: - expected = _expected_annotations(context) + expected = self._expected(context) records = context.load_table(SchemaName.INSTANCE).unwrap() for record in records: count, first, last = expected.get(record["token"], (0, None, None)) @@ -67,21 +65,19 @@ def fix(self, context: SanityContext) -> bool: record["last_annotation_token"] = last return context.save_table(SchemaName.INSTANCE, records) - -def _expected_annotations(context: SanityContext) -> dict[str, tuple[int, str, str]]: - """Return the count, first and last annotation token in time of each instance with annotations. - - A `SampleAnnotation` is at its sample's time, an `ObjectAnn` at its frame's time. - """ - samples = {sample.token: sample for sample in context.load_records(SchemaName.SAMPLE).unwrap()} - frames = {frame.token: frame for frame in context.load_records(SchemaName.SAMPLE_DATA).unwrap()} - times: dict[str, list[tuple[int, str]]] = defaultdict(list) - for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap(): - if ann.sample_token in samples: - times[ann.instance_token].append((samples[ann.sample_token].timestamp, ann.token)) - for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap(): - if ann.instance_token is not None and ann.sample_data_token in frames: - times[ann.instance_token].append((frames[ann.sample_data_token].timestamp, ann.token)) - return { - token: (len(entries), min(entries)[1], max(entries)[1]) for token, entries in times.items() - } + @staticmethod + def _expected(context: SanityContext) -> dict[str, tuple[int, str, str]]: + """Return the count and first / last label token of each instance with labels.""" + samples = {s.token: s for s in context.load_records(SchemaName.SAMPLE).unwrap()} + frames = {f.token: f for f in context.load_records(SchemaName.SAMPLE_DATA).unwrap()} + boxes = [ + (ann.instance_token, samples[ann.sample_token].timestamp, ann.token) + for ann in context.load_records(SchemaName.SAMPLE_ANNOTATION).unwrap() + if ann.sample_token in samples + ] + labels_2d = [ + (ann.instance_token, frames[ann.sample_data_token].timestamp, ann.token) + for ann in context.load_records(SchemaName.OBJECT_ANN).unwrap() + if ann.instance_token is not None and ann.sample_data_token in frames + ] + return Instance.annotation_spans(boxes + labels_2d) diff --git a/t4_devkit/sanity/reference/ref207.py b/t4_devkit/sanity/reference/ref207.py index ccf04fdc..15057dff 100644 --- a/t4_devkit/sanity/reference/ref207.py +++ b/t4_devkit/sanity/reference/ref207.py @@ -2,8 +2,7 @@ from typing import TYPE_CHECKING -import hashlib - +from t4_devkit.common.io import file_sha256 from t4_devkit.schema.v1 import SchemaName from ..checker import RecordsChecker, RuleID, RuleName, Severity @@ -34,6 +33,6 @@ def check(self, context: SanityContext) -> list[Reason] | None: path = data_root.joinpath(map_.filename) if not path.exists(): failures.append(Reason(f"Map {map_.token}: '{map_.filename}' not found")) - elif hashlib.sha256(path.read_bytes()).hexdigest() != map_.sha256: + elif file_sha256(path) != map_.sha256: failures.append(Reason(f"Map {map_.token}: SHA-256 of '{map_.filename}' differs")) return failures or None diff --git a/t4_devkit/sanity/structure/str012.py b/t4_devkit/sanity/structure/str012.py index 76da8761..4c376b18 100644 --- a/t4_devkit/sanity/structure/str012.py +++ b/t4_devkit/sanity/structure/str012.py @@ -1,8 +1,8 @@ from __future__ import annotations -import hashlib from typing import TYPE_CHECKING +from t4_devkit.common.io import file_sha256 from t4_devkit.schema.v1 import AssetEntry from ..checker import ManifestChecker, RuleID, RuleName, Severity @@ -10,8 +10,6 @@ from ..result import Reason, make_report if TYPE_CHECKING: - from pathlib import Path - from ..context import SanityContext from ..result import Report @@ -54,14 +52,6 @@ def check(self, context: SanityContext) -> list[Reason] | None: failures.append( Reason(f"{entry.path}: {filepath.stat().st_size} bytes, listed {entry.bytes}") ) - elif _sha256(filepath) != entry.sha256: + elif file_sha256(filepath) != entry.sha256: failures.append(Reason(f"{entry.path}: SHA-256 differs from 'manifest.json'")) return failures or None - - -def _sha256(filepath: Path) -> str: - digest = hashlib.sha256() - with filepath.open("rb") as f: - for chunk in iter(lambda: f.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() diff --git a/t4_devkit/sanity/structure/str013.py b/t4_devkit/sanity/structure/str013.py index 1db29f65..4c7a6e01 100644 --- a/t4_devkit/sanity/structure/str013.py +++ b/t4_devkit/sanity/structure/str013.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING +from t4_devkit.schema.v1 import Manifest + from ..checker import ManifestChecker, RuleID, RuleName, Severity from ..registry import CHECKERS from ..result import Reason @@ -12,9 +14,6 @@ __all__ = ["STR013"] -# Directories whose files must all be listed in `manifest.json`. -LISTED_DIRS = ("annotation", "data", "map", "input_bag") - @CHECKERS.register() class STR013(ManifestChecker): @@ -33,7 +32,7 @@ def check(self, context: SanityContext) -> list[Reason] | None: listed = set(context.manifest.unwrap().listed_paths()) return [ Reason(f"File not listed: {path}") - for directory in LISTED_DIRS + for directory in ("annotation", *Manifest.ASSET_DIRS) for path in sorted( filepath.relative_to(data_root).as_posix() for filepath in data_root.joinpath(directory).rglob("*") diff --git a/t4_devkit/schema/v1/convert_v0.py b/t4_devkit/schema/v1/convert_v0.py index 402721bb..583ba8d3 100644 --- a/t4_devkit/schema/v1/convert_v0.py +++ b/t4_devkit/schema/v1/convert_v0.py @@ -15,9 +15,10 @@ from PIL import Image from pycocotools import mask as cocomask -from t4_devkit.common.io import V0_BIN_FIELDS, load_json +from t4_devkit.common.io import V0_BIN_FIELDS, file_sha256, load_json from t4_devkit.schema.tables.visibility import VisibilityLevel as V0VisibilityLevel +from .instance import Instance from .name import SchemaName from .sensor import DistortionModel @@ -153,8 +154,7 @@ def convert_map(self, record: Record) -> Record | None: filepath = osp.join(self.data_root, filename) if not osp.exists(filepath): return None - with open(filepath, "rb") as f: - sha256 = hashlib.sha256(f.read()).hexdigest() + sha256 = file_sha256(filepath) fallbacks = self.FALLBACKS["map"] return { "token": record["token"], @@ -575,31 +575,24 @@ def _is_key_frame(self, record: Record) -> bool: def _instance_annotations( self, tables: dict[SchemaName, list[Record]] ) -> dict[str, tuple[int, str, str]]: - """Return the number of labels and the first / last label token in time of each instance. - - A `SampleAnnotation` is at its sample's time, an `ObjectAnn` at its frame's time (as - sanity `REC013` checks). - """ + """Return the number of labels and the first / last label token in time of each instance + (`Instance.annotation_spans`).""" sample_times = { sample["token"]: sample["timestamp"] for sample in tables[SchemaName.SAMPLE] } frame_times = { frame["token"]: frame["timestamp"] for frame in tables[SchemaName.SAMPLE_DATA] } - times: dict[str, list[tuple[int, str]]] = {} - for ann in tables[SchemaName.SAMPLE_ANNOTATION]: - times.setdefault(ann["instance_token"], []).append( - (sample_times[ann["sample_token"]], ann["token"]) - ) - for ann in tables[SchemaName.OBJECT_ANN]: - if ann["instance_token"] is not None: - times.setdefault(ann["instance_token"], []).append( - (frame_times[ann["sample_data_token"]], ann["token"]) - ) - return { - token: (len(entries), min(entries)[1], max(entries)[1]) - for token, entries in times.items() - } + boxes = [ + (ann["instance_token"], sample_times[ann["sample_token"]], ann["token"]) + for ann in tables[SchemaName.SAMPLE_ANNOTATION] + ] + labels_2d = [ + (ann["instance_token"], frame_times[ann["sample_data_token"]], ann["token"]) + for ann in tables[SchemaName.OBJECT_ANN] + if ann["instance_token"] is not None + ] + return Instance.annotation_spans(boxes + labels_2d) def _read_projector(self) -> Record: """Read Autoware's `map_projector_info.yaml`, if any.""" diff --git a/t4_devkit/schema/v1/instance.py b/t4_devkit/schema/v1/instance.py index f4879a64..641127cf 100644 --- a/t4_devkit/schema/v1/instance.py +++ b/t4_devkit/schema/v1/instance.py @@ -1,5 +1,8 @@ from __future__ import annotations +from collections import defaultdict +from typing import Iterable + from attrs import define, field, validators from t4_devkit.schema.tables.base import SchemaBase, impossible_empty @@ -52,3 +55,27 @@ class Instance(SchemaBase): converter=lambda x: MapElement(**x) if isinstance(x, dict) else x, validator=validators.instance_of(MapElement), ) + + @staticmethod + def annotation_spans( + labels: Iterable[tuple[str, int, str]], + ) -> dict[str, tuple[int, str, str]]: + """Return the number of labels and the first / last label in time of each instance. + + This is what `nbr_annotations` and `first_annotation_token` / `last_annotation_token` + hold. + + Args: + labels (Iterable[tuple[str, int, str]]): (instance token, time, label token) of each + 3D box (at its sample's time) and 2D label (at its frame's time). + + Returns: + Count, first and last label token, keyed by instance token. + """ + times: dict[str, list[tuple[int, str]]] = defaultdict(list) + for instance_token, timestamp, token in labels: + times[instance_token].append((timestamp, token)) + return { + token: (len(entries), min(entries)[1], max(entries)[1]) + for token, entries in times.items() + } diff --git a/t4_devkit/schema/v1/manifest.py b/t4_devkit/schema/v1/manifest.py index c1e97f57..755d9280 100644 --- a/t4_devkit/schema/v1/manifest.py +++ b/t4_devkit/schema/v1/manifest.py @@ -2,7 +2,7 @@ import os.path as osp from enum import Enum, unique -from typing import TYPE_CHECKING, Any, Mapping, Sequence +from typing import TYPE_CHECKING, Any, ClassVar, Mapping, Sequence from attrs import converters, define, field, validators @@ -304,6 +304,9 @@ class Manifest: migration_notes (list[str] | None): Notes on anything lost or changed during migration. """ + # Folders whose files are listed in `assets` (the tables, under `annotation/`, in `tables`). + ASSET_DIRS: ClassVar[tuple[str, ...]] = ("data", "map", "input_bag") + schema_version: str = field(validator=validators.instance_of(str)) dataset_id: str = field(validator=validators.instance_of(str)) version: int = field(validator=(validators.instance_of(int), validators.ge(0))) diff --git a/t4_devkit/schema/v1/migrate_v0.py b/t4_devkit/schema/v1/migrate_v0.py index c938253f..f5c03ef9 100644 --- a/t4_devkit/schema/v1/migrate_v0.py +++ b/t4_devkit/schema/v1/migrate_v0.py @@ -7,7 +7,7 @@ import numpy as np -from t4_devkit.common.io import V0_BIN_FIELDS, load_image, load_json, load_pointcloud +from t4_devkit.common.io import load_image, load_json, load_pointcloud from .loader import load_dataset from .name import SchemaName @@ -89,7 +89,8 @@ def migrate_lidar(self, sensor: Sensor) -> None: """ records = self._channel_frames(sensor) self._relink(sensor, records) - source_fields = self._source_fields(records[0]) + # the 0.x layout, as loading read it (`.info` field count, or the `.pcd` header) + source_fields = [(field.name, field.dtype) for field in sensor.lidar.fields] fields = self._point_fields(sensor.channel, records, source_fields) if self._v0_frames[records[0].token].get("info_filename"): fields += [("source_index", "uint8"), ("source_timestamp", "int64")] @@ -390,15 +391,6 @@ def _sources(self, info: dict, num_points: int) -> dict[str, np.ndarray]: raise ValueError(f"{(~covered).sum()} points are outside the .info sources") return {"source_index": source_index, "source_timestamp": source_timestamp} - def _source_fields(self, record: SampleData) -> list[tuple[str, str]]: - """Return the 0.x point fields of a channel, from its first frame.""" - if record.filename.endswith(".pcd"): - points = load_pointcloud(osp.join(self.data_root, record.filename)) - return [(name, values.dtype.name) for name, values in points.items()] - info_filename = self._v0_frames[record.token].get("info_filename") - info = load_json(osp.join(self.data_root, info_filename)) if info_filename else {} - return [(name, "float32") for name in V0_BIN_FIELDS[: info.get("num_pts_feats", 5)]] - def _read_points( self, record: SampleData, source_fields: list[tuple[str, str]] ) -> dict[str, np.ndarray]: diff --git a/t4_devkit/schema/v1/writer.py b/t4_devkit/schema/v1/writer.py index 7da8152a..0f91b893 100644 --- a/t4_devkit/schema/v1/writer.py +++ b/t4_devkit/schema/v1/writer.py @@ -1,14 +1,12 @@ from __future__ import annotations -import hashlib import importlib.metadata import json -import os import shutil from pathlib import Path from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence -from t4_devkit.common.io import save_pointcloud, save_video +from t4_devkit.common.io import file_sha256, save_pointcloud, save_video from t4_devkit.common.serialize import serialize_dataclass, serialize_dataclasses from .manifest import MANIFEST_FILENAME, Capabilities, Manifest @@ -36,9 +34,6 @@ class DatasetWriter: >>> manifest = writer.save(tables, manifest_info) """ - # Folders whose files are listed in `manifest.json` as assets. - ASSET_DIRS: tuple[str, ...] = ("data", "map", "input_bag") - def __init__(self, output_root: PathLike) -> None: """ Args: @@ -181,7 +176,7 @@ def _asset_paths(self) -> list[str]: """Every file under the asset folders, relative to the dataset root, sorted.""" return sorted( filepath.relative_to(self.output_root).as_posix() - for directory in self.ASSET_DIRS + for directory in Manifest.ASSET_DIRS for filepath in (self.output_root / directory).rglob("*") if filepath.is_file() ) @@ -192,8 +187,4 @@ def _write_json(self, path: str, data: Any) -> None: filepath.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n") def _sha256(self, path: str) -> str: - digest = hashlib.sha256() - with open(os.fspath(self.output_root / path), "rb") as f: - for chunk in iter(lambda: f.read(1 << 20), b""): - digest.update(chunk) - return digest.hexdigest() + return file_sha256(self.output_root / path) From 820224f1c0d8aae85971d2f40b57e285e87a24c4 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 17:05:59 +0900 Subject: [PATCH 03/10] style: local constants, match on Maybe, helper methods --- t4_devkit/dataclass/pointcloud.py | 10 ++++---- t4_devkit/sanity/checker.py | 11 +++++---- t4_devkit/sanity/context.py | 3 +-- t4_devkit/sanity/format/base.py | 6 ++--- t4_devkit/sanity/record/rec012.py | 15 ++++++------ t4_devkit/sanity/record/rec022.py | 34 ++++++++++++---------------- t4_devkit/sanity/record/rec027.py | 6 ++--- t4_devkit/sanity/reference/ref302.py | 16 +++++++------ t4_devkit/sanity/reference/ref303.py | 16 +++++++------ t4_devkit/sanity/reference/ref304.py | 16 +++++++------ t4_devkit/schema/v1/convert_v0.py | 12 +++++----- 11 files changed, 73 insertions(+), 72 deletions(-) diff --git a/t4_devkit/dataclass/pointcloud.py b/t4_devkit/dataclass/pointcloud.py index 436a0a52..0c61f3c0 100644 --- a/t4_devkit/dataclass/pointcloud.py +++ b/t4_devkit/dataclass/pointcloud.py @@ -28,9 +28,6 @@ "Stamp", ] -# Point fields of `LidarPointCloud.points`, in row order. -XYZI = ("x", "y", "z", "intensity") - @define class PointCloudSourceInfo: @@ -274,9 +271,10 @@ def from_file( raise ValueError(f"Unknown 0.x point layout with {num_pts_feats} fields") fields = [(name, "float32") for name in V0_BIN_FIELDS[:num_pts_feats]] - columns = load_pointcloud(filepath, frame_index=frame_index, fields=fields, columns=XYZI) - points = np.empty((len(XYZI), len(columns["x"])), dtype=np.float32) - for i, name in enumerate(XYZI): + xyzi = ("x", "y", "z", "intensity") # rows of `points` + columns = load_pointcloud(filepath, frame_index=frame_index, fields=fields, columns=xyzi) + points = np.empty((len(xyzi), len(columns["x"])), dtype=np.float32) + for i, name in enumerate(xyzi): points[i] = columns[name] return cls(points, metainfo=metainfo) diff --git a/t4_devkit/sanity/checker.py b/t4_devkit/sanity/checker.py index 53af1e98..2d0038f6 100644 --- a/t4_devkit/sanity/checker.py +++ b/t4_devkit/sanity/checker.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, NewType from returns.maybe import Maybe, Nothing, Some +from returns.pipeline import is_successful from t4_devkit.schema.v1 import SCHEMAS as V1_SCHEMAS @@ -141,10 +142,12 @@ class RecordsChecker(ManifestChecker): tables: tuple[V1SchemaName, ...] def can_skip(self, context: SanityContext) -> Maybe[Reason]: - skip = super().can_skip(context) - if skip != Nothing: - return skip - invalid = [table.value for table in self.tables if context.load_records(table) == Nothing] + match super().can_skip(context): + case Some(reason): + return Some(reason) + invalid = [ + table.value for table in self.tables if not is_successful(context.load_records(table)) + ] if invalid: return Maybe.from_value(Reason(f"Tables not listed or not valid: {', '.join(invalid)}")) return Nothing diff --git a/t4_devkit/sanity/context.py b/t4_devkit/sanity/context.py index bcb866ed..5ea15624 100644 --- a/t4_devkit/sanity/context.py +++ b/t4_devkit/sanity/context.py @@ -104,8 +104,7 @@ def raw_manifest(self) -> Maybe[dict]: """Return `manifest.json` as read, or Nothing if it cannot be read.""" match self.manifest_json: case Some(x) if x.exists(): - result = load_json_safe(x) - return Some(result.unwrap()) if is_successful(result) else Nothing + return result_to_maybe(load_json_safe(x)) return Nothing @cached_property diff --git a/t4_devkit/sanity/format/base.py b/t4_devkit/sanity/format/base.py index ae6977cc..bde7afbd 100644 --- a/t4_devkit/sanity/format/base.py +++ b/t4_devkit/sanity/format/base.py @@ -69,9 +69,9 @@ class TableFieldChecker(ManifestChecker): table: V1SchemaName def can_skip(self, context: SanityContext) -> Maybe[Reason]: - skip = super().can_skip(context) - if skip != Nothing: - return skip + match super().can_skip(context): + case Some(reason): + return Some(reason) if ( not self.table.is_optional() and self.table.value not in context.manifest.unwrap().tables diff --git a/t4_devkit/sanity/record/rec012.py b/t4_devkit/sanity/record/rec012.py index ae0c2485..3cacd465 100644 --- a/t4_devkit/sanity/record/rec012.py +++ b/t4_devkit/sanity/record/rec012.py @@ -31,7 +31,7 @@ class REC012(RecordsChecker): ) def check(self, context: SanityContext) -> list[Reason] | None: - counts = _sample_counts(context) + counts = self._sample_counts(context) return [ Reason( f"Scene {scene.token}: 'nbr_samples' {scene.nbr_samples}, has {counts[scene.token]}" @@ -41,14 +41,15 @@ def check(self, context: SanityContext) -> list[Reason] | None: ] or None def fix(self, context: SanityContext) -> bool: - counts = _sample_counts(context) + counts = self._sample_counts(context) records = context.load_table(SchemaName.SCENE).unwrap() for record in records: record["nbr_samples"] = counts[record["token"]] return context.save_table(SchemaName.SCENE, records) - -def _sample_counts(context: SanityContext) -> Counter[str]: - return Counter( - sample.scene_token for sample in context.load_records(SchemaName.SAMPLE).unwrap() - ) + @staticmethod + def _sample_counts(context: SanityContext) -> Counter[str]: + """Return the number of samples of each scene.""" + return Counter( + sample.scene_token for sample in context.load_records(SchemaName.SAMPLE).unwrap() + ) diff --git a/t4_devkit/sanity/record/rec022.py b/t4_devkit/sanity/record/rec022.py index fd1f135d..073a1551 100644 --- a/t4_devkit/sanity/record/rec022.py +++ b/t4_devkit/sanity/record/rec022.py @@ -38,29 +38,25 @@ class REC022(RecordsChecker): ) def check(self, context: SanityContext) -> list[Reason] | None: + # 2000-01-01 and 2100-01-01 in Unix microseconds + low, high = 946_684_800_000_000, 4_102_444_800_000_000 + fields_of = { + SchemaName.SAMPLE: ("timestamp",), + SchemaName.SAMPLE_DATA: ("timestamp",), + SchemaName.EGO_POSE: ("timestamp",), + SchemaName.VEHICLE_STATE: ("timestamp",), + SchemaName.TRAFFIC_LIGHT: ("timestamp",), + SchemaName.ROUTE: ("valid_from", "valid_until"), + SchemaName.SCENARIO: ("start_timestamp", "end_timestamp", "key_timestamp"), + SchemaName.CAPTION: ("start_timestamp", "end_timestamp"), + } failures = [] - for table, fields in TIMESTAMP_FIELDS.items(): + for table in self.tables: for record in context.load_records(table).unwrap(): - for name in fields: + for name in fields_of[table]: value = getattr(record, name) - if value is not None and not MIN_TIMESTAMP <= value < MAX_TIMESTAMP: + if value is not None and not low <= value < high: failures.append( Reason(f"{record_name(table)} {record.token}: '{name}' {value}") ) return failures or None - - -# 2000-01-01 and 2100-01-01 in Unix microseconds. -MIN_TIMESTAMP = 946_684_800_000_000 -MAX_TIMESTAMP = 4_102_444_800_000_000 - -TIMESTAMP_FIELDS = { - SchemaName.SAMPLE: ("timestamp",), - SchemaName.SAMPLE_DATA: ("timestamp",), - SchemaName.EGO_POSE: ("timestamp",), - SchemaName.VEHICLE_STATE: ("timestamp",), - SchemaName.TRAFFIC_LIGHT: ("timestamp",), - SchemaName.ROUTE: ("valid_from", "valid_until"), - SchemaName.SCENARIO: ("start_timestamp", "end_timestamp", "key_timestamp"), - SchemaName.CAPTION: ("start_timestamp", "end_timestamp"), -} diff --git a/t4_devkit/sanity/record/rec027.py b/t4_devkit/sanity/record/rec027.py index 929296ef..172cb7ab 100644 --- a/t4_devkit/sanity/record/rec027.py +++ b/t4_devkit/sanity/record/rec027.py @@ -42,11 +42,9 @@ def check(self, context: SanityContext) -> list[Reason] | None: for route in context.load_records(SchemaName.ROUTE).unwrap() for pose in (route.start_pose, route.goal_pose) ] + tolerance = 1e-3 return [ Reason(f"{table} {token}: rotation norm {rotation.norm:.6f}") for table, token, rotation in rotations - if abs(rotation.norm - 1.0) > NORM_TOLERANCE + if abs(rotation.norm - 1.0) > tolerance ] or None - - -NORM_TOLERANCE = 1e-3 diff --git a/t4_devkit/sanity/reference/ref302.py b/t4_devkit/sanity/reference/ref302.py index aee269ae..fe4faebb 100644 --- a/t4_devkit/sanity/reference/ref302.py +++ b/t4_devkit/sanity/reference/ref302.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from returns.maybe import Maybe, Nothing +from returns.maybe import Maybe, Nothing, Some from t4_devkit.schema.v1 import SchemaName @@ -31,12 +31,14 @@ class REF302(RecordsChecker): ) def can_skip(self, context: SanityContext) -> Maybe[Reason]: - skip = super().can_skip(context) - if skip != Nothing: - return skip - if context.lanelet_map == Nothing: - return Maybe.from_value(Reason("No lanelet2 map to check against")) - return Nothing + match super().can_skip(context): + case Some(reason): + return Some(reason) + match context.lanelet_map: + case Some(_): + return Nothing + case _: + return Maybe.from_value(Reason("No lanelet2 map to check against")) def check(self, context: SanityContext) -> list[Reason] | None: lanelet = context.lanelet_map.unwrap() diff --git a/t4_devkit/sanity/reference/ref303.py b/t4_devkit/sanity/reference/ref303.py index 936ead3f..5fe66a32 100644 --- a/t4_devkit/sanity/reference/ref303.py +++ b/t4_devkit/sanity/reference/ref303.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from returns.maybe import Maybe, Nothing +from returns.maybe import Maybe, Nothing, Some from t4_devkit.schema.v1 import SchemaName @@ -34,12 +34,14 @@ class REF303(RecordsChecker): ) def can_skip(self, context: SanityContext) -> Maybe[Reason]: - skip = super().can_skip(context) - if skip != Nothing: - return skip - if context.lanelet_map == Nothing: - return Maybe.from_value(Reason("No lanelet2 map to check against")) - return Nothing + match super().can_skip(context): + case Some(reason): + return Some(reason) + match context.lanelet_map: + case Some(_): + return Nothing + case _: + return Maybe.from_value(Reason("No lanelet2 map to check against")) def check(self, context: SanityContext) -> list[Reason] | None: regulatory_elements = { diff --git a/t4_devkit/sanity/reference/ref304.py b/t4_devkit/sanity/reference/ref304.py index a8e861ce..6ca91b9a 100644 --- a/t4_devkit/sanity/reference/ref304.py +++ b/t4_devkit/sanity/reference/ref304.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING -from returns.maybe import Maybe, Nothing +from returns.maybe import Maybe, Nothing, Some from t4_devkit.schema.v1 import SchemaName @@ -31,12 +31,14 @@ class REF304(RecordsChecker): ) def can_skip(self, context: SanityContext) -> Maybe[Reason]: - skip = super().can_skip(context) - if skip != Nothing: - return skip - if context.lanelet_map == Nothing: - return Maybe.from_value(Reason("No lanelet2 map to check against")) - return Nothing + match super().can_skip(context): + case Some(reason): + return Some(reason) + match context.lanelet_map: + case Some(_): + return Nothing + case _: + return Maybe.from_value(Reason("No lanelet2 map to check against")) def check(self, context: SanityContext) -> list[Reason] | None: lanelets = { diff --git a/t4_devkit/schema/v1/convert_v0.py b/t4_devkit/schema/v1/convert_v0.py index 583ba8d3..49991f04 100644 --- a/t4_devkit/schema/v1/convert_v0.py +++ b/t4_devkit/schema/v1/convert_v0.py @@ -60,10 +60,6 @@ class V0Converter: "camera": {"is_rectified": True}, } - # The 0.x converter names the files of one frame by its frame number, e.g. `00042.jpg` and - # `00042.pcd.bin`: it groups the channels of a frame (their timestamps differ by up to ~90 ms). - FRAME_NUMBER: ClassVar[re.Pattern] = re.compile(r"^(\d+)\.") - def __init__(self, data_root: PathLike) -> None: """ Args: @@ -555,10 +551,14 @@ def _frame_samples(self) -> tuple[list[Record], dict[str, str], dict[str, str]]: return [row for chain in by_scene.values() for row in chain], sample_of_frame, remap def _frame_groups(self) -> dict[int, list[Record]]: - """0.x `sample_data` grouped by frame number (`FRAME_NUMBER` in the file name), or by + """0.x `sample_data` grouped by frame number (the digits starting the file name), or by position in each channel's frames in time if a file name has no number.""" + # The 0.x converter names the files of one frame by its frame number, e.g. `00042.jpg` + # and `00042.pcd.bin`: it groups the channels of a frame (their timestamps differ by up + # to ~90 ms). + frame_number = re.compile(r"^(\d+)\.") records = self.v0["sample_data"] - numbers = [self.FRAME_NUMBER.match(osp.basename(record["filename"])) for record in records] + numbers = [frame_number.match(osp.basename(record["filename"])) for record in records] groups: dict[int, list[Record]] = defaultdict(list) if all(numbers): for record, number in zip(records, numbers): From 3a0620e23f7665d24edcbdfd31304c77bda9a1e4 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 17:09:22 +0900 Subject: [PATCH 04/10] docs: Return-form docstrings, with Args/Returns on the 0.x converter --- t4_devkit/common/io.py | 2 +- t4_devkit/helper/rendering.py | 4 +- t4_devkit/schema/v1/convert_v0.py | 230 +++++++++++++++++++++++++----- t4_devkit/schema/v1/sensor.py | 2 +- t4_devkit/schema/v1/writer.py | 2 +- t4_devkit/tier4.py | 11 +- 6 files changed, 207 insertions(+), 44 deletions(-) diff --git a/t4_devkit/common/io.py b/t4_devkit/common/io.py index 4372be3d..fc7412b8 100644 --- a/t4_devkit/common/io.py +++ b/t4_devkit/common/io.py @@ -311,7 +311,7 @@ def _add_video_stream( preset: int, crf: int, ) -> av.video.stream.VideoStream: - """The AV1 stream of a camera video, sized by its first image.""" + """Return the AV1 stream of a camera video, sized by its first image.""" stream = container.add_stream("libsvtav1", rate=frame_rate) stream.height, stream.width = image.shape[:2] stream.pix_fmt = pixel_format diff --git a/t4_devkit/helper/rendering.py b/t4_devkit/helper/rendering.py index 123b1378..ec3b9124 100644 --- a/t4_devkit/helper/rendering.py +++ b/t4_devkit/helper/rendering.py @@ -65,7 +65,7 @@ def __init__(self, t4: T4Devkit) -> None: self._executor = concurrent.futures.ThreadPoolExecutor() def _first_sample_data_tokens(self, modality: SensorModality) -> list[str]: - """The first frame of each channel of a modality, in sensor order. + """Return the first frame of each channel of a modality, in sensor order. A recalibration adds a `sensor` row with the same channel, and source LiDARs of a merged channel have no frames, so channels are taken once and sensors without frames skipped. @@ -581,7 +581,7 @@ def _render_annotation2ds( current_sample_token = sample.next def _annotation_sample(self, annotation_token: str | None) -> Sample | None: - """The sample of an instance's first or last annotation, or None if there is none. + """Return the sample of an instance's first or last annotation, or None if there is none. The annotation is a `sample_annotation`, or an `object_ann` for instances labeled only in 2D, whose sample is that of its frame. diff --git a/t4_devkit/schema/v1/convert_v0.py b/t4_devkit/schema/v1/convert_v0.py index 49991f04..6d44b238 100644 --- a/t4_devkit/schema/v1/convert_v0.py +++ b/t4_devkit/schema/v1/convert_v0.py @@ -92,7 +92,7 @@ def __init__(self, data_root: PathLike) -> None: self.traffic_light_ways = self._traffic_light_ways() def convert(self) -> dict[SchemaName, list[Record]]: - """Convert every table. + """Return every table converted to 1.0 records. Returns: Raw 1.0 records per table; empty for tables not converted. @@ -145,7 +145,16 @@ def convert(self) -> dict[SchemaName, list[Record]]: # --- core tables --- def convert_map(self, record: Record) -> Record | None: - """`sha256` from the map file, `projector` from its yaml; None if the map file is missing.""" + """Return the 1.0 `map` row of a 0.x map, or None if the map file is missing. + + `sha256` comes from the map file, `projector` from its yaml. + + Args: + record (Record): 0.x `map` row. + + Returns: + The 1.0 row, or None. + """ filename = record["filename"] or self.FALLBACKS["map_filename"] filepath = osp.join(self.data_root, filename) if not osp.exists(filepath): @@ -166,7 +175,17 @@ def convert_map(self, record: Record) -> Record | None: } def convert_vehicle(self, log: Record) -> Record: - """One vehicle per log, named after the 0.x `log.vehicle`, with placeholder dimensions.""" + """Return the vehicle of a log: named after the 0.x `log.vehicle`, with placeholder + dimensions. + + 0.x has no vehicle table, so there is one vehicle per log. + + Args: + log (Record): 0.x `log` row. + + Returns: + The 1.0 `vehicle` row. + """ fallbacks = self.FALLBACKS["vehicle"] return { "token": self._vehicle_token(log), @@ -181,6 +200,15 @@ def convert_vehicle(self, log: Record) -> Record: } def convert_log(self, record: Record, log2map: dict[str, str]) -> Record: + """Return the 1.0 `log` row of a 0.x log. + + Args: + record (Record): 0.x `log` row. + log2map (dict[str, str]): Map token of each log token. + + Returns: + The 1.0 row. + """ return { "token": record["token"], "logfile": self._none_if_empty(record["logfile"]), @@ -191,7 +219,14 @@ def convert_log(self, record: Record, log2map: dict[str, str]) -> Record: } def convert_scene(self, record: Record) -> Record: - """Counts and first / last sample over the 1.0 samples (every frame).""" + """Return the 1.0 `scene` row, with counts and first / last sample over the 1.0 samples. + + Args: + record (Record): 0.x `scene` row. + + Returns: + The 1.0 row. + """ samples = [sample for sample in self.samples if sample["scene_token"] == record["token"]] return { "token": record["token"], @@ -204,6 +239,14 @@ def convert_scene(self, record: Record) -> Record: } def convert_ego_pose(self, record: Record) -> Record: + """Return the 1.0 `ego_pose` row of a 0.x ego pose. + + Args: + record (Record): 0.x `ego_pose` row. + + Returns: + The 1.0 row. + """ fallbacks = self.FALLBACKS["ego_pose"] return { "token": record["token"], @@ -219,9 +262,16 @@ def convert_ego_pose(self, record: Record) -> Record: } def convert_sensor(self, index: int, calibration: Record) -> Record: - """Merge 0.x `sensor` (per channel) and `calibrated_sensor` (per calibration). + """Return the 1.0 `sensor` row of a 0.x calibration, merged with its 0.x `sensor` row. One row per calibration, keeping the `calibrated_sensor` token, indexed in file order. + + Args: + index (int): Sensor index, the position of the calibration in the file. + calibration (Record): 0.x `calibrated_sensor` row. + + Returns: + The 1.0 row. """ sensor = self.sensors[calibration["sensor_token"]] modality = sensor["modality"] @@ -239,7 +289,15 @@ def convert_sensor(self, index: int, calibration: Record) -> Record: } def convert_camera(self, calibration: Record) -> Record: - """Size from `sample_data`, or from the first image if 0.x stored none.""" + """Return the camera block of a calibration: size from `sample_data`, or from the first + image if 0.x stored none. + + Args: + calibration (Record): 0.x `calibrated_sensor` row of a camera. + + Returns: + The camera block. + """ distortion = calibration["camera_distortion"] distortion_model = DistortionModel.from_num_coefficients(len(distortion)) frames = self.frames[calibration["token"]] @@ -260,7 +318,15 @@ def convert_camera(self, calibration: Record) -> Record: } def convert_lidar(self, calibration: Record) -> Record: - """From the first frame: its format, its `.info` file (if any) and its header.""" + """Return the LiDAR block of a calibration, from its first frame: its format, its `.info` + file (if any) and its header. + + Args: + calibration (Record): 0.x `calibrated_sensor` row of a LiDAR. + + Returns: + The LiDAR block. + """ frames = self.frames[calibration["token"]] if not frames: return self._convert_source_lidar(calibration) @@ -293,6 +359,14 @@ def convert_lidar(self, calibration: Record) -> Record: } def convert_sample_data(self, record: Record) -> Record: + """Return the 1.0 `sample_data` row of a 0.x frame, in the 1.0 sample of its frame. + + Args: + record (Record): 0.x `sample_data` row. + + Returns: + The 1.0 row. + """ return { "token": record["token"], "sample_token": self.sample_of_frame[record["token"]], @@ -313,7 +387,11 @@ def convert_sample_data(self, record: Record) -> Record: # --- label tables --- def convert_categories(self) -> list[Record]: - """Categories without `index` get their list position, as the 0.x devkit reads them. + """Return the 1.0 `category` rows. Categories without `index` get their list position, as + the 0.x devkit reads them. + + Returns: + The 1.0 rows. Raises: ValueError: If some categories have an `index` and others do not. @@ -327,7 +405,16 @@ def convert_categories(self) -> list[Record]: return [self.convert_category(record, index) for record, index in zip(records, indices)] def convert_category(self, record: Record, index: int) -> Record: - """`is_thing` is false only for categories used by `surface_ann` (road, sky) and nothing else.""" + """Return the 1.0 `category` row. `is_thing` is false only for categories used by + `surface_ann` (road, sky) and nothing else. + + Args: + record (Record): 0.x `category` row. + index (int): Category index. + + Returns: + The 1.0 row. + """ token = record["token"] return { "token": token, @@ -340,6 +427,14 @@ def convert_category(self, record: Record, index: int) -> Record: } def convert_attribute(self, record: Record) -> Record: + """Return the 1.0 `attribute` row of a 0.x attribute. + + Args: + record (Record): 0.x `attribute` row. + + Returns: + The 1.0 row. + """ return { "token": record["token"], "name": record["name"], @@ -347,7 +442,12 @@ def convert_attribute(self, record: Record) -> Record: } def convert_visibilities(self) -> list[Record]: - """Levels as read by 0.x (aliases mapped); `unavailable` rows are dropped (visibility unknown).""" + """Return the 1.0 `visibility` rows: levels as read by 0.x (aliases mapped); `unavailable` + rows are dropped (visibility unknown). + + Returns: + The 1.0 rows. + """ rows = [] for record in self.v0["visibility"]: level = V0VisibilityLevel.from_value(record["level"]) @@ -364,10 +464,19 @@ def convert_visibilities(self) -> list[Record]: def convert_instance( self, record: Record, annotations: dict[str, tuple[int, str, str]] ) -> Record: - """A lamp listed in 0.x traffic_light points to its lanelet2 way in `map_element`. + """Return the 1.0 `instance` row. A lamp listed in 0.x traffic_light points to its lanelet2 + way in `map_element`. + + `nbr_annotations` and the first / last annotation come from `annotations`: 1.0 counts 3D + boxes and 2D labels, and 0.x tools count them differently (some write 0 for 2D). - `nbr_annotations` and the first / last annotation come from `annotations`: 1.0 counts - 3D boxes and 2D labels, and 0.x tools count them differently (some write 0 for 2D). + Args: + record (Record): 0.x `instance` row. + annotations (dict[str, tuple[int, str, str]]): Label count and first / last label token + of each instance (`Instance.annotation_spans`). + + Returns: + The 1.0 row. """ way_id = self.traffic_light_ways.get(record["token"]) count, first, last = annotations.get(record["token"], (0, None, None)) @@ -383,7 +492,16 @@ def convert_instance( } def convert_sample_annotation(self, record: Record, known_visibility: set[str]) -> Record: - """Visibility levels dropped by `convert_visibilities` become a null `visibility_token`.""" + """Return the 1.0 `sample_annotation` row. Visibility levels dropped by + `convert_visibilities` become a null `visibility_token`. + + Args: + record (Record): 0.x `sample_annotation` row. + known_visibility (set[str]): Tokens of the visibility levels kept. + + Returns: + The 1.0 row. + """ visibility_token = record["visibility_token"] annotation_type, provenance, confidence = self._label_origin(record) return { @@ -407,6 +525,14 @@ def convert_sample_annotation(self, record: Record, known_visibility: set[str]) } def convert_object_ann(self, record: Record) -> Record: + """Return the 1.0 `object_ann` row of a 0.x 2D label. + + Args: + record (Record): 0.x `object_ann` row. + + Returns: + The 1.0 row. + """ annotation_type, provenance, confidence = self._label_origin(record) return { "token": record["token"], @@ -425,7 +551,15 @@ def convert_object_ann(self, record: Record) -> Record: } def convert_surface_ann(self, record: Record) -> Record: - """A 0.x semantic mask becomes an `object_ann` without instance; its box is the mask's.""" + """Return a 0.x semantic mask as an `object_ann` row without instance; its box is the + mask's. + + Args: + record (Record): 0.x `surface_ann` row. + + Returns: + The 1.0 `object_ann` row. + """ mask = record["mask"] annotation_type, provenance, confidence = self._label_origin(record) return { @@ -444,6 +578,14 @@ def convert_surface_ann(self, record: Record) -> Record: } def convert_keypoint(self, record: Record) -> Record: + """Return the 1.0 `keypoint` row of a 0.x keypoint label. + + Args: + record (Record): 0.x `keypoint` row. + + Returns: + The 1.0 row. + """ annotation_type, provenance, confidence = self._label_origin(record) return { "token": record["token"], @@ -460,10 +602,17 @@ def convert_keypoint(self, record: Record) -> Record: # --- planning tables --- def convert_vehicle_state(self, record: Record) -> Record: - """0.x vehicle states come from the Autoware vehicle interface (`/vehicle/status/*`). + """Return the 1.0 `vehicle_state` row. 0.x vehicle states come from the Autoware vehicle + interface (`/vehicle/status/*`). `additional_info.speed` (norm of the longitudinal and lateral velocity) is dropped: it can't be split back into the 1.0 velocities. + + Args: + record (Record): 0.x `vehicle_state` row. + + Returns: + The 1.0 row. """ shift_state = record.get("shift_state") # The 0.x devkit names forward gears FORWARD or HIGH; the 0.x converter writes DRIVE. @@ -488,7 +637,8 @@ def convert_vehicle_state(self, record: Record) -> Record: # --- helpers --- def _frame_samples(self) -> tuple[list[Record], dict[str, str], dict[str, str]]: - """One 1.0 sample per 0.x frame: the frames of all channels with the same frame number. + """Return one 1.0 sample per 0.x frame: the frames of all channels with the same frame + number. A frame with 0.x key samples keeps the earliest one's token; camera-only 0.x datasets have one sample per camera image, merged here. Other frames get a new deterministic token. @@ -551,8 +701,9 @@ def _frame_samples(self) -> tuple[list[Record], dict[str, str], dict[str, str]]: return [row for chain in by_scene.values() for row in chain], sample_of_frame, remap def _frame_groups(self) -> dict[int, list[Record]]: - """0.x `sample_data` grouped by frame number (the digits starting the file name), or by - position in each channel's frames in time if a file name has no number.""" + """Return 0.x `sample_data` grouped by frame number (the digits starting the file name), or + by position in each channel's frames in time if a file name has no number. + """ # The 0.x converter names the files of one frame by its frame number, e.g. `00042.jpg` # and `00042.pcd.bin`: it groups the channels of a frame (their timestamps differ by up # to ~90 ms). @@ -570,6 +721,9 @@ def _frame_groups(self) -> dict[int, list[Record]]: return groups def _is_key_frame(self, record: Record) -> bool: + """Return whether a 0.x frame is a key frame: marked as one, or carrying a 2D label or + keypoint. + """ return record["is_key_frame"] or record["token"] in self.labeled_frames def _instance_annotations( @@ -595,7 +749,9 @@ def _instance_annotations( return Instance.annotation_spans(boxes + labels_2d) def _read_projector(self) -> Record: - """Read Autoware's `map_projector_info.yaml`, if any.""" + """Return the `projector` block from Autoware's `map_projector_info.yaml` (empty values + without it). + """ filepath = osp.join(self.data_root, self.FALLBACKS["map_projector_filename"]) info = {} if osp.exists(filepath): @@ -613,7 +769,8 @@ def _read_projector(self) -> Record: } def _convert_source_lidar(self, calibration: Record) -> Record: - """A LiDAR without frames of its own: a source of a concatenated channel. + """Return the LiDAR block of a LiDAR without frames of its own: a source of a concatenated + channel. Its points are stored in the concatenated files, so it has their format and fields; it counts a frame wherever the `.info` file of a concatenated frame lists points from it. @@ -651,7 +808,7 @@ def _convert_source_lidar(self, calibration: Record) -> Record: } def _capture_time(self, log_token: str) -> str: - """The log's first sample time, ISO 8601 UTC, for logs without `data_captured`.""" + """Return the log's first sample time, ISO 8601 UTC, for logs without `data_captured`.""" scenes = { record["token"] for record in self.v0["scene"] if record["log_token"] == log_token } @@ -661,12 +818,13 @@ def _capture_time(self, log_token: str) -> str: return datetime.fromtimestamp(timestamp / 1e6, tz=timezone.utc).isoformat() def _traffic_light_ways(self) -> dict[str, int]: - """The lanelet2 way (`type=traffic_light`) each lamp instance shows, by instance token. + """Return the lanelet2 way (`type=traffic_light`) each lamp instance shows, by instance + token. Two 0.x tables hold this link: `traffic_light` (`primitive_id`, #288) and `traffic_light_instance_map` (`traffic_light_linestring_id`, from the unmerged - `feat/traffic-light-instance-map` branch), which wins when both list a lamp. - `traffic_light` rows without `primitive_id` are skipped, as no 0.x devkit reads them. + `feat/traffic-light-instance-map` branch), which wins when both list a lamp. `traffic_light` + rows without `primitive_id` are skipped, as no 0.x devkit reads them. """ ways = {} skipped = 0 @@ -682,7 +840,9 @@ def _traffic_light_ways(self) -> dict[str, int]: return ways def _frame_infos(self, calibration_token: str) -> list[Record]: - """The 0.x `.info` file of each frame of a channel ({} where there is none), read once.""" + """Return the 0.x `.info` file of each frame of a channel ({} where there is none), read + once. + """ if calibration_token not in self.infos: self.infos[calibration_token] = [ load_json(osp.join(self.data_root, frame["info_filename"])) @@ -695,10 +855,10 @@ def _frame_infos(self, calibration_token: str) -> list[Record]: @staticmethod def _non_empty_mask(mask: Record | None) -> Record | None: - """None for a mask without pixels. + """Return the mask, or None for a mask without pixels. - Some 0.x tools give box-only labels a placeholder: an empty RLE whose `size` is - (width, height) instead of the (height, width) of real masks. + Some 0.x tools give box-only labels a placeholder: an empty RLE whose `size` is (width, + height) instead of the (height, width) of real masks. """ if mask is None or "counts" not in mask: return None @@ -707,10 +867,10 @@ def _non_empty_mask(mask: Record | None) -> Record | None: @staticmethod def _mask_bbox(mask: Record) -> list[float]: - """Box around a 0.x RLE mask, as (xmin, ymin, xmax, ymax) on pixel edges. + """Return the box around a 0.x RLE mask, as (xmin, ymin, xmax, ymax) on pixel edges. - Same convention as 0.x boxes, which the 0.x converter writes as (x, y, x + w, y + h) - clipped to the image size. Mask `size` is (height, width), as in COCO. + Same convention as 0.x boxes, which the 0.x converter writes as (x, y, x + w, y + h) clipped + to the image size. Mask `size` is (height, width), as in COCO. """ rle = {"counts": base64.b64decode(mask["counts"]), "size": mask["size"]} x, y, w, h = (float(v) for v in cocomask.toBbox(rle)) @@ -718,17 +878,17 @@ def _mask_bbox(mask: Record) -> list[float]: @staticmethod def _vehicle_token(log: Record) -> str: - """Deterministic token: 0.x has no vehicle table.""" + """Return a deterministic vehicle token for a log: 0.x has no vehicle table.""" return hashlib.md5(f"{log['token']}/vehicle".encode()).hexdigest() @staticmethod def _none_if_empty(value: Any) -> Any: - """0.x writes "" for missing values; 1.0 uses None.""" + """Return None for "", which 0.x writes for missing values; other values unchanged.""" return None if value == "" else value @staticmethod def _label_origin(record: Record) -> tuple[str, str | None, float | None]: - """`annotation_type`, `provenance` and `confidence` from the 0.x autolabel fields. + """Return `annotation_type`, `provenance` and `confidence` from the 0.x autolabel fields. Manual unless 0.x marks the label as automatic. `provenance` lists the autolabel models; `confidence` is the model score when there is exactly one model. diff --git a/t4_devkit/schema/v1/sensor.py b/t4_devkit/schema/v1/sensor.py index 5cd96b82..ec0d4be8 100644 --- a/t4_devkit/schema/v1/sensor.py +++ b/t4_devkit/schema/v1/sensor.py @@ -43,7 +43,7 @@ def num_coefficients(self) -> int: @classmethod def from_num_coefficients(cls, num_coefficients: int) -> DistortionModel: - """The model with this number of coefficients (each model has a different count).""" + """Return the model with this number of coefficients (each model has a different count).""" for model in cls: if model.num_coefficients == num_coefficients: return model diff --git a/t4_devkit/schema/v1/writer.py b/t4_devkit/schema/v1/writer.py index 0f91b893..dc8a605e 100644 --- a/t4_devkit/schema/v1/writer.py +++ b/t4_devkit/schema/v1/writer.py @@ -173,7 +173,7 @@ def _data_path(self, channel: str, extension: str) -> str: return path def _asset_paths(self) -> list[str]: - """Every file under the asset folders, relative to the dataset root, sorted.""" + """Return every file under the asset folders, relative to the dataset root, sorted.""" return sorted( filepath.relative_to(self.output_root).as_posix() for directory in Manifest.ASSET_DIRS diff --git a/t4_devkit/tier4.py b/t4_devkit/tier4.py index bb1ea950..fe179288 100644 --- a/t4_devkit/tier4.py +++ b/t4_devkit/tier4.py @@ -695,8 +695,9 @@ def render_pointcloud( ) def _key_sample(self, sample_token: str | None, *, forward: bool) -> Sample | None: - """The first sample with a key frame from a sample (included), following `next` or - `prev`, or None.""" + """Return the first sample with a key frame from a sample (included), following `next` or + `prev`, or None. + """ while sample_token is not None: sample: Sample = self.get("sample", sample_token) if sample.has_key_frame: @@ -705,7 +706,9 @@ def _key_sample(self, sample_token: str | None, *, forward: bool) -> Sample | No return None def _box_visibility(self, ann: SampleAnnotation) -> VisibilityLevel: - """Visibility of a 3D box; `UNAVAILABLE` when the annotation's visibility is unknown.""" + """Return the visibility of a 3D box; `UNAVAILABLE` when the annotation's visibility is + unknown. + """ if ann.visibility_token is None: return VisibilityLevel.UNAVAILABLE visibility: Visibility = self.get("visibility", ann.visibility_token) @@ -713,7 +716,7 @@ def _box_visibility(self, ann: SampleAnnotation) -> VisibilityLevel: @staticmethod def _box_confidence(ann: SampleAnnotation | ObjectAnn) -> float: - """Confidence of a box: the label's, or 1.0 for labels without one (manual).""" + """Return the confidence of a box: the label's, or 1.0 for labels without one (manual).""" return 1.0 if ann.confidence is None else ann.confidence From df1fd1f35c8b83881f4d3a8b005bfd350d9453f4 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 17:13:43 +0900 Subject: [PATCH 05/10] style: no ** dict spreading in tests and the sample generator --- tests/sample/t4dataset_v1/make_sample_v1.py | 28 +++++++++------------ tests/sanity/test_v1_checkers.py | 8 +++--- tests/sanity/test_v1_record_checkers.py | 2 +- tests/sanity/test_v1_reference_checkers.py | 2 +- tests/schema/v1/test_convert_v0.py | 6 ++--- tests/schema/v1/test_core_tables.py | 2 +- 6 files changed, 22 insertions(+), 26 deletions(-) diff --git a/tests/sample/t4dataset_v1/make_sample_v1.py b/tests/sample/t4dataset_v1/make_sample_v1.py index ebb97aff..0bd10532 100644 --- a/tests/sample/t4dataset_v1/make_sample_v1.py +++ b/tests/sample/t4dataset_v1/make_sample_v1.py @@ -143,8 +143,8 @@ def link(tokens: list[str], i: int) -> dict: "timestamp": timestamps[i], "is_key_frame": i < NUM_FRAMES, "is_valid": True, - **link(tokens, i), } + | link(tokens, i) ) ego_pose.append( { @@ -169,8 +169,8 @@ def link(tokens: list[str], i: int) -> dict: "timestamp": timestamps[i], "scene_token": "scene", "has_key_frame": i < NUM_FRAMES, - **link(samples, i), } + | link(samples, i) for i, s in enumerate(samples) ], "sample_data": sample_data, @@ -350,13 +350,9 @@ def link(tokens: list[str], i: int) -> dict: "acceleration": None, "num_lidar_pts": 0, "num_radar_pts": None, - **link(sample_annotations, i), - **( - label() - if i == 0 - else label("interpolated", provenance="t4-devkit interpolation") - ), } + | link(sample_annotations, i) + | (label() if i == 0 else label("interpolated", provenance="t4-devkit interpolation")) for i, token in enumerate(sample_annotations) ], "object_ann": [ @@ -370,8 +366,8 @@ def link(tokens: list[str], i: int) -> dict: "mask": None, "orientation": None, "number": None, - **label(), - }, + } + | label(), { "token": "object_ann1", "sample_data_token": "cam0", @@ -382,8 +378,8 @@ def link(tokens: list[str], i: int) -> dict: "mask": road_mask(), "orientation": None, "number": None, - **label(), - }, + } + | label(), ], "keypoint": [ { @@ -393,8 +389,8 @@ def link(tokens: list[str], i: int) -> dict: "category_tokens": ["category_car"], "keypoints": [[12.0, 18.0]], "num_keypoints": 1, - **label(), } + | label() ], "traffic_light": [ { @@ -406,8 +402,8 @@ def link(tokens: list[str], i: int) -> dict: ], "object_ann_token": None, "instance_token": None, - **label("auto", provenance="sample recognizer", confidence=0.9), } + | label("auto", provenance="sample recognizer", confidence=0.9) ], "route": [ { @@ -440,9 +436,9 @@ def link(tokens: list[str], i: int) -> dict: "dynamic_entities": None, "justification": None, "taxonomy": {"id": "cod-scene", "version": "0.1.1"}, - **label("auto", provenance="sample classifier", confidence=0.9), "raw": None, } + | label("auto", provenance="sample classifier", confidence=0.9) ], "caption": [ { @@ -456,8 +452,8 @@ def link(tokens: list[str], i: int) -> dict: "caption_type": "description", "text": "The ego vehicle follows a car.", "language": "en", - **label(), } + | label() ], } diff --git a/tests/sanity/test_v1_checkers.py b/tests/sanity/test_v1_checkers.py index abdc4a8e..0a094273 100644 --- a/tests/sanity/test_v1_checkers.py +++ b/tests/sanity/test_v1_checkers.py @@ -61,7 +61,7 @@ def test_v1_sample_passes(tmp_path: Path) -> None: def test_str010_unsupported_version(tmp_path: Path) -> None: """An unsupported schema version fails, and the other 1.0 rules are skipped.""" data_root = _copy_v1(tmp_path) - _edit_json(data_root / "manifest.json", lambda m: {**m, "schema_version": "2.0.0"}) + _edit_json(data_root / "manifest.json", lambda m: dict(m, schema_version="2.0.0")) reports = _reports(data_root) assert reports["STR010"].is_failed() assert reports["FMT020"].is_failed() @@ -96,7 +96,7 @@ def test_str014_required_table(tmp_path: Path) -> None: data_root = _copy_v1(tmp_path) _edit_json( data_root / "manifest.json", - lambda m: {**m, "tables": {k: v for k, v in m["tables"].items() if k != "vehicle"}}, + lambda m: dict(m, tables={k: v for k, v in m["tables"].items() if k != "vehicle"}), ) reports = _reports(data_root) assert reports["STR014"].is_failed() and "vehicle" in reports["STR014"].reasons[0] @@ -113,8 +113,8 @@ def test_str015_identity(tmp_path: Path) -> None: @pytest.mark.parametrize( ("table", "rule", "change"), [ - ("sensor", "FMT024", lambda records: [{**records[0], "index": 300}, *records[1:]]), - ("sample", "FMT021", lambda records: [{**records[0], "prev": ""}, *records[1:]]), + ("sensor", "FMT024", lambda records: [dict(records[0], index=300), *records[1:]]), + ("sample", "FMT021", lambda records: [dict(records[0], prev=""), *records[1:]]), ], ) def test_fmt_invalid_record(tmp_path: Path, table: str, rule: str, change) -> None: diff --git a/tests/sanity/test_v1_record_checkers.py b/tests/sanity/test_v1_record_checkers.py index 4bb40b5b..167ee5b0 100644 --- a/tests/sanity/test_v1_record_checkers.py +++ b/tests/sanity/test_v1_record_checkers.py @@ -35,7 +35,7 @@ def _reports(data_root: Path, **kwargs) -> dict: def _second_instance(records: list[dict]) -> list[dict]: - return [*_set(records, 0, index=5), {**records[0], "token": "instance_other"}] + return [*_set(records, 0, index=5), dict(records[0], token="instance_other")] def _add_label_column(records: list[dict]) -> list[dict]: diff --git a/tests/sanity/test_v1_reference_checkers.py b/tests/sanity/test_v1_reference_checkers.py index 2d845564..4812b1cf 100644 --- a/tests/sanity/test_v1_reference_checkers.py +++ b/tests/sanity/test_v1_reference_checkers.py @@ -77,7 +77,7 @@ def _reports(data_root: Path) -> dict: ( "REF206", "sample_data", - lambda r: [{**x, "num_points": 99} if x["num_points"] else x for x in r], + lambda r: [dict(x, num_points=99) if x["num_points"] else x for x in r], ), ("REF207", "map", lambda r: _set(r, 0, sha256="0" * 64)), ("REF302", "instance", lambda r: _set(r, 0, map_element={"id": 99999, "type": "way"})), diff --git a/tests/schema/v1/test_convert_v0.py b/tests/schema/v1/test_convert_v0.py index 5f1619fb..351919e8 100644 --- a/tests/schema/v1/test_convert_v0.py +++ b/tests/schema/v1/test_convert_v0.py @@ -166,13 +166,13 @@ def test_load_v0_per_camera_samples(tmp_path: Path) -> None: data_root = _copy_v0(tmp_path) samples = json.loads((data_root / "annotation" / "sample.json").read_text()) first = samples[0]["token"] - extra = {**samples[0], "token": "f" * 32} # sorts after `first` + extra = dict(samples[0], token="f" * 32) # sorts after `first` _edit_v0(data_root, "sample", lambda records: [*records, extra]) _edit_v0( data_root, "sample_data", lambda records: [ - {**r, "sample_token": extra["token"]} if r["filename"] == "data/CAM_BACK/001.jpg" else r + dict(r, sample_token=extra["token"]) if r["filename"] == "data/CAM_BACK/001.jpg" else r for r in records ], ) @@ -211,7 +211,7 @@ def test_load_v0_labeled_frame_is_key(tmp_path: Path) -> None: data_root, "sample_data", lambda records: [ - {**r, "is_key_frame": False} if r["token"] == labeled["sample_data_token"] else r + dict(r, is_key_frame=False) if r["token"] == labeled["sample_data_token"] else r for r in records ], ) diff --git a/tests/schema/v1/test_core_tables.py b/tests/schema/v1/test_core_tables.py index fdc7ce4c..9972f58d 100644 --- a/tests/schema/v1/test_core_tables.py +++ b/tests/schema/v1/test_core_tables.py @@ -43,7 +43,7 @@ def test_sample_data_needs_sample() -> None: non_key = next(record for record in records if not record["is_key_frame"]) SampleData.from_dict(non_key) with pytest.raises(TypeError): - SampleData.from_dict({**non_key, "sample_token": None}) + SampleData.from_dict(dict(non_key, sample_token=None)) def test_invalid_covariance_length() -> None: From 456dfea46877fe9a3cc7d0feee5f9e4d0d228721 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 17:22:12 +0900 Subject: [PATCH 06/10] test: shared SampleCopy helper, merge round-trip and invalid-record tests, drop redundant tests --- tests/cli/test_migrate.py | 2 +- tests/common/test_io.py | 34 +---- tests/helpers.py | 87 +++++++++++ tests/sample/t4dataset_v1/make_sample_v1.py | 27 ++-- tests/sample/t4dataset_v1/test_sample_v1.py | 26 ++-- tests/sanity/test_v1_checkers.py | 104 +++++-------- tests/sanity/test_v1_record_checkers.py | 130 ++++++---------- tests/sanity/test_v1_reference_checkers.py | 96 ++++-------- tests/schema/v1/test_caption_table.py | 28 ---- tests/schema/v1/test_convert_v0.py | 157 ++++++-------------- tests/schema/v1/test_core_tables.py | 53 ------- tests/schema/v1/test_label_tables.py | 56 ------- tests/schema/v1/test_loader.py | 25 ++-- tests/schema/v1/test_manifest.py | 13 +- tests/schema/v1/test_migrate_v0.py | 91 ++++++------ tests/schema/v1/test_planning_tables.py | 42 ------ tests/schema/v1/test_registry.py | 5 - tests/schema/v1/test_sensor.py | 58 -------- tests/schema/v1/test_tables.py | 71 +++++++++ tests/test_tier4.py | 52 ++++--- 20 files changed, 429 insertions(+), 728 deletions(-) create mode 100644 tests/helpers.py delete mode 100644 tests/schema/v1/test_caption_table.py delete mode 100644 tests/schema/v1/test_core_tables.py delete mode 100644 tests/schema/v1/test_label_tables.py delete mode 100644 tests/schema/v1/test_planning_tables.py delete mode 100644 tests/schema/v1/test_sensor.py create mode 100644 tests/schema/v1/test_tables.py diff --git a/tests/cli/test_migrate.py b/tests/cli/test_migrate.py index dd1123a7..d917e0d5 100644 --- a/tests/cli/test_migrate.py +++ b/tests/cli/test_migrate.py @@ -16,7 +16,7 @@ def test_t4migrate(tmp_path: Path) -> None: with pytest.warns(UserWarning, match="kept as float32"): result = CliRunner().invoke(cli, [SAMPLE_V0.as_posix(), tmp_path.as_posix(), "-d", "3"]) - assert result.exit_code == 0, result.output + assert result.exit_code == 0, result.output # t4migrate exits 1 if a sanity rule fails manifest = json.loads((tmp_path / "t4dataset" / "3" / "manifest.json").read_text()) assert (manifest["dataset_id"], manifest["version"]) == ("t4dataset", 3) assert manifest["migrated_from_schema_version"] == "0.x" diff --git a/tests/common/test_io.py b/tests/common/test_io.py index e7bc77bd..7830f841 100644 --- a/tests/common/test_io.py +++ b/tests/common/test_io.py @@ -1,8 +1,9 @@ +from __future__ import annotations + import tempfile from pathlib import Path import numpy as np -import pyarrow as pa import pyarrow.parquet as pq import pypcd4 import pytest @@ -18,7 +19,6 @@ ) SAMPLE_PARQUET = Path(__file__).parents[1] / "sample/t4dataset_v1/data/LIDAR_TOP/LIDAR_TOP.parquet" -SAMPLE_VIDEO = Path(__file__).parents[1] / "sample/t4dataset_v1/data/CAM_FRONT/CAM_FRONT.mp4" def test_json_io() -> None: @@ -46,20 +46,6 @@ def test_load_pointcloud_parquet() -> None: load_pointcloud(SAMPLE_PARQUET) -def test_load_pointcloud_parquet_null_column(tmp_path: Path) -> None: - """A column null on every point of a frame (a frame without labels) is left out.""" - schema = pa.schema([("x", pa.float32()), ("label", pa.uint8())]) - path = tmp_path / "LIDAR_TOP.parquet" - with pq.ParquetWriter(path, schema) as writer: - writer.write_table(pa.table([pa.array([1.0, 2.0]), pa.array([0, 3])], schema=schema)) - writer.write_table(pa.table([pa.array([1.0, 2.0]), pa.nulls(2, pa.uint8())], schema=schema)) - - labeled = load_pointcloud(path, frame_index=0) - assert labeled["label"].tolist() == [0, 3] - assert labeled["label"].dtype == np.uint8 - assert list(load_pointcloud(path, frame_index=1)) == ["x"] - - def test_load_pointcloud_bin(tmp_path: Path) -> None: """A .bin file is read with the layout given in `fields`.""" fields = [("x", "float32"), ("y", "float32"), ("z", "float32"), ("ring", "uint16")] @@ -84,19 +70,6 @@ def test_load_pointcloud_pcd(tmp_path: Path) -> None: assert frame["ring"].dtype == np.uint16 -def test_load_image_video() -> None: - """Sample frames: red ramps 0..255 across the width, green is 128 * frame_index.""" - for frame_index in (1, 0): - image = load_image(SAMPLE_VIDEO, frame_index=frame_index) - assert image.shape == (48, 64, 3) - assert image.dtype == np.uint8 - assert abs(image[..., 1].mean() - 128 * frame_index) < 4 - assert image[:, 0, 0].mean() < 8 and image[:, -1, 0].mean() > 240 - - with pytest.raises(IndexError): - load_image(SAMPLE_VIDEO, frame_index=2) - - def test_load_image_file(tmp_path: Path) -> None: path = tmp_path / "frame.png" pixels = np.random.default_rng(0).integers(0, 256, (4, 6, 3), dtype=np.uint8) @@ -164,6 +137,9 @@ def test_save_video(tmp_path: Path, dtype, bit_depth: int, max_value: int) -> No green = image[..., 1].mean() / max_value assert abs(green - 0.25 * frame_index) < 0.03 + with pytest.raises(IndexError): + load_image(path, frame_index=3) + def test_save_pointcloud_null_column(tmp_path: Path) -> None: """A field that is None in a frame is written null and left out when read.""" diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 00000000..db963d6e --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any, Callable + +import pyarrow as pa +import pyarrow.parquet as pq + +from t4_devkit.sanity import sanity_check + +SAMPLE_ROOT = Path(__file__).parent / "sample" +SAMPLE_V0 = SAMPLE_ROOT / "t4dataset" +SAMPLE_V1 = SAMPLE_ROOT / "t4dataset_v1" + + +class SampleCopy: + """A writable copy of a sample dataset, for tests that change or break it. + + Examples: + >>> sample = SampleCopy(tmp_path) # the 1.0 sample, as `/sample` + >>> sample.edit("scene", lambda records: SampleCopy.set_field(records, 0, nbr_samples=5)) + >>> sample.reports()["REC012"].is_failed() + True + """ + + def __init__(self, tmp_path: Path, source: Path = SAMPLE_V1, name: str = "sample") -> None: + """ + Args: + tmp_path (Path): Directory to copy into. + source (Path, optional): Sample to copy, the 1.0 sample by default. + name (str, optional): Folder name; "sample" is the 1.0 sample's `dataset_id`. + """ + self.root = tmp_path / name + shutil.copytree(source, self.root) + + def read(self, table: str) -> Any: + """Return a table (or "manifest") as stored.""" + return json.loads(self._path(table).read_text()) + + def edit(self, table: str, change: Callable[[Any], Any]) -> None: + """Replace a table (or "manifest") with `change(records)`.""" + self._path(table).write_text(json.dumps(change(self.read(table)))) + + def reports(self, **kwargs: Any) -> dict[str, Any]: + """Return the sanity reports by rule ID; `kwargs` go to `sanity_check`.""" + result = sanity_check(self.root.as_posix(), **kwargs) + return {report.id: report for report in result.reports} + + def write_labels(self, frames: list[list[int | None] | None]) -> None: + """Add a `label` column to the 1.0 sample's LiDAR file and to its `sensor.json` fields. + + Args: + frames (list[list[int | None] | None]): Labels per frame; None for a frame without + labels. + """ + path = self.root / "data/LIDAR_TOP/LIDAR_TOP.parquet" + parquet = pq.ParquetFile(path) + schema = parquet.schema_arrow.append(pa.field("label", pa.uint8())) + with pq.ParquetWriter(path.with_suffix(".tmp"), schema) as writer: + for i, labels in enumerate(frames): + table = parquet.read_row_group(i) + column = ( + pa.nulls(table.num_rows, pa.uint8()) + if labels is None + else pa.array(labels, pa.uint8()) + ) + writer.write_table(table.append_column("label", column)) + path.with_suffix(".tmp").replace(path) + + def declare(records: list[dict]) -> list[dict]: + records[1]["lidar"]["fields"].append({"name": "label", "dtype": "uint8"}) + return records + + self.edit("sensor", declare) + + @staticmethod + def set_field(records: list[dict], position: int, **fields: Any) -> list[dict]: + """Return the records with `fields` set on the record at `position`.""" + records[position].update(fields) + return records + + def _path(self, table: str) -> Path: + if table == "manifest": + return self.root / "manifest.json" + return self.root / "annotation" / f"{table}.json" diff --git a/tests/sample/t4dataset_v1/make_sample_v1.py b/tests/sample/t4dataset_v1/make_sample_v1.py index 0bd10532..9383d4bd 100644 --- a/tests/sample/t4dataset_v1/make_sample_v1.py +++ b/tests/sample/t4dataset_v1/make_sample_v1.py @@ -489,23 +489,28 @@ def link(tokens: list[str], i: int) -> dict: } -def main() -> None: +def write(output_root: Path) -> None: + """Write the sample dataset to `output_root`, which must not exist.""" + writer = DatasetWriter(output_root) + camera_video = writer.write_camera("CAM_FRONT", camera_frames(), FPS) + lidar_path, _ = writer.write_lidar("LIDAR_TOP", lidar_frames(), LIDAR_FIELDS) with tempfile.TemporaryDirectory() as tmp: - output_root = Path(tmp) / "sample" - writer = DatasetWriter(output_root) - camera_video = writer.write_camera("CAM_FRONT", camera_frames(), FPS) - lidar_path, _ = writer.write_lidar("LIDAR_TOP", lidar_frames(), LIDAR_FIELDS) map_file = Path(tmp) / "lanelet2_map.osm" write_map(map_file) writer.copy(map_file, MAP_PATH) + map_sha256 = hashlib.sha256((output_root / MAP_PATH).read_bytes()).hexdigest() + records = { + SchemaName(name): [SCHEMAS.get(SchemaName(name)).from_dict(data) for data in rows] + for name, rows in tables(camera_video, lidar_path, map_sha256).items() + } + writer.save(records, MANIFEST_INFO) - map_sha256 = hashlib.sha256(map_file.read_bytes()).hexdigest() - records = { - SchemaName(name): [SCHEMAS.get(SchemaName(name)).from_dict(data) for data in rows] - for name, rows in tables(camera_video, lidar_path, map_sha256).items() - } - writer.save(records, MANIFEST_INFO) +def main() -> None: + """Write the sample dataset in place of this directory's.""" + with tempfile.TemporaryDirectory() as tmp: + output_root = Path(tmp) / "sample" + write(output_root) for name in ("annotation", "data", "map", "manifest.json"): target = ROOT / name if target.is_dir(): diff --git a/tests/sample/t4dataset_v1/test_sample_v1.py b/tests/sample/t4dataset_v1/test_sample_v1.py index 41696480..736d9ba7 100644 --- a/tests/sample/t4dataset_v1/test_sample_v1.py +++ b/tests/sample/t4dataset_v1/test_sample_v1.py @@ -1,19 +1,23 @@ from __future__ import annotations -import hashlib -import json from pathlib import Path +import make_sample_v1 + SAMPLE_V1 = Path(__file__).parent -def test_sample_v1_manifest_matches_files() -> None: - """Every table and asset listed in manifest.json exists and has the listed hash.""" - manifest = json.loads((SAMPLE_V1 / "manifest.json").read_text()) - assert manifest["schema_version"] == "1.0.0" +def test_generator_reproduces_sample(tmp_path: Path) -> None: + """`make_sample_v1.py` writes the committed sample: run it after changing either. - for entry in [*manifest["tables"].values(), *manifest["assets"]]: - data = (SAMPLE_V1 / entry["path"]).read_bytes() - assert hashlib.sha256(data).hexdigest() == entry["sha256"], entry["path"] - if "bytes" in entry: - assert len(data) == entry["bytes"], entry["path"] + The video and the manifest (its hash and the t4-devkit version) are left out: their bytes + depend on the encoder build and the installed version. + """ + make_sample_v1.write(tmp_path / "sample") + paths = [ + *(f"annotation/{path.name}" for path in (SAMPLE_V1 / "annotation").glob("*.json")), + "data/LIDAR_TOP/LIDAR_TOP.parquet", + "map/lanelet2_map.osm", + ] + for path in paths: + assert (tmp_path / "sample" / path).read_bytes() == (SAMPLE_V1 / path).read_bytes(), path diff --git a/tests/sanity/test_v1_checkers.py b/tests/sanity/test_v1_checkers.py index 0a094273..0cafb30d 100644 --- a/tests/sanity/test_v1_checkers.py +++ b/tests/sanity/test_v1_checkers.py @@ -1,36 +1,11 @@ from __future__ import annotations -import json -import shutil from pathlib import Path -import pytest - from t4_devkit.sanity import sanity_check from t4_devkit.sanity.checker import SchemaVersion from t4_devkit.sanity.registry import CHECKERS - -SAMPLE_ROOT = Path(__file__).parents[1] / "sample" -SAMPLE_V0 = SAMPLE_ROOT / "t4dataset" -SAMPLE_V1 = SAMPLE_ROOT / "t4dataset_v1" - - -def _reports(data_root: Path, **kwargs) -> dict: - return {report.id: report for report in sanity_check(data_root.as_posix(), **kwargs).reports} - - -def _failed(reports: dict) -> set[str]: - return {id for id, report in reports.items() if report.is_failed()} - - -def _copy_v1(tmp_path: Path) -> Path: - data_root = tmp_path / "sample" # named as the sample's `dataset_id` - shutil.copytree(SAMPLE_V1, data_root) - return data_root - - -def _edit_json(path: Path, change) -> None: - path.write_text(json.dumps(change(json.loads(path.read_text())))) +from tests.helpers import SAMPLE_V0, SAMPLE_V1, SampleCopy def test_rules_by_version() -> None: @@ -41,94 +16,83 @@ def test_rules_by_version() -> None: for id, checker in rules.items() if SchemaVersion.V0 in checker.versions } - assert set(_reports(SAMPLE_V0)) == v0_rules - assert len(v0_rules) == 60 + assert {report.id for report in sanity_check(SAMPLE_V0.as_posix()).reports} == v0_rules - v1_reports = _reports(SAMPLE_V1) + v1_reports = {report.id for report in sanity_check(SAMPLE_V1.as_posix()).reports} assert "STR006" not in v1_reports and "FMT001" not in v1_reports - assert {"STR001", "TIV001", "STR010", "FMT020", "FMT040"} <= set(v1_reports) + assert {"STR001", "TIV001", "STR010", "FMT020", "FMT040"} <= v1_reports def test_v1_sample_passes(tmp_path: Path) -> None: - """The 1.0 sample passes; the warnings are the missing version, bag and pointcloud map - directories, and hashes not verified (opt-in).""" - result = sanity_check(_copy_v1(tmp_path).as_posix()) + """The 1.0 sample passes every rule, hashes included; the only warnings are the missing + version, bag and pointcloud map directories.""" + result = sanity_check(SampleCopy(tmp_path).root.as_posix(), checksum=True) assert result.is_passed() warnings = {report.id for report in result.reports if not report.is_passed(strict=True)} - assert warnings == {"STR001", "STR005", "STR009", "STR012"} + assert warnings == {"STR001", "STR005", "STR009"} def test_str010_unsupported_version(tmp_path: Path) -> None: """An unsupported schema version fails, and the other 1.0 rules are skipped.""" - data_root = _copy_v1(tmp_path) - _edit_json(data_root / "manifest.json", lambda m: dict(m, schema_version="2.0.0")) - reports = _reports(data_root) + sample = SampleCopy(tmp_path) + sample.edit("manifest", lambda m: dict(m, schema_version="2.0.0")) + reports = sample.reports() assert reports["STR010"].is_failed() assert reports["FMT020"].is_failed() assert reports["STR011"].is_skipped() and reports["FMT021"].is_skipped() def test_str011_missing_file(tmp_path: Path) -> None: - data_root = _copy_v1(tmp_path) - (data_root / "map" / "lanelet2_map.osm").unlink() - assert "STR011" in _failed(_reports(data_root)) + sample = SampleCopy(tmp_path) + (sample.root / "map" / "lanelet2_map.osm").unlink() + assert sample.reports()["STR011"].is_failed() def test_str012_checksum(tmp_path: Path) -> None: """Hashes are verified only with `checksum`; a changed file then fails.""" - data_root = _copy_v1(tmp_path) - with (data_root / "map" / "lanelet2_map.osm").open("a") as f: + sample = SampleCopy(tmp_path) + with (sample.root / "map" / "lanelet2_map.osm").open("a") as f: f.write(" ") - assert _reports(data_root)["STR012"].severity == "WARNING" - report = _reports(data_root, checksum=True)["STR012"] + assert sample.reports()["STR012"].severity == "WARNING" + report = sample.reports(checksum=True)["STR012"] assert report.is_failed() and "lanelet2_map.osm" in report.reasons[0] def test_str013_unlisted_file(tmp_path: Path) -> None: - data_root = _copy_v1(tmp_path) - (data_root / "data" / "extra.txt").write_text("not listed") - report = _reports(data_root)["STR013"] + sample = SampleCopy(tmp_path) + (sample.root / "data" / "extra.txt").write_text("not listed") + report = sample.reports()["STR013"] assert not report.is_passed(strict=True) and "data/extra.txt" in report.reasons[0] def test_str014_required_table(tmp_path: Path) -> None: """An unlisted required table fails STR014, and its format rule is skipped.""" - data_root = _copy_v1(tmp_path) - _edit_json( - data_root / "manifest.json", + sample = SampleCopy(tmp_path) + sample.edit( + "manifest", lambda m: dict(m, tables={k: v for k, v in m["tables"].items() if k != "vehicle"}), ) - reports = _reports(data_root) + reports = sample.reports() assert reports["STR014"].is_failed() and "vehicle" in reports["STR014"].reasons[0] assert reports["FMT027"].is_skipped() def test_str015_identity(tmp_path: Path) -> None: - data_root = tmp_path / "other_id" - shutil.copytree(SAMPLE_V1, data_root) - report = _reports(data_root)["STR015"] + report = SampleCopy(tmp_path, name="other_id").reports()["STR015"] assert not report.is_passed(strict=True) and "'sample'" in report.reasons[0] -@pytest.mark.parametrize( - ("table", "rule", "change"), - [ - ("sensor", "FMT024", lambda records: [dict(records[0], index=300), *records[1:]]), - ("sample", "FMT021", lambda records: [dict(records[0], prev=""), *records[1:]]), - ], -) -def test_fmt_invalid_record(tmp_path: Path, table: str, rule: str, change) -> None: +def test_fmt_invalid_record(tmp_path: Path) -> None: """A record that does not build into its 1.0 class fails its table's format rule.""" - data_root = _copy_v1(tmp_path) - path = data_root / "annotation" / f"{table}.json" - token = json.loads(path.read_text())[0]["token"] - _edit_json(path, change) - report = _reports(data_root)[rule] + sample = SampleCopy(tmp_path) + token = sample.read("sensor")[0]["token"] + sample.edit("sensor", lambda records: SampleCopy.set_field(records, 0, index=300)) + report = sample.reports()["FMT024"] assert report.is_failed() and token in report.reasons[0] def test_fmt_unreadable_table(tmp_path: Path) -> None: - data_root = _copy_v1(tmp_path) - (data_root / "annotation" / "scene.json").write_text("not json") - report = _reports(data_root)["FMT025"] + sample = SampleCopy(tmp_path) + (sample.root / "annotation" / "scene.json").write_text("not json") + report = sample.reports()["FMT025"] assert report.is_failed() and "cannot be read" in report.reasons[0] diff --git a/tests/sanity/test_v1_record_checkers.py b/tests/sanity/test_v1_record_checkers.py index 167ee5b0..f455c3c7 100644 --- a/tests/sanity/test_v1_record_checkers.py +++ b/tests/sanity/test_v1_record_checkers.py @@ -1,44 +1,19 @@ from __future__ import annotations -import json -import shutil from pathlib import Path -import pyarrow as pa -import pyarrow.parquet as pq import pytest -from t4_devkit.sanity import sanity_check +from tests.helpers import SampleCopy -SAMPLE_V1 = Path(__file__).parents[1] / "sample" / "t4dataset_v1" - - -def _copy_v1(tmp_path: Path) -> Path: - data_root = tmp_path / "sample" # named as the sample's `dataset_id` - shutil.copytree(SAMPLE_V1, data_root) - return data_root - - -def _edit(data_root: Path, name: str, change) -> None: - """Edit a table (or `manifest`) of the copy in place.""" - path = data_root / ("manifest.json" if name == "manifest" else f"annotation/{name}.json") - path.write_text(json.dumps(change(json.loads(path.read_text())))) - - -def _set(records: list[dict], position: int, **fields) -> list[dict]: - records[position].update(fields) - return records - - -def _reports(data_root: Path, **kwargs) -> dict: - return {report.id: report for report in sanity_check(data_root.as_posix(), **kwargs).reports} +set_field = SampleCopy.set_field def _second_instance(records: list[dict]) -> list[dict]: - return [*_set(records, 0, index=5), dict(records[0], token="instance_other")] + return [*set_field(records, 0, index=5), dict(records[0], token="instance_other")] -def _add_label_column(records: list[dict]) -> list[dict]: +def _declare_label_field(records: list[dict]) -> list[dict]: records[1]["lidar"]["fields"].append({"name": "label", "dtype": "uint8"}) return records @@ -54,39 +29,34 @@ def edit(manifest: dict) -> dict: # (rule, table, change): each change breaks only what the rule checks. CASES = [ ("REC008", "sensor", lambda r: []), - ("REC009", "sensor", lambda r: _set(r, 1, index=0)), + ("REC009", "sensor", lambda r: set_field(r, 1, index=0)), ("REC010", "instance", _second_instance), - ("REC011", "sensor", _add_label_column), - ("REC012", "scene", lambda r: _set(r, 0, nbr_samples=5)), - ("REC013", "instance", lambda r: _set(r, 0, nbr_annotations=2)), - ("REC014", "object_ann", lambda r: _set(r, 1, instance_token="instance_car")), - ("REC015", "object_ann", lambda r: _set(r, 0, orientation=1.0)), - ("REC016", "sample_annotation", lambda r: _set(r, 0, confidence=0.5)), - ("REC017", "sample_annotation", lambda r: _set(r, 1, provenance=None)), - ("REC018", "sample", lambda r: _set(r, 0, has_key_frame=False)), + ("REC011", "sensor", _declare_label_field), + ("REC012", "scene", lambda r: set_field(r, 0, nbr_samples=5)), + ("REC013", "instance", lambda r: set_field(r, 0, nbr_annotations=2)), + ("REC014", "object_ann", lambda r: set_field(r, 1, instance_token="instance_car")), + ("REC015", "object_ann", lambda r: set_field(r, 0, orientation=1.0)), + ("REC016", "sample_annotation", lambda r: set_field(r, 0, confidence=0.5)), + ("REC017", "sample_annotation", lambda r: set_field(r, 1, provenance=None)), + ("REC018", "sample", lambda r: set_field(r, 0, has_key_frame=False)), ("REC019", "sensor", lambda r: (r[0]["camera"]["video"].update(frame_count=3), r)[1]), - ("REC020", "sample_data", lambda r: _set(r, 0, fileformat="jpg")), - ("REC021", "sample", lambda r: _set(r, 1, timestamp=r[0]["timestamp"] - 1)), - ("REC022", "ego_pose", lambda r: _set(r, 0, timestamp=1_704_067_200)), + ("REC020", "sample_data", lambda r: set_field(r, 0, fileformat="jpg")), + ("REC021", "sample", lambda r: set_field(r, 1, timestamp=r[0]["timestamp"] - 1)), + ("REC022", "ego_pose", lambda r: set_field(r, 0, timestamp=1_704_067_200)), ("REC023", "manifest", _capabilities(lambda c: c["modalities"]["camera"].append("CAM_BACK"))), ("REC024", "manifest", _capabilities(lambda c: c.update(route=False))), - ("REC025", "vehicle", lambda r: _set(r, 0, model="unknown")), - ("REC026", "route", lambda r: _set(r, 0, valid_until=r[0]["valid_from"])), - ("REC027", "sensor", lambda r: _set(r, 0, rotation=[2.0, 0.0, 0.0, 0.0])), - ("REC028", "sample_data", lambda r: _set(r, 0, is_key_frame=False)), + ("REC025", "vehicle", lambda r: set_field(r, 0, model="unknown")), + ("REC026", "route", lambda r: set_field(r, 0, valid_until=r[0]["valid_from"])), + ("REC027", "sensor", lambda r: set_field(r, 0, rotation=[2.0, 0.0, 0.0, 0.0])), + ("REC028", "sample_data", lambda r: set_field(r, 0, is_key_frame=False)), ] -def test_v1_sample_passes_record_rules() -> None: - reports = _reports(SAMPLE_V1) - assert all(reports[rule].is_passed(strict=True) for rule, *_ in CASES) - - @pytest.mark.parametrize(("rule", "table", "change"), CASES, ids=[case[0] for case in CASES]) def test_record_rule_fails(tmp_path: Path, rule: str, table: str, change) -> None: - data_root = _copy_v1(tmp_path) - _edit(data_root, table, change) - report = _reports(data_root)[rule] + sample = SampleCopy(tmp_path) + sample.edit(table, change) + report = sample.reports()[rule] assert not report.is_passed(strict=True), report assert report.reasons @@ -94,49 +64,37 @@ def test_record_rule_fails(tmp_path: Path, rule: str, table: str, change) -> Non @pytest.mark.parametrize( ("rule", "table", "change"), [ - ("REC012", "scene", lambda r: _set(r, 0, nbr_samples=5)), - ("REC013", "instance", lambda r: _set(r, 0, nbr_annotations=1, last_annotation_token=None)), + ("REC012", "scene", lambda r: set_field(r, 0, nbr_samples=5)), + ( + "REC013", + "instance", + lambda r: set_field(r, 0, nbr_annotations=1, last_annotation_token=None), + ), ], ) def test_record_rule_fix(tmp_path: Path, rule: str, table: str, change) -> None: """`fix` recomputes the values and updates the table's hash in `manifest.json`.""" - data_root = _copy_v1(tmp_path) - _edit(data_root, table, change) - assert _reports(data_root, fix=True)[rule].fixed - reports = _reports(data_root, checksum=True) + sample = SampleCopy(tmp_path) + sample.edit(table, change) + assert sample.reports(fix=True)[rule].fixed + reports = sample.reports(checksum=True) assert reports[rule].is_passed(strict=True) assert reports["STR012"].is_passed(strict=True) -def _write_labels(data_root: Path, frames: list[list[int | None] | None]) -> None: - """Add a `label` column to the LiDAR file and `sensor.json`; `None` = no labels.""" - path = data_root / "data/LIDAR_TOP/LIDAR_TOP.parquet" - parquet = pq.ParquetFile(path) - schema = parquet.schema_arrow.append(pa.field("label", pa.uint8())) - with pq.ParquetWriter(path.with_suffix(".tmp"), schema) as writer: - for i, labels in enumerate(frames): - table = parquet.read_row_group(i) - column = ( - pa.nulls(table.num_rows, pa.uint8()) - if labels is None - else pa.array(labels, pa.uint8()) - ) - writer.write_table(table.append_column("label", column)) - path.with_suffix(".tmp").replace(path) - _edit(data_root, "sensor", _add_label_column) - - def test_rec029_point_labels(tmp_path: Path) -> None: """Category 0 must be `unpainted`, and a frame's labels are all null or none null.""" - data_root = _copy_v1(tmp_path) - _write_labels(data_root, [[0] * 8, [1] * 7 + [None], None]) - _edit(data_root, "category", lambda r: _set(_set(r, 0, index=1), 1, index=0)) - - reasons = _reports(data_root)["REC029"].reasons + sample = SampleCopy(tmp_path / "bad") + sample.write_labels([[0] * 8, [1] * 7 + [None], None]) + sample.edit("category", lambda r: set_field(set_field(r, 0, index=1), 1, index=0)) + reasons = sample.reports()["REC029"].reasons assert "not 'unpainted'" in reasons[0] assert "frames [1]" in reasons[1] - _write_labels(_copy_v1(tmp_path / "ok"), [[0] * 8, [1] * 8, None]) - data_root = tmp_path / "ok" / "sample" - _edit(data_root, "category", lambda r: _set(_set(r, 0, index=1), 1, index=0, name="unpainted")) - assert _reports(data_root)["REC029"].is_passed(strict=True) + sample = SampleCopy(tmp_path / "ok") + sample.write_labels([[0] * 8, [1] * 8, None]) + sample.edit( + "category", + lambda r: set_field(set_field(r, 0, index=1), 1, index=0, name="unpainted"), + ) + assert sample.reports()["REC029"].is_passed(strict=True) diff --git a/tests/sanity/test_v1_reference_checkers.py b/tests/sanity/test_v1_reference_checkers.py index 4812b1cf..b0c7d79c 100644 --- a/tests/sanity/test_v1_reference_checkers.py +++ b/tests/sanity/test_v1_reference_checkers.py @@ -1,33 +1,12 @@ from __future__ import annotations -import json -import shutil from pathlib import Path -import pyarrow as pa -import pyarrow.parquet as pq import pytest -from t4_devkit.sanity import sanity_check +from tests.helpers import SampleCopy -SAMPLE_V1 = Path(__file__).parents[1] / "sample" / "t4dataset_v1" -LIDAR_FILE = "data/LIDAR_TOP/LIDAR_TOP.parquet" - - -def _copy_v1(tmp_path: Path) -> Path: - data_root = tmp_path / "sample" # named as the sample's `dataset_id` - shutil.copytree(SAMPLE_V1, data_root) - return data_root - - -def _edit(data_root: Path, name: str, change) -> None: - path = data_root / "annotation" / f"{name}.json" - path.write_text(json.dumps(change(json.loads(path.read_text())))) - - -def _set(records: list[dict], position: int, **fields) -> list[dict]: - records[position].update(fields) - return records +set_field = SampleCopy.set_field def _lidar(records: list[dict], **fields) -> list[dict]: @@ -36,56 +15,37 @@ def _lidar(records: list[dict], **fields) -> list[dict]: return records -def _add_label_column(data_root: Path) -> None: - """Add a `label` column whose values no category has.""" - path = data_root / LIDAR_FILE - parquet = pq.ParquetFile(path) - with pq.ParquetWriter( - path.with_suffix(".tmp"), parquet.schema_arrow.append(pa.field("label", pa.uint8())) - ) as writer: - for i in range(parquet.num_row_groups): - table = parquet.read_row_group(i) - writer.write_table( - table.append_column("label", pa.array([7] * table.num_rows, pa.uint8())) - ) - path.with_suffix(".tmp").replace(path) - - -def _reports(data_root: Path) -> dict: - return {report.id: report for report in sanity_check(data_root.as_posix()).reports} - - MISSING = "0" * 32 # (rule, table, change): each change breaks what the rule checks. CASES = [ - ("REF015", "sample_data", lambda r: _set(r, 0, sensor_token=MISSING)), - ("REF016", "log", lambda r: _set(r, 0, vehicle_token=MISSING)), - ("REF017", "log", lambda r: _set(r, 0, map_token=None)), - ("REF018", "sample_annotation", lambda r: _set(r, 0, visibility_token=MISSING)), - ("REF019", "object_ann", lambda r: _set(r, 0, category_token=MISSING)), - ("REF020", "keypoint", lambda r: _set(r, 0, category_tokens=[MISSING])), - ("REF021", "traffic_light", lambda r: _set(r, 0, object_ann_token=MISSING)), - ("REF022", "route", lambda r: _set(r, 0, map_token=MISSING)), - ("REF023", "scenario", lambda r: _set(r, 0, log_token=MISSING)), - ("REF024", "caption", lambda r: _set(r, 0, scenario_token=MISSING)), + ("REF015", "sample_data", lambda r: set_field(r, 0, sensor_token=MISSING)), + ("REF016", "log", lambda r: set_field(r, 0, vehicle_token=MISSING)), + ("REF017", "log", lambda r: set_field(r, 0, map_token=None)), + ("REF018", "sample_annotation", lambda r: set_field(r, 0, visibility_token=MISSING)), + ("REF019", "object_ann", lambda r: set_field(r, 0, category_token=MISSING)), + ("REF020", "keypoint", lambda r: set_field(r, 0, category_tokens=[MISSING])), + ("REF021", "traffic_light", lambda r: set_field(r, 0, object_ann_token=MISSING)), + ("REF022", "route", lambda r: set_field(r, 0, map_token=MISSING)), + ("REF023", "scenario", lambda r: set_field(r, 0, log_token=MISSING)), + ("REF024", "caption", lambda r: set_field(r, 0, scenario_token=MISSING)), ("REF025", "sensor", lambda r: _lidar(r, source_channels=["CAM_FRONT"])), - ("REF107", "sample", lambda r: _set(r, 0, next=r[2]["token"])), - ("REF108", "sample_data", lambda r: _set(r, 0, next=r[3]["token"])), - ("REF204", "sample_data", lambda r: _set(r, 0, filename="data/CAM_FRONT/other.mp4")), + ("REF107", "sample", lambda r: set_field(r, 0, next=r[2]["token"])), + ("REF108", "sample_data", lambda r: set_field(r, 0, next=r[3]["token"])), + ("REF204", "sample_data", lambda r: set_field(r, 0, filename="data/CAM_FRONT/other.mp4")), ("REF205", "sensor", lambda r: (r[1]["lidar"]["fields"][3].update(dtype="float64"), r)[1]), ( "REF206", "sample_data", lambda r: [dict(x, num_points=99) if x["num_points"] else x for x in r], ), - ("REF207", "map", lambda r: _set(r, 0, sha256="0" * 64)), - ("REF302", "instance", lambda r: _set(r, 0, map_element={"id": 99999, "type": "way"})), - ("REF303", "traffic_light", lambda r: _set(r, 0, regulatory_element_id=99999)), + ("REF207", "map", lambda r: set_field(r, 0, sha256="0" * 64)), + ("REF302", "instance", lambda r: set_field(r, 0, map_element={"id": 99999, "type": "way"})), + ("REF303", "traffic_light", lambda r: set_field(r, 0, regulatory_element_id=99999)), ( "REF304", "route", - lambda r: _set( + lambda r: set_field( r, 0, segments=[{"preferred_primitive_id": 99999, "primitive_ids": [99999]}] ), ), @@ -93,23 +53,17 @@ def _reports(data_root: Path) -> dict: ] -def test_v1_sample_passes_reference_rules() -> None: - reports = _reports(SAMPLE_V1) - rules = {rule for rule, *_ in CASES} | {"REF306"} - assert all(reports[rule].is_passed(strict=True) for rule in rules) - - @pytest.mark.parametrize(("rule", "table", "change"), CASES, ids=[case[0] for case in CASES]) def test_reference_rule_fails(tmp_path: Path, rule: str, table: str, change) -> None: - data_root = _copy_v1(tmp_path) - _edit(data_root, table, change) - report = _reports(data_root)[rule] + sample = SampleCopy(tmp_path) + sample.edit(table, change) + report = sample.reports()[rule] assert report.is_failed(), report def test_ref306_point_labels(tmp_path: Path) -> None: """A LiDAR `label` value that no category index has fails.""" - data_root = _copy_v1(tmp_path) - _add_label_column(data_root) - report = _reports(data_root)["REF306"] + sample = SampleCopy(tmp_path) + sample.write_labels([[7] * 8] * 3) + report = sample.reports()["REF306"] assert report.is_failed() and "[7]" in report.reasons[0] diff --git a/tests/schema/v1/test_caption_table.py b/tests/schema/v1/test_caption_table.py deleted file mode 100644 index 13d300f8..00000000 --- a/tests/schema/v1/test_caption_table.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from t4_devkit.common.serialize import serialize_dataclasses -from t4_devkit.schema.v1 import Caption - -CAPTION_JSON = Path(__file__).parents[2] / "sample" / "t4dataset_v1" / "annotation" / "caption.json" - - -def test_caption_json() -> None: - """The sample's captions load, and every field round-trips.""" - records = Caption.from_json(CAPTION_JSON) - assert serialize_dataclasses(records) == json.loads(CAPTION_JSON.read_text()) - - -@pytest.mark.parametrize( - ("key", "value"), - [("caption_type", "summary"), ("text", ""), ("end_timestamp", 0)], -) -def test_invalid_caption(key: str, value: object) -> None: - data = json.loads(CAPTION_JSON.read_text())[0] - data[key] = value - with pytest.raises(ValueError): - Caption.from_dict(data) diff --git a/tests/schema/v1/test_convert_v0.py b/tests/schema/v1/test_convert_v0.py index 351919e8..3ba6757d 100644 --- a/tests/schema/v1/test_convert_v0.py +++ b/tests/schema/v1/test_convert_v0.py @@ -2,7 +2,6 @@ import base64 import json -import shutil from datetime import datetime, timezone from pathlib import Path @@ -11,8 +10,9 @@ from pycocotools import mask as cocomask from t4_devkit.schema.v1 import SchemaName, V0Converter, load_dataset +from tests.helpers import SAMPLE_V0, SampleCopy -SAMPLE_V0 = Path(__file__).parents[2] / "sample" / "t4dataset" +set_field = SampleCopy.set_field def test_load_v0_dataset() -> None: @@ -81,66 +81,16 @@ def test_load_v0_category_index(tmp_path: Path) -> None: categories = load_dataset(SAMPLE_V0).tables[SchemaName.CATEGORY] assert [category.index for category in categories] == list(range(len(categories))) - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V0, data_root) - category_path = data_root / "annotation" / "category.json" - records = json.loads(category_path.read_text()) - records[0]["index"] = 5 - category_path.write_text(json.dumps(records)) + sample = SampleCopy(tmp_path, SAMPLE_V0) + sample.edit("category", lambda records: set_field(records, 0, index=5)) with pytest.raises(ValueError, match="index"): - load_dataset(data_root) - - -def test_load_v0_autolabel(tmp_path: Path) -> None: - """A 0.x autolabel becomes annotation_type auto, with the model as provenance.""" - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V0, data_root) - path = data_root / "annotation" / "sample_annotation.json" - records = json.loads(path.read_text()) - records[0]["automatic_annotation"] = True - records[0]["autolabel_metadata"] = [{"name": "centerpoint-v2", "score": 0.8}] - path.write_text(json.dumps(records)) - - annotation = load_dataset(data_root).tables[SchemaName.SAMPLE_ANNOTATION][0] - assert annotation.annotation_type == "auto" - assert annotation.provenance == "centerpoint-v2" - assert annotation.confidence == 0.8 - - -def test_load_v0_visibility(tmp_path: Path) -> None: - """0.x level aliases are mapped; `unavailable` is dropped and becomes a null visibility_token.""" - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V0, data_root) - path = data_root / "annotation" / "visibility.json" - records = json.loads(path.read_text()) - records[0]["level"] = "v80-100" - records[1]["level"] = "unavailable" - path.write_text(json.dumps(records)) - path = data_root / "annotation" / "sample_annotation.json" - annotations = json.loads(path.read_text()) - annotations[0]["visibility_token"] = records[1]["token"] - path.write_text(json.dumps(annotations)) - - tables = load_dataset(data_root).tables - assert [r.level for r in tables[SchemaName.VISIBILITY]] == ["full", "partial", "none"] - assert tables[SchemaName.SAMPLE_ANNOTATION][0].visibility_token is None - - -def _copy_v0(tmp_path: Path) -> Path: - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V0, data_root) - return data_root - - -def _edit_v0(data_root: Path, name: str, change) -> None: - path = data_root / "annotation" / f"{name}.json" - path.write_text(json.dumps(change(json.loads(path.read_text())))) + load_dataset(sample.root) def test_load_v0_frames(tmp_path: Path) -> None: """Every 0.x frame is a 1.0 sample, grouped by frame number; 0.x key samples keep their token.""" - data_root = _copy_v0(tmp_path) - sample_data = json.loads((data_root / "annotation" / "sample_data.json").read_text()) + sample = SampleCopy(tmp_path, SAMPLE_V0) + sample_data = sample.read("sample_data") lidar2 = next(r for r in sample_data if r["filename"] == "data/LIDAR_TOP/002.pcd.bin") lidar3 = next(r for r in sample_data if r["filename"] == "data/LIDAR_TOP/003.pcd.bin") @@ -151,25 +101,23 @@ def sweep(records: list[dict]) -> list[dict]: record.update(is_key_frame=False, sample_token=lidar3["sample_token"]) return records - _edit_v0(data_root, "sample_data", sweep) - dataset = load_dataset(data_root) + sample.edit("sample_data", sweep) + dataset = load_dataset(sample.root) frame = dataset.get(SchemaName.SAMPLE_DATA, lidar2["token"]) assert frame.sample_token == lidar2["sample_token"] # its own frame's sample, not the next assert not frame.is_key_frame assert dataset.get(SchemaName.SAMPLE, frame.sample_token).has_key_frame # cameras are key samples = dataset.tables[SchemaName.SAMPLE] - assert len(samples) == 3 and all(len(sample.data) == 3 for sample in samples) + assert len(samples) == 3 and all(len(s.data) == 3 for s in samples) def test_load_v0_per_camera_samples(tmp_path: Path) -> None: """0.x samples of one frame (one per camera image) merge into one 1.0 sample.""" - data_root = _copy_v0(tmp_path) - samples = json.loads((data_root / "annotation" / "sample.json").read_text()) - first = samples[0]["token"] - extra = dict(samples[0], token="f" * 32) # sorts after `first` - _edit_v0(data_root, "sample", lambda records: [*records, extra]) - _edit_v0( - data_root, + sample = SampleCopy(tmp_path, SAMPLE_V0) + first = sample.read("sample")[0] + extra = dict(first, token="f" * 32) # sorts after `first` + sample.edit("sample", lambda records: [*records, extra]) + sample.edit( "sample_data", lambda records: [ dict(r, sample_token=extra["token"]) if r["filename"] == "data/CAM_BACK/001.jpg" else r @@ -177,9 +125,10 @@ def test_load_v0_per_camera_samples(tmp_path: Path) -> None: ], ) - dataset = load_dataset(data_root) + dataset = load_dataset(sample.root) assert len(dataset.tables[SchemaName.SAMPLE]) == 3 - assert set(dataset.get(SchemaName.SAMPLE, first).data) == {"LIDAR_TOP", "CAM_FRONT", "CAM_BACK"} + channels = set(dataset.get(SchemaName.SAMPLE, first["token"]).data) + assert channels == {"LIDAR_TOP", "CAM_FRONT", "CAM_BACK"} def test_load_v0_mask_only_frame_is_key(tmp_path: Path) -> None: @@ -205,17 +154,16 @@ def test_load_v0_mask_only_frame_is_key(tmp_path: Path) -> None: def test_load_v0_labeled_frame_is_key(tmp_path: Path) -> None: """A 0.x 2D label on a non-key frame makes it a key frame.""" - data_root = _copy_v0(tmp_path) - labeled = json.loads((data_root / "annotation" / "object_ann.json").read_text())[0] - _edit_v0( - data_root, + sample = SampleCopy(tmp_path, SAMPLE_V0) + labeled = sample.read("object_ann")[0] + sample.edit( "sample_data", lambda records: [ dict(r, is_key_frame=False) if r["token"] == labeled["sample_data_token"] else r for r in records ], ) - frame = load_dataset(data_root).get(SchemaName.SAMPLE_DATA, labeled["sample_data_token"]) + frame = load_dataset(sample.root).get(SchemaName.SAMPLE_DATA, labeled["sample_data_token"]) assert frame.is_key_frame @@ -230,15 +178,11 @@ def test_load_v0_vehicle_state() -> None: def test_load_v0_capture_time(tmp_path: Path) -> None: """An empty 0.x `data_captured` becomes the first sample time.""" - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V0, data_root) - log_path = data_root / "annotation" / "log.json" - logs = json.loads(log_path.read_text()) - logs[0]["data_captured"] = "" - log_path.write_text(json.dumps(logs)) - - tables = load_dataset(data_root).tables - first = min(sample.timestamp for sample in tables[SchemaName.SAMPLE]) + sample = SampleCopy(tmp_path, SAMPLE_V0) + sample.edit("log", lambda records: set_field(records, 0, data_captured="")) + + tables = load_dataset(sample.root).tables + first = min(s.timestamp for s in tables[SchemaName.SAMPLE]) expected = datetime.fromtimestamp(first / 1e6, tz=timezone.utc).isoformat() assert tables[SchemaName.LOG][0].data_captured == expected @@ -252,15 +196,12 @@ def test_load_v0_capture_time(tmp_path: Path) -> None: ) def test_load_v0_traffic_light(tmp_path: Path, table: str, way_key: str) -> None: """A 0.x lamp-to-way row sets the instance's lanelet2 way.""" - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V0, data_root) - instance_token = json.loads((data_root / "annotation" / "instance.json").read_text())[0][ - "token" - ] - rows = [{"token": "tl0", "instance_token": instance_token, way_key: "1234"}] - (data_root / "annotation" / f"{table}.json").write_text(json.dumps(rows)) + sample = SampleCopy(tmp_path, SAMPLE_V0) + instance_token = sample.read("instance")[0]["token"] + row = {"token": "tl0", "instance_token": instance_token, way_key: "1234"} + (sample.root / "annotation" / f"{table}.json").write_text(json.dumps([row])) - tables = load_dataset(data_root).tables + tables = load_dataset(sample.root).tables assert tables[SchemaName.TRAFFIC_LIGHT] == [] map_element = tables[SchemaName.INSTANCE][0].map_element assert (map_element.id, map_element.type) == (1234, "way") @@ -268,20 +209,13 @@ def test_load_v0_traffic_light(tmp_path: Path, table: str, way_key: str) -> None def test_load_v0_source_lidar(tmp_path: Path) -> None: """A source LiDAR takes the concatenated channel's fields and counts frames with its points.""" - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V0, data_root) - annotation = data_root / "annotation" - - def edit(name: str, change) -> None: - path = annotation / f"{name}.json" - path.write_text(json.dumps(change(json.loads(path.read_text())))) - + sample = SampleCopy(tmp_path, SAMPLE_V0) source = {"token": "src", "channel": "LIDAR_SOURCE", "modality": "lidar"} calibration = {"token": "src_calib", "sensor_token": "src"} calibration |= {"translation": [0.0, 0.0, 0.0], "rotation": [1.0, 0.0, 0.0, 0.0]} calibration |= {"camera_intrinsic": [], "camera_distortion": []} - edit("sensor", lambda records: [*records, source]) - edit("calibrated_sensor", lambda records: [*records, calibration]) + sample.edit("sensor", lambda records: [*records, source]) + sample.edit("calibrated_sensor", lambda records: [*records, calibration]) def add_info(records: list[dict]) -> list[dict]: frames = [r for r in records if r["filename"].startswith("data/LIDAR_TOP")] @@ -290,13 +224,13 @@ def add_info(records: list[dict]) -> list[dict]: # points from the source in the first two frames only sources = [{"sensor_token": "src", "idx_begin": 0, "length": 10 if i < 2 else 0}] info = {"num_pts_feats": 7, "sources": sources} - (data_root / record["info_filename"]).parent.mkdir(exist_ok=True) - (data_root / record["info_filename"]).write_text(json.dumps(info)) + (sample.root / record["info_filename"]).parent.mkdir(exist_ok=True) + (sample.root / record["info_filename"]).write_text(json.dumps(info)) return records - edit("sample_data", add_info) + sample.edit("sample_data", add_info) - sensors = {s.channel: s for s in load_dataset(data_root).tables[SchemaName.SENSOR]} + sensors = {s.channel: s for s in load_dataset(sample.root).tables[SchemaName.SENSOR]} concat, source_lidar = sensors["LIDAR_TOP"].lidar, sensors["LIDAR_SOURCE"].lidar assert concat.source_channels == ["LIDAR_SOURCE"] assert source_lidar.fields == concat.fields and len(concat.fields) == 7 @@ -305,15 +239,12 @@ def add_info(records: list[dict]) -> list[dict]: def test_load_v0_empty_mask(tmp_path: Path) -> None: """A box-only 0.x label with an empty placeholder mask has no mask in 1.0.""" - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V0, data_root) - path = data_root / "annotation" / "object_ann.json" - records = json.loads(path.read_text()) + sample = SampleCopy(tmp_path, SAMPLE_V0) empty = cocomask.encode(np.zeros((600, 800), dtype=np.uint8, order="F")) # as written by some 0.x tools: (width, height) - records[0]["mask"] = {"size": [800, 600], "counts": base64.b64encode(empty["counts"]).decode()} - path.write_text(json.dumps(records)) + mask = {"size": [800, 600], "counts": base64.b64encode(empty["counts"]).decode()} + sample.edit("object_ann", lambda records: set_field(records, 0, mask=mask)) - annotation = load_dataset(data_root).tables[SchemaName.OBJECT_ANN][0] + annotation = load_dataset(sample.root).tables[SchemaName.OBJECT_ANN][0] assert annotation.mask is None - assert list(annotation.bbox) == records[0]["bbox"] + assert list(annotation.bbox) == sample.read("object_ann")[0]["bbox"] diff --git a/tests/schema/v1/test_core_tables.py b/tests/schema/v1/test_core_tables.py deleted file mode 100644 index 9972f58d..00000000 --- a/tests/schema/v1/test_core_tables.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from t4_devkit.common.serialize import serialize_dataclasses -from t4_devkit.schema.v1 import EgoPose, Log, Map, Sample, SampleData, Scene - -ANNOTATION_DIR = Path(__file__).parents[2] / "sample" / "t4dataset_v1" / "annotation" - - -@pytest.mark.parametrize( - ("schema", "name"), - [ - (Sample, "sample"), - (SampleData, "sample_data"), - (EgoPose, "ego_pose"), - (Scene, "scene"), - (Log, "log"), - (Map, "map"), - ], -) -def test_table_json(schema, name: str) -> None: - """The sample's table loads, and every field round-trips.""" - filepath = ANNOTATION_DIR / f"{name}.json" - records = schema.from_json(filepath) - assert serialize_dataclasses(records) == json.loads(filepath.read_text()) - - -def test_empty_string_link_is_rejected() -> None: - """Missing links are null in 1.0, never an empty string.""" - data = json.loads((ANNOTATION_DIR / "sample.json").read_text())[0] - data["prev"] = "" - with pytest.raises(ValueError): - Sample.from_dict(data) - - -def test_sample_data_needs_sample() -> None: - """Every frame belongs to a sample, key frame or not.""" - records = json.loads((ANNOTATION_DIR / "sample_data.json").read_text()) - non_key = next(record for record in records if not record["is_key_frame"]) - SampleData.from_dict(non_key) - with pytest.raises(TypeError): - SampleData.from_dict(dict(non_key, sample_token=None)) - - -def test_invalid_covariance_length() -> None: - data = json.loads((ANNOTATION_DIR / "ego_pose.json").read_text())[0] - data["covariance"] = [0.0] * 35 - with pytest.raises(ValueError): - EgoPose.from_dict(data) diff --git a/tests/schema/v1/test_label_tables.py b/tests/schema/v1/test_label_tables.py deleted file mode 100644 index b09e6bdc..00000000 --- a/tests/schema/v1/test_label_tables.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from t4_devkit.common.serialize import serialize_dataclasses -from t4_devkit.schema.v1 import ( - Attribute, - Category, - Instance, - Keypoint, - ObjectAnn, - SampleAnnotation, - Visibility, -) - -ANNOTATION_DIR = Path(__file__).parents[2] / "sample" / "t4dataset_v1" / "annotation" - - -@pytest.mark.parametrize( - ("schema", "name"), - [ - (Category, "category"), - (Attribute, "attribute"), - (Visibility, "visibility"), - (Instance, "instance"), - (SampleAnnotation, "sample_annotation"), - (ObjectAnn, "object_ann"), - (Keypoint, "keypoint"), - ], -) -def test_table_json(schema, name: str) -> None: - """The sample's table loads, and every field round-trips.""" - filepath = ANNOTATION_DIR / f"{name}.json" - records = schema.from_json(filepath) - assert serialize_dataclasses(records) == json.loads(filepath.read_text()) - - -@pytest.mark.parametrize( - ("key", "value"), - [("annotation_type", "human"), ("confidence", 1.5)], -) -def test_invalid_label_provenance(key: str, value: object) -> None: - data = json.loads((ANNOTATION_DIR / "sample_annotation.json").read_text())[0] - data[key] = value - with pytest.raises(ValueError): - SampleAnnotation.from_dict(data) - - -def test_invalid_keypoints_shape() -> None: - data = json.loads((ANNOTATION_DIR / "keypoint.json").read_text())[0] - data["keypoints"] = [1.0, 2.0] - with pytest.raises(ValueError): - Keypoint.from_dict(data) diff --git a/tests/schema/v1/test_loader.py b/tests/schema/v1/test_loader.py index 6de940e3..b378da76 100644 --- a/tests/schema/v1/test_loader.py +++ b/tests/schema/v1/test_loader.py @@ -1,35 +1,34 @@ from __future__ import annotations import json -import shutil from pathlib import Path import pytest +from t4_devkit.common.serialize import serialize_dataclass, serialize_dataclasses from t4_devkit.schema.v1 import SCHEMAS, SchemaName, Sensor, load_dataset - -SAMPLE_ROOT = Path(__file__).parents[2] / "sample" -SAMPLE_V1 = SAMPLE_ROOT / "t4dataset_v1" +from tests.helpers import SAMPLE_V1, SampleCopy def _copy_without_table(tmp_path: Path, name: str) -> Path: - data_root = tmp_path / "dataset" - shutil.copytree(SAMPLE_V1, data_root) - manifest = json.loads((data_root / "manifest.json").read_text()) - del manifest["tables"][name] - (data_root / "manifest.json").write_text(json.dumps(manifest)) - return data_root + sample = SampleCopy(tmp_path) + sample.edit( + "manifest", lambda m: dict(m, tables={k: v for k, v in m["tables"].items() if k != name}) + ) + return sample.root def test_load_dataset() -> None: - """Every table is loaded with its 1.0 class, and lookups by token work.""" + """Every table loads with its 1.0 class and every field round-trips, the manifest too; + lookups by token work.""" dataset = load_dataset(SAMPLE_V1) - assert dataset.manifest.schema_version == "1.0.0" + manifest = json.loads((SAMPLE_V1 / "manifest.json").read_text()) + assert serialize_dataclass(dataset.manifest) == manifest for name in SchemaName: records = json.loads((SAMPLE_V1 / "annotation" / name.filename).read_text()) - assert len(dataset.tables[name]) == len(records) assert all(isinstance(r, SCHEMAS.get(name)) for r in dataset.tables[name]) + assert serialize_dataclasses(dataset.tables[name]) == records, name assert isinstance(dataset.get("sensor", "sensor_cam"), Sensor) diff --git a/tests/schema/v1/test_manifest.py b/tests/schema/v1/test_manifest.py index 19a68588..d85cfe8f 100644 --- a/tests/schema/v1/test_manifest.py +++ b/tests/schema/v1/test_manifest.py @@ -6,7 +6,6 @@ import pytest from attrs import evolve -from t4_devkit.common.serialize import serialize_dataclass from t4_devkit.schema.v1 import ( Capabilities, Manifest, @@ -19,16 +18,6 @@ SAMPLE_ROOT = Path(__file__).parents[2] / "sample" -def test_load_manifest_v1() -> None: - """A 1.0 dataset has a manifest, and every field round-trips.""" - data_root = SAMPLE_ROOT / "t4dataset_v1" - manifest = load_manifest(data_root) - - assert isinstance(manifest, Manifest) - assert manifest.schema_version == "1.0.0" - assert serialize_dataclass(manifest) == json.loads((data_root / "manifest.json").read_text()) - - def test_load_manifest_v0() -> None: """A 0.x dataset has no manifest.""" assert load_manifest(SAMPLE_ROOT / "t4dataset") is None @@ -43,7 +32,7 @@ def test_unsupported_schema_version() -> None: @pytest.mark.parametrize( ("key", "value"), - [("is_synthetic", "yes"), ("version", -1), ("migration_notes", "note")], + [("is_synthetic", "yes"), ("version", -1)], ) def test_invalid_field_type(key: str, value: object) -> None: data = json.loads((SAMPLE_ROOT / "t4dataset_v1" / "manifest.json").read_text()) diff --git a/tests/schema/v1/test_migrate_v0.py b/tests/schema/v1/test_migrate_v0.py index c5350513..08267b7a 100644 --- a/tests/schema/v1/test_migrate_v0.py +++ b/tests/schema/v1/test_migrate_v0.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import shutil from pathlib import Path import numpy as np @@ -11,8 +10,7 @@ from t4_devkit.common.io import load_image, load_pointcloud from t4_devkit.sanity import sanity_check from t4_devkit.schema.v1 import SchemaName, V0Migrator - -SAMPLE_V0 = Path(__file__).parents[2] / "sample" / "t4dataset" +from tests.helpers import SAMPLE_V0, SampleCopy def _migrate_lidar(data_root: Path, output_root: Path) -> V0Migrator: @@ -62,15 +60,10 @@ def test_migrate_lidar(tmp_path: Path) -> None: def _semseg_copy(tmp_path: Path) -> Path: """The 0.x sample with integer intensity, ring -1 (not recorded) and labels on frame 0.""" - data_root = tmp_path / "v0" - shutil.copytree(SAMPLE_V0, data_root) - annotation = data_root / "annotation" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") + data_root = sample.root frames = sorted( - ( - r - for r in json.loads((annotation / "sample_data.json").read_text()) - if "LIDAR" in r["filename"] - ), + (r for r in sample.read("sample_data") if "LIDAR" in r["filename"]), key=lambda r: r["timestamp"], ) for record in frames: @@ -90,7 +83,7 @@ def _semseg_copy(tmp_path: Path) -> Path: "sample_data_token": frames[0]["token"], } ] - (annotation / "lidarseg.json").write_text(json.dumps(lidarseg)) + (data_root / "annotation/lidarseg.json").write_text(json.dumps(lidarseg)) return data_root @@ -132,8 +125,7 @@ def test_migrate_lidar_labels(tmp_path: Path) -> None: def test_migrate_camera(tmp_path: Path) -> None: """A camera channel becomes one video; each frame is its own image, in timestamp order.""" - data_root = tmp_path / "v0" - shutil.copytree(SAMPLE_V0, data_root) + data_root = SampleCopy(tmp_path, SAMPLE_V0, name="v0").root migrator = V0Migrator(data_root, tmp_path / "sample") sensor = migrator.cameras()[0] frames = migrator._channel_frames(sensor) @@ -172,18 +164,23 @@ def test_copy_images(tmp_path: Path) -> None: def test_relink_half_linked_frame(tmp_path: Path) -> None: """A frame its neighbors skip (as 0.x placeholders can be) is linked back in time order.""" - data_root = tmp_path / "v0" - shutil.copytree(SAMPLE_V0, data_root) - path = data_root / "annotation/sample_data.json" - records = json.loads(path.read_text()) + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") first, middle, last = sorted( - (r for r in records if r["filename"].startswith("data/CAM_FRONT/")), + (r for r in sample.read("sample_data") if r["filename"].startswith("data/CAM_FRONT/")), key=lambda r: r["timestamp"], ) - first["next"], last["prev"] = last["token"], first["token"] # both skip `middle` - path.write_text(json.dumps(records)) - migrator = V0Migrator(data_root, tmp_path / "sample") + def skip_middle(records: list[dict]) -> list[dict]: + for record in records: + if record["token"] == first["token"]: + record["next"] = last["token"] + if record["token"] == last["token"]: + record["prev"] = first["token"] + return records + + sample.edit("sample_data", skip_middle) + + migrator = V0Migrator(sample.root, tmp_path / "sample") sensor = migrator.cameras()[0] migrator.copy_images(sensor) @@ -208,19 +205,17 @@ def test_migrate_lidarseg_passes_sanity(tmp_path: Path) -> None: def test_migrate_lidar_sources(tmp_path: Path) -> None: """`.info` sources become `source_index` (the source's `sensor.index`) and µs timestamps.""" - data_root = tmp_path / "v0" - shutil.copytree(SAMPLE_V0, data_root) - annotation = data_root / "annotation" - sensors = json.loads((annotation / "sensor.json").read_text()) - top = next(sensor for sensor in sensors if sensor["channel"] == "LIDAR_TOP") - sensors.append(dict(top, token="rear", channel="LIDAR_REAR")) - (annotation / "sensor.json").write_text(json.dumps(sensors)) - calibrations = json.loads((annotation / "calibrated_sensor.json").read_text()) - top_calibration = next(c for c in calibrations if c["sensor_token"] == top["token"]) - calibrations.append(dict(top_calibration, token="rear_calibration", sensor_token="rear")) - (annotation / "calibrated_sensor.json").write_text(json.dumps(calibrations)) - - records = json.loads((annotation / "sample_data.json").read_text()) + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") + top = next(s for s in sample.read("sensor") if s["channel"] == "LIDAR_TOP") + top_calibration = next( + c for c in sample.read("calibrated_sensor") if c["sensor_token"] == top["token"] + ) + rear_calibration = dict(top_calibration, token="rear_calibration", sensor_token="rear") + sample.edit("sensor", lambda r: [*r, dict(top, token="rear", channel="LIDAR_REAR")]) + sample.edit("calibrated_sensor", lambda r: [*r, rear_calibration]) + + data_root = sample.root + records = sample.read("sample_data") (data_root / "data/LIDAR_TOP_INFO").mkdir() for i, record in enumerate(r for r in records if "LIDAR" in r["filename"]): record["info_filename"] = f"data/LIDAR_TOP_INFO/{i}.json" @@ -241,7 +236,7 @@ def test_migrate_lidar_sources(tmp_path: Path) -> None: ] } (data_root / record["info_filename"]).write_text(json.dumps(info)) - (annotation / "sample_data.json").write_text(json.dumps(records)) + sample.edit("sample_data", lambda _: records) with pytest.warns(UserWarning): migrator = _migrate_lidar(data_root, tmp_path / "sample") @@ -257,16 +252,18 @@ def test_migrate_lidar_sources(tmp_path: Path) -> None: def test_migrate_recalibrated_channel(tmp_path: Path) -> None: """Two calibrations of one channel would share its file: migration stops instead.""" - data_root = tmp_path / "v0" - shutil.copytree(SAMPLE_V0, data_root) - annotation = data_root / "annotation" - calibrations = json.loads((annotation / "calibrated_sensor.json").read_text()) - calibrations.append(dict(calibrations[0], token="recalibrated")) # LIDAR_TOP again - (annotation / "calibrated_sensor.json").write_text(json.dumps(calibrations)) - records = json.loads((annotation / "sample_data.json").read_text()) - last = max((r for r in records if "LIDAR" in r["filename"]), key=lambda r: r["timestamp"]) - last["calibrated_sensor_token"] = "recalibrated" - (annotation / "sample_data.json").write_text(json.dumps(records)) + sample = SampleCopy(tmp_path, SAMPLE_V0, name="v0") + # LIDAR_TOP again, used by its last frame + sample.edit("calibrated_sensor", lambda r: [*r, dict(r[0], token="recalibrated")]) + lidar = [r for r in sample.read("sample_data") if "LIDAR" in r["filename"]] + last = max(lidar, key=lambda r: r["timestamp"])["token"] + sample.edit( + "sample_data", + lambda records: [ + dict(r, calibrated_sensor_token="recalibrated") if r["token"] == last else r + for r in records + ], + ) with pytest.warns(UserWarning), pytest.raises(FileExistsError, match="LIDAR_TOP"): - _migrate_lidar(data_root, tmp_path / "sample") + _migrate_lidar(sample.root, tmp_path / "sample") diff --git a/tests/schema/v1/test_planning_tables.py b/tests/schema/v1/test_planning_tables.py deleted file mode 100644 index 4c3b65be..00000000 --- a/tests/schema/v1/test_planning_tables.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from t4_devkit.common.serialize import serialize_dataclasses -from t4_devkit.schema.v1 import Route, Scenario, TrafficLight, Vehicle, VehicleState - -ANNOTATION_DIR = Path(__file__).parents[2] / "sample" / "t4dataset_v1" / "annotation" - - -@pytest.mark.parametrize( - ("schema", "name"), - [ - (Vehicle, "vehicle"), - (VehicleState, "vehicle_state"), - (Route, "route"), - (Scenario, "scenario"), - (TrafficLight, "traffic_light"), - ], -) -def test_table_json(schema, name: str) -> None: - """The sample's table loads, and every field round-trips.""" - filepath = ANNOTATION_DIR / f"{name}.json" - records = schema.from_json(filepath) - assert serialize_dataclasses(records) == json.loads(filepath.read_text()) - - -def test_scenario_end_before_start() -> None: - data = json.loads((ANNOTATION_DIR / "scenario.json").read_text())[0] - data["end_timestamp"] = data["start_timestamp"] - 1 - with pytest.raises(ValueError): - Scenario.from_dict(data) - - -def test_invalid_traffic_light_color() -> None: - data = json.loads((ANNOTATION_DIR / "traffic_light.json").read_text())[0] - data["elements"][0]["color"] = "yellow" - with pytest.raises(ValueError): - TrafficLight.from_dict(data) diff --git a/tests/schema/v1/test_registry.py b/tests/schema/v1/test_registry.py index 17304381..532886cb 100644 --- a/tests/schema/v1/test_registry.py +++ b/tests/schema/v1/test_registry.py @@ -7,11 +7,6 @@ from t4_devkit.schema.v1 import SCHEMAS, Attribute, SchemaName -def test_every_table_is_registered() -> None: - assert set(SCHEMAS.keys()) == {name.value for name in SchemaName} - assert SchemaName.SAMPLE.filename == "sample.json" - - def test_required_tables() -> None: required = {name.value for name in SchemaName if not name.is_optional()} assert required == {"sample", "sample_data", "ego_pose", "sensor", "scene", "log", "vehicle"} diff --git a/tests/schema/v1/test_sensor.py b/tests/schema/v1/test_sensor.py deleted file mode 100644 index bc839fea..00000000 --- a/tests/schema/v1/test_sensor.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -import copy -import json -from pathlib import Path - -import pytest - -from t4_devkit.common.serialize import serialize_dataclasses -from t4_devkit.schema.v1 import Camera, Lidar, Sensor - -SENSOR_JSON = Path(__file__).parents[2] / "sample" / "t4dataset_v1" / "annotation" / "sensor.json" - - -@pytest.fixture -def sensor_records() -> list[dict]: - return json.loads(SENSOR_JSON.read_text()) - - -def test_sensor_json(sensor_records) -> None: - """The sample's sensors load, and every field round-trips.""" - sensors = Sensor.from_json(SENSOR_JSON) - - assert isinstance(sensors[0].camera, Camera) - assert isinstance(sensors[1].lidar, Lidar) - assert serialize_dataclasses(sensors) == sensor_records - - -def test_block_must_match_modality(sensor_records) -> None: - camera = copy.deepcopy(sensor_records[0]) - camera["lidar"] = sensor_records[1]["lidar"] - with pytest.raises(ValueError, match="only the 'camera' block"): - Sensor.from_dict(camera) - - -@pytest.mark.parametrize("index", [-1, 256, None]) -def test_invalid_index(sensor_records, index) -> None: - camera = copy.deepcopy(sensor_records[0]) - camera["index"] = index - with pytest.raises((TypeError, ValueError)): - Sensor(**camera) - - -@pytest.mark.parametrize( - ("block", "key", "value"), - [ - ("video", "bit_depth", 12), - ("video", "format", "h264"), - (None, "intrinsic", []), - (None, "distortion_model", "rational_polynomial"), - ], -) -def test_invalid_camera(sensor_records, block, key, value) -> None: - camera = copy.deepcopy(sensor_records[0]) - target = camera["camera"][block] if block else camera["camera"] - target[key] = value - with pytest.raises(ValueError): - Sensor.from_dict(camera) diff --git a/tests/schema/v1/test_tables.py b/tests/schema/v1/test_tables.py new file mode 100644 index 00000000..2361edfa --- /dev/null +++ b/tests/schema/v1/test_tables.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import copy +import json + +import pytest + +from t4_devkit.schema.v1 import SCHEMAS, SchemaName +from tests.helpers import SAMPLE_V1 + + +def _set(path: tuple, value: object): + """Return an edit that sets `value` at `path` (keys / indices) in the record.""" + + def edit(record: dict, records: list[dict]) -> None: + target = record + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + + return edit + + +def _other_lidar_block(record: dict, records: list[dict]) -> None: + record["lidar"] = records[1]["lidar"] + + +# (table, position, edit, error): one invalid value per case; the whole table round-trips in +# test_loader.py::test_load_dataset. +CASES = { + "empty-string-link": ("sample", 0, _set(("prev",), ""), ValueError), + "frame-without-sample": ("sample_data", -1, _set(("sample_token",), None), TypeError), + "covariance-length": ("ego_pose", 0, _set(("covariance",), [0.0] * 35), ValueError), + "annotation-type": ("sample_annotation", 0, _set(("annotation_type",), "human"), ValueError), + "confidence-range": ("sample_annotation", 0, _set(("confidence",), 1.5), ValueError), + "keypoints-shape": ("keypoint", 0, _set(("keypoints",), [1.0, 2.0]), ValueError), + "scenario-end-before-start": ("scenario", 0, _set(("end_timestamp",), 0), ValueError), + "traffic-light-color": ( + "traffic_light", + 0, + _set(("elements", 0, "color"), "yellow"), + ValueError, + ), + "caption-type": ("caption", 0, _set(("caption_type",), "summary"), ValueError), + "caption-empty-text": ("caption", 0, _set(("text",), ""), ValueError), + "caption-end-before-start": ("caption", 0, _set(("end_timestamp",), 0), ValueError), + "sensor-index-range": ("sensor", 0, _set(("index",), 256), ValueError), + "sensor-index-missing": ("sensor", 0, _set(("index",), None), TypeError), + "sensor-block-modality": ("sensor", 0, _other_lidar_block, ValueError), + "video-bit-depth": ("sensor", 0, _set(("camera", "video", "bit_depth"), 12), ValueError), + "video-format": ("sensor", 0, _set(("camera", "video", "format"), "h264"), ValueError), + "camera-intrinsic": ("sensor", 0, _set(("camera", "intrinsic"), []), ValueError), + "distortion-model": ( + "sensor", + 0, + _set(("camera", "distortion_model"), "rational_polynomial"), + ValueError, + ), +} + + +@pytest.mark.parametrize(("table", "position", "edit", "error"), CASES.values(), ids=CASES.keys()) +def test_invalid_record(table: str, position: int, edit, error: type[Exception]) -> None: + """A record with one invalid value is rejected when built; unedited, it builds.""" + schema = SCHEMAS.get(SchemaName(table)) + records = json.loads((SAMPLE_V1 / "annotation" / f"{table}.json").read_text()) + record = records[position] + schema.from_dict(copy.deepcopy(record)) + edit(record, records) + with pytest.raises(error): + schema.from_dict(record) diff --git a/tests/test_tier4.py b/tests/test_tier4.py index a8d5460c..0b6e9438 100644 --- a/tests/test_tier4.py +++ b/tests/test_tier4.py @@ -1,8 +1,6 @@ from __future__ import annotations -import json import os -import shutil from pathlib import Path from unittest.mock import patch @@ -13,6 +11,7 @@ from t4_devkit.schema import VisibilityLevel from t4_devkit.tier4 import DBMetadata, T4Devkit, load_metadata from t4_devkit.typing import Quaternion +from tests.helpers import SAMPLE_V0, SampleCopy @pytest.fixture(scope="session") @@ -70,32 +69,41 @@ def test_load_metadata_with_specific_revision(self, tmp_path): def test_unknown_visibility(tmp_path): - """A box whose visibility is unknown (0.x `unavailable`) gets `UNAVAILABLE`.""" - data_root = tmp_path / "t4dataset" - shutil.copytree(Path(__file__).parent / "sample/t4dataset", data_root) - visibility_path = data_root / "annotation" / "visibility.json" - visibilities = json.loads(visibility_path.read_text()) - visibilities[0]["level"] = "unavailable" # the level of the first annotation - visibility_path.write_text(json.dumps(visibilities)) - - t4 = T4Devkit(data_root.as_posix(), verbose=False) + """0.x level aliases are mapped; `unavailable` is dropped, and a box with it gets + `UNAVAILABLE`.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="t4dataset") + sample.edit( + "visibility", + lambda records: SampleCopy.set_field( + SampleCopy.set_field(records, 0, level="unavailable"), # the first annotation's + 1, + level="v60-80", + ), + ) + + t4 = T4Devkit(sample.root.as_posix(), verbose=False) + assert [visibility.level for visibility in t4.visibility] == ["most", "partial", "none"] annotation = t4.sample_annotation[0] assert annotation.visibility_token is None assert t4.get_box3d(annotation.token).visibility == VisibilityLevel.UNAVAILABLE def test_autolabel_confidence(tmp_path): - """A box takes the confidence of its label; manual labels keep 1.0.""" - data_root = tmp_path / "t4dataset" - shutil.copytree(Path(__file__).parent / "sample/t4dataset", data_root) - annotation_path = data_root / "annotation" / "sample_annotation.json" - annotations = json.loads(annotation_path.read_text()) - annotations[0]["automatic_annotation"] = True - annotations[0]["autolabel_metadata"] = [{"name": "centerpoint-v2", "score": 0.8}] - annotation_path.write_text(json.dumps(annotations)) - - t4 = T4Devkit(data_root.as_posix(), verbose=False) - assert t4.get_box3d(t4.sample_annotation[0].token).confidence == 0.8 + """A 0.x autolabel becomes `auto` with the model as provenance; a box takes the confidence + of its label, and manual labels keep 1.0.""" + sample = SampleCopy(tmp_path, SAMPLE_V0, name="t4dataset") + metadata = [{"name": "centerpoint-v2", "score": 0.8}] + sample.edit( + "sample_annotation", + lambda r: SampleCopy.set_field( + r, 0, automatic_annotation=True, autolabel_metadata=metadata + ), + ) + + t4 = T4Devkit(sample.root.as_posix(), verbose=False) + annotation = t4.sample_annotation[0] + assert (annotation.annotation_type, annotation.provenance) == ("auto", "centerpoint-v2") + assert t4.get_box3d(annotation.token).confidence == 0.8 assert t4.get_box3d(t4.sample_annotation[1].token).confidence == 1.0 From 1f9631c005c5511c4251b49af0522428768c9917 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 17:23:49 +0900 Subject: [PATCH 07/10] fix(cli): create the version console at call time, so captured output has no colors --- t4_devkit/cli/version.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/t4_devkit/cli/version.py b/t4_devkit/cli/version.py index 3080a24e..bb738e81 100644 --- a/t4_devkit/cli/version.py +++ b/t4_devkit/cli/version.py @@ -6,11 +6,10 @@ import typer from rich.console import Console -console = Console() - def version_callback(ctx: typer.Context, value: bool) -> None: if value: version = importlib.metadata.version("t4-devkit") + console = Console() console.print(f"[bold green]{ctx.info_name}[/bold green]: [cyan]{version}[/cyan]") raise typer.Exit() From 42ccadc6acdb112afe60db602aa82d5fa600f37b Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 19:34:18 +0900 Subject: [PATCH 08/10] test(schema): instance index above int16 is rejected --- tests/schema/v1/test_tables.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/schema/v1/test_tables.py b/tests/schema/v1/test_tables.py index 2361edfa..65b0436c 100644 --- a/tests/schema/v1/test_tables.py +++ b/tests/schema/v1/test_tables.py @@ -33,6 +33,7 @@ def _other_lidar_block(record: dict, records: list[dict]) -> None: "covariance-length": ("ego_pose", 0, _set(("covariance",), [0.0] * 35), ValueError), "annotation-type": ("sample_annotation", 0, _set(("annotation_type",), "human"), ValueError), "confidence-range": ("sample_annotation", 0, _set(("confidence",), 1.5), ValueError), + "instance-index-range": ("instance", 0, _set(("index",), 2**15), ValueError), "keypoints-shape": ("keypoint", 0, _set(("keypoints",), [1.0, 2.0]), ValueError), "scenario-end-before-start": ("scenario", 0, _set(("end_timestamp",), 0), ValueError), "traffic-light-color": ( From 7f82dc02092fea2d35e5b9064fb0724b4b8b82f4 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Thu, 15 Oct 2026 23:00:38 +0900 Subject: [PATCH 09/10] fix(schema): sample_data.filename is required, as in the master sheet --- docs/schema/v1/table.md | 2 +- t4_devkit/sanity/reference/ref201.py | 2 +- t4_devkit/schema/v1/sample_data.py | 4 ++-- tests/schema/v1/test_tables.py | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/schema/v1/table.md b/docs/schema/v1/table.md index 499a23a5..322cb042 100644 --- a/docs/schema/v1/table.md +++ b/docs/schema/v1/table.md @@ -68,7 +68,7 @@ sample_data { "sample_token": -- Foreign key to the `Sample` (frame) this observation belongs to. "ego_pose_token": -- Foreign key to the `EgoPose` table. "sensor_token": -- Foreign key to the `Sensor` table. Replaces `calibrated_sensor_token`. - "filename": -- Path to the channel file (video or point cloud container), or to a per-frame file for datasets that keep 0.x files. + "filename": -- Path to the channel file (video or point cloud container), or to a per-frame file for datasets that keep 0.x files. "frame_index": -- Index of the frame inside the channel file, starting at 0. `null` for per-frame files. "fileformat": -- Format of the file. "num_points": -- Number of points in the frame. LiDAR only. diff --git a/t4_devkit/sanity/reference/ref201.py b/t4_devkit/sanity/reference/ref201.py index e0fdf35d..ca1b682d 100644 --- a/t4_devkit/sanity/reference/ref201.py +++ b/t4_devkit/sanity/reference/ref201.py @@ -36,6 +36,6 @@ def check(self, context: SanityContext) -> list[Reason] | None: Reason(f"File not found: {record['filename']}") for record in records if record.get("is_valid", True) - and record["filename"] is not None # 1.0: no file + and record["filename"] is not None # invalid (reported by FMT022), not a missing file and not data_root.joinpath(record["filename"]).exists() ] or None diff --git a/t4_devkit/schema/v1/sample_data.py b/t4_devkit/schema/v1/sample_data.py index 9a0d75f2..3922eda5 100644 --- a/t4_devkit/schema/v1/sample_data.py +++ b/t4_devkit/schema/v1/sample_data.py @@ -46,7 +46,7 @@ class SampleData(SchemaBase): sample_token (str): Foreign key pointing to the sample (frame) this observation belongs to. ego_pose_token (str): Foreign key pointing to the ego pose. sensor_token (str): Foreign key pointing to the sensor. - filename (str | None): Path to the channel file, or to a per-frame file. + filename (str): Path to the channel file, or to a per-frame file. frame_index (int | None): Index of the frame inside the channel file, or None for per-frame files. fileformat (FileFormat): Format of the file. @@ -66,7 +66,7 @@ class SampleData(SchemaBase): sample_token: str = field(validator=(validators.instance_of(str), impossible_empty())) ego_pose_token: str = field(validator=(validators.instance_of(str), impossible_empty())) sensor_token: str = field(validator=(validators.instance_of(str), impossible_empty())) - filename: str | None = field(validator=validators.optional(validators.instance_of(str))) + filename: str = field(validator=(validators.instance_of(str), impossible_empty())) frame_index: int | None = field( validator=validators.optional( validators.and_(validators.instance_of(int), validators.ge(0)) diff --git a/tests/schema/v1/test_tables.py b/tests/schema/v1/test_tables.py index 65b0436c..e0e59d52 100644 --- a/tests/schema/v1/test_tables.py +++ b/tests/schema/v1/test_tables.py @@ -29,6 +29,7 @@ def _other_lidar_block(record: dict, records: list[dict]) -> None: # test_loader.py::test_load_dataset. CASES = { "empty-string-link": ("sample", 0, _set(("prev",), ""), ValueError), + "frame-without-file": ("sample_data", 0, _set(("filename",), None), TypeError), "frame-without-sample": ("sample_data", -1, _set(("sample_token",), None), TypeError), "covariance-length": ("ego_pose", 0, _set(("covariance",), [0.0] * 35), ValueError), "annotation-type": ("sample_annotation", 0, _set(("annotation_type",), "human"), ValueError), From f014293bc61d69c33806c38aadf0be56c20e6cb4 Mon Sep 17 00:00:00 2001 From: Jacob Lambert Date: Wed, 21 Oct 2026 10:31:05 +0900 Subject: [PATCH 10/10] test(schema): SampleCopy in the mask-only key frame test --- tests/schema/v1/test_convert_v0.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tests/schema/v1/test_convert_v0.py b/tests/schema/v1/test_convert_v0.py index 3ba6757d..6b922a7f 100644 --- a/tests/schema/v1/test_convert_v0.py +++ b/tests/schema/v1/test_convert_v0.py @@ -133,23 +133,18 @@ def test_load_v0_per_camera_samples(tmp_path: Path) -> None: def test_load_v0_mask_only_frame_is_key(tmp_path: Path) -> None: """A non-key 0.x frame whose only label is a semantic mask (`surface_ann`) is a key frame.""" - data_root = _copy_v0(tmp_path) - token = json.loads((data_root / "annotation" / "surface_ann.json").read_text())[0][ - "sample_data_token" - ] - _edit_v0( - data_root, + sample = SampleCopy(tmp_path, SAMPLE_V0) + token = sample.read("surface_ann")[0]["sample_data_token"] + sample.edit( "sample_data", lambda records: [ dict(r, is_key_frame=False) if r["token"] == token else r for r in records ], ) - _edit_v0( - data_root, - "object_ann", - lambda records: [r for r in records if r["sample_data_token"] != token], + sample.edit( + "object_ann", lambda records: [r for r in records if r["sample_data_token"] != token] ) - assert load_dataset(data_root).get(SchemaName.SAMPLE_DATA, token).is_key_frame + assert load_dataset(sample.root).get(SchemaName.SAMPLE_DATA, token).is_key_frame def test_load_v0_labeled_frame_is_key(tmp_path: Path) -> None: