diff --git a/docs/schema/v1/index.md b/docs/schema/v1/index.md index 713d2b5f..84c03dc9 100644 --- a/docs/schema/v1/index.md +++ b/docs/schema/v1/index.md @@ -156,6 +156,8 @@ Provenance { >>> manifest = V0Migrator("<0.x dataset>", "out//").migrate("", ) ``` +Metadata 0.x does not store (vehicle, map ids, provenance, routes, ...) can be added by the caller before writing: change records through `migrator.tables(...)`, and pass `manifest_fields` (e.g. `provenance`, `project_id`) to `migrate()`. + ## Loading 0.x Datasets `load_dataset()` reads a 0.x dataset (no `manifest.json`) by converting its tables to 1.0 records when loading, on a best-effort basis. Sensor data files stay as they are: one file per frame, and `sample_data.frame_index` is `null`. diff --git a/docs/schema/v1/table.md b/docs/schema/v1/table.md index 322cb042..524795b4 100644 --- a/docs/schema/v1/table.md +++ b/docs/schema/v1/table.md @@ -93,7 +93,7 @@ ego_pose { "rotation": <[float;4]> -- Orientation as quaternion (w, x, y, z). "timestamp": -- Unix timestamp in microseconds. "twist": -- (vx, vy, vz, yaw_rate, pitch_rate, roll_rate) in the vehicle frame. - "acceleration": -- (ax, ay, az) in the vehicle frame. + "acceleration": -- (ax, ay, az) in the vehicle frame, as an IMU measures it: gravity included (az ≈ +9.8 m/s² at rest). "geocoordinate": -- (latitude, longitude, altitude). "source": -- Where the pose came from. "frame": -- Coordinate frame, e.g. "map". diff --git a/t4_devkit/schema/v1/migrate_v0.py b/t4_devkit/schema/v1/migrate_v0.py index f5c03ef9..4ab8a075 100644 --- a/t4_devkit/schema/v1/migrate_v0.py +++ b/t4_devkit/schema/v1/migrate_v0.py @@ -3,7 +3,7 @@ import os.path as osp import warnings from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Mapping import numpy as np @@ -168,15 +168,28 @@ def copy_images(self, sensor: Sensor) -> None: for record in records: self.writer.copy(osp.join(self.data_root, record.filename), record.filename) - def migrate(self, dataset_id: str, version: int, *, keep_images: bool = False) -> Manifest: + def migrate( + self, + dataset_id: str, + version: int, + *, + keep_images: bool = False, + manifest_fields: Mapping[str, Any] | None = None, + ) -> Manifest: """Migrate every channel, copy `map/` and `input_bag/`, then write the tables and `manifest.json`. + What 0.x does not store can be given by the caller: records through `tables`, and + manifest fields (e.g. `provenance`, `project_id`, `generators`) through + `manifest_fields`. + Args: dataset_id (str): Dataset ID, as in the output folder. version (int): Dataset version, as in the output folder (given by Web.Auto). keep_images (bool, optional): Copy camera images as they are instead of encoding one video per channel. + manifest_fields (Mapping[str, Any] | None, optional): Manifest fields that replace + the defaults; `migration_notes` are added after the migration's own. Returns: The manifest written. @@ -191,19 +204,25 @@ def migrate(self, dataset_id: str, version: int, *, keep_images: bool = False) - for directory in ("map", "input_bag"): if osp.isdir(osp.join(self.data_root, directory)): self.writer.copy(osp.join(self.data_root, directory), directory) - return self.writer.save(self.dataset.tables, self._manifest_info(dataset_id, version)) + fields = dict(manifest_fields or {}) + if "provenance" not in fields: + self.notes.append( + "provenance.rosbag.record_start / record_end are the first / last sample times" + ) + manifest_info = self._manifest_info(dataset_id, version) + manifest_info.update(fields) + manifest_info["migration_notes"] = self.notes + list(fields.get("migration_notes", [])) + return self.writer.save(self.dataset.tables, manifest_info) def tables(self, table: SchemaName) -> list[Any]: """Return the 1.0 records of a table.""" return self.dataset.tables[table] def _manifest_info(self, dataset_id: str, version: int) -> dict[str, Any]: - """Return the manifest fields the writer does not compute. 0.x has no provenance, so - it is left empty for backfilling, except the recording time (from the samples).""" + """Return the default manifest fields the writer does not compute. 0.x has no + provenance, so it is left empty for backfilling, except the recording time (from the + samples).""" timestamps = [sample.timestamp for sample in self.tables(SchemaName.SAMPLE)] - self.notes.append( - "provenance.rosbag.record_start / record_end are the first / last sample times" - ) def utc(timestamp: int) -> str: return datetime.fromtimestamp(timestamp / 1e6, tz=timezone.utc).strftime( @@ -233,7 +252,6 @@ def utc(timestamp: int) -> str: "is_synthetic": False, "is_discontinuous": None, "migrated_from_schema_version": "0.x", - "migration_notes": self.notes, } def _channel_frames(self, sensor: Sensor) -> list[SampleData]: diff --git a/tests/schema/v1/test_migrate_v0.py b/tests/schema/v1/test_migrate_v0.py index 08267b7a..27370fb3 100644 --- a/tests/schema/v1/test_migrate_v0.py +++ b/tests/schema/v1/test_migrate_v0.py @@ -267,3 +267,42 @@ def test_migrate_recalibrated_channel(tmp_path: Path) -> None: with pytest.warns(UserWarning), pytest.raises(FileExistsError, match="LIDAR_TOP"): _migrate_lidar(sample.root, tmp_path / "sample") + + +def test_migrate_with_backfilled_metadata(tmp_path: Path) -> None: + """Records changed through `tables` and caller-given manifest fields are written.""" + output_root = tmp_path / "t4dataset" / "1" + migrator = V0Migrator(SAMPLE_V0, output_root) + vehicle = migrator.tables(SchemaName.VEHICLE)[0] + vehicle.model = "j6_gen2" + provenance = { + "rosbag": { + "log_file_ids": ["3145801d-4bdd-486f-b0e6-9253fca050de"], + "storage": "sqlite3", + "record_start": "2025-11-13T07:26:15Z", + "record_end": "2025-11-13T07:27:13Z", + }, + "topics": {"LIDAR_TOP": "/sensing/lidar/concatenated/pointcloud"}, + "process_config_id": None, + "process_config_version_id": None, + } + fields = { + "project_id": "x2_dev", + "provenance": provenance, + "generators": [{"name": "autoware", "version": "20250715-v4.2.1"}], + "migration_notes": ["route: dropped, lanelets not in the dataset's map"], + } + with pytest.warns(UserWarning): # the sample's fractional intensity stays float32 + migrator.migrate("t4dataset", 1, manifest_fields=fields) + + manifest = json.loads((output_root / "manifest.json").read_text()) + assert manifest["project_id"] == "x2_dev" + assert manifest["provenance"] == provenance + assert [g["name"] for g in manifest["generators"]] == ["autoware", "t4-devkit"] + # the caller's notes come after the migration's; no "first / last sample times" note + assert manifest["migration_notes"][-1].startswith("route: dropped") + assert not any("sample times" in note for note in manifest["migration_notes"]) + vehicles = json.loads((output_root / "annotation" / "vehicle.json").read_text()) + assert vehicles[0]["model"] == "j6_gen2" + reports = sanity_check(output_root.parent.as_posix(), revision="1").reports + assert not [report.id for report in reports if report.is_failed()]