feat(schema): caller-given metadata in V0Migrator.migrate (backfill)

docs/schema/v1/index.md

@@ -156,6 +156,8 @@ Provenance {
 >>> manifest = V0Migrator("<0.x dataset>", "out/<DATASET_ID>/<DATASET_VERSION>").migrate("<DATASET_ID>", <DATASET_VERSION>)
 ```
 
+Metadata 0.x does not store (vehicle, map ids, provenance, routes, ...) can be added by the caller before writing: change records through `migrator.tables(...)`, and pass `manifest_fields` (e.g. `provenance`, `project_id`) to `migrate()`.
+
 ## Loading 0.x Datasets
 
 `load_dataset()` reads a 0.x dataset (no `manifest.json`) by converting its tables to 1.0 records when loading, on a best-effort basis. Sensor data files stay as they are: one file per frame, and `sample_data.frame_index` is `null`.

docs/schema/v1/table.md

@@ -93,7 +93,7 @@ ego_pose {
   "rotation":         <[float;4]> -- Orientation as quaternion (w, x, y, z).
   "timestamp":        <int> -- Unix timestamp in microseconds.
   "twist":            <option[[float;6]]> -- (vx, vy, vz, yaw_rate, pitch_rate, roll_rate) in the vehicle frame.
-  "acceleration":     <option[[float;3]]> -- (ax, ay, az) in the vehicle frame.
+  "acceleration":     <option[[float;3]]> -- (ax, ay, az) in the vehicle frame, as an IMU measures it: gravity included (az ≈ +9.8 m/s² at rest).
   "geocoordinate":    <option[[float;3]]> -- (latitude, longitude, altitude).
   "source":           <enum["localization", "post_processed", "gnss_ins"]> -- Where the pose came from.
   "frame":            <str> -- Coordinate frame, e.g. "map".

t4_devkit/schema/v1/migrate_v0.py

@@ -3,7 +3,7 @@
 import os.path as osp
 import warnings
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Mapping
 
 import numpy as np
 
@@ -168,15 +168,28 @@ def copy_images(self, sensor: Sensor) -> None:
         for record in records:
             self.writer.copy(osp.join(self.data_root, record.filename), record.filename)
 
-    def migrate(self, dataset_id: str, version: int, *, keep_images: bool = False) -> Manifest:
+    def migrate(
+        self,
+        dataset_id: str,
+        version: int,
+        *,
+        keep_images: bool = False,
+        manifest_fields: Mapping[str, Any] | None = None,
+    ) -> Manifest:
         """Migrate every channel, copy `map/` and `input_bag/`, then write the tables and
         `manifest.json`.
 
+        What 0.x does not store can be given by the caller: records through `tables`, and
+        manifest fields (e.g. `provenance`, `project_id`, `generators`) through
+        `manifest_fields`.
+
         Args:
             dataset_id (str): Dataset ID, as in the output folder.
             version (int): Dataset version, as in the output folder (given by Web.Auto).
             keep_images (bool, optional): Copy camera images as they are instead of encoding
                 one video per channel.
+            manifest_fields (Mapping[str, Any] | None, optional): Manifest fields that replace
+                the defaults; `migration_notes` are added after the migration's own.
 
         Returns:
             The manifest written.
@@ -191,19 +204,25 @@ def migrate(self, dataset_id: str, version: int, *, keep_images: bool = False) -
         for directory in ("map", "input_bag"):
             if osp.isdir(osp.join(self.data_root, directory)):
                 self.writer.copy(osp.join(self.data_root, directory), directory)
-        return self.writer.save(self.dataset.tables, self._manifest_info(dataset_id, version))
+        fields = dict(manifest_fields or {})
+        if "provenance" not in fields:
+            self.notes.append(
+                "provenance.rosbag.record_start / record_end are the first / last sample times"
+            )
+        manifest_info = self._manifest_info(dataset_id, version)
+        manifest_info.update(fields)
+        manifest_info["migration_notes"] = self.notes + list(fields.get("migration_notes", []))
+        return self.writer.save(self.dataset.tables, manifest_info)
 
     def tables(self, table: SchemaName) -> list[Any]:
         """Return the 1.0 records of a table."""
         return self.dataset.tables[table]
 
     def _manifest_info(self, dataset_id: str, version: int) -> dict[str, Any]:
-        """Return the manifest fields the writer does not compute. 0.x has no provenance, so
-        it is left empty for backfilling, except the recording time (from the samples)."""
+        """Return the default manifest fields the writer does not compute. 0.x has no
+        provenance, so it is left empty for backfilling, except the recording time (from the
+        samples)."""
         timestamps = [sample.timestamp for sample in self.tables(SchemaName.SAMPLE)]
-        self.notes.append(
-            "provenance.rosbag.record_start / record_end are the first / last sample times"
-        )
 
         def utc(timestamp: int) -> str:
             return datetime.fromtimestamp(timestamp / 1e6, tz=timezone.utc).strftime(
@@ -233,7 +252,6 @@ def utc(timestamp: int) -> str:
             "is_synthetic": False,
             "is_discontinuous": None,
             "migrated_from_schema_version": "0.x",
-            "migration_notes": self.notes,
         }
 
     def _channel_frames(self, sensor: Sensor) -> list[SampleData]:

tests/schema/v1/test_migrate_v0.py

@@ -267,3 +267,42 @@ def test_migrate_recalibrated_channel(tmp_path: Path) -> None:
 
     with pytest.warns(UserWarning), pytest.raises(FileExistsError, match="LIDAR_TOP"):
         _migrate_lidar(sample.root, tmp_path / "sample")
+
+
+def test_migrate_with_backfilled_metadata(tmp_path: Path) -> None:
+    """Records changed through `tables` and caller-given manifest fields are written."""
+    output_root = tmp_path / "t4dataset" / "1"
+    migrator = V0Migrator(SAMPLE_V0, output_root)
+    vehicle = migrator.tables(SchemaName.VEHICLE)[0]
+    vehicle.model = "j6_gen2"
+    provenance = {
+        "rosbag": {
+            "log_file_ids": ["3145801d-4bdd-486f-b0e6-9253fca050de"],
+            "storage": "sqlite3",
+            "record_start": "2025-11-13T07:26:15Z",
+            "record_end": "2025-11-13T07:27:13Z",
+        },
+        "topics": {"LIDAR_TOP": "/sensing/lidar/concatenated/pointcloud"},
+        "process_config_id": None,
+        "process_config_version_id": None,
+    }
+    fields = {
+        "project_id": "x2_dev",
+        "provenance": provenance,
+        "generators": [{"name": "autoware", "version": "20250715-v4.2.1"}],
+        "migration_notes": ["route: dropped, lanelets not in the dataset's map"],
+    }
+    with pytest.warns(UserWarning):  # the sample's fractional intensity stays float32
+        migrator.migrate("t4dataset", 1, manifest_fields=fields)
+
+    manifest = json.loads((output_root / "manifest.json").read_text())
+    assert manifest["project_id"] == "x2_dev"
+    assert manifest["provenance"] == provenance
+    assert [g["name"] for g in manifest["generators"]] == ["autoware", "t4-devkit"]
+    # the caller's notes come after the migration's; no "first / last sample times" note
+    assert manifest["migration_notes"][-1].startswith("route: dropped")
+    assert not any("sample times" in note for note in manifest["migration_notes"])
+    vehicles = json.loads((output_root / "annotation" / "vehicle.json").read_text())
+    assert vehicles[0]["model"] == "j6_gen2"
+    reports = sanity_check(output_root.parent.as_posix(), revision="1").reports
+    assert not [report.id for report in reports if report.is_failed()]