From e7675435894f25d10ad7d4f4426d72e120dbe08f Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 18:32:33 -0500 Subject: [PATCH] egocap: drop the capture_unix_us timestamp basis No publisher stamps it any more: the camera bridge publishes the frame-ready time or the chrony-qualified capture time, and the Jetson sender's choices are monotonic_us, capture_monotonic_us and processing_unix_us. Streams files and the emulator no longer accept it, and wire value 3 is no longer assigned. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/egocap/LIVEKIT_PREVIEW.md | 2 +- examples/egocap/capture_server/README.md | 8 ++++---- .../capture_server/livekit_streams.py | 1 - .../capture_server/capture_server/protocol.py | 4 ---- .../egocap/tests/backend/test_livekit_recording.py | 9 ++++----- .../tests/backend/test_livekit_recording_source.py | 2 -- .../tests/capture_server/test_livekit_streams.py | 4 +--- .../tests/tools/test_livekit_orin_emulator.py | 14 ++++---------- examples/egocap/tools/livekit_orin_emulator.py | 6 +++--- 9 files changed, 17 insertions(+), 33 deletions(-) diff --git a/examples/egocap/LIVEKIT_PREVIEW.md b/examples/egocap/LIVEKIT_PREVIEW.md index 26d2a795da..47ff7bd226 100644 --- a/examples/egocap/LIVEKIT_PREVIEW.md +++ b/examples/egocap/LIVEKIT_PREVIEW.md @@ -73,7 +73,7 @@ uv run python tools/livekit_orin_emulator.py \ ``` `--add-stream sensor=N,livekit=NAME` is repeatable. `--timestamp-basis` takes `monotonic_us` (default), -`capture_monotonic_us`, `capture_unix_us` or `processing_unix_us`; set the same value in the streams file. The Orin's +`capture_monotonic_us` or `processing_unix_us`; set the same value in the streams file. The Orin's sender (`--timestamp-basis=` on the sensing app) takes `monotonic_us` (default), `capture_monotonic_us` or `processing_unix_us`. Both publishers answer the PC's clock-sync requests, which recording relies on. Frames go through a software encoder, so this checks the PC side and the metadata, not the Jetson encoder. diff --git a/examples/egocap/capture_server/README.md b/examples/egocap/capture_server/README.md index 881fe056fa..115ef610d0 100644 --- a/examples/egocap/capture_server/README.md +++ b/examples/egocap/capture_server/README.md @@ -594,7 +594,7 @@ All enums are IntEnum classes. Supply enum members to the Python API; incoming n | `ImageCodec` | `JPEG=1` | | `ClockDomain` | `CAPTURE_HOST_MONOTONIC=1` | | `TimeUnit` | `NANOSECONDS=1` | -| `LiveKitTimestampBasis` | `PUBLISHER_MONOTONIC_US=1`, `PUBLISHER_PROCESSING_UNIX_US=2`, `SENSOR_CAPTURE_UNIX_US=3`, `SENSOR_CAPTURE_MONOTONIC_US=4` | +| `LiveKitTimestampBasis` | `PUBLISHER_MONOTONIC_US=1`, `PUBLISHER_PROCESSING_UNIX_US=2`, `SENSOR_CAPTURE_MONOTONIC_US=4` | | `SampleTimeSemantics` | `HOST_POLL=1` | | `PoseRepresentation` | `POSITION_QUATERNION=1` | | `QuaternionOrder` | `XYZW=1` | @@ -1229,7 +1229,7 @@ A `streams` entry with `kind` `LIVEKIT_IMAGE=3` says where a camera's video is p | `room` | Nonempty string of at most 512 characters. | The room to join. | | `publisher` | Nonempty string of at most 512 characters. | The publishing participant's identity. | | `track` | Nonempty string of at most 512 characters. | The published track's name; unique per `publisher`. | -| `timestamp_basis` | `LiveKitTimestampBasis`: `PUBLISHER_MONOTONIC_US=1`, `PUBLISHER_PROCESSING_UNIX_US=2`, `SENSOR_CAPTURE_UNIX_US=3`, `SENSOR_CAPTURE_MONOTONIC_US=4`. | What the frame's `user_timestamp_us` means. 1: the publisher's monotonic clock when the frame was ready to encode. 2: Unix processing time on the publisher. 3: Unix time of the sensor's capture event. 4: the sensor's capture event on the publisher's monotonic clock. Only 1 and 4 are recordable, using the publisher's clock-sync replies; 2 and 3 are preview only. | +| `timestamp_basis` | `LiveKitTimestampBasis`: `PUBLISHER_MONOTONIC_US=1`, `PUBLISHER_PROCESSING_UNIX_US=2`, `SENSOR_CAPTURE_MONOTONIC_US=4`. | What the frame's `user_timestamp_us` means. 1: the publisher's monotonic clock when the frame was ready to encode. 2: Unix processing time on the publisher. 4: the sensor's capture event on the publisher's monotonic clock. Only 1 and 4 are recordable, using the publisher's clock-sync replies; 2 is preview only. | | `width`, `height` | Positive integer, or null (the key may be omitted). | The configured frame size, or null when not known. | | `fps` | Finite positive number, or null (the key may be omitted). | The configured frame rate, or null when not known. | @@ -1244,13 +1244,13 @@ livekit: url: wss://LIVEKIT_HOST # ws:// or wss:// room: sensing-preview publisher: orin-sensing - timestamp_basis: monotonic_us # monotonic_us (default), capture_monotonic_us, processing_unix_us or capture_unix_us + timestamp_basis: monotonic_us # monotonic_us (default), capture_monotonic_us or processing_unix_us streams: # may be empty - {name: Head left, track: sensing-left, width: 2560, height: 1984, fps: 30} - {name: Head right, track: sensing-right} # width, height and fps are optional ``` -`timestamp_basis` applies to every stream and maps to the `LiveKitTimestampBasis` values above (`monotonic_us`=1, `processing_unix_us`=2, `capture_unix_us`=3, `capture_monotonic_us`=4). `url`, `room` and `publisher` are set once under `livekit`; each stream supplies `name`, `track` and optionally `width`, `height` and `fps`. A LiveKit stream name that equals a camera name configured with `--gmsl-camera` is rejected. +`timestamp_basis` applies to every stream and maps to the `LiveKitTimestampBasis` values above (`monotonic_us`=1, `processing_unix_us`=2, `capture_monotonic_us`=4). `url`, `room` and `publisher` are set once under `livekit`; each stream supplies `name`, `track` and optionally `width`, `height` and `fps`. A LiveKit stream name that equals a camera name configured with `--gmsl-camera` is rejected. Each WebSocket binary message consists of a 32-byte header followed immediately by a payload. For CaptureFrame, that payload is the packed tracking blob defined by SessionInit. For CaptureImage, it is the JPEG bytes. The header itself contains no JSON and no field-name strings. diff --git a/examples/egocap/capture_server/capture_server/livekit_streams.py b/examples/egocap/capture_server/capture_server/livekit_streams.py index ad44cecc05..0a9ed70a7b 100644 --- a/examples/egocap/capture_server/capture_server/livekit_streams.py +++ b/examples/egocap/capture_server/capture_server/livekit_streams.py @@ -30,7 +30,6 @@ _BASES = { "monotonic_us": p.LiveKitTimestampBasis.PUBLISHER_MONOTONIC_US, "processing_unix_us": p.LiveKitTimestampBasis.PUBLISHER_PROCESSING_UNIX_US, - "capture_unix_us": p.LiveKitTimestampBasis.SENSOR_CAPTURE_UNIX_US, "capture_monotonic_us": p.LiveKitTimestampBasis.SENSOR_CAPTURE_MONOTONIC_US, } _STREAM_KEYS = {"name", "track", "width", "height", "fps"} diff --git a/examples/egocap/capture_server/capture_server/protocol.py b/examples/egocap/capture_server/capture_server/protocol.py index 3736266cef..9f48c6d2ab 100644 --- a/examples/egocap/capture_server/capture_server/protocol.py +++ b/examples/egocap/capture_server/capture_server/protocol.py @@ -214,10 +214,6 @@ class LiveKitTimestampBasis(IntEnum): # Unix time in microseconds on the publisher host when the frame finished conversion: processing time, # later than capture by the capture and conversion delay. PUBLISHER_PROCESSING_UNIX_US = 2 - # Unix time in microseconds of the sensor's capture event, mapped from the sensor's own counter by a sender - # that has verified the mapping. Not accepted for synchronized recording: no Unix-clock mapping is part of the - # approved synchronization path, so the value is kept for the wire's sake only. - SENSOR_CAPTURE_UNIX_US = 3 # The sensor's capture event in microseconds on the publisher host's monotonic clock, mapped from the sensor's # own counter by a sender that has verified the mapping. Recordable the same way as PUBLISHER_MONOTONIC_US. SENSOR_CAPTURE_MONOTONIC_US = 4 diff --git a/examples/egocap/tests/backend/test_livekit_recording.py b/examples/egocap/tests/backend/test_livekit_recording.py index 8e8421b696..d8afa871a4 100644 --- a/examples/egocap/tests/backend/test_livekit_recording.py +++ b/examples/egocap/tests/backend/test_livekit_recording.py @@ -27,7 +27,6 @@ from recording_service import PixelFormat, SubmissionResult, SubmissionStatus, VideoSample CAPTURE = p.LiveKitTimestampBasis.SENSOR_CAPTURE_MONOTONIC_US -UNIX_CAPTURE = p.LiveKitTimestampBasis.SENSOR_CAPTURE_UNIX_US PROCESSING = p.LiveKitTimestampBasis.PUBLISHER_PROCESSING_UNIX_US MONO = p.LiveKitTimestampBasis.PUBLISHER_MONOTONIC_US @@ -412,8 +411,8 @@ def submit(sample): assert submitted[0].image is frame.image -@pytest.mark.parametrize("basis", [PROCESSING, UNIX_CAPTURE]) -async def test_non_capture_stream_fails_at_once_and_never_offers(basis): +async def test_non_capture_stream_fails_at_once_and_never_offers(): + basis = PROCESSING reasons: list[str] = [] streams = [_stream(16, "Head left", basis)] receiver = FakeReceiver({16: [_received(16)]}) @@ -851,8 +850,8 @@ def test_publisher_monotonic_stream_is_recordable(): assert unrecordable_reason(_started(_geometry(16, MONO), _geometry(17))) is None -@pytest.mark.parametrize("basis", [PROCESSING, UNIX_CAPTURE]) -def test_non_monotonic_bases_are_still_refused(basis): +def test_non_monotonic_basis_is_still_refused(): + basis = PROCESSING assert unrecordable_reason(_started(_geometry(16, basis))) == ( "LiveKit stream 'cam16': its timestamps are not on the publisher's monotonic clock, " "so it cannot be recorded" diff --git a/examples/egocap/tests/backend/test_livekit_recording_source.py b/examples/egocap/tests/backend/test_livekit_recording_source.py index 677a756c56..b12e7576b6 100644 --- a/examples/egocap/tests/backend/test_livekit_recording_source.py +++ b/examples/egocap/tests/backend/test_livekit_recording_source.py @@ -23,7 +23,6 @@ from egocap_web.livekit_recording import LiveKitRecordingSource, recordable_livekit_streams CAPTURE = p.LiveKitTimestampBasis.SENSOR_CAPTURE_MONOTONIC_US -UNIX_CAPTURE = p.LiveKitTimestampBasis.SENSOR_CAPTURE_UNIX_US PROCESSING = p.LiveKitTimestampBasis.PUBLISHER_PROCESSING_UNIX_US MONO = p.LiveKitTimestampBasis.PUBLISHER_MONOTONIC_US @@ -86,7 +85,6 @@ def test_only_monotonic_streams_with_full_geometry_are_recordable(): keep, frame_ready, livekit_stream(17, basis=PROCESSING), - livekit_stream(22, basis=UNIX_CAPTURE), livekit_stream(19, width=None), livekit_stream(20, height=None), livekit_stream(21, fps=None), diff --git a/examples/egocap/tests/capture_server/test_livekit_streams.py b/examples/egocap/tests/capture_server/test_livekit_streams.py index 7b48b9da8e..55fc6d069b 100644 --- a/examples/egocap/tests/capture_server/test_livekit_streams.py +++ b/examples/egocap/tests/capture_server/test_livekit_streams.py @@ -479,7 +479,6 @@ async def test_cli_serves_the_streams_from_the_file(tmp_path, unused_tcp_port): def test_timestamp_basis_values_are_wire_stable(): assert int(p.LiveKitTimestampBasis.PUBLISHER_MONOTONIC_US) == 1 assert int(p.LiveKitTimestampBasis.PUBLISHER_PROCESSING_UNIX_US) == 2 - assert int(p.LiveKitTimestampBasis.SENSOR_CAPTURE_UNIX_US) == 3 assert int(p.LiveKitTimestampBasis.SENSOR_CAPTURE_MONOTONIC_US) == 4 @@ -487,7 +486,6 @@ def test_timestamp_basis_values_are_wire_stable(): ("value", "expected"), [ (2, p.LiveKitTimestampBasis.PUBLISHER_PROCESSING_UNIX_US), - (3, p.LiveKitTimestampBasis.SENSOR_CAPTURE_UNIX_US), (4, p.LiveKitTimestampBasis.SENSOR_CAPTURE_MONOTONIC_US), ], ) @@ -503,7 +501,6 @@ def test_non_default_bases_round_trip_through_json(value, expected): ("", p.LiveKitTimestampBasis.PUBLISHER_MONOTONIC_US), ("timestamp_basis: monotonic_us,", p.LiveKitTimestampBasis.PUBLISHER_MONOTONIC_US), ("timestamp_basis: processing_unix_us,", p.LiveKitTimestampBasis.PUBLISHER_PROCESSING_UNIX_US), - ("timestamp_basis: capture_unix_us,", p.LiveKitTimestampBasis.SENSOR_CAPTURE_UNIX_US), ("timestamp_basis: capture_monotonic_us,", p.LiveKitTimestampBasis.SENSOR_CAPTURE_MONOTONIC_US), ], ) @@ -522,6 +519,7 @@ def test_loader_puts_the_room_basis_in_every_stream(tmp_path, basis_line, expect "wall_clock", "unix_us", "UNIX_US", + "capture_unix_us", "CAPTURE_UNIX_US", "CAPTURE_MONOTONIC_US", "capture_monotonic", diff --git a/examples/egocap/tests/tools/test_livekit_orin_emulator.py b/examples/egocap/tests/tools/test_livekit_orin_emulator.py index 997e05ead3..cf9b8839b8 100644 --- a/examples/egocap/tests/tools/test_livekit_orin_emulator.py +++ b/examples/egocap/tests/tools/test_livekit_orin_emulator.py @@ -71,9 +71,7 @@ def test_missing_or_empty_token_file_is_rejected(tmp_path, token_file): emu.parse_args(argv(empty, stream)) -@pytest.mark.parametrize( - "basis", ["capture_monotonic_us", "capture_unix_us", "processing_unix_us", "monotonic_us"] -) +@pytest.mark.parametrize("basis", ["capture_monotonic_us", "processing_unix_us", "monotonic_us"]) def test_each_timestamp_basis_choice_is_accepted(token_file, basis): args = emu.parse_args(argv(token_file, "--add-stream=sensor=0,livekit=a", f"--timestamp-basis={basis}")) assert args.timestamp_basis == basis @@ -90,7 +88,6 @@ def test_bad_options_are_rejected(token_file, extra): def test_user_timestamp_basis(monkeypatch): monkeypatch.setattr(emu.time, "time_ns", lambda: 5_000_000) monkeypatch.setattr(emu.time, "monotonic_ns", lambda: 9_000_000) - assert emu.user_timestamp_us("capture_unix_us") == 5_000 assert emu.user_timestamp_us("processing_unix_us") == 5_000 assert emu.user_timestamp_us("monotonic_us") == 9_000 assert emu.user_timestamp_us("capture_monotonic_us") == 9_000 @@ -266,7 +263,6 @@ def slow_pattern(width, height, sensor, frame_id): ("basis", "expected_us"), [ ("capture_monotonic_us", 1_000), - ("capture_unix_us", 1_000), ("monotonic_us", 1_500), ("processing_unix_us", 1_500), ], @@ -305,11 +301,10 @@ def _model(now_ns, **kw): return emu.ClockModel(started_ns=0, monotonic_ns=lambda: now_ns[0], **kw) -def test_offset_only_applies_to_unix_bases_at_any_run_time(): +def test_offset_only_applies_to_the_unix_basis_at_any_run_time(): now_ns = [0] model = _model(now_ns, offset_us=250_000) for now_ns[0] in (0, 10_000_000_000): - assert model.adjustment_us("capture_unix_us", at_capture=True) == 250_000 assert model.adjustment_us("processing_unix_us", at_capture=False) == 250_000 @@ -318,7 +313,7 @@ def test_drift_grows_linearly_with_injected_run_time(): model = _model(now_ns, drift_ppm=15.0) seen = [] for now_ns[0] in (0, 10_000_000_000, 20_000_000_000): - seen.append(model.adjustment_us("capture_unix_us", at_capture=True)) + seen.append(model.adjustment_us("processing_unix_us", at_capture=False)) assert seen == [0, 150, 300] @@ -327,7 +322,6 @@ def test_pipeline_delay_applies_at_capture_only_and_monotonic_follows_its_own_of model = _model( now_ns, offset_us=1_000, drift_ppm=15.0, pipeline_delay_us=40_000, monotonic_offset_ns=3_000_000_000 ) - assert model.adjustment_us("capture_unix_us", at_capture=True) == 1_000 + 150 - 40_000 assert model.adjustment_us("processing_unix_us", at_capture=False) == 1_000 + 150 # The Unix offset does not leak into the monotonic basis. assert model.adjustment_us("monotonic_us", at_capture=True) == 3_000_000 + 150 - 40_000 @@ -448,7 +442,7 @@ async def test_tracks_sharing_one_model_get_equal_stamps_and_keep_their_identity async def test_all_zero_model_reproduces_this_hosts_readings(token_file, monkeypatch): model = emu.ClockModel() - for basis in ("capture_monotonic_us", "capture_unix_us", "processing_unix_us", "monotonic_us"): + for basis in ("capture_monotonic_us", "processing_unix_us", "monotonic_us"): assert model.adjustment_us(basis, at_capture=True) == model.adjustment_us(basis, at_capture=False) == 0 _, timestamp_us, metadata = await _publish_one(token_file, monkeypatch, 0, model) assert timestamp_us == metadata.user_timestamp == 5_000_000 diff --git a/examples/egocap/tools/livekit_orin_emulator.py b/examples/egocap/tools/livekit_orin_emulator.py index 2db8f77fa6..c88ae32e4f 100644 --- a/examples/egocap/tools/livekit_orin_emulator.py +++ b/examples/egocap/tools/livekit_orin_emulator.py @@ -37,7 +37,7 @@ STATS_INTERVAL_S = 5.0 #: Bases on the emulated publisher's monotonic clock, and bases that stamp the capture event (not processing time). _MONOTONIC_BASES = ("capture_monotonic_us", "monotonic_us") -_CAPTURE_BASES = ("capture_monotonic_us", "capture_unix_us") +_CAPTURE_BASES = ("capture_monotonic_us",) #: Width of the frame-counter strip, in bits (one coloured block per bit). _COUNTER_BITS = 24 @@ -87,12 +87,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--fps", type=float, default=30.0) parser.add_argument( "--timestamp-basis", - choices=("capture_monotonic_us", "capture_unix_us", "processing_unix_us", "monotonic_us"), + choices=("capture_monotonic_us", "processing_unix_us", "monotonic_us"), default="monotonic_us", help="what user_timestamp carries: monotonic_us is the emulated Orin's monotonic time when the frame is " "ready (what the native sender stamps by default; recordable, mapped by clock-sync); " "capture_monotonic_us is the same clock declared as the sensor capture event (also recordable); " - "capture_unix_us and processing_unix_us are Unix-time stamps, which cannot be recorded", + "processing_unix_us is a Unix-time stamp, which cannot be recorded", ) parser.add_argument( "--clock-offset-ms",