From 61bdb489e0b3b82003064af26f6612def0b80053 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 12 Nov 2026 18:31:11 -0500 Subject: [PATCH 01/13] egocap: set up chrony time sync from the launcher instead of the UDP clock exchange Once the page is up, the launcher installs chrony on the PC if missing, starts the timesync module's unprivileged follower, configures the Orin's chrony through the module tool, and waits a bounded time for the clocks to settle. A failure there leaves the preview running and capture times unavailable. The bridge gets the chrony addresses and the Orin's reference id. The sender profile and launcher no longer carry the UDP clock port or the timing-extension switch. The launcher also runs the receiver's setup.sh when its venv or decoder is missing, frees the cameras from nvargus-daemon before the sender starts, stops the remote sender on exit, and puts the Manus calibration worker on CaptureServer's path. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/egocap/CAMERA_SETTINGS.md | 19 +- examples/egocap/README.md | 13 +- .../configs/default-camera-settings.yaml | 5 - .../egocap/tests/tools/test_start_system.py | 62 +++++- examples/egocap/tools/start_system.py | 202 +++++++++++++++--- 5 files changed, 244 insertions(+), 57 deletions(-) diff --git a/examples/egocap/CAMERA_SETTINGS.md b/examples/egocap/CAMERA_SETTINGS.md index ef50361baf..77ff2d144a 100644 --- a/examples/egocap/CAMERA_SETTINGS.md +++ b/examples/egocap/CAMERA_SETTINGS.md @@ -17,7 +17,7 @@ pass it with `--sender-config`. | SIPL platform file | Sensor geometry, sensor rate, trigger mode, link wiring | `sipl.platform_config_json` on the Orin (installed by the BSP or the camera deploy) | | Sender (Orin) | Output frame rate and bitrate, codec, timing records, clock responder, TCP port | The sender profile, `configs/default-camera-settings.yaml` | | Receiver (PC) | Decoding, the CUDA IPC output and its stream names | Generated by the launcher from the sender profile | -| Bridge (PC) | Publishing into LiveKit; timestamp basis; the sender-clock address; output rate | Command-line flags, built by the launcher | +| Bridge (PC) | Publishing into LiveKit; timestamp basis; the chrony addresses; output rate | Command-line flags, built by the launcher | | CaptureServer | The stream descriptors the page reads (room, track, size, nominal fps, timestamp basis) | Generated by the launcher | The sender never reads exposure, gain, white balance, trigger frequency or clock-pin settings from its YAML, and there @@ -26,17 +26,16 @@ is no runtime settings API: add such keys to the file and nothing happens. ## The default sender profile `configs/default-camera-settings.yaml` is the profile the launcher uses unless told otherwise. It describes one SHW5G head -camera on the SIPL backend: HEVC at 30 fps and 4 Mbit/s, -the AVS1 timing extension on, and the UDP clock responder on port 5611. +camera on the SIPL backend: HEVC at 30 fps and 4 Mbit/s. The file is specific to that rig. A different SIPL platform or camera changes the `sipl` block (platform file, config name, link masks, tuning directory) and possibly the stream id and size. The Argus backend has no `sipl` block: it selects a sensor mode with `cameras[].mode`, can set `capture.mode` and `alignment`, enforces `output.fps` with a rate limiter and can set output dimensions. Keep one file per rig and pass the right one with `--sender-config`. The launcher itself supports one stream per -file and requires `network.timing_extension: true`, `network.clock_port` and an explicit `output.fps`. +file and requires an explicit `output.fps`. `tools/start_system.py` loads this file by default. Command-line overrides (`--fps`, `--bitrate-mbps`, -`--clock-port`, `--sender-port`) are applied to the loaded mapping, and the result is what is uploaded. The same +`--sender-port`) are applied to the loaded mapping, and the result is what is uploaded. The same values feed the bridge and the CaptureServer descriptor, so they cannot disagree. ### How the file reaches the sender @@ -68,10 +67,10 @@ effective `output.fps`; the descriptor value is nominal and cannot make the came The page shows latency from the sensor's capture time only when all of these hold: -1. The sender runs with `network.timing_extension: true` and `network.clock_port` set (the default profile does both), - from a release that includes the timing extension and the clock responder. +1. The sender is a release that includes the AVS1 timing extension; it sends the sensor capture time on its + `CLOCK_REALTIME` with every frame. 2. The receiver passes the timing records through to the CUDA IPC output. -3. The bridge runs with `--timestamp-basis capture_monotonic_us --sender-clock ORIN_HOST:CLOCK_PORT`. For a SIPL profile +3. The bridge runs with `--timestamp-basis capture_monotonic_us --chrony-sender ORIN_HOST --chrony-sender-refid ID --chrony-pc-port PORT`. For a SIPL profile the launcher also passes `--source-full-range`, because SIPL pictures use the full 0-255 range; without it the encoded stream signals limited range and the page's blacks and whites are wrong. The bridge binary must support that flag. @@ -99,13 +98,11 @@ shipped profile. | `sipl.link_masks` | Empty; required for SIPL | Sequence of unsigned masks, one per deserializer in transport order; passed to `ApplyMask`. Not an FPS knob. | | `sipl.nito_dir` | `/var/nvidia/nvcam/settings/sipl` | ISP tuning directory; loads `.nito` on the Orin. Not a YAML sensor-mode selector. | | `sipl.isp0_buffers` | `6`; at least 2 | Number of ISP0 surfaces allocated/registered per sensor. Surface ownership and return timing affect pressure; increasing the count does not directly select FPS. | -| `network.bind` | `0.0.0.0`; nonempty | Sender TCP listening address; also used by the timing UDP responder, whose inspected implementation requires IPv4. | +| `network.bind` | `0.0.0.0`; nonempty | Sender TCP listening address. | | `network.port` | `5000`; 1–65535 | AVS1 TCP listening port. Receiver must connect here. | | `network.reconnect_ms` | `1000` | Retry delay for failed listen setup. Not a per-frame sleep or FPS limiter. | | `network.dump_wire` | Empty / disabled | Writes the transmitted framed stream to a diagnostic file on the sender. Avoid enabling by default; storage can add work. | | `network.send_buffer_kb` | `0` | Kernel TCP send-buffer request in KiB. Zero leaves OS behavior; positive values set `SO_SNDBUF`. Bounds one source of hidden buffering. | -| `network.timing_extension` | `false` | Emits AVS1 sensor timing qualification records; use `true` for the capture-time chain. Requires a compatible sender and receiver. | -| `network.clock_port` | `0` / disabled | UDP clock-exchange responder; 1–65535 enables it. Use 5611 in this profile and the same port in the bridge's sender-clock address. | | `runtime.frames` | `0` / unlimited | Finite run limit. Argus counts bundles; SIPL counts successful frame submissions across streams. Not a time duration or FPS setting. | | `runtime.kpi` | `true` | Enables periodic KPI collection/reporting; detailed output also depends on `--verbose`. | | `runtime.log_interval_s` | `1` | Periodic logging interval in seconds; zero disables interval-based summaries at sites guarded by a positive interval. | diff --git a/examples/egocap/README.md b/examples/egocap/README.md index 4b83d227d7..5149643cdb 100644 --- a/examples/egocap/README.md +++ b/examples/egocap/README.md @@ -106,15 +106,16 @@ Without `--orin` or `--mock` only the PC side starts. `--skip-deploy` reuses the for `--seconds` (default one hour) and the launcher stops the whole stack when it ends, so a take in progress is cut off; the launcher prints the stop time at startup. The sender release the profile describes, its deploy tool, the LiveKit server and the bridge are not in this tree, so pass their paths (or set `EGOCAP_DEPLOY_SCRIPT`, -`EGOCAP_LIVEKIT_SERVER`, `EGOCAP_BRIDGE`). `--orin` needs an IPv4 address, because the bridge measures the sender's -clock at that address. +`EGOCAP_LIVEKIT_SERVER`, `EGOCAP_BRIDGE`). `--orin` needs an IPv4 address, because the bridge finds the sender's chrony by it. The launcher starts an +unprivileged chronyd on this PC that measures the Orin's clock and configures the Orin's chrony through +`src/core/timesync/tools/timesync_tool.py`; see that module's README. The page needs the frontend built once (`cd frontend && npm install && npm run build`); the launcher refuses to start -without `frontend/dist`. +without `frontend/dist`. For the default receiver, the launcher runs the receiver's `setup.sh` (venv, requirements, +native decoder) when the venv or decoder is missing. -The sender settings come from the checked-in profile `configs/default-camera-settings.yaml` (30 fps, 4 Mbit/s HEVC, -timing extension and clock responder on). Pass another file with `--sender-config`, or override single values with -`--fps`, `--bitrate-mbps`, `--clock-port` and `--sender-port`; the receiver, bridge and CaptureServer descriptor are +The sender settings come from the checked-in profile `configs/default-camera-settings.yaml` (30 fps, 4 Mbit/s HEVC). Pass another file with `--sender-config`, or override single values with +`--fps`, `--bitrate-mbps` and `--sender-port`; the receiver, bridge and CaptureServer descriptor are built from the same effective values. [`CAMERA_SETTINGS.md`](CAMERA_SETTINGS.md) says which component owns which setting and lists every key. diff --git a/examples/egocap/configs/default-camera-settings.yaml b/examples/egocap/configs/default-camera-settings.yaml index eddd396809..e5cbd27cc7 100644 --- a/examples/egocap/configs/default-camera-settings.yaml +++ b/examples/egocap/configs/default-camera-settings.yaml @@ -22,11 +22,6 @@ sipl: network: bind: 0.0.0.0 port: 5000 # TCP port the PC receiver connects to - # The two settings below are what let the page show latency measured from the sensor's capture time: the timing - # records carry the capture instant, and the clock responder lets the PC map the sender's clock onto its own. - # The bridge connects to clock_port, so the launcher takes it from here. - timing_extension: true - clock_port: 5611 # UDP runtime: frames: 0 # 0 = run until stopped diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index e1c93101d1..0918834b8d 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -128,7 +128,7 @@ def fail_start(*_, **__): def test_default_profile_gives_the_head_camera_settings(): config = ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG) assert ss.sender_settings(config) == ss.SenderSettings( - port=5000, clock_port=5611, fps=30, bitrate_mbps=4, stream_id="head", full_range=True, sipl=True + port=5000, fps=30, bitrate_mbps=4, stream_id="head", full_range=True, sipl=True ) @@ -139,19 +139,17 @@ def test_argus_backend_is_not_full_range(): def test_overrides_replace_only_their_values(): - config = ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG, 60, 9, 6000, 5100) + config = ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG, 60, 9, 5100) assert ss.sender_settings(config) == ss.SenderSettings( - port=5100, clock_port=6000, fps=60, bitrate_mbps=9, stream_id="head", full_range=True, sipl=True + port=5100, fps=60, bitrate_mbps=9, stream_id="head", full_range=True, sipl=True ) partial = ss.sender_settings(ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG, fps=15)) - assert (partial.fps, partial.bitrate_mbps, partial.clock_port, partial.port) == (15, 4, 5611, 5000) + assert (partial.fps, partial.bitrate_mbps, partial.port) == (15, 4, 5000) @pytest.mark.parametrize( "mutate", [ - lambda c: c["network"].pop("timing_extension"), - lambda c: c["network"].pop("clock_port"), lambda c: c["streams"][0]["output"].update(fps=0), lambda c: c["streams"][0]["output"].pop("fps"), lambda c: c.update(capture_backend="auto"), @@ -638,3 +636,55 @@ def test_sighup_during_stop_kills_everything_at_once_and_restores_the_handler(st assert time.monotonic() - started < 5 assert [c.process.returncode for c in stack.children] == [-signal.SIGKILL] * 2 assert signal.getsignal(signal.SIGHUP) is before + + +def test_default_profile_has_no_legacy_clock_keys(): + network = ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG)["network"] + assert "timing_extension" not in network and "clock_port" not in network + + +def test_parse_args_timesync_defaults(monkeypatch): + monkeypatch.delenv("EGOCAP_TIMESYNC_TOOL", raising=False) + args = ss.parse_args([]) + assert args.chrony_cmd_port == 12301 + assert args.chrony_wait == 120.0 + assert args.require_chrony is False + assert args.timesync_tool.as_posix().endswith("src/core/timesync/tools/timesync_tool.py") + + +def test_load_timesync_tool_exposes_module_functions(tmp_path: Path): + tool = tmp_path / "fake_tool.py" + tool.write_text("def lan_address_toward(h):\n return h\n\ndef reference_id_for(a):\n return 'ID' + a\n") + module = ss.load_timesync_tool(tool) + assert module.reference_id_for("y") == "IDy" + + +def test_load_timesync_tool_missing_file_raises(tmp_path: Path): + with pytest.raises(ss.StartupError, match="does not exist"): + ss.load_timesync_tool(tmp_path / "absent.py") + + +def test_default_receiver_uses_its_venv_python_when_present(monkeypatch, tmp_path: Path): + receiver_dir = tmp_path / "examples/camera_streamer/camera_receiver" + (receiver_dir / ".venv/bin").mkdir(parents=True) + (receiver_dir / ".venv/bin/python").write_text("") + monkeypatch.setattr(ss, "REPO_ROOT", tmp_path) + assert ss.default_receiver_command().startswith(f"{receiver_dir}/.venv/bin/python ") + (receiver_dir / ".venv/bin/python").unlink() + assert ss.default_receiver_command().startswith("python3 ") + + +def test_ensure_receiver_environment_runs_setup_only_when_something_is_missing(monkeypatch, tmp_path: Path): + receiver_dir = tmp_path / "examples/camera_streamer/camera_receiver" + (receiver_dir / ".venv/bin").mkdir(parents=True) + (receiver_dir / "codec").mkdir() + monkeypatch.setattr(ss, "REPO_ROOT", tmp_path) + calls = [] + monkeypatch.setattr(ss.subprocess, "run", lambda cmd, **kw: calls.append(cmd)) + ss.ensure_receiver_environment() + assert calls == [["bash", str(receiver_dir / "setup.sh")]] + calls.clear() + (receiver_dir / ".venv/bin/python").write_text("") + (receiver_dir / "codec/_camera_receiver_codec.cpython-313-x86_64-linux-gnu.so").write_text("") + ss.ensure_receiver_environment() + assert calls == [] diff --git a/examples/egocap/tools/start_system.py b/examples/egocap/tools/start_system.py index 05e7bfe05e..b9e81a9c7f 100644 --- a/examples/egocap/tools/start_system.py +++ b/examples/egocap/tools/start_system.py @@ -51,6 +51,7 @@ LIVEKIT_DEV_KEY = "devkey" LIVEKIT_DEV_SECRET = "secret" CONTROL_TOKEN_MARKER = "Recording control token for this run:" +TIMESYNC_TOOL = REPO_ROOT / "src/core/timesync/tools/timesync_tool.py" DEFAULT_SENDER_CONFIG = EGOCAP_ROOT / "configs/default-camera-settings.yaml" DEFAULT_STATION_CONFIG = EGOCAP_ROOT / "recording/egocap/stations/ego_stations.example.yaml" # The page reads latency from the sensor's capture time; see CAMERA_SETTINGS.md. @@ -152,7 +153,6 @@ class SenderSettings: """The effective sender values the rest of the stack must agree with.""" port: int - clock_port: int fps: int bitrate_mbps: int stream_id: str @@ -174,7 +174,6 @@ def load_sender_config( path: Path, fps: int | None = None, bitrate_mbps: int | None = None, - clock_port: int | None = None, sender_port: int | None = None, ) -> dict: """Read the checked-in sender profile and apply the command-line overrides to the loaded mapping. @@ -198,8 +197,6 @@ def load_sender_config( output["fps"] = fps if bitrate_mbps is not None: output["bitrate_mbps"] = bitrate_mbps - if clock_port is not None: - network["clock_port"] = clock_port if sender_port is not None: network["port"] = sender_port return config @@ -221,12 +218,8 @@ def sender_settings(config: dict) -> SenderSettings: raise StartupError( "the sender config must set streams[0].output.fps; the bridge and descriptor need a rate" ) - # Without both, the page has no sensor capture time and no clock to map it with. - if not network.get("timing_extension") or not network.get("clock_port"): - raise StartupError("the sender config must set network.timing_extension: true and network.clock_port") return SenderSettings( port=int(network.get("port", 5000)), - clock_port=int(network["clock_port"]), fps=int(output["fps"]), bitrate_mbps=int(output.get("bitrate_mbps", 0)), stream_id=str(stream["id"]), @@ -248,12 +241,23 @@ def mock_sender_command(python: str, script: Path, settings: SenderSettings, wid ] -def bridge_timing_arguments(settings: SenderSettings, mock: bool, orin_host: str | None) -> list[str]: - """The bridge flags that say what the frame timestamps mean and where the sender's clock can be measured.""" +def bridge_timing_arguments( + settings: SenderSettings, mock: bool, orin_host: str | None, reference_id: str | None, chrony_port: int +) -> list[str]: + """The bridge flags that say what the frame timestamps mean and which chrony instances can vouch for them.""" if mock: - # No sender clock to measure and no SIPL range to signal. + # No sender clock to qualify and no SIPL range to signal. return ["--timestamp-basis", MOCK_TIMESTAMP_BASIS] - flags = ["--timestamp-basis", TIMESTAMP_BASIS, "--sender-clock", f"{orin_host}:{settings.clock_port}"] + flags = [ + "--timestamp-basis", + TIMESTAMP_BASIS, + "--chrony-sender", + str(orin_host), + "--chrony-sender-refid", + str(reference_id), + "--chrony-pc-port", + str(chrony_port), + ] # SIPL pictures are full range; the bridge must say so or the encoded stream signals limited range. return [*flags, "--source-full-range"] if settings.full_range else flags @@ -277,6 +281,26 @@ def check_station_cameras(station_config: Path, capture_cameras: Sequence[str]) ) +def load_timesync_tool(path: Path): + """The timesync module's tool, imported so its address and reference-id helpers have one copy.""" + import importlib.util + + if not path.is_file(): + raise StartupError(f"the timesync tool {path} does not exist (--timesync-tool or EGOCAP_TIMESYNC_TOOL)") + spec = importlib.util.spec_from_file_location("timesync_tool", path) + if spec is None or spec.loader is None: + raise StartupError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + # The tool's dataclasses look their module up by name, so it is registered while it runs. + sys.modules["timesync_tool"] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop("timesync_tool", None) + raise + return module + + def port_open(host: str, port: int) -> bool: """Whether something accepts TCP connections at host:port.""" try: @@ -453,6 +477,25 @@ def control_token(log: Path) -> str | None: return None +def ensure_receiver_environment() -> None: + """Run the receiver's own ``setup.sh`` when its venv or its native decoder is missing; it is idempotent.""" + receiver_dir = REPO_ROOT / "examples/camera_streamer/camera_receiver" + has_venv = (receiver_dir / ".venv/bin/python").is_file() + has_codec = any((receiver_dir / "codec").glob("_camera_receiver_codec*.so")) + if has_venv and has_codec: + return + print("setting up the camera receiver environment (first run)", flush=True) + subprocess.run(["bash", str(receiver_dir / "setup.sh")], check=True) + + +def default_receiver_command() -> str: + """Run the receiver with the Python from its own ``setup.sh`` environment, which has CuPy, when it exists.""" + receiver_dir = REPO_ROOT / "examples/camera_streamer/camera_receiver" + venv_python = receiver_dir / ".venv/bin/python" + python = str(venv_python) if venv_python.is_file() else "python3" + return f"{python} {receiver_dir / 'camera_receiver.py'}" + + def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: """The command line; every external tool's path may also come from an environment variable.""" parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) @@ -466,12 +509,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--bridge", default=os.environ.get("EGOCAP_BRIDGE"), help="camera_livekit_bridge binary") parser.add_argument( "--receiver", - default=os.environ.get( - "EGOCAP_RECEIVER", - shlex.join( - ["python3", str(REPO_ROOT / "examples/camera_streamer/camera_receiver/camera_receiver.py")] - ), - ), + default=os.environ.get("EGOCAP_RECEIVER", default_receiver_command()), help="command that runs the camera receiver (takes --config FILE)", ) parser.add_argument("--release", default="egocap-start", help="release name for the Orin deploy") @@ -486,8 +524,24 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: ) parser.add_argument("--fps", type=int, help="override the profile's output.fps") parser.add_argument("--bitrate-mbps", type=int, help="override the profile's HEVC bitrate") - parser.add_argument("--clock-port", type=int, help="override the profile's UDP clock port (sender and bridge)") parser.add_argument("--sender-port", type=int, help="override the profile's TCP video port") + parser.add_argument( + "--timesync-tool", + type=Path, + default=Path(os.environ.get("EGOCAP_TIMESYNC_TOOL", TIMESYNC_TOOL)), + help="the timesync module's timesync_tool.py", + ) + parser.add_argument("--chrony-cmd-port", type=int, default=12301, help="command port of this PC's chronyd") + parser.add_argument( + "--chrony-wait", + type=float, + default=120.0, + help="seconds to wait for this PC to settle on the Orin's clock", + ) + parser.add_argument( + "--require-chrony", action="store_true", help="stop instead of starting before the clocks have settled" + ) + parser.add_argument("--host-lan-ip", help="this PC's address toward the Orin (default: found from the route)") parser.add_argument( "--mock", action="store_true", @@ -552,18 +606,14 @@ def main(argv: Sequence[str] | None = None) -> int: check_station_cameras(args.station_config, args.capture_camera) basis = MOCK_TIMESTAMP_BASIS if args.mock else TIMESTAMP_BASIS # One loaded profile is the source for the sender, the receiver, the bridge and the descriptor. - sender_config = load_sender_config( - args.sender_config, args.fps, args.bitrate_mbps, args.clock_port, args.sender_port - ) + sender_config = load_sender_config(args.sender_config, args.fps, args.bitrate_mbps, args.sender_port) sender = sender_settings(sender_config) - # The bridge's sender-clock address must be a literal IPv4 address, not a name. + # The bridge identifies the Orin's chrony by the reference id of its IPv4 address. if args.orin: try: ipaddress.IPv4Address(orin_host) except ValueError as error: - raise StartupError( - f"--orin needs an IPv4 address for the sender clock, got {orin_host!r}" - ) from error + raise StartupError(f"--orin needs an IPv4 address for the Orin, got {orin_host!r}") from error (workdir / "sender.yaml").write_text(yaml.safe_dump(sender_config, sort_keys=False)) (workdir / "streams.yaml").write_text( streams_yaml( @@ -580,7 +630,7 @@ def main(argv: Sequence[str] | None = None) -> int: else: print( f"sender config {args.sender_config}: {sender.fps} fps, {sender.bitrate_mbps} Mbit/s, " - f"timing extension on, clock port {sender.clock_port}, release {args.release}" + f"release {args.release}" ) if args.orin: stop_at = time.strftime("%H:%M:%S", time.localtime(time.time() + args.seconds)) @@ -596,12 +646,26 @@ def main(argv: Sequence[str] | None = None) -> int: token_file.chmod(0o600) python = sys.executable pythonpath = os.pathsep.join( - str(EGOCAP_ROOT / sub) for sub in ("backend", "recording", "capture_server", ".") + [ + *(str(EGOCAP_ROOT / sub) for sub in ("backend", "recording", "capture_server", ".")), + # CaptureServer imports the Manus calibration worker, which is not part of this folder. + str(REPO_ROOT / "src/plugins/manus_calibration/python"), + ] ) + if args.orin and args.receiver == default_receiver_command(): + ensure_receiver_environment() + # Time sync is set up after the page is up (below), so a slow or failed install never holds the preview. + if args.orin: + tool = load_timesync_tool(args.timesync_tool) + lan_ip = args.host_lan_ip or tool.lan_address_toward(orin_host) # 1. The Orin sender: deploy a release (never activating a service), then run it for a bounded time. if args.orin: deploy = [python, args.deploy_script, args.orin] + # nvargus-daemon returns at boot and holds the cameras, which SIPL capture needs free; a sender left + # by an earlier run outlives its ssh session. + subprocess.run([*deploy, "stop"], check=False) + subprocess.run([*deploy, "stop-daemon"], check=True) if not args.skip_deploy: print("deploying the sender to", args.orin) subprocess.run( @@ -698,9 +762,86 @@ def main(argv: Sequence[str] | None = None) -> int: f"{sender.stream_id}={track}:2", "--fps", str(sender.fps), - *bridge_timing_arguments(sender, args.mock, orin_host), + *bridge_timing_arguments( + sender, + args.mock, + orin_host, + tool.reference_id_for(orin_host) if args.orin else None, + args.chrony_cmd_port, + ), ], ) + if args.orin: + # Every step is bounded: the page and preview do not need qualified times, and a slow Orin or a missing + # package must not keep them down. The bridge refuses unqualified times, so a take cannot start until + # time sync is up. + print("setting up time sync", flush=True) + settled = False + try: + provisioned = ( + subprocess.run([python, str(args.timesync_tool), "ensure-local"], check=False).returncode == 0 + ) + except OSError: + provisioned = False + if provisioned: + # An unprivileged chronyd on this PC measures the Orin's clock and never adjusts this PC's. Nothing + # has to reach this PC from the Orin, so a firewall here does not matter. + stack.start( + "timesync-follower", + [ + python, + str(args.timesync_tool), + "follow", + "--server", + orin_host, + "--cmd-port", + str(args.chrony_cmd_port), + "--workdir", + str(workdir / "chrony"), + ], + ) + print("configuring the Orin's chrony to serve its clock to", lan_ip, flush=True) + try: + configured = ( + subprocess.run( + [ + python, + str(args.timesync_tool), + "configure-remote", + args.orin, + "--host-lan-ip", + lan_ip, + ], + check=False, + timeout=90, + ).returncode + == 0 + ) + except subprocess.TimeoutExpired: + configured = False + settled = ( + configured + and subprocess.run( + [ + python, + str(args.timesync_tool), + "status", + "--remote", + orin_host, + "--cmd-port", + str(args.chrony_cmd_port), + "--wait", + str(args.chrony_wait), + ], + check=False, + ).returncode + == 0 + ) + if not settled: + message = "the clocks have not settled, so the bridge refuses capture times until they do" + if args.require_chrony: + raise StartupError(message) + print("WARNING:", message) url = f"http://127.0.0.1:{args.backend_port}/" token = control_token(backend.log) print(f"\nEgo Capture is up at {url}\nrecording control token: {token}\nlogs: {workdir}") @@ -726,6 +867,9 @@ def main(argv: Sequence[str] | None = None) -> int: return 1 finally: stack.stop_all() + if args.orin and args.deploy_script: + # Ending the local ssh does not end the remote sender; ask it to stop so it frees the cameras. + subprocess.run([sys.executable, args.deploy_script, args.orin, "stop"], check=False) if __name__ == "__main__": From ef131b50816dc1d4dcd7918a3a9e4897863475dc Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 10:59:56 -0500 Subject: [PATCH 02/13] egocap: show no latency for a frame without a capture time The browser SDK reports a frame that carries only a frame id as a stamp of 0, and the page turned that into the publisher's monotonic uptime. Treat a stamp of 0 or below as no stamp. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/egocap/frontend/src/displayLatency.test.ts | 6 ++++++ examples/egocap/frontend/src/displayLatency.ts | 2 ++ 2 files changed, 8 insertions(+) diff --git a/examples/egocap/frontend/src/displayLatency.test.ts b/examples/egocap/frontend/src/displayLatency.test.ts index 5baec9f676..da6e358f7c 100644 --- a/examples/egocap/frontend/src/displayLatency.test.ts +++ b/examples/egocap/frontend/src/displayLatency.test.ts @@ -28,6 +28,12 @@ describe("latencySample", () => { expect(latencySample(Number.NaN, { userTimestamp: 1_000_000_000n }, mapping())).toBeNull(); }); + it("is null for a stamp of zero, which the browser SDK reports for a frame that carries no capture time", () => { + // Without this a frame with only a frame id gives the publisher's whole monotonic uptime as its latency. + expect(latencySample(997_072, { userTimestamp: 0n }, mapping())).toBeNull(); + expect(latencySample(997_072, { userTimestamp: -1n }, mapping())).toBeNull(); + }); + it("is null when display precedes capture by more than the uncertainty, but keeps a small negative", () => { expect(latencySample(996_990, { userTimestamp: 1_000_000_000n }, mapping())).toBeNull(); // -10 ms expect(latencySample(996_999, { userTimestamp: 1_000_000_000n }, mapping())?.ms).toBeCloseTo(-1, 6); diff --git a/examples/egocap/frontend/src/displayLatency.ts b/examples/egocap/frontend/src/displayLatency.ts index 352ad0f00e..ea756effad 100644 --- a/examples/egocap/frontend/src/displayLatency.ts +++ b/examples/egocap/frontend/src/displayLatency.ts @@ -35,6 +35,8 @@ export function latencySample( mapping: ClockMapping | null, ): LatencySample | null { if (stamp === undefined || mapping === null || !Number.isFinite(expectedDisplayTime)) return null; + // The browser SDK reports a frame that carries no capture time (only a frame id) as a stamp of 0, not as absent. + if (stamp.userTimestamp <= 0n) return null; const displayRemote = toRemoteNs(mapping, BigInt(Math.round(expectedDisplayTime * 1e6))); const ms = Number(displayRemote - stamp.userTimestamp * 1000n) / 1e6; const uncertaintyMs = Number(mapping.uncertaintyNs) / 1e6; From ff92e844c161ba8a3f83a6317389ec8c11fa1bf4 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:16:55 -0500 Subject: [PATCH 03/13] egocap: refuse a take whose first frame has no capture time; clarify the latency label A take records the publisher's capture time, so prepare() now refuses a stream whose first frame carries none before storage opens. The latency tooltip says its +- covers only the browser's clock mapping, not the sensor-to-publisher conversion or the meaning of the sensor start-of-frame. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../backend/egocap_web/livekit_recording.py | 5 ++ .../egocap/frontend/src/LiveKitCameraTile.tsx | 2 +- .../backend/test_livekit_recording_source.py | 20 ++++- .../backend/test_recording_session_livekit.py | 75 +++++++++++++++++++ 4 files changed, 99 insertions(+), 3 deletions(-) diff --git a/examples/egocap/backend/egocap_web/livekit_recording.py b/examples/egocap/backend/egocap_web/livekit_recording.py index 3a236b5ed4..d93632987d 100644 --- a/examples/egocap/backend/egocap_web/livekit_recording.py +++ b/examples/egocap/backend/egocap_web/livekit_recording.py @@ -386,6 +386,11 @@ async def prepare(self) -> None: ) if frame is None: raise RuntimeError(f"{stream.name}: the LiveKit track was lost before recording started") + # A take records the publisher's capture time, so a source that sends none cannot start one. + if frame.user_timestamp_us is None: + raise RuntimeError( + f"{stream.name}: the first frame carries no capture time, so it cannot be recorded" + ) if frame.image.shape[:2] != (stream.height, stream.width): raise RuntimeError( f"{stream.name}: frames are {frame.image.shape[1]}x{frame.image.shape[0]}, " diff --git a/examples/egocap/frontend/src/LiveKitCameraTile.tsx b/examples/egocap/frontend/src/LiveKitCameraTile.tsx index 72c5c8820c..bf3cd2a1e3 100644 --- a/examples/egocap/frontend/src/LiveKitCameraTile.tsx +++ b/examples/egocap/frontend/src/LiveKitCameraTile.tsx @@ -209,7 +209,7 @@ export function LiveKitCameraTile({ title={ latencyLabel === NO_LATENCY ? "No latency for the displayed frame: it needs the publisher's per-frame stamp (matched by RTP timestamp), a current clock mapping to the publisher, and a stream whose timestamp is on the publisher's monotonic clock." - : `The publisher's frame stamp (frame-ready or sensor capture, per the stream) to the frame's expected display on this page, including receive, decode and render (${latencyDetail}); ± is half the clock-exchange round trip, the drift the exchange's age allows, and this browser's timer resolution` + : `The publisher's frame stamp (frame-ready or sensor capture, per the stream) to the frame's expected display on this page, including receive, decode and render (${latencyDetail}); ± is only this browser's clock mapping to the publisher (half the clock-exchange round trip, the drift the exchange's age allows, and this browser's timer resolution). It does not include the sensor-to-publisher clock conversion or how closely the sensor's start-of-frame marks the exposure` } > {latencyLabel} diff --git a/examples/egocap/tests/backend/test_livekit_recording_source.py b/examples/egocap/tests/backend/test_livekit_recording_source.py index b12e7576b6..d82cff1abf 100644 --- a/examples/egocap/tests/backend/test_livekit_recording_source.py +++ b/examples/egocap/tests/backend/test_livekit_recording_source.py @@ -58,14 +58,14 @@ def jpeg_camera(stream_id: int = 1) -> p.CameraStream: ) -def frame(stream_id: int, *, width=6, height=4) -> ReceivedFrame: +def frame(stream_id: int, *, width=6, height=4, stamped=True) -> ReceivedFrame: return ReceivedFrame( stream_id=stream_id, image=np.zeros((height, width, 3), dtype=np.uint8), # Captured just now on this host's clock (the fake mapping is the identity): passes the frame-age check. receive_time_ns=time.monotonic_ns(), transport_timestamp_us=1, - user_timestamp_us=time.monotonic_ns() // 1000, + user_timestamp_us=time.monotonic_ns() // 1000 if stamped else None, frame_id=None, user_data=None, ) @@ -225,6 +225,22 @@ async def test_a_wrong_sized_first_frame_fails_naming_the_stream_and_leaves_the_ assert FakeReceiver.instances[0].stops == 1 +async def test_a_first_frame_without_a_capture_time_fails_prepare_and_leaves_the_room(): + streams = [livekit_stream(16)] + source = make_source(streams) + with pytest.raises(RuntimeError, match=r"cam16: the first frame carries no capture time"): + await prepare_with_frames(source, streams, stamped=False) + assert FakeReceiver.instances[0].stops == 1 + + +async def test_a_first_frame_with_a_capture_time_prepares(): + streams = [livekit_stream(16)] + source = make_source(streams) + await prepare_with_frames(source, streams, stamped=True) + assert FakeReceiver.instances[0].stops == 0 + await source.stop() + + async def test_a_lost_track_fails_naming_the_stream_and_leaves_the_room(): streams = [livekit_stream(16)] source = make_source(streams) diff --git a/examples/egocap/tests/backend/test_recording_session_livekit.py b/examples/egocap/tests/backend/test_recording_session_livekit.py index 8ea2d5ce32..e9c64f459a 100644 --- a/examples/egocap/tests/backend/test_recording_session_livekit.py +++ b/examples/egocap/tests/backend/test_recording_session_livekit.py @@ -10,8 +10,12 @@ import asyncio from pathlib import Path +import numpy as np import pytest +from capture_server import protocol as p +from egocap_web.livekit_receiver import ReceivedFrame, StreamCounters +from egocap_web.livekit_recording import LiveKitRecordingFeed from egocap_web.recording_session import CoordinatorBusyError, RecordingSessionCoordinator from recording_service import ( Disposition, @@ -254,3 +258,74 @@ async def test_an_interrupted_takes_evidence_does_not_describe_the_next_take(tmp assert calls == [(two.episode_id, {"event_time": tracking})] await coordinator.close() + + +class _UnstampedReceiver: + """Delivers one frame with no publisher timestamp, then stays quiet.""" + + class _Clock: + discontinuity = False + + def mapping(self): + return None + + def __init__(self) -> None: + self.clock_sync = self._Clock() + self.counters = {16: StreamCounters()} + self._sent = False + + async def next_frame_or_lost(self, stream_id, timeout=None): + if self._sent: + await asyncio.Event().wait() + self._sent = True + return ReceivedFrame( + stream_id=stream_id, + image=np.zeros((4, 6, 3), dtype=np.uint8), + receive_time_ns=0, + transport_timestamp_us=1, + user_timestamp_us=None, + frame_id=None, + user_data=None, + ) + + +class _RealFeedSource(FakeSource): + """A source whose ``start`` runs the real feed, so its own failure reason reaches the coordinator.""" + + def start(self, ingress, on_failure) -> None: + super().start(ingress, on_failure) + stream = p.LiveKitCameraStream( + stream_id=16, + name="head", + url="ws://lk", + room="r", + publisher="o", + track="t", + timestamp_basis=p.LiveKitTimestampBasis.SENSOR_CAPTURE_MONOTONIC_US, + ) + self.feed = LiveKitRecordingFeed( + _UnstampedReceiver(), [stream], ingress, {16: "head"}, on_failure=on_failure + ) + self.feed.start() + + async def stop(self) -> None: + await super().stop() + await self.feed.stop() + + +async def test_a_frame_without_a_publisher_timestamp_ends_the_take_and_leaves_it_recoverable(tmp_path): + source = _RealFeedSource([]) + coordinator = make_coordinator(tmp_path, source) + await coordinator.open() + coordinator.set_task("t") + await coordinator.start_episode() + for _ in range(300): + snap = coordinator.snapshot() + if snap.service.phase is RecordingPhase.RECOVERY_REQUIRED and snap.active_operation is None: + break + await asyncio.sleep(0.01) + snap = coordinator.snapshot() + assert snap.service.phase is RecordingPhase.RECOVERY_REQUIRED + assert "a frame arrived without the publisher's frame timestamp" in (coordinator._last_issue or "") + assert snap.service.last_result is None or snap.service.last_result.disposition is not Disposition.KEPT + await coordinator.close() From 1f7c3e7cf51b1fab2f597199529c07a90df9dbd5 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:18:26 -0500 Subject: [PATCH 04/13] egocap: declare the bridge's shrunken size in the stream descriptor The launcher starts the bridge with a 2x shrink (--stream NAME=TRACK:2) but wrote the sensor size into the LiveKit streams file, so the recorder refused every take: it checks the first frame against the declared size. Derive both from one BRIDGE_DOWNSCALE constant; the receiver keeps the full sensor size. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../egocap/tests/tools/test_start_system.py | 39 +++++++++++++++++++ examples/egocap/tools/start_system.py | 27 +++++++++++-- 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index 0918834b8d..73bdf000a4 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -125,6 +125,45 @@ def fail_start(*_, **__): assert "frontend is not built" in capsys.readouterr().err +def test_descriptor_declares_the_bridge_downscaled_size_and_the_receiver_the_sensor_size( + monkeypatch, capsys, tmp_path: Path +): + commands: dict[str, list[str]] = {} + + def record_start(self, name, command, *_, **__): + commands[name] = list(command) + if name == "bridge": + # The bridge is the last thing the files under test feed; stop the run once it is launched. + raise ss.StartupError("stop after the bridge") + + workdir = tmp_path / "work" + workdir.mkdir() + index = tmp_path / "dist" / "index.html" + index.parent.mkdir() + index.write_text("") + timesync = tmp_path / "timesync_tool.py" + timesync.write_text("def reference_id_for(host):\n return 'REFID'\n") + monkeypatch.setattr(signal, "signal", lambda *_: None) + monkeypatch.setattr(ss.tempfile, "mkdtemp", lambda **_: str(workdir)) + monkeypatch.setattr(ss, "FRONTEND_INDEX", index) + monkeypatch.setattr(ss.Stack, "start", record_start) + monkeypatch.setattr(ss.Stack, "wait_until", lambda *_, **__: None) + monkeypatch.setattr(ss.Stack, "stop_all", lambda *_: None) + monkeypatch.setattr(ss.subprocess, "run", lambda *_, **__: None) + argv = ["--orin", "user@10.0.0.5", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] + argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "10.0.0.2"] + assert ss.main(argv) == 1 + assert "stop after the bridge" in capsys.readouterr().err + + declared = yaml.safe_load((workdir / "streams.yaml").read_text())["livekit"]["streams"][0] + received = yaml.safe_load((workdir / "receiver.yaml").read_text())["streams"][0] + assert (declared["width"], declared["height"]) == (2560 // ss.BRIDGE_DOWNSCALE, 1984 // ss.BRIDGE_DOWNSCALE) + assert (declared["width"], declared["height"]) == (1280, 992) + assert (received["width"], received["height"]) == (2560, 1984) + bridge = commands["bridge"] + assert bridge[bridge.index("--stream") + 1].endswith(f":{ss.BRIDGE_DOWNSCALE}") + + def test_default_profile_gives_the_head_camera_settings(): config = ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG) assert ss.sender_settings(config) == ss.SenderSettings( diff --git a/examples/egocap/tools/start_system.py b/examples/egocap/tools/start_system.py index b9e81a9c7f..426f41be8b 100644 --- a/examples/egocap/tools/start_system.py +++ b/examples/egocap/tools/start_system.py @@ -64,6 +64,9 @@ STOP_SIGNALS = tuple(sig for sig in (signal.SIGTERM, getattr(signal, "SIGHUP", None)) if sig is not None) # The mock sender has no clock responder, so the bridge stamps the frame-ready time on its own clock instead. MOCK_TIMESTAMP_BASIS = "monotonic_us" +# The bridge shrinks the picture by this integer factor before encoding, so the descriptor declares the shrunken +# size: the recorder refuses a take whose frames differ from the declared one. +BRIDGE_DOWNSCALE = 2 # The backend serves the page from this build output and mounts nothing if it is missing. FRONTEND_INDEX = EGOCAP_ROOT / "frontend" / "dist" / "index.html" @@ -518,9 +521,17 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "--sender-config", type=Path, default=DEFAULT_SENDER_CONFIG, help="sender profile uploaded to the Orin" ) parser.add_argument("--station-config", type=Path, default=DEFAULT_STATION_CONFIG) - parser.add_argument("--width", type=int, default=2560, help="sensor width the receiver and descriptor expect") parser.add_argument( - "--height", type=int, default=1984, help="sensor height the receiver and descriptor expect" + "--width", + type=int, + default=2560, + help="sensor width; the receiver expects it and the descriptor declares it divided by the bridge shrink", + ) + parser.add_argument( + "--height", + type=int, + default=1984, + help="sensor height; the receiver expects it and the descriptor declares it divided by the bridge shrink", ) parser.add_argument("--fps", type=int, help="override the profile's output.fps") parser.add_argument("--bitrate-mbps", type=int, help="override the profile's HEVC bitrate") @@ -617,7 +628,15 @@ def main(argv: Sequence[str] | None = None) -> int: (workdir / "sender.yaml").write_text(yaml.safe_dump(sender_config, sort_keys=False)) (workdir / "streams.yaml").write_text( streams_yaml( - livekit_url, room, publisher, track, args.width, args.height, sender.fps, basis, sender.stream_id + livekit_url, + room, + publisher, + track, + args.width // BRIDGE_DOWNSCALE, + args.height // BRIDGE_DOWNSCALE, + sender.fps, + basis, + sender.stream_id, ) ) (workdir / "receiver.yaml").write_text( @@ -759,7 +778,7 @@ def main(argv: Sequence[str] | None = None) -> int: "--socket", ipc_socket, "--stream", - f"{sender.stream_id}={track}:2", + f"{sender.stream_id}={track}:{BRIDGE_DOWNSCALE}", "--fps", str(sender.fps), *bridge_timing_arguments( From d0c538cce894db655e6b46f37f485112716e4823 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:29:53 -0500 Subject: [PATCH 05/13] egocap: start the receiver from the environment the first-run setup just made Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/egocap/tools/start_system.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/examples/egocap/tools/start_system.py b/examples/egocap/tools/start_system.py index 426f41be8b..e25cb55a4a 100644 --- a/examples/egocap/tools/start_system.py +++ b/examples/egocap/tools/start_system.py @@ -173,6 +173,16 @@ def _section(parent: dict, key: str, path: Path) -> dict: return value +def _section(parent: dict, key: str, path: Path) -> dict: + """The mapping under ``parent[key]``, created when absent or null; another type is a startup error.""" + value = parent.get(key) + if value is None: + value = parent[key] = {} + if not isinstance(value, dict): + raise StartupError(f"{path}: {key} must be a mapping") + return value + + def load_sender_config( path: Path, fps: int | None = None, @@ -674,6 +684,9 @@ def main(argv: Sequence[str] | None = None) -> int: if args.orin and args.receiver == default_receiver_command(): ensure_receiver_environment() + # The default was resolved before this setup created the receiver's environment, so resolve it again. + # An explicit --receiver or EGOCAP_RECEIVER is not the default and is never replaced. + args.receiver = default_receiver_command() # Time sync is set up after the page is up (below), so a slow or failed install never holds the preview. if args.orin: tool = load_timesync_tool(args.timesync_tool) From 258cc052b2fd70c3d52e05f915922626b08712a5 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:30:03 -0500 Subject: [PATCH 06/13] egocap: test the first-run receiver command, the clean sender exit and malformed profile sections Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../egocap/tests/tools/test_start_system.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index 73bdf000a4..c733918447 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -727,3 +727,133 @@ def test_ensure_receiver_environment_runs_setup_only_when_something_is_missing(m (receiver_dir / "codec/_camera_receiver_codec.cpython-313-x86_64-linux-gnu.so").write_text("") ss.ensure_receiver_environment() assert calls == [] + + +class _FakeProcess: + def __init__(self, returncode: int | None) -> None: + self.returncode = returncode + self.pid = 0 + + def poll(self): + return self.returncode + + +def _run_main_with_stub_children(monkeypatch, tmp_path: Path, extra_argv: list[str], exit_codes=None): + """Run main() against a REPO_ROOT without a receiver venv; returns (exit code, argv per child, run calls). + + ``exit_codes`` maps a child name to the code its poll() reports; other children keep running. + """ + exit_codes = exit_codes or {} + started: dict[str, list[str]] = {} + runs: list = [] + workdir = tmp_path / "work" + workdir.mkdir() + index = tmp_path / "dist" / "index.html" + index.parent.mkdir() + index.write_text("") + timesync = tmp_path / "timesync_tool.py" + timesync.write_text("def reference_id_for(host):\n return 'REFID'\n") + receiver_dir = tmp_path / "repo/examples/camera_streamer/camera_receiver" + receiver_dir.mkdir(parents=True) + + def fake_start(self, name, command, *_, **__): + started[name] = list(command) + log = self.workdir / f"{name}.log" + log.write_text("") + child = ss.Managed(name, _FakeProcess(exit_codes.get(name)), log) + self.children.append(child) + return child + + class _Done: + returncode = 0 + + monkeypatch.delenv("EGOCAP_RECEIVER", raising=False) + monkeypatch.setattr(signal, "signal", lambda *_: None) + monkeypatch.setattr(ss, "REPO_ROOT", tmp_path / "repo") + monkeypatch.setattr(ss.tempfile, "mkdtemp", lambda **_: str(workdir)) + monkeypatch.setattr(ss, "FRONTEND_INDEX", index) + monkeypatch.setattr(ss.Stack, "start", fake_start) + monkeypatch.setattr(ss.Stack, "wait_until", lambda *_, **__: None) + monkeypatch.setattr(ss.Stack, "stop_all", lambda *_: None) + monkeypatch.setattr(ss.subprocess, "run", lambda cmd, **__: runs.append(list(cmd)) or _Done()) + monkeypatch.setattr(ss.time, "sleep", lambda *_: None) + argv = ["--orin", "user@10.0.0.5", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] + argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "10.0.0.2", *extra_argv] + return ss.main(argv), started, runs, receiver_dir + + +def test_first_run_starts_the_receiver_with_the_venv_python_that_setup_created(monkeypatch, tmp_path: Path): + venv_python = tmp_path / "repo/examples/camera_streamer/camera_receiver/.venv/bin/python" + setups = [] + + def fake_setup(): + setups.append(True) + venv_python.parent.mkdir(parents=True) + venv_python.write_text("") + + monkeypatch.setattr(ss, "ensure_receiver_environment", fake_setup) + # The sender exits cleanly as soon as the stack is up, which ends the run. + code, started, _, _ = _run_main_with_stub_children(monkeypatch, tmp_path, [], {"sender": 0}) + assert code == 0 + assert setups == [True] + assert started["receiver"][0] == str(venv_python) + assert "python3" not in started["receiver"][0] + + +def test_explicit_receiver_is_passed_through_without_setup(monkeypatch, tmp_path: Path): + def fail_setup(): + raise AssertionError("setup ran for an explicit receiver") + + monkeypatch.setattr(ss, "ensure_receiver_environment", fail_setup) + code, started, _, _ = _run_main_with_stub_children( + monkeypatch, tmp_path, ["--receiver", "custom-python recv.py"], {"sender": 0} + ) + assert code == 0 + assert started["receiver"][:2] == ["custom-python", "recv.py"] + + +@pytest.mark.parametrize( + ("exit_codes", "expected"), + [ + ({"sender": 0}, 0), + ({"sender": 3}, 1), + ({"bridge": 0}, 1), + ({"receiver": 0}, 1), + ], +) +def test_only_a_clean_sender_exit_ends_the_run_successfully(monkeypatch, tmp_path: Path, exit_codes, expected): + monkeypatch.setattr(ss, "ensure_receiver_environment", lambda: None) + code, _, _, _ = _run_main_with_stub_children(monkeypatch, tmp_path, ["--receiver", "r.py"], exit_codes) + assert code == expected + + +def _config_file(tmp_path: Path, text: str) -> Path: + path = tmp_path / "sender.yaml" + path.write_text(text) + return path + + +@pytest.mark.parametrize( + "text", + [ + "network: null\nstreams:\n - {id: head, output: {fps: 30, bitrate_mbps: 4}}\n", + "streams:\n - {id: head, output: null}\n", + ], +) +def test_load_sender_config_accepts_a_null_section(tmp_path: Path, text: str): + config = ss.load_sender_config(_config_file(tmp_path, text), fps=20, sender_port=5001) + assert config["network"]["port"] == 5001 + assert config["streams"][0]["output"]["fps"] == 20 + + +@pytest.mark.parametrize( + "text", + [ + "network: 5\nstreams:\n - {id: head}\n", + "streams:\n - {id: head, output: [1]}\n", + "streams: [3]\n", + ], +) +def test_load_sender_config_rejects_a_section_that_is_not_a_mapping(tmp_path: Path, text: str): + with pytest.raises(ss.StartupError): + ss.load_sender_config(_config_file(tmp_path, text)) From 79cc6e6b7632f1dc242eed8ac24ff2f2fdd793eb Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:41:42 -0500 Subject: [PATCH 07/13] egocap: fit the launcher tests and code to the chrony setup after the rebase Drop the review fixes that the base launcher now carries, and update the launcher tests for the chrony flags, the extra deploy and timesync steps and the timesync follower child. Add a test for the chrony wiring. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../egocap/tests/tools/test_start_system.py | 102 +++++++----------- examples/egocap/tools/start_system.py | 10 -- 2 files changed, 39 insertions(+), 73 deletions(-) diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index c733918447..6c6d06631b 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -28,6 +28,9 @@ source["name"] for source in next(iter(yaml.safe_load(EXAMPLE_STATION.read_text())["stations"].values()))["video_sources"] ] +TIMESYNC_STUB = ( + "def lan_address_toward(host):\n return '10.0.0.2'\n\n\ndef reference_id_for(host):\n return 'REFID'\n" +) CAMERA_ARGS = [arg for name in EXAMPLE_CAMERAS for arg in ("--capture-camera", f"{name}=dev0")] @@ -142,7 +145,7 @@ def record_start(self, name, command, *_, **__): index.parent.mkdir() index.write_text("") timesync = tmp_path / "timesync_tool.py" - timesync.write_text("def reference_id_for(host):\n return 'REFID'\n") + timesync.write_text(TIMESYNC_STUB, encoding="utf-8") monkeypatch.setattr(signal, "signal", lambda *_: None) monkeypatch.setattr(ss.tempfile, "mkdtemp", lambda **_: str(workdir)) monkeypatch.setattr(ss, "FRONTEND_INDEX", index) @@ -151,7 +154,7 @@ def record_start(self, name, command, *_, **__): monkeypatch.setattr(ss.Stack, "stop_all", lambda *_: None) monkeypatch.setattr(ss.subprocess, "run", lambda *_, **__: None) argv = ["--orin", "user@10.0.0.5", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] - argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "10.0.0.2"] + argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "10.0.0.2", *CAMERA_ARGS] assert ss.main(argv) == 1 assert "stop after the bridge" in capsys.readouterr().err @@ -229,12 +232,12 @@ def null_it(config): else: config["streams"][0]["output"] = None - config = ss.load_sender_config(_profile_with(tmp_path, null_it), 30, 12, 5700, 5701) + config = ss.load_sender_config(_profile_with(tmp_path, null_it), 30, 12, 5701) output = config["streams"][0]["output"] assert output["fps"] == 30 and output["bitrate_mbps"] == 12 - assert config["network"]["clock_port"] == 5700 and config["network"]["port"] == 5701 + assert config["network"]["port"] == 5701 if section == "network": - assert config["network"] == {"clock_port": 5700, "port": 5701} + assert config["network"] == {"port": 5701} else: assert output == {"fps": 30, "bitrate_mbps": 12} @@ -442,14 +445,15 @@ def test_mock_sender_command_follows_the_profile(): def test_bridge_timing_arguments_for_mock_and_real_sender(): sipl = ss.sender_settings(ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG)) - assert ss.bridge_timing_arguments(sipl, True, None) == ["--timestamp-basis", "monotonic_us"] - real = " ".join(ss.bridge_timing_arguments(sipl, False, "192.0.2.10")) + assert ss.bridge_timing_arguments(sipl, True, None, None, 12301) == ["--timestamp-basis", "monotonic_us"] + real = " ".join(ss.bridge_timing_arguments(sipl, False, "192.0.2.10", "C000020A", 12301)) assert "--timestamp-basis capture_monotonic_us" in real - assert "--sender-clock 192.0.2.10:5611" in real + assert "--chrony-sender 192.0.2.10 --chrony-sender-refid C000020A --chrony-pc-port 12301" in real + assert "--sender-clock" not in real assert "--source-full-range" in real argus_config = {**ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG), "capture_backend": "argus"} argus = ss.sender_settings(argus_config) - assert "--source-full-range" not in ss.bridge_timing_arguments(argus, False, "192.0.2.10") + assert "--source-full-range" not in ss.bridge_timing_arguments(argus, False, "192.0.2.10", "C000020A", 12301) def test_main_refuses_orin_together_with_mock(monkeypatch, capsys, tmp_path: Path): @@ -514,7 +518,17 @@ def interrupt(_seconds): monkeypatch.setattr(ss.Stack, "wait_until", lambda *_, **__: None) # The fake children are not real processes, so there is nothing to signal. monkeypatch.setattr(ss.Stack, "stop_all", lambda self: None) - monkeypatch.setattr(ss.subprocess, "run", lambda command, **_: deploys.append(list(command))) + # Every deploy, stop and timesync step is recorded and reports success. + monkeypatch.setattr( + ss.subprocess, "run", lambda command, **_: deploys.append(list(command)) or SimpleNamespace(returncode=0) + ) + monkeypatch.setattr( + ss, + "load_timesync_tool", + lambda _path: SimpleNamespace( + lan_address_toward=lambda host: "192.0.2.1", reference_id_for=lambda host: "C000020A" + ), + ) monkeypatch.setattr(ss.time, "sleep", interrupt) return SimpleNamespace(children=children, deploys=deploys, workdir=workdir, installed=installed) @@ -527,7 +541,7 @@ def test_main_starts_the_orin_chain_in_order(recorded_main): argv = ["--orin", "user@192.0.2.10", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] assert ss.main([*argv, *CAMERA_ARGS]) == 0 names = [c.name for c in recorded_main.children] - assert names == ["sender", "livekit", "capture-server", "backend", "receiver", "bridge"] + assert names == ["sender", "livekit", "capture-server", "backend", "receiver", "bridge", "timesync-follower"] capture = _command(recorded_main, "capture-server") assert "--camera" in capture assert all(f"{name}=dev0" in capture for name in EXAMPLE_CAMERAS) @@ -535,11 +549,20 @@ def test_main_starts_the_orin_chain_in_order(recorded_main): assert "--mock" not in capture bridge = _command(recorded_main, "bridge") assert "--timestamp-basis capture_monotonic_us" in bridge - assert "--sender-clock 192.0.2.10:5611" in bridge + assert "--sender-clock" not in bridge assert "--source-full-range" in bridge assert "--config" in _command(recorded_main, "receiver") - assert len(recorded_main.deploys) == 1 - assert "--with-sipl" in recorded_main.deploys[0] + release_deploys = [d for d in recorded_main.deploys if "--with-sipl" in d] + assert len(release_deploys) == 1 + assert release_deploys[0][1:3] == ["x", "user@192.0.2.10"] + + +def test_main_wires_chrony_into_the_bridge_and_starts_the_follower_after_it(recorded_main): + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS]) == 0 + names = [c.name for c in recorded_main.children] + assert names.index("timesync-follower") > names.index("bridge") + bridge = _command(recorded_main, "bridge") + assert "--chrony-sender 192.0.2.10 --chrony-sender-refid C000020A --chrony-pc-port 12301" in bridge def test_main_starts_the_mock_chain_in_order(recorded_main, tmp_path: Path): @@ -752,7 +775,7 @@ def _run_main_with_stub_children(monkeypatch, tmp_path: Path, extra_argv: list[s index.parent.mkdir() index.write_text("") timesync = tmp_path / "timesync_tool.py" - timesync.write_text("def reference_id_for(host):\n return 'REFID'\n") + timesync.write_text(TIMESYNC_STUB, encoding="utf-8") receiver_dir = tmp_path / "repo/examples/camera_streamer/camera_receiver" receiver_dir.mkdir(parents=True) @@ -778,7 +801,7 @@ class _Done: monkeypatch.setattr(ss.subprocess, "run", lambda cmd, **__: runs.append(list(cmd)) or _Done()) monkeypatch.setattr(ss.time, "sleep", lambda *_: None) argv = ["--orin", "user@10.0.0.5", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] - argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "10.0.0.2", *extra_argv] + argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "10.0.0.2", *CAMERA_ARGS, *extra_argv] return ss.main(argv), started, runs, receiver_dir @@ -810,50 +833,3 @@ def fail_setup(): ) assert code == 0 assert started["receiver"][:2] == ["custom-python", "recv.py"] - - -@pytest.mark.parametrize( - ("exit_codes", "expected"), - [ - ({"sender": 0}, 0), - ({"sender": 3}, 1), - ({"bridge": 0}, 1), - ({"receiver": 0}, 1), - ], -) -def test_only_a_clean_sender_exit_ends_the_run_successfully(monkeypatch, tmp_path: Path, exit_codes, expected): - monkeypatch.setattr(ss, "ensure_receiver_environment", lambda: None) - code, _, _, _ = _run_main_with_stub_children(monkeypatch, tmp_path, ["--receiver", "r.py"], exit_codes) - assert code == expected - - -def _config_file(tmp_path: Path, text: str) -> Path: - path = tmp_path / "sender.yaml" - path.write_text(text) - return path - - -@pytest.mark.parametrize( - "text", - [ - "network: null\nstreams:\n - {id: head, output: {fps: 30, bitrate_mbps: 4}}\n", - "streams:\n - {id: head, output: null}\n", - ], -) -def test_load_sender_config_accepts_a_null_section(tmp_path: Path, text: str): - config = ss.load_sender_config(_config_file(tmp_path, text), fps=20, sender_port=5001) - assert config["network"]["port"] == 5001 - assert config["streams"][0]["output"]["fps"] == 20 - - -@pytest.mark.parametrize( - "text", - [ - "network: 5\nstreams:\n - {id: head}\n", - "streams:\n - {id: head, output: [1]}\n", - "streams: [3]\n", - ], -) -def test_load_sender_config_rejects_a_section_that_is_not_a_mapping(tmp_path: Path, text: str): - with pytest.raises(ss.StartupError): - ss.load_sender_config(_config_file(tmp_path, text)) diff --git a/examples/egocap/tools/start_system.py b/examples/egocap/tools/start_system.py index e25cb55a4a..1be6195c88 100644 --- a/examples/egocap/tools/start_system.py +++ b/examples/egocap/tools/start_system.py @@ -173,16 +173,6 @@ def _section(parent: dict, key: str, path: Path) -> dict: return value -def _section(parent: dict, key: str, path: Path) -> dict: - """The mapping under ``parent[key]``, created when absent or null; another type is a startup error.""" - value = parent.get(key) - if value is None: - value = parent[key] = {} - if not isinstance(value, dict): - raise StartupError(f"{path}: {key} must be a mapping") - return value - - def load_sender_config( path: Path, fps: int | None = None, From dc77b6ce45606aea7bf24e31828f301a9015137d Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 18:26:05 -0500 Subject: [PATCH 08/13] egocap: name the sender-profile clock test after the behaviour it checks Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/egocap/tests/tools/test_start_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index 6c6d06631b..686c714372 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -700,7 +700,7 @@ def test_sighup_during_stop_kills_everything_at_once_and_restores_the_handler(st assert signal.getsignal(signal.SIGHUP) is before -def test_default_profile_has_no_legacy_clock_keys(): +def test_default_profile_declares_no_sender_clock_responder(): network = ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG)["network"] assert "timing_extension" not in network and "clock_port" not in network From dd8632ac7c82bbd04b144ee3baeb6cb485a53bfe Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 10:53:12 -0500 Subject: [PATCH 09/13] examples/egocap: keep the preview up when the optional timing follower exits The Chrony follower is an optional child unless --require-chrony is given: its exit is reported once and the stack keeps running, while the bridge keeps refusing capture times. With --require-chrony its exit still stops the stack. nvargus-daemon is stopped only for the SIPL backend, which is the one that needs the cameras free. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../egocap/tests/tools/test_start_system.py | 76 ++++++++++++++++++- examples/egocap/tools/start_system.py | 39 ++++++++-- 2 files changed, 106 insertions(+), 9 deletions(-) diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index 686c714372..513bf0874d 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -499,11 +499,15 @@ def recorded_main(monkeypatch, tmp_path: Path): children: list[SimpleNamespace] = [] deploys: list[list[str]] = [] - def fake_start(self, name, command, env=None, cwd=None, grace_s=ss.DEFAULT_GRACE_S, leader_only=False): + def fake_start( + self, name, command, env=None, cwd=None, grace_s=ss.DEFAULT_GRACE_S, leader_only=False, optional=False + ): log = self.workdir / f"{name}.log" log.write_text("") - children.append(SimpleNamespace(name=name, command=list(command), env=env, leader_only=leader_only)) - managed = ss.Managed(name, SimpleNamespace(poll=lambda: None, pid=0), log, grace_s, leader_only) + children.append( + SimpleNamespace(name=name, command=list(command), env=env, leader_only=leader_only, optional=optional) + ) + managed = ss.Managed(name, SimpleNamespace(poll=lambda: None, pid=0), log, grace_s, leader_only, optional) self.children.append(managed) return managed @@ -565,6 +569,72 @@ def test_main_wires_chrony_into_the_bridge_and_starts_the_follower_after_it(reco assert "--chrony-sender 192.0.2.10 --chrony-sender-refid C000020A --chrony-pc-port 12301" in bridge +def test_the_timing_follower_is_optional_unless_chrony_is_required(recorded_main): + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS]) == 0 + follower = next(c for c in recorded_main.children if c.name == "timesync-follower") + assert follower.optional + others = [c for c in recorded_main.children if c.name != "timesync-follower"] + assert not any(c.optional for c in others) + + +def test_the_timing_follower_is_required_with_require_chrony(recorded_main): + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS, "--require-chrony"]) == 0 + follower = next(c for c in recorded_main.children if c.name == "timesync-follower") + assert not follower.optional + + +def _exited(name: str, optional: bool, tmp_path: Path, code: int = 1) -> ss.Managed: + return ss.Managed( + name, + SimpleNamespace(poll=lambda: code, returncode=code, pid=0), + tmp_path / f"{name}.log", + optional=optional, + ) + + +def _running(name: str, tmp_path: Path) -> ss.Managed: + return ss.Managed(name, SimpleNamespace(poll=lambda: None, returncode=None, pid=0), tmp_path / f"{name}.log") + + +def test_watch_keeps_the_stack_up_when_an_optional_child_exits(monkeypatch, capsys, tmp_path: Path): + stack = SimpleNamespace(children=[_running("backend", tmp_path), _exited("timesync-follower", True, tmp_path)]) + ticks: list[int] = [] + + def sleep(_seconds): + ticks.append(1) + if len(ticks) == 3: + raise KeyboardInterrupt + + monkeypatch.setattr(ss.time, "sleep", sleep) + with pytest.raises(KeyboardInterrupt): + ss.watch(stack) + out = capsys.readouterr().out + assert out.count("timesync-follower exited with 1") == 1 + assert "preview stays up" in out + + +def test_watch_returns_a_required_child_that_exits(tmp_path: Path): + backend = _exited("backend", False, tmp_path, code=3) + stack = SimpleNamespace(children=[_exited("timesync-follower", True, tmp_path), backend]) + assert ss.watch(stack) is backend + + +def test_the_required_follower_exit_stops_the_stack(tmp_path: Path): + follower = _exited("timesync-follower", False, tmp_path) + assert ss.watch(SimpleNamespace(children=[_running("backend", tmp_path), follower])) is follower + + +def test_stop_daemon_runs_only_for_the_sipl_backend(recorded_main, tmp_path: Path): + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS]) == 0 + assert any(d[-1] == "stop-daemon" for d in recorded_main.deploys) + recorded_main.deploys.clear() + argus = {**ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG), "capture_backend": "argus"} + config = tmp_path / "argus.yaml" + config.write_text(yaml.safe_dump(argus)) + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS, "--sender-config", str(config)]) == 0 + assert not any(d[-1] == "stop-daemon" for d in recorded_main.deploys) + + def test_main_starts_the_mock_chain_in_order(recorded_main, tmp_path: Path): mock_sender = tmp_path / "mock_sender.py" mock_sender.write_text("") diff --git a/examples/egocap/tools/start_system.py b/examples/egocap/tools/start_system.py index 1be6195c88..4b8e3b9862 100644 --- a/examples/egocap/tools/start_system.py +++ b/examples/egocap/tools/start_system.py @@ -322,6 +322,29 @@ class Managed: log: Path grace_s: float = DEFAULT_GRACE_S leader_only: bool = False + # An optional child may exit without stopping the stack; see watch(). + optional: bool = False + + +def watch(stack: "Stack") -> Managed: + """Block until a required child exits and return it. + + An optional child that exits is reported once and the stack keeps running. + """ + reported: set[str] = set() + while True: + for child in stack.children: + if child.process.poll() is None: + continue + if not child.optional: + return child + if child.name not in reported: + reported.add(child.name) + print( + f"WARNING: {child.name} exited with {child.process.returncode}; the preview stays up and " + f"capture times are unavailable; see {child.log}" + ) + time.sleep(1) class Stack: @@ -340,11 +363,13 @@ def start( cwd: Path | None = None, grace_s: float = DEFAULT_GRACE_S, leader_only: bool = False, + optional: bool = False, ) -> Managed: """Launch a child in its own process group so stopping it also stops anything it spawned. ``leader_only`` sends the polite SIGINT to the child itself and not to its whole group, for a child that - closes its own helper processes in an order of its own (the backend's video writers). + closes its own helper processes in an order of its own (the backend's video writers). ``optional`` marks a + child whose exit is reported but does not stop the stack. """ # Each child writes to its own file so a failed step can point the operator at the output to read. log = self.workdir / f"{name}.log" @@ -364,7 +389,7 @@ def start( # A wrong tool path is an operator mistake, so say which tool instead of printing a traceback. raise StartupError(f"cannot start {name} ({command[0]}): {error}") from error # Registering the child here is what makes stop_all() cover it even if a later step fails. - child = Managed(name, process, log, grace_s, leader_only) + child = Managed(name, process, log, grace_s, leader_only, optional) self.children.append(child) print(f"started {name} (pid {process.pid}); log {log}") return child @@ -687,7 +712,8 @@ def main(argv: Sequence[str] | None = None) -> int: # nvargus-daemon returns at boot and holds the cameras, which SIPL capture needs free; a sender left # by an earlier run outlives its ssh session. subprocess.run([*deploy, "stop"], check=False) - subprocess.run([*deploy, "stop-daemon"], check=True) + if sender.sipl: + subprocess.run([*deploy, "stop-daemon"], check=True) if not args.skip_deploy: print("deploying the sender to", args.orin) subprocess.run( @@ -821,6 +847,9 @@ def main(argv: Sequence[str] | None = None) -> int: "--workdir", str(workdir / "chrony"), ], + # Without --require-chrony a failed follower leaves the preview up and the bridge refusing + # capture times, which is what an unavailable clock measurement means. + optional=not args.require_chrony, ) print("configuring the Orin's chrony to serve its clock to", lan_ip, flush=True) try: @@ -874,9 +903,7 @@ def main(argv: Sequence[str] | None = None) -> int: if args.open: webbrowser.open(url) # Stay up until a child dies or the operator interrupts. - while all(child.process.poll() is None for child in stack.children): - time.sleep(1) - dead = next(child for child in stack.children if child.process.poll() is not None) + dead = watch(stack) if dead.name == "sender" and dead.process.returncode == 0: # The sender runs for --seconds and then exits cleanly: that is the requested end, not a failure. print(f"the Orin sender finished its {args.seconds} s run; stopping the stack") From b3aab071e93243b194e6c3563ba70067c04025c3 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 12:11:34 -0500 Subject: [PATCH 10/13] examples/egocap: check the launcher's deploy commands against the real deploy tool The test runs every command the launcher issues for the deploy tool through the tool's own parser and dispatch, so a command the tool does not have fails here instead of at startup. It is skipped in a tree that does not contain the tool. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../egocap/tests/tools/test_start_system.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index 513bf0874d..53b60da2c9 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -7,6 +7,7 @@ import base64 import hashlib import hmac +import importlib.util import json import os import signal @@ -635,6 +636,32 @@ def test_stop_daemon_runs_only_for_the_sipl_backend(recorded_main, tmp_path: Pat assert not any(d[-1] == "stop-daemon" for d in recorded_main.deploys) +DEPLOY_TOOL = ss.REPO_ROOT / "examples/camera_streamer/argus_sender/deploy/deploy_sender.py" + + +@pytest.mark.skipif( + not DEPLOY_TOOL.is_file(), + reason="the deploy tool comes from the camera-streaming tools PR, absent from this tree", +) +def test_every_deploy_command_the_launcher_issues_is_accepted_by_the_real_deploy_tool(recorded_main, monkeypatch): + spec = importlib.util.spec_from_file_location("deploy_sender_real", DEPLOY_TOOL) + tool = importlib.util.module_from_spec(spec) + spec.loader.exec_module(tool) + dispatched: list[str] = [] + for name in dir(tool): + if name.startswith("cmd_"): + monkeypatch.setattr(tool, name, lambda *a, _name=name, **k: dispatched.append(_name)) + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS]) == 0 + issued = [d[2:] for d in recorded_main.deploys if d[1] == "x"] + issued += [c.command[2:] for c in recorded_main.children if c.command[1] == "x"] + assert any(command[1] == "stop-daemon" for command in issued) + for command in issued: + # `target command ...` is exactly how the launcher calls the tool; the real parser must accept it. + assert tool.main(command) == 0, command + assert "cmd_stop_daemon" in dispatched + assert len(dispatched) == len(issued) + + def test_main_starts_the_mock_chain_in_order(recorded_main, tmp_path: Path): mock_sender = tmp_path / "mock_sender.py" mock_sender.write_text("") From a332c7bf5592ab7518380eacf58e6d4e462b7719 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 13:51:33 -0500 Subject: [PATCH 11/13] examples/egocap: address review suggestions on the timing launcher Docs no longer describe the removed clock responder, say that time sync runs before the page address is printed and that the launcher stops nvargus-daemon and may install chrony, and state that the timing record is sent with every frame. The receiver helpers follow CAMERA_RECEIVER_VENV/CAMERA_RECEIVER_PYTHON and keep their quoting, receiver setup runs for any camera chain, deploy-tool calls have timeouts, and the remote stop on exit is skipped when the Orin was never contacted and cannot raise. A contract test runs the launcher's timesync commands through the real tool's parser when the tool is present. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/egocap/CAMERA_SETTINGS.md | 9 +- examples/egocap/README.md | 17 ++-- .../backend/egocap_web/livekit_recording.py | 3 +- .../backend/test_recording_session_livekit.py | 2 +- .../egocap/tests/tools/test_start_system.py | 79 +++++++++++++++-- examples/egocap/tools/start_system.py | 84 +++++++++++++------ 6 files changed, 153 insertions(+), 41 deletions(-) diff --git a/examples/egocap/CAMERA_SETTINGS.md b/examples/egocap/CAMERA_SETTINGS.md index 77ff2d144a..4bbde3eaf4 100644 --- a/examples/egocap/CAMERA_SETTINGS.md +++ b/examples/egocap/CAMERA_SETTINGS.md @@ -15,7 +15,7 @@ pass it with `--sender-config`. | Component | Owns | Where it is set | |---|---|---| | SIPL platform file | Sensor geometry, sensor rate, trigger mode, link wiring | `sipl.platform_config_json` on the Orin (installed by the BSP or the camera deploy) | -| Sender (Orin) | Output frame rate and bitrate, codec, timing records, clock responder, TCP port | The sender profile, `configs/default-camera-settings.yaml` | +| Sender (Orin) | Output frame rate and bitrate, codec, timing records, TCP port | The sender profile, `configs/default-camera-settings.yaml` | | Receiver (PC) | Decoding, the CUDA IPC output and its stream names | Generated by the launcher from the sender profile | | Bridge (PC) | Publishing into LiveKit; timestamp basis; the chrony addresses; output rate | Command-line flags, built by the launcher | | CaptureServer | The stream descriptors the page reads (room, track, size, nominal fps, timestamp basis) | Generated by the launcher | @@ -45,8 +45,8 @@ The sender does not search for a camera file. The deploy tool uploads the chosen it that are relative to the file (the remaps) resolve inside the release, and starts `./build/argus_sender --config src/configs/run-config.yaml`. The file is read once at start. Editing it while the sender runs changes nothing: stop the sender and start it again. A reconnecting receiver reuses the same in-memory configuration. Uploading a new file does -not rebuild the binary, so the release must already support the timing extension and the clock responder; an older -parser rejects keys it does not know instead of ignoring them. The platform JSON and tuning files are not uploaded +not rebuild the binary, so the release must already send the timing record (it is sent with every frame and has no +setting); an older parser rejects keys it does not know instead of ignoring them. The platform JSON and tuning files are not uploaded and must already exist at the configured paths on the Orin. ### Frame rate @@ -68,7 +68,8 @@ effective `output.fps`; the descriptor value is nominal and cannot make the came The page shows latency from the sensor's capture time only when all of these hold: 1. The sender is a release that includes the AVS1 timing extension; it sends the sensor capture time on its - `CLOCK_REALTIME` with every frame. + `CLOCK_REALTIME` with every frame. A release whose parser still reads `network.timing_extension` is older than this profile and sends + no capture time unless that key is set to `true`, which this profile no longer carries; every take with it is refused. 2. The receiver passes the timing records through to the CUDA IPC output. 3. The bridge runs with `--timestamp-basis capture_monotonic_us --chrony-sender ORIN_HOST --chrony-sender-refid ID --chrony-pc-port PORT`. For a SIPL profile the launcher also passes `--source-full-range`, because SIPL pictures use the full 0-255 range; without it the diff --git a/examples/egocap/README.md b/examples/egocap/README.md index 5149643cdb..20bd712845 100644 --- a/examples/egocap/README.md +++ b/examples/egocap/README.md @@ -105,14 +105,21 @@ timestamp because the mock sender has no clock to measure. A recording made in t Without `--orin` or `--mock` only the PC side starts. `--skip-deploy` reuses the previous release. The sender runs for `--seconds` (default one hour) and the launcher stops the whole stack when it ends, so a take in progress is cut off; the launcher prints the stop time at startup. The sender release the profile describes, its deploy tool, the -LiveKit server and the bridge are not in this tree, so pass their paths (or set `EGOCAP_DEPLOY_SCRIPT`, -`EGOCAP_LIVEKIT_SERVER`, `EGOCAP_BRIDGE`). `--orin` needs an IPv4 address, because the bridge finds the sender's chrony by it. The launcher starts an -unprivileged chronyd on this PC that measures the Orin's clock and configures the Orin's chrony through -`src/core/timesync/tools/timesync_tool.py`; see that module's README. +timesync tool, the LiveKit server and the bridge are not in this tree's base, so pass their paths (or set +`EGOCAP_DEPLOY_SCRIPT`, `EGOCAP_TIMESYNC_TOOL`, `EGOCAP_LIVEKIT_SERVER`, `EGOCAP_BRIDGE`); `--orin` needs the timesync +tool and refuses to start without it. `--orin` also needs an IPv4 address, because the bridge finds the sender's chrony +by it. After the bridge starts, the launcher runs time sync through `src/core/timesync/tools/timesync_tool.py` (see that +module's README): `ensure-local` (which may install chrony on this PC), an unprivileged chronyd on this PC that measures +the Orin's clock, and `configure-remote` on the Orin; each step has a timeout. These steps run before the page address +is printed, and a failure leaves the preview up with capture times unavailable (`--require-chrony` makes it fatal). For +the SIPL backend the launcher also stops `nvargus-daemon` on the Orin before starting the sender, since it holds the +cameras; it does not restart it. The page needs the frontend built once (`cd frontend && npm install && npm run build`); the launcher refuses to start without `frontend/dist`. For the default receiver, the launcher runs the receiver's `setup.sh` (venv, requirements, -native decoder) when the venv or decoder is missing. +native decoder) when the venv or decoder is missing and the camera chain starts (`--orin`, or `--mock` with the +bridge); `CAMERA_RECEIVER_VENV` and `CAMERA_RECEIVER_PYTHON` select the environment as they do for the receiver's own +scripts. The sender settings come from the checked-in profile `configs/default-camera-settings.yaml` (30 fps, 4 Mbit/s HEVC). Pass another file with `--sender-config`, or override single values with `--fps`, `--bitrate-mbps` and `--sender-port`; the receiver, bridge and CaptureServer descriptor are diff --git a/examples/egocap/backend/egocap_web/livekit_recording.py b/examples/egocap/backend/egocap_web/livekit_recording.py index d93632987d..421e8cd46c 100644 --- a/examples/egocap/backend/egocap_web/livekit_recording.py +++ b/examples/egocap/backend/egocap_web/livekit_recording.py @@ -389,7 +389,8 @@ async def prepare(self) -> None: # A take records the publisher's capture time, so a source that sends none cannot start one. if frame.user_timestamp_us is None: raise RuntimeError( - f"{stream.name}: the first frame carries no capture time, so it cannot be recorded" + f"{stream.name}: the first frame carries no capture time, so it cannot be recorded " + "(the bridge publishes none while time sync is not up)" ) if frame.image.shape[:2] != (stream.height, stream.width): raise RuntimeError( diff --git a/examples/egocap/tests/backend/test_recording_session_livekit.py b/examples/egocap/tests/backend/test_recording_session_livekit.py index e9c64f459a..6daad7796d 100644 --- a/examples/egocap/tests/backend/test_recording_session_livekit.py +++ b/examples/egocap/tests/backend/test_recording_session_livekit.py @@ -326,6 +326,6 @@ async def test_a_frame_without_a_publisher_timestamp_ends_the_take_and_leaves_it await asyncio.sleep(0.01) snap = coordinator.snapshot() assert snap.service.phase is RecordingPhase.RECOVERY_REQUIRED - assert "a frame arrived without the publisher's frame timestamp" in (coordinator._last_issue or "") + assert "a frame arrived without the publisher's frame timestamp" in (snap.issue or "") assert snap.service.last_result is None or snap.service.last_result.disposition is not Disposition.KEPT await coordinator.close() diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index 53b60da2c9..47df6b6738 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -10,6 +10,7 @@ import importlib.util import json import os +import shlex import signal import sys import threading @@ -30,7 +31,7 @@ for source in next(iter(yaml.safe_load(EXAMPLE_STATION.read_text())["stations"].values()))["video_sources"] ] TIMESYNC_STUB = ( - "def lan_address_toward(host):\n return '10.0.0.2'\n\n\ndef reference_id_for(host):\n return 'REFID'\n" + "def lan_address_toward(host):\n return '192.0.2.2'\n\n\ndef reference_id_for(host):\n return 'REFID'\n" ) CAMERA_ARGS = [arg for name in EXAMPLE_CAMERAS for arg in ("--capture-camera", f"{name}=dev0")] @@ -154,8 +155,8 @@ def record_start(self, name, command, *_, **__): monkeypatch.setattr(ss.Stack, "wait_until", lambda *_, **__: None) monkeypatch.setattr(ss.Stack, "stop_all", lambda *_: None) monkeypatch.setattr(ss.subprocess, "run", lambda *_, **__: None) - argv = ["--orin", "user@10.0.0.5", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] - argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "10.0.0.2", *CAMERA_ARGS] + argv = ["--orin", "user@192.0.2.5", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] + argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "192.0.2.2", *CAMERA_ARGS] assert ss.main(argv) == 1 assert "stop after the bridge" in capsys.readouterr().err @@ -636,6 +637,70 @@ def test_stop_daemon_runs_only_for_the_sipl_backend(recorded_main, tmp_path: Pat assert not any(d[-1] == "stop-daemon" for d in recorded_main.deploys) +TIMESYNC_TOOL = ss.REPO_ROOT / "src/core/timesync/tools/timesync_tool.py" + + +@pytest.mark.skipif( + not TIMESYNC_TOOL.is_file(), reason="the timesync tool comes from the timesync PR, absent from this tree" +) +def test_every_timesync_command_the_launcher_issues_is_accepted_by_the_real_timesync_tool( + recorded_main, monkeypatch +): + spec = importlib.util.spec_from_file_location("timesync_tool_real", TIMESYNC_TOOL) + tool = importlib.util.module_from_spec(spec) + spec.loader.exec_module(tool) + dispatched: list[str] = [] + # The tool looks its commands up by module-level name when it runs, so replacing them records the dispatch + # without installing or configuring anything. + for name in ("cmd_ensure_local", "cmd_follow", "cmd_configure_remote", "cmd_status"): + monkeypatch.setattr(tool, name, lambda args, _name=name: dispatched.append(_name) or 0) + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS, "--timesync-tool", str(TIMESYNC_TOOL)]) == 0 + issued = [d[2:] for d in recorded_main.deploys if d[1] == str(TIMESYNC_TOOL)] + issued += [c.command[2:] for c in recorded_main.children if c.command[1] == str(TIMESYNC_TOOL)] + assert {command[0] for command in issued} == {"ensure-local", "follow", "configure-remote", "status"} + for command in issued: + assert tool.main(command) == 0, command + assert len(dispatched) == len(issued) + + +def test_the_receiver_python_follows_the_receivers_own_environment_variables(monkeypatch, tmp_path: Path): + spaced = tmp_path / "a dir" / "python" + spaced.parent.mkdir() + spaced.write_text("") + monkeypatch.setenv("CAMERA_RECEIVER_PYTHON", str(spaced)) + words = shlex.split(ss.default_receiver_command()) + assert words[0] == str(spaced) + assert words[1].endswith("camera_receiver.py") + monkeypatch.delenv("CAMERA_RECEIVER_PYTHON") + venv = tmp_path / "venv" + (venv / "bin").mkdir(parents=True) + (venv / "bin/python").write_text("") + monkeypatch.setenv("CAMERA_RECEIVER_VENV", str(venv)) + assert shlex.split(ss.default_receiver_command())[0] == str(venv / "bin/python") + + +def test_the_remote_stop_on_exit_is_skipped_when_the_orin_was_never_contacted(recorded_main, monkeypatch): + monkeypatch.setattr(ss, "FRONTEND_INDEX", Path("/nonexistent/index.html")) + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS]) == 1 + assert not any(d[-1] == "stop" for d in recorded_main.deploys) + + +def test_a_remote_stop_that_times_out_on_exit_does_not_raise(recorded_main, monkeypatch, capsys): + stops: list[int] = [] + real_run = ss.subprocess.run + + def run(command, **kwargs): + if command[-1] == "stop": + stops.append(1) + if len(stops) == 2: # the one on exit; the first clears a sender an earlier run left + raise ss.subprocess.TimeoutExpired(command, kwargs.get("timeout", 0)) + return real_run(command, **kwargs) + + monkeypatch.setattr(ss.subprocess, "run", run) + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS]) == 0 + assert "could not confirm that the Orin sender stopped" in capsys.readouterr().err + + DEPLOY_TOOL = ss.REPO_ROOT / "examples/camera_streamer/argus_sender/deploy/deploy_sender.py" @@ -859,7 +924,9 @@ def poll(self): def _run_main_with_stub_children(monkeypatch, tmp_path: Path, extra_argv: list[str], exit_codes=None): - """Run main() against a REPO_ROOT without a receiver venv; returns (exit code, argv per child, run calls). + """Run main() against a REPO_ROOT without a receiver venv. + + Returns (exit code, argv per child, run calls, receiver dir). ``exit_codes`` maps a child name to the code its poll() reports; other children keep running. """ @@ -897,8 +964,8 @@ class _Done: monkeypatch.setattr(ss.Stack, "stop_all", lambda *_: None) monkeypatch.setattr(ss.subprocess, "run", lambda cmd, **__: runs.append(list(cmd)) or _Done()) monkeypatch.setattr(ss.time, "sleep", lambda *_: None) - argv = ["--orin", "user@10.0.0.5", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] - argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "10.0.0.2", *CAMERA_ARGS, *extra_argv] + argv = ["--orin", "user@192.0.2.5", "--deploy-script", "x", "--bridge", "b", "--livekit-server", "l"] + argv += ["--timesync-tool", str(timesync), "--host-lan-ip", "192.0.2.2", *CAMERA_ARGS, *extra_argv] return ss.main(argv), started, runs, receiver_dir diff --git a/examples/egocap/tools/start_system.py b/examples/egocap/tools/start_system.py index 4b8e3b9862..9aad557bdc 100644 --- a/examples/egocap/tools/start_system.py +++ b/examples/egocap/tools/start_system.py @@ -6,20 +6,24 @@ uv run python tools/start_system.py --orin user@orin --deploy-script PATH --bridge PATH --livekit-server PATH -In order: deploy and start the Orin sender over ssh, a LiveKit server, CaptureServer, the backend (which owns -the recorder), the camera receiver and the camera-to-LiveKit bridge. Without ``--orin`` no sender is started, so -only the PC side comes up. The servers, the backend and the receiver are waited on until they are ready before the -next one starts, and all children are stopped, last started first, when this script ends on Ctrl-C, on SIGTERM or -after an error. The Orin sender runs for ``--seconds``; when it ends the whole stack is stopped. +In order: deploy and start the Orin sender over ssh (for the SIPL backend this first stops nvargus-daemon, which +holds the cameras), a LiveKit server, CaptureServer, the backend (which owns the recorder), the camera receiver +and the camera-to-LiveKit bridge, then time sync through the timesync tool: ``ensure-local`` (which may install +chrony on this PC), an unprivileged follower here that measures the Orin's clock, and ``configure-remote`` on +the Orin. Without ``--orin`` no sender is started, so only the PC side comes up. +The servers, the backend and the receiver are waited on until they are ready before the next one starts, and all +children are stopped, last started first, when this script ends on Ctrl-C, on SIGTERM or after an error. The +Orin sender runs for ``--seconds``; when it ends the whole stack is stopped. The sender settings come from the checked-in profile ``configs/default-camera-settings.yaml`` (see CAMERA_SETTINGS.md). CaptureServer runs real acquisition (a built IsaacCapture checkout and tracking hardware) unless ``--mock`` gives it synthetic tracking and cameras. ``--mock`` also replaces the Orin sender with a local mock sender (no ``--orin``), so nothing real is started; the receiver, bridge and page are the same. -The sender release the profile describes, its deploy tool, the LiveKit server, the bridge and the mock sender are -not part of this tree, so the tools' paths are options (or ``EGOCAP_DEPLOY_SCRIPT``, ``EGOCAP_LIVEKIT_SERVER``, -``EGOCAP_BRIDGE``, ``EGOCAP_MOCK_SENDER``). The pure helpers sit above the supervisor so they can be tested without +The sender release the profile describes, its deploy tool, the timesync tool, the LiveKit server, the bridge and +the mock sender are not part of this tree's base, so the tools' paths are options (or ``EGOCAP_DEPLOY_SCRIPT``, +``EGOCAP_TIMESYNC_TOOL``, ``EGOCAP_LIVEKIT_SERVER``, ``EGOCAP_BRIDGE``, ``EGOCAP_MOCK_SENDER``). The pure +helpers sit above the supervisor so they can be tested without starting anything. """ @@ -58,11 +62,15 @@ TIMESTAMP_BASIS = "capture_monotonic_us" # Seconds a child gets to exit after SIGINT before it is killed. The backend closes an in-progress take, which its # coordinator bounds at 35 s and the aligner bounds again afterwards, so it gets a margin over both. +# A deploy-tool command that only signals or queries the Orin; the deploy itself is not bounded by this. +ORIN_COMMAND_TIMEOUT_S = 60.0 +# Installing chrony may download packages, so it gets longer than a query. +ENSURE_LOCAL_TIMEOUT_S = 300.0 DEFAULT_GRACE_S = 15.0 BACKEND_GRACE_S = 90.0 # Signals that mean "stop the stack"; while it is stopping they mean "stop faster". STOP_SIGNALS = tuple(sig for sig in (signal.SIGTERM, getattr(signal, "SIGHUP", None)) if sig is not None) -# The mock sender has no clock responder, so the bridge stamps the frame-ready time on its own clock instead. +# The mock sender sends no capture time, so the bridge stamps the frame-ready time on its own clock instead. MOCK_TIMESTAMP_BASIS = "monotonic_us" # The bridge shrinks the picture by this integer factor before encoding, so the descriptor declares the shrunken # size: the recorder refuses a take whose frames differ from the declared one. @@ -508,7 +516,7 @@ def control_token(log: Path) -> str | None: def ensure_receiver_environment() -> None: """Run the receiver's own ``setup.sh`` when its venv or its native decoder is missing; it is idempotent.""" receiver_dir = REPO_ROOT / "examples/camera_streamer/camera_receiver" - has_venv = (receiver_dir / ".venv/bin/python").is_file() + has_venv = receiver_venv_python(receiver_dir).is_file() has_codec = any((receiver_dir / "codec").glob("_camera_receiver_codec*.so")) if has_venv and has_codec: return @@ -516,12 +524,24 @@ def ensure_receiver_environment() -> None: subprocess.run(["bash", str(receiver_dir / "setup.sh")], check=True) +def receiver_venv_python(receiver_dir: Path) -> Path: + """The Python of the receiver's own environment, where its ``setup.sh`` put it. + + ``setup.sh`` honours ``CAMERA_RECEIVER_VENV`` and ``run.sh`` ``CAMERA_RECEIVER_PYTHON``, so they decide here + too. + """ + explicit = os.environ.get("CAMERA_RECEIVER_PYTHON") + if explicit: + return Path(explicit) + return Path(os.environ.get("CAMERA_RECEIVER_VENV", receiver_dir / ".venv")) / "bin/python" + + def default_receiver_command() -> str: """Run the receiver with the Python from its own ``setup.sh`` environment, which has CuPy, when it exists.""" receiver_dir = REPO_ROOT / "examples/camera_streamer/camera_receiver" - venv_python = receiver_dir / ".venv/bin/python" + venv_python = receiver_venv_python(receiver_dir) python = str(venv_python) if venv_python.is_file() else "python3" - return f"{python} {receiver_dir / 'camera_receiver.py'}" + return shlex.join([python, str(receiver_dir / "camera_receiver.py")]) def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: @@ -617,6 +637,7 @@ def main(argv: Sequence[str] | None = None) -> int: workdir = Path(tempfile.mkdtemp(prefix="egocap-start-")) ipc_socket = str(workdir / "camera_receiver.sock") stack = Stack(workdir) + orin_contacted = False # Children run in their own sessions, so a signal to this script reaches none of them; turn the ones that mean # stop into the Ctrl-C path so the finally below always runs stop_all(). for sig in STOP_SIGNALS: @@ -697,23 +718,23 @@ def main(argv: Sequence[str] | None = None) -> int: ] ) - if args.orin and args.receiver == default_receiver_command(): + if chain and args.receiver == default_receiver_command(): ensure_receiver_environment() # The default was resolved before this setup created the receiver's environment, so resolve it again. # An explicit --receiver or EGOCAP_RECEIVER is not the default and is never replaced. args.receiver = default_receiver_command() - # Time sync is set up after the page is up (below), so a slow or failed install never holds the preview. if args.orin: tool = load_timesync_tool(args.timesync_tool) lan_ip = args.host_lan_ip or tool.lan_address_toward(orin_host) # 1. The Orin sender: deploy a release (never activating a service), then run it for a bounded time. if args.orin: + orin_contacted = True deploy = [python, args.deploy_script, args.orin] # nvargus-daemon returns at boot and holds the cameras, which SIPL capture needs free; a sender left # by an earlier run outlives its ssh session. - subprocess.run([*deploy, "stop"], check=False) + subprocess.run([*deploy, "stop"], check=False, timeout=ORIN_COMMAND_TIMEOUT_S) if sender.sipl: - subprocess.run([*deploy, "stop-daemon"], check=True) + subprocess.run([*deploy, "stop-daemon"], check=True, timeout=ORIN_COMMAND_TIMEOUT_S) if not args.skip_deploy: print("deploying the sender to", args.orin) subprocess.run( @@ -820,16 +841,21 @@ def main(argv: Sequence[str] | None = None) -> int: ], ) if args.orin: - # Every step is bounded: the page and preview do not need qualified times, and a slow Orin or a missing - # package must not keep them down. The bridge refuses unqualified times, so a take cannot start until - # time sync is up. + # Each step has a timeout, and a failure only degrades capture times (the bridge refuses unqualified + # times, so a take cannot start until time sync is up); but the banner, --open and supervision wait for + # these steps, so the preview comes up after them, not before. print("setting up time sync", flush=True) settled = False try: provisioned = ( - subprocess.run([python, str(args.timesync_tool), "ensure-local"], check=False).returncode == 0 + subprocess.run( + [python, str(args.timesync_tool), "ensure-local"], + check=False, + timeout=ENSURE_LOCAL_TIMEOUT_S, + ).returncode + == 0 ) - except OSError: + except (OSError, subprocess.TimeoutExpired): provisioned = False if provisioned: # An unprivileged chronyd on this PC measures the Orin's clock and never adjusts this PC's. Nothing @@ -911,14 +937,24 @@ def main(argv: Sequence[str] | None = None) -> int: raise StartupError(f"{dead.name} exited with {dead.process.returncode}; see {dead.log}") except KeyboardInterrupt: return 0 - except (StartupError, subprocess.CalledProcessError) as error: + except (StartupError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: print(f"error: {error}", file=sys.stderr) return 1 finally: stack.stop_all() - if args.orin and args.deploy_script: + if orin_contacted: # Ending the local ssh does not end the remote sender; ask it to stop so it frees the cameras. - subprocess.run([sys.executable, args.deploy_script, args.orin, "stop"], check=False) + try: + subprocess.run( + [sys.executable, args.deploy_script, args.orin, "stop"], + check=False, + timeout=ORIN_COMMAND_TIMEOUT_S, + ) + except (subprocess.TimeoutExpired, KeyboardInterrupt): + print( + "could not confirm that the Orin sender stopped; check it with the deploy tool", + file=sys.stderr, + ) if __name__ == "__main__": From a0c075c74fc2e6ba10535734723de53e6ec36ae9 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 13:55:00 -0500 Subject: [PATCH 12/13] examples/egocap: print the page address and open the browser before time sync Time sync can take minutes (installing chrony, configuring the Orin), and it held the banner, --open and the supervision loop. The address is now printed and the browser opened first; time sync runs afterwards, and a required child that has died is noticed between its steps. A sender that ended cleanly is still the requested end of its run, not a failure. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../egocap/tests/tools/test_start_system.py | 28 ++++++++++++ examples/egocap/tools/start_system.py | 44 +++++++++++++------ 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index 47df6b6738..3f76530f49 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -524,6 +524,7 @@ def interrupt(_seconds): monkeypatch.setattr(ss.Stack, "wait_until", lambda *_, **__: None) # The fake children are not real processes, so there is nothing to signal. monkeypatch.setattr(ss.Stack, "stop_all", lambda self: None) + monkeypatch.setattr(ss, "ensure_receiver_environment", lambda: None) # Every deploy, stop and timesync step is recorded and reports success. monkeypatch.setattr( ss.subprocess, "run", lambda command, **_: deploys.append(list(command)) or SimpleNamespace(returncode=0) @@ -648,6 +649,8 @@ def test_every_timesync_command_the_launcher_issues_is_accepted_by_the_real_time ): spec = importlib.util.spec_from_file_location("timesync_tool_real", TIMESYNC_TOOL) tool = importlib.util.module_from_spec(spec) + # The tool's dataclasses look their module up by name while the class bodies run. + monkeypatch.setitem(sys.modules, spec.name, tool) spec.loader.exec_module(tool) dispatched: list[str] = [] # The tool looks its commands up by module-level name when it runs, so replacing them records the dispatch @@ -663,6 +666,31 @@ def test_every_timesync_command_the_launcher_issues_is_accepted_by_the_real_time assert len(dispatched) == len(issued) +def test_the_page_address_is_printed_before_time_sync_starts(recorded_main, capsys): + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS]) == 0 + out = capsys.readouterr().out + assert out.index("Ego Capture is up") < out.index("setting up time sync") + assert "time sync: settled" in out + + +def test_a_child_that_dies_during_time_sync_stops_startup(recorded_main, monkeypatch, capsys): + real_start = ss.Stack.start + + def start(self, name, command, *args, **kwargs): + managed = real_start(self, name, command, *args, **kwargs) + if name == "bridge": + managed.process.poll = lambda: 3 + managed.process.returncode = 3 + return managed + + monkeypatch.setattr(ss.Stack, "start", start) + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS]) == 1 + captured = capsys.readouterr() + assert "bridge exited with 3" in captured.err + # The banner was printed first: the preview is up while time sync runs. + assert "Ego Capture is up" in captured.out + + def test_the_receiver_python_follows_the_receivers_own_environment_variables(monkeypatch, tmp_path: Path): spaced = tmp_path / "a dir" / "python" spaced.parent.mkdir() diff --git a/examples/egocap/tools/start_system.py b/examples/egocap/tools/start_system.py index 9aad557bdc..09fd233ca2 100644 --- a/examples/egocap/tools/start_system.py +++ b/examples/egocap/tools/start_system.py @@ -334,6 +334,19 @@ class Managed: optional: bool = False +def raise_if_a_child_exited(stack: "Stack") -> None: + """Stop startup when a required child has already exited; an optional one is left to ``watch``. + + A sender that ended cleanly is the requested end of its run, which ``watch`` reports, not a startup failure. + """ + for child in stack.children: + if child.optional or child.process.poll() is None: + continue + if child.name == "sender" and child.process.returncode == 0: + continue + raise StartupError(f"{child.name} exited with {child.process.returncode}; see {child.log}") + + def watch(stack: "Stack") -> Managed: """Block until a required child exits and return it. @@ -840,11 +853,21 @@ def main(argv: Sequence[str] | None = None) -> int: ), ], ) + url = f"http://127.0.0.1:{args.backend_port}/" + token = control_token(backend.log) + print(f"\nEgo Capture is up at {url}\nrecording control token: {token}\nlogs: {workdir}") + if args.mock: + # Everything the page shows and records in this mode is synthetic. + print("--mock: tracking, the CaptureServer cameras and the camera stream are synthetic.") + print("Ctrl-C stops everything.") + if args.open: + webbrowser.open(url) if args.orin: - # Each step has a timeout, and a failure only degrades capture times (the bridge refuses unqualified - # times, so a take cannot start until time sync is up); but the banner, --open and supervision wait for - # these steps, so the preview comes up after them, not before. - print("setting up time sync", flush=True) + # The page address is already printed and the browser open, so a slow or failed step never holds the + # preview. Each step has a timeout; a failure only degrades capture times (the bridge refuses + # unqualified times, so a take cannot start until time sync is up). A child that dies meanwhile is + # noticed between steps. + print("setting up time sync; capture times are unavailable until it finishes", flush=True) settled = False try: provisioned = ( @@ -857,6 +880,7 @@ def main(argv: Sequence[str] | None = None) -> int: ) except (OSError, subprocess.TimeoutExpired): provisioned = False + raise_if_a_child_exited(stack) if provisioned: # An unprivileged chronyd on this PC measures the Orin's clock and never adjusts this PC's. Nothing # has to reach this PC from the Orin, so a firewall here does not matter. @@ -896,6 +920,7 @@ def main(argv: Sequence[str] | None = None) -> int: ) except subprocess.TimeoutExpired: configured = False + raise_if_a_child_exited(stack) settled = ( configured and subprocess.run( @@ -919,15 +944,8 @@ def main(argv: Sequence[str] | None = None) -> int: if args.require_chrony: raise StartupError(message) print("WARNING:", message) - url = f"http://127.0.0.1:{args.backend_port}/" - token = control_token(backend.log) - print(f"\nEgo Capture is up at {url}\nrecording control token: {token}\nlogs: {workdir}") - if args.mock: - # Everything the page shows and records in this mode is synthetic. - print("--mock: tracking, the CaptureServer cameras and the camera stream are synthetic.") - print("Ctrl-C stops everything.") - if args.open: - webbrowser.open(url) + else: + print("time sync: settled", flush=True) # Stay up until a child dies or the operator interrupts. dead = watch(stack) if dead.name == "sender" and dead.process.returncode == 0: From e222728bfcdd964cf02d5d5a7287607305e92c88 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 19:34:20 -0500 Subject: [PATCH 13/13] examples/egocap: size the stack from the profile, refuse stereo, supervise the sync wait The receiver, bridge descriptor and mock sender now use the sender profile's output size when it sets one, and --width/--height only describe the sensor when it does not; a flag that contradicts the profile, a profile that sets one dimension, or a non-positive flag is refused. A profile whose stream is not mono is refused, since the receiver stream the launcher writes is mono. The time-sync `status --wait` now runs under a helper that checks the children once a second, so a required child that dies during the wait stops startup. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../egocap/tests/tools/test_start_system.py | 88 +++++++++++++++++++ examples/egocap/tools/start_system.py | 86 +++++++++++++++--- 2 files changed, 161 insertions(+), 13 deletions(-) diff --git a/examples/egocap/tests/tools/test_start_system.py b/examples/egocap/tests/tools/test_start_system.py index 3f76530f49..1faa14fceb 100644 --- a/examples/egocap/tests/tools/test_start_system.py +++ b/examples/egocap/tests/tools/test_start_system.py @@ -529,6 +529,8 @@ def interrupt(_seconds): monkeypatch.setattr( ss.subprocess, "run", lambda command, **_: deploys.append(list(command)) or SimpleNamespace(returncode=0) ) + # The supervised helper (time-sync status) is recorded like the other commands. + monkeypatch.setattr(ss, "run_supervised", lambda _stack, command: deploys.append(list(command)) or 0) monkeypatch.setattr( ss, "load_timesync_tool", @@ -1025,3 +1027,89 @@ def fail_setup(): ) assert code == 0 assert started["receiver"][:2] == ["custom-python", "recv.py"] + + +def _stereo_profile(config: dict) -> None: + config["streams"][0]["type"] = "stereo" + + +def _sized(width, height): + def mutate(config: dict) -> None: + output = config["streams"][0].setdefault("output", {}) + output["width"], output["height"] = width, height + + return mutate + + +def test_load_sender_config_refuses_a_stream_that_is_not_mono(tmp_path: Path): + with pytest.raises(ss.StartupError, match="mono"): + ss.load_sender_config(_profile_with(tmp_path, _stereo_profile)) + + +def test_frame_size_defaults_to_the_sensor_size_and_follows_the_flags(): + settings = ss.sender_settings(ss.load_sender_config(ss.DEFAULT_SENDER_CONFIG)) + assert ss.frame_size(settings, None, None) == (2560, 1984) + assert ss.frame_size(settings, 640, 480) == (640, 480) + with pytest.raises(ss.StartupError, match="divide"): + ss.frame_size(settings, 641, 480) + + +def test_frame_size_follows_the_profile_and_refuses_contradicting_flags(tmp_path: Path): + config = yaml.safe_load(_profile_with(tmp_path, _sized(1280, 960)).read_text()) + settings = ss.sender_settings(config) + assert ss.frame_size(settings, None, None) == (1280, 960) + assert ss.frame_size(settings, 1280, 960) == (1280, 960) + with pytest.raises(ss.StartupError, match="--width 2560 contradicts"): + ss.frame_size(settings, 2560, None) + with pytest.raises(ss.StartupError, match="--height"): + ss.frame_size(settings, None, 1984) + + +def test_frame_size_refuses_a_profile_that_sets_only_one_dimension(tmp_path: Path): + config = ss.load_sender_config(_profile_with(tmp_path, _sized(1280, None))) + with pytest.raises(ss.StartupError, match="both"): + ss.frame_size(ss.sender_settings(config), None, None) + + +def test_main_writes_the_profile_output_size_into_the_receiver_and_the_descriptor(recorded_main, tmp_path: Path): + profile = _profile_with(tmp_path, _sized(1280, 960)) + assert ss.main([*ORIN_ARGV, *CAMERA_ARGS, "--sender-config", str(profile)]) == 0 + declared = yaml.safe_load((recorded_main.workdir / "streams.yaml").read_text())["livekit"]["streams"][0] + received = yaml.safe_load((recorded_main.workdir / "receiver.yaml").read_text())["streams"][0] + assert (received["width"], received["height"]) == (1280, 960) + assert (declared["width"], declared["height"]) == (1280 // ss.BRIDGE_DOWNSCALE, 960 // ss.BRIDGE_DOWNSCALE) + + +def test_main_refuses_a_width_that_contradicts_the_profile(recorded_main, capsys, tmp_path: Path): + profile = _profile_with(tmp_path, _sized(1280, 960)) + argv = [*ORIN_ARGV, *CAMERA_ARGS, "--sender-config", str(profile), "--width", "2560"] + assert ss.main(argv) == 1 + assert "contradicts" in capsys.readouterr().err + + +def _stack_with_child_polling(tmp_path: Path, code): + stack = ss.Stack(tmp_path) + stack.children.append( + ss.Managed("bridge", SimpleNamespace(poll=lambda: code, returncode=code, pid=0), tmp_path / "bridge.log") + ) + return stack + + +def test_run_supervised_returns_the_helpers_exit_code(tmp_path: Path): + stack = _stack_with_child_polling(tmp_path, None) + assert ss.run_supervised(stack, [sys.executable, "-c", "raise SystemExit(3)"]) == 3 + + +def test_run_supervised_kills_the_helper_when_a_required_child_exits(monkeypatch, tmp_path: Path): + stack = _stack_with_child_polling(tmp_path, 1) + helpers = [] + real_popen = ss.subprocess.Popen + + def popen(command, **kwargs): + helpers.append(real_popen(command, **kwargs)) + return helpers[-1] + + monkeypatch.setattr(ss.subprocess, "Popen", popen) + with pytest.raises(ss.StartupError, match="bridge exited with 1"): + ss.run_supervised(stack, ["sleep", "30"]) + assert helpers[0].poll() is not None diff --git a/examples/egocap/tools/start_system.py b/examples/egocap/tools/start_system.py index 09fd233ca2..cdbac17e43 100644 --- a/examples/egocap/tools/start_system.py +++ b/examples/egocap/tools/start_system.py @@ -75,6 +75,8 @@ # The bridge shrinks the picture by this integer factor before encoding, so the descriptor declares the shrunken # size: the recorder refuses a take whose frames differ from the declared one. BRIDGE_DOWNSCALE = 2 +DEFAULT_SENSOR_WIDTH = 2560 +DEFAULT_SENSOR_HEIGHT = 1984 # The backend serves the page from this build output and mounts nothing if it is missing. FRONTEND_INDEX = EGOCAP_ROOT / "frontend" / "dist" / "index.html" @@ -169,6 +171,9 @@ class SenderSettings: stream_id: str full_range: bool sipl: bool + # The profile's output size; None leaves the sensor mode's native size, which --width and --height describe. + width: int | None = None + height: int | None = None def _section(parent: dict, key: str, path: Path) -> dict: @@ -201,6 +206,9 @@ def load_sender_config( stream = config["streams"][0] if not isinstance(stream, dict): raise StartupError(f"{path}: streams[0] must be a mapping") + # The receiver stream the launcher writes is mono, so another type would deploy a sender it cannot receive. + if stream.get("type") != "mono": + raise StartupError(f"{path}: the launcher supports only a mono stream, got type {stream.get('type')!r}") network = _section(config, "network", path) output = _section(stream, "output", path) # An override replaces the profile's value; the profile stays the source of everything else. @@ -237,9 +245,45 @@ def sender_settings(config: dict) -> SenderSettings: # SIPL produces full-range pictures; the bridge must say so or the encoded stream signals limited range. full_range=config["capture_backend"] == "sipl", sipl=config["capture_backend"] == "sipl", + width=_output_dimension(output, "width"), + height=_output_dimension(output, "height"), ) +def _output_dimension(output: dict, key: str) -> int | None: + value = output.get(key) + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise StartupError( + f"the sender config's streams[0].output.{key} must be a positive integer, got {value!r}" + ) + return value + + +def frame_size(settings: SenderSettings, width: int | None, height: int | None) -> tuple[int, int]: + """The picture size the sender delivers: the profile's output size when it sets one, else the sensor's. + + The receiver, the bridge and the descriptor all follow this, so a profile that resizes its output cannot + disagree with them. A --width or --height that contradicts the profile is refused. + """ + for name, flag in (("--width", width), ("--height", height)): + if flag is not None and flag <= 0: + raise StartupError(f"{name} must be positive, got {flag}") + if (settings.width is None) != (settings.height is None): + raise StartupError("the sender config must set both streams[0].output.width and height, or neither") + if settings.width is not None: + for name, flag, value in (("--width", width, settings.width), ("--height", height, settings.height)): + if flag is not None and flag != value: + raise StartupError(f"{name} {flag} contradicts the sender config's output size {value}") + size = (settings.width, settings.height) + else: + size = (width or DEFAULT_SENSOR_WIDTH, height or DEFAULT_SENSOR_HEIGHT) + if size[0] % (2 * BRIDGE_DOWNSCALE) or size[1] % (2 * BRIDGE_DOWNSCALE): + raise StartupError(f"the picture size {size[0]}x{size[1]} must divide by {2 * BRIDGE_DOWNSCALE}") + return size + + def mock_sender_command(python: str, script: Path, settings: SenderSettings, width: int, height: int) -> list[str]: """The local mock sender that stands in for the Orin: one HEVC stream shaped like the real one.""" return [ @@ -347,6 +391,23 @@ def raise_if_a_child_exited(stack: "Stack") -> None: raise StartupError(f"{child.name} exited with {child.process.returncode}; see {child.log}") +def run_supervised(stack: "Stack", command: list[str]) -> int: + """Run a helper to completion while checking the stack's children once a second. + + A required child that dies meanwhile stops the helper and raises, instead of going unnoticed until it returns. + """ + process = subprocess.Popen(command) + try: + while process.poll() is None: + raise_if_a_child_exited(stack) + time.sleep(1) + finally: + if process.poll() is None: + process.kill() + process.wait() + return process.returncode + + def watch(stack: "Stack") -> Managed: """Block until a required child exits and return it. @@ -582,14 +643,14 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument( "--width", type=int, - default=2560, - help="sensor width; the receiver expects it and the descriptor declares it divided by the bridge shrink", + help="sensor width when the profile sets no output size (default 2560); the receiver expects it and the " + "descriptor declares it divided by the bridge shrink", ) parser.add_argument( "--height", type=int, - default=1984, - help="sensor height; the receiver expects it and the descriptor declares it divided by the bridge shrink", + help="sensor height when the profile sets no output size (default 1984); the receiver expects it and the " + "descriptor declares it divided by the bridge shrink", ) parser.add_argument("--fps", type=int, help="override the profile's output.fps") parser.add_argument("--bitrate-mbps", type=int, help="override the profile's HEVC bitrate") @@ -678,6 +739,7 @@ def main(argv: Sequence[str] | None = None) -> int: # One loaded profile is the source for the sender, the receiver, the bridge and the descriptor. sender_config = load_sender_config(args.sender_config, args.fps, args.bitrate_mbps, args.sender_port) sender = sender_settings(sender_config) + width, height = frame_size(sender, args.width, args.height) # The bridge identifies the Orin's chrony by the reference id of its IPv4 address. if args.orin: try: @@ -691,17 +753,15 @@ def main(argv: Sequence[str] | None = None) -> int: room, publisher, track, - args.width // BRIDGE_DOWNSCALE, - args.height // BRIDGE_DOWNSCALE, + width // BRIDGE_DOWNSCALE, + height // BRIDGE_DOWNSCALE, sender.fps, basis, sender.stream_id, ) ) (workdir / "receiver.yaml").write_text( - receiver_yaml( - orin_host or "127.0.0.1", sender.port, ipc_socket, args.width, args.height, sender.stream_id - ) + receiver_yaml(orin_host or "127.0.0.1", sender.port, ipc_socket, width, height, sender.stream_id) ) if args.mock: print(f"mock sender: {sender.fps} fps stream {sender.stream_id!r}, timestamps are frame-ready time") @@ -771,7 +831,7 @@ def main(argv: Sequence[str] | None = None) -> int: mock_sender = None if chain and args.mock: mock_sender = stack.start( - "mock-sender", mock_sender_command(python, args.mock_sender, sender, args.width, args.height) + "mock-sender", mock_sender_command(python, args.mock_sender, sender, width, height) ) # 2. LiveKit carries the video from the bridge to the browser and the recorder. if args.livekit_server: @@ -923,7 +983,8 @@ def main(argv: Sequence[str] | None = None) -> int: raise_if_a_child_exited(stack) settled = ( configured - and subprocess.run( + and run_supervised( + stack, [ python, str(args.timesync_tool), @@ -935,8 +996,7 @@ def main(argv: Sequence[str] | None = None) -> int: "--wait", str(args.chrony_wait), ], - check=False, - ).returncode + ) == 0 ) if not settled: