diff --git a/docs/source/references/oob_teleop_control.rst b/docs/source/references/oob_teleop_control.rst index 25d69ee59f..d4e09b589b 100644 --- a/docs/source/references/oob_teleop_control.rst +++ b/docs/source/references/oob_teleop_control.rst @@ -38,6 +38,17 @@ This will: 4. Accept the self-signed certificate and click CONNECT automatically via Chrome DevTools Protocol (CDP) +When the OpenXR application starts after the service, add +``--defer-oob-connect``: steps 2-4 wait for ``GET /api/oob/v1/connect``, so +the headset connects once there is a stream instead of retrying until one +appears. + +.. code-block:: bash + + python -m isaaccapture.cloudxr.service run --accept-eula --setup-oob --defer-oob-connect + # once the application has its OpenXR session: + curl -k -H "X-Control-Token: " https://localhost:48322/api/oob/v1/connect + Ordinary headset absence, an ``offline`` or ``unauthorized`` transport, and browser recovery failures leave the host running with a degraded OOB status. The lifecycle retries for 60 seconds at a 5-second cadence, then keeps @@ -498,6 +509,14 @@ Supported query keys: ``serverIP``, ``port``, ``panelHiddenAtStart``, ``codec``. Optional ``targetClientId`` restricts the push to a single headset (returns 404 if not connected). +``GET /api/oob/v1/connect`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Releases headset setup on a service started with ``--defer-oob-connect``. +Returns ``{"connectRequested": true, "alreadyRequested": false}``; repeating +it is harmless (``"alreadyRequested": true``). A service that was not started +deferred answers 409. + Authentication -------------- @@ -574,6 +593,16 @@ Environment variables - Default video codec for headset bookmarks * - ``TELEOP_CLIENT_PANEL_HIDDEN_AT_START`` - Hide control panel on load (``true`` / ``false``) + * - ``TELEOP_CLIENT_IMMERSIVE_MODE`` + - WebXR session mode: ``vr`` or ``ar``. PICO 4 Ultra needs ``vr``. + * - ``TELEOP_CLIENT_REFERENCE_SPACE`` + - XR reference space: ``auto``, ``local-floor``, ``local``, ``viewer`` or + ``unbounded``. Pin it to compare replays across runs. + * - ``TELEOP_CLIENT_REPLAY_PACING`` + - Replay pacing: ``frame`` (one recorded frame per XR frame, independent + of server speed) or ``time``. + * - ``TELEOP_CLIENT_SHOW_RECORDING_CONTROLS`` + - Enable the recording and replay controls (``true`` / ``false``). * - ``TELEOP_CLIENT_ROUTE`` - HashRouter fragment appended to the bookmark URL. Default empty (no fragment — the web client picks its own landing route). Set @@ -608,6 +637,16 @@ Environment variables Before each new browser launch, ADB must confirm ``mWakefulness=Awake``; wear or unlock the headset (or cover its proximity sensor) when the service reports that it is asleep. + * - ``TELEOP_OOB_RECORDING`` + - Recording to select in the client before CONNECT: a host path, pushed to + the headset, or ``latest``, the newest + ``isaacteleop-input-recording*.json`` (the client's Save name) in its + Download folder. + Starting the replay is left to the operator. Unset by default. + * - ``TELEOP_HEADSET_BROWSER_PACKAGE`` + - Headset browser package. Unset, the vendor's browser is used. On PICO, + set ``com.pico.browser``: in hand-tracking mode the OS holds VR from + non-system browsers behind a "requires controllers" dialog. * - ``TELEOP_OOB_RECOVERY_TIMEOUT_SEC`` - Positive finite recovery-episode duration in seconds (default ``60``). Expiry changes to observation mode; it never stops the host. If this is diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py index 5bf23f2d00..e367f490df 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py @@ -38,12 +38,14 @@ from inspect import isawaitable from dataclasses import dataclass from enum import Enum +from pathlib import Path from .oob_teleop_env import ( default_web_client_origin, parse_env_port, build_headset_bookmark_url, client_ui_fields_from_env, oob_progress, + oob_replay_recording_from_env, redact_control_token, resolve_control_token, resolve_lan_host_for_oob, @@ -1225,6 +1227,8 @@ def stop_coturn(proc: subprocess.Popen | None) -> None: def _discover_devtools_socket() -> str | None: """Return the bare name of the browser's DevTools abstract socket, or None. + When several tie on prefix, the one holding a teleop tab wins. + Chromium-based Android browsers expose a Unix abstract socket matching ``@_devtools_remote[_]`` in ``/proc/net/unix``. Known prefixes in the wild: @@ -1258,11 +1262,33 @@ def _discover_devtools_socket() -> str | None: # Prefer teleop-relevant prefixes in this order, otherwise fall back to # the first discovered socket. priority = ("weblayer", "com.oculus.browser", "chrome", "webview") - for prefix in priority: - for cand in candidates: - if cand.startswith(prefix): - return cand - return candidates[0] + + def rank(name: str) -> int: + return next( + (i for i, pre in enumerate(priority) if name.startswith(pre)), + len(priority), + ) + + ranked = sorted(candidates, key=rank) + tied = [c for c in ranked if rank(c) == rank(ranked[0])] + if len(tied) == 1: + return tied[0] + # PICO exposes one weblayer_devtools_remote_ per browser process, and + # the teleop tab can be on any of them: ask each. + for cand in tied: + try: + _adb_forward_cdp(cand, _CDP_LOCAL_PORT) + except (OobAdbError, subprocess.CalledProcessError, OSError) as exc: + log.debug("CDP: cannot forward to @%s: %s", cand, exc) + continue + if any( + "oobEnable=" in (t.get("url") or "") + for t in _cdp_list_tabs(_CDP_LOCAL_PORT) + ): + log.info("CDP: teleop tab found on @%s", cand) + return cand + # None has one yet, which is normal before the page is opened. + return tied[0] def _adb_forward_cdp(socket_name: str, local_port: int) -> None: @@ -1522,12 +1548,89 @@ async def send(ws, method: str, params: dict | None = None) -> None: log.info("CDP: sent reset-panel key (%s)", _RESET_PANEL_KEY) +_HEADSET_RECORDING_DIR = "/sdcard/Download" +LATEST_RECORDING = "latest" +# The client's Save names files isaacteleop-input-recording.json; the browser +# adds " (N)" to repeats. ``latest`` only looks at these. +_RECORDING_PREFIX = "isaacteleop-input-recording" +# How long the client gets to read and validate a recording set over CDP. +_RECORDING_LOAD_TIMEOUT = 10.0 +# The client reports each import in #loadRecordingStatus, as class +# success or error plus a message. +_CLEAR_LOAD_STATUS_JS = """(() => { + const el = document.getElementById('loadRecordingStatus'); + if (el) { el.textContent = ''; el.className = 'load-recording-status'; } +})()""" +_READ_LOAD_STATUS_JS = """(() => { + const el = document.getElementById('loadRecordingStatus'); + const state = ['success', 'error'].find((c) => el?.classList.contains(c)); + return state ? { state, text: el.textContent } : null; +})()""" + + +def _newest_recording_on_headset() -> str: + """Path of the newest client recording in the headset's Download folder.""" + out = _run_adb( + "newest recording", + [ + "adb", + "shell", + f"ls -t {_HEADSET_RECORDING_DIR}/{_RECORDING_PREFIX}*.json 2>/dev/null" + " | head -n 1", + ], + timeout=10.0, + ) + path = (out or "").strip() + if not path or not path.startswith(_HEADSET_RECORDING_DIR): + raise OobAdbError( + f"no recording found in {_HEADSET_RECORDING_DIR} on the headset; " + "record one first, or pass a host path instead of 'latest'" + ) + log.info("newest recording on headset: %s", path) + return path + + +def _resolve_recording(spec: str) -> str: + """Turn the configured recording into a path on the headset.""" + if spec == LATEST_RECORDING: + return _newest_recording_on_headset() + return _push_recording_to_headset(spec) + + +def _push_recording_to_headset(host_path: str) -> str: + """Copy a recording onto the headset and return its on-device path. + + DOM.setFileInputFiles names a path in the *browser's* filesystem, so the + file has to exist on the headset first. + """ + src = Path(host_path) + if not src.is_file(): + raise OobAdbError(f"recording not found: {src}") + device_path = f"{_HEADSET_RECORDING_DIR}/{src.name}" + try: + _adb_run( + ["adb", "push", str(src), device_path], + capture_output=True, + text=True, + timeout=120, + check=True, + ) + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or exc.stdout or "").strip() or "(no adb output)" + raise OobAdbError( + f"adb push {src} -> {device_path} failed (rc={exc.returncode}): {detail}" + ) from exc + log.info("pushed recording to headset: %s", device_path) + return device_path + + async def _cdp_session_click_connect( ws_url: str, *, refresh_static_assets: bool = False, allow_navigation: bool = True, clear_stale_error: bool = False, + recording_device_path: str | None = None, on_dispatched: Callable[[], Awaitable[None] | None] | None = None, on_client_loaded: Callable[[], Awaitable[None] | None] | None = None, ) -> None: @@ -1544,7 +1647,7 @@ async def _cdp_session_click_connect( _seq = 0 - async def send(ws, method, params=None): + async def send(ws, method, params=None, *, check=False): nonlocal _seq _seq += 1 req_id = _seq @@ -1554,6 +1657,8 @@ async def send(ws, method, params=None): while True: msg = json.loads(await asyncio.wait_for(ws.recv(), timeout=10.0)) if msg.get("id") == req_id: + if check and "error" in msg: + raise OobAdbError(f"CDP: {method} failed: {msg['error']}") return msg.get("result", {}) async with ws_connect(ws_url) as ws: @@ -1745,6 +1850,56 @@ async def send(ws, method, params=None): if isawaitable(result): await result + # The file input is in the page from first render; fill it before + # the session goes immersive. + if recording_device_path: + await send(ws, "DOM.enable") + doc = await send(ws, "DOM.getDocument", {"depth": -1}) + node = await send( + ws, + "DOM.querySelector", + { + "nodeId": doc["root"]["nodeId"], + "selector": "input[type=file]", + }, + ) + node_id = node.get("nodeId") + if not node_id: + raise OobAdbError( + "CDP: the teleop page has no recording file input; " + "the web client is older than the recorder UI" + ) + await send(ws, "Runtime.evaluate", {"expression": _CLEAR_LOAD_STATUS_JS}) + await send( + ws, + "DOM.setFileInputFiles", + {"nodeId": node_id, "files": [recording_device_path]}, + check=True, + ) + # Setting the input only starts the import; CONNECT without a + # replayable recording would make an unattended run worthless. + load_deadline = loop.time() + _RECORDING_LOAD_TIMEOUT + while True: + r = await send( + ws, + "Runtime.evaluate", + {"expression": _READ_LOAD_STATUS_JS, "returnByValue": True}, + ) + status = (r.get("result") or {}).get("value") or {} + if status.get("state") == "success": + break + if status.get("state") == "error": + raise OobAdbError( + f"CDP: the client rejected the recording: {status.get('text')}" + ) + if loop.time() > load_deadline: + raise OobAdbError( + "CDP: the client did not confirm the recording within " + f"{_RECORDING_LOAD_TIMEOUT:.0f}s" + ) + await asyncio.sleep(1.0) + log.info("CDP: %s", status.get("text")) + # Extra grace period: the CONNECT button can become enabled before the # React store is fully mounted, which causes "XR is not available" # errors if clicked immediately. @@ -1934,6 +2089,7 @@ async def _find_and_click_teleop_tab( usb_local: bool = False, host_client: bool = False, refresh_static_assets: bool = False, + recording_device_path: str | None = None, on_dispatched: Callable[[], Awaitable[None] | None] | None = None, on_client_loaded: Callable[[], Awaitable[None] | None] | None = None, ) -> str: @@ -1985,8 +2141,23 @@ def _is_candidate_tab(tab: dict) -> bool: # (browser was being woken up), and a second intent reliably # navigates an already-warm tab. retry_at = time.monotonic() + (timeout / 2) + rescan_tick = 0 while ws_url is None and time.monotonic() < deadline: await asyncio.sleep(1.0) + # The forward was chosen before the page existed, when stale-tab + # cleanup had left no teleop tab to tell the sockets apart. Re-probe + # every third pass so it follows the page once it appears. + rescan_tick += 1 + if rescan_tick % 3 == 0: + rescanned = await asyncio.to_thread(_discover_devtools_socket) + # Discovery probes by forwarding, so the port may not point at it. + if rescanned: + try: + await asyncio.to_thread( + _adb_forward_cdp, rescanned, _CDP_LOCAL_PORT + ) + except (OobAdbError, subprocess.CalledProcessError, OSError) as exc: + log.debug("CDP: re-forward to @%s failed: %s", rescanned, exc) for tab in await asyncio.to_thread(_cdp_list_tabs, _CDP_LOCAL_PORT): if "id" not in tab or not tab.get("webSocketDebuggerUrl"): continue @@ -2080,6 +2251,7 @@ def _is_candidate_tab(tab: dict) -> bool: await _cdp_session_click_connect( ws_url, refresh_static_assets=refresh_static_assets, + recording_device_path=recording_device_path, on_dispatched=on_dispatched, on_client_loaded=on_client_loaded, ) @@ -2138,6 +2310,12 @@ async def run_oob_connect( phase; callers should treat this as non-fatal and ask the user to tap CONNECT manually. """ + # Before the deadline: a push can be slow, and a bad path should fail + # before the browser is launched. + recording = oob_replay_recording_from_env() + recording_device_path = ( + await asyncio.to_thread(_resolve_recording, recording) if recording else None + ) deadline = time.monotonic() + timeout # --- Step 0: close any stale teleop tabs from a prior session ----------- @@ -2218,7 +2396,7 @@ async def run_oob_connect( ) from exc try: - # --- Steps 3+4: find the teleop tab and click CONNECT ------------------ + # --- Steps 3+4: find the teleop tab, load any recording, click CONNECT - ws_url = await _find_and_click_teleop_tab( resolved_port=resolved_port, deadline=deadline, @@ -2226,6 +2404,7 @@ async def run_oob_connect( usb_local=usb_local, host_client=host_client, refresh_static_assets=usb_local or host_client, + recording_device_path=recording_device_path, on_dispatched=on_dispatched, on_client_loaded=on_client_loaded, ) diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_env.py b/src/python/isaaccapture/cloudxr/oob_teleop_env.py index d4d56a34cb..993f6f5b47 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_env.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_env.py @@ -439,7 +439,8 @@ def client_ui_fields_from_env() -> dict: Keys match query params the WebXR client reads on page load (``serverIP``, ``port``, ``codec``, ``panelHiddenAtStart``, - ``reconnectMaxAttempts``, ``reconnectDelayMs``, + ``immersiveMode``, ``referenceSpace``, ``replayPacing``, + ``showRecordingControls``, ``reconnectMaxAttempts``, ``reconnectDelayMs``, ``streamAttachTimeoutMs``, ``warmupBeginTimeoutMs``, ``warmupEndTimeoutMs``). Deliberately excludes ``reconnectEnabled`` - OOB never sets it; the client's own reconnect loop is opt-in via a URL param the user supplies directly. @@ -454,6 +455,20 @@ def client_ui_fields_from_env() -> dict: out["panelHiddenAtStart"] = True elif ph in ("0", "false", "no", "off"): out["panelHiddenAtStart"] = False + # Passed through like codec; the client validates the value. + for env_name, key in ( + ("TELEOP_CLIENT_IMMERSIVE_MODE", "immersiveMode"), + ("TELEOP_CLIENT_REFERENCE_SPACE", "referenceSpace"), + ("TELEOP_CLIENT_REPLAY_PACING", "replayPacing"), + ): + val = os.environ.get(env_name, "").strip() + if val: + out[key] = val + rc = os.environ.get("TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", "").strip().lower() + if rc in ("1", "true", "yes", "on"): + out["showRecordingControls"] = True + elif rc in ("0", "false", "no", "off"): + out["showRecordingControls"] = False # reconnectMaxAttempts/reconnectDelayMs are already set above by # client_reconnect_config_from_env(), which validates them and raises on bad input - do not # re-parse them here with this loop's silent-skip-on-ValueError semantics. @@ -472,6 +487,11 @@ def client_ui_fields_from_env() -> dict: return out +def oob_replay_recording_from_env() -> str: + """``TELEOP_OOB_RECORDING``: host path or ``latest``, or ``""`` when unset.""" + return os.environ.get("TELEOP_OOB_RECORDING", "").strip() + + def teleop_client_route_from_env() -> str: """Return the HashRouter fragment to append to the headset bookmark URL. @@ -505,7 +525,8 @@ def build_headset_bookmark_url( Set *oob_enable* to ``False`` to omit ``oobEnable`` (and the control token, which only authenticates hub operations). The remaining params — - ``serverIP``, ``port``, ``codec``, ``panelHiddenAtStart``, ``reconnectEnabled``, + ``serverIP``, ``port``, ``codec``, ``panelHiddenAtStart``, ``immersiveMode``, + ``referenceSpace``, ``replayPacing``, ``showRecordingControls``, ``reconnectEnabled``, ``reconnectMaxAttempts``, ``reconnectDelayMs``, ``streamAttachTimeoutMs``, ``warmupBeginTimeoutMs``, ``warmupEndTimeoutMs`` — are plain form overrides the client honours either way, so the headset still lands with the right @@ -535,6 +556,13 @@ def build_headset_bookmark_url( v = cfg.get("panelHiddenAtStart") if isinstance(v, bool): params["panelHiddenAtStart"] = "true" if v else "false" + for key in ("immersiveMode", "referenceSpace", "replayPacing"): + v = cfg.get(key) + if v is not None and str(v).strip() != "": + params[key] = str(v).strip() + v = cfg.get("showRecordingControls") + if isinstance(v, bool): + params["showRecordingControls"] = "true" if v else "false" v = cfg.get("reconnectEnabled") if isinstance(v, bool): params["reconnectEnabled"] = "true" if v else "false" diff --git a/src/python/isaaccapture/cloudxr/service/__main__.py b/src/python/isaaccapture/cloudxr/service/__main__.py index 96fae05df2..81108bce9e 100644 --- a/src/python/isaaccapture/cloudxr/service/__main__.py +++ b/src/python/isaaccapture/cloudxr/service/__main__.py @@ -80,6 +80,17 @@ def _add_run_arguments(parser: argparse.ArgumentParser) -> None: "TELEOP_WEB_CLIENT_STATIC_DIR or ~/.cloudxr/static-client." ), ) + parser.add_argument( + "--defer-oob-connect", + action="store_true", + default=False, + help=( + "Hold headset setup (opening the teleop page, clicking CONNECT, " + "relaying its signalling) until GET /api/oob/v1/connect, for an " + "OpenXR application that starts after the service. Requires " + "--setup-oob." + ), + ) def _run_flags(args: argparse.Namespace) -> list[str]: @@ -94,7 +105,7 @@ def _run_flags(args: argparse.Namespace) -> list[str]: flags += ["--cloudxr-install-dir", args.cloudxr_install_dir] if args.cloudxr_env_config: flags += ["--cloudxr-env-config", args.cloudxr_env_config] - for name in ("setup_oob", "usb_local", "host_client"): + for name in ("setup_oob", "usb_local", "host_client", "defer_oob_connect"): if getattr(args, name): flags.append("--" + name.replace("_", "-")) return flags @@ -255,6 +266,11 @@ def _print_service_summary( print_oob_hub_startup_banner( lan_host=oob_lan_host, web_client_base=hosted_client_url ) + if args.defer_oob_connect: + _out( + "\033[33mHeadset setup is deferred: request /api/oob/v1/connect " + "(with the control token) once the application is running.\033[0m" + ) elif hosted_client_url is not None: print_hosted_client_line(hosted_client_url) else: @@ -272,6 +288,8 @@ def _cmd_run(args: argparse.Namespace) -> int: """Run the service in the foreground until interrupted.""" if args.usb_local and not args.setup_oob: _fail("--usb-local requires --setup-oob.") + if args.defer_oob_connect and not args.setup_oob: + _fail("--defer-oob-connect requires --setup-oob.") if args.usb_local and os.getenv("TELEOP_OOB_HUB_ONLY"): _fail( "TELEOP_OOB_HUB_ONLY is not compatible with --usb-local " @@ -289,6 +307,7 @@ def _cmd_run(args: argparse.Namespace) -> int: usb_local=args.usb_local, host_client=args.host_client, recovery_config=getattr(args, "_oob_recovery_config", None), + defer_oob_connect=args.defer_oob_connect, ) except RuntimeError as exc: _fail(str(exc)) @@ -370,6 +389,9 @@ def _print_summary_for(args: argparse.Namespace, run_dir: str, logs_dir: Path) - def _cmd_start(args: argparse.Namespace) -> int: """Start the service detached from this terminal.""" + # Checked here too, so start says why instead of only that the child exited. + if args.defer_oob_connect and not args.setup_oob: + _fail("--defer-oob-connect requires --setup-oob.") from .. import background # noqa: PLC0415 from ..runtime import is_runtime_live # noqa: PLC0415 diff --git a/src/python/isaaccapture/cloudxr/wss.py b/src/python/isaaccapture/cloudxr/wss.py index a0a7d84d70..24b97aa412 100755 --- a/src/python/isaaccapture/cloudxr/wss.py +++ b/src/python/isaaccapture/cloudxr/wss.py @@ -239,6 +239,7 @@ def _is_oob_hub_http_path(path: str) -> bool: return path in ( "/api/oob/v1/state", "/api/oob/v1/config", + "/api/oob/v1/connect", ) @@ -305,8 +306,14 @@ def _json_response(status: int, phrase: str, body: dict) -> Response: ) -def _make_http_handler(backend_host, backend_port, hub=None, static_dir=None): - """Return the WSS HTTP request handler (OOB hub API, static ``/client/``, cert page).""" +def _make_http_handler( + backend_host, backend_port, hub=None, static_dir=None, connect_requested=None +): + """Return the WSS HTTP request handler (OOB hub API, static ``/client/``, cert page). + + *connect_requested* is the gate a service started with ``defer_oob_connect`` + holds headset setup behind; ``/api/oob/v1/connect`` opens it. + """ async def handle_http_request(connection, request): """Dispatch non-WebSocket HTTP: CORS preflight, OOB APIs, static client, or cert HTML.""" @@ -362,6 +369,28 @@ async def handle_http_request(connection, request): }.get(status, "Error") return _json_response(status, phrase, body) + if path == "/api/oob/v1/connect": + if not hub.check_token(token): + return _json_response( + 401, "Unauthorized", {"error": "Unauthorized"} + ) + # No gate: headset setup already began with the service. + if connect_requested is None: + return _json_response( + 409, + "Conflict", + {"error": "headset setup is not deferred on this service"}, + ) + already = connect_requested.is_set() + if not already: + log.info("OOB headset setup released by /api/oob/v1/connect") + connect_requested.set() + return _json_response( + 200, + "OK", + {"connectRequested": True, "alreadyRequested": already}, + ) + if hub is None and _is_oob_hub_http_path(path): return Response( 404, @@ -719,7 +748,11 @@ async def handler(ws) -> None: ) http_handler = _make_http_handler( - backend_host, backend_port, hub=hub, static_dir=_host_client_static_dir + backend_host, + backend_port, + hub=hub, + static_dir=_host_client_static_dir, + connect_requested=connect_requested, ) async with ws_serve( diff --git a/tests/python/core/cloudxr/test_oob_teleop_adb.py b/tests/python/core/cloudxr/test_oob_teleop_adb.py index efb131b5af..92cb855764 100644 --- a/tests/python/core/cloudxr/test_oob_teleop_adb.py +++ b/tests/python/core/cloudxr/test_oob_teleop_adb.py @@ -1025,6 +1025,44 @@ def test_discover_devtools_socket_prefers_weblayer_over_generic( assert _discover_devtools_socket() == "weblayer_devtools_remote_777" +_THREE_WEBLAYER_SOCKETS = ( + "0: 00000003 00000000 00010000 0001 01 12345 @weblayer_devtools_remote_111\n" + "1: 00000003 00000000 00010000 0001 01 12346 @weblayer_devtools_remote_222\n" + "2: 00000003 00000000 00010000 0001 01 12347 @weblayer_devtools_remote_333\n" +) + + +@patch("cloudxr_py_test_ns.oob_teleop_adb._cdp_list_tabs") +@patch("cloudxr_py_test_ns.oob_teleop_adb._adb_forward_cdp") +@patch("cloudxr_py_test_ns.oob_teleop_adb._run_adb") +def test_discover_devtools_socket_picks_the_one_holding_the_teleop_tab( + mock_run_adb: MagicMock, + mock_forward: MagicMock, + mock_list_tabs: MagicMock, +) -> None: + """PICO runs several WebLayer processes: skip a dead one, pass one without the page.""" + mock_run_adb.return_value = _THREE_WEBLAYER_SOCKETS + mock_forward.side_effect = [OobAdbError("gone"), None, None] + mock_list_tabs.side_effect = [ + [{"url": "about:blank"}], + [{"url": "https://h.test/?oobEnable=1&serverIP=10.0.0.1"}], + ] + assert _discover_devtools_socket() == "weblayer_devtools_remote_333" + + +@patch( + "cloudxr_py_test_ns.oob_teleop_adb._cdp_list_tabs", + return_value=[{"url": "about:blank"}], +) +@patch("cloudxr_py_test_ns.oob_teleop_adb._adb_forward_cdp") +@patch( + "cloudxr_py_test_ns.oob_teleop_adb._run_adb", return_value=_THREE_WEBLAYER_SOCKETS +) +def test_discover_devtools_socket_without_teleop_tab_keeps_first(*_mocks) -> None: + """No teleop tab anywhere is the normal state before ``am start``.""" + assert _discover_devtools_socket() == "weblayer_devtools_remote_111" + + @patch("cloudxr_py_test_ns.oob_teleop_adb._run_adb") def test_discover_devtools_socket_falls_back_to_first_candidate( mock_run_adb: MagicMock, @@ -1267,6 +1305,38 @@ def respond(self, method: str, params: dict) -> dict: raise AssertionError(f"unscripted CDP call: {method} {params}") +class _CdpError(Exception): + """Returned by a script to answer with a CDP protocol error.""" + + +class _RecordingCdpScript(_CdpScript): + """Adds the recording file input and the client's #loadRecordingStatus.""" + + def __init__(self, statuses: list[dict | None], *, set_files_fails=False): + super().__init__() + self.statuses = statuses + self.set_files_fails = set_files_fails + + def respond(self, method: str, params: dict): + expr = params.get("expression", "") + if method.startswith("DOM.") or "loadRecordingStatus" in expr: + self.calls.append((method, params)) + if method == "DOM.enable" or "el.textContent = ''" in expr: + return {} + if method == "DOM.getDocument": + return {"root": {"nodeId": 1}} + if method == "DOM.querySelector": + return {"nodeId": 2} + if method == "DOM.setFileInputFiles": + return _CdpError("file not found") if self.set_files_fails else {} + if "loadRecordingStatus" in expr: + status = ( + self.statuses.pop(0) if len(self.statuses) > 1 else self.statuses[0] + ) + return {"result": {"value": status}} + return super().respond(method, params) + + @asynccontextmanager async def _fake_cdp_ws(script: _CdpScript): """Serves *script*'s responses on an OS-assigned port; yields the `ws://` URL to connect to.""" @@ -1275,7 +1345,11 @@ async def handler(websocket): async for raw in websocket: msg = json.loads(raw) result = script.respond(msg["method"], msg.get("params", {})) - await websocket.send(json.dumps({"id": msg["id"], "result": result})) + if isinstance(result, _CdpError): + reply = {"id": msg["id"], "error": {"message": str(result)}} + else: + reply = {"id": msg["id"], "result": result} + await websocket.send(json.dumps(reply)) async with ws_serve(handler, "localhost", 0) as server: port = server.sockets[0].getsockname()[1] @@ -1320,6 +1394,47 @@ async def test_cdp_session_click_connect_no_interstitial_reaches_ready_and_click assert click_index > dispatch_index +async def test_cdp_recording_load_waits_for_the_client() -> None: + """A slow import is waited for, and CONNECT comes only after it succeeded.""" + loaded = {"state": "success", "text": 'Loaded 3 frames from "r.json".'} + script = _RecordingCdpScript([None, None, loaded]) + async with _fake_cdp_ws(script) as ws_url: + await _cdp_session_click_connect( + ws_url, recording_device_path="/sdcard/Download/r.json" + ) + methods = [m for m, _ in script.calls] + assert methods.index("DOM.setFileInputFiles") < methods.index( + "Input.dispatchMouseEvent" + ) + + +@pytest.mark.parametrize( + ("statuses", "set_files_fails", "match"), + [ + ([None], True, "setFileInputFiles failed"), + ( + [{"state": "error", "text": 'Could not load "r.json": bad version'}], + False, + "rejected the recording: Could not load", + ), + ([None], False, "did not confirm the recording"), + ], +) +async def test_cdp_recording_load_failure_stops_before_connect( + statuses, set_files_fails, match +) -> None: + script = _RecordingCdpScript(statuses, set_files_fails=set_files_fails) + async with _fake_cdp_ws(script) as ws_url: + with ( + patch.object(adb_module, "_RECORDING_LOAD_TIMEOUT", 0.3), + pytest.raises(OobAdbError, match=match), + ): + await _cdp_session_click_connect( + ws_url, recording_device_path="/sdcard/Download/r.json" + ) + assert "Input.dispatchMouseEvent" not in [m for m, _ in script.calls] + + async def test_cdp_connect_waits_for_awake_before_mouse_dispatch() -> None: asleep = _CdpScript(interstitial=False) async with _fake_cdp_ws(asleep) as ws_url: @@ -1711,3 +1826,99 @@ async def test_monitor_without_callback_still_logs_and_keeps_watching() -> None: task.cancel() with pytest.raises(asyncio.CancelledError): await task + + +@patch("cloudxr_py_test_ns.oob_teleop_adb._adb_run") +def test_recording_host_path_is_pushed_to_download(mock_adb, tmp_path): + with pytest.raises(OobAdbError, match="recording not found"): + adb_module._resolve_recording(str(tmp_path / "nope.json")) + mock_adb.assert_not_called() + + rec = tmp_path / "pickplace-run1.json" + rec.write_text("{}", encoding="utf-8") + assert ( + adb_module._resolve_recording(str(rec)) + == "/sdcard/Download/pickplace-run1.json" + ) + assert mock_adb.call_args.args[0][:2] == ["adb", "push"] + + +@patch("cloudxr_py_test_ns.oob_teleop_adb._run_adb") +def test_recording_latest_is_the_newest_client_save(mock_run): + mock_run.return_value = "/sdcard/Download/isaacteleop-input-recording (9).json\n" + assert ( + adb_module._resolve_recording("latest") + == "/sdcard/Download/isaacteleop-input-recording (9).json" + ) + assert "isaacteleop-input-recording*.json" in mock_run.call_args.args[1][2] + + mock_run.return_value = "" + with pytest.raises(OobAdbError, match="no recording found"): + adb_module._resolve_recording("latest") + + +# --------------------------------------------------------------------------- +# CDP: _find_and_click_teleop_tab re-scans the DevTools sockets +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_find_teleop_tab_follows_the_page_to_another_socket() -> None: + """The forward was chosen before the page existed; the wait loop must follow it.""" + tabs_by_socket = { + "weblayer_devtools_remote_111": [ + { + "id": "ntp", + "url": "https://browser.example/new", + "webSocketDebuggerUrl": "ws://ntp", + } + ], + "weblayer_devtools_remote_222": [ + { + "id": "teleop", + "url": "https://10.0.0.2:48322/client/?oobEnable=1", + "webSocketDebuggerUrl": "ws://teleop", + } + ], + } + forwarded = {"socket": "weblayer_devtools_remote_111"} + + def forward(socket_name: str, _port: int) -> None: + forwarded["socket"] = socket_name + + def list_tabs(_port: int) -> list[dict]: + return tabs_by_socket[forwarded["socket"]] + + real_sleep = asyncio.sleep + + async def no_wait(_seconds: float) -> None: + await real_sleep(0) + + async def immediate(fn, *args, **kwargs): + return fn(*args, **kwargs) + + with ( + patch.object(adb_module.asyncio, "to_thread", side_effect=immediate), + patch.object(adb_module.asyncio, "sleep", side_effect=no_wait), + patch.object( + adb_module, + "_discover_devtools_socket", + return_value="weblayer_devtools_remote_222", + ), + patch.object(adb_module, "_adb_forward_cdp", side_effect=forward), + patch.object(adb_module, "_cdp_list_tabs", side_effect=list_tabs), + patch.object(adb_module, "_cdp_session_click_connect") as click, + patch.object(adb_module, "run_adb_headset_bookmark") as relaunch, + ): + ws_url = await adb_module._find_and_click_teleop_tab( + resolved_port=48322, + deadline=time.monotonic() + 5.0, + timeout=60.0, + ) + + assert ws_url == "ws://teleop" + assert forwarded["socket"] == "weblayer_devtools_remote_222" + click.assert_awaited_once() + assert click.await_args.args == ("ws://teleop",) + # Found by moving the forward, not by the mid-timeout am start re-fire. + relaunch.assert_not_called() diff --git a/tests/python/core/cloudxr/test_oob_teleop_env.py b/tests/python/core/cloudxr/test_oob_teleop_env.py index c9138c214a..954259fe00 100644 --- a/tests/python/core/cloudxr/test_oob_teleop_env.py +++ b/tests/python/core/cloudxr/test_oob_teleop_env.py @@ -25,6 +25,7 @@ default_initial_stream_config, default_web_client_origin, guess_lan_ipv4, + oob_replay_recording_from_env, versioned_web_client_url, print_hosted_client_line, print_oob_hub_startup_banner, @@ -49,6 +50,11 @@ def clear_teleop_env(monkeypatch: pytest.MonkeyPatch) -> None: "TELEOP_STREAM_PORT", "TELEOP_CLIENT_CODEC", "TELEOP_CLIENT_PANEL_HIDDEN_AT_START", + "TELEOP_CLIENT_IMMERSIVE_MODE", + "TELEOP_CLIENT_REFERENCE_SPACE", + "TELEOP_CLIENT_REPLAY_PACING", + "TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", + "TELEOP_OOB_RECORDING", "TELEOP_CLIENT_RECONNECT_MAX_ATTEMPTS", "TELEOP_CLIENT_RECONNECT_DELAY_MS", "TELEOP_CLIENT_STREAM_ATTACH_TIMEOUT_MS", @@ -210,6 +216,32 @@ def test_build_headset_bookmark_url_with_token() -> None: assert q["controlToken"] == ["secret123"] +def test_build_headset_bookmark_url_with_replay_overrides() -> None: + """The replay fields reach the headset URL, and are left out when unset.""" + base = {"serverIP": "10.0.0.1", "port": 48322} + overrides = { + "immersiveMode": "vr", + "referenceSpace": "local", + "replayPacing": "frame", + "showRecordingControls": True, + } + + def query(stream_config): + url = build_headset_bookmark_url( + web_client_base="https://h.test/", stream_config=stream_config + ) + return parse_qs(urlparse(url).query) + + q = query({**base, **overrides}) + assert {k: q[k] for k in overrides} == { + "immersiveMode": ["vr"], + "referenceSpace": ["local"], + "replayPacing": ["frame"], + "showRecordingControls": ["true"], + } + assert not set(overrides) & set(query(base)) + + def test_build_headset_bookmark_url_with_codec() -> None: """codec from stream_config is forwarded as a query param in the bookmark URL.""" u = build_headset_bookmark_url( @@ -372,6 +404,28 @@ def test_client_ui_fields_from_env_panel_hidden( assert fields["panelHiddenAtStart"] is True +def test_client_ui_fields_from_env_replay_settings( + clear_teleop_env: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """The four replay settings map onto client fields; unset ones add no keys.""" + keys = ("immersiveMode", "referenceSpace", "replayPacing", "showRecordingControls") + assert not set(keys) & set(client_ui_fields_from_env()) + + monkeypatch.setenv("TELEOP_CLIENT_IMMERSIVE_MODE", "vr") + monkeypatch.setenv("TELEOP_CLIENT_REFERENCE_SPACE", "local") + monkeypatch.setenv("TELEOP_CLIENT_REPLAY_PACING", "frame") + monkeypatch.setenv("TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", "on") + fields = client_ui_fields_from_env() + assert {k: fields[k] for k in keys} == { + "immersiveMode": "vr", + "referenceSpace": "local", + "replayPacing": "frame", + "showRecordingControls": True, + } + monkeypatch.setenv("TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", "off") + assert client_ui_fields_from_env()["showRecordingControls"] is False + + def test_client_ui_fields_from_env_never_sets_reconnect_enabled( clear_teleop_env: None, ) -> None: @@ -869,3 +923,12 @@ def test_print_hosted_client_line_hides_credentials() -> None: assert "controlToken=" in out assert "turnCredential=" in out assert "port=48322" in out + + +def test_oob_replay_recording_from_env( + clear_teleop_env: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """Unset means no recording; a set value is returned stripped.""" + assert oob_replay_recording_from_env() == "" + monkeypatch.setenv("TELEOP_OOB_RECORDING", " /tmp/take-7.json ") + assert oob_replay_recording_from_env() == "/tmp/take-7.json" diff --git a/tests/python/core/cloudxr/test_service_cli.py b/tests/python/core/cloudxr/test_service_cli.py index 26734b6ae8..b88470d795 100644 --- a/tests/python/core/cloudxr/test_service_cli.py +++ b/tests/python/core/cloudxr/test_service_cli.py @@ -51,6 +51,11 @@ def test_install_dir_emitted_only_when_changed(self): default = os.path.expanduser("~/.cloudxr") assert cli._run_flags(_run_args(cloudxr_install_dir=default)) == [] + def test_defer_oob_connect_round_trips(self): + """A detached `run` must hold headset setup back just like a foreground one.""" + args = _run_args(setup_oob=True, defer_oob_connect=True) + assert cli._run_flags(args) == ["--setup-oob", "--defer-oob-connect"] + def test_accept_eula_is_never_baked_into_the_unit(self): """Acceptance is a marker file written at install time, not a unit flag.""" assert "--accept-eula" not in cli._run_flags(_run_args(accept_eula=True)) @@ -114,6 +119,40 @@ def test_foreground_fatal_with_stalled_writer_returns_error(tmp_path, capsys): service.stop.assert_called_once() +@pytest.mark.parametrize("command", ["_cmd_run", "_cmd_start"]) +def test_defer_oob_connect_requires_setup_oob(capsys, command): + """``start`` refuses before it writes the EULA marker or spawns ``run``.""" + with ( + patch("isaaccapture.cloudxr.background.start_and_wait") as spawn, + pytest.raises(SystemExit), + ): + getattr(cli, command)(_run_args(defer_oob_connect=True)) + assert "--defer-oob-connect requires --setup-oob" in capsys.readouterr().err + spawn.assert_not_called() + + +def test_defer_oob_connect_reaches_the_service(tmp_path): + """The flag maps onto the service's existing defer_oob_connect gate.""" + args = cli._build_parser().parse_args( + [ + "run", + "--setup-oob", + "--defer-oob-connect", + "--cloudxr-install-dir", + str(tmp_path), + ] + ) + with ( + patch.object(cli, "_oob_preflight", return_value=None), + patch.object( + cli, "CloudXRService", side_effect=RuntimeError("stop here") + ) as service, + ): + with pytest.raises(SystemExit): + cli._cmd_run(args) + assert service.call_args.kwargs["defer_oob_connect"] is True + + class TestStartEula: """The EULA gate on start.""" diff --git a/tests/python/core/cloudxr/test_wss_oob_connect.py b/tests/python/core/cloudxr/test_wss_oob_connect.py new file mode 100644 index 0000000000..52bd742841 --- /dev/null +++ b/tests/python/core/cloudxr/test_wss_oob_connect.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``/api/oob/v1/connect``: releasing a deferred headset setup over HTTP.""" + +from __future__ import annotations + +import asyncio +import json + +import pytest +from cloudxr_py_test_ns.oob_teleop_hub import OOBControlHub +from cloudxr_py_test_ns.wss import CORS_HEADERS, _make_http_handler, add_cors_headers + +TOKEN = "test-control-token" +PATH = "/api/oob/v1/connect" + + +class FakeRequest: + def __init__(self, path: str, headers: dict[str, str] | None = None) -> None: + self.path = path + self.headers = headers or {} + + +def _handler(connect_requested: asyncio.Event | None): + hub = OOBControlHub(control_token=TOKEN) + return _make_http_handler( + "localhost", 49100, hub=hub, connect_requested=connect_requested + ) + + +@pytest.mark.asyncio +async def test_connect_opens_the_deferred_gate_once() -> None: + """A retrying orchestrator is told it already succeeded, not that it failed.""" + gate = asyncio.Event() + handler = _handler(gate) + request = FakeRequest(PATH, {"X-Control-Token": TOKEN}) + + first = await handler(None, request) + second = await handler(None, request) + + assert gate.is_set() + assert [r.status_code for r in (first, second)] == [200, 200] + assert [json.loads(r.body)["alreadyRequested"] for r in (first, second)] == [ + False, + True, + ] + + +@pytest.mark.asyncio +async def test_connect_refusals() -> None: + """Bad token: 401 and the gate stays shut. No deferral: 409.""" + gate = asyncio.Event() + handler = _handler(gate) + for headers in ({}, {"X-Control-Token": "wrong"}): + response = await handler(None, FakeRequest(PATH, headers)) + assert response.status_code == 401 + assert not gate.is_set() + + response = await _handler(None)(None, FakeRequest(PATH, {"X-Control-Token": TOKEN})) + assert response.status_code == 409 + + +@pytest.mark.asyncio +async def test_connect_is_a_hub_endpoint() -> None: + """Like the other hub endpoints: 404 without a hub, no CORS headers.""" + response = await _make_http_handler("localhost", 49100, hub=None)( + None, FakeRequest(PATH) + ) + assert response.status_code == 404 + + request = FakeRequest(PATH, {"X-Control-Token": TOKEN}) + response = await _handler(asyncio.Event())(None, request) + add_cors_headers(None, request, response) + assert not any(k in response.headers for k in CORS_HEADERS)