From e6da8e16892e6ce58f2d27b6ff133f314f70bcee Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 12 Nov 2026 18:32:14 -0500 Subject: [PATCH 01/10] camera_livekit_bridge: qualify capture times against chrony The bridge's capture_monotonic_us basis now converts the sender's wall-clock capture time with the timesync library: a background poll of this PC's and the sender's chrony, a per-frame qualification against a time budget, and a refusal that publishes the frame without a timestamp. Verdict counts are reported in the stream statistics. Frames carry no capture time unless qualified, so a recording take fails rather than recording a guessed one. The mock sender can emit the realtime timing record so the path runs without a camera. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/camera_livekit_bridge/CMakeLists.txt | 8 + examples/camera_livekit_bridge/README.md | 23 +- .../bridge/CMakeLists.txt | 1 + .../camera_livekit_bridge/bridge/main.cpp | 74 +++- .../bridge/stream_worker.cpp | 16 + .../bridge/stream_worker.hpp | 8 + .../chrony_clock/CMakeLists.txt | 19 + .../chrony_clock/chrony_capture_clock.cpp | 106 ++++++ .../inc/chrony_clock/chrony_capture_clock.hpp | 76 ++++ .../tests/chrony_clock/CMakeLists.txt | 14 + .../test_chrony_capture_clock.cpp | 358 ++++++++++++++++++ .../tests/python/test_bridge_timing.py | 48 ++- .../timestamp_basis/test_timestamp_basis.cpp | 14 +- .../inc/video_types/timestamp_basis.hpp | 16 +- .../camera_streamer/mock_sender/README.md | 1 + .../mock_sender/mock_sender.py | 41 +- .../mock_sender/tests/test_mock_sender.py | 38 ++ 17 files changed, 820 insertions(+), 41 deletions(-) create mode 100644 examples/camera_livekit_bridge/chrony_clock/CMakeLists.txt create mode 100644 examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp create mode 100644 examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp create mode 100644 examples/camera_livekit_bridge/tests/chrony_clock/CMakeLists.txt create mode 100644 examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp diff --git a/examples/camera_livekit_bridge/CMakeLists.txt b/examples/camera_livekit_bridge/CMakeLists.txt index 0c8bb954fa..8669783950 100644 --- a/examples/camera_livekit_bridge/CMakeLists.txt +++ b/examples/camera_livekit_bridge/CMakeLists.txt @@ -56,10 +56,18 @@ endif() find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH) message(STATUS "LiveKit SDK=${LIVEKIT_SDK_DIR}") +# The time-sync library comes from the product tree; a build inside it already has the target. +if(NOT TARGET timesync::timesync) + cmake_path(GET CMAKE_CURRENT_SOURCE_DIR PARENT_PATH _examples_dir) + cmake_path(GET _examples_dir PARENT_PATH _repo_dir) + add_subdirectory(${_repo_dir}/src/core/timesync ${CMAKE_BINARY_DIR}/timesync) +endif() + add_subdirectory(video_types) add_subdirectory(ipc_client) add_subdirectory(gpu_ops) add_subdirectory(nvenc) +add_subdirectory(chrony_clock) add_subdirectory(livekit_publisher) add_subdirectory(probe) add_subdirectory(bridge) diff --git a/examples/camera_livekit_bridge/README.md b/examples/camera_livekit_bridge/README.md index 6540688247..a3b1f11130 100644 --- a/examples/camera_livekit_bridge/README.md +++ b/examples/camera_livekit_bridge/README.md @@ -80,11 +80,30 @@ the same basis: | --- | --- | | `capture_unix_us` (default) | The sensor start-of-frame as Unix time on the sender's clock. | | `monotonic_us` | This host's monotonic clock at the instant the picture is ready to encode (before encoding): frame-ready time, later than the sensor capture. | +| `capture_monotonic_us` | The sensor start-of-frame on this host's monotonic clock, converted from the sender's `CLOCK_REALTIME` with chrony evidence for both hosts (below). | `capture_unix_us` is used only when the receiver reports a realtime sender clock; a frame with no usable sensor start-of-frame (missing, in the future, older than 5 s, or on another clock domain) carries no `user_timestamp`, so -a consumer that needs a capture time says so instead of using a guess. `capture_monotonic_us` is refused: it would -need a measured sender-to-host clock mapping with a stated uncertainty, which does not exist yet. +a consumer that needs a capture time says so instead of using a guess. + +`capture_monotonic_us` uses the sensor start-of-frame that the receiver reads from the sender's AVS1 timing record +(`sensor_capture_realtime_ns`, `sensor_capture_uncertainty_ns`, `sensor_capture_valid`) and the time-sync library in +`src/core/timesync`. Each frame is converted from the sender's `CLOCK_REALTIME` to this host's monotonic clock, or +refused with a typed reason, using chronyd's own report for both hosts, a bracketed read of this host's clocks, and the +sender's stated uncertainty. A refused frame is published without a timestamp, and the frame-ready time is never +offered in its place. It needs: + +| Option | Meaning | +| --- | --- | +| `--chrony-sender HOST[:PORT]` | The camera host's chronyd, read over chrony's monitoring interface (it must `cmdallow` this host). | +| `--chrony-sender-refid HEX` | The reference id this host's chronyd must have selected (an NTP source's reference id is its IPv4 address in hex). | +| `--chrony-pc-port PORT` | Read this host's chronyd on that command port over loopback, for a dedicated instance next to the system one. | +| `--chronyc PATH` | The `chronyc` binary (default `chronyc` on `PATH`). | + +`src/core/timesync/README.md` describes the arrangement (this host measures the camera host, which serves its own clock) and how to set it up with `timesync_tool.py`. The +`[stats]` line gains the number of frames that qualified, a count per refusal reason, the last combined uncertainty and +the clock generation the frame was converted under. The caller of a recording should treat a frame without a +timestamp as a timing fault. `--source-full-range` says the receiver's pictures use the full 0-255 range, as SIPL sources do; the encoded stream then signals full range, so players and recordings keep the black and white levels. Without it the stream signals diff --git a/examples/camera_livekit_bridge/bridge/CMakeLists.txt b/examples/camera_livekit_bridge/bridge/CMakeLists.txt index c647bca7ef..1dbf931a71 100644 --- a/examples/camera_livekit_bridge/bridge/CMakeLists.txt +++ b/examples/camera_livekit_bridge/bridge/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries(camera_livekit_bridge camera_livekit_bridge::gpu_ops camera_livekit_bridge::nvenc camera_livekit_bridge::livekit_publisher + camera_livekit_bridge::chrony_clock camera_livekit_bridge::video_types CUDA::cudart Threads::Threads diff --git a/examples/camera_livekit_bridge/bridge/main.cpp b/examples/camera_livekit_bridge/bridge/main.cpp index 9c7360373b..6b23a97b10 100644 --- a/examples/camera_livekit_bridge/bridge/main.cpp +++ b/examples/camera_livekit_bridge/bridge/main.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,8 @@ void usage() std::cerr << "usage: camera_livekit_bridge --livekit-url URL --livekit-token-file FILE --stream NAME=TRACK[:DIV] [--stream ...]\n" " [--socket PATH] [--fps N] [--bitrate-mbps N] [--idr-period N] [--max-notice-age-ms N] [--gpu N] [--source-full-range]\n [--timestamp-basis monotonic_us|capture_monotonic_us|capture_unix_us]\n" + " capture_monotonic_us: --chrony-sender HOST[:PORT] --chrony-sender-refid HEX [--chrony-pc-port PORT]\n" + " [--chronyc PATH]\n" " NAME is a stream in camera_receiver's IPC catalog; TRACK is the LiveKit track name egocap expects;\n DIV shrinks the picture on the GPU by that integer factor before encoding.\n"; } @@ -67,6 +70,8 @@ int main(int argc, char** argv) std::vector streams; WorkerConfig defaults; int gpu = 0; + ChronyCaptureClockConfig chrony_config; + bool use_chrony = false; for (int i = 1; i < argc; ++i) { const std::string arg = argv[i]; @@ -103,15 +108,32 @@ int main(int argc, char** argv) std::cerr << "--timestamp-basis must be monotonic_us, capture_monotonic_us or capture_unix_us\n"; return 2; } - if (*basis == TimestampBasis::CaptureMonotonicUs) + defaults.timestamp_basis = *basis; + } + else if (arg == "--chrony-sender") + { + const std::string endpoint = next(); + const auto colon = endpoint.rfind(':'); + chrony_config.sender_host = colon == std::string::npos ? endpoint : endpoint.substr(0, colon); + if (colon != std::string::npos) + chrony_config.sender_port = std::atoi(endpoint.substr(colon + 1).c_str()); + if (chrony_config.sender_host.empty()) { - std::cerr - << "--timestamp-basis capture_monotonic_us is not available: it needs a measured sender-to-host clock " - "mapping, which is not implemented. Use monotonic_us (frame-ready time) or capture_unix_us.\n"; + std::cerr << "--chrony-sender needs HOST or HOST:PORT, got '" << endpoint << "'\n"; return 2; } - defaults.timestamp_basis = *basis; + use_chrony = true; } + else if (arg == "--chrony-sender-refid") + { + chrony_config.expected_sender_reference_id = next(); + for (auto& c : chrony_config.expected_sender_reference_id) + c = static_cast(std::toupper(static_cast(c))); + } + else if (arg == "--chrony-pc-port") + chrony_config.pc_port = std::atoi(next().c_str()); + else if (arg == "--chronyc") + chrony_config.chronyc_path = next(); else if (arg == "--gpu") gpu = std::atoi(next().c_str()); else if (arg == "--stream") @@ -146,6 +168,21 @@ int main(int argc, char** argv) return 2; } + if (defaults.timestamp_basis == TimestampBasis::CaptureMonotonicUs && !use_chrony) + { + std::cerr + << "--timestamp-basis capture_monotonic_us needs --chrony-sender HOST: the sender's capture time is only " + "used with chrony evidence for both hosts. Use monotonic_us for the frame-ready time or " + "capture_unix_us for a Unix-time sensor stamp.\n"; + return 2; + } + if (use_chrony && chrony_config.expected_sender_reference_id.empty()) + { + std::cerr << "--chrony-sender needs --chrony-sender-refid HEX: the reference id chronyd shows for its source " + "(chronyc tracking), so a host following another source is refused.\n"; + return 2; + } + std::signal(SIGINT, on_signal); std::signal(SIGTERM, on_signal); @@ -153,6 +190,14 @@ int main(int argc, char** argv) { CudaContext context(gpu); auto session = LiveKitSession::connect(url, read_token(token_file)); + std::unique_ptr chrony_clock; + if (use_chrony) + { + chrony_clock = std::make_unique(chrony_config); + chrony_clock->poll_once(); + chrony_clock->start(); + defaults.chrony_clock = chrony_clock.get(); + } std::vector> workers; for (const auto& spec : streams) { @@ -213,6 +258,25 @@ int main(int argc, char** argv) << static_cast(stats.capture_to_ready_us_total.load()) / ready_samples / 1000.0 << " ms mean " << stats.capture_to_ready_max_us.load() / 1000.0 << " ms max"; } + if (defaults.timestamp_basis == TimestampBasis::CaptureMonotonicUs) + { + std::cerr + << ", capture claims: " + << stats + .chrony_verdicts[static_cast(isaaccapture::timesync::Verdict::Qualified)] + .load() + << " qualified"; + for (std::size_t v = 1; v < 10; ++v) + { + if (stats.chrony_verdicts[v].load() > 0) + std::cerr + << ", " + << isaaccapture::timesync::to_string(static_cast(v)) + << " " << stats.chrony_verdicts[v].load(); + } + std::cerr << ", combined uncertainty " << stats.last_uncertainty_ns.load() / 1000.0 + << " us, clock generation " << stats.last_generation.load(); + } std::cerr << "\n"; last_frames[i] = frames; } diff --git a/examples/camera_livekit_bridge/bridge/stream_worker.cpp b/examples/camera_livekit_bridge/bridge/stream_worker.cpp index 77f5cb8f58..9ef8abffbc 100644 --- a/examples/camera_livekit_bridge/bridge/stream_worker.cpp +++ b/examples/camera_livekit_bridge/bridge/stream_worker.cpp @@ -328,6 +328,22 @@ struct StreamWorker::Impl const uint64_t sensor_sof_ns = wall_is_realtime ? unsigned_field(notice.metadata, "sensor_sof_ns") : 0; frame.user_timestamp_us = user_timestamp_us(config.timestamp_basis, sensor_sof_ns, ready_realtime_ns, ready_monotonic_ns); + if (config.timestamp_basis == TimestampBasis::CaptureMonotonicUs && config.chrony_clock != nullptr) + { + // The receiver's timing record: the sensor start of frame on the sender's CLOCK_REALTIME with the + // sender's stated uncertainty. A refusal publishes no capture time; the frame-ready time is never + // offered in its place. + const auto valid = notice.metadata.find("sensor_capture_valid"); + const bool sender_valid = valid != notice.metadata.end() && valid->is_boolean() && valid->get(); + const auto result = config.chrony_clock->qualify( + sender_valid, static_cast(unsigned_field(notice.metadata, "sensor_capture_realtime_ns")), + static_cast(unsigned_field(notice.metadata, "sensor_capture_uncertainty_ns"))); + ++stats.chrony_verdicts[static_cast(result.verdict)]; + stats.last_uncertainty_ns = result.uncertainty_ns; + stats.last_generation = result.generation; + if (result.verdict == isaaccapture::timesync::Verdict::Qualified) + frame.user_timestamp_us = static_cast(result.capture_monotonic_ns) / 1000; + } if (sensor_sof_ns != 0) { const int64_t age_ns = ready_realtime_ns - static_cast(sensor_sof_ns); diff --git a/examples/camera_livekit_bridge/bridge/stream_worker.hpp b/examples/camera_livekit_bridge/bridge/stream_worker.hpp index 2d9cf6c4e1..3ac9b158ea 100644 --- a/examples/camera_livekit_bridge/bridge/stream_worker.hpp +++ b/examples/camera_livekit_bridge/bridge/stream_worker.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -36,6 +37,8 @@ struct WorkerConfig bool source_full_range = false; /// What each frame's user_timestamp means; the consumer's streams file must name the same basis. TimestampBasis timestamp_basis = TimestampBasis::CaptureUnixUs; + /// Chrony evidence for capture_monotonic_us; shared by every stream and owned by the caller. + ChronyCaptureClock* chrony_clock = nullptr; }; struct WorkerStats @@ -59,6 +62,11 @@ struct WorkerStats std::atomic capture_to_ready_us_total{ 0 }; std::atomic capture_to_ready_samples{ 0 }; std::atomic capture_to_ready_max_us{ 0 }; + /// capture_monotonic_us: how many frames got each timesync::Verdict, and the last frame's combined uncertainty and + /// the PC clock generation it was converted under. + std::atomic chrony_verdicts[16] = {}; + std::atomic last_uncertainty_ns{ 0 }; + std::atomic last_generation{ 0 }; /// The latest target the transport reported (0 until it reports one). std::atomic target_bps{ 0 }; }; diff --git a/examples/camera_livekit_bridge/chrony_clock/CMakeLists.txt b/examples/camera_livekit_bridge/chrony_clock/CMakeLists.txt new file mode 100644 index 0000000000..c877afd849 --- /dev/null +++ b/examples/camera_livekit_bridge/chrony_clock/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Qualifies a sender CLOCK_REALTIME capture time against chrony's evidence for both hosts. +add_library(chrony_clock STATIC chrony_capture_clock.cpp) + +target_include_directories(chrony_clock + INTERFACE + $ +) + +target_link_libraries(chrony_clock + PUBLIC + timesync::timesync +) + +target_compile_options(chrony_clock PRIVATE -Wall -Wextra) + +add_library(camera_livekit_bridge::chrony_clock ALIAS chrony_clock) diff --git a/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp b/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp new file mode 100644 index 0000000000..b3da795b20 --- /dev/null +++ b/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/chrony_clock/chrony_capture_clock.hpp" + +#include + +namespace camera_bridge +{ + +namespace +{ + +isaaccapture::timesync::ChronyMonitorConfig monitor_config(const ChronyCaptureClockConfig& c, + const std::string& host, + int port) +{ + isaaccapture::timesync::ChronyMonitorConfig m; + m.chronyc_path = c.chronyc_path; + m.host = host; + m.port = port; + m.poll_interval = c.poll_interval; + m.command_timeout = c.command_timeout; + return m; +} + +isaaccapture::timesync::ClockReader resolve(isaaccapture::timesync::ClockReader clocks) +{ + if (!clocks.monotonic_ns || !clocks.realtime_ns) + return isaaccapture::timesync::system_clock_reader(); + return clocks; +} + +} // namespace + +ChronyCaptureClock::ChronyCaptureClock(ChronyCaptureClockConfig config, + isaaccapture::timesync::CommandRunner runner, + isaaccapture::timesync::ClockReader clocks) + : m_config(std::move(config)), + m_clocks(resolve(std::move(clocks))), + m_pc_monitor(monitor_config(m_config, m_config.pc_port != 0 ? "127.0.0.1" : "", m_config.pc_port), + runner, + m_clocks.monotonic_ns, + m_clocks.realtime_ns), + m_sender_monitor(monitor_config(m_config, m_config.sender_host, m_config.sender_port), + runner, + m_clocks.monotonic_ns, + m_clocks.realtime_ns) +{ + m_qualifier_config.budget_ns = m_config.budget_ns; + m_qualifier_config.expected_sender_reference_id = m_config.expected_sender_reference_id; +} + +ChronyCaptureClock::~ChronyCaptureClock() +{ + stop(); +} + +void ChronyCaptureClock::start() +{ + m_pc_monitor.start(); + m_sender_monitor.start(); +} + +void ChronyCaptureClock::stop() +{ + m_pc_monitor.stop(); + m_sender_monitor.stop(); +} + +void ChronyCaptureClock::poll_once() +{ + m_pc_monitor.poll_once(); + m_sender_monitor.poll_once(); +} + +isaaccapture::timesync::Qualification ChronyCaptureClock::qualify(bool sender_valid, + int64_t sender_realtime_ns, + int64_t upstream_uncertainty_ns) +{ + isaaccapture::timesync::SenderCapture capture; + capture.valid = sender_valid; + capture.realtime_ns = sender_realtime_ns; + capture.mapping_uncertainty_ns = upstream_uncertainty_ns; + + isaaccapture::timesync::LocalClockObservation observation; + int64_t now_ns = 0; + { + std::lock_guard lock(m_mutex); + observation = m_tracker.observe(isaaccapture::timesync::sample_local_clock(m_clocks)); + now_ns = m_clocks.monotonic_ns(); + } + if (observation.stepped) + { + // The frame that revealed a realtime step on this host makes no claim. + isaaccapture::timesync::Qualification refused; + refused.verdict = isaaccapture::timesync::Verdict::NoLocalClock; + refused.generation = observation.generation; + return refused; + } + return isaaccapture::timesync::qualify_capture(capture, m_sender_monitor.snapshot().evidence, + m_pc_monitor.snapshot().evidence, observation, now_ns, + m_qualifier_config); +} + +} // namespace camera_bridge diff --git a/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp b/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp new file mode 100644 index 0000000000..74826dfded --- /dev/null +++ b/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace camera_bridge +{ + +struct ChronyCaptureClockConfig +{ + std::string chronyc_path = "chronyc"; + /// The camera host's chronyd, queried over chronyc's read-only monitoring interface (the camera host must + /// `cmdallow` this machine). Required: without it the sender's synchronization is unknown and nothing qualifies. + std::string sender_host; + int sender_port = 0; + /// Upper-case hex reference id this machine's chronyd must have selected: the camera host as its NTP source. + std::string expected_sender_reference_id; + /// The command port of this machine's chronyd that measures the camera host, queried on the loopback address. 0 + /// queries the system chronyd over its local socket. + int pc_port = 0; + int64_t budget_ns = 5'000'000; + std::chrono::milliseconds poll_interval{ 1000 }; + std::chrono::milliseconds command_timeout{ 2000 }; +}; + +/** + * @brief Qualifies a sender CLOCK_REALTIME capture time against chrony's evidence and moves it onto this host's + * monotonic clock. + * + * It polls this host's chronyd and the camera host's in the background and samples the local realtime/monotonic pair + * per frame. A frame the evidence cannot vouch for gets a refusal verdict and no time, and the publisher sends it + * without a timestamp, which a recording consumer treats as a timing fault. A detected realtime step on this host + * refuses the frame that revealed it; frames after it qualify against the new clock generation. Thread-safe. + */ +class ChronyCaptureClock +{ +public: + /// The runner and clocks default to the real process runner and the system clocks; tests substitute fakes. + explicit ChronyCaptureClock(ChronyCaptureClockConfig config, + isaaccapture::timesync::CommandRunner runner = {}, + isaaccapture::timesync::ClockReader clocks = {}); + ~ChronyCaptureClock(); + + ChronyCaptureClock(const ChronyCaptureClock&) = delete; + ChronyCaptureClock& operator=(const ChronyCaptureClock&) = delete; + + void start(); + void stop(); + + /// One immediate poll of both monitors on the calling thread, for startup and tests. + void poll_once(); + + isaaccapture::timesync::Qualification qualify(bool sender_valid, + int64_t sender_realtime_ns, + int64_t upstream_uncertainty_ns); + +private: + ChronyCaptureClockConfig m_config; + isaaccapture::timesync::ClockReader m_clocks; + isaaccapture::timesync::QualifierConfig m_qualifier_config; + isaaccapture::timesync::ChronyMonitor m_pc_monitor; + isaaccapture::timesync::ChronyMonitor m_sender_monitor; + std::mutex m_mutex; + isaaccapture::timesync::LocalClockTracker m_tracker; +}; + +} // namespace camera_bridge diff --git a/examples/camera_livekit_bridge/tests/chrony_clock/CMakeLists.txt b/examples/camera_livekit_bridge/tests/chrony_clock/CMakeLists.txt new file mode 100644 index 0000000000..36d34069f0 --- /dev/null +++ b/examples/camera_livekit_bridge/tests/chrony_clock/CMakeLists.txt @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_executable(chrony_clock_tests test_chrony_capture_clock.cpp) + +target_link_libraries(chrony_clock_tests + PRIVATE + camera_livekit_bridge::chrony_clock + Threads::Threads +) + +target_compile_options(chrony_clock_tests PRIVATE -Wall -Wextra) + +add_test(NAME chrony_clock COMMAND chrony_clock_tests) diff --git a/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp b/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp new file mode 100644 index 0000000000..c12e6c517b --- /dev/null +++ b/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp @@ -0,0 +1,358 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using camera_bridge::ChronyCaptureClock; +using camera_bridge::ChronyCaptureClockConfig; +using isaaccapture::timesync::ClockReader; +using isaaccapture::timesync::CommandResult; +using isaaccapture::timesync::CommandRunner; +using isaaccapture::timesync::Qualification; +using isaaccapture::timesync::Verdict; + +static int g_failures = 0; + +#define CHECK(cond) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +namespace +{ + +constexpr int64_t kMs = 1'000'000; +constexpr int64_t kSec = 1'000'000'000; +const char* const kExpectedRefId = "C0A80001"; + +// chronyc -c tracking: refid,name,stratum,ref time,system time,last offset,rms offset,frequency,residual freq,skew, +// root delay,root dispersion,update interval,leap status. +std::string tracking_line(const std::string& refid, + int stratum, + double ref_time_s, + double root_dispersion_s, + const std::string& leap = "Normal") +{ + char buf[512]; + std::snprintf(buf, sizeof(buf), + "%s,host,%d,%.9f,0.000000011,-0.000001098,0.000008212,1.382,0.001,0.019," + "0.000100,%.9f,2.0,%s", + refid.c_str(), stratum, ref_time_s, root_dispersion_s, leap.c_str()); + return buf; +} + +// Answers by argv: no "-h", or "-h 127.0.0.1", is the PC's chronyd; any other "-h HOST" is the camera host's. +struct FakeChrony +{ + std::mutex mutex; + CommandResult pc; + CommandResult sender; + std::atomic pc_calls{ 0 }; + std::atomic sender_calls{ 0 }; + + CommandRunner runner() + { + return [this](const std::vector& argv, std::chrono::milliseconds) + { + bool remote = false; + for (size_t i = 0; i + 1 < argv.size(); ++i) + remote = remote || (argv[i] == "-h" && argv[i + 1] != "127.0.0.1"); + (remote ? sender_calls : pc_calls)++; + std::lock_guard lock(mutex); + return remote ? sender : pc; + }; + } +}; + +struct FakeClocks +{ + std::atomic monotonic{ 1'000 * kSec }; + std::atomic realtime{ 1'800'000'000 * kSec + 123'456'789 }; + + ClockReader reader() + { + return ClockReader{ [this] { return monotonic.load(); }, [this] { return realtime.load(); } }; + } + void advance(int64_t ns) + { + monotonic += ns; + realtime += ns; + } +}; + +CommandResult ok(const std::string& line) +{ + CommandResult r; + r.started = true; + r.exit_code = 0; + r.output = line + "\n"; + return r; +} + +ChronyCaptureClockConfig make_config() +{ + ChronyCaptureClockConfig c; + c.sender_host = "camera-host"; + c.expected_sender_reference_id = kExpectedRefId; + c.pc_port = 3323; + c.poll_interval = std::chrono::milliseconds(50); + return c; +} + +struct Rig +{ + FakeChrony chrony; + FakeClocks clocks; + ChronyCaptureClockConfig config = make_config(); + std::unique_ptr clock; + + double now_s() const + { + return static_cast(clocks.realtime.load()) * 1e-9; + } + void set_sender(const std::string& refid, int stratum, double disp_s, const std::string& leap = "Normal") + { + std::lock_guard lock(chrony.mutex); + chrony.sender = ok(tracking_line(refid, stratum, now_s() - 1.0, disp_s, leap)); + } + void set_pc(const std::string& refid, int stratum, double disp_s) + { + std::lock_guard lock(chrony.mutex); + chrony.pc = ok(tracking_line(refid, stratum, now_s() - 1.0, disp_s)); + } + void healthy() + { + set_sender("A9FEA97B", 3, 0.0005); // the camera host, following its own upstream + set_pc(kExpectedRefId, 4, 0.0005); // this machine's chronyd, following the camera host + } + void build() + { + clock = std::make_unique(config, chrony.runner(), clocks.reader()); + } + Qualification qualify(int64_t age_ns = 10 * kMs, bool valid = true, int64_t upstream_ns = 100'000) + { + return clock->qualify(valid, clocks.realtime.load() - age_ns, upstream_ns); + } +}; + +void test_happy_path() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->poll_once(); + CHECK(r.chrony.pc_calls == 1 && r.chrony.sender_calls == 1); + const int64_t realtime = r.clocks.realtime.load(); + const Qualification q = r.clock->qualify(true, realtime - 10 * kMs, 100'000); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.capture_monotonic_ns == (realtime - 10 * kMs) - (realtime - r.clocks.monotonic.load())); + CHECK(q.uncertainty_ns > 0 && q.uncertainty_ns <= r.config.budget_ns); + CHECK(q.generation >= 1); +} + +void test_camera_host_local_reference() +{ + // The camera host may serve its own clock (chrony `local`); only this machine has to follow it. + Rig r; + r.healthy(); + r.set_sender("7F7F0101", 10, 0.0005); + r.build(); + r.clock->poll_once(); + CHECK(r.qualify().verdict == Verdict::Qualified); +} + +void test_pc_local_reference_refused() +{ + Rig r; + r.healthy(); + r.set_pc("7F7F0101", 10, 0.0005); + r.build(); + r.clock->poll_once(); + CHECK(r.qualify().verdict == Verdict::Unsynchronised); +} + +void test_sender_invalid() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->poll_once(); + CHECK(r.qualify(10 * kMs, false).verdict == Verdict::SenderInvalid); +} + +void test_no_poll_yet() +{ + Rig r; + r.healthy(); + r.build(); + CHECK(r.qualify().verdict == Verdict::NoSenderEvidence); +} + +void test_sender_unreachable() +{ + Rig r; + r.healthy(); + r.chrony.sender = CommandResult{}; // started == false + r.build(); + r.clock->poll_once(); + CHECK(r.chrony.pc_calls == 1 && r.chrony.sender_calls == 1); + CHECK(r.qualify().verdict == Verdict::NoSenderEvidence); +} + +void test_reference_mismatch() +{ + Rig r; + r.healthy(); + r.set_pc("C0A80002", 4, 0.0005); // this machine follows something other than the camera host + r.build(); + r.clock->poll_once(); + CHECK(r.qualify().verdict == Verdict::ReferenceMismatch); +} + +void test_unsynchronised() +{ + Rig r; + r.healthy(); + r.set_sender("A9FEA97B", 3, 0.0005, "Not synchronised"); + r.build(); + r.clock->poll_once(); + CHECK(r.qualify().verdict == Verdict::Unsynchronised); +} + +void test_stale_evidence() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->poll_once(); + r.clocks.advance(11 * kSec); // max_evidence_age is 10 s + CHECK(r.qualify().verdict == Verdict::StaleEvidence); +} + +void test_realtime_step() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->poll_once(); + const Qualification first = r.qualify(); + CHECK(first.verdict == Verdict::Qualified); + + r.clocks.realtime += 50 * kMs; + const Qualification stepped = r.qualify(); + CHECK(stepped.verdict == Verdict::NoLocalClock); + CHECK(stepped.generation > first.generation); + + const int64_t realtime = r.clocks.realtime.load(); + const Qualification after = r.clock->qualify(true, realtime - 10 * kMs, 100'000); + CHECK(after.verdict == Verdict::Qualified); + CHECK(after.generation == stepped.generation); + CHECK(after.capture_monotonic_ns == (realtime - 10 * kMs) - (realtime - r.clocks.monotonic.load())); +} + +void test_over_budget() +{ + Rig r; + r.healthy(); + r.set_pc(kExpectedRefId, 4, 0.010); // the link dispersion is what this machine adds beyond the camera host + r.build(); + r.clock->poll_once(); + const Qualification q = r.qualify(); + CHECK(q.verdict == Verdict::OverBudget); + CHECK(q.uncertainty_ns > r.config.budget_ns); +} + +void test_start_stop() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->start(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + while ((r.chrony.pc_calls < 2 || r.chrony.sender_calls < 2) && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + CHECK(r.chrony.pc_calls >= 2); + CHECK(r.chrony.sender_calls >= 2); + CHECK(r.qualify().verdict == Verdict::Qualified); + + const auto t0 = std::chrono::steady_clock::now(); + r.clock->stop(); + CHECK(std::chrono::steady_clock::now() - t0 < std::chrono::milliseconds(500)); + const int polls = r.chrony.pc_calls; + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + CHECK(r.chrony.pc_calls == polls); +} + +void test_concurrent_qualify() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->poll_once(); + std::atomic bad{ 0 }; + std::vector threads; + for (int t = 0; t < 4; ++t) + { + threads.emplace_back( + [&] + { + for (int i = 0; i < 200; ++i) + { + const Qualification q = r.qualify(); + if (q.verdict == Verdict::Qualified) + { + if (q.uncertainty_ns <= 0) + ++bad; + } + else if (std::string(isaaccapture::timesync::to_string(q.verdict)) == "unknown") + { + ++bad; + } + } + }); + } + for (std::thread& t : threads) + t.join(); + CHECK(bad == 0); +} + +} // namespace + +int main() +{ + test_happy_path(); + test_camera_host_local_reference(); + test_pc_local_reference_refused(); + test_sender_invalid(); + test_no_poll_yet(); + test_sender_unreachable(); + test_reference_mismatch(); + test_unsynchronised(); + test_stale_evidence(); + test_realtime_step(); + test_over_budget(); + test_start_stop(); + test_concurrent_qualify(); + if (g_failures != 0) + { + std::fprintf(stderr, "%d check(s) failed\n", g_failures); + return 1; + } + std::printf("chrony_clock tests passed\n"); + return 0; +} diff --git a/examples/camera_livekit_bridge/tests/python/test_bridge_timing.py b/examples/camera_livekit_bridge/tests/python/test_bridge_timing.py index d74d8e8fa8..60d2c90bda 100644 --- a/examples/camera_livekit_bridge/tests/python/test_bridge_timing.py +++ b/examples/camera_livekit_bridge/tests/python/test_bridge_timing.py @@ -442,30 +442,50 @@ def test_monotonic_us_clock_and_feed(self): self._check_clock("monotonic_us") self._check_feed("monotonic_us") - def test_capture_monotonic_us_is_refused_by_bridge(self): - # Needs a measured sender-to-PC clock mapping the bridge does not have, so it must fail at startup. - token = self._tmp / "refuse.token" - token.write_text(make_token("unused", "bridge-refuse", True)) + def test_capture_unix_us_is_refused_by_egocap(self): + reason = self.refuse["reason"] + self.assertIsNotNone( + reason, "egocap accepted a capture_unix_us stream for recording" + ) + self.assertIn("monotonic clock", reason) + self.assertEqual(self.refuse["recordable"], 0) + + +@unittest.skipUnless( + BRIDGE_BIN and Path(str(BRIDGE_BIN)).exists(), "camera_livekit_bridge not built" +) +class BridgeCaptureBasisCliTest(unittest.TestCase): + """Startup refusals; the bridge exits before connecting, so no servers or GPU.""" + + def setUp(self): + self._td = tempfile.TemporaryDirectory(prefix="bridge_cli_") + self.addCleanup(self._td.cleanup) + self._tmp_dir = Path(self._td.name) + + def _run_bridge_cli(self, name: str, *extra: str): + token = self._tmp_dir / f"{name}.token" + token.write_text(make_token("unused", f"bridge-{name}", True)) cmd = [ str(BRIDGE_BIN), "--livekit-url", "ws://127.0.0.1:1", "--livekit-token-file", str(token), - "--socket", str(self._tmp / "none.sock"), "--stream", "head=head_track", - "--timestamp-basis", "capture_monotonic_us", + "--socket", str(self._tmp_dir / "none.sock"), "--stream", "head=head_track", + "--timestamp-basis", "capture_monotonic_us", *extra, ] # fmt: skip started = time.monotonic() out = subprocess.run( cmd, capture_output=True, text=True, check=False, timeout=30 ) self.assertLess(time.monotonic() - started, 15) + return out + + def test_capture_monotonic_us_requires_chrony_sender(self): + out = self._run_bridge_cli("no-sender") self.assertEqual(out.returncode, 2, out.stdout + out.stderr) - self.assertIn("monotonic_us", out.stderr) + self.assertIn("--chrony-sender", out.stderr) - def test_capture_unix_us_is_refused_by_egocap(self): - reason = self.refuse["reason"] - self.assertIsNotNone( - reason, "egocap accepted a capture_unix_us stream for recording" - ) - self.assertIn("monotonic clock", reason) - self.assertEqual(self.refuse["recordable"], 0) + def test_capture_monotonic_us_requires_sender_refid(self): + out = self._run_bridge_cli("no-refid", "--chrony-sender", "127.0.0.1") + self.assertEqual(out.returncode, 2, out.stdout + out.stderr) + self.assertIn("--chrony-sender-refid", out.stderr) if __name__ == "__main__": diff --git a/examples/camera_livekit_bridge/tests/timestamp_basis/test_timestamp_basis.cpp b/examples/camera_livekit_bridge/tests/timestamp_basis/test_timestamp_basis.cpp index 0ec75b4c02..946c2589be 100644 --- a/examples/camera_livekit_bridge/tests/timestamp_basis/test_timestamp_basis.cpp +++ b/examples/camera_livekit_bridge/tests/timestamp_basis/test_timestamp_basis.cpp @@ -48,13 +48,15 @@ int main() const uint64_t sof = static_cast(realtime - 80'000'000); check(user_timestamp_us(TimestampBasis::CaptureUnixUs, sof, realtime, monotonic) == sof / 1000, "capture_unix_us is the sensor time"); - check(user_timestamp_us(TimestampBasis::CaptureMonotonicUs, sof, realtime, monotonic) == - (5'000'000'000'000 - 80'000'000) / 1000, - "capture_monotonic_us is now minus the age"); + + // The adapter produces this basis; user_timestamp_us never does. + for (const uint64_t sensor : { uint64_t{ 0 }, sof, static_cast(realtime + 1'000'000) }) + check(user_timestamp_us(TimestampBasis::CaptureMonotonicUs, sensor, realtime, monotonic) == 0, + "capture_monotonic_us gives 0"); // No sensor time, a future one and an implausibly old one are not vouched for. - for (const auto basis : { TimestampBasis::CaptureUnixUs, TimestampBasis::CaptureMonotonicUs }) { + const auto basis = TimestampBasis::CaptureUnixUs; check(user_timestamp_us(basis, 0, realtime, monotonic) == 0, "no sensor time gives 0"); check(user_timestamp_us(basis, static_cast(realtime + 1'000'000), realtime, monotonic) == 0, "future sensor time gives 0"); @@ -66,10 +68,6 @@ int main() "sensor time at the age limit is accepted"); } - // An age larger than the monotonic clock itself cannot be placed on it. - check(user_timestamp_us(TimestampBasis::CaptureMonotonicUs, sof, realtime, 10'000'000) == 0, - "age beyond the monotonic clock gives 0"); - if (g_failures == 0) std::cout << "timestamp_basis: all checks passed\n"; return g_failures == 0 ? 0 : 1; diff --git a/examples/camera_livekit_bridge/video_types/inc/video_types/timestamp_basis.hpp b/examples/camera_livekit_bridge/video_types/inc/video_types/timestamp_basis.hpp index e2cca3ea5c..6ce3ed3103 100644 --- a/examples/camera_livekit_bridge/video_types/inc/video_types/timestamp_basis.hpp +++ b/examples/camera_livekit_bridge/video_types/inc/video_types/timestamp_basis.hpp @@ -15,9 +15,9 @@ enum class TimestampBasis { /// The publisher's CLOCK_MONOTONIC when the frame is published (frame-ready time). MonotonicUs, - /// The sensor's capture event on the publisher's CLOCK_MONOTONIC. The sensor time arrives as sender wall-clock - /// time, so this assumes the sender's clock agrees with this host's wall clock and carries no bound on that error. - /// The bridge does not offer it until a measured sender-to-host clock mapping exists. + /// The sensor's capture event on the publisher's CLOCK_MONOTONIC, converted from the sender's CLOCK_REALTIME with + /// chrony's synchronisation evidence for both hosts and refused when that evidence cannot vouch for it. The + /// conversion is made by the chrony adapter, not by `user_timestamp_us`. CaptureMonotonicUs, /// The sensor's capture event as Unix time on the sender's clock. CaptureUnixUs, @@ -42,7 +42,8 @@ inline constexpr int64_t kMaxCaptureAgeNs = 5'000'000'000; * * `sensor_sof_ns` is the sensor start of frame as Unix nanoseconds on the sender's clock (0 when unknown); * `realtime_now_ns` and `monotonic_now_ns` are this host's CLOCK_REALTIME and CLOCK_MONOTONIC read together. - * For the capture bases a missing, future or implausibly old sensor time gives 0, never a guess. + * For the capture bases a missing, future or implausibly old sensor time gives 0, never a guess. The monotonic capture + * basis needs measured synchronisation evidence, which this function does not have, so it always gives 0. */ inline uint64_t user_timestamp_us(TimestampBasis basis, uint64_t sensor_sof_ns, @@ -51,15 +52,12 @@ inline uint64_t user_timestamp_us(TimestampBasis basis, { if (basis == TimestampBasis::MonotonicUs) return monotonic_now_ns > 0 ? static_cast(monotonic_now_ns) / 1000 : 0; - if (sensor_sof_ns == 0) + if (basis == TimestampBasis::CaptureMonotonicUs || sensor_sof_ns == 0) return 0; const int64_t age_ns = realtime_now_ns - static_cast(sensor_sof_ns); if (age_ns < 0 || age_ns > kMaxCaptureAgeNs) return 0; - if (basis == TimestampBasis::CaptureUnixUs) - return sensor_sof_ns / 1000; - const int64_t capture_ns = monotonic_now_ns - age_ns; - return capture_ns > 0 ? static_cast(capture_ns) / 1000 : 0; + return sensor_sof_ns / 1000; } } // namespace camera_bridge diff --git a/examples/camera_streamer/mock_sender/README.md b/examples/camera_streamer/mock_sender/README.md index 754a3dd0f1..3faeeb4beb 100644 --- a/examples/camera_streamer/mock_sender/README.md +++ b/examples/camera_streamer/mock_sender/README.md @@ -30,6 +30,7 @@ tick, as in the real sender. | `--disconnect-at SECONDS` | Close the receiver connection after that long. | | `--clock-offset-ms`, `--clock-drift-ppm` | Offset (default 0, as with chrony-synced clocks) and drift of the synthetic sender clock. | | `--sof-lag-ms` | Synthetic sensor start-of-frame to capture-ready lag (default 20, range 0-500). | +| `--timing-ext`, `--sensor-uncertainty-us` | Send the AVS1 timing record on every bundle: the synthetic sensor start of frame (capture-ready minus the lag) on the mock's wall clock, with the stated uncertainty (default 500 us). The mock's wall clock is this host's clock plus `--clock-offset-ms` and `--clock-drift-ppm`. | Timing fields come from the synthetic clock only. Every frame carries a synthetic sensor start-of-frame TSC, in a TSC domain of the mock's own, so the diff --git a/examples/camera_streamer/mock_sender/mock_sender.py b/examples/camera_streamer/mock_sender/mock_sender.py index bc5c895d71..7e492bb828 100755 --- a/examples/camera_streamer/mock_sender/mock_sender.py +++ b/examples/camera_streamer/mock_sender/mock_sender.py @@ -43,6 +43,13 @@ SAMPLE_SOF_TSC_VALID = 0x0002 BUNDLE_SENDER_CLOCK_VALID = 0x0001 # realtime domain; the TAI flag stays clear BUNDLE_SENDER_TSC_VALID = 0x0004 +BUNDLE_TIMING_EXT = ( + 0x0008 # one TIMING record per sample follows the sample descriptors +) +# Capture time, uncertainty, anchor age, flags, clock domain, reserved. +TIMING = struct.Struct("!QIIHHI") +TIMING_VALID = 0x0001 +TIMING_DOMAIN_REALTIME = 2 TSC_BASE_NS = 7_000_000_000_000 # arbitrary origin of the synthetic TSC domain LOOP_SECONDS = 12 GOP_SECONDS = 2 @@ -200,9 +207,14 @@ def build_record( enqueue_ns: int, ready_tsc_ns: int, sof_lag_ns: int, + timing_uncertainty_ns: int | None = None, ) -> bytes: - """One length-prefixed AVS1 record: every stream has a descriptor, present or not.""" - streams, samples, body = [], [], b"" + """One length-prefixed AVS1 record: every stream has a descriptor, present or not. + + With ``timing_uncertainty_ns`` every sample also carries a timing record: the synthetic sensor start of frame on the + mock's wall clock (the ready time minus the lag) for a present sample, all zero for an absent one. + """ + streams, samples, timings, body = [], [], [], b"" for index, (cursor, payload) in enumerate(zip(cursors, payloads)): present = payload is not None streams.append( @@ -222,13 +234,24 @@ def build_record( ) ) # fmt: skip body += payload + timings.append( + TIMING.pack( + ready_ns - sof_lag_ns, timing_uncertainty_ns or 0, 0, + TIMING_VALID, TIMING_DOMAIN_REALTIME, 0, + ) + ) # fmt: skip else: samples.append( SAMPLE.pack(index, 0, 0, 0, 0, index, 0, 0, 0, 0, 0, 0, 0, 0, 0) ) + timings.append(TIMING.pack(0, 0, 0, 0, 0, 0)) descriptors = b"".join(streams) + b"".join(samples) + bundle_flags = BUNDLE_SENDER_CLOCK_VALID | BUNDLE_SENDER_TSC_VALID + if timing_uncertainty_ns is not None: + descriptors += b"".join(timings) + bundle_flags |= BUNDLE_TIMING_EXT header = HEADER.pack( - b"AVS1", 1, BUNDLE_SENDER_CLOCK_VALID | BUNDLE_SENDER_TSC_VALID, HEADER.size, + b"AVS1", 1, bundle_flags, HEADER.size, len(streams), len(samples), bundle_sequence, session_id, ready_ns, enqueue_ns, len(descriptors), len(body), ready_tsc_ns, ) # fmt: skip @@ -248,10 +271,12 @@ def __init__( pauses: list[Pause], disconnects_s: list[float], sof_lag_ns: int = 20_000_000, + timing_uncertainty_ns: int | None = None, ) -> None: self.specs = specs self.clock = clock self.sof_lag_ns = sof_lag_ns + self.timing_uncertainty_ns = timing_uncertainty_ns self.pauses = pauses self.disconnects_s = sorted(disconnects_s) self._units = { @@ -337,6 +362,7 @@ def _stream_to(self, conn: socket.socket, stop_at: float | None = None) -> None: self.clock.now_ns(), ready_tsc_ns, self.sof_lag_ns, + self.timing_uncertainty_ns, ) conn.sendall(record) self.bundles_sent += 1 @@ -400,6 +426,14 @@ def main(argv: list[str] | None = None) -> int: help="synthetic sensor start-of-frame to capture-ready lag; must be in [0, 500]", ) # fmt: skip parser.add_argument("--clock-drift-ppm", type=float, default=0.0) + parser.add_argument( + "--timing-ext", action="store_true", + help="send the AVS1 timing record: the synthetic sensor start of frame on the mock's wall clock", + ) # fmt: skip + parser.add_argument( + "--sensor-uncertainty-us", type=float, default=500.0, + help="the uncertainty the timing record states for the sensor-to-wall-clock mapping", + ) # fmt: skip parser.add_argument( "--pause", type=parse_pause, action="append", default=[], metavar="NAME@START+SECONDS", help="stop one stream for a while; it resumes at an IDR", @@ -435,6 +469,7 @@ def main(argv: list[str] | None = None) -> int: args.pause, args.disconnect_at, int(args.sof_lag_ms * 1e6), + int(args.sensor_uncertainty_us * 1000) if args.timing_ext else None, ) listener = socket.create_server((args.bind, args.port), reuse_port=False) logger.info( diff --git a/examples/camera_streamer/mock_sender/tests/test_mock_sender.py b/examples/camera_streamer/mock_sender/tests/test_mock_sender.py index 1a5c5f062b..28f458348c 100755 --- a/examples/camera_streamer/mock_sender/tests/test_mock_sender.py +++ b/examples/camera_streamer/mock_sender/tests/test_mock_sender.py @@ -194,6 +194,44 @@ def padded(*args, **kwargs): self.assertGreater(sender.bundles_sent, 0) +class TimingRecordTest(unittest.TestCase): + def _record(self, uncertainty, payloads): + cursors = [ms._Cursor(SPEC, [(b"x", True)]) for _ in payloads] + return ms.build_record( + cursors, + payloads, + 1, + 7, + 2_000_000_000, + 2_000_000_100, + 5_000_000_000, + 20_000_000, + uncertainty, + ) + + def _header(self, record): + return ms.HEADER.unpack_from(record, 4) + + def test_no_timing_extension_by_default(self): + header = self._header(self._record(None, [b"abc"])) + self.assertFalse(header[2] & ms.BUNDLE_TIMING_EXT) + + def test_timing_extension_carries_realtime_capture_per_sample(self): + record = self._record(3000, [b"abc", None]) + header = self._header(record) + self.assertTrue(header[2] & ms.BUNDLE_TIMING_EXT) + descriptors = record[4 + ms.HEADER.size :][: header[10]] + timings = descriptors[-2 * ms.TIMING.size :] + present = ms.TIMING.unpack_from(timings, 0) + self.assertEqual( + present, + (1_980_000_000, 3000, 0, ms.TIMING_VALID, ms.TIMING_DOMAIN_REALTIME, 0), + ) + self.assertEqual( + ms.TIMING.unpack_from(timings, ms.TIMING.size), (0, 0, 0, 0, 0, 0) + ) + + class ParseArgsTest(unittest.TestCase): def test_stream_spec(self): self.assertEqual( From a9273f12cd64de5a7069bfe767584414c3e9c131 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 10:59:51 -0500 Subject: [PATCH 02/10] camera_livekit_bridge: publish the capture time of a frame that misses the precision target A frame over the precision budget kept no capture time, so a consumer could not use it. Publish the converted instant for every usable verdict; only a conversion that cannot be vouched for publishes none. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/camera_livekit_bridge/bridge/stream_worker.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/camera_livekit_bridge/bridge/stream_worker.cpp b/examples/camera_livekit_bridge/bridge/stream_worker.cpp index 9ef8abffbc..cd617fe790 100644 --- a/examples/camera_livekit_bridge/bridge/stream_worker.cpp +++ b/examples/camera_livekit_bridge/bridge/stream_worker.cpp @@ -331,8 +331,8 @@ struct StreamWorker::Impl if (config.timestamp_basis == TimestampBasis::CaptureMonotonicUs && config.chrony_clock != nullptr) { // The receiver's timing record: the sensor start of frame on the sender's CLOCK_REALTIME with the - // sender's stated uncertainty. A refusal publishes no capture time; the frame-ready time is never - // offered in its place. + // sender's stated uncertainty. A conversion that cannot be vouched for publishes no capture time; the + // frame-ready time is never offered in its place. const auto valid = notice.metadata.find("sensor_capture_valid"); const bool sender_valid = valid != notice.metadata.end() && valid->is_boolean() && valid->get(); const auto result = config.chrony_clock->qualify( @@ -341,7 +341,9 @@ struct StreamWorker::Impl ++stats.chrony_verdicts[static_cast(result.verdict)]; stats.last_uncertainty_ns = result.uncertainty_ns; stats.last_generation = result.generation; - if (result.verdict == isaaccapture::timesync::Verdict::Qualified) + // A precision miss still has a converted instant with a stated bound, so the frame keeps its capture + // time; only a conversion that cannot be vouched for publishes none. + if (result.usable()) frame.user_timestamp_us = static_cast(result.capture_monotonic_ns) / 1000; } if (sensor_sof_ns != 0) From 3a9d4bff7e9b1ba9ced022795827251ea9482d3c Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:16:52 -0500 Subject: [PATCH 03/10] camera_livekit_bridge: document a capture over the precision target as published A conversion that cannot be vouched for publishes no timestamp; one that passes every check but is over the precision target keeps its timestamp and is only counted differently in the statistics. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/camera_livekit_bridge/README.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/camera_livekit_bridge/README.md b/examples/camera_livekit_bridge/README.md index a3b1f11130..f1b1200dae 100644 --- a/examples/camera_livekit_bridge/README.md +++ b/examples/camera_livekit_bridge/README.md @@ -90,8 +90,12 @@ a consumer that needs a capture time says so instead of using a guess. (`sensor_capture_realtime_ns`, `sensor_capture_uncertainty_ns`, `sensor_capture_valid`) and the time-sync library in `src/core/timesync`. Each frame is converted from the sender's `CLOCK_REALTIME` to this host's monotonic clock, or refused with a typed reason, using chronyd's own report for both hosts, a bracketed read of this host's clocks, and the -sender's stated uncertainty. A refused frame is published without a timestamp, and the frame-ready time is never -offered in its place. It needs: +sender's stated uncertainty. Two outcomes are different in kind. A conversion that cannot be vouched for (a stale or +unsynchronised chrony report, another reference, an implausible age, an invalid sender record) is published without a +timestamp, and the frame-ready time is never offered in its place. A conversion that passes every one of those checks +but whose combined uncertainty is over the precision target (`over_budget`) still has a capture instant with a known +bound, so it is published with that timestamp; the target classifies the frame in the statistics and does not remove its +time. It needs: | Option | Meaning | | --- | --- | @@ -101,9 +105,10 @@ offered in its place. It needs: | `--chronyc PATH` | The `chronyc` binary (default `chronyc` on `PATH`). | `src/core/timesync/README.md` describes the arrangement (this host measures the camera host, which serves its own clock) and how to set it up with `timesync_tool.py`. The -`[stats]` line gains the number of frames that qualified, a count per refusal reason, the last combined uncertainty and -the clock generation the frame was converted under. The caller of a recording should treat a frame without a -timestamp as a timing fault. +`[stats]` line gains the number of frames that met the precision target, a count per other verdict (`over_budget` frames +carry a timestamp; the rest do not), the last combined uncertainty and the clock generation the frame was converted +under. These are runtime diagnostics only and are not recorded with the video. The caller of a recording should treat a +frame without a timestamp as a timing fault. `--source-full-range` says the receiver's pictures use the full 0-255 range, as SIPL sources do; the encoded stream then signals full range, so players and recordings keep the black and white levels. Without it the stream signals From 8233b63fb6daa52ae2705f0d3105348f34845781 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:29:42 -0500 Subject: [PATCH 04/10] camera_livekit_bridge: do not reuse chrony evidence read before a clock step A detected PC wall-clock step refused only the frame that revealed it; the next frame reused chrony snapshots collected before the step and qualified with a capture time off by the step. Record the step time and ignore evidence read at or before it, so frames stay without a timestamp until both hosts have been read again. A take in progress then ends with the missing-timestamp fault. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../chrony_clock/chrony_capture_clock.cpp | 20 ++++++++++++++++--- .../inc/chrony_clock/chrony_capture_clock.hpp | 8 +++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp b/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp index b3da795b20..0e8044ef1e 100644 --- a/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp +++ b/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp @@ -85,10 +85,14 @@ isaaccapture::timesync::Qualification ChronyCaptureClock::qualify(bool sender_va isaaccapture::timesync::LocalClockObservation observation; int64_t now_ns = 0; + int64_t evidence_floor_ns = 0; { std::lock_guard lock(m_mutex); observation = m_tracker.observe(isaaccapture::timesync::sample_local_clock(m_clocks)); now_ns = m_clocks.monotonic_ns(); + if (observation.stepped) + m_evidence_floor_ns = now_ns; + evidence_floor_ns = m_evidence_floor_ns; } if (observation.stepped) { @@ -98,9 +102,19 @@ isaaccapture::timesync::Qualification ChronyCaptureClock::qualify(bool sender_va refused.generation = observation.generation; return refused; } - return isaaccapture::timesync::qualify_capture(capture, m_sender_monitor.snapshot().evidence, - m_pc_monitor.snapshot().evidence, observation, now_ns, - m_qualifier_config); + // Chrony evidence read at or before a step describes the clock as it was, so it cannot support a frame converted + // against the clock as it is. Frames stay unqualified until both hosts have been read again after the step. + auto sender_evidence = m_sender_monitor.snapshot().evidence; + auto pc_evidence = m_pc_monitor.snapshot().evidence; + if (evidence_floor_ns != 0) + { + if (sender_evidence && sender_evidence->observed_monotonic_ns <= evidence_floor_ns) + sender_evidence.reset(); + if (pc_evidence && pc_evidence->observed_monotonic_ns <= evidence_floor_ns) + pc_evidence.reset(); + } + return isaaccapture::timesync::qualify_capture( + capture, sender_evidence, pc_evidence, observation, now_ns, m_qualifier_config); } } // namespace camera_bridge diff --git a/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp b/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp index 74826dfded..4dc6c34419 100644 --- a/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp +++ b/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp @@ -39,7 +39,10 @@ struct ChronyCaptureClockConfig * It polls this host's chronyd and the camera host's in the background and samples the local realtime/monotonic pair * per frame. A frame the evidence cannot vouch for gets a refusal verdict and no time, and the publisher sends it * without a timestamp, which a recording consumer treats as a timing fault. A detected realtime step on this host - * refuses the frame that revealed it; frames after it qualify against the new clock generation. Thread-safe. + * refuses the frame that revealed it and every frame after it until both chronyds have been polled again after the + * step: evidence read before it described the old clock and is never reused, so the frames in between carry no + * timestamp and a take in progress ends with that fault. Frames after that qualify against the new clock generation. + * Thread-safe. */ class ChronyCaptureClock { @@ -71,6 +74,9 @@ class ChronyCaptureClock isaaccapture::timesync::ChronyMonitor m_sender_monitor; std::mutex m_mutex; isaaccapture::timesync::LocalClockTracker m_tracker; + /// Monotonic time of the latest detected realtime step on this host; chrony evidence read at or before it is not + /// used. + int64_t m_evidence_floor_ns = 0; }; } // namespace camera_bridge From 0d9e60653291ff7e852354b8cf594e093444626a Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:31:03 -0500 Subject: [PATCH 05/10] camera_livekit_bridge: test that evidence read before a clock step is not reused With the sender capture timestamp not moving with the PC clock: a step is refused, the next frames without a new poll are not usable, and qualification recovers only after both hosts are read again, with the capture time correct on the new clock. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../test_chrony_capture_clock.cpp | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp b/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp index c12e6c517b..066d5c3120 100644 --- a/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp +++ b/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp @@ -258,6 +258,9 @@ void test_realtime_step() CHECK(stepped.verdict == Verdict::NoLocalClock); CHECK(stepped.generation > first.generation); + // Evidence read before the step is not reused; both hosts are read again after it. + r.clocks.advance(kMs); + r.clock->poll_once(); const int64_t realtime = r.clocks.realtime.load(); const Qualification after = r.clock->qualify(true, realtime - 10 * kMs, 100'000); CHECK(after.verdict == Verdict::Qualified); @@ -265,6 +268,41 @@ void test_realtime_step() CHECK(after.capture_monotonic_ns == (realtime - 10 * kMs) - (realtime - r.clocks.monotonic.load())); } +void test_realtime_step_discards_evidence_read_before_it() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->poll_once(); + // The sender's capture timestamp is fixed in realtime and does not move with this machine's clock. + const int64_t capture_realtime = r.clocks.realtime.load() - 80 * kMs; + const Qualification first = r.clock->qualify(true, capture_realtime, 100'000); + CHECK(first.verdict == Verdict::Qualified); + + r.clocks.realtime -= 20 * kMs; // only this machine's realtime steps + const Qualification stepped = r.clock->qualify(true, capture_realtime, 100'000); + CHECK(stepped.verdict == Verdict::NoLocalClock); + CHECK(stepped.generation > first.generation); + + // No poll since the step: the evidence describes the old clock and must not qualify the frame. + const Qualification again = r.clock->qualify(true, capture_realtime, 100'000); + CHECK(!again.usable()); + CHECK(again.verdict == Verdict::NoSenderEvidence || again.verdict == Verdict::NoPcEvidence); + const Qualification third = r.clock->qualify(true, capture_realtime, 100'000); + CHECK(!third.usable()); + + // Both hosts read again after the step: qualification recovers on the new clock. + r.clocks.monotonic += kMs; + r.clocks.realtime += kMs; + r.healthy(); + r.clock->poll_once(); + const Qualification recovered = r.clock->qualify(true, capture_realtime, 100'000); + CHECK(recovered.verdict == Verdict::Qualified); + CHECK(recovered.generation == stepped.generation); + const int64_t offset = r.clocks.realtime.load() - r.clocks.monotonic.load(); + CHECK(recovered.capture_monotonic_ns == capture_realtime - offset); +} + void test_over_budget() { Rig r; @@ -345,6 +383,7 @@ int main() test_unsynchronised(); test_stale_evidence(); test_realtime_step(); + test_realtime_step_discards_evidence_read_before_it(); test_over_budget(); test_start_stop(); test_concurrent_qualify(); From 32928816ede2f0e1a781f52cbed10ad5a858de60 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:53:49 -0500 Subject: [PATCH 06/10] camera_livekit_bridge: hold timestamps off for a while after a wall-clock step A take can only notice a step on this host through a frame without a timestamp or through silence, and the recorder fails a take after 5 s of silence. Refuse every frame for step_hold_ns (8 s) after a step, whatever the chrony evidence says, so a take cannot continue across the step even if the media path drops the refused frames. Evidence read before the step is still never reused. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/camera_livekit_bridge/README.md | 1 + .../chrony_clock/chrony_capture_clock.cpp | 10 ++- .../inc/chrony_clock/chrony_capture_clock.hpp | 12 ++- .../test_chrony_capture_clock.cpp | 86 +++++++++++++++++-- 4 files changed, 98 insertions(+), 11 deletions(-) diff --git a/examples/camera_livekit_bridge/README.md b/examples/camera_livekit_bridge/README.md index f1b1200dae..48814c3413 100644 --- a/examples/camera_livekit_bridge/README.md +++ b/examples/camera_livekit_bridge/README.md @@ -105,6 +105,7 @@ time. It needs: | `--chronyc PATH` | The `chronyc` binary (default `chronyc` on `PATH`). | `src/core/timesync/README.md` describes the arrangement (this host measures the camera host, which serves its own clock) and how to set it up with `timesync_tool.py`. The +A realtime step of this host's wall clock is a timing fault: the frame that reveals it and every frame for the next 8 s are published without a timestamp (a longer silence than a recording take tolerates, so a take cannot continue across the step even if some of those frames are lost on the way), and chrony readings taken before the step are never reused. The `[stats]` line gains the number of frames that met the precision target, a count per other verdict (`over_budget` frames carry a timestamp; the rest do not), the last combined uncertainty and the clock generation the frame was converted under. These are runtime diagnostics only and are not recorded with the video. The caller of a recording should treat a diff --git a/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp b/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp index 0e8044ef1e..819568cb8f 100644 --- a/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp +++ b/examples/camera_livekit_bridge/chrony_clock/chrony_capture_clock.cpp @@ -86,17 +86,23 @@ isaaccapture::timesync::Qualification ChronyCaptureClock::qualify(bool sender_va isaaccapture::timesync::LocalClockObservation observation; int64_t now_ns = 0; int64_t evidence_floor_ns = 0; + bool held = false; { std::lock_guard lock(m_mutex); observation = m_tracker.observe(isaaccapture::timesync::sample_local_clock(m_clocks)); now_ns = m_clocks.monotonic_ns(); if (observation.stepped) + { m_evidence_floor_ns = now_ns; + m_hold_until_ns = now_ns + m_config.step_hold_ns; + } evidence_floor_ns = m_evidence_floor_ns; + held = now_ns < m_hold_until_ns; } - if (observation.stepped) + if (observation.stepped || held) { - // The frame that revealed a realtime step on this host makes no claim. + // The frame that revealed a realtime step on this host makes no claim, and neither does any frame within the + // hold after it. isaaccapture::timesync::Qualification refused; refused.verdict = isaaccapture::timesync::Verdict::NoLocalClock; refused.generation = observation.generation; diff --git a/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp b/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp index 4dc6c34419..82416cea45 100644 --- a/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp +++ b/examples/camera_livekit_bridge/chrony_clock/inc/chrony_clock/chrony_capture_clock.hpp @@ -28,6 +28,11 @@ struct ChronyCaptureClockConfig /// queries the system chronyd over its local socket. int pc_port = 0; int64_t budget_ns = 5'000'000; + /// After a realtime step on this host no frame is given a timestamp for this long, whatever the evidence says. A + /// take can only notice the step through a frame that has no timestamp, or through silence; holding timestamps + /// off for longer than a recording consumer tolerates silence (5 s) means a take cannot continue across a step even + /// if the media path drops the frames that were refused. Frames still flow without timestamps meanwhile. + int64_t step_hold_ns = 8'000'000'000; std::chrono::milliseconds poll_interval{ 1000 }; std::chrono::milliseconds command_timeout{ 2000 }; }; @@ -40,8 +45,9 @@ struct ChronyCaptureClockConfig * per frame. A frame the evidence cannot vouch for gets a refusal verdict and no time, and the publisher sends it * without a timestamp, which a recording consumer treats as a timing fault. A detected realtime step on this host * refuses the frame that revealed it and every frame after it until both chronyds have been polled again after the - * step: evidence read before it described the old clock and is never reused, so the frames in between carry no - * timestamp and a take in progress ends with that fault. Frames after that qualify against the new clock generation. + * step: evidence read before it described the old clock and is never reused, and for `step_hold_ns` after it no frame + * is timestamped at all, so the frames in between carry no timestamp and a take in progress ends with that fault even + * if some of those frames are lost on the way to it. Frames after that qualify against the new clock generation. * Thread-safe. */ class ChronyCaptureClock @@ -77,6 +83,8 @@ class ChronyCaptureClock /// Monotonic time of the latest detected realtime step on this host; chrony evidence read at or before it is not /// used. int64_t m_evidence_floor_ns = 0; + /// Monotonic time until which no frame is timestamped after a detected step. + int64_t m_hold_until_ns = 0; }; } // namespace camera_bridge diff --git a/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp b/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp index 066d5c3120..fc80fe2749 100644 --- a/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp +++ b/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp @@ -258,9 +258,14 @@ void test_realtime_step() CHECK(stepped.verdict == Verdict::NoLocalClock); CHECK(stepped.generation > first.generation); - // Evidence read before the step is not reused; both hosts are read again after it. + // Evidence read before the step is not reused; both hosts are read again after it. The hold still refuses frames. r.clocks.advance(kMs); r.clock->poll_once(); + CHECK(r.qualify().verdict == Verdict::NoLocalClock); + + // Once the hold has elapsed and the evidence is fresh, qualification recovers on the new clock. + r.clocks.advance(r.config.step_hold_ns); + r.clock->poll_once(); const int64_t realtime = r.clocks.realtime.load(); const Qualification after = r.clock->qualify(true, realtime - 10 * kMs, 100'000); CHECK(after.verdict == Verdict::Qualified); @@ -287,20 +292,85 @@ void test_realtime_step_discards_evidence_read_before_it() // No poll since the step: the evidence describes the old clock and must not qualify the frame. const Qualification again = r.clock->qualify(true, capture_realtime, 100'000); CHECK(!again.usable()); - CHECK(again.verdict == Verdict::NoSenderEvidence || again.verdict == Verdict::NoPcEvidence); + CHECK(again.generation == stepped.generation); const Qualification third = r.clock->qualify(true, capture_realtime, 100'000); CHECK(!third.usable()); - // Both hosts read again after the step: qualification recovers on the new clock. - r.clocks.monotonic += kMs; - r.clocks.realtime += kMs; + // Both hosts read again after the step, and the hold has elapsed: qualification recovers on the new clock. + r.clocks.advance(r.config.step_hold_ns + kMs); r.healthy(); r.clock->poll_once(); - const Qualification recovered = r.clock->qualify(true, capture_realtime, 100'000); + const int64_t later_capture_realtime = r.clocks.realtime.load() - 80 * kMs; + const Qualification recovered = r.clock->qualify(true, later_capture_realtime, 100'000); CHECK(recovered.verdict == Verdict::Qualified); CHECK(recovered.generation == stepped.generation); const int64_t offset = r.clocks.realtime.load() - r.clocks.monotonic.load(); - CHECK(recovered.capture_monotonic_ns == capture_realtime - offset); + CHECK(recovered.capture_monotonic_ns == later_capture_realtime - offset); +} + +// No frame is timestamped for step_hold_ns after a step, whatever the evidence says, and every refusal carries the +// post-step generation. +void test_step_hold() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->poll_once(); + const Qualification first = r.qualify(); + CHECK(first.verdict == Verdict::Qualified); + + r.clocks.realtime += 50 * kMs; + const Qualification stepped = r.qualify(); + CHECK(stepped.verdict == Verdict::NoLocalClock); + CHECK(stepped.generation > first.generation); + + // Fresh post-step evidence does not lift the hold. + r.clocks.advance(kSec); + r.healthy(); + r.clock->poll_once(); + const Qualification during = r.qualify(); + CHECK(during.verdict == Verdict::NoLocalClock); + CHECK(!during.usable()); + CHECK(during.generation == stepped.generation); + + // Just before the hold ends, still refused. + r.clocks.advance(r.config.step_hold_ns - kSec - 10 * kMs); + r.healthy(); + r.clock->poll_once(); + const Qualification before_end = r.qualify(); + CHECK(!before_end.usable()); + CHECK(before_end.verdict == Verdict::NoLocalClock); + CHECK(before_end.generation == stepped.generation); + + // Just after, with fresh evidence, usable and consistent with the new clock. + r.clocks.advance(20 * kMs); + r.healthy(); + r.clock->poll_once(); + const int64_t realtime = r.clocks.realtime.load(); + const Qualification after = r.clock->qualify(true, realtime - 10 * kMs, 100'000); + CHECK(after.verdict == Verdict::Qualified); + CHECK(after.generation == stepped.generation); + CHECK(after.capture_monotonic_ns == (realtime - 10 * kMs) - (realtime - r.clocks.monotonic.load())); +} + +void test_step_hold_configurable() +{ + Rig r; + r.config.step_hold_ns = kMs; + r.healthy(); + r.build(); + r.clock->poll_once(); + CHECK(r.qualify().verdict == Verdict::Qualified); + + r.clocks.realtime += 50 * kMs; + const Qualification stepped = r.qualify(); + CHECK(stepped.verdict == Verdict::NoLocalClock); + + r.clocks.advance(2 * kMs); + r.clock->poll_once(); + const Qualification after = r.qualify(); + CHECK(after.verdict == Verdict::Qualified); + CHECK(after.generation == stepped.generation); } void test_over_budget() @@ -384,6 +454,8 @@ int main() test_stale_evidence(); test_realtime_step(); test_realtime_step_discards_evidence_read_before_it(); + test_step_hold(); + test_step_hold_configurable(); test_over_budget(); test_start_stop(); test_concurrent_qualify(); From 8508ad3d221ca4e67b4a73c29a59adf75ffd2ad1 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 18:25:07 -0500 Subject: [PATCH 07/10] camera_livekit_bridge: drop the Unix-time timestamp basis The bridge publishes either the frame-ready time or the chrony-qualified capture time, both on the publisher's monotonic clock. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/camera_livekit_bridge/README.md | 9 +---- .../camera_livekit_bridge/bridge/main.cpp | 7 ++-- .../bridge/stream_worker.cpp | 5 ++- .../bridge/stream_worker.hpp | 2 +- .../tests/python/test_bridge_timing.py | 32 ---------------- .../timestamp_basis/test_timestamp_basis.cpp | 38 +------------------ .../inc/video_types/timestamp_basis.hpp | 30 +-------------- 7 files changed, 11 insertions(+), 112 deletions(-) diff --git a/examples/camera_livekit_bridge/README.md b/examples/camera_livekit_bridge/README.md index 48814c3413..3b3fdad286 100644 --- a/examples/camera_livekit_bridge/README.md +++ b/examples/camera_livekit_bridge/README.md @@ -19,7 +19,7 @@ Each stream runs on its own thread. A frame is copied (or box-averaged down) on the GPU from the IPC slot into an NVENC input surface, encoded, and submitted as a pre-encoded access unit. Pixels never reach host memory. The frame's source sequence is the track's `frame_id`, the sensor start time (when the receiver -derived one) is the `user_timestamp` in Unix microseconds, and the sensor +derived one) is the `user_timestamp` (what it means is set by `--timestamp-basis`, below), and the sensor counter is the 8-byte `user_data`. The newest picture is encoded and older queued ones are skipped. The receiver overwrites a slot without notice once its ring wraps, so after the GPU copy the bridge checks for a newer notice for the same slot, or as many newer notices as the ring has other slots, and drops the @@ -78,14 +78,9 @@ the same basis: | Basis | Value | | --- | --- | -| `capture_unix_us` (default) | The sensor start-of-frame as Unix time on the sender's clock. | -| `monotonic_us` | This host's monotonic clock at the instant the picture is ready to encode (before encoding): frame-ready time, later than the sensor capture. | +| `monotonic_us` (default) | This host's monotonic clock at the instant the picture is ready to encode (before encoding): frame-ready time, later than the sensor capture. | | `capture_monotonic_us` | The sensor start-of-frame on this host's monotonic clock, converted from the sender's `CLOCK_REALTIME` with chrony evidence for both hosts (below). | -`capture_unix_us` is used only when the receiver reports a realtime sender clock; a frame with no usable sensor -start-of-frame (missing, in the future, older than 5 s, or on another clock domain) carries no `user_timestamp`, so -a consumer that needs a capture time says so instead of using a guess. - `capture_monotonic_us` uses the sensor start-of-frame that the receiver reads from the sender's AVS1 timing record (`sensor_capture_realtime_ns`, `sensor_capture_uncertainty_ns`, `sensor_capture_valid`) and the time-sync library in `src/core/timesync`. Each frame is converted from the sender's `CLOCK_REALTIME` to this host's monotonic clock, or diff --git a/examples/camera_livekit_bridge/bridge/main.cpp b/examples/camera_livekit_bridge/bridge/main.cpp index 6b23a97b10..77fcc4746d 100644 --- a/examples/camera_livekit_bridge/bridge/main.cpp +++ b/examples/camera_livekit_bridge/bridge/main.cpp @@ -34,7 +34,7 @@ void usage() { std::cerr << "usage: camera_livekit_bridge --livekit-url URL --livekit-token-file FILE --stream NAME=TRACK[:DIV] [--stream ...]\n" - " [--socket PATH] [--fps N] [--bitrate-mbps N] [--idr-period N] [--max-notice-age-ms N] [--gpu N] [--source-full-range]\n [--timestamp-basis monotonic_us|capture_monotonic_us|capture_unix_us]\n" + " [--socket PATH] [--fps N] [--bitrate-mbps N] [--idr-period N] [--max-notice-age-ms N] [--gpu N] [--source-full-range]\n [--timestamp-basis monotonic_us|capture_monotonic_us]\n" " capture_monotonic_us: --chrony-sender HOST[:PORT] --chrony-sender-refid HEX [--chrony-pc-port PORT]\n" " [--chronyc PATH]\n" " NAME is a stream in camera_receiver's IPC catalog; TRACK is the LiveKit track name egocap expects;\n DIV shrinks the picture on the GPU by that integer factor before encoding.\n"; @@ -105,7 +105,7 @@ int main(int argc, char** argv) const auto basis = parse_timestamp_basis(next()); if (!basis) { - std::cerr << "--timestamp-basis must be monotonic_us, capture_monotonic_us or capture_unix_us\n"; + std::cerr << "--timestamp-basis must be monotonic_us or capture_monotonic_us\n"; return 2; } defaults.timestamp_basis = *basis; @@ -172,8 +172,7 @@ int main(int argc, char** argv) { std::cerr << "--timestamp-basis capture_monotonic_us needs --chrony-sender HOST: the sender's capture time is only " - "used with chrony evidence for both hosts. Use monotonic_us for the frame-ready time or " - "capture_unix_us for a Unix-time sensor stamp.\n"; + "used with chrony evidence for both hosts. Use monotonic_us for the frame-ready time.\n"; return 2; } if (use_chrony && chrony_config.expected_sender_reference_id.empty()) diff --git a/examples/camera_livekit_bridge/bridge/stream_worker.cpp b/examples/camera_livekit_bridge/bridge/stream_worker.cpp index cd617fe790..5411fa419c 100644 --- a/examples/camera_livekit_bridge/bridge/stream_worker.cpp +++ b/examples/camera_livekit_bridge/bridge/stream_worker.cpp @@ -326,8 +326,9 @@ struct StreamWorker::Impl const bool wall_is_realtime = wall_domain != notice.metadata.end() && wall_domain->is_string() && wall_domain->get() == "realtime"; const uint64_t sensor_sof_ns = wall_is_realtime ? unsigned_field(notice.metadata, "sensor_sof_ns") : 0; - frame.user_timestamp_us = - user_timestamp_us(config.timestamp_basis, sensor_sof_ns, ready_realtime_ns, ready_monotonic_ns); + frame.user_timestamp_us = config.timestamp_basis == TimestampBasis::MonotonicUs ? + static_cast(ready_monotonic_ns / 1000) : + 0; if (config.timestamp_basis == TimestampBasis::CaptureMonotonicUs && config.chrony_clock != nullptr) { // The receiver's timing record: the sensor start of frame on the sender's CLOCK_REALTIME with the diff --git a/examples/camera_livekit_bridge/bridge/stream_worker.hpp b/examples/camera_livekit_bridge/bridge/stream_worker.hpp index 3ac9b158ea..bace7dbc9b 100644 --- a/examples/camera_livekit_bridge/bridge/stream_worker.hpp +++ b/examples/camera_livekit_bridge/bridge/stream_worker.hpp @@ -36,7 +36,7 @@ struct WorkerConfig /// receiver's IPC catalog does not carry the range, so the operator says it here. bool source_full_range = false; /// What each frame's user_timestamp means; the consumer's streams file must name the same basis. - TimestampBasis timestamp_basis = TimestampBasis::CaptureUnixUs; + TimestampBasis timestamp_basis = TimestampBasis::MonotonicUs; /// Chrony evidence for capture_monotonic_us; shared by every stream and owned by the caller. ChronyCaptureClock* chrony_clock = nullptr; }; diff --git a/examples/camera_livekit_bridge/tests/python/test_bridge_timing.py b/examples/camera_livekit_bridge/tests/python/test_bridge_timing.py index 60d2c90bda..9fae18a175 100644 --- a/examples/camera_livekit_bridge/tests/python/test_bridge_timing.py +++ b/examples/camera_livekit_bridge/tests/python/test_bridge_timing.py @@ -143,22 +143,6 @@ def counting_add(exchange): asyncio.run(main()) """ -# Runs under EGOCAP_PY: only the loader and recordability check, no network. -REFUSE_SCRIPT = r""" -import json, sys -from pathlib import Path -from types import SimpleNamespace -from capture_server.livekit_streams import load_livekit_streams -from egocap_web.livekit_recording import recordable_livekit_streams, unrecordable_reason - -streams = load_livekit_streams(Path(sys.argv[1])) -started = SimpleNamespace(streams=streams) -print("RESULT " + json.dumps({ - "reason": unrecordable_reason(started, "http://127.0.0.1:8765"), - "recordable": len(recordable_livekit_streams(started)), -})) -""" - def make_receiver_token(room: str, identity: str) -> str: """Subscribe and publish-data: clock sync sends its requests as data messages.""" @@ -258,7 +242,6 @@ def _report(r: dict) -> str: @unittest.skipIf(_skip_reason() is not None, _skip_reason() or "") class BridgeTimingTest(unittest.TestCase): results: dict[str, dict] - refuse: dict @classmethod def setUpClass(cls): @@ -393,13 +376,6 @@ def _run_all(cls): for basis in BASES: cls.results[basis] = cls._run_basis(url, sock, room, basis) print(f"\n[{basis}] {_report(cls.results[basis])}", file=sys.stderr) - unix_yaml = cls._tmp / "streams-unix.yaml" - unix_yaml.write_text(_streams_yaml(url, room, "bridge-unix", "capture_unix_us")) - out = subprocess.run( - [str(EGOCAP_PY), "-c", REFUSE_SCRIPT, str(unix_yaml)], - capture_output=True, text=True, check=False, env=_egocap_env(), timeout=120, - ) # fmt: skip - cls.refuse = _result(out.stdout, "refuse check", out.stderr[-2000:]) def _check_feed(self, basis: str): r = self.results[basis] @@ -442,14 +418,6 @@ def test_monotonic_us_clock_and_feed(self): self._check_clock("monotonic_us") self._check_feed("monotonic_us") - def test_capture_unix_us_is_refused_by_egocap(self): - reason = self.refuse["reason"] - self.assertIsNotNone( - reason, "egocap accepted a capture_unix_us stream for recording" - ) - self.assertIn("monotonic clock", reason) - self.assertEqual(self.refuse["recordable"], 0) - @unittest.skipUnless( BRIDGE_BIN and Path(str(BRIDGE_BIN)).exists(), "camera_livekit_bridge not built" diff --git a/examples/camera_livekit_bridge/tests/timestamp_basis/test_timestamp_basis.cpp b/examples/camera_livekit_bridge/tests/timestamp_basis/test_timestamp_basis.cpp index 946c2589be..e8a2409e1a 100644 --- a/examples/camera_livekit_bridge/tests/timestamp_basis/test_timestamp_basis.cpp +++ b/examples/camera_livekit_bridge/tests/timestamp_basis/test_timestamp_basis.cpp @@ -3,7 +3,6 @@ #include -#include #include namespace @@ -25,49 +24,14 @@ void check(bool condition, const char* what) int main() { using camera_bridge::TimestampBasis; - using camera_bridge::user_timestamp_us; check(camera_bridge::parse_timestamp_basis("monotonic_us") == TimestampBasis::MonotonicUs, "parse monotonic_us"); check(camera_bridge::parse_timestamp_basis("capture_monotonic_us") == TimestampBasis::CaptureMonotonicUs, "parse capture_monotonic_us"); - check(camera_bridge::parse_timestamp_basis("capture_unix_us") == TimestampBasis::CaptureUnixUs, - "parse capture_unix_us"); + check(!camera_bridge::parse_timestamp_basis("capture_unix_us").has_value(), "reject a Unix-time basis"); check(!camera_bridge::parse_timestamp_basis("processing_unix_us").has_value(), "reject an unsupported basis"); check(!camera_bridge::parse_timestamp_basis("").has_value(), "reject an empty basis"); - const int64_t realtime = 1'800'000'000'000'000'000; - const int64_t monotonic = 5'000'000'000'000; - - // Frame-ready time ignores the sensor time and is the monotonic clock in microseconds. - check(user_timestamp_us(TimestampBasis::MonotonicUs, 0, realtime, monotonic) == 5'000'000'000, - "monotonic_us without sensor time"); - check(user_timestamp_us(TimestampBasis::MonotonicUs, 123, realtime, monotonic) == 5'000'000'000, - "monotonic_us ignores sensor time"); - - // Capture time 80 ms ago on the sender's wall clock. - const uint64_t sof = static_cast(realtime - 80'000'000); - check(user_timestamp_us(TimestampBasis::CaptureUnixUs, sof, realtime, monotonic) == sof / 1000, - "capture_unix_us is the sensor time"); - - // The adapter produces this basis; user_timestamp_us never does. - for (const uint64_t sensor : { uint64_t{ 0 }, sof, static_cast(realtime + 1'000'000) }) - check(user_timestamp_us(TimestampBasis::CaptureMonotonicUs, sensor, realtime, monotonic) == 0, - "capture_monotonic_us gives 0"); - - // No sensor time, a future one and an implausibly old one are not vouched for. - { - const auto basis = TimestampBasis::CaptureUnixUs; - check(user_timestamp_us(basis, 0, realtime, monotonic) == 0, "no sensor time gives 0"); - check(user_timestamp_us(basis, static_cast(realtime + 1'000'000), realtime, monotonic) == 0, - "future sensor time gives 0"); - check(user_timestamp_us(basis, static_cast(realtime - camera_bridge::kMaxCaptureAgeNs - 1), realtime, - monotonic) == 0, - "too old sensor time gives 0"); - check(user_timestamp_us( - basis, static_cast(realtime - camera_bridge::kMaxCaptureAgeNs), realtime, monotonic) != 0, - "sensor time at the age limit is accepted"); - } - if (g_failures == 0) std::cout << "timestamp_basis: all checks passed\n"; return g_failures == 0 ? 0 : 1; diff --git a/examples/camera_livekit_bridge/video_types/inc/video_types/timestamp_basis.hpp b/examples/camera_livekit_bridge/video_types/inc/video_types/timestamp_basis.hpp index 6ce3ed3103..980519eebb 100644 --- a/examples/camera_livekit_bridge/video_types/inc/video_types/timestamp_basis.hpp +++ b/examples/camera_livekit_bridge/video_types/inc/video_types/timestamp_basis.hpp @@ -16,11 +16,8 @@ enum class TimestampBasis /// The publisher's CLOCK_MONOTONIC when the frame is published (frame-ready time). MonotonicUs, /// The sensor's capture event on the publisher's CLOCK_MONOTONIC, converted from the sender's CLOCK_REALTIME with - /// chrony's synchronisation evidence for both hosts and refused when that evidence cannot vouch for it. The - /// conversion is made by the chrony adapter, not by `user_timestamp_us`. + /// chrony's synchronisation evidence for both hosts and refused when that evidence cannot vouch for it. CaptureMonotonicUs, - /// The sensor's capture event as Unix time on the sender's clock. - CaptureUnixUs, }; inline std::optional parse_timestamp_basis(const std::string& name) @@ -29,35 +26,10 @@ inline std::optional parse_timestamp_basis(const std::string& na return TimestampBasis::MonotonicUs; if (name == "capture_monotonic_us") return TimestampBasis::CaptureMonotonicUs; - if (name == "capture_unix_us") - return TimestampBasis::CaptureUnixUs; return std::nullopt; } /// A sensor time older than this, or in the future, is not a capture time this host can vouch for. inline constexpr int64_t kMaxCaptureAgeNs = 5'000'000'000; -/** - * @brief The frame's `user_timestamp` in microseconds on `basis`, or 0 when the publisher cannot vouch for one. - * - * `sensor_sof_ns` is the sensor start of frame as Unix nanoseconds on the sender's clock (0 when unknown); - * `realtime_now_ns` and `monotonic_now_ns` are this host's CLOCK_REALTIME and CLOCK_MONOTONIC read together. - * For the capture bases a missing, future or implausibly old sensor time gives 0, never a guess. The monotonic capture - * basis needs measured synchronisation evidence, which this function does not have, so it always gives 0. - */ -inline uint64_t user_timestamp_us(TimestampBasis basis, - uint64_t sensor_sof_ns, - int64_t realtime_now_ns, - int64_t monotonic_now_ns) -{ - if (basis == TimestampBasis::MonotonicUs) - return monotonic_now_ns > 0 ? static_cast(monotonic_now_ns) / 1000 : 0; - if (basis == TimestampBasis::CaptureMonotonicUs || sensor_sof_ns == 0) - return 0; - const int64_t age_ns = realtime_now_ns - static_cast(sensor_sof_ns); - if (age_ns < 0 || age_ns > kMaxCaptureAgeNs) - return 0; - return sensor_sof_ns / 1000; -} - } // namespace camera_bridge From f996f72101ce8d0a35ebdc57863ff43d152c19b4 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 18:51:11 -0500 Subject: [PATCH 08/10] camera_livekit_bridge: state the clock adapter's place in the integration boundary Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/camera_livekit_bridge/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/camera_livekit_bridge/README.md b/examples/camera_livekit_bridge/README.md index 3b3fdad286..d3a16c1376 100644 --- a/examples/camera_livekit_bridge/README.md +++ b/examples/camera_livekit_bridge/README.md @@ -40,6 +40,13 @@ slots" applies. | --- | --- | --- | --- | | `livekit_publisher/` and the session, publication and feedback wiring in `bridge/` | Transport adapters | Adapt the useful implementation; retire the hard-coded transport glue | Map sessions, publications, encoder feedback and reconnect behaviour to the adapter boundary. Lands when real web clients receive streams, and feedback and recovery work, through an adapter. | | `ipc_client/` and the slot read in `bridge/stream_worker.cpp` | Camera reader views and leases | Retire | Read frames through a lease held until the final GPU read. Lands with the receiver's lease support. | +| `chrony_clock/` and the capture-time publication in `bridge/stream_worker.cpp` | The receiver backend's published clock mapping (offset, uncertainty, discontinuity epoch) | Transitional: adapt, then retire the bridge-specific adapter | Carry mapped capture time, validity and discontinuity behaviour through the receiver contract. Lands when consumers obtain equivalent timing without the bridge-only adapter. | +| `src/core/timesync/` (used by `chrony_clock/`) | The estimator behind the receiver's clock mapping | Reuse | Keep the chrony evidence handling and qualification behind the destination interface. | + +The `chrony_clock/` adapter connects the reusable timing library to the current video publication path and is intended +to converge on the receiver backend's clock-mapping contract. With `capture_monotonic_us`, a mapping that cannot be +vouched for publishes no timestamp, and the frame-ready time is never offered in its place; a capture over the +precision target keeps its timestamp. `monotonic_us` is a separate, declared basis. ## Build From cbceade08e63d113fa498178ee1c71a994b8d87f Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 12:07:31 -0500 Subject: [PATCH 09/10] camera_livekit_bridge: state which frames a recording can accept Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- examples/camera_livekit_bridge/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/camera_livekit_bridge/README.md b/examples/camera_livekit_bridge/README.md index d3a16c1376..515dc85279 100644 --- a/examples/camera_livekit_bridge/README.md +++ b/examples/camera_livekit_bridge/README.md @@ -113,6 +113,13 @@ carry a timestamp; the rest do not), the last combined uncertainty and the clock under. These are runtime diagnostics only and are not recorded with the video. The caller of a recording should treat a frame without a timestamp as a timing fault. +Recording eligibility is permissive in this bridge: an `over_budget` frame is converted validly, keeps its capture +time and is published like a `qualified` one, so a recording that accepts every timestamped frame records it. The +timestamp alone does not say the precision target was met, and the verdict and bound are not carried with the video. +A frame without a timestamp (an invalid mapping, stale or unsynchronised evidence, or a wall-clock step) is the timing +fault. A consumer that must reject frames above the precision target needs the bound from the bridge's diagnostics +or a stricter publication policy; this bridge does not apply one. + `--source-full-range` says the receiver's pictures use the full 0-255 range, as SIPL sources do; the encoded stream then signals full range, so players and recordings keep the black and white levels. Without it the stream signals limited range (16-235). The receiver's IPC catalog does not carry the range, so it is set here and must match the From e9a6282df9078cecae81c5f32e5cd75f96a6c711 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 12:12:09 -0500 Subject: [PATCH 10/10] camera_livekit_bridge: test that a PC follower that stops answering stops qualifying Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../test_chrony_capture_clock.cpp | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp b/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp index fc80fe2749..17be64d299 100644 --- a/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp +++ b/examples/camera_livekit_bridge/tests/chrony_clock/test_chrony_capture_clock.cpp @@ -140,6 +140,10 @@ struct Rig set_sender("A9FEA97B", 3, 0.0005); // the camera host, following its own upstream set_pc(kExpectedRefId, 4, 0.0005); // this machine's chronyd, following the camera host } + void healthy_sender_only() + { + set_sender("A9FEA97B", 3, 0.0005); // the camera host keeps answering + } void build() { clock = std::make_unique(config, chrony.runner(), clocks.reader()); @@ -214,6 +218,24 @@ void test_sender_unreachable() CHECK(r.qualify().verdict == Verdict::NoSenderEvidence); } +// The PC's measuring instance can exit after it was ready (it is optional in the launcher). Its last report is not +// extended: once it is older than the evidence limit every frame is refused, so a recording gets no timestamps. +void test_pc_follower_that_stops_answering_stops_qualifying() +{ + Rig r; + r.healthy(); + r.build(); + r.clock->poll_once(); + CHECK(r.qualify().verdict == Verdict::Qualified); + r.chrony.pc = CommandResult{}; // the follower is gone: the command never starts + r.clocks.advance(5 * kSec); + r.clock->poll_once(); + r.healthy_sender_only(); + r.clocks.advance(6 * kSec); // the PC's last report is now 11 s old + r.clock->poll_once(); + CHECK(r.qualify().verdict == Verdict::StaleEvidence); +} + void test_reference_mismatch() { Rig r; @@ -449,6 +471,7 @@ int main() test_sender_invalid(); test_no_poll_yet(); test_sender_unreachable(); + test_pc_follower_that_stops_answering_stops_qualifying(); test_reference_mismatch(); test_unsynchronised(); test_stale_evidence();