From 4fe62ef4a414b3cae0a08ed06ca4e14950840bd3 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 12 Nov 2026 18:31:15 -0500 Subject: [PATCH 01/10] timesync: qualify camera capture times against chrony's own error report Adds a POSIX library that converts a camera host's wall-clock capture time to the PC's monotonic clock and refuses a frame it cannot vouch for. It parses `chronyc -c tracking`, polls it in the background without a shell, tracks wall-clock steps on the local host, and sums a relative inter-host uncertainty bound or returns a typed refusal. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/core/CMakeLists.txt | 6 + src/core/timesync/CMakeLists.txt | 25 + src/core/timesync/cpp/CMakeLists.txt | 29 + src/core/timesync/cpp/capture_qualifier.cpp | 197 +++++ src/core/timesync/cpp/chrony_monitor.cpp | 147 ++++ src/core/timesync/cpp/chrony_tracking.cpp | 136 ++++ src/core/timesync/cpp/command_runner.cpp | 142 ++++ .../cpp/inc/timesync/capture_qualifier.hpp | 121 +++ .../cpp/inc/timesync/chrony_monitor.hpp | 93 +++ .../cpp/inc/timesync/chrony_tracking.hpp | 57 ++ .../cpp/inc/timesync/command_runner.hpp | 33 + .../timesync/cpp/inc/timesync/local_clock.hpp | 76 ++ src/core/timesync/cpp/local_clock.cpp | 115 +++ tests/cpp/core/CMakeLists.txt | 3 + tests/cpp/core/timesync/CMakeLists.txt | 8 + .../timesync/capture_qualifier/CMakeLists.txt | 12 + .../test_capture_qualifier.cpp | 753 ++++++++++++++++++ .../timesync/chrony_monitor/CMakeLists.txt | 14 + .../chrony_monitor/test_chrony_monitor.cpp | 344 ++++++++ .../timesync/chrony_tracking/CMakeLists.txt | 12 + .../chrony_tracking/test_chrony_tracking.cpp | 289 +++++++ .../timesync/command_runner/CMakeLists.txt | 14 + .../command_runner/test_command_runner.cpp | 119 +++ .../core/timesync/local_clock/CMakeLists.txt | 12 + .../timesync/local_clock/test_local_clock.cpp | 343 ++++++++ 25 files changed, 3100 insertions(+) create mode 100644 src/core/timesync/CMakeLists.txt create mode 100644 src/core/timesync/cpp/CMakeLists.txt create mode 100644 src/core/timesync/cpp/capture_qualifier.cpp create mode 100644 src/core/timesync/cpp/chrony_monitor.cpp create mode 100644 src/core/timesync/cpp/chrony_tracking.cpp create mode 100644 src/core/timesync/cpp/command_runner.cpp create mode 100644 src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp create mode 100644 src/core/timesync/cpp/inc/timesync/chrony_monitor.hpp create mode 100644 src/core/timesync/cpp/inc/timesync/chrony_tracking.hpp create mode 100644 src/core/timesync/cpp/inc/timesync/command_runner.hpp create mode 100644 src/core/timesync/cpp/inc/timesync/local_clock.hpp create mode 100644 src/core/timesync/cpp/local_clock.cpp create mode 100644 tests/cpp/core/timesync/CMakeLists.txt create mode 100644 tests/cpp/core/timesync/capture_qualifier/CMakeLists.txt create mode 100644 tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp create mode 100644 tests/cpp/core/timesync/chrony_monitor/CMakeLists.txt create mode 100644 tests/cpp/core/timesync/chrony_monitor/test_chrony_monitor.cpp create mode 100644 tests/cpp/core/timesync/chrony_tracking/CMakeLists.txt create mode 100644 tests/cpp/core/timesync/chrony_tracking/test_chrony_tracking.cpp create mode 100644 tests/cpp/core/timesync/command_runner/CMakeLists.txt create mode 100644 tests/cpp/core/timesync/command_runner/test_command_runner.cpp create mode 100644 tests/cpp/core/timesync/local_clock/CMakeLists.txt create mode 100644 tests/cpp/core/timesync/local_clock/test_local_clock.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 9cc4772373..a9e2ce55a6 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -13,6 +13,12 @@ add_subdirectory(schema_compat) # log during its own construction. add_subdirectory(log_bridge) +# Chrony evidence parsing and capture-time qualification (no dependency on the rest of the product). +# POSIX clocks only, so the Windows source build skips it. +if(NOT WIN32) + add_subdirectory(timesync) +endif() + # Build OXR Utils first - header-only utilities (no dependencies) add_subdirectory(oxr_utils) diff --git a/src/core/timesync/CMakeLists.txt b/src/core/timesync/CMakeLists.txt new file mode 100644 index 0000000000..3e55149981 --- /dev/null +++ b/src/core/timesync/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Chrony evidence parsing, wall-to-monotonic conversion and capture qualification. Standard library and POSIX +# clocks only, so it also configures on its own (cmake -S src/core/timesync) and builds without CUDA, LiveKit +# or Python. +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(isaaccapture_timesync LANGUAGES CXX) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) + include(CTest) +endif() + +add_subdirectory(cpp) + +# Standalone configure runs the module's tests; in the product tree tests/cpp/core adds them. +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR AND BUILD_TESTING) + cmake_path(GET CMAKE_CURRENT_SOURCE_DIR PARENT_PATH _core_dir) + cmake_path(GET _core_dir PARENT_PATH _src_dir) + cmake_path(GET _src_dir PARENT_PATH _repo_dir) + add_subdirectory(${_repo_dir}/tests/cpp/core/timesync tests) +endif() diff --git a/src/core/timesync/cpp/CMakeLists.txt b/src/core/timesync/cpp/CMakeLists.txt new file mode 100644 index 0000000000..2bc0e90e3c --- /dev/null +++ b/src/core/timesync/cpp/CMakeLists.txt @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_library(timesync STATIC + chrony_tracking.cpp + local_clock.cpp + capture_qualifier.cpp + command_runner.cpp + chrony_monitor.cpp +) + +target_include_directories(timesync + INTERFACE + $ +) + +target_link_libraries(timesync + PRIVATE + Threads::Threads +) + +target_compile_features(timesync PUBLIC cxx_std_17) +target_compile_options(timesync PRIVATE -Wall -Wextra) + +add_library(timesync::timesync ALIAS timesync) diff --git a/src/core/timesync/cpp/capture_qualifier.cpp b/src/core/timesync/cpp/capture_qualifier.cpp new file mode 100644 index 0000000000..e1b144376a --- /dev/null +++ b/src/core/timesync/cpp/capture_qualifier.cpp @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/timesync/capture_qualifier.hpp" + +#include +#include +#include +#include + +namespace isaaccapture::timesync +{ + +namespace +{ + +constexpr int64_t kSaturated = std::numeric_limits::max() / 4; + +int64_t to_ns_saturating(double seconds) +{ + const double ns = seconds * 1e9; + if (!(ns < static_cast(kSaturated))) + { + return kSaturated; + } + return static_cast(std::ceil(ns)); +} + +int64_t add_saturating(int64_t a, int64_t b) +{ + return (a > kSaturated - b) ? kSaturated : a + b; +} + +std::string upper(std::string s) +{ + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::toupper(c); }); + return s; +} + +// Age of a report; an unrepresentable difference reads as maximal, so it is refused as stale. +int64_t evidence_age_ns(const ChronyEvidence& e, int64_t now_monotonic_ns) +{ + int64_t age = 0; + return __builtin_sub_overflow(now_monotonic_ns, e.observed_monotonic_ns, &age) ? INT64_MAX : age; +} + +bool stale(const ChronyEvidence& e, int64_t now_monotonic_ns, int64_t max_age_ns) +{ + const int64_t age = evidence_age_ns(e, now_monotonic_ns); + return age < 0 || age > max_age_ns; +} + +// True when chronyd's last source update is too old at the moment it was queried. +bool source_quiet(const ChronyEvidence& e, int64_t max_age_ns, int64_t future_tolerance_ns) +{ + if (max_age_ns <= 0 || e.observed_realtime_ns <= 0 || e.tracking.reference_id == kChronyLocalReferenceId) + { + return false; + } + const double age_s = static_cast(e.observed_realtime_ns) * 1e-9 - e.tracking.ref_time_unix_s; + // A source update later than the query by more than the tolerance is impossible evidence, not fresh evidence. + const double lead_s = static_cast(std::max(future_tolerance_ns, 0)) * 1e-9; + return !(age_s <= static_cast(max_age_ns) * 1e-9 && age_s >= -lead_s); // also true for a NaN +} + +} // namespace + +const char* to_string(Verdict v) +{ + switch (v) + { + case Verdict::Qualified: + return "qualified"; + case Verdict::SenderInvalid: + return "sender_invalid"; + case Verdict::NoSenderEvidence: + return "no_sender_evidence"; + case Verdict::NoPcEvidence: + return "no_pc_evidence"; + case Verdict::StaleEvidence: + return "stale_evidence"; + case Verdict::Unsynchronised: + return "unsynchronised"; + case Verdict::ReferenceMismatch: + return "reference_mismatch"; + case Verdict::NoLocalClock: + return "no_local_clock"; + case Verdict::ImplausibleAge: + return "implausible_age"; + case Verdict::OverBudget: + return "over_budget"; + } + return "unknown"; +} + +int64_t chrony_error_bound_ns(const ChronyTracking& t, int64_t age_ns) +{ + const double age_s = static_cast(std::max(age_ns, 0)) * 1e-9; + const double bound_s = + std::fabs(t.system_time_s) + t.root_dispersion_s + 0.5 * t.root_delay_s + t.skew_ppm * 1e-6 * age_s; + return to_ns_saturating(bound_s); // rounded up, so the bound never understates +} + +int64_t inter_host_bound_ns(const ChronyEvidence& follower, + const ChronyEvidence& server, + int64_t follower_age_ns, + int64_t server_age_ns) +{ + const ChronyTracking& s = follower.tracking; + const ChronyTracking& p = server.tracking; + // Root delay and dispersion are cumulative along the chain to the reference, so what the follower adds beyond + // the server's own values is the link between them; the shared upstream part cancels. + const double link_dispersion_s = std::max(s.root_dispersion_s - p.root_dispersion_s, 0.0); + const double link_delay_s = std::max(s.root_delay_s - p.root_delay_s, 0.0); + // The follower copied the server's values when it last polled; the server's dispersion has grown since. + const double copy_slack_s = p.skew_ppm * 1e-6 * s.update_interval_s; + const double follower_age_s = static_cast(std::max(follower_age_ns, 0)) * 1e-9; + const double server_age_s = static_cast(std::max(server_age_ns, 0)) * 1e-9; + const double bound_s = std::fabs(s.system_time_s) + std::fabs(p.system_time_s) + link_dispersion_s + + 0.5 * link_delay_s + copy_slack_s + s.skew_ppm * 1e-6 * follower_age_s + + p.skew_ppm * 1e-6 * server_age_s; + return to_ns_saturating(bound_s); +} + +Qualification qualify_capture(const SenderCapture& capture, + const std::optional& sender_host, + const std::optional& pc_host, + const LocalClockObservation& pc_clock, + int64_t now_monotonic_ns, + const QualifierConfig& config) +{ + Qualification q; + auto refuse = [&q](Verdict v) + { + q.verdict = v; + return q; + }; + + if (!capture.valid || capture.mapping_uncertainty_ns < 0) + { + return refuse(Verdict::SenderInvalid); + } + if (!sender_host) + { + return refuse(Verdict::NoSenderEvidence); + } + if (!pc_host) + { + return refuse(Verdict::NoPcEvidence); + } + if (stale(*sender_host, now_monotonic_ns, config.max_evidence_age_ns) || + stale(*pc_host, now_monotonic_ns, config.max_evidence_age_ns) || + source_quiet(*sender_host, config.max_source_update_age_ns, config.source_update_future_tolerance_ns) || + source_quiet(*pc_host, config.max_source_update_age_ns, config.source_update_future_tolerance_ns)) + { + return refuse(Verdict::StaleEvidence); + } + // The PC follows the camera host. A local reference on the PC says nothing about the camera host; the camera + // host may be one, because its own clock is what the PC measures. + const ChronyEvidence& follower = *pc_host; + const ChronyEvidence& server = *sender_host; + if (!is_synchronised(follower.tracking, false) || !is_synchronised(server.tracking, true)) + { + return refuse(Verdict::Unsynchronised); + } + if (config.expected_sender_reference_id.empty() || + upper(follower.tracking.reference_id) != upper(config.expected_sender_reference_id)) + { + return refuse(Verdict::ReferenceMismatch); + } + if (!pc_clock.usable) + { + return refuse(Verdict::NoLocalClock); + } + + q.generation = pc_clock.generation; + int64_t offset = 0; + int64_t capture_age = 0; + if (__builtin_sub_overflow(pc_clock.sample.realtime_ns, pc_clock.sample.monotonic_ns, &offset) || + __builtin_sub_overflow(capture.realtime_ns, offset, &q.capture_monotonic_ns) || + __builtin_sub_overflow(now_monotonic_ns, q.capture_monotonic_ns, &capture_age) || + capture_age < -std::max(config.future_tolerance_ns, 0) || capture_age > config.max_capture_age_ns) + { + return refuse(Verdict::ImplausibleAge); + } + + q.terms.sensor_ns = capture.mapping_uncertainty_ns; + q.terms.inter_host_ns = inter_host_bound_ns( + follower, server, evidence_age_ns(follower, now_monotonic_ns), evidence_age_ns(server, now_monotonic_ns)); + q.terms.pc_local_ns = + add_saturating(pc_clock.sample.read_uncertainty_ns, std::max(pc_clock.step_tolerance_ns, 0)); + q.uncertainty_ns = add_saturating(add_saturating(q.terms.sensor_ns, q.terms.inter_host_ns), q.terms.pc_local_ns); + q.verdict = q.uncertainty_ns > config.budget_ns ? Verdict::OverBudget : Verdict::Qualified; + return q; +} + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/chrony_monitor.cpp b/src/core/timesync/cpp/chrony_monitor.cpp new file mode 100644 index 0000000000..6eb07134a7 --- /dev/null +++ b/src/core/timesync/cpp/chrony_monitor.cpp @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/timesync/chrony_monitor.hpp" + +#include "inc/timesync/chrony_tracking.hpp" +#include "inc/timesync/local_clock.hpp" + +#include + +namespace isaaccapture::timesync +{ + +const char* to_string(MonitorStatus s) +{ + switch (s) + { + case MonitorStatus::NotPolled: + return "not_polled"; + case MonitorStatus::Ok: + return "ok"; + case MonitorStatus::CommandFailed: + return "command_failed"; + case MonitorStatus::TimedOut: + return "timed_out"; + case MonitorStatus::ParseFailed: + return "parse_failed"; + } + return "unknown"; +} + +ChronyMonitor::ChronyMonitor(ChronyMonitorConfig config, CommandRunner runner, NowFn now_monotonic_ns, NowFn now_realtime_ns) + : config_(std::move(config)), + runner_(runner ? std::move(runner) : posix_command_runner()), + now_(now_monotonic_ns ? std::move(now_monotonic_ns) : system_clock_reader().monotonic_ns), + now_realtime_(now_realtime_ns ? std::move(now_realtime_ns) : system_clock_reader().realtime_ns) +{ +} + +ChronyMonitor::~ChronyMonitor() +{ + stop(); +} + +MonitorStatus ChronyMonitor::poll_once() +{ + std::vector argv{ config_.chronyc_path, "-c" }; + if (!config_.host.empty()) + { + argv.push_back("-h"); + argv.push_back(config_.host); + } + if (config_.port != 0) + { + argv.push_back("-p"); + argv.push_back(std::to_string(config_.port)); + } + argv.push_back("tracking"); + + const int64_t started_ns = now_(); + const int64_t started_realtime_ns = now_realtime_(); + const CommandResult r = runner_(argv, config_.command_timeout); + + MonitorStatus status = MonitorStatus::Ok; + std::optional fresh; + if (r.timed_out) + { + status = MonitorStatus::TimedOut; + } + else if (!r.started || r.truncated || r.exit_code != 0) + { + status = MonitorStatus::CommandFailed; + } + else + { + ChronyParseResult parsed = parse_tracking_csv(r.output); + if (!parsed.tracking) + { + status = MonitorStatus::ParseFailed; + } + else + { + // Stamped when the query began: the report is no newer than that, so its age is never understated. + fresh = ChronyEvidence{ std::move(*parsed.tracking), started_ns, started_realtime_ns }; + } + } + + std::lock_guard lock(mutex_); + ++snapshot_.polls; + snapshot_.last_status = status; + if (fresh) + { + snapshot_.evidence = std::move(fresh); + } + else + { + ++snapshot_.failures; + } + return status; +} + +MonitorSnapshot ChronyMonitor::snapshot() const +{ + std::lock_guard lock(mutex_); + return snapshot_; +} + +void ChronyMonitor::start() +{ + std::lock_guard lock(mutex_); + if (thread_.joinable()) + { + return; + } + stopping_ = false; + thread_ = std::thread([this] { run(); }); +} + +void ChronyMonitor::stop() +{ + std::thread joinable; + { + std::lock_guard lock(mutex_); + stopping_ = true; + joinable = std::move(thread_); + } + wake_.notify_all(); + if (joinable.joinable()) + { + joinable.join(); + } +} + +void ChronyMonitor::run() +{ + for (;;) + { + poll_once(); + std::unique_lock lock(mutex_); + if (wake_.wait_for(lock, config_.poll_interval, [this] { return stopping_; })) + { + return; + } + } +} + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/chrony_tracking.cpp b/src/core/timesync/cpp/chrony_tracking.cpp new file mode 100644 index 0000000000..6dab044e90 --- /dev/null +++ b/src/core/timesync/cpp/chrony_tracking.cpp @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/timesync/chrony_tracking.hpp" + +#include +#include +#include +#include +#include + +namespace isaaccapture::timesync +{ + +namespace +{ + +constexpr size_t kFieldCount = 14; + +std::string_view trim(std::string_view s) +{ + while (!s.empty() && std::isspace(static_cast(s.front()))) + { + s.remove_prefix(1); + } + while (!s.empty() && std::isspace(static_cast(s.back()))) + { + s.remove_suffix(1); + } + return s; +} + +bool parse_double(std::string_view s, double& out) +{ + s = trim(s); + if (s.empty()) + { + return false; + } + if (s.front() == '+') + { + s.remove_prefix(1); + } + auto [end, ec] = std::from_chars(s.data(), s.data() + s.size(), out); + return ec == std::errc() && end == s.data() + s.size() && std::isfinite(out); +} + +bool parse_int(std::string_view s, long long& out) +{ + s = trim(s); + auto [end, ec] = std::from_chars(s.data(), s.data() + s.size(), out); + return !s.empty() && ec == std::errc() && end == s.data() + s.size(); +} + +std::string upper(std::string_view s) +{ + std::string r(trim(s)); + std::transform(r.begin(), r.end(), r.begin(), [](unsigned char c) { return std::toupper(c); }); + return r; +} + +} // namespace + +ChronyParseResult parse_tracking_csv(std::string_view line) +{ + line = trim(line); + if (line.empty()) + { + return { std::nullopt, ChronyParseError::Empty }; + } + + std::vector f; + size_t start = 0; + for (;;) + { + const size_t comma = line.find(',', start); + if (comma == std::string_view::npos) + { + f.push_back(line.substr(start)); + break; + } + f.push_back(line.substr(start, comma - start)); + start = comma + 1; + } + if (f.size() != kFieldCount) + { + return { std::nullopt, ChronyParseError::FieldCount }; + } + + ChronyTracking t; + t.reference_id = upper(f[0]); + t.reference_name = std::string(trim(f[1])); + t.leap_status = std::string(trim(f[13])); + + long long stratum = 0; + if (!parse_int(f[2], stratum)) + { + return { std::nullopt, ChronyParseError::BadNumber }; + } + if (stratum < 0 || stratum > 255) + { + return { std::nullopt, ChronyParseError::BadStratum }; + } + t.stratum = static_cast(stratum); + + double* const numeric[] = { + &t.ref_time_unix_s, &t.system_time_s, &t.last_offset_s, &t.rms_offset_s, &t.frequency_ppm, + &t.residual_freq_ppm, &t.skew_ppm, &t.root_delay_s, &t.root_dispersion_s, &t.update_interval_s, + }; + for (size_t i = 0; i < std::size(numeric); ++i) + { + if (!parse_double(f[3 + i], *numeric[i])) + { + return { std::nullopt, ChronyParseError::BadNumber }; + } + } + if (t.rms_offset_s < 0.0 || t.skew_ppm < 0.0 || t.root_delay_s < 0.0 || t.root_dispersion_s < 0.0 || + t.update_interval_s < 0.0) + { + return { std::nullopt, ChronyParseError::BadNumber }; + } + return { std::move(t), ChronyParseError::None }; +} + +bool is_synchronised(const ChronyTracking& t, bool allow_local_reference) +{ + const bool leap_ok = + t.leap_status == "Normal" || t.leap_status == "Insert second" || t.leap_status == "Delete second"; + if (!leap_ok || t.stratum < 1 || t.stratum > 15 || t.reference_id == "00000000" || t.reference_id.empty()) + { + return false; + } + return allow_local_reference || t.reference_id != kChronyLocalReferenceId; +} + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/command_runner.cpp b/src/core/timesync/cpp/command_runner.cpp new file mode 100644 index 0000000000..6a1e912e07 --- /dev/null +++ b/src/core/timesync/cpp/command_runner.cpp @@ -0,0 +1,142 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/timesync/command_runner.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +extern char** environ; + +namespace isaaccapture::timesync +{ + +namespace +{ + +struct Fd +{ + int fd = -1; + ~Fd() + { + if (fd >= 0) + { + close(fd); + } + } +}; + +} // namespace + +CommandRunner posix_command_runner() +{ + return [](const std::vector& argv, std::chrono::milliseconds timeout) + { + CommandResult result; + if (argv.empty()) + { + return result; + } + + int fds[2]; + if (pipe(fds) != 0) + { + return result; + } + Fd read_end{ fds[0] }; + Fd write_end{ fds[1] }; + + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_adddup2(&actions, write_end.fd, STDOUT_FILENO); + posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, "/dev/null", O_WRONLY, 0); + posix_spawn_file_actions_addopen(&actions, STDIN_FILENO, "/dev/null", O_RDONLY, 0); + posix_spawn_file_actions_addclose(&actions, read_end.fd); + posix_spawn_file_actions_addclose(&actions, write_end.fd); + + std::vector args; + for (const auto& a : argv) + { + args.push_back(const_cast(a.c_str())); + } + args.push_back(nullptr); + + pid_t pid = 0; + const int rc = posix_spawnp(&pid, args[0], &actions, nullptr, args.data(), environ); + posix_spawn_file_actions_destroy(&actions); + if (rc != 0) + { + return result; + } + result.started = true; + close(write_end.fd); + write_end.fd = -1; + + const auto deadline = std::chrono::steady_clock::now() + timeout; + char buffer[4096]; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + if (left.count() <= 0) + { + result.timed_out = true; + break; + } + pollfd p{ read_end.fd, POLLIN, 0 }; + const int pr = poll(&p, 1, static_cast(left.count())); + if (pr < 0 && errno == EINTR) + { + continue; + } + if (pr == 0) + { + result.timed_out = true; + break; + } + if (pr < 0) + { + break; + } + const ssize_t n = read(read_end.fd, buffer, sizeof(buffer)); + if (n < 0 && errno == EINTR) + { + continue; + } + if (n <= 0) + { + break; // end of output + } + const size_t room = kMaxCommandOutputBytes - result.output.size(); + result.output.append(buffer, std::min(static_cast(n), room)); + if (static_cast(n) > room) + { + result.truncated = true; + break; + } + } + + if (result.timed_out || result.truncated) + { + kill(pid, SIGKILL); + } + int status = 0; + while (waitpid(pid, &status, 0) < 0 && errno == EINTR) + { + } + if (!result.timed_out && !result.truncated && WIFEXITED(status)) + { + result.exit_code = WEXITSTATUS(status); + } + return result; + }; +} + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp new file mode 100644 index 0000000000..ac5b4b6ea6 --- /dev/null +++ b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "chrony_tracking.hpp" +#include "local_clock.hpp" + +#include +#include +#include + +namespace isaaccapture::timesync +{ + +/// A chrony report plus when it was read, on the PC's monotonic clock. Evidence for the camera host is stamped +/// on receipt at the PC, so its age includes the delay in getting it here. +struct ChronyEvidence +{ + ChronyTracking tracking; + int64_t observed_monotonic_ns = 0; + int64_t observed_realtime_ns = 0; ///< The PC's CLOCK_REALTIME at the same instant; 0 when unknown. +}; + +/// What the sender claims about one frame: the capture instant in its CLOCK_REALTIME, and how far off its own +/// sensor-to-realtime mapping can be. +struct SenderCapture +{ + int64_t realtime_ns = 0; + int64_t mapping_uncertainty_ns = 0; + bool valid = false; +}; + +struct QualifierConfig +{ + int64_t budget_ns = 5'000'000; + /// A report older than this is refused, never extrapolated. This bounds how long ago chronyd was asked, not how + /// long ago it last heard from its source; see max_source_update_age_ns. + int64_t max_evidence_age_ns = 10'000'000'000; + /// Refuse when chronyd's last source update (Ref time) is older than this at the time of the query, so frequent + /// polling cannot hide a reference that went quiet. 0 disables it. A local reference has no source updates and + /// is exempt. The default is twice chrony's longest default poll interval. + int64_t max_source_update_age_ns = 2'048'000'000'000; + /// A source update stamped later than the query by more than this is impossible evidence and is refused. It + /// allows for the two hosts' clocks disagreeing by a little. + int64_t source_update_future_tolerance_ns = 1'000'000'000; + /// A capture older than this, or later than the future tolerance, is implausible; both bounds are inclusive. + int64_t max_capture_age_ns = 1'000'000'000; + /// Negative values are treated as 0. + int64_t future_tolerance_ns = 1'000'000; + /// Upper-case hex reference id the PC's chronyd must have selected: the camera host. The PC follows the camera + /// host, which may serve its own clock (chrony `local`) or follow UTC. This is a topology check, not + /// authentication or a quality measure (an IPv6 source is reported as a 32-bit hash). + std::string expected_sender_reference_id; +}; + +enum class Verdict +{ + Qualified, + SenderInvalid, + NoSenderEvidence, + NoPcEvidence, + StaleEvidence, + Unsynchronised, + ReferenceMismatch, + NoLocalClock, ///< Wide bracket or no sample. + ImplausibleAge, + OverBudget, +}; + +const char* to_string(Verdict v); + +struct UncertaintyTerms +{ + int64_t sensor_ns = 0; ///< The sender's own sensor-to-realtime mapping. + int64_t inter_host_ns = 0; ///< Camera host versus PC wall clock, from both chrony reports and their age. + int64_t pc_local_ns = 0; ///< The PC's wall/monotonic bracket. +}; + +struct Qualification +{ + Verdict verdict = Verdict::SenderInvalid; + int64_t capture_monotonic_ns = 0; ///< PC CLOCK_MONOTONIC; meaningful only when Qualified. + int64_t uncertainty_ns = 0; ///< Sum of the terms; filled for OverBudget as well, for diagnosis. + uint64_t generation = 0; ///< The PC local-clock generation the conversion used. + UncertaintyTerms terms; +}; + +/// chrony's bound on a host clock against its reference, |System time| + Root dispersion + Root delay / 2, plus +/// skew * age for the time since the report. Always non-negative; rounded up to the next nanosecond so it never +/// understates, and saturating. +int64_t chrony_error_bound_ns(const ChronyTracking& t, int64_t age_ns); + +/// The bound between a following host's wall clock and the host it follows, as `qualify_capture` charges it (see +/// below). The ages are of each report, on the PC's monotonic clock. Rounded up and saturating. +int64_t inter_host_bound_ns(const ChronyEvidence& follower, + const ChronyEvidence& server, + int64_t follower_age_ns, + int64_t server_age_ns); + +/// Converts a sender realtime capture stamp to PC monotonic and decides whether it may be recorded. +/// +/// The inter-host term bounds sender realtime against PC realtime along camera host system clock -> camera host +/// NTP clock -> PC chronyd's measurement -> PC system clock: both hosts' |System time|, the PC's Root dispersion and +/// Root delay / 2 minus the camera host's (those fields are cumulative to the reference, so the difference is the +/// link between the two and the camera host's own upstream error cancels), a slack for the camera host's dispersion +/// growing since the PC copied it, and each report's skew times its age. The assumptions are that the PC's source is +/// the camera host and that its root fields are cumulative through the camera host's; a differing reference id is +/// refused, and the subtraction is clamped at zero. The bound is derived from chrony's own reports, not measured +/// independently of them. +/// +/// `Qualified` carries the PC clock generation. The caller must pin the generation when a take starts and end the +/// take, not continue it, when a later frame reports a different one. +Qualification qualify_capture(const SenderCapture& capture, + const std::optional& sender_host, + const std::optional& pc_host, + const LocalClockObservation& pc_clock, + int64_t now_monotonic_ns, + const QualifierConfig& config); + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/inc/timesync/chrony_monitor.hpp b/src/core/timesync/cpp/inc/timesync/chrony_monitor.hpp new file mode 100644 index 0000000000..24a006257c --- /dev/null +++ b/src/core/timesync/cpp/inc/timesync/chrony_monitor.hpp @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "capture_qualifier.hpp" +#include "command_runner.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace isaaccapture::timesync +{ + +struct ChronyMonitorConfig +{ + std::string chronyc_path = "chronyc"; + /// Empty queries the local chronyd. A host queries a remote one, which must allow this client (cmdallow); + /// chronyd accepts only monitoring commands from the network. + std::string host; + int port = 0; ///< 0 uses chronyc's default. + std::chrono::milliseconds poll_interval{ 1000 }; + std::chrono::milliseconds command_timeout{ 2000 }; +}; + +enum class MonitorStatus +{ + NotPolled, + Ok, + CommandFailed, ///< Could not start, non-zero exit, or truncated output. + TimedOut, + ParseFailed, +}; + +const char* to_string(MonitorStatus s); + +struct MonitorSnapshot +{ + /// Newest good report, stamped when its query began on the monitor's monotonic clock. It is kept across + /// failures so that it simply ages into staleness; check `last_status` for why it stopped updating. + std::optional evidence; + MonitorStatus last_status = MonitorStatus::NotPolled; + uint64_t polls = 0; + uint64_t failures = 0; +}; + +/// Polls `chronyc -c tracking` on a background thread, outside any frame path. One report per poll, no queueing. +class ChronyMonitor +{ +public: + using NowFn = std::function; + + /// The runner and clocks default to the real process runner, CLOCK_MONOTONIC and CLOCK_REALTIME. + explicit ChronyMonitor(ChronyMonitorConfig config, + CommandRunner runner = {}, + NowFn now_monotonic_ns = {}, + NowFn now_realtime_ns = {}); + ~ChronyMonitor(); + + ChronyMonitor(const ChronyMonitor&) = delete; + ChronyMonitor& operator=(const ChronyMonitor&) = delete; + + /// Starts the polling thread; a second call does nothing. + void start(); + /// Wakes and joins the thread; safe to call repeatedly. Returns within the command timeout at worst. + void stop(); + + /// One poll on the calling thread; this is what the thread runs, and what tests drive directly. + MonitorStatus poll_once(); + + MonitorSnapshot snapshot() const; + +private: + void run(); + + ChronyMonitorConfig config_; + CommandRunner runner_; + NowFn now_; + NowFn now_realtime_; + mutable std::mutex mutex_; + MonitorSnapshot snapshot_; + std::condition_variable wake_; + bool stopping_ = false; + std::thread thread_; +}; + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/inc/timesync/chrony_tracking.hpp b/src/core/timesync/cpp/inc/timesync/chrony_tracking.hpp new file mode 100644 index 0000000000..49b25f43c4 --- /dev/null +++ b/src/core/timesync/cpp/inc/timesync/chrony_tracking.hpp @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include + +namespace isaaccapture::timesync +{ + +/// One `chronyc -c tracking` report (chrony 4.x, 14 CSV fields). Times are seconds, rates are ppm. +struct ChronyTracking +{ + std::string reference_id; ///< Upper-case hex; an NTP source is its IPv4 address, 7F7F0101 is a local reference. + std::string reference_name; + int stratum = 0; + double ref_time_unix_s = 0.0; + double system_time_s = 0.0; ///< Signed; only the magnitude is used, so the sign convention does not matter. + double last_offset_s = 0.0; + double rms_offset_s = 0.0; + double frequency_ppm = 0.0; + double residual_freq_ppm = 0.0; + double skew_ppm = 0.0; + double root_delay_s = 0.0; + double root_dispersion_s = 0.0; + double update_interval_s = 0.0; + std::string leap_status; ///< "Normal", "Insert second", "Delete second" or "Not synchronised". +}; + +enum class ChronyParseError +{ + None, + Empty, + FieldCount, + BadNumber, ///< Not a finite number, or a field that cannot be negative is. + BadStratum, +}; + +struct ChronyParseResult +{ + std::optional tracking; + ChronyParseError error = ChronyParseError::None; +}; + +/// Parses one CSV line. Anything unexpected is an error; nothing is guessed. +ChronyParseResult parse_tracking_csv(std::string_view line); + +/// Reference id chrony reports when the host serves its own clock (`local` directive). +inline constexpr std::string_view kChronyLocalReferenceId = "7F7F0101"; + +/// True when chrony reports a usable reference: a normal or leap-pending status and a stratum of 1 to 15. +/// A local reference passes only when `allow_local_reference` is set, because it says nothing about UTC. +bool is_synchronised(const ChronyTracking& t, bool allow_local_reference); + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/inc/timesync/command_runner.hpp b/src/core/timesync/cpp/inc/timesync/command_runner.hpp new file mode 100644 index 0000000000..a6ff140280 --- /dev/null +++ b/src/core/timesync/cpp/inc/timesync/command_runner.hpp @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include + +namespace isaaccapture::timesync +{ + +struct CommandResult +{ + bool started = false; ///< The program was found and launched. + bool timed_out = false; ///< It was killed for exceeding the timeout. + bool truncated = false; ///< Output beyond the cap was discarded. + int exit_code = -1; ///< Valid only when it ran to completion. + std::string output; ///< Standard output; standard error is discarded. +}; + +/// Runs a program with an argument vector, never through a shell. Tests substitute fakes. +using CommandRunner = + std::function& argv, std::chrono::milliseconds timeout)>; + +/// Output cap for the default runner. +inline constexpr size_t kMaxCommandOutputBytes = 64 * 1024; + +/// posix_spawnp the program, read its stdout up to the cap, kill it when the timeout passes. +CommandRunner posix_command_runner(); + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/inc/timesync/local_clock.hpp b/src/core/timesync/cpp/inc/timesync/local_clock.hpp new file mode 100644 index 0000000000..0c9f612bdd --- /dev/null +++ b/src/core/timesync/cpp/inc/timesync/local_clock.hpp @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +namespace isaaccapture::timesync +{ + +/// Where a host reads its clocks. Tests substitute fakes; the default reads CLOCK_MONOTONIC and CLOCK_REALTIME. +/// A failed read returns kClockReadFailed. +inline constexpr int64_t kClockReadFailed = INT64_MIN; + +struct ClockReader +{ + std::function monotonic_ns; + std::function realtime_ns; +}; + +ClockReader system_clock_reader(); + +/// One paired reading: the realtime value is taken between two monotonic reads, and the monotonic instant is the +/// bracket midpoint, so the pair is off by at most `read_uncertainty_ns`. A failed or inconsistent read gives +/// `read_uncertainty_ns == INT64_MAX`, which no tracker accepts. +struct LocalClockSample +{ + int64_t monotonic_ns = 0; + int64_t realtime_ns = 0; + int64_t read_uncertainty_ns = 0; +}; + +LocalClockSample sample_local_clock(const ClockReader& reader); + +/// realtime minus monotonic at the sample, the offset that maps a realtime stamp onto this host's monotonic clock. +constexpr int64_t wall_minus_monotonic_ns(const LocalClockSample& s) +{ + return s.realtime_ns - s.monotonic_ns; +} + +struct LocalClockConfig +{ + /// A sample whose bracket is wider than this is unusable. + int64_t max_read_uncertainty_ns = 200'000; + /// CLOCK_REALTIME and CLOCK_MONOTONIC share frequency adjustment on Linux, so their difference changes only on + /// a step. A change beyond both bracket widths plus this margin is a step. A step smaller than that goes + /// undetected, so the margin is also charged to every conversion as uncertainty (`step_tolerance_ns`). + int64_t step_margin_ns = 100'000; +}; + +struct LocalClockObservation +{ + LocalClockSample sample; + uint64_t generation = 0; ///< Changes whenever the realtime clock stepped, so older mappings are not reused. + bool usable = false; ///< False when the bracket was too wide or the readings were invalid. + int64_t step_tolerance_ns = 0; ///< Largest step this tracker can miss; added to the conversion's uncertainty. + bool stepped = false; ///< True for the sample that revealed a step. +}; + +/// Watches successive samples of one host and numbers the intervals between detected realtime steps. +class LocalClockTracker +{ +public: + explicit LocalClockTracker(LocalClockConfig config = {}); + + LocalClockObservation observe(const LocalClockSample& sample); + +private: + LocalClockConfig config_; + bool have_previous_ = false; + LocalClockSample previous_{}; + uint64_t generation_ = 1; +}; + +} // namespace isaaccapture::timesync diff --git a/src/core/timesync/cpp/local_clock.cpp b/src/core/timesync/cpp/local_clock.cpp new file mode 100644 index 0000000000..effea619ef --- /dev/null +++ b/src/core/timesync/cpp/local_clock.cpp @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/timesync/local_clock.hpp" + +#include + +namespace isaaccapture::timesync +{ + +namespace +{ + +int64_t read_clock_ns(clockid_t id) +{ + timespec ts{}; + int64_t ns = 0; + if (clock_gettime(id, &ts) != 0 || __builtin_mul_overflow(static_cast(ts.tv_sec), 1'000'000'000LL, &ns) || + __builtin_add_overflow(ns, static_cast(ts.tv_nsec), &ns)) + { + return kClockReadFailed; + } + return ns; +} + +// Magnitude of a difference that cannot overflow, including INT64_MIN. +bool abs_diff(int64_t a, int64_t b, uint64_t& out) +{ + int64_t d = 0; + if (__builtin_sub_overflow(a, b, &d)) + { + return false; + } + out = d < 0 ? ~static_cast(d) + 1 : static_cast(d); + return true; +} + +} // namespace + +ClockReader system_clock_reader() +{ + return { [] { return read_clock_ns(CLOCK_MONOTONIC); }, [] { return read_clock_ns(CLOCK_REALTIME); } }; +} + +LocalClockSample sample_local_clock(const ClockReader& reader) +{ + const int64_t before = reader.monotonic_ns(); + const int64_t wall = reader.realtime_ns(); + const int64_t after = reader.monotonic_ns(); + LocalClockSample s; + s.read_uncertainty_ns = INT64_MAX; + int64_t width = 0; + if (before == kClockReadFailed || wall == kClockReadFailed || after == kClockReadFailed || + __builtin_sub_overflow(after, before, &width) || width < 0 || width == INT64_MAX) + { + return s; + } + s.monotonic_ns = before + width / 2; + s.realtime_ns = wall; + s.read_uncertainty_ns = (width + 1) / 2; // rounded up, so the pair is never claimed tighter than it is + return s; +} + +LocalClockTracker::LocalClockTracker(LocalClockConfig config) : config_(config) +{ +} + +LocalClockObservation LocalClockTracker::observe(const LocalClockSample& sample) +{ + LocalClockObservation obs; + obs.sample = sample; + obs.generation = generation_; + if (sample.read_uncertainty_ns < 0 || sample.read_uncertainty_ns > config_.max_read_uncertainty_ns) + { + return obs; // unusable; the previous sample stays the reference + } + obs.usable = true; + obs.step_tolerance_ns = config_.step_margin_ns; + + int64_t offset = 0; + if (__builtin_sub_overflow(sample.realtime_ns, sample.monotonic_ns, &offset)) + { + obs.usable = false; + obs.step_tolerance_ns = 0; + return obs; + } + + if (have_previous_) + { + // CLOCK_MONOTONIC and CLOCK_REALTIME follow the same frequency adjustment, so their difference changes only + // on a step (or a suspend, which pauses only the monotonic clock). + uint64_t moved = 0; + int64_t previous_offset = 0; + if (__builtin_sub_overflow(previous_.realtime_ns, previous_.monotonic_ns, &previous_offset) || + !abs_diff(offset, previous_offset, moved)) + { + obs.usable = false; + obs.step_tolerance_ns = 0; + return obs; + } + const __int128 allowed = + static_cast<__int128>(sample.read_uncertainty_ns) + previous_.read_uncertainty_ns + config_.step_margin_ns; + if (sample.monotonic_ns < previous_.monotonic_ns || static_cast<__int128>(moved) > allowed) + { + ++generation_; + obs.stepped = true; + obs.generation = generation_; + } + } + previous_ = sample; + have_previous_ = true; + return obs; +} + +} // namespace isaaccapture::timesync diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 54c6e1e0d5..68fe2c6c95 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -12,6 +12,9 @@ add_subdirectory(replay_deviceio_session) add_subdirectory(live_trackers) add_subdirectory(tracker_metadata) add_subdirectory(log_bridge) +if(NOT WIN32) + add_subdirectory(timesync) +endif() add_subdirectory(plugin_manager) if(BUILD_OVXR_CAMERA_QUADS) diff --git a/tests/cpp/core/timesync/CMakeLists.txt b/tests/cpp/core/timesync/CMakeLists.txt new file mode 100644 index 0000000000..170c57f55d --- /dev/null +++ b/tests/cpp/core/timesync/CMakeLists.txt @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_subdirectory(chrony_tracking) +add_subdirectory(local_clock) +add_subdirectory(capture_qualifier) +add_subdirectory(command_runner) +add_subdirectory(chrony_monitor) diff --git a/tests/cpp/core/timesync/capture_qualifier/CMakeLists.txt b/tests/cpp/core/timesync/capture_qualifier/CMakeLists.txt new file mode 100644 index 0000000000..4bc7e72404 --- /dev/null +++ b/tests/cpp/core/timesync/capture_qualifier/CMakeLists.txt @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(timesync_capture_qualifier_tests test_capture_qualifier.cpp) + +target_link_libraries(timesync_capture_qualifier_tests PRIVATE timesync::timesync) + +target_compile_options(timesync_capture_qualifier_tests PRIVATE -Wall -Wextra) + +add_test(NAME timesync_capture_qualifier COMMAND timesync_capture_qualifier_tests) diff --git a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp new file mode 100644 index 0000000000..a046f2565b --- /dev/null +++ b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp @@ -0,0 +1,753 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace isaaccapture::timesync; + +static int g_failures = 0; + +#define CHECK(cond) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +namespace +{ + +constexpr int64_t kMs = 1'000'000; +constexpr int64_t kSecond = 1'000'000'000; +constexpr int64_t kPcMono = 50 * kSecond; +constexpr int64_t kWallMinusMono = 1'700'000'000'000'000'000; +constexpr int64_t kNow = kPcMono + kSecond; +constexpr int64_t kCaptureAge = 10 * kMs; +constexpr int64_t kSensorNs = 30'000; +constexpr int64_t kPcLocalNs = 20'000; +const char* const kSenderId = "A9FEA97B"; // the camera host, as the PC's chronyd reports it +const char* const kUpstreamId = "5BBD5B9D"; // whatever the camera host itself follows + +// Double-to-ns conversion: the header does not state the rounding, so allow 1 ns per converted bound. +bool within(int64_t actual, int64_t expected, int64_t tol) +{ + return std::llabs(actual - expected) <= tol; +} + +ChronyTracking synced(const std::string& id = kSenderId) +{ + ChronyTracking t; + t.reference_id = id; + t.stratum = 3; + t.leap_status = "Normal"; + return t; +} + +struct Fixture +{ + SenderCapture capture; + std::optional sender; + std::optional pc; + LocalClockObservation clock; + int64_t now = kNow; + QualifierConfig config; + + Fixture() + { + config.expected_sender_reference_id = kSenderId; + capture.valid = true; + capture.mapping_uncertainty_ns = kSensorNs; + capture.realtime_ns = kWallMinusMono + (kNow - kCaptureAge); + // The PC's chronyd follows the camera host: pc is the follower and carries the expected reference id; sender + // is the server and has an arbitrary upstream id. + sender = ChronyEvidence{ synced(kUpstreamId), kNow - kSecond }; + pc = ChronyEvidence{ synced(), kNow - kSecond }; + clock.sample.monotonic_ns = kPcMono; + clock.sample.realtime_ns = kPcMono + kWallMinusMono; + clock.sample.read_uncertainty_ns = kPcLocalNs; + clock.generation = 1; + clock.usable = true; + } + + Qualification run() const + { + return qualify_capture(capture, sender, pc, clock, now, config); + } + + void set_capture_monotonic(int64_t mono) + { + capture.realtime_ns = kWallMinusMono + mono; + } +}; + +void test_to_string() +{ + const Verdict all[] = { Verdict::Qualified, Verdict::SenderInvalid, Verdict::NoSenderEvidence, + Verdict::NoPcEvidence, Verdict::StaleEvidence, Verdict::Unsynchronised, + Verdict::ReferenceMismatch, Verdict::NoLocalClock, Verdict::ImplausibleAge, + Verdict::OverBudget }; + std::set names; + for (Verdict v : all) + { + const char* s = to_string(v); + CHECK(s != nullptr && std::strlen(s) > 0); + if (s) + names.insert(s); + } + CHECK(names.size() == sizeof(all) / sizeof(all[0])); +} + +void test_chrony_error_bound() +{ + ChronyTracking t; + CHECK(chrony_error_bound_ns(t, 0) == 0); + CHECK(chrony_error_bound_ns(t, 5 * kSecond) == 0); + + t.system_time_s = 1e-4; + t.root_dispersion_s = 2e-4; + t.root_delay_s = 4e-4; + // 100 us + 200 us + 400 us / 2 + CHECK(within(chrony_error_bound_ns(t, 0), 500'000, 1)); + + t.skew_ppm = 10.0; + // skew term: 10e-6 * 2 s = 20 us + CHECK(within(chrony_error_bound_ns(t, 2 * kSecond), 520'000, 1)); + + ChronyTracking pos; + pos.system_time_s = 3e-4; + ChronyTracking neg; + neg.system_time_s = -3e-4; + CHECK(within(chrony_error_bound_ns(pos, 0), 300'000, 1)); + CHECK(chrony_error_bound_ns(neg, 0) == chrony_error_bound_ns(pos, 0)); + + ChronyTracking skew_only; + skew_only.skew_ppm = 10.0; + CHECK(chrony_error_bound_ns(skew_only, -kSecond) >= 0); +} + +void test_qualified_happy_path() +{ + Fixture f; + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.capture_monotonic_ns == f.capture.realtime_ns - kWallMinusMono); + CHECK(q.capture_monotonic_ns == kNow - kCaptureAge); + CHECK(q.generation == 1); + CHECK(q.terms.sensor_ns == kSensorNs); + CHECK(q.terms.pc_local_ns == kPcLocalNs); + CHECK(q.terms.inter_host_ns == 0); + CHECK(q.uncertainty_ns == q.terms.sensor_ns + q.terms.inter_host_ns + q.terms.pc_local_ns); +} + +void test_conversion_uses_pc_offset_not_sample_time() +{ + Fixture f; + f.now = kNow + 5 * kMs; // later than the clock sample; the offset still applies + f.set_capture_monotonic(f.now - kMs); + f.sender->observed_monotonic_ns = f.now - kSecond; + f.pc->observed_monotonic_ns = f.now - kSecond; + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.capture_monotonic_ns == f.now - kMs); +} + +void test_inter_host_terms() +{ + Fixture f; + f.pc->tracking.system_time_s = 1e-4; + f.pc->tracking.root_dispersion_s = 2e-4; + f.pc->tracking.root_delay_s = 4e-4; + f.pc->tracking.skew_ppm = 10.0; + f.sender->tracking.system_time_s = -2e-4; + f.sender->tracking.root_dispersion_s = 1e-4; + f.sender->tracking.root_delay_s = 2e-4; + f.sender->tracking.skew_ppm = 5.0; + f.config.budget_ns = 10 * kMs; + const Qualification q = f.run(); + // Both reports are one second old, follower (PC) update interval 0. |system time|: 100 us + 200 us; root dispersion + // difference 200 - 100 = 100 us; root delay difference (400 - 200) / 2 = 100 us; skew * age: 10 us + 5 us. + CHECK(q.verdict == Verdict::Qualified); + CHECK(within(q.terms.inter_host_ns, 515'000, 2)); + CHECK(q.uncertainty_ns == q.terms.sensor_ns + q.terms.inter_host_ns + q.terms.pc_local_ns); +} + +// Evidence with every chrony term zero, so each test sets only the term under test. +ChronyEvidence zero_evidence() +{ + return ChronyEvidence{ synced(), 0 }; +} + +void test_upstream_cancels() +{ + // The camera host is internet-disciplined: large shared root fields, small link added by the PC. + Fixture f; + f.sender->tracking.root_delay_s = 0.1; + f.sender->tracking.root_dispersion_s = 0.01; + f.pc->tracking.root_delay_s = 0.1 + 200e-6; + f.pc->tracking.root_dispersion_s = 0.01 + 50e-6; + const Qualification q = f.run(); + // Link only: 50 us + 200 us / 2 = 150 us. The summed model would charge about 0.11 s + 0.11 s. + CHECK(within(q.terms.inter_host_ns, 150'000, 50)); // double error on the 0.1 s operands + CHECK(q.terms.inter_host_ns < kMs); + CHECK(q.verdict == Verdict::Qualified); + CHECK(f.config.budget_ns == 5 * kMs); + CHECK(q.uncertainty_ns == q.terms.sensor_ns + q.terms.inter_host_ns + q.terms.pc_local_ns); +} + +void test_pc_below_camera_host_clamps_to_zero() +{ + Fixture f; + f.sender->tracking.root_delay_s = 0.1; + f.sender->tracking.root_dispersion_s = 0.01; + f.pc->tracking.root_delay_s = 0.05; + f.pc->tracking.root_dispersion_s = 0.005; + CHECK(f.run().terms.inter_host_ns == 0); + CHECK(f.run().verdict == Verdict::Qualified); + // Only the dispersion is below: the delay difference still counts, the dispersion one does not. + f.pc->tracking.root_delay_s = 0.1 + 400e-6; + CHECK(within(f.run().terms.inter_host_ns, 200'000, 50)); + // Only the delay is below. + f.pc->tracking.root_delay_s = 0.05; + f.pc->tracking.root_dispersion_s = 0.01 + 80e-6; + CHECK(within(f.run().terms.inter_host_ns, 80'000, 50)); +} + +void test_copy_slack() +{ + // In the direct-call tests below, s is the follower (the PC) and p the server (the camera host). + ChronyEvidence s = zero_evidence(); + ChronyEvidence p = zero_evidence(); + p.tracking.skew_ppm = 1.0; + s.tracking.update_interval_s = 16.0; + // 1e-6 * 16 s + CHECK(inter_host_bound_ns(s, p, 0, 0) == 16'000); + // The follower's skew and the server's update interval do not enter the slack. + ChronyEvidence s2 = zero_evidence(); + s2.tracking.skew_ppm = 1.0; + ChronyEvidence p2 = zero_evidence(); + p2.tracking.update_interval_s = 16.0; + CHECK(inter_host_bound_ns(s2, p2, 0, 0) == 0); +} + +void test_aging_charged_per_host() +{ + ChronyEvidence s = zero_evidence(); + ChronyEvidence p = zero_evidence(); + s.tracking.skew_ppm = 2.0; + p.tracking.skew_ppm = 1.0; + // follower 2 ppm * 3 s = 6 us; server 1 ppm * 4 s = 4 us. + CHECK(inter_host_bound_ns(s, p, 3 * kSecond, 4 * kSecond) == 10'000); + CHECK(inter_host_bound_ns(s, p, 3 * kSecond, 0) == 6'000); + CHECK(inter_host_bound_ns(s, p, 0, 4 * kSecond) == 4'000); + // Negative ages are treated as 0. + CHECK(inter_host_bound_ns(s, p, -kSecond, -kSecond) == 0); + + // Through qualify_capture, with the fixture's reports one second old: 2 us + 1 us. + Fixture f; + f.pc->tracking.skew_ppm = 2.0; + f.sender->tracking.skew_ppm = 1.0; + CHECK(f.run().terms.inter_host_ns == 3'000); + f.pc->observed_monotonic_ns = f.now - 3 * kSecond; + f.sender->observed_monotonic_ns = f.now - 4 * kSecond; + CHECK(f.run().terms.inter_host_ns == 10'000); +} + +void test_both_system_times_charged() +{ + ChronyEvidence s = zero_evidence(); + ChronyEvidence p = zero_evidence(); + s.tracking.system_time_s = 100e-6; + CHECK(within(inter_host_bound_ns(s, p, 0, 0), 100'000, 1)); + s.tracking.system_time_s = 0.0; + p.tracking.system_time_s = 250e-6; + CHECK(within(inter_host_bound_ns(s, p, 0, 0), 250'000, 1)); + s.tracking.system_time_s = 100e-6; + CHECK(within(inter_host_bound_ns(s, p, 0, 0), 350'000, 1)); + // The sign does not matter: only the magnitude is charged. + s.tracking.system_time_s = -100e-6; + p.tracking.system_time_s = -250e-6; + CHECK(within(inter_host_bound_ns(s, p, 0, 0), 350'000, 1)); + s.tracking.system_time_s = 100e-6; + p.tracking.system_time_s = -250e-6; + CHECK(within(inter_host_bound_ns(s, p, 0, 0), 350'000, 1)); +} + +void test_inter_host_bound_direct() +{ + ChronyEvidence s = zero_evidence(); + ChronyEvidence p = zero_evidence(); + CHECK(inter_host_bound_ns(s, p, 0, 0) == 0); + CHECK(inter_host_bound_ns(s, p, 100 * kSecond, 100 * kSecond) == 0); + + // Root fields equal on both sides cancel completely, however large. + s.tracking.root_delay_s = p.tracking.root_delay_s = 0.1; + s.tracking.root_dispersion_s = p.tracking.root_dispersion_s = 0.01; + CHECK(inter_host_bound_ns(s, p, 0, 0) == 0); + + // Dispersion difference counts in full, delay difference at half; exact in binary doubles. + ChronyEvidence s2 = zero_evidence(); + ChronyEvidence p2 = zero_evidence(); + s2.tracking.root_dispersion_s = 0.5; + s2.tracking.root_delay_s = 0.25; + CHECK(inter_host_bound_ns(s2, p2, 0, 0) == 500'000'000 + 125'000'000); + + // A follower's root fields below the server's contribute nothing, never a negative term. + CHECK(inter_host_bound_ns(p2, s2, 0, 0) == 0); + + // Combined: 1 us slack + 2 us follower aging + 3 us server aging = 6 us. + ChronyEvidence s3 = zero_evidence(); + ChronyEvidence p3 = zero_evidence(); + p3.tracking.skew_ppm = 1.0; + s3.tracking.update_interval_s = 1.0; + s3.tracking.skew_ppm = 2.0; + CHECK(inter_host_bound_ns(s3, p3, kSecond, 3 * kSecond) == 1'000 + 2'000 + 3'000); + + // Rounds up and saturates. + ChronyEvidence s4 = zero_evidence(); + s4.tracking.system_time_s = 1.5e-9; + CHECK(inter_host_bound_ns(s4, zero_evidence(), 0, 0) == 2); + s4.tracking.system_time_s = 1e12; + CHECK(inter_host_bound_ns(s4, zero_evidence(), 0, 0) == INT64_MAX / 4); // the saturation value +} + +void test_simple_rejections() +{ + { + Fixture f; + f.capture.valid = false; + CHECK(f.run().verdict == Verdict::SenderInvalid); + } + { + Fixture f; + f.sender.reset(); + CHECK(f.run().verdict == Verdict::NoSenderEvidence); + } + { + Fixture f; + f.pc.reset(); + CHECK(f.run().verdict == Verdict::NoPcEvidence); + } + { + Fixture f; + f.clock.usable = false; + CHECK(f.run().verdict == Verdict::NoLocalClock); + } +} + +void test_evidence_age_boundary() +{ + const int64_t max_age = QualifierConfig{}.max_evidence_age_ns; + for (bool sender_side : { true, false }) + { + Fixture f; + auto& e = sender_side ? f.sender : f.pc; + e->observed_monotonic_ns = f.now - max_age; + CHECK(f.run().verdict == Verdict::Qualified); + e->observed_monotonic_ns = f.now - max_age - 1; + CHECK(f.run().verdict == Verdict::StaleEvidence); + } +} + +void test_unsynchronised() +{ + { + Fixture f; + f.sender->tracking.leap_status = "Not synchronised"; + CHECK(f.run().verdict == Verdict::Unsynchronised); + } + { + Fixture f; + f.pc->tracking.leap_status = "Not synchronised"; + CHECK(f.run().verdict == Verdict::Unsynchronised); + } + { + Fixture f; + f.sender->tracking.leap_status = "Insert second"; + f.pc->tracking.leap_status = "Delete second"; + CHECK(f.run().verdict == Verdict::Qualified); + } +} + +void test_reference_matching() +{ + { + Fixture f; + f.pc->tracking.reference_id = "DEADBEEF"; + CHECK(f.run().verdict == Verdict::ReferenceMismatch); + } + { + // Ids compare case-insensitively (the header documents upper-case expected ids only). + Fixture f; + f.pc->tracking.reference_id = "a9fea97b"; + CHECK(f.run().verdict == Verdict::Qualified); + } + { + Fixture f; + f.config.expected_sender_reference_id = "a9fea97b"; + CHECK(f.run().verdict == Verdict::Qualified); + } +} + +void test_reference_is_the_pcs_not_the_camera_hosts() +{ + // The camera host's own upstream is not what is checked. + Fixture g; + g.sender->tracking.reference_id = "0A0B0C0D"; + CHECK(g.run().verdict == Verdict::Qualified); + + Fixture h; + h.config.expected_sender_reference_id.clear(); + CHECK(h.run().verdict == Verdict::ReferenceMismatch); + + // Swapped evidence: the PC reports the camera host's upstream, not the camera host. + Fixture f; + std::swap(f.sender, f.pc); + CHECK(f.run().verdict == Verdict::ReferenceMismatch); +} + +void test_unsynchronised_pc() +{ + Fixture f; + f.pc->tracking.leap_status = "Not synchronised"; + CHECK(f.run().verdict == Verdict::Unsynchronised); + + Fixture g; + g.pc->tracking.stratum = 0; + CHECK(g.run().verdict == Verdict::Unsynchronised); +} + +void test_local_reference_pc_refused() +{ + Fixture f; + f.pc->tracking.reference_id = std::string(kChronyLocalReferenceId); + CHECK(f.run().verdict == Verdict::Unsynchronised); +} + +void test_local_reference_camera_host_accepted() +{ + // The camera host may serve its own clock; only the PC has to follow it. + Fixture f; + f.sender->tracking.reference_id = std::string(kChronyLocalReferenceId); + CHECK(f.run().verdict == Verdict::Qualified); +} + +void test_unsynchronised_camera_host() +{ + Fixture f; + f.sender->tracking.leap_status = "Not synchronised"; + CHECK(f.run().verdict == Verdict::Unsynchronised); +} + +void test_implausible_age() +{ + const QualifierConfig def; + { + Fixture f; + f.set_capture_monotonic(f.now - def.max_capture_age_ns - 1); + CHECK(f.run().verdict == Verdict::ImplausibleAge); + } + { + Fixture f; + f.set_capture_monotonic(f.now + def.future_tolerance_ns + 1); + CHECK(f.run().verdict == Verdict::ImplausibleAge); + } + { + Fixture f; + f.set_capture_monotonic(f.now + def.future_tolerance_ns / 2); + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.capture_monotonic_ns == f.now + def.future_tolerance_ns / 2); + } +} + +void test_budget_boundary() +{ + Fixture f; + const int64_t unc = kSensorNs + kPcLocalNs; // chrony terms are zero, so the total is exact integers + + f.config.budget_ns = unc; + Qualification q = f.run(); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.uncertainty_ns == unc); + + f.config.budget_ns = unc - 1; + q = f.run(); + CHECK(q.verdict == Verdict::OverBudget); + CHECK(q.uncertainty_ns == unc); + CHECK(q.terms.sensor_ns == kSensorNs); + CHECK(q.terms.pc_local_ns == kPcLocalNs); + CHECK(q.generation == f.clock.generation); +} + +void test_generation_carried_through() +{ + Fixture f; + f.clock.generation = 3; + CHECK(f.run().generation == 3); + // A newer observation is simply reported; the qualifier does not compare generations. + f.clock.generation = 4; + f.clock.sample.realtime_ns += 7 * kSecond; // step already folded into the observation's own offset + f.set_capture_monotonic(f.now - kCaptureAge); + f.capture.realtime_ns += 7 * kSecond; + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.generation == 4); + CHECK(q.capture_monotonic_ns == f.now - kCaptureAge); +} + +void test_error_bound_rounds_up() +{ + ChronyTracking t; + t.root_dispersion_s = 1.5e-9; + CHECK(chrony_error_bound_ns(t, 0) == 2); + t.root_dispersion_s = 1e-9; + CHECK(chrony_error_bound_ns(t, 0) == 1); + t.root_dispersion_s = 0.0; + CHECK(chrony_error_bound_ns(t, 0) == 0); +} + +void test_overflowing_inputs_are_refused() +{ + { + Fixture f; + f.sender->observed_monotonic_ns = INT64_MIN; // now - observed overflows + CHECK(f.run().verdict == Verdict::StaleEvidence); + f.sender->observed_monotonic_ns = f.now - kSecond; + f.pc->observed_monotonic_ns = INT64_MIN; + CHECK(f.run().verdict == Verdict::StaleEvidence); + } + { + Fixture f; + f.capture.realtime_ns = INT64_MIN; // realtime - offset overflows + CHECK(f.run().verdict == Verdict::ImplausibleAge); + } + { + Fixture f; + f.clock.sample.realtime_ns = INT64_MAX; // realtime - monotonic overflows + f.clock.sample.monotonic_ns = -5; + CHECK(f.run().verdict == Verdict::ImplausibleAge); + } +} + +void test_newer_generation_is_reported_for_the_caller_to_pin() +{ + // The qualifier does not compare generations: a take must pin the generation of its first frame and end + // when a later Qualified result carries a different one. + Fixture f; + f.clock.generation = 1; + const Qualification first = f.run(); + f.clock.generation = 2; + const Qualification later = f.run(); + CHECK(first.verdict == Verdict::Qualified); + CHECK(later.verdict == Verdict::Qualified); + CHECK(first.generation == 1); + CHECK(later.generation == 2); + CHECK(later.generation != first.generation); +} + +void test_pc_local_term_includes_step_tolerance() +{ + Fixture f; + f.clock.step_tolerance_ns = 70'000; + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.terms.pc_local_ns == kPcLocalNs + 70'000); + CHECK(q.uncertainty_ns == q.terms.sensor_ns + q.terms.inter_host_ns + q.terms.pc_local_ns); + + f.config.budget_ns = kSensorNs + kPcLocalNs + 70'000 - 1; + CHECK(f.run().verdict == Verdict::OverBudget); +} + +// ref_time is a small whole number of seconds so a 1 ns difference survives the double arithmetic. +constexpr double kRefTimeS = 1000.0; +constexpr int64_t kObservedAtLimit = 1000 * kSecond + 2'048 * kSecond; + +void test_source_update_age_gate() +{ + CHECK(QualifierConfig{}.max_source_update_age_ns == 2'048 * kSecond); + for (bool sender_side : { true, false }) + { + Fixture f; + f.sender->tracking.ref_time_unix_s = kRefTimeS; + f.pc->tracking.ref_time_unix_s = kRefTimeS; + f.sender->observed_realtime_ns = kObservedAtLimit; + f.pc->observed_realtime_ns = kObservedAtLimit; + CHECK(f.run().verdict == Verdict::Qualified); + auto& e = sender_side ? f.sender : f.pc; + e->observed_realtime_ns = kObservedAtLimit + 1; + CHECK(f.run().verdict == Verdict::StaleEvidence); + } +} + +void test_source_update_gate_exemptions() +{ + { + // A local reference has no source updates. + Fixture f; + f.sender->tracking.reference_id = std::string(kChronyLocalReferenceId); + f.sender->tracking.ref_time_unix_s = 1.0; + f.sender->observed_realtime_ns = kWallMinusMono; + CHECK(f.run().verdict == Verdict::Qualified); + } + { + Fixture f; + f.sender->tracking.ref_time_unix_s = kRefTimeS; + f.sender->observed_realtime_ns = kObservedAtLimit + 1; + CHECK(f.run().verdict == Verdict::StaleEvidence); + f.config.max_source_update_age_ns = 0; + CHECK(f.run().verdict == Verdict::Qualified); + } + { + Fixture f; + f.pc->tracking.ref_time_unix_s = kRefTimeS; + f.pc->observed_realtime_ns = kObservedAtLimit + 1; + CHECK(f.run().verdict == Verdict::StaleEvidence); + f.pc->observed_realtime_ns = 0; + CHECK(f.run().verdict == Verdict::Qualified); + } +} + +void test_future_tolerance_min_behaves_like_zero() +{ + Fixture f; + f.config.future_tolerance_ns = INT64_MIN; + f.set_capture_monotonic(f.now + 1); + CHECK(f.run().verdict == Verdict::ImplausibleAge); + f.set_capture_monotonic(f.now); + CHECK(f.run().verdict == Verdict::Qualified); +} + +void test_negative_future_tolerance_behaves_like_zero() +{ + Fixture f; + f.config.future_tolerance_ns = -5; + f.set_capture_monotonic(f.now + 1); + CHECK(f.run().verdict == Verdict::ImplausibleAge); + f.set_capture_monotonic(f.now); + CHECK(f.run().verdict == Verdict::Qualified); +} + +void test_max_future_tolerance_accepts_far_future_capture() +{ + Fixture f; + f.config.future_tolerance_ns = INT64_MAX; + f.set_capture_monotonic(f.now + 100 * kSecond); + CHECK(f.run().verdict == Verdict::Qualified); +} + +// Observation exactly 1 s before ref_time, so ref_time is exactly at the default future tolerance. +constexpr int64_t kObservedOneSecBeforeRef = 999 * kSecond; + +void test_source_update_future_gate() +{ + CHECK(QualifierConfig{}.source_update_future_tolerance_ns == kSecond); + for (bool sender_side : { true, false }) + { + Fixture f; + f.sender->tracking.ref_time_unix_s = kRefTimeS; + f.pc->tracking.ref_time_unix_s = kRefTimeS; + f.sender->observed_realtime_ns = kRefTimeS * kSecond; + f.pc->observed_realtime_ns = kRefTimeS * kSecond; + auto& e = sender_side ? f.sender : f.pc; + e->observed_realtime_ns = kObservedOneSecBeforeRef; + CHECK(f.run().verdict == Verdict::Qualified); + e->observed_realtime_ns = kObservedOneSecBeforeRef - 1'000; + CHECK(f.run().verdict == Verdict::StaleEvidence); + } +} + +void test_source_update_future_gate_exemptions_and_zero_tolerance() +{ + { + // A local reference is exempt. + Fixture f; + f.sender->tracking.reference_id = std::string(kChronyLocalReferenceId); + f.sender->tracking.ref_time_unix_s = kRefTimeS; + f.sender->observed_realtime_ns = kSecond; + CHECK(f.run().verdict == Verdict::Qualified); + } + { + // An unknown observation time skips the gate. + Fixture f; + f.sender->tracking.ref_time_unix_s = kRefTimeS; + f.sender->observed_realtime_ns = 0; + CHECK(f.run().verdict == Verdict::Qualified); + f.sender->observed_realtime_ns = -1; + CHECK(f.run().verdict == Verdict::Qualified); + } + for (int64_t tol : { int64_t{ 0 }, int64_t{ -5 }, INT64_MIN }) + { + Fixture f; + f.config.source_update_future_tolerance_ns = tol; + f.sender->tracking.ref_time_unix_s = kRefTimeS; + f.pc->tracking.ref_time_unix_s = kRefTimeS; + f.sender->observed_realtime_ns = kRefTimeS * kSecond; + f.pc->observed_realtime_ns = kRefTimeS * kSecond; + CHECK(f.run().verdict == Verdict::Qualified); + f.pc->observed_realtime_ns = kRefTimeS * kSecond - 1'000; + CHECK(f.run().verdict == Verdict::StaleEvidence); + f.pc->observed_realtime_ns = kRefTimeS * kSecond; + f.sender->observed_realtime_ns = kRefTimeS * kSecond - 1'000; + CHECK(f.run().verdict == Verdict::StaleEvidence); + } +} + +} // namespace + +int main() +{ + test_to_string(); + test_chrony_error_bound(); + test_qualified_happy_path(); + test_conversion_uses_pc_offset_not_sample_time(); + test_inter_host_terms(); + test_upstream_cancels(); + test_pc_below_camera_host_clamps_to_zero(); + test_copy_slack(); + test_aging_charged_per_host(); + test_both_system_times_charged(); + test_inter_host_bound_direct(); + test_simple_rejections(); + test_evidence_age_boundary(); + test_unsynchronised(); + test_reference_matching(); + test_reference_is_the_pcs_not_the_camera_hosts(); + test_unsynchronised_pc(); + test_local_reference_pc_refused(); + test_local_reference_camera_host_accepted(); + test_unsynchronised_camera_host(); + test_implausible_age(); + test_budget_boundary(); + test_generation_carried_through(); + test_error_bound_rounds_up(); + test_overflowing_inputs_are_refused(); + test_newer_generation_is_reported_for_the_caller_to_pin(); + test_pc_local_term_includes_step_tolerance(); + test_source_update_age_gate(); + test_source_update_gate_exemptions(); + test_future_tolerance_min_behaves_like_zero(); + test_negative_future_tolerance_behaves_like_zero(); + test_max_future_tolerance_accepts_far_future_capture(); + test_source_update_future_gate(); + test_source_update_future_gate_exemptions_and_zero_tolerance(); + if (g_failures) + { + std::fprintf(stderr, "%d check(s) failed\n", g_failures); + return 1; + } + std::printf("timesync_capture_qualifier: all checks passed\n"); + return 0; +} diff --git a/tests/cpp/core/timesync/chrony_monitor/CMakeLists.txt b/tests/cpp/core/timesync/chrony_monitor/CMakeLists.txt new file mode 100644 index 0000000000..7952e19b5a --- /dev/null +++ b/tests/cpp/core/timesync/chrony_monitor/CMakeLists.txt @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(timesync_chrony_monitor_tests test_chrony_monitor.cpp) + +target_link_libraries(timesync_chrony_monitor_tests PRIVATE timesync::timesync Threads::Threads) + +target_compile_options(timesync_chrony_monitor_tests PRIVATE -Wall -Wextra) + +add_test(NAME timesync_chrony_monitor COMMAND timesync_chrony_monitor_tests) diff --git a/tests/cpp/core/timesync/chrony_monitor/test_chrony_monitor.cpp b/tests/cpp/core/timesync/chrony_monitor/test_chrony_monitor.cpp new file mode 100644 index 0000000000..496391a355 --- /dev/null +++ b/tests/cpp/core/timesync/chrony_monitor/test_chrony_monitor.cpp @@ -0,0 +1,344 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace isaaccapture::timesync; +using std::chrono::milliseconds; + +static int g_failures = 0; + +#define CHECK(cond) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +namespace +{ + +const char* const kGoodLine = + "A9FEA97B,169.254.169.123,4,1707763422.384912731,0.000000011,-0.000001098,0.000008212,1.382,0.001,0.019," + "0.000289,0.000357,2.0,Normal"; + +using Clock = std::chrono::steady_clock; + +// Fake runner state shared with the test; the runner may be called from the monitor thread. +struct Fake +{ + std::mutex mutex; + CommandResult result; + std::vector last_argv; + milliseconds last_timeout{ 0 }; + std::atomic calls{ 0 }; + std::set threads; + std::atomic clock{ 1000 }; + std::atomic realtime{ 7'000'000'000'000 }; + int64_t advance_by = 0; // clock advance while the "command" runs + milliseconds block{ 0 }; // real sleep inside the runner + + CommandRunner runner() + { + return [this](const std::vector& argv, milliseconds timeout) + { + CommandResult r; + milliseconds b; + { + std::lock_guard lock(mutex); + last_argv = argv; + last_timeout = timeout; + threads.insert(std::this_thread::get_id()); + r = result; + b = block; + } + clock += advance_by; + realtime += advance_by; + ++calls; + if (b.count() > 0) + { + std::this_thread::sleep_for(b); + } + return r; + }; + } + ChronyMonitor::NowFn now() + { + return [this] { return clock.load(); }; + } + ChronyMonitor::NowFn now_realtime() + { + return [this] { return realtime.load(); }; + } +}; + +CommandResult good() +{ + CommandResult r; + r.started = true; + r.exit_code = 0; + r.output = std::string(kGoodLine) + "\n"; + return r; +} + +void test_argv_local() +{ + Fake f; + f.result = good(); + ChronyMonitorConfig cfg; + cfg.chronyc_path = "/usr/bin/chronyc"; + ChronyMonitor m(cfg, f.runner(), f.now()); + m.poll_once(); + CHECK((f.last_argv == std::vector{ "/usr/bin/chronyc", "-c", "tracking" })); +} + +void test_argv_remote() +{ + Fake f; + f.result = good(); + ChronyMonitorConfig cfg; + cfg.chronyc_path = "/usr/bin/chronyc"; + cfg.host = "10.0.0.5"; + cfg.port = 323; + ChronyMonitor m(cfg, f.runner(), f.now()); + m.poll_once(); + CHECK((f.last_argv == + std::vector{ "/usr/bin/chronyc", "-c", "-h", "10.0.0.5", "-p", "323", "tracking" })); +} + +void test_good_poll() +{ + Fake f; + f.result = good(); + f.advance_by = 500; + ChronyMonitorConfig cfg; + cfg.command_timeout = milliseconds(1234); + ChronyMonitor m(cfg, f.runner(), f.now()); + CHECK(m.snapshot().last_status == MonitorStatus::NotPolled); + CHECK(m.poll_once() == MonitorStatus::Ok); + const auto s = m.snapshot(); + CHECK(s.last_status == MonitorStatus::Ok); + CHECK(s.polls == 1); + CHECK(s.failures == 0); + CHECK(s.evidence.has_value()); + if (s.evidence) + { + CHECK(s.evidence->tracking.reference_id == "A9FEA97B"); + CHECK(s.evidence->tracking.stratum == 4); + CHECK(s.evidence->observed_monotonic_ns == 1000); // read before the runner advanced the clock + } + CHECK(f.clock.load() == 1500); + CHECK(f.last_timeout == milliseconds(1234)); +} + +void test_realtime_stamp_read_before_runner() +{ + Fake f; + f.result = good(); + f.advance_by = 500; // the runner advances both clocks + ChronyMonitor m({}, f.runner(), f.now(), f.now_realtime()); + CHECK(m.poll_once() == MonitorStatus::Ok); + const auto s = m.snapshot(); + CHECK(s.evidence.has_value()); + if (s.evidence) + { + CHECK(s.evidence->observed_monotonic_ns == 1000); + CHECK(s.evidence->observed_realtime_ns == 7'000'000'000'000); + } + CHECK(f.realtime.load() == 7'000'000'000'500); +} + +void test_default_realtime_clock_is_system_realtime() +{ + Fake f; + f.result = good(); + ChronyMonitor m({}, f.runner(), f.now()); + CHECK(m.poll_once() == MonitorStatus::Ok); + const auto s = m.snapshot(); + CHECK(s.evidence.has_value()); + if (s.evidence) + { + CHECK(s.evidence->observed_realtime_ns > 0); + CHECK(s.evidence->observed_monotonic_ns == 1000); + } +} + +void test_failure_statuses() +{ + Fake f; + ChronyMonitor m({}, f.runner(), f.now()); + + f.result = CommandResult{}; + f.result.started = true; + f.result.timed_out = true; + CHECK(m.poll_once() == MonitorStatus::TimedOut); + + f.result = CommandResult{}; // started=false + CHECK(m.poll_once() == MonitorStatus::CommandFailed); + + f.result = good(); + f.result.exit_code = 2; + CHECK(m.poll_once() == MonitorStatus::CommandFailed); + + f.result = good(); + f.result.truncated = true; + CHECK(m.poll_once() == MonitorStatus::CommandFailed); + + f.result = good(); + f.result.output = "garbage\n"; + CHECK(m.poll_once() == MonitorStatus::ParseFailed); + + const auto s = m.snapshot(); + CHECK(s.polls == 5); + CHECK(s.failures == 5); + CHECK(!s.evidence.has_value()); +} + +void test_evidence_kept_across_failure() +{ + Fake f; + f.result = good(); + ChronyMonitor m({}, f.runner(), f.now()); + CHECK(m.poll_once() == MonitorStatus::Ok); + f.clock = 9000; + f.result = CommandResult{}; + f.result.started = true; + f.result.timed_out = true; + CHECK(m.poll_once() == MonitorStatus::TimedOut); + const auto s = m.snapshot(); + CHECK(s.last_status == MonitorStatus::TimedOut); + CHECK(s.polls == 2); + CHECK(s.failures == 1); + CHECK(s.evidence.has_value()); + if (s.evidence) + { + CHECK(s.evidence->observed_monotonic_ns == 1000); + } +} + +void test_to_string_distinct() +{ + const MonitorStatus all[] = { MonitorStatus::NotPolled, MonitorStatus::Ok, MonitorStatus::CommandFailed, + MonitorStatus::TimedOut, MonitorStatus::ParseFailed }; + std::set names; + for (auto s : all) + { + names.insert(to_string(s)); + } + CHECK(names.size() == 5); + CHECK(names.count("unknown") == 0); +} + +bool wait_for_calls(const Fake& f, int n, milliseconds limit) +{ + const auto end = Clock::now() + limit; + while (Clock::now() < end) + { + if (f.calls.load() >= n) + { + return true; + } + std::this_thread::sleep_for(milliseconds(2)); + } + return f.calls.load() >= n; +} + +void test_thread_polls_and_single_start() +{ + Fake f; + f.result = good(); + ChronyMonitorConfig cfg; + cfg.poll_interval = milliseconds(10); + ChronyMonitor m(cfg, f.runner(), f.now()); + m.start(); + m.start(); + CHECK(wait_for_calls(f, 3, milliseconds(2000))); + m.stop(); + CHECK(m.snapshot().polls >= 3); + std::lock_guard lock(f.mutex); + CHECK(f.threads.size() == 1); + CHECK(f.threads.count(std::this_thread::get_id()) == 0); +} + +void test_stop_prompt_and_idempotent() +{ + Fake f; + f.result = good(); + ChronyMonitorConfig cfg; + cfg.poll_interval = std::chrono::seconds(60); + ChronyMonitor m(cfg, f.runner(), f.now()); + m.start(); + CHECK(wait_for_calls(f, 1, milliseconds(2000))); + const auto t0 = Clock::now(); + m.stop(); + m.stop(); + CHECK(Clock::now() - t0 < std::chrono::seconds(1)); +} + +void test_destructor_stops() +{ + Fake f; + f.result = good(); + const auto t0 = Clock::now(); + { + ChronyMonitorConfig cfg; + cfg.poll_interval = std::chrono::seconds(60); + ChronyMonitor m(cfg, f.runner(), f.now()); + m.start(); + CHECK(wait_for_calls(f, 1, milliseconds(2000))); + } + CHECK(Clock::now() - t0 < std::chrono::seconds(2)); +} + +void test_stop_during_blocking_runner() +{ + Fake f; + f.result = good(); + f.block = milliseconds(300); + ChronyMonitorConfig cfg; + cfg.poll_interval = std::chrono::seconds(60); + ChronyMonitor m(cfg, f.runner(), f.now()); + m.start(); + // Wait until the runner has been entered (calls increments just before it sleeps). + CHECK(wait_for_calls(f, 1, milliseconds(2000))); + const auto t0 = Clock::now(); + m.stop(); + CHECK(Clock::now() - t0 < milliseconds(300 + 700)); +} + +} // namespace + +int main() +{ + test_argv_local(); + test_argv_remote(); + test_good_poll(); + test_realtime_stamp_read_before_runner(); + test_default_realtime_clock_is_system_realtime(); + test_failure_statuses(); + test_evidence_kept_across_failure(); + test_to_string_distinct(); + test_thread_polls_and_single_start(); + test_stop_prompt_and_idempotent(); + test_destructor_stops(); + test_stop_during_blocking_runner(); + if (g_failures) + { + std::fprintf(stderr, "%d check(s) failed\n", g_failures); + return 1; + } + std::printf("timesync_chrony_monitor: all checks passed\n"); + return 0; +} diff --git a/tests/cpp/core/timesync/chrony_tracking/CMakeLists.txt b/tests/cpp/core/timesync/chrony_tracking/CMakeLists.txt new file mode 100644 index 0000000000..617c4473e0 --- /dev/null +++ b/tests/cpp/core/timesync/chrony_tracking/CMakeLists.txt @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(timesync_chrony_tracking_tests test_chrony_tracking.cpp) + +target_link_libraries(timesync_chrony_tracking_tests PRIVATE timesync::timesync) + +target_compile_options(timesync_chrony_tracking_tests PRIVATE -Wall -Wextra) + +add_test(NAME timesync_chrony_tracking COMMAND timesync_chrony_tracking_tests) diff --git a/tests/cpp/core/timesync/chrony_tracking/test_chrony_tracking.cpp b/tests/cpp/core/timesync/chrony_tracking/test_chrony_tracking.cpp new file mode 100644 index 0000000000..e2a069ec77 --- /dev/null +++ b/tests/cpp/core/timesync/chrony_tracking/test_chrony_tracking.cpp @@ -0,0 +1,289 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include +#include + +using namespace isaaccapture::timesync; + +static int g_failures = 0; + +#define CHECK(cond) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +namespace +{ + +const char* const kGoodLine = + "A9FEA97B,169.254.169.123,4,1707763422.384912731,0.000000011,-0.000001098,0.000008212,1.382,0.001,0.019," + "0.000289,0.000357,2.0,Normal"; + +bool near(double a, double b) +{ + return std::fabs(a - b) <= 1e-12 * std::fmax(1.0, std::fabs(b)); +} + +// Builds a 14-field line with one field replaced (index 0-based). +std::string with_field(int index, const std::string& value) +{ + const char* names[14] = { "A9FEA97B", "169.254.169.123", "4", "1707763422.384912731", + "0.000000011", "-0.000001098", "0.000008212", "1.382", + "0.001", "0.019", "0.000289", "0.000357", + "2.0", "Normal" }; + std::string out; + for (int i = 0; i < 14; ++i) + { + if (i) + out += ','; + out += (i == index) ? value : names[i]; + } + return out; +} + +ChronyTracking make(int stratum, const std::string& id, const std::string& leap) +{ + ChronyTracking t; + t.reference_id = id; + t.stratum = stratum; + t.leap_status = leap; + return t; +} + +void test_valid_line() +{ + auto r = parse_tracking_csv(kGoodLine); + CHECK(r.error == ChronyParseError::None); + CHECK(r.tracking.has_value()); + if (!r.tracking) + return; + const auto& t = *r.tracking; + CHECK(t.reference_id == "A9FEA97B"); + CHECK(t.reference_name == "169.254.169.123"); + CHECK(t.stratum == 4); + CHECK(near(t.ref_time_unix_s, 1707763422.384912731)); + CHECK(near(t.system_time_s, 0.000000011)); + CHECK(near(t.last_offset_s, -0.000001098)); + CHECK(near(t.rms_offset_s, 0.000008212)); + CHECK(near(t.frequency_ppm, 1.382)); + CHECK(near(t.residual_freq_ppm, 0.001)); + CHECK(near(t.skew_ppm, 0.019)); + CHECK(near(t.root_delay_s, 0.000289)); + CHECK(near(t.root_dispersion_s, 0.000357)); + CHECK(near(t.update_interval_s, 2.0)); + CHECK(t.leap_status == "Normal"); + CHECK(is_synchronised(t, false)); +} + +void test_local_reference_line() +{ + auto r = parse_tracking_csv( + "7F7F0101,,10,1707763422.000000000,0.000000000,0.000000000,0.000000000,0.000,0.000,0.000,0.000000," + "0.000000,64.0,Normal"); + CHECK(r.error == ChronyParseError::None); + CHECK(r.tracking.has_value()); + if (!r.tracking) + return; + CHECK(r.tracking->reference_id == "7F7F0101"); + CHECK(r.tracking->reference_id == kChronyLocalReferenceId); + CHECK(r.tracking->reference_name.empty()); + CHECK(r.tracking->stratum == 10); + CHECK(!is_synchronised(*r.tracking, false)); + CHECK(is_synchronised(*r.tracking, true)); +} + +void test_not_synchronised_line() +{ + auto r = parse_tracking_csv( + "00000000,,0,0.000000000,0.000000000,0.000000000,0.000000000,0.000,0.000,1000000.000,0.000000," + "0.000000,0.0,Not synchronised"); + CHECK(r.error == ChronyParseError::None); + CHECK(r.tracking.has_value()); + if (!r.tracking) + return; + CHECK(r.tracking->leap_status == "Not synchronised"); + CHECK(!is_synchronised(*r.tracking, false)); + CHECK(!is_synchronised(*r.tracking, true)); +} + +void test_empty_and_field_count() +{ + auto e = parse_tracking_csv(""); + CHECK(e.error == ChronyParseError::Empty); + CHECK(!e.tracking.has_value()); + + const std::string good = kGoodLine; + auto few = parse_tracking_csv(good.substr(0, good.rfind(','))); + CHECK(few.error == ChronyParseError::FieldCount); + CHECK(!few.tracking.has_value()); + + auto many = parse_tracking_csv(good + ",extra"); + CHECK(many.error == ChronyParseError::FieldCount); + CHECK(!many.tracking.has_value()); + + auto one = parse_tracking_csv("A9FEA97B"); + CHECK(one.error == ChronyParseError::FieldCount); +} + +void test_bad_numbers() +{ + // Every numeric field: non-numeric, NaN and inf are rejected. + const int numeric[] = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }; + for (int idx : numeric) + { + for (const char* bad : { "abc", "nan", "inf", "-inf", "" }) + { + auto r = parse_tracking_csv(with_field(idx, bad)); + CHECK(r.error == ChronyParseError::BadNumber); + CHECK(!r.tracking.has_value()); + } + } + // Fields that cannot be negative: skew, root delay, root dispersion. + for (int idx : { 9, 10, 11 }) + { + auto r = parse_tracking_csv(with_field(idx, "-0.000001")); + CHECK(r.error == ChronyParseError::BadNumber); + CHECK(!r.tracking.has_value()); + } + // Signed fields stay signed. + CHECK(parse_tracking_csv(with_field(4, "-0.5")).error == ChronyParseError::None); + CHECK(parse_tracking_csv(with_field(5, "-0.5")).error == ChronyParseError::None); +} + +void test_stratum_bounds() +{ + // Header: parse may reject (BadStratum) or accept, but a stratum outside 1..15 is never synchronised. + for (const char* s : { "0", "16", "17", "255" }) + { + auto r = parse_tracking_csv(with_field(2, s)); + CHECK(r.error == ChronyParseError::None || r.error == ChronyParseError::BadStratum); + CHECK(r.tracking.has_value() == (r.error == ChronyParseError::None)); + if (r.tracking) + { + CHECK(!is_synchronised(*r.tracking, false)); + CHECK(!is_synchronised(*r.tracking, true)); + } + } + for (const char* s : { "-1", "x", "4.5", "" }) + { + auto r = parse_tracking_csv(with_field(2, s)); + CHECK(r.error != ChronyParseError::None); + CHECK(!r.tracking.has_value()); + } + auto ok = parse_tracking_csv(with_field(2, "15")); + CHECK(ok.error == ChronyParseError::None); + CHECK(ok.tracking.has_value() && is_synchronised(*ok.tracking, false)); +} + +void test_line_endings() +{ + for (const char* suffix : { "\n", "\r\n" }) + { + auto r = parse_tracking_csv(std::string(kGoodLine) + suffix); + CHECK(r.error == ChronyParseError::None); + CHECK(r.tracking.has_value()); + if (r.tracking) + CHECK(r.tracking->leap_status == "Normal"); + } +} + +void test_reference_id_case() +{ + auto r = parse_tracking_csv(with_field(0, "a9fea97b")); + CHECK(r.error == ChronyParseError::None); + CHECK(r.tracking.has_value() && r.tracking->reference_id == "A9FEA97B"); + auto l = parse_tracking_csv(with_field(0, "7f7f0101")); + CHECK(l.tracking.has_value() && l.tracking->reference_id == "7F7F0101"); +} + +void test_is_synchronised() +{ + CHECK(is_synchronised(make(4, "A9FEA97B", "Normal"), false)); + CHECK(is_synchronised(make(4, "A9FEA97B", "Normal"), true)); + CHECK(is_synchronised(make(1, "A9FEA97B", "Normal"), false)); + CHECK(is_synchronised(make(15, "A9FEA97B", "Normal"), false)); + CHECK(!is_synchronised(make(0, "A9FEA97B", "Normal"), false)); + CHECK(!is_synchronised(make(16, "A9FEA97B", "Normal"), false)); + CHECK(is_synchronised(make(4, "A9FEA97B", "Insert second"), false)); + CHECK(is_synchronised(make(4, "A9FEA97B", "Delete second"), false)); + CHECK(!is_synchronised(make(4, "A9FEA97B", "Not synchronised"), false)); + CHECK(!is_synchronised(make(4, "A9FEA97B", "Not synchronised"), true)); + + const auto local = make(10, std::string(kChronyLocalReferenceId), "Normal"); + CHECK(!is_synchronised(local, false)); + CHECK(is_synchronised(local, true)); + CHECK(is_synchronised(make(10, std::string(kChronyLocalReferenceId), "Insert second"), true)); + CHECK(!is_synchronised(make(10, std::string(kChronyLocalReferenceId), "Not synchronised"), true)); +} + +// Captured from `chronyc -c tracking` on chrony 4.5-1ubuntu4.2 while synchronised to an internet NTP server. +void test_real_chrony_4_5_capture() +{ + const char* line = + "5BBD5B9D,91.189.91.157,3,1794516440.890180266,-0.000037218,-0.000042871,0.001530292,-12.973,-0.003,0.278," + "0.055693440,0.001789562,258.4,Normal"; + const ChronyParseResult r = parse_tracking_csv(line); + CHECK(r.error == ChronyParseError::None); + CHECK(r.tracking.has_value()); + if (!r.tracking) + return; + const ChronyTracking& t = *r.tracking; + CHECK(t.reference_id == "5BBD5B9D"); + CHECK(t.reference_name == "91.189.91.157"); + CHECK(t.stratum == 3); + CHECK(near(t.ref_time_unix_s, 1794516440.890180266)); + CHECK(near(t.system_time_s, -0.000037218)); + CHECK(near(t.last_offset_s, -0.000042871)); + CHECK(near(t.rms_offset_s, 0.001530292)); + CHECK(near(t.frequency_ppm, -12.973)); + CHECK(near(t.residual_freq_ppm, -0.003)); + CHECK(near(t.skew_ppm, 0.278)); + CHECK(near(t.root_delay_s, 0.055693440)); + CHECK(near(t.root_dispersion_s, 0.001789562)); + CHECK(near(t.update_interval_s, 258.4)); + CHECK(t.leap_status == "Normal"); + // The id is the upper-case hex of the IPv4 address 91.189.91.157. + char hex[9]; + std::snprintf(hex, sizeof(hex), "%02X%02X%02X%02X", 91, 189, 91, 157); + CHECK(t.reference_id == hex); + CHECK(is_synchronised(t, false)); + // An internet-disciplined host's absolute bound is far above a 5 ms budget, hence the relative composition. + // 27.84672 ms (root delay / 2) + 1.789562 ms + 0.037218 ms + const int64_t bound = chrony_error_bound_ns(t, 0); + CHECK(std::llabs(bound - 29'673'500) <= 1'000); +} + +} // namespace + +int main() +{ + test_valid_line(); + test_local_reference_line(); + test_not_synchronised_line(); + test_empty_and_field_count(); + test_bad_numbers(); + test_stratum_bounds(); + test_line_endings(); + test_reference_id_case(); + test_is_synchronised(); + test_real_chrony_4_5_capture(); + if (g_failures) + { + std::fprintf(stderr, "%d check(s) failed\n", g_failures); + return 1; + } + std::printf("timesync_chrony_tracking: all checks passed\n"); + return 0; +} diff --git a/tests/cpp/core/timesync/command_runner/CMakeLists.txt b/tests/cpp/core/timesync/command_runner/CMakeLists.txt new file mode 100644 index 0000000000..75cfd66315 --- /dev/null +++ b/tests/cpp/core/timesync/command_runner/CMakeLists.txt @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(timesync_command_runner_tests test_command_runner.cpp) + +target_link_libraries(timesync_command_runner_tests PRIVATE timesync::timesync Threads::Threads) + +target_compile_options(timesync_command_runner_tests PRIVATE -Wall -Wextra) + +add_test(NAME timesync_command_runner COMMAND timesync_command_runner_tests) diff --git a/tests/cpp/core/timesync/command_runner/test_command_runner.cpp b/tests/cpp/core/timesync/command_runner/test_command_runner.cpp new file mode 100644 index 0000000000..7693ac0009 --- /dev/null +++ b/tests/cpp/core/timesync/command_runner/test_command_runner.cpp @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include + +using namespace isaaccapture::timesync; +using std::chrono::milliseconds; + +static int g_failures = 0; + +#define CHECK(cond) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +namespace +{ + +CommandResult run(const std::vector& argv, milliseconds timeout = milliseconds(5000)) +{ + return posix_command_runner()(argv, timeout); +} + +void test_echo() +{ + const auto r = run({ "/bin/echo", "hello" }); + CHECK(r.started); + CHECK(!r.timed_out); + CHECK(!r.truncated); + CHECK(r.exit_code == 0); + CHECK(r.output == "hello\n"); +} + +void test_no_shell() +{ + const auto r = run({ "/bin/echo", "$HOME; echo x" }); + CHECK(r.started); + CHECK(r.output == "$HOME; echo x\n"); +} + +void test_missing_program() +{ + const auto r = run({ "/nonexistent/isaaccapture-no-such-program" }); + CHECK(!r.started); + CHECK(r.exit_code == -1); +} + +void test_exit_code() +{ + const auto r = run({ "/bin/false" }); + CHECK(r.started); + CHECK(r.exit_code == 1); + CHECK(!r.timed_out); +} + +void test_timeout() +{ + const auto t0 = std::chrono::steady_clock::now(); + const auto r = run({ "/bin/sleep", "5" }, milliseconds(200)); + const auto elapsed = std::chrono::steady_clock::now() - t0; + CHECK(r.started); + CHECK(r.timed_out); + CHECK(r.exit_code == -1); + CHECK(elapsed < std::chrono::seconds(2)); +} + +void test_output_cap() +{ + // Shell pipeline is a test fixture only; the runner under test does not use a shell. + const auto r = run({ "/bin/sh", "-c", "yes | head -c 200000" }); + CHECK(r.started); + CHECK(r.truncated); + CHECK(r.output.size() == kMaxCommandOutputBytes); +} + +void test_empty_argv() +{ + const auto r = run({}); + CHECK(!r.started); +} + +void test_stderr_discarded() +{ + const auto r = run({ "/bin/sh", "-c", "echo err 1>&2; echo out" }); + CHECK(r.started); + CHECK(r.exit_code == 0); + CHECK(r.output == "out\n"); +} + +} // namespace + +int main() +{ + test_echo(); + test_no_shell(); + test_missing_program(); + test_exit_code(); + test_timeout(); + test_output_cap(); + test_empty_argv(); + test_stderr_discarded(); + if (g_failures) + { + std::fprintf(stderr, "%d check(s) failed\n", g_failures); + return 1; + } + std::printf("timesync_command_runner: all checks passed\n"); + return 0; +} diff --git a/tests/cpp/core/timesync/local_clock/CMakeLists.txt b/tests/cpp/core/timesync/local_clock/CMakeLists.txt new file mode 100644 index 0000000000..5a620e9251 --- /dev/null +++ b/tests/cpp/core/timesync/local_clock/CMakeLists.txt @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(timesync_local_clock_tests test_local_clock.cpp) + +target_link_libraries(timesync_local_clock_tests PRIVATE timesync::timesync) + +target_compile_options(timesync_local_clock_tests PRIVATE -Wall -Wextra) + +add_test(NAME timesync_local_clock COMMAND timesync_local_clock_tests) diff --git a/tests/cpp/core/timesync/local_clock/test_local_clock.cpp b/tests/cpp/core/timesync/local_clock/test_local_clock.cpp new file mode 100644 index 0000000000..90d9ab6745 --- /dev/null +++ b/tests/cpp/core/timesync/local_clock/test_local_clock.cpp @@ -0,0 +1,343 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include +#include +#include + +using namespace isaaccapture::timesync; + +static int g_failures = 0; + +#define CHECK(cond) \ + do \ + { \ + if (!(cond)) \ + { \ + std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +namespace +{ + +constexpr int64_t kSecond = 1'000'000'000; +constexpr int64_t kBase = 1'700'000'000'000'000'000; + +LocalClockSample make_sample(int64_t mono, int64_t wall_minus_mono, int64_t uncertainty = 1000) +{ + LocalClockSample s; + s.monotonic_ns = mono; + s.realtime_ns = mono + wall_minus_mono; + s.read_uncertainty_ns = uncertainty; + return s; +} + +void test_sample_local_clock() +{ + std::vector mono_script = { 1000, 1400 }; + size_t mono_calls = 0; + std::string order; + ClockReader reader; + reader.monotonic_ns = [&] + { + order += 'm'; + return mono_script.at(mono_calls++); + }; + reader.realtime_ns = [&] + { + order += 'r'; + return int64_t{ 5000 }; + }; + + const LocalClockSample s = sample_local_clock(reader); + CHECK(order == "mrm"); + CHECK(s.monotonic_ns == 1200); + CHECK(s.realtime_ns == 5000); + CHECK(s.read_uncertainty_ns == 200); + CHECK(wall_minus_monotonic_ns(s) == 3800); +} + +void test_system_reader_is_populated() +{ + const ClockReader r = system_clock_reader(); + CHECK(static_cast(r.monotonic_ns)); + CHECK(static_cast(r.realtime_ns)); + if (r.monotonic_ns && r.realtime_ns) + { + const LocalClockSample s = sample_local_clock(r); + CHECK(s.read_uncertainty_ns >= 0); + CHECK(s.realtime_ns > 0); + } +} + +void test_first_sample() +{ + LocalClockTracker tracker; + const auto o = tracker.observe(make_sample(10 * kSecond, kBase)); + CHECK(o.usable); + CHECK(o.generation == 1); + CHECK(!o.stepped); +} + +void test_steady_drift_does_not_step() +{ + LocalClockTracker tracker; + int64_t offset = kBase; + tracker.observe(make_sample(10 * kSecond, offset)); + for (int i = 1; i <= 20; ++i) + { + // 50 us of offset change per sample, inside the default step margin. + offset += 50'000; + const auto o = tracker.observe(make_sample((10 + i) * kSecond, offset)); + CHECK(o.usable); + CHECK(!o.stepped); + CHECK(o.generation == 1); + } +} + +void test_forward_and_backward_step() +{ + LocalClockTracker tracker; + int64_t offset = kBase; + tracker.observe(make_sample(10 * kSecond, offset)); + + offset += 2 * kSecond; // forward step + auto fwd = tracker.observe(make_sample(11 * kSecond, offset)); + CHECK(fwd.usable); + CHECK(fwd.stepped); + CHECK(fwd.generation == 2); + + auto steady = tracker.observe(make_sample(12 * kSecond, offset)); + CHECK(!steady.stepped); + CHECK(steady.generation == 2); + + offset -= 5 * kSecond; // backward step + auto back = tracker.observe(make_sample(13 * kSecond, offset)); + CHECK(back.usable); + CHECK(back.stepped); + CHECK(back.generation == 3); + + auto after = tracker.observe(make_sample(14 * kSecond, offset)); + CHECK(!after.stepped); + CHECK(after.generation == 3); +} + +void test_wide_bracket_is_unusable_and_harmless() +{ + const LocalClockConfig cfg; // max_read_uncertainty_ns = 200'000 + LocalClockTracker tracker(cfg); + tracker.observe(make_sample(10 * kSecond, kBase)); + + // Unusable sample whose offset would look like a step: no generation change, no step flag. + auto wide = tracker.observe(make_sample(11 * kSecond, kBase + 3 * kSecond, cfg.max_read_uncertainty_ns + 1)); + CHECK(!wide.usable); + CHECK(!wide.stepped); + CHECK(wide.generation == 1); + + // The next narrow sample is compared with the last usable one, not with the wide one. + auto next = tracker.observe(make_sample(12 * kSecond, kBase + 1000)); + CHECK(next.usable); + CHECK(!next.stepped); + CHECK(next.generation == 1); + + // A bracket exactly at the limit is still usable. + auto edge = tracker.observe(make_sample(13 * kSecond, kBase + 1000, cfg.max_read_uncertainty_ns)); + CHECK(edge.usable); + CHECK(edge.generation == 1); +} + +void test_wide_bracket_does_not_hide_a_real_step() +{ + LocalClockTracker tracker; + tracker.observe(make_sample(10 * kSecond, kBase)); + tracker.observe(make_sample(11 * kSecond, kBase, 5'000'000)); // unusable + auto o = tracker.observe(make_sample(12 * kSecond, kBase + 4 * kSecond)); + CHECK(o.usable); + CHECK(o.stepped); + CHECK(o.generation == 2); +} + +void test_small_jitter_within_uncertainty_is_not_a_step() +{ + LocalClockTracker tracker; + tracker.observe(make_sample(10 * kSecond, kBase, 50'000)); + // Offset moves by 80 us: inside both bracket widths plus the margin. + auto o = tracker.observe(make_sample(11 * kSecond, kBase + 80'000, 50'000)); + CHECK(!o.stepped); + CHECK(o.generation == 1); +} + +// Reader that replays scripted monotonic values around one realtime value. +ClockReader scripted_reader(std::vector mono, int64_t wall) +{ + auto calls = std::make_shared(0); + ClockReader r; + r.monotonic_ns = [mono, calls] { return mono.at((*calls)++); }; + r.realtime_ns = [wall] { return wall; }; + return r; +} + +void test_step_threshold_is_exact() +{ + const LocalClockConfig cfg; + // Offset change `delta` between samples `gap` apart on the monotonic clock, with the given bracket widths. + auto stepped = [&](int64_t delta, int64_t gap, int64_t width) + { + LocalClockTracker tracker(cfg); + tracker.observe(make_sample(10 * kSecond, kBase, width)); + return tracker.observe(make_sample(10 * kSecond + gap, kBase + delta, width)).stepped; + }; + for (int64_t gap : { int64_t{ 1'000'000 }, 1000 * kSecond }) + { + // Zero widths: the threshold is the margin. + CHECK(!stepped(cfg.step_margin_ns, gap, 0)); + CHECK(stepped(cfg.step_margin_ns + 1, gap, 0)); + CHECK(stepped(-cfg.step_margin_ns - 1, gap, 0)); + // Non-zero widths: the margin plus both of them. + const int64_t threshold = cfg.step_margin_ns + 2 * 30'000; + CHECK(!stepped(threshold, gap, 30'000)); + CHECK(stepped(threshold + 1, gap, 30'000)); + } +} + +void test_long_idle_without_offset_change_is_not_a_step() +{ + LocalClockTracker tracker; + tracker.observe(make_sample(10 * kSecond, kBase)); + const auto o = tracker.observe(make_sample(10 * kSecond + 100'000 * kSecond, kBase)); + CHECK(o.usable); + CHECK(!o.stepped); + CHECK(o.generation == 1); +} + +void test_step_tolerance_reported() +{ + LocalClockConfig cfg; + cfg.step_margin_ns = 123'456; + LocalClockTracker tracker(cfg); + CHECK(tracker.observe(make_sample(10 * kSecond, kBase)).step_tolerance_ns == cfg.step_margin_ns); + CHECK(tracker.observe(make_sample(11 * kSecond, kBase + 2 * kSecond)).step_tolerance_ns == cfg.step_margin_ns); + const auto wide = tracker.observe(make_sample(12 * kSecond, kBase, cfg.max_read_uncertainty_ns + 1)); + CHECK(!wide.usable); + CHECK(wide.step_tolerance_ns == 0); +} + +void test_failed_reads_are_unusable_and_harmless() +{ + std::vector readers; + readers.push_back(scripted_reader({ kClockReadFailed, 1400 }, 5000)); + readers.push_back(scripted_reader({ 1000, kClockReadFailed }, 5000)); + readers.push_back(scripted_reader({ 1000, 1400 }, kClockReadFailed)); + readers.push_back(scripted_reader({ 1000, 999 }, 5000)); // after = before - 1 + for (const ClockReader& reader : readers) + { + LocalClockTracker tracker; + tracker.observe(make_sample(10 * kSecond, kBase)); + const LocalClockSample s = sample_local_clock(reader); + CHECK(s.read_uncertainty_ns == INT64_MAX); + const auto bad = tracker.observe(s); + CHECK(!bad.usable); + CHECK(!bad.stepped); + CHECK(bad.generation == 1); + // The reference is still the first sample. + const auto next = tracker.observe(make_sample(11 * kSecond, kBase + 80'000)); + CHECK(next.usable); + CHECK(!next.stepped); + CHECK(next.generation == 1); + } +} + +void test_offset_overflow_is_unusable() +{ + { + LocalClockTracker tracker; + const LocalClockSample s = sample_local_clock(scripted_reader({ -5, -5 }, INT64_MAX)); + CHECK(s.read_uncertainty_ns == 0); + const auto o = tracker.observe(s); + CHECK(!o.usable); + CHECK(!o.stepped); + CHECK(o.generation == 1); + } + { + // Overflow on a later sample leaves the earlier reference in place. + LocalClockTracker tracker; + tracker.observe(make_sample(10 * kSecond, kBase)); + const auto bad = tracker.observe(sample_local_clock(scripted_reader({ -5, -5 }, INT64_MAX))); + CHECK(!bad.usable); + CHECK(!bad.stepped); + const auto next = tracker.observe(make_sample(11 * kSecond, kBase)); + CHECK(next.usable); + CHECK(!next.stepped); + } +} + +void test_extreme_monotonic_values() +{ + auto raw = [](int64_t mono, int64_t wall) + { + LocalClockSample s; + s.monotonic_ns = mono; + s.realtime_ns = wall; + s.read_uncertainty_ns = 0; + return s; + }; + { + // Monotonic time far behind the previous sample with an unchanged offset: a backward step. + LocalClockTracker tracker; + tracker.observe(make_sample(10 * kSecond, kBase)); + const auto o = tracker.observe(raw(INT64_MIN, INT64_MIN + kBase)); + CHECK(o.usable); + CHECK(o.stepped); + CHECK(o.generation == 2); + } + { + // Representable but enormous offset change. + LocalClockTracker tracker; + tracker.observe(make_sample(10 * kSecond, kBase)); + const auto o = tracker.observe(raw(INT64_MAX, INT64_MAX)); + CHECK(o.usable); + CHECK(o.stepped); + CHECK(o.generation == 2); + } + { + // Offset difference overflows. + LocalClockTracker tracker; + tracker.observe(raw(0, INT64_MAX / 2 + 10)); + const auto o = tracker.observe(raw(1, -(INT64_MAX / 2) - 10)); + CHECK(!o.usable || o.stepped); + } +} + +} // namespace + +int main() +{ + test_sample_local_clock(); + test_system_reader_is_populated(); + test_first_sample(); + test_steady_drift_does_not_step(); + test_forward_and_backward_step(); + test_wide_bracket_is_unusable_and_harmless(); + test_wide_bracket_does_not_hide_a_real_step(); + test_small_jitter_within_uncertainty_is_not_a_step(); + test_step_threshold_is_exact(); + test_long_idle_without_offset_change_is_not_a_step(); + test_step_tolerance_reported(); + test_failed_reads_are_unusable_and_harmless(); + test_offset_overflow_is_unusable(); + test_extreme_monotonic_values(); + if (g_failures) + { + std::fprintf(stderr, "%d check(s) failed\n", g_failures); + return 1; + } + std::printf("timesync_local_clock: all checks passed\n"); + return 0; +} From a36b166a248e723a07b20eba571a1f18560f41a6 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Thu, 12 Nov 2026 18:31:16 -0500 Subject: [PATCH 02/10] timesync: add the chrony setup tool and README timesync_tool.py runs an unprivileged chronyd on the PC that follows or serves the camera host's clock, configures the camera host's chrony over ssh, and reports whether the setup has settled. Both topologies are documented. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/core/timesync/README.md | 96 +++++ src/core/timesync/tools/test_timesync_tool.py | 393 +++++++++++++++++ src/core/timesync/tools/timesync_tool.py | 399 ++++++++++++++++++ 3 files changed, 888 insertions(+) create mode 100644 src/core/timesync/README.md create mode 100644 src/core/timesync/tools/test_timesync_tool.py create mode 100755 src/core/timesync/tools/timesync_tool.py diff --git a/src/core/timesync/README.md b/src/core/timesync/README.md new file mode 100644 index 0000000000..771a5bdd5f --- /dev/null +++ b/src/core/timesync/README.md @@ -0,0 +1,96 @@ + + +# timesync + +Converts a camera host's wall-clock capture time to the PC's monotonic clock and decides whether +the result may be recorded. The two hosts keep their wall clocks aligned with `chronyd`; this +library reads chrony's own report of how well it is doing and refuses a frame it cannot vouch for. + +Target `timesync::timesync` (consumers include ``). POSIX only, so the Windows source +build skips it. It builds without CUDA, LiveKit or Python, and on its own: + +```bash +cmake -S src/core/timesync -B build -DBUILD_TESTING=ON && cmake --build build && ctest --test-dir build +``` + +## Pieces + +| Header | Role | +| --- | --- | +| `chrony_tracking.hpp` | Strict parser for one `chronyc -c tracking` line; `is_synchronised`. | +| `local_clock.hpp` | Bracketed realtime/monotonic sample and a tracker that numbers the intervals between realtime steps. | +| `capture_qualifier.hpp` | `qualify_capture`: PC-monotonic capture time, summed uncertainty, or a typed refusal. | +| `chrony_monitor.hpp`, `command_runner.hpp` | Background `chronyc` poll: no shell, a timeout, an output cap, never in the frame path. | +| `tools/timesync_tool.py` | Sets up the chrony the library reads (below). Python standard library only. | + +## Usage + +A consumer keeps a `ChronyMonitor` for the PC and one for the camera host, a `LocalClockTracker` +fed by `sample_local_clock`, and calls `qualify_capture` per frame. The caller pins the clock +generation when a take starts and ends the take if a frame reports another. A `ChronyMonitor` can +query a chronyd over its command port (`host` and `port` in `ChronyMonitorConfig`), which is how a +dedicated unprivileged instance next to the system chronyd is read. + +## Arrangement + +The camera host keeps its own clock and serves it (it may follow UTC or serve its own clock). A second, +unprivileged chronyd on the PC measures it and never adjusts the PC's system clock, so nothing has to reach the +PC (a PC firewall does not matter) and the PC needs no root beyond installing chrony. `qualify_capture` charges +the link between the two. + +## Running and deploying it + +`tools/timesync_tool.py` (Python standard library only): + +```bash +# On the PC: install chrony if it is missing (needs sudo). +python3 src/core/timesync/tools/timesync_tool.py ensure-local + +# Install and start chrony on the camera host, let this PC measure it and read its state +# (needs passwordless sudo and the network there; bounded by --timeout, default 90 s). +python3 src/core/timesync/tools/timesync_tool.py configure-remote user@camera-host --host-lan-ip PC_LAN_IP + +# On the PC, in the foreground: measure the camera host's clock. +python3 src/core/timesync/tools/timesync_tool.py follow --server CAMERA_HOST_IP + +# Read-only: has this PC settled on the camera host? Exits non-zero with the reasons when not. +python3 src/core/timesync/tools/timesync_tool.py status --remote CAMERA_HOST_IP --wait 120 +``` + +The PC instance copies the system `chronyd` binary into its work directory, because the distribution's AppArmor +profile is tied to `/usr/sbin/chronyd` and would stop it reading a configuration outside `/etc/chrony`. Deploy and +launch tools call this tool; they do not carry their own chrony setup. + +## What the bound assumes + +The inter-host term is both hosts' `System time`, the PC's root dispersion and root delay / 2 minus the camera +host's, a slack for the camera host's dispersion growing since the PC copied it, and each report's skew times its +age. + +Why the subtraction is used: chrony's root delay and root dispersion are cumulative along the chain to the +reference. When the PC's source is the camera host, the PC's values are the camera host's plus the link between +them, so the difference isolates the link and the camera host's own upstream error (large when it follows an +internet source) cancels. That is why the bound is relative and not the sum of both hosts' absolute bounds. + +What is and is not checked: the PC's selected reference id must be the camera host's address, so a different source +is refused as `ReferenceMismatch`. The assumption that the root fields are cumulative through the camera host is +chrony's documented behaviour, not something this library measures. The two reports are read at different instants, +so the difference can be slightly off; it is clamped at zero, and the copy slack and the skew-times-age terms cover +the drift between reads. The bound is derived from chrony's own reports. It has not been validated against an +independent reference such as a hardware sync pulse, so a qualified frame means "within chrony's claimed error", +not "measured within". Refusal policy: any report that is stale, has no recent source update, is unsynchronised, or +names another source is refused and the frame carries no time; nothing is estimated. + +Step detection compares successive wall-minus-monotonic offsets. On Linux the two clocks share +frequency adjustment, so the difference changes only on a step. A change beyond both bracket widths +plus a margin (100 us by default) counts as one; a smaller step goes undetected, so that margin is +added to every conversion's uncertainty. Detection also takes one sample interval. + +A report is refused when chronyd was last asked too long ago, and also when its last source update +is older than `max_source_update_age_ns`, so frequent polling cannot hide a quiet reference. + +The parser expects the 14-field `chronyc -c tracking` layout of chrony 4.5 and fails closed on any +other field count. diff --git a/src/core/timesync/tools/test_timesync_tool.py b/src/core/timesync/tools/test_timesync_tool.py new file mode 100644 index 0000000000..ceb6de88be --- /dev/null +++ b/src/core/timesync/tools/test_timesync_tool.py @@ -0,0 +1,393 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the timesync chrony setup tool (no real chrony, ssh or network).""" + +from __future__ import annotations + +import contextlib +import io +import stat +import subprocess +import tempfile +import unittest +from unittest import mock +from pathlib import Path + +import timesync_tool as tool +from timesync_tool import ToolError, Tracking + +REAL_LINE = ( + "5BBD5B9D,91.189.91.157,3,1794516440.890180266,-0.000037218,-0.000042871," + "0.001530292,-12.973,-0.003,0.278,0.055693440,0.001789562,258.4,Normal" +) +LOCAL_LINE = ( + "7F7F0101,,10,1794518542.68,0.000000000,0.000000000,0.000000000,0.000,0.000," + "0.000,0.000000000,0.000000000,0.0,Normal" +) +HOST_IP = "192.168.1.221" +HOST_ID = "C0A801DD" + + +def remote_line(ref_id: str = HOST_ID, dispersion: str = "0.000300000") -> str: + return ( + f"{ref_id},,4,1794518542.68,-0.000001000,0.000000000,0.000000000,0.000," + f"0.000,0.050,0.000400000,{dispersion},4.0,Normal" + ) + + +def make_tracking(**overrides: object) -> Tracking: + fields: dict[str, object] = dict( + reference_id=HOST_ID, + reference_name="", + stratum=4, + ref_time=1.0, + system_time=0.0, + root_delay=0.0004, + root_dispersion=0.0003, + skew_ppm=0.05, + update_interval=4.0, + leap_status="Normal", + ) + fields.update(overrides) + return Tracking(**fields) # type: ignore[arg-type] + + +class ParseTrackingTest(unittest.TestCase): + def test_real_chrony_line(self) -> None: + t = tool.parse_tracking(REAL_LINE) + self.assertEqual(t.reference_id, "5BBD5B9D") + self.assertEqual(t.reference_name, "91.189.91.157") + self.assertEqual(t.stratum, 3) + self.assertAlmostEqual(t.ref_time, 1794516440.890180266) + self.assertAlmostEqual(t.system_time, -0.000037218) + self.assertAlmostEqual(t.root_delay, 0.05569344) + self.assertAlmostEqual(t.root_dispersion, 0.001789562) + self.assertAlmostEqual(t.skew_ppm, 0.278) + self.assertAlmostEqual(t.update_interval, 258.4) + self.assertEqual(t.leap_status, "Normal") + self.assertTrue(t.synchronised) + + def test_local_reference_line(self) -> None: + t = tool.parse_tracking(LOCAL_LINE + "\n") + self.assertEqual(t.reference_id, "7F7F0101") + self.assertEqual(t.reference_name, "") + self.assertEqual(t.stratum, 10) + self.assertTrue(t.synchronised) + + def test_not_synchronised(self) -> None: + self.assertFalse(make_tracking(leap_status="Not synchronised").synchronised) + + def test_stratum_zero_not_synchronised(self) -> None: + self.assertFalse(make_tracking(stratum=0).synchronised) + + def test_wrong_field_counts(self) -> None: + fields = REAL_LINE.split(",") + for bad in (",".join(fields[:13]), ",".join(fields + ["x"])): + with self.assertRaises(ToolError): + tool.parse_tracking(bad) + + def test_non_numeric_fields(self) -> None: + for index in (2, 3, 4, 9, 10, 11, 12): + fields = REAL_LINE.split(",") + fields[index] = "abc" + with self.subTest(index=index), self.assertRaises(ToolError): + tool.parse_tracking(",".join(fields)) + + +class ReferenceIdTest(unittest.TestCase): + def test_known_addresses(self) -> None: + self.assertEqual(tool.reference_id_for(HOST_IP), HOST_ID) + self.assertEqual(tool.reference_id_for("91.189.91.157"), "5BBD5B9D") + + def test_bad_addresses(self) -> None: + for bad in ("999.1.1.1", "a.b.c.d", "1.2.3"): + with self.subTest(bad=bad), self.assertRaises(ToolError): + tool.reference_id_for(bad) + + +class LanAddressTest(unittest.TestCase): + def test_loopback(self) -> None: + self.assertEqual(tool.lan_address_toward("127.0.0.1"), "127.0.0.1") + + +class RemoteDropInTest(unittest.TestCase): + def test_serves_the_pc_and_lets_it_read_state(self) -> None: + self.assertEqual( + tool.remote_drop_in(HOST_IP).splitlines(), + [ + f"allow {HOST_IP}", + "bindcmdaddress 0.0.0.0", + f"cmdallow {HOST_IP}", + ], + ) + + +class RemoteScriptTest(unittest.TestCase): + def setUp(self) -> None: + self.script = tool.remote_script(HOST_IP) + + def test_install_guard_and_include_check(self) -> None: + self.assertIn("command -v chronyd", self.script) + self.assertIn("apt-get install -y chrony", self.script) + self.assertIn("conf\\.d", self.script) + self.assertIn("/etc/chrony/chrony.conf", self.script) + + def test_writes_only_the_drop_in(self) -> None: + self.assertEqual(self.script.count("tee "), 1) + self.assertIn(f"tee {tool.DROP_IN}", self.script) + self.assertNotIn(">/etc", self.script.replace(">/dev/null", "")) + + def test_idempotent_compare_before_write(self) -> None: + compare = self.script.index('"$current" != "$wanted"') + self.assertLess(compare, self.script.index("tee ")) + self.assertIn("is already as wanted", self.script) + + def test_unchanged_drop_in_starts_a_stopped_chrony(self) -> None: + start = self.script.index("elif ! sudo -n systemctl is-active --quiet chrony") + branch = self.script[start : self.script.index("else", start)] + self.assertIn("systemctl start chrony", branch) + self.assertNotIn("restart", branch) + + def test_serves_the_pc(self) -> None: + self.assertIn(f"allow {HOST_IP}", self.script) + self.assertIn(f"cmdallow {HOST_IP}", self.script) + self.assertIn("bindcmdaddress 0.0.0.0", self.script) + self.assertNotIn("server ", self.script) + + def test_wanted_text_is_quoted(self) -> None: + script = tool.remote_script("1.2.3.4'; rm -rf x; '") + self.assertNotIn("\nrm -rf x", script) + + def test_bash_accepts_script(self) -> None: + result = subprocess.run( + ["bash", "-n"], + input=self.script, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +class JudgeTest(unittest.TestCase): + def judge(self, local: Tracking) -> list[str]: + return tool.judge(local, HOST_ID, 0.002) + + def test_usable(self) -> None: + self.assertEqual(self.judge(make_tracking()), []) + + def test_unsynchronised(self) -> None: + problems = self.judge(make_tracking(leap_status="Not synchronised")) + self.assertEqual(len(problems), 1) + self.assertIn("not synchronised", problems[0]) + + def test_wrong_reference(self) -> None: + problems = self.judge(make_tracking(reference_id="5BBD5B9D")) + self.assertEqual(len(problems), 1) + self.assertIn("5BBD5B9D", problems[0]) + + def test_high_dispersion(self) -> None: + problems = self.judge(make_tracking(root_dispersion=0.01)) + self.assertEqual(len(problems), 1) + self.assertIn("dispersion", problems[0]) + + def test_problems_accumulate(self) -> None: + bad = make_tracking( + leap_status="Not synchronised", + reference_id="5BBD5B9D", + root_dispersion=0.01, + ) + self.assertEqual(len(self.judge(bad)), 3) + + +def run_main(argv: list[str]) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = tool.main(argv) + return code, out.getvalue(), err.getvalue() + + +class MainConfigureRemoteTest(unittest.TestCase): + def test_dry_run_prints_script(self) -> None: + code, out, _ = run_main( + ["configure-remote", "user@host", "--host-lan-ip", HOST_IP, "--dry-run"] + ) + self.assertEqual(code, 0) + self.assertIn("DRY-RUN", out) + self.assertIn(f"allow {HOST_IP}", out) + self.assertIn(f"cmdallow {HOST_IP}", out) + self.assertIn("bindcmdaddress 0.0.0.0", out) + + def test_bad_host_lan_ip(self) -> None: + code, _, err = run_main( + ["configure-remote", "user@host", "--host-lan-ip", "nope", "--dry-run"] + ) + self.assertEqual(code, 1) + self.assertIn("error:", err) + + def test_timeout_is_an_error(self) -> None: + def expire(*args: object, **kwargs: object) -> None: + raise subprocess.TimeoutExpired(cmd="ssh", timeout=5) + + with mock.patch.object(tool.subprocess, "run", expire): + code, _, err = run_main( + [ + "configure-remote", + "user@host", + "--host-lan-ip", + HOST_IP, + "--timeout", + "5", + ] + ) + self.assertEqual(code, 1) + self.assertIn("did not finish within 5 s", err) + + +class EnsureLocalTest(unittest.TestCase): + def test_does_nothing_when_chronyd_is_found(self) -> None: + with ( + mock.patch.object(tool.shutil, "which", return_value="/usr/sbin/chronyd"), + mock.patch.object(tool.subprocess, "run") as run, + ): + code, out, _ = run_main(["ensure-local"]) + self.assertEqual(code, 0) + run.assert_not_called() + self.assertIn("already installed", out) + + def test_installs_chrony_when_missing(self) -> None: + done = subprocess.CompletedProcess([], 0) + with ( + mock.patch.object(tool.shutil, "which", return_value=None), + mock.patch.object(Path, "is_file", return_value=False), + mock.patch.object(tool.subprocess, "run", return_value=done) as run, + ): + code, _, _ = run_main(["ensure-local"]) + self.assertEqual(code, 0) + commands = [call.args[0] for call in run.call_args_list] + self.assertTrue(any("apt-get" in c and "install" in c for c in commands)) + self.assertTrue(all(c[0] == "sudo" for c in commands)) + + def test_failed_install_is_an_error(self) -> None: + failed = subprocess.CompletedProcess([], 100) + with ( + mock.patch.object(tool.shutil, "which", return_value=None), + mock.patch.object(Path, "is_file", return_value=False), + mock.patch.object(tool.subprocess, "run", return_value=failed), + ): + code, _, err = run_main(["ensure-local"]) + self.assertEqual(code, 1) + self.assertIn("exited 100", err) + + +class FollowerConfigTest(unittest.TestCase): + def test_follower_config(self) -> None: + text = tool.follower_config("192.168.1.50", 123, 12301, Path("/w")) + lines = text.splitlines() + self.assertIn("server 192.168.1.50 port 123 iburst minpoll 0 maxpoll 3", lines) + self.assertIn("port 0", lines) + self.assertIn("cmdport 12301", lines) + self.assertIn("bindcmdaddress 127.0.0.1", lines) + for line in lines: + self.assertFalse(line.startswith(("allow", "local")), line) + + def test_follower_config_uses_given_server_port(self) -> None: + text = tool.follower_config("10.0.0.2", 12300, 9, Path("/w")) + self.assertIn("server 10.0.0.2 port 12300 iburst", text) + self.assertIn("cmdport 9\n", text) + + +CAMERA_IP = "192.168.1.50" +CAMERA_ID = "C0A80132" + + +class StatusPcFollowsTest(unittest.TestCase): + """The PC's own instance (127.0.0.1) must follow the camera host, which is read for its own state.""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.tmp = Path(self._tmp.name) + + def fake_chronyc(self, pc: str, camera: str | None) -> str: + camera_branch = f"echo '{camera}'" if camera else "echo 'no reply' >&2; exit 1" + path = self.tmp / "chronyc" + path.write_text( + "#!/bin/bash\n" + 'case "$*" in\n' + f" *'-h 127.0.0.1'*) echo '{pc}' ;;\n" + f" *) {camera_branch} ;;\n" + "esac\n", + encoding="utf-8", + ) + path.chmod(path.stat().st_mode | stat.S_IXUSR) + return str(path) + + def status(self, pc: str, camera: str | None) -> tuple[int, str, str]: + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = tool.main( + [ + "status", + "--remote", + CAMERA_IP, + "--wait", + "0", + "--chronyc", + self.fake_chronyc(pc, camera), + ] + ) + return code, out.getvalue(), err.getvalue() + + def test_good(self) -> None: + code, out, err = self.status( + remote_line(ref_id=CAMERA_ID), remote_line(ref_id="5BBD5B9D") + ) + self.assertEqual(code, 0, err) + self.assertIn("ok: this machine follows " + CAMERA_ID, out) + + def test_pc_follows_another_reference_id(self) -> None: + code, _, err = self.status( + remote_line(ref_id="5BBD5B9D"), remote_line(ref_id="5BBD5B9D") + ) + self.assertEqual(code, 1) + self.assertIn("not ready:", err) + self.assertIn("this machine follows 5BBD5B9D", err) + self.assertIn(CAMERA_ID, err) + + def test_pc_instance_unreadable(self) -> None: + path = self.tmp / "chronyc" + path.write_text("#!/bin/bash\nexit 1\n", encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + code = tool.main(["status", "--remote", CAMERA_IP, "--chronyc", str(path)]) + self.assertEqual(code, 1) + self.assertIn("cannot be read", err.getvalue()) + + def test_camera_host_unreadable(self) -> None: + code, _, err = self.status(remote_line(ref_id=CAMERA_ID), None) + self.assertEqual(code, 1) + self.assertIn("not ready:", err) + self.assertIn("camera host's chronyd cannot be read", err) + + def test_camera_host_unsynchronised(self) -> None: + unsynchronised = remote_line(ref_id="5BBD5B9D").replace( + "Normal", "Not synchronised" + ) + code, _, err = self.status(remote_line(ref_id=CAMERA_ID), unsynchronised) + self.assertEqual(code, 1) + self.assertIn("the camera host is not synchronised", err) + + def test_pc_high_dispersion(self) -> None: + code, _, err = self.status( + remote_line(ref_id=CAMERA_ID, dispersion="0.010000000"), + remote_line(ref_id="5BBD5B9D"), + ) + self.assertEqual(code, 1) + self.assertIn("this machine's root dispersion", err) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/core/timesync/tools/timesync_tool.py b/src/core/timesync/tools/timesync_tool.py new file mode 100755 index 0000000000..493b7da6eb --- /dev/null +++ b/src/core/timesync/tools/timesync_tool.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Runs and deploys the chrony setup that the timesync library reads: the camera host serves its clock and the PC measures it. + + timesync_tool.py ensure-local + timesync_tool.py configure-remote user@host --host-lan-ip IP [--timeout S] [--dry-run] + timesync_tool.py follow --server HOST [--cmd-port N] + timesync_tool.py status --remote HOST [--cmd-port N] [--wait S] + +``ensure-local`` installs chrony on this machine if its ``chronyd`` is missing (needs sudo). ``configure-remote`` installs +chrony on the camera host if it is missing, lets this machine sync from it and read its state, and makes sure it is +running (it needs passwordless sudo and the network there). ``follow`` runs a second, unprivileged chronyd on this +machine in the foreground until interrupted: it measures the camera host's clock and never adjusts the system clock, so +it needs no root, leaves the system chronyd alone and needs no inbound path to this machine. ``status`` is read-only: it +succeeds when that instance has selected the camera host and is settled. POSIX only. +""" + +from __future__ import annotations + +import argparse +import shlex +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_CMD_PORT = 12301 +DROP_IN = "/etc/chrony/conf.d/camera-agx.conf" +TRACKING_FIELDS = 14 + + +class ToolError(Exception): + """A failure with a message worth showing as is.""" + + +@dataclass(frozen=True) +class Tracking: + """One ``chronyc -c tracking`` line; seconds and ppm, as chrony prints them.""" + + reference_id: str + reference_name: str + stratum: int + ref_time: float + system_time: float + root_delay: float + root_dispersion: float + skew_ppm: float + update_interval: float + leap_status: str + + @property + def synchronised(self) -> bool: + return ( + self.leap_status in ("Normal", "Insert second", "Delete second") + and 1 <= self.stratum <= 15 + and self.reference_id not in ("", "00000000") + ) + + +def parse_tracking(line: str) -> Tracking: + """Parse one chrony 4.x CSV tracking line; anything unexpected is an error.""" + fields = line.strip().split(",") + if len(fields) != TRACKING_FIELDS: + raise ToolError( + f"expected {TRACKING_FIELDS} tracking fields, got {len(fields)}: {line.strip()!r}" + ) + try: + return Tracking( + reference_id=fields[0].strip().upper(), + reference_name=fields[1].strip(), + stratum=int(fields[2]), + ref_time=float(fields[3]), + system_time=float(fields[4]), + root_delay=float(fields[10]), + root_dispersion=float(fields[11]), + skew_ppm=float(fields[9]), + update_interval=float(fields[12]), + leap_status=fields[13].strip(), + ) + except ValueError as error: + raise ToolError( + f"unparseable tracking line {line.strip()!r}: {error}" + ) from error + + +def reference_id_for(ip: str) -> str: + """The reference id chrony shows for an NTP source at an IPv4 address: its four bytes in upper-case hex.""" + parts = ip.split(".") + if len(parts) != 4 or not all(p.isdigit() and 0 <= int(p) <= 255 for p in parts): + raise ToolError(f"{ip!r} is not an IPv4 address") + return "".join(f"{int(p):02X}" for p in parts) + + +def lan_address_toward(host: str) -> str: + """This machine's address on the route to ``host`` (a UDP connect sends nothing).""" + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.connect((host, 9)) + return sock.getsockname()[0] + + +def follower_config(server: str, server_port: int, cmd_port: int, workdir: Path) -> str: + """The unprivileged follower instance: it measures ``server``'s clock, serves no NTP and adjusts nothing.""" + return ( + f"server {server} port {server_port} iburst minpoll 0 maxpoll 3\n" + "port 0\n" + f"cmdport {cmd_port}\n" + "bindcmdaddress 127.0.0.1\n" + f"pidfile {workdir / 'chronyd.pid'}\n" + f"driftfile {workdir / 'chronyd.drift'}\n" + f"dumpdir {workdir}\n" + ) + + +def remote_drop_in(host_lan_ip: str) -> str: + """The camera host's chrony drop-in: serve this PC and let it read this chronyd's state.""" + return f"allow {host_lan_ip}\nbindcmdaddress 0.0.0.0\ncmdallow {host_lan_ip}\n" + + +def remote_script(host_lan_ip: str) -> str: + """The bash that ``configure-remote`` runs on the camera host.""" + wanted = remote_drop_in(host_lan_ip) + return f"""set -eu +sudo -n true || {{ echo "passwordless sudo is required on the target" >&2; exit 1; }} +if ! command -v chronyd >/dev/null; then + echo "chrony is not installed: installing it" + sudo -n apt-get update -qq + sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y chrony +fi +grep -Eqs '^[[:space:]]*(confdir|include)[[:space:]].*conf\\.d' /etc/chrony/chrony.conf \\ + || {{ echo "/etc/chrony/chrony.conf does not include conf.d" >&2; exit 1; }} +wanted={shlex.quote(wanted.rstrip())} +current=$(sudo -n cat {DROP_IN} 2>/dev/null || true) +if [[ "$current" != "$wanted" ]]; then + printf '%s\\n' "$wanted" | sudo -n tee {DROP_IN} >/dev/null + echo "wrote {DROP_IN}; restarting chrony" + sudo -n systemctl restart chrony +elif ! sudo -n systemctl is-active --quiet chrony; then + echo "chrony is not running: starting it" + sudo -n systemctl start chrony +else + echo "{DROP_IN} is already as wanted" +fi +sudo -n systemctl enable chrony >/dev/null 2>&1 || true +""" + + +def chronyc_tracking( + chronyc: str, host: str | None, port: int | None, timeout: float = 5.0 +) -> Tracking: + """Ask a chronyd for its tracking report: a remote one over UDP when ``host`` is given, else the system's.""" + command = [chronyc, "-c"] + if host: + command += ["-h", host] + if port: + command += ["-p", str(port)] + command.append("tracking") + try: + result = subprocess.run( + command, capture_output=True, text=True, timeout=timeout, check=False + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise ToolError(f"{shlex.join(command)} failed: {error}") from error + if result.returncode != 0: + raise ToolError( + f"{shlex.join(command)} exited {result.returncode}: {result.stdout.strip() or result.stderr.strip()}" + ) + return parse_tracking(result.stdout) + + +def judge(local: Tracking, expected_id: str, max_dispersion_s: float) -> list[str]: + """What keeps this machine's measuring instance from being usable, one message each; empty when it is usable.""" + problems = [] + if not local.synchronised: + problems.append( + f"this machine is not synchronised (stratum {local.stratum}, {local.leap_status})" + ) + if local.reference_id != expected_id: + problems.append( + f"this machine follows {local.reference_id} ({local.reference_name or 'unnamed'}), " + f"not the camera host ({expected_id})" + ) + if local.root_dispersion > max_dispersion_s: + problems.append( + f"this machine's root dispersion is {local.root_dispersion * 1e3:.1f} ms, " + f"above {max_dispersion_s * 1e3:.1f} ms; it is still settling" + ) + return problems + + +def run_chronyd(args: argparse.Namespace, make_config, what: str) -> int: + """Run an unprivileged, clock-neutral chronyd in the foreground until interrupted.""" + chronyd = args.chronyd or shutil.which("chronyd") or "/usr/sbin/chronyd" + if not Path(chronyd).is_file(): + raise ToolError(f"{chronyd} not found; install chrony") + workdir = ( + Path(args.workdir) + if args.workdir + else Path(tempfile.mkdtemp(prefix="timesync-")) + ) + workdir.mkdir(parents=True, exist_ok=True) + # A copy of the binary: the distribution's AppArmor profile is tied to /usr/sbin/chronyd and would stop it reading a + # configuration outside /etc/chrony. + binary = workdir / "chronyd" + shutil.copy2(chronyd, binary) + config = workdir / "chrony.conf" + config.write_text(make_config(workdir)) + command = [str(binary), "-x", "-U", "-d", "-f", str(config)] + print(what, shlex.join(command), flush=True) + process = subprocess.Popen(command) + + def stop(*_: object) -> None: + process.send_signal(signal.SIGINT) + + signal.signal(signal.SIGINT, stop) + signal.signal(signal.SIGTERM, stop) + return process.wait() + + +def cmd_follow(args: argparse.Namespace) -> int: + reference_id_for(args.server) + return run_chronyd( + args, + lambda workdir: follower_config( + args.server, args.server_ntp_port, args.cmd_port, workdir + ), + f"measuring {args.server}'s clock:", + ) + + +def cmd_ensure_local(args: argparse.Namespace) -> int: + """Install chrony here when ``chronyd`` is missing; the follower and the PC's chronyc need it.""" + if shutil.which("chronyd") or Path("/usr/sbin/chronyd").is_file(): + print("chrony is already installed") + return 0 + print("chrony is not installed: installing it (sudo)", flush=True) + for command in ( + ["sudo", "apt-get", "update", "-qq"], + [ + "sudo", + "env", + "DEBIAN_FRONTEND=noninteractive", + "apt-get", + "install", + "-y", + "chrony", + ], + ): + try: + returncode = subprocess.run( + command, check=False, timeout=args.timeout + ).returncode + except subprocess.TimeoutExpired as error: + raise ToolError( + f"{shlex.join(command)} did not finish within {args.timeout:.0f} s" + ) from error + if returncode != 0: + raise ToolError(f"{shlex.join(command)} exited {returncode}") + return 0 + + +def cmd_configure_remote(args: argparse.Namespace) -> int: + reference_id_for(args.host_lan_ip) + script = remote_script(args.host_lan_ip) + command = [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "ConnectTimeout=10", + args.target, + "bash -s", + ] + if args.dry_run: + print("DRY-RUN", shlex.join(command), "<<<") + print(script) + return 0 + try: + return subprocess.run( + command, input=script, text=True, check=False, timeout=args.timeout + ).returncode + except subprocess.TimeoutExpired as error: + raise ToolError( + f"configuring {args.target} did not finish within {args.timeout:.0f} s" + ) from error + + +def cmd_status(args: argparse.Namespace) -> int: + """Succeeds when this machine's measuring instance follows the camera host and has settled.""" + expected = reference_id_for(args.remote) + deadline = time.monotonic() + args.wait + while True: + problems: list[str] = [] + try: + local = chronyc_tracking(args.chronyc, "127.0.0.1", args.cmd_port) + except ToolError as error: + local = None + problems.append( + f"this machine's chronyd on command port {args.cmd_port} cannot be read: {error}" + ) + try: + remote = chronyc_tracking(args.chronyc, args.remote, None) + except ToolError: + remote = None + if remote is None: + problems.append( + "the camera host's chronyd cannot be read (not installed, not running, or it does not cmdallow this machine)" + ) + elif not remote.synchronised: + problems.append( + f"the camera host is not synchronised (stratum {remote.stratum}, {remote.leap_status})" + ) + if local is not None: + problems += judge(local, expected, args.max_dispersion_ms / 1e3) + if not problems: + assert local is not None + print( + f"ok: this machine follows {local.reference_id} (stratum {local.stratum}), " + f"root delay {local.root_delay * 1e3:.3f} ms, root dispersion {local.root_dispersion * 1e3:.3f} ms, " + f"System time {abs(local.system_time) * 1e6:.1f} us" + ) + return 0 + if time.monotonic() >= deadline: + for problem in problems: + print("not ready:", problem, file=sys.stderr) + return 1 + time.sleep(2.0) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + ensure = sub.add_parser( + "ensure-local", help="install chrony on this machine if it is missing" + ) + ensure.add_argument( + "--timeout", + type=float, + default=300.0, + help="seconds each install step may take", + ) + follow = sub.add_parser( + "follow", + help="run the unprivileged instance that measures the camera host's clock, in the foreground", + ) + follow.add_argument( + "--server", required=True, help="the camera host's IPv4 address" + ) + follow.add_argument("--server-ntp-port", type=int, default=123) + follow.add_argument("--cmd-port", type=int, default=DEFAULT_CMD_PORT) + follow.add_argument("--workdir") + follow.add_argument("--chronyd") + configure = sub.add_parser( + "configure-remote", + help="install and start chrony on the camera host and let this machine measure it", + ) + configure.add_argument("target", help="user@host of the camera host") + configure.add_argument( + "--host-lan-ip", required=True, help="this machine's address on the camera LAN" + ) + configure.add_argument( + "--timeout", type=float, default=90.0, help="seconds the whole setup may take" + ) + configure.add_argument("--dry-run", action="store_true") + status = sub.add_parser( + "status", + help="read-only: does this machine follow the camera host, and is it settled", + ) + status.add_argument("--remote", required=True, help="the camera host's address") + status.add_argument("--cmd-port", type=int, default=DEFAULT_CMD_PORT) + status.add_argument( + "--wait", + type=float, + default=0.0, + help="keep checking this many seconds before giving up", + ) + status.add_argument("--max-dispersion-ms", type=float, default=2.0) + status.add_argument("--chronyc", default=shutil.which("chronyc") or "chronyc") + args = parser.parse_args(argv) + commands = { + "ensure-local": cmd_ensure_local, + "follow": cmd_follow, + "configure-remote": cmd_configure_remote, + "status": cmd_status, + } + try: + return commands[args.command](args) + except ToolError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) From aacd2f1ee8c394b48fc256cb657d6c2ef4d8de82 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 10:59:46 -0500 Subject: [PATCH 03/10] timesync: a capture that misses the precision target is still usable OverBudget already carries the converted capture instant and its bound. Add Qualification::usable() so a caller can keep the instant for every verdict that passed the validity checks and treat Qualified as only the statement that the bound met the budget. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../cpp/inc/timesync/capture_qualifier.hpp | 17 +++++++- .../test_capture_qualifier.cpp | 40 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp index ac5b4b6ea6..1039e59403 100644 --- a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp +++ b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp @@ -80,10 +80,23 @@ struct UncertaintyTerms struct Qualification { Verdict verdict = Verdict::SenderInvalid; - int64_t capture_monotonic_ns = 0; ///< PC CLOCK_MONOTONIC; meaningful only when Qualified. - int64_t uncertainty_ns = 0; ///< Sum of the terms; filled for OverBudget as well, for diagnosis. + /// PC CLOCK_MONOTONIC; meaningful when `usable()`. + int64_t capture_monotonic_ns = 0; + /// Sum of the terms, filled whenever `usable()`. It is the honest bound on `capture_monotonic_ns`, whether or not + /// it met the budget. + int64_t uncertainty_ns = 0; uint64_t generation = 0; ///< The PC local-clock generation the conversion used. UncertaintyTerms terms; + + /// Whether `capture_monotonic_ns` is a converted capture instant with a stated bound. `OverBudget` is usable: every + /// validity check passed and only the precision target was missed, so the instant is still known to within + /// `uncertainty_ns`. Every other verdict means the conversion could not be vouched for at all. A caller that + /// records frames publishes the instant and the bound when this holds and `Qualified` only says the bound met + /// the budget. + bool usable() const + { + return verdict == Verdict::Qualified || verdict == Verdict::OverBudget; + } }; /// chrony's bound on a host clock against its reference, |System time| + Root dispersion + Root delay / 2, plus diff --git a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp index a046f2565b..25d8f028ab 100644 --- a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp +++ b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp @@ -492,6 +492,44 @@ void test_budget_boundary() CHECK(q.generation == f.clock.generation); } +void test_usable_verdicts() +{ + const Verdict usable[] = { Verdict::Qualified, Verdict::OverBudget }; + const Verdict unusable[] = { Verdict::SenderInvalid, Verdict::NoSenderEvidence, Verdict::NoPcEvidence, + Verdict::StaleEvidence, Verdict::Unsynchronised, Verdict::ReferenceMismatch, + Verdict::NoLocalClock, Verdict::ImplausibleAge }; + for (Verdict v : usable) + { + Qualification q; + q.verdict = v; + CHECK(q.usable()); + } + for (Verdict v : unusable) + { + Qualification q; + q.verdict = v; + CHECK(!q.usable()); + } +} + +void test_over_budget_is_usable_with_the_same_instant() +{ + Fixture wide; + wide.config.budget_ns = 100 * kMs; + const Qualification ok = wide.run(); + CHECK(ok.verdict == Verdict::Qualified); + + Fixture tight; + tight.config.budget_ns = 1; + const Qualification over = tight.run(); + CHECK(over.verdict == Verdict::OverBudget); + CHECK(over.usable()); + CHECK(over.capture_monotonic_ns == ok.capture_monotonic_ns); + CHECK(over.capture_monotonic_ns != 0); + CHECK(over.uncertainty_ns > tight.config.budget_ns); + CHECK(over.uncertainty_ns == ok.uncertainty_ns); +} + void test_generation_carried_through() { Fixture f; @@ -731,6 +769,8 @@ int main() test_unsynchronised_camera_host(); test_implausible_age(); test_budget_boundary(); + test_usable_verdicts(); + test_over_budget_is_usable_with_the_same_instant(); test_generation_carried_through(); test_error_bound_rounds_up(); test_overflowing_inputs_are_refused(); From 4eb7b29589a3500a0c8f908ee4a70055a83e0002 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:16:48 -0500 Subject: [PATCH 04/10] timesync: say that a capture over the precision target keeps its time Document that OverBudget is usable and that the budget is a precision target for the diagnostics, not a condition for having a capture time. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/core/timesync/README.md | 6 ++++-- src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/core/timesync/README.md b/src/core/timesync/README.md index 771a5bdd5f..d96ae17d62 100644 --- a/src/core/timesync/README.md +++ b/src/core/timesync/README.md @@ -22,7 +22,7 @@ cmake -S src/core/timesync -B build -DBUILD_TESTING=ON && cmake --build build && | --- | --- | | `chrony_tracking.hpp` | Strict parser for one `chronyc -c tracking` line; `is_synchronised`. | | `local_clock.hpp` | Bracketed realtime/monotonic sample and a tracker that numbers the intervals between realtime steps. | -| `capture_qualifier.hpp` | `qualify_capture`: PC-monotonic capture time, summed uncertainty, or a typed refusal. | +| `capture_qualifier.hpp` | `qualify_capture`: PC-monotonic capture time with its summed uncertainty (`usable()`; `Qualified` also met the budget), or a typed refusal. | | `chrony_monitor.hpp`, `command_runner.hpp` | Background `chronyc` poll: no shell, a timeout, an output cap, never in the frame path. | | `tools/timesync_tool.py` | Sets up the chrony the library reads (below). Python standard library only. | @@ -82,7 +82,9 @@ so the difference can be slightly off; it is clamped at zero, and the copy slack the drift between reads. The bound is derived from chrony's own reports. It has not been validated against an independent reference such as a hardware sync pulse, so a qualified frame means "within chrony's claimed error", not "measured within". Refusal policy: any report that is stale, has no recent source update, is unsynchronised, or -names another source is refused and the frame carries no time; nothing is estimated. +names another source is refused and the frame carries no time; nothing is estimated. A frame that passes every check but +whose bound is over the budget is `OverBudget`, which is still `usable()`: it keeps its capture time and the bound, and the +budget is a precision target for the diagnostics, not a condition for having a time. Step detection compares successive wall-minus-monotonic offsets. On Linux the two clocks share frequency adjustment, so the difference changes only on a step. A change beyond both bracket widths diff --git a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp index 1039e59403..cff43b3358 100644 --- a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp +++ b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp @@ -111,7 +111,8 @@ int64_t inter_host_bound_ns(const ChronyEvidence& follower, int64_t follower_age_ns, int64_t server_age_ns); -/// Converts a sender realtime capture stamp to PC monotonic and decides whether it may be recorded. +/// Converts a sender realtime capture stamp to PC monotonic and says how well the conversion can be vouched for: not +/// at all (a refusal), vouched for but over the budget, or within it. /// /// The inter-host term bounds sender realtime against PC realtime along camera host system clock -> camera host /// NTP clock -> PC chronyd's measurement -> PC system clock: both hosts' |System time|, the PC's Root dispersion and From 395b68ecdee041e421112757f248e5bd9cc1cf40 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:29:39 -0500 Subject: [PATCH 05/10] timesync: cancel the camera host's root values only for the same upstream sample The inter-host bound subtracted the camera host's root delay and dispersion from the PC's on the assumption that both reports describe the same upstream sample. When the camera host has updated from its own source since the PC last updated from it, its newer root values are not what the PC copied, and the subtraction shrank the bound with no new measurement of the link. Subtract only when the camera host's last source update is not newer than the PC's; otherwise charge the PC's full root values, which is larger and never smaller. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/core/timesync/README.md | 5 ++++- src/core/timesync/cpp/capture_qualifier.cpp | 13 ++++++++++--- .../timesync/cpp/inc/timesync/capture_qualifier.hpp | 6 ++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/core/timesync/README.md b/src/core/timesync/README.md index d96ae17d62..64a403e4f1 100644 --- a/src/core/timesync/README.md +++ b/src/core/timesync/README.md @@ -77,7 +77,10 @@ internet source) cancels. That is why the bound is relative and not the sum of b What is and is not checked: the PC's selected reference id must be the camera host's address, so a different source is refused as `ReferenceMismatch`. The assumption that the root fields are cumulative through the camera host is -chrony's documented behaviour, not something this library measures. The two reports are read at different instants, +chrony's documented behaviour, not something this library measures. The camera host's root fields are subtracted only +when its last source update is not newer than the PC's (`Ref time`): then the PC's copy and the camera host's report +describe the same upstream sample. If the camera host has updated since, its newer root values are not what the PC +copied, nothing is subtracted, and the PC's full root values are charged instead; the bound is then larger, not smaller. The two reports are read at different instants, so the difference can be slightly off; it is clamped at zero, and the copy slack and the skew-times-age terms cover the drift between reads. The bound is derived from chrony's own reports. It has not been validated against an independent reference such as a hardware sync pulse, so a qualified frame means "within chrony's claimed error", diff --git a/src/core/timesync/cpp/capture_qualifier.cpp b/src/core/timesync/cpp/capture_qualifier.cpp index e1b144376a..5a956f5693 100644 --- a/src/core/timesync/cpp/capture_qualifier.cpp +++ b/src/core/timesync/cpp/capture_qualifier.cpp @@ -109,9 +109,16 @@ int64_t inter_host_bound_ns(const ChronyEvidence& follower, const ChronyTracking& s = follower.tracking; const ChronyTracking& p = server.tracking; // Root delay and dispersion are cumulative along the chain to the reference, so what the follower adds beyond - // the server's own values is the link between them; the shared upstream part cancels. - const double link_dispersion_s = std::max(s.root_dispersion_s - p.root_dispersion_s, 0.0); - const double link_delay_s = std::max(s.root_delay_s - p.root_delay_s, 0.0); + // the server's own values is the link between them, and the shared upstream part cancels. That holds only when + // the follower's copy of the server's values and the server's report describe the same upstream sample: the + // server has not updated from its own source since the follower last updated from it. A server whose last update + // is newer than the follower's may have different root values than the ones the follower copied, so subtracting + // them would shrink the bound without any new measurement of the link. Then nothing cancels and the follower's + // full root values are charged, which is larger and never smaller than the truth. + const bool same_upstream_sample = p.ref_time_unix_s > 0.0 && p.ref_time_unix_s <= s.ref_time_unix_s; + const double link_dispersion_s = + same_upstream_sample ? std::max(s.root_dispersion_s - p.root_dispersion_s, 0.0) : s.root_dispersion_s; + const double link_delay_s = same_upstream_sample ? std::max(s.root_delay_s - p.root_delay_s, 0.0) : s.root_delay_s; // The follower copied the server's values when it last polled; the server's dispersion has grown since. const double copy_slack_s = p.skew_ppm * 1e-6 * s.update_interval_s; const double follower_age_s = static_cast(std::max(follower_age_ns, 0)) * 1e-9; diff --git a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp index cff43b3358..acce06a111 100644 --- a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp +++ b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp @@ -120,8 +120,10 @@ int64_t inter_host_bound_ns(const ChronyEvidence& follower, /// link between the two and the camera host's own upstream error cancels), a slack for the camera host's dispersion /// growing since the PC copied it, and each report's skew times its age. The assumptions are that the PC's source is /// the camera host and that its root fields are cumulative through the camera host's; a differing reference id is -/// refused, and the subtraction is clamped at zero. The bound is derived from chrony's own reports, not measured -/// independently of them. +/// refused, and the subtraction is clamped at zero. The subtraction is made only when the camera host's last source +/// update is not newer than the PC's; otherwise the two reports are not known to describe the same upstream sample +/// and the PC's full root delay and dispersion are charged. The bound is derived from chrony's own reports, not +/// measured independently of them. /// /// `Qualified` carries the PC clock generation. The caller must pin the generation when a take starts and end the /// take, not continue it, when a later frame reports a different one. From 53c8a6367feac067870f65c2bd6f591f85c40120 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:30:59 -0500 Subject: [PATCH 06/10] timesync: test that cancellation needs a shared upstream sample Cover a camera host whose last update is newer than the PC's (including grown root values and an unknown Ref time), the compatible case that still cancels, and the unchanged bound floor; give the existing cancellation tests evidence that shares an upstream sample. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- .../test_capture_qualifier.cpp | 121 +++++++++++++++++- 1 file changed, 115 insertions(+), 6 deletions(-) diff --git a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp index 25d8f028ab..fea3fa0f28 100644 --- a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp +++ b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp @@ -164,6 +164,19 @@ void test_conversion_uses_pc_offset_not_sample_time() CHECK(q.capture_monotonic_ns == f.now - kMs); } +// Evidence with every chrony term zero, so each test sets only the term under test. +ChronyEvidence zero_evidence() +{ + return ChronyEvidence{ synced(), 0 }; +} + +// The camera host's last source update is not newer than the PC's copy of it, so the shared upstream part cancels. +void share_upstream_sample(ChronyEvidence& follower, ChronyEvidence& server) +{ + server.tracking.ref_time_unix_s = 100.0; + follower.tracking.ref_time_unix_s = 100.0; +} + void test_inter_host_terms() { Fixture f; @@ -176,6 +189,7 @@ void test_inter_host_terms() f.sender->tracking.root_delay_s = 2e-4; f.sender->tracking.skew_ppm = 5.0; f.config.budget_ns = 10 * kMs; + share_upstream_sample(*f.pc, *f.sender); const Qualification q = f.run(); // Both reports are one second old, follower (PC) update interval 0. |system time|: 100 us + 200 us; root dispersion // difference 200 - 100 = 100 us; root delay difference (400 - 200) / 2 = 100 us; skew * age: 10 us + 5 us. @@ -184,12 +198,6 @@ void test_inter_host_terms() CHECK(q.uncertainty_ns == q.terms.sensor_ns + q.terms.inter_host_ns + q.terms.pc_local_ns); } -// Evidence with every chrony term zero, so each test sets only the term under test. -ChronyEvidence zero_evidence() -{ - return ChronyEvidence{ synced(), 0 }; -} - void test_upstream_cancels() { // The camera host is internet-disciplined: large shared root fields, small link added by the PC. @@ -198,6 +206,7 @@ void test_upstream_cancels() f.sender->tracking.root_dispersion_s = 0.01; f.pc->tracking.root_delay_s = 0.1 + 200e-6; f.pc->tracking.root_dispersion_s = 0.01 + 50e-6; + share_upstream_sample(*f.pc, *f.sender); const Qualification q = f.run(); // Link only: 50 us + 200 us / 2 = 150 us. The summed model would charge about 0.11 s + 0.11 s. CHECK(within(q.terms.inter_host_ns, 150'000, 50)); // double error on the 0.1 s operands @@ -210,6 +219,7 @@ void test_upstream_cancels() void test_pc_below_camera_host_clamps_to_zero() { Fixture f; + share_upstream_sample(*f.pc, *f.sender); f.sender->tracking.root_delay_s = 0.1; f.sender->tracking.root_dispersion_s = 0.01; f.pc->tracking.root_delay_s = 0.05; @@ -295,6 +305,7 @@ void test_inter_host_bound_direct() // Root fields equal on both sides cancel completely, however large. s.tracking.root_delay_s = p.tracking.root_delay_s = 0.1; s.tracking.root_dispersion_s = p.tracking.root_dispersion_s = 0.01; + share_upstream_sample(s, p); CHECK(inter_host_bound_ns(s, p, 0, 0) == 0); // Dispersion difference counts in full, delay difference at half; exact in binary doubles. @@ -305,6 +316,7 @@ void test_inter_host_bound_direct() CHECK(inter_host_bound_ns(s2, p2, 0, 0) == 500'000'000 + 125'000'000); // A follower's root fields below the server's contribute nothing, never a negative term. + share_upstream_sample(p2, s2); CHECK(inter_host_bound_ns(p2, s2, 0, 0) == 0); // Combined: 1 us slack + 2 us follower aging + 3 us server aging = 6 us. @@ -323,6 +335,101 @@ void test_inter_host_bound_direct() CHECK(inter_host_bound_ns(s4, zero_evidence(), 0, 0) == INT64_MAX / 4); // the saturation value } +// Follower (PC) root fields 0.5 s dispersion and 0.25 s delay, server's 0.25 s and 0.125 s; all exact in binary +// doubles. +struct RootFieldPair +{ + ChronyEvidence follower = zero_evidence(); + ChronyEvidence server = zero_evidence(); + + RootFieldPair() + { + follower.tracking.root_dispersion_s = 0.5; + follower.tracking.root_delay_s = 0.25; + server.tracking.root_dispersion_s = 0.25; + server.tracking.root_delay_s = 0.125; + follower.tracking.ref_time_unix_s = 100.0; + } + + int64_t bound() const + { + return inter_host_bound_ns(follower, server, 0, 0); + } +}; + +constexpr int64_t kCancelledBoundNs = 250'000'000 + 62'500'000; // (0.5 - 0.25) + (0.25 - 0.125) / 2 +constexpr int64_t kUncancelledBoundNs = 500'000'000 + 125'000'000; // 0.5 + 0.25 / 2 + +void test_cancellation_requires_a_shared_upstream_sample() +{ + // Server last updated no later than the follower copied it: cancellation as before. + RootFieldPair same; + same.server.tracking.ref_time_unix_s = 100.0; + CHECK(same.bound() == kCancelledBoundNs); + same.server.tracking.ref_time_unix_s = 40.0; + CHECK(same.bound() == kCancelledBoundNs); + + // Server updated after the follower: its root fields may differ from the copy, so nothing cancels. + RootFieldPair newer; + newer.server.tracking.ref_time_unix_s = 101.0; + CHECK(newer.bound() == kUncancelledBoundNs); + CHECK(newer.bound() > same.bound()); + // Growing the newer server's root fields changes nothing: the bound is the follower's full values. + newer.server.tracking.root_dispersion_s = 0.4; + newer.server.tracking.root_delay_s = 0.3; + CHECK(newer.bound() == kUncancelledBoundNs); + + // Unknown server update time: nothing cancels. + RootFieldPair unknown; + unknown.server.tracking.ref_time_unix_s = 0.0; + CHECK(unknown.bound() == kUncancelledBoundNs); + + // Unknown follower update time with a known server one: the server is newer, nothing cancels. + RootFieldPair no_follower_time; + no_follower_time.follower.tracking.ref_time_unix_s = 0.0; + no_follower_time.server.tracking.ref_time_unix_s = 100.0; + CHECK(no_follower_time.bound() == kUncancelledBoundNs); + + // The other terms are charged on top in both cases. + Fixture f; + f.pc->tracking = same.follower.tracking; + f.pc->tracking.system_time_s = 1e-4; + f.sender->tracking = same.server.tracking; + f.sender->tracking.system_time_s = -2e-4; + f.config.budget_ns = 10 * kSecond; + CHECK(within(f.run().terms.inter_host_ns, kCancelledBoundNs + 300'000, 2)); + f.sender->tracking.ref_time_unix_s = 101.0; + CHECK(within(f.run().terms.inter_host_ns, kUncancelledBoundNs + 300'000, 2)); +} + +void test_newer_server_root_growth_never_shrinks_the_bound() +{ + // Follower evidence fixed; only the server's root fields grow, with a newer update time. + RootFieldPair original; + original.server.tracking.ref_time_unix_s = 100.0; + const int64_t before = original.bound(); + + for (double growth : { 0.01, 0.1, 0.25, 0.4, 1.0 }) + { + RootFieldPair grown; + grown.server.tracking.ref_time_unix_s = 100.5; + grown.server.tracking.root_dispersion_s += growth; + grown.server.tracking.root_delay_s += growth; + CHECK(grown.bound() >= before); + } + + // The same growth through qualify_capture. + Fixture f; + f.pc->tracking = original.follower.tracking; + f.sender->tracking = original.server.tracking; + f.config.budget_ns = 10 * kSecond; + const int64_t before_run = f.run().terms.inter_host_ns; + f.sender->tracking.ref_time_unix_s = 100.5; + f.sender->tracking.root_dispersion_s = 0.6; + f.sender->tracking.root_delay_s = 0.6; + CHECK(f.run().terms.inter_host_ns >= before_run); +} + void test_simple_rejections() { { @@ -758,6 +865,8 @@ int main() test_aging_charged_per_host(); test_both_system_times_charged(); test_inter_host_bound_direct(); + test_cancellation_requires_a_shared_upstream_sample(); + test_newer_server_root_growth_never_shrinks_the_bound(); test_simple_rejections(); test_evidence_age_boundary(); test_unsynchronised(); From 35da2a64ebf7bd1fdbfc9c06ab166827e3a26f4c Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 11:54:57 -0500 Subject: [PATCH 07/10] timesync: charge the PC's full root values, bound the command wait, serialise the monitor Answers review of the capture qualifier, command runner, monitor and tool: - Charge the PC's full root delay and dispersion. Subtracting the camera host's needs both reports to describe the same upstream sample, and chrony's report has no sample identifier (Ref time is each host's own). - Refuse a local-clock observation older than the evidence limit, with an overflow-safe age. - Keep the command deadline through reaping after stdout closes, treat a failed waitpid as a failure, and close pipe ends in the child only above stderr. - Serialise ChronyMonitor start() and stop() so a start cannot spawn a second worker while stop() joins the first. - status also checks the System time offset; configure-remote --offline adds a chrony local reference for a camera host with no upstream source. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/core/timesync/README.md | 38 ++-- src/core/timesync/cpp/capture_qualifier.cpp | 27 ++- src/core/timesync/cpp/chrony_monitor.cpp | 4 + src/core/timesync/cpp/command_runner.cpp | 66 +++++- .../cpp/inc/timesync/capture_qualifier.hpp | 17 +- .../cpp/inc/timesync/chrony_monitor.hpp | 2 + src/core/timesync/tools/test_timesync_tool.py | 85 +++++-- src/core/timesync/tools/timesync_tool.py | 37 ++- .../test_capture_qualifier.cpp | 213 ++++++++---------- .../chrony_monitor/test_chrony_monitor.cpp | 58 +++++ .../command_runner/test_command_runner.cpp | 39 ++++ 11 files changed, 402 insertions(+), 184 deletions(-) diff --git a/src/core/timesync/README.md b/src/core/timesync/README.md index 64a403e4f1..48b5e3b82d 100644 --- a/src/core/timesync/README.md +++ b/src/core/timesync/README.md @@ -39,7 +39,13 @@ dedicated unprivileged instance next to the system chronyd is read. The camera host keeps its own clock and serves it (it may follow UTC or serve its own clock). A second, unprivileged chronyd on the PC measures it and never adjusts the PC's system clock, so nothing has to reach the PC (a PC firewall does not matter) and the PC needs no root beyond installing chrony. `qualify_capture` charges -the link between the two. +the PC's full root delay and dispersion, which include the link between the two and the camera host's own upstream +error (see "What the bound assumes"). + +A camera host with no upstream source is served as synchronised only if its chrony has a `local` reference; pass +`configure-remote --offline` to write one. It is off by default, so a host that follows UTC keeps its configuration. +`status` is a readiness check on the PC's follower (synchronised, the camera host's reference id, root dispersion and +System time offset under `--max-dispersion-ms`); it does not state the per-frame bound. ## Running and deploying it @@ -66,28 +72,24 @@ launch tools call this tool; they do not carry their own chrony setup. ## What the bound assumes -The inter-host term is both hosts' `System time`, the PC's root dispersion and root delay / 2 minus the camera -host's, a slack for the camera host's dispersion growing since the PC copied it, and each report's skew times its -age. +The inter-host term is both hosts' `System time`, the PC's full root dispersion and root delay / 2, a slack for the +camera host's dispersion growing since the PC copied it, and each report's skew times its age. -Why the subtraction is used: chrony's root delay and root dispersion are cumulative along the chain to the -reference. When the PC's source is the camera host, the PC's values are the camera host's plus the link between -them, so the difference isolates the link and the camera host's own upstream error (large when it follows an -internet source) cancels. That is why the bound is relative and not the sum of both hosts' absolute bounds. +The camera host's own root values are not subtracted. They would cancel its upstream error only if both hosts' root +fields described the same upstream sample, and chrony's report carries no identifier that shows it: `Ref time` is when +each host processed its own latest measurement, so comparing the two says nothing about which samples the root fields +came from. Subtracting without that correspondence could shrink the bound with no new measurement of the link, so the +PC's full values are charged. They include the camera host's upstream error, so the bound is larger than the link +alone and never smaller; a camera host disciplined from the internet (large root delay) will usually be over the +budget, and those frames keep their capture time. What is and is not checked: the PC's selected reference id must be the camera host's address, so a different source -is refused as `ReferenceMismatch`. The assumption that the root fields are cumulative through the camera host is -chrony's documented behaviour, not something this library measures. The camera host's root fields are subtracted only -when its last source update is not newer than the PC's (`Ref time`): then the PC's copy and the camera host's report -describe the same upstream sample. If the camera host has updated since, its newer root values are not what the PC -copied, nothing is subtracted, and the PC's full root values are charged instead; the bound is then larger, not smaller. The two reports are read at different instants, -so the difference can be slightly off; it is clamped at zero, and the copy slack and the skew-times-age terms cover -the drift between reads. The bound is derived from chrony's own reports. It has not been validated against an +is refused as `ReferenceMismatch`. The bound is derived from chrony's own reports. It has not been validated against an independent reference such as a hardware sync pulse, so a qualified frame means "within chrony's claimed error", not "measured within". Refusal policy: any report that is stale, has no recent source update, is unsynchronised, or -names another source is refused and the frame carries no time; nothing is estimated. A frame that passes every check but -whose bound is over the budget is `OverBudget`, which is still `usable()`: it keeps its capture time and the bound, and the -budget is a precision target for the diagnostics, not a condition for having a time. +names another source is refused and the frame carries no time; nothing is estimated. A frame that passes every check +but whose bound is over the budget is `OverBudget`, which is still `usable()`: it keeps its capture time and the +bound, and the budget is a precision target for the diagnostics, not a condition for having a time. Step detection compares successive wall-minus-monotonic offsets. On Linux the two clocks share frequency adjustment, so the difference changes only on a step. A change beyond both bracket widths diff --git a/src/core/timesync/cpp/capture_qualifier.cpp b/src/core/timesync/cpp/capture_qualifier.cpp index 5a956f5693..f6f46d310b 100644 --- a/src/core/timesync/cpp/capture_qualifier.cpp +++ b/src/core/timesync/cpp/capture_qualifier.cpp @@ -108,17 +108,14 @@ int64_t inter_host_bound_ns(const ChronyEvidence& follower, { const ChronyTracking& s = follower.tracking; const ChronyTracking& p = server.tracking; - // Root delay and dispersion are cumulative along the chain to the reference, so what the follower adds beyond - // the server's own values is the link between them, and the shared upstream part cancels. That holds only when - // the follower's copy of the server's values and the server's report describe the same upstream sample: the - // server has not updated from its own source since the follower last updated from it. A server whose last update - // is newer than the follower's may have different root values than the ones the follower copied, so subtracting - // them would shrink the bound without any new measurement of the link. Then nothing cancels and the follower's - // full root values are charged, which is larger and never smaller than the truth. - const bool same_upstream_sample = p.ref_time_unix_s > 0.0 && p.ref_time_unix_s <= s.ref_time_unix_s; - const double link_dispersion_s = - same_upstream_sample ? std::max(s.root_dispersion_s - p.root_dispersion_s, 0.0) : s.root_dispersion_s; - const double link_delay_s = same_upstream_sample ? std::max(s.root_delay_s - p.root_delay_s, 0.0) : s.root_delay_s; + // The follower's root delay and dispersion are charged in full. Subtracting the camera host's own values, so that + // only the link between the two is charged, would need both reports to describe the same upstream sample, and + // chrony's tracking output has no sample identifier to show that: `Ref time` is when each host processed its own + // latest measurement, so comparing the two proves nothing about which samples the root fields came from. Without + // that correspondence a subtraction can shrink the bound with no new measurement of the link, so none is made. + // The charge includes the camera host's upstream error, so it is larger than the link alone and never smaller. + const double link_dispersion_s = s.root_dispersion_s; + const double link_delay_s = s.root_delay_s; // The follower copied the server's values when it last polled; the server's dispersion has grown since. const double copy_slack_s = p.skew_ppm * 1e-6 * s.update_interval_s; const double follower_age_s = static_cast(std::max(follower_age_ns, 0)) * 1e-9; @@ -179,6 +176,14 @@ Qualification qualify_capture(const SenderCapture& capture, { return refuse(Verdict::NoLocalClock); } + // The offset between this host's clocks is read from this sample; one older than the evidence limit may predate a + // wall-clock step and carries the old generation. + int64_t sample_age = 0; + if (__builtin_sub_overflow(now_monotonic_ns, pc_clock.sample.monotonic_ns, &sample_age) || + sample_age > config.max_evidence_age_ns) + { + return refuse(Verdict::StaleEvidence); + } q.generation = pc_clock.generation; int64_t offset = 0; diff --git a/src/core/timesync/cpp/chrony_monitor.cpp b/src/core/timesync/cpp/chrony_monitor.cpp index 6eb07134a7..3490bf8116 100644 --- a/src/core/timesync/cpp/chrony_monitor.cpp +++ b/src/core/timesync/cpp/chrony_monitor.cpp @@ -107,6 +107,7 @@ MonitorSnapshot ChronyMonitor::snapshot() const void ChronyMonitor::start() { + std::lock_guard lifecycle(lifecycle_mutex_); std::lock_guard lock(mutex_); if (thread_.joinable()) { @@ -118,6 +119,9 @@ void ChronyMonitor::start() void ChronyMonitor::stop() { + // Held until the worker has been joined: a start() meanwhile would clear stopping_ and spawn a second worker, and + // this join would then never return. + std::lock_guard lifecycle(lifecycle_mutex_); std::thread joinable; { std::lock_guard lock(mutex_); diff --git a/src/core/timesync/cpp/command_runner.cpp b/src/core/timesync/cpp/command_runner.cpp index 6a1e912e07..ba5b013325 100644 --- a/src/core/timesync/cpp/command_runner.cpp +++ b/src/core/timesync/cpp/command_runner.cpp @@ -58,8 +58,16 @@ CommandRunner posix_command_runner() posix_spawn_file_actions_adddup2(&actions, write_end.fd, STDOUT_FILENO); posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, "/dev/null", O_WRONLY, 0); posix_spawn_file_actions_addopen(&actions, STDIN_FILENO, "/dev/null", O_RDONLY, 0); - posix_spawn_file_actions_addclose(&actions, read_end.fd); - posix_spawn_file_actions_addclose(&actions, write_end.fd); + // Descriptors 0-2 were set up above; closing a pipe end that landed on one of them (the parent started with + // it closed) would undo that. Only higher descriptors need closing in the child. + if (read_end.fd > STDERR_FILENO) + { + posix_spawn_file_actions_addclose(&actions, read_end.fd); + } + if (write_end.fd > STDERR_FILENO) + { + posix_spawn_file_actions_addclose(&actions, write_end.fd); + } std::vector args; for (const auto& a : argv) @@ -123,15 +131,59 @@ CommandRunner posix_command_runner() } } - if (result.timed_out || result.truncated) + int status = 0; + bool reaped = false; + bool wait_failed = false; + if (!result.timed_out && !result.truncated) { - kill(pid, SIGKILL); + // The output ended, which does not mean the program did: one that closed stdout and hangs must still be + // bounded by the same deadline, so reap it without blocking until the deadline passes. + for (;;) + { + const pid_t waited = waitpid(pid, &status, WNOHANG); + if (waited == pid) + { + reaped = true; + break; + } + if (waited < 0 && errno == EINTR) + { + continue; + } + if (waited < 0) + { + wait_failed = true; + break; + } + if (std::chrono::steady_clock::now() >= deadline) + { + result.timed_out = true; + break; + } + poll(nullptr, 0, 5); + } } - int status = 0; - while (waitpid(pid, &status, 0) < 0 && errno == EINTR) + if (!reaped && !wait_failed) { + kill(pid, SIGKILL); + for (;;) + { + const pid_t waited = waitpid(pid, &status, 0); + if (waited == pid) + { + reaped = true; + break; + } + if (waited < 0 && errno == EINTR) + { + continue; + } + break; // not reapable: the exit status is unknown + } } - if (!result.timed_out && !result.truncated && WIFEXITED(status)) + // The exit code is read only from a status that waitpid actually returned: a failed wait leaves it at -1 + // instead of reporting a zero-initialised status as success. + if (reaped && !result.timed_out && !result.truncated && WIFEXITED(status)) { result.exit_code = WEXITSTATUS(status); } diff --git a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp index acce06a111..36f1849eca 100644 --- a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp +++ b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp @@ -115,15 +115,14 @@ int64_t inter_host_bound_ns(const ChronyEvidence& follower, /// at all (a refusal), vouched for but over the budget, or within it. /// /// The inter-host term bounds sender realtime against PC realtime along camera host system clock -> camera host -/// NTP clock -> PC chronyd's measurement -> PC system clock: both hosts' |System time|, the PC's Root dispersion and -/// Root delay / 2 minus the camera host's (those fields are cumulative to the reference, so the difference is the -/// link between the two and the camera host's own upstream error cancels), a slack for the camera host's dispersion -/// growing since the PC copied it, and each report's skew times its age. The assumptions are that the PC's source is -/// the camera host and that its root fields are cumulative through the camera host's; a differing reference id is -/// refused, and the subtraction is clamped at zero. The subtraction is made only when the camera host's last source -/// update is not newer than the PC's; otherwise the two reports are not known to describe the same upstream sample -/// and the PC's full root delay and dispersion are charged. The bound is derived from chrony's own reports, not -/// measured independently of them. +/// NTP clock -> PC chronyd's measurement -> PC system clock: both hosts' |System time|, the PC's full Root dispersion +/// and Root delay / 2, a slack for the camera host's dispersion growing since the PC copied it, and each report's +/// skew times its age. The camera host's own root values are not subtracted: nothing in chrony's report shows that its +/// root fields and the PC's copy describe the same upstream sample (`Ref time` is each host's own), and a subtraction +/// without that correspondence can shrink the bound with no new measurement of the link. The charge therefore +/// includes the camera host's upstream error and is larger than the link alone, never smaller. The assumption is +/// that the PC's source is the camera host; a differing reference id is refused. The bound is derived from chrony's +/// own reports, not measured independently of them. /// /// `Qualified` carries the PC clock generation. The caller must pin the generation when a take starts and end the /// take, not continue it, when a later frame reports a different one. diff --git a/src/core/timesync/cpp/inc/timesync/chrony_monitor.hpp b/src/core/timesync/cpp/inc/timesync/chrony_monitor.hpp index 24a006257c..4aeec64fcf 100644 --- a/src/core/timesync/cpp/inc/timesync/chrony_monitor.hpp +++ b/src/core/timesync/cpp/inc/timesync/chrony_monitor.hpp @@ -83,6 +83,8 @@ class ChronyMonitor CommandRunner runner_; NowFn now_; NowFn now_realtime_; + /// Serialises start() and stop() so a start cannot slip in while a stop is still joining the worker. + std::mutex lifecycle_mutex_; mutable std::mutex mutex_; MonitorSnapshot snapshot_; std::condition_variable wake_; diff --git a/src/core/timesync/tools/test_timesync_tool.py b/src/core/timesync/tools/test_timesync_tool.py index ceb6de88be..9e5869b9a6 100644 --- a/src/core/timesync/tools/test_timesync_tool.py +++ b/src/core/timesync/tools/test_timesync_tool.py @@ -11,11 +11,10 @@ import subprocess import tempfile import unittest -from unittest import mock +import unittest.mock from pathlib import Path import timesync_tool as tool -from timesync_tool import ToolError, Tracking REAL_LINE = ( "5BBD5B9D,91.189.91.157,3,1794516440.890180266,-0.000037218,-0.000042871," @@ -36,7 +35,7 @@ def remote_line(ref_id: str = HOST_ID, dispersion: str = "0.000300000") -> str: ) -def make_tracking(**overrides: object) -> Tracking: +def make_tracking(**overrides: object) -> tool.Tracking: fields: dict[str, object] = dict( reference_id=HOST_ID, reference_name="", @@ -50,7 +49,7 @@ def make_tracking(**overrides: object) -> Tracking: leap_status="Normal", ) fields.update(overrides) - return Tracking(**fields) # type: ignore[arg-type] + return tool.Tracking(**fields) # type: ignore[arg-type] class ParseTrackingTest(unittest.TestCase): @@ -84,14 +83,14 @@ def test_stratum_zero_not_synchronised(self) -> None: def test_wrong_field_counts(self) -> None: fields = REAL_LINE.split(",") for bad in (",".join(fields[:13]), ",".join(fields + ["x"])): - with self.assertRaises(ToolError): + with self.assertRaises(tool.ToolError): tool.parse_tracking(bad) def test_non_numeric_fields(self) -> None: for index in (2, 3, 4, 9, 10, 11, 12): fields = REAL_LINE.split(",") fields[index] = "abc" - with self.subTest(index=index), self.assertRaises(ToolError): + with self.subTest(index=index), self.assertRaises(tool.ToolError): tool.parse_tracking(",".join(fields)) @@ -102,7 +101,7 @@ def test_known_addresses(self) -> None: def test_bad_addresses(self) -> None: for bad in ("999.1.1.1", "a.b.c.d", "1.2.3"): - with self.subTest(bad=bad), self.assertRaises(ToolError): + with self.subTest(bad=bad), self.assertRaises(tool.ToolError): tool.reference_id_for(bad) @@ -123,6 +122,22 @@ def test_serves_the_pc_and_lets_it_read_state(self) -> None: ) +class RemoteDropInOfflineTest(unittest.TestCase): + def test_default_has_no_local_reference(self) -> None: + self.assertEqual( + tool.remote_drop_in(HOST_IP), + f"allow {HOST_IP}\nbindcmdaddress 0.0.0.0\ncmdallow {HOST_IP}\n", + ) + self.assertNotIn("local", tool.remote_drop_in(HOST_IP, False)) + self.assertNotIn("local stratum", tool.remote_script(HOST_IP)) + + def test_offline_adds_local_stratum(self) -> None: + drop_in = tool.remote_drop_in(HOST_IP, True) + self.assertIn("local stratum 10", drop_in.splitlines()) + self.assertIn(f"allow {HOST_IP}", drop_in) + self.assertIn("local stratum 10", tool.remote_script(HOST_IP, True)) + + class RemoteScriptTest(unittest.TestCase): def setUp(self) -> None: self.script = tool.remote_script(HOST_IP) @@ -171,7 +186,7 @@ def test_bash_accepts_script(self) -> None: class JudgeTest(unittest.TestCase): - def judge(self, local: Tracking) -> list[str]: + def judge(self, local: tool.Tracking) -> list[str]: return tool.judge(local, HOST_ID, 0.002) def test_usable(self) -> None: @@ -192,6 +207,19 @@ def test_high_dispersion(self) -> None: self.assertEqual(len(problems), 1) self.assertIn("dispersion", problems[0]) + def test_large_system_time_is_not_ready(self) -> None: + problems = self.judge(make_tracking(root_dispersion=0.0003, system_time=0.050)) + self.assertEqual(len(problems), 1) + self.assertIn("System time", problems[0]) + # The sign does not matter. + problems = self.judge(make_tracking(root_dispersion=0.0003, system_time=-0.050)) + self.assertEqual(len(problems), 1) + self.assertIn("System time", problems[0]) + + def test_small_system_time_stays_ready(self) -> None: + self.assertEqual(self.judge(make_tracking(system_time=0.0005)), []) + self.assertEqual(self.judge(make_tracking(system_time=-0.0005)), []) + def test_problems_accumulate(self) -> None: bad = make_tracking( leap_status="Not synchronised", @@ -219,6 +247,25 @@ def test_dry_run_prints_script(self) -> None: self.assertIn(f"cmdallow {HOST_IP}", out) self.assertIn("bindcmdaddress 0.0.0.0", out) + def test_dry_run_offline_adds_local_reference(self) -> None: + code, out, _ = run_main( + [ + "configure-remote", + "user@host", + "--host-lan-ip", + HOST_IP, + "--offline", + "--dry-run", + ] + ) + self.assertEqual(code, 0) + self.assertIn("local stratum 10", out) + code, out, _ = run_main( + ["configure-remote", "user@host", "--host-lan-ip", HOST_IP, "--dry-run"] + ) + self.assertEqual(code, 0) + self.assertNotIn("local stratum", out) + def test_bad_host_lan_ip(self) -> None: code, _, err = run_main( ["configure-remote", "user@host", "--host-lan-ip", "nope", "--dry-run"] @@ -230,7 +277,7 @@ def test_timeout_is_an_error(self) -> None: def expire(*args: object, **kwargs: object) -> None: raise subprocess.TimeoutExpired(cmd="ssh", timeout=5) - with mock.patch.object(tool.subprocess, "run", expire): + with unittest.mock.patch.object(tool.subprocess, "run", expire): code, _, err = run_main( [ "configure-remote", @@ -248,8 +295,10 @@ def expire(*args: object, **kwargs: object) -> None: class EnsureLocalTest(unittest.TestCase): def test_does_nothing_when_chronyd_is_found(self) -> None: with ( - mock.patch.object(tool.shutil, "which", return_value="/usr/sbin/chronyd"), - mock.patch.object(tool.subprocess, "run") as run, + unittest.mock.patch.object( + tool.shutil, "which", return_value="/usr/sbin/chronyd" + ), + unittest.mock.patch.object(tool.subprocess, "run") as run, ): code, out, _ = run_main(["ensure-local"]) self.assertEqual(code, 0) @@ -259,9 +308,11 @@ def test_does_nothing_when_chronyd_is_found(self) -> None: def test_installs_chrony_when_missing(self) -> None: done = subprocess.CompletedProcess([], 0) with ( - mock.patch.object(tool.shutil, "which", return_value=None), - mock.patch.object(Path, "is_file", return_value=False), - mock.patch.object(tool.subprocess, "run", return_value=done) as run, + unittest.mock.patch.object(tool.shutil, "which", return_value=None), + unittest.mock.patch.object(Path, "is_file", return_value=False), + unittest.mock.patch.object( + tool.subprocess, "run", return_value=done + ) as run, ): code, _, _ = run_main(["ensure-local"]) self.assertEqual(code, 0) @@ -272,9 +323,9 @@ def test_installs_chrony_when_missing(self) -> None: def test_failed_install_is_an_error(self) -> None: failed = subprocess.CompletedProcess([], 100) with ( - mock.patch.object(tool.shutil, "which", return_value=None), - mock.patch.object(Path, "is_file", return_value=False), - mock.patch.object(tool.subprocess, "run", return_value=failed), + unittest.mock.patch.object(tool.shutil, "which", return_value=None), + unittest.mock.patch.object(Path, "is_file", return_value=False), + unittest.mock.patch.object(tool.subprocess, "run", return_value=failed), ): code, _, err = run_main(["ensure-local"]) self.assertEqual(code, 1) diff --git a/src/core/timesync/tools/timesync_tool.py b/src/core/timesync/tools/timesync_tool.py index 493b7da6eb..6e22a0ce25 100755 --- a/src/core/timesync/tools/timesync_tool.py +++ b/src/core/timesync/tools/timesync_tool.py @@ -4,7 +4,7 @@ """Runs and deploys the chrony setup that the timesync library reads: the camera host serves its clock and the PC measures it. timesync_tool.py ensure-local - timesync_tool.py configure-remote user@host --host-lan-ip IP [--timeout S] [--dry-run] + timesync_tool.py configure-remote user@host --host-lan-ip IP [--offline] [--timeout S] [--dry-run] timesync_tool.py follow --server HOST [--cmd-port N] timesync_tool.py status --remote HOST [--cmd-port N] [--wait S] @@ -117,14 +117,19 @@ def follower_config(server: str, server_port: int, cmd_port: int, workdir: Path) ) -def remote_drop_in(host_lan_ip: str) -> str: - """The camera host's chrony drop-in: serve this PC and let it read this chronyd's state.""" - return f"allow {host_lan_ip}\nbindcmdaddress 0.0.0.0\ncmdallow {host_lan_ip}\n" +def remote_drop_in(host_lan_ip: str, offline: bool = False) -> str: + """The camera host's chrony drop-in: serve this PC and let it read this chronyd's state. + ``offline`` adds a ``local`` reference, which an isolated camera host (no upstream source) needs to be served as + synchronised; a host that follows UTC must not have one. + """ + drop_in = f"allow {host_lan_ip}\nbindcmdaddress 0.0.0.0\ncmdallow {host_lan_ip}\n" + return drop_in + "local stratum 10\n" if offline else drop_in -def remote_script(host_lan_ip: str) -> str: + +def remote_script(host_lan_ip: str, offline: bool = False) -> str: """The bash that ``configure-remote`` runs on the camera host.""" - wanted = remote_drop_in(host_lan_ip) + wanted = remote_drop_in(host_lan_ip, offline) return f"""set -eu sudo -n true || {{ echo "passwordless sudo is required on the target" >&2; exit 1; }} if ! command -v chronyd >/dev/null; then @@ -174,7 +179,13 @@ def chronyc_tracking( def judge(local: Tracking, expected_id: str, max_dispersion_s: float) -> list[str]: - """What keeps this machine's measuring instance from being usable, one message each; empty when it is usable.""" + """What keeps this machine's measuring instance from being usable, one message each; empty when it is usable. + + The same limit applies to the root dispersion and to the size of the System time offset, so a follower that reports + a small dispersion while its clock is tens of milliseconds off is not called ready. This is a readiness check on + the follower only: it does not state the inter-host bound, which the capture qualifier computes per frame from both + hosts' reports. + """ problems = [] if not local.synchronised: problems.append( @@ -190,6 +201,11 @@ def judge(local: Tracking, expected_id: str, max_dispersion_s: float) -> list[st f"this machine's root dispersion is {local.root_dispersion * 1e3:.1f} ms, " f"above {max_dispersion_s * 1e3:.1f} ms; it is still settling" ) + if abs(local.system_time) > max_dispersion_s: + problems.append( + f"this machine's System time offset is {abs(local.system_time) * 1e3:.1f} ms, " + f"above {max_dispersion_s * 1e3:.1f} ms; it is still settling" + ) return problems @@ -266,7 +282,7 @@ def cmd_ensure_local(args: argparse.Namespace) -> int: def cmd_configure_remote(args: argparse.Namespace) -> int: reference_id_for(args.host_lan_ip) - script = remote_script(args.host_lan_ip) + script = remote_script(args.host_lan_ip, args.offline) command = [ "ssh", "-o", @@ -366,6 +382,11 @@ def main(argv: list[str] | None = None) -> int: configure.add_argument( "--timeout", type=float, default=90.0, help="seconds the whole setup may take" ) + configure.add_argument( + "--offline", + action="store_true", + help="the camera host has no upstream time source: add a chrony `local` reference so it is served as synchronised", + ) configure.add_argument("--dry-run", action="store_true") status = sub.add_parser( "status", diff --git a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp index fea3fa0f28..b30afa881a 100644 --- a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp +++ b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp @@ -3,6 +3,7 @@ #include +#include #include #include #include @@ -170,11 +171,13 @@ ChronyEvidence zero_evidence() return ChronyEvidence{ synced(), 0 }; } -// The camera host's last source update is not newer than the PC's copy of it, so the shared upstream part cancels. -void share_upstream_sample(ChronyEvidence& follower, ChronyEvidence& server) +// What the inter-host term used to be when the camera host's root values were subtracted from the PC's; the +// charge must never fall below it. +int64_t old_cancelled_bound_ns(const ChronyTracking& follower, const ChronyTracking& server) { - server.tracking.ref_time_unix_s = 100.0; - follower.tracking.ref_time_unix_s = 100.0; + const double disp = std::max(follower.root_dispersion_s - server.root_dispersion_s, 0.0); + const double delay = std::max(follower.root_delay_s - server.root_delay_s, 0.0); + return static_cast((disp + delay / 2.0) * 1e9); } void test_inter_host_terms() @@ -189,50 +192,50 @@ void test_inter_host_terms() f.sender->tracking.root_delay_s = 2e-4; f.sender->tracking.skew_ppm = 5.0; f.config.budget_ns = 10 * kMs; - share_upstream_sample(*f.pc, *f.sender); const Qualification q = f.run(); - // Both reports are one second old, follower (PC) update interval 0. |system time|: 100 us + 200 us; root dispersion - // difference 200 - 100 = 100 us; root delay difference (400 - 200) / 2 = 100 us; skew * age: 10 us + 5 us. + // Both reports are one second old, follower (PC) update interval 0. |system time|: 100 us + 200 us; the PC's + // full root dispersion 200 us; its root delay / 2 = 200 us; skew * age: 10 us + 5 us. CHECK(q.verdict == Verdict::Qualified); - CHECK(within(q.terms.inter_host_ns, 515'000, 2)); + CHECK(within(q.terms.inter_host_ns, 715'000, 2)); + CHECK(q.terms.inter_host_ns >= old_cancelled_bound_ns(f.pc->tracking, f.sender->tracking) + 300'000); CHECK(q.uncertainty_ns == q.terms.sensor_ns + q.terms.inter_host_ns + q.terms.pc_local_ns); } -void test_upstream_cancels() +void test_upstream_error_is_charged() { - // The camera host is internet-disciplined: large shared root fields, small link added by the PC. + // The camera host is internet-disciplined: large root fields on both hosts. The PC's full values are charged, + // the camera host's own are not subtracted, so the term is far above the link alone. Fixture f; f.sender->tracking.root_delay_s = 0.1; f.sender->tracking.root_dispersion_s = 0.01; f.pc->tracking.root_delay_s = 0.1 + 200e-6; f.pc->tracking.root_dispersion_s = 0.01 + 50e-6; - share_upstream_sample(*f.pc, *f.sender); const Qualification q = f.run(); - // Link only: 50 us + 200 us / 2 = 150 us. The summed model would charge about 0.11 s + 0.11 s. - CHECK(within(q.terms.inter_host_ns, 150'000, 50)); // double error on the 0.1 s operands - CHECK(q.terms.inter_host_ns < kMs); - CHECK(q.verdict == Verdict::Qualified); + // 0.01005 s + 0.1002 s / 2 = 60.15 ms. + CHECK(within(q.terms.inter_host_ns, 60'150'000, 100)); // double error on the 0.1 s operands + CHECK(q.terms.inter_host_ns >= old_cancelled_bound_ns(f.pc->tracking, f.sender->tracking)); + CHECK(q.verdict == Verdict::OverBudget); + CHECK(q.usable()); CHECK(f.config.budget_ns == 5 * kMs); CHECK(q.uncertainty_ns == q.terms.sensor_ns + q.terms.inter_host_ns + q.terms.pc_local_ns); + f.config.budget_ns = kSecond; + CHECK(f.run().verdict == Verdict::Qualified); } -void test_pc_below_camera_host_clamps_to_zero() +void test_pc_below_camera_host_is_still_charged_in_full() { Fixture f; - share_upstream_sample(*f.pc, *f.sender); f.sender->tracking.root_delay_s = 0.1; f.sender->tracking.root_dispersion_s = 0.01; f.pc->tracking.root_delay_s = 0.05; f.pc->tracking.root_dispersion_s = 0.005; - CHECK(f.run().terms.inter_host_ns == 0); - CHECK(f.run().verdict == Verdict::Qualified); - // Only the dispersion is below: the delay difference still counts, the dispersion one does not. - f.pc->tracking.root_delay_s = 0.1 + 400e-6; - CHECK(within(f.run().terms.inter_host_ns, 200'000, 50)); - // Only the delay is below. - f.pc->tracking.root_delay_s = 0.05; - f.pc->tracking.root_dispersion_s = 0.01 + 80e-6; + // No clamp to zero: 0.005 s + 0.05 s / 2 = 30 ms, whatever the camera host reports. + CHECK(within(f.run().terms.inter_host_ns, 30'000'000, 50)); + CHECK(f.run().verdict == Verdict::OverBudget); + f.pc->tracking.root_delay_s = 0.0; + f.pc->tracking.root_dispersion_s = 80e-6; CHECK(within(f.run().terms.inter_host_ns, 80'000, 50)); + CHECK(f.run().verdict == Verdict::Qualified); } void test_copy_slack() @@ -302,21 +305,19 @@ void test_inter_host_bound_direct() CHECK(inter_host_bound_ns(s, p, 0, 0) == 0); CHECK(inter_host_bound_ns(s, p, 100 * kSecond, 100 * kSecond) == 0); - // Root fields equal on both sides cancel completely, however large. + // Root fields equal on both sides are charged in full: 0.01 s + 0.1 s / 2. s.tracking.root_delay_s = p.tracking.root_delay_s = 0.1; s.tracking.root_dispersion_s = p.tracking.root_dispersion_s = 0.01; - share_upstream_sample(s, p); - CHECK(inter_host_bound_ns(s, p, 0, 0) == 0); + CHECK(within(inter_host_bound_ns(s, p, 0, 0), 60'000'000, 50)); - // Dispersion difference counts in full, delay difference at half; exact in binary doubles. + // Dispersion counts in full, delay at half; exact in binary doubles. ChronyEvidence s2 = zero_evidence(); ChronyEvidence p2 = zero_evidence(); s2.tracking.root_dispersion_s = 0.5; s2.tracking.root_delay_s = 0.25; CHECK(inter_host_bound_ns(s2, p2, 0, 0) == 500'000'000 + 125'000'000); - // A follower's root fields below the server's contribute nothing, never a negative term. - share_upstream_sample(p2, s2); + // The server's root fields are not charged: a follower with none reports zero against a server with many. CHECK(inter_host_bound_ns(p2, s2, 0, 0) == 0); // Combined: 1 us slack + 2 us follower aging + 3 us server aging = 6 us. @@ -335,99 +336,60 @@ void test_inter_host_bound_direct() CHECK(inter_host_bound_ns(s4, zero_evidence(), 0, 0) == INT64_MAX / 4); // the saturation value } -// Follower (PC) root fields 0.5 s dispersion and 0.25 s delay, server's 0.25 s and 0.125 s; all exact in binary -// doubles. -struct RootFieldPair +// Follower (PC) root fields 0.5 s dispersion and 0.25 s delay; all exact in binary doubles. +constexpr int64_t kFollowerBoundNs = 500'000'000 + 125'000'000; // 0.5 + 0.25 / 2 + +void test_server_root_fields_and_ref_time_do_not_change_the_bound() { ChronyEvidence follower = zero_evidence(); + follower.tracking.root_dispersion_s = 0.5; + follower.tracking.root_delay_s = 0.25; ChronyEvidence server = zero_evidence(); + server.tracking.root_dispersion_s = 0.25; + server.tracking.root_delay_s = 0.125; + CHECK(inter_host_bound_ns(follower, server, 0, 0) == kFollowerBoundNs); - RootFieldPair() + // The server's root fields moving either way, to zero, to just under the follower's, above it, or far above it. + for (double disp : { 0.0, 0.25, 0.4999, 0.5, 0.75, 10.0 }) { - follower.tracking.root_dispersion_s = 0.5; - follower.tracking.root_delay_s = 0.25; - server.tracking.root_dispersion_s = 0.25; - server.tracking.root_delay_s = 0.125; - follower.tracking.ref_time_unix_s = 100.0; + for (double delay : { 0.0, 0.125, 0.2499, 0.25, 0.5, 10.0 }) + { + server.tracking.root_dispersion_s = disp; + server.tracking.root_delay_s = delay; + CHECK(inter_host_bound_ns(follower, server, 0, 0) == kFollowerBoundNs); + CHECK(inter_host_bound_ns(follower, server, 0, 0) >= + old_cancelled_bound_ns(follower.tracking, server.tracking)); + } } - int64_t bound() const + // Neither host's Ref time matters, whichever is newer, and whether or not it is known. + for (double follower_ref : { 0.0, 40.0, 100.0, 101.0 }) { - return inter_host_bound_ns(follower, server, 0, 0); + for (double server_ref : { 0.0, 40.0, 100.0, 101.0 }) + { + follower.tracking.ref_time_unix_s = follower_ref; + server.tracking.ref_time_unix_s = server_ref; + CHECK(inter_host_bound_ns(follower, server, 0, 0) == kFollowerBoundNs); + } } -}; -constexpr int64_t kCancelledBoundNs = 250'000'000 + 62'500'000; // (0.5 - 0.25) + (0.25 - 0.125) / 2 -constexpr int64_t kUncancelledBoundNs = 500'000'000 + 125'000'000; // 0.5 + 0.25 / 2 - -void test_cancellation_requires_a_shared_upstream_sample() -{ - // Server last updated no later than the follower copied it: cancellation as before. - RootFieldPair same; - same.server.tracking.ref_time_unix_s = 100.0; - CHECK(same.bound() == kCancelledBoundNs); - same.server.tracking.ref_time_unix_s = 40.0; - CHECK(same.bound() == kCancelledBoundNs); - - // Server updated after the follower: its root fields may differ from the copy, so nothing cancels. - RootFieldPair newer; - newer.server.tracking.ref_time_unix_s = 101.0; - CHECK(newer.bound() == kUncancelledBoundNs); - CHECK(newer.bound() > same.bound()); - // Growing the newer server's root fields changes nothing: the bound is the follower's full values. - newer.server.tracking.root_dispersion_s = 0.4; - newer.server.tracking.root_delay_s = 0.3; - CHECK(newer.bound() == kUncancelledBoundNs); - - // Unknown server update time: nothing cancels. - RootFieldPair unknown; - unknown.server.tracking.ref_time_unix_s = 0.0; - CHECK(unknown.bound() == kUncancelledBoundNs); - - // Unknown follower update time with a known server one: the server is newer, nothing cancels. - RootFieldPair no_follower_time; - no_follower_time.follower.tracking.ref_time_unix_s = 0.0; - no_follower_time.server.tracking.ref_time_unix_s = 100.0; - CHECK(no_follower_time.bound() == kUncancelledBoundNs); - - // The other terms are charged on top in both cases. + // The same through qualify_capture, with the other terms charged on top. Fixture f; - f.pc->tracking = same.follower.tracking; + f.pc->tracking.root_dispersion_s = 0.5; + f.pc->tracking.root_delay_s = 0.25; f.pc->tracking.system_time_s = 1e-4; - f.sender->tracking = same.server.tracking; f.sender->tracking.system_time_s = -2e-4; f.config.budget_ns = 10 * kSecond; - CHECK(within(f.run().terms.inter_host_ns, kCancelledBoundNs + 300'000, 2)); - f.sender->tracking.ref_time_unix_s = 101.0; - CHECK(within(f.run().terms.inter_host_ns, kUncancelledBoundNs + 300'000, 2)); -} - -void test_newer_server_root_growth_never_shrinks_the_bound() -{ - // Follower evidence fixed; only the server's root fields grow, with a newer update time. - RootFieldPair original; - original.server.tracking.ref_time_unix_s = 100.0; - const int64_t before = original.bound(); - - for (double growth : { 0.01, 0.1, 0.25, 0.4, 1.0 }) + const int64_t base = f.run().terms.inter_host_ns; + CHECK(within(base, kFollowerBoundNs + 300'000, 2)); + for (double v : { 0.0, 0.1, 0.5, 0.6, 5.0 }) { - RootFieldPair grown; - grown.server.tracking.ref_time_unix_s = 100.5; - grown.server.tracking.root_dispersion_s += growth; - grown.server.tracking.root_delay_s += growth; - CHECK(grown.bound() >= before); + f.sender->tracking.root_dispersion_s = v; + f.sender->tracking.root_delay_s = v; + f.sender->tracking.ref_time_unix_s = 100.0 + v; + f.pc->tracking.ref_time_unix_s = 100.0; + CHECK(f.run().terms.inter_host_ns == base); } - - // The same growth through qualify_capture. - Fixture f; - f.pc->tracking = original.follower.tracking; - f.sender->tracking = original.server.tracking; - f.config.budget_ns = 10 * kSecond; - const int64_t before_run = f.run().terms.inter_host_ns; - f.sender->tracking.ref_time_unix_s = 100.5; - f.sender->tracking.root_dispersion_s = 0.6; - f.sender->tracking.root_delay_s = 0.6; - CHECK(f.run().terms.inter_host_ns >= before_run); } void test_simple_rejections() @@ -468,6 +430,25 @@ void test_evidence_age_boundary() } } +void test_pc_clock_observation_age() +{ + const int64_t max_age = QualifierConfig{}.max_evidence_age_ns; + Fixture f; + f.config.budget_ns = 10 * kSecond; + // A fresh observation is used; the limit itself is inclusive, as for the chrony evidence. + f.clock.sample.monotonic_ns = f.now - max_age; + f.clock.sample.realtime_ns = f.clock.sample.monotonic_ns + kWallMinusMono; + CHECK(f.run().verdict == Verdict::Qualified); + f.clock.sample.monotonic_ns = f.now - max_age - 1; + f.clock.sample.realtime_ns = f.clock.sample.monotonic_ns + kWallMinusMono; + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::StaleEvidence); + CHECK(!q.usable()); + // An unusable clock is still reported as such, ahead of its age. + f.clock.usable = false; + CHECK(f.run().verdict == Verdict::NoLocalClock); +} + void test_unsynchronised() { { @@ -681,10 +662,14 @@ void test_overflowing_inputs_are_refused() } { Fixture f; - f.clock.sample.realtime_ns = INT64_MAX; // realtime - monotonic overflows - f.clock.sample.monotonic_ns = -5; + f.clock.sample.realtime_ns = INT64_MIN; // realtime - monotonic overflows CHECK(f.run().verdict == Verdict::ImplausibleAge); } + { + Fixture f; + f.clock.sample.monotonic_ns = INT64_MIN; // now - observed overflows: refused as stale, not wrapped + CHECK(f.run().verdict == Verdict::StaleEvidence); + } } void test_newer_generation_is_reported_for_the_caller_to_pin() @@ -859,16 +844,16 @@ int main() test_qualified_happy_path(); test_conversion_uses_pc_offset_not_sample_time(); test_inter_host_terms(); - test_upstream_cancels(); - test_pc_below_camera_host_clamps_to_zero(); + test_upstream_error_is_charged(); + test_pc_below_camera_host_is_still_charged_in_full(); test_copy_slack(); test_aging_charged_per_host(); test_both_system_times_charged(); test_inter_host_bound_direct(); - test_cancellation_requires_a_shared_upstream_sample(); - test_newer_server_root_growth_never_shrinks_the_bound(); + test_server_root_fields_and_ref_time_do_not_change_the_bound(); test_simple_rejections(); test_evidence_age_boundary(); + test_pc_clock_observation_age(); test_unsynchronised(); test_reference_matching(); test_reference_is_the_pcs_not_the_camera_hosts(); diff --git a/tests/cpp/core/timesync/chrony_monitor/test_chrony_monitor.cpp b/tests/cpp/core/timesync/chrony_monitor/test_chrony_monitor.cpp index 496391a355..988a814638 100644 --- a/tests/cpp/core/timesync/chrony_monitor/test_chrony_monitor.cpp +++ b/tests/cpp/core/timesync/chrony_monitor/test_chrony_monitor.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -318,6 +319,61 @@ void test_stop_during_blocking_runner() CHECK(Clock::now() - t0 < milliseconds(300 + 700)); } +void test_restart_after_stop() +{ + Fake f; + f.result = good(); + ChronyMonitorConfig cfg; + cfg.poll_interval = milliseconds(5); + ChronyMonitor m(cfg, f.runner(), f.now()); + for (int i = 0; i < 3; ++i) + { + const int before = f.calls.load(); + m.start(); + CHECK(wait_for_calls(f, before + 2, milliseconds(2000))); + m.stop(); + const int after_stop = f.calls.load(); + std::this_thread::sleep_for(milliseconds(30)); + CHECK(f.calls.load() == after_stop); // no worker left polling + } +} + +void test_concurrent_start_stop_never_hangs_or_leaks_a_worker() +{ + // Runs on a helper thread so a deadlock fails the test instead of hanging the run; a hung helper is detached. + auto body = [] + { + Fake f; + f.result = good(); + ChronyMonitorConfig cfg; + cfg.poll_interval = milliseconds(1); + ChronyMonitor m(cfg, f.runner(), f.now()); + for (int i = 0; i < 200; ++i) + { + std::thread starter([&m] { m.start(); }); + std::thread stopper([&m] { m.stop(); }); + starter.join(); + stopper.join(); + } + m.stop(); + const int after_stop = f.calls.load(); + std::this_thread::sleep_for(milliseconds(30)); + return f.calls.load() == after_stop; + }; + std::packaged_task task(body); + auto fut = task.get_future(); + std::thread t(std::move(task)); + if (fut.wait_for(std::chrono::seconds(30)) != std::future_status::ready) + { + std::fprintf(stderr, "concurrent start/stop hung\n"); + ++g_failures; + t.detach(); + return; + } + t.join(); + CHECK(fut.get()); // no worker still polling after the final stop +} + } // namespace int main() @@ -334,6 +390,8 @@ int main() test_stop_prompt_and_idempotent(); test_destructor_stops(); test_stop_during_blocking_runner(); + test_restart_after_stop(); + test_concurrent_start_stop_never_hangs_or_leaks_a_worker(); if (g_failures) { std::fprintf(stderr, "%d check(s) failed\n", g_failures); diff --git a/tests/cpp/core/timesync/command_runner/test_command_runner.cpp b/tests/cpp/core/timesync/command_runner/test_command_runner.cpp index 7693ac0009..7767263b87 100644 --- a/tests/cpp/core/timesync/command_runner/test_command_runner.cpp +++ b/tests/cpp/core/timesync/command_runner/test_command_runner.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -97,6 +98,41 @@ void test_stderr_discarded() CHECK(r.output == "out\n"); } +void test_closed_stdout_then_hang_is_killed_at_deadline() +{ + // The child closes stdout (EOF for the runner) and then hangs; the deadline must still apply. + const auto t0 = std::chrono::steady_clock::now(); + const auto r = run({ "/bin/sh", "-c", "exec 1>&-; sleep 30" }, milliseconds(300)); + const auto elapsed = std::chrono::steady_clock::now() - t0; + CHECK(r.started); + CHECK(r.timed_out); + CHECK(r.exit_code == -1); + CHECK(elapsed < std::chrono::seconds(3)); +} + +void test_normal_exit_code_reported() +{ + const auto r = run({ "/bin/sh", "-c", "exit 7" }); + CHECK(r.started); + CHECK(!r.timed_out); + CHECK(r.exit_code == 7); +} + +void test_ignored_sigchld_never_reports_success() +{ + // With SIGCHLD ignored the kernel auto-reaps, so waitpid fails with ECHILD and no status is known. A failing + // command must not come back as exit_code 0. + struct sigaction ign = {}; + ign.sa_handler = SIG_IGN; + sigemptyset(&ign.sa_mask); + struct sigaction old = {}; + CHECK(sigaction(SIGCHLD, &ign, &old) == 0); + const auto r = run({ "/bin/false" }); + sigaction(SIGCHLD, &old, nullptr); + CHECK(r.started); + CHECK(r.exit_code != 0); +} + } // namespace int main() @@ -109,6 +145,9 @@ int main() test_output_cap(); test_empty_argv(); test_stderr_discarded(); + test_closed_stdout_then_hang_is_killed_at_deadline(); + test_normal_exit_code_reported(); + test_ignored_sigchld_never_reports_success(); if (g_failures) { std::fprintf(stderr, "%d check(s) failed\n", g_failures); From f75f5b27023954bed8804cbbe57163f8fc818688 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Fri, 13 Nov 2026 18:51:11 -0500 Subject: [PATCH 08/10] timesync: state the integration boundary Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/core/timesync/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/core/timesync/README.md b/src/core/timesync/README.md index 48b5e3b82d..8b0b3dad71 100644 --- a/src/core/timesync/README.md +++ b/src/core/timesync/README.md @@ -16,6 +16,17 @@ build skips it. It builds without CUDA, LiveKit or Python, and on its own: cmake -S src/core/timesync -B build -DBUILD_TESTING=ON && cmake --build build && ctest --test-dir build ``` +## Integration boundary + +This module supplies reusable clock evidence and capture-time qualification. It does not publish anything: +publication is done by the consumer, currently the camera_livekit_bridge `chrony_clock` adapter. The module is +intended to remain the estimator behind the receiver backend's clock mapping. Reported uncertainty describes the +supported measurement model; it is not independent proof of physical timestamp accuracy. + +| Current path | Intended owner or replacement | Reuse, adapt or retire | Follow-up and landing condition | +| --- | --- | --- | --- | +| `src/core/timesync/` | The estimator behind the receiver backend's clock mapping | Reuse the chrony evidence handling and qualification; adapt the output to the mapping | Preserve qualified mappings and validity and uncertainty behaviour behind the destination interface. Lands when the receiver backend publishes the mapping. | + ## Pieces | Header | Role | From 41c1f6274fcbf11a54e484f2db3f400458c4e171 Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 12:05:52 -0500 Subject: [PATCH 09/10] timesync: apply the measured inter-host offset and stop calling it a settling fault The PC's follower only measures the camera host's clock and never adjusts the PC's, so a fixed offset between the wall clocks stays. qualify_capture now converts the sender's wall stamp by that offset (the PC's System time minus the camera host's, in chrony's sign: positive when the system clock is slow) and reports it separately from the uncertainty. The bound no longer charges the offset; it charges each report's frequency drift over its age with the existing dispersion, delay and skew terms. status no longer treats a large offset as still settling. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/core/timesync/README.md | 14 +- src/core/timesync/cpp/capture_qualifier.cpp | 33 ++++- .../cpp/inc/timesync/capture_qualifier.hpp | 18 ++- .../cpp/inc/timesync/chrony_tracking.hpp | 3 +- src/core/timesync/tools/test_timesync_tool.py | 24 ++-- src/core/timesync/tools/timesync_tool.py | 22 ++-- .../test_capture_qualifier.cpp | 120 ++++++++++++++---- 7 files changed, 174 insertions(+), 60 deletions(-) diff --git a/src/core/timesync/README.md b/src/core/timesync/README.md index 8b0b3dad71..56b93258fb 100644 --- a/src/core/timesync/README.md +++ b/src/core/timesync/README.md @@ -55,8 +55,10 @@ error (see "What the bound assumes"). A camera host with no upstream source is served as synchronised only if its chrony has a `local` reference; pass `configure-remote --offline` to write one. It is off by default, so a host that follows UTC keeps its configuration. -`status` is a readiness check on the PC's follower (synchronised, the camera host's reference id, root dispersion and -System time offset under `--max-dispersion-ms`); it does not state the per-frame bound. +`status` is a readiness check on the PC's follower (synchronised, the camera host's reference id and root dispersion under +`--max-dispersion-ms`, finite values). The measured offset to the camera host is reported, not limited: the follower +never adjusts the PC's clock, so it stays, and `qualify_capture` applies it to each capture. `status` does not state +the per-frame bound. ## Running and deploying it @@ -83,8 +85,12 @@ launch tools call this tool; they do not carry their own chrony setup. ## What the bound assumes -The inter-host term is both hosts' `System time`, the PC's full root dispersion and root delay / 2, a slack for the -camera host's dispersion growing since the PC copied it, and each report's skew times its age. +`qualify_capture` first moves the sender's wall stamp onto the PC's wall clock by the measured offset, the difference +of the two hosts' `System time` (chrony's NTP clock minus the system clock; in `chronyc -c` output a positive value means +the system clock is slow). The PC's NTP clock is the camera host's, so the offset is the PC's `System time` minus the +camera host's, and a fixed offset of either sign is converted correctly. The inter-host term bounds what is left: the +PC's full root dispersion and root delay / 2, a slack for the camera host's dispersion growing since the PC copied it, +and each report's skew and frequency times its age (how far the correction can have moved since the report). The camera host's own root values are not subtracted. They would cancel its upstream error only if both hosts' root fields described the same upstream sample, and chrony's report carries no identifier that shows it: `Ref time` is when diff --git a/src/core/timesync/cpp/capture_qualifier.cpp b/src/core/timesync/cpp/capture_qualifier.cpp index f6f46d310b..08e762d9da 100644 --- a/src/core/timesync/cpp/capture_qualifier.cpp +++ b/src/core/timesync/cpp/capture_qualifier.cpp @@ -63,6 +63,21 @@ bool source_quiet(const ChronyEvidence& e, int64_t max_age_ns, int64_t future_to return !(age_s <= static_cast(max_age_ns) * 1e-9 && age_s >= -lead_s); // also true for a NaN } +// Sender CLOCK_REALTIME minus PC CLOCK_REALTIME, from the two hosts' reports. chrony's CSV `System time` is the NTP +// clock minus the system clock (positive when the system clock is slow). The PC follows the camera host, so the two NTP +// clocks agree: PC_sys + system_time(PC) = sender_sys + system_time(sender). False when the value is not finite or +// does not fit. +bool inter_host_offset_ns(const ChronyEvidence& follower, const ChronyEvidence& server, int64_t& offset_ns) +{ + const double ns = (follower.tracking.system_time_s - server.tracking.system_time_s) * 1e9; + if (!std::isfinite(ns) || std::fabs(ns) >= static_cast(kSaturated)) + { + return false; + } + offset_ns = static_cast(std::llround(ns)); + return true; +} + } // namespace const char* to_string(Verdict v) @@ -120,9 +135,12 @@ int64_t inter_host_bound_ns(const ChronyEvidence& follower, const double copy_slack_s = p.skew_ppm * 1e-6 * s.update_interval_s; const double follower_age_s = static_cast(std::max(follower_age_ns, 0)) * 1e-9; const double server_age_s = static_cast(std::max(server_age_ns, 0)) * 1e-9; - const double bound_s = std::fabs(s.system_time_s) + std::fabs(p.system_time_s) + link_dispersion_s + - 0.5 * link_delay_s + copy_slack_s + s.skew_ppm * 1e-6 * follower_age_s + - p.skew_ppm * 1e-6 * server_age_s; + // The hosts' System time offsets are applied as a correction (see qualify_capture) and are not part of the bound. + // What remains is how far each correction can have moved since its report: the system clock's own rate error + // times the report's age, charged by magnitude. + const double bound_s = link_dispersion_s + 0.5 * link_delay_s + copy_slack_s + s.skew_ppm * 1e-6 * follower_age_s + + p.skew_ppm * 1e-6 * server_age_s + std::fabs(s.frequency_ppm) * 1e-6 * follower_age_s + + std::fabs(p.frequency_ppm) * 1e-6 * server_age_s; return to_ns_saturating(bound_s); } @@ -187,9 +205,14 @@ Qualification qualify_capture(const SenderCapture& capture, q.generation = pc_clock.generation; int64_t offset = 0; + int64_t capture_pc_realtime = 0; int64_t capture_age = 0; - if (__builtin_sub_overflow(pc_clock.sample.realtime_ns, pc_clock.sample.monotonic_ns, &offset) || - __builtin_sub_overflow(capture.realtime_ns, offset, &q.capture_monotonic_ns) || + // The sender's wall time is moved onto the PC's wall clock by the measured inter-host offset, then onto the PC's + // monotonic clock by the PC's own realtime-minus-monotonic offset. + if (!inter_host_offset_ns(follower, server, q.inter_host_offset_ns) || + __builtin_sub_overflow(pc_clock.sample.realtime_ns, pc_clock.sample.monotonic_ns, &offset) || + __builtin_sub_overflow(capture.realtime_ns, q.inter_host_offset_ns, &capture_pc_realtime) || + __builtin_sub_overflow(capture_pc_realtime, offset, &q.capture_monotonic_ns) || __builtin_sub_overflow(now_monotonic_ns, q.capture_monotonic_ns, &capture_age) || capture_age < -std::max(config.future_tolerance_ns, 0) || capture_age > config.max_capture_age_ns) { diff --git a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp index 36f1849eca..8b20b3013e 100644 --- a/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp +++ b/src/core/timesync/cpp/inc/timesync/capture_qualifier.hpp @@ -86,6 +86,9 @@ struct Qualification /// it met the budget. int64_t uncertainty_ns = 0; uint64_t generation = 0; ///< The PC local-clock generation the conversion used. + /// The correction applied: sender CLOCK_REALTIME minus PC CLOCK_REALTIME, from the two hosts' `System time`. + /// It is a measured offset, not an uncertainty, and may be large; `uncertainty_ns` bounds what is left after it. + int64_t inter_host_offset_ns = 0; UncertaintyTerms terms; /// Whether `capture_monotonic_ns` is a converted capture instant with a stated bound. `OverBudget` is usable: every @@ -104,8 +107,9 @@ struct Qualification /// understates, and saturating. int64_t chrony_error_bound_ns(const ChronyTracking& t, int64_t age_ns); -/// The bound between a following host's wall clock and the host it follows, as `qualify_capture` charges it (see -/// below). The ages are of each report, on the PC's monotonic clock. Rounded up and saturating. +/// The residual bound between a following host's wall clock and the host it follows, once `qualify_capture` has +/// applied their measured offset (see below). The ages are of each report, on the PC's monotonic clock. Rounded up and +/// saturating. int64_t inter_host_bound_ns(const ChronyEvidence& follower, const ChronyEvidence& server, int64_t follower_age_ns, @@ -114,10 +118,12 @@ int64_t inter_host_bound_ns(const ChronyEvidence& follower, /// Converts a sender realtime capture stamp to PC monotonic and says how well the conversion can be vouched for: not /// at all (a refusal), vouched for but over the budget, or within it. /// -/// The inter-host term bounds sender realtime against PC realtime along camera host system clock -> camera host -/// NTP clock -> PC chronyd's measurement -> PC system clock: both hosts' |System time|, the PC's full Root dispersion -/// and Root delay / 2, a slack for the camera host's dispersion growing since the PC copied it, and each report's -/// skew times its age. The camera host's own root values are not subtracted: nothing in chrony's report shows that its +/// The sender's realtime stamp is first moved onto the PC's realtime clock by the measured offset along camera host +/// system clock -> camera host NTP clock -> PC chronyd's measurement -> PC system clock, which is the difference of the +/// two hosts' `System time` (a measuring-only follower never removes it, and a fixed offset of either sign is +/// converted correctly). The inter-host term bounds what is left: the PC's full Root dispersion and Root delay / 2, a +/// slack for the camera host's dispersion growing since the PC copied it, and each report's skew and frequency times +/// its age. The camera host's own root values are not subtracted: nothing in chrony's report shows that its /// root fields and the PC's copy describe the same upstream sample (`Ref time` is each host's own), and a subtraction /// without that correspondence can shrink the bound with no new measurement of the link. The charge therefore /// includes the camera host's upstream error and is larger than the link alone, never smaller. The assumption is diff --git a/src/core/timesync/cpp/inc/timesync/chrony_tracking.hpp b/src/core/timesync/cpp/inc/timesync/chrony_tracking.hpp index 49b25f43c4..8fb989c0e6 100644 --- a/src/core/timesync/cpp/inc/timesync/chrony_tracking.hpp +++ b/src/core/timesync/cpp/inc/timesync/chrony_tracking.hpp @@ -17,7 +17,8 @@ struct ChronyTracking std::string reference_name; int stratum = 0; double ref_time_unix_s = 0.0; - double system_time_s = 0.0; ///< Signed; only the magnitude is used, so the sign convention does not matter. + double system_time_s = 0.0; ///< chrony's NTP clock minus the system clock, signed: positive when the system clock + ///< is slow. double last_offset_s = 0.0; double rms_offset_s = 0.0; double frequency_ppm = 0.0; diff --git a/src/core/timesync/tools/test_timesync_tool.py b/src/core/timesync/tools/test_timesync_tool.py index 9e5869b9a6..7ab03927e6 100644 --- a/src/core/timesync/tools/test_timesync_tool.py +++ b/src/core/timesync/tools/test_timesync_tool.py @@ -207,18 +207,20 @@ def test_high_dispersion(self) -> None: self.assertEqual(len(problems), 1) self.assertIn("dispersion", problems[0]) - def test_large_system_time_is_not_ready(self) -> None: - problems = self.judge(make_tracking(root_dispersion=0.0003, system_time=0.050)) - self.assertEqual(len(problems), 1) - self.assertIn("System time", problems[0]) - # The sign does not matter. - problems = self.judge(make_tracking(root_dispersion=0.0003, system_time=-0.050)) - self.assertEqual(len(problems), 1) - self.assertIn("System time", problems[0]) + def test_a_large_measured_offset_of_either_sign_is_ready(self) -> None: + # A measuring-only follower never removes the offset; it is applied per frame, so it is not a readiness fault. + for offset in (0.100, -0.100, 0.0005): + self.assertEqual( + self.judge(make_tracking(root_dispersion=0.0003, system_time=offset)), + [], + ) - def test_small_system_time_stays_ready(self) -> None: - self.assertEqual(self.judge(make_tracking(system_time=0.0005)), []) - self.assertEqual(self.judge(make_tracking(system_time=-0.0005)), []) + def test_non_finite_values_are_not_ready(self) -> None: + for field in ("system_time", "root_dispersion", "root_delay"): + problems = self.judge(make_tracking(**{field: float("nan")})) + self.assertTrue( + any("non-finite" in p or "dispersion" in p for p in problems) + ) def test_problems_accumulate(self) -> None: bad = make_tracking( diff --git a/src/core/timesync/tools/timesync_tool.py b/src/core/timesync/tools/timesync_tool.py index 6e22a0ce25..c110f309a5 100755 --- a/src/core/timesync/tools/timesync_tool.py +++ b/src/core/timesync/tools/timesync_tool.py @@ -19,6 +19,7 @@ from __future__ import annotations import argparse +import math import shlex import shutil import signal @@ -181,10 +182,11 @@ def chronyc_tracking( def judge(local: Tracking, expected_id: str, max_dispersion_s: float) -> list[str]: """What keeps this machine's measuring instance from being usable, one message each; empty when it is usable. - The same limit applies to the root dispersion and to the size of the System time offset, so a follower that reports - a small dispersion while its clock is tens of milliseconds off is not called ready. This is a readiness check on - the follower only: it does not state the inter-host bound, which the capture qualifier computes per frame from both - hosts' reports. + The follower only measures the camera host's clock; it never adjusts this machine's, so a settled follower reports + the whole fixed offset as its System time. That offset is a measurement the capture qualifier applies, not a fault, + and is not limited here; readiness requires a synchronised follower of the expected reference whose root dispersion + is small and whose reported values are finite. This is a readiness check on the follower only: it does not state the + inter-host bound, which the capture qualifier computes per frame from both hosts' reports. """ problems = [] if not local.synchronised: @@ -201,11 +203,11 @@ def judge(local: Tracking, expected_id: str, max_dispersion_s: float) -> list[st f"this machine's root dispersion is {local.root_dispersion * 1e3:.1f} ms, " f"above {max_dispersion_s * 1e3:.1f} ms; it is still settling" ) - if abs(local.system_time) > max_dispersion_s: - problems.append( - f"this machine's System time offset is {abs(local.system_time) * 1e3:.1f} ms, " - f"above {max_dispersion_s * 1e3:.1f} ms; it is still settling" - ) + if not all( + math.isfinite(value) + for value in (local.system_time, local.root_dispersion, local.root_delay) + ): + problems.append("this machine's tracking report has a non-finite value") return problems @@ -338,7 +340,7 @@ def cmd_status(args: argparse.Namespace) -> int: print( f"ok: this machine follows {local.reference_id} (stratum {local.stratum}), " f"root delay {local.root_delay * 1e3:.3f} ms, root dispersion {local.root_dispersion * 1e3:.3f} ms, " - f"System time {abs(local.system_time) * 1e6:.1f} us" + f"measured offset to the camera host {local.system_time * 1e3:+.3f} ms (applied per frame)" ) return 0 if time.monotonic() >= deadline: diff --git a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp index b30afa881a..7abe32c34b 100644 --- a/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp +++ b/tests/cpp/core/timesync/capture_qualifier/test_capture_qualifier.cpp @@ -5,10 +5,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -193,11 +195,12 @@ void test_inter_host_terms() f.sender->tracking.skew_ppm = 5.0; f.config.budget_ns = 10 * kMs; const Qualification q = f.run(); - // Both reports are one second old, follower (PC) update interval 0. |system time|: 100 us + 200 us; the PC's - // full root dispersion 200 us; its root delay / 2 = 200 us; skew * age: 10 us + 5 us. + // Both reports are one second old, follower (PC) update interval 0. The System time offsets are applied as a + // correction and are not charged: the PC's full root dispersion 200 us; its root delay / 2 = 200 us; skew * age: + // 10 us + 5 us. CHECK(q.verdict == Verdict::Qualified); - CHECK(within(q.terms.inter_host_ns, 715'000, 2)); - CHECK(q.terms.inter_host_ns >= old_cancelled_bound_ns(f.pc->tracking, f.sender->tracking) + 300'000); + CHECK(within(q.terms.inter_host_ns, 415'000, 2)); + CHECK(q.terms.inter_host_ns >= old_cancelled_bound_ns(f.pc->tracking, f.sender->tracking)); CHECK(q.uncertainty_ns == q.terms.sensor_ns + q.terms.inter_host_ns + q.terms.pc_local_ns); } @@ -278,24 +281,88 @@ void test_aging_charged_per_host() CHECK(f.run().terms.inter_host_ns == 10'000); } -void test_both_system_times_charged() +void test_system_time_is_corrected_not_charged() +{ + ChronyEvidence s = zero_evidence(); + ChronyEvidence p = zero_evidence(); + s.tracking.system_time_s = 0.1; + p.tracking.system_time_s = -0.25; + CHECK(inter_host_bound_ns(s, p, kSecond, kSecond) == 0); +} + +void test_frequency_charged_by_age_per_host() { ChronyEvidence s = zero_evidence(); ChronyEvidence p = zero_evidence(); - s.tracking.system_time_s = 100e-6; - CHECK(within(inter_host_bound_ns(s, p, 0, 0), 100'000, 1)); - s.tracking.system_time_s = 0.0; - p.tracking.system_time_s = 250e-6; - CHECK(within(inter_host_bound_ns(s, p, 0, 0), 250'000, 1)); - s.tracking.system_time_s = 100e-6; - CHECK(within(inter_host_bound_ns(s, p, 0, 0), 350'000, 1)); - // The sign does not matter: only the magnitude is charged. - s.tracking.system_time_s = -100e-6; - p.tracking.system_time_s = -250e-6; - CHECK(within(inter_host_bound_ns(s, p, 0, 0), 350'000, 1)); - s.tracking.system_time_s = 100e-6; - p.tracking.system_time_s = -250e-6; - CHECK(within(inter_host_bound_ns(s, p, 0, 0), 350'000, 1)); + s.tracking.frequency_ppm = -20.0; + p.tracking.frequency_ppm = 5.0; + // The correction can have moved by the rate error times the report's age; the sign does not matter. + CHECK(inter_host_bound_ns(s, p, 2 * kSecond, 4 * kSecond) == 40'000 + 20'000); +} + +// A camera wall clock off from the PC's by a known amount, with the PC's measuring-only follower reporting that +// offset as its System time: the capture is converted by the measured correction and the offset is not uncertainty. +void test_fixed_offset_of_either_sign_is_corrected() +{ + for (const int64_t ahead_ns : { 100 * kMs, -100 * kMs }) + { + Fixture f; + // The camera clock reads `ahead_ns` more than the PC's; the real capture age is kCaptureAge. + f.capture.realtime_ns = kWallMinusMono + (kNow - kCaptureAge) + ahead_ns; + // The PC's NTP clock (the camera host's) is ahead of the PC system clock by ahead_ns: PC system time is slow. + f.pc->tracking.system_time_s = static_cast(ahead_ns) * 1e-9; + f.pc->tracking.root_dispersion_s = 50e-6; + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.capture_monotonic_ns == kNow - kCaptureAge); + CHECK(q.inter_host_offset_ns == ahead_ns); + CHECK(within(q.terms.inter_host_ns, 50'000, 2)); // the offset is not charged as uncertainty + } +} + +void test_offset_without_correction_would_be_implausible() +{ + // The same camera-ahead capture with no measured offset is refused: the conversion has nothing to correct with. + Fixture f; + f.capture.realtime_ns = kWallMinusMono + (kNow - kCaptureAge) + 100 * kMs; + CHECK(f.run().verdict == Verdict::ImplausibleAge); +} + +void test_camera_host_own_system_time_is_part_of_the_offset() +{ + // The camera host's system clock is itself 30 ms slow of the NTP clock it serves, so its wall stamps trail that + // clock. + Fixture f; + f.sender->tracking.system_time_s = 0.030; + f.pc->tracking.system_time_s = 0.0; // the PC's system clock agrees with the served NTP clock + f.capture.realtime_ns = kWallMinusMono + (kNow - kCaptureAge) - 30 * kMs; + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::Qualified); + CHECK(q.capture_monotonic_ns == kNow - kCaptureAge); + CHECK(q.inter_host_offset_ns == -30 * kMs); +} + +void test_unrepresentable_or_nonfinite_offset_is_refused() +{ + Fixture f; + f.pc->tracking.system_time_s = std::numeric_limits::infinity(); + CHECK(f.run().verdict == Verdict::ImplausibleAge); + f.pc->tracking.system_time_s = std::nan(""); + CHECK(f.run().verdict == Verdict::ImplausibleAge); + f.pc->tracking.system_time_s = 1e12; + CHECK(f.run().verdict == Verdict::ImplausibleAge); +} + +void test_large_correct_offset_with_high_uncertainty_is_over_budget_not_invalid() +{ + Fixture f; + f.capture.realtime_ns = kWallMinusMono + (kNow - kCaptureAge) + 100 * kMs; + f.pc->tracking.system_time_s = 0.1; + f.pc->tracking.root_dispersion_s = 0.02; + const Qualification q = f.run(); + CHECK(q.verdict == Verdict::OverBudget); + CHECK(q.usable()); + CHECK(q.capture_monotonic_ns == kNow - kCaptureAge); } void test_inter_host_bound_direct() @@ -330,9 +397,9 @@ void test_inter_host_bound_direct() // Rounds up and saturates. ChronyEvidence s4 = zero_evidence(); - s4.tracking.system_time_s = 1.5e-9; + s4.tracking.root_dispersion_s = 1.5e-9; CHECK(inter_host_bound_ns(s4, zero_evidence(), 0, 0) == 2); - s4.tracking.system_time_s = 1e12; + s4.tracking.root_dispersion_s = 1e12; CHECK(inter_host_bound_ns(s4, zero_evidence(), 0, 0) == INT64_MAX / 4); // the saturation value } @@ -381,7 +448,8 @@ void test_server_root_fields_and_ref_time_do_not_change_the_bound() f.sender->tracking.system_time_s = -2e-4; f.config.budget_ns = 10 * kSecond; const int64_t base = f.run().terms.inter_host_ns; - CHECK(within(base, kFollowerBoundNs + 300'000, 2)); + // The System time offsets are corrected, not charged. + CHECK(within(base, kFollowerBoundNs, 2)); for (double v : { 0.0, 0.1, 0.5, 0.6, 5.0 }) { f.sender->tracking.root_dispersion_s = v; @@ -848,7 +916,13 @@ int main() test_pc_below_camera_host_is_still_charged_in_full(); test_copy_slack(); test_aging_charged_per_host(); - test_both_system_times_charged(); + test_system_time_is_corrected_not_charged(); + test_frequency_charged_by_age_per_host(); + test_fixed_offset_of_either_sign_is_corrected(); + test_offset_without_correction_would_be_implausible(); + test_camera_host_own_system_time_is_part_of_the_offset(); + test_unrepresentable_or_nonfinite_offset_is_refused(); + test_large_correct_offset_with_high_uncertainty_is_over_budget_not_invalid(); test_inter_host_bound_direct(); test_server_root_fields_and_ref_time_do_not_change_the_bound(); test_simple_rejections(); From abe8f87b047e54fd241b011e45b4a5ef26e9153d Mon Sep 17 00:00:00 2001 From: Gareth Morgan Date: Mon, 16 Nov 2026 14:29:46 -0500 Subject: [PATCH 10/10] timesync: say that the PC's follower measures the offset and the library applies it Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: Gareth Morgan --- src/core/timesync/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/timesync/README.md b/src/core/timesync/README.md index 56b93258fb..6a70056959 100644 --- a/src/core/timesync/README.md +++ b/src/core/timesync/README.md @@ -6,8 +6,10 @@ SPDX-License-Identifier: Apache-2.0 # timesync Converts a camera host's wall-clock capture time to the PC's monotonic clock and decides whether -the result may be recorded. The two hosts keep their wall clocks aligned with `chronyd`; this -library reads chrony's own report of how well it is doing and refuses a frame it cannot vouch for. +the result may be recorded. The PC runs a measuring-only `chronyd` that follows the camera host and never adjusts +the PC's clock, so the two wall clocks keep their offset; the library applies that measured offset (the difference +of the two hosts' `System time`) to each capture time. It reads chrony's own report of how well it is doing and +refuses a frame it cannot vouch for. Target `timesync::timesync` (consumers include ``). POSIX only, so the Windows source build skips it. It builds without CUDA, LiveKit or Python, and on its own: