From 5856c4c961d3a2bf07739ee274641f22b2accd0b Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 12:16:48 -0800 Subject: [PATCH 01/16] feat(camera): add camera_core, the slot and window core of the camera capture plugin First step of the unified camera capture plugin (milestone M0). camera_core tracks which pool slots hold which frame sets and who may still read them. It uses the standard library only and never touches slot memory, so it builds and is tested on every platform. - A group is one or more hardware-synchronised streams published as one frame set. - A reader subscribes with a window (frame sets kept for it at a time) and a delivery order, newest or in order. - A subscription is accepted only when the pool covers it: slots >= ring history + sum of reader windows + backend reserve. - A backend either takes a free slot and writes it, or hands over a slot the camera SDK filled and is told the moment nobody holds it any more. - Every frame event carries its timeline, the group's generation and the number of frame sets the reader skipped. Tests: tests/cpp/core/camera_core (Catch2), run with `ctest -L camera_core`. Signed-off-by: Farbod Motlagh Co-Authored-By: Claude --- src/core/CMakeLists.txt | 3 + src/core/camera_core/CMakeLists.txt | 6 + src/core/camera_core/cpp/CMakeLists.txt | 16 + .../cpp/inc/camera_core/publisher.hpp | 222 ++++++++ .../camera_core/cpp/inc/camera_core/types.hpp | 116 ++++ src/core/camera_core/cpp/publisher.cpp | 445 +++++++++++++++ tests/cpp/core/CMakeLists.txt | 1 + tests/cpp/core/camera_core/CMakeLists.txt | 16 + tests/cpp/core/camera_core/test_publisher.cpp | 510 ++++++++++++++++++ 9 files changed, 1335 insertions(+) create mode 100644 src/core/camera_core/CMakeLists.txt create mode 100644 src/core/camera_core/cpp/CMakeLists.txt create mode 100644 src/core/camera_core/cpp/inc/camera_core/publisher.hpp create mode 100644 src/core/camera_core/cpp/inc/camera_core/types.hpp create mode 100644 src/core/camera_core/cpp/publisher.cpp create mode 100644 tests/cpp/core/camera_core/CMakeLists.txt create mode 100644 tests/cpp/core/camera_core/test_publisher.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 9cc4772373..3393c107b0 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -19,6 +19,9 @@ add_subdirectory(oxr_utils) # Build Plugin Manager add_subdirectory(plugin_manager) +# Camera capture core (standard library only) +add_subdirectory(camera_core) + # Build OXR library (depends on oxr_utils for types) add_subdirectory(oxr) diff --git a/src/core/camera_core/CMakeLists.txt b/src/core/camera_core/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_core/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_core/cpp/CMakeLists.txt b/src/core/camera_core/cpp/CMakeLists.txt new file mode 100644 index 0000000000..0728a6df2c --- /dev/null +++ b/src/core/camera_core/cpp/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Camera capture core: which pool slots hold which frame sets and who may still read them. +# Standard library only, so it builds and is tested on every platform. +add_library(camera_core STATIC + publisher.cpp +) + +target_include_directories(camera_core INTERFACE + $ +) + +add_library(camera::camera_core ALIAS camera_core) diff --git a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp new file mode 100644 index 0000000000..3b7ee5dd87 --- /dev/null +++ b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp @@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +using GroupId = std::uint32_t; +using SubscriptionId = std::uint32_t; // 0 is never a valid subscription + +/** + * @brief Where a reader's frame events go: a socket connection or an in-process queue. + */ +class EventSink +{ +public: + virtual ~EventSink() = default; + + /** + * @brief Hand one event to the reader. Called on the capture path with the publisher's + * lock held, so it must not block and must not call back into the publisher. + * @return false when the event could not be handed over; the reader then gets a later one. + */ + virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) = 0; +}; + +struct PublisherStats +{ + std::uint64_t published = 0; // frame sets published + std::uint64_t events_sent = 0; + std::uint64_t send_failures = 0; + std::uint64_t no_free_slot = 0; // acquire_slot calls that found every slot in use +}; + +struct SubscribeResult +{ + SubscriptionId id = 0; + std::string error; // why the subscription was refused, when id is 0 + + explicit operator bool() const + { + return id != 0; + } +}; + +/** + * @brief Tracks which pool slots hold which frame sets and who may still read them. + * + * A group keeps a short ring of recent frame sets; a reader's window is how many are kept for + * it at a time, unread and held together. A subscription is accepted only when + * slots >= ring history + sum of reader windows + backend reserve. Events carry slot indices; + * slot memory is never touched here. Thread-safe; no call blocks on a reader. + */ +class Publisher +{ +public: + using Clock = std::function; + using SlotFreeCallback = std::function; + + /** + * @param clock Host monotonic time in nanoseconds; defaults to std::chrono::steady_clock. + */ + explicit Publisher(Clock clock = {}); + + // ---- backend side ---------------------------------------------------------------------- + + /** + * @brief Declare a group and the size of its pools. + * @param backend_reserve Slots the backend needs beyond the ring and the readers: one to + * write ahead, more for an SDK that keeps several captures in flight in its own queue. + * @throws std::invalid_argument if the description is not usable. + */ + GroupId add_group(const GroupDescription& description, std::uint32_t backend_reserve = 1); + + /** + * @brief Take a free slot to write the next image into. + * @return The slot, or nothing when every slot is in use (the frame should be dropped). + */ + std::optional acquire_slot(GroupId group, std::size_t stream); + + /** + * @brief Mark a slot the SDK chose and filled, for a pool the SDK owns. + * @throws std::invalid_argument if the slot is still held by the ring or a reader. + */ + void claim_slot(GroupId group, std::size_t stream, std::uint32_t slot); + + /** + * @brief Give back a slot taken with acquire_slot or claim_slot without publishing it. + */ + void abandon_slot(GroupId group, std::size_t stream, std::uint32_t slot); + + /** + * @brief Be told when a published slot is no longer held by the ring or any reader, so a + * buffer the SDK owns can go back to it at once. Called outside the publisher's lock, on + * the thread whose call released the last hold. + */ + void set_slot_free_callback(GroupId group, SlotFreeCallback callback); + + /** + * @brief Publish one frame set. images[i] belongs to stream i and names a slot taken with + * acquire_slot or claim_slot. + * @param timeline capture_ns and sdk_ready_ns from the backend; the rest is filled in here. + * @return The frame set's sequence number. + * @throws std::invalid_argument if the images do not match the group. + */ + std::uint64_t publish(GroupId group, std::span images, const FrameTimeline& timeline); + + /** + * @brief Tell readers the camera restarted: the generation changes and the ring is emptied. + * Frame sets readers already hold stay valid until released. + */ + void restart_group(GroupId group); + + // ---- reader side ----------------------------------------------------------------------- + + /** + * @brief Start sending a group's frame events to a sink. + * @return The subscription, or the reason it does not fit the pool. + */ + SubscribeResult subscribe(EventSink& sink, GroupId group, const ReaderOptions& options); + + /** + * @brief Return a frame set. The reader's next one, if any is waiting, is sent at once. + */ + void release(SubscriptionId subscription, std::uint64_t lease_id); + + /** + * @brief End a subscription and take back every frame set it still holds. + */ + void unsubscribe(SubscriptionId subscription); + + // ---- description ----------------------------------------------------------------------- + + std::size_t group_count() const; + GroupDescription group_description(GroupId group) const; + std::optional find_group(std::string_view id) const; + std::uint32_t generation(GroupId group) const; + PublisherStats stats() const; + +private: + struct FrameSet + { + std::uint64_t sequence = 0; + std::uint32_t generation = 0; + FrameTimeline timeline; + std::array images; + }; + + struct Subscription + { + GroupId group = 0; + std::uint32_t window = 1; + DeliveryOrder order = DeliveryOrder::NEWEST; + EventSink* sink = nullptr; + std::uint64_t cursor = 0; // sequence of the last frame set sent to this reader + bool started = false; + std::map leases; // sent and not yet released + }; + + struct Slot + { + std::uint32_t holds = 0; // the ring and every reader lease that names this slot + bool writing = false; // taken by the backend and not yet published + }; + + struct Group + { + GroupDescription description; + std::uint32_t reserve = 1; + std::uint32_t slot_count = 0; // the smallest pool of the group + std::uint32_t generation = 0; + std::uint64_t sequence = 0; + std::vector> slots; // per stream + std::deque ring; + std::vector subscriptions; + std::shared_ptr on_free; + }; + + struct FreedSlot + { + std::shared_ptr callback; + std::size_t stream = 0; + std::uint32_t slot = 0; + }; + + Group& group_at(GroupId group); + const Group& group_at(GroupId group) const; + static Slot& slot_at(Group& group, std::size_t stream, std::uint32_t slot); + std::size_t history_of(const Group& group) const; + void hold(Group& group, const FrameSet& set); + void unhold(Group& group, const FrameSet& set, std::vector& freed); + void trim_ring(Group& group, std::vector& freed); + void deliver(Group& group, SubscriptionId id, Subscription& subscription, std::vector& freed); + static void notify(const std::vector& freed); + + Clock m_clock; + mutable std::mutex m_mutex; + std::vector> m_groups; + std::map m_subscriptions; + SubscriptionId m_next_subscription = 1; + std::uint64_t m_next_lease = 1; + PublisherStats m_stats; +}; + +} // namespace core::camera diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp new file mode 100644 index 0000000000..8e919e35ec --- /dev/null +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// A group publishes one image per stream, all captured together. +inline constexpr std::size_t MAX_STREAMS_PER_GROUP = 8; +inline constexpr std::uint32_t MAX_SLOTS_PER_STREAM = 64; + +// Marks a time that is not known. All times are nanoseconds on the host's monotonic clock. +inline constexpr std::int64_t TIME_UNKNOWN = -1; + +enum class PixelFormat : std::uint32_t +{ + BGRA8 = 1, + RGBA8 = 2, + NV12 = 3, + GRAY8 = 4, + // Encoded: a slot holds one access unit instead of pixels. + H264 = 16, + H265 = 17, +}; + +constexpr bool is_encoded(PixelFormat format) +{ + return format == PixelFormat::H264 || format == PixelFormat::H265; +} + +// How a slot's memory reaches another process. The core tracks slots by index only and never +// touches their memory, so a new kind needs no change here. +enum class MemoryKind : std::uint32_t +{ + HOST = 1, // shared host memory, passed as a file descriptor + CUDA = 2, // CUDA memory on a discrete GPU + DMA_BUF = 3, // Jetson buffer, passed as a file descriptor + NVSCI_BUF = 4, // reserved for SIPL; nothing implements it yet +}; + +struct StreamDescription +{ + std::string id; + PixelFormat format = PixelFormat::BGRA8; + MemoryKind memory = MemoryKind::HOST; + std::uint32_t width = 0; + std::uint32_t height = 0; + std::uint32_t stride_bytes = 0; // row pitch of the first plane; 0 for an encoded stream + std::uint64_t slot_bytes = 0; // capacity of one slot + std::uint32_t slot_count = 0; + double frame_rate = 0.0; // configured or measured, not the sensor mode's maximum +}; + +// One or more hardware-synchronised streams that are published as one frame set. +struct GroupDescription +{ + std::string id; + std::vector streams; +}; + +enum ImageFlag : std::uint32_t +{ + IMAGE_FLAG_KEYFRAME = 1u << 0, // encoded stream: this access unit decodes on its own + IMAGE_FLAG_SDK_WARNING = 1u << 1, // the camera SDK delivered the frame but flagged it as suspect +}; + +struct ImageInfo +{ + std::uint32_t slot = 0; + std::uint32_t flags = 0; // ImageFlag bits + std::uint64_t bytes_used = 0; // encoded stream: size of the access unit + std::uint64_t device_frame_id = 0; // the SDK's own frame number + std::int64_t device_time_ns = TIME_UNKNOWN; // the SDK's own timestamp, in the SDK's clock +}; + +// Where a frame set was at each step, so latency can be read off every frame. +struct FrameTimeline +{ + std::int64_t capture_ns = TIME_UNKNOWN; // sensor capture, when the backend can map it to the host clock + std::int64_t sdk_ready_ns = TIME_UNKNOWN; // the SDK handed the frame to the backend + std::int64_t published_ns = TIME_UNKNOWN; // set by Publisher::publish + std::int64_t sent_ns = TIME_UNKNOWN; // set when the event is handed to a reader +}; + +struct FrameEvent +{ + std::uint64_t lease_id = 0; // return it with Publisher::release + std::uint64_t sequence = 0; // counts frame sets of the group; never restarts + std::uint32_t generation = 0; // changes when the camera restarts + std::uint32_t skipped = 0; // frame sets this reader did not receive since its previous one + std::uint32_t image_count = 0; + FrameTimeline timeline; + std::array images; // images[i] belongs to stream i +}; + +enum class DeliveryOrder +{ + DEFAULT, // NEWEST for a window of one, IN_ORDER otherwise + NEWEST, // each time the reader has room it gets the newest frame set and skips older ones + IN_ORDER, // the reader gets every frame set in sequence while it keeps up +}; + +struct ReaderOptions +{ + std::uint32_t window = 1; // frame sets kept for this reader at a time, unread and held together + DeliveryOrder order = DeliveryOrder::DEFAULT; +}; + +} // namespace core::camera diff --git a/src/core/camera_core/cpp/publisher.cpp b/src/core/camera_core/cpp/publisher.cpp new file mode 100644 index 0000000000..5cb3c8562c --- /dev/null +++ b/src/core/camera_core/cpp/publisher.cpp @@ -0,0 +1,445 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_core/publisher.hpp" + +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +std::int64_t steady_now_ns() +{ + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +DeliveryOrder resolve_order(const ReaderOptions& options) +{ + if (options.order != DeliveryOrder::DEFAULT) + { + return options.order; + } + return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; +} + +} // namespace + +Publisher::Publisher(Clock clock) : m_clock(clock ? std::move(clock) : Clock(steady_now_ns)) +{ +} + +GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t backend_reserve) +{ + if (description.id.empty()) + { + throw std::invalid_argument("camera group needs an id"); + } + if (description.streams.empty() || description.streams.size() > MAX_STREAMS_PER_GROUP) + { + throw std::invalid_argument("camera group '" + description.id + "' needs between 1 and " + + std::to_string(MAX_STREAMS_PER_GROUP) + " streams"); + } + if (backend_reserve == 0) + { + throw std::invalid_argument("camera group '" + description.id + "' needs a backend reserve of at least 1"); + } + + auto group = std::make_unique(); + group->description = description; + group->reserve = backend_reserve; + group->slot_count = std::numeric_limits::max(); + for (const StreamDescription& stream : description.streams) + { + if (stream.id.empty() || stream.slot_bytes == 0 || stream.slot_count == 0 || + stream.slot_count > MAX_SLOTS_PER_STREAM) + { + throw std::invalid_argument("camera group '" + description.id + "' has a stream without an id, a slot " + + "size or between 1 and " + std::to_string(MAX_SLOTS_PER_STREAM) + " slots"); + } + group->slot_count = std::min(group->slot_count, stream.slot_count); + group->slots.emplace_back(stream.slot_count); + } + + std::lock_guard lock(m_mutex); + for (const auto& existing : m_groups) + { + if (existing->description.id == description.id) + { + throw std::invalid_argument("camera group '" + description.id + "' already exists"); + } + } + m_groups.push_back(std::move(group)); + return static_cast(m_groups.size() - 1); +} + +std::optional Publisher::acquire_slot(GroupId group, std::size_t stream) +{ + std::lock_guard lock(m_mutex); + std::vector& slots = group_at(group).slots.at(stream); + for (std::uint32_t i = 0; i < slots.size(); ++i) + { + if (slots[i].holds == 0 && !slots[i].writing) + { + slots[i].writing = true; + return i; + } + } + ++m_stats.no_free_slot; + return std::nullopt; +} + +void Publisher::claim_slot(GroupId group, std::size_t stream, std::uint32_t slot) +{ + std::lock_guard lock(m_mutex); + Slot& state = slot_at(group_at(group), stream, slot); + if (state.holds != 0 || state.writing) + { + throw std::invalid_argument("camera slot " + std::to_string(slot) + " is still in use and cannot be claimed"); + } + state.writing = true; +} + +void Publisher::abandon_slot(GroupId group, std::size_t stream, std::uint32_t slot) +{ + std::lock_guard lock(m_mutex); + Slot& state = slot_at(group_at(group), stream, slot); + if (!state.writing) + { + throw std::invalid_argument("camera slot " + std::to_string(slot) + " was not acquired and cannot be abandoned"); + } + state.writing = false; +} + +void Publisher::set_slot_free_callback(GroupId group, SlotFreeCallback callback) +{ + std::lock_guard lock(m_mutex); + group_at(group).on_free = callback ? std::make_shared(std::move(callback)) : nullptr; +} + +std::uint64_t Publisher::publish(GroupId group, std::span images, const FrameTimeline& timeline) +{ + std::vector freed; + std::uint64_t sequence = 0; + { + std::lock_guard lock(m_mutex); + Group& target = group_at(group); + if (images.size() != target.description.streams.size()) + { + throw std::invalid_argument("camera group '" + target.description.id + "' publishes " + + std::to_string(target.description.streams.size()) + " images per frame set"); + } + for (std::size_t i = 0; i < images.size(); ++i) + { + if (!slot_at(target, i, images[i].slot).writing) + { + throw std::invalid_argument("camera slot " + std::to_string(images[i].slot) + + " was not acquired before it was published"); + } + } + + FrameSet set; + set.sequence = ++target.sequence; + set.generation = target.generation; + set.timeline = timeline; + set.timeline.published_ns = m_clock(); + for (std::size_t i = 0; i < images.size(); ++i) + { + set.images[i] = images[i]; + slot_at(target, i, images[i].slot).writing = false; // the ring holds it from here + } + hold(target, set); + target.ring.push_back(set); + ++m_stats.published; + sequence = set.sequence; + + trim_ring(target, freed); + for (SubscriptionId id : target.subscriptions) + { + deliver(target, id, m_subscriptions.at(id), freed); + } + } + notify(freed); + return sequence; +} + +void Publisher::restart_group(GroupId group) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + Group& target = group_at(group); + ++target.generation; + while (!target.ring.empty()) + { + unhold(target, target.ring.front(), freed); + target.ring.pop_front(); + } + } + notify(freed); +} + +SubscribeResult Publisher::subscribe(EventSink& sink, GroupId group, const ReaderOptions& options) +{ + SubscribeResult result; + std::lock_guard lock(m_mutex); + if (group >= m_groups.size()) + { + result.error = "unknown camera group"; + return result; + } + if (options.window == 0) + { + result.error = "a reader's window must be at least 1"; + return result; + } + Group& target = *m_groups[group]; + const DeliveryOrder order = resolve_order(options); + + // The ring keeps what in-order readers may still be waiting for; newest-only readers + // need the newest frame set and nothing older. + std::size_t history = order == DeliveryOrder::IN_ORDER ? options.window : 1; + std::size_t windows = options.window; + for (SubscriptionId id : target.subscriptions) + { + const Subscription& other = m_subscriptions.at(id); + if (other.order == DeliveryOrder::IN_ORDER) + { + history = std::max(history, other.window); + } + windows += other.window; + } + const std::size_t needed = history + windows + target.reserve; + if (needed > target.slot_count) + { + result.error = "subscription does not fit the pool of camera group '" + target.description.id + "': it needs " + + std::to_string(needed) + " slots (" + std::to_string(history) + " ring history, " + + std::to_string(windows) + " for reader windows, " + std::to_string(target.reserve) + + " for the backend) and the pool has " + std::to_string(target.slot_count); + return result; + } + + Subscription subscription; + subscription.group = group; + subscription.window = options.window; + subscription.order = order; + subscription.sink = &sink; + result.id = m_next_subscription++; + m_subscriptions.emplace(result.id, std::move(subscription)); + target.subscriptions.push_back(result.id); + return result; +} + +void Publisher::release(SubscriptionId subscription, std::uint64_t lease_id) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + const auto found = m_subscriptions.find(subscription); + if (found == m_subscriptions.end()) + { + return; + } + const auto lease = found->second.leases.find(lease_id); + if (lease == found->second.leases.end()) + { + return; + } + Group& target = *m_groups[found->second.group]; + unhold(target, lease->second, freed); + found->second.leases.erase(lease); + deliver(target, subscription, found->second, freed); + } + notify(freed); +} + +void Publisher::unsubscribe(SubscriptionId subscription) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + const auto found = m_subscriptions.find(subscription); + if (found == m_subscriptions.end()) + { + return; + } + Group& target = *m_groups[found->second.group]; + for (const auto& lease : found->second.leases) + { + unhold(target, lease.second, freed); + } + std::erase(target.subscriptions, subscription); + m_subscriptions.erase(found); + trim_ring(target, freed); + } + notify(freed); +} + +std::size_t Publisher::group_count() const +{ + std::lock_guard lock(m_mutex); + return m_groups.size(); +} + +GroupDescription Publisher::group_description(GroupId group) const +{ + std::lock_guard lock(m_mutex); + return group_at(group).description; +} + +std::optional Publisher::find_group(std::string_view id) const +{ + std::lock_guard lock(m_mutex); + for (std::size_t i = 0; i < m_groups.size(); ++i) + { + if (m_groups[i]->description.id == id) + { + return static_cast(i); + } + } + return std::nullopt; +} + +std::uint32_t Publisher::generation(GroupId group) const +{ + std::lock_guard lock(m_mutex); + return group_at(group).generation; +} + +PublisherStats Publisher::stats() const +{ + std::lock_guard lock(m_mutex); + return m_stats; +} + +Publisher::Group& Publisher::group_at(GroupId group) +{ + if (group >= m_groups.size()) + { + throw std::out_of_range("unknown camera group"); + } + return *m_groups[group]; +} + +const Publisher::Group& Publisher::group_at(GroupId group) const +{ + if (group >= m_groups.size()) + { + throw std::out_of_range("unknown camera group"); + } + return *m_groups[group]; +} + +Publisher::Slot& Publisher::slot_at(Group& group, std::size_t stream, std::uint32_t slot) +{ + if (stream >= group.slots.size() || slot >= group.slots[stream].size()) + { + throw std::invalid_argument("camera group '" + group.description.id + "' has no stream " + + std::to_string(stream) + " slot " + std::to_string(slot)); + } + return group.slots[stream][slot]; +} + +std::size_t Publisher::history_of(const Group& group) const +{ + std::size_t history = 1; // the newest frame set is always kept + for (SubscriptionId id : group.subscriptions) + { + const Subscription& subscription = m_subscriptions.at(id); + if (subscription.order == DeliveryOrder::IN_ORDER) + { + history = std::max(history, subscription.window); + } + } + return history; +} + +void Publisher::hold(Group& group, const FrameSet& set) +{ + for (std::size_t i = 0; i < group.slots.size(); ++i) + { + ++slot_at(group, i, set.images[i].slot).holds; + } +} + +void Publisher::unhold(Group& group, const FrameSet& set, std::vector& freed) +{ + for (std::size_t i = 0; i < group.slots.size(); ++i) + { + Slot& slot = slot_at(group, i, set.images[i].slot); + if (--slot.holds == 0 && group.on_free) + { + freed.push_back({ group.on_free, i, set.images[i].slot }); + } + } +} + +void Publisher::trim_ring(Group& group, std::vector& freed) +{ + const std::size_t history = history_of(group); + while (group.ring.size() > history) + { + unhold(group, group.ring.front(), freed); + group.ring.pop_front(); + } +} + +void Publisher::deliver(Group& group, SubscriptionId id, Subscription& subscription, std::vector& freed) +{ + while (subscription.leases.size() < subscription.window) + { + if (group.ring.empty() || group.ring.back().sequence <= subscription.cursor) + { + return; + } + const FrameSet* target = &group.ring.back(); + if (subscription.order == DeliveryOrder::IN_ORDER) + { + target = &*std::find_if(group.ring.begin(), group.ring.end(), + [&](const FrameSet& set) { return set.sequence > subscription.cursor; }); + } + + FrameEvent event; + event.lease_id = m_next_lease++; + event.sequence = target->sequence; + event.generation = target->generation; + if (subscription.started) + { + const std::uint64_t skipped = target->sequence - subscription.cursor - 1; + event.skipped = + static_cast(std::min(skipped, std::numeric_limits::max())); + } + event.image_count = static_cast(group.slots.size()); + event.images = target->images; + event.timeline = target->timeline; + event.timeline.sent_ns = m_clock(); + + hold(group, *target); + if (!subscription.sink->send_event(id, event)) + { + unhold(group, *target, freed); + ++m_stats.send_failures; + return; + } + ++m_stats.events_sent; + subscription.leases.emplace(event.lease_id, *target); + subscription.cursor = target->sequence; + subscription.started = true; + } +} + +void Publisher::notify(const std::vector& freed) +{ + for (const FreedSlot& slot : freed) + { + (*slot.callback)(slot.stream, slot.slot); + } +} + +} // namespace core::camera diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 54c6e1e0d5..e21545f14f 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -13,6 +13,7 @@ add_subdirectory(live_trackers) add_subdirectory(tracker_metadata) add_subdirectory(log_bridge) add_subdirectory(plugin_manager) +add_subdirectory(camera_core) if(BUILD_OVXR_CAMERA_QUADS) add_subdirectory(ovxr_pip_layer) diff --git a/tests/cpp/core/camera_core/CMakeLists.txt b/tests/cpp/core/camera_core/CMakeLists.txt new file mode 100644 index 0000000000..aa26bc438f --- /dev/null +++ b/tests/cpp/core/camera_core/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_core_tests + test_publisher.cpp +) + +target_link_libraries(camera_core_tests PRIVATE + camera::camera_core + Catch2::Catch2WithMain +) + +message(STATUS "camera_core_tests target enabled with Catch2") +catch_discover_tests(camera_core_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp new file mode 100644 index 0000000000..5084581600 --- /dev/null +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -0,0 +1,510 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +// A reader that keeps every event it is sent. +struct RecordingSink : EventSink +{ + bool send_event(SubscriptionId, const FrameEvent& event) override + { + if (!accept) + { + return false; + } + events.push_back(event); + return true; + } + + std::vector events; + bool accept = true; +}; + +GroupDescription make_group(std::uint32_t slots, std::size_t streams = 2, const std::string& id = "head") +{ + GroupDescription group; + group.id = id; + for (std::size_t i = 0; i < streams; ++i) + { + StreamDescription stream; + stream.id = id + "/" + std::to_string(i); + stream.width = 640; + stream.height = 400; + stream.stride_bytes = 640 * 4; + stream.slot_bytes = 640 * 400 * 4; + stream.slot_count = slots; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// Publishes one frame set the way a backend does: take a free slot per stream, then publish. +std::uint64_t publish_next(Publisher& publisher, GroupId group, const FrameTimeline& timeline = {}) +{ + const std::size_t streams = publisher.group_description(group).streams.size(); + std::vector images(streams); + for (std::size_t i = 0; i < streams; ++i) + { + const auto slot = publisher.acquire_slot(group, i); + REQUIRE(slot.has_value()); + images[i].slot = *slot; + } + return publisher.publish(group, images, timeline); +} + +std::vector sequences(const RecordingSink& sink) +{ + std::vector result; + for (const FrameEvent& event : sink.events) + { + result.push_back(event.sequence); + } + return result; +} + +} // namespace + +TEST_CASE("a window of one always gets the newest frame set", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); + REQUIRE(sequences(sink) == std::vector{ 1 }); + REQUIRE(sink.events[0].skipped == 0); + REQUIRE(sink.events[0].image_count == 2); + + // The reader still holds frame set 1, so the next ones wait. + publish_next(publisher, group); + publish_next(publisher, group); + REQUIRE(sink.events.size() == 1); + + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 3 }); + REQUIRE(sink.events[1].skipped == 1); +} + +TEST_CASE("a larger window gets every frame set in order", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 3 }); + REQUIRE(reader); + + for (int i = 0; i < 4; ++i) + { + publish_next(publisher, group); + } + // Three are out, the fourth waits in the ring until the reader has room. + REQUIRE(sequences(sink) == std::vector{ 1, 2, 3 }); + + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 3, 4 }); + REQUIRE(sink.events[3].skipped == 0); +} + +TEST_CASE("an in-order reader that falls behind is told what it skipped", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 2 }); + REQUIRE(reader); + + for (int i = 0; i < 6; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sequences(sink) == std::vector{ 1, 2 }); + + // The ring kept the last two (5 and 6); 3 and 4 are gone. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 5 }); + REQUIRE(sink.events[2].skipped == 2); + + publisher.release(reader.id, sink.events[1].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 5, 6 }); + REQUIRE(sink.events[3].skipped == 0); +} + +TEST_CASE("newest order with a larger window skips to the newest frame set", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 2, .order = DeliveryOrder::NEWEST }); + REQUIRE(reader); + + for (int i = 0; i < 4; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sequences(sink) == std::vector{ 1, 2 }); + + // A viewer that holds one frame and wants the newest must not be handed frame set 3. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 4 }); + REQUIRE(sink.events[2].skipped == 1); +} + +TEST_CASE("a subscription that does not fit the pool is refused", "[camera_core][budget]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4)); + RecordingSink first; + RecordingSink second; + + // 1 ring history + 1 window + 1 backend reserve = 3 of 4 slots. + REQUIRE(publisher.subscribe(first, group, { .window = 1 })); + + // An in-order reader with a window of two needs 2 ring history + 3 windows + 1 reserve = 6. + const SubscribeResult refused = publisher.subscribe(second, group, { .window = 2 }); + REQUIRE_FALSE(refused); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("needs 6 slots")); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("pool has 4")); + + // A second newest-only reader needs 1 + 2 + 1 = 4 and fits. + REQUIRE(publisher.subscribe(second, group, { .window = 1 })); + + REQUIRE_FALSE(publisher.subscribe(second, group, { .window = 0 })); + REQUIRE_FALSE(publisher.subscribe(second, 99, { .window = 1 })); +} + +TEST_CASE("the backend reserve counts against the pool", "[camera_core][budget]") +{ + Publisher publisher; + // An SDK that keeps four captures in flight, on a pool of eight. + const GroupId group = publisher.add_group(make_group(8), 4); + RecordingSink sinks[4]; + + // 1 ring history + 3 windows + 4 reserve = 8. + for (int i = 0; i < 3; ++i) + { + REQUIRE(publisher.subscribe(sinks[i], group, { .window = 1 })); + } + const SubscribeResult refused = publisher.subscribe(sinks[3], group, { .window = 1 }); + REQUIRE_FALSE(refused); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("4 for the backend")); +} + +TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", "[camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(3, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); // slot 0: held by the reader + publish_next(publisher, group); // slot 1: held by the ring + REQUIRE(sink.events.size() == 1); + REQUIRE(sink.events[0].images[0].slot == 0); + + const auto third = publisher.acquire_slot(group, 0); + REQUIRE(third == 2u); + REQUIRE_FALSE(publisher.acquire_slot(group, 0).has_value()); + REQUIRE(publisher.stats().no_free_slot == 1); + publisher.abandon_slot(group, 0, *third); + + // Releasing frame set 1 frees slot 0 and hands the reader frame set 2. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events.size() == 2); + REQUIRE(sink.events[1].images[0].slot == 1); + REQUIRE(publisher.acquire_slot(group, 0) == 0u); +} + +TEST_CASE("the slot-free callback fires when the last holder lets go", "[camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + std::vector> freed; + publisher.set_slot_free_callback(group, + [&](std::size_t stream, std::uint32_t slot) + { + // Runs outside the publisher's lock, so calling back in must not deadlock. + (void)publisher.stats(); + freed.emplace_back(stream, slot); + }); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + // The SDK owns the buffers and picks the slot it filled. + const auto publish_slot = [&](std::uint32_t slot) + { + publisher.claim_slot(group, 0, slot); + ImageInfo image; + image.slot = slot; + publisher.publish(group, { &image, 1 }, {}); + }; + + publish_slot(0); // ring + reader + publish_slot(1); // the ring lets slot 0 go, the reader still has it + REQUIRE(freed.empty()); + REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 0), std::invalid_argument); + + publisher.release(reader.id, sink.events[0].lease_id); // reader moves on to slot 1 + REQUIRE(freed == std::vector>{ { 0, 0 } }); + + publish_slot(2); // the ring lets slot 1 go, the reader still has it + REQUIRE(freed.size() == 1); + publisher.unsubscribe(reader.id); + REQUIRE(freed == std::vector>{ { 0, 0 }, { 0, 1 } }); + + // A slot the backend gives up itself needs no notification. + publisher.claim_slot(group, 0, 0); + publisher.abandon_slot(group, 0, 0); + REQUIRE(freed.size() == 2); +} + +TEST_CASE("unsubscribing returns every frame set the reader held", "[camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 3 }); + REQUIRE(reader); + for (int i = 0; i < 3; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sink.events.size() == 3); + + publisher.unsubscribe(reader.id); + // Only the newest frame set stays in the ring: seven of the eight slots are free again. + for (int i = 0; i < 7; ++i) + { + REQUIRE(publisher.acquire_slot(group, 0).has_value()); + } + REQUIRE_FALSE(publisher.acquire_slot(group, 0).has_value()); + + // Releasing after the subscription ended is harmless. + publisher.release(reader.id, sink.events[0].lease_id); +} + +TEST_CASE("a frame set that could not be sent is not counted as held", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(3, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + sink.accept = false; + publish_next(publisher, group); + REQUIRE(publisher.stats().send_failures == 1); + REQUIRE(sink.events.empty()); + + sink.accept = true; + publish_next(publisher, group); + REQUIRE(sequences(sink) == std::vector{ 2 }); + REQUIRE(sink.events[0].skipped == 0); + REQUIRE(publisher.stats().events_sent == 1); + + // Frame set 1 left the ring and no reader holds it: its slot is free. + REQUIRE(publisher.acquire_slot(group, 0) == 0u); +} + +TEST_CASE("a restart changes the generation and empties the ring", "[camera_core][lifecycle]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); + REQUIRE(sink.events[0].generation == 0); + + publisher.restart_group(group); + REQUIRE(publisher.generation(group) == 1); + + // The frame set the reader holds stays valid; the sequence keeps counting. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events.size() == 1); + publish_next(publisher, group); + REQUIRE(sink.events.size() == 2); + REQUIRE(sink.events[1].generation == 1); + REQUIRE(sink.events[1].sequence == 2); +} + +TEST_CASE("a group publishes one image per stream", "[camera_core][groups]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 3, "rig")); + REQUIRE(publisher.find_group("rig") == group); + REQUIRE_FALSE(publisher.find_group("head").has_value()); + REQUIRE(publisher.group_count() == 1); + + RecordingSink sink; + REQUIRE(publisher.subscribe(sink, group, { .window = 1 })); + + std::vector images(3); + for (std::size_t i = 0; i < 3; ++i) + { + images[i].slot = *publisher.acquire_slot(group, i); + images[i].device_frame_id = 100 + i; + } + images[1].flags = IMAGE_FLAG_KEYFRAME; + images[1].bytes_used = 4096; + images[2].flags = IMAGE_FLAG_SDK_WARNING; + publisher.publish(group, images, {}); + + REQUIRE(sink.events.size() == 1); + const FrameEvent& event = sink.events[0]; + REQUIRE(event.image_count == 3); + REQUIRE(event.images[0].device_frame_id == 100); + REQUIRE(event.images[1].flags == IMAGE_FLAG_KEYFRAME); + REQUIRE(event.images[1].bytes_used == 4096); + REQUIRE(event.images[2].flags == IMAGE_FLAG_SDK_WARNING); +} + +TEST_CASE("every frame set carries its timeline", "[camera_core][timeline]") +{ + std::int64_t now = 1000; + Publisher publisher([&] { return now; }); + const GroupId group = publisher.add_group(make_group(4, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + FrameTimeline timeline; + timeline.capture_ns = 100; + timeline.sdk_ready_ns = 900; + publish_next(publisher, group, timeline); + REQUIRE(sink.events[0].timeline.capture_ns == 100); + REQUIRE(sink.events[0].timeline.sdk_ready_ns == 900); + REQUIRE(sink.events[0].timeline.published_ns == 1000); + REQUIRE(sink.events[0].timeline.sent_ns == 1000); + + // A frame set that waits for the reader is stamped as sent when it leaves. + now = 2000; + publish_next(publisher, group); + now = 2500; + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events[1].timeline.capture_ns == TIME_UNKNOWN); + REQUIRE(sink.events[1].timeline.published_ns == 2000); + REQUIRE(sink.events[1].timeline.sent_ns == 2500); +} + +TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") +{ + Publisher publisher; + REQUIRE_THROWS_AS(publisher.add_group(make_group(4, 0)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4, MAX_STREAMS_PER_GROUP + 1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(0)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(MAX_SLOTS_PER_STREAM + 1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 0), std::invalid_argument); + + const GroupId group = publisher.add_group(make_group(4)); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4)), std::invalid_argument); // same id + + // Publishing needs one image per stream, each in a slot the backend took first. + std::vector images(2); + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); + images[0].slot = *publisher.acquire_slot(group, 0); + REQUIRE_THROWS_AS(publisher.publish(group, { images.data(), 1 }, {}), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); // stream 1 not acquired + REQUIRE(publisher.stats().published == 0); + + REQUIRE_THROWS_AS(publisher.abandon_slot(group, 1, 0), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.acquire_slot(99, 0), std::out_of_range); + REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 99), std::invalid_argument); +} + +TEST_CASE("capture and a reader on different threads keep every frame set in order", "[camera_core][threads]") +{ + // A reader as the endpoint will drive it: events arrive on the capture thread, the + // reader's own thread takes them and releases. + struct QueueSink : EventSink + { + bool send_event(SubscriptionId, const FrameEvent& event) override + { + { + std::lock_guard lock(mutex); + queue.push_back(event); + } + ready.notify_one(); + return true; + } + + std::mutex mutex; + std::condition_variable ready; + std::deque queue; + }; + + constexpr std::uint64_t FRAME_SETS = 2000; + Publisher publisher; + const GroupId group = publisher.add_group(make_group(16)); + QueueSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 4 }); + REQUIRE(reader); + + std::atomic received{ 0 }; + std::atomic out_of_order{ 0 }; + std::thread consumer( + [&] + { + std::uint64_t last = 0; + while (last < FRAME_SETS) + { + FrameEvent event; + { + std::unique_lock lock(sink.mutex); + sink.ready.wait(lock, [&] { return !sink.queue.empty(); }); + event = sink.queue.front(); + sink.queue.pop_front(); + } + out_of_order += event.sequence <= last ? 1 : 0; + last = event.sequence; + ++received; + publisher.release(reader.id, event.lease_id); + } + }); + + std::uint64_t dropped = 0; + for (std::uint64_t i = 0; i < FRAME_SETS; ++i) + { + std::vector images(2); + const auto left = publisher.acquire_slot(group, 0); + const auto right = publisher.acquire_slot(group, 1); + if (!left || !right) + { + // Cannot happen while the budget holds: a free slot is always left for the backend. + ++dropped; + continue; + } + images[0].slot = *left; + images[1].slot = *right; + publisher.publish(group, images, {}); + } + consumer.join(); + + REQUIRE(dropped == 0); + REQUIRE(out_of_order == 0); + REQUIRE(received > 0); + REQUIRE(publisher.stats().published == FRAME_SETS); +} From 0a04690261eac93cc7f5e4f1cb747f1fa0cfbef9 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 12:51:40 -0800 Subject: [PATCH 02/16] fix(camera): describe an encoded stream by its codec, not as a pixel format H.264 and H.265 are codecs, so they do not belong in PixelFormat. A stream description now has a Codec (NONE for pixels) next to its PixelFormat, and is_encoded() reads the codec. Signed-off-by: Farbod Motlagh Co-Authored-By: Claude --- .../camera_core/cpp/inc/camera_core/types.hpp | 20 ++++++++++++------- tests/cpp/core/camera_core/test_publisher.cpp | 17 ++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp index 8e919e35ec..efba4d0b64 100644 --- a/src/core/camera_core/cpp/inc/camera_core/types.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -25,15 +25,15 @@ enum class PixelFormat : std::uint32_t RGBA8 = 2, NV12 = 3, GRAY8 = 4, - // Encoded: a slot holds one access unit instead of pixels. - H264 = 16, - H265 = 17, }; -constexpr bool is_encoded(PixelFormat format) +// Whether a stream's slots hold pixels or one access unit of encoded video each. +enum class Codec : std::uint32_t { - return format == PixelFormat::H264 || format == PixelFormat::H265; -} + NONE = 0, // pixels, laid out as the stream's pixel format says + H264 = 1, + H265 = 2, +}; // How a slot's memory reaches another process. The core tracks slots by index only and never // touches their memory, so a new kind needs no change here. @@ -48,7 +48,8 @@ enum class MemoryKind : std::uint32_t struct StreamDescription { std::string id; - PixelFormat format = PixelFormat::BGRA8; + Codec codec = Codec::NONE; + PixelFormat pixel_format = PixelFormat::BGRA8; // of a pixel stream; unused when the stream is encoded MemoryKind memory = MemoryKind::HOST; std::uint32_t width = 0; std::uint32_t height = 0; @@ -56,6 +57,11 @@ struct StreamDescription std::uint64_t slot_bytes = 0; // capacity of one slot std::uint32_t slot_count = 0; double frame_rate = 0.0; // configured or measured, not the sensor mode's maximum + + constexpr bool is_encoded() const + { + return codec != Codec::NONE; + } }; // One or more hardware-synchronised streams that are published as one frame set. diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp index 5084581600..46f6c00212 100644 --- a/tests/cpp/core/camera_core/test_publisher.cpp +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -382,6 +382,23 @@ TEST_CASE("a group publishes one image per stream", "[camera_core][groups]") REQUIRE(event.images[2].flags == IMAGE_FLAG_SDK_WARNING); } +TEST_CASE("an encoded stream is described by its codec, not by a pixel format", "[camera_core][groups]") +{ + GroupDescription description = make_group(4, 1, "head.enc"); + StreamDescription& stream = description.streams[0]; + REQUIRE_FALSE(stream.is_encoded()); + stream.codec = Codec::H265; + stream.stride_bytes = 0; + stream.slot_bytes = 2 * 1024 * 1024; // room for one access unit + + Publisher publisher; + const GroupId group = publisher.add_group(description); + const StreamDescription stored = publisher.group_description(group).streams[0]; + REQUIRE(stored.is_encoded()); + REQUIRE(stored.codec == Codec::H265); + REQUIRE(stored.slot_bytes == 2 * 1024 * 1024); +} + TEST_CASE("every frame set carries its timeline", "[camera_core][timeline]") { std::int64_t now = 1000; From cdb81280fb71015fd2e51c6380fc18c3cfabb1a5 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 12:53:26 -0800 Subject: [PATCH 03/16] fix(camera): describe plane layout per stream and count per group Two more places where the stream description and the publisher were too coarse for real cameras: - A stream had a single stride. NV12 has two planes, and on Jetson each plane has its own offset and stride, so a stream now lists its planes. add_group rejects a pixel stream without a size or without one stride per plane of its format, and an encoded stream that lists planes. - Counters were totals over every group. They are now kept per group, so a tool can tell which camera drops frames. Signed-off-by: Farbod Motlagh Co-Authored-By: Claude --- .../cpp/inc/camera_core/publisher.hpp | 7 ++-- .../camera_core/cpp/inc/camera_core/types.hpp | 18 +++++++- src/core/camera_core/cpp/publisher.cpp | 41 +++++++++++++++---- tests/cpp/core/camera_core/test_publisher.cpp | 31 ++++++++++---- 4 files changed, 78 insertions(+), 19 deletions(-) diff --git a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp index 3b7ee5dd87..6f98df88cb 100644 --- a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp @@ -41,6 +41,7 @@ class EventSink virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) = 0; }; +// Counters of one group since it was added. struct PublisherStats { std::uint64_t published = 0; // frame sets published @@ -131,7 +132,7 @@ class Publisher // ---- reader side ----------------------------------------------------------------------- /** - * @brief Start sending a group's frame events to a sink. + * @brief Start sending a group's frame events to a sink, which must outlive the subscription. * @return The subscription, or the reason it does not fit the pool. */ SubscribeResult subscribe(EventSink& sink, GroupId group, const ReaderOptions& options); @@ -152,7 +153,7 @@ class Publisher GroupDescription group_description(GroupId group) const; std::optional find_group(std::string_view id) const; std::uint32_t generation(GroupId group) const; - PublisherStats stats() const; + PublisherStats stats(GroupId group) const; private: struct FrameSet @@ -191,6 +192,7 @@ class Publisher std::deque ring; std::vector subscriptions; std::shared_ptr on_free; + PublisherStats stats; }; struct FreedSlot @@ -216,7 +218,6 @@ class Publisher std::map m_subscriptions; SubscriptionId m_next_subscription = 1; std::uint64_t m_next_lease = 1; - PublisherStats m_stats; }; } // namespace core::camera diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp index efba4d0b64..34b8fb11fb 100644 --- a/src/core/camera_core/cpp/inc/camera_core/types.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -27,6 +27,21 @@ enum class PixelFormat : std::uint32_t GRAY8 = 4, }; +// NV12 has two planes (luma, then interleaved chroma); the other formats have one. +inline constexpr std::size_t MAX_PLANES_PER_IMAGE = 2; + +constexpr std::uint32_t plane_count_of(PixelFormat format) +{ + return format == PixelFormat::NV12 ? 2 : 1; +} + +// Where one plane of an image sits inside a slot. +struct PlaneLayout +{ + std::uint32_t offset_bytes = 0; + std::uint32_t stride_bytes = 0; // from the start of one row to the start of the next +}; + // Whether a stream's slots hold pixels or one access unit of encoded video each. enum class Codec : std::uint32_t { @@ -53,7 +68,8 @@ struct StreamDescription MemoryKind memory = MemoryKind::HOST; std::uint32_t width = 0; std::uint32_t height = 0; - std::uint32_t stride_bytes = 0; // row pitch of the first plane; 0 for an encoded stream + std::uint32_t plane_count = 0; // 0 for an encoded stream + std::array planes; // the same for every slot of the stream std::uint64_t slot_bytes = 0; // capacity of one slot std::uint32_t slot_count = 0; double frame_rate = 0.0; // configured or measured, not the sensor mode's maximum diff --git a/src/core/camera_core/cpp/publisher.cpp b/src/core/camera_core/cpp/publisher.cpp index 5cb3c8562c..b3b3ab0766 100644 --- a/src/core/camera_core/cpp/publisher.cpp +++ b/src/core/camera_core/cpp/publisher.cpp @@ -29,6 +29,27 @@ DeliveryOrder resolve_order(const ReaderOptions& options) return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; } +// A pixel stream names every plane of its format; an encoded stream has none. +bool layout_is_valid(const StreamDescription& stream) +{ + if (stream.is_encoded()) + { + return stream.plane_count == 0; + } + if (stream.width == 0 || stream.height == 0 || stream.plane_count != plane_count_of(stream.pixel_format)) + { + return false; + } + for (std::uint32_t i = 0; i < stream.plane_count; ++i) + { + if (stream.planes[i].stride_bytes == 0) + { + return false; + } + } + return true; +} + } // namespace Publisher::Publisher(Clock clock) : m_clock(clock ? std::move(clock) : Clock(steady_now_ns)) @@ -63,6 +84,11 @@ GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t throw std::invalid_argument("camera group '" + description.id + "' has a stream without an id, a slot " + "size or between 1 and " + std::to_string(MAX_SLOTS_PER_STREAM) + " slots"); } + if (!layout_is_valid(stream)) + { + throw std::invalid_argument("camera stream '" + stream.id + "' needs a size and one stride per plane of " + + "its pixel format, or no planes at all when it is encoded"); + } group->slot_count = std::min(group->slot_count, stream.slot_count); group->slots.emplace_back(stream.slot_count); } @@ -82,7 +108,8 @@ GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t std::optional Publisher::acquire_slot(GroupId group, std::size_t stream) { std::lock_guard lock(m_mutex); - std::vector& slots = group_at(group).slots.at(stream); + Group& target = group_at(group); + std::vector& slots = target.slots.at(stream); for (std::uint32_t i = 0; i < slots.size(); ++i) { if (slots[i].holds == 0 && !slots[i].writing) @@ -91,7 +118,7 @@ std::optional Publisher::acquire_slot(GroupId group, std::size_t return i; } } - ++m_stats.no_free_slot; + ++target.stats.no_free_slot; return std::nullopt; } @@ -156,7 +183,7 @@ std::uint64_t Publisher::publish(GroupId group, std::span image } hold(target, set); target.ring.push_back(set); - ++m_stats.published; + ++target.stats.published; sequence = set.sequence; trim_ring(target, freed); @@ -312,10 +339,10 @@ std::uint32_t Publisher::generation(GroupId group) const return group_at(group).generation; } -PublisherStats Publisher::stats() const +PublisherStats Publisher::stats(GroupId group) const { std::lock_guard lock(m_mutex); - return m_stats; + return group_at(group).stats; } Publisher::Group& Publisher::group_at(GroupId group) @@ -424,10 +451,10 @@ void Publisher::deliver(Group& group, SubscriptionId id, Subscription& subscript if (!subscription.sink->send_event(id, event)) { unhold(group, *target, freed); - ++m_stats.send_failures; + ++group.stats.send_failures; return; } - ++m_stats.events_sent; + ++group.stats.events_sent; subscription.leases.emplace(event.lease_id, *target); subscription.cursor = target->sequence; subscription.started = true; diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp index 46f6c00212..e13ed8e051 100644 --- a/tests/cpp/core/camera_core/test_publisher.cpp +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -48,7 +48,8 @@ GroupDescription make_group(std::uint32_t slots, std::size_t streams = 2, const stream.id = id + "/" + std::to_string(i); stream.width = 640; stream.height = 400; - stream.stride_bytes = 640 * 4; + stream.plane_count = 1; + stream.planes[0].stride_bytes = 640 * 4; stream.slot_bytes = 640 * 400 * 4; stream.slot_count = slots; stream.frame_rate = 60.0; @@ -226,7 +227,7 @@ TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", " const auto third = publisher.acquire_slot(group, 0); REQUIRE(third == 2u); REQUIRE_FALSE(publisher.acquire_slot(group, 0).has_value()); - REQUIRE(publisher.stats().no_free_slot == 1); + REQUIRE(publisher.stats(group).no_free_slot == 1); publisher.abandon_slot(group, 0, *third); // Releasing frame set 1 frees slot 0 and hands the reader frame set 2. @@ -245,7 +246,7 @@ TEST_CASE("the slot-free callback fires when the last holder lets go", "[camera_ [&](std::size_t stream, std::uint32_t slot) { // Runs outside the publisher's lock, so calling back in must not deadlock. - (void)publisher.stats(); + (void)publisher.stats(group); freed.emplace_back(stream, slot); }); RecordingSink sink; @@ -315,14 +316,14 @@ TEST_CASE("a frame set that could not be sent is not counted as held", "[camera_ sink.accept = false; publish_next(publisher, group); - REQUIRE(publisher.stats().send_failures == 1); + REQUIRE(publisher.stats(group).send_failures == 1); REQUIRE(sink.events.empty()); sink.accept = true; publish_next(publisher, group); REQUIRE(sequences(sink) == std::vector{ 2 }); REQUIRE(sink.events[0].skipped == 0); - REQUIRE(publisher.stats().events_sent == 1); + REQUIRE(publisher.stats(group).events_sent == 1); // Frame set 1 left the ring and no reader holds it: its slot is free. REQUIRE(publisher.acquire_slot(group, 0) == 0u); @@ -388,7 +389,7 @@ TEST_CASE("an encoded stream is described by its codec, not by a pixel format", StreamDescription& stream = description.streams[0]; REQUIRE_FALSE(stream.is_encoded()); stream.codec = Codec::H265; - stream.stride_bytes = 0; + stream.plane_count = 0; stream.slot_bytes = 2 * 1024 * 1024; // room for one access unit Publisher publisher; @@ -436,6 +437,20 @@ TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") REQUIRE_THROWS_AS(publisher.add_group(make_group(MAX_SLOTS_PER_STREAM + 1)), std::invalid_argument); REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 0), std::invalid_argument); + // A pixel stream names every plane of its format; an encoded stream has none. + GroupDescription nv12 = make_group(4, 1, "nv12"); + nv12.streams[0].pixel_format = PixelFormat::NV12; + REQUIRE_THROWS_AS(publisher.add_group(nv12), std::invalid_argument); + nv12.streams[0].plane_count = 2; + nv12.streams[0].planes[1] = { .offset_bytes = 640 * 400, .stride_bytes = 640 }; + REQUIRE_NOTHROW(publisher.add_group(nv12)); + GroupDescription encoded_with_planes = make_group(4, 1, "enc"); + encoded_with_planes.streams[0].codec = Codec::H264; + REQUIRE_THROWS_AS(publisher.add_group(encoded_with_planes), std::invalid_argument); + GroupDescription no_size = make_group(4, 1, "empty"); + no_size.streams[0].width = 0; + REQUIRE_THROWS_AS(publisher.add_group(no_size), std::invalid_argument); + const GroupId group = publisher.add_group(make_group(4)); REQUIRE_THROWS_AS(publisher.add_group(make_group(4)), std::invalid_argument); // same id @@ -445,7 +460,7 @@ TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") images[0].slot = *publisher.acquire_slot(group, 0); REQUIRE_THROWS_AS(publisher.publish(group, { images.data(), 1 }, {}), std::invalid_argument); REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); // stream 1 not acquired - REQUIRE(publisher.stats().published == 0); + REQUIRE(publisher.stats(group).published == 0); REQUIRE_THROWS_AS(publisher.abandon_slot(group, 1, 0), std::invalid_argument); REQUIRE_THROWS_AS(publisher.acquire_slot(99, 0), std::out_of_range); @@ -523,5 +538,5 @@ TEST_CASE("capture and a reader on different threads keep every frame set in ord REQUIRE(dropped == 0); REQUIRE(out_of_order == 0); REQUIRE(received > 0); - REQUIRE(publisher.stats().published == FRAME_SETS); + REQUIRE(publisher.stats(group).published == FRAME_SETS); } From be73673097abcb17f9666e886f2f49db3a07a160 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 13:40:09 -0800 Subject: [PATCH 04/16] docs(camera): add a README for camera_core and keep only the host memory kind The README explains the terms (stream, group, frame set, window, order, lease, generation), the two ways a backend gets a slot, and the pool budget, so a reviewer does not need anything outside the repository. MemoryKind listed kinds that nothing implements yet. It now has HOST only; each further kind is added together with its pool. Signed-off-by: Farbod Motlagh Co-Authored-By: Claude --- src/core/camera_core/README.md | 69 +++++++++++++++++++ .../camera_core/cpp/inc/camera_core/types.hpp | 9 +-- 2 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 src/core/camera_core/README.md diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md new file mode 100644 index 0000000000..ad50e60f2f --- /dev/null +++ b/src/core/camera_core/README.md @@ -0,0 +1,69 @@ + + +# Camera interface: core + +The camera interface is the one place Isaac Capture applications get camera frames from. One +process owns the cameras and keeps their frames in fixed pools of shared buffers; viewers, +encoders and recorders in other processes read those buffers in place. + +This module is the part with no memory and no sockets in it: it decides which pool slot holds +which frame set and who may still read it. It uses the standard library only. The shared memory +behind the slots and the connection to other processes live in separate modules. + +## Terms + +- **Stream**: one sensor's images, or one encoded video stream. It has a pool of **slots**; a + slot holds one image, or one access unit when the stream is encoded. +- **Group**: one or more streams captured together, such as a stereo pair or a triggered rig. + A group publishes one **frame set** at a time: one image per stream, under one sequence number. +- **Window**: how many frame sets are kept for a reader at a time, unread and held together. +- **Order**: `NEWEST` hands the reader the latest frame set each time it has room and skips the + rest (a viewer, a live encoder). `IN_ORDER` hands over every frame set in sequence (a recorder). + The default is `NEWEST` for a window of one and `IN_ORDER` otherwise. +- **Lease**: a reader's hold on a frame set. Its slots are not reused until the reader releases + it or goes away. +- **Generation**: changes when a camera restarts, so a reader can tell a restart from a skip. + +## How frames move + +A backend gets a slot in one of two ways, then publishes: + +```cpp +// The backend writes the image itself. +auto slot = publisher.acquire_slot(group, stream); // nothing: every slot is in use, drop the frame +// ... write the pixels into that slot ... +publisher.publish(group, images, timeline); + +// The camera SDK owns the buffers and picked one. +publisher.claim_slot(group, stream, slot_the_sdk_filled); +publisher.publish(group, images, timeline); +``` + +For the second way, `set_slot_free_callback` tells the backend the moment a slot is held by +nobody, so the buffer can go back to the SDK. + +A reader subscribes with a window and an order, receives `FrameEvent`s through an `EventSink`, +and returns each one with `release`. Events carry slot indices, the frame set's timeline and the +number of frame sets the reader skipped. A slow reader skips; it never slows capture or another +reader, and no call blocks on a reader. + +## Pool budget + +A slot is in use while the ring of recent frame sets or any lease refers to it, so a subscription +is accepted only when the pool can cover it: + +```text +slots >= ring history + sum of reader windows + backend reserve +``` + +Ring history is one frame set, or the largest window among in-order readers. The backend reserve +is the number of slots the backend needs for itself: one to write ahead, more for an SDK that +keeps several captures in flight. A subscription that does not fit is refused with a message that +gives these numbers. + +## Tests + +`tests/cpp/core/camera_core`, run with `ctest -L camera_core`. diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp index 34b8fb11fb..1e66086a1a 100644 --- a/src/core/camera_core/cpp/inc/camera_core/types.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -50,14 +50,11 @@ enum class Codec : std::uint32_t H265 = 2, }; -// How a slot's memory reaches another process. The core tracks slots by index only and never -// touches their memory, so a new kind needs no change here. +// Where a stream's slots live, which tells a reader how to map them. The core tracks slots by +// index only and never touches their memory; each further kind is added with its pool. enum class MemoryKind : std::uint32_t { - HOST = 1, // shared host memory, passed as a file descriptor - CUDA = 2, // CUDA memory on a discrete GPU - DMA_BUF = 3, // Jetson buffer, passed as a file descriptor - NVSCI_BUF = 4, // reserved for SIPL; nothing implements it yet + HOST = 1, // shared host memory }; struct StreamDescription From 9977ee4a8ef3b40b9de97411b33d8070d5eb7ff0 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 13:47:59 -0800 Subject: [PATCH 05/16] test(camera): tag the camera_core tests as unit tests Core tests carry the unit tag, which becomes the CTest label that `ctest -L unit` selects. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/camera_core/README.md | 3 +- tests/cpp/core/camera_core/test_publisher.cpp | 32 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md index ad50e60f2f..bc5e640787 100644 --- a/src/core/camera_core/README.md +++ b/src/core/camera_core/README.md @@ -66,4 +66,5 @@ gives these numbers. ## Tests -`tests/cpp/core/camera_core`, run with `ctest -L camera_core`. +`tests/cpp/core/camera_core`, run with `ctest -L camera_core`. They are tagged `unit`, so +`ctest -L unit` runs them too. diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp index e13ed8e051..5d8f7c946d 100644 --- a/tests/cpp/core/camera_core/test_publisher.cpp +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -84,7 +84,7 @@ std::vector sequences(const RecordingSink& sink) } // namespace -TEST_CASE("a window of one always gets the newest frame set", "[camera_core][delivery]") +TEST_CASE("a window of one always gets the newest frame set", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8)); @@ -107,7 +107,7 @@ TEST_CASE("a window of one always gets the newest frame set", "[camera_core][del REQUIRE(sink.events[1].skipped == 1); } -TEST_CASE("a larger window gets every frame set in order", "[camera_core][delivery]") +TEST_CASE("a larger window gets every frame set in order", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8)); @@ -127,7 +127,7 @@ TEST_CASE("a larger window gets every frame set in order", "[camera_core][delive REQUIRE(sink.events[3].skipped == 0); } -TEST_CASE("an in-order reader that falls behind is told what it skipped", "[camera_core][delivery]") +TEST_CASE("an in-order reader that falls behind is told what it skipped", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8)); @@ -151,7 +151,7 @@ TEST_CASE("an in-order reader that falls behind is told what it skipped", "[came REQUIRE(sink.events[3].skipped == 0); } -TEST_CASE("newest order with a larger window skips to the newest frame set", "[camera_core][delivery]") +TEST_CASE("newest order with a larger window skips to the newest frame set", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8)); @@ -171,7 +171,7 @@ TEST_CASE("newest order with a larger window skips to the newest frame set", "[c REQUIRE(sink.events[2].skipped == 1); } -TEST_CASE("a subscription that does not fit the pool is refused", "[camera_core][budget]") +TEST_CASE("a subscription that does not fit the pool is refused", "[unit][camera_core][budget]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(4)); @@ -194,7 +194,7 @@ TEST_CASE("a subscription that does not fit the pool is refused", "[camera_core] REQUIRE_FALSE(publisher.subscribe(second, 99, { .window = 1 })); } -TEST_CASE("the backend reserve counts against the pool", "[camera_core][budget]") +TEST_CASE("the backend reserve counts against the pool", "[unit][camera_core][budget]") { Publisher publisher; // An SDK that keeps four captures in flight, on a pool of eight. @@ -211,7 +211,7 @@ TEST_CASE("the backend reserve counts against the pool", "[camera_core][budget]" REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("4 for the backend")); } -TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", "[camera_core][slots]") +TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", "[unit][camera_core][slots]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(3, 1)); @@ -237,7 +237,7 @@ TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", " REQUIRE(publisher.acquire_slot(group, 0) == 0u); } -TEST_CASE("the slot-free callback fires when the last holder lets go", "[camera_core][slots]") +TEST_CASE("the slot-free callback fires when the last holder lets go", "[unit][camera_core][slots]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(4, 1)); @@ -281,7 +281,7 @@ TEST_CASE("the slot-free callback fires when the last holder lets go", "[camera_ REQUIRE(freed.size() == 2); } -TEST_CASE("unsubscribing returns every frame set the reader held", "[camera_core][slots]") +TEST_CASE("unsubscribing returns every frame set the reader held", "[unit][camera_core][slots]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8, 1)); @@ -306,7 +306,7 @@ TEST_CASE("unsubscribing returns every frame set the reader held", "[camera_core publisher.release(reader.id, sink.events[0].lease_id); } -TEST_CASE("a frame set that could not be sent is not counted as held", "[camera_core][delivery]") +TEST_CASE("a frame set that could not be sent is not counted as held", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(3, 1)); @@ -329,7 +329,7 @@ TEST_CASE("a frame set that could not be sent is not counted as held", "[camera_ REQUIRE(publisher.acquire_slot(group, 0) == 0u); } -TEST_CASE("a restart changes the generation and empties the ring", "[camera_core][lifecycle]") +TEST_CASE("a restart changes the generation and empties the ring", "[unit][camera_core][lifecycle]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(4, 1)); @@ -352,7 +352,7 @@ TEST_CASE("a restart changes the generation and empties the ring", "[camera_core REQUIRE(sink.events[1].sequence == 2); } -TEST_CASE("a group publishes one image per stream", "[camera_core][groups]") +TEST_CASE("a group publishes one image per stream", "[unit][camera_core][groups]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(4, 3, "rig")); @@ -383,7 +383,7 @@ TEST_CASE("a group publishes one image per stream", "[camera_core][groups]") REQUIRE(event.images[2].flags == IMAGE_FLAG_SDK_WARNING); } -TEST_CASE("an encoded stream is described by its codec, not by a pixel format", "[camera_core][groups]") +TEST_CASE("an encoded stream is described by its codec, not by a pixel format", "[unit][camera_core][groups]") { GroupDescription description = make_group(4, 1, "head.enc"); StreamDescription& stream = description.streams[0]; @@ -400,7 +400,7 @@ TEST_CASE("an encoded stream is described by its codec, not by a pixel format", REQUIRE(stored.slot_bytes == 2 * 1024 * 1024); } -TEST_CASE("every frame set carries its timeline", "[camera_core][timeline]") +TEST_CASE("every frame set carries its timeline", "[unit][camera_core][timeline]") { std::int64_t now = 1000; Publisher publisher([&] { return now; }); @@ -428,7 +428,7 @@ TEST_CASE("every frame set carries its timeline", "[camera_core][timeline]") REQUIRE(sink.events[1].timeline.sent_ns == 2500); } -TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") +TEST_CASE("misuse by a backend is rejected", "[unit][camera_core][validation]") { Publisher publisher; REQUIRE_THROWS_AS(publisher.add_group(make_group(4, 0)), std::invalid_argument); @@ -467,7 +467,7 @@ TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 99), std::invalid_argument); } -TEST_CASE("capture and a reader on different threads keep every frame set in order", "[camera_core][threads]") +TEST_CASE("capture and a reader on different threads keep every frame set in order", "[unit][camera_core][threads]") { // A reader as the endpoint will drive it: events arrive on the capture thread, the // reader's own thread takes them and releases. From 8240723c7869532cc1b856191f65c1772035458a Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 13:54:22 -0800 Subject: [PATCH 06/16] build(camera): link the threads library into the camera_core tests The tests start threads. Without the link, std::thread fails at run time on C libraries that keep threads in a separate library. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- tests/cpp/core/camera_core/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/cpp/core/camera_core/CMakeLists.txt b/tests/cpp/core/camera_core/CMakeLists.txt index aa26bc438f..f74232fd6b 100644 --- a/tests/cpp/core/camera_core/CMakeLists.txt +++ b/tests/cpp/core/camera_core/CMakeLists.txt @@ -3,6 +3,8 @@ cmake_minimum_required(VERSION 3.20) +find_package(Threads REQUIRED) + add_executable(camera_core_tests test_publisher.cpp ) @@ -10,6 +12,7 @@ add_executable(camera_core_tests target_link_libraries(camera_core_tests PRIVATE camera::camera_core Catch2::Catch2WithMain + Threads::Threads ) message(STATUS "camera_core_tests target enabled with Catch2") From 7894a112bdcd8f28a077fa1e64cd25e6b698b545 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:31:03 -0800 Subject: [PATCH 07/16] fix(camera): reject descriptions and images that do not fit their slots add_group now checks that every plane of a pixel stream lies inside the slot (row length against stride, last row against the slot size) and that a pool can hold one frame set on top of the backend's reserve. publish refuses an image larger than its slot. EventSink::send_event is noexcept: it runs on the capture path with the publisher's lock held, and a throwing sink would have left a slot held. A group with a slot-free callback hands out its slots with claim_slot only, so a slot cannot be taken again before its callback has run. Adds agent notes for the module. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/camera_core/AGENTS.md | 16 ++++ src/core/camera_core/README.md | 3 +- .../cpp/inc/camera_core/publisher.hpp | 11 ++- src/core/camera_core/cpp/publisher.cpp | 56 ++++++++++++- tests/cpp/core/camera_core/test_publisher.cpp | 82 ++++++++++++++++++- 5 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 src/core/camera_core/AGENTS.md diff --git a/src/core/camera_core/AGENTS.md b/src/core/camera_core/AGENTS.md new file mode 100644 index 0000000000..4fb79450a3 --- /dev/null +++ b/src/core/camera_core/AGENTS.md @@ -0,0 +1,16 @@ + + +# Agent notes — `camera_core` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) for the terms (stream, group, window, order, lease) before changing behaviour. + +- **Standard library only.** No slot memory, sockets, logging or platform headers here; that keeps the module building and tested on every platform. Memory and connections live in `camera_ipc`. +- **One type or field per concept.** A codec is not a pixel format, each plane has its own layout, counters are per group. Do not give a field a second meaning, and do not carry a type over from a prototype without modelling it for this interface. +- **Nothing ahead of its user.** A memory kind, pixel format or flag is added in the change that brings the pool or backend that needs it. +- **`EventSink::send_event` runs on the capture path with the publisher's lock held.** It is `noexcept` and must not block or call back into the publisher. Callbacks to a backend (`SlotFreeCallback`) run outside the lock. +- **Reject bad input where it enters.** `add_group` and `publish` refuse a description or an image that a reader could run off the end of a slot with; readers do not re-check. diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md index bc5e640787..cc0d4fbe2f 100644 --- a/src/core/camera_core/README.md +++ b/src/core/camera_core/README.md @@ -43,7 +43,8 @@ publisher.publish(group, images, timeline); ``` For the second way, `set_slot_free_callback` tells the backend the moment a slot is held by -nobody, so the buffer can go back to the SDK. +nobody, so the buffer can go back to the SDK. A group uses one way or the other: once it has +that callback, its slots are handed out with `claim_slot` only. A reader subscribes with a window and an order, receives `FrameEvent`s through an `EventSink`, and returns each one with `release`. Events carry slot indices, the frame set's timeline and the diff --git a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp index 6f98df88cb..44ed310fec 100644 --- a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp @@ -35,10 +35,10 @@ class EventSink /** * @brief Hand one event to the reader. Called on the capture path with the publisher's - * lock held, so it must not block and must not call back into the publisher. + * lock held, so it must not block, throw or call back into the publisher. * @return false when the event could not be handed over; the reader then gets a later one. */ - virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) = 0; + virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) noexcept = 0; }; // Counters of one group since it was added. @@ -93,6 +93,8 @@ class Publisher /** * @brief Take a free slot to write the next image into. * @return The slot, or nothing when every slot is in use (the frame should be dropped). + * @throws std::logic_error for a group that has a slot-free callback: its backend owns the + * pool and names slots with claim_slot. */ std::optional acquire_slot(GroupId group, std::size_t stream); @@ -110,7 +112,8 @@ class Publisher /** * @brief Be told when a published slot is no longer held by the ring or any reader, so a * buffer the SDK owns can go back to it at once. Called outside the publisher's lock, on - * the thread whose call released the last hold. + * the thread whose call released the last hold. A group with this callback hands out its + * slots with claim_slot only, so a slot cannot be taken again before its callback has run. */ void set_slot_free_callback(GroupId group, SlotFreeCallback callback); @@ -119,7 +122,7 @@ class Publisher * acquire_slot or claim_slot. * @param timeline capture_ns and sdk_ready_ns from the backend; the rest is filled in here. * @return The frame set's sequence number. - * @throws std::invalid_argument if the images do not match the group. + * @throws std::invalid_argument if the images do not match the group or do not fit its slots. */ std::uint64_t publish(GroupId group, std::span images, const FrameTimeline& timeline); diff --git a/src/core/camera_core/cpp/publisher.cpp b/src/core/camera_core/cpp/publisher.cpp index b3b3ab0766..5222ae681a 100644 --- a/src/core/camera_core/cpp/publisher.cpp +++ b/src/core/camera_core/cpp/publisher.cpp @@ -29,7 +29,33 @@ DeliveryOrder resolve_order(const ReaderOptions& options) return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; } -// A pixel stream names every plane of its format; an encoded stream has none. +// The bytes one row of a plane holds and the number of its rows. +struct PlaneSize +{ + std::uint64_t row_bytes = 0; + std::uint64_t rows = 0; +}; + +PlaneSize plane_size_of(const StreamDescription& stream, std::uint32_t plane) +{ + const std::uint64_t width = stream.width; + const std::uint64_t height = stream.height; + switch (stream.pixel_format) + { + case PixelFormat::BGRA8: + case PixelFormat::RGBA8: + return { width * 4, height }; + case PixelFormat::GRAY8: + return { width, height }; + case PixelFormat::NV12: + // Luma at full size, then one interleaved chroma pair per two by two pixels. + return plane == 0 ? PlaneSize{ width, height } : PlaneSize{ (width + 1) / 2 * 2, (height + 1) / 2 }; + } + return {}; +} + +// A pixel stream names every plane of its format and every plane lies inside the slot; an +// encoded stream has no planes. bool layout_is_valid(const StreamDescription& stream) { if (stream.is_encoded()) @@ -42,7 +68,11 @@ bool layout_is_valid(const StreamDescription& stream) } for (std::uint32_t i = 0; i < stream.plane_count; ++i) { - if (stream.planes[i].stride_bytes == 0) + const PlaneSize size = plane_size_of(stream, i); + const std::uint64_t stride = stream.planes[i].stride_bytes; + // 64-bit sums of 32-bit fields, so nothing here can wrap. + const std::uint64_t end = stream.planes[i].offset_bytes + stride * (size.rows - 1) + size.row_bytes; + if (size.rows == 0 || stride < size.row_bytes || end > stream.slot_bytes) { return false; } @@ -86,12 +116,19 @@ GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t } if (!layout_is_valid(stream)) { - throw std::invalid_argument("camera stream '" + stream.id + "' needs a size and one stride per plane of " + - "its pixel format, or no planes at all when it is encoded"); + throw std::invalid_argument("camera stream '" + stream.id + "' needs a size and every plane of its " + + "pixel format inside the slot, or no planes at all when it is encoded"); } group->slot_count = std::min(group->slot_count, stream.slot_count); group->slots.emplace_back(stream.slot_count); } + // Before any reader, the ring keeps one frame set and the backend keeps its reserve. + if (group->slot_count <= backend_reserve) + { + throw std::invalid_argument("camera group '" + description.id + "' has pools of " + + std::to_string(group->slot_count) + " slots, too few for one frame set and a " + + "backend reserve of " + std::to_string(backend_reserve)); + } std::lock_guard lock(m_mutex); for (const auto& existing : m_groups) @@ -109,6 +146,11 @@ std::optional Publisher::acquire_slot(GroupId group, std::size_t { std::lock_guard lock(m_mutex); Group& target = group_at(group); + if (target.on_free) + { + throw std::logic_error("camera group '" + target.description.id + + "' has a slot-free callback, so its backend names slots with claim_slot"); + } std::vector& slots = target.slots.at(stream); for (std::uint32_t i = 0; i < slots.size(); ++i) { @@ -169,6 +211,12 @@ std::uint64_t Publisher::publish(GroupId group, std::span image throw std::invalid_argument("camera slot " + std::to_string(images[i].slot) + " was not acquired before it was published"); } + if (images[i].bytes_used > target.description.streams[i].slot_bytes) + { + throw std::invalid_argument("camera stream '" + target.description.streams[i].id + "' published " + + std::to_string(images[i].bytes_used) + " bytes into a slot of " + + std::to_string(target.description.streams[i].slot_bytes)); + } } FrameSet set; diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp index 5d8f7c946d..f3bc370d4a 100644 --- a/tests/cpp/core/camera_core/test_publisher.cpp +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -24,7 +24,7 @@ namespace // A reader that keeps every event it is sent. struct RecordingSink : EventSink { - bool send_event(SubscriptionId, const FrameEvent& event) override + bool send_event(SubscriptionId, const FrameEvent& event) noexcept override { if (!accept) { @@ -437,6 +437,11 @@ TEST_CASE("misuse by a backend is rejected", "[unit][camera_core][validation]") REQUIRE_THROWS_AS(publisher.add_group(make_group(MAX_SLOTS_PER_STREAM + 1)), std::invalid_argument); REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 0), std::invalid_argument); + // A pool holds at least one frame set for the ring on top of the backend's reserve. + REQUIRE_THROWS_AS(publisher.add_group(make_group(1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 4), std::invalid_argument); + REQUIRE_NOTHROW(publisher.add_group(make_group(4, 2, "tight"), 3)); + // A pixel stream names every plane of its format; an encoded stream has none. GroupDescription nv12 = make_group(4, 1, "nv12"); nv12.streams[0].pixel_format = PixelFormat::NV12; @@ -460,20 +465,93 @@ TEST_CASE("misuse by a backend is rejected", "[unit][camera_core][validation]") images[0].slot = *publisher.acquire_slot(group, 0); REQUIRE_THROWS_AS(publisher.publish(group, { images.data(), 1 }, {}), std::invalid_argument); REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); // stream 1 not acquired + + // An image cannot be larger than the slot it sits in. + images[1].slot = *publisher.acquire_slot(group, 1); + images[1].bytes_used = publisher.group_description(group).streams[1].slot_bytes + 1; + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); REQUIRE(publisher.stats(group).published == 0); + images[1].bytes_used = 0; + REQUIRE(publisher.publish(group, images, {}) == 1); // the slots were still the backend's to publish REQUIRE_THROWS_AS(publisher.abandon_slot(group, 1, 0), std::invalid_argument); REQUIRE_THROWS_AS(publisher.acquire_slot(99, 0), std::out_of_range); REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 99), std::invalid_argument); } +TEST_CASE("a plane that does not lie inside the slot is rejected", "[unit][camera_core][validation]") +{ + Publisher publisher; + const auto with = [](auto&& change) + { + GroupDescription group = make_group(4, 1); + change(group.streams[0]); + return group; + }; + + // A row is as long as the format needs, and the last row ends inside the slot. + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].stride_bytes = 640 * 4 - 1; })), + std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].offset_bytes = 1; })), + std::invalid_argument); + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.slot_bytes -= 1; })), std::invalid_argument); + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].stride_bytes = 0xFFFFFFFF; })), + std::invalid_argument); + + // Padding between rows is fine as long as the last row still fits. + const auto padded = [](StreamDescription& stream) + { + stream.planes[0].stride_bytes = 4096; + stream.slot_bytes = 4096 * 399 + 640 * 4; + }; + REQUIRE_NOTHROW(publisher.add_group(with(padded))); + + // NV12 with odd sides: chroma rows cover the rounded-up half, and both planes must fit. + const auto nv12 = [](std::uint64_t slot_bytes) + { + return [slot_bytes](StreamDescription& stream) + { + stream.id = "nv12"; + stream.pixel_format = PixelFormat::NV12; + stream.width = 5; + stream.height = 3; + stream.plane_count = 2; + stream.planes[0] = { .offset_bytes = 0, .stride_bytes = 8 }; + stream.planes[1] = { .offset_bytes = 24, .stride_bytes = 8 }; + stream.slot_bytes = slot_bytes; + }; + }; + REQUIRE_THROWS_AS(publisher.add_group(with(nv12(24 + 8 + 5))), std::invalid_argument); // chroma row needs 6 + GroupDescription odd = with(nv12(24 + 8 + 6)); + odd.id = "odd"; + REQUIRE_NOTHROW(publisher.add_group(odd)); +} + +TEST_CASE("a group with a slot-free callback hands out slots by claim only", "[unit][camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + REQUIRE(publisher.acquire_slot(group, 0) == 0u); + publisher.abandon_slot(group, 0, 0); + + publisher.set_slot_free_callback(group, [](std::size_t, std::uint32_t) {}); + REQUIRE_THROWS_AS(publisher.acquire_slot(group, 0), std::logic_error); + REQUIRE_NOTHROW(publisher.claim_slot(group, 0, 0)); + + publisher.set_slot_free_callback(group, {}); + REQUIRE(publisher.acquire_slot(group, 0) == 1u); +} + TEST_CASE("capture and a reader on different threads keep every frame set in order", "[unit][camera_core][threads]") { // A reader as the endpoint will drive it: events arrive on the capture thread, the // reader's own thread takes them and releases. struct QueueSink : EventSink { - bool send_event(SubscriptionId, const FrameEvent& event) override + bool send_event(SubscriptionId, const FrameEvent& event) noexcept override { { std::lock_guard lock(mutex); From 0bd42c220b6c42042b025386bfc59335a96935a6 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:31:40 -0800 Subject: [PATCH 08/16] feat(camera): add camera_ipc with a shared host-memory pool HostPool is one stream's slots in shared host memory: a single anonymous memory file with the slots a page-aligned stride apart. The owner maps it writable and then seals it, so a process that is sent the descriptor can map it for reading only and cannot resize it. HostPoolView is that reader's mapping; both sides derive the layout from the slot count and slot size. UniqueFd owns a file descriptor. An anonymous sealed file is used rather than named shared memory: there is no name to collide or to clean up after a crash, and only a process that was handed the descriptor can map it. Linux only, so the module and its tests are added under a platform check. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/CMakeLists.txt | 5 + src/core/camera_core/README.md | 2 +- src/core/camera_ipc/AGENTS.md | 15 ++ src/core/camera_ipc/CMakeLists.txt | 6 + src/core/camera_ipc/README.md | 43 +++++ src/core/camera_ipc/cpp/CMakeLists.txt | 16 ++ src/core/camera_ipc/cpp/host_pool.cpp | 162 ++++++++++++++++++ .../cpp/inc/camera_ipc/host_pool.hpp | 109 ++++++++++++ .../cpp/inc/camera_ipc/unique_fd.hpp | 73 ++++++++ tests/cpp/core/CMakeLists.txt | 4 + tests/cpp/core/camera_ipc/CMakeLists.txt | 16 ++ tests/cpp/core/camera_ipc/test_host_pool.cpp | 123 +++++++++++++ 12 files changed, 573 insertions(+), 1 deletion(-) create mode 100644 src/core/camera_ipc/AGENTS.md create mode 100644 src/core/camera_ipc/CMakeLists.txt create mode 100644 src/core/camera_ipc/README.md create mode 100644 src/core/camera_ipc/cpp/CMakeLists.txt create mode 100644 src/core/camera_ipc/cpp/host_pool.cpp create mode 100644 src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp create mode 100644 src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp create mode 100644 tests/cpp/core/camera_ipc/CMakeLists.txt create mode 100644 tests/cpp/core/camera_ipc/test_host_pool.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 3393c107b0..a2d49ed641 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -22,6 +22,11 @@ add_subdirectory(plugin_manager) # Camera capture core (standard library only) add_subdirectory(camera_core) +# Camera frames between processes (shared host memory) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_subdirectory(camera_ipc) +endif() + # Build OXR library (depends on oxr_utils for types) add_subdirectory(oxr) diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md index cc0d4fbe2f..d601eaa7a9 100644 --- a/src/core/camera_core/README.md +++ b/src/core/camera_core/README.md @@ -11,7 +11,7 @@ encoders and recorders in other processes read those buffers in place. This module is the part with no memory and no sockets in it: it decides which pool slot holds which frame set and who may still read it. It uses the standard library only. The shared memory -behind the slots and the connection to other processes live in separate modules. +behind the slots and the connection to other processes live in [`camera_ipc`](../camera_ipc). ## Terms diff --git a/src/core/camera_ipc/AGENTS.md b/src/core/camera_ipc/AGENTS.md new file mode 100644 index 0000000000..124e3963a0 --- /dev/null +++ b/src/core/camera_ipc/AGENTS.md @@ -0,0 +1,15 @@ + + +# Agent notes — `camera_ipc` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) first; it explains why frames cross between processes this way. + +- **Linux only.** The module and its tests are added under `CMAKE_SYSTEM_NAME STREQUAL "Linux"`. Platform calls stay here and out of `camera_core`. +- **Readers cannot write.** A pool is sealed before its descriptor leaves the process, and a reader maps it read-only. Do not hand a reader a writable mapping. +- **Derive what both sides can compute.** A pool's layout follows from the slot count and slot size in the stream's description; do not send it separately. +- **Tests need no camera and no GPU**, and carry the `unit` tag. diff --git a/src/core/camera_ipc/CMakeLists.txt b/src/core/camera_ipc/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_ipc/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_ipc/README.md b/src/core/camera_ipc/README.md new file mode 100644 index 0000000000..f97feaa046 --- /dev/null +++ b/src/core/camera_ipc/README.md @@ -0,0 +1,43 @@ + + +# Camera interface: sharing frames between processes + +The process that owns the cameras and the processes that read them are on one machine. This +module is what crosses between them. Linux only. + +[`camera_core`](../camera_core) decides which slot holds which frame; this module provides the +memory behind the slots. + +## Host pool + +`HostPool` is one stream's slots in host memory: a single anonymous file in memory +(`memfd_create`), with the slots laid out one page-aligned stride apart. The owner maps it +writable, then seals it. A reader is sent the descriptor and maps it with `HostPoolView`, read +only. Both sides derive the layout from the slot count and slot size, which the stream's +description carries. + +```cpp +HostPool pool(stream.slot_count, stream.slot_bytes, "camera.left"); +std::span pixels = pool.slot(index); // the owner writes here + +HostPoolView view(std::move(received_descriptor), stream.slot_count, stream.slot_bytes); +std::span image = view.slot(index); // a reader sees the same bytes +``` + +Why an anonymous sealed file and not named shared memory (`shm_open`): + +- **Nothing to clean up.** There is no name to collide with another instance and no file left in + `/dev/shm` after a crash. The memory is freed when the last process that holds it goes away. +- **Readers cannot damage it.** The seals stop any process that receives the descriptor from + mapping it writable, writing to it, or resizing it. One misbehaving reader cannot change what + the others see, and cannot make the owner fault by shrinking the file under it. +- **Access follows the connection.** Only a process that was handed the descriptor can map the + pool. + +## Tests + +`tests/cpp/core/camera_ipc`, run with `ctest -L camera_ipc`. They are tagged `unit`, so +`ctest -L unit` runs them too. diff --git a/src/core/camera_ipc/cpp/CMakeLists.txt b/src/core/camera_ipc/cpp/CMakeLists.txt new file mode 100644 index 0000000000..a17f6255fa --- /dev/null +++ b/src/core/camera_ipc/cpp/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# How camera frames cross between processes on one machine: pools of shared host memory that +# readers can only map read-only. Linux only. +add_library(camera_ipc STATIC + host_pool.cpp +) + +target_include_directories(camera_ipc INTERFACE + $ +) + +add_library(camera::camera_ipc ALIAS camera_ipc) diff --git a/src/core/camera_ipc/cpp/host_pool.cpp b/src/core/camera_ipc/cpp/host_pool.cpp new file mode 100644 index 0000000000..7df3ae0776 --- /dev/null +++ b/src/core/camera_ipc/cpp/host_pool.cpp @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_ipc/host_pool.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// Older C library headers lack this seal although the kernel has it. +#ifndef F_SEAL_FUTURE_WRITE +# define F_SEAL_FUTURE_WRITE 0x0010 +#endif + +namespace core::camera +{ + +namespace +{ + +[[noreturn]] void throw_errno(const std::string& what) +{ + throw std::system_error(errno, std::generic_category(), what); +} + +// Slots start on page boundaries. Both sides derive the layout from the slot size alone. +std::uint64_t slot_stride_for(std::uint64_t slot_bytes) +{ + const auto page = static_cast(::sysconf(_SC_PAGESIZE)); + return (slot_bytes + page - 1) / page * page; +} + +std::uint64_t pool_bytes_for(std::uint32_t slot_count, std::uint64_t slot_stride) +{ + const auto limit = static_cast(std::numeric_limits::max()); + if (slot_count == 0 || slot_stride == 0 || slot_stride > limit / slot_count) + { + throw std::invalid_argument( + "a host pool needs at least one slot of at least one byte, and a size a file can have"); + } + return slot_stride * slot_count; +} + +} // namespace + +HostPool::HostPool(std::uint32_t slot_count, std::uint64_t slot_bytes, const std::string& name) + : m_slot_count(slot_count), m_slot_bytes(slot_bytes), m_slot_stride(slot_stride_for(slot_bytes)) +{ + const std::uint64_t pool_bytes = pool_bytes_for(slot_count, m_slot_stride); + m_fd.reset(::memfd_create(name.c_str(), MFD_CLOEXEC | MFD_ALLOW_SEALING)); + if (!m_fd.valid()) + { + throw_errno("host pool '" + name + "': memfd_create"); + } + if (::ftruncate(m_fd.get(), static_cast(pool_bytes)) != 0) + { + throw_errno("host pool '" + name + "': sizing the pool"); + } + void* memory = ::mmap(nullptr, pool_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, m_fd.get(), 0); + if (memory == MAP_FAILED) + { + throw_errno("host pool '" + name + "': mapping the pool"); + } + // From here on nobody can resize the pool or map it writable; the mapping above stays. + if (::fcntl(m_fd.get(), F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_FUTURE_WRITE | F_SEAL_SEAL) != 0) + { + const int error = errno; + ::munmap(memory, pool_bytes); + throw std::system_error(error, std::generic_category(), "host pool '" + name + "': sealing the pool"); + } + m_memory = static_cast(memory); +} + +HostPool::~HostPool() +{ + ::munmap(m_memory, m_slot_stride * m_slot_count); +} + +std::span HostPool::slot(std::uint32_t index) +{ + if (index >= m_slot_count) + { + throw std::out_of_range("host pool has no slot " + std::to_string(index)); + } + return { m_memory + m_slot_stride * index, m_slot_bytes }; +} + +HostPoolView::HostPoolView(UniqueFd descriptor, std::uint32_t slot_count, std::uint64_t slot_bytes) + : m_slot_count(slot_count), m_slot_bytes(slot_bytes), m_slot_stride(slot_stride_for(slot_bytes)) +{ + const std::uint64_t pool_bytes = pool_bytes_for(slot_count, m_slot_stride); + struct stat info = {}; + if (::fstat(descriptor.get(), &info) != 0) + { + throw_errno("host pool view: inspecting the pool"); + } + // Mapping more than the descriptor holds would fault on first access instead of failing here. + if (static_cast(info.st_size) < pool_bytes) + { + throw std::invalid_argument("host pool view: the descriptor holds less than the pool needs"); + } + void* memory = ::mmap(nullptr, pool_bytes, PROT_READ, MAP_SHARED, descriptor.get(), 0); + if (memory == MAP_FAILED) + { + throw_errno("host pool view: mapping the pool"); + } + m_memory = static_cast(memory); +} + +HostPoolView::~HostPoolView() +{ + unmap(); +} + +HostPoolView::HostPoolView(HostPoolView&& other) noexcept + : m_memory(std::exchange(other.m_memory, nullptr)), + m_slot_count(std::exchange(other.m_slot_count, 0)), + m_slot_bytes(other.m_slot_bytes), + m_slot_stride(other.m_slot_stride) +{ +} + +HostPoolView& HostPoolView::operator=(HostPoolView&& other) noexcept +{ + if (this != &other) + { + unmap(); + m_memory = std::exchange(other.m_memory, nullptr); + m_slot_count = std::exchange(other.m_slot_count, 0); + m_slot_bytes = other.m_slot_bytes; + m_slot_stride = other.m_slot_stride; + } + return *this; +} + +std::span HostPoolView::slot(std::uint32_t index) const +{ + if (index >= m_slot_count) + { + throw std::out_of_range("host pool view has no slot " + std::to_string(index)); + } + return { m_memory + m_slot_stride * index, m_slot_bytes }; +} + +void HostPoolView::unmap() +{ + if (m_memory != nullptr) + { + ::munmap(const_cast(m_memory), m_slot_stride * m_slot_count); + m_memory = nullptr; + m_slot_count = 0; + } +} + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp b/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp new file mode 100644 index 0000000000..54694f1436 --- /dev/null +++ b/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "unique_fd.hpp" + +#include +#include +#include +#include + +namespace core::camera +{ + +/** + * @brief A pool of equally sized slots in one block of shared host memory. + * + * The process that creates the pool writes the slots. Any process that is sent the pool's + * descriptor can map it, but only for reading: the memory is sealed against new writable + * mappings, writes and resizing. + */ +class HostPool +{ +public: + /** + * @param name Shows up in /proc for debugging; it does not have to be unique. + * @throws std::invalid_argument for an empty pool. + * @throws std::system_error if the memory cannot be allocated or sealed. + */ + HostPool(std::uint32_t slot_count, std::uint64_t slot_bytes, const std::string& name); + ~HostPool(); + + HostPool(const HostPool&) = delete; + HostPool& operator=(const HostPool&) = delete; + + std::uint32_t slot_count() const + { + return m_slot_count; + } + + std::uint64_t slot_bytes() const + { + return m_slot_bytes; + } + + /** + * @brief The writable memory of one slot. + * @throws std::out_of_range for a slot the pool does not have. + */ + std::span slot(std::uint32_t index); + + /** + * @brief The pool's descriptor, to send to another process. The pool keeps owning it. + */ + int descriptor() const + { + return m_fd.get(); + } + +private: + UniqueFd m_fd; + std::byte* m_memory = nullptr; + std::uint32_t m_slot_count = 0; + std::uint64_t m_slot_bytes = 0; + std::uint64_t m_slot_stride = 0; // from the start of one slot to the start of the next +}; + +/** + * @brief Another process's read-only mapping of a HostPool, made from the descriptor it was sent. + */ +class HostPoolView +{ +public: + /** + * @param descriptor Of the pool. It is closed once the pool is mapped. + * @param slot_count, slot_bytes As the pool was created with. + * @throws std::system_error if the descriptor cannot be mapped. + * @throws std::invalid_argument if the descriptor holds less than the pool needs. + */ + HostPoolView(UniqueFd descriptor, std::uint32_t slot_count, std::uint64_t slot_bytes); + ~HostPoolView(); + + HostPoolView(HostPoolView&& other) noexcept; + HostPoolView& operator=(HostPoolView&& other) noexcept; + HostPoolView(const HostPoolView&) = delete; + HostPoolView& operator=(const HostPoolView&) = delete; + + std::uint32_t slot_count() const + { + return m_slot_count; + } + + /** + * @brief The memory of one slot. + * @throws std::out_of_range for a slot the pool does not have. + */ + std::span slot(std::uint32_t index) const; + +private: + void unmap(); + + const std::byte* m_memory = nullptr; + std::uint32_t m_slot_count = 0; + std::uint64_t m_slot_bytes = 0; + std::uint64_t m_slot_stride = 0; +}; + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp b/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp new file mode 100644 index 0000000000..387bf46515 --- /dev/null +++ b/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace core::camera +{ + +// Owns one file descriptor and closes it. +class UniqueFd +{ +public: + UniqueFd() = default; + + explicit UniqueFd(int fd) : m_fd(fd) + { + } + + UniqueFd(UniqueFd&& other) noexcept : m_fd(other.release()) + { + } + + UniqueFd& operator=(UniqueFd&& other) noexcept + { + if (this != &other) + { + reset(other.release()); + } + return *this; + } + + UniqueFd(const UniqueFd&) = delete; + UniqueFd& operator=(const UniqueFd&) = delete; + + ~UniqueFd() + { + reset(); + } + + int get() const + { + return m_fd; + } + + bool valid() const + { + return m_fd >= 0; + } + + // Gives the descriptor up without closing it. + int release() + { + const int fd = m_fd; + m_fd = -1; + return fd; + } + + void reset(int fd = -1) + { + if (m_fd >= 0) + { + ::close(m_fd); + } + m_fd = fd; + } + +private: + int m_fd = -1; +}; + +} // namespace core::camera diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index e21545f14f..705bead328 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -15,6 +15,10 @@ add_subdirectory(log_bridge) add_subdirectory(plugin_manager) add_subdirectory(camera_core) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_subdirectory(camera_ipc) +endif() + if(BUILD_OVXR_CAMERA_QUADS) add_subdirectory(ovxr_pip_layer) endif() diff --git a/tests/cpp/core/camera_ipc/CMakeLists.txt b/tests/cpp/core/camera_ipc/CMakeLists.txt new file mode 100644 index 0000000000..b26697a7fe --- /dev/null +++ b/tests/cpp/core/camera_ipc/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_ipc_tests + test_host_pool.cpp +) + +target_link_libraries(camera_ipc_tests PRIVATE + camera::camera_ipc + Catch2::Catch2WithMain +) + +message(STATUS "camera_ipc_tests target enabled with Catch2") +catch_discover_tests(camera_ipc_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_ipc/test_host_pool.cpp b/tests/cpp/core/camera_ipc/test_host_pool.cpp new file mode 100644 index 0000000000..6b2e4fca94 --- /dev/null +++ b/tests/cpp/core/camera_ipc/test_host_pool.cpp @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +// Not a whole number of pages, so slots are padded apart. +constexpr std::uint64_t SLOT_BYTES = 5000; + +// What another process ends up with after the pool's descriptor was sent to it. +UniqueFd duplicate_descriptor(const HostPool& pool) +{ + UniqueFd result(::fcntl(pool.descriptor(), F_DUPFD_CLOEXEC, 0)); + REQUIRE(result.valid()); + return result; +} + +} // namespace + +TEST_CASE("a view sees what the owner writes", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(3, SLOT_BYTES, "camera_ipc_test"); + REQUIRE(pool.slot_count() == 3); + REQUIRE(pool.slot_bytes() == SLOT_BYTES); + + const HostPoolView view(duplicate_descriptor(pool), 3, SLOT_BYTES); + REQUIRE(view.slot_count() == 3); + + // Slots start zeroed and are separate from each other. + const auto is_zero = [](std::byte value) { return value == std::byte{ 0 }; }; + REQUIRE(std::all_of(view.slot(0).begin(), view.slot(0).end(), is_zero)); + + std::fill(pool.slot(1).begin(), pool.slot(1).end(), std::byte{ 0x5a }); + REQUIRE(view.slot(1).size() == SLOT_BYTES); + REQUIRE(view.slot(1).front() == std::byte{ 0x5a }); + REQUIRE(view.slot(1).back() == std::byte{ 0x5a }); + REQUIRE(std::all_of(view.slot(0).begin(), view.slot(0).end(), is_zero)); + REQUIRE(std::all_of(view.slot(2).begin(), view.slot(2).end(), is_zero)); + + // The mapping is live: a later write shows up without mapping again. + pool.slot(2)[100] = std::byte{ 0x11 }; + REQUIRE(view.slot(2)[100] == std::byte{ 0x11 }); +} + +TEST_CASE("slots start on page boundaries", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + const auto page = static_cast(::sysconf(_SC_PAGESIZE)); + REQUIRE(reinterpret_cast(pool.slot(0).data()) % page == 0); + REQUIRE(reinterpret_cast(pool.slot(1).data()) % page == 0); + REQUIRE(pool.slot(1).data() >= pool.slot(0).data() + SLOT_BYTES); +} + +TEST_CASE("a process that is sent the pool cannot write to it or resize it", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(1, SLOT_BYTES, "camera_ipc_test"); + const UniqueFd received = duplicate_descriptor(pool); + + void* writable = ::mmap(nullptr, SLOT_BYTES, PROT_READ | PROT_WRITE, MAP_SHARED, received.get(), 0); + REQUIRE(writable == MAP_FAILED); + REQUIRE(errno == EPERM); + + const char byte = 1; + REQUIRE(::write(received.get(), &byte, 1) == -1); + REQUIRE(::ftruncate(received.get(), 16) == -1); + REQUIRE(::ftruncate(received.get(), static_cast(16 * SLOT_BYTES)) == -1); + + // The owner still can. + pool.slot(0)[0] = std::byte{ 0x7f }; + const HostPoolView view(duplicate_descriptor(pool), 1, SLOT_BYTES); + REQUIRE(view.slot(0)[0] == std::byte{ 0x7f }); +} + +TEST_CASE("a view outlives the pool and can be moved", "[unit][camera_ipc][host_pool]") +{ + std::vector views; + { + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + pool.slot(1)[7] = std::byte{ 0x42 }; + views.emplace_back(duplicate_descriptor(pool), 2, SLOT_BYTES); + } + // The memory lives as long as anything maps it. + HostPoolView moved = std::move(views.front()); + REQUIRE(views.front().slot_count() == 0); + REQUIRE(moved.slot_count() == 2); + REQUIRE(moved.slot(1)[7] == std::byte{ 0x42 }); +} + +TEST_CASE("misuse of a host pool is rejected", "[unit][camera_ipc][host_pool]") +{ + REQUIRE_THROWS_AS(HostPool(0, SLOT_BYTES, "camera_ipc_test"), std::invalid_argument); + REQUIRE_THROWS_AS(HostPool(1, 0, "camera_ipc_test"), std::invalid_argument); + + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + REQUIRE_THROWS_AS(pool.slot(2), std::out_of_range); + + const HostPoolView view(duplicate_descriptor(pool), 2, SLOT_BYTES); + REQUIRE_THROWS_AS(view.slot(2), std::out_of_range); + + // A descriptor that holds less than the pool claims would fault on access, so it is refused. + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 3, SLOT_BYTES), std::invalid_argument); + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 2, 4 * SLOT_BYTES), std::invalid_argument); + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 0, SLOT_BYTES), std::invalid_argument); + + REQUIRE_THROWS_AS(HostPoolView(UniqueFd(-1), 2, SLOT_BYTES), std::system_error); +} From 6a873c2fecaca1aaba10ca5cec322283297fa2dd Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:32:40 -0800 Subject: [PATCH 09/16] feat(camera): add a message socket that carries file descriptors MessageSocket and MessageListener are a connection between two processes on one machine: a Unix domain socket of the sequenced-packet kind that carries whole messages in order, each with optional file descriptors. This is how a pool's descriptor, and later the per-frame events, reach a reader. - Sending never blocks. A full queue and a closed peer are statuses, so capture cannot stall on a reader that stopped reading. - Receiving waits with a timeout and returns the descriptors as owned UniqueFds. A message that does not fit is an error, not a silent cut. - A lock file next to the socket file decides which process owns the path. The kernel drops the lock when its holder dies, so a socket file found while holding it is stale and is replaced; anything that is not a socket is left alone. - The socket file is created with mode 0600. The README explains why a Unix socket and shared memory are used rather than a messaging library. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/CMakeLists.txt | 2 +- src/core/camera_ipc/AGENTS.md | 3 + src/core/camera_ipc/README.md | 48 ++- src/core/camera_ipc/cpp/CMakeLists.txt | 6 +- .../cpp/inc/camera_ipc/message_socket.hpp | 129 +++++++ src/core/camera_ipc/cpp/message_socket.cpp | 283 +++++++++++++++ tests/cpp/core/camera_ipc/CMakeLists.txt | 4 + .../core/camera_ipc/test_message_socket.cpp | 337 ++++++++++++++++++ 8 files changed, 808 insertions(+), 4 deletions(-) create mode 100644 src/core/camera_ipc/cpp/inc/camera_ipc/message_socket.hpp create mode 100644 src/core/camera_ipc/cpp/message_socket.cpp create mode 100644 tests/cpp/core/camera_ipc/test_message_socket.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index a2d49ed641..602cc1969a 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -22,7 +22,7 @@ add_subdirectory(plugin_manager) # Camera capture core (standard library only) add_subdirectory(camera_core) -# Camera frames between processes (shared host memory) +# Camera frames and messages between processes (shared host memory, descriptor passing) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) endif() diff --git a/src/core/camera_ipc/AGENTS.md b/src/core/camera_ipc/AGENTS.md index 124e3963a0..70eb3dce2c 100644 --- a/src/core/camera_ipc/AGENTS.md +++ b/src/core/camera_ipc/AGENTS.md @@ -10,6 +10,9 @@ SPDX-License-Identifier: Apache-2.0 Read [`README.md`](README.md) first; it explains why frames cross between processes this way. - **Linux only.** The module and its tests are added under `CMAKE_SYSTEM_NAME STREQUAL "Linux"`. Platform calls stay here and out of `camera_core`. +- **Frame memory crosses as a file descriptor on the module's Unix socket.** Do not add a messaging or shared-memory library for it; a new kind of memory is a new pool type behind the same connection. +- **Nothing on the capture path waits for a reader.** Sends do not block; a full queue and a closed peer are statuses, not exceptions. +- **A lock file owns the socket path.** Remove a socket file only while holding it, and never remove anything that is not a socket. - **Readers cannot write.** A pool is sealed before its descriptor leaves the process, and a reader maps it read-only. Do not hand a reader a writable mapping. - **Derive what both sides can compute.** A pool's layout follows from the slot count and slot size in the stream's description; do not send it separately. - **Tests need no camera and no GPU**, and carry the `unit` tag. diff --git a/src/core/camera_ipc/README.md b/src/core/camera_ipc/README.md index f97feaa046..f0a20e551f 100644 --- a/src/core/camera_ipc/README.md +++ b/src/core/camera_ipc/README.md @@ -9,7 +9,7 @@ The process that owns the cameras and the processes that read them are on one ma module is what crosses between them. Linux only. [`camera_core`](../camera_core) decides which slot holds which frame; this module provides the -memory behind the slots. +memory behind the slots and the connection that tells a reader which slot to look at. ## Host pool @@ -37,6 +37,52 @@ Why an anonymous sealed file and not named shared memory (`shm_open`): - **Access follows the connection.** Only a process that was handed the descriptor can map the pool. +## Message socket + +`MessageListener` and `MessageSocket` are the connection: a Unix domain socket of the +sequenced-packet kind (`SOCK_SEQPACKET`). The owner listens on a socket file, each reader +connects, and either side sends messages with file descriptors attached. + +```cpp +MessageListener listener(path); // the owner +auto reader = listener.accept(timeout); +const int descriptors[] = { pool.descriptor() }; +reader->send(description_bytes, descriptors); + +MessageSocket socket = MessageSocket::connect(path); // a reader +ReceivedMessage message = socket.receive(buffer, timeout); +HostPoolView view(std::move(message.descriptors[0]), slot_count, slot_bytes); +``` + +- **Whole messages.** One `send` is one `receive`, in order, so there is no framing to get wrong. +- **Sending never blocks.** If a reader stops draining its queue the sender gets `WOULD_BLOCK` + and moves on; capture never waits for a reader. +- **A dead peer is seen at once.** When a process exits or crashes the other end reads + `CLOSED`, which is how the frames a crashed reader held are taken back. +- **Owner only.** The socket file is created with mode `0600`. +- **One listener per path.** A lock file next to the socket file (`.lock`) decides who + owns the path, so of two processes that start together exactly one listens, and a socket + file left by a crashed process is replaced safely. Anything else found at the path is left + alone. The lock file stays behind, empty. + +## Why a socket and shared memory, and not a messaging library + +Three things have to cross between the processes: the frame memory, a small event per frame +set, and the news that a peer has gone. + +Frame memory is named by a file descriptor: a sealed memory file here, and the same holds for +the buffers of hardware capture and codec stacks (DMA-BUF) and for GPU memory that can be +exported. Attaching a descriptor to a message on a Unix socket is the standard way to hand one +to another process, so a socket is needed whatever else is used. Once it is there it also +carries the events and reports dead peers, which leaves a messaging library nothing to do. + +| Considered | Why it is not used | +|---|---| +| iceoryx2 | It moves self-contained data through host memory that it allocates. Frame memory that exists only as a descriptor would still need a socket beside it, and its C++ binding adds a Rust toolchain to the build. | +| CUDA inter-process handles | A way to share one kind of GPU memory, not a way to connect processes: the handle still has to travel over some connection, and host memory and DMA-BUF are not covered. | +| ZeroMQ, gRPC, DDS | They copy the payload or share host memory only, and none of them pass descriptors. | +| PipeWire, GStreamer | The same mechanism underneath, wrapped in a media graph and, for PipeWire, a daemon. A reader here needs neither. | + ## Tests `tests/cpp/core/camera_ipc`, run with `ctest -L camera_ipc`. They are tagged `unit`, so diff --git a/src/core/camera_ipc/cpp/CMakeLists.txt b/src/core/camera_ipc/cpp/CMakeLists.txt index a17f6255fa..69db298406 100644 --- a/src/core/camera_ipc/cpp/CMakeLists.txt +++ b/src/core/camera_ipc/cpp/CMakeLists.txt @@ -3,10 +3,12 @@ cmake_minimum_required(VERSION 3.20) -# How camera frames cross between processes on one machine: pools of shared host memory that -# readers can only map read-only. Linux only. +# How camera frames and messages cross between processes on one machine: pools of shared host +# memory that readers can only map read-only, and a socket that carries whole messages with +# file descriptors. Linux only. add_library(camera_ipc STATIC host_pool.cpp + message_socket.cpp ) target_include_directories(camera_ipc INTERFACE diff --git a/src/core/camera_ipc/cpp/inc/camera_ipc/message_socket.hpp b/src/core/camera_ipc/cpp/inc/camera_ipc/message_socket.hpp new file mode 100644 index 0000000000..0a5954dd08 --- /dev/null +++ b/src/core/camera_ipc/cpp/inc/camera_ipc/message_socket.hpp @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "unique_fd.hpp" + +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// A message carries at most this many descriptors. +inline constexpr std::size_t MAX_DESCRIPTORS_PER_MESSAGE = 64; + +enum class SendStatus +{ + SENT, + WOULD_BLOCK, // the peer's queue is full; nothing was sent + CLOSED, // the peer is gone +}; + +enum class ReceiveStatus +{ + MESSAGE, + TIMEOUT, + CLOSED, // the peer is gone +}; + +struct ReceivedMessage +{ + ReceiveStatus status = ReceiveStatus::TIMEOUT; + std::size_t bytes = 0; // of the payload, when status is MESSAGE + std::vector descriptors; // the ones that came with the message +}; + +/** + * @brief One end of a connection between two processes on this machine. It carries whole + * messages in order, each with optional file descriptors. + */ +class MessageSocket +{ +public: + /** + * @brief Connect to a MessageListener. + * @throws std::system_error if nothing listens on the path. + */ + static MessageSocket connect(const std::string& path); + + explicit MessageSocket(UniqueFd fd); + + /** + * @brief Send one message. Never blocks. + * @throws std::invalid_argument for an empty payload (it would read as a closed connection) + * or more than MAX_DESCRIPTORS_PER_MESSAGE descriptors. + * @throws std::system_error for a failure other than a full queue or a closed peer. + */ + SendStatus send(std::span payload, std::span descriptors = {}); + + /** + * @brief Wait up to `timeout` for one message and copy its payload into `buffer`. + * @throws std::length_error if the message does not fit the buffer. + * @throws std::system_error for a failure other than a closed peer. + */ + ReceivedMessage receive(std::span buffer, std::chrono::milliseconds timeout); + + // For waiting on several sockets at once. + int descriptor() const + { + return m_fd.get(); + } + +private: + UniqueFd m_fd; +}; + +/** + * @brief Accepts connections on a socket file. Only processes of the same user can connect. + * + * A lock file next to the socket file (`.lock`) decides which process owns the path, so + * of two processes that start at the same moment exactly one listens. The lock file stays + * behind, empty, for the next listener to lock. + */ +class MessageListener +{ +public: + /** + * @brief Start listening on `path`. A socket file left behind by a process that has gone is + * replaced; anything else found at the path is left alone. + * @throws std::system_error with EADDRINUSE if another process listens on the path, with + * EEXIST if the path holds something that is not a socket, or with the cause if the path + * cannot be locked or bound. + */ + explicit MessageListener(const std::string& path); + + // Removes the socket file. + ~MessageListener(); + + MessageListener(const MessageListener&) = delete; + MessageListener& operator=(const MessageListener&) = delete; + + /** + * @brief Wait up to `timeout` for a connection. + * @return The connection, or nothing on timeout. + */ + std::optional accept(std::chrono::milliseconds timeout); + + int descriptor() const + { + return m_fd.get(); + } + + const std::string& path() const + { + return m_path; + } + +private: + UniqueFd m_lock; // declared first, so it is released after the socket is closed + UniqueFd m_fd; + std::string m_path; +}; + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/message_socket.cpp b/src/core/camera_ipc/cpp/message_socket.cpp new file mode 100644 index 0000000000..05425aa36d --- /dev/null +++ b/src/core/camera_ipc/cpp/message_socket.cpp @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_ipc/message_socket.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +[[noreturn]] void throw_errno(const std::string& what) +{ + throw std::system_error(errno, std::generic_category(), what); +} + +sockaddr_un address_of(const std::string& path) +{ + sockaddr_un address = {}; + address.sun_family = AF_UNIX; + if (path.empty() || path.size() >= sizeof(address.sun_path)) + { + throw std::invalid_argument("socket path '" + path + "' is empty or too long"); + } + std::memcpy(address.sun_path, path.data(), path.size()); + return address; +} + +UniqueFd new_socket(int extra_flags = 0) +{ + UniqueFd fd(::socket(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC | extra_flags, 0)); + if (!fd.valid()) + { + throw_errno("creating a socket"); + } + return fd; +} + +// Waits until the descriptor is readable. Returns false on timeout. +bool wait_readable(int fd, std::chrono::milliseconds timeout) +{ + using Milliseconds = std::chrono::milliseconds; + const Milliseconds longest(std::numeric_limits::max()); // what poll() can wait + const auto deadline = std::chrono::steady_clock::now() + std::clamp(timeout, Milliseconds::zero(), longest); + pollfd entry = { fd, POLLIN, 0 }; + for (;;) + { + // An interrupted wait resumes with the time that is left, not the whole timeout again. + const auto left = std::chrono::ceil(deadline - std::chrono::steady_clock::now()); + const int ready = ::poll(&entry, 1, static_cast(std::max(left, Milliseconds::zero()).count())); + if (ready > 0) + { + return true; + } + if (ready == 0) + { + return false; + } + if (errno != EINTR) + { + throw_errno("waiting on a socket"); + } + } +} + +} // namespace + +MessageSocket MessageSocket::connect(const std::string& path) +{ + const sockaddr_un address = address_of(path); + UniqueFd fd = new_socket(); + if (::connect(fd.get(), reinterpret_cast(&address), sizeof(address)) != 0) + { + throw_errno("connecting to '" + path + "'"); + } + return MessageSocket(std::move(fd)); +} + +MessageSocket::MessageSocket(UniqueFd fd) : m_fd(std::move(fd)) +{ +} + +SendStatus MessageSocket::send(std::span payload, std::span descriptors) +{ + if (payload.empty() || descriptors.size() > MAX_DESCRIPTORS_PER_MESSAGE) + { + throw std::invalid_argument("a message needs a payload and carries at most " + + std::to_string(MAX_DESCRIPTORS_PER_MESSAGE) + " descriptors"); + } + iovec data = { const_cast(payload.data()), payload.size() }; + msghdr message = {}; + message.msg_iov = &data; + message.msg_iovlen = 1; + + alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int) * MAX_DESCRIPTORS_PER_MESSAGE)]; + if (!descriptors.empty()) + { + message.msg_control = control; + message.msg_controllen = CMSG_SPACE(descriptors.size_bytes()); + cmsghdr* header = CMSG_FIRSTHDR(&message); + header->cmsg_level = SOL_SOCKET; + header->cmsg_type = SCM_RIGHTS; + header->cmsg_len = CMSG_LEN(descriptors.size_bytes()); + std::memcpy(CMSG_DATA(header), descriptors.data(), descriptors.size_bytes()); + } + + for (;;) + { + if (::sendmsg(m_fd.get(), &message, MSG_DONTWAIT | MSG_NOSIGNAL) >= 0) + { + return SendStatus::SENT; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + return SendStatus::WOULD_BLOCK; + } + if (errno == EPIPE || errno == ECONNRESET || errno == ENOTCONN) + { + return SendStatus::CLOSED; + } + if (errno != EINTR) + { + throw_errno("sending a message"); + } + } +} + +ReceivedMessage MessageSocket::receive(std::span buffer, std::chrono::milliseconds timeout) +{ + ReceivedMessage result; + if (!wait_readable(m_fd.get(), timeout)) + { + return result; + } + iovec data = { buffer.data(), buffer.size() }; + alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int) * MAX_DESCRIPTORS_PER_MESSAGE)]; + msghdr message = {}; + message.msg_iov = &data; + message.msg_iovlen = 1; + message.msg_control = control; + message.msg_controllen = sizeof(control); + + ssize_t received = 0; + do + { + received = ::recvmsg(m_fd.get(), &message, MSG_DONTWAIT | MSG_CMSG_CLOEXEC); + } while (received < 0 && errno == EINTR); + if (received < 0) + { + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + return result; + } + if (errno == ECONNRESET) + { + result.status = ReceiveStatus::CLOSED; + return result; + } + throw_errno("receiving a message"); + } + + // Take ownership of every descriptor first, so none leaks if the message is rejected below. + for (cmsghdr* header = CMSG_FIRSTHDR(&message); header != nullptr; header = CMSG_NXTHDR(&message, header)) + { + if (header->cmsg_level != SOL_SOCKET || header->cmsg_type != SCM_RIGHTS) + { + continue; + } + const std::size_t count = (header->cmsg_len - CMSG_LEN(0)) / sizeof(int); + for (std::size_t i = 0; i < count; ++i) + { + int fd = -1; + std::memcpy(&fd, CMSG_DATA(header) + i * sizeof(int), sizeof(int)); + result.descriptors.emplace_back(fd); + } + } + // Checked before the empty read below: a cut message can also come back with no bytes. + if ((message.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) != 0) + { + throw std::length_error("received a message larger than the buffer or with too many descriptors"); + } + if (received == 0 && result.descriptors.empty()) + { + result.status = ReceiveStatus::CLOSED; // an orderly shutdown reads as an empty message + return result; + } + result.status = ReceiveStatus::MESSAGE; + result.bytes = static_cast(received); + return result; +} + +MessageListener::MessageListener(const std::string& path) : m_path(path) +{ + const sockaddr_un address = address_of(path); + + // The lock decides who owns the path. The kernel drops it when its holder dies, so a socket + // file found while holding it was left by a process that is gone. + const std::string lock_path = path + ".lock"; + m_lock.reset(::open(lock_path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR)); + if (!m_lock.valid()) + { + throw_errno("opening '" + lock_path + "'"); + } + if (::flock(m_lock.get(), LOCK_EX | LOCK_NB) != 0) + { + if (errno == EWOULDBLOCK) + { + throw std::system_error(EADDRINUSE, std::generic_category(), "another process listens on '" + path + "'"); + } + throw_errno("locking '" + lock_path + "'"); + } + struct stat entry = {}; + if (::lstat(path.c_str(), &entry) == 0) + { + if (!S_ISSOCK(entry.st_mode)) + { + throw std::system_error(EEXIST, std::generic_category(), "'" + path + "' exists and is not a socket"); + } + ::unlink(path.c_str()); + } + else if (errno != ENOENT) + { + throw_errno("inspecting '" + path + "'"); + } + + // Not blocking: a connection that goes away between the wait and the accept must not stall accept(). + m_fd = new_socket(SOCK_NONBLOCK); + // The socket file is created with the socket's own mode: only this user may connect. + if (::fchmod(m_fd.get(), S_IRUSR | S_IWUSR) != 0) + { + throw_errno("restricting '" + path + "'"); + } + if (::bind(m_fd.get(), reinterpret_cast(&address), sizeof(address)) != 0) + { + throw_errno("binding '" + path + "'"); + } + if (::listen(m_fd.get(), SOMAXCONN) != 0) + { + const int error = errno; + ::unlink(path.c_str()); + throw std::system_error(error, std::generic_category(), "listening on '" + path + "'"); + } +} + +MessageListener::~MessageListener() +{ + ::unlink(m_path.c_str()); +} + +std::optional MessageListener::accept(std::chrono::milliseconds timeout) +{ + if (!wait_readable(m_fd.get(), timeout)) + { + return std::nullopt; + } + UniqueFd connection(::accept4(m_fd.get(), nullptr, nullptr, SOCK_CLOEXEC)); + if (!connection.valid()) + { + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED || errno == EINTR) + { + return std::nullopt; + } + throw_errno("accepting a connection on '" + m_path + "'"); + } + return MessageSocket(std::move(connection)); +} + +} // namespace core::camera diff --git a/tests/cpp/core/camera_ipc/CMakeLists.txt b/tests/cpp/core/camera_ipc/CMakeLists.txt index b26697a7fe..ea7c1cacd4 100644 --- a/tests/cpp/core/camera_ipc/CMakeLists.txt +++ b/tests/cpp/core/camera_ipc/CMakeLists.txt @@ -3,13 +3,17 @@ cmake_minimum_required(VERSION 3.20) +find_package(Threads REQUIRED) + add_executable(camera_ipc_tests test_host_pool.cpp + test_message_socket.cpp ) target_link_libraries(camera_ipc_tests PRIVATE camera::camera_ipc Catch2::Catch2WithMain + Threads::Threads ) message(STATUS "camera_ipc_tests target enabled with Catch2") diff --git a/tests/cpp/core/camera_ipc/test_message_socket.cpp b/tests/cpp/core/camera_ipc/test_message_socket.cpp new file mode 100644 index 0000000000..6154c63dd6 --- /dev/null +++ b/tests/cpp/core/camera_ipc/test_message_socket.cpp @@ -0,0 +1,337 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; + +namespace +{ + +// A directory for socket files that is removed with everything in it. +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_ipc_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "endpoint.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +std::span bytes_of(std::string_view text) +{ + return std::as_bytes(std::span(text.data(), text.size())); +} + +std::string text_of(std::span bytes, std::size_t count) +{ + return { reinterpret_cast(bytes.data()), count }; +} + +struct Connection +{ + TemporaryDirectory directory; + MessageListener listener{ directory.socket_path() }; + MessageSocket client = MessageSocket::connect(directory.socket_path()); + MessageSocket server = accept(); + + MessageSocket accept() + { + auto accepted = listener.accept(1s); + REQUIRE(accepted.has_value()); + return std::move(*accepted); + } +}; + +} // namespace + +TEST_CASE("messages arrive whole and in order", "[unit][camera_ipc][socket]") +{ + Connection connection; + REQUIRE(connection.client.send(bytes_of("first")) == SendStatus::SENT); + REQUIRE(connection.client.send(bytes_of("second message")) == SendStatus::SENT); + + std::array buffer; + ReceivedMessage received = connection.server.receive(buffer, 1s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(text_of(buffer, received.bytes) == "first"); + REQUIRE(received.descriptors.empty()); + + received = connection.server.receive(buffer, 1s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(text_of(buffer, received.bytes) == "second message"); + + // And the other way. + REQUIRE(connection.server.send(bytes_of("reply")) == SendStatus::SENT); + received = connection.client.receive(buffer, 1s); + REQUIRE(text_of(buffer, received.bytes) == "reply"); +} + +TEST_CASE("descriptors travel with a message", "[unit][camera_ipc][socket]") +{ + Connection connection; + HostPool left(2, 4096, "camera_ipc_test_left"); + HostPool right(2, 4096, "camera_ipc_test_right"); + left.slot(1)[5] = std::byte{ 0x33 }; + right.slot(0)[9] = std::byte{ 0x44 }; + + const std::array descriptors = { left.descriptor(), right.descriptor() }; + REQUIRE(connection.server.send(bytes_of("pools"), descriptors) == SendStatus::SENT); + + std::array buffer; + ReceivedMessage received = connection.client.receive(buffer, 1s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(text_of(buffer, received.bytes) == "pools"); + REQUIRE(received.descriptors.size() == 2); + + // The receiver's descriptors are its own, arrive in order and map the same memory. + REQUIRE(received.descriptors[0].get() != left.descriptor()); + REQUIRE(received.descriptors[1].get() != right.descriptor()); + const HostPoolView left_view(std::move(received.descriptors[0]), 2, 4096); + const HostPoolView right_view(std::move(received.descriptors[1]), 2, 4096); + REQUIRE(left_view.slot(1)[5] == std::byte{ 0x33 }); + REQUIRE(right_view.slot(0)[9] == std::byte{ 0x44 }); +} + +TEST_CASE("send never blocks when the peer does not read", "[unit][camera_ipc][socket]") +{ + Connection connection; + const std::vector payload(1024, std::byte{ 1 }); + + SendStatus status = SendStatus::SENT; + int sent = 0; + while (status == SendStatus::SENT && sent < 1'000'000) + { + status = connection.client.send(payload); + sent += status == SendStatus::SENT ? 1 : 0; + } + REQUIRE(status == SendStatus::WOULD_BLOCK); + REQUIRE(sent > 0); + + // Room appears as soon as the peer reads. + std::array buffer; + REQUIRE(connection.server.receive(buffer, 1s).status == ReceiveStatus::MESSAGE); + REQUIRE(connection.client.send(payload) == SendStatus::SENT); +} + +TEST_CASE("a closed peer is reported, not thrown", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + MessageListener listener(directory.socket_path()); + std::optional server; + { + MessageSocket client = MessageSocket::connect(directory.socket_path()); + server = listener.accept(1s); + REQUIRE(server.has_value()); + REQUIRE(client.send(bytes_of("last words")) == SendStatus::SENT); + } + + // What was sent before the peer closed is still delivered. + std::array buffer; + ReceivedMessage received = server->receive(buffer, 1s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(text_of(buffer, received.bytes) == "last words"); + + REQUIRE(server->receive(buffer, 1s).status == ReceiveStatus::CLOSED); + REQUIRE(server->send(bytes_of("anyone?")) == SendStatus::CLOSED); +} + +TEST_CASE("receive and accept time out when nothing arrives", "[unit][camera_ipc][socket]") +{ + Connection connection; + std::array buffer; + REQUIRE(connection.server.receive(buffer, 10ms).status == ReceiveStatus::TIMEOUT); + REQUIRE(connection.server.receive(buffer, 0ms).status == ReceiveStatus::TIMEOUT); + REQUIRE_FALSE(connection.listener.accept(10ms).has_value()); +} + +TEST_CASE("a listener replaces a stale socket file but not a live one", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + { + MessageListener live(path); + try + { + MessageListener second(path); + FAIL("a second listener took over a live endpoint"); + } + catch (const std::system_error& error) + { + REQUIRE(error.code().value() == EADDRINUSE); + } + // The live listener still works after the refused attempt. + MessageSocket client = MessageSocket::connect(path); + REQUIRE(live.accept(1s).has_value()); + } + REQUIRE_FALSE(std::filesystem::exists(path)); // a listener removes its file + + // A process that died without cleaning up leaves the file behind. + { + sockaddr_un address = {}; + address.sun_family = AF_UNIX; + std::memcpy(address.sun_path, path.data(), path.size()); + const UniqueFd dead(::socket(AF_UNIX, SOCK_SEQPACKET, 0)); + REQUIRE(::bind(dead.get(), reinterpret_cast(&address), sizeof(address)) == 0); + } + REQUIRE(std::filesystem::exists(path)); + MessageListener replacement(path); + MessageSocket client = MessageSocket::connect(path); + REQUIRE(replacement.accept(1s).has_value()); +} + +TEST_CASE("the lock decides who listens, not the socket file", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + { + MessageListener live(path); + // Even with the socket file gone, the path still belongs to the live listener. + REQUIRE(std::filesystem::remove(path)); + REQUIRE_THROWS_AS(MessageListener(path), std::system_error); + // A refused attempt does not loosen the owner's lock, even from the same process. + REQUIRE_THROWS_AS(MessageListener(path), std::system_error); + REQUIRE_FALSE(std::filesystem::exists(path)); + } + // The lock file stays for the next listener, which takes the path over. + REQUIRE(std::filesystem::exists(path + ".lock")); + MessageListener next(path); + MessageSocket client = MessageSocket::connect(path); + REQUIRE(next.accept(1s).has_value()); +} + +TEST_CASE("of listeners that start at the same moment exactly one wins", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + + std::array, 8> contenders; + std::atomic start{ false }; + std::vector threads; + for (auto& contender : contenders) + { + threads.emplace_back( + [&contender, &start, &path] + { + while (!start.load()) + { + std::this_thread::yield(); + } + try + { + contender = std::make_unique(path); + } + catch (const std::system_error&) + { + // Lost to another contender. + } + }); + } + start = true; + for (auto& thread : threads) + { + thread.join(); + } + + const auto is_listening = [](const std::unique_ptr& contender) { return contender != nullptr; }; + REQUIRE(std::count_if(contenders.begin(), contenders.end(), is_listening) == 1); + + // The winner owns the socket file: a client reaches it. + MessageListener& winner = **std::find_if(contenders.begin(), contenders.end(), is_listening); + MessageSocket client = MessageSocket::connect(path); + REQUIRE(winner.accept(1s).has_value()); +} + +TEST_CASE("a path that holds something other than a socket is left alone", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + std::ofstream(path) << "not a socket"; + + try + { + MessageListener listener(path); + FAIL("a listener replaced a file that is not a socket"); + } + catch (const std::system_error& error) + { + REQUIRE(error.code().value() == EEXIST); + } + REQUIRE(std::filesystem::is_regular_file(path)); + REQUIRE(std::filesystem::file_size(path) == 12); +} + +TEST_CASE("only the owner can connect to a listener", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const MessageListener listener(directory.socket_path()); + + const std::filesystem::file_status status = std::filesystem::status(directory.socket_path()); + REQUIRE(std::filesystem::is_socket(status)); + REQUIRE(status.permissions() == (std::filesystem::perms::owner_read | std::filesystem::perms::owner_write)); +} + +TEST_CASE("misuse of a message socket is rejected", "[unit][camera_ipc][socket]") +{ + Connection connection; + REQUIRE_THROWS_AS(connection.client.send({}), std::invalid_argument); + + const std::vector too_many(MAX_DESCRIPTORS_PER_MESSAGE + 1, 0); + REQUIRE_THROWS_AS(connection.client.send(bytes_of("x"), too_many), std::invalid_argument); + + // A message that does not fit the receiver's buffer is an error, not a silent cut. + REQUIRE(connection.client.send(bytes_of("sixteen bytes!!!")) == SendStatus::SENT); + std::array small; + REQUIRE_THROWS_AS(connection.server.receive(small, 1s), std::length_error); + // Also with no room at all, which must not read as a closed connection. + REQUIRE(connection.client.send(bytes_of("x")) == SendStatus::SENT); + REQUIRE_THROWS_AS(connection.server.receive({}, 1s), std::length_error); + + TemporaryDirectory directory; + REQUIRE_THROWS_AS(MessageSocket::connect(directory.socket_path()), std::system_error); + REQUIRE_THROWS_AS(MessageSocket::connect(std::string(200, 'a')), std::invalid_argument); + REQUIRE_THROWS_AS(MessageListener(""), std::invalid_argument); +} From 1572f956b642f3758f724f1d9a07ed29421d6b93 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:45:19 -0800 Subject: [PATCH 10/16] feat(camera): add camera_wire, the messages between the plugin and a reader The plugin and a reader exchange six messages over their socket: Hello and Welcome (the catalog of what is published), Subscribe and Subscribed (with one pool descriptor per stream), then Frame per frame set and Release when the reader is done with it. A connection carries one subscription. The messages are FlatBuffers tables in a schema kept inside the module, so a reader and a plugin from different builds keep understanding each other: the schema only grows, and a message type a build does not know is ignored. It is not under src/core/schema because these messages are never recorded. WireEncoder and decode_wire_message convert to and from camera_core types, so nothing outside the module sees FlatBuffers. Encoding a frame event allocates nothing once the encoder's buffer has grown. Decoding treats its input as coming from another process and checks it in full. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/CMakeLists.txt | 1 + src/core/camera_wire/AGENTS.md | 16 + src/core/camera_wire/CMakeLists.txt | 7 + src/core/camera_wire/README.md | 58 +++ src/core/camera_wire/cpp/CMakeLists.txt | 23 ++ src/core/camera_wire/cpp/codec.cpp | 341 ++++++++++++++++++ .../cpp/inc/camera_wire/messages.hpp | 113 ++++++ src/core/camera_wire/fbs/CMakeLists.txt | 34 ++ src/core/camera_wire/fbs/camera_wire.fbs | 108 ++++++ tests/cpp/core/CMakeLists.txt | 1 + tests/cpp/core/camera_wire/CMakeLists.txt | 19 + tests/cpp/core/camera_wire/test_codec.cpp | 298 +++++++++++++++ 12 files changed, 1019 insertions(+) create mode 100644 src/core/camera_wire/AGENTS.md create mode 100644 src/core/camera_wire/CMakeLists.txt create mode 100644 src/core/camera_wire/README.md create mode 100644 src/core/camera_wire/cpp/CMakeLists.txt create mode 100644 src/core/camera_wire/cpp/codec.cpp create mode 100644 src/core/camera_wire/cpp/inc/camera_wire/messages.hpp create mode 100644 src/core/camera_wire/fbs/CMakeLists.txt create mode 100644 src/core/camera_wire/fbs/camera_wire.fbs create mode 100644 tests/cpp/core/camera_wire/CMakeLists.txt create mode 100644 tests/cpp/core/camera_wire/test_codec.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 602cc1969a..7851685616 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -25,6 +25,7 @@ add_subdirectory(camera_core) # Camera frames and messages between processes (shared host memory, descriptor passing) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) + add_subdirectory(camera_wire) endif() # Build OXR library (depends on oxr_utils for types) diff --git a/src/core/camera_wire/AGENTS.md b/src/core/camera_wire/AGENTS.md new file mode 100644 index 0000000000..a7596019c1 --- /dev/null +++ b/src/core/camera_wire/AGENTS.md @@ -0,0 +1,16 @@ + + +# Agent notes — `camera_wire` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) before changing `fbs/camera_wire.fbs`: a reader and a plugin from different builds must keep understanding each other. + +- **The schema only grows.** Append fields with fresh ids, optional; append union members; tables, never structs. Do not reuse or renumber. +- **FlatBuffers stays inside this module.** Its users see `camera_core` types; only the codec and its tests include the generated header. +- **Decoded bytes are untrusted.** Verify, then bound every count against the fixed arrays it fills, and check for a body behind a body type. +- **A new enum value needs both sides.** The core enum and the schema enum carry the same numbers (pinned by `static_assert`), and the decoder must accept the value before a plugin sends it. +- **The frame path does not allocate.** Keep `encode(const FrameEvent&)` free of containers that do. diff --git a/src/core/camera_wire/CMakeLists.txt b/src/core/camera_wire/CMakeLists.txt new file mode 100644 index 0000000000..e162dea055 --- /dev/null +++ b/src/core/camera_wire/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(fbs) +add_subdirectory(cpp) diff --git a/src/core/camera_wire/README.md b/src/core/camera_wire/README.md new file mode 100644 index 0000000000..20effdcb4b --- /dev/null +++ b/src/core/camera_wire/README.md @@ -0,0 +1,58 @@ + + +# Camera interface: messages between the plugin and a reader + +What the camera plugin and a reader say to each other over their socket +([`camera_ipc`](../camera_ipc)). Each socket message is one of these: + +```text +reader plugin + | Hello (protocol version) | + |------------------------------------>| + | Welcome (what is published) | + |<------------------------------------| + | Subscribe (group, window, order) | + |------------------------------------>| + | Subscribed + one pool descriptor | + | per stream | + |<------------------------------------| + | Frame (lease, slot per image) | once per frame set + |<------------------------------------| + | Release (lease) | + |------------------------------------>| +``` + +A connection carries one subscription; a reader that wants two groups opens two. Closing the +connection ends the subscription and returns every frame set the reader still held. A `Frame` +can overtake `Subscribed`, because capture does not wait for the answer to be sent; it belongs +to the subscription all the same. + +## Schema + +[`fbs/camera_wire.fbs`](fbs/camera_wire.fbs) defines the messages as FlatBuffers tables. It is +kept here and not in [`schema`](../schema): that directory holds what goes into recordings, +and these messages are never recorded. + +A reader and a plugin may come from different builds, so the schema only grows: + +- Append fields with fresh ids and leave them optional; append union members. Never reuse or + renumber an id, and use tables, not structs. +- A message type this build does not know decodes to nothing and is ignored. +- A stream whose format or kind of memory this build does not know makes its group + unreadable, and the reader is told so; other groups are unaffected. +- `WIRE_PROTOCOL_VERSION` changes only when the two sides would misread each other. The + plugin refuses a reader with a different version in `Welcome`. + +## Codec + +`WireEncoder` and `decode_wire_message` turn [`camera_core`](../camera_core) types into bytes +and back, so nothing outside this module sees FlatBuffers. The encoder reuses its buffer; a +frame event is encoded without allocating. The decoder treats its input as coming from another +process: it verifies the buffer and refuses counts that would not fit a group. + +## Tests + +`tests/cpp/core/camera_wire`, run with `ctest -L camera_wire`. They are tagged `unit`. diff --git a/src/core/camera_wire/cpp/CMakeLists.txt b/src/core/camera_wire/cpp/CMakeLists.txt new file mode 100644 index 0000000000..a4d4122dbb --- /dev/null +++ b/src/core/camera_wire/cpp/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Turns the messages between the camera plugin and its readers into bytes and back. The +# FlatBuffers types stay inside this library; its users see camera_core types. +add_library(camera_wire STATIC + codec.cpp +) + +target_include_directories(camera_wire INTERFACE + $ +) + +target_link_libraries(camera_wire + PUBLIC + camera::camera_core + PRIVATE + camera::camera_wire_schema +) + +add_library(camera::camera_wire ALIAS camera_wire) diff --git a/src/core/camera_wire/cpp/codec.cpp b/src/core/camera_wire/cpp/codec.cpp new file mode 100644 index 0000000000..d0856fb521 --- /dev/null +++ b/src/core/camera_wire/cpp/codec.cpp @@ -0,0 +1,341 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_wire/messages.hpp" + +#include + +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// The schema's enums carry the same numbers as the core's, so a value converts by a cast. +static_assert(static_cast(PixelFormat::BGRA8) == wire::PixelFormat_Bgra8); +static_assert(static_cast(PixelFormat::RGBA8) == wire::PixelFormat_Rgba8); +static_assert(static_cast(PixelFormat::NV12) == wire::PixelFormat_Nv12); +static_assert(static_cast(PixelFormat::GRAY8) == wire::PixelFormat_Gray8); +static_assert(static_cast(Codec::NONE) == wire::Codec_None); +static_assert(static_cast(Codec::H264) == wire::Codec_H264); +static_assert(static_cast(Codec::H265) == wire::Codec_H265); +static_assert(static_cast(MemoryKind::HOST) == wire::MemoryKind_Host); +static_assert(static_cast(DeliveryOrder::DEFAULT) == wire::DeliveryOrder_Default); +static_assert(static_cast(DeliveryOrder::NEWEST) == wire::DeliveryOrder_Newest); +static_assert(static_cast(DeliveryOrder::IN_ORDER) == wire::DeliveryOrder_InOrder); + +using Builder = flatbuffers::FlatBufferBuilder; + +flatbuffers::Offset build_group(Builder& builder, const GroupDescription& group) +{ + std::vector> streams; + streams.reserve(group.streams.size()); + for (const StreamDescription& stream : group.streams) + { + std::array, MAX_PLANES_PER_IMAGE> planes; + const std::size_t plane_count = std::min(stream.plane_count, MAX_PLANES_PER_IMAGE); + for (std::size_t i = 0; i < plane_count; ++i) + { + planes[i] = wire::CreatePlaneLayout(builder, stream.planes[i].offset_bytes, stream.planes[i].stride_bytes); + } + const auto id = builder.CreateString(stream.id); + const auto plane_vector = builder.CreateVector(planes.data(), plane_count); + streams.push_back(wire::CreateStream(builder, id, static_cast(stream.codec), + static_cast(stream.pixel_format), + static_cast(stream.memory), stream.width, stream.height, + plane_vector, stream.slot_bytes, stream.slot_count, stream.frame_rate)); + } + const auto id = builder.CreateString(group.id); + const auto stream_vector = builder.CreateVector(streams); + return wire::CreateGroup(builder, id, stream_vector); +} + +std::string text_of(const flatbuffers::String* string) +{ + return string != nullptr ? string->str() : std::string(); +} + +std::optional pixel_format_of(wire::PixelFormat format) +{ + switch (format) + { + case wire::PixelFormat_Bgra8: + case wire::PixelFormat_Rgba8: + case wire::PixelFormat_Nv12: + case wire::PixelFormat_Gray8: + return static_cast(format); + default: + return std::nullopt; + } +} + +std::optional codec_of(wire::Codec codec) +{ + switch (codec) + { + case wire::Codec_None: + case wire::Codec_H264: + case wire::Codec_H265: + return static_cast(codec); + default: + return std::nullopt; + } +} + +std::optional order_of(wire::DeliveryOrder order) +{ + switch (order) + { + case wire::DeliveryOrder_Default: + case wire::DeliveryOrder_Newest: + case wire::DeliveryOrder_InOrder: + return static_cast(order); + default: + return std::nullopt; + } +} + +GroupDescription read_group(const wire::Group* group) +{ + GroupDescription result; + if (group == nullptr) + { + return result; + } + result.id = text_of(group->id()); + const auto* streams = group->streams(); + if (streams == nullptr) + { + return result; + } + if (streams->size() > MAX_STREAMS_PER_GROUP) + { + throw WireError("camera group '" + result.id + "' has more streams than a group can have"); + } + for (const wire::Stream* stream : *streams) + { + StreamDescription description; + description.id = text_of(stream->id()); + const auto codec = codec_of(stream->codec()); + const auto pixel_format = pixel_format_of(stream->pixel_format()); + // An encoded stream has no pixel format to know. + if (!codec || stream->memory() != wire::MemoryKind_Host || (*codec == Codec::NONE && !pixel_format)) + { + throw WireError("camera stream '" + description.id + + "' uses a format or a kind of memory this build does not know"); + } + description.codec = *codec; + description.pixel_format = pixel_format.value_or(PixelFormat::BGRA8); + description.memory = MemoryKind::HOST; + description.width = stream->width(); + description.height = stream->height(); + if (const auto* planes = stream->planes()) + { + if (planes->size() > MAX_PLANES_PER_IMAGE) + { + throw WireError("camera stream '" + description.id + "' has more planes than an image can have"); + } + description.plane_count = planes->size(); + for (std::uint32_t i = 0; i < planes->size(); ++i) + { + description.planes[i] = { planes->Get(i)->offset_bytes(), planes->Get(i)->stride_bytes() }; + } + } + description.slot_bytes = stream->slot_bytes(); + description.slot_count = stream->slot_count(); + description.frame_rate = stream->frame_rate(); + result.streams.push_back(std::move(description)); + } + return result; +} + +FrameEvent read_frame(const wire::Frame& frame) +{ + FrameEvent event; + event.lease_id = frame.lease_id(); + event.sequence = frame.sequence(); + event.generation = frame.generation(); + event.skipped = frame.skipped(); + if (const wire::Timeline* timeline = frame.timeline()) + { + event.timeline = { timeline->capture_ns(), timeline->sdk_ready_ns(), timeline->published_ns(), + timeline->sent_ns() }; + } + if (const auto* images = frame.images()) + { + if (images->size() > MAX_STREAMS_PER_GROUP) + { + throw WireError("a camera frame set has more images than a group has streams"); + } + event.image_count = images->size(); + for (std::uint32_t i = 0; i < images->size(); ++i) + { + const wire::Image* image = images->Get(i); + event.images[i] = { image->slot(), image->flags(), image->bytes_used(), image->device_frame_id(), + image->device_time_ns() }; + } + } + return event; +} + +} // namespace + +struct WireEncoder::Impl +{ + Builder builder{ 1024 }; + + template + std::span finish(wire::Body type, flatbuffers::Offset body) + { + builder.Finish(wire::CreateMessage(builder, type, body.Union())); + if (builder.GetSize() > MAX_WIRE_MESSAGE_BYTES) + { + throw std::length_error("a camera wire message of " + std::to_string(builder.GetSize()) + + " bytes is over the limit of " + std::to_string(MAX_WIRE_MESSAGE_BYTES)); + } + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; + } +}; + +WireEncoder::WireEncoder() : m_impl(std::make_unique()) +{ +} + +WireEncoder::~WireEncoder() = default; + +std::span WireEncoder::encode(const HelloMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto name = builder.CreateString(message.reader_name); + return m_impl->finish(wire::Body_Hello, wire::CreateHello(builder, message.protocol_version, name)); +} + +std::span WireEncoder::encode(const WelcomeMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::vector> groups; + groups.reserve(message.groups.size()); + for (const GroupDescription& group : message.groups) + { + groups.push_back(build_group(builder, group)); + } + const auto group_vector = builder.CreateVector(groups); + const auto error = builder.CreateString(message.error); + return m_impl->finish( + wire::Body_Welcome, wire::CreateWelcome(builder, message.protocol_version, group_vector, error)); +} + +std::span WireEncoder::encode(const SubscribeMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto group = builder.CreateString(message.group); + return m_impl->finish( + wire::Body_Subscribe, wire::CreateSubscribe(builder, group, message.options.window, + static_cast(message.options.order))); +} + +std::span WireEncoder::encode(const SubscribedMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto error = builder.CreateString(message.error); + const auto group = build_group(builder, message.group); + return m_impl->finish(wire::Body_Subscribed, wire::CreateSubscribed(builder, error, group)); +} + +// The per-frame path: nothing here allocates once the builder has grown to a frame's size. +std::span WireEncoder::encode(const FrameEvent& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::array, MAX_STREAMS_PER_GROUP> images; + const std::size_t image_count = std::min(message.image_count, MAX_STREAMS_PER_GROUP); + for (std::size_t i = 0; i < image_count; ++i) + { + const ImageInfo& image = message.images[i]; + images[i] = wire::CreateImage( + builder, image.slot, image.flags, image.bytes_used, image.device_frame_id, image.device_time_ns); + } + const auto image_vector = builder.CreateVector(images.data(), image_count); + const auto timeline = wire::CreateTimeline(builder, message.timeline.capture_ns, message.timeline.sdk_ready_ns, + message.timeline.published_ns, message.timeline.sent_ns); + return m_impl->finish( + wire::Body_Frame, wire::CreateFrame(builder, message.lease_id, message.sequence, message.generation, + message.skipped, timeline, image_vector)); +} + +std::span WireEncoder::encode(const ReleaseMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + return m_impl->finish(wire::Body_Release, wire::CreateRelease(builder, message.lease_id)); +} + +WireMessage decode_wire_message(std::span bytes) +{ + const auto* data = reinterpret_cast(bytes.data()); + flatbuffers::Verifier verifier(data, bytes.size()); + if (bytes.empty() || !wire::VerifyMessageBuffer(verifier)) + { + throw WireError("received bytes that are not a camera wire message"); + } + const wire::Message* message = wire::GetMessage(data); + // The verifier accepts a body type without a body. + if (message->body_type() != wire::Body_NONE && message->body() == nullptr) + { + throw WireError("received a camera wire message without a body"); + } + switch (message->body_type()) + { + case wire::Body_Hello: + { + const wire::Hello* hello = message->body_as_Hello(); + return HelloMessage{ hello->protocol_version(), text_of(hello->reader_name()) }; + } + case wire::Body_Welcome: + { + const wire::Welcome* welcome = message->body_as_Welcome(); + WelcomeMessage result; + result.protocol_version = welcome->protocol_version(); + result.error = text_of(welcome->error()); + if (const auto* groups = welcome->groups()) + { + for (const wire::Group* group : *groups) + { + result.groups.push_back(read_group(group)); + } + } + return result; + } + case wire::Body_Subscribe: + { + const wire::Subscribe* subscribe = message->body_as_Subscribe(); + const auto order = order_of(subscribe->order()); + if (!order) + { + throw WireError("a camera reader asked for a delivery order this build does not know"); + } + return SubscribeMessage{ text_of(subscribe->group()), { subscribe->window(), *order } }; + } + case wire::Body_Subscribed: + { + const wire::Subscribed* subscribed = message->body_as_Subscribed(); + return SubscribedMessage{ text_of(subscribed->error()), read_group(subscribed->group()) }; + } + case wire::Body_Frame: + return read_frame(*message->body_as_Frame()); + case wire::Body_Release: + return ReleaseMessage{ message->body_as_Release()->lease_id() }; + default: + return std::monostate{}; + } +} + +} // namespace core::camera diff --git a/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp b/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp new file mode 100644 index 0000000000..e2626c10ee --- /dev/null +++ b/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// Changes only when a reader and a plugin of different versions could misread each other. +// A field or a message added to the schema does not change it. +inline constexpr std::uint32_t WIRE_PROTOCOL_VERSION = 1; + +// No message is larger than this, so a receive buffer of this size always fits one. +inline constexpr std::size_t MAX_WIRE_MESSAGE_BYTES = 64 * 1024; + +// Reader to plugin, first message on a connection. +struct HelloMessage +{ + std::uint32_t protocol_version = WIRE_PROTOCOL_VERSION; + std::string reader_name; // for the plugin's log +}; + +// Plugin to reader, in answer to HelloMessage. +struct WelcomeMessage +{ + std::uint32_t protocol_version = WIRE_PROTOCOL_VERSION; + std::vector groups; // what the plugin publishes + std::string error; // why the plugin refuses the reader, if it does +}; + +// Reader to plugin: start sending a group's frame sets. One subscription per connection. +struct SubscribeMessage +{ + std::string group; + ReaderOptions options; +}; + +// Plugin to reader, in answer to SubscribeMessage. On success one file descriptor per stream +// travels with it, in stream order, for the stream's pool. +struct SubscribedMessage +{ + std::string error; // why the subscription was refused, if it was + GroupDescription group; +}; + +// Reader to plugin: done with a frame set. +struct ReleaseMessage +{ + std::uint64_t lease_id = 0; +}; + +// What decode_wire_message returns. A frame set travels as a FrameEvent. std::monostate is a +// message from a newer peer that this build does not know; ignore it. +using WireMessage = + std::variant; + +/** + * @brief Bytes that are not a message, or a message this build cannot represent. + */ +class WireError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +/** + * @brief Turns messages into bytes. It reuses one buffer, so after the first few messages + * encoding allocates nothing. Not thread-safe. + */ +class WireEncoder +{ +public: + WireEncoder(); + ~WireEncoder(); + + WireEncoder(const WireEncoder&) = delete; + WireEncoder& operator=(const WireEncoder&) = delete; + + /** + * @return The message's bytes, valid until the next call on this encoder. + * @throws std::length_error if the message is larger than MAX_WIRE_MESSAGE_BYTES. + */ + std::span encode(const HelloMessage& message); + std::span encode(const WelcomeMessage& message); + std::span encode(const SubscribeMessage& message); + std::span encode(const SubscribedMessage& message); + std::span encode(const FrameEvent& message); + std::span encode(const ReleaseMessage& message); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +/** + * @brief Read one message. The bytes come from another process and are checked in full. + * @throws WireError if they are not a message, or name more streams, planes or images than a + * group can have, or a format this build does not know. + */ +WireMessage decode_wire_message(std::span bytes); + +} // namespace core::camera diff --git a/src/core/camera_wire/fbs/CMakeLists.txt b/src/core/camera_wire/fbs/CMakeLists.txt new file mode 100644 index 0000000000..02239af735 --- /dev/null +++ b/src/core/camera_wire/fbs/CMakeLists.txt @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# The messages between the camera plugin and its readers, as a FlatBuffer schema. The generated +# header is reached as ; only the codec and its +# tests include it. + +include(${CMAKE_SOURCE_DIR}/cmake/GenerateFlatBuffers.cmake) + +generate_flatbuffer_headers( + CAMERA_WIRE_GENERATED_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/generated/camera_wire_schema + CAMERA_WIRE_GENERATED_BFBS +) + +add_custom_target(camera_wire_schema_generation + DEPENDS ${CAMERA_WIRE_GENERATED_HEADERS} ${CAMERA_WIRE_GENERATED_BFBS} +) + +add_library(camera_wire_schema INTERFACE) +add_dependencies(camera_wire_schema camera_wire_schema_generation) + +target_include_directories(camera_wire_schema INTERFACE + $ +) + +target_link_libraries(camera_wire_schema INTERFACE + flatbuffers +) + +add_library(camera::camera_wire_schema ALIAS camera_wire_schema) diff --git a/src/core/camera_wire/fbs/camera_wire.fbs b/src/core/camera_wire/fbs/camera_wire.fbs new file mode 100644 index 0000000000..5f6bc1dd5c --- /dev/null +++ b/src/core/camera_wire/fbs/camera_wire.fbs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The messages between the camera plugin and a reader, one per socket message. A reader and a +// plugin may be built at different times, so this file only grows: append fields with fresh +// ids and leave them optional, append union members, never reuse or renumber. See README.md. + +namespace core.camera.wire; + +// Zero means "a value this build does not know" for the kinds a newer peer may extend. +enum PixelFormat : uint32 { Unknown = 0, Bgra8 = 1, Rgba8 = 2, Nv12 = 3, Gray8 = 4 } +enum Codec : uint32 { None = 0, H264 = 1, H265 = 2 } +enum MemoryKind : uint32 { Unknown = 0, Host = 1 } +enum DeliveryOrder : uint32 { Default = 0, Newest = 1, InOrder = 2 } + +// Where one plane of an image sits inside a slot. +table PlaneLayout { + offset_bytes: uint32 (id: 0); + stride_bytes: uint32 (id: 1); +} + +// One sensor's images or one encoded video stream, and the pool its frames are in. +table Stream { + id: string (id: 0); + codec: Codec (id: 1); + pixel_format: PixelFormat (id: 2); + memory: MemoryKind (id: 3); + width: uint32 (id: 4); + height: uint32 (id: 5); + planes: [PlaneLayout] (id: 6); + slot_bytes: uint64 (id: 7); + slot_count: uint32 (id: 8); + frame_rate: float64 (id: 9); +} + +// Streams that are captured together and published as one frame set. +table Group { + id: string (id: 0); + streams: [Stream] (id: 1); +} + +// Reader to plugin, first message on a connection. +table Hello { + protocol_version: uint32 (id: 0); + reader_name: string (id: 1); +} + +// Plugin to reader, in answer to Hello. `error` is set when the plugin refuses the reader. +table Welcome { + protocol_version: uint32 (id: 0); + groups: [Group] (id: 1); + error: string (id: 2); +} + +// Reader to plugin: start sending this group's frame sets. One subscription per connection. +table Subscribe { + group: string (id: 0); + window: uint32 = 1 (id: 1); + order: DeliveryOrder (id: 2); +} + +// Plugin to reader, in answer to Subscribe. On success the message carries one file descriptor +// per stream of the group, in stream order, for the stream's pool. `error` is set otherwise. +table Subscribed { + error: string (id: 0); + group: Group (id: 1); +} + +// One image of a frame set: the slot it is in and what the camera SDK said about it. +table Image { + slot: uint32 (id: 0); + flags: uint32 (id: 1); + bytes_used: uint64 (id: 2); + device_frame_id: uint64 (id: 3); + device_time_ns: int64 = -1 (id: 4); +} + +// Host monotonic times in nanoseconds; -1 where a time is not known. +table Timeline { + capture_ns: int64 = -1 (id: 0); + sdk_ready_ns: int64 = -1 (id: 1); + published_ns: int64 = -1 (id: 2); + sent_ns: int64 = -1 (id: 3); +} + +// Plugin to reader: one frame set, held for the reader until it sends Release with the same +// lease. Frames may arrive before Subscribed; they belong to the subscription all the same. +table Frame { + lease_id: uint64 (id: 0); + sequence: uint64 (id: 1); + generation: uint32 (id: 2); + skipped: uint32 (id: 3); + timeline: Timeline (id: 4); + images: [Image] (id: 5); +} + +// Reader to plugin: done with a frame set. +table Release { + lease_id: uint64 (id: 0); +} + +union Body { Hello, Welcome, Subscribe, Subscribed, Frame, Release } + +table Message { + body: Body (id: 1); +} + +root_type Message; diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 705bead328..35bbcad055 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -17,6 +17,7 @@ add_subdirectory(camera_core) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) + add_subdirectory(camera_wire) endif() if(BUILD_OVXR_CAMERA_QUADS) diff --git a/tests/cpp/core/camera_wire/CMakeLists.txt b/tests/cpp/core/camera_wire/CMakeLists.txt new file mode 100644 index 0000000000..69b9c4a7b1 --- /dev/null +++ b/tests/cpp/core/camera_wire/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_wire_tests + test_codec.cpp +) + +# The schema is linked to build messages the encoder never would: ones from a newer peer and +# malformed ones. +target_link_libraries(camera_wire_tests PRIVATE + camera::camera_wire + camera::camera_wire_schema + Catch2::Catch2WithMain +) + +message(STATUS "camera_wire_tests target enabled with Catch2") +catch_discover_tests(camera_wire_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_wire/test_codec.cpp b/tests/cpp/core/camera_wire/test_codec.cpp new file mode 100644 index 0000000000..e3b33b516f --- /dev/null +++ b/tests/cpp/core/camera_wire/test_codec.cpp @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +namespace wire = core::camera::wire; + +namespace +{ + +GroupDescription stereo_group() +{ + GroupDescription group; + group.id = "head"; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = std::string("head/") + eye; + stream.pixel_format = PixelFormat::NV12; + stream.width = 1920; + stream.height = 1080; + stream.plane_count = 2; + stream.planes[0] = { .offset_bytes = 0, .stride_bytes = 2048 }; + stream.planes[1] = { .offset_bytes = 2048 * 1080, .stride_bytes = 2048 }; + stream.slot_bytes = 2048 * 1620; + stream.slot_count = 8; + stream.frame_rate = 59.94; + group.streams.push_back(stream); + } + return group; +} + +GroupDescription encoded_group() +{ + GroupDescription group; + group.id = "head.h265"; + StreamDescription stream; + stream.id = "head/left.h265"; + stream.codec = Codec::H265; + stream.width = 1920; + stream.height = 1080; + stream.slot_bytes = 1 << 20; + stream.slot_count = 16; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + return group; +} + +void require_same(const GroupDescription& actual, const GroupDescription& expected) +{ + REQUIRE(actual.id == expected.id); + REQUIRE(actual.streams.size() == expected.streams.size()); + for (std::size_t i = 0; i < expected.streams.size(); ++i) + { + const StreamDescription& a = actual.streams[i]; + const StreamDescription& e = expected.streams[i]; + REQUIRE(a.id == e.id); + REQUIRE(a.codec == e.codec); + REQUIRE(a.memory == e.memory); + REQUIRE(a.width == e.width); + REQUIRE(a.height == e.height); + REQUIRE(a.plane_count == e.plane_count); + for (std::uint32_t plane = 0; plane < e.plane_count; ++plane) + { + REQUIRE(a.planes[plane].offset_bytes == e.planes[plane].offset_bytes); + REQUIRE(a.planes[plane].stride_bytes == e.planes[plane].stride_bytes); + } + REQUIRE(a.slot_bytes == e.slot_bytes); + REQUIRE(a.slot_count == e.slot_count); + REQUIRE(a.frame_rate == e.frame_rate); + if (!e.is_encoded()) + { + REQUIRE(a.pixel_format == e.pixel_format); + } + } +} + +// What the other process receives: a copy of the bytes, decoded. +template +Message round_trip(WireEncoder& encoder, const Message& message) +{ + const std::span bytes = encoder.encode(message); + const std::vector received(bytes.begin(), bytes.end()); + WireMessage decoded = decode_wire_message(received); + REQUIRE(std::holds_alternative(decoded)); + return std::get(std::move(decoded)); +} + +std::span bytes_of(const flatbuffers::FlatBufferBuilder& builder) +{ + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; +} + +} // namespace + +TEST_CASE("the handshake messages survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + const HelloMessage hello = round_trip(encoder, HelloMessage{ .protocol_version = 7, .reader_name = "viewer" }); + REQUIRE(hello.protocol_version == 7); + REQUIRE(hello.reader_name == "viewer"); + + WelcomeMessage welcome; + welcome.groups = { stereo_group(), encoded_group() }; + const WelcomeMessage received = round_trip(encoder, welcome); + REQUIRE(received.protocol_version == WIRE_PROTOCOL_VERSION); + REQUIRE(received.error.empty()); + REQUIRE(received.groups.size() == 2); + require_same(received.groups[0], stereo_group()); + require_same(received.groups[1], encoded_group()); + + const WelcomeMessage refused = round_trip(encoder, WelcomeMessage{ .error = "protocol version 7 is not supported" }); + REQUIRE(refused.error == "protocol version 7 is not supported"); + REQUIRE(refused.groups.empty()); +} + +TEST_CASE("a subscription and its answer survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + const SubscribeMessage subscribe = + round_trip(encoder, SubscribeMessage{ "head", { .window = 4, .order = DeliveryOrder::IN_ORDER } }); + REQUIRE(subscribe.group == "head"); + REQUIRE(subscribe.options.window == 4); + REQUIRE(subscribe.options.order == DeliveryOrder::IN_ORDER); + + // What a reader that sets nothing asks for. + const SubscribeMessage defaults = round_trip(encoder, SubscribeMessage{ "head", {} }); + REQUIRE(defaults.options.window == 1); + REQUIRE(defaults.options.order == DeliveryOrder::DEFAULT); + + const SubscribedMessage accepted = round_trip(encoder, SubscribedMessage{ "", stereo_group() }); + REQUIRE(accepted.error.empty()); + require_same(accepted.group, stereo_group()); + + const SubscribedMessage refused = round_trip(encoder, SubscribedMessage{ "does not fit the pool", {} }); + REQUIRE(refused.error == "does not fit the pool"); + REQUIRE(refused.group.streams.empty()); +} + +TEST_CASE("a frame set and its release survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + FrameEvent event; + event.lease_id = 0x1122334455667788; + event.sequence = 123456789012; + event.generation = 3; + event.skipped = 2; + event.image_count = 2; + event.timeline = { .capture_ns = 1000, .sdk_ready_ns = 2000, .published_ns = 3000, .sent_ns = 4000 }; + event.images[0] = { + .slot = 5, .flags = IMAGE_FLAG_SDK_WARNING, .bytes_used = 0, .device_frame_id = 77, .device_time_ns = 99 + }; + event.images[1] = { .slot = 6, .flags = IMAGE_FLAG_KEYFRAME, .bytes_used = 4096, .device_frame_id = 78 }; + + const FrameEvent received = round_trip(encoder, event); + REQUIRE(received.lease_id == event.lease_id); + REQUIRE(received.sequence == event.sequence); + REQUIRE(received.generation == 3); + REQUIRE(received.skipped == 2); + REQUIRE(received.image_count == 2); + REQUIRE(received.timeline.capture_ns == 1000); + REQUIRE(received.timeline.sdk_ready_ns == 2000); + REQUIRE(received.timeline.published_ns == 3000); + REQUIRE(received.timeline.sent_ns == 4000); + REQUIRE(received.images[0].slot == 5); + REQUIRE(received.images[0].flags == IMAGE_FLAG_SDK_WARNING); + REQUIRE(received.images[0].device_frame_id == 77); + REQUIRE(received.images[0].device_time_ns == 99); + REQUIRE(received.images[1].slot == 6); + REQUIRE(received.images[1].flags == IMAGE_FLAG_KEYFRAME); + REQUIRE(received.images[1].bytes_used == 4096); + REQUIRE(received.images[1].device_time_ns == TIME_UNKNOWN); + + // Times that are not known stay unknown. + const FrameEvent bare = round_trip(encoder, FrameEvent{}); + REQUIRE(bare.image_count == 0); + REQUIRE(bare.timeline.capture_ns == TIME_UNKNOWN); + REQUIRE(bare.timeline.sent_ns == TIME_UNKNOWN); + + REQUIRE(round_trip(encoder, ReleaseMessage{ 42 }).lease_id == 42); +} + +TEST_CASE("an encoder is reused from message to message", "[unit][camera_wire]") +{ + WireEncoder encoder; + FrameEvent event; + event.image_count = 2; + for (std::uint64_t sequence = 1; sequence <= 100; ++sequence) + { + event.sequence = sequence; + REQUIRE(round_trip(encoder, event).sequence == sequence); + } + // A small message after a large one is not padded to the large one's size. + WelcomeMessage welcome; + welcome.groups = { stereo_group(), encoded_group() }; + const std::size_t large = encoder.encode(welcome).size(); + REQUIRE(encoder.encode(ReleaseMessage{ 1 }).size() < large); +} + +TEST_CASE("bytes that are not a message are rejected", "[unit][camera_wire]") +{ + REQUIRE_THROWS_AS(decode_wire_message({}), WireError); + + const std::vector garbage(64, std::byte{ 0xa5 }); + REQUIRE_THROWS_AS(decode_wire_message(garbage), WireError); + + WireEncoder encoder; + WelcomeMessage welcome; + welcome.groups = { stereo_group() }; + const std::span bytes = encoder.encode(welcome); + const std::vector whole(bytes.begin(), bytes.end()); + REQUIRE_THROWS_AS(decode_wire_message({ whole.data(), whole.size() / 2 }), WireError); + + // No cut of a message may be read out of bounds; each is either refused or read in full. + for (std::size_t length = 0; length < whole.size(); ++length) + { + const std::vector cut(whole.begin(), whole.begin() + static_cast(length)); + try + { + (void)decode_wire_message(cut); + } + catch (const WireError&) + { + } + } +} + +TEST_CASE("a message that does not fit this build's limits is rejected", "[unit][camera_wire]") +{ + WireEncoder encoder; + + GroupDescription crowded = stereo_group(); + crowded.streams.resize(MAX_STREAMS_PER_GROUP + 1, crowded.streams.front()); + const std::span bytes = encoder.encode(SubscribedMessage{ "", crowded }); + REQUIRE_THROWS_AS(decode_wire_message(bytes), WireError); + + // A pixel format added by a newer plugin cannot be read, so the group is refused as a whole. + GroupDescription newer = stereo_group(); + newer.streams[1].pixel_format = static_cast(99); + REQUIRE_THROWS_AS(decode_wire_message(encoder.encode(SubscribedMessage{ "", newer })), WireError); + + flatbuffers::FlatBufferBuilder builder; + + // More planes than an image can have. + const std::vector> planes( + MAX_PLANES_PER_IMAGE + 1, wire::CreatePlaneLayout(builder, 0, 64)); + const auto stream = + wire::CreateStream(builder, builder.CreateString("s"), wire::Codec_None, wire::PixelFormat_Gray8, + wire::MemoryKind_Host, 64, 64, builder.CreateVector(planes), 64 * 64, 4, 30.0); + const auto group = wire::CreateGroup(builder, builder.CreateString("g"), builder.CreateVector(&stream, 1)); + builder.Finish(wire::CreateMessage(builder, wire::Body_Subscribed, wire::CreateSubscribed(builder, 0, group).Union())); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // More images than a group has streams. + builder.Clear(); + const std::vector> images(MAX_STREAMS_PER_GROUP + 1, wire::CreateImage(builder)); + builder.Finish(wire::CreateMessage( + builder, wire::Body_Frame, wire::CreateFrame(builder, 1, 1, 0, 0, 0, builder.CreateVector(images)).Union())); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // A body type with no body behind it. + builder.Clear(); + builder.Finish(wire::CreateMessage(builder, wire::Body_Release, 0)); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // The largest message a receive buffer has to hold. + REQUIRE_THROWS_AS( + encoder.encode(HelloMessage{ .reader_name = std::string(MAX_WIRE_MESSAGE_BYTES, 'x') }), std::length_error); +} + +TEST_CASE("a message from a newer peer is ignored, not an error", "[unit][camera_wire]") +{ + flatbuffers::FlatBufferBuilder builder; + + // A union member this build does not have. + const auto body = wire::CreateRelease(builder, 1); + builder.Finish(wire::CreateMessage(builder, static_cast(wire::Body_MAX + 1), body.Union())); + REQUIRE(std::holds_alternative(decode_wire_message(bytes_of(builder)))); + + // No body at all. + builder.Clear(); + builder.Finish(wire::CreateMessage(builder)); + REQUIRE(std::holds_alternative(decode_wire_message(bytes_of(builder)))); +} From ec188ab08a55f70513bcc0dbf504971ac3c2a684 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:52:38 -0800 Subject: [PATCH 11/16] feat(camera): add camera_endpoint, which serves a publisher to readers Endpoint puts a Publisher on a socket and speaks the camera_wire messages: it tells each reader what is published, and on a subscription sends the group's pool descriptors and from then on its frame events. One thread accepts readers and handles the handshake, subscriptions and releases. Frame events are not sent from it: they are encoded and sent on the publishing thread, inside Publisher::publish, with a send that does not block, so nothing sits between a frame set being published and its event being on the socket. A closed connection ends the reader's subscription and returns every frame set it held, whether the reader exited or was killed. A reader that breaks the protocol is dropped without affecting the others. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/CMakeLists.txt | 1 + src/core/camera_endpoint/AGENTS.md | 15 + src/core/camera_endpoint/CMakeLists.txt | 6 + src/core/camera_endpoint/README.md | 49 +++ src/core/camera_endpoint/cpp/CMakeLists.txt | 27 ++ src/core/camera_endpoint/cpp/endpoint.cpp | 397 +++++++++++++++++ .../cpp/inc/camera_endpoint/endpoint.hpp | 65 +++ tests/cpp/core/CMakeLists.txt | 1 + tests/cpp/core/camera_endpoint/CMakeLists.txt | 22 + .../core/camera_endpoint/test_endpoint.cpp | 416 ++++++++++++++++++ 10 files changed, 999 insertions(+) create mode 100644 src/core/camera_endpoint/AGENTS.md create mode 100644 src/core/camera_endpoint/CMakeLists.txt create mode 100644 src/core/camera_endpoint/README.md create mode 100644 src/core/camera_endpoint/cpp/CMakeLists.txt create mode 100644 src/core/camera_endpoint/cpp/endpoint.cpp create mode 100644 src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp create mode 100644 tests/cpp/core/camera_endpoint/CMakeLists.txt create mode 100644 tests/cpp/core/camera_endpoint/test_endpoint.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 7851685616..bafa5c8a6b 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -26,6 +26,7 @@ add_subdirectory(camera_core) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) add_subdirectory(camera_wire) + add_subdirectory(camera_endpoint) endif() # Build OXR library (depends on oxr_utils for types) diff --git a/src/core/camera_endpoint/AGENTS.md b/src/core/camera_endpoint/AGENTS.md new file mode 100644 index 0000000000..92ff196bc1 --- /dev/null +++ b/src/core/camera_endpoint/AGENTS.md @@ -0,0 +1,15 @@ + + +# Agent notes — `camera_endpoint` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) for what runs on which thread before changing it. + +- **Nothing sits between publish and the socket.** A frame event is encoded and sent inside `send_event`, on the publishing thread. Do not add a queue or a thread hop there, and do not log per frame. +- **`send_event` holds the publisher's lock.** It must not take another lock, block or call the publisher. That lock is also what serialises each connection's event encoder. +- **Unsubscribe before a connection is destroyed.** `Publisher::unsubscribe` returning is the guarantee that no `send_event` is running or will start for that sink. +- **A reader is untrusted.** Whatever it sends can at most get it dropped; it must not stop the thread or affect other readers. diff --git a/src/core/camera_endpoint/CMakeLists.txt b/src/core/camera_endpoint/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_endpoint/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_endpoint/README.md b/src/core/camera_endpoint/README.md new file mode 100644 index 0000000000..a32ab763c4 --- /dev/null +++ b/src/core/camera_endpoint/README.md @@ -0,0 +1,49 @@ + + +# Camera interface: the endpoint + +The camera plugin's side of the conversation with readers. It puts a +[`camera_core`](../camera_core) `Publisher` on a socket ([`camera_ipc`](../camera_ipc)) and +speaks the messages of [`camera_wire`](../camera_wire). + +```cpp +Publisher publisher; +const GroupId head = publisher.add_group(description); +HostPool left(slots, bytes, "head.left"), right(slots, bytes, "head.right"); + +Endpoint endpoint(publisher, socket_path); +endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); +endpoint.start(); +// ... the backend acquires slots, writes pixels and publishes; readers come and go ... +``` + +## What runs where + +- **One thread of its own** accepts readers and handles what they send: the handshake, + subscriptions and releases. It sleeps in `poll` until a reader sends something. +- **Frame events are sent from the thread that publishes**, straight from + `Publisher::publish`, with a send that does not block. There is no queue and no hand-over + to another thread between a frame set being published and its event being on the socket. + The event that follows a release is sent from the endpoint's thread for the same reason. +- A reader's subscription takes effect before the answer to it is sent, so a `Frame` can + reach the reader ahead of `Subscribed`. + +## When a reader goes away + +A closed connection is the signal, whether the reader exited, crashed or was killed. Its +subscription ends and every frame set it held goes back to the pool. A reader that sends +bytes that are not a message, or a message only a plugin sends, is dropped the same way; +other readers are not affected. + +## Logging + +`isaaccapture.camera_endpoint.Endpoint` logs readers arriving, subscribing, being refused and +leaving. Nothing is logged per frame. + +## Tests + +`tests/cpp/core/camera_endpoint`, run with `ctest -L camera_endpoint`. They are tagged `unit` +and play the reader's part with the socket and the codec directly. diff --git a/src/core/camera_endpoint/cpp/CMakeLists.txt b/src/core/camera_endpoint/cpp/CMakeLists.txt new file mode 100644 index 0000000000..4bcdfb1e78 --- /dev/null +++ b/src/core/camera_endpoint/cpp/CMakeLists.txt @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +# The plugin's side of the conversation with readers: serves a Publisher's groups over a socket. +add_library(camera_endpoint STATIC + endpoint.cpp +) + +target_include_directories(camera_endpoint INTERFACE + $ +) + +target_link_libraries(camera_endpoint + PUBLIC + camera::camera_core + PRIVATE + camera::camera_ipc + camera::camera_wire + log_bridge::log_bridge_core + Threads::Threads +) + +add_library(camera::camera_endpoint ALIAS camera_endpoint) diff --git a/src/core/camera_endpoint/cpp/endpoint.cpp b/src/core/camera_endpoint/cpp/endpoint.cpp new file mode 100644 index 0000000000..380265eff9 --- /dev/null +++ b/src/core/camera_endpoint/cpp/endpoint.cpp @@ -0,0 +1,397 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_endpoint/endpoint.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// One reader. As an EventSink it is called by the publisher, on whichever thread publishes or +// releases, with the publisher's lock held; everything else runs on the endpoint's thread. +class Connection final : public EventSink +{ +public: + explicit Connection(MessageSocket connected) : socket(std::move(connected)) + { + } + + bool send_event(SubscriptionId /*subscription*/, const FrameEvent& event) noexcept override + { + try + { + return socket.send(m_event_encoder.encode(event)) == SendStatus::SENT; + } + catch (const std::exception&) + { + return false; // the reader gets a later frame set, or its closed socket is noticed + } + } + + MessageSocket socket; + std::string name = "unnamed"; + bool greeted = false; + SubscriptionId subscription = 0; + +private: + WireEncoder m_event_encoder; // used inside send_event only, which the publisher's lock serialises +}; + +const char* name_of(DeliveryOrder order) +{ + switch (order) + { + case DeliveryOrder::NEWEST: + return "newest"; + case DeliveryOrder::IN_ORDER: + return "in order"; + default: + return "default order"; + } +} + +} // namespace + +class Endpoint::Impl +{ +public: + Impl(Publisher& publisher, const std::string& path) + : m_publisher(publisher), + m_listener(path), + m_wake(::eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK)), + m_buffer(MAX_WIRE_MESSAGE_BYTES), + m_logger(isaaccapture::Logger::get("isaaccapture.camera_endpoint.Endpoint")) + { + if (!m_wake.valid()) + { + throw std::system_error(errno, std::generic_category(), "creating the camera endpoint's wake-up descriptor"); + } + } + + ~Impl() + { + stop(); + } + + void set_group_memory(GroupId group, std::vector descriptors) + { + if (m_thread.joinable()) + { + throw std::logic_error("a camera group's memory is named before the endpoint starts"); + } + const GroupDescription description = m_publisher.group_description(group); + if (descriptors.size() != description.streams.size()) + { + throw std::invalid_argument("camera group '" + description.id + "' needs one descriptor per stream"); + } + m_memory[group] = std::move(descriptors); + } + + void start() + { + if (m_thread.joinable()) + { + return; + } + m_stopping = false; + m_thread = std::thread([this] { run(); }); + m_logger->info("serving cameras on '{}'", m_listener.path()); + } + + void stop() + { + if (!m_thread.joinable()) + { + return; + } + m_stopping = true; + const std::uint64_t one = 1; + // Cannot fail in a way that matters: the counter only overflows after 2^64 wake-ups. + [[maybe_unused]] const ssize_t written = ::write(m_wake.get(), &one, sizeof(one)); + m_thread.join(); + } + + std::size_t reader_count() const + { + return m_reader_count.load(); + } + +private: + void run() + { + std::vector waits; + while (!m_stopping) + { + waits.assign({ { m_wake.get(), POLLIN, 0 }, { m_listener.descriptor(), POLLIN, 0 } }); + for (const auto& connection : m_connections) + { + waits.push_back({ connection->socket.descriptor(), POLLIN, 0 }); + } + if (::poll(waits.data(), waits.size(), -1) < 0) + { + if (errno == EINTR) + { + continue; + } + m_logger->error("camera endpoint stops serving: poll failed with errno {}", errno); + break; + } + + // Back to front, so dropping a connection leaves the indices still to visit alone. + for (std::size_t i = m_connections.size(); i-- > 0;) + { + if (waits[i + 2].revents != 0 && !serve(*m_connections[i])) + { + drop(i); + } + } + if ((waits[1].revents & POLLIN) != 0) + { + accept_readers(); + } + } + while (!m_connections.empty()) + { + drop(m_connections.size() - 1); + } + // Empty the wake-up counter, so that serving can start again. + std::uint64_t wakes = 0; + [[maybe_unused]] const ssize_t drained = ::read(m_wake.get(), &wakes, sizeof(wakes)); + } + + void accept_readers() + { + try + { + while (auto socket = m_listener.accept(std::chrono::milliseconds(0))) + { + m_connections.push_back(std::make_unique(std::move(*socket))); + m_reader_count = m_connections.size(); + } + } + catch (const std::exception& error) + { + m_logger->warn("could not accept a camera reader: {}", error.what()); + } + } + + // Ends the reader's subscription before its sink goes away. unsubscribe() returns only once + // no send_event is running or can start for it. + void drop(std::size_t index) + { + Connection& connection = *m_connections[index]; + if (connection.subscription != 0) + { + m_publisher.unsubscribe(connection.subscription); + } + m_logger->info("camera reader '{}' is gone", connection.name); + m_connections.erase(m_connections.begin() + static_cast(index)); + m_reader_count = m_connections.size(); + } + + // Handles everything the reader has sent. Returns false when the connection is to be dropped. + bool serve(Connection& connection) + { + try + { + for (;;) + { + const ReceivedMessage received = connection.socket.receive(m_buffer, std::chrono::milliseconds(0)); + if (received.status == ReceiveStatus::TIMEOUT) + { + return true; + } + if (received.status == ReceiveStatus::CLOSED) + { + return false; + } + const WireMessage message = decode_wire_message({ m_buffer.data(), received.bytes }); + if (!handle(connection, message)) + { + return false; + } + } + } + catch (const std::exception& error) + { + m_logger->warn("dropping camera reader '{}': {}", connection.name, error.what()); + return false; + } + } + + bool handle(Connection& connection, const WireMessage& message) + { + if (const auto* hello = std::get_if(&message)) + { + return greet(connection, *hello); + } + if (const auto* subscribe = std::get_if(&message)) + { + if (!connection.greeted) + { + m_logger->warn("dropping a camera reader that subscribed before it said hello"); + return false; + } + return answer(connection, *subscribe); + } + if (const auto* release = std::get_if(&message)) + { + if (connection.subscription != 0) + { + m_publisher.release(connection.subscription, release->lease_id); + } + return true; + } + if (std::holds_alternative(message)) + { + return true; // from a newer reader; not for this build + } + m_logger->warn("dropping camera reader '{}': it sent a message only a plugin sends", connection.name); + return false; + } + + bool greet(Connection& connection, const HelloMessage& hello) + { + connection.name = hello.reader_name.empty() ? "unnamed" : hello.reader_name; + WelcomeMessage welcome; + if (hello.protocol_version != WIRE_PROTOCOL_VERSION) + { + welcome.error = "this camera plugin speaks protocol version " + std::to_string(WIRE_PROTOCOL_VERSION) + + " and the reader version " + std::to_string(hello.protocol_version); + m_logger->warn("refusing camera reader '{}': {}", connection.name, welcome.error); + (void)send(connection, m_encoder.encode(welcome)); + return false; + } + const std::size_t group_count = m_publisher.group_count(); + for (std::size_t i = 0; i < group_count; ++i) + { + welcome.groups.push_back(m_publisher.group_description(static_cast(i))); + } + connection.greeted = true; + m_logger->info("camera reader '{}' connected", connection.name); + return send(connection, m_encoder.encode(welcome)); + } + + bool answer(Connection& connection, const SubscribeMessage& subscribe) + { + SubscribedMessage reply; + std::span descriptors; + const std::optional group = m_publisher.find_group(subscribe.group); + if (connection.subscription != 0) + { + reply.error = "a connection carries one subscription; open another connection for a second group"; + } + else if (!group) + { + reply.error = "the camera plugin publishes no group '" + subscribe.group + "'"; + } + else if (!m_memory.contains(*group)) + { + reply.error = "camera group '" + subscribe.group + "' has no memory a reader can map"; + } + else + { + // From here on the publisher may send this reader frame events at any moment, so a + // Frame can reach it before the Subscribed below. + const SubscribeResult result = m_publisher.subscribe(connection, *group, subscribe.options); + if (result) + { + connection.subscription = result.id; + reply.group = m_publisher.group_description(*group); + descriptors = m_memory.at(*group); + } + else + { + reply.error = result.error; + } + } + + if (reply.error.empty()) + { + m_logger->info("camera reader '{}' subscribed to '{}' (window {}, {})", connection.name, subscribe.group, + subscribe.options.window, name_of(subscribe.options.order)); + } + else + { + m_logger->warn("refusing camera reader '{}': {}", connection.name, reply.error); + } + return send(connection, m_encoder.encode(reply), descriptors); + } + + static bool send(Connection& connection, std::span bytes, std::span descriptors = {}) + { + return connection.socket.send(bytes, descriptors) == SendStatus::SENT; + } + + Publisher& m_publisher; + MessageListener m_listener; + UniqueFd m_wake; + std::map> m_memory; + + std::thread m_thread; + std::atomic m_stopping{ false }; + std::atomic m_reader_count{ 0 }; + + // The endpoint's thread only. + std::vector> m_connections; + std::vector m_buffer; + WireEncoder m_encoder; + + std::shared_ptr m_logger; +}; + +Endpoint::Endpoint(Publisher& publisher, const std::string& path) : m_impl(std::make_unique(publisher, path)) +{ +} + +Endpoint::~Endpoint() = default; + +void Endpoint::set_group_memory(GroupId group, std::vector descriptors) +{ + m_impl->set_group_memory(group, std::move(descriptors)); +} + +void Endpoint::start() +{ + m_impl->start(); +} + +void Endpoint::stop() +{ + m_impl->stop(); +} + +std::size_t Endpoint::reader_count() const +{ + return m_impl->reader_count(); +} + +} // namespace core::camera diff --git a/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp b/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp new file mode 100644 index 0000000000..3d68fa052f --- /dev/null +++ b/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include + +namespace core::camera +{ + +/** + * @brief Serves a Publisher's groups to readers in other processes. + * + * Listens on a socket file. Each reader is told what is published; when it subscribes it is + * sent the group's pool descriptors and from then on the group's frame events. A reader that + * goes away, cleanly or not, has every frame set it held taken back. + */ +class Endpoint +{ +public: + /** + * @brief Start listening on `path`. Readers are answered once start() has been called. + * @param publisher Must outlive the endpoint. + * @throws std::system_error if another process listens on the path or it cannot be bound. + */ + Endpoint(Publisher& publisher, const std::string& path); + + // Stops serving. + ~Endpoint(); + + Endpoint(const Endpoint&) = delete; + Endpoint& operator=(const Endpoint&) = delete; + + /** + * @brief Name what a reader maps for a group: one descriptor per stream, in stream order, + * for the stream's pool. The endpoint does not own them; they stay open while it serves. + * A group without them cannot be subscribed to. + * @throws std::logic_error after start(). + * @throws std::out_of_range for a group the publisher does not have. + * @throws std::invalid_argument if the count does not match the group's streams. + */ + void set_group_memory(GroupId group, std::vector descriptors); + + void start(); + + /** + * @brief Stop serving: every subscription ends and every reader sees its connection close. + * Does nothing if the endpoint is not serving. + */ + void stop(); + + // Readers connected right now. + std::size_t reader_count() const; + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace core::camera diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 35bbcad055..803dfcffbf 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -18,6 +18,7 @@ add_subdirectory(camera_core) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) add_subdirectory(camera_wire) + add_subdirectory(camera_endpoint) endif() if(BUILD_OVXR_CAMERA_QUADS) diff --git a/tests/cpp/core/camera_endpoint/CMakeLists.txt b/tests/cpp/core/camera_endpoint/CMakeLists.txt new file mode 100644 index 0000000000..5a8373c671 --- /dev/null +++ b/tests/cpp/core/camera_endpoint/CMakeLists.txt @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(camera_endpoint_tests + test_endpoint.cpp +) + +# The tests play the reader's part with the socket and the codec themselves. +target_link_libraries(camera_endpoint_tests PRIVATE + camera::camera_endpoint + camera::camera_ipc + camera::camera_wire + Catch2::Catch2WithMain + Threads::Threads +) + +message(STATUS "camera_endpoint_tests target enabled with Catch2") +catch_discover_tests(camera_endpoint_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_endpoint/test_endpoint.cpp b/tests/cpp/core/camera_endpoint/test_endpoint.cpp new file mode 100644 index 0000000000..fb92b8611b --- /dev/null +++ b/tests/cpp/core/camera_endpoint/test_endpoint.cpp @@ -0,0 +1,416 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; +constexpr std::uint32_t SLOTS = 8; + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_endpoint_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +GroupDescription gray_group(const std::string& id, std::size_t streams) +{ + GroupDescription group; + group.id = id; + for (std::size_t i = 0; i < streams; ++i) + { + StreamDescription stream; + stream.id = id + "/" + std::to_string(i); + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = WIDTH; + stream.slot_bytes = WIDTH * HEIGHT; + stream.slot_count = SLOTS; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// A plugin in miniature: a stereo group and a single camera, in host memory, behind an endpoint. +struct Served +{ + TemporaryDirectory directory; + Publisher publisher; + GroupId head = publisher.add_group(gray_group("head", 2)); + GroupId wrist = publisher.add_group(gray_group("wrist", 1)); + HostPool left{ SLOTS, WIDTH* HEIGHT, "test_left" }; + HostPool right{ SLOTS, WIDTH* HEIGHT, "test_right" }; + HostPool hand{ SLOTS, WIDTH* HEIGHT, "test_wrist" }; + Endpoint endpoint{ publisher, directory.socket_path() }; + + Served() + { + endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); + endpoint.set_group_memory(wrist, { hand.descriptor() }); + endpoint.start(); + } + + // Publishes one stereo frame set whose every pixel is `value`. + std::uint64_t publish(std::uint8_t value) + { + std::vector images(2); + HostPool* pools[] = { &left, &right }; + for (std::size_t i = 0; i < 2; ++i) + { + const auto slot = publisher.acquire_slot(head, i); + REQUIRE(slot.has_value()); + std::fill(pools[i]->slot(*slot).begin(), pools[i]->slot(*slot).end(), std::byte{ value }); + images[i].slot = *slot; + } + return publisher.publish(head, images, {}); + } +}; + +// The reader's part, played with the socket and the codec directly. +struct Reader +{ + struct Received + { + ReceiveStatus status = ReceiveStatus::TIMEOUT; + WireMessage message; + std::vector descriptors; + }; + + explicit Reader(const Served& served) : socket(MessageSocket::connect(served.directory.socket_path())) + { + } + + template + void send(const Message& message) + { + REQUIRE(socket.send(encoder.encode(message)) == SendStatus::SENT); + } + + Received receive(std::chrono::milliseconds timeout = 2s) + { + Received result; + ReceivedMessage received = socket.receive(buffer, timeout); + result.status = received.status; + if (received.status == ReceiveStatus::MESSAGE) + { + result.message = decode_wire_message({ buffer.data(), received.bytes }); + result.descriptors = std::move(received.descriptors); + } + return result; + } + + template + Message expect(std::chrono::milliseconds timeout = 2s) + { + Received received = receive(timeout); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(std::holds_alternative(received.message)); + return std::get(std::move(received.message)); + } + + WelcomeMessage hello(const std::string& name = "test reader") + { + send(HelloMessage{ .reader_name = name }); + return expect(); + } + + // Subscribes and returns the answer with the pool descriptors that came with it. + std::pair> subscribe(const std::string& group, ReaderOptions options = {}) + { + send(SubscribeMessage{ group, options }); + Received received = receive(); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(std::holds_alternative(received.message)); + return { std::get(std::move(received.message)), std::move(received.descriptors) }; + } + + MessageSocket socket; + WireEncoder encoder; + std::vector buffer = std::vector(MAX_WIRE_MESSAGE_BYTES); +}; + +bool wait_until(const std::function& condition, std::chrono::milliseconds timeout = 2s) +{ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (!condition()) + { + if (std::chrono::steady_clock::now() > deadline) + { + return false; + } + std::this_thread::sleep_for(1ms); + } + return true; +} + +} // namespace + +TEST_CASE("a reader is told what is published", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + + const WelcomeMessage welcome = reader.hello(); + REQUIRE(welcome.error.empty()); + REQUIRE(welcome.protocol_version == WIRE_PROTOCOL_VERSION); + REQUIRE(welcome.groups.size() == 2); + REQUIRE(welcome.groups[0].id == "head"); + REQUIRE(welcome.groups[0].streams.size() == 2); + REQUIRE(welcome.groups[0].streams[1].id == "head/1"); + REQUIRE(welcome.groups[0].streams[1].width == WIDTH); + REQUIRE(welcome.groups[1].id == "wrist"); + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 1; })); +} + +TEST_CASE("a reader that speaks another protocol version is refused", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + + reader.send(HelloMessage{ .protocol_version = WIRE_PROTOCOL_VERSION + 1, .reader_name = "from the future" }); + const WelcomeMessage welcome = reader.expect(); + REQUIRE_THAT(welcome.error, ContainsSubstring("protocol version")); + REQUIRE(welcome.groups.empty()); + REQUIRE(reader.receive().status == ReceiveStatus::CLOSED); +} + +TEST_CASE("a subscriber maps the pools and is sent frame sets", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + reader.hello(); + + auto [subscribed, descriptors] = reader.subscribe("head"); + REQUIRE(subscribed.error.empty()); + REQUIRE(subscribed.group.id == "head"); + REQUIRE(descriptors.size() == 2); + const HostPoolView left(std::move(descriptors[0]), SLOTS, WIDTH * HEIGHT); + const HostPoolView right(std::move(descriptors[1]), SLOTS, WIDTH * HEIGHT); + + served.publish(0x11); + const FrameEvent first = reader.expect(); + REQUIRE(first.sequence == 1); + REQUIRE(first.image_count == 2); + REQUIRE(first.skipped == 0); + REQUIRE(first.timeline.published_ns != TIME_UNKNOWN); + REQUIRE(first.timeline.sent_ns >= first.timeline.published_ns); + REQUIRE(left.slot(first.images[0].slot).front() == std::byte{ 0x11 }); + REQUIRE(right.slot(first.images[1].slot).back() == std::byte{ 0x11 }); + + // A window of one holds one frame set; the next comes with the release, and is the newest. + served.publish(0x22); + served.publish(0x33); + REQUIRE(reader.receive(20ms).status == ReceiveStatus::TIMEOUT); + REQUIRE(left.slot(first.images[0].slot).front() == std::byte{ 0x11 }); // still held, so not rewritten + + reader.send(ReleaseMessage{ first.lease_id }); + const FrameEvent newest = reader.expect(); + REQUIRE(newest.sequence == 3); + REQUIRE(newest.skipped == 1); + REQUIRE(left.slot(newest.images[0].slot).front() == std::byte{ 0x33 }); +} + +TEST_CASE("a subscription that cannot be served is refused and the connection stays usable", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + reader.hello(); + + auto [unknown, no_descriptors] = reader.subscribe("chest"); + REQUIRE_THAT(unknown.error, ContainsSubstring("no group 'chest'")); + REQUIRE(no_descriptors.empty()); + + auto [too_large, none] = reader.subscribe("head", { .window = SLOTS }); + REQUIRE_THAT(too_large.error, ContainsSubstring("does not fit the pool")); + REQUIRE(none.empty()); + + auto [accepted, descriptors] = reader.subscribe("wrist"); + REQUIRE(accepted.error.empty()); + REQUIRE(descriptors.size() == 1); + + auto [second, extra] = reader.subscribe("head"); + REQUIRE_THAT(second.error, ContainsSubstring("one subscription")); + REQUIRE(extra.empty()); +} + +TEST_CASE("a reader that disappears returns what it held", "[unit][camera_endpoint]") +{ + Served served; + { + Reader reader(served); + reader.hello(); + auto [subscribed, descriptors] = reader.subscribe("head", { .window = 2, .order = DeliveryOrder::IN_ORDER }); + REQUIRE(subscribed.error.empty()); + served.publish(1); + served.publish(2); + REQUIRE(reader.expect().sequence == 1); + REQUIRE(reader.expect().sequence == 2); + REQUIRE(served.endpoint.reader_count() == 1); + // Gone without releasing either frame set. + } + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 0; })); + + // With the held slots back, capture runs through the whole pool again and again. + for (std::uint8_t i = 0; i < 3 * SLOTS; ++i) + { + served.publish(i); + } + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + + // And the room in the pool is there for the next reader. + Reader next(served); + next.hello(); + auto [again, descriptors] = next.subscribe("head", { .window = 2, .order = DeliveryOrder::IN_ORDER }); + REQUIRE(again.error.empty()); +} + +TEST_CASE("a reader that breaks the protocol is dropped and the others are not", "[unit][camera_endpoint]") +{ + Served served; + Reader good(served); + good.hello(); + auto [subscribed, descriptors] = good.subscribe("head"); + REQUIRE(subscribed.error.empty()); + + Reader garbage(served); + const std::vector noise(32, std::byte{ 0x5a }); + REQUIRE(garbage.socket.send(noise) == SendStatus::SENT); + REQUIRE(garbage.receive().status == ReceiveStatus::CLOSED); + + Reader impostor(served); + impostor.hello(); + impostor.send(WelcomeMessage{}); + REQUIRE(impostor.receive().status == ReceiveStatus::CLOSED); + + Reader impatient(served); + impatient.send(SubscribeMessage{ "head", {} }); + REQUIRE(impatient.receive().status == ReceiveStatus::CLOSED); + + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 1; })); + served.publish(7); + REQUIRE(good.expect().sequence == 1); +} + +TEST_CASE("readers do not slow each other", "[unit][camera_endpoint]") +{ + Served served; + Reader stalled(served); + stalled.hello(); + REQUIRE(stalled.subscribe("head").first.error.empty()); + Reader live(served); + live.hello(); + REQUIRE(live.subscribe("head").first.error.empty()); + + // One reader never releases; the other still gets every frame set. + for (std::uint64_t sequence = 1; sequence <= 5; ++sequence) + { + REQUIRE(served.publish(static_cast(sequence)) == sequence); + const FrameEvent event = live.expect(); + REQUIRE(event.sequence == sequence); + REQUIRE(event.skipped == 0); + // Whether the release is seen before or after the next publish, that frame set is next. + live.send(ReleaseMessage{ event.lease_id }); + } + REQUIRE(stalled.expect().sequence == 1); + REQUIRE(stalled.receive(20ms).status == ReceiveStatus::TIMEOUT); +} + +TEST_CASE("stopping the endpoint closes every reader, and it can serve again", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + reader.hello(); + REQUIRE(reader.subscribe("head").first.error.empty()); + + served.endpoint.stop(); + REQUIRE(reader.receive().status == ReceiveStatus::CLOSED); + REQUIRE(served.endpoint.reader_count() == 0); + served.endpoint.stop(); // nothing to stop + + served.endpoint.start(); + Reader next(served); + REQUIRE(next.hello().groups.size() == 2); + REQUIRE(next.subscribe("head").first.error.empty()); + served.publish(9); + REQUIRE(next.expect().sequence == 1); +} + +TEST_CASE("a group needs its memory named before readers can subscribe", "[unit][camera_endpoint]") +{ + TemporaryDirectory directory; + Publisher publisher; + const GroupId head = publisher.add_group(gray_group("head", 2)); + HostPool pool(SLOTS, WIDTH * HEIGHT, "test"); + Endpoint endpoint(publisher, directory.socket_path()); + + REQUIRE_THROWS_AS(endpoint.set_group_memory(head, { pool.descriptor() }), std::invalid_argument); + REQUIRE_THROWS_AS(endpoint.set_group_memory(head + 1, {}), std::out_of_range); + endpoint.start(); + REQUIRE_THROWS_AS(endpoint.set_group_memory(head, { pool.descriptor(), pool.descriptor() }), std::logic_error); + + MessageSocket socket = MessageSocket::connect(directory.socket_path()); + WireEncoder encoder; + std::vector buffer(MAX_WIRE_MESSAGE_BYTES); + REQUIRE(socket.send(encoder.encode(HelloMessage{})) == SendStatus::SENT); + REQUIRE(socket.receive(buffer, 2s).status == ReceiveStatus::MESSAGE); + REQUIRE(socket.send(encoder.encode(SubscribeMessage{ "head", {} })) == SendStatus::SENT); + const ReceivedMessage received = socket.receive(buffer, 2s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + const WireMessage answer = decode_wire_message({ buffer.data(), received.bytes }); + REQUIRE_THAT(std::get(answer).error, ContainsSubstring("no memory a reader can map")); + + // Another endpoint cannot take the same path while this one lives. + REQUIRE_THROWS_AS(Endpoint(publisher, directory.socket_path()), std::system_error); +} From 8fb524a1e9b100bfe6c0b9894a61f09bad546e34 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 17:17:17 -0800 Subject: [PATCH 12/16] fix(camera): a stopped endpoint gives up its path stop() ended the endpoint's thread but kept the listening socket, so a reader that connected afterwards was accepted by the kernel and then waited for an answer nobody would send. stop() now closes the listener, which removes the socket file and releases the path: a reader that tries to connect is refused at once. start() takes the path again. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/camera_endpoint/cpp/endpoint.cpp | 19 ++++++++++++++----- .../cpp/inc/camera_endpoint/endpoint.hpp | 9 +++++++-- .../core/camera_endpoint/test_endpoint.cpp | 3 +++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/core/camera_endpoint/cpp/endpoint.cpp b/src/core/camera_endpoint/cpp/endpoint.cpp index 380265eff9..652a06afaf 100644 --- a/src/core/camera_endpoint/cpp/endpoint.cpp +++ b/src/core/camera_endpoint/cpp/endpoint.cpp @@ -85,7 +85,8 @@ class Endpoint::Impl public: Impl(Publisher& publisher, const std::string& path) : m_publisher(publisher), - m_listener(path), + m_path(path), + m_listener(std::in_place, path), m_wake(::eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK)), m_buffer(MAX_WIRE_MESSAGE_BYTES), m_logger(isaaccapture::Logger::get("isaaccapture.camera_endpoint.Endpoint")) @@ -121,9 +122,13 @@ class Endpoint::Impl { return; } + if (!m_listener) + { + m_listener.emplace(m_path); + } m_stopping = false; m_thread = std::thread([this] { run(); }); - m_logger->info("serving cameras on '{}'", m_listener.path()); + m_logger->info("serving cameras on '{}'", m_path); } void stop() @@ -137,6 +142,9 @@ class Endpoint::Impl // Cannot fail in a way that matters: the counter only overflows after 2^64 wake-ups. [[maybe_unused]] const ssize_t written = ::write(m_wake.get(), &one, sizeof(one)); m_thread.join(); + // Without the listener a reader that tries to connect is refused at once, instead of + // waiting in a queue nobody serves. + m_listener.reset(); } std::size_t reader_count() const @@ -150,7 +158,7 @@ class Endpoint::Impl std::vector waits; while (!m_stopping) { - waits.assign({ { m_wake.get(), POLLIN, 0 }, { m_listener.descriptor(), POLLIN, 0 } }); + waits.assign({ { m_wake.get(), POLLIN, 0 }, { m_listener->descriptor(), POLLIN, 0 } }); for (const auto& connection : m_connections) { waits.push_back({ connection->socket.descriptor(), POLLIN, 0 }); @@ -191,7 +199,7 @@ class Endpoint::Impl { try { - while (auto socket = m_listener.accept(std::chrono::milliseconds(0))) + while (auto socket = m_listener->accept(std::chrono::milliseconds(0))) { m_connections.push_back(std::make_unique(std::move(*socket))); m_reader_count = m_connections.size(); @@ -352,7 +360,8 @@ class Endpoint::Impl } Publisher& m_publisher; - MessageListener m_listener; + std::string m_path; + std::optional m_listener; // absent between stop() and the next start() UniqueFd m_wake; std::map> m_memory; diff --git a/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp b/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp index 3d68fa052f..7183340c5a 100644 --- a/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp +++ b/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp @@ -46,11 +46,16 @@ class Endpoint */ void set_group_memory(GroupId group, std::vector descriptors); + /** + * @brief Serve readers. After a stop() this takes the path again. + * @throws std::system_error if another process took the path meanwhile. + */ void start(); /** - * @brief Stop serving: every subscription ends and every reader sees its connection close. - * Does nothing if the endpoint is not serving. + * @brief Stop serving: every subscription ends, every reader sees its connection close, and + * the path is given up, so a reader that tries to connect is refused at once. Does nothing + * if the endpoint is not serving. */ void stop(); diff --git a/tests/cpp/core/camera_endpoint/test_endpoint.cpp b/tests/cpp/core/camera_endpoint/test_endpoint.cpp index fb92b8611b..d99eba0ad5 100644 --- a/tests/cpp/core/camera_endpoint/test_endpoint.cpp +++ b/tests/cpp/core/camera_endpoint/test_endpoint.cpp @@ -379,6 +379,9 @@ TEST_CASE("stopping the endpoint closes every reader, and it can serve again", " REQUIRE(served.endpoint.reader_count() == 0); served.endpoint.stop(); // nothing to stop + // A stopped endpoint refuses a reader at once; it does not leave it waiting for an answer. + REQUIRE_THROWS_AS(MessageSocket::connect(served.directory.socket_path()), std::system_error); + served.endpoint.start(); Reader next(served); REQUIRE(next.hello().groups.size() == 2); From 363e246333c8fadab36f8aa2e0e7883e5be0672f Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:58:08 -0800 Subject: [PATCH 13/16] feat(camera): add camera_reader, what an application links to read cameras CameraReader::open subscribes to one group of the camera plugin and next() hands out frame sets. A FrameSet is a hold on the frames: its images are the plugin's own buffers, mapped read-only, and stay as they are until the object is destroyed or released. Nothing is copied. With newest order and a window larger than one, next() takes the newest frame set that is waiting, gives the older ones back and counts them as skipped. Frame sets that arrive before the plugin's answer to the subscription are kept and handed out first. A frame set can be given back from another thread and after its reader is gone. A plugin that goes away is reported as CameraDisconnected; opening fails with the plugin's own reason when a subscription is refused. camera_core now exposes the two rules the reader shares with the publisher, resolved_order and plane_size_of, instead of keeping them private. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/CMakeLists.txt | 1 + .../camera_core/cpp/inc/camera_core/types.hpp | 35 ++ src/core/camera_core/cpp/publisher.cpp | 38 +- src/core/camera_reader/AGENTS.md | 14 + src/core/camera_reader/CMakeLists.txt | 6 + src/core/camera_reader/README.md | 40 ++ src/core/camera_reader/cpp/CMakeLists.txt | 23 ++ src/core/camera_reader/cpp/camera_reader.cpp | 364 +++++++++++++++++ .../cpp/inc/camera_reader/camera_reader.hpp | 174 ++++++++ tests/cpp/core/CMakeLists.txt | 1 + tests/cpp/core/camera_reader/CMakeLists.txt | 24 ++ .../core/camera_reader/test_camera_reader.cpp | 378 ++++++++++++++++++ 12 files changed, 1062 insertions(+), 36 deletions(-) create mode 100644 src/core/camera_reader/AGENTS.md create mode 100644 src/core/camera_reader/CMakeLists.txt create mode 100644 src/core/camera_reader/README.md create mode 100644 src/core/camera_reader/cpp/CMakeLists.txt create mode 100644 src/core/camera_reader/cpp/camera_reader.cpp create mode 100644 src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp create mode 100644 tests/cpp/core/camera_reader/CMakeLists.txt create mode 100644 tests/cpp/core/camera_reader/test_camera_reader.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index bafa5c8a6b..3a606f2e8b 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -27,6 +27,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) add_subdirectory(camera_wire) add_subdirectory(camera_endpoint) + add_subdirectory(camera_reader) endif() # Build OXR library (depends on oxr_utils for types) diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp index 1e66086a1a..9aa4567509 100644 --- a/src/core/camera_core/cpp/inc/camera_core/types.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -42,6 +42,31 @@ struct PlaneLayout std::uint32_t stride_bytes = 0; // from the start of one row to the start of the next }; +// The pixels of one plane: the bytes one row holds, without padding, and the number of rows. +struct PlaneSize +{ + std::uint64_t row_bytes = 0; + std::uint64_t rows = 0; +}; + +constexpr PlaneSize plane_size_of(PixelFormat format, std::uint32_t width, std::uint32_t height, std::uint32_t plane) +{ + const std::uint64_t columns = width; + const std::uint64_t rows = height; + switch (format) + { + case PixelFormat::BGRA8: + case PixelFormat::RGBA8: + return { columns * 4, rows }; + case PixelFormat::GRAY8: + return { columns, rows }; + case PixelFormat::NV12: + // Luma at full size, then one interleaved chroma pair per two by two pixels. + return plane == 0 ? PlaneSize{ columns, rows } : PlaneSize{ (columns + 1) / 2 * 2, (rows + 1) / 2 }; + } + return {}; +} + // Whether a stream's slots hold pixels or one access unit of encoded video each. enum class Codec : std::uint32_t { @@ -132,4 +157,14 @@ struct ReaderOptions DeliveryOrder order = DeliveryOrder::DEFAULT; }; +// The order a reader with these options gets: what it asked for, or the default for its window. +constexpr DeliveryOrder resolved_order(const ReaderOptions& options) +{ + if (options.order != DeliveryOrder::DEFAULT) + { + return options.order; + } + return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; +} + } // namespace core::camera diff --git a/src/core/camera_core/cpp/publisher.cpp b/src/core/camera_core/cpp/publisher.cpp index 5222ae681a..37cd59e85c 100644 --- a/src/core/camera_core/cpp/publisher.cpp +++ b/src/core/camera_core/cpp/publisher.cpp @@ -20,40 +20,6 @@ std::int64_t steady_now_ns() return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); } -DeliveryOrder resolve_order(const ReaderOptions& options) -{ - if (options.order != DeliveryOrder::DEFAULT) - { - return options.order; - } - return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; -} - -// The bytes one row of a plane holds and the number of its rows. -struct PlaneSize -{ - std::uint64_t row_bytes = 0; - std::uint64_t rows = 0; -}; - -PlaneSize plane_size_of(const StreamDescription& stream, std::uint32_t plane) -{ - const std::uint64_t width = stream.width; - const std::uint64_t height = stream.height; - switch (stream.pixel_format) - { - case PixelFormat::BGRA8: - case PixelFormat::RGBA8: - return { width * 4, height }; - case PixelFormat::GRAY8: - return { width, height }; - case PixelFormat::NV12: - // Luma at full size, then one interleaved chroma pair per two by two pixels. - return plane == 0 ? PlaneSize{ width, height } : PlaneSize{ (width + 1) / 2 * 2, (height + 1) / 2 }; - } - return {}; -} - // A pixel stream names every plane of its format and every plane lies inside the slot; an // encoded stream has no planes. bool layout_is_valid(const StreamDescription& stream) @@ -68,7 +34,7 @@ bool layout_is_valid(const StreamDescription& stream) } for (std::uint32_t i = 0; i < stream.plane_count; ++i) { - const PlaneSize size = plane_size_of(stream, i); + const PlaneSize size = plane_size_of(stream.pixel_format, stream.width, stream.height, i); const std::uint64_t stride = stream.planes[i].stride_bytes; // 64-bit sums of 32-bit fields, so nothing here can wrap. const std::uint64_t end = stream.planes[i].offset_bytes + stride * (size.rows - 1) + size.row_bytes; @@ -275,7 +241,7 @@ SubscribeResult Publisher::subscribe(EventSink& sink, GroupId group, const Reade return result; } Group& target = *m_groups[group]; - const DeliveryOrder order = resolve_order(options); + const DeliveryOrder order = resolved_order(options); // The ring keeps what in-order readers may still be waiting for; newest-only readers // need the newest frame set and nothing older. diff --git a/src/core/camera_reader/AGENTS.md b/src/core/camera_reader/AGENTS.md new file mode 100644 index 0000000000..732ad6314a --- /dev/null +++ b/src/core/camera_reader/AGENTS.md @@ -0,0 +1,14 @@ + + +# Agent notes — `camera_reader` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +- **The public header shows `camera_core` types only.** The socket, the pools and the wire messages stay in the `.cpp`; an application that links the reader does not see them. +- **No copies and no conversions here.** A view is the plugin's buffer. Pixel conversion, if a consumer wants it, is a separate step on top. +- **A frame set outlives its reader safely.** Both share one state object; keep it that way rather than making `FrameSet` point back at the reader. +- **Releasing must never throw or block.** It runs in destructors, possibly on another thread; a plugin that is gone takes its frames back by itself. +- **The plugin is trusted for content, not for bounds.** Check a frame event against the group before handing out views. diff --git a/src/core/camera_reader/CMakeLists.txt b/src/core/camera_reader/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_reader/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_reader/README.md b/src/core/camera_reader/README.md new file mode 100644 index 0000000000..605857a925 --- /dev/null +++ b/src/core/camera_reader/README.md @@ -0,0 +1,40 @@ + + +# Camera interface: the reader + +What an application links to get camera frames. It talks to the camera plugin's +[endpoint](../camera_endpoint) and hands out frame sets whose pixels are the plugin's own +buffers, mapped read-only; nothing is copied. + +```cpp +#include + +CameraReader reader = CameraReader::open(endpoint, "head"); +while (auto frames = reader.next(std::chrono::seconds(1))) +{ + const ImageView left = frames->image(0); + process(left.plane(0), left.stream->width, left.stream->height, left.stream->planes[0].stride_bytes); +} +``` + +- **A `FrameSet` is a hold on the frames.** While it exists the plugin does not reuse those + buffers, so the pixels stay as they are. It gives them back when it is destroyed, or + earlier with `release()`. Hold at most as many as the reader's window. +- **Window and order** are chosen in `open` ([terms](../camera_core/README.md)). The default, + a window of one, always gives the newest frame set and says in `skipped()` how many were + passed over. A recorder asks for a larger window and gets every frame set in order. +- **`next` returns nothing on timeout** and throws `CameraDisconnected` when the plugin is + gone. Opening fails with a `CameraReaderError` that says why: nothing listens on the + endpoint, the group does not exist, or the subscription does not fit the pool. +- **One reader reads one group.** For two groups, open two readers; `descriptor()` lets an + application wait on several at once. +- A `FrameSet` may be given back from another thread than the one that calls `next`, and + after its reader is gone. + +## Tests + +`tests/cpp/core/camera_reader`, run with `ctest -L camera_reader`. They are tagged `unit` +and read from a real endpoint in the same process. diff --git a/src/core/camera_reader/cpp/CMakeLists.txt b/src/core/camera_reader/cpp/CMakeLists.txt new file mode 100644 index 0000000000..c2ad541a18 --- /dev/null +++ b/src/core/camera_reader/cpp/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# What an application links to read cameras from the camera plugin. +add_library(camera_reader STATIC + camera_reader.cpp +) + +target_include_directories(camera_reader INTERFACE + $ +) + +target_link_libraries(camera_reader + PUBLIC + camera::camera_core + PRIVATE + camera::camera_ipc + camera::camera_wire +) + +add_library(camera::camera_reader ALIAS camera_reader) diff --git a/src/core/camera_reader/cpp/camera_reader.cpp b/src/core/camera_reader/cpp/camera_reader.cpp new file mode 100644 index 0000000000..a09a445916 --- /dev/null +++ b/src/core/camera_reader/cpp/camera_reader.cpp @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_reader/camera_reader.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// How long the plugin gets to answer the handshake and the subscription. +constexpr std::chrono::milliseconds ANSWER_TIMEOUT(2000); + +struct Received +{ + WireMessage message; + std::vector descriptors; +}; + +// Waits for one message. Nothing on timeout. +std::optional receive(MessageSocket& socket, std::span buffer, std::chrono::milliseconds timeout) +{ + ReceivedMessage received = socket.receive(buffer, timeout); + if (received.status == ReceiveStatus::TIMEOUT) + { + return std::nullopt; + } + if (received.status == ReceiveStatus::CLOSED) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + try + { + return Received{ decode_wire_message(buffer.first(received.bytes)), std::move(received.descriptors) }; + } + catch (const WireError& error) + { + throw CameraReaderError(std::string("the camera plugin sent something that is not a message: ") + error.what()); + } +} + +MessageSocket connect_to(const std::string& endpoint) +{ + try + { + return MessageSocket::connect(endpoint); + } + catch (const std::system_error& error) + { + throw CameraReaderError("no camera plugin answers on '" + endpoint + "': " + error.code().message()); + } +} + +// Says hello and returns what the plugin publishes. +std::vector greet(MessageSocket& socket, WireEncoder& encoder, std::span buffer) +{ + HelloMessage hello; + hello.reader_name = program_invocation_short_name; + if (socket.send(encoder.encode(hello)) != SendStatus::SENT) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + const auto deadline = std::chrono::steady_clock::now() + ANSWER_TIMEOUT; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + std::optional received = receive(socket, buffer, left); + if (!received) + { + throw CameraReaderError("the camera plugin did not answer the reader's hello"); + } + if (auto* welcome = std::get_if(&received->message)) + { + if (!welcome->error.empty()) + { + throw CameraReaderError("the camera plugin refused this reader: " + welcome->error); + } + return std::move(welcome->groups); + } + } +} + +} // namespace + +namespace detail +{ + +// Shared by a reader and the frame sets it handed out, so a frame set can be given back after +// its reader is gone. +class ReaderState +{ +public: + explicit ReaderState(MessageSocket connected) : socket(std::move(connected)), buffer(MAX_WIRE_MESSAGE_BYTES) + { + } + + // Called from a FrameSet, possibly on another thread than the one that reads. + void release(std::uint64_t lease_id) noexcept + { + try + { + const std::lock_guard lock(m_release_mutex); + (void)socket.send(m_release_encoder.encode(ReleaseMessage{ lease_id })); + } + catch (const std::exception&) + { + // The plugin is gone or will drop this reader; either way it takes the frame set back. + } + } + + // The next frame event, from the ones that arrived early or from the socket. + std::optional take(std::chrono::milliseconds timeout) + { + if (!early.empty()) + { + const FrameEvent event = early.front(); + early.pop_front(); + return event; + } + const auto deadline = std::chrono::steady_clock::now() + timeout; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + const std::optional received = receive(socket, buffer, left); + if (!received) + { + return std::nullopt; + } + if (const auto* event = std::get_if(&received->message)) + { + return *event; + } + // Anything else is not for a subscribed reader of this build; keep waiting. + } + } + + // A frame set that names a slot or a size the pools do not have cannot be read safely. + void check(const FrameEvent& event) const + { + bool fits = event.image_count == group.streams.size(); + for (std::size_t i = 0; fits && i < event.image_count; ++i) + { + fits = event.images[i].slot < group.streams[i].slot_count && + event.images[i].bytes_used <= group.streams[i].slot_bytes; + } + if (!fits) + { + throw CameraReaderError("the camera plugin sent a frame set that does not match group '" + group.id + "'"); + } + } + + MessageSocket socket; + GroupDescription group; + ReaderOptions options; + std::vector pools; // one per stream + std::deque early; // frame events that arrived before the answer to the subscription + std::vector buffer; // the reading thread only + +private: + std::mutex m_release_mutex; + WireEncoder m_release_encoder; +}; + +} // namespace detail + +std::span ImageView::plane(std::uint32_t index) const +{ + if (stream == nullptr || index >= stream->plane_count) + { + throw std::out_of_range("the image has no plane " + std::to_string(index)); + } + const PlaneSize size = plane_size_of(stream->pixel_format, stream->width, stream->height, index); + const std::uint64_t offset = stream->planes[index].offset_bytes; + const std::uint64_t bytes = stream->planes[index].stride_bytes * (size.rows - 1) + size.row_bytes; + if (size.rows == 0 || offset + bytes > data.size()) + { + throw std::out_of_range("plane " + std::to_string(index) + " does not lie inside the image's slot"); + } + return data.subspan(offset, bytes); +} + +FrameSet::FrameSet(std::shared_ptr state, const FrameEvent& event) + : m_state(std::move(state)), m_event(event) +{ +} + +FrameSet::FrameSet(FrameSet&& other) noexcept : m_state(std::move(other.m_state)), m_event(other.m_event) +{ +} + +FrameSet& FrameSet::operator=(FrameSet&& other) noexcept +{ + if (this != &other) + { + release(); + m_state = std::move(other.m_state); + m_event = other.m_event; + } + return *this; +} + +FrameSet::~FrameSet() +{ + release(); +} + +ImageView FrameSet::image(std::size_t stream) const +{ + if (!m_state) + { + throw std::logic_error("the frame set was given back"); + } + if (stream >= m_event.image_count) + { + throw std::out_of_range("the frame set has no image " + std::to_string(stream)); + } + const StreamDescription& description = m_state->group.streams[stream]; + const ImageInfo& info = m_event.images[stream]; + const std::span slot = m_state->pools[stream].slot(info.slot); + return { &description, info, description.is_encoded() ? slot.first(info.bytes_used) : slot }; +} + +void FrameSet::release() noexcept +{ + if (m_state) + { + m_state->release(m_event.lease_id); + m_state.reset(); + } +} + +std::vector CameraReader::list(const std::string& endpoint) +{ + MessageSocket socket = connect_to(endpoint); + WireEncoder encoder; + std::vector buffer(MAX_WIRE_MESSAGE_BYTES); + return greet(socket, encoder, buffer); +} + +CameraReader CameraReader::open(const std::string& endpoint, const std::string& group, const ReaderOptions& options) +{ + auto state = std::make_shared(connect_to(endpoint)); + WireEncoder encoder; + greet(state->socket, encoder, state->buffer); + + if (state->socket.send(encoder.encode(SubscribeMessage{ group, options })) != SendStatus::SENT) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + const auto deadline = std::chrono::steady_clock::now() + ANSWER_TIMEOUT; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + std::optional received = receive(state->socket, state->buffer, left); + if (!received) + { + throw CameraReaderError("the camera plugin did not answer the subscription to '" + group + "'"); + } + // The plugin starts sending frame sets as soon as the subscription exists, which can be + // before its answer is on the way. + if (const auto* event = std::get_if(&received->message)) + { + state->early.push_back(*event); + continue; + } + auto* subscribed = std::get_if(&received->message); + if (subscribed == nullptr) + { + continue; + } + if (!subscribed->error.empty()) + { + throw CameraReaderError(subscribed->error); + } + if (received->descriptors.size() != subscribed->group.streams.size()) + { + throw CameraReaderError("the camera plugin did not send one pool per stream of '" + group + "'"); + } + state->group = std::move(subscribed->group); + state->options = { options.window, resolved_order(options) }; + try + { + for (std::size_t i = 0; i < state->group.streams.size(); ++i) + { + const StreamDescription& stream = state->group.streams[i]; + state->pools.emplace_back(std::move(received->descriptors[i]), stream.slot_count, stream.slot_bytes); + } + } + catch (const std::exception& error) + { + throw CameraReaderError("cannot map the pools of '" + group + "': " + error.what()); + } + return CameraReader(std::move(state)); + } +} + +CameraReader::CameraReader(std::shared_ptr state) : m_state(std::move(state)) +{ +} + +CameraReader::CameraReader(CameraReader&& other) noexcept = default; +CameraReader& CameraReader::operator=(CameraReader&& other) noexcept = default; +CameraReader::~CameraReader() = default; + +const GroupDescription& CameraReader::group() const +{ + return m_state->group; +} + +std::optional CameraReader::next(std::chrono::milliseconds timeout) +{ + std::optional event = m_state->take(timeout); + if (!event) + { + return std::nullopt; + } + // With a window of one nothing can be waiting behind this event, so the extra look at the + // socket is skipped. + if (m_state->options.order == DeliveryOrder::NEWEST && m_state->options.window > 1) + { + while (const std::optional newer = m_state->take(std::chrono::milliseconds(0))) + { + m_state->release(event->lease_id); + const std::uint64_t skipped = std::uint64_t{ event->skipped } + newer->skipped + 1; + event = newer; + event->skipped = static_cast(std::min(skipped, UINT32_MAX)); + } + } + try + { + m_state->check(*event); + } + catch (const CameraReaderError&) + { + m_state->release(event->lease_id); + throw; + } + return FrameSet(m_state, *event); +} + +int CameraReader::descriptor() const +{ + return m_state->socket.descriptor(); +} + +} // namespace core::camera diff --git a/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp b/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp new file mode 100644 index 0000000000..0920638a28 --- /dev/null +++ b/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace detail +{ +class ReaderState; +} + +/** + * @brief A reader could not be opened or cannot go on: nothing listens on the endpoint, the + * plugin refused the reader, or it sent something a reader cannot use. + */ +class CameraReaderError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +/** + * @brief The camera plugin closed the connection or is gone. + */ +class CameraDisconnected : public CameraReaderError +{ +public: + using CameraReaderError::CameraReaderError; +}; + +/** + * @brief One image of a frame set. The memory is the plugin's pool, mapped read-only, and + * stays as it is for as long as the FrameSet the view came from is held. + */ +struct ImageView +{ + const StreamDescription* stream = nullptr; // what the bytes are; owned by the reader + ImageInfo info; + std::span data; // pixels: the whole slot; encoded video: the access unit + + /** + * @brief The bytes of one plane of a pixel image, from its first row to the end of its last. + * Rows are stream->planes[index].stride_bytes apart. + * @throws std::out_of_range for a plane the image does not have. + */ + std::span plane(std::uint32_t index) const; +}; + +/** + * @brief One frame set, held for this reader: the plugin does not reuse its slots until the + * object is destroyed or release() is called. Hold at most as many as the reader's window. + */ +class FrameSet +{ +public: + FrameSet(FrameSet&& other) noexcept; + FrameSet& operator=(FrameSet&& other) noexcept; + FrameSet(const FrameSet&) = delete; + FrameSet& operator=(const FrameSet&) = delete; + ~FrameSet(); + + std::uint64_t sequence() const + { + return m_event.sequence; + } + + // Changes when the camera restarted. + std::uint32_t generation() const + { + return m_event.generation; + } + + // Frame sets published since the previous one this reader was given, and not given to it. + std::uint32_t skipped() const + { + return m_event.skipped; + } + + const FrameTimeline& timeline() const + { + return m_event.timeline; + } + + std::size_t image_count() const + { + return m_event.image_count; + } + + /** + * @brief The image of one stream, in the order of the group's streams. + * @throws std::out_of_range for a stream the group does not have. + * @throws std::logic_error after release(). + */ + ImageView image(std::size_t stream) const; + + // Gives the frame set back before the object is destroyed. Its views are no longer valid. + void release() noexcept; + +private: + friend class CameraReader; + FrameSet(std::shared_ptr state, const FrameEvent& event); + + std::shared_ptr m_state; + FrameEvent m_event; +}; + +/** + * @brief Reads one group of cameras from the camera plugin. + * + * @code + * CameraReader reader = CameraReader::open(endpoint, "head"); + * while (auto frames = reader.next(std::chrono::seconds(1))) + * { + * std::span left = frames->image(0).data; + * } + * @endcode + */ +class CameraReader +{ +public: + /** + * @brief What the plugin behind `endpoint` publishes. + * @throws CameraReaderError if nothing listens there or the plugin refuses this reader. + */ + static std::vector list(const std::string& endpoint); + + /** + * @brief Subscribe to one group. A reader for a second group is a second CameraReader. + * @throws CameraReaderError if nothing listens on the endpoint or the plugin refuses the + * subscription; the message says why. + */ + static CameraReader open(const std::string& endpoint, const std::string& group, const ReaderOptions& options = {}); + + CameraReader(CameraReader&& other) noexcept; + CameraReader& operator=(CameraReader&& other) noexcept; + CameraReader(const CameraReader&) = delete; + CameraReader& operator=(const CameraReader&) = delete; + ~CameraReader(); + + const GroupDescription& group() const; + + /** + * @brief Wait up to `timeout` for a frame set. With newest order this is the newest one the + * plugin has sent; older ones waiting behind it are given back and counted as skipped. + * @return The frame set, or nothing on timeout. + * @throws CameraDisconnected if the plugin is gone. + * @throws CameraReaderError if the plugin sent a frame set that does not match the group. + */ + std::optional next(std::chrono::milliseconds timeout); + + // Becomes readable when a frame set may be waiting; for waiting on several readers at once. + int descriptor() const; + +private: + explicit CameraReader(std::shared_ptr state); + + std::shared_ptr m_state; +}; + +} // namespace core::camera diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 803dfcffbf..d8efd2cdb3 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -19,6 +19,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) add_subdirectory(camera_wire) add_subdirectory(camera_endpoint) + add_subdirectory(camera_reader) endif() if(BUILD_OVXR_CAMERA_QUADS) diff --git a/tests/cpp/core/camera_reader/CMakeLists.txt b/tests/cpp/core/camera_reader/CMakeLists.txt new file mode 100644 index 0000000000..4c14425de0 --- /dev/null +++ b/tests/cpp/core/camera_reader/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(camera_reader_tests + test_camera_reader.cpp +) + +# The reader is tested against a real endpoint, and against a stand-in plugin built from the +# socket and the codec for what a real one does not do on demand. +target_link_libraries(camera_reader_tests PRIVATE + camera::camera_reader + camera::camera_endpoint + camera::camera_ipc + camera::camera_wire + Catch2::Catch2WithMain + Threads::Threads +) + +message(STATUS "camera_reader_tests target enabled with Catch2") +catch_discover_tests(camera_reader_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_reader/test_camera_reader.cpp b/tests/cpp/core/camera_reader/test_camera_reader.cpp new file mode 100644 index 0000000000..34965d34e6 --- /dev/null +++ b/tests/cpp/core/camera_reader/test_camera_reader.cpp @@ -0,0 +1,378 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; +constexpr std::uint32_t STRIDE = 80; // rows are padded +constexpr std::uint32_t SLOTS = 8; +constexpr std::uint64_t SLOT_BYTES = STRIDE * HEIGHT; + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_reader_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +GroupDescription stereo_group() +{ + GroupDescription group; + group.id = "head"; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = std::string("head/") + eye; + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = STRIDE; + stream.slot_bytes = SLOT_BYTES; + stream.slot_count = SLOTS; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// A plugin in miniature: a stereo group in host memory behind an endpoint. +struct Served +{ + TemporaryDirectory directory; + std::string endpoint_path = directory.socket_path(); + Publisher publisher; + GroupId head = publisher.add_group(stereo_group()); + HostPool left{ SLOTS, SLOT_BYTES, "test_left" }; + HostPool right{ SLOTS, SLOT_BYTES, "test_right" }; + Endpoint endpoint{ publisher, endpoint_path }; + + Served() + { + endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); + endpoint.start(); + } + + // Publishes one frame set: every left pixel is `value`, every right pixel `value + 1`. + std::uint64_t publish(std::uint8_t value, const FrameTimeline& timeline = {}) + { + std::vector images(2); + HostPool* pools[] = { &left, &right }; + for (std::size_t i = 0; i < 2; ++i) + { + const auto slot = publisher.acquire_slot(head, i); + REQUIRE(slot.has_value()); + const std::byte fill{ static_cast(value + i) }; + std::fill(pools[i]->slot(*slot).begin(), pools[i]->slot(*slot).end(), fill); + images[i].slot = *slot; + images[i].device_frame_id = 1000 + value; + } + return publisher.publish(head, images, timeline); + } +}; + +std::byte first_pixel(const FrameSet& frames, std::size_t stream) +{ + return frames.image(stream).data.front(); +} + +} // namespace + +TEST_CASE("a reader lists what a plugin publishes", "[unit][camera_reader]") +{ + Served served; + const std::vector groups = CameraReader::list(served.endpoint_path); + REQUIRE(groups.size() == 1); + REQUIRE(groups[0].id == "head"); + REQUIRE(groups[0].streams.size() == 2); + REQUIRE(groups[0].streams[0].id == "head/left"); +} + +TEST_CASE("a reader gets each frame set with its pixels and what is known about it", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open(served.endpoint_path, "head"); + REQUIRE(reader.group().id == "head"); + REQUIRE(reader.group().streams.size() == 2); + REQUIRE(reader.descriptor() >= 0); + + REQUIRE_FALSE(reader.next(10ms).has_value()); // nothing published yet + + served.publish(0x10, { .capture_ns = 111, .sdk_ready_ns = 222 }); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 1); + REQUIRE(frames->skipped() == 0); + REQUIRE(frames->image_count() == 2); + REQUIRE(frames->timeline().capture_ns == 111); + REQUIRE(frames->timeline().sdk_ready_ns == 222); + REQUIRE(frames->timeline().sent_ns >= frames->timeline().published_ns); + + const ImageView left = frames->image(0); + const ImageView right = frames->image(1); + REQUIRE(left.stream->id == "head/left"); + REQUIRE(left.info.device_frame_id == 1000 + 0x10); + REQUIRE(left.data.size() == SLOT_BYTES); + REQUIRE(left.data.front() == std::byte{ 0x10 }); + REQUIRE(right.data.back() == std::byte{ 0x11 }); + + // A plane runs from its first row to the end of its last, without the last row's padding. + REQUIRE(left.plane(0).size() == STRIDE * (HEIGHT - 1) + WIDTH); + REQUIRE(left.plane(0).data() == left.data.data()); + REQUIRE_THROWS_AS(left.plane(1), std::out_of_range); + REQUIRE_THROWS_AS(frames->image(2), std::out_of_range); +} + +TEST_CASE("a held frame set is not rewritten and the next one is the newest", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open(served.endpoint_path, "head"); + + served.publish(1); + std::optional held = reader.next(2s); + REQUIRE(held.has_value()); + + // Capture runs through the pool several times while the reader holds on. + for (std::uint8_t value = 2; value <= 3 * SLOTS; ++value) + { + served.publish(value); + } + REQUIRE(first_pixel(*held, 0) == std::byte{ 1 }); + REQUIRE(first_pixel(*held, 1) == std::byte{ 2 }); + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + + held.reset(); // gives it back + std::optional newest = reader.next(2s); + REQUIRE(newest.has_value()); + REQUIRE(newest->sequence() == 3 * SLOTS); + REQUIRE(newest->skipped() == 3 * SLOTS - 2); + REQUIRE(first_pixel(*newest, 0) == std::byte{ 3 * SLOTS }); + + // release() gives a frame set back before it goes out of scope. + newest->release(); + REQUIRE_THROWS_AS(newest->image(0), std::logic_error); + served.publish(99); + REQUIRE(reader.next(2s)->sequence() == 3 * SLOTS + 1); +} + +TEST_CASE("an in-order reader gets every frame set", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open(served.endpoint_path, "head", { .window = 3 }); + + served.publish(1); + served.publish(2); + served.publish(3); + std::vector held; + for (std::uint64_t sequence = 1; sequence <= 3; ++sequence) + { + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == sequence); + REQUIRE(frames->skipped() == 0); + held.push_back(std::move(*frames)); + } + // The window is full: three are held together, and all three are still readable. + served.publish(4); + REQUIRE_FALSE(reader.next(20ms).has_value()); + REQUIRE(first_pixel(held[0], 0) == std::byte{ 1 }); + REQUIRE(first_pixel(held[2], 0) == std::byte{ 3 }); + + held.erase(held.begin()); + REQUIRE(reader.next(2s)->sequence() == 4); +} + +TEST_CASE("a newest-order reader with a larger window skips to the newest waiting frame set", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = + CameraReader::open(served.endpoint_path, "head", { .window = 3, .order = DeliveryOrder::NEWEST }); + + served.publish(1); + served.publish(2); + served.publish(3); + // Three events are waiting; the reader takes the last and gives the others back. + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 3); + REQUIRE(frames->skipped() == 2); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 3 }); + + // Giving those two back made room in the window again, with the third still held: once the + // plugin has seen the releases, a newly published frame set reaches the reader. + std::optional more; + for (std::uint8_t value = 4; value < 200 && !more; ++value) + { + served.publish(value); + more = reader.next(20ms); + } + REQUIRE(more.has_value()); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 3 }); +} + +TEST_CASE("a frame set can be given back after its reader is gone", "[unit][camera_reader]") +{ + Served served; + std::optional frames; + { + CameraReader reader = CameraReader::open(served.endpoint_path, "head"); + served.publish(5); + frames = reader.next(2s); + REQUIRE(frames.has_value()); + CameraReader moved = std::move(reader); + REQUIRE(moved.group().id == "head"); + } + // The pixels stay readable for as long as the frame set is held. + REQUIRE(first_pixel(*frames, 0) == std::byte{ 5 }); + frames.reset(); +} + +TEST_CASE("a reader says why it could not be opened", "[unit][camera_reader]") +{ + TemporaryDirectory directory; + REQUIRE_THROWS_WITH( + CameraReader::open(directory.socket_path(), "head"), ContainsSubstring("no camera plugin answers on")); + REQUIRE_THROWS_AS(CameraReader::list(directory.socket_path()), CameraReaderError); + + Served served; + REQUIRE_THROWS_WITH(CameraReader::open(served.endpoint_path, "chest"), ContainsSubstring("no group 'chest'")); + REQUIRE_THROWS_WITH(CameraReader::open(served.endpoint_path, "head", { .window = SLOTS }), + ContainsSubstring("does not fit the pool")); + + // A refusal leaves nothing behind: the pool still has room for a reader that fits. + REQUIRE_NOTHROW(CameraReader::open(served.endpoint_path, "head", { .window = 2 })); +} + +TEST_CASE("a reader is told when the plugin is gone", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open(served.endpoint_path, "head"); + served.publish(1); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + + served.endpoint.stop(); + REQUIRE_THROWS_AS(reader.next(2s), CameraDisconnected); + // What the reader holds is still mapped, and giving it back to nobody is harmless. + REQUIRE(first_pixel(*frames, 0) == std::byte{ 1 }); + frames.reset(); +} + +TEST_CASE("a reader copes with what a plugin may send", "[unit][camera_reader]") +{ + // A stand-in plugin, to send what a real one does not send on demand. + TemporaryDirectory directory; + MessageListener listener(directory.socket_path()); + HostPool left(SLOTS, SLOT_BYTES, "test_left"); + HostPool right(SLOTS, SLOT_BYTES, "test_right"); + left.slot(2)[0] = std::byte{ 0x42 }; + + FrameEvent early; + early.lease_id = 7; + early.sequence = 1; + early.image_count = 2; + early.images[0].slot = 2; + FrameEvent bad_slot = early; + bad_slot.lease_id = 8; + bad_slot.sequence = 2; + bad_slot.images[1].slot = SLOTS; + + std::vector released; + std::thread plugin( + [&] + { + try + { + WireEncoder encoder; + std::vector buffer(MAX_WIRE_MESSAGE_BYTES); + std::optional reader = listener.accept(2s); + if (!reader) + { + return; + } + const auto receive = [&] { + return decode_wire_message({ buffer.data(), reader->receive(buffer, 2s).bytes }); + }; + (void)receive(); // hello + WelcomeMessage welcome; + welcome.groups = { stereo_group() }; + reader->send(encoder.encode(welcome)); + (void)receive(); // subscribe + + // A frame set overtakes the answer to the subscription. + reader->send(encoder.encode(early)); + const std::array descriptors = { left.descriptor(), right.descriptor() }; + reader->send(encoder.encode(SubscribedMessage{ "", stereo_group() }), descriptors); + reader->send(encoder.encode(bad_slot)); + for (int i = 0; i < 2; ++i) + { + released.push_back(std::get(receive()).lease_id); + } + } + catch (const std::exception&) + { + // The checks on `released` below fail. + } + }); + + CameraReader reader = CameraReader::open(directory.socket_path(), "head"); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 1); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 0x42 }); + frames.reset(); + + // A frame set that names a slot the pool does not have is refused and given back. + REQUIRE_THROWS_WITH(reader.next(2s), ContainsSubstring("does not match group 'head'")); + plugin.join(); + REQUIRE(released == std::vector{ 7, 8 }); +} From d11f9135a163984c5a43efbfd3d17406a37d47eb Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 17:09:01 -0800 Subject: [PATCH 14/16] feat(camera): add the camera plugin's core: backend interface and host A backend opens cameras through one SDK. It implements CameraSource for a configured camera: describe() says what the camera publishes, start() gets a FrameOutput to acquire slots from and publish frame sets to. CameraHost is the plugin process without its main(): it allocates each camera's pools in shared host memory, registers the camera with one publisher, and serves all cameras to readers on one endpoint. A camera is configured as ID:BACKEND[:KEY=VALUE,...]. OptionReader makes a backend refuse options it does not read, so a misspelt option is an error instead of a setting that is silently ignored. A camera that names a backend the build does not have is refused with the list of those it has. The executable, the list of built backends and the synthetic backend follow in the next pull request. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/plugins/CMakeLists.txt | 1 + src/plugins/camera/AGENTS.md | 14 + src/plugins/camera/CMakeLists.txt | 10 + src/plugins/camera/core/CMakeLists.txt | 27 ++ src/plugins/camera/core/camera_host.cpp | 194 +++++++++++++ .../core/inc/camera_plugin/camera_host.hpp | 71 +++++ .../core/inc/camera_plugin/camera_source.hpp | 95 +++++++ .../camera/core/inc/camera_plugin/options.hpp | 55 ++++ src/plugins/camera/core/options.cpp | 107 +++++++ tests/cpp/plugins/CMakeLists.txt | 1 + tests/cpp/plugins/camera/CMakeLists.txt | 17 ++ tests/cpp/plugins/camera/test_camera_host.cpp | 265 ++++++++++++++++++ 12 files changed, 857 insertions(+) create mode 100644 src/plugins/camera/AGENTS.md create mode 100644 src/plugins/camera/CMakeLists.txt create mode 100644 src/plugins/camera/core/CMakeLists.txt create mode 100644 src/plugins/camera/core/camera_host.cpp create mode 100644 src/plugins/camera/core/inc/camera_plugin/camera_host.hpp create mode 100644 src/plugins/camera/core/inc/camera_plugin/camera_source.hpp create mode 100644 src/plugins/camera/core/inc/camera_plugin/options.hpp create mode 100644 src/plugins/camera/core/options.cpp create mode 100644 tests/cpp/plugins/camera/CMakeLists.txt create mode 100644 tests/cpp/plugins/camera/test_camera_host.cpp diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index c26d9384c2..929e096f65 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -26,6 +26,7 @@ add_subdirectory(rebot_devarm_leader) add_subdirectory(manus) add_subdirectory(manus_calibration) add_subdirectory(haptikos) +add_subdirectory(camera) if(BUILD_PLUGIN_NOITOM_MOCAP) add_subdirectory(noitom_mocap) diff --git a/src/plugins/camera/AGENTS.md b/src/plugins/camera/AGENTS.md new file mode 100644 index 0000000000..08a391fda7 --- /dev/null +++ b/src/plugins/camera/AGENTS.md @@ -0,0 +1,14 @@ + + +# Agent notes — camera plugin + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +- **Backends are chosen at build time.** One CMake option per backend, off by default; a backend that is off is not configured, so its SDK is never looked for. Do not load backends at run time and do not add a backend ABI. +- **No SDK types outside a backend's directory.** `core/` and `app/` see `CameraSource` and `camera_core` types only, and readers never link a camera SDK. +- **A backend refuses what it does not understand.** Read options through `OptionReader` and call `finish()`, so a misspelt option is an error. +- **A camera drops a frame rather than wait.** When `acquire` has no free slot, abandon what was taken for that frame set and carry on; nothing on the capture path blocks on readers. +- **One plugin process hosts every backend.** A second instance on another endpoint is a configuration, not a different design. diff --git a/src/plugins/camera/CMakeLists.txt b/src/plugins/camera/CMakeLists.txt new file mode 100644 index 0000000000..c65feb2e17 --- /dev/null +++ b/src/plugins/camera/CMakeLists.txt @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The camera plugin: one process that owns the cameras and serves their frames to readers. +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(STATUS "Skipping camera plugin (Linux only)") + return() +endif() + +add_subdirectory(core) diff --git a/src/plugins/camera/core/CMakeLists.txt b/src/plugins/camera/core/CMakeLists.txt new file mode 100644 index 0000000000..c84fda947e --- /dev/null +++ b/src/plugins/camera/core/CMakeLists.txt @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Threads REQUIRED) + +# What a camera backend implements, and the host that gives each camera its pools and puts +# them all behind one endpoint. +add_library(camera_plugin_core STATIC + camera_host.cpp + options.cpp +) + +target_include_directories(camera_plugin_core INTERFACE + $ +) + +target_link_libraries(camera_plugin_core + PUBLIC + camera::camera_core + PRIVATE + camera::camera_endpoint + camera::camera_ipc + log_bridge::log_bridge_core + Threads::Threads +) + +add_library(camera::plugin_core ALIAS camera_plugin_core) diff --git a/src/plugins/camera/core/camera_host.cpp b/src/plugins/camera/core/camera_host.cpp new file mode 100644 index 0000000000..1eb59e4ee7 --- /dev/null +++ b/src/plugins/camera/core/camera_host.cpp @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_plugin/camera_host.hpp" + +#include +#include +#include + +#include +#include +#include + +namespace plugins::camera +{ + +using core::camera::Endpoint; +using core::camera::FrameTimeline; +using core::camera::GroupDescription; +using core::camera::GroupId; +using core::camera::HostPool; +using core::camera::ImageInfo; +using core::camera::Publisher; +using core::camera::StreamDescription; + +namespace +{ + +// One camera's way into the publisher: slots come from the camera's own pools. +class PoolOutput final : public FrameOutput +{ +public: + PoolOutput(Publisher& publisher, GroupId group, std::vector>& pools) + : m_publisher(publisher), m_group(group), m_pools(pools) + { + } + + std::optional acquire(std::size_t stream) override + { + const std::optional slot = m_publisher.acquire_slot(m_group, stream); + if (!slot) + { + return std::nullopt; + } + return WritableImage{ *slot, m_pools.at(stream)->slot(*slot) }; + } + + void abandon(std::size_t stream, std::uint32_t slot) override + { + m_publisher.abandon_slot(m_group, stream, slot); + } + + std::uint64_t publish(std::span images, const FrameTimeline& timeline) override + { + return m_publisher.publish(m_group, images, timeline); + } + +private: + Publisher& m_publisher; + GroupId m_group; + std::vector>& m_pools; +}; + +struct Camera +{ + std::unique_ptr source; + std::string id; + std::vector> pools; // one per stream + std::unique_ptr output; +}; + +} // namespace + +const Backend& find_backend(std::span backends, const std::string& name) +{ + std::string built; + for (const Backend& backend : backends) + { + if (backend.name == name) + { + return backend; + } + built += (built.empty() ? "" : ", ") + backend.name; + } + throw std::invalid_argument("this camera plugin was built without a '" + name + "' backend (it has: " + built + + "). Backends are chosen when the plugin is built; see the plugin's README"); +} + +class CameraHost::Impl +{ +public: + explicit Impl(const std::string& endpoint_path) + : endpoint(publisher, endpoint_path), logger(isaaccapture::Logger::get("isaaccapture.plugins.camera.CameraHost")) + { + } + + Publisher publisher; + Endpoint endpoint; + // Stable addresses: an output points at its camera's pools. + std::vector> cameras; + bool started = false; + std::shared_ptr logger; +}; + +CameraHost::CameraHost(const std::string& endpoint_path) : m_impl(std::make_unique(endpoint_path)) +{ +} + +CameraHost::~CameraHost() +{ + stop(); +} + +void CameraHost::add_camera(std::unique_ptr source, std::uint32_t slot_count) +{ + if (m_impl->started) + { + throw std::logic_error("cameras are added before the camera host starts"); + } + GroupDescription description = source->describe(); + for (StreamDescription& stream : description.streams) + { + if (stream.slot_count == 0) + { + stream.slot_count = slot_count; + } + } + + auto camera = std::make_unique(); + camera->id = description.id; + // Refuses a description that is not usable before any memory is allocated for it. + const GroupId group = m_impl->publisher.add_group(description); + std::vector descriptors; + for (const StreamDescription& stream : description.streams) + { + camera->pools.push_back(std::make_unique(stream.slot_count, stream.slot_bytes, "camera." + stream.id)); + descriptors.push_back(camera->pools.back()->descriptor()); + } + m_impl->endpoint.set_group_memory(group, std::move(descriptors)); + camera->output = std::make_unique(m_impl->publisher, group, camera->pools); + camera->source = std::move(source); + + const StreamDescription& first = description.streams.front(); + m_impl->logger->info("camera '{}': {} stream(s), {}x{} at {:.2f} fps, {} slots of {} bytes each", description.id, + description.streams.size(), first.width, first.height, first.frame_rate, first.slot_count, + first.slot_bytes); + m_impl->cameras.push_back(std::move(camera)); +} + +void CameraHost::start() +{ + if (m_impl->started) + { + return; + } + m_impl->started = true; + m_impl->endpoint.start(); + for (const auto& camera : m_impl->cameras) + { + camera->source->start(*camera->output); + } +} + +void CameraHost::stop() +{ + if (!m_impl->started) + { + return; + } + // Cameras first: nothing publishes into a pool or the endpoint once they have stopped. + for (const auto& camera : m_impl->cameras) + { + camera->source->stop(); + } + m_impl->endpoint.stop(); + m_impl->started = false; +} + +std::size_t CameraHost::camera_count() const +{ + return m_impl->cameras.size(); +} + +std::size_t CameraHost::reader_count() const +{ + return m_impl->endpoint.reader_count(); +} + +const Publisher& CameraHost::publisher() const +{ + return m_impl->publisher; +} + +} // namespace plugins::camera diff --git a/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp b/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp new file mode 100644 index 0000000000..d0818a2234 --- /dev/null +++ b/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "camera_source.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace plugins::camera +{ + +// Slots per stream unless a camera says otherwise: one for the newest frame set, one for the +// camera to write ahead, and six for readers' windows. +inline constexpr std::uint32_t DEFAULT_SLOT_COUNT = 8; + +/** + * @brief Finds a backend by name. + * @throws std::invalid_argument naming the backends this build has, and how a build gets more. + */ +const Backend& find_backend(std::span backends, const std::string& name); + +/** + * @brief The cameras of one plugin process: gives each its pools and serves them all to + * readers on one endpoint. + */ +class CameraHost +{ +public: + /** + * @throws std::system_error if another process serves the endpoint or it cannot be bound. + */ + explicit CameraHost(const std::string& endpoint_path); + + // Stops the cameras and the endpoint. + ~CameraHost(); + + CameraHost(const CameraHost&) = delete; + CameraHost& operator=(const CameraHost&) = delete; + + /** + * @brief Add a camera and allocate its pools. Call before start(). + * @param slot_count For streams the camera described without one. + * @throws std::invalid_argument if the camera's description is not usable or its id is taken. + */ + void add_camera(std::unique_ptr source, std::uint32_t slot_count = DEFAULT_SLOT_COUNT); + + // Starts serving readers, then the cameras. + void start(); + + // Stops the cameras, then closes every reader. + void stop(); + + std::size_t camera_count() const; + std::size_t reader_count() const; + + // For counters and descriptions; publishing goes through each camera's FrameOutput. + const core::camera::Publisher& publisher() const; + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace plugins::camera diff --git a/src/plugins/camera/core/inc/camera_plugin/camera_source.hpp b/src/plugins/camera/core/inc/camera_plugin/camera_source.hpp new file mode 100644 index 0000000000..8ea82cf085 --- /dev/null +++ b/src/plugins/camera/core/inc/camera_plugin/camera_source.hpp @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins::camera +{ + +// One camera as the configuration names it. +struct CameraConfig +{ + std::string id; // the group readers ask for + std::string backend; // the backend that opens it + std::map options; // the backend's own settings +}; + +// A free slot of one stream, for the next image. +struct WritableImage +{ + std::uint32_t slot = 0; + std::span data; +}; + +/** + * @brief Where a camera's frames go. Safe to call from the camera's capture thread while + * readers come and go. + */ +class FrameOutput +{ +public: + virtual ~FrameOutput() = default; + + /** + * @brief A free slot of one stream to write the next image into. + * @return Nothing when every slot is in use: drop the frame, and abandon() what was already + * acquired for it. + */ + virtual std::optional acquire(std::size_t stream) = 0; + + // Gives back an acquired slot that will not be published. + virtual void abandon(std::size_t stream, std::uint32_t slot) = 0; + + /** + * @brief Publish one frame set: images[i] is stream i's, in a slot from acquire(). + * @param timeline capture_ns and sdk_ready_ns, on the host's monotonic clock. + * @return The frame set's sequence number. + */ + virtual std::uint64_t publish(std::span images, + const core::camera::FrameTimeline& timeline) = 0; +}; + +/** + * @brief One configured camera: one or more streams captured together. A backend implements + * this for its SDK. + */ +class CameraSource +{ +public: + virtual ~CameraSource() = default; + + /** + * @brief What the camera publishes: the group, its streams and the size of a slot. A + * stream's slot_count may be left 0 to take the plugin's pool size. + */ + virtual core::camera::GroupDescription describe() const = 0; + + // Start capturing into `output`, on a thread of the source's own. + virtual void start(FrameOutput& output) = 0; + + // Stop capturing. The output is not called once this has returned. + virtual void stop() = 0; +}; + +// One camera SDK as the plugin sees it. +struct Backend +{ + std::string name; // what a camera's configuration names + std::string description; + + // Opens one camera. Throws std::invalid_argument for options the backend does not accept. + std::function(const CameraConfig&)> open; +}; + +} // namespace plugins::camera diff --git a/src/plugins/camera/core/inc/camera_plugin/options.hpp b/src/plugins/camera/core/inc/camera_plugin/options.hpp new file mode 100644 index 0000000000..a1cf26ec00 --- /dev/null +++ b/src/plugins/camera/core/inc/camera_plugin/options.hpp @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "camera_source.hpp" + +#include +#include +#include +#include +#include + +namespace plugins::camera +{ + +/** + * @brief Reads a camera's options for a backend and refuses what the backend does not know, + * so a misspelt option is an error and not a silently ignored setting. + */ +class OptionReader +{ +public: + explicit OptionReader(const CameraConfig& config); + + /** + * @brief The option's value, or `fallback` when it is not set. + * @throws std::invalid_argument if the value is not a number between `minimum` and `maximum`. + */ + std::uint32_t number(std::string_view key, std::uint32_t fallback, std::uint32_t minimum, std::uint32_t maximum); + + // The option's text, or `fallback` when it is not set. + std::string text(std::string_view key, std::string_view fallback); + + /** + * @brief Call once every option was read. + * @throws std::invalid_argument naming an option that was set but never read. + */ + void finish() const; + + // For a backend's own complaints: "camera 'head' (synthetic backend): ". + [[noreturn]] void fail(const std::string& what) const; + +private: + const CameraConfig& m_config; + std::set> m_read; +}; + +/** + * @brief Parse one `--camera` argument: ID:BACKEND[:KEY=VALUE[,KEY=VALUE]...]. + * @throws std::invalid_argument if it does not have that form. + */ +CameraConfig parse_camera_argument(std::string_view argument); + +} // namespace plugins::camera diff --git a/src/plugins/camera/core/options.cpp b/src/plugins/camera/core/options.cpp new file mode 100644 index 0000000000..328996764d --- /dev/null +++ b/src/plugins/camera/core/options.cpp @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_plugin/options.hpp" + +#include +#include +#include + +namespace plugins::camera +{ + +namespace +{ + +constexpr std::string_view CAMERA_ARGUMENT_FORM = "ID:BACKEND[:KEY=VALUE[,KEY=VALUE]...]"; + +} + +OptionReader::OptionReader(const CameraConfig& config) : m_config(config) +{ +} + +std::uint32_t OptionReader::number(std::string_view key, std::uint32_t fallback, std::uint32_t minimum, std::uint32_t maximum) +{ + m_read.emplace(key); + const auto found = m_config.options.find(std::string(key)); + if (found == m_config.options.end()) + { + return fallback; + } + const std::string& value = found->second; + std::uint32_t result = 0; + const auto [end, error] = std::from_chars(value.data(), value.data() + value.size(), result); + if (error != std::errc() || end != value.data() + value.size() || result < minimum || result > maximum) + { + fail("option '" + std::string(key) + "' is '" + value + "'; it takes a whole number from " + + std::to_string(minimum) + " to " + std::to_string(maximum)); + } + return result; +} + +std::string OptionReader::text(std::string_view key, std::string_view fallback) +{ + m_read.emplace(key); + const auto found = m_config.options.find(std::string(key)); + return found == m_config.options.end() ? std::string(fallback) : found->second; +} + +void OptionReader::finish() const +{ + for (const auto& [key, value] : m_config.options) + { + if (!m_read.contains(key)) + { + fail("there is no option '" + key + "'"); + } + } +} + +void OptionReader::fail(const std::string& what) const +{ + throw std::invalid_argument("camera '" + m_config.id + "' (" + m_config.backend + " backend): " + what); +} + +CameraConfig parse_camera_argument(std::string_view argument) +{ + const auto fail = [&](const std::string& what) -> void + { + throw std::invalid_argument("--camera=" + std::string(argument) + ": " + what + "; the form is " + + std::string(CAMERA_ARGUMENT_FORM)); + }; + + CameraConfig config; + const std::size_t first = argument.find(':'); + if (first == std::string_view::npos) + { + fail("a camera needs an id and a backend"); + } + const std::size_t second = argument.find(':', first + 1); + config.id = argument.substr(0, first); + config.backend = argument.substr(first + 1, second == std::string_view::npos ? second : second - first - 1); + if (config.id.empty() || config.backend.empty()) + { + fail("a camera needs an id and a backend"); + } + + std::string_view rest = second == std::string_view::npos ? std::string_view() : argument.substr(second + 1); + while (!rest.empty()) + { + const std::size_t comma = rest.find(','); + const std::string_view pair = rest.substr(0, comma); + const std::size_t equals = pair.find('='); + if (equals == std::string_view::npos || equals == 0) + { + fail("'" + std::string(pair) + "' is not KEY=VALUE"); + } + if (!config.options.emplace(pair.substr(0, equals), pair.substr(equals + 1)).second) + { + fail("option '" + std::string(pair.substr(0, equals)) + "' is given twice"); + } + rest = comma == std::string_view::npos ? std::string_view() : rest.substr(comma + 1); + } + return config; +} + +} // namespace plugins::camera diff --git a/tests/cpp/plugins/CMakeLists.txt b/tests/cpp/plugins/CMakeLists.txt index fd7e2d2e68..8e22e0f06c 100644 --- a/tests/cpp/plugins/CMakeLists.txt +++ b/tests/cpp/plugins/CMakeLists.txt @@ -11,6 +11,7 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(spacemouse) add_subdirectory(gamepad) + add_subdirectory(camera) endif() if(BUILD_PLUGIN_OGLO) diff --git a/tests/cpp/plugins/camera/CMakeLists.txt b/tests/cpp/plugins/camera/CMakeLists.txt new file mode 100644 index 0000000000..cda01eed40 --- /dev/null +++ b/tests/cpp/plugins/camera/CMakeLists.txt @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_plugin_tests + test_camera_host.cpp +) + +target_link_libraries(camera_plugin_tests PRIVATE + camera::plugin_core + camera::camera_reader + Catch2::Catch2WithMain +) + +message(STATUS "camera_plugin_tests target enabled with Catch2") +catch_discover_tests(camera_plugin_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/plugins/camera/test_camera_host.cpp b/tests/cpp/plugins/camera/test_camera_host.cpp new file mode 100644 index 0000000000..6b5983c828 --- /dev/null +++ b/tests/cpp/plugins/camera/test_camera_host.cpp @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; +using core::camera::CameraReader; +using core::camera::FrameSet; +using core::camera::GroupDescription; +using core::camera::ImageInfo; +using core::camera::PixelFormat; +using core::camera::StreamDescription; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_host_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +// A camera that publishes when the test says so. `control` outlives the source, which the +// host owns. +struct ManualControl +{ + FrameOutput* output = nullptr; // set while the camera runs + std::size_t streams = 0; + + // Publishes one frame set whose every pixel is `value`. False when it had to be dropped. + bool publish(std::uint8_t value) + { + std::vector images(streams); + for (std::size_t stream = 0; stream < streams; ++stream) + { + const std::optional image = output->acquire(stream); + if (!image) + { + for (std::size_t taken = 0; taken < stream; ++taken) + { + output->abandon(taken, images[taken].slot); + } + return false; + } + std::fill(image->data.begin(), image->data.end(), std::byte{ value }); + images[stream].slot = image->slot; + } + output->publish(images, {}); + return true; + } +}; + +class ManualSource final : public CameraSource +{ +public: + ManualSource(std::string id, std::size_t streams, ManualControl& control, std::uint32_t slot_count = 0) + : m_id(std::move(id)), m_slot_count(slot_count), m_control(control) + { + m_control.streams = streams; + } + + GroupDescription describe() const override + { + GroupDescription group; + group.id = m_id; + for (std::size_t i = 0; i < m_control.streams; ++i) + { + StreamDescription stream; + stream.id = m_id + "/" + std::to_string(i); + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = WIDTH; + stream.slot_bytes = WIDTH * HEIGHT; + stream.slot_count = m_slot_count; + stream.frame_rate = 30.0; + group.streams.push_back(stream); + } + return group; + } + + void start(FrameOutput& output) override + { + m_control.output = &output; + } + + void stop() override + { + m_control.output = nullptr; + } + +private: + std::string m_id; + std::uint32_t m_slot_count; + ManualControl& m_control; +}; + +} // namespace + +TEST_CASE("a camera argument names an id, a backend and the backend's options", "[unit][camera_plugin]") +{ + const CameraConfig bare = parse_camera_argument("head:synthetic"); + REQUIRE(bare.id == "head"); + REQUIRE(bare.backend == "synthetic"); + REQUIRE(bare.options.empty()); + + const CameraConfig full = parse_camera_argument("wrist:synthetic:width=640,height=480,format=gray8"); + REQUIRE(full.id == "wrist"); + REQUIRE(full.options.size() == 3); + REQUIRE(full.options.at("width") == "640"); + REQUIRE(full.options.at("format") == "gray8"); + + for (const char* malformed : { "head", ":synthetic", "head:", "head:synthetic:width", "head:synthetic:=3", + "head:synthetic:width=1,width=2" }) + { + REQUIRE_THROWS_WITH(parse_camera_argument(malformed), ContainsSubstring("ID:BACKEND")); + } +} + +TEST_CASE("options a backend does not read are refused", "[unit][camera_plugin]") +{ + const CameraConfig config = parse_camera_argument("head:zed:width=640,serial=1234,widht=7"); + OptionReader options(config); + REQUIRE(options.number("width", 1280, 16, 8192) == 640); + REQUIRE(options.number("height", 720, 16, 8192) == 720); // not set: the fallback + REQUIRE(options.text("serial", "") == "1234"); + REQUIRE(options.text("mode", "hd720") == "hd720"); + REQUIRE_THROWS_WITH(options.finish(), ContainsSubstring("camera 'head' (zed backend): there is no option 'widht'")); + + REQUIRE_THROWS_WITH(options.number("serial", 0, 0, 100), ContainsSubstring("whole number from 0 to 100")); + const CameraConfig words = parse_camera_argument("head:zed:width=wide,height=-3,fps=60.5"); + OptionReader strict(words); + REQUIRE_THROWS_AS(strict.number("width", 1, 0, 100), std::invalid_argument); + REQUIRE_THROWS_AS(strict.number("height", 1, 0, 100), std::invalid_argument); + REQUIRE_THROWS_AS(strict.number("fps", 1, 0, 100), std::invalid_argument); +} + +TEST_CASE("a backend that is not in the build is named with the ones that are", "[unit][camera_plugin]") +{ + const std::vector backends = { { "synthetic", "", {} }, { "zed", "", {} } }; + REQUIRE(find_backend(backends, "zed").name == "zed"); + REQUIRE_THROWS_WITH(find_backend(backends, "oak"), ContainsSubstring("built without a 'oak' backend") && + ContainsSubstring("synthetic, zed") && + ContainsSubstring("chosen when the plugin is built")); +} + +TEST_CASE("a host gives each camera its pools and serves them all on one endpoint", "[unit][camera_plugin]") +{ + TemporaryDirectory directory; + ManualControl head; + ManualControl wrist; + CameraHost host(directory.socket_path()); + host.add_camera(std::make_unique("head", 2, head)); + host.add_camera(std::make_unique("wrist", 1, wrist, 4), 16); // the camera's own count wins + REQUIRE(host.camera_count() == 2); + + ManualControl unused; + REQUIRE_THROWS_AS(host.add_camera(std::make_unique("head", 1, unused)), std::invalid_argument); + REQUIRE_THROWS_AS(host.add_camera(std::make_unique("empty", 0, unused)), std::invalid_argument); + + host.start(); + REQUIRE(head.output != nullptr); + REQUIRE_THROWS_AS(host.add_camera(std::make_unique("late", 1, unused)), std::logic_error); + + const std::vector groups = CameraReader::list(directory.socket_path()); + REQUIRE(groups.size() == 2); + REQUIRE(groups[0].id == "head"); + REQUIRE(groups[0].streams[0].slot_count == DEFAULT_SLOT_COUNT); + REQUIRE(groups[1].id == "wrist"); + REQUIRE(groups[1].streams[0].slot_count == 4); + + CameraReader head_reader = CameraReader::open(directory.socket_path(), "head"); + CameraReader wrist_reader = CameraReader::open(directory.socket_path(), "wrist"); + REQUIRE(host.reader_count() == 2); + + REQUIRE(head.publish(0x21)); + REQUIRE(wrist.publish(0x42)); + const std::optional pair = head_reader.next(2s); + REQUIRE(pair.has_value()); + REQUIRE(pair->image_count() == 2); + REQUIRE(pair->image(0).data.front() == std::byte{ 0x21 }); + REQUIRE(pair->image(1).data.back() == std::byte{ 0x21 }); + const std::optional single = wrist_reader.next(2s); + REQUIRE(single.has_value()); + REQUIRE(single->image(0).data.front() == std::byte{ 0x42 }); + REQUIRE(host.publisher().stats(0).published == 1); + + host.stop(); + REQUIRE(head.output == nullptr); + REQUIRE(host.reader_count() == 0); +} + +TEST_CASE("a camera drops a frame set when its pool is full, and never overwrites a held one", "[unit][camera_plugin]") +{ + TemporaryDirectory directory; + ManualControl camera; + CameraHost host(directory.socket_path()); + host.add_camera(std::make_unique("head", 2, camera), 3); + host.start(); + + // A window of one in a pool of three: the newest, the reader's, and one to write ahead. + CameraReader reader = CameraReader::open(directory.socket_path(), "head"); + REQUIRE(camera.publish(1)); + const std::optional held = reader.next(2s); + REQUIRE(held.has_value()); + for (std::uint8_t value = 2; value < 20; ++value) + { + REQUIRE(camera.publish(value)); // the ring lets go of the frame set before, so there is room + } + REQUIRE(held->image(0).data.front() == std::byte{ 1 }); + REQUIRE(host.publisher().stats(0).no_free_slot == 0); + + // A backend that takes a slot and neither publishes nor abandons it is what fills a pool. + const std::optional forgotten = camera.output->acquire(0); + REQUIRE(forgotten.has_value()); + REQUIRE_FALSE(camera.publish(99)); // stream 0 has no free slot left; the frame set is dropped + REQUIRE(host.publisher().stats(0).no_free_slot == 1); + REQUIRE(held->image(0).data.front() == std::byte{ 1 }); + + camera.output->abandon(0, forgotten->slot); + REQUIRE(camera.publish(100)); +} From 8e6a6a1ca6463664e2fb31e9a9443f7cbd3eb944 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 17:12:46 -0800 Subject: [PATCH 15/16] feat(camera): add the camera_plugin executable and a synthetic backend camera_plugin owns the cameras named on its command line and serves their frames to readers on one endpoint: camera_plugin --endpoint=PATH --camera=ID:BACKEND[:KEY=VALUE,...] SIGINT and SIGTERM stop it cleanly. A misspelt argument is an error, and a second plugin cannot take the endpoint of a running one. Which backends a plugin has is decided when it is built: backends/ lists them, each behind its own CMake option, and a backend that is off is not configured at all, so its SDK is neither looked for nor built. The synthetic backend needs no SDK and is always built. It publishes frames whose every byte is known from the frame number, so a reader can check that it received the whole of the frame it was told about. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/plugins/camera/AGENTS.md | 2 +- src/plugins/camera/CMakeLists.txt | 4 + src/plugins/camera/README.md | 72 ++++ src/plugins/camera/app/CMakeLists.txt | 17 + src/plugins/camera/app/main.cpp | 168 ++++++++++ src/plugins/camera/backends/CMakeLists.txt | 25 ++ src/plugins/camera/backends/backends.cpp | 18 + .../inc/camera_plugin_backends/backends.hpp | 16 + .../camera/backends/synthetic/CMakeLists.txt | 22 ++ .../synthetic_backend.hpp | 34 ++ .../backends/synthetic/synthetic_backend.cpp | 204 ++++++++++++ src/plugins/camera/plugin.yaml | 7 + tests/cpp/plugins/camera/CMakeLists.txt | 12 + .../cpp/plugins/camera/test_camera_plugin.cpp | 308 ++++++++++++++++++ 14 files changed, 908 insertions(+), 1 deletion(-) create mode 100644 src/plugins/camera/README.md create mode 100644 src/plugins/camera/app/CMakeLists.txt create mode 100644 src/plugins/camera/app/main.cpp create mode 100644 src/plugins/camera/backends/CMakeLists.txt create mode 100644 src/plugins/camera/backends/backends.cpp create mode 100644 src/plugins/camera/backends/inc/camera_plugin_backends/backends.hpp create mode 100644 src/plugins/camera/backends/synthetic/CMakeLists.txt create mode 100644 src/plugins/camera/backends/synthetic/inc/camera_backend_synthetic/synthetic_backend.hpp create mode 100644 src/plugins/camera/backends/synthetic/synthetic_backend.cpp create mode 100644 src/plugins/camera/plugin.yaml create mode 100644 tests/cpp/plugins/camera/test_camera_plugin.cpp diff --git a/src/plugins/camera/AGENTS.md b/src/plugins/camera/AGENTS.md index 08a391fda7..07a1aa889d 100644 --- a/src/plugins/camera/AGENTS.md +++ b/src/plugins/camera/AGENTS.md @@ -7,7 +7,7 @@ SPDX-License-Identifier: Apache-2.0 **CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). -- **Backends are chosen at build time.** One CMake option per backend, off by default; a backend that is off is not configured, so its SDK is never looked for. Do not load backends at run time and do not add a backend ABI. +- **Backends are chosen at build time.** One CMake option per backend, off by default; a backend that is off is not configured, so its SDK is never looked for. Do not load backends at run time and do not add a backend ABI. The steps are in [`README.md`](README.md), "Adding a backend". - **No SDK types outside a backend's directory.** `core/` and `app/` see `CameraSource` and `camera_core` types only, and readers never link a camera SDK. - **A backend refuses what it does not understand.** Read options through `OptionReader` and call `finish()`, so a misspelt option is an error. - **A camera drops a frame rather than wait.** When `acquire` has no free slot, abandon what was taken for that frame set and carry on; nothing on the capture path blocks on readers. diff --git a/src/plugins/camera/CMakeLists.txt b/src/plugins/camera/CMakeLists.txt index c65feb2e17..6114c5c5c7 100644 --- a/src/plugins/camera/CMakeLists.txt +++ b/src/plugins/camera/CMakeLists.txt @@ -8,3 +8,7 @@ if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") endif() add_subdirectory(core) +add_subdirectory(backends) +add_subdirectory(app) + +install(FILES plugin.yaml README.md DESTINATION plugins/camera) diff --git a/src/plugins/camera/README.md b/src/plugins/camera/README.md new file mode 100644 index 0000000000..3d373bb1ee --- /dev/null +++ b/src/plugins/camera/README.md @@ -0,0 +1,72 @@ + + +# Camera plugin + +One process that owns the cameras and serves their frames to any number of readers in other +processes. Viewers, encoders and recorders read the plugin's buffers in place through +[`camera_reader`](../../core/camera_reader); none of them links a camera SDK. + +```bash +camera_plugin --endpoint=/tmp/cameras.sock --camera=head:synthetic +``` + +```cpp +CameraReader reader = CameraReader::open("/tmp/cameras.sock", "head"); +while (auto frames = reader.next(std::chrono::seconds(1))) +{ + const ImageView left = frames->image(0); +} +``` + +## Running it + +| Argument | Meaning | +|---|---| +| `--endpoint=PATH` | The socket file readers connect to. Only one plugin serves a path at a time. | +| `--camera=ID:BACKEND[:KEY=VALUE,...]` | One camera: the id readers ask for, the backend that opens it, and that backend's options. Repeat it for each camera. | +| `--slots=N` | Frames kept per stream, 8 by default. A pool holds the newest frame set, one slot for the camera to write ahead, and the windows of all readers; a reader that does not fit is refused with the numbers. | +| `--list-backends` | Print the backends this build has. | + +A misspelt argument or option is an error, not something ignored. `SIGINT` and `SIGTERM` stop +the plugin cleanly; readers see their connection close. + +## Backends + +A backend opens cameras through one SDK. **Which backends a plugin has is decided when it is +built**, so nobody builds or installs an SDK they do not use. + +| Backend | Needs | Options | +|---|---|---| +| `synthetic` | nothing; always built | `width` (1280), `height` (720), `fps` (60), `streams` (2), `format` (`bgra8`, `rgba8`, `gray8`, `nv12`) | + +The synthetic camera writes frames whose every byte is known from the frame number, so a +reader can check that it received the whole of the frame it was told about. It is what the +tests and the latency bench read. + +### Adding a backend + +1. Put it in `backends//` as a library of its own that implements `CameraSource` + (`core/inc/camera_plugin/camera_source.hpp`) and returns a `Backend` from one function. +2. Add an option `BUILD_CAMERA_BACKEND_`, off by default, next to the other build + options in the top-level `CMakeLists.txt`. +3. In `backends/CMakeLists.txt`, add the directory and link the library only when the option + is on, and pass the option to `backends.cpp` as a compile definition; there, add the + backend to `built_backends()` under that definition. + +A backend that is off is not configured at all, so its SDK is neither looked for nor built. A +camera that names a backend the build does not have is refused with the list of those it has. + +## Layout + +- `core/`: what a backend implements, and `CameraHost`, which gives each camera its pools and + puts all of them behind one endpoint. +- `backends/`: the backends of this build. +- `app/`: the `camera_plugin` executable. + +## Tests + +`tests/cpp/plugins/camera`, run with `ctest -L camera_plugin`. One case starts the plugin as a +process of its own and reads from it, kills it, and starts another on the same path. diff --git a/src/plugins/camera/app/CMakeLists.txt b/src/plugins/camera/app/CMakeLists.txt new file mode 100644 index 0000000000..cb767f7074 --- /dev/null +++ b/src/plugins/camera/app/CMakeLists.txt @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Threads REQUIRED) + +add_executable(camera_plugin + main.cpp +) + +target_link_libraries(camera_plugin PRIVATE + camera::plugin_core + camera::plugin_backends + log_bridge::log_bridge_core + Threads::Threads +) + +install(TARGETS camera_plugin RUNTIME DESTINATION plugins/camera) diff --git a/src/plugins/camera/app/main.cpp b/src/plugins/camera/app/main.cpp new file mode 100644 index 0000000000..b87cd09312 --- /dev/null +++ b/src/plugins/camera/app/main.cpp @@ -0,0 +1,168 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::camera; + +namespace +{ + +constexpr std::string_view USAGE = + "Usage: camera_plugin --endpoint=PATH --camera=ID:BACKEND[:KEY=VALUE[,KEY=VALUE]...] [--camera=...]\n" + " [--slots=N]\n" + " camera_plugin --list-backends\n" + "\n" + "Owns the cameras it is given and serves their frames to readers that connect to PATH.\n" + " --endpoint=PATH the socket file readers connect to\n" + " --camera=... one camera: the id readers ask for, the backend that opens it, and the\n" + " backend's options; give it once per camera\n" + " --slots=N frames kept per stream (default 8); a reader with a larger window needs more\n" + " --list-backends print the backends this build has and exit\n"; + +// The value of `--name=value` in `arg`, or nothing when `arg` is another flag. +std::optional flag_value(std::string_view arg, std::string_view name) +{ + if (arg.starts_with(name) && arg.size() > name.size() && arg[name.size()] == '=') + { + return arg.substr(name.size() + 1); + } + return std::nullopt; +} + +struct Arguments +{ + std::string endpoint; + std::vector cameras; + std::uint32_t slots = DEFAULT_SLOT_COUNT; + bool list_backends = false; + bool help = false; +}; + +Arguments parse(int argc, char** argv) +{ + Arguments arguments; + for (int i = 1; i < argc; ++i) + { + const std::string_view arg = argv[i]; + if (arg == "--help" || arg == "-h") + { + arguments.help = true; + } + else if (arg == "--list-backends") + { + arguments.list_backends = true; + } + else if (const auto endpoint = flag_value(arg, "--endpoint")) + { + arguments.endpoint = *endpoint; + } + else if (const auto camera = flag_value(arg, "--camera")) + { + arguments.cameras.push_back(parse_camera_argument(*camera)); + } + else if (const auto slots = flag_value(arg, "--slots")) + { + const auto [end, error] = std::from_chars(slots->data(), slots->data() + slots->size(), arguments.slots); + if (error != std::errc() || end != slots->data() + slots->size()) + { + throw std::invalid_argument("--slots takes a whole number, not '" + std::string(*slots) + "'"); + } + } + else if (flag_value(arg, "--plugin-root-id")) + { + // Added by the plugin manager for plugins with a device tree; cameras have none. + } + else + { + throw std::invalid_argument("unknown argument '" + std::string(arg) + "'"); + } + } + if (!arguments.help && !arguments.list_backends) + { + if (arguments.endpoint.empty()) + { + throw std::invalid_argument("--endpoint is required"); + } + if (arguments.cameras.empty()) + { + throw std::invalid_argument("at least one --camera is required"); + } + } + return arguments; +} + +} // namespace + +int main(int argc, char** argv) +{ + Arguments arguments; + try + { + arguments = parse(argc, argv); + } + catch (const std::exception& error) + { + // The refusal and the usage that follows are one message to the operator, on one channel. + std::cerr << "camera_plugin: " << error.what() << "\n\n" << USAGE; + return 2; + } + const std::vector backends = built_backends(); + if (arguments.help) + { + std::cout << USAGE; + return 0; + } + if (arguments.list_backends) + { + for (const Backend& backend : backends) + { + std::cout << backend.name << ": " << backend.description << "\n"; + } + return 0; + } + + const auto logger = isaaccapture::Logger::get("isaaccapture.plugins.camera.main"); + try + { + // Every thread started below inherits this mask, so the signals wait for sigwait() here. + sigset_t stop_signals; + sigemptyset(&stop_signals); + sigaddset(&stop_signals, SIGINT); + sigaddset(&stop_signals, SIGTERM); + pthread_sigmask(SIG_BLOCK, &stop_signals, nullptr); + + CameraHost host(arguments.endpoint); + for (const CameraConfig& camera : arguments.cameras) + { + host.add_camera(find_backend(backends, camera.backend).open(camera), arguments.slots); + } + host.start(); + logger->info("camera plugin serves {} camera(s) on '{}'", host.camera_count(), arguments.endpoint); + + int received = 0; + sigwait(&stop_signals, &received); + logger->info("camera plugin stops on signal {}", received); + host.stop(); + return 0; + } + catch (const std::exception& error) + { + logger->error("camera plugin cannot run: {}", error.what()); + return 1; + } +} diff --git a/src/plugins/camera/backends/CMakeLists.txt b/src/plugins/camera/backends/CMakeLists.txt new file mode 100644 index 0000000000..fdb1991a40 --- /dev/null +++ b/src/plugins/camera/backends/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Which cameras this build of the plugin can open is decided here, when the build is configured. +# The synthetic backend needs no SDK and is always built. Every other backend sits behind its +# own option, off by default, and a backend that is off is not configured at all: its SDK is +# neither looked for nor built. See ../README.md, "Adding a backend". +add_subdirectory(synthetic) + +add_library(camera_plugin_backends STATIC + backends.cpp +) + +target_include_directories(camera_plugin_backends INTERFACE + $ +) + +target_link_libraries(camera_plugin_backends + PUBLIC + camera::plugin_core + PRIVATE + camera::backend_synthetic +) + +add_library(camera::plugin_backends ALIAS camera_plugin_backends) diff --git a/src/plugins/camera/backends/backends.cpp b/src/plugins/camera/backends/backends.cpp new file mode 100644 index 0000000000..12e7de1677 --- /dev/null +++ b/src/plugins/camera/backends/backends.cpp @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_plugin_backends/backends.hpp" + +#include + +namespace plugins::camera +{ + +std::vector built_backends() +{ + std::vector backends; + backends.push_back(synthetic_backend()); + return backends; +} + +} // namespace plugins::camera diff --git a/src/plugins/camera/backends/inc/camera_plugin_backends/backends.hpp b/src/plugins/camera/backends/inc/camera_plugin_backends/backends.hpp new file mode 100644 index 0000000000..92e90576d8 --- /dev/null +++ b/src/plugins/camera/backends/inc/camera_plugin_backends/backends.hpp @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include + +namespace plugins::camera +{ + +// The backends this build of the plugin has. Which ones is decided when the build is configured. +std::vector built_backends(); + +} // namespace plugins::camera diff --git a/src/plugins/camera/backends/synthetic/CMakeLists.txt b/src/plugins/camera/backends/synthetic/CMakeLists.txt new file mode 100644 index 0000000000..15776ea70b --- /dev/null +++ b/src/plugins/camera/backends/synthetic/CMakeLists.txt @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Threads REQUIRED) + +# A camera that needs no hardware: frames with known content at a set rate. +add_library(camera_backend_synthetic STATIC + synthetic_backend.cpp +) + +target_include_directories(camera_backend_synthetic INTERFACE + $ +) + +target_link_libraries(camera_backend_synthetic + PUBLIC + camera::plugin_core + PRIVATE + Threads::Threads +) + +add_library(camera::backend_synthetic ALIAS camera_backend_synthetic) diff --git a/src/plugins/camera/backends/synthetic/inc/camera_backend_synthetic/synthetic_backend.hpp b/src/plugins/camera/backends/synthetic/inc/camera_backend_synthetic/synthetic_backend.hpp new file mode 100644 index 0000000000..df2cc64879 --- /dev/null +++ b/src/plugins/camera/backends/synthetic/inc/camera_backend_synthetic/synthetic_backend.hpp @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include + +namespace plugins::camera +{ + +/** + * @brief A camera that needs no hardware. Options: `width`, `height`, `fps`, `streams` and + * `format` (bgra8, rgba8, gray8 or nv12). + * + * Frame n of stream s starts with n as eight little-endian bytes; every other byte of the + * image is synthetic_fill(n, s). A reader can therefore check that it got the whole of the + * frame it was told about. + */ +Backend synthetic_backend(); + +// The value of every byte of a synthetic image after its first eight. +constexpr std::byte synthetic_fill(std::uint64_t frame, std::size_t stream) +{ + return static_cast((frame + 17 * stream) & 0xff); +} + +// The frame number a synthetic image starts with. +std::uint64_t synthetic_frame_number(std::span image); + +} // namespace plugins::camera diff --git a/src/plugins/camera/backends/synthetic/synthetic_backend.cpp b/src/plugins/camera/backends/synthetic/synthetic_backend.cpp new file mode 100644 index 0000000000..58d88430fd --- /dev/null +++ b/src/plugins/camera/backends/synthetic/synthetic_backend.cpp @@ -0,0 +1,204 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_backend_synthetic/synthetic_backend.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace plugins::camera +{ + +using core::camera::FrameTimeline; +using core::camera::GroupDescription; +using core::camera::ImageInfo; +using core::camera::PixelFormat; +using core::camera::PlaneSize; +using core::camera::StreamDescription; + +namespace +{ + +std::int64_t now_ns() +{ + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +class SyntheticSource final : public CameraSource +{ +public: + explicit SyntheticSource(const CameraConfig& config) + { + OptionReader options(config); + const std::uint32_t width = options.number("width", 1280, 16, 8192); + const std::uint32_t height = options.number("height", 720, 16, 8192); + const std::uint32_t fps = options.number("fps", 60, 1, 1000); + const std::uint32_t streams = options.number("streams", 2, 1, core::camera::MAX_STREAMS_PER_GROUP); + const std::string format_name = options.text("format", "bgra8"); + options.finish(); + + PixelFormat format = PixelFormat::BGRA8; + if (format_name == "rgba8") + { + format = PixelFormat::RGBA8; + } + else if (format_name == "gray8") + { + format = PixelFormat::GRAY8; + } + else if (format_name == "nv12") + { + format = PixelFormat::NV12; + } + else if (format_name != "bgra8") + { + options.fail("option 'format' is '" + format_name + "'; it takes bgra8, rgba8, gray8 or nv12"); + } + + m_period = std::chrono::nanoseconds(1'000'000'000 / fps); + m_group.id = config.id; + for (std::uint32_t i = 0; i < streams; ++i) + { + StreamDescription stream; + // A pair reads better as left and right than as 0 and 1. + stream.id = config.id + "/" + (streams == 2 ? (i == 0 ? "left" : "right") : std::to_string(i)); + stream.pixel_format = format; + stream.width = width; + stream.height = height; + stream.plane_count = core::camera::plane_count_of(format); + std::uint64_t offset = 0; + for (std::uint32_t plane = 0; plane < stream.plane_count; ++plane) + { + const PlaneSize size = core::camera::plane_size_of(format, width, height, plane); + stream.planes[plane] = { static_cast(offset), static_cast(size.row_bytes) }; + offset += size.row_bytes * size.rows; + } + stream.slot_bytes = offset; + stream.frame_rate = fps; + m_group.streams.push_back(stream); + } + } + + ~SyntheticSource() override + { + stop(); + } + + GroupDescription describe() const override + { + return m_group; + } + + void start(FrameOutput& output) override + { + if (m_thread.joinable()) + { + return; + } + m_stopping = false; + m_thread = std::thread([this, &output] { run(output); }); + } + + void stop() override + { + if (!m_thread.joinable()) + { + return; + } + m_stopping = true; + m_thread.join(); + } + +private: + void run(FrameOutput& output) + { + std::vector images(m_group.streams.size()); + auto tick = std::chrono::steady_clock::now(); + for (std::uint64_t frame = 0;; ++frame) + { + if (!sleep_until(tick)) + { + return; + } + FrameTimeline timeline; + timeline.capture_ns = now_ns(); + if (write(output, frame, images)) + { + timeline.sdk_ready_ns = now_ns(); + output.publish(images, timeline); + } + tick += m_period; + // After a stall, carry on from now instead of bursting to catch up. + tick = std::max(tick, std::chrono::steady_clock::now() - m_period); + } + } + + // Sleeps until `tick` in steps short enough for stop() to be prompt at any frame rate. + // False when the camera was stopped meanwhile. A flag and short sleeps, not a condition + // variable: GCC 11's ThreadSanitizer misreads a wait on the steady clock as a double lock. + bool sleep_until(std::chrono::steady_clock::time_point tick) const + { + constexpr std::chrono::milliseconds LONGEST_STEP(50); + while (!m_stopping) + { + const auto now = std::chrono::steady_clock::now(); + if (now >= tick) + { + return true; + } + std::this_thread::sleep_until(std::min(tick, now + LONGEST_STEP)); + } + return false; + } + + // Writes one frame set. False when a stream has no free slot: the frame set is dropped. + bool write(FrameOutput& output, std::uint64_t frame, std::vector& images) + { + for (std::size_t stream = 0; stream < images.size(); ++stream) + { + const std::optional image = output.acquire(stream); + if (!image) + { + for (std::size_t taken = 0; taken < stream; ++taken) + { + output.abandon(taken, images[taken].slot); + } + return false; + } + std::memset(image->data.data(), std::to_integer(synthetic_fill(frame, stream)), image->data.size()); + std::memcpy(image->data.data(), &frame, sizeof(frame)); + images[stream] = {}; + images[stream].slot = image->slot; + images[stream].device_frame_id = frame; + } + return true; + } + + GroupDescription m_group; + std::chrono::nanoseconds m_period{}; + std::thread m_thread; + std::atomic m_stopping{ false }; +}; + +} // namespace + +Backend synthetic_backend() +{ + return { "synthetic", "frames with known content at a set rate; needs no hardware", + [](const CameraConfig& config) { return std::make_unique(config); } }; +} + +std::uint64_t synthetic_frame_number(std::span image) +{ + std::uint64_t frame = 0; + std::memcpy(&frame, image.data(), std::min(image.size(), sizeof(frame))); + return frame; +} + +} // namespace plugins::camera diff --git a/src/plugins/camera/plugin.yaml b/src/plugins/camera/plugin.yaml new file mode 100644 index 0000000000..14f27bc4cf --- /dev/null +++ b/src/plugins/camera/plugin.yaml @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: camera +description: "Owns the cameras and serves their frames to readers in other processes" +command: "./camera_plugin" +version: "0.1.0" diff --git a/tests/cpp/plugins/camera/CMakeLists.txt b/tests/cpp/plugins/camera/CMakeLists.txt index cda01eed40..8cbddebef4 100644 --- a/tests/cpp/plugins/camera/CMakeLists.txt +++ b/tests/cpp/plugins/camera/CMakeLists.txt @@ -3,15 +3,27 @@ cmake_minimum_required(VERSION 3.20) +find_package(Threads REQUIRED) + add_executable(camera_plugin_tests test_camera_host.cpp + test_camera_plugin.cpp ) target_link_libraries(camera_plugin_tests PRIVATE camera::plugin_core + camera::plugin_backends + camera::backend_synthetic camera::camera_reader Catch2::Catch2WithMain + Threads::Threads +) + +# Some cases start the plugin as a process of its own and read from it. +target_compile_definitions(camera_plugin_tests PRIVATE + CAMERA_PLUGIN_EXECUTABLE="$" ) +add_dependencies(camera_plugin_tests camera_plugin) message(STATUS "camera_plugin_tests target enabled with Catch2") catch_discover_tests(camera_plugin_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/plugins/camera/test_camera_plugin.cpp b/tests/cpp/plugins/camera/test_camera_plugin.cpp new file mode 100644 index 0000000000..e566ae6884 --- /dev/null +++ b/tests/cpp/plugins/camera/test_camera_plugin.cpp @@ -0,0 +1,308 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; +using core::camera::CameraReader; +using core::camera::CameraReaderError; +using core::camera::DeliveryOrder; +using core::camera::FrameSet; +using core::camera::ImageView; +using core::camera::PixelFormat; + +namespace +{ + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_plugin_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +std::unique_ptr open_synthetic(const std::string& argument) +{ + const CameraConfig config = parse_camera_argument(argument); + const std::vector backends = built_backends(); + return find_backend(backends, config.backend).open(config); +} + +// Every byte of a synthetic image is known from its frame number and its stream. +void require_synthetic(const ImageView& image, std::size_t stream) +{ + const std::uint64_t frame = synthetic_frame_number(image.data); + REQUIRE(image.info.device_frame_id == frame); + const std::byte fill = synthetic_fill(frame, stream); + const auto rest = image.data.subspan(sizeof(frame)); + REQUIRE(std::all_of(rest.begin(), rest.end(), [fill](std::byte value) { return value == fill; })); +} + +// The plugin as a process of its own. +class PluginProcess +{ +public: + explicit PluginProcess(const std::vector& arguments) + { + std::vector owned = { CAMERA_PLUGIN_EXECUTABLE }; + owned.insert(owned.end(), arguments.begin(), arguments.end()); + std::vector argv; + for (std::string& argument : owned) + { + argv.push_back(argument.data()); + } + argv.push_back(nullptr); + m_pid = ::fork(); + REQUIRE(m_pid >= 0); + if (m_pid == 0) + { + ::execv(argv[0], argv.data()); + ::_exit(127); + } + } + + ~PluginProcess() + { + if (m_pid > 0) + { + ::kill(m_pid, SIGKILL); + ::waitpid(m_pid, nullptr, 0); + } + } + + // Waits for the process to end and returns its exit code, or -1 if it did not exit by itself. + int wait() + { + int status = 0; + REQUIRE(::waitpid(m_pid, &status, 0) == m_pid); + m_pid = -1; + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; + } + + int stop(int signal) + { + REQUIRE(::kill(m_pid, signal) == 0); + return wait(); + } + +private: + pid_t m_pid = -1; +}; + +// A plugin that was just started needs a moment before it listens. +std::optional open_when_ready(const std::string& endpoint, + const std::string& group, + const core::camera::ReaderOptions& options = {}) +{ + const auto deadline = std::chrono::steady_clock::now() + 10s; + while (std::chrono::steady_clock::now() < deadline) + { + try + { + return CameraReader::open(endpoint, group, options); + } + catch (const CameraReaderError&) + { + std::this_thread::sleep_for(10ms); + } + } + return std::nullopt; +} + +// Reads whatever is still on its way; ends by throwing when the plugin's side has closed. +void read_until_closed(CameraReader& reader) +{ + for (int i = 0; i < 1000; ++i) + { + (void)reader.next(2s); + } +} + +} // namespace + +TEST_CASE("the synthetic backend is in every build", "[unit][camera_plugin]") +{ + const std::vector backends = built_backends(); + REQUIRE(find_backend(backends, "synthetic").name == "synthetic"); + REQUIRE_THROWS_WITH( + find_backend(backends, "no_such_sdk"), ContainsSubstring("built without a 'no_such_sdk' backend")); +} + +TEST_CASE("the synthetic backend refuses options it does not know", "[unit][camera_plugin]") +{ + REQUIRE_THROWS_WITH(open_synthetic("head:synthetic:widht=640"), ContainsSubstring("no option 'widht'")); + REQUIRE_THROWS_WITH(open_synthetic("head:synthetic:width=wide"), ContainsSubstring("whole number from 16 to 8192")); + REQUIRE_THROWS_WITH(open_synthetic("head:synthetic:fps=0"), ContainsSubstring("camera 'head' (synthetic backend)")); + REQUIRE_THROWS_WITH(open_synthetic("head:synthetic:streams=9"), ContainsSubstring("option 'streams'")); + REQUIRE_THROWS_WITH(open_synthetic("head:synthetic:format=jpeg"), ContainsSubstring("bgra8, rgba8, gray8 or nv12")); + + const core::camera::GroupDescription pair = open_synthetic("head:synthetic")->describe(); + REQUIRE(pair.id == "head"); + REQUIRE(pair.streams.size() == 2); + REQUIRE(pair.streams[0].id == "head/left"); + REQUIRE(pair.streams[1].id == "head/right"); + REQUIRE(pair.streams[0].slot_bytes == 1280u * 720u * 4u); + + const core::camera::GroupDescription nv12 = + open_synthetic("rig:synthetic:streams=3,format=nv12,width=64,height=48")->describe(); + REQUIRE(nv12.streams[2].id == "rig/2"); + REQUIRE(nv12.streams[2].pixel_format == PixelFormat::NV12); + REQUIRE(nv12.streams[2].plane_count == 2); + REQUIRE(nv12.streams[2].planes[1].offset_bytes == 64u * 48u); + REQUIRE(nv12.streams[2].slot_bytes == 64u * 48u * 3u / 2u); +} + +TEST_CASE("synthetic cameras reach readers whole and in step", "[unit][camera_plugin]") +{ + TemporaryDirectory directory; + CameraHost host(directory.socket_path()); + host.add_camera(open_synthetic("head:synthetic:width=160,height=120,fps=200")); + host.add_camera(open_synthetic("wrist:synthetic:width=64,height=48,fps=200,streams=1,format=gray8"), 6); + REQUIRE_THROWS_AS(host.add_camera(open_synthetic("head:synthetic")), std::invalid_argument); // id taken + REQUIRE(host.camera_count() == 2); + host.start(); + REQUIRE_THROWS_AS(host.add_camera(open_synthetic("late:synthetic")), std::logic_error); + + const auto groups = CameraReader::list(directory.socket_path()); + REQUIRE(groups.size() == 2); + REQUIRE(groups[0].streams[0].slot_count == DEFAULT_SLOT_COUNT); + REQUIRE(groups[1].streams[0].slot_count == 6); + + // A pair arrives as one frame set: the same frame number in both eyes, each with its own fill. + CameraReader head = CameraReader::open(directory.socket_path(), "head"); + CameraReader wrist = CameraReader::open(directory.socket_path(), "wrist", { .window = 2 }); + std::uint64_t previous = 0; + for (int i = 0; i < 20; ++i) + { + const std::optional frames = head.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->image_count() == 2); + require_synthetic(frames->image(0), 0); + require_synthetic(frames->image(1), 1); + REQUIRE(frames->image(0).info.device_frame_id == frames->image(1).info.device_frame_id); + REQUIRE(frames->sequence() > previous); + REQUIRE(frames->timeline().capture_ns <= frames->timeline().sdk_ready_ns); + REQUIRE(frames->timeline().sdk_ready_ns <= frames->timeline().published_ns); + previous = frames->sequence(); + } + + // Whatever a reader was not given is accounted for: no frame set goes missing unreported, + // however far the reader fell behind while the loop above ran. + std::optional first = wrist.next(2s); + REQUIRE(first.has_value()); + std::uint64_t last = first->sequence(); + first.reset(); + for (int i = 0; i < 20; ++i) + { + const std::optional frames = wrist.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == last + 1 + frames->skipped()); + require_synthetic(frames->image(0), 0); + last = frames->sequence(); + } + REQUIRE(host.reader_count() == 2); + + host.stop(); + // What was already on its way is still delivered; then the reader is told. + REQUIRE_THROWS_AS(read_until_closed(head), core::camera::CameraDisconnected); +} + +TEST_CASE("a reader in another process gets frames from the plugin", "[unit][camera_plugin][process]") +{ + TemporaryDirectory directory; + const std::string endpoint = directory.socket_path(); + PluginProcess plugin({ "--endpoint=" + endpoint, "--camera=head:synthetic:width=320,height=240,fps=120", + "--plugin-root-id=ignored" }); + + std::optional reader = open_when_ready(endpoint, "head"); + REQUIRE(reader.has_value()); + std::uint64_t previous = 0; + for (int i = 0; i < 30; ++i) + { + const std::optional frames = reader->next(2s); + REQUIRE(frames.has_value()); + require_synthetic(frames->image(0), 0); + require_synthetic(frames->image(1), 1); + REQUIRE(frames->sequence() > previous); + previous = frames->sequence(); + } + + // A second plugin cannot take the endpoint of a running one. + PluginProcess second({ "--endpoint=" + endpoint, "--camera=head:synthetic" }); + REQUIRE(second.wait() == 1); + REQUIRE(reader->next(2s).has_value()); + + // SIGTERM is a clean stop; the reader is told. + REQUIRE(plugin.stop(SIGTERM) == 0); + REQUIRE_THROWS_AS(read_until_closed(*reader), core::camera::CameraDisconnected); + + // A plugin that was killed leaves a socket file behind; the next one replaces it. + { + PluginProcess killed({ "--endpoint=" + endpoint, "--camera=head:synthetic:width=64,height=48" }); + REQUIRE(open_when_ready(endpoint, "head").has_value()); + REQUIRE(killed.stop(SIGKILL) == -1); + } + REQUIRE(std::filesystem::exists(endpoint)); + PluginProcess next({ "--endpoint=" + endpoint, "--camera=head:synthetic:width=64,height=48" }); + std::optional again = open_when_ready(endpoint, "head"); + REQUIRE(again.has_value()); + REQUIRE(again->next(2s).has_value()); +} + +TEST_CASE("the plugin refuses arguments it cannot use", "[unit][camera_plugin][process]") +{ + TemporaryDirectory directory; + const std::string endpoint = "--endpoint=" + directory.socket_path(); + REQUIRE(PluginProcess({ "--list-backends" }).wait() == 0); + REQUIRE(PluginProcess({ "--help" }).wait() == 0); + REQUIRE(PluginProcess({}).wait() == 2); + REQUIRE(PluginProcess({ endpoint }).wait() == 2); + REQUIRE(PluginProcess({ endpoint, "--camera=head:synthetic", "--frobnicate" }).wait() == 2); + REQUIRE(PluginProcess({ endpoint, "--camera=head" }).wait() == 2); + // Arguments that parse but cannot be honoured end the plugin with an error in its log. + REQUIRE(PluginProcess({ endpoint, "--camera=head:zed" }).wait() == 1); + REQUIRE(PluginProcess({ endpoint, "--camera=head:synthetic:colour=red" }).wait() == 1); + REQUIRE(PluginProcess({ endpoint, "--camera=head:synthetic", "--slots=1" }).wait() == 1); +} From 8e29524cd6067a8a7f1636d188d5c454573fb3cc Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 17:20:16 -0800 Subject: [PATCH 16/16] feat(camera): add camera_bench and log what each camera does camera_bench reads one group from a running camera plugin, like any other reader, and prints where each frame set spent its time: capture to SDK ready, to published, to sent, to the reader, as percentiles. Without a group it lists what the plugin publishes. It can read with a larger window or in order, and check every byte against the synthetic backend's pattern. The plugin now logs every ten seconds how many frame sets each camera published, and warns when a camera dropped frame sets for lack of a free slot or a reader did not take its events. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/plugins/camera/CMakeLists.txt | 1 + src/plugins/camera/README.md | 32 +- src/plugins/camera/app/main.cpp | 15 +- src/plugins/camera/bench/CMakeLists.txt | 14 + src/plugins/camera/bench/main.cpp | 293 ++++++++++++++++++ src/plugins/camera/core/camera_host.cpp | 30 ++ .../core/inc/camera_plugin/camera_host.hpp | 6 + tests/cpp/plugins/camera/CMakeLists.txt | 6 +- .../cpp/plugins/camera/test_camera_bench.cpp | 137 ++++++++ 9 files changed, 530 insertions(+), 4 deletions(-) create mode 100644 src/plugins/camera/bench/CMakeLists.txt create mode 100644 src/plugins/camera/bench/main.cpp create mode 100644 tests/cpp/plugins/camera/test_camera_bench.cpp diff --git a/src/plugins/camera/CMakeLists.txt b/src/plugins/camera/CMakeLists.txt index 6114c5c5c7..ffbd4ebe0d 100644 --- a/src/plugins/camera/CMakeLists.txt +++ b/src/plugins/camera/CMakeLists.txt @@ -10,5 +10,6 @@ endif() add_subdirectory(core) add_subdirectory(backends) add_subdirectory(app) +add_subdirectory(bench) install(FILES plugin.yaml README.md DESTINATION plugins/camera) diff --git a/src/plugins/camera/README.md b/src/plugins/camera/README.md index 3d373bb1ee..64a5ef8514 100644 --- a/src/plugins/camera/README.md +++ b/src/plugins/camera/README.md @@ -33,6 +33,34 @@ while (auto frames = reader.next(std::chrono::seconds(1))) A misspelt argument or option is an error, not something ignored. `SIGINT` and `SIGTERM` stop the plugin cleanly; readers see their connection close. +Every ten seconds the plugin logs, per camera, how many frame sets it published. It warns +when a camera had to drop frame sets because its pool was full, or when a reader did not take +its events; both mean the pool or a reader's window needs another look. + +## Measuring + +Every frame set carries the times it passed each step, on the host's monotonic clock. +`camera_bench` reads a group like any other reader and prints them: + +```text +$ camera_bench --endpoint=/tmp/cameras.sock --group=head --frames=600 +head: 2 stream(s) + head/left 1280x720 bgra8, 60.00 fps, 8 slots of 3686400 bytes + head/right 1280x720 bgra8, 60.00 fps, 8 slots of 3686400 bytes +read 600 frame sets in 10.00 s (60.0 per second), skipped 0 + p50 p90 p99 max (microseconds) + capture -> sdk ready ... + sdk ready -> published ... + published -> sent ... + sent -> reader ... + capture -> reader ... +``` + +`published -> sent` and `sent -> reader` are what the plugin and the process boundary add; the +steps before them belong to the camera and its SDK. Without `--group` the bench lists what the +plugin publishes. `--window` and `--order` read as a recorder would, and `--check-synthetic` +checks every byte of every image against the synthetic backend's pattern. + ## Backends A backend opens cameras through one SDK. **Which backends a plugin has is decided when it is @@ -65,8 +93,10 @@ camera that names a backend the build does not have is refused with the list of puts all of them behind one endpoint. - `backends/`: the backends of this build. - `app/`: the `camera_plugin` executable. +- `bench/`: the `camera_bench` executable. ## Tests `tests/cpp/plugins/camera`, run with `ctest -L camera_plugin`. One case starts the plugin as a -process of its own and reads from it, kills it, and starts another on the same path. +process of its own and reads from it, kills it, and starts another on the same path; another +runs the bench against a camera in the test process. diff --git a/src/plugins/camera/app/main.cpp b/src/plugins/camera/app/main.cpp index b87cd09312..353369730c 100644 --- a/src/plugins/camera/app/main.cpp +++ b/src/plugins/camera/app/main.cpp @@ -6,9 +6,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -34,6 +36,9 @@ constexpr std::string_view USAGE = " --slots=N frames kept per stream (default 8); a reader with a larger window needs more\n" " --list-backends print the backends this build has and exit\n"; +// How often the plugin logs what each camera did. +constexpr time_t ACTIVITY_LOG_SECONDS = 10; + // The value of `--name=value` in `arg`, or nothing when `arg` is another flag. std::optional flag_value(std::string_view arg, std::string_view name) { @@ -154,8 +159,16 @@ int main(int argc, char** argv) host.start(); logger->info("camera plugin serves {} camera(s) on '{}'", host.camera_count(), arguments.endpoint); + // Sleeps until a signal says stop, waking now and then to log what the cameras did. + const timespec activity_interval = { ACTIVITY_LOG_SECONDS, 0 }; int received = 0; - sigwait(&stop_signals, &received); + while ((received = sigtimedwait(&stop_signals, nullptr, &activity_interval)) <= 0) + { + if (errno == EAGAIN) + { + host.log_activity(); + } + } logger->info("camera plugin stops on signal {}", received); host.stop(); return 0; diff --git a/src/plugins/camera/bench/CMakeLists.txt b/src/plugins/camera/bench/CMakeLists.txt new file mode 100644 index 0000000000..06f2637023 --- /dev/null +++ b/src/plugins/camera/bench/CMakeLists.txt @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Reads one group from a running camera plugin and prints where each frame set spent its time. +add_executable(camera_bench + main.cpp +) + +target_link_libraries(camera_bench PRIVATE + camera::camera_reader + camera::backend_synthetic +) + +install(TARGETS camera_bench RUNTIME DESTINATION plugins/camera) diff --git a/src/plugins/camera/bench/main.cpp b/src/plugins/camera/bench/main.cpp new file mode 100644 index 0000000000..211d0c76d8 --- /dev/null +++ b/src/plugins/camera/bench/main.cpp @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +constexpr std::string_view USAGE = + "Usage: camera_bench --endpoint=PATH [--group=ID] [--frames=N] [--window=N] [--order=newest|in_order]\n" + " [--check-synthetic]\n" + "\n" + "Reads one group from a running camera plugin and prints where each frame set spent its time.\n" + "Without --group it lists what the plugin publishes.\n" + " --frames=N frame sets to read (default 300)\n" + " --window=N frame sets held for this reader at a time (default 1)\n" + " --order=... newest skips to the latest frame set, in_order reads every one\n" + " --check-synthetic check every byte of every image against the synthetic backend's pattern\n"; + +// How long this reader waits for a frame set before it gives up on the camera. +constexpr std::chrono::seconds FRAME_TIMEOUT(5); + +std::optional flag_value(std::string_view arg, std::string_view name) +{ + if (arg.starts_with(name) && arg.size() > name.size() && arg[name.size()] == '=') + { + return arg.substr(name.size() + 1); + } + return std::nullopt; +} + +std::uint32_t number_of(std::string_view flag, std::string_view text) +{ + std::uint32_t value = 0; + const auto [end, error] = std::from_chars(text.data(), text.data() + text.size(), value); + if (error != std::errc() || end != text.data() + text.size() || value == 0) + { + throw std::invalid_argument(std::string(flag) + " takes a whole number above zero, not '" + std::string(text) + + "'"); + } + return value; +} + +struct Arguments +{ + std::string endpoint; + std::string group; + std::uint32_t frames = 300; + ReaderOptions options; + bool check_synthetic = false; + bool help = false; +}; + +Arguments parse(int argc, char** argv) +{ + Arguments arguments; + for (int i = 1; i < argc; ++i) + { + const std::string_view arg = argv[i]; + if (arg == "--help" || arg == "-h") + { + arguments.help = true; + } + else if (arg == "--check-synthetic") + { + arguments.check_synthetic = true; + } + else if (const auto endpoint = flag_value(arg, "--endpoint")) + { + arguments.endpoint = *endpoint; + } + else if (const auto group = flag_value(arg, "--group")) + { + arguments.group = *group; + } + else if (const auto frames = flag_value(arg, "--frames")) + { + arguments.frames = number_of("--frames", *frames); + } + else if (const auto window = flag_value(arg, "--window")) + { + arguments.options.window = number_of("--window", *window); + } + else if (const auto order = flag_value(arg, "--order")) + { + if (*order != "newest" && *order != "in_order") + { + throw std::invalid_argument("--order takes newest or in_order, not '" + std::string(*order) + "'"); + } + arguments.options.order = *order == "newest" ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; + } + else + { + throw std::invalid_argument("unknown argument '" + std::string(arg) + "'"); + } + } + if (!arguments.help && arguments.endpoint.empty()) + { + throw std::invalid_argument("--endpoint is required"); + } + return arguments; +} + +const char* name_of(const StreamDescription& stream) +{ + if (stream.codec == Codec::H264) + { + return "h264"; + } + if (stream.codec == Codec::H265) + { + return "h265"; + } + switch (stream.pixel_format) + { + case PixelFormat::BGRA8: + return "bgra8"; + case PixelFormat::RGBA8: + return "rgba8"; + case PixelFormat::NV12: + return "nv12"; + case PixelFormat::GRAY8: + return "gray8"; + } + return "unknown"; +} + +void print_group(const GroupDescription& group) +{ + std::printf("%s: %zu stream(s)\n", group.id.c_str(), group.streams.size()); + for (const StreamDescription& stream : group.streams) + { + std::printf(" %-24s %ux%u %s, %.2f fps, %u slots of %llu bytes\n", stream.id.c_str(), stream.width, + stream.height, name_of(stream), stream.frame_rate, stream.slot_count, + static_cast(stream.slot_bytes)); + } +} + +std::int64_t now_ns() +{ + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +// One step of a frame set's way from the sensor to this reader. +struct Stage +{ + const char* name; + std::vector nanoseconds; + + // Skips a step whose start or end the camera did not report. + void add(std::int64_t from, std::int64_t to) + { + if (from != TIME_UNKNOWN && to != TIME_UNKNOWN) + { + nanoseconds.push_back(to - from); + } + } + + void print() + { + if (nanoseconds.empty()) + { + std::printf(" %-22s not reported by this camera\n", name); + return; + } + std::sort(nanoseconds.begin(), nanoseconds.end()); + const auto at = [&](double fraction) { + return nanoseconds[static_cast(fraction * static_cast(nanoseconds.size() - 1))] / 1000.0; + }; + std::printf(" %-22s %10.1f %10.1f %10.1f %10.1f\n", name, at(0.5), at(0.9), at(0.99), at(1.0)); + } +}; + +// Whether every byte of the image is what the synthetic backend wrote for its frame number. +bool is_synthetic(const ImageView& image, std::size_t stream) +{ + const std::uint64_t frame = plugins::camera::synthetic_frame_number(image.data); + const std::byte fill = plugins::camera::synthetic_fill(frame, stream); + const auto rest = image.data.subspan(std::min(image.data.size(), sizeof(frame))); + return image.info.device_frame_id == frame && + std::all_of(rest.begin(), rest.end(), [fill](std::byte value) { return value == fill; }); +} + +int run(const Arguments& arguments) +{ + if (arguments.group.empty()) + { + for (const GroupDescription& group : CameraReader::list(arguments.endpoint)) + { + print_group(group); + } + return 0; + } + + CameraReader reader = CameraReader::open(arguments.endpoint, arguments.group, arguments.options); + print_group(reader.group()); + + std::array stages = { { { "capture -> sdk ready", {} }, + { "sdk ready -> published", {} }, + { "published -> sent", {} }, + { "sent -> reader", {} }, + { "capture -> reader", {} } } }; + std::uint64_t skipped = 0; + std::uint64_t corrupt = 0; + const std::int64_t started = now_ns(); + for (std::uint32_t i = 0; i < arguments.frames; ++i) + { + const std::optional frames = reader.next(FRAME_TIMEOUT); + const std::int64_t received = now_ns(); + if (!frames) + { + std::fprintf(stderr, "camera_bench: no frame set from '%s' for %lld seconds\n", arguments.group.c_str(), + static_cast(FRAME_TIMEOUT.count())); + return 1; + } + const FrameTimeline& timeline = frames->timeline(); + stages[0].add(timeline.capture_ns, timeline.sdk_ready_ns); + stages[1].add(timeline.sdk_ready_ns, timeline.published_ns); + stages[2].add(timeline.published_ns, timeline.sent_ns); + stages[3].add(timeline.sent_ns, received); + stages[4].add(timeline.capture_ns, received); + skipped += frames->skipped(); + if (arguments.check_synthetic) + { + for (std::size_t stream = 0; stream < frames->image_count(); ++stream) + { + corrupt += is_synthetic(frames->image(stream), stream) ? 0 : 1; + } + } + } + const double seconds = static_cast(now_ns() - started) / 1e9; + + std::printf("read %u frame sets in %.2f s (%.1f per second), skipped %llu\n", arguments.frames, seconds, + arguments.frames / seconds, static_cast(skipped)); + std::printf(" %-22s %10s %10s %10s %10s (microseconds)\n", "", "p50", "p90", "p99", "max"); + for (Stage& stage : stages) + { + stage.print(); + } + if (arguments.check_synthetic) + { + std::printf("synthetic pattern: %llu image(s) did not match\n", static_cast(corrupt)); + } + return corrupt == 0 ? 0 : 1; +} + +} // namespace + +// Everything this tool prints is its result or its usage, for a terminal; it logs nothing. +int main(int argc, char** argv) +{ + Arguments arguments; + try + { + arguments = parse(argc, argv); + } + catch (const std::exception& error) + { + std::cerr << "camera_bench: " << error.what() << "\n\n" << USAGE; + return 2; + } + if (arguments.help) + { + std::cout << USAGE; + return 0; + } + try + { + return run(arguments); + } + catch (const std::exception& error) + { + std::cerr << "camera_bench: " << error.what() << "\n"; + return 1; + } +} diff --git a/src/plugins/camera/core/camera_host.cpp b/src/plugins/camera/core/camera_host.cpp index 1eb59e4ee7..91c4d8a826 100644 --- a/src/plugins/camera/core/camera_host.cpp +++ b/src/plugins/camera/core/camera_host.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -65,8 +66,10 @@ struct Camera { std::unique_ptr source; std::string id; + GroupId group = 0; std::vector> pools; // one per stream std::unique_ptr output; + core::camera::PublisherStats logged; // the counters at the previous log_activity() }; } // namespace @@ -99,6 +102,7 @@ class CameraHost::Impl // Stable addresses: an output points at its camera's pools. std::vector> cameras; bool started = false; + std::chrono::steady_clock::time_point activity_logged = std::chrono::steady_clock::now(); std::shared_ptr logger; }; @@ -130,6 +134,7 @@ void CameraHost::add_camera(std::unique_ptr source, std::uint32_t camera->id = description.id; // Refuses a description that is not usable before any memory is allocated for it. const GroupId group = m_impl->publisher.add_group(description); + camera->group = group; std::vector descriptors; for (const StreamDescription& stream : description.streams) { @@ -191,4 +196,29 @@ const Publisher& CameraHost::publisher() const return m_impl->publisher; } +void CameraHost::log_activity() +{ + const auto now = std::chrono::steady_clock::now(); + const double seconds = std::chrono::duration(now - m_impl->activity_logged).count(); + m_impl->activity_logged = now; + for (const auto& camera : m_impl->cameras) + { + const core::camera::PublisherStats stats = m_impl->publisher.stats(camera->group); + const std::uint64_t published = stats.published - camera->logged.published; + const std::uint64_t dropped = stats.no_free_slot - camera->logged.no_free_slot; + const std::uint64_t unsent = stats.send_failures - camera->logged.send_failures; + camera->logged = stats; + m_impl->logger->info("camera '{}': {} frame sets in {:.1f} s ({:.1f} per second), {} reader(s) connected", + camera->id, published, seconds, seconds > 0.0 ? published / seconds : 0.0, + m_impl->endpoint.reader_count()); + if (dropped != 0 || unsent != 0) + { + m_impl->logger->warn( + "camera '{}': dropped {} frame set(s) for lack of a free slot; {} event(s) " + "were not taken by a reader", + camera->id, dropped, unsent); + } + } +} + } // namespace plugins::camera diff --git a/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp b/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp index d0818a2234..0e53f3ea77 100644 --- a/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp +++ b/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp @@ -60,6 +60,12 @@ class CameraHost std::size_t camera_count() const; std::size_t reader_count() const; + /** + * @brief Log what each camera did since the previous call: frame sets published, and as a + * warning, frame sets it had to drop for lack of a free slot and events a reader did not take. + */ + void log_activity(); + // For counters and descriptions; publishing goes through each camera's FrameOutput. const core::camera::Publisher& publisher() const; diff --git a/tests/cpp/plugins/camera/CMakeLists.txt b/tests/cpp/plugins/camera/CMakeLists.txt index 8cbddebef4..3c6c5f9816 100644 --- a/tests/cpp/plugins/camera/CMakeLists.txt +++ b/tests/cpp/plugins/camera/CMakeLists.txt @@ -6,6 +6,7 @@ cmake_minimum_required(VERSION 3.20) find_package(Threads REQUIRED) add_executable(camera_plugin_tests + test_camera_bench.cpp test_camera_host.cpp test_camera_plugin.cpp ) @@ -19,11 +20,12 @@ target_link_libraries(camera_plugin_tests PRIVATE Threads::Threads ) -# Some cases start the plugin as a process of its own and read from it. +# Some cases start the plugin or the bench as a process of its own. target_compile_definitions(camera_plugin_tests PRIVATE CAMERA_PLUGIN_EXECUTABLE="$" + CAMERA_BENCH_EXECUTABLE="$" ) -add_dependencies(camera_plugin_tests camera_plugin) +add_dependencies(camera_plugin_tests camera_plugin camera_bench) message(STATUS "camera_plugin_tests target enabled with Catch2") catch_discover_tests(camera_plugin_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/plugins/camera/test_camera_bench.cpp b/tests/cpp/plugins/camera/test_camera_bench.cpp new file mode 100644 index 0000000000..e6d65f6214 --- /dev/null +++ b/tests/cpp/plugins/camera/test_camera_bench.cpp @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::camera; +using Catch::Matchers::ContainsSubstring; + +namespace +{ + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_bench_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string file(const std::string& name) const + { + return (m_path / name).string(); + } + +private: + std::filesystem::path m_path; +}; + +struct BenchRun +{ + int exit_code = -1; + std::string output; // what the bench printed, results and errors together +}; + +// Runs camera_bench to its end with its output going to a file. +BenchRun run_bench(const TemporaryDirectory& directory, const std::vector& arguments) +{ + const std::string capture = directory.file("bench_output.txt"); + std::vector owned = { CAMERA_BENCH_EXECUTABLE }; + owned.insert(owned.end(), arguments.begin(), arguments.end()); + std::vector argv; + for (std::string& argument : owned) + { + argv.push_back(argument.data()); + } + argv.push_back(nullptr); + + const pid_t pid = ::fork(); + REQUIRE(pid >= 0); + if (pid == 0) + { + const int file = ::open(capture.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + ::dup2(file, STDOUT_FILENO); + ::dup2(file, STDERR_FILENO); + ::execv(argv[0], argv.data()); + ::_exit(127); + } + int status = 0; + REQUIRE(::waitpid(pid, &status, 0) == pid); + + BenchRun result; + result.exit_code = WIFEXITED(status) ? WEXITSTATUS(status) : -1; + std::ifstream stream(capture); + result.output.assign(std::istreambuf_iterator(stream), std::istreambuf_iterator()); + return result; +} + +} // namespace + +TEST_CASE("the bench reads a camera from another process and reports each step", "[unit][camera_plugin][process]") +{ + TemporaryDirectory directory; + const std::string endpoint = directory.file("cameras.sock"); + const std::vector backends = built_backends(); + const CameraConfig config = parse_camera_argument("head:synthetic:width=320,height=240,fps=200"); + CameraHost host(endpoint); + host.add_camera(find_backend(backends, config.backend).open(config)); + host.start(); + + const BenchRun listing = run_bench(directory, { "--endpoint=" + endpoint }); + REQUIRE(listing.exit_code == 0); + REQUIRE_THAT(listing.output, ContainsSubstring("head: 2 stream(s)") && ContainsSubstring("head/left") && + ContainsSubstring("320x240 bgra8, 200.00 fps")); + + const BenchRun run = + run_bench(directory, { "--endpoint=" + endpoint, "--group=head", "--frames=50", "--check-synthetic" }); + REQUIRE(run.exit_code == 0); + REQUIRE_THAT(run.output, ContainsSubstring("read 50 frame sets")); + REQUIRE_THAT(run.output, ContainsSubstring("capture -> sdk ready") && ContainsSubstring("sent -> reader") && + ContainsSubstring("capture -> reader")); + REQUIRE_THAT(run.output, ContainsSubstring("synthetic pattern: 0 image(s) did not match")); + + const BenchRun in_order = run_bench( + directory, { "--endpoint=" + endpoint, "--group=head", "--frames=20", "--window=3", "--order=in_order" }); + REQUIRE(in_order.exit_code == 0); + REQUIRE_THAT(in_order.output, ContainsSubstring("read 20 frame sets")); + + // The plugin's own counters saw the bench's readers come, read and go. + REQUIRE(host.publisher().stats(0).events_sent >= 70); + host.log_activity(); + + const BenchRun unknown = run_bench(directory, { "--endpoint=" + endpoint, "--group=chest" }); + REQUIRE(unknown.exit_code == 1); + REQUIRE_THAT(unknown.output, ContainsSubstring("no group 'chest'")); + + REQUIRE(run_bench(directory, { "--group=head" }).exit_code == 2); + REQUIRE(run_bench(directory, { "--endpoint=" + endpoint, "--order=backwards" }).exit_code == 2); + REQUIRE(run_bench(directory, { "--help" }).exit_code == 0); + + host.stop(); + const BenchRun nobody = run_bench(directory, { "--endpoint=" + endpoint, "--group=head" }); + REQUIRE(nobody.exit_code == 1); + REQUIRE_THAT(nobody.output, ContainsSubstring("no camera plugin answers on")); +}