From 5856c4c961d3a2bf07739ee274641f22b2accd0b Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 12:16:48 -0800 Subject: [PATCH 1/8] feat(camera): add camera_core, the slot and window core of the camera capture plugin First step of the unified camera capture plugin (milestone M0). camera_core tracks which pool slots hold which frame sets and who may still read them. It uses the standard library only and never touches slot memory, so it builds and is tested on every platform. - A group is one or more hardware-synchronised streams published as one frame set. - A reader subscribes with a window (frame sets kept for it at a time) and a delivery order, newest or in order. - A subscription is accepted only when the pool covers it: slots >= ring history + sum of reader windows + backend reserve. - A backend either takes a free slot and writes it, or hands over a slot the camera SDK filled and is told the moment nobody holds it any more. - Every frame event carries its timeline, the group's generation and the number of frame sets the reader skipped. Tests: tests/cpp/core/camera_core (Catch2), run with `ctest -L camera_core`. Signed-off-by: Farbod Motlagh Co-Authored-By: Claude --- src/core/CMakeLists.txt | 3 + src/core/camera_core/CMakeLists.txt | 6 + src/core/camera_core/cpp/CMakeLists.txt | 16 + .../cpp/inc/camera_core/publisher.hpp | 222 ++++++++ .../camera_core/cpp/inc/camera_core/types.hpp | 116 ++++ src/core/camera_core/cpp/publisher.cpp | 445 +++++++++++++++ tests/cpp/core/CMakeLists.txt | 1 + tests/cpp/core/camera_core/CMakeLists.txt | 16 + tests/cpp/core/camera_core/test_publisher.cpp | 510 ++++++++++++++++++ 9 files changed, 1335 insertions(+) create mode 100644 src/core/camera_core/CMakeLists.txt create mode 100644 src/core/camera_core/cpp/CMakeLists.txt create mode 100644 src/core/camera_core/cpp/inc/camera_core/publisher.hpp create mode 100644 src/core/camera_core/cpp/inc/camera_core/types.hpp create mode 100644 src/core/camera_core/cpp/publisher.cpp create mode 100644 tests/cpp/core/camera_core/CMakeLists.txt create mode 100644 tests/cpp/core/camera_core/test_publisher.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 9cc4772373..3393c107b0 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -19,6 +19,9 @@ add_subdirectory(oxr_utils) # Build Plugin Manager add_subdirectory(plugin_manager) +# Camera capture core (standard library only) +add_subdirectory(camera_core) + # Build OXR library (depends on oxr_utils for types) add_subdirectory(oxr) diff --git a/src/core/camera_core/CMakeLists.txt b/src/core/camera_core/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_core/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_core/cpp/CMakeLists.txt b/src/core/camera_core/cpp/CMakeLists.txt new file mode 100644 index 0000000000..0728a6df2c --- /dev/null +++ b/src/core/camera_core/cpp/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Camera capture core: which pool slots hold which frame sets and who may still read them. +# Standard library only, so it builds and is tested on every platform. +add_library(camera_core STATIC + publisher.cpp +) + +target_include_directories(camera_core INTERFACE + $ +) + +add_library(camera::camera_core ALIAS camera_core) diff --git a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp new file mode 100644 index 0000000000..3b7ee5dd87 --- /dev/null +++ b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp @@ -0,0 +1,222 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +using GroupId = std::uint32_t; +using SubscriptionId = std::uint32_t; // 0 is never a valid subscription + +/** + * @brief Where a reader's frame events go: a socket connection or an in-process queue. + */ +class EventSink +{ +public: + virtual ~EventSink() = default; + + /** + * @brief Hand one event to the reader. Called on the capture path with the publisher's + * lock held, so it must not block and must not call back into the publisher. + * @return false when the event could not be handed over; the reader then gets a later one. + */ + virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) = 0; +}; + +struct PublisherStats +{ + std::uint64_t published = 0; // frame sets published + std::uint64_t events_sent = 0; + std::uint64_t send_failures = 0; + std::uint64_t no_free_slot = 0; // acquire_slot calls that found every slot in use +}; + +struct SubscribeResult +{ + SubscriptionId id = 0; + std::string error; // why the subscription was refused, when id is 0 + + explicit operator bool() const + { + return id != 0; + } +}; + +/** + * @brief Tracks which pool slots hold which frame sets and who may still read them. + * + * A group keeps a short ring of recent frame sets; a reader's window is how many are kept for + * it at a time, unread and held together. A subscription is accepted only when + * slots >= ring history + sum of reader windows + backend reserve. Events carry slot indices; + * slot memory is never touched here. Thread-safe; no call blocks on a reader. + */ +class Publisher +{ +public: + using Clock = std::function; + using SlotFreeCallback = std::function; + + /** + * @param clock Host monotonic time in nanoseconds; defaults to std::chrono::steady_clock. + */ + explicit Publisher(Clock clock = {}); + + // ---- backend side ---------------------------------------------------------------------- + + /** + * @brief Declare a group and the size of its pools. + * @param backend_reserve Slots the backend needs beyond the ring and the readers: one to + * write ahead, more for an SDK that keeps several captures in flight in its own queue. + * @throws std::invalid_argument if the description is not usable. + */ + GroupId add_group(const GroupDescription& description, std::uint32_t backend_reserve = 1); + + /** + * @brief Take a free slot to write the next image into. + * @return The slot, or nothing when every slot is in use (the frame should be dropped). + */ + std::optional acquire_slot(GroupId group, std::size_t stream); + + /** + * @brief Mark a slot the SDK chose and filled, for a pool the SDK owns. + * @throws std::invalid_argument if the slot is still held by the ring or a reader. + */ + void claim_slot(GroupId group, std::size_t stream, std::uint32_t slot); + + /** + * @brief Give back a slot taken with acquire_slot or claim_slot without publishing it. + */ + void abandon_slot(GroupId group, std::size_t stream, std::uint32_t slot); + + /** + * @brief Be told when a published slot is no longer held by the ring or any reader, so a + * buffer the SDK owns can go back to it at once. Called outside the publisher's lock, on + * the thread whose call released the last hold. + */ + void set_slot_free_callback(GroupId group, SlotFreeCallback callback); + + /** + * @brief Publish one frame set. images[i] belongs to stream i and names a slot taken with + * acquire_slot or claim_slot. + * @param timeline capture_ns and sdk_ready_ns from the backend; the rest is filled in here. + * @return The frame set's sequence number. + * @throws std::invalid_argument if the images do not match the group. + */ + std::uint64_t publish(GroupId group, std::span images, const FrameTimeline& timeline); + + /** + * @brief Tell readers the camera restarted: the generation changes and the ring is emptied. + * Frame sets readers already hold stay valid until released. + */ + void restart_group(GroupId group); + + // ---- reader side ----------------------------------------------------------------------- + + /** + * @brief Start sending a group's frame events to a sink. + * @return The subscription, or the reason it does not fit the pool. + */ + SubscribeResult subscribe(EventSink& sink, GroupId group, const ReaderOptions& options); + + /** + * @brief Return a frame set. The reader's next one, if any is waiting, is sent at once. + */ + void release(SubscriptionId subscription, std::uint64_t lease_id); + + /** + * @brief End a subscription and take back every frame set it still holds. + */ + void unsubscribe(SubscriptionId subscription); + + // ---- description ----------------------------------------------------------------------- + + std::size_t group_count() const; + GroupDescription group_description(GroupId group) const; + std::optional find_group(std::string_view id) const; + std::uint32_t generation(GroupId group) const; + PublisherStats stats() const; + +private: + struct FrameSet + { + std::uint64_t sequence = 0; + std::uint32_t generation = 0; + FrameTimeline timeline; + std::array images; + }; + + struct Subscription + { + GroupId group = 0; + std::uint32_t window = 1; + DeliveryOrder order = DeliveryOrder::NEWEST; + EventSink* sink = nullptr; + std::uint64_t cursor = 0; // sequence of the last frame set sent to this reader + bool started = false; + std::map leases; // sent and not yet released + }; + + struct Slot + { + std::uint32_t holds = 0; // the ring and every reader lease that names this slot + bool writing = false; // taken by the backend and not yet published + }; + + struct Group + { + GroupDescription description; + std::uint32_t reserve = 1; + std::uint32_t slot_count = 0; // the smallest pool of the group + std::uint32_t generation = 0; + std::uint64_t sequence = 0; + std::vector> slots; // per stream + std::deque ring; + std::vector subscriptions; + std::shared_ptr on_free; + }; + + struct FreedSlot + { + std::shared_ptr callback; + std::size_t stream = 0; + std::uint32_t slot = 0; + }; + + Group& group_at(GroupId group); + const Group& group_at(GroupId group) const; + static Slot& slot_at(Group& group, std::size_t stream, std::uint32_t slot); + std::size_t history_of(const Group& group) const; + void hold(Group& group, const FrameSet& set); + void unhold(Group& group, const FrameSet& set, std::vector& freed); + void trim_ring(Group& group, std::vector& freed); + void deliver(Group& group, SubscriptionId id, Subscription& subscription, std::vector& freed); + static void notify(const std::vector& freed); + + Clock m_clock; + mutable std::mutex m_mutex; + std::vector> m_groups; + std::map m_subscriptions; + SubscriptionId m_next_subscription = 1; + std::uint64_t m_next_lease = 1; + PublisherStats m_stats; +}; + +} // namespace core::camera diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp new file mode 100644 index 0000000000..8e919e35ec --- /dev/null +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// A group publishes one image per stream, all captured together. +inline constexpr std::size_t MAX_STREAMS_PER_GROUP = 8; +inline constexpr std::uint32_t MAX_SLOTS_PER_STREAM = 64; + +// Marks a time that is not known. All times are nanoseconds on the host's monotonic clock. +inline constexpr std::int64_t TIME_UNKNOWN = -1; + +enum class PixelFormat : std::uint32_t +{ + BGRA8 = 1, + RGBA8 = 2, + NV12 = 3, + GRAY8 = 4, + // Encoded: a slot holds one access unit instead of pixels. + H264 = 16, + H265 = 17, +}; + +constexpr bool is_encoded(PixelFormat format) +{ + return format == PixelFormat::H264 || format == PixelFormat::H265; +} + +// How a slot's memory reaches another process. The core tracks slots by index only and never +// touches their memory, so a new kind needs no change here. +enum class MemoryKind : std::uint32_t +{ + HOST = 1, // shared host memory, passed as a file descriptor + CUDA = 2, // CUDA memory on a discrete GPU + DMA_BUF = 3, // Jetson buffer, passed as a file descriptor + NVSCI_BUF = 4, // reserved for SIPL; nothing implements it yet +}; + +struct StreamDescription +{ + std::string id; + PixelFormat format = PixelFormat::BGRA8; + MemoryKind memory = MemoryKind::HOST; + std::uint32_t width = 0; + std::uint32_t height = 0; + std::uint32_t stride_bytes = 0; // row pitch of the first plane; 0 for an encoded stream + std::uint64_t slot_bytes = 0; // capacity of one slot + std::uint32_t slot_count = 0; + double frame_rate = 0.0; // configured or measured, not the sensor mode's maximum +}; + +// One or more hardware-synchronised streams that are published as one frame set. +struct GroupDescription +{ + std::string id; + std::vector streams; +}; + +enum ImageFlag : std::uint32_t +{ + IMAGE_FLAG_KEYFRAME = 1u << 0, // encoded stream: this access unit decodes on its own + IMAGE_FLAG_SDK_WARNING = 1u << 1, // the camera SDK delivered the frame but flagged it as suspect +}; + +struct ImageInfo +{ + std::uint32_t slot = 0; + std::uint32_t flags = 0; // ImageFlag bits + std::uint64_t bytes_used = 0; // encoded stream: size of the access unit + std::uint64_t device_frame_id = 0; // the SDK's own frame number + std::int64_t device_time_ns = TIME_UNKNOWN; // the SDK's own timestamp, in the SDK's clock +}; + +// Where a frame set was at each step, so latency can be read off every frame. +struct FrameTimeline +{ + std::int64_t capture_ns = TIME_UNKNOWN; // sensor capture, when the backend can map it to the host clock + std::int64_t sdk_ready_ns = TIME_UNKNOWN; // the SDK handed the frame to the backend + std::int64_t published_ns = TIME_UNKNOWN; // set by Publisher::publish + std::int64_t sent_ns = TIME_UNKNOWN; // set when the event is handed to a reader +}; + +struct FrameEvent +{ + std::uint64_t lease_id = 0; // return it with Publisher::release + std::uint64_t sequence = 0; // counts frame sets of the group; never restarts + std::uint32_t generation = 0; // changes when the camera restarts + std::uint32_t skipped = 0; // frame sets this reader did not receive since its previous one + std::uint32_t image_count = 0; + FrameTimeline timeline; + std::array images; // images[i] belongs to stream i +}; + +enum class DeliveryOrder +{ + DEFAULT, // NEWEST for a window of one, IN_ORDER otherwise + NEWEST, // each time the reader has room it gets the newest frame set and skips older ones + IN_ORDER, // the reader gets every frame set in sequence while it keeps up +}; + +struct ReaderOptions +{ + std::uint32_t window = 1; // frame sets kept for this reader at a time, unread and held together + DeliveryOrder order = DeliveryOrder::DEFAULT; +}; + +} // namespace core::camera diff --git a/src/core/camera_core/cpp/publisher.cpp b/src/core/camera_core/cpp/publisher.cpp new file mode 100644 index 0000000000..5cb3c8562c --- /dev/null +++ b/src/core/camera_core/cpp/publisher.cpp @@ -0,0 +1,445 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_core/publisher.hpp" + +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +std::int64_t steady_now_ns() +{ + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +DeliveryOrder resolve_order(const ReaderOptions& options) +{ + if (options.order != DeliveryOrder::DEFAULT) + { + return options.order; + } + return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; +} + +} // namespace + +Publisher::Publisher(Clock clock) : m_clock(clock ? std::move(clock) : Clock(steady_now_ns)) +{ +} + +GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t backend_reserve) +{ + if (description.id.empty()) + { + throw std::invalid_argument("camera group needs an id"); + } + if (description.streams.empty() || description.streams.size() > MAX_STREAMS_PER_GROUP) + { + throw std::invalid_argument("camera group '" + description.id + "' needs between 1 and " + + std::to_string(MAX_STREAMS_PER_GROUP) + " streams"); + } + if (backend_reserve == 0) + { + throw std::invalid_argument("camera group '" + description.id + "' needs a backend reserve of at least 1"); + } + + auto group = std::make_unique(); + group->description = description; + group->reserve = backend_reserve; + group->slot_count = std::numeric_limits::max(); + for (const StreamDescription& stream : description.streams) + { + if (stream.id.empty() || stream.slot_bytes == 0 || stream.slot_count == 0 || + stream.slot_count > MAX_SLOTS_PER_STREAM) + { + throw std::invalid_argument("camera group '" + description.id + "' has a stream without an id, a slot " + + "size or between 1 and " + std::to_string(MAX_SLOTS_PER_STREAM) + " slots"); + } + group->slot_count = std::min(group->slot_count, stream.slot_count); + group->slots.emplace_back(stream.slot_count); + } + + std::lock_guard lock(m_mutex); + for (const auto& existing : m_groups) + { + if (existing->description.id == description.id) + { + throw std::invalid_argument("camera group '" + description.id + "' already exists"); + } + } + m_groups.push_back(std::move(group)); + return static_cast(m_groups.size() - 1); +} + +std::optional Publisher::acquire_slot(GroupId group, std::size_t stream) +{ + std::lock_guard lock(m_mutex); + std::vector& slots = group_at(group).slots.at(stream); + for (std::uint32_t i = 0; i < slots.size(); ++i) + { + if (slots[i].holds == 0 && !slots[i].writing) + { + slots[i].writing = true; + return i; + } + } + ++m_stats.no_free_slot; + return std::nullopt; +} + +void Publisher::claim_slot(GroupId group, std::size_t stream, std::uint32_t slot) +{ + std::lock_guard lock(m_mutex); + Slot& state = slot_at(group_at(group), stream, slot); + if (state.holds != 0 || state.writing) + { + throw std::invalid_argument("camera slot " + std::to_string(slot) + " is still in use and cannot be claimed"); + } + state.writing = true; +} + +void Publisher::abandon_slot(GroupId group, std::size_t stream, std::uint32_t slot) +{ + std::lock_guard lock(m_mutex); + Slot& state = slot_at(group_at(group), stream, slot); + if (!state.writing) + { + throw std::invalid_argument("camera slot " + std::to_string(slot) + " was not acquired and cannot be abandoned"); + } + state.writing = false; +} + +void Publisher::set_slot_free_callback(GroupId group, SlotFreeCallback callback) +{ + std::lock_guard lock(m_mutex); + group_at(group).on_free = callback ? std::make_shared(std::move(callback)) : nullptr; +} + +std::uint64_t Publisher::publish(GroupId group, std::span images, const FrameTimeline& timeline) +{ + std::vector freed; + std::uint64_t sequence = 0; + { + std::lock_guard lock(m_mutex); + Group& target = group_at(group); + if (images.size() != target.description.streams.size()) + { + throw std::invalid_argument("camera group '" + target.description.id + "' publishes " + + std::to_string(target.description.streams.size()) + " images per frame set"); + } + for (std::size_t i = 0; i < images.size(); ++i) + { + if (!slot_at(target, i, images[i].slot).writing) + { + throw std::invalid_argument("camera slot " + std::to_string(images[i].slot) + + " was not acquired before it was published"); + } + } + + FrameSet set; + set.sequence = ++target.sequence; + set.generation = target.generation; + set.timeline = timeline; + set.timeline.published_ns = m_clock(); + for (std::size_t i = 0; i < images.size(); ++i) + { + set.images[i] = images[i]; + slot_at(target, i, images[i].slot).writing = false; // the ring holds it from here + } + hold(target, set); + target.ring.push_back(set); + ++m_stats.published; + sequence = set.sequence; + + trim_ring(target, freed); + for (SubscriptionId id : target.subscriptions) + { + deliver(target, id, m_subscriptions.at(id), freed); + } + } + notify(freed); + return sequence; +} + +void Publisher::restart_group(GroupId group) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + Group& target = group_at(group); + ++target.generation; + while (!target.ring.empty()) + { + unhold(target, target.ring.front(), freed); + target.ring.pop_front(); + } + } + notify(freed); +} + +SubscribeResult Publisher::subscribe(EventSink& sink, GroupId group, const ReaderOptions& options) +{ + SubscribeResult result; + std::lock_guard lock(m_mutex); + if (group >= m_groups.size()) + { + result.error = "unknown camera group"; + return result; + } + if (options.window == 0) + { + result.error = "a reader's window must be at least 1"; + return result; + } + Group& target = *m_groups[group]; + const DeliveryOrder order = resolve_order(options); + + // The ring keeps what in-order readers may still be waiting for; newest-only readers + // need the newest frame set and nothing older. + std::size_t history = order == DeliveryOrder::IN_ORDER ? options.window : 1; + std::size_t windows = options.window; + for (SubscriptionId id : target.subscriptions) + { + const Subscription& other = m_subscriptions.at(id); + if (other.order == DeliveryOrder::IN_ORDER) + { + history = std::max(history, other.window); + } + windows += other.window; + } + const std::size_t needed = history + windows + target.reserve; + if (needed > target.slot_count) + { + result.error = "subscription does not fit the pool of camera group '" + target.description.id + "': it needs " + + std::to_string(needed) + " slots (" + std::to_string(history) + " ring history, " + + std::to_string(windows) + " for reader windows, " + std::to_string(target.reserve) + + " for the backend) and the pool has " + std::to_string(target.slot_count); + return result; + } + + Subscription subscription; + subscription.group = group; + subscription.window = options.window; + subscription.order = order; + subscription.sink = &sink; + result.id = m_next_subscription++; + m_subscriptions.emplace(result.id, std::move(subscription)); + target.subscriptions.push_back(result.id); + return result; +} + +void Publisher::release(SubscriptionId subscription, std::uint64_t lease_id) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + const auto found = m_subscriptions.find(subscription); + if (found == m_subscriptions.end()) + { + return; + } + const auto lease = found->second.leases.find(lease_id); + if (lease == found->second.leases.end()) + { + return; + } + Group& target = *m_groups[found->second.group]; + unhold(target, lease->second, freed); + found->second.leases.erase(lease); + deliver(target, subscription, found->second, freed); + } + notify(freed); +} + +void Publisher::unsubscribe(SubscriptionId subscription) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + const auto found = m_subscriptions.find(subscription); + if (found == m_subscriptions.end()) + { + return; + } + Group& target = *m_groups[found->second.group]; + for (const auto& lease : found->second.leases) + { + unhold(target, lease.second, freed); + } + std::erase(target.subscriptions, subscription); + m_subscriptions.erase(found); + trim_ring(target, freed); + } + notify(freed); +} + +std::size_t Publisher::group_count() const +{ + std::lock_guard lock(m_mutex); + return m_groups.size(); +} + +GroupDescription Publisher::group_description(GroupId group) const +{ + std::lock_guard lock(m_mutex); + return group_at(group).description; +} + +std::optional Publisher::find_group(std::string_view id) const +{ + std::lock_guard lock(m_mutex); + for (std::size_t i = 0; i < m_groups.size(); ++i) + { + if (m_groups[i]->description.id == id) + { + return static_cast(i); + } + } + return std::nullopt; +} + +std::uint32_t Publisher::generation(GroupId group) const +{ + std::lock_guard lock(m_mutex); + return group_at(group).generation; +} + +PublisherStats Publisher::stats() const +{ + std::lock_guard lock(m_mutex); + return m_stats; +} + +Publisher::Group& Publisher::group_at(GroupId group) +{ + if (group >= m_groups.size()) + { + throw std::out_of_range("unknown camera group"); + } + return *m_groups[group]; +} + +const Publisher::Group& Publisher::group_at(GroupId group) const +{ + if (group >= m_groups.size()) + { + throw std::out_of_range("unknown camera group"); + } + return *m_groups[group]; +} + +Publisher::Slot& Publisher::slot_at(Group& group, std::size_t stream, std::uint32_t slot) +{ + if (stream >= group.slots.size() || slot >= group.slots[stream].size()) + { + throw std::invalid_argument("camera group '" + group.description.id + "' has no stream " + + std::to_string(stream) + " slot " + std::to_string(slot)); + } + return group.slots[stream][slot]; +} + +std::size_t Publisher::history_of(const Group& group) const +{ + std::size_t history = 1; // the newest frame set is always kept + for (SubscriptionId id : group.subscriptions) + { + const Subscription& subscription = m_subscriptions.at(id); + if (subscription.order == DeliveryOrder::IN_ORDER) + { + history = std::max(history, subscription.window); + } + } + return history; +} + +void Publisher::hold(Group& group, const FrameSet& set) +{ + for (std::size_t i = 0; i < group.slots.size(); ++i) + { + ++slot_at(group, i, set.images[i].slot).holds; + } +} + +void Publisher::unhold(Group& group, const FrameSet& set, std::vector& freed) +{ + for (std::size_t i = 0; i < group.slots.size(); ++i) + { + Slot& slot = slot_at(group, i, set.images[i].slot); + if (--slot.holds == 0 && group.on_free) + { + freed.push_back({ group.on_free, i, set.images[i].slot }); + } + } +} + +void Publisher::trim_ring(Group& group, std::vector& freed) +{ + const std::size_t history = history_of(group); + while (group.ring.size() > history) + { + unhold(group, group.ring.front(), freed); + group.ring.pop_front(); + } +} + +void Publisher::deliver(Group& group, SubscriptionId id, Subscription& subscription, std::vector& freed) +{ + while (subscription.leases.size() < subscription.window) + { + if (group.ring.empty() || group.ring.back().sequence <= subscription.cursor) + { + return; + } + const FrameSet* target = &group.ring.back(); + if (subscription.order == DeliveryOrder::IN_ORDER) + { + target = &*std::find_if(group.ring.begin(), group.ring.end(), + [&](const FrameSet& set) { return set.sequence > subscription.cursor; }); + } + + FrameEvent event; + event.lease_id = m_next_lease++; + event.sequence = target->sequence; + event.generation = target->generation; + if (subscription.started) + { + const std::uint64_t skipped = target->sequence - subscription.cursor - 1; + event.skipped = + static_cast(std::min(skipped, std::numeric_limits::max())); + } + event.image_count = static_cast(group.slots.size()); + event.images = target->images; + event.timeline = target->timeline; + event.timeline.sent_ns = m_clock(); + + hold(group, *target); + if (!subscription.sink->send_event(id, event)) + { + unhold(group, *target, freed); + ++m_stats.send_failures; + return; + } + ++m_stats.events_sent; + subscription.leases.emplace(event.lease_id, *target); + subscription.cursor = target->sequence; + subscription.started = true; + } +} + +void Publisher::notify(const std::vector& freed) +{ + for (const FreedSlot& slot : freed) + { + (*slot.callback)(slot.stream, slot.slot); + } +} + +} // namespace core::camera diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 54c6e1e0d5..e21545f14f 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -13,6 +13,7 @@ add_subdirectory(live_trackers) add_subdirectory(tracker_metadata) add_subdirectory(log_bridge) add_subdirectory(plugin_manager) +add_subdirectory(camera_core) if(BUILD_OVXR_CAMERA_QUADS) add_subdirectory(ovxr_pip_layer) diff --git a/tests/cpp/core/camera_core/CMakeLists.txt b/tests/cpp/core/camera_core/CMakeLists.txt new file mode 100644 index 0000000000..aa26bc438f --- /dev/null +++ b/tests/cpp/core/camera_core/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_core_tests + test_publisher.cpp +) + +target_link_libraries(camera_core_tests PRIVATE + camera::camera_core + Catch2::Catch2WithMain +) + +message(STATUS "camera_core_tests target enabled with Catch2") +catch_discover_tests(camera_core_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp new file mode 100644 index 0000000000..5084581600 --- /dev/null +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -0,0 +1,510 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +// A reader that keeps every event it is sent. +struct RecordingSink : EventSink +{ + bool send_event(SubscriptionId, const FrameEvent& event) override + { + if (!accept) + { + return false; + } + events.push_back(event); + return true; + } + + std::vector events; + bool accept = true; +}; + +GroupDescription make_group(std::uint32_t slots, std::size_t streams = 2, const std::string& id = "head") +{ + GroupDescription group; + group.id = id; + for (std::size_t i = 0; i < streams; ++i) + { + StreamDescription stream; + stream.id = id + "/" + std::to_string(i); + stream.width = 640; + stream.height = 400; + stream.stride_bytes = 640 * 4; + stream.slot_bytes = 640 * 400 * 4; + stream.slot_count = slots; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// Publishes one frame set the way a backend does: take a free slot per stream, then publish. +std::uint64_t publish_next(Publisher& publisher, GroupId group, const FrameTimeline& timeline = {}) +{ + const std::size_t streams = publisher.group_description(group).streams.size(); + std::vector images(streams); + for (std::size_t i = 0; i < streams; ++i) + { + const auto slot = publisher.acquire_slot(group, i); + REQUIRE(slot.has_value()); + images[i].slot = *slot; + } + return publisher.publish(group, images, timeline); +} + +std::vector sequences(const RecordingSink& sink) +{ + std::vector result; + for (const FrameEvent& event : sink.events) + { + result.push_back(event.sequence); + } + return result; +} + +} // namespace + +TEST_CASE("a window of one always gets the newest frame set", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); + REQUIRE(sequences(sink) == std::vector{ 1 }); + REQUIRE(sink.events[0].skipped == 0); + REQUIRE(sink.events[0].image_count == 2); + + // The reader still holds frame set 1, so the next ones wait. + publish_next(publisher, group); + publish_next(publisher, group); + REQUIRE(sink.events.size() == 1); + + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 3 }); + REQUIRE(sink.events[1].skipped == 1); +} + +TEST_CASE("a larger window gets every frame set in order", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 3 }); + REQUIRE(reader); + + for (int i = 0; i < 4; ++i) + { + publish_next(publisher, group); + } + // Three are out, the fourth waits in the ring until the reader has room. + REQUIRE(sequences(sink) == std::vector{ 1, 2, 3 }); + + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 3, 4 }); + REQUIRE(sink.events[3].skipped == 0); +} + +TEST_CASE("an in-order reader that falls behind is told what it skipped", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 2 }); + REQUIRE(reader); + + for (int i = 0; i < 6; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sequences(sink) == std::vector{ 1, 2 }); + + // The ring kept the last two (5 and 6); 3 and 4 are gone. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 5 }); + REQUIRE(sink.events[2].skipped == 2); + + publisher.release(reader.id, sink.events[1].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 5, 6 }); + REQUIRE(sink.events[3].skipped == 0); +} + +TEST_CASE("newest order with a larger window skips to the newest frame set", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 2, .order = DeliveryOrder::NEWEST }); + REQUIRE(reader); + + for (int i = 0; i < 4; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sequences(sink) == std::vector{ 1, 2 }); + + // A viewer that holds one frame and wants the newest must not be handed frame set 3. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 4 }); + REQUIRE(sink.events[2].skipped == 1); +} + +TEST_CASE("a subscription that does not fit the pool is refused", "[camera_core][budget]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4)); + RecordingSink first; + RecordingSink second; + + // 1 ring history + 1 window + 1 backend reserve = 3 of 4 slots. + REQUIRE(publisher.subscribe(first, group, { .window = 1 })); + + // An in-order reader with a window of two needs 2 ring history + 3 windows + 1 reserve = 6. + const SubscribeResult refused = publisher.subscribe(second, group, { .window = 2 }); + REQUIRE_FALSE(refused); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("needs 6 slots")); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("pool has 4")); + + // A second newest-only reader needs 1 + 2 + 1 = 4 and fits. + REQUIRE(publisher.subscribe(second, group, { .window = 1 })); + + REQUIRE_FALSE(publisher.subscribe(second, group, { .window = 0 })); + REQUIRE_FALSE(publisher.subscribe(second, 99, { .window = 1 })); +} + +TEST_CASE("the backend reserve counts against the pool", "[camera_core][budget]") +{ + Publisher publisher; + // An SDK that keeps four captures in flight, on a pool of eight. + const GroupId group = publisher.add_group(make_group(8), 4); + RecordingSink sinks[4]; + + // 1 ring history + 3 windows + 4 reserve = 8. + for (int i = 0; i < 3; ++i) + { + REQUIRE(publisher.subscribe(sinks[i], group, { .window = 1 })); + } + const SubscribeResult refused = publisher.subscribe(sinks[3], group, { .window = 1 }); + REQUIRE_FALSE(refused); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("4 for the backend")); +} + +TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", "[camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(3, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); // slot 0: held by the reader + publish_next(publisher, group); // slot 1: held by the ring + REQUIRE(sink.events.size() == 1); + REQUIRE(sink.events[0].images[0].slot == 0); + + const auto third = publisher.acquire_slot(group, 0); + REQUIRE(third == 2u); + REQUIRE_FALSE(publisher.acquire_slot(group, 0).has_value()); + REQUIRE(publisher.stats().no_free_slot == 1); + publisher.abandon_slot(group, 0, *third); + + // Releasing frame set 1 frees slot 0 and hands the reader frame set 2. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events.size() == 2); + REQUIRE(sink.events[1].images[0].slot == 1); + REQUIRE(publisher.acquire_slot(group, 0) == 0u); +} + +TEST_CASE("the slot-free callback fires when the last holder lets go", "[camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + std::vector> freed; + publisher.set_slot_free_callback(group, + [&](std::size_t stream, std::uint32_t slot) + { + // Runs outside the publisher's lock, so calling back in must not deadlock. + (void)publisher.stats(); + freed.emplace_back(stream, slot); + }); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + // The SDK owns the buffers and picks the slot it filled. + const auto publish_slot = [&](std::uint32_t slot) + { + publisher.claim_slot(group, 0, slot); + ImageInfo image; + image.slot = slot; + publisher.publish(group, { &image, 1 }, {}); + }; + + publish_slot(0); // ring + reader + publish_slot(1); // the ring lets slot 0 go, the reader still has it + REQUIRE(freed.empty()); + REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 0), std::invalid_argument); + + publisher.release(reader.id, sink.events[0].lease_id); // reader moves on to slot 1 + REQUIRE(freed == std::vector>{ { 0, 0 } }); + + publish_slot(2); // the ring lets slot 1 go, the reader still has it + REQUIRE(freed.size() == 1); + publisher.unsubscribe(reader.id); + REQUIRE(freed == std::vector>{ { 0, 0 }, { 0, 1 } }); + + // A slot the backend gives up itself needs no notification. + publisher.claim_slot(group, 0, 0); + publisher.abandon_slot(group, 0, 0); + REQUIRE(freed.size() == 2); +} + +TEST_CASE("unsubscribing returns every frame set the reader held", "[camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 3 }); + REQUIRE(reader); + for (int i = 0; i < 3; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sink.events.size() == 3); + + publisher.unsubscribe(reader.id); + // Only the newest frame set stays in the ring: seven of the eight slots are free again. + for (int i = 0; i < 7; ++i) + { + REQUIRE(publisher.acquire_slot(group, 0).has_value()); + } + REQUIRE_FALSE(publisher.acquire_slot(group, 0).has_value()); + + // Releasing after the subscription ended is harmless. + publisher.release(reader.id, sink.events[0].lease_id); +} + +TEST_CASE("a frame set that could not be sent is not counted as held", "[camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(3, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + sink.accept = false; + publish_next(publisher, group); + REQUIRE(publisher.stats().send_failures == 1); + REQUIRE(sink.events.empty()); + + sink.accept = true; + publish_next(publisher, group); + REQUIRE(sequences(sink) == std::vector{ 2 }); + REQUIRE(sink.events[0].skipped == 0); + REQUIRE(publisher.stats().events_sent == 1); + + // Frame set 1 left the ring and no reader holds it: its slot is free. + REQUIRE(publisher.acquire_slot(group, 0) == 0u); +} + +TEST_CASE("a restart changes the generation and empties the ring", "[camera_core][lifecycle]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); + REQUIRE(sink.events[0].generation == 0); + + publisher.restart_group(group); + REQUIRE(publisher.generation(group) == 1); + + // The frame set the reader holds stays valid; the sequence keeps counting. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events.size() == 1); + publish_next(publisher, group); + REQUIRE(sink.events.size() == 2); + REQUIRE(sink.events[1].generation == 1); + REQUIRE(sink.events[1].sequence == 2); +} + +TEST_CASE("a group publishes one image per stream", "[camera_core][groups]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 3, "rig")); + REQUIRE(publisher.find_group("rig") == group); + REQUIRE_FALSE(publisher.find_group("head").has_value()); + REQUIRE(publisher.group_count() == 1); + + RecordingSink sink; + REQUIRE(publisher.subscribe(sink, group, { .window = 1 })); + + std::vector images(3); + for (std::size_t i = 0; i < 3; ++i) + { + images[i].slot = *publisher.acquire_slot(group, i); + images[i].device_frame_id = 100 + i; + } + images[1].flags = IMAGE_FLAG_KEYFRAME; + images[1].bytes_used = 4096; + images[2].flags = IMAGE_FLAG_SDK_WARNING; + publisher.publish(group, images, {}); + + REQUIRE(sink.events.size() == 1); + const FrameEvent& event = sink.events[0]; + REQUIRE(event.image_count == 3); + REQUIRE(event.images[0].device_frame_id == 100); + REQUIRE(event.images[1].flags == IMAGE_FLAG_KEYFRAME); + REQUIRE(event.images[1].bytes_used == 4096); + REQUIRE(event.images[2].flags == IMAGE_FLAG_SDK_WARNING); +} + +TEST_CASE("every frame set carries its timeline", "[camera_core][timeline]") +{ + std::int64_t now = 1000; + Publisher publisher([&] { return now; }); + const GroupId group = publisher.add_group(make_group(4, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + FrameTimeline timeline; + timeline.capture_ns = 100; + timeline.sdk_ready_ns = 900; + publish_next(publisher, group, timeline); + REQUIRE(sink.events[0].timeline.capture_ns == 100); + REQUIRE(sink.events[0].timeline.sdk_ready_ns == 900); + REQUIRE(sink.events[0].timeline.published_ns == 1000); + REQUIRE(sink.events[0].timeline.sent_ns == 1000); + + // A frame set that waits for the reader is stamped as sent when it leaves. + now = 2000; + publish_next(publisher, group); + now = 2500; + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events[1].timeline.capture_ns == TIME_UNKNOWN); + REQUIRE(sink.events[1].timeline.published_ns == 2000); + REQUIRE(sink.events[1].timeline.sent_ns == 2500); +} + +TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") +{ + Publisher publisher; + REQUIRE_THROWS_AS(publisher.add_group(make_group(4, 0)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4, MAX_STREAMS_PER_GROUP + 1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(0)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(MAX_SLOTS_PER_STREAM + 1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 0), std::invalid_argument); + + const GroupId group = publisher.add_group(make_group(4)); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4)), std::invalid_argument); // same id + + // Publishing needs one image per stream, each in a slot the backend took first. + std::vector images(2); + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); + images[0].slot = *publisher.acquire_slot(group, 0); + REQUIRE_THROWS_AS(publisher.publish(group, { images.data(), 1 }, {}), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); // stream 1 not acquired + REQUIRE(publisher.stats().published == 0); + + REQUIRE_THROWS_AS(publisher.abandon_slot(group, 1, 0), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.acquire_slot(99, 0), std::out_of_range); + REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 99), std::invalid_argument); +} + +TEST_CASE("capture and a reader on different threads keep every frame set in order", "[camera_core][threads]") +{ + // A reader as the endpoint will drive it: events arrive on the capture thread, the + // reader's own thread takes them and releases. + struct QueueSink : EventSink + { + bool send_event(SubscriptionId, const FrameEvent& event) override + { + { + std::lock_guard lock(mutex); + queue.push_back(event); + } + ready.notify_one(); + return true; + } + + std::mutex mutex; + std::condition_variable ready; + std::deque queue; + }; + + constexpr std::uint64_t FRAME_SETS = 2000; + Publisher publisher; + const GroupId group = publisher.add_group(make_group(16)); + QueueSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 4 }); + REQUIRE(reader); + + std::atomic received{ 0 }; + std::atomic out_of_order{ 0 }; + std::thread consumer( + [&] + { + std::uint64_t last = 0; + while (last < FRAME_SETS) + { + FrameEvent event; + { + std::unique_lock lock(sink.mutex); + sink.ready.wait(lock, [&] { return !sink.queue.empty(); }); + event = sink.queue.front(); + sink.queue.pop_front(); + } + out_of_order += event.sequence <= last ? 1 : 0; + last = event.sequence; + ++received; + publisher.release(reader.id, event.lease_id); + } + }); + + std::uint64_t dropped = 0; + for (std::uint64_t i = 0; i < FRAME_SETS; ++i) + { + std::vector images(2); + const auto left = publisher.acquire_slot(group, 0); + const auto right = publisher.acquire_slot(group, 1); + if (!left || !right) + { + // Cannot happen while the budget holds: a free slot is always left for the backend. + ++dropped; + continue; + } + images[0].slot = *left; + images[1].slot = *right; + publisher.publish(group, images, {}); + } + consumer.join(); + + REQUIRE(dropped == 0); + REQUIRE(out_of_order == 0); + REQUIRE(received > 0); + REQUIRE(publisher.stats().published == FRAME_SETS); +} From 0a04690261eac93cc7f5e4f1cb747f1fa0cfbef9 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 12:51:40 -0800 Subject: [PATCH 2/8] fix(camera): describe an encoded stream by its codec, not as a pixel format H.264 and H.265 are codecs, so they do not belong in PixelFormat. A stream description now has a Codec (NONE for pixels) next to its PixelFormat, and is_encoded() reads the codec. Signed-off-by: Farbod Motlagh Co-Authored-By: Claude --- .../camera_core/cpp/inc/camera_core/types.hpp | 20 ++++++++++++------- tests/cpp/core/camera_core/test_publisher.cpp | 17 ++++++++++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp index 8e919e35ec..efba4d0b64 100644 --- a/src/core/camera_core/cpp/inc/camera_core/types.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -25,15 +25,15 @@ enum class PixelFormat : std::uint32_t RGBA8 = 2, NV12 = 3, GRAY8 = 4, - // Encoded: a slot holds one access unit instead of pixels. - H264 = 16, - H265 = 17, }; -constexpr bool is_encoded(PixelFormat format) +// Whether a stream's slots hold pixels or one access unit of encoded video each. +enum class Codec : std::uint32_t { - return format == PixelFormat::H264 || format == PixelFormat::H265; -} + NONE = 0, // pixels, laid out as the stream's pixel format says + H264 = 1, + H265 = 2, +}; // How a slot's memory reaches another process. The core tracks slots by index only and never // touches their memory, so a new kind needs no change here. @@ -48,7 +48,8 @@ enum class MemoryKind : std::uint32_t struct StreamDescription { std::string id; - PixelFormat format = PixelFormat::BGRA8; + Codec codec = Codec::NONE; + PixelFormat pixel_format = PixelFormat::BGRA8; // of a pixel stream; unused when the stream is encoded MemoryKind memory = MemoryKind::HOST; std::uint32_t width = 0; std::uint32_t height = 0; @@ -56,6 +57,11 @@ struct StreamDescription std::uint64_t slot_bytes = 0; // capacity of one slot std::uint32_t slot_count = 0; double frame_rate = 0.0; // configured or measured, not the sensor mode's maximum + + constexpr bool is_encoded() const + { + return codec != Codec::NONE; + } }; // One or more hardware-synchronised streams that are published as one frame set. diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp index 5084581600..46f6c00212 100644 --- a/tests/cpp/core/camera_core/test_publisher.cpp +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -382,6 +382,23 @@ TEST_CASE("a group publishes one image per stream", "[camera_core][groups]") REQUIRE(event.images[2].flags == IMAGE_FLAG_SDK_WARNING); } +TEST_CASE("an encoded stream is described by its codec, not by a pixel format", "[camera_core][groups]") +{ + GroupDescription description = make_group(4, 1, "head.enc"); + StreamDescription& stream = description.streams[0]; + REQUIRE_FALSE(stream.is_encoded()); + stream.codec = Codec::H265; + stream.stride_bytes = 0; + stream.slot_bytes = 2 * 1024 * 1024; // room for one access unit + + Publisher publisher; + const GroupId group = publisher.add_group(description); + const StreamDescription stored = publisher.group_description(group).streams[0]; + REQUIRE(stored.is_encoded()); + REQUIRE(stored.codec == Codec::H265); + REQUIRE(stored.slot_bytes == 2 * 1024 * 1024); +} + TEST_CASE("every frame set carries its timeline", "[camera_core][timeline]") { std::int64_t now = 1000; From cdb81280fb71015fd2e51c6380fc18c3cfabb1a5 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 12:53:26 -0800 Subject: [PATCH 3/8] fix(camera): describe plane layout per stream and count per group Two more places where the stream description and the publisher were too coarse for real cameras: - A stream had a single stride. NV12 has two planes, and on Jetson each plane has its own offset and stride, so a stream now lists its planes. add_group rejects a pixel stream without a size or without one stride per plane of its format, and an encoded stream that lists planes. - Counters were totals over every group. They are now kept per group, so a tool can tell which camera drops frames. Signed-off-by: Farbod Motlagh Co-Authored-By: Claude --- .../cpp/inc/camera_core/publisher.hpp | 7 ++-- .../camera_core/cpp/inc/camera_core/types.hpp | 18 +++++++- src/core/camera_core/cpp/publisher.cpp | 41 +++++++++++++++---- tests/cpp/core/camera_core/test_publisher.cpp | 31 ++++++++++---- 4 files changed, 78 insertions(+), 19 deletions(-) diff --git a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp index 3b7ee5dd87..6f98df88cb 100644 --- a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp @@ -41,6 +41,7 @@ class EventSink virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) = 0; }; +// Counters of one group since it was added. struct PublisherStats { std::uint64_t published = 0; // frame sets published @@ -131,7 +132,7 @@ class Publisher // ---- reader side ----------------------------------------------------------------------- /** - * @brief Start sending a group's frame events to a sink. + * @brief Start sending a group's frame events to a sink, which must outlive the subscription. * @return The subscription, or the reason it does not fit the pool. */ SubscribeResult subscribe(EventSink& sink, GroupId group, const ReaderOptions& options); @@ -152,7 +153,7 @@ class Publisher GroupDescription group_description(GroupId group) const; std::optional find_group(std::string_view id) const; std::uint32_t generation(GroupId group) const; - PublisherStats stats() const; + PublisherStats stats(GroupId group) const; private: struct FrameSet @@ -191,6 +192,7 @@ class Publisher std::deque ring; std::vector subscriptions; std::shared_ptr on_free; + PublisherStats stats; }; struct FreedSlot @@ -216,7 +218,6 @@ class Publisher std::map m_subscriptions; SubscriptionId m_next_subscription = 1; std::uint64_t m_next_lease = 1; - PublisherStats m_stats; }; } // namespace core::camera diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp index efba4d0b64..34b8fb11fb 100644 --- a/src/core/camera_core/cpp/inc/camera_core/types.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -27,6 +27,21 @@ enum class PixelFormat : std::uint32_t GRAY8 = 4, }; +// NV12 has two planes (luma, then interleaved chroma); the other formats have one. +inline constexpr std::size_t MAX_PLANES_PER_IMAGE = 2; + +constexpr std::uint32_t plane_count_of(PixelFormat format) +{ + return format == PixelFormat::NV12 ? 2 : 1; +} + +// Where one plane of an image sits inside a slot. +struct PlaneLayout +{ + std::uint32_t offset_bytes = 0; + std::uint32_t stride_bytes = 0; // from the start of one row to the start of the next +}; + // Whether a stream's slots hold pixels or one access unit of encoded video each. enum class Codec : std::uint32_t { @@ -53,7 +68,8 @@ struct StreamDescription MemoryKind memory = MemoryKind::HOST; std::uint32_t width = 0; std::uint32_t height = 0; - std::uint32_t stride_bytes = 0; // row pitch of the first plane; 0 for an encoded stream + std::uint32_t plane_count = 0; // 0 for an encoded stream + std::array planes; // the same for every slot of the stream std::uint64_t slot_bytes = 0; // capacity of one slot std::uint32_t slot_count = 0; double frame_rate = 0.0; // configured or measured, not the sensor mode's maximum diff --git a/src/core/camera_core/cpp/publisher.cpp b/src/core/camera_core/cpp/publisher.cpp index 5cb3c8562c..b3b3ab0766 100644 --- a/src/core/camera_core/cpp/publisher.cpp +++ b/src/core/camera_core/cpp/publisher.cpp @@ -29,6 +29,27 @@ DeliveryOrder resolve_order(const ReaderOptions& options) return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; } +// A pixel stream names every plane of its format; an encoded stream has none. +bool layout_is_valid(const StreamDescription& stream) +{ + if (stream.is_encoded()) + { + return stream.plane_count == 0; + } + if (stream.width == 0 || stream.height == 0 || stream.plane_count != plane_count_of(stream.pixel_format)) + { + return false; + } + for (std::uint32_t i = 0; i < stream.plane_count; ++i) + { + if (stream.planes[i].stride_bytes == 0) + { + return false; + } + } + return true; +} + } // namespace Publisher::Publisher(Clock clock) : m_clock(clock ? std::move(clock) : Clock(steady_now_ns)) @@ -63,6 +84,11 @@ GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t throw std::invalid_argument("camera group '" + description.id + "' has a stream without an id, a slot " + "size or between 1 and " + std::to_string(MAX_SLOTS_PER_STREAM) + " slots"); } + if (!layout_is_valid(stream)) + { + throw std::invalid_argument("camera stream '" + stream.id + "' needs a size and one stride per plane of " + + "its pixel format, or no planes at all when it is encoded"); + } group->slot_count = std::min(group->slot_count, stream.slot_count); group->slots.emplace_back(stream.slot_count); } @@ -82,7 +108,8 @@ GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t std::optional Publisher::acquire_slot(GroupId group, std::size_t stream) { std::lock_guard lock(m_mutex); - std::vector& slots = group_at(group).slots.at(stream); + Group& target = group_at(group); + std::vector& slots = target.slots.at(stream); for (std::uint32_t i = 0; i < slots.size(); ++i) { if (slots[i].holds == 0 && !slots[i].writing) @@ -91,7 +118,7 @@ std::optional Publisher::acquire_slot(GroupId group, std::size_t return i; } } - ++m_stats.no_free_slot; + ++target.stats.no_free_slot; return std::nullopt; } @@ -156,7 +183,7 @@ std::uint64_t Publisher::publish(GroupId group, std::span image } hold(target, set); target.ring.push_back(set); - ++m_stats.published; + ++target.stats.published; sequence = set.sequence; trim_ring(target, freed); @@ -312,10 +339,10 @@ std::uint32_t Publisher::generation(GroupId group) const return group_at(group).generation; } -PublisherStats Publisher::stats() const +PublisherStats Publisher::stats(GroupId group) const { std::lock_guard lock(m_mutex); - return m_stats; + return group_at(group).stats; } Publisher::Group& Publisher::group_at(GroupId group) @@ -424,10 +451,10 @@ void Publisher::deliver(Group& group, SubscriptionId id, Subscription& subscript if (!subscription.sink->send_event(id, event)) { unhold(group, *target, freed); - ++m_stats.send_failures; + ++group.stats.send_failures; return; } - ++m_stats.events_sent; + ++group.stats.events_sent; subscription.leases.emplace(event.lease_id, *target); subscription.cursor = target->sequence; subscription.started = true; diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp index 46f6c00212..e13ed8e051 100644 --- a/tests/cpp/core/camera_core/test_publisher.cpp +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -48,7 +48,8 @@ GroupDescription make_group(std::uint32_t slots, std::size_t streams = 2, const stream.id = id + "/" + std::to_string(i); stream.width = 640; stream.height = 400; - stream.stride_bytes = 640 * 4; + stream.plane_count = 1; + stream.planes[0].stride_bytes = 640 * 4; stream.slot_bytes = 640 * 400 * 4; stream.slot_count = slots; stream.frame_rate = 60.0; @@ -226,7 +227,7 @@ TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", " const auto third = publisher.acquire_slot(group, 0); REQUIRE(third == 2u); REQUIRE_FALSE(publisher.acquire_slot(group, 0).has_value()); - REQUIRE(publisher.stats().no_free_slot == 1); + REQUIRE(publisher.stats(group).no_free_slot == 1); publisher.abandon_slot(group, 0, *third); // Releasing frame set 1 frees slot 0 and hands the reader frame set 2. @@ -245,7 +246,7 @@ TEST_CASE("the slot-free callback fires when the last holder lets go", "[camera_ [&](std::size_t stream, std::uint32_t slot) { // Runs outside the publisher's lock, so calling back in must not deadlock. - (void)publisher.stats(); + (void)publisher.stats(group); freed.emplace_back(stream, slot); }); RecordingSink sink; @@ -315,14 +316,14 @@ TEST_CASE("a frame set that could not be sent is not counted as held", "[camera_ sink.accept = false; publish_next(publisher, group); - REQUIRE(publisher.stats().send_failures == 1); + REQUIRE(publisher.stats(group).send_failures == 1); REQUIRE(sink.events.empty()); sink.accept = true; publish_next(publisher, group); REQUIRE(sequences(sink) == std::vector{ 2 }); REQUIRE(sink.events[0].skipped == 0); - REQUIRE(publisher.stats().events_sent == 1); + REQUIRE(publisher.stats(group).events_sent == 1); // Frame set 1 left the ring and no reader holds it: its slot is free. REQUIRE(publisher.acquire_slot(group, 0) == 0u); @@ -388,7 +389,7 @@ TEST_CASE("an encoded stream is described by its codec, not by a pixel format", StreamDescription& stream = description.streams[0]; REQUIRE_FALSE(stream.is_encoded()); stream.codec = Codec::H265; - stream.stride_bytes = 0; + stream.plane_count = 0; stream.slot_bytes = 2 * 1024 * 1024; // room for one access unit Publisher publisher; @@ -436,6 +437,20 @@ TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") REQUIRE_THROWS_AS(publisher.add_group(make_group(MAX_SLOTS_PER_STREAM + 1)), std::invalid_argument); REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 0), std::invalid_argument); + // A pixel stream names every plane of its format; an encoded stream has none. + GroupDescription nv12 = make_group(4, 1, "nv12"); + nv12.streams[0].pixel_format = PixelFormat::NV12; + REQUIRE_THROWS_AS(publisher.add_group(nv12), std::invalid_argument); + nv12.streams[0].plane_count = 2; + nv12.streams[0].planes[1] = { .offset_bytes = 640 * 400, .stride_bytes = 640 }; + REQUIRE_NOTHROW(publisher.add_group(nv12)); + GroupDescription encoded_with_planes = make_group(4, 1, "enc"); + encoded_with_planes.streams[0].codec = Codec::H264; + REQUIRE_THROWS_AS(publisher.add_group(encoded_with_planes), std::invalid_argument); + GroupDescription no_size = make_group(4, 1, "empty"); + no_size.streams[0].width = 0; + REQUIRE_THROWS_AS(publisher.add_group(no_size), std::invalid_argument); + const GroupId group = publisher.add_group(make_group(4)); REQUIRE_THROWS_AS(publisher.add_group(make_group(4)), std::invalid_argument); // same id @@ -445,7 +460,7 @@ TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") images[0].slot = *publisher.acquire_slot(group, 0); REQUIRE_THROWS_AS(publisher.publish(group, { images.data(), 1 }, {}), std::invalid_argument); REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); // stream 1 not acquired - REQUIRE(publisher.stats().published == 0); + REQUIRE(publisher.stats(group).published == 0); REQUIRE_THROWS_AS(publisher.abandon_slot(group, 1, 0), std::invalid_argument); REQUIRE_THROWS_AS(publisher.acquire_slot(99, 0), std::out_of_range); @@ -523,5 +538,5 @@ TEST_CASE("capture and a reader on different threads keep every frame set in ord REQUIRE(dropped == 0); REQUIRE(out_of_order == 0); REQUIRE(received > 0); - REQUIRE(publisher.stats().published == FRAME_SETS); + REQUIRE(publisher.stats(group).published == FRAME_SETS); } From be73673097abcb17f9666e886f2f49db3a07a160 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 13:40:09 -0800 Subject: [PATCH 4/8] docs(camera): add a README for camera_core and keep only the host memory kind The README explains the terms (stream, group, frame set, window, order, lease, generation), the two ways a backend gets a slot, and the pool budget, so a reviewer does not need anything outside the repository. MemoryKind listed kinds that nothing implements yet. It now has HOST only; each further kind is added together with its pool. Signed-off-by: Farbod Motlagh Co-Authored-By: Claude --- src/core/camera_core/README.md | 69 +++++++++++++++++++ .../camera_core/cpp/inc/camera_core/types.hpp | 9 +-- 2 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 src/core/camera_core/README.md diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md new file mode 100644 index 0000000000..ad50e60f2f --- /dev/null +++ b/src/core/camera_core/README.md @@ -0,0 +1,69 @@ + + +# Camera interface: core + +The camera interface is the one place Isaac Capture applications get camera frames from. One +process owns the cameras and keeps their frames in fixed pools of shared buffers; viewers, +encoders and recorders in other processes read those buffers in place. + +This module is the part with no memory and no sockets in it: it decides which pool slot holds +which frame set and who may still read it. It uses the standard library only. The shared memory +behind the slots and the connection to other processes live in separate modules. + +## Terms + +- **Stream**: one sensor's images, or one encoded video stream. It has a pool of **slots**; a + slot holds one image, or one access unit when the stream is encoded. +- **Group**: one or more streams captured together, such as a stereo pair or a triggered rig. + A group publishes one **frame set** at a time: one image per stream, under one sequence number. +- **Window**: how many frame sets are kept for a reader at a time, unread and held together. +- **Order**: `NEWEST` hands the reader the latest frame set each time it has room and skips the + rest (a viewer, a live encoder). `IN_ORDER` hands over every frame set in sequence (a recorder). + The default is `NEWEST` for a window of one and `IN_ORDER` otherwise. +- **Lease**: a reader's hold on a frame set. Its slots are not reused until the reader releases + it or goes away. +- **Generation**: changes when a camera restarts, so a reader can tell a restart from a skip. + +## How frames move + +A backend gets a slot in one of two ways, then publishes: + +```cpp +// The backend writes the image itself. +auto slot = publisher.acquire_slot(group, stream); // nothing: every slot is in use, drop the frame +// ... write the pixels into that slot ... +publisher.publish(group, images, timeline); + +// The camera SDK owns the buffers and picked one. +publisher.claim_slot(group, stream, slot_the_sdk_filled); +publisher.publish(group, images, timeline); +``` + +For the second way, `set_slot_free_callback` tells the backend the moment a slot is held by +nobody, so the buffer can go back to the SDK. + +A reader subscribes with a window and an order, receives `FrameEvent`s through an `EventSink`, +and returns each one with `release`. Events carry slot indices, the frame set's timeline and the +number of frame sets the reader skipped. A slow reader skips; it never slows capture or another +reader, and no call blocks on a reader. + +## Pool budget + +A slot is in use while the ring of recent frame sets or any lease refers to it, so a subscription +is accepted only when the pool can cover it: + +```text +slots >= ring history + sum of reader windows + backend reserve +``` + +Ring history is one frame set, or the largest window among in-order readers. The backend reserve +is the number of slots the backend needs for itself: one to write ahead, more for an SDK that +keeps several captures in flight. A subscription that does not fit is refused with a message that +gives these numbers. + +## Tests + +`tests/cpp/core/camera_core`, run with `ctest -L camera_core`. diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp index 34b8fb11fb..1e66086a1a 100644 --- a/src/core/camera_core/cpp/inc/camera_core/types.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -50,14 +50,11 @@ enum class Codec : std::uint32_t H265 = 2, }; -// How a slot's memory reaches another process. The core tracks slots by index only and never -// touches their memory, so a new kind needs no change here. +// Where a stream's slots live, which tells a reader how to map them. The core tracks slots by +// index only and never touches their memory; each further kind is added with its pool. enum class MemoryKind : std::uint32_t { - HOST = 1, // shared host memory, passed as a file descriptor - CUDA = 2, // CUDA memory on a discrete GPU - DMA_BUF = 3, // Jetson buffer, passed as a file descriptor - NVSCI_BUF = 4, // reserved for SIPL; nothing implements it yet + HOST = 1, // shared host memory }; struct StreamDescription From 9977ee4a8ef3b40b9de97411b33d8070d5eb7ff0 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 13:47:59 -0800 Subject: [PATCH 5/8] test(camera): tag the camera_core tests as unit tests Core tests carry the unit tag, which becomes the CTest label that `ctest -L unit` selects. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/camera_core/README.md | 3 +- tests/cpp/core/camera_core/test_publisher.cpp | 32 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md index ad50e60f2f..bc5e640787 100644 --- a/src/core/camera_core/README.md +++ b/src/core/camera_core/README.md @@ -66,4 +66,5 @@ gives these numbers. ## Tests -`tests/cpp/core/camera_core`, run with `ctest -L camera_core`. +`tests/cpp/core/camera_core`, run with `ctest -L camera_core`. They are tagged `unit`, so +`ctest -L unit` runs them too. diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp index e13ed8e051..5d8f7c946d 100644 --- a/tests/cpp/core/camera_core/test_publisher.cpp +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -84,7 +84,7 @@ std::vector sequences(const RecordingSink& sink) } // namespace -TEST_CASE("a window of one always gets the newest frame set", "[camera_core][delivery]") +TEST_CASE("a window of one always gets the newest frame set", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8)); @@ -107,7 +107,7 @@ TEST_CASE("a window of one always gets the newest frame set", "[camera_core][del REQUIRE(sink.events[1].skipped == 1); } -TEST_CASE("a larger window gets every frame set in order", "[camera_core][delivery]") +TEST_CASE("a larger window gets every frame set in order", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8)); @@ -127,7 +127,7 @@ TEST_CASE("a larger window gets every frame set in order", "[camera_core][delive REQUIRE(sink.events[3].skipped == 0); } -TEST_CASE("an in-order reader that falls behind is told what it skipped", "[camera_core][delivery]") +TEST_CASE("an in-order reader that falls behind is told what it skipped", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8)); @@ -151,7 +151,7 @@ TEST_CASE("an in-order reader that falls behind is told what it skipped", "[came REQUIRE(sink.events[3].skipped == 0); } -TEST_CASE("newest order with a larger window skips to the newest frame set", "[camera_core][delivery]") +TEST_CASE("newest order with a larger window skips to the newest frame set", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8)); @@ -171,7 +171,7 @@ TEST_CASE("newest order with a larger window skips to the newest frame set", "[c REQUIRE(sink.events[2].skipped == 1); } -TEST_CASE("a subscription that does not fit the pool is refused", "[camera_core][budget]") +TEST_CASE("a subscription that does not fit the pool is refused", "[unit][camera_core][budget]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(4)); @@ -194,7 +194,7 @@ TEST_CASE("a subscription that does not fit the pool is refused", "[camera_core] REQUIRE_FALSE(publisher.subscribe(second, 99, { .window = 1 })); } -TEST_CASE("the backend reserve counts against the pool", "[camera_core][budget]") +TEST_CASE("the backend reserve counts against the pool", "[unit][camera_core][budget]") { Publisher publisher; // An SDK that keeps four captures in flight, on a pool of eight. @@ -211,7 +211,7 @@ TEST_CASE("the backend reserve counts against the pool", "[camera_core][budget]" REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("4 for the backend")); } -TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", "[camera_core][slots]") +TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", "[unit][camera_core][slots]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(3, 1)); @@ -237,7 +237,7 @@ TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", " REQUIRE(publisher.acquire_slot(group, 0) == 0u); } -TEST_CASE("the slot-free callback fires when the last holder lets go", "[camera_core][slots]") +TEST_CASE("the slot-free callback fires when the last holder lets go", "[unit][camera_core][slots]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(4, 1)); @@ -281,7 +281,7 @@ TEST_CASE("the slot-free callback fires when the last holder lets go", "[camera_ REQUIRE(freed.size() == 2); } -TEST_CASE("unsubscribing returns every frame set the reader held", "[camera_core][slots]") +TEST_CASE("unsubscribing returns every frame set the reader held", "[unit][camera_core][slots]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(8, 1)); @@ -306,7 +306,7 @@ TEST_CASE("unsubscribing returns every frame set the reader held", "[camera_core publisher.release(reader.id, sink.events[0].lease_id); } -TEST_CASE("a frame set that could not be sent is not counted as held", "[camera_core][delivery]") +TEST_CASE("a frame set that could not be sent is not counted as held", "[unit][camera_core][delivery]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(3, 1)); @@ -329,7 +329,7 @@ TEST_CASE("a frame set that could not be sent is not counted as held", "[camera_ REQUIRE(publisher.acquire_slot(group, 0) == 0u); } -TEST_CASE("a restart changes the generation and empties the ring", "[camera_core][lifecycle]") +TEST_CASE("a restart changes the generation and empties the ring", "[unit][camera_core][lifecycle]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(4, 1)); @@ -352,7 +352,7 @@ TEST_CASE("a restart changes the generation and empties the ring", "[camera_core REQUIRE(sink.events[1].sequence == 2); } -TEST_CASE("a group publishes one image per stream", "[camera_core][groups]") +TEST_CASE("a group publishes one image per stream", "[unit][camera_core][groups]") { Publisher publisher; const GroupId group = publisher.add_group(make_group(4, 3, "rig")); @@ -383,7 +383,7 @@ TEST_CASE("a group publishes one image per stream", "[camera_core][groups]") REQUIRE(event.images[2].flags == IMAGE_FLAG_SDK_WARNING); } -TEST_CASE("an encoded stream is described by its codec, not by a pixel format", "[camera_core][groups]") +TEST_CASE("an encoded stream is described by its codec, not by a pixel format", "[unit][camera_core][groups]") { GroupDescription description = make_group(4, 1, "head.enc"); StreamDescription& stream = description.streams[0]; @@ -400,7 +400,7 @@ TEST_CASE("an encoded stream is described by its codec, not by a pixel format", REQUIRE(stored.slot_bytes == 2 * 1024 * 1024); } -TEST_CASE("every frame set carries its timeline", "[camera_core][timeline]") +TEST_CASE("every frame set carries its timeline", "[unit][camera_core][timeline]") { std::int64_t now = 1000; Publisher publisher([&] { return now; }); @@ -428,7 +428,7 @@ TEST_CASE("every frame set carries its timeline", "[camera_core][timeline]") REQUIRE(sink.events[1].timeline.sent_ns == 2500); } -TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") +TEST_CASE("misuse by a backend is rejected", "[unit][camera_core][validation]") { Publisher publisher; REQUIRE_THROWS_AS(publisher.add_group(make_group(4, 0)), std::invalid_argument); @@ -467,7 +467,7 @@ TEST_CASE("misuse by a backend is rejected", "[camera_core][validation]") REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 99), std::invalid_argument); } -TEST_CASE("capture and a reader on different threads keep every frame set in order", "[camera_core][threads]") +TEST_CASE("capture and a reader on different threads keep every frame set in order", "[unit][camera_core][threads]") { // A reader as the endpoint will drive it: events arrive on the capture thread, the // reader's own thread takes them and releases. From 8240723c7869532cc1b856191f65c1772035458a Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 13:54:22 -0800 Subject: [PATCH 6/8] build(camera): link the threads library into the camera_core tests The tests start threads. Without the link, std::thread fails at run time on C libraries that keep threads in a separate library. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- tests/cpp/core/camera_core/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/cpp/core/camera_core/CMakeLists.txt b/tests/cpp/core/camera_core/CMakeLists.txt index aa26bc438f..f74232fd6b 100644 --- a/tests/cpp/core/camera_core/CMakeLists.txt +++ b/tests/cpp/core/camera_core/CMakeLists.txt @@ -3,6 +3,8 @@ cmake_minimum_required(VERSION 3.20) +find_package(Threads REQUIRED) + add_executable(camera_core_tests test_publisher.cpp ) @@ -10,6 +12,7 @@ add_executable(camera_core_tests target_link_libraries(camera_core_tests PRIVATE camera::camera_core Catch2::Catch2WithMain + Threads::Threads ) message(STATUS "camera_core_tests target enabled with Catch2") From 7894a112bdcd8f28a077fa1e64cd25e6b698b545 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:31:03 -0800 Subject: [PATCH 7/8] fix(camera): reject descriptions and images that do not fit their slots add_group now checks that every plane of a pixel stream lies inside the slot (row length against stride, last row against the slot size) and that a pool can hold one frame set on top of the backend's reserve. publish refuses an image larger than its slot. EventSink::send_event is noexcept: it runs on the capture path with the publisher's lock held, and a throwing sink would have left a slot held. A group with a slot-free callback hands out its slots with claim_slot only, so a slot cannot be taken again before its callback has run. Adds agent notes for the module. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/camera_core/AGENTS.md | 16 ++++ src/core/camera_core/README.md | 3 +- .../cpp/inc/camera_core/publisher.hpp | 11 ++- src/core/camera_core/cpp/publisher.cpp | 56 ++++++++++++- tests/cpp/core/camera_core/test_publisher.cpp | 82 ++++++++++++++++++- 5 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 src/core/camera_core/AGENTS.md diff --git a/src/core/camera_core/AGENTS.md b/src/core/camera_core/AGENTS.md new file mode 100644 index 0000000000..4fb79450a3 --- /dev/null +++ b/src/core/camera_core/AGENTS.md @@ -0,0 +1,16 @@ + + +# Agent notes — `camera_core` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) for the terms (stream, group, window, order, lease) before changing behaviour. + +- **Standard library only.** No slot memory, sockets, logging or platform headers here; that keeps the module building and tested on every platform. Memory and connections live in `camera_ipc`. +- **One type or field per concept.** A codec is not a pixel format, each plane has its own layout, counters are per group. Do not give a field a second meaning, and do not carry a type over from a prototype without modelling it for this interface. +- **Nothing ahead of its user.** A memory kind, pixel format or flag is added in the change that brings the pool or backend that needs it. +- **`EventSink::send_event` runs on the capture path with the publisher's lock held.** It is `noexcept` and must not block or call back into the publisher. Callbacks to a backend (`SlotFreeCallback`) run outside the lock. +- **Reject bad input where it enters.** `add_group` and `publish` refuse a description or an image that a reader could run off the end of a slot with; readers do not re-check. diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md index bc5e640787..cc0d4fbe2f 100644 --- a/src/core/camera_core/README.md +++ b/src/core/camera_core/README.md @@ -43,7 +43,8 @@ publisher.publish(group, images, timeline); ``` For the second way, `set_slot_free_callback` tells the backend the moment a slot is held by -nobody, so the buffer can go back to the SDK. +nobody, so the buffer can go back to the SDK. A group uses one way or the other: once it has +that callback, its slots are handed out with `claim_slot` only. A reader subscribes with a window and an order, receives `FrameEvent`s through an `EventSink`, and returns each one with `release`. Events carry slot indices, the frame set's timeline and the diff --git a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp index 6f98df88cb..44ed310fec 100644 --- a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp +++ b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp @@ -35,10 +35,10 @@ class EventSink /** * @brief Hand one event to the reader. Called on the capture path with the publisher's - * lock held, so it must not block and must not call back into the publisher. + * lock held, so it must not block, throw or call back into the publisher. * @return false when the event could not be handed over; the reader then gets a later one. */ - virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) = 0; + virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) noexcept = 0; }; // Counters of one group since it was added. @@ -93,6 +93,8 @@ class Publisher /** * @brief Take a free slot to write the next image into. * @return The slot, or nothing when every slot is in use (the frame should be dropped). + * @throws std::logic_error for a group that has a slot-free callback: its backend owns the + * pool and names slots with claim_slot. */ std::optional acquire_slot(GroupId group, std::size_t stream); @@ -110,7 +112,8 @@ class Publisher /** * @brief Be told when a published slot is no longer held by the ring or any reader, so a * buffer the SDK owns can go back to it at once. Called outside the publisher's lock, on - * the thread whose call released the last hold. + * the thread whose call released the last hold. A group with this callback hands out its + * slots with claim_slot only, so a slot cannot be taken again before its callback has run. */ void set_slot_free_callback(GroupId group, SlotFreeCallback callback); @@ -119,7 +122,7 @@ class Publisher * acquire_slot or claim_slot. * @param timeline capture_ns and sdk_ready_ns from the backend; the rest is filled in here. * @return The frame set's sequence number. - * @throws std::invalid_argument if the images do not match the group. + * @throws std::invalid_argument if the images do not match the group or do not fit its slots. */ std::uint64_t publish(GroupId group, std::span images, const FrameTimeline& timeline); diff --git a/src/core/camera_core/cpp/publisher.cpp b/src/core/camera_core/cpp/publisher.cpp index b3b3ab0766..5222ae681a 100644 --- a/src/core/camera_core/cpp/publisher.cpp +++ b/src/core/camera_core/cpp/publisher.cpp @@ -29,7 +29,33 @@ DeliveryOrder resolve_order(const ReaderOptions& options) return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; } -// A pixel stream names every plane of its format; an encoded stream has none. +// The bytes one row of a plane holds and the number of its rows. +struct PlaneSize +{ + std::uint64_t row_bytes = 0; + std::uint64_t rows = 0; +}; + +PlaneSize plane_size_of(const StreamDescription& stream, std::uint32_t plane) +{ + const std::uint64_t width = stream.width; + const std::uint64_t height = stream.height; + switch (stream.pixel_format) + { + case PixelFormat::BGRA8: + case PixelFormat::RGBA8: + return { width * 4, height }; + case PixelFormat::GRAY8: + return { width, height }; + case PixelFormat::NV12: + // Luma at full size, then one interleaved chroma pair per two by two pixels. + return plane == 0 ? PlaneSize{ width, height } : PlaneSize{ (width + 1) / 2 * 2, (height + 1) / 2 }; + } + return {}; +} + +// A pixel stream names every plane of its format and every plane lies inside the slot; an +// encoded stream has no planes. bool layout_is_valid(const StreamDescription& stream) { if (stream.is_encoded()) @@ -42,7 +68,11 @@ bool layout_is_valid(const StreamDescription& stream) } for (std::uint32_t i = 0; i < stream.plane_count; ++i) { - if (stream.planes[i].stride_bytes == 0) + const PlaneSize size = plane_size_of(stream, i); + const std::uint64_t stride = stream.planes[i].stride_bytes; + // 64-bit sums of 32-bit fields, so nothing here can wrap. + const std::uint64_t end = stream.planes[i].offset_bytes + stride * (size.rows - 1) + size.row_bytes; + if (size.rows == 0 || stride < size.row_bytes || end > stream.slot_bytes) { return false; } @@ -86,12 +116,19 @@ GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t } if (!layout_is_valid(stream)) { - throw std::invalid_argument("camera stream '" + stream.id + "' needs a size and one stride per plane of " + - "its pixel format, or no planes at all when it is encoded"); + throw std::invalid_argument("camera stream '" + stream.id + "' needs a size and every plane of its " + + "pixel format inside the slot, or no planes at all when it is encoded"); } group->slot_count = std::min(group->slot_count, stream.slot_count); group->slots.emplace_back(stream.slot_count); } + // Before any reader, the ring keeps one frame set and the backend keeps its reserve. + if (group->slot_count <= backend_reserve) + { + throw std::invalid_argument("camera group '" + description.id + "' has pools of " + + std::to_string(group->slot_count) + " slots, too few for one frame set and a " + + "backend reserve of " + std::to_string(backend_reserve)); + } std::lock_guard lock(m_mutex); for (const auto& existing : m_groups) @@ -109,6 +146,11 @@ std::optional Publisher::acquire_slot(GroupId group, std::size_t { std::lock_guard lock(m_mutex); Group& target = group_at(group); + if (target.on_free) + { + throw std::logic_error("camera group '" + target.description.id + + "' has a slot-free callback, so its backend names slots with claim_slot"); + } std::vector& slots = target.slots.at(stream); for (std::uint32_t i = 0; i < slots.size(); ++i) { @@ -169,6 +211,12 @@ std::uint64_t Publisher::publish(GroupId group, std::span image throw std::invalid_argument("camera slot " + std::to_string(images[i].slot) + " was not acquired before it was published"); } + if (images[i].bytes_used > target.description.streams[i].slot_bytes) + { + throw std::invalid_argument("camera stream '" + target.description.streams[i].id + "' published " + + std::to_string(images[i].bytes_used) + " bytes into a slot of " + + std::to_string(target.description.streams[i].slot_bytes)); + } } FrameSet set; diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp index 5d8f7c946d..f3bc370d4a 100644 --- a/tests/cpp/core/camera_core/test_publisher.cpp +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -24,7 +24,7 @@ namespace // A reader that keeps every event it is sent. struct RecordingSink : EventSink { - bool send_event(SubscriptionId, const FrameEvent& event) override + bool send_event(SubscriptionId, const FrameEvent& event) noexcept override { if (!accept) { @@ -437,6 +437,11 @@ TEST_CASE("misuse by a backend is rejected", "[unit][camera_core][validation]") REQUIRE_THROWS_AS(publisher.add_group(make_group(MAX_SLOTS_PER_STREAM + 1)), std::invalid_argument); REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 0), std::invalid_argument); + // A pool holds at least one frame set for the ring on top of the backend's reserve. + REQUIRE_THROWS_AS(publisher.add_group(make_group(1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 4), std::invalid_argument); + REQUIRE_NOTHROW(publisher.add_group(make_group(4, 2, "tight"), 3)); + // A pixel stream names every plane of its format; an encoded stream has none. GroupDescription nv12 = make_group(4, 1, "nv12"); nv12.streams[0].pixel_format = PixelFormat::NV12; @@ -460,20 +465,93 @@ TEST_CASE("misuse by a backend is rejected", "[unit][camera_core][validation]") images[0].slot = *publisher.acquire_slot(group, 0); REQUIRE_THROWS_AS(publisher.publish(group, { images.data(), 1 }, {}), std::invalid_argument); REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); // stream 1 not acquired + + // An image cannot be larger than the slot it sits in. + images[1].slot = *publisher.acquire_slot(group, 1); + images[1].bytes_used = publisher.group_description(group).streams[1].slot_bytes + 1; + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); REQUIRE(publisher.stats(group).published == 0); + images[1].bytes_used = 0; + REQUIRE(publisher.publish(group, images, {}) == 1); // the slots were still the backend's to publish REQUIRE_THROWS_AS(publisher.abandon_slot(group, 1, 0), std::invalid_argument); REQUIRE_THROWS_AS(publisher.acquire_slot(99, 0), std::out_of_range); REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 99), std::invalid_argument); } +TEST_CASE("a plane that does not lie inside the slot is rejected", "[unit][camera_core][validation]") +{ + Publisher publisher; + const auto with = [](auto&& change) + { + GroupDescription group = make_group(4, 1); + change(group.streams[0]); + return group; + }; + + // A row is as long as the format needs, and the last row ends inside the slot. + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].stride_bytes = 640 * 4 - 1; })), + std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].offset_bytes = 1; })), + std::invalid_argument); + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.slot_bytes -= 1; })), std::invalid_argument); + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].stride_bytes = 0xFFFFFFFF; })), + std::invalid_argument); + + // Padding between rows is fine as long as the last row still fits. + const auto padded = [](StreamDescription& stream) + { + stream.planes[0].stride_bytes = 4096; + stream.slot_bytes = 4096 * 399 + 640 * 4; + }; + REQUIRE_NOTHROW(publisher.add_group(with(padded))); + + // NV12 with odd sides: chroma rows cover the rounded-up half, and both planes must fit. + const auto nv12 = [](std::uint64_t slot_bytes) + { + return [slot_bytes](StreamDescription& stream) + { + stream.id = "nv12"; + stream.pixel_format = PixelFormat::NV12; + stream.width = 5; + stream.height = 3; + stream.plane_count = 2; + stream.planes[0] = { .offset_bytes = 0, .stride_bytes = 8 }; + stream.planes[1] = { .offset_bytes = 24, .stride_bytes = 8 }; + stream.slot_bytes = slot_bytes; + }; + }; + REQUIRE_THROWS_AS(publisher.add_group(with(nv12(24 + 8 + 5))), std::invalid_argument); // chroma row needs 6 + GroupDescription odd = with(nv12(24 + 8 + 6)); + odd.id = "odd"; + REQUIRE_NOTHROW(publisher.add_group(odd)); +} + +TEST_CASE("a group with a slot-free callback hands out slots by claim only", "[unit][camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + REQUIRE(publisher.acquire_slot(group, 0) == 0u); + publisher.abandon_slot(group, 0, 0); + + publisher.set_slot_free_callback(group, [](std::size_t, std::uint32_t) {}); + REQUIRE_THROWS_AS(publisher.acquire_slot(group, 0), std::logic_error); + REQUIRE_NOTHROW(publisher.claim_slot(group, 0, 0)); + + publisher.set_slot_free_callback(group, {}); + REQUIRE(publisher.acquire_slot(group, 0) == 1u); +} + TEST_CASE("capture and a reader on different threads keep every frame set in order", "[unit][camera_core][threads]") { // A reader as the endpoint will drive it: events arrive on the capture thread, the // reader's own thread takes them and releases. struct QueueSink : EventSink { - bool send_event(SubscriptionId, const FrameEvent& event) override + bool send_event(SubscriptionId, const FrameEvent& event) noexcept override { { std::lock_guard lock(mutex); From 0bd42c220b6c42042b025386bfc59335a96935a6 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:31:40 -0800 Subject: [PATCH 8/8] feat(camera): add camera_ipc with a shared host-memory pool HostPool is one stream's slots in shared host memory: a single anonymous memory file with the slots a page-aligned stride apart. The owner maps it writable and then seals it, so a process that is sent the descriptor can map it for reading only and cannot resize it. HostPoolView is that reader's mapping; both sides derive the layout from the slot count and slot size. UniqueFd owns a file descriptor. An anonymous sealed file is used rather than named shared memory: there is no name to collide or to clean up after a crash, and only a process that was handed the descriptor can map it. Linux only, so the module and its tests are added under a platform check. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/CMakeLists.txt | 5 + src/core/camera_core/README.md | 2 +- src/core/camera_ipc/AGENTS.md | 15 ++ src/core/camera_ipc/CMakeLists.txt | 6 + src/core/camera_ipc/README.md | 43 +++++ src/core/camera_ipc/cpp/CMakeLists.txt | 16 ++ src/core/camera_ipc/cpp/host_pool.cpp | 162 ++++++++++++++++++ .../cpp/inc/camera_ipc/host_pool.hpp | 109 ++++++++++++ .../cpp/inc/camera_ipc/unique_fd.hpp | 73 ++++++++ tests/cpp/core/CMakeLists.txt | 4 + tests/cpp/core/camera_ipc/CMakeLists.txt | 16 ++ tests/cpp/core/camera_ipc/test_host_pool.cpp | 123 +++++++++++++ 12 files changed, 573 insertions(+), 1 deletion(-) create mode 100644 src/core/camera_ipc/AGENTS.md create mode 100644 src/core/camera_ipc/CMakeLists.txt create mode 100644 src/core/camera_ipc/README.md create mode 100644 src/core/camera_ipc/cpp/CMakeLists.txt create mode 100644 src/core/camera_ipc/cpp/host_pool.cpp create mode 100644 src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp create mode 100644 src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp create mode 100644 tests/cpp/core/camera_ipc/CMakeLists.txt create mode 100644 tests/cpp/core/camera_ipc/test_host_pool.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 3393c107b0..a2d49ed641 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -22,6 +22,11 @@ add_subdirectory(plugin_manager) # Camera capture core (standard library only) add_subdirectory(camera_core) +# Camera frames between processes (shared host memory) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_subdirectory(camera_ipc) +endif() + # Build OXR library (depends on oxr_utils for types) add_subdirectory(oxr) diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md index cc0d4fbe2f..d601eaa7a9 100644 --- a/src/core/camera_core/README.md +++ b/src/core/camera_core/README.md @@ -11,7 +11,7 @@ encoders and recorders in other processes read those buffers in place. This module is the part with no memory and no sockets in it: it decides which pool slot holds which frame set and who may still read it. It uses the standard library only. The shared memory -behind the slots and the connection to other processes live in separate modules. +behind the slots and the connection to other processes live in [`camera_ipc`](../camera_ipc). ## Terms diff --git a/src/core/camera_ipc/AGENTS.md b/src/core/camera_ipc/AGENTS.md new file mode 100644 index 0000000000..124e3963a0 --- /dev/null +++ b/src/core/camera_ipc/AGENTS.md @@ -0,0 +1,15 @@ + + +# Agent notes — `camera_ipc` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) first; it explains why frames cross between processes this way. + +- **Linux only.** The module and its tests are added under `CMAKE_SYSTEM_NAME STREQUAL "Linux"`. Platform calls stay here and out of `camera_core`. +- **Readers cannot write.** A pool is sealed before its descriptor leaves the process, and a reader maps it read-only. Do not hand a reader a writable mapping. +- **Derive what both sides can compute.** A pool's layout follows from the slot count and slot size in the stream's description; do not send it separately. +- **Tests need no camera and no GPU**, and carry the `unit` tag. diff --git a/src/core/camera_ipc/CMakeLists.txt b/src/core/camera_ipc/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_ipc/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_ipc/README.md b/src/core/camera_ipc/README.md new file mode 100644 index 0000000000..f97feaa046 --- /dev/null +++ b/src/core/camera_ipc/README.md @@ -0,0 +1,43 @@ + + +# Camera interface: sharing frames between processes + +The process that owns the cameras and the processes that read them are on one machine. This +module is what crosses between them. Linux only. + +[`camera_core`](../camera_core) decides which slot holds which frame; this module provides the +memory behind the slots. + +## Host pool + +`HostPool` is one stream's slots in host memory: a single anonymous file in memory +(`memfd_create`), with the slots laid out one page-aligned stride apart. The owner maps it +writable, then seals it. A reader is sent the descriptor and maps it with `HostPoolView`, read +only. Both sides derive the layout from the slot count and slot size, which the stream's +description carries. + +```cpp +HostPool pool(stream.slot_count, stream.slot_bytes, "camera.left"); +std::span pixels = pool.slot(index); // the owner writes here + +HostPoolView view(std::move(received_descriptor), stream.slot_count, stream.slot_bytes); +std::span image = view.slot(index); // a reader sees the same bytes +``` + +Why an anonymous sealed file and not named shared memory (`shm_open`): + +- **Nothing to clean up.** There is no name to collide with another instance and no file left in + `/dev/shm` after a crash. The memory is freed when the last process that holds it goes away. +- **Readers cannot damage it.** The seals stop any process that receives the descriptor from + mapping it writable, writing to it, or resizing it. One misbehaving reader cannot change what + the others see, and cannot make the owner fault by shrinking the file under it. +- **Access follows the connection.** Only a process that was handed the descriptor can map the + pool. + +## Tests + +`tests/cpp/core/camera_ipc`, run with `ctest -L camera_ipc`. They are tagged `unit`, so +`ctest -L unit` runs them too. diff --git a/src/core/camera_ipc/cpp/CMakeLists.txt b/src/core/camera_ipc/cpp/CMakeLists.txt new file mode 100644 index 0000000000..a17f6255fa --- /dev/null +++ b/src/core/camera_ipc/cpp/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# How camera frames cross between processes on one machine: pools of shared host memory that +# readers can only map read-only. Linux only. +add_library(camera_ipc STATIC + host_pool.cpp +) + +target_include_directories(camera_ipc INTERFACE + $ +) + +add_library(camera::camera_ipc ALIAS camera_ipc) diff --git a/src/core/camera_ipc/cpp/host_pool.cpp b/src/core/camera_ipc/cpp/host_pool.cpp new file mode 100644 index 0000000000..7df3ae0776 --- /dev/null +++ b/src/core/camera_ipc/cpp/host_pool.cpp @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_ipc/host_pool.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// Older C library headers lack this seal although the kernel has it. +#ifndef F_SEAL_FUTURE_WRITE +# define F_SEAL_FUTURE_WRITE 0x0010 +#endif + +namespace core::camera +{ + +namespace +{ + +[[noreturn]] void throw_errno(const std::string& what) +{ + throw std::system_error(errno, std::generic_category(), what); +} + +// Slots start on page boundaries. Both sides derive the layout from the slot size alone. +std::uint64_t slot_stride_for(std::uint64_t slot_bytes) +{ + const auto page = static_cast(::sysconf(_SC_PAGESIZE)); + return (slot_bytes + page - 1) / page * page; +} + +std::uint64_t pool_bytes_for(std::uint32_t slot_count, std::uint64_t slot_stride) +{ + const auto limit = static_cast(std::numeric_limits::max()); + if (slot_count == 0 || slot_stride == 0 || slot_stride > limit / slot_count) + { + throw std::invalid_argument( + "a host pool needs at least one slot of at least one byte, and a size a file can have"); + } + return slot_stride * slot_count; +} + +} // namespace + +HostPool::HostPool(std::uint32_t slot_count, std::uint64_t slot_bytes, const std::string& name) + : m_slot_count(slot_count), m_slot_bytes(slot_bytes), m_slot_stride(slot_stride_for(slot_bytes)) +{ + const std::uint64_t pool_bytes = pool_bytes_for(slot_count, m_slot_stride); + m_fd.reset(::memfd_create(name.c_str(), MFD_CLOEXEC | MFD_ALLOW_SEALING)); + if (!m_fd.valid()) + { + throw_errno("host pool '" + name + "': memfd_create"); + } + if (::ftruncate(m_fd.get(), static_cast(pool_bytes)) != 0) + { + throw_errno("host pool '" + name + "': sizing the pool"); + } + void* memory = ::mmap(nullptr, pool_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, m_fd.get(), 0); + if (memory == MAP_FAILED) + { + throw_errno("host pool '" + name + "': mapping the pool"); + } + // From here on nobody can resize the pool or map it writable; the mapping above stays. + if (::fcntl(m_fd.get(), F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_FUTURE_WRITE | F_SEAL_SEAL) != 0) + { + const int error = errno; + ::munmap(memory, pool_bytes); + throw std::system_error(error, std::generic_category(), "host pool '" + name + "': sealing the pool"); + } + m_memory = static_cast(memory); +} + +HostPool::~HostPool() +{ + ::munmap(m_memory, m_slot_stride * m_slot_count); +} + +std::span HostPool::slot(std::uint32_t index) +{ + if (index >= m_slot_count) + { + throw std::out_of_range("host pool has no slot " + std::to_string(index)); + } + return { m_memory + m_slot_stride * index, m_slot_bytes }; +} + +HostPoolView::HostPoolView(UniqueFd descriptor, std::uint32_t slot_count, std::uint64_t slot_bytes) + : m_slot_count(slot_count), m_slot_bytes(slot_bytes), m_slot_stride(slot_stride_for(slot_bytes)) +{ + const std::uint64_t pool_bytes = pool_bytes_for(slot_count, m_slot_stride); + struct stat info = {}; + if (::fstat(descriptor.get(), &info) != 0) + { + throw_errno("host pool view: inspecting the pool"); + } + // Mapping more than the descriptor holds would fault on first access instead of failing here. + if (static_cast(info.st_size) < pool_bytes) + { + throw std::invalid_argument("host pool view: the descriptor holds less than the pool needs"); + } + void* memory = ::mmap(nullptr, pool_bytes, PROT_READ, MAP_SHARED, descriptor.get(), 0); + if (memory == MAP_FAILED) + { + throw_errno("host pool view: mapping the pool"); + } + m_memory = static_cast(memory); +} + +HostPoolView::~HostPoolView() +{ + unmap(); +} + +HostPoolView::HostPoolView(HostPoolView&& other) noexcept + : m_memory(std::exchange(other.m_memory, nullptr)), + m_slot_count(std::exchange(other.m_slot_count, 0)), + m_slot_bytes(other.m_slot_bytes), + m_slot_stride(other.m_slot_stride) +{ +} + +HostPoolView& HostPoolView::operator=(HostPoolView&& other) noexcept +{ + if (this != &other) + { + unmap(); + m_memory = std::exchange(other.m_memory, nullptr); + m_slot_count = std::exchange(other.m_slot_count, 0); + m_slot_bytes = other.m_slot_bytes; + m_slot_stride = other.m_slot_stride; + } + return *this; +} + +std::span HostPoolView::slot(std::uint32_t index) const +{ + if (index >= m_slot_count) + { + throw std::out_of_range("host pool view has no slot " + std::to_string(index)); + } + return { m_memory + m_slot_stride * index, m_slot_bytes }; +} + +void HostPoolView::unmap() +{ + if (m_memory != nullptr) + { + ::munmap(const_cast(m_memory), m_slot_stride * m_slot_count); + m_memory = nullptr; + m_slot_count = 0; + } +} + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp b/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp new file mode 100644 index 0000000000..54694f1436 --- /dev/null +++ b/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "unique_fd.hpp" + +#include +#include +#include +#include + +namespace core::camera +{ + +/** + * @brief A pool of equally sized slots in one block of shared host memory. + * + * The process that creates the pool writes the slots. Any process that is sent the pool's + * descriptor can map it, but only for reading: the memory is sealed against new writable + * mappings, writes and resizing. + */ +class HostPool +{ +public: + /** + * @param name Shows up in /proc for debugging; it does not have to be unique. + * @throws std::invalid_argument for an empty pool. + * @throws std::system_error if the memory cannot be allocated or sealed. + */ + HostPool(std::uint32_t slot_count, std::uint64_t slot_bytes, const std::string& name); + ~HostPool(); + + HostPool(const HostPool&) = delete; + HostPool& operator=(const HostPool&) = delete; + + std::uint32_t slot_count() const + { + return m_slot_count; + } + + std::uint64_t slot_bytes() const + { + return m_slot_bytes; + } + + /** + * @brief The writable memory of one slot. + * @throws std::out_of_range for a slot the pool does not have. + */ + std::span slot(std::uint32_t index); + + /** + * @brief The pool's descriptor, to send to another process. The pool keeps owning it. + */ + int descriptor() const + { + return m_fd.get(); + } + +private: + UniqueFd m_fd; + std::byte* m_memory = nullptr; + std::uint32_t m_slot_count = 0; + std::uint64_t m_slot_bytes = 0; + std::uint64_t m_slot_stride = 0; // from the start of one slot to the start of the next +}; + +/** + * @brief Another process's read-only mapping of a HostPool, made from the descriptor it was sent. + */ +class HostPoolView +{ +public: + /** + * @param descriptor Of the pool. It is closed once the pool is mapped. + * @param slot_count, slot_bytes As the pool was created with. + * @throws std::system_error if the descriptor cannot be mapped. + * @throws std::invalid_argument if the descriptor holds less than the pool needs. + */ + HostPoolView(UniqueFd descriptor, std::uint32_t slot_count, std::uint64_t slot_bytes); + ~HostPoolView(); + + HostPoolView(HostPoolView&& other) noexcept; + HostPoolView& operator=(HostPoolView&& other) noexcept; + HostPoolView(const HostPoolView&) = delete; + HostPoolView& operator=(const HostPoolView&) = delete; + + std::uint32_t slot_count() const + { + return m_slot_count; + } + + /** + * @brief The memory of one slot. + * @throws std::out_of_range for a slot the pool does not have. + */ + std::span slot(std::uint32_t index) const; + +private: + void unmap(); + + const std::byte* m_memory = nullptr; + std::uint32_t m_slot_count = 0; + std::uint64_t m_slot_bytes = 0; + std::uint64_t m_slot_stride = 0; +}; + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp b/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp new file mode 100644 index 0000000000..387bf46515 --- /dev/null +++ b/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace core::camera +{ + +// Owns one file descriptor and closes it. +class UniqueFd +{ +public: + UniqueFd() = default; + + explicit UniqueFd(int fd) : m_fd(fd) + { + } + + UniqueFd(UniqueFd&& other) noexcept : m_fd(other.release()) + { + } + + UniqueFd& operator=(UniqueFd&& other) noexcept + { + if (this != &other) + { + reset(other.release()); + } + return *this; + } + + UniqueFd(const UniqueFd&) = delete; + UniqueFd& operator=(const UniqueFd&) = delete; + + ~UniqueFd() + { + reset(); + } + + int get() const + { + return m_fd; + } + + bool valid() const + { + return m_fd >= 0; + } + + // Gives the descriptor up without closing it. + int release() + { + const int fd = m_fd; + m_fd = -1; + return fd; + } + + void reset(int fd = -1) + { + if (m_fd >= 0) + { + ::close(m_fd); + } + m_fd = fd; + } + +private: + int m_fd = -1; +}; + +} // namespace core::camera diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index e21545f14f..705bead328 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -15,6 +15,10 @@ add_subdirectory(log_bridge) add_subdirectory(plugin_manager) add_subdirectory(camera_core) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_subdirectory(camera_ipc) +endif() + if(BUILD_OVXR_CAMERA_QUADS) add_subdirectory(ovxr_pip_layer) endif() diff --git a/tests/cpp/core/camera_ipc/CMakeLists.txt b/tests/cpp/core/camera_ipc/CMakeLists.txt new file mode 100644 index 0000000000..b26697a7fe --- /dev/null +++ b/tests/cpp/core/camera_ipc/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_ipc_tests + test_host_pool.cpp +) + +target_link_libraries(camera_ipc_tests PRIVATE + camera::camera_ipc + Catch2::Catch2WithMain +) + +message(STATUS "camera_ipc_tests target enabled with Catch2") +catch_discover_tests(camera_ipc_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_ipc/test_host_pool.cpp b/tests/cpp/core/camera_ipc/test_host_pool.cpp new file mode 100644 index 0000000000..6b2e4fca94 --- /dev/null +++ b/tests/cpp/core/camera_ipc/test_host_pool.cpp @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +// Not a whole number of pages, so slots are padded apart. +constexpr std::uint64_t SLOT_BYTES = 5000; + +// What another process ends up with after the pool's descriptor was sent to it. +UniqueFd duplicate_descriptor(const HostPool& pool) +{ + UniqueFd result(::fcntl(pool.descriptor(), F_DUPFD_CLOEXEC, 0)); + REQUIRE(result.valid()); + return result; +} + +} // namespace + +TEST_CASE("a view sees what the owner writes", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(3, SLOT_BYTES, "camera_ipc_test"); + REQUIRE(pool.slot_count() == 3); + REQUIRE(pool.slot_bytes() == SLOT_BYTES); + + const HostPoolView view(duplicate_descriptor(pool), 3, SLOT_BYTES); + REQUIRE(view.slot_count() == 3); + + // Slots start zeroed and are separate from each other. + const auto is_zero = [](std::byte value) { return value == std::byte{ 0 }; }; + REQUIRE(std::all_of(view.slot(0).begin(), view.slot(0).end(), is_zero)); + + std::fill(pool.slot(1).begin(), pool.slot(1).end(), std::byte{ 0x5a }); + REQUIRE(view.slot(1).size() == SLOT_BYTES); + REQUIRE(view.slot(1).front() == std::byte{ 0x5a }); + REQUIRE(view.slot(1).back() == std::byte{ 0x5a }); + REQUIRE(std::all_of(view.slot(0).begin(), view.slot(0).end(), is_zero)); + REQUIRE(std::all_of(view.slot(2).begin(), view.slot(2).end(), is_zero)); + + // The mapping is live: a later write shows up without mapping again. + pool.slot(2)[100] = std::byte{ 0x11 }; + REQUIRE(view.slot(2)[100] == std::byte{ 0x11 }); +} + +TEST_CASE("slots start on page boundaries", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + const auto page = static_cast(::sysconf(_SC_PAGESIZE)); + REQUIRE(reinterpret_cast(pool.slot(0).data()) % page == 0); + REQUIRE(reinterpret_cast(pool.slot(1).data()) % page == 0); + REQUIRE(pool.slot(1).data() >= pool.slot(0).data() + SLOT_BYTES); +} + +TEST_CASE("a process that is sent the pool cannot write to it or resize it", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(1, SLOT_BYTES, "camera_ipc_test"); + const UniqueFd received = duplicate_descriptor(pool); + + void* writable = ::mmap(nullptr, SLOT_BYTES, PROT_READ | PROT_WRITE, MAP_SHARED, received.get(), 0); + REQUIRE(writable == MAP_FAILED); + REQUIRE(errno == EPERM); + + const char byte = 1; + REQUIRE(::write(received.get(), &byte, 1) == -1); + REQUIRE(::ftruncate(received.get(), 16) == -1); + REQUIRE(::ftruncate(received.get(), static_cast(16 * SLOT_BYTES)) == -1); + + // The owner still can. + pool.slot(0)[0] = std::byte{ 0x7f }; + const HostPoolView view(duplicate_descriptor(pool), 1, SLOT_BYTES); + REQUIRE(view.slot(0)[0] == std::byte{ 0x7f }); +} + +TEST_CASE("a view outlives the pool and can be moved", "[unit][camera_ipc][host_pool]") +{ + std::vector views; + { + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + pool.slot(1)[7] = std::byte{ 0x42 }; + views.emplace_back(duplicate_descriptor(pool), 2, SLOT_BYTES); + } + // The memory lives as long as anything maps it. + HostPoolView moved = std::move(views.front()); + REQUIRE(views.front().slot_count() == 0); + REQUIRE(moved.slot_count() == 2); + REQUIRE(moved.slot(1)[7] == std::byte{ 0x42 }); +} + +TEST_CASE("misuse of a host pool is rejected", "[unit][camera_ipc][host_pool]") +{ + REQUIRE_THROWS_AS(HostPool(0, SLOT_BYTES, "camera_ipc_test"), std::invalid_argument); + REQUIRE_THROWS_AS(HostPool(1, 0, "camera_ipc_test"), std::invalid_argument); + + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + REQUIRE_THROWS_AS(pool.slot(2), std::out_of_range); + + const HostPoolView view(duplicate_descriptor(pool), 2, SLOT_BYTES); + REQUIRE_THROWS_AS(view.slot(2), std::out_of_range); + + // A descriptor that holds less than the pool claims would fault on access, so it is refused. + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 3, SLOT_BYTES), std::invalid_argument); + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 2, 4 * SLOT_BYTES), std::invalid_argument); + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 0, SLOT_BYTES), std::invalid_argument); + + REQUIRE_THROWS_AS(HostPoolView(UniqueFd(-1), 2, SLOT_BYTES), std::system_error); +}