diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 9cc4772373..3a606f2e8b 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -19,6 +19,17 @@ add_subdirectory(oxr_utils) # Build Plugin Manager add_subdirectory(plugin_manager) +# Camera capture core (standard library only) +add_subdirectory(camera_core) + +# Camera frames and messages between processes (shared host memory, descriptor passing) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_subdirectory(camera_ipc) + add_subdirectory(camera_wire) + add_subdirectory(camera_endpoint) + add_subdirectory(camera_reader) +endif() + # Build OXR library (depends on oxr_utils for types) add_subdirectory(oxr) diff --git a/src/core/camera_core/AGENTS.md b/src/core/camera_core/AGENTS.md new file mode 100644 index 0000000000..4fb79450a3 --- /dev/null +++ b/src/core/camera_core/AGENTS.md @@ -0,0 +1,16 @@ + + +# Agent notes — `camera_core` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) for the terms (stream, group, window, order, lease) before changing behaviour. + +- **Standard library only.** No slot memory, sockets, logging or platform headers here; that keeps the module building and tested on every platform. Memory and connections live in `camera_ipc`. +- **One type or field per concept.** A codec is not a pixel format, each plane has its own layout, counters are per group. Do not give a field a second meaning, and do not carry a type over from a prototype without modelling it for this interface. +- **Nothing ahead of its user.** A memory kind, pixel format or flag is added in the change that brings the pool or backend that needs it. +- **`EventSink::send_event` runs on the capture path with the publisher's lock held.** It is `noexcept` and must not block or call back into the publisher. Callbacks to a backend (`SlotFreeCallback`) run outside the lock. +- **Reject bad input where it enters.** `add_group` and `publish` refuse a description or an image that a reader could run off the end of a slot with; readers do not re-check. diff --git a/src/core/camera_core/CMakeLists.txt b/src/core/camera_core/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_core/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_core/README.md b/src/core/camera_core/README.md new file mode 100644 index 0000000000..d601eaa7a9 --- /dev/null +++ b/src/core/camera_core/README.md @@ -0,0 +1,71 @@ + + +# Camera interface: core + +The camera interface is the one place Isaac Capture applications get camera frames from. One +process owns the cameras and keeps their frames in fixed pools of shared buffers; viewers, +encoders and recorders in other processes read those buffers in place. + +This module is the part with no memory and no sockets in it: it decides which pool slot holds +which frame set and who may still read it. It uses the standard library only. The shared memory +behind the slots and the connection to other processes live in [`camera_ipc`](../camera_ipc). + +## Terms + +- **Stream**: one sensor's images, or one encoded video stream. It has a pool of **slots**; a + slot holds one image, or one access unit when the stream is encoded. +- **Group**: one or more streams captured together, such as a stereo pair or a triggered rig. + A group publishes one **frame set** at a time: one image per stream, under one sequence number. +- **Window**: how many frame sets are kept for a reader at a time, unread and held together. +- **Order**: `NEWEST` hands the reader the latest frame set each time it has room and skips the + rest (a viewer, a live encoder). `IN_ORDER` hands over every frame set in sequence (a recorder). + The default is `NEWEST` for a window of one and `IN_ORDER` otherwise. +- **Lease**: a reader's hold on a frame set. Its slots are not reused until the reader releases + it or goes away. +- **Generation**: changes when a camera restarts, so a reader can tell a restart from a skip. + +## How frames move + +A backend gets a slot in one of two ways, then publishes: + +```cpp +// The backend writes the image itself. +auto slot = publisher.acquire_slot(group, stream); // nothing: every slot is in use, drop the frame +// ... write the pixels into that slot ... +publisher.publish(group, images, timeline); + +// The camera SDK owns the buffers and picked one. +publisher.claim_slot(group, stream, slot_the_sdk_filled); +publisher.publish(group, images, timeline); +``` + +For the second way, `set_slot_free_callback` tells the backend the moment a slot is held by +nobody, so the buffer can go back to the SDK. A group uses one way or the other: once it has +that callback, its slots are handed out with `claim_slot` only. + +A reader subscribes with a window and an order, receives `FrameEvent`s through an `EventSink`, +and returns each one with `release`. Events carry slot indices, the frame set's timeline and the +number of frame sets the reader skipped. A slow reader skips; it never slows capture or another +reader, and no call blocks on a reader. + +## Pool budget + +A slot is in use while the ring of recent frame sets or any lease refers to it, so a subscription +is accepted only when the pool can cover it: + +```text +slots >= ring history + sum of reader windows + backend reserve +``` + +Ring history is one frame set, or the largest window among in-order readers. The backend reserve +is the number of slots the backend needs for itself: one to write ahead, more for an SDK that +keeps several captures in flight. A subscription that does not fit is refused with a message that +gives these numbers. + +## Tests + +`tests/cpp/core/camera_core`, run with `ctest -L camera_core`. They are tagged `unit`, so +`ctest -L unit` runs them too. diff --git a/src/core/camera_core/cpp/CMakeLists.txt b/src/core/camera_core/cpp/CMakeLists.txt new file mode 100644 index 0000000000..0728a6df2c --- /dev/null +++ b/src/core/camera_core/cpp/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Camera capture core: which pool slots hold which frame sets and who may still read them. +# Standard library only, so it builds and is tested on every platform. +add_library(camera_core STATIC + publisher.cpp +) + +target_include_directories(camera_core INTERFACE + $ +) + +add_library(camera::camera_core ALIAS camera_core) diff --git a/src/core/camera_core/cpp/inc/camera_core/publisher.hpp b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp new file mode 100644 index 0000000000..44ed310fec --- /dev/null +++ b/src/core/camera_core/cpp/inc/camera_core/publisher.hpp @@ -0,0 +1,226 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "types.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +using GroupId = std::uint32_t; +using SubscriptionId = std::uint32_t; // 0 is never a valid subscription + +/** + * @brief Where a reader's frame events go: a socket connection or an in-process queue. + */ +class EventSink +{ +public: + virtual ~EventSink() = default; + + /** + * @brief Hand one event to the reader. Called on the capture path with the publisher's + * lock held, so it must not block, throw or call back into the publisher. + * @return false when the event could not be handed over; the reader then gets a later one. + */ + virtual bool send_event(SubscriptionId subscription, const FrameEvent& event) noexcept = 0; +}; + +// Counters of one group since it was added. +struct PublisherStats +{ + std::uint64_t published = 0; // frame sets published + std::uint64_t events_sent = 0; + std::uint64_t send_failures = 0; + std::uint64_t no_free_slot = 0; // acquire_slot calls that found every slot in use +}; + +struct SubscribeResult +{ + SubscriptionId id = 0; + std::string error; // why the subscription was refused, when id is 0 + + explicit operator bool() const + { + return id != 0; + } +}; + +/** + * @brief Tracks which pool slots hold which frame sets and who may still read them. + * + * A group keeps a short ring of recent frame sets; a reader's window is how many are kept for + * it at a time, unread and held together. A subscription is accepted only when + * slots >= ring history + sum of reader windows + backend reserve. Events carry slot indices; + * slot memory is never touched here. Thread-safe; no call blocks on a reader. + */ +class Publisher +{ +public: + using Clock = std::function; + using SlotFreeCallback = std::function; + + /** + * @param clock Host monotonic time in nanoseconds; defaults to std::chrono::steady_clock. + */ + explicit Publisher(Clock clock = {}); + + // ---- backend side ---------------------------------------------------------------------- + + /** + * @brief Declare a group and the size of its pools. + * @param backend_reserve Slots the backend needs beyond the ring and the readers: one to + * write ahead, more for an SDK that keeps several captures in flight in its own queue. + * @throws std::invalid_argument if the description is not usable. + */ + GroupId add_group(const GroupDescription& description, std::uint32_t backend_reserve = 1); + + /** + * @brief Take a free slot to write the next image into. + * @return The slot, or nothing when every slot is in use (the frame should be dropped). + * @throws std::logic_error for a group that has a slot-free callback: its backend owns the + * pool and names slots with claim_slot. + */ + std::optional acquire_slot(GroupId group, std::size_t stream); + + /** + * @brief Mark a slot the SDK chose and filled, for a pool the SDK owns. + * @throws std::invalid_argument if the slot is still held by the ring or a reader. + */ + void claim_slot(GroupId group, std::size_t stream, std::uint32_t slot); + + /** + * @brief Give back a slot taken with acquire_slot or claim_slot without publishing it. + */ + void abandon_slot(GroupId group, std::size_t stream, std::uint32_t slot); + + /** + * @brief Be told when a published slot is no longer held by the ring or any reader, so a + * buffer the SDK owns can go back to it at once. Called outside the publisher's lock, on + * the thread whose call released the last hold. A group with this callback hands out its + * slots with claim_slot only, so a slot cannot be taken again before its callback has run. + */ + void set_slot_free_callback(GroupId group, SlotFreeCallback callback); + + /** + * @brief Publish one frame set. images[i] belongs to stream i and names a slot taken with + * acquire_slot or claim_slot. + * @param timeline capture_ns and sdk_ready_ns from the backend; the rest is filled in here. + * @return The frame set's sequence number. + * @throws std::invalid_argument if the images do not match the group or do not fit its slots. + */ + std::uint64_t publish(GroupId group, std::span images, const FrameTimeline& timeline); + + /** + * @brief Tell readers the camera restarted: the generation changes and the ring is emptied. + * Frame sets readers already hold stay valid until released. + */ + void restart_group(GroupId group); + + // ---- reader side ----------------------------------------------------------------------- + + /** + * @brief Start sending a group's frame events to a sink, which must outlive the subscription. + * @return The subscription, or the reason it does not fit the pool. + */ + SubscribeResult subscribe(EventSink& sink, GroupId group, const ReaderOptions& options); + + /** + * @brief Return a frame set. The reader's next one, if any is waiting, is sent at once. + */ + void release(SubscriptionId subscription, std::uint64_t lease_id); + + /** + * @brief End a subscription and take back every frame set it still holds. + */ + void unsubscribe(SubscriptionId subscription); + + // ---- description ----------------------------------------------------------------------- + + std::size_t group_count() const; + GroupDescription group_description(GroupId group) const; + std::optional find_group(std::string_view id) const; + std::uint32_t generation(GroupId group) const; + PublisherStats stats(GroupId group) const; + +private: + struct FrameSet + { + std::uint64_t sequence = 0; + std::uint32_t generation = 0; + FrameTimeline timeline; + std::array images; + }; + + struct Subscription + { + GroupId group = 0; + std::uint32_t window = 1; + DeliveryOrder order = DeliveryOrder::NEWEST; + EventSink* sink = nullptr; + std::uint64_t cursor = 0; // sequence of the last frame set sent to this reader + bool started = false; + std::map leases; // sent and not yet released + }; + + struct Slot + { + std::uint32_t holds = 0; // the ring and every reader lease that names this slot + bool writing = false; // taken by the backend and not yet published + }; + + struct Group + { + GroupDescription description; + std::uint32_t reserve = 1; + std::uint32_t slot_count = 0; // the smallest pool of the group + std::uint32_t generation = 0; + std::uint64_t sequence = 0; + std::vector> slots; // per stream + std::deque ring; + std::vector subscriptions; + std::shared_ptr on_free; + PublisherStats stats; + }; + + struct FreedSlot + { + std::shared_ptr callback; + std::size_t stream = 0; + std::uint32_t slot = 0; + }; + + Group& group_at(GroupId group); + const Group& group_at(GroupId group) const; + static Slot& slot_at(Group& group, std::size_t stream, std::uint32_t slot); + std::size_t history_of(const Group& group) const; + void hold(Group& group, const FrameSet& set); + void unhold(Group& group, const FrameSet& set, std::vector& freed); + void trim_ring(Group& group, std::vector& freed); + void deliver(Group& group, SubscriptionId id, Subscription& subscription, std::vector& freed); + static void notify(const std::vector& freed); + + Clock m_clock; + mutable std::mutex m_mutex; + std::vector> m_groups; + std::map m_subscriptions; + SubscriptionId m_next_subscription = 1; + std::uint64_t m_next_lease = 1; +}; + +} // namespace core::camera diff --git a/src/core/camera_core/cpp/inc/camera_core/types.hpp b/src/core/camera_core/cpp/inc/camera_core/types.hpp new file mode 100644 index 0000000000..9aa4567509 --- /dev/null +++ b/src/core/camera_core/cpp/inc/camera_core/types.hpp @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// A group publishes one image per stream, all captured together. +inline constexpr std::size_t MAX_STREAMS_PER_GROUP = 8; +inline constexpr std::uint32_t MAX_SLOTS_PER_STREAM = 64; + +// Marks a time that is not known. All times are nanoseconds on the host's monotonic clock. +inline constexpr std::int64_t TIME_UNKNOWN = -1; + +enum class PixelFormat : std::uint32_t +{ + BGRA8 = 1, + RGBA8 = 2, + NV12 = 3, + GRAY8 = 4, +}; + +// NV12 has two planes (luma, then interleaved chroma); the other formats have one. +inline constexpr std::size_t MAX_PLANES_PER_IMAGE = 2; + +constexpr std::uint32_t plane_count_of(PixelFormat format) +{ + return format == PixelFormat::NV12 ? 2 : 1; +} + +// Where one plane of an image sits inside a slot. +struct PlaneLayout +{ + std::uint32_t offset_bytes = 0; + std::uint32_t stride_bytes = 0; // from the start of one row to the start of the next +}; + +// The pixels of one plane: the bytes one row holds, without padding, and the number of rows. +struct PlaneSize +{ + std::uint64_t row_bytes = 0; + std::uint64_t rows = 0; +}; + +constexpr PlaneSize plane_size_of(PixelFormat format, std::uint32_t width, std::uint32_t height, std::uint32_t plane) +{ + const std::uint64_t columns = width; + const std::uint64_t rows = height; + switch (format) + { + case PixelFormat::BGRA8: + case PixelFormat::RGBA8: + return { columns * 4, rows }; + case PixelFormat::GRAY8: + return { columns, rows }; + case PixelFormat::NV12: + // Luma at full size, then one interleaved chroma pair per two by two pixels. + return plane == 0 ? PlaneSize{ columns, rows } : PlaneSize{ (columns + 1) / 2 * 2, (rows + 1) / 2 }; + } + return {}; +} + +// Whether a stream's slots hold pixels or one access unit of encoded video each. +enum class Codec : std::uint32_t +{ + NONE = 0, // pixels, laid out as the stream's pixel format says + H264 = 1, + H265 = 2, +}; + +// Where a stream's slots live, which tells a reader how to map them. The core tracks slots by +// index only and never touches their memory; each further kind is added with its pool. +enum class MemoryKind : std::uint32_t +{ + HOST = 1, // shared host memory +}; + +struct StreamDescription +{ + std::string id; + Codec codec = Codec::NONE; + PixelFormat pixel_format = PixelFormat::BGRA8; // of a pixel stream; unused when the stream is encoded + MemoryKind memory = MemoryKind::HOST; + std::uint32_t width = 0; + std::uint32_t height = 0; + std::uint32_t plane_count = 0; // 0 for an encoded stream + std::array planes; // the same for every slot of the stream + std::uint64_t slot_bytes = 0; // capacity of one slot + std::uint32_t slot_count = 0; + double frame_rate = 0.0; // configured or measured, not the sensor mode's maximum + + constexpr bool is_encoded() const + { + return codec != Codec::NONE; + } +}; + +// One or more hardware-synchronised streams that are published as one frame set. +struct GroupDescription +{ + std::string id; + std::vector streams; +}; + +enum ImageFlag : std::uint32_t +{ + IMAGE_FLAG_KEYFRAME = 1u << 0, // encoded stream: this access unit decodes on its own + IMAGE_FLAG_SDK_WARNING = 1u << 1, // the camera SDK delivered the frame but flagged it as suspect +}; + +struct ImageInfo +{ + std::uint32_t slot = 0; + std::uint32_t flags = 0; // ImageFlag bits + std::uint64_t bytes_used = 0; // encoded stream: size of the access unit + std::uint64_t device_frame_id = 0; // the SDK's own frame number + std::int64_t device_time_ns = TIME_UNKNOWN; // the SDK's own timestamp, in the SDK's clock +}; + +// Where a frame set was at each step, so latency can be read off every frame. +struct FrameTimeline +{ + std::int64_t capture_ns = TIME_UNKNOWN; // sensor capture, when the backend can map it to the host clock + std::int64_t sdk_ready_ns = TIME_UNKNOWN; // the SDK handed the frame to the backend + std::int64_t published_ns = TIME_UNKNOWN; // set by Publisher::publish + std::int64_t sent_ns = TIME_UNKNOWN; // set when the event is handed to a reader +}; + +struct FrameEvent +{ + std::uint64_t lease_id = 0; // return it with Publisher::release + std::uint64_t sequence = 0; // counts frame sets of the group; never restarts + std::uint32_t generation = 0; // changes when the camera restarts + std::uint32_t skipped = 0; // frame sets this reader did not receive since its previous one + std::uint32_t image_count = 0; + FrameTimeline timeline; + std::array images; // images[i] belongs to stream i +}; + +enum class DeliveryOrder +{ + DEFAULT, // NEWEST for a window of one, IN_ORDER otherwise + NEWEST, // each time the reader has room it gets the newest frame set and skips older ones + IN_ORDER, // the reader gets every frame set in sequence while it keeps up +}; + +struct ReaderOptions +{ + std::uint32_t window = 1; // frame sets kept for this reader at a time, unread and held together + DeliveryOrder order = DeliveryOrder::DEFAULT; +}; + +// The order a reader with these options gets: what it asked for, or the default for its window. +constexpr DeliveryOrder resolved_order(const ReaderOptions& options) +{ + if (options.order != DeliveryOrder::DEFAULT) + { + return options.order; + } + return options.window == 1 ? DeliveryOrder::NEWEST : DeliveryOrder::IN_ORDER; +} + +} // namespace core::camera diff --git a/src/core/camera_core/cpp/publisher.cpp b/src/core/camera_core/cpp/publisher.cpp new file mode 100644 index 0000000000..37cd59e85c --- /dev/null +++ b/src/core/camera_core/cpp/publisher.cpp @@ -0,0 +1,486 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_core/publisher.hpp" + +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +std::int64_t steady_now_ns() +{ + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} + +// A pixel stream names every plane of its format and every plane lies inside the slot; an +// encoded stream has no planes. +bool layout_is_valid(const StreamDescription& stream) +{ + if (stream.is_encoded()) + { + return stream.plane_count == 0; + } + if (stream.width == 0 || stream.height == 0 || stream.plane_count != plane_count_of(stream.pixel_format)) + { + return false; + } + for (std::uint32_t i = 0; i < stream.plane_count; ++i) + { + const PlaneSize size = plane_size_of(stream.pixel_format, stream.width, stream.height, i); + const std::uint64_t stride = stream.planes[i].stride_bytes; + // 64-bit sums of 32-bit fields, so nothing here can wrap. + const std::uint64_t end = stream.planes[i].offset_bytes + stride * (size.rows - 1) + size.row_bytes; + if (size.rows == 0 || stride < size.row_bytes || end > stream.slot_bytes) + { + return false; + } + } + return true; +} + +} // namespace + +Publisher::Publisher(Clock clock) : m_clock(clock ? std::move(clock) : Clock(steady_now_ns)) +{ +} + +GroupId Publisher::add_group(const GroupDescription& description, std::uint32_t backend_reserve) +{ + if (description.id.empty()) + { + throw std::invalid_argument("camera group needs an id"); + } + if (description.streams.empty() || description.streams.size() > MAX_STREAMS_PER_GROUP) + { + throw std::invalid_argument("camera group '" + description.id + "' needs between 1 and " + + std::to_string(MAX_STREAMS_PER_GROUP) + " streams"); + } + if (backend_reserve == 0) + { + throw std::invalid_argument("camera group '" + description.id + "' needs a backend reserve of at least 1"); + } + + auto group = std::make_unique(); + group->description = description; + group->reserve = backend_reserve; + group->slot_count = std::numeric_limits::max(); + for (const StreamDescription& stream : description.streams) + { + if (stream.id.empty() || stream.slot_bytes == 0 || stream.slot_count == 0 || + stream.slot_count > MAX_SLOTS_PER_STREAM) + { + throw std::invalid_argument("camera group '" + description.id + "' has a stream without an id, a slot " + + "size or between 1 and " + std::to_string(MAX_SLOTS_PER_STREAM) + " slots"); + } + if (!layout_is_valid(stream)) + { + throw std::invalid_argument("camera stream '" + stream.id + "' needs a size and every plane of its " + + "pixel format inside the slot, or no planes at all when it is encoded"); + } + group->slot_count = std::min(group->slot_count, stream.slot_count); + group->slots.emplace_back(stream.slot_count); + } + // Before any reader, the ring keeps one frame set and the backend keeps its reserve. + if (group->slot_count <= backend_reserve) + { + throw std::invalid_argument("camera group '" + description.id + "' has pools of " + + std::to_string(group->slot_count) + " slots, too few for one frame set and a " + + "backend reserve of " + std::to_string(backend_reserve)); + } + + std::lock_guard lock(m_mutex); + for (const auto& existing : m_groups) + { + if (existing->description.id == description.id) + { + throw std::invalid_argument("camera group '" + description.id + "' already exists"); + } + } + m_groups.push_back(std::move(group)); + return static_cast(m_groups.size() - 1); +} + +std::optional Publisher::acquire_slot(GroupId group, std::size_t stream) +{ + std::lock_guard lock(m_mutex); + Group& target = group_at(group); + if (target.on_free) + { + throw std::logic_error("camera group '" + target.description.id + + "' has a slot-free callback, so its backend names slots with claim_slot"); + } + std::vector& slots = target.slots.at(stream); + for (std::uint32_t i = 0; i < slots.size(); ++i) + { + if (slots[i].holds == 0 && !slots[i].writing) + { + slots[i].writing = true; + return i; + } + } + ++target.stats.no_free_slot; + return std::nullopt; +} + +void Publisher::claim_slot(GroupId group, std::size_t stream, std::uint32_t slot) +{ + std::lock_guard lock(m_mutex); + Slot& state = slot_at(group_at(group), stream, slot); + if (state.holds != 0 || state.writing) + { + throw std::invalid_argument("camera slot " + std::to_string(slot) + " is still in use and cannot be claimed"); + } + state.writing = true; +} + +void Publisher::abandon_slot(GroupId group, std::size_t stream, std::uint32_t slot) +{ + std::lock_guard lock(m_mutex); + Slot& state = slot_at(group_at(group), stream, slot); + if (!state.writing) + { + throw std::invalid_argument("camera slot " + std::to_string(slot) + " was not acquired and cannot be abandoned"); + } + state.writing = false; +} + +void Publisher::set_slot_free_callback(GroupId group, SlotFreeCallback callback) +{ + std::lock_guard lock(m_mutex); + group_at(group).on_free = callback ? std::make_shared(std::move(callback)) : nullptr; +} + +std::uint64_t Publisher::publish(GroupId group, std::span images, const FrameTimeline& timeline) +{ + std::vector freed; + std::uint64_t sequence = 0; + { + std::lock_guard lock(m_mutex); + Group& target = group_at(group); + if (images.size() != target.description.streams.size()) + { + throw std::invalid_argument("camera group '" + target.description.id + "' publishes " + + std::to_string(target.description.streams.size()) + " images per frame set"); + } + for (std::size_t i = 0; i < images.size(); ++i) + { + if (!slot_at(target, i, images[i].slot).writing) + { + throw std::invalid_argument("camera slot " + std::to_string(images[i].slot) + + " was not acquired before it was published"); + } + if (images[i].bytes_used > target.description.streams[i].slot_bytes) + { + throw std::invalid_argument("camera stream '" + target.description.streams[i].id + "' published " + + std::to_string(images[i].bytes_used) + " bytes into a slot of " + + std::to_string(target.description.streams[i].slot_bytes)); + } + } + + FrameSet set; + set.sequence = ++target.sequence; + set.generation = target.generation; + set.timeline = timeline; + set.timeline.published_ns = m_clock(); + for (std::size_t i = 0; i < images.size(); ++i) + { + set.images[i] = images[i]; + slot_at(target, i, images[i].slot).writing = false; // the ring holds it from here + } + hold(target, set); + target.ring.push_back(set); + ++target.stats.published; + sequence = set.sequence; + + trim_ring(target, freed); + for (SubscriptionId id : target.subscriptions) + { + deliver(target, id, m_subscriptions.at(id), freed); + } + } + notify(freed); + return sequence; +} + +void Publisher::restart_group(GroupId group) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + Group& target = group_at(group); + ++target.generation; + while (!target.ring.empty()) + { + unhold(target, target.ring.front(), freed); + target.ring.pop_front(); + } + } + notify(freed); +} + +SubscribeResult Publisher::subscribe(EventSink& sink, GroupId group, const ReaderOptions& options) +{ + SubscribeResult result; + std::lock_guard lock(m_mutex); + if (group >= m_groups.size()) + { + result.error = "unknown camera group"; + return result; + } + if (options.window == 0) + { + result.error = "a reader's window must be at least 1"; + return result; + } + Group& target = *m_groups[group]; + const DeliveryOrder order = resolved_order(options); + + // The ring keeps what in-order readers may still be waiting for; newest-only readers + // need the newest frame set and nothing older. + std::size_t history = order == DeliveryOrder::IN_ORDER ? options.window : 1; + std::size_t windows = options.window; + for (SubscriptionId id : target.subscriptions) + { + const Subscription& other = m_subscriptions.at(id); + if (other.order == DeliveryOrder::IN_ORDER) + { + history = std::max(history, other.window); + } + windows += other.window; + } + const std::size_t needed = history + windows + target.reserve; + if (needed > target.slot_count) + { + result.error = "subscription does not fit the pool of camera group '" + target.description.id + "': it needs " + + std::to_string(needed) + " slots (" + std::to_string(history) + " ring history, " + + std::to_string(windows) + " for reader windows, " + std::to_string(target.reserve) + + " for the backend) and the pool has " + std::to_string(target.slot_count); + return result; + } + + Subscription subscription; + subscription.group = group; + subscription.window = options.window; + subscription.order = order; + subscription.sink = &sink; + result.id = m_next_subscription++; + m_subscriptions.emplace(result.id, std::move(subscription)); + target.subscriptions.push_back(result.id); + return result; +} + +void Publisher::release(SubscriptionId subscription, std::uint64_t lease_id) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + const auto found = m_subscriptions.find(subscription); + if (found == m_subscriptions.end()) + { + return; + } + const auto lease = found->second.leases.find(lease_id); + if (lease == found->second.leases.end()) + { + return; + } + Group& target = *m_groups[found->second.group]; + unhold(target, lease->second, freed); + found->second.leases.erase(lease); + deliver(target, subscription, found->second, freed); + } + notify(freed); +} + +void Publisher::unsubscribe(SubscriptionId subscription) +{ + std::vector freed; + { + std::lock_guard lock(m_mutex); + const auto found = m_subscriptions.find(subscription); + if (found == m_subscriptions.end()) + { + return; + } + Group& target = *m_groups[found->second.group]; + for (const auto& lease : found->second.leases) + { + unhold(target, lease.second, freed); + } + std::erase(target.subscriptions, subscription); + m_subscriptions.erase(found); + trim_ring(target, freed); + } + notify(freed); +} + +std::size_t Publisher::group_count() const +{ + std::lock_guard lock(m_mutex); + return m_groups.size(); +} + +GroupDescription Publisher::group_description(GroupId group) const +{ + std::lock_guard lock(m_mutex); + return group_at(group).description; +} + +std::optional Publisher::find_group(std::string_view id) const +{ + std::lock_guard lock(m_mutex); + for (std::size_t i = 0; i < m_groups.size(); ++i) + { + if (m_groups[i]->description.id == id) + { + return static_cast(i); + } + } + return std::nullopt; +} + +std::uint32_t Publisher::generation(GroupId group) const +{ + std::lock_guard lock(m_mutex); + return group_at(group).generation; +} + +PublisherStats Publisher::stats(GroupId group) const +{ + std::lock_guard lock(m_mutex); + return group_at(group).stats; +} + +Publisher::Group& Publisher::group_at(GroupId group) +{ + if (group >= m_groups.size()) + { + throw std::out_of_range("unknown camera group"); + } + return *m_groups[group]; +} + +const Publisher::Group& Publisher::group_at(GroupId group) const +{ + if (group >= m_groups.size()) + { + throw std::out_of_range("unknown camera group"); + } + return *m_groups[group]; +} + +Publisher::Slot& Publisher::slot_at(Group& group, std::size_t stream, std::uint32_t slot) +{ + if (stream >= group.slots.size() || slot >= group.slots[stream].size()) + { + throw std::invalid_argument("camera group '" + group.description.id + "' has no stream " + + std::to_string(stream) + " slot " + std::to_string(slot)); + } + return group.slots[stream][slot]; +} + +std::size_t Publisher::history_of(const Group& group) const +{ + std::size_t history = 1; // the newest frame set is always kept + for (SubscriptionId id : group.subscriptions) + { + const Subscription& subscription = m_subscriptions.at(id); + if (subscription.order == DeliveryOrder::IN_ORDER) + { + history = std::max(history, subscription.window); + } + } + return history; +} + +void Publisher::hold(Group& group, const FrameSet& set) +{ + for (std::size_t i = 0; i < group.slots.size(); ++i) + { + ++slot_at(group, i, set.images[i].slot).holds; + } +} + +void Publisher::unhold(Group& group, const FrameSet& set, std::vector& freed) +{ + for (std::size_t i = 0; i < group.slots.size(); ++i) + { + Slot& slot = slot_at(group, i, set.images[i].slot); + if (--slot.holds == 0 && group.on_free) + { + freed.push_back({ group.on_free, i, set.images[i].slot }); + } + } +} + +void Publisher::trim_ring(Group& group, std::vector& freed) +{ + const std::size_t history = history_of(group); + while (group.ring.size() > history) + { + unhold(group, group.ring.front(), freed); + group.ring.pop_front(); + } +} + +void Publisher::deliver(Group& group, SubscriptionId id, Subscription& subscription, std::vector& freed) +{ + while (subscription.leases.size() < subscription.window) + { + if (group.ring.empty() || group.ring.back().sequence <= subscription.cursor) + { + return; + } + const FrameSet* target = &group.ring.back(); + if (subscription.order == DeliveryOrder::IN_ORDER) + { + target = &*std::find_if(group.ring.begin(), group.ring.end(), + [&](const FrameSet& set) { return set.sequence > subscription.cursor; }); + } + + FrameEvent event; + event.lease_id = m_next_lease++; + event.sequence = target->sequence; + event.generation = target->generation; + if (subscription.started) + { + const std::uint64_t skipped = target->sequence - subscription.cursor - 1; + event.skipped = + static_cast(std::min(skipped, std::numeric_limits::max())); + } + event.image_count = static_cast(group.slots.size()); + event.images = target->images; + event.timeline = target->timeline; + event.timeline.sent_ns = m_clock(); + + hold(group, *target); + if (!subscription.sink->send_event(id, event)) + { + unhold(group, *target, freed); + ++group.stats.send_failures; + return; + } + ++group.stats.events_sent; + subscription.leases.emplace(event.lease_id, *target); + subscription.cursor = target->sequence; + subscription.started = true; + } +} + +void Publisher::notify(const std::vector& freed) +{ + for (const FreedSlot& slot : freed) + { + (*slot.callback)(slot.stream, slot.slot); + } +} + +} // namespace core::camera diff --git a/src/core/camera_endpoint/AGENTS.md b/src/core/camera_endpoint/AGENTS.md new file mode 100644 index 0000000000..92ff196bc1 --- /dev/null +++ b/src/core/camera_endpoint/AGENTS.md @@ -0,0 +1,15 @@ + + +# Agent notes — `camera_endpoint` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) for what runs on which thread before changing it. + +- **Nothing sits between publish and the socket.** A frame event is encoded and sent inside `send_event`, on the publishing thread. Do not add a queue or a thread hop there, and do not log per frame. +- **`send_event` holds the publisher's lock.** It must not take another lock, block or call the publisher. That lock is also what serialises each connection's event encoder. +- **Unsubscribe before a connection is destroyed.** `Publisher::unsubscribe` returning is the guarantee that no `send_event` is running or will start for that sink. +- **A reader is untrusted.** Whatever it sends can at most get it dropped; it must not stop the thread or affect other readers. diff --git a/src/core/camera_endpoint/CMakeLists.txt b/src/core/camera_endpoint/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_endpoint/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_endpoint/README.md b/src/core/camera_endpoint/README.md new file mode 100644 index 0000000000..a32ab763c4 --- /dev/null +++ b/src/core/camera_endpoint/README.md @@ -0,0 +1,49 @@ + + +# Camera interface: the endpoint + +The camera plugin's side of the conversation with readers. It puts a +[`camera_core`](../camera_core) `Publisher` on a socket ([`camera_ipc`](../camera_ipc)) and +speaks the messages of [`camera_wire`](../camera_wire). + +```cpp +Publisher publisher; +const GroupId head = publisher.add_group(description); +HostPool left(slots, bytes, "head.left"), right(slots, bytes, "head.right"); + +Endpoint endpoint(publisher, socket_path); +endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); +endpoint.start(); +// ... the backend acquires slots, writes pixels and publishes; readers come and go ... +``` + +## What runs where + +- **One thread of its own** accepts readers and handles what they send: the handshake, + subscriptions and releases. It sleeps in `poll` until a reader sends something. +- **Frame events are sent from the thread that publishes**, straight from + `Publisher::publish`, with a send that does not block. There is no queue and no hand-over + to another thread between a frame set being published and its event being on the socket. + The event that follows a release is sent from the endpoint's thread for the same reason. +- A reader's subscription takes effect before the answer to it is sent, so a `Frame` can + reach the reader ahead of `Subscribed`. + +## When a reader goes away + +A closed connection is the signal, whether the reader exited, crashed or was killed. Its +subscription ends and every frame set it held goes back to the pool. A reader that sends +bytes that are not a message, or a message only a plugin sends, is dropped the same way; +other readers are not affected. + +## Logging + +`isaaccapture.camera_endpoint.Endpoint` logs readers arriving, subscribing, being refused and +leaving. Nothing is logged per frame. + +## Tests + +`tests/cpp/core/camera_endpoint`, run with `ctest -L camera_endpoint`. They are tagged `unit` +and play the reader's part with the socket and the codec directly. diff --git a/src/core/camera_endpoint/cpp/CMakeLists.txt b/src/core/camera_endpoint/cpp/CMakeLists.txt new file mode 100644 index 0000000000..4bcdfb1e78 --- /dev/null +++ b/src/core/camera_endpoint/cpp/CMakeLists.txt @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +# The plugin's side of the conversation with readers: serves a Publisher's groups over a socket. +add_library(camera_endpoint STATIC + endpoint.cpp +) + +target_include_directories(camera_endpoint INTERFACE + $ +) + +target_link_libraries(camera_endpoint + PUBLIC + camera::camera_core + PRIVATE + camera::camera_ipc + camera::camera_wire + log_bridge::log_bridge_core + Threads::Threads +) + +add_library(camera::camera_endpoint ALIAS camera_endpoint) diff --git a/src/core/camera_endpoint/cpp/endpoint.cpp b/src/core/camera_endpoint/cpp/endpoint.cpp new file mode 100644 index 0000000000..652a06afaf --- /dev/null +++ b/src/core/camera_endpoint/cpp/endpoint.cpp @@ -0,0 +1,406 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_endpoint/endpoint.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// One reader. As an EventSink it is called by the publisher, on whichever thread publishes or +// releases, with the publisher's lock held; everything else runs on the endpoint's thread. +class Connection final : public EventSink +{ +public: + explicit Connection(MessageSocket connected) : socket(std::move(connected)) + { + } + + bool send_event(SubscriptionId /*subscription*/, const FrameEvent& event) noexcept override + { + try + { + return socket.send(m_event_encoder.encode(event)) == SendStatus::SENT; + } + catch (const std::exception&) + { + return false; // the reader gets a later frame set, or its closed socket is noticed + } + } + + MessageSocket socket; + std::string name = "unnamed"; + bool greeted = false; + SubscriptionId subscription = 0; + +private: + WireEncoder m_event_encoder; // used inside send_event only, which the publisher's lock serialises +}; + +const char* name_of(DeliveryOrder order) +{ + switch (order) + { + case DeliveryOrder::NEWEST: + return "newest"; + case DeliveryOrder::IN_ORDER: + return "in order"; + default: + return "default order"; + } +} + +} // namespace + +class Endpoint::Impl +{ +public: + Impl(Publisher& publisher, const std::string& path) + : m_publisher(publisher), + m_path(path), + m_listener(std::in_place, path), + m_wake(::eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK)), + m_buffer(MAX_WIRE_MESSAGE_BYTES), + m_logger(isaaccapture::Logger::get("isaaccapture.camera_endpoint.Endpoint")) + { + if (!m_wake.valid()) + { + throw std::system_error(errno, std::generic_category(), "creating the camera endpoint's wake-up descriptor"); + } + } + + ~Impl() + { + stop(); + } + + void set_group_memory(GroupId group, std::vector descriptors) + { + if (m_thread.joinable()) + { + throw std::logic_error("a camera group's memory is named before the endpoint starts"); + } + const GroupDescription description = m_publisher.group_description(group); + if (descriptors.size() != description.streams.size()) + { + throw std::invalid_argument("camera group '" + description.id + "' needs one descriptor per stream"); + } + m_memory[group] = std::move(descriptors); + } + + void start() + { + if (m_thread.joinable()) + { + return; + } + if (!m_listener) + { + m_listener.emplace(m_path); + } + m_stopping = false; + m_thread = std::thread([this] { run(); }); + m_logger->info("serving cameras on '{}'", m_path); + } + + void stop() + { + if (!m_thread.joinable()) + { + return; + } + m_stopping = true; + const std::uint64_t one = 1; + // Cannot fail in a way that matters: the counter only overflows after 2^64 wake-ups. + [[maybe_unused]] const ssize_t written = ::write(m_wake.get(), &one, sizeof(one)); + m_thread.join(); + // Without the listener a reader that tries to connect is refused at once, instead of + // waiting in a queue nobody serves. + m_listener.reset(); + } + + std::size_t reader_count() const + { + return m_reader_count.load(); + } + +private: + void run() + { + std::vector waits; + while (!m_stopping) + { + waits.assign({ { m_wake.get(), POLLIN, 0 }, { m_listener->descriptor(), POLLIN, 0 } }); + for (const auto& connection : m_connections) + { + waits.push_back({ connection->socket.descriptor(), POLLIN, 0 }); + } + if (::poll(waits.data(), waits.size(), -1) < 0) + { + if (errno == EINTR) + { + continue; + } + m_logger->error("camera endpoint stops serving: poll failed with errno {}", errno); + break; + } + + // Back to front, so dropping a connection leaves the indices still to visit alone. + for (std::size_t i = m_connections.size(); i-- > 0;) + { + if (waits[i + 2].revents != 0 && !serve(*m_connections[i])) + { + drop(i); + } + } + if ((waits[1].revents & POLLIN) != 0) + { + accept_readers(); + } + } + while (!m_connections.empty()) + { + drop(m_connections.size() - 1); + } + // Empty the wake-up counter, so that serving can start again. + std::uint64_t wakes = 0; + [[maybe_unused]] const ssize_t drained = ::read(m_wake.get(), &wakes, sizeof(wakes)); + } + + void accept_readers() + { + try + { + while (auto socket = m_listener->accept(std::chrono::milliseconds(0))) + { + m_connections.push_back(std::make_unique(std::move(*socket))); + m_reader_count = m_connections.size(); + } + } + catch (const std::exception& error) + { + m_logger->warn("could not accept a camera reader: {}", error.what()); + } + } + + // Ends the reader's subscription before its sink goes away. unsubscribe() returns only once + // no send_event is running or can start for it. + void drop(std::size_t index) + { + Connection& connection = *m_connections[index]; + if (connection.subscription != 0) + { + m_publisher.unsubscribe(connection.subscription); + } + m_logger->info("camera reader '{}' is gone", connection.name); + m_connections.erase(m_connections.begin() + static_cast(index)); + m_reader_count = m_connections.size(); + } + + // Handles everything the reader has sent. Returns false when the connection is to be dropped. + bool serve(Connection& connection) + { + try + { + for (;;) + { + const ReceivedMessage received = connection.socket.receive(m_buffer, std::chrono::milliseconds(0)); + if (received.status == ReceiveStatus::TIMEOUT) + { + return true; + } + if (received.status == ReceiveStatus::CLOSED) + { + return false; + } + const WireMessage message = decode_wire_message({ m_buffer.data(), received.bytes }); + if (!handle(connection, message)) + { + return false; + } + } + } + catch (const std::exception& error) + { + m_logger->warn("dropping camera reader '{}': {}", connection.name, error.what()); + return false; + } + } + + bool handle(Connection& connection, const WireMessage& message) + { + if (const auto* hello = std::get_if(&message)) + { + return greet(connection, *hello); + } + if (const auto* subscribe = std::get_if(&message)) + { + if (!connection.greeted) + { + m_logger->warn("dropping a camera reader that subscribed before it said hello"); + return false; + } + return answer(connection, *subscribe); + } + if (const auto* release = std::get_if(&message)) + { + if (connection.subscription != 0) + { + m_publisher.release(connection.subscription, release->lease_id); + } + return true; + } + if (std::holds_alternative(message)) + { + return true; // from a newer reader; not for this build + } + m_logger->warn("dropping camera reader '{}': it sent a message only a plugin sends", connection.name); + return false; + } + + bool greet(Connection& connection, const HelloMessage& hello) + { + connection.name = hello.reader_name.empty() ? "unnamed" : hello.reader_name; + WelcomeMessage welcome; + if (hello.protocol_version != WIRE_PROTOCOL_VERSION) + { + welcome.error = "this camera plugin speaks protocol version " + std::to_string(WIRE_PROTOCOL_VERSION) + + " and the reader version " + std::to_string(hello.protocol_version); + m_logger->warn("refusing camera reader '{}': {}", connection.name, welcome.error); + (void)send(connection, m_encoder.encode(welcome)); + return false; + } + const std::size_t group_count = m_publisher.group_count(); + for (std::size_t i = 0; i < group_count; ++i) + { + welcome.groups.push_back(m_publisher.group_description(static_cast(i))); + } + connection.greeted = true; + m_logger->info("camera reader '{}' connected", connection.name); + return send(connection, m_encoder.encode(welcome)); + } + + bool answer(Connection& connection, const SubscribeMessage& subscribe) + { + SubscribedMessage reply; + std::span descriptors; + const std::optional group = m_publisher.find_group(subscribe.group); + if (connection.subscription != 0) + { + reply.error = "a connection carries one subscription; open another connection for a second group"; + } + else if (!group) + { + reply.error = "the camera plugin publishes no group '" + subscribe.group + "'"; + } + else if (!m_memory.contains(*group)) + { + reply.error = "camera group '" + subscribe.group + "' has no memory a reader can map"; + } + else + { + // From here on the publisher may send this reader frame events at any moment, so a + // Frame can reach it before the Subscribed below. + const SubscribeResult result = m_publisher.subscribe(connection, *group, subscribe.options); + if (result) + { + connection.subscription = result.id; + reply.group = m_publisher.group_description(*group); + descriptors = m_memory.at(*group); + } + else + { + reply.error = result.error; + } + } + + if (reply.error.empty()) + { + m_logger->info("camera reader '{}' subscribed to '{}' (window {}, {})", connection.name, subscribe.group, + subscribe.options.window, name_of(subscribe.options.order)); + } + else + { + m_logger->warn("refusing camera reader '{}': {}", connection.name, reply.error); + } + return send(connection, m_encoder.encode(reply), descriptors); + } + + static bool send(Connection& connection, std::span bytes, std::span descriptors = {}) + { + return connection.socket.send(bytes, descriptors) == SendStatus::SENT; + } + + Publisher& m_publisher; + std::string m_path; + std::optional m_listener; // absent between stop() and the next start() + UniqueFd m_wake; + std::map> m_memory; + + std::thread m_thread; + std::atomic m_stopping{ false }; + std::atomic m_reader_count{ 0 }; + + // The endpoint's thread only. + std::vector> m_connections; + std::vector m_buffer; + WireEncoder m_encoder; + + std::shared_ptr m_logger; +}; + +Endpoint::Endpoint(Publisher& publisher, const std::string& path) : m_impl(std::make_unique(publisher, path)) +{ +} + +Endpoint::~Endpoint() = default; + +void Endpoint::set_group_memory(GroupId group, std::vector descriptors) +{ + m_impl->set_group_memory(group, std::move(descriptors)); +} + +void Endpoint::start() +{ + m_impl->start(); +} + +void Endpoint::stop() +{ + m_impl->stop(); +} + +std::size_t Endpoint::reader_count() const +{ + return m_impl->reader_count(); +} + +} // namespace core::camera diff --git a/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp b/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp new file mode 100644 index 0000000000..7183340c5a --- /dev/null +++ b/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include + +namespace core::camera +{ + +/** + * @brief Serves a Publisher's groups to readers in other processes. + * + * Listens on a socket file. Each reader is told what is published; when it subscribes it is + * sent the group's pool descriptors and from then on the group's frame events. A reader that + * goes away, cleanly or not, has every frame set it held taken back. + */ +class Endpoint +{ +public: + /** + * @brief Start listening on `path`. Readers are answered once start() has been called. + * @param publisher Must outlive the endpoint. + * @throws std::system_error if another process listens on the path or it cannot be bound. + */ + Endpoint(Publisher& publisher, const std::string& path); + + // Stops serving. + ~Endpoint(); + + Endpoint(const Endpoint&) = delete; + Endpoint& operator=(const Endpoint&) = delete; + + /** + * @brief Name what a reader maps for a group: one descriptor per stream, in stream order, + * for the stream's pool. The endpoint does not own them; they stay open while it serves. + * A group without them cannot be subscribed to. + * @throws std::logic_error after start(). + * @throws std::out_of_range for a group the publisher does not have. + * @throws std::invalid_argument if the count does not match the group's streams. + */ + void set_group_memory(GroupId group, std::vector descriptors); + + /** + * @brief Serve readers. After a stop() this takes the path again. + * @throws std::system_error if another process took the path meanwhile. + */ + void start(); + + /** + * @brief Stop serving: every subscription ends, every reader sees its connection close, and + * the path is given up, so a reader that tries to connect is refused at once. Does nothing + * if the endpoint is not serving. + */ + void stop(); + + // Readers connected right now. + std::size_t reader_count() const; + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace core::camera diff --git a/src/core/camera_ipc/AGENTS.md b/src/core/camera_ipc/AGENTS.md new file mode 100644 index 0000000000..70eb3dce2c --- /dev/null +++ b/src/core/camera_ipc/AGENTS.md @@ -0,0 +1,18 @@ + + +# Agent notes — `camera_ipc` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) first; it explains why frames cross between processes this way. + +- **Linux only.** The module and its tests are added under `CMAKE_SYSTEM_NAME STREQUAL "Linux"`. Platform calls stay here and out of `camera_core`. +- **Frame memory crosses as a file descriptor on the module's Unix socket.** Do not add a messaging or shared-memory library for it; a new kind of memory is a new pool type behind the same connection. +- **Nothing on the capture path waits for a reader.** Sends do not block; a full queue and a closed peer are statuses, not exceptions. +- **A lock file owns the socket path.** Remove a socket file only while holding it, and never remove anything that is not a socket. +- **Readers cannot write.** A pool is sealed before its descriptor leaves the process, and a reader maps it read-only. Do not hand a reader a writable mapping. +- **Derive what both sides can compute.** A pool's layout follows from the slot count and slot size in the stream's description; do not send it separately. +- **Tests need no camera and no GPU**, and carry the `unit` tag. diff --git a/src/core/camera_ipc/CMakeLists.txt b/src/core/camera_ipc/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_ipc/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_ipc/README.md b/src/core/camera_ipc/README.md new file mode 100644 index 0000000000..f0a20e551f --- /dev/null +++ b/src/core/camera_ipc/README.md @@ -0,0 +1,89 @@ + + +# Camera interface: sharing frames between processes + +The process that owns the cameras and the processes that read them are on one machine. This +module is what crosses between them. Linux only. + +[`camera_core`](../camera_core) decides which slot holds which frame; this module provides the +memory behind the slots and the connection that tells a reader which slot to look at. + +## Host pool + +`HostPool` is one stream's slots in host memory: a single anonymous file in memory +(`memfd_create`), with the slots laid out one page-aligned stride apart. The owner maps it +writable, then seals it. A reader is sent the descriptor and maps it with `HostPoolView`, read +only. Both sides derive the layout from the slot count and slot size, which the stream's +description carries. + +```cpp +HostPool pool(stream.slot_count, stream.slot_bytes, "camera.left"); +std::span pixels = pool.slot(index); // the owner writes here + +HostPoolView view(std::move(received_descriptor), stream.slot_count, stream.slot_bytes); +std::span image = view.slot(index); // a reader sees the same bytes +``` + +Why an anonymous sealed file and not named shared memory (`shm_open`): + +- **Nothing to clean up.** There is no name to collide with another instance and no file left in + `/dev/shm` after a crash. The memory is freed when the last process that holds it goes away. +- **Readers cannot damage it.** The seals stop any process that receives the descriptor from + mapping it writable, writing to it, or resizing it. One misbehaving reader cannot change what + the others see, and cannot make the owner fault by shrinking the file under it. +- **Access follows the connection.** Only a process that was handed the descriptor can map the + pool. + +## Message socket + +`MessageListener` and `MessageSocket` are the connection: a Unix domain socket of the +sequenced-packet kind (`SOCK_SEQPACKET`). The owner listens on a socket file, each reader +connects, and either side sends messages with file descriptors attached. + +```cpp +MessageListener listener(path); // the owner +auto reader = listener.accept(timeout); +const int descriptors[] = { pool.descriptor() }; +reader->send(description_bytes, descriptors); + +MessageSocket socket = MessageSocket::connect(path); // a reader +ReceivedMessage message = socket.receive(buffer, timeout); +HostPoolView view(std::move(message.descriptors[0]), slot_count, slot_bytes); +``` + +- **Whole messages.** One `send` is one `receive`, in order, so there is no framing to get wrong. +- **Sending never blocks.** If a reader stops draining its queue the sender gets `WOULD_BLOCK` + and moves on; capture never waits for a reader. +- **A dead peer is seen at once.** When a process exits or crashes the other end reads + `CLOSED`, which is how the frames a crashed reader held are taken back. +- **Owner only.** The socket file is created with mode `0600`. +- **One listener per path.** A lock file next to the socket file (`.lock`) decides who + owns the path, so of two processes that start together exactly one listens, and a socket + file left by a crashed process is replaced safely. Anything else found at the path is left + alone. The lock file stays behind, empty. + +## Why a socket and shared memory, and not a messaging library + +Three things have to cross between the processes: the frame memory, a small event per frame +set, and the news that a peer has gone. + +Frame memory is named by a file descriptor: a sealed memory file here, and the same holds for +the buffers of hardware capture and codec stacks (DMA-BUF) and for GPU memory that can be +exported. Attaching a descriptor to a message on a Unix socket is the standard way to hand one +to another process, so a socket is needed whatever else is used. Once it is there it also +carries the events and reports dead peers, which leaves a messaging library nothing to do. + +| Considered | Why it is not used | +|---|---| +| iceoryx2 | It moves self-contained data through host memory that it allocates. Frame memory that exists only as a descriptor would still need a socket beside it, and its C++ binding adds a Rust toolchain to the build. | +| CUDA inter-process handles | A way to share one kind of GPU memory, not a way to connect processes: the handle still has to travel over some connection, and host memory and DMA-BUF are not covered. | +| ZeroMQ, gRPC, DDS | They copy the payload or share host memory only, and none of them pass descriptors. | +| PipeWire, GStreamer | The same mechanism underneath, wrapped in a media graph and, for PipeWire, a daemon. A reader here needs neither. | + +## Tests + +`tests/cpp/core/camera_ipc`, run with `ctest -L camera_ipc`. They are tagged `unit`, so +`ctest -L unit` runs them too. diff --git a/src/core/camera_ipc/cpp/CMakeLists.txt b/src/core/camera_ipc/cpp/CMakeLists.txt new file mode 100644 index 0000000000..69db298406 --- /dev/null +++ b/src/core/camera_ipc/cpp/CMakeLists.txt @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# How camera frames and messages cross between processes on one machine: pools of shared host +# memory that readers can only map read-only, and a socket that carries whole messages with +# file descriptors. Linux only. +add_library(camera_ipc STATIC + host_pool.cpp + message_socket.cpp +) + +target_include_directories(camera_ipc INTERFACE + $ +) + +add_library(camera::camera_ipc ALIAS camera_ipc) diff --git a/src/core/camera_ipc/cpp/host_pool.cpp b/src/core/camera_ipc/cpp/host_pool.cpp new file mode 100644 index 0000000000..7df3ae0776 --- /dev/null +++ b/src/core/camera_ipc/cpp/host_pool.cpp @@ -0,0 +1,162 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_ipc/host_pool.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// Older C library headers lack this seal although the kernel has it. +#ifndef F_SEAL_FUTURE_WRITE +# define F_SEAL_FUTURE_WRITE 0x0010 +#endif + +namespace core::camera +{ + +namespace +{ + +[[noreturn]] void throw_errno(const std::string& what) +{ + throw std::system_error(errno, std::generic_category(), what); +} + +// Slots start on page boundaries. Both sides derive the layout from the slot size alone. +std::uint64_t slot_stride_for(std::uint64_t slot_bytes) +{ + const auto page = static_cast(::sysconf(_SC_PAGESIZE)); + return (slot_bytes + page - 1) / page * page; +} + +std::uint64_t pool_bytes_for(std::uint32_t slot_count, std::uint64_t slot_stride) +{ + const auto limit = static_cast(std::numeric_limits::max()); + if (slot_count == 0 || slot_stride == 0 || slot_stride > limit / slot_count) + { + throw std::invalid_argument( + "a host pool needs at least one slot of at least one byte, and a size a file can have"); + } + return slot_stride * slot_count; +} + +} // namespace + +HostPool::HostPool(std::uint32_t slot_count, std::uint64_t slot_bytes, const std::string& name) + : m_slot_count(slot_count), m_slot_bytes(slot_bytes), m_slot_stride(slot_stride_for(slot_bytes)) +{ + const std::uint64_t pool_bytes = pool_bytes_for(slot_count, m_slot_stride); + m_fd.reset(::memfd_create(name.c_str(), MFD_CLOEXEC | MFD_ALLOW_SEALING)); + if (!m_fd.valid()) + { + throw_errno("host pool '" + name + "': memfd_create"); + } + if (::ftruncate(m_fd.get(), static_cast(pool_bytes)) != 0) + { + throw_errno("host pool '" + name + "': sizing the pool"); + } + void* memory = ::mmap(nullptr, pool_bytes, PROT_READ | PROT_WRITE, MAP_SHARED, m_fd.get(), 0); + if (memory == MAP_FAILED) + { + throw_errno("host pool '" + name + "': mapping the pool"); + } + // From here on nobody can resize the pool or map it writable; the mapping above stays. + if (::fcntl(m_fd.get(), F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW | F_SEAL_FUTURE_WRITE | F_SEAL_SEAL) != 0) + { + const int error = errno; + ::munmap(memory, pool_bytes); + throw std::system_error(error, std::generic_category(), "host pool '" + name + "': sealing the pool"); + } + m_memory = static_cast(memory); +} + +HostPool::~HostPool() +{ + ::munmap(m_memory, m_slot_stride * m_slot_count); +} + +std::span HostPool::slot(std::uint32_t index) +{ + if (index >= m_slot_count) + { + throw std::out_of_range("host pool has no slot " + std::to_string(index)); + } + return { m_memory + m_slot_stride * index, m_slot_bytes }; +} + +HostPoolView::HostPoolView(UniqueFd descriptor, std::uint32_t slot_count, std::uint64_t slot_bytes) + : m_slot_count(slot_count), m_slot_bytes(slot_bytes), m_slot_stride(slot_stride_for(slot_bytes)) +{ + const std::uint64_t pool_bytes = pool_bytes_for(slot_count, m_slot_stride); + struct stat info = {}; + if (::fstat(descriptor.get(), &info) != 0) + { + throw_errno("host pool view: inspecting the pool"); + } + // Mapping more than the descriptor holds would fault on first access instead of failing here. + if (static_cast(info.st_size) < pool_bytes) + { + throw std::invalid_argument("host pool view: the descriptor holds less than the pool needs"); + } + void* memory = ::mmap(nullptr, pool_bytes, PROT_READ, MAP_SHARED, descriptor.get(), 0); + if (memory == MAP_FAILED) + { + throw_errno("host pool view: mapping the pool"); + } + m_memory = static_cast(memory); +} + +HostPoolView::~HostPoolView() +{ + unmap(); +} + +HostPoolView::HostPoolView(HostPoolView&& other) noexcept + : m_memory(std::exchange(other.m_memory, nullptr)), + m_slot_count(std::exchange(other.m_slot_count, 0)), + m_slot_bytes(other.m_slot_bytes), + m_slot_stride(other.m_slot_stride) +{ +} + +HostPoolView& HostPoolView::operator=(HostPoolView&& other) noexcept +{ + if (this != &other) + { + unmap(); + m_memory = std::exchange(other.m_memory, nullptr); + m_slot_count = std::exchange(other.m_slot_count, 0); + m_slot_bytes = other.m_slot_bytes; + m_slot_stride = other.m_slot_stride; + } + return *this; +} + +std::span HostPoolView::slot(std::uint32_t index) const +{ + if (index >= m_slot_count) + { + throw std::out_of_range("host pool view has no slot " + std::to_string(index)); + } + return { m_memory + m_slot_stride * index, m_slot_bytes }; +} + +void HostPoolView::unmap() +{ + if (m_memory != nullptr) + { + ::munmap(const_cast(m_memory), m_slot_stride * m_slot_count); + m_memory = nullptr; + m_slot_count = 0; + } +} + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp b/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp new file mode 100644 index 0000000000..54694f1436 --- /dev/null +++ b/src/core/camera_ipc/cpp/inc/camera_ipc/host_pool.hpp @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "unique_fd.hpp" + +#include +#include +#include +#include + +namespace core::camera +{ + +/** + * @brief A pool of equally sized slots in one block of shared host memory. + * + * The process that creates the pool writes the slots. Any process that is sent the pool's + * descriptor can map it, but only for reading: the memory is sealed against new writable + * mappings, writes and resizing. + */ +class HostPool +{ +public: + /** + * @param name Shows up in /proc for debugging; it does not have to be unique. + * @throws std::invalid_argument for an empty pool. + * @throws std::system_error if the memory cannot be allocated or sealed. + */ + HostPool(std::uint32_t slot_count, std::uint64_t slot_bytes, const std::string& name); + ~HostPool(); + + HostPool(const HostPool&) = delete; + HostPool& operator=(const HostPool&) = delete; + + std::uint32_t slot_count() const + { + return m_slot_count; + } + + std::uint64_t slot_bytes() const + { + return m_slot_bytes; + } + + /** + * @brief The writable memory of one slot. + * @throws std::out_of_range for a slot the pool does not have. + */ + std::span slot(std::uint32_t index); + + /** + * @brief The pool's descriptor, to send to another process. The pool keeps owning it. + */ + int descriptor() const + { + return m_fd.get(); + } + +private: + UniqueFd m_fd; + std::byte* m_memory = nullptr; + std::uint32_t m_slot_count = 0; + std::uint64_t m_slot_bytes = 0; + std::uint64_t m_slot_stride = 0; // from the start of one slot to the start of the next +}; + +/** + * @brief Another process's read-only mapping of a HostPool, made from the descriptor it was sent. + */ +class HostPoolView +{ +public: + /** + * @param descriptor Of the pool. It is closed once the pool is mapped. + * @param slot_count, slot_bytes As the pool was created with. + * @throws std::system_error if the descriptor cannot be mapped. + * @throws std::invalid_argument if the descriptor holds less than the pool needs. + */ + HostPoolView(UniqueFd descriptor, std::uint32_t slot_count, std::uint64_t slot_bytes); + ~HostPoolView(); + + HostPoolView(HostPoolView&& other) noexcept; + HostPoolView& operator=(HostPoolView&& other) noexcept; + HostPoolView(const HostPoolView&) = delete; + HostPoolView& operator=(const HostPoolView&) = delete; + + std::uint32_t slot_count() const + { + return m_slot_count; + } + + /** + * @brief The memory of one slot. + * @throws std::out_of_range for a slot the pool does not have. + */ + std::span slot(std::uint32_t index) const; + +private: + void unmap(); + + const std::byte* m_memory = nullptr; + std::uint32_t m_slot_count = 0; + std::uint64_t m_slot_bytes = 0; + std::uint64_t m_slot_stride = 0; +}; + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/inc/camera_ipc/message_socket.hpp b/src/core/camera_ipc/cpp/inc/camera_ipc/message_socket.hpp new file mode 100644 index 0000000000..0a5954dd08 --- /dev/null +++ b/src/core/camera_ipc/cpp/inc/camera_ipc/message_socket.hpp @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "unique_fd.hpp" + +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// A message carries at most this many descriptors. +inline constexpr std::size_t MAX_DESCRIPTORS_PER_MESSAGE = 64; + +enum class SendStatus +{ + SENT, + WOULD_BLOCK, // the peer's queue is full; nothing was sent + CLOSED, // the peer is gone +}; + +enum class ReceiveStatus +{ + MESSAGE, + TIMEOUT, + CLOSED, // the peer is gone +}; + +struct ReceivedMessage +{ + ReceiveStatus status = ReceiveStatus::TIMEOUT; + std::size_t bytes = 0; // of the payload, when status is MESSAGE + std::vector descriptors; // the ones that came with the message +}; + +/** + * @brief One end of a connection between two processes on this machine. It carries whole + * messages in order, each with optional file descriptors. + */ +class MessageSocket +{ +public: + /** + * @brief Connect to a MessageListener. + * @throws std::system_error if nothing listens on the path. + */ + static MessageSocket connect(const std::string& path); + + explicit MessageSocket(UniqueFd fd); + + /** + * @brief Send one message. Never blocks. + * @throws std::invalid_argument for an empty payload (it would read as a closed connection) + * or more than MAX_DESCRIPTORS_PER_MESSAGE descriptors. + * @throws std::system_error for a failure other than a full queue or a closed peer. + */ + SendStatus send(std::span payload, std::span descriptors = {}); + + /** + * @brief Wait up to `timeout` for one message and copy its payload into `buffer`. + * @throws std::length_error if the message does not fit the buffer. + * @throws std::system_error for a failure other than a closed peer. + */ + ReceivedMessage receive(std::span buffer, std::chrono::milliseconds timeout); + + // For waiting on several sockets at once. + int descriptor() const + { + return m_fd.get(); + } + +private: + UniqueFd m_fd; +}; + +/** + * @brief Accepts connections on a socket file. Only processes of the same user can connect. + * + * A lock file next to the socket file (`.lock`) decides which process owns the path, so + * of two processes that start at the same moment exactly one listens. The lock file stays + * behind, empty, for the next listener to lock. + */ +class MessageListener +{ +public: + /** + * @brief Start listening on `path`. A socket file left behind by a process that has gone is + * replaced; anything else found at the path is left alone. + * @throws std::system_error with EADDRINUSE if another process listens on the path, with + * EEXIST if the path holds something that is not a socket, or with the cause if the path + * cannot be locked or bound. + */ + explicit MessageListener(const std::string& path); + + // Removes the socket file. + ~MessageListener(); + + MessageListener(const MessageListener&) = delete; + MessageListener& operator=(const MessageListener&) = delete; + + /** + * @brief Wait up to `timeout` for a connection. + * @return The connection, or nothing on timeout. + */ + std::optional accept(std::chrono::milliseconds timeout); + + int descriptor() const + { + return m_fd.get(); + } + + const std::string& path() const + { + return m_path; + } + +private: + UniqueFd m_lock; // declared first, so it is released after the socket is closed + UniqueFd m_fd; + std::string m_path; +}; + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp b/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp new file mode 100644 index 0000000000..387bf46515 --- /dev/null +++ b/src/core/camera_ipc/cpp/inc/camera_ipc/unique_fd.hpp @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +namespace core::camera +{ + +// Owns one file descriptor and closes it. +class UniqueFd +{ +public: + UniqueFd() = default; + + explicit UniqueFd(int fd) : m_fd(fd) + { + } + + UniqueFd(UniqueFd&& other) noexcept : m_fd(other.release()) + { + } + + UniqueFd& operator=(UniqueFd&& other) noexcept + { + if (this != &other) + { + reset(other.release()); + } + return *this; + } + + UniqueFd(const UniqueFd&) = delete; + UniqueFd& operator=(const UniqueFd&) = delete; + + ~UniqueFd() + { + reset(); + } + + int get() const + { + return m_fd; + } + + bool valid() const + { + return m_fd >= 0; + } + + // Gives the descriptor up without closing it. + int release() + { + const int fd = m_fd; + m_fd = -1; + return fd; + } + + void reset(int fd = -1) + { + if (m_fd >= 0) + { + ::close(m_fd); + } + m_fd = fd; + } + +private: + int m_fd = -1; +}; + +} // namespace core::camera diff --git a/src/core/camera_ipc/cpp/message_socket.cpp b/src/core/camera_ipc/cpp/message_socket.cpp new file mode 100644 index 0000000000..05425aa36d --- /dev/null +++ b/src/core/camera_ipc/cpp/message_socket.cpp @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_ipc/message_socket.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +[[noreturn]] void throw_errno(const std::string& what) +{ + throw std::system_error(errno, std::generic_category(), what); +} + +sockaddr_un address_of(const std::string& path) +{ + sockaddr_un address = {}; + address.sun_family = AF_UNIX; + if (path.empty() || path.size() >= sizeof(address.sun_path)) + { + throw std::invalid_argument("socket path '" + path + "' is empty or too long"); + } + std::memcpy(address.sun_path, path.data(), path.size()); + return address; +} + +UniqueFd new_socket(int extra_flags = 0) +{ + UniqueFd fd(::socket(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC | extra_flags, 0)); + if (!fd.valid()) + { + throw_errno("creating a socket"); + } + return fd; +} + +// Waits until the descriptor is readable. Returns false on timeout. +bool wait_readable(int fd, std::chrono::milliseconds timeout) +{ + using Milliseconds = std::chrono::milliseconds; + const Milliseconds longest(std::numeric_limits::max()); // what poll() can wait + const auto deadline = std::chrono::steady_clock::now() + std::clamp(timeout, Milliseconds::zero(), longest); + pollfd entry = { fd, POLLIN, 0 }; + for (;;) + { + // An interrupted wait resumes with the time that is left, not the whole timeout again. + const auto left = std::chrono::ceil(deadline - std::chrono::steady_clock::now()); + const int ready = ::poll(&entry, 1, static_cast(std::max(left, Milliseconds::zero()).count())); + if (ready > 0) + { + return true; + } + if (ready == 0) + { + return false; + } + if (errno != EINTR) + { + throw_errno("waiting on a socket"); + } + } +} + +} // namespace + +MessageSocket MessageSocket::connect(const std::string& path) +{ + const sockaddr_un address = address_of(path); + UniqueFd fd = new_socket(); + if (::connect(fd.get(), reinterpret_cast(&address), sizeof(address)) != 0) + { + throw_errno("connecting to '" + path + "'"); + } + return MessageSocket(std::move(fd)); +} + +MessageSocket::MessageSocket(UniqueFd fd) : m_fd(std::move(fd)) +{ +} + +SendStatus MessageSocket::send(std::span payload, std::span descriptors) +{ + if (payload.empty() || descriptors.size() > MAX_DESCRIPTORS_PER_MESSAGE) + { + throw std::invalid_argument("a message needs a payload and carries at most " + + std::to_string(MAX_DESCRIPTORS_PER_MESSAGE) + " descriptors"); + } + iovec data = { const_cast(payload.data()), payload.size() }; + msghdr message = {}; + message.msg_iov = &data; + message.msg_iovlen = 1; + + alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int) * MAX_DESCRIPTORS_PER_MESSAGE)]; + if (!descriptors.empty()) + { + message.msg_control = control; + message.msg_controllen = CMSG_SPACE(descriptors.size_bytes()); + cmsghdr* header = CMSG_FIRSTHDR(&message); + header->cmsg_level = SOL_SOCKET; + header->cmsg_type = SCM_RIGHTS; + header->cmsg_len = CMSG_LEN(descriptors.size_bytes()); + std::memcpy(CMSG_DATA(header), descriptors.data(), descriptors.size_bytes()); + } + + for (;;) + { + if (::sendmsg(m_fd.get(), &message, MSG_DONTWAIT | MSG_NOSIGNAL) >= 0) + { + return SendStatus::SENT; + } + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + return SendStatus::WOULD_BLOCK; + } + if (errno == EPIPE || errno == ECONNRESET || errno == ENOTCONN) + { + return SendStatus::CLOSED; + } + if (errno != EINTR) + { + throw_errno("sending a message"); + } + } +} + +ReceivedMessage MessageSocket::receive(std::span buffer, std::chrono::milliseconds timeout) +{ + ReceivedMessage result; + if (!wait_readable(m_fd.get(), timeout)) + { + return result; + } + iovec data = { buffer.data(), buffer.size() }; + alignas(cmsghdr) char control[CMSG_SPACE(sizeof(int) * MAX_DESCRIPTORS_PER_MESSAGE)]; + msghdr message = {}; + message.msg_iov = &data; + message.msg_iovlen = 1; + message.msg_control = control; + message.msg_controllen = sizeof(control); + + ssize_t received = 0; + do + { + received = ::recvmsg(m_fd.get(), &message, MSG_DONTWAIT | MSG_CMSG_CLOEXEC); + } while (received < 0 && errno == EINTR); + if (received < 0) + { + if (errno == EAGAIN || errno == EWOULDBLOCK) + { + return result; + } + if (errno == ECONNRESET) + { + result.status = ReceiveStatus::CLOSED; + return result; + } + throw_errno("receiving a message"); + } + + // Take ownership of every descriptor first, so none leaks if the message is rejected below. + for (cmsghdr* header = CMSG_FIRSTHDR(&message); header != nullptr; header = CMSG_NXTHDR(&message, header)) + { + if (header->cmsg_level != SOL_SOCKET || header->cmsg_type != SCM_RIGHTS) + { + continue; + } + const std::size_t count = (header->cmsg_len - CMSG_LEN(0)) / sizeof(int); + for (std::size_t i = 0; i < count; ++i) + { + int fd = -1; + std::memcpy(&fd, CMSG_DATA(header) + i * sizeof(int), sizeof(int)); + result.descriptors.emplace_back(fd); + } + } + // Checked before the empty read below: a cut message can also come back with no bytes. + if ((message.msg_flags & (MSG_TRUNC | MSG_CTRUNC)) != 0) + { + throw std::length_error("received a message larger than the buffer or with too many descriptors"); + } + if (received == 0 && result.descriptors.empty()) + { + result.status = ReceiveStatus::CLOSED; // an orderly shutdown reads as an empty message + return result; + } + result.status = ReceiveStatus::MESSAGE; + result.bytes = static_cast(received); + return result; +} + +MessageListener::MessageListener(const std::string& path) : m_path(path) +{ + const sockaddr_un address = address_of(path); + + // The lock decides who owns the path. The kernel drops it when its holder dies, so a socket + // file found while holding it was left by a process that is gone. + const std::string lock_path = path + ".lock"; + m_lock.reset(::open(lock_path.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, S_IRUSR | S_IWUSR)); + if (!m_lock.valid()) + { + throw_errno("opening '" + lock_path + "'"); + } + if (::flock(m_lock.get(), LOCK_EX | LOCK_NB) != 0) + { + if (errno == EWOULDBLOCK) + { + throw std::system_error(EADDRINUSE, std::generic_category(), "another process listens on '" + path + "'"); + } + throw_errno("locking '" + lock_path + "'"); + } + struct stat entry = {}; + if (::lstat(path.c_str(), &entry) == 0) + { + if (!S_ISSOCK(entry.st_mode)) + { + throw std::system_error(EEXIST, std::generic_category(), "'" + path + "' exists and is not a socket"); + } + ::unlink(path.c_str()); + } + else if (errno != ENOENT) + { + throw_errno("inspecting '" + path + "'"); + } + + // Not blocking: a connection that goes away between the wait and the accept must not stall accept(). + m_fd = new_socket(SOCK_NONBLOCK); + // The socket file is created with the socket's own mode: only this user may connect. + if (::fchmod(m_fd.get(), S_IRUSR | S_IWUSR) != 0) + { + throw_errno("restricting '" + path + "'"); + } + if (::bind(m_fd.get(), reinterpret_cast(&address), sizeof(address)) != 0) + { + throw_errno("binding '" + path + "'"); + } + if (::listen(m_fd.get(), SOMAXCONN) != 0) + { + const int error = errno; + ::unlink(path.c_str()); + throw std::system_error(error, std::generic_category(), "listening on '" + path + "'"); + } +} + +MessageListener::~MessageListener() +{ + ::unlink(m_path.c_str()); +} + +std::optional MessageListener::accept(std::chrono::milliseconds timeout) +{ + if (!wait_readable(m_fd.get(), timeout)) + { + return std::nullopt; + } + UniqueFd connection(::accept4(m_fd.get(), nullptr, nullptr, SOCK_CLOEXEC)); + if (!connection.valid()) + { + if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ECONNABORTED || errno == EINTR) + { + return std::nullopt; + } + throw_errno("accepting a connection on '" + m_path + "'"); + } + return MessageSocket(std::move(connection)); +} + +} // namespace core::camera diff --git a/src/core/camera_reader/AGENTS.md b/src/core/camera_reader/AGENTS.md new file mode 100644 index 0000000000..732ad6314a --- /dev/null +++ b/src/core/camera_reader/AGENTS.md @@ -0,0 +1,14 @@ + + +# Agent notes — `camera_reader` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +- **The public header shows `camera_core` types only.** The socket, the pools and the wire messages stay in the `.cpp`; an application that links the reader does not see them. +- **No copies and no conversions here.** A view is the plugin's buffer. Pixel conversion, if a consumer wants it, is a separate step on top. +- **A frame set outlives its reader safely.** Both share one state object; keep it that way rather than making `FrameSet` point back at the reader. +- **Releasing must never throw or block.** It runs in destructors, possibly on another thread; a plugin that is gone takes its frames back by itself. +- **The plugin is trusted for content, not for bounds.** Check a frame event against the group before handing out views. diff --git a/src/core/camera_reader/CMakeLists.txt b/src/core/camera_reader/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_reader/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_reader/README.md b/src/core/camera_reader/README.md new file mode 100644 index 0000000000..605857a925 --- /dev/null +++ b/src/core/camera_reader/README.md @@ -0,0 +1,40 @@ + + +# Camera interface: the reader + +What an application links to get camera frames. It talks to the camera plugin's +[endpoint](../camera_endpoint) and hands out frame sets whose pixels are the plugin's own +buffers, mapped read-only; nothing is copied. + +```cpp +#include + +CameraReader reader = CameraReader::open(endpoint, "head"); +while (auto frames = reader.next(std::chrono::seconds(1))) +{ + const ImageView left = frames->image(0); + process(left.plane(0), left.stream->width, left.stream->height, left.stream->planes[0].stride_bytes); +} +``` + +- **A `FrameSet` is a hold on the frames.** While it exists the plugin does not reuse those + buffers, so the pixels stay as they are. It gives them back when it is destroyed, or + earlier with `release()`. Hold at most as many as the reader's window. +- **Window and order** are chosen in `open` ([terms](../camera_core/README.md)). The default, + a window of one, always gives the newest frame set and says in `skipped()` how many were + passed over. A recorder asks for a larger window and gets every frame set in order. +- **`next` returns nothing on timeout** and throws `CameraDisconnected` when the plugin is + gone. Opening fails with a `CameraReaderError` that says why: nothing listens on the + endpoint, the group does not exist, or the subscription does not fit the pool. +- **One reader reads one group.** For two groups, open two readers; `descriptor()` lets an + application wait on several at once. +- A `FrameSet` may be given back from another thread than the one that calls `next`, and + after its reader is gone. + +## Tests + +`tests/cpp/core/camera_reader`, run with `ctest -L camera_reader`. They are tagged `unit` +and read from a real endpoint in the same process. diff --git a/src/core/camera_reader/cpp/CMakeLists.txt b/src/core/camera_reader/cpp/CMakeLists.txt new file mode 100644 index 0000000000..c2ad541a18 --- /dev/null +++ b/src/core/camera_reader/cpp/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# What an application links to read cameras from the camera plugin. +add_library(camera_reader STATIC + camera_reader.cpp +) + +target_include_directories(camera_reader INTERFACE + $ +) + +target_link_libraries(camera_reader + PUBLIC + camera::camera_core + PRIVATE + camera::camera_ipc + camera::camera_wire +) + +add_library(camera::camera_reader ALIAS camera_reader) diff --git a/src/core/camera_reader/cpp/camera_reader.cpp b/src/core/camera_reader/cpp/camera_reader.cpp new file mode 100644 index 0000000000..a09a445916 --- /dev/null +++ b/src/core/camera_reader/cpp/camera_reader.cpp @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_reader/camera_reader.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// How long the plugin gets to answer the handshake and the subscription. +constexpr std::chrono::milliseconds ANSWER_TIMEOUT(2000); + +struct Received +{ + WireMessage message; + std::vector descriptors; +}; + +// Waits for one message. Nothing on timeout. +std::optional receive(MessageSocket& socket, std::span buffer, std::chrono::milliseconds timeout) +{ + ReceivedMessage received = socket.receive(buffer, timeout); + if (received.status == ReceiveStatus::TIMEOUT) + { + return std::nullopt; + } + if (received.status == ReceiveStatus::CLOSED) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + try + { + return Received{ decode_wire_message(buffer.first(received.bytes)), std::move(received.descriptors) }; + } + catch (const WireError& error) + { + throw CameraReaderError(std::string("the camera plugin sent something that is not a message: ") + error.what()); + } +} + +MessageSocket connect_to(const std::string& endpoint) +{ + try + { + return MessageSocket::connect(endpoint); + } + catch (const std::system_error& error) + { + throw CameraReaderError("no camera plugin answers on '" + endpoint + "': " + error.code().message()); + } +} + +// Says hello and returns what the plugin publishes. +std::vector greet(MessageSocket& socket, WireEncoder& encoder, std::span buffer) +{ + HelloMessage hello; + hello.reader_name = program_invocation_short_name; + if (socket.send(encoder.encode(hello)) != SendStatus::SENT) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + const auto deadline = std::chrono::steady_clock::now() + ANSWER_TIMEOUT; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + std::optional received = receive(socket, buffer, left); + if (!received) + { + throw CameraReaderError("the camera plugin did not answer the reader's hello"); + } + if (auto* welcome = std::get_if(&received->message)) + { + if (!welcome->error.empty()) + { + throw CameraReaderError("the camera plugin refused this reader: " + welcome->error); + } + return std::move(welcome->groups); + } + } +} + +} // namespace + +namespace detail +{ + +// Shared by a reader and the frame sets it handed out, so a frame set can be given back after +// its reader is gone. +class ReaderState +{ +public: + explicit ReaderState(MessageSocket connected) : socket(std::move(connected)), buffer(MAX_WIRE_MESSAGE_BYTES) + { + } + + // Called from a FrameSet, possibly on another thread than the one that reads. + void release(std::uint64_t lease_id) noexcept + { + try + { + const std::lock_guard lock(m_release_mutex); + (void)socket.send(m_release_encoder.encode(ReleaseMessage{ lease_id })); + } + catch (const std::exception&) + { + // The plugin is gone or will drop this reader; either way it takes the frame set back. + } + } + + // The next frame event, from the ones that arrived early or from the socket. + std::optional take(std::chrono::milliseconds timeout) + { + if (!early.empty()) + { + const FrameEvent event = early.front(); + early.pop_front(); + return event; + } + const auto deadline = std::chrono::steady_clock::now() + timeout; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + const std::optional received = receive(socket, buffer, left); + if (!received) + { + return std::nullopt; + } + if (const auto* event = std::get_if(&received->message)) + { + return *event; + } + // Anything else is not for a subscribed reader of this build; keep waiting. + } + } + + // A frame set that names a slot or a size the pools do not have cannot be read safely. + void check(const FrameEvent& event) const + { + bool fits = event.image_count == group.streams.size(); + for (std::size_t i = 0; fits && i < event.image_count; ++i) + { + fits = event.images[i].slot < group.streams[i].slot_count && + event.images[i].bytes_used <= group.streams[i].slot_bytes; + } + if (!fits) + { + throw CameraReaderError("the camera plugin sent a frame set that does not match group '" + group.id + "'"); + } + } + + MessageSocket socket; + GroupDescription group; + ReaderOptions options; + std::vector pools; // one per stream + std::deque early; // frame events that arrived before the answer to the subscription + std::vector buffer; // the reading thread only + +private: + std::mutex m_release_mutex; + WireEncoder m_release_encoder; +}; + +} // namespace detail + +std::span ImageView::plane(std::uint32_t index) const +{ + if (stream == nullptr || index >= stream->plane_count) + { + throw std::out_of_range("the image has no plane " + std::to_string(index)); + } + const PlaneSize size = plane_size_of(stream->pixel_format, stream->width, stream->height, index); + const std::uint64_t offset = stream->planes[index].offset_bytes; + const std::uint64_t bytes = stream->planes[index].stride_bytes * (size.rows - 1) + size.row_bytes; + if (size.rows == 0 || offset + bytes > data.size()) + { + throw std::out_of_range("plane " + std::to_string(index) + " does not lie inside the image's slot"); + } + return data.subspan(offset, bytes); +} + +FrameSet::FrameSet(std::shared_ptr state, const FrameEvent& event) + : m_state(std::move(state)), m_event(event) +{ +} + +FrameSet::FrameSet(FrameSet&& other) noexcept : m_state(std::move(other.m_state)), m_event(other.m_event) +{ +} + +FrameSet& FrameSet::operator=(FrameSet&& other) noexcept +{ + if (this != &other) + { + release(); + m_state = std::move(other.m_state); + m_event = other.m_event; + } + return *this; +} + +FrameSet::~FrameSet() +{ + release(); +} + +ImageView FrameSet::image(std::size_t stream) const +{ + if (!m_state) + { + throw std::logic_error("the frame set was given back"); + } + if (stream >= m_event.image_count) + { + throw std::out_of_range("the frame set has no image " + std::to_string(stream)); + } + const StreamDescription& description = m_state->group.streams[stream]; + const ImageInfo& info = m_event.images[stream]; + const std::span slot = m_state->pools[stream].slot(info.slot); + return { &description, info, description.is_encoded() ? slot.first(info.bytes_used) : slot }; +} + +void FrameSet::release() noexcept +{ + if (m_state) + { + m_state->release(m_event.lease_id); + m_state.reset(); + } +} + +std::vector CameraReader::list(const std::string& endpoint) +{ + MessageSocket socket = connect_to(endpoint); + WireEncoder encoder; + std::vector buffer(MAX_WIRE_MESSAGE_BYTES); + return greet(socket, encoder, buffer); +} + +CameraReader CameraReader::open(const std::string& endpoint, const std::string& group, const ReaderOptions& options) +{ + auto state = std::make_shared(connect_to(endpoint)); + WireEncoder encoder; + greet(state->socket, encoder, state->buffer); + + if (state->socket.send(encoder.encode(SubscribeMessage{ group, options })) != SendStatus::SENT) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + const auto deadline = std::chrono::steady_clock::now() + ANSWER_TIMEOUT; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + std::optional received = receive(state->socket, state->buffer, left); + if (!received) + { + throw CameraReaderError("the camera plugin did not answer the subscription to '" + group + "'"); + } + // The plugin starts sending frame sets as soon as the subscription exists, which can be + // before its answer is on the way. + if (const auto* event = std::get_if(&received->message)) + { + state->early.push_back(*event); + continue; + } + auto* subscribed = std::get_if(&received->message); + if (subscribed == nullptr) + { + continue; + } + if (!subscribed->error.empty()) + { + throw CameraReaderError(subscribed->error); + } + if (received->descriptors.size() != subscribed->group.streams.size()) + { + throw CameraReaderError("the camera plugin did not send one pool per stream of '" + group + "'"); + } + state->group = std::move(subscribed->group); + state->options = { options.window, resolved_order(options) }; + try + { + for (std::size_t i = 0; i < state->group.streams.size(); ++i) + { + const StreamDescription& stream = state->group.streams[i]; + state->pools.emplace_back(std::move(received->descriptors[i]), stream.slot_count, stream.slot_bytes); + } + } + catch (const std::exception& error) + { + throw CameraReaderError("cannot map the pools of '" + group + "': " + error.what()); + } + return CameraReader(std::move(state)); + } +} + +CameraReader::CameraReader(std::shared_ptr state) : m_state(std::move(state)) +{ +} + +CameraReader::CameraReader(CameraReader&& other) noexcept = default; +CameraReader& CameraReader::operator=(CameraReader&& other) noexcept = default; +CameraReader::~CameraReader() = default; + +const GroupDescription& CameraReader::group() const +{ + return m_state->group; +} + +std::optional CameraReader::next(std::chrono::milliseconds timeout) +{ + std::optional event = m_state->take(timeout); + if (!event) + { + return std::nullopt; + } + // With a window of one nothing can be waiting behind this event, so the extra look at the + // socket is skipped. + if (m_state->options.order == DeliveryOrder::NEWEST && m_state->options.window > 1) + { + while (const std::optional newer = m_state->take(std::chrono::milliseconds(0))) + { + m_state->release(event->lease_id); + const std::uint64_t skipped = std::uint64_t{ event->skipped } + newer->skipped + 1; + event = newer; + event->skipped = static_cast(std::min(skipped, UINT32_MAX)); + } + } + try + { + m_state->check(*event); + } + catch (const CameraReaderError&) + { + m_state->release(event->lease_id); + throw; + } + return FrameSet(m_state, *event); +} + +int CameraReader::descriptor() const +{ + return m_state->socket.descriptor(); +} + +} // namespace core::camera diff --git a/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp b/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp new file mode 100644 index 0000000000..0920638a28 --- /dev/null +++ b/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace detail +{ +class ReaderState; +} + +/** + * @brief A reader could not be opened or cannot go on: nothing listens on the endpoint, the + * plugin refused the reader, or it sent something a reader cannot use. + */ +class CameraReaderError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +/** + * @brief The camera plugin closed the connection or is gone. + */ +class CameraDisconnected : public CameraReaderError +{ +public: + using CameraReaderError::CameraReaderError; +}; + +/** + * @brief One image of a frame set. The memory is the plugin's pool, mapped read-only, and + * stays as it is for as long as the FrameSet the view came from is held. + */ +struct ImageView +{ + const StreamDescription* stream = nullptr; // what the bytes are; owned by the reader + ImageInfo info; + std::span data; // pixels: the whole slot; encoded video: the access unit + + /** + * @brief The bytes of one plane of a pixel image, from its first row to the end of its last. + * Rows are stream->planes[index].stride_bytes apart. + * @throws std::out_of_range for a plane the image does not have. + */ + std::span plane(std::uint32_t index) const; +}; + +/** + * @brief One frame set, held for this reader: the plugin does not reuse its slots until the + * object is destroyed or release() is called. Hold at most as many as the reader's window. + */ +class FrameSet +{ +public: + FrameSet(FrameSet&& other) noexcept; + FrameSet& operator=(FrameSet&& other) noexcept; + FrameSet(const FrameSet&) = delete; + FrameSet& operator=(const FrameSet&) = delete; + ~FrameSet(); + + std::uint64_t sequence() const + { + return m_event.sequence; + } + + // Changes when the camera restarted. + std::uint32_t generation() const + { + return m_event.generation; + } + + // Frame sets published since the previous one this reader was given, and not given to it. + std::uint32_t skipped() const + { + return m_event.skipped; + } + + const FrameTimeline& timeline() const + { + return m_event.timeline; + } + + std::size_t image_count() const + { + return m_event.image_count; + } + + /** + * @brief The image of one stream, in the order of the group's streams. + * @throws std::out_of_range for a stream the group does not have. + * @throws std::logic_error after release(). + */ + ImageView image(std::size_t stream) const; + + // Gives the frame set back before the object is destroyed. Its views are no longer valid. + void release() noexcept; + +private: + friend class CameraReader; + FrameSet(std::shared_ptr state, const FrameEvent& event); + + std::shared_ptr m_state; + FrameEvent m_event; +}; + +/** + * @brief Reads one group of cameras from the camera plugin. + * + * @code + * CameraReader reader = CameraReader::open(endpoint, "head"); + * while (auto frames = reader.next(std::chrono::seconds(1))) + * { + * std::span left = frames->image(0).data; + * } + * @endcode + */ +class CameraReader +{ +public: + /** + * @brief What the plugin behind `endpoint` publishes. + * @throws CameraReaderError if nothing listens there or the plugin refuses this reader. + */ + static std::vector list(const std::string& endpoint); + + /** + * @brief Subscribe to one group. A reader for a second group is a second CameraReader. + * @throws CameraReaderError if nothing listens on the endpoint or the plugin refuses the + * subscription; the message says why. + */ + static CameraReader open(const std::string& endpoint, const std::string& group, const ReaderOptions& options = {}); + + CameraReader(CameraReader&& other) noexcept; + CameraReader& operator=(CameraReader&& other) noexcept; + CameraReader(const CameraReader&) = delete; + CameraReader& operator=(const CameraReader&) = delete; + ~CameraReader(); + + const GroupDescription& group() const; + + /** + * @brief Wait up to `timeout` for a frame set. With newest order this is the newest one the + * plugin has sent; older ones waiting behind it are given back and counted as skipped. + * @return The frame set, or nothing on timeout. + * @throws CameraDisconnected if the plugin is gone. + * @throws CameraReaderError if the plugin sent a frame set that does not match the group. + */ + std::optional next(std::chrono::milliseconds timeout); + + // Becomes readable when a frame set may be waiting; for waiting on several readers at once. + int descriptor() const; + +private: + explicit CameraReader(std::shared_ptr state); + + std::shared_ptr m_state; +}; + +} // namespace core::camera diff --git a/src/core/camera_wire/AGENTS.md b/src/core/camera_wire/AGENTS.md new file mode 100644 index 0000000000..a7596019c1 --- /dev/null +++ b/src/core/camera_wire/AGENTS.md @@ -0,0 +1,16 @@ + + +# Agent notes — `camera_wire` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) before changing `fbs/camera_wire.fbs`: a reader and a plugin from different builds must keep understanding each other. + +- **The schema only grows.** Append fields with fresh ids, optional; append union members; tables, never structs. Do not reuse or renumber. +- **FlatBuffers stays inside this module.** Its users see `camera_core` types; only the codec and its tests include the generated header. +- **Decoded bytes are untrusted.** Verify, then bound every count against the fixed arrays it fills, and check for a body behind a body type. +- **A new enum value needs both sides.** The core enum and the schema enum carry the same numbers (pinned by `static_assert`), and the decoder must accept the value before a plugin sends it. +- **The frame path does not allocate.** Keep `encode(const FrameEvent&)` free of containers that do. diff --git a/src/core/camera_wire/CMakeLists.txt b/src/core/camera_wire/CMakeLists.txt new file mode 100644 index 0000000000..e162dea055 --- /dev/null +++ b/src/core/camera_wire/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(fbs) +add_subdirectory(cpp) diff --git a/src/core/camera_wire/README.md b/src/core/camera_wire/README.md new file mode 100644 index 0000000000..20effdcb4b --- /dev/null +++ b/src/core/camera_wire/README.md @@ -0,0 +1,58 @@ + + +# Camera interface: messages between the plugin and a reader + +What the camera plugin and a reader say to each other over their socket +([`camera_ipc`](../camera_ipc)). Each socket message is one of these: + +```text +reader plugin + | Hello (protocol version) | + |------------------------------------>| + | Welcome (what is published) | + |<------------------------------------| + | Subscribe (group, window, order) | + |------------------------------------>| + | Subscribed + one pool descriptor | + | per stream | + |<------------------------------------| + | Frame (lease, slot per image) | once per frame set + |<------------------------------------| + | Release (lease) | + |------------------------------------>| +``` + +A connection carries one subscription; a reader that wants two groups opens two. Closing the +connection ends the subscription and returns every frame set the reader still held. A `Frame` +can overtake `Subscribed`, because capture does not wait for the answer to be sent; it belongs +to the subscription all the same. + +## Schema + +[`fbs/camera_wire.fbs`](fbs/camera_wire.fbs) defines the messages as FlatBuffers tables. It is +kept here and not in [`schema`](../schema): that directory holds what goes into recordings, +and these messages are never recorded. + +A reader and a plugin may come from different builds, so the schema only grows: + +- Append fields with fresh ids and leave them optional; append union members. Never reuse or + renumber an id, and use tables, not structs. +- A message type this build does not know decodes to nothing and is ignored. +- A stream whose format or kind of memory this build does not know makes its group + unreadable, and the reader is told so; other groups are unaffected. +- `WIRE_PROTOCOL_VERSION` changes only when the two sides would misread each other. The + plugin refuses a reader with a different version in `Welcome`. + +## Codec + +`WireEncoder` and `decode_wire_message` turn [`camera_core`](../camera_core) types into bytes +and back, so nothing outside this module sees FlatBuffers. The encoder reuses its buffer; a +frame event is encoded without allocating. The decoder treats its input as coming from another +process: it verifies the buffer and refuses counts that would not fit a group. + +## Tests + +`tests/cpp/core/camera_wire`, run with `ctest -L camera_wire`. They are tagged `unit`. diff --git a/src/core/camera_wire/cpp/CMakeLists.txt b/src/core/camera_wire/cpp/CMakeLists.txt new file mode 100644 index 0000000000..a4d4122dbb --- /dev/null +++ b/src/core/camera_wire/cpp/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Turns the messages between the camera plugin and its readers into bytes and back. The +# FlatBuffers types stay inside this library; its users see camera_core types. +add_library(camera_wire STATIC + codec.cpp +) + +target_include_directories(camera_wire INTERFACE + $ +) + +target_link_libraries(camera_wire + PUBLIC + camera::camera_core + PRIVATE + camera::camera_wire_schema +) + +add_library(camera::camera_wire ALIAS camera_wire) diff --git a/src/core/camera_wire/cpp/codec.cpp b/src/core/camera_wire/cpp/codec.cpp new file mode 100644 index 0000000000..d0856fb521 --- /dev/null +++ b/src/core/camera_wire/cpp/codec.cpp @@ -0,0 +1,341 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_wire/messages.hpp" + +#include + +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// The schema's enums carry the same numbers as the core's, so a value converts by a cast. +static_assert(static_cast(PixelFormat::BGRA8) == wire::PixelFormat_Bgra8); +static_assert(static_cast(PixelFormat::RGBA8) == wire::PixelFormat_Rgba8); +static_assert(static_cast(PixelFormat::NV12) == wire::PixelFormat_Nv12); +static_assert(static_cast(PixelFormat::GRAY8) == wire::PixelFormat_Gray8); +static_assert(static_cast(Codec::NONE) == wire::Codec_None); +static_assert(static_cast(Codec::H264) == wire::Codec_H264); +static_assert(static_cast(Codec::H265) == wire::Codec_H265); +static_assert(static_cast(MemoryKind::HOST) == wire::MemoryKind_Host); +static_assert(static_cast(DeliveryOrder::DEFAULT) == wire::DeliveryOrder_Default); +static_assert(static_cast(DeliveryOrder::NEWEST) == wire::DeliveryOrder_Newest); +static_assert(static_cast(DeliveryOrder::IN_ORDER) == wire::DeliveryOrder_InOrder); + +using Builder = flatbuffers::FlatBufferBuilder; + +flatbuffers::Offset build_group(Builder& builder, const GroupDescription& group) +{ + std::vector> streams; + streams.reserve(group.streams.size()); + for (const StreamDescription& stream : group.streams) + { + std::array, MAX_PLANES_PER_IMAGE> planes; + const std::size_t plane_count = std::min(stream.plane_count, MAX_PLANES_PER_IMAGE); + for (std::size_t i = 0; i < plane_count; ++i) + { + planes[i] = wire::CreatePlaneLayout(builder, stream.planes[i].offset_bytes, stream.planes[i].stride_bytes); + } + const auto id = builder.CreateString(stream.id); + const auto plane_vector = builder.CreateVector(planes.data(), plane_count); + streams.push_back(wire::CreateStream(builder, id, static_cast(stream.codec), + static_cast(stream.pixel_format), + static_cast(stream.memory), stream.width, stream.height, + plane_vector, stream.slot_bytes, stream.slot_count, stream.frame_rate)); + } + const auto id = builder.CreateString(group.id); + const auto stream_vector = builder.CreateVector(streams); + return wire::CreateGroup(builder, id, stream_vector); +} + +std::string text_of(const flatbuffers::String* string) +{ + return string != nullptr ? string->str() : std::string(); +} + +std::optional pixel_format_of(wire::PixelFormat format) +{ + switch (format) + { + case wire::PixelFormat_Bgra8: + case wire::PixelFormat_Rgba8: + case wire::PixelFormat_Nv12: + case wire::PixelFormat_Gray8: + return static_cast(format); + default: + return std::nullopt; + } +} + +std::optional codec_of(wire::Codec codec) +{ + switch (codec) + { + case wire::Codec_None: + case wire::Codec_H264: + case wire::Codec_H265: + return static_cast(codec); + default: + return std::nullopt; + } +} + +std::optional order_of(wire::DeliveryOrder order) +{ + switch (order) + { + case wire::DeliveryOrder_Default: + case wire::DeliveryOrder_Newest: + case wire::DeliveryOrder_InOrder: + return static_cast(order); + default: + return std::nullopt; + } +} + +GroupDescription read_group(const wire::Group* group) +{ + GroupDescription result; + if (group == nullptr) + { + return result; + } + result.id = text_of(group->id()); + const auto* streams = group->streams(); + if (streams == nullptr) + { + return result; + } + if (streams->size() > MAX_STREAMS_PER_GROUP) + { + throw WireError("camera group '" + result.id + "' has more streams than a group can have"); + } + for (const wire::Stream* stream : *streams) + { + StreamDescription description; + description.id = text_of(stream->id()); + const auto codec = codec_of(stream->codec()); + const auto pixel_format = pixel_format_of(stream->pixel_format()); + // An encoded stream has no pixel format to know. + if (!codec || stream->memory() != wire::MemoryKind_Host || (*codec == Codec::NONE && !pixel_format)) + { + throw WireError("camera stream '" + description.id + + "' uses a format or a kind of memory this build does not know"); + } + description.codec = *codec; + description.pixel_format = pixel_format.value_or(PixelFormat::BGRA8); + description.memory = MemoryKind::HOST; + description.width = stream->width(); + description.height = stream->height(); + if (const auto* planes = stream->planes()) + { + if (planes->size() > MAX_PLANES_PER_IMAGE) + { + throw WireError("camera stream '" + description.id + "' has more planes than an image can have"); + } + description.plane_count = planes->size(); + for (std::uint32_t i = 0; i < planes->size(); ++i) + { + description.planes[i] = { planes->Get(i)->offset_bytes(), planes->Get(i)->stride_bytes() }; + } + } + description.slot_bytes = stream->slot_bytes(); + description.slot_count = stream->slot_count(); + description.frame_rate = stream->frame_rate(); + result.streams.push_back(std::move(description)); + } + return result; +} + +FrameEvent read_frame(const wire::Frame& frame) +{ + FrameEvent event; + event.lease_id = frame.lease_id(); + event.sequence = frame.sequence(); + event.generation = frame.generation(); + event.skipped = frame.skipped(); + if (const wire::Timeline* timeline = frame.timeline()) + { + event.timeline = { timeline->capture_ns(), timeline->sdk_ready_ns(), timeline->published_ns(), + timeline->sent_ns() }; + } + if (const auto* images = frame.images()) + { + if (images->size() > MAX_STREAMS_PER_GROUP) + { + throw WireError("a camera frame set has more images than a group has streams"); + } + event.image_count = images->size(); + for (std::uint32_t i = 0; i < images->size(); ++i) + { + const wire::Image* image = images->Get(i); + event.images[i] = { image->slot(), image->flags(), image->bytes_used(), image->device_frame_id(), + image->device_time_ns() }; + } + } + return event; +} + +} // namespace + +struct WireEncoder::Impl +{ + Builder builder{ 1024 }; + + template + std::span finish(wire::Body type, flatbuffers::Offset body) + { + builder.Finish(wire::CreateMessage(builder, type, body.Union())); + if (builder.GetSize() > MAX_WIRE_MESSAGE_BYTES) + { + throw std::length_error("a camera wire message of " + std::to_string(builder.GetSize()) + + " bytes is over the limit of " + std::to_string(MAX_WIRE_MESSAGE_BYTES)); + } + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; + } +}; + +WireEncoder::WireEncoder() : m_impl(std::make_unique()) +{ +} + +WireEncoder::~WireEncoder() = default; + +std::span WireEncoder::encode(const HelloMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto name = builder.CreateString(message.reader_name); + return m_impl->finish(wire::Body_Hello, wire::CreateHello(builder, message.protocol_version, name)); +} + +std::span WireEncoder::encode(const WelcomeMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::vector> groups; + groups.reserve(message.groups.size()); + for (const GroupDescription& group : message.groups) + { + groups.push_back(build_group(builder, group)); + } + const auto group_vector = builder.CreateVector(groups); + const auto error = builder.CreateString(message.error); + return m_impl->finish( + wire::Body_Welcome, wire::CreateWelcome(builder, message.protocol_version, group_vector, error)); +} + +std::span WireEncoder::encode(const SubscribeMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto group = builder.CreateString(message.group); + return m_impl->finish( + wire::Body_Subscribe, wire::CreateSubscribe(builder, group, message.options.window, + static_cast(message.options.order))); +} + +std::span WireEncoder::encode(const SubscribedMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto error = builder.CreateString(message.error); + const auto group = build_group(builder, message.group); + return m_impl->finish(wire::Body_Subscribed, wire::CreateSubscribed(builder, error, group)); +} + +// The per-frame path: nothing here allocates once the builder has grown to a frame's size. +std::span WireEncoder::encode(const FrameEvent& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::array, MAX_STREAMS_PER_GROUP> images; + const std::size_t image_count = std::min(message.image_count, MAX_STREAMS_PER_GROUP); + for (std::size_t i = 0; i < image_count; ++i) + { + const ImageInfo& image = message.images[i]; + images[i] = wire::CreateImage( + builder, image.slot, image.flags, image.bytes_used, image.device_frame_id, image.device_time_ns); + } + const auto image_vector = builder.CreateVector(images.data(), image_count); + const auto timeline = wire::CreateTimeline(builder, message.timeline.capture_ns, message.timeline.sdk_ready_ns, + message.timeline.published_ns, message.timeline.sent_ns); + return m_impl->finish( + wire::Body_Frame, wire::CreateFrame(builder, message.lease_id, message.sequence, message.generation, + message.skipped, timeline, image_vector)); +} + +std::span WireEncoder::encode(const ReleaseMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + return m_impl->finish(wire::Body_Release, wire::CreateRelease(builder, message.lease_id)); +} + +WireMessage decode_wire_message(std::span bytes) +{ + const auto* data = reinterpret_cast(bytes.data()); + flatbuffers::Verifier verifier(data, bytes.size()); + if (bytes.empty() || !wire::VerifyMessageBuffer(verifier)) + { + throw WireError("received bytes that are not a camera wire message"); + } + const wire::Message* message = wire::GetMessage(data); + // The verifier accepts a body type without a body. + if (message->body_type() != wire::Body_NONE && message->body() == nullptr) + { + throw WireError("received a camera wire message without a body"); + } + switch (message->body_type()) + { + case wire::Body_Hello: + { + const wire::Hello* hello = message->body_as_Hello(); + return HelloMessage{ hello->protocol_version(), text_of(hello->reader_name()) }; + } + case wire::Body_Welcome: + { + const wire::Welcome* welcome = message->body_as_Welcome(); + WelcomeMessage result; + result.protocol_version = welcome->protocol_version(); + result.error = text_of(welcome->error()); + if (const auto* groups = welcome->groups()) + { + for (const wire::Group* group : *groups) + { + result.groups.push_back(read_group(group)); + } + } + return result; + } + case wire::Body_Subscribe: + { + const wire::Subscribe* subscribe = message->body_as_Subscribe(); + const auto order = order_of(subscribe->order()); + if (!order) + { + throw WireError("a camera reader asked for a delivery order this build does not know"); + } + return SubscribeMessage{ text_of(subscribe->group()), { subscribe->window(), *order } }; + } + case wire::Body_Subscribed: + { + const wire::Subscribed* subscribed = message->body_as_Subscribed(); + return SubscribedMessage{ text_of(subscribed->error()), read_group(subscribed->group()) }; + } + case wire::Body_Frame: + return read_frame(*message->body_as_Frame()); + case wire::Body_Release: + return ReleaseMessage{ message->body_as_Release()->lease_id() }; + default: + return std::monostate{}; + } +} + +} // namespace core::camera diff --git a/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp b/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp new file mode 100644 index 0000000000..e2626c10ee --- /dev/null +++ b/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// Changes only when a reader and a plugin of different versions could misread each other. +// A field or a message added to the schema does not change it. +inline constexpr std::uint32_t WIRE_PROTOCOL_VERSION = 1; + +// No message is larger than this, so a receive buffer of this size always fits one. +inline constexpr std::size_t MAX_WIRE_MESSAGE_BYTES = 64 * 1024; + +// Reader to plugin, first message on a connection. +struct HelloMessage +{ + std::uint32_t protocol_version = WIRE_PROTOCOL_VERSION; + std::string reader_name; // for the plugin's log +}; + +// Plugin to reader, in answer to HelloMessage. +struct WelcomeMessage +{ + std::uint32_t protocol_version = WIRE_PROTOCOL_VERSION; + std::vector groups; // what the plugin publishes + std::string error; // why the plugin refuses the reader, if it does +}; + +// Reader to plugin: start sending a group's frame sets. One subscription per connection. +struct SubscribeMessage +{ + std::string group; + ReaderOptions options; +}; + +// Plugin to reader, in answer to SubscribeMessage. On success one file descriptor per stream +// travels with it, in stream order, for the stream's pool. +struct SubscribedMessage +{ + std::string error; // why the subscription was refused, if it was + GroupDescription group; +}; + +// Reader to plugin: done with a frame set. +struct ReleaseMessage +{ + std::uint64_t lease_id = 0; +}; + +// What decode_wire_message returns. A frame set travels as a FrameEvent. std::monostate is a +// message from a newer peer that this build does not know; ignore it. +using WireMessage = + std::variant; + +/** + * @brief Bytes that are not a message, or a message this build cannot represent. + */ +class WireError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +/** + * @brief Turns messages into bytes. It reuses one buffer, so after the first few messages + * encoding allocates nothing. Not thread-safe. + */ +class WireEncoder +{ +public: + WireEncoder(); + ~WireEncoder(); + + WireEncoder(const WireEncoder&) = delete; + WireEncoder& operator=(const WireEncoder&) = delete; + + /** + * @return The message's bytes, valid until the next call on this encoder. + * @throws std::length_error if the message is larger than MAX_WIRE_MESSAGE_BYTES. + */ + std::span encode(const HelloMessage& message); + std::span encode(const WelcomeMessage& message); + std::span encode(const SubscribeMessage& message); + std::span encode(const SubscribedMessage& message); + std::span encode(const FrameEvent& message); + std::span encode(const ReleaseMessage& message); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +/** + * @brief Read one message. The bytes come from another process and are checked in full. + * @throws WireError if they are not a message, or name more streams, planes or images than a + * group can have, or a format this build does not know. + */ +WireMessage decode_wire_message(std::span bytes); + +} // namespace core::camera diff --git a/src/core/camera_wire/fbs/CMakeLists.txt b/src/core/camera_wire/fbs/CMakeLists.txt new file mode 100644 index 0000000000..02239af735 --- /dev/null +++ b/src/core/camera_wire/fbs/CMakeLists.txt @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# The messages between the camera plugin and its readers, as a FlatBuffer schema. The generated +# header is reached as ; only the codec and its +# tests include it. + +include(${CMAKE_SOURCE_DIR}/cmake/GenerateFlatBuffers.cmake) + +generate_flatbuffer_headers( + CAMERA_WIRE_GENERATED_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/generated/camera_wire_schema + CAMERA_WIRE_GENERATED_BFBS +) + +add_custom_target(camera_wire_schema_generation + DEPENDS ${CAMERA_WIRE_GENERATED_HEADERS} ${CAMERA_WIRE_GENERATED_BFBS} +) + +add_library(camera_wire_schema INTERFACE) +add_dependencies(camera_wire_schema camera_wire_schema_generation) + +target_include_directories(camera_wire_schema INTERFACE + $ +) + +target_link_libraries(camera_wire_schema INTERFACE + flatbuffers +) + +add_library(camera::camera_wire_schema ALIAS camera_wire_schema) diff --git a/src/core/camera_wire/fbs/camera_wire.fbs b/src/core/camera_wire/fbs/camera_wire.fbs new file mode 100644 index 0000000000..5f6bc1dd5c --- /dev/null +++ b/src/core/camera_wire/fbs/camera_wire.fbs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The messages between the camera plugin and a reader, one per socket message. A reader and a +// plugin may be built at different times, so this file only grows: append fields with fresh +// ids and leave them optional, append union members, never reuse or renumber. See README.md. + +namespace core.camera.wire; + +// Zero means "a value this build does not know" for the kinds a newer peer may extend. +enum PixelFormat : uint32 { Unknown = 0, Bgra8 = 1, Rgba8 = 2, Nv12 = 3, Gray8 = 4 } +enum Codec : uint32 { None = 0, H264 = 1, H265 = 2 } +enum MemoryKind : uint32 { Unknown = 0, Host = 1 } +enum DeliveryOrder : uint32 { Default = 0, Newest = 1, InOrder = 2 } + +// Where one plane of an image sits inside a slot. +table PlaneLayout { + offset_bytes: uint32 (id: 0); + stride_bytes: uint32 (id: 1); +} + +// One sensor's images or one encoded video stream, and the pool its frames are in. +table Stream { + id: string (id: 0); + codec: Codec (id: 1); + pixel_format: PixelFormat (id: 2); + memory: MemoryKind (id: 3); + width: uint32 (id: 4); + height: uint32 (id: 5); + planes: [PlaneLayout] (id: 6); + slot_bytes: uint64 (id: 7); + slot_count: uint32 (id: 8); + frame_rate: float64 (id: 9); +} + +// Streams that are captured together and published as one frame set. +table Group { + id: string (id: 0); + streams: [Stream] (id: 1); +} + +// Reader to plugin, first message on a connection. +table Hello { + protocol_version: uint32 (id: 0); + reader_name: string (id: 1); +} + +// Plugin to reader, in answer to Hello. `error` is set when the plugin refuses the reader. +table Welcome { + protocol_version: uint32 (id: 0); + groups: [Group] (id: 1); + error: string (id: 2); +} + +// Reader to plugin: start sending this group's frame sets. One subscription per connection. +table Subscribe { + group: string (id: 0); + window: uint32 = 1 (id: 1); + order: DeliveryOrder (id: 2); +} + +// Plugin to reader, in answer to Subscribe. On success the message carries one file descriptor +// per stream of the group, in stream order, for the stream's pool. `error` is set otherwise. +table Subscribed { + error: string (id: 0); + group: Group (id: 1); +} + +// One image of a frame set: the slot it is in and what the camera SDK said about it. +table Image { + slot: uint32 (id: 0); + flags: uint32 (id: 1); + bytes_used: uint64 (id: 2); + device_frame_id: uint64 (id: 3); + device_time_ns: int64 = -1 (id: 4); +} + +// Host monotonic times in nanoseconds; -1 where a time is not known. +table Timeline { + capture_ns: int64 = -1 (id: 0); + sdk_ready_ns: int64 = -1 (id: 1); + published_ns: int64 = -1 (id: 2); + sent_ns: int64 = -1 (id: 3); +} + +// Plugin to reader: one frame set, held for the reader until it sends Release with the same +// lease. Frames may arrive before Subscribed; they belong to the subscription all the same. +table Frame { + lease_id: uint64 (id: 0); + sequence: uint64 (id: 1); + generation: uint32 (id: 2); + skipped: uint32 (id: 3); + timeline: Timeline (id: 4); + images: [Image] (id: 5); +} + +// Reader to plugin: done with a frame set. +table Release { + lease_id: uint64 (id: 0); +} + +union Body { Hello, Welcome, Subscribe, Subscribed, Frame, Release } + +table Message { + body: Body (id: 1); +} + +root_type Message; diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index c26d9384c2..929e096f65 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -26,6 +26,7 @@ add_subdirectory(rebot_devarm_leader) add_subdirectory(manus) add_subdirectory(manus_calibration) add_subdirectory(haptikos) +add_subdirectory(camera) if(BUILD_PLUGIN_NOITOM_MOCAP) add_subdirectory(noitom_mocap) diff --git a/src/plugins/camera/AGENTS.md b/src/plugins/camera/AGENTS.md new file mode 100644 index 0000000000..08a391fda7 --- /dev/null +++ b/src/plugins/camera/AGENTS.md @@ -0,0 +1,14 @@ + + +# Agent notes — camera plugin + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +- **Backends are chosen at build time.** One CMake option per backend, off by default; a backend that is off is not configured, so its SDK is never looked for. Do not load backends at run time and do not add a backend ABI. +- **No SDK types outside a backend's directory.** `core/` and `app/` see `CameraSource` and `camera_core` types only, and readers never link a camera SDK. +- **A backend refuses what it does not understand.** Read options through `OptionReader` and call `finish()`, so a misspelt option is an error. +- **A camera drops a frame rather than wait.** When `acquire` has no free slot, abandon what was taken for that frame set and carry on; nothing on the capture path blocks on readers. +- **One plugin process hosts every backend.** A second instance on another endpoint is a configuration, not a different design. diff --git a/src/plugins/camera/CMakeLists.txt b/src/plugins/camera/CMakeLists.txt new file mode 100644 index 0000000000..c65feb2e17 --- /dev/null +++ b/src/plugins/camera/CMakeLists.txt @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The camera plugin: one process that owns the cameras and serves their frames to readers. +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(STATUS "Skipping camera plugin (Linux only)") + return() +endif() + +add_subdirectory(core) diff --git a/src/plugins/camera/core/CMakeLists.txt b/src/plugins/camera/core/CMakeLists.txt new file mode 100644 index 0000000000..c84fda947e --- /dev/null +++ b/src/plugins/camera/core/CMakeLists.txt @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Threads REQUIRED) + +# What a camera backend implements, and the host that gives each camera its pools and puts +# them all behind one endpoint. +add_library(camera_plugin_core STATIC + camera_host.cpp + options.cpp +) + +target_include_directories(camera_plugin_core INTERFACE + $ +) + +target_link_libraries(camera_plugin_core + PUBLIC + camera::camera_core + PRIVATE + camera::camera_endpoint + camera::camera_ipc + log_bridge::log_bridge_core + Threads::Threads +) + +add_library(camera::plugin_core ALIAS camera_plugin_core) diff --git a/src/plugins/camera/core/camera_host.cpp b/src/plugins/camera/core/camera_host.cpp new file mode 100644 index 0000000000..1eb59e4ee7 --- /dev/null +++ b/src/plugins/camera/core/camera_host.cpp @@ -0,0 +1,194 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_plugin/camera_host.hpp" + +#include +#include +#include + +#include +#include +#include + +namespace plugins::camera +{ + +using core::camera::Endpoint; +using core::camera::FrameTimeline; +using core::camera::GroupDescription; +using core::camera::GroupId; +using core::camera::HostPool; +using core::camera::ImageInfo; +using core::camera::Publisher; +using core::camera::StreamDescription; + +namespace +{ + +// One camera's way into the publisher: slots come from the camera's own pools. +class PoolOutput final : public FrameOutput +{ +public: + PoolOutput(Publisher& publisher, GroupId group, std::vector>& pools) + : m_publisher(publisher), m_group(group), m_pools(pools) + { + } + + std::optional acquire(std::size_t stream) override + { + const std::optional slot = m_publisher.acquire_slot(m_group, stream); + if (!slot) + { + return std::nullopt; + } + return WritableImage{ *slot, m_pools.at(stream)->slot(*slot) }; + } + + void abandon(std::size_t stream, std::uint32_t slot) override + { + m_publisher.abandon_slot(m_group, stream, slot); + } + + std::uint64_t publish(std::span images, const FrameTimeline& timeline) override + { + return m_publisher.publish(m_group, images, timeline); + } + +private: + Publisher& m_publisher; + GroupId m_group; + std::vector>& m_pools; +}; + +struct Camera +{ + std::unique_ptr source; + std::string id; + std::vector> pools; // one per stream + std::unique_ptr output; +}; + +} // namespace + +const Backend& find_backend(std::span backends, const std::string& name) +{ + std::string built; + for (const Backend& backend : backends) + { + if (backend.name == name) + { + return backend; + } + built += (built.empty() ? "" : ", ") + backend.name; + } + throw std::invalid_argument("this camera plugin was built without a '" + name + "' backend (it has: " + built + + "). Backends are chosen when the plugin is built; see the plugin's README"); +} + +class CameraHost::Impl +{ +public: + explicit Impl(const std::string& endpoint_path) + : endpoint(publisher, endpoint_path), logger(isaaccapture::Logger::get("isaaccapture.plugins.camera.CameraHost")) + { + } + + Publisher publisher; + Endpoint endpoint; + // Stable addresses: an output points at its camera's pools. + std::vector> cameras; + bool started = false; + std::shared_ptr logger; +}; + +CameraHost::CameraHost(const std::string& endpoint_path) : m_impl(std::make_unique(endpoint_path)) +{ +} + +CameraHost::~CameraHost() +{ + stop(); +} + +void CameraHost::add_camera(std::unique_ptr source, std::uint32_t slot_count) +{ + if (m_impl->started) + { + throw std::logic_error("cameras are added before the camera host starts"); + } + GroupDescription description = source->describe(); + for (StreamDescription& stream : description.streams) + { + if (stream.slot_count == 0) + { + stream.slot_count = slot_count; + } + } + + auto camera = std::make_unique(); + camera->id = description.id; + // Refuses a description that is not usable before any memory is allocated for it. + const GroupId group = m_impl->publisher.add_group(description); + std::vector descriptors; + for (const StreamDescription& stream : description.streams) + { + camera->pools.push_back(std::make_unique(stream.slot_count, stream.slot_bytes, "camera." + stream.id)); + descriptors.push_back(camera->pools.back()->descriptor()); + } + m_impl->endpoint.set_group_memory(group, std::move(descriptors)); + camera->output = std::make_unique(m_impl->publisher, group, camera->pools); + camera->source = std::move(source); + + const StreamDescription& first = description.streams.front(); + m_impl->logger->info("camera '{}': {} stream(s), {}x{} at {:.2f} fps, {} slots of {} bytes each", description.id, + description.streams.size(), first.width, first.height, first.frame_rate, first.slot_count, + first.slot_bytes); + m_impl->cameras.push_back(std::move(camera)); +} + +void CameraHost::start() +{ + if (m_impl->started) + { + return; + } + m_impl->started = true; + m_impl->endpoint.start(); + for (const auto& camera : m_impl->cameras) + { + camera->source->start(*camera->output); + } +} + +void CameraHost::stop() +{ + if (!m_impl->started) + { + return; + } + // Cameras first: nothing publishes into a pool or the endpoint once they have stopped. + for (const auto& camera : m_impl->cameras) + { + camera->source->stop(); + } + m_impl->endpoint.stop(); + m_impl->started = false; +} + +std::size_t CameraHost::camera_count() const +{ + return m_impl->cameras.size(); +} + +std::size_t CameraHost::reader_count() const +{ + return m_impl->endpoint.reader_count(); +} + +const Publisher& CameraHost::publisher() const +{ + return m_impl->publisher; +} + +} // namespace plugins::camera diff --git a/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp b/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp new file mode 100644 index 0000000000..d0818a2234 --- /dev/null +++ b/src/plugins/camera/core/inc/camera_plugin/camera_host.hpp @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "camera_source.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace plugins::camera +{ + +// Slots per stream unless a camera says otherwise: one for the newest frame set, one for the +// camera to write ahead, and six for readers' windows. +inline constexpr std::uint32_t DEFAULT_SLOT_COUNT = 8; + +/** + * @brief Finds a backend by name. + * @throws std::invalid_argument naming the backends this build has, and how a build gets more. + */ +const Backend& find_backend(std::span backends, const std::string& name); + +/** + * @brief The cameras of one plugin process: gives each its pools and serves them all to + * readers on one endpoint. + */ +class CameraHost +{ +public: + /** + * @throws std::system_error if another process serves the endpoint or it cannot be bound. + */ + explicit CameraHost(const std::string& endpoint_path); + + // Stops the cameras and the endpoint. + ~CameraHost(); + + CameraHost(const CameraHost&) = delete; + CameraHost& operator=(const CameraHost&) = delete; + + /** + * @brief Add a camera and allocate its pools. Call before start(). + * @param slot_count For streams the camera described without one. + * @throws std::invalid_argument if the camera's description is not usable or its id is taken. + */ + void add_camera(std::unique_ptr source, std::uint32_t slot_count = DEFAULT_SLOT_COUNT); + + // Starts serving readers, then the cameras. + void start(); + + // Stops the cameras, then closes every reader. + void stop(); + + std::size_t camera_count() const; + std::size_t reader_count() const; + + // For counters and descriptions; publishing goes through each camera's FrameOutput. + const core::camera::Publisher& publisher() const; + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace plugins::camera diff --git a/src/plugins/camera/core/inc/camera_plugin/camera_source.hpp b/src/plugins/camera/core/inc/camera_plugin/camera_source.hpp new file mode 100644 index 0000000000..8ea82cf085 --- /dev/null +++ b/src/plugins/camera/core/inc/camera_plugin/camera_source.hpp @@ -0,0 +1,95 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace plugins::camera +{ + +// One camera as the configuration names it. +struct CameraConfig +{ + std::string id; // the group readers ask for + std::string backend; // the backend that opens it + std::map options; // the backend's own settings +}; + +// A free slot of one stream, for the next image. +struct WritableImage +{ + std::uint32_t slot = 0; + std::span data; +}; + +/** + * @brief Where a camera's frames go. Safe to call from the camera's capture thread while + * readers come and go. + */ +class FrameOutput +{ +public: + virtual ~FrameOutput() = default; + + /** + * @brief A free slot of one stream to write the next image into. + * @return Nothing when every slot is in use: drop the frame, and abandon() what was already + * acquired for it. + */ + virtual std::optional acquire(std::size_t stream) = 0; + + // Gives back an acquired slot that will not be published. + virtual void abandon(std::size_t stream, std::uint32_t slot) = 0; + + /** + * @brief Publish one frame set: images[i] is stream i's, in a slot from acquire(). + * @param timeline capture_ns and sdk_ready_ns, on the host's monotonic clock. + * @return The frame set's sequence number. + */ + virtual std::uint64_t publish(std::span images, + const core::camera::FrameTimeline& timeline) = 0; +}; + +/** + * @brief One configured camera: one or more streams captured together. A backend implements + * this for its SDK. + */ +class CameraSource +{ +public: + virtual ~CameraSource() = default; + + /** + * @brief What the camera publishes: the group, its streams and the size of a slot. A + * stream's slot_count may be left 0 to take the plugin's pool size. + */ + virtual core::camera::GroupDescription describe() const = 0; + + // Start capturing into `output`, on a thread of the source's own. + virtual void start(FrameOutput& output) = 0; + + // Stop capturing. The output is not called once this has returned. + virtual void stop() = 0; +}; + +// One camera SDK as the plugin sees it. +struct Backend +{ + std::string name; // what a camera's configuration names + std::string description; + + // Opens one camera. Throws std::invalid_argument for options the backend does not accept. + std::function(const CameraConfig&)> open; +}; + +} // namespace plugins::camera diff --git a/src/plugins/camera/core/inc/camera_plugin/options.hpp b/src/plugins/camera/core/inc/camera_plugin/options.hpp new file mode 100644 index 0000000000..a1cf26ec00 --- /dev/null +++ b/src/plugins/camera/core/inc/camera_plugin/options.hpp @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include "camera_source.hpp" + +#include +#include +#include +#include +#include + +namespace plugins::camera +{ + +/** + * @brief Reads a camera's options for a backend and refuses what the backend does not know, + * so a misspelt option is an error and not a silently ignored setting. + */ +class OptionReader +{ +public: + explicit OptionReader(const CameraConfig& config); + + /** + * @brief The option's value, or `fallback` when it is not set. + * @throws std::invalid_argument if the value is not a number between `minimum` and `maximum`. + */ + std::uint32_t number(std::string_view key, std::uint32_t fallback, std::uint32_t minimum, std::uint32_t maximum); + + // The option's text, or `fallback` when it is not set. + std::string text(std::string_view key, std::string_view fallback); + + /** + * @brief Call once every option was read. + * @throws std::invalid_argument naming an option that was set but never read. + */ + void finish() const; + + // For a backend's own complaints: "camera 'head' (synthetic backend): ". + [[noreturn]] void fail(const std::string& what) const; + +private: + const CameraConfig& m_config; + std::set> m_read; +}; + +/** + * @brief Parse one `--camera` argument: ID:BACKEND[:KEY=VALUE[,KEY=VALUE]...]. + * @throws std::invalid_argument if it does not have that form. + */ +CameraConfig parse_camera_argument(std::string_view argument); + +} // namespace plugins::camera diff --git a/src/plugins/camera/core/options.cpp b/src/plugins/camera/core/options.cpp new file mode 100644 index 0000000000..328996764d --- /dev/null +++ b/src/plugins/camera/core/options.cpp @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_plugin/options.hpp" + +#include +#include +#include + +namespace plugins::camera +{ + +namespace +{ + +constexpr std::string_view CAMERA_ARGUMENT_FORM = "ID:BACKEND[:KEY=VALUE[,KEY=VALUE]...]"; + +} + +OptionReader::OptionReader(const CameraConfig& config) : m_config(config) +{ +} + +std::uint32_t OptionReader::number(std::string_view key, std::uint32_t fallback, std::uint32_t minimum, std::uint32_t maximum) +{ + m_read.emplace(key); + const auto found = m_config.options.find(std::string(key)); + if (found == m_config.options.end()) + { + return fallback; + } + const std::string& value = found->second; + std::uint32_t result = 0; + const auto [end, error] = std::from_chars(value.data(), value.data() + value.size(), result); + if (error != std::errc() || end != value.data() + value.size() || result < minimum || result > maximum) + { + fail("option '" + std::string(key) + "' is '" + value + "'; it takes a whole number from " + + std::to_string(minimum) + " to " + std::to_string(maximum)); + } + return result; +} + +std::string OptionReader::text(std::string_view key, std::string_view fallback) +{ + m_read.emplace(key); + const auto found = m_config.options.find(std::string(key)); + return found == m_config.options.end() ? std::string(fallback) : found->second; +} + +void OptionReader::finish() const +{ + for (const auto& [key, value] : m_config.options) + { + if (!m_read.contains(key)) + { + fail("there is no option '" + key + "'"); + } + } +} + +void OptionReader::fail(const std::string& what) const +{ + throw std::invalid_argument("camera '" + m_config.id + "' (" + m_config.backend + " backend): " + what); +} + +CameraConfig parse_camera_argument(std::string_view argument) +{ + const auto fail = [&](const std::string& what) -> void + { + throw std::invalid_argument("--camera=" + std::string(argument) + ": " + what + "; the form is " + + std::string(CAMERA_ARGUMENT_FORM)); + }; + + CameraConfig config; + const std::size_t first = argument.find(':'); + if (first == std::string_view::npos) + { + fail("a camera needs an id and a backend"); + } + const std::size_t second = argument.find(':', first + 1); + config.id = argument.substr(0, first); + config.backend = argument.substr(first + 1, second == std::string_view::npos ? second : second - first - 1); + if (config.id.empty() || config.backend.empty()) + { + fail("a camera needs an id and a backend"); + } + + std::string_view rest = second == std::string_view::npos ? std::string_view() : argument.substr(second + 1); + while (!rest.empty()) + { + const std::size_t comma = rest.find(','); + const std::string_view pair = rest.substr(0, comma); + const std::size_t equals = pair.find('='); + if (equals == std::string_view::npos || equals == 0) + { + fail("'" + std::string(pair) + "' is not KEY=VALUE"); + } + if (!config.options.emplace(pair.substr(0, equals), pair.substr(equals + 1)).second) + { + fail("option '" + std::string(pair.substr(0, equals)) + "' is given twice"); + } + rest = comma == std::string_view::npos ? std::string_view() : rest.substr(comma + 1); + } + return config; +} + +} // namespace plugins::camera diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 54c6e1e0d5..d8efd2cdb3 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -13,6 +13,14 @@ add_subdirectory(live_trackers) add_subdirectory(tracker_metadata) add_subdirectory(log_bridge) add_subdirectory(plugin_manager) +add_subdirectory(camera_core) + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_subdirectory(camera_ipc) + add_subdirectory(camera_wire) + add_subdirectory(camera_endpoint) + add_subdirectory(camera_reader) +endif() if(BUILD_OVXR_CAMERA_QUADS) add_subdirectory(ovxr_pip_layer) diff --git a/tests/cpp/core/camera_core/CMakeLists.txt b/tests/cpp/core/camera_core/CMakeLists.txt new file mode 100644 index 0000000000..f74232fd6b --- /dev/null +++ b/tests/cpp/core/camera_core/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(camera_core_tests + test_publisher.cpp +) + +target_link_libraries(camera_core_tests PRIVATE + camera::camera_core + Catch2::Catch2WithMain + Threads::Threads +) + +message(STATUS "camera_core_tests target enabled with Catch2") +catch_discover_tests(camera_core_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_core/test_publisher.cpp b/tests/cpp/core/camera_core/test_publisher.cpp new file mode 100644 index 0000000000..f3bc370d4a --- /dev/null +++ b/tests/cpp/core/camera_core/test_publisher.cpp @@ -0,0 +1,620 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +// A reader that keeps every event it is sent. +struct RecordingSink : EventSink +{ + bool send_event(SubscriptionId, const FrameEvent& event) noexcept override + { + if (!accept) + { + return false; + } + events.push_back(event); + return true; + } + + std::vector events; + bool accept = true; +}; + +GroupDescription make_group(std::uint32_t slots, std::size_t streams = 2, const std::string& id = "head") +{ + GroupDescription group; + group.id = id; + for (std::size_t i = 0; i < streams; ++i) + { + StreamDescription stream; + stream.id = id + "/" + std::to_string(i); + stream.width = 640; + stream.height = 400; + stream.plane_count = 1; + stream.planes[0].stride_bytes = 640 * 4; + stream.slot_bytes = 640 * 400 * 4; + stream.slot_count = slots; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// Publishes one frame set the way a backend does: take a free slot per stream, then publish. +std::uint64_t publish_next(Publisher& publisher, GroupId group, const FrameTimeline& timeline = {}) +{ + const std::size_t streams = publisher.group_description(group).streams.size(); + std::vector images(streams); + for (std::size_t i = 0; i < streams; ++i) + { + const auto slot = publisher.acquire_slot(group, i); + REQUIRE(slot.has_value()); + images[i].slot = *slot; + } + return publisher.publish(group, images, timeline); +} + +std::vector sequences(const RecordingSink& sink) +{ + std::vector result; + for (const FrameEvent& event : sink.events) + { + result.push_back(event.sequence); + } + return result; +} + +} // namespace + +TEST_CASE("a window of one always gets the newest frame set", "[unit][camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); + REQUIRE(sequences(sink) == std::vector{ 1 }); + REQUIRE(sink.events[0].skipped == 0); + REQUIRE(sink.events[0].image_count == 2); + + // The reader still holds frame set 1, so the next ones wait. + publish_next(publisher, group); + publish_next(publisher, group); + REQUIRE(sink.events.size() == 1); + + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 3 }); + REQUIRE(sink.events[1].skipped == 1); +} + +TEST_CASE("a larger window gets every frame set in order", "[unit][camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 3 }); + REQUIRE(reader); + + for (int i = 0; i < 4; ++i) + { + publish_next(publisher, group); + } + // Three are out, the fourth waits in the ring until the reader has room. + REQUIRE(sequences(sink) == std::vector{ 1, 2, 3 }); + + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 3, 4 }); + REQUIRE(sink.events[3].skipped == 0); +} + +TEST_CASE("an in-order reader that falls behind is told what it skipped", "[unit][camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 2 }); + REQUIRE(reader); + + for (int i = 0; i < 6; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sequences(sink) == std::vector{ 1, 2 }); + + // The ring kept the last two (5 and 6); 3 and 4 are gone. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 5 }); + REQUIRE(sink.events[2].skipped == 2); + + publisher.release(reader.id, sink.events[1].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 5, 6 }); + REQUIRE(sink.events[3].skipped == 0); +} + +TEST_CASE("newest order with a larger window skips to the newest frame set", "[unit][camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 2, .order = DeliveryOrder::NEWEST }); + REQUIRE(reader); + + for (int i = 0; i < 4; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sequences(sink) == std::vector{ 1, 2 }); + + // A viewer that holds one frame and wants the newest must not be handed frame set 3. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sequences(sink) == std::vector{ 1, 2, 4 }); + REQUIRE(sink.events[2].skipped == 1); +} + +TEST_CASE("a subscription that does not fit the pool is refused", "[unit][camera_core][budget]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4)); + RecordingSink first; + RecordingSink second; + + // 1 ring history + 1 window + 1 backend reserve = 3 of 4 slots. + REQUIRE(publisher.subscribe(first, group, { .window = 1 })); + + // An in-order reader with a window of two needs 2 ring history + 3 windows + 1 reserve = 6. + const SubscribeResult refused = publisher.subscribe(second, group, { .window = 2 }); + REQUIRE_FALSE(refused); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("needs 6 slots")); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("pool has 4")); + + // A second newest-only reader needs 1 + 2 + 1 = 4 and fits. + REQUIRE(publisher.subscribe(second, group, { .window = 1 })); + + REQUIRE_FALSE(publisher.subscribe(second, group, { .window = 0 })); + REQUIRE_FALSE(publisher.subscribe(second, 99, { .window = 1 })); +} + +TEST_CASE("the backend reserve counts against the pool", "[unit][camera_core][budget]") +{ + Publisher publisher; + // An SDK that keeps four captures in flight, on a pool of eight. + const GroupId group = publisher.add_group(make_group(8), 4); + RecordingSink sinks[4]; + + // 1 ring history + 3 windows + 4 reserve = 8. + for (int i = 0; i < 3; ++i) + { + REQUIRE(publisher.subscribe(sinks[i], group, { .window = 1 })); + } + const SubscribeResult refused = publisher.subscribe(sinks[3], group, { .window = 1 }); + REQUIRE_FALSE(refused); + REQUIRE_THAT(refused.error, Catch::Matchers::ContainsSubstring("4 for the backend")); +} + +TEST_CASE("a slot is reused only when neither the ring nor a reader holds it", "[unit][camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(3, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); // slot 0: held by the reader + publish_next(publisher, group); // slot 1: held by the ring + REQUIRE(sink.events.size() == 1); + REQUIRE(sink.events[0].images[0].slot == 0); + + const auto third = publisher.acquire_slot(group, 0); + REQUIRE(third == 2u); + REQUIRE_FALSE(publisher.acquire_slot(group, 0).has_value()); + REQUIRE(publisher.stats(group).no_free_slot == 1); + publisher.abandon_slot(group, 0, *third); + + // Releasing frame set 1 frees slot 0 and hands the reader frame set 2. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events.size() == 2); + REQUIRE(sink.events[1].images[0].slot == 1); + REQUIRE(publisher.acquire_slot(group, 0) == 0u); +} + +TEST_CASE("the slot-free callback fires when the last holder lets go", "[unit][camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + std::vector> freed; + publisher.set_slot_free_callback(group, + [&](std::size_t stream, std::uint32_t slot) + { + // Runs outside the publisher's lock, so calling back in must not deadlock. + (void)publisher.stats(group); + freed.emplace_back(stream, slot); + }); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + // The SDK owns the buffers and picks the slot it filled. + const auto publish_slot = [&](std::uint32_t slot) + { + publisher.claim_slot(group, 0, slot); + ImageInfo image; + image.slot = slot; + publisher.publish(group, { &image, 1 }, {}); + }; + + publish_slot(0); // ring + reader + publish_slot(1); // the ring lets slot 0 go, the reader still has it + REQUIRE(freed.empty()); + REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 0), std::invalid_argument); + + publisher.release(reader.id, sink.events[0].lease_id); // reader moves on to slot 1 + REQUIRE(freed == std::vector>{ { 0, 0 } }); + + publish_slot(2); // the ring lets slot 1 go, the reader still has it + REQUIRE(freed.size() == 1); + publisher.unsubscribe(reader.id); + REQUIRE(freed == std::vector>{ { 0, 0 }, { 0, 1 } }); + + // A slot the backend gives up itself needs no notification. + publisher.claim_slot(group, 0, 0); + publisher.abandon_slot(group, 0, 0); + REQUIRE(freed.size() == 2); +} + +TEST_CASE("unsubscribing returns every frame set the reader held", "[unit][camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(8, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 3 }); + REQUIRE(reader); + for (int i = 0; i < 3; ++i) + { + publish_next(publisher, group); + } + REQUIRE(sink.events.size() == 3); + + publisher.unsubscribe(reader.id); + // Only the newest frame set stays in the ring: seven of the eight slots are free again. + for (int i = 0; i < 7; ++i) + { + REQUIRE(publisher.acquire_slot(group, 0).has_value()); + } + REQUIRE_FALSE(publisher.acquire_slot(group, 0).has_value()); + + // Releasing after the subscription ended is harmless. + publisher.release(reader.id, sink.events[0].lease_id); +} + +TEST_CASE("a frame set that could not be sent is not counted as held", "[unit][camera_core][delivery]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(3, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + sink.accept = false; + publish_next(publisher, group); + REQUIRE(publisher.stats(group).send_failures == 1); + REQUIRE(sink.events.empty()); + + sink.accept = true; + publish_next(publisher, group); + REQUIRE(sequences(sink) == std::vector{ 2 }); + REQUIRE(sink.events[0].skipped == 0); + REQUIRE(publisher.stats(group).events_sent == 1); + + // Frame set 1 left the ring and no reader holds it: its slot is free. + REQUIRE(publisher.acquire_slot(group, 0) == 0u); +} + +TEST_CASE("a restart changes the generation and empties the ring", "[unit][camera_core][lifecycle]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + publish_next(publisher, group); + REQUIRE(sink.events[0].generation == 0); + + publisher.restart_group(group); + REQUIRE(publisher.generation(group) == 1); + + // The frame set the reader holds stays valid; the sequence keeps counting. + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events.size() == 1); + publish_next(publisher, group); + REQUIRE(sink.events.size() == 2); + REQUIRE(sink.events[1].generation == 1); + REQUIRE(sink.events[1].sequence == 2); +} + +TEST_CASE("a group publishes one image per stream", "[unit][camera_core][groups]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 3, "rig")); + REQUIRE(publisher.find_group("rig") == group); + REQUIRE_FALSE(publisher.find_group("head").has_value()); + REQUIRE(publisher.group_count() == 1); + + RecordingSink sink; + REQUIRE(publisher.subscribe(sink, group, { .window = 1 })); + + std::vector images(3); + for (std::size_t i = 0; i < 3; ++i) + { + images[i].slot = *publisher.acquire_slot(group, i); + images[i].device_frame_id = 100 + i; + } + images[1].flags = IMAGE_FLAG_KEYFRAME; + images[1].bytes_used = 4096; + images[2].flags = IMAGE_FLAG_SDK_WARNING; + publisher.publish(group, images, {}); + + REQUIRE(sink.events.size() == 1); + const FrameEvent& event = sink.events[0]; + REQUIRE(event.image_count == 3); + REQUIRE(event.images[0].device_frame_id == 100); + REQUIRE(event.images[1].flags == IMAGE_FLAG_KEYFRAME); + REQUIRE(event.images[1].bytes_used == 4096); + REQUIRE(event.images[2].flags == IMAGE_FLAG_SDK_WARNING); +} + +TEST_CASE("an encoded stream is described by its codec, not by a pixel format", "[unit][camera_core][groups]") +{ + GroupDescription description = make_group(4, 1, "head.enc"); + StreamDescription& stream = description.streams[0]; + REQUIRE_FALSE(stream.is_encoded()); + stream.codec = Codec::H265; + stream.plane_count = 0; + stream.slot_bytes = 2 * 1024 * 1024; // room for one access unit + + Publisher publisher; + const GroupId group = publisher.add_group(description); + const StreamDescription stored = publisher.group_description(group).streams[0]; + REQUIRE(stored.is_encoded()); + REQUIRE(stored.codec == Codec::H265); + REQUIRE(stored.slot_bytes == 2 * 1024 * 1024); +} + +TEST_CASE("every frame set carries its timeline", "[unit][camera_core][timeline]") +{ + std::int64_t now = 1000; + Publisher publisher([&] { return now; }); + const GroupId group = publisher.add_group(make_group(4, 1)); + RecordingSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 1 }); + REQUIRE(reader); + + FrameTimeline timeline; + timeline.capture_ns = 100; + timeline.sdk_ready_ns = 900; + publish_next(publisher, group, timeline); + REQUIRE(sink.events[0].timeline.capture_ns == 100); + REQUIRE(sink.events[0].timeline.sdk_ready_ns == 900); + REQUIRE(sink.events[0].timeline.published_ns == 1000); + REQUIRE(sink.events[0].timeline.sent_ns == 1000); + + // A frame set that waits for the reader is stamped as sent when it leaves. + now = 2000; + publish_next(publisher, group); + now = 2500; + publisher.release(reader.id, sink.events[0].lease_id); + REQUIRE(sink.events[1].timeline.capture_ns == TIME_UNKNOWN); + REQUIRE(sink.events[1].timeline.published_ns == 2000); + REQUIRE(sink.events[1].timeline.sent_ns == 2500); +} + +TEST_CASE("misuse by a backend is rejected", "[unit][camera_core][validation]") +{ + Publisher publisher; + REQUIRE_THROWS_AS(publisher.add_group(make_group(4, 0)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4, MAX_STREAMS_PER_GROUP + 1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(0)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(MAX_SLOTS_PER_STREAM + 1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 0), std::invalid_argument); + + // A pool holds at least one frame set for the ring on top of the backend's reserve. + REQUIRE_THROWS_AS(publisher.add_group(make_group(1)), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4), 4), std::invalid_argument); + REQUIRE_NOTHROW(publisher.add_group(make_group(4, 2, "tight"), 3)); + + // A pixel stream names every plane of its format; an encoded stream has none. + GroupDescription nv12 = make_group(4, 1, "nv12"); + nv12.streams[0].pixel_format = PixelFormat::NV12; + REQUIRE_THROWS_AS(publisher.add_group(nv12), std::invalid_argument); + nv12.streams[0].plane_count = 2; + nv12.streams[0].planes[1] = { .offset_bytes = 640 * 400, .stride_bytes = 640 }; + REQUIRE_NOTHROW(publisher.add_group(nv12)); + GroupDescription encoded_with_planes = make_group(4, 1, "enc"); + encoded_with_planes.streams[0].codec = Codec::H264; + REQUIRE_THROWS_AS(publisher.add_group(encoded_with_planes), std::invalid_argument); + GroupDescription no_size = make_group(4, 1, "empty"); + no_size.streams[0].width = 0; + REQUIRE_THROWS_AS(publisher.add_group(no_size), std::invalid_argument); + + const GroupId group = publisher.add_group(make_group(4)); + REQUIRE_THROWS_AS(publisher.add_group(make_group(4)), std::invalid_argument); // same id + + // Publishing needs one image per stream, each in a slot the backend took first. + std::vector images(2); + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); + images[0].slot = *publisher.acquire_slot(group, 0); + REQUIRE_THROWS_AS(publisher.publish(group, { images.data(), 1 }, {}), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); // stream 1 not acquired + + // An image cannot be larger than the slot it sits in. + images[1].slot = *publisher.acquire_slot(group, 1); + images[1].bytes_used = publisher.group_description(group).streams[1].slot_bytes + 1; + REQUIRE_THROWS_AS(publisher.publish(group, images, {}), std::invalid_argument); + REQUIRE(publisher.stats(group).published == 0); + images[1].bytes_used = 0; + REQUIRE(publisher.publish(group, images, {}) == 1); // the slots were still the backend's to publish + + REQUIRE_THROWS_AS(publisher.abandon_slot(group, 1, 0), std::invalid_argument); + REQUIRE_THROWS_AS(publisher.acquire_slot(99, 0), std::out_of_range); + REQUIRE_THROWS_AS(publisher.claim_slot(group, 0, 99), std::invalid_argument); +} + +TEST_CASE("a plane that does not lie inside the slot is rejected", "[unit][camera_core][validation]") +{ + Publisher publisher; + const auto with = [](auto&& change) + { + GroupDescription group = make_group(4, 1); + change(group.streams[0]); + return group; + }; + + // A row is as long as the format needs, and the last row ends inside the slot. + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].stride_bytes = 640 * 4 - 1; })), + std::invalid_argument); + REQUIRE_THROWS_AS(publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].offset_bytes = 1; })), + std::invalid_argument); + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.slot_bytes -= 1; })), std::invalid_argument); + REQUIRE_THROWS_AS( + publisher.add_group(with([](StreamDescription& stream) { stream.planes[0].stride_bytes = 0xFFFFFFFF; })), + std::invalid_argument); + + // Padding between rows is fine as long as the last row still fits. + const auto padded = [](StreamDescription& stream) + { + stream.planes[0].stride_bytes = 4096; + stream.slot_bytes = 4096 * 399 + 640 * 4; + }; + REQUIRE_NOTHROW(publisher.add_group(with(padded))); + + // NV12 with odd sides: chroma rows cover the rounded-up half, and both planes must fit. + const auto nv12 = [](std::uint64_t slot_bytes) + { + return [slot_bytes](StreamDescription& stream) + { + stream.id = "nv12"; + stream.pixel_format = PixelFormat::NV12; + stream.width = 5; + stream.height = 3; + stream.plane_count = 2; + stream.planes[0] = { .offset_bytes = 0, .stride_bytes = 8 }; + stream.planes[1] = { .offset_bytes = 24, .stride_bytes = 8 }; + stream.slot_bytes = slot_bytes; + }; + }; + REQUIRE_THROWS_AS(publisher.add_group(with(nv12(24 + 8 + 5))), std::invalid_argument); // chroma row needs 6 + GroupDescription odd = with(nv12(24 + 8 + 6)); + odd.id = "odd"; + REQUIRE_NOTHROW(publisher.add_group(odd)); +} + +TEST_CASE("a group with a slot-free callback hands out slots by claim only", "[unit][camera_core][slots]") +{ + Publisher publisher; + const GroupId group = publisher.add_group(make_group(4, 1)); + REQUIRE(publisher.acquire_slot(group, 0) == 0u); + publisher.abandon_slot(group, 0, 0); + + publisher.set_slot_free_callback(group, [](std::size_t, std::uint32_t) {}); + REQUIRE_THROWS_AS(publisher.acquire_slot(group, 0), std::logic_error); + REQUIRE_NOTHROW(publisher.claim_slot(group, 0, 0)); + + publisher.set_slot_free_callback(group, {}); + REQUIRE(publisher.acquire_slot(group, 0) == 1u); +} + +TEST_CASE("capture and a reader on different threads keep every frame set in order", "[unit][camera_core][threads]") +{ + // A reader as the endpoint will drive it: events arrive on the capture thread, the + // reader's own thread takes them and releases. + struct QueueSink : EventSink + { + bool send_event(SubscriptionId, const FrameEvent& event) noexcept override + { + { + std::lock_guard lock(mutex); + queue.push_back(event); + } + ready.notify_one(); + return true; + } + + std::mutex mutex; + std::condition_variable ready; + std::deque queue; + }; + + constexpr std::uint64_t FRAME_SETS = 2000; + Publisher publisher; + const GroupId group = publisher.add_group(make_group(16)); + QueueSink sink; + const SubscribeResult reader = publisher.subscribe(sink, group, { .window = 4 }); + REQUIRE(reader); + + std::atomic received{ 0 }; + std::atomic out_of_order{ 0 }; + std::thread consumer( + [&] + { + std::uint64_t last = 0; + while (last < FRAME_SETS) + { + FrameEvent event; + { + std::unique_lock lock(sink.mutex); + sink.ready.wait(lock, [&] { return !sink.queue.empty(); }); + event = sink.queue.front(); + sink.queue.pop_front(); + } + out_of_order += event.sequence <= last ? 1 : 0; + last = event.sequence; + ++received; + publisher.release(reader.id, event.lease_id); + } + }); + + std::uint64_t dropped = 0; + for (std::uint64_t i = 0; i < FRAME_SETS; ++i) + { + std::vector images(2); + const auto left = publisher.acquire_slot(group, 0); + const auto right = publisher.acquire_slot(group, 1); + if (!left || !right) + { + // Cannot happen while the budget holds: a free slot is always left for the backend. + ++dropped; + continue; + } + images[0].slot = *left; + images[1].slot = *right; + publisher.publish(group, images, {}); + } + consumer.join(); + + REQUIRE(dropped == 0); + REQUIRE(out_of_order == 0); + REQUIRE(received > 0); + REQUIRE(publisher.stats(group).published == FRAME_SETS); +} diff --git a/tests/cpp/core/camera_endpoint/CMakeLists.txt b/tests/cpp/core/camera_endpoint/CMakeLists.txt new file mode 100644 index 0000000000..5a8373c671 --- /dev/null +++ b/tests/cpp/core/camera_endpoint/CMakeLists.txt @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(camera_endpoint_tests + test_endpoint.cpp +) + +# The tests play the reader's part with the socket and the codec themselves. +target_link_libraries(camera_endpoint_tests PRIVATE + camera::camera_endpoint + camera::camera_ipc + camera::camera_wire + Catch2::Catch2WithMain + Threads::Threads +) + +message(STATUS "camera_endpoint_tests target enabled with Catch2") +catch_discover_tests(camera_endpoint_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_endpoint/test_endpoint.cpp b/tests/cpp/core/camera_endpoint/test_endpoint.cpp new file mode 100644 index 0000000000..d99eba0ad5 --- /dev/null +++ b/tests/cpp/core/camera_endpoint/test_endpoint.cpp @@ -0,0 +1,419 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; +constexpr std::uint32_t SLOTS = 8; + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_endpoint_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +GroupDescription gray_group(const std::string& id, std::size_t streams) +{ + GroupDescription group; + group.id = id; + for (std::size_t i = 0; i < streams; ++i) + { + StreamDescription stream; + stream.id = id + "/" + std::to_string(i); + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = WIDTH; + stream.slot_bytes = WIDTH * HEIGHT; + stream.slot_count = SLOTS; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// A plugin in miniature: a stereo group and a single camera, in host memory, behind an endpoint. +struct Served +{ + TemporaryDirectory directory; + Publisher publisher; + GroupId head = publisher.add_group(gray_group("head", 2)); + GroupId wrist = publisher.add_group(gray_group("wrist", 1)); + HostPool left{ SLOTS, WIDTH* HEIGHT, "test_left" }; + HostPool right{ SLOTS, WIDTH* HEIGHT, "test_right" }; + HostPool hand{ SLOTS, WIDTH* HEIGHT, "test_wrist" }; + Endpoint endpoint{ publisher, directory.socket_path() }; + + Served() + { + endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); + endpoint.set_group_memory(wrist, { hand.descriptor() }); + endpoint.start(); + } + + // Publishes one stereo frame set whose every pixel is `value`. + std::uint64_t publish(std::uint8_t value) + { + std::vector images(2); + HostPool* pools[] = { &left, &right }; + for (std::size_t i = 0; i < 2; ++i) + { + const auto slot = publisher.acquire_slot(head, i); + REQUIRE(slot.has_value()); + std::fill(pools[i]->slot(*slot).begin(), pools[i]->slot(*slot).end(), std::byte{ value }); + images[i].slot = *slot; + } + return publisher.publish(head, images, {}); + } +}; + +// The reader's part, played with the socket and the codec directly. +struct Reader +{ + struct Received + { + ReceiveStatus status = ReceiveStatus::TIMEOUT; + WireMessage message; + std::vector descriptors; + }; + + explicit Reader(const Served& served) : socket(MessageSocket::connect(served.directory.socket_path())) + { + } + + template + void send(const Message& message) + { + REQUIRE(socket.send(encoder.encode(message)) == SendStatus::SENT); + } + + Received receive(std::chrono::milliseconds timeout = 2s) + { + Received result; + ReceivedMessage received = socket.receive(buffer, timeout); + result.status = received.status; + if (received.status == ReceiveStatus::MESSAGE) + { + result.message = decode_wire_message({ buffer.data(), received.bytes }); + result.descriptors = std::move(received.descriptors); + } + return result; + } + + template + Message expect(std::chrono::milliseconds timeout = 2s) + { + Received received = receive(timeout); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(std::holds_alternative(received.message)); + return std::get(std::move(received.message)); + } + + WelcomeMessage hello(const std::string& name = "test reader") + { + send(HelloMessage{ .reader_name = name }); + return expect(); + } + + // Subscribes and returns the answer with the pool descriptors that came with it. + std::pair> subscribe(const std::string& group, ReaderOptions options = {}) + { + send(SubscribeMessage{ group, options }); + Received received = receive(); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(std::holds_alternative(received.message)); + return { std::get(std::move(received.message)), std::move(received.descriptors) }; + } + + MessageSocket socket; + WireEncoder encoder; + std::vector buffer = std::vector(MAX_WIRE_MESSAGE_BYTES); +}; + +bool wait_until(const std::function& condition, std::chrono::milliseconds timeout = 2s) +{ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (!condition()) + { + if (std::chrono::steady_clock::now() > deadline) + { + return false; + } + std::this_thread::sleep_for(1ms); + } + return true; +} + +} // namespace + +TEST_CASE("a reader is told what is published", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + + const WelcomeMessage welcome = reader.hello(); + REQUIRE(welcome.error.empty()); + REQUIRE(welcome.protocol_version == WIRE_PROTOCOL_VERSION); + REQUIRE(welcome.groups.size() == 2); + REQUIRE(welcome.groups[0].id == "head"); + REQUIRE(welcome.groups[0].streams.size() == 2); + REQUIRE(welcome.groups[0].streams[1].id == "head/1"); + REQUIRE(welcome.groups[0].streams[1].width == WIDTH); + REQUIRE(welcome.groups[1].id == "wrist"); + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 1; })); +} + +TEST_CASE("a reader that speaks another protocol version is refused", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + + reader.send(HelloMessage{ .protocol_version = WIRE_PROTOCOL_VERSION + 1, .reader_name = "from the future" }); + const WelcomeMessage welcome = reader.expect(); + REQUIRE_THAT(welcome.error, ContainsSubstring("protocol version")); + REQUIRE(welcome.groups.empty()); + REQUIRE(reader.receive().status == ReceiveStatus::CLOSED); +} + +TEST_CASE("a subscriber maps the pools and is sent frame sets", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + reader.hello(); + + auto [subscribed, descriptors] = reader.subscribe("head"); + REQUIRE(subscribed.error.empty()); + REQUIRE(subscribed.group.id == "head"); + REQUIRE(descriptors.size() == 2); + const HostPoolView left(std::move(descriptors[0]), SLOTS, WIDTH * HEIGHT); + const HostPoolView right(std::move(descriptors[1]), SLOTS, WIDTH * HEIGHT); + + served.publish(0x11); + const FrameEvent first = reader.expect(); + REQUIRE(first.sequence == 1); + REQUIRE(first.image_count == 2); + REQUIRE(first.skipped == 0); + REQUIRE(first.timeline.published_ns != TIME_UNKNOWN); + REQUIRE(first.timeline.sent_ns >= first.timeline.published_ns); + REQUIRE(left.slot(first.images[0].slot).front() == std::byte{ 0x11 }); + REQUIRE(right.slot(first.images[1].slot).back() == std::byte{ 0x11 }); + + // A window of one holds one frame set; the next comes with the release, and is the newest. + served.publish(0x22); + served.publish(0x33); + REQUIRE(reader.receive(20ms).status == ReceiveStatus::TIMEOUT); + REQUIRE(left.slot(first.images[0].slot).front() == std::byte{ 0x11 }); // still held, so not rewritten + + reader.send(ReleaseMessage{ first.lease_id }); + const FrameEvent newest = reader.expect(); + REQUIRE(newest.sequence == 3); + REQUIRE(newest.skipped == 1); + REQUIRE(left.slot(newest.images[0].slot).front() == std::byte{ 0x33 }); +} + +TEST_CASE("a subscription that cannot be served is refused and the connection stays usable", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + reader.hello(); + + auto [unknown, no_descriptors] = reader.subscribe("chest"); + REQUIRE_THAT(unknown.error, ContainsSubstring("no group 'chest'")); + REQUIRE(no_descriptors.empty()); + + auto [too_large, none] = reader.subscribe("head", { .window = SLOTS }); + REQUIRE_THAT(too_large.error, ContainsSubstring("does not fit the pool")); + REQUIRE(none.empty()); + + auto [accepted, descriptors] = reader.subscribe("wrist"); + REQUIRE(accepted.error.empty()); + REQUIRE(descriptors.size() == 1); + + auto [second, extra] = reader.subscribe("head"); + REQUIRE_THAT(second.error, ContainsSubstring("one subscription")); + REQUIRE(extra.empty()); +} + +TEST_CASE("a reader that disappears returns what it held", "[unit][camera_endpoint]") +{ + Served served; + { + Reader reader(served); + reader.hello(); + auto [subscribed, descriptors] = reader.subscribe("head", { .window = 2, .order = DeliveryOrder::IN_ORDER }); + REQUIRE(subscribed.error.empty()); + served.publish(1); + served.publish(2); + REQUIRE(reader.expect().sequence == 1); + REQUIRE(reader.expect().sequence == 2); + REQUIRE(served.endpoint.reader_count() == 1); + // Gone without releasing either frame set. + } + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 0; })); + + // With the held slots back, capture runs through the whole pool again and again. + for (std::uint8_t i = 0; i < 3 * SLOTS; ++i) + { + served.publish(i); + } + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + + // And the room in the pool is there for the next reader. + Reader next(served); + next.hello(); + auto [again, descriptors] = next.subscribe("head", { .window = 2, .order = DeliveryOrder::IN_ORDER }); + REQUIRE(again.error.empty()); +} + +TEST_CASE("a reader that breaks the protocol is dropped and the others are not", "[unit][camera_endpoint]") +{ + Served served; + Reader good(served); + good.hello(); + auto [subscribed, descriptors] = good.subscribe("head"); + REQUIRE(subscribed.error.empty()); + + Reader garbage(served); + const std::vector noise(32, std::byte{ 0x5a }); + REQUIRE(garbage.socket.send(noise) == SendStatus::SENT); + REQUIRE(garbage.receive().status == ReceiveStatus::CLOSED); + + Reader impostor(served); + impostor.hello(); + impostor.send(WelcomeMessage{}); + REQUIRE(impostor.receive().status == ReceiveStatus::CLOSED); + + Reader impatient(served); + impatient.send(SubscribeMessage{ "head", {} }); + REQUIRE(impatient.receive().status == ReceiveStatus::CLOSED); + + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 1; })); + served.publish(7); + REQUIRE(good.expect().sequence == 1); +} + +TEST_CASE("readers do not slow each other", "[unit][camera_endpoint]") +{ + Served served; + Reader stalled(served); + stalled.hello(); + REQUIRE(stalled.subscribe("head").first.error.empty()); + Reader live(served); + live.hello(); + REQUIRE(live.subscribe("head").first.error.empty()); + + // One reader never releases; the other still gets every frame set. + for (std::uint64_t sequence = 1; sequence <= 5; ++sequence) + { + REQUIRE(served.publish(static_cast(sequence)) == sequence); + const FrameEvent event = live.expect(); + REQUIRE(event.sequence == sequence); + REQUIRE(event.skipped == 0); + // Whether the release is seen before or after the next publish, that frame set is next. + live.send(ReleaseMessage{ event.lease_id }); + } + REQUIRE(stalled.expect().sequence == 1); + REQUIRE(stalled.receive(20ms).status == ReceiveStatus::TIMEOUT); +} + +TEST_CASE("stopping the endpoint closes every reader, and it can serve again", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + reader.hello(); + REQUIRE(reader.subscribe("head").first.error.empty()); + + served.endpoint.stop(); + REQUIRE(reader.receive().status == ReceiveStatus::CLOSED); + REQUIRE(served.endpoint.reader_count() == 0); + served.endpoint.stop(); // nothing to stop + + // A stopped endpoint refuses a reader at once; it does not leave it waiting for an answer. + REQUIRE_THROWS_AS(MessageSocket::connect(served.directory.socket_path()), std::system_error); + + served.endpoint.start(); + Reader next(served); + REQUIRE(next.hello().groups.size() == 2); + REQUIRE(next.subscribe("head").first.error.empty()); + served.publish(9); + REQUIRE(next.expect().sequence == 1); +} + +TEST_CASE("a group needs its memory named before readers can subscribe", "[unit][camera_endpoint]") +{ + TemporaryDirectory directory; + Publisher publisher; + const GroupId head = publisher.add_group(gray_group("head", 2)); + HostPool pool(SLOTS, WIDTH * HEIGHT, "test"); + Endpoint endpoint(publisher, directory.socket_path()); + + REQUIRE_THROWS_AS(endpoint.set_group_memory(head, { pool.descriptor() }), std::invalid_argument); + REQUIRE_THROWS_AS(endpoint.set_group_memory(head + 1, {}), std::out_of_range); + endpoint.start(); + REQUIRE_THROWS_AS(endpoint.set_group_memory(head, { pool.descriptor(), pool.descriptor() }), std::logic_error); + + MessageSocket socket = MessageSocket::connect(directory.socket_path()); + WireEncoder encoder; + std::vector buffer(MAX_WIRE_MESSAGE_BYTES); + REQUIRE(socket.send(encoder.encode(HelloMessage{})) == SendStatus::SENT); + REQUIRE(socket.receive(buffer, 2s).status == ReceiveStatus::MESSAGE); + REQUIRE(socket.send(encoder.encode(SubscribeMessage{ "head", {} })) == SendStatus::SENT); + const ReceivedMessage received = socket.receive(buffer, 2s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + const WireMessage answer = decode_wire_message({ buffer.data(), received.bytes }); + REQUIRE_THAT(std::get(answer).error, ContainsSubstring("no memory a reader can map")); + + // Another endpoint cannot take the same path while this one lives. + REQUIRE_THROWS_AS(Endpoint(publisher, directory.socket_path()), std::system_error); +} diff --git a/tests/cpp/core/camera_ipc/CMakeLists.txt b/tests/cpp/core/camera_ipc/CMakeLists.txt new file mode 100644 index 0000000000..ea7c1cacd4 --- /dev/null +++ b/tests/cpp/core/camera_ipc/CMakeLists.txt @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(camera_ipc_tests + test_host_pool.cpp + test_message_socket.cpp +) + +target_link_libraries(camera_ipc_tests PRIVATE + camera::camera_ipc + Catch2::Catch2WithMain + Threads::Threads +) + +message(STATUS "camera_ipc_tests target enabled with Catch2") +catch_discover_tests(camera_ipc_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_ipc/test_host_pool.cpp b/tests/cpp/core/camera_ipc/test_host_pool.cpp new file mode 100644 index 0000000000..6b2e4fca94 --- /dev/null +++ b/tests/cpp/core/camera_ipc/test_host_pool.cpp @@ -0,0 +1,123 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +// Not a whole number of pages, so slots are padded apart. +constexpr std::uint64_t SLOT_BYTES = 5000; + +// What another process ends up with after the pool's descriptor was sent to it. +UniqueFd duplicate_descriptor(const HostPool& pool) +{ + UniqueFd result(::fcntl(pool.descriptor(), F_DUPFD_CLOEXEC, 0)); + REQUIRE(result.valid()); + return result; +} + +} // namespace + +TEST_CASE("a view sees what the owner writes", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(3, SLOT_BYTES, "camera_ipc_test"); + REQUIRE(pool.slot_count() == 3); + REQUIRE(pool.slot_bytes() == SLOT_BYTES); + + const HostPoolView view(duplicate_descriptor(pool), 3, SLOT_BYTES); + REQUIRE(view.slot_count() == 3); + + // Slots start zeroed and are separate from each other. + const auto is_zero = [](std::byte value) { return value == std::byte{ 0 }; }; + REQUIRE(std::all_of(view.slot(0).begin(), view.slot(0).end(), is_zero)); + + std::fill(pool.slot(1).begin(), pool.slot(1).end(), std::byte{ 0x5a }); + REQUIRE(view.slot(1).size() == SLOT_BYTES); + REQUIRE(view.slot(1).front() == std::byte{ 0x5a }); + REQUIRE(view.slot(1).back() == std::byte{ 0x5a }); + REQUIRE(std::all_of(view.slot(0).begin(), view.slot(0).end(), is_zero)); + REQUIRE(std::all_of(view.slot(2).begin(), view.slot(2).end(), is_zero)); + + // The mapping is live: a later write shows up without mapping again. + pool.slot(2)[100] = std::byte{ 0x11 }; + REQUIRE(view.slot(2)[100] == std::byte{ 0x11 }); +} + +TEST_CASE("slots start on page boundaries", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + const auto page = static_cast(::sysconf(_SC_PAGESIZE)); + REQUIRE(reinterpret_cast(pool.slot(0).data()) % page == 0); + REQUIRE(reinterpret_cast(pool.slot(1).data()) % page == 0); + REQUIRE(pool.slot(1).data() >= pool.slot(0).data() + SLOT_BYTES); +} + +TEST_CASE("a process that is sent the pool cannot write to it or resize it", "[unit][camera_ipc][host_pool]") +{ + HostPool pool(1, SLOT_BYTES, "camera_ipc_test"); + const UniqueFd received = duplicate_descriptor(pool); + + void* writable = ::mmap(nullptr, SLOT_BYTES, PROT_READ | PROT_WRITE, MAP_SHARED, received.get(), 0); + REQUIRE(writable == MAP_FAILED); + REQUIRE(errno == EPERM); + + const char byte = 1; + REQUIRE(::write(received.get(), &byte, 1) == -1); + REQUIRE(::ftruncate(received.get(), 16) == -1); + REQUIRE(::ftruncate(received.get(), static_cast(16 * SLOT_BYTES)) == -1); + + // The owner still can. + pool.slot(0)[0] = std::byte{ 0x7f }; + const HostPoolView view(duplicate_descriptor(pool), 1, SLOT_BYTES); + REQUIRE(view.slot(0)[0] == std::byte{ 0x7f }); +} + +TEST_CASE("a view outlives the pool and can be moved", "[unit][camera_ipc][host_pool]") +{ + std::vector views; + { + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + pool.slot(1)[7] = std::byte{ 0x42 }; + views.emplace_back(duplicate_descriptor(pool), 2, SLOT_BYTES); + } + // The memory lives as long as anything maps it. + HostPoolView moved = std::move(views.front()); + REQUIRE(views.front().slot_count() == 0); + REQUIRE(moved.slot_count() == 2); + REQUIRE(moved.slot(1)[7] == std::byte{ 0x42 }); +} + +TEST_CASE("misuse of a host pool is rejected", "[unit][camera_ipc][host_pool]") +{ + REQUIRE_THROWS_AS(HostPool(0, SLOT_BYTES, "camera_ipc_test"), std::invalid_argument); + REQUIRE_THROWS_AS(HostPool(1, 0, "camera_ipc_test"), std::invalid_argument); + + HostPool pool(2, SLOT_BYTES, "camera_ipc_test"); + REQUIRE_THROWS_AS(pool.slot(2), std::out_of_range); + + const HostPoolView view(duplicate_descriptor(pool), 2, SLOT_BYTES); + REQUIRE_THROWS_AS(view.slot(2), std::out_of_range); + + // A descriptor that holds less than the pool claims would fault on access, so it is refused. + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 3, SLOT_BYTES), std::invalid_argument); + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 2, 4 * SLOT_BYTES), std::invalid_argument); + REQUIRE_THROWS_AS(HostPoolView(duplicate_descriptor(pool), 0, SLOT_BYTES), std::invalid_argument); + + REQUIRE_THROWS_AS(HostPoolView(UniqueFd(-1), 2, SLOT_BYTES), std::system_error); +} diff --git a/tests/cpp/core/camera_ipc/test_message_socket.cpp b/tests/cpp/core/camera_ipc/test_message_socket.cpp new file mode 100644 index 0000000000..6154c63dd6 --- /dev/null +++ b/tests/cpp/core/camera_ipc/test_message_socket.cpp @@ -0,0 +1,337 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; + +namespace +{ + +// A directory for socket files that is removed with everything in it. +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_ipc_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "endpoint.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +std::span bytes_of(std::string_view text) +{ + return std::as_bytes(std::span(text.data(), text.size())); +} + +std::string text_of(std::span bytes, std::size_t count) +{ + return { reinterpret_cast(bytes.data()), count }; +} + +struct Connection +{ + TemporaryDirectory directory; + MessageListener listener{ directory.socket_path() }; + MessageSocket client = MessageSocket::connect(directory.socket_path()); + MessageSocket server = accept(); + + MessageSocket accept() + { + auto accepted = listener.accept(1s); + REQUIRE(accepted.has_value()); + return std::move(*accepted); + } +}; + +} // namespace + +TEST_CASE("messages arrive whole and in order", "[unit][camera_ipc][socket]") +{ + Connection connection; + REQUIRE(connection.client.send(bytes_of("first")) == SendStatus::SENT); + REQUIRE(connection.client.send(bytes_of("second message")) == SendStatus::SENT); + + std::array buffer; + ReceivedMessage received = connection.server.receive(buffer, 1s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(text_of(buffer, received.bytes) == "first"); + REQUIRE(received.descriptors.empty()); + + received = connection.server.receive(buffer, 1s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(text_of(buffer, received.bytes) == "second message"); + + // And the other way. + REQUIRE(connection.server.send(bytes_of("reply")) == SendStatus::SENT); + received = connection.client.receive(buffer, 1s); + REQUIRE(text_of(buffer, received.bytes) == "reply"); +} + +TEST_CASE("descriptors travel with a message", "[unit][camera_ipc][socket]") +{ + Connection connection; + HostPool left(2, 4096, "camera_ipc_test_left"); + HostPool right(2, 4096, "camera_ipc_test_right"); + left.slot(1)[5] = std::byte{ 0x33 }; + right.slot(0)[9] = std::byte{ 0x44 }; + + const std::array descriptors = { left.descriptor(), right.descriptor() }; + REQUIRE(connection.server.send(bytes_of("pools"), descriptors) == SendStatus::SENT); + + std::array buffer; + ReceivedMessage received = connection.client.receive(buffer, 1s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(text_of(buffer, received.bytes) == "pools"); + REQUIRE(received.descriptors.size() == 2); + + // The receiver's descriptors are its own, arrive in order and map the same memory. + REQUIRE(received.descriptors[0].get() != left.descriptor()); + REQUIRE(received.descriptors[1].get() != right.descriptor()); + const HostPoolView left_view(std::move(received.descriptors[0]), 2, 4096); + const HostPoolView right_view(std::move(received.descriptors[1]), 2, 4096); + REQUIRE(left_view.slot(1)[5] == std::byte{ 0x33 }); + REQUIRE(right_view.slot(0)[9] == std::byte{ 0x44 }); +} + +TEST_CASE("send never blocks when the peer does not read", "[unit][camera_ipc][socket]") +{ + Connection connection; + const std::vector payload(1024, std::byte{ 1 }); + + SendStatus status = SendStatus::SENT; + int sent = 0; + while (status == SendStatus::SENT && sent < 1'000'000) + { + status = connection.client.send(payload); + sent += status == SendStatus::SENT ? 1 : 0; + } + REQUIRE(status == SendStatus::WOULD_BLOCK); + REQUIRE(sent > 0); + + // Room appears as soon as the peer reads. + std::array buffer; + REQUIRE(connection.server.receive(buffer, 1s).status == ReceiveStatus::MESSAGE); + REQUIRE(connection.client.send(payload) == SendStatus::SENT); +} + +TEST_CASE("a closed peer is reported, not thrown", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + MessageListener listener(directory.socket_path()); + std::optional server; + { + MessageSocket client = MessageSocket::connect(directory.socket_path()); + server = listener.accept(1s); + REQUIRE(server.has_value()); + REQUIRE(client.send(bytes_of("last words")) == SendStatus::SENT); + } + + // What was sent before the peer closed is still delivered. + std::array buffer; + ReceivedMessage received = server->receive(buffer, 1s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(text_of(buffer, received.bytes) == "last words"); + + REQUIRE(server->receive(buffer, 1s).status == ReceiveStatus::CLOSED); + REQUIRE(server->send(bytes_of("anyone?")) == SendStatus::CLOSED); +} + +TEST_CASE("receive and accept time out when nothing arrives", "[unit][camera_ipc][socket]") +{ + Connection connection; + std::array buffer; + REQUIRE(connection.server.receive(buffer, 10ms).status == ReceiveStatus::TIMEOUT); + REQUIRE(connection.server.receive(buffer, 0ms).status == ReceiveStatus::TIMEOUT); + REQUIRE_FALSE(connection.listener.accept(10ms).has_value()); +} + +TEST_CASE("a listener replaces a stale socket file but not a live one", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + { + MessageListener live(path); + try + { + MessageListener second(path); + FAIL("a second listener took over a live endpoint"); + } + catch (const std::system_error& error) + { + REQUIRE(error.code().value() == EADDRINUSE); + } + // The live listener still works after the refused attempt. + MessageSocket client = MessageSocket::connect(path); + REQUIRE(live.accept(1s).has_value()); + } + REQUIRE_FALSE(std::filesystem::exists(path)); // a listener removes its file + + // A process that died without cleaning up leaves the file behind. + { + sockaddr_un address = {}; + address.sun_family = AF_UNIX; + std::memcpy(address.sun_path, path.data(), path.size()); + const UniqueFd dead(::socket(AF_UNIX, SOCK_SEQPACKET, 0)); + REQUIRE(::bind(dead.get(), reinterpret_cast(&address), sizeof(address)) == 0); + } + REQUIRE(std::filesystem::exists(path)); + MessageListener replacement(path); + MessageSocket client = MessageSocket::connect(path); + REQUIRE(replacement.accept(1s).has_value()); +} + +TEST_CASE("the lock decides who listens, not the socket file", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + { + MessageListener live(path); + // Even with the socket file gone, the path still belongs to the live listener. + REQUIRE(std::filesystem::remove(path)); + REQUIRE_THROWS_AS(MessageListener(path), std::system_error); + // A refused attempt does not loosen the owner's lock, even from the same process. + REQUIRE_THROWS_AS(MessageListener(path), std::system_error); + REQUIRE_FALSE(std::filesystem::exists(path)); + } + // The lock file stays for the next listener, which takes the path over. + REQUIRE(std::filesystem::exists(path + ".lock")); + MessageListener next(path); + MessageSocket client = MessageSocket::connect(path); + REQUIRE(next.accept(1s).has_value()); +} + +TEST_CASE("of listeners that start at the same moment exactly one wins", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + + std::array, 8> contenders; + std::atomic start{ false }; + std::vector threads; + for (auto& contender : contenders) + { + threads.emplace_back( + [&contender, &start, &path] + { + while (!start.load()) + { + std::this_thread::yield(); + } + try + { + contender = std::make_unique(path); + } + catch (const std::system_error&) + { + // Lost to another contender. + } + }); + } + start = true; + for (auto& thread : threads) + { + thread.join(); + } + + const auto is_listening = [](const std::unique_ptr& contender) { return contender != nullptr; }; + REQUIRE(std::count_if(contenders.begin(), contenders.end(), is_listening) == 1); + + // The winner owns the socket file: a client reaches it. + MessageListener& winner = **std::find_if(contenders.begin(), contenders.end(), is_listening); + MessageSocket client = MessageSocket::connect(path); + REQUIRE(winner.accept(1s).has_value()); +} + +TEST_CASE("a path that holds something other than a socket is left alone", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + std::ofstream(path) << "not a socket"; + + try + { + MessageListener listener(path); + FAIL("a listener replaced a file that is not a socket"); + } + catch (const std::system_error& error) + { + REQUIRE(error.code().value() == EEXIST); + } + REQUIRE(std::filesystem::is_regular_file(path)); + REQUIRE(std::filesystem::file_size(path) == 12); +} + +TEST_CASE("only the owner can connect to a listener", "[unit][camera_ipc][socket]") +{ + TemporaryDirectory directory; + const MessageListener listener(directory.socket_path()); + + const std::filesystem::file_status status = std::filesystem::status(directory.socket_path()); + REQUIRE(std::filesystem::is_socket(status)); + REQUIRE(status.permissions() == (std::filesystem::perms::owner_read | std::filesystem::perms::owner_write)); +} + +TEST_CASE("misuse of a message socket is rejected", "[unit][camera_ipc][socket]") +{ + Connection connection; + REQUIRE_THROWS_AS(connection.client.send({}), std::invalid_argument); + + const std::vector too_many(MAX_DESCRIPTORS_PER_MESSAGE + 1, 0); + REQUIRE_THROWS_AS(connection.client.send(bytes_of("x"), too_many), std::invalid_argument); + + // A message that does not fit the receiver's buffer is an error, not a silent cut. + REQUIRE(connection.client.send(bytes_of("sixteen bytes!!!")) == SendStatus::SENT); + std::array small; + REQUIRE_THROWS_AS(connection.server.receive(small, 1s), std::length_error); + // Also with no room at all, which must not read as a closed connection. + REQUIRE(connection.client.send(bytes_of("x")) == SendStatus::SENT); + REQUIRE_THROWS_AS(connection.server.receive({}, 1s), std::length_error); + + TemporaryDirectory directory; + REQUIRE_THROWS_AS(MessageSocket::connect(directory.socket_path()), std::system_error); + REQUIRE_THROWS_AS(MessageSocket::connect(std::string(200, 'a')), std::invalid_argument); + REQUIRE_THROWS_AS(MessageListener(""), std::invalid_argument); +} diff --git a/tests/cpp/core/camera_reader/CMakeLists.txt b/tests/cpp/core/camera_reader/CMakeLists.txt new file mode 100644 index 0000000000..4c14425de0 --- /dev/null +++ b/tests/cpp/core/camera_reader/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(camera_reader_tests + test_camera_reader.cpp +) + +# The reader is tested against a real endpoint, and against a stand-in plugin built from the +# socket and the codec for what a real one does not do on demand. +target_link_libraries(camera_reader_tests PRIVATE + camera::camera_reader + camera::camera_endpoint + camera::camera_ipc + camera::camera_wire + Catch2::Catch2WithMain + Threads::Threads +) + +message(STATUS "camera_reader_tests target enabled with Catch2") +catch_discover_tests(camera_reader_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_reader/test_camera_reader.cpp b/tests/cpp/core/camera_reader/test_camera_reader.cpp new file mode 100644 index 0000000000..34965d34e6 --- /dev/null +++ b/tests/cpp/core/camera_reader/test_camera_reader.cpp @@ -0,0 +1,378 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; +constexpr std::uint32_t STRIDE = 80; // rows are padded +constexpr std::uint32_t SLOTS = 8; +constexpr std::uint64_t SLOT_BYTES = STRIDE * HEIGHT; + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_reader_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +GroupDescription stereo_group() +{ + GroupDescription group; + group.id = "head"; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = std::string("head/") + eye; + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = STRIDE; + stream.slot_bytes = SLOT_BYTES; + stream.slot_count = SLOTS; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// A plugin in miniature: a stereo group in host memory behind an endpoint. +struct Served +{ + TemporaryDirectory directory; + std::string endpoint_path = directory.socket_path(); + Publisher publisher; + GroupId head = publisher.add_group(stereo_group()); + HostPool left{ SLOTS, SLOT_BYTES, "test_left" }; + HostPool right{ SLOTS, SLOT_BYTES, "test_right" }; + Endpoint endpoint{ publisher, endpoint_path }; + + Served() + { + endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); + endpoint.start(); + } + + // Publishes one frame set: every left pixel is `value`, every right pixel `value + 1`. + std::uint64_t publish(std::uint8_t value, const FrameTimeline& timeline = {}) + { + std::vector images(2); + HostPool* pools[] = { &left, &right }; + for (std::size_t i = 0; i < 2; ++i) + { + const auto slot = publisher.acquire_slot(head, i); + REQUIRE(slot.has_value()); + const std::byte fill{ static_cast(value + i) }; + std::fill(pools[i]->slot(*slot).begin(), pools[i]->slot(*slot).end(), fill); + images[i].slot = *slot; + images[i].device_frame_id = 1000 + value; + } + return publisher.publish(head, images, timeline); + } +}; + +std::byte first_pixel(const FrameSet& frames, std::size_t stream) +{ + return frames.image(stream).data.front(); +} + +} // namespace + +TEST_CASE("a reader lists what a plugin publishes", "[unit][camera_reader]") +{ + Served served; + const std::vector groups = CameraReader::list(served.endpoint_path); + REQUIRE(groups.size() == 1); + REQUIRE(groups[0].id == "head"); + REQUIRE(groups[0].streams.size() == 2); + REQUIRE(groups[0].streams[0].id == "head/left"); +} + +TEST_CASE("a reader gets each frame set with its pixels and what is known about it", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open(served.endpoint_path, "head"); + REQUIRE(reader.group().id == "head"); + REQUIRE(reader.group().streams.size() == 2); + REQUIRE(reader.descriptor() >= 0); + + REQUIRE_FALSE(reader.next(10ms).has_value()); // nothing published yet + + served.publish(0x10, { .capture_ns = 111, .sdk_ready_ns = 222 }); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 1); + REQUIRE(frames->skipped() == 0); + REQUIRE(frames->image_count() == 2); + REQUIRE(frames->timeline().capture_ns == 111); + REQUIRE(frames->timeline().sdk_ready_ns == 222); + REQUIRE(frames->timeline().sent_ns >= frames->timeline().published_ns); + + const ImageView left = frames->image(0); + const ImageView right = frames->image(1); + REQUIRE(left.stream->id == "head/left"); + REQUIRE(left.info.device_frame_id == 1000 + 0x10); + REQUIRE(left.data.size() == SLOT_BYTES); + REQUIRE(left.data.front() == std::byte{ 0x10 }); + REQUIRE(right.data.back() == std::byte{ 0x11 }); + + // A plane runs from its first row to the end of its last, without the last row's padding. + REQUIRE(left.plane(0).size() == STRIDE * (HEIGHT - 1) + WIDTH); + REQUIRE(left.plane(0).data() == left.data.data()); + REQUIRE_THROWS_AS(left.plane(1), std::out_of_range); + REQUIRE_THROWS_AS(frames->image(2), std::out_of_range); +} + +TEST_CASE("a held frame set is not rewritten and the next one is the newest", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open(served.endpoint_path, "head"); + + served.publish(1); + std::optional held = reader.next(2s); + REQUIRE(held.has_value()); + + // Capture runs through the pool several times while the reader holds on. + for (std::uint8_t value = 2; value <= 3 * SLOTS; ++value) + { + served.publish(value); + } + REQUIRE(first_pixel(*held, 0) == std::byte{ 1 }); + REQUIRE(first_pixel(*held, 1) == std::byte{ 2 }); + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + + held.reset(); // gives it back + std::optional newest = reader.next(2s); + REQUIRE(newest.has_value()); + REQUIRE(newest->sequence() == 3 * SLOTS); + REQUIRE(newest->skipped() == 3 * SLOTS - 2); + REQUIRE(first_pixel(*newest, 0) == std::byte{ 3 * SLOTS }); + + // release() gives a frame set back before it goes out of scope. + newest->release(); + REQUIRE_THROWS_AS(newest->image(0), std::logic_error); + served.publish(99); + REQUIRE(reader.next(2s)->sequence() == 3 * SLOTS + 1); +} + +TEST_CASE("an in-order reader gets every frame set", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open(served.endpoint_path, "head", { .window = 3 }); + + served.publish(1); + served.publish(2); + served.publish(3); + std::vector held; + for (std::uint64_t sequence = 1; sequence <= 3; ++sequence) + { + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == sequence); + REQUIRE(frames->skipped() == 0); + held.push_back(std::move(*frames)); + } + // The window is full: three are held together, and all three are still readable. + served.publish(4); + REQUIRE_FALSE(reader.next(20ms).has_value()); + REQUIRE(first_pixel(held[0], 0) == std::byte{ 1 }); + REQUIRE(first_pixel(held[2], 0) == std::byte{ 3 }); + + held.erase(held.begin()); + REQUIRE(reader.next(2s)->sequence() == 4); +} + +TEST_CASE("a newest-order reader with a larger window skips to the newest waiting frame set", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = + CameraReader::open(served.endpoint_path, "head", { .window = 3, .order = DeliveryOrder::NEWEST }); + + served.publish(1); + served.publish(2); + served.publish(3); + // Three events are waiting; the reader takes the last and gives the others back. + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 3); + REQUIRE(frames->skipped() == 2); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 3 }); + + // Giving those two back made room in the window again, with the third still held: once the + // plugin has seen the releases, a newly published frame set reaches the reader. + std::optional more; + for (std::uint8_t value = 4; value < 200 && !more; ++value) + { + served.publish(value); + more = reader.next(20ms); + } + REQUIRE(more.has_value()); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 3 }); +} + +TEST_CASE("a frame set can be given back after its reader is gone", "[unit][camera_reader]") +{ + Served served; + std::optional frames; + { + CameraReader reader = CameraReader::open(served.endpoint_path, "head"); + served.publish(5); + frames = reader.next(2s); + REQUIRE(frames.has_value()); + CameraReader moved = std::move(reader); + REQUIRE(moved.group().id == "head"); + } + // The pixels stay readable for as long as the frame set is held. + REQUIRE(first_pixel(*frames, 0) == std::byte{ 5 }); + frames.reset(); +} + +TEST_CASE("a reader says why it could not be opened", "[unit][camera_reader]") +{ + TemporaryDirectory directory; + REQUIRE_THROWS_WITH( + CameraReader::open(directory.socket_path(), "head"), ContainsSubstring("no camera plugin answers on")); + REQUIRE_THROWS_AS(CameraReader::list(directory.socket_path()), CameraReaderError); + + Served served; + REQUIRE_THROWS_WITH(CameraReader::open(served.endpoint_path, "chest"), ContainsSubstring("no group 'chest'")); + REQUIRE_THROWS_WITH(CameraReader::open(served.endpoint_path, "head", { .window = SLOTS }), + ContainsSubstring("does not fit the pool")); + + // A refusal leaves nothing behind: the pool still has room for a reader that fits. + REQUIRE_NOTHROW(CameraReader::open(served.endpoint_path, "head", { .window = 2 })); +} + +TEST_CASE("a reader is told when the plugin is gone", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open(served.endpoint_path, "head"); + served.publish(1); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + + served.endpoint.stop(); + REQUIRE_THROWS_AS(reader.next(2s), CameraDisconnected); + // What the reader holds is still mapped, and giving it back to nobody is harmless. + REQUIRE(first_pixel(*frames, 0) == std::byte{ 1 }); + frames.reset(); +} + +TEST_CASE("a reader copes with what a plugin may send", "[unit][camera_reader]") +{ + // A stand-in plugin, to send what a real one does not send on demand. + TemporaryDirectory directory; + MessageListener listener(directory.socket_path()); + HostPool left(SLOTS, SLOT_BYTES, "test_left"); + HostPool right(SLOTS, SLOT_BYTES, "test_right"); + left.slot(2)[0] = std::byte{ 0x42 }; + + FrameEvent early; + early.lease_id = 7; + early.sequence = 1; + early.image_count = 2; + early.images[0].slot = 2; + FrameEvent bad_slot = early; + bad_slot.lease_id = 8; + bad_slot.sequence = 2; + bad_slot.images[1].slot = SLOTS; + + std::vector released; + std::thread plugin( + [&] + { + try + { + WireEncoder encoder; + std::vector buffer(MAX_WIRE_MESSAGE_BYTES); + std::optional reader = listener.accept(2s); + if (!reader) + { + return; + } + const auto receive = [&] { + return decode_wire_message({ buffer.data(), reader->receive(buffer, 2s).bytes }); + }; + (void)receive(); // hello + WelcomeMessage welcome; + welcome.groups = { stereo_group() }; + reader->send(encoder.encode(welcome)); + (void)receive(); // subscribe + + // A frame set overtakes the answer to the subscription. + reader->send(encoder.encode(early)); + const std::array descriptors = { left.descriptor(), right.descriptor() }; + reader->send(encoder.encode(SubscribedMessage{ "", stereo_group() }), descriptors); + reader->send(encoder.encode(bad_slot)); + for (int i = 0; i < 2; ++i) + { + released.push_back(std::get(receive()).lease_id); + } + } + catch (const std::exception&) + { + // The checks on `released` below fail. + } + }); + + CameraReader reader = CameraReader::open(directory.socket_path(), "head"); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 1); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 0x42 }); + frames.reset(); + + // A frame set that names a slot the pool does not have is refused and given back. + REQUIRE_THROWS_WITH(reader.next(2s), ContainsSubstring("does not match group 'head'")); + plugin.join(); + REQUIRE(released == std::vector{ 7, 8 }); +} diff --git a/tests/cpp/core/camera_wire/CMakeLists.txt b/tests/cpp/core/camera_wire/CMakeLists.txt new file mode 100644 index 0000000000..69b9c4a7b1 --- /dev/null +++ b/tests/cpp/core/camera_wire/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_wire_tests + test_codec.cpp +) + +# The schema is linked to build messages the encoder never would: ones from a newer peer and +# malformed ones. +target_link_libraries(camera_wire_tests PRIVATE + camera::camera_wire + camera::camera_wire_schema + Catch2::Catch2WithMain +) + +message(STATUS "camera_wire_tests target enabled with Catch2") +catch_discover_tests(camera_wire_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_wire/test_codec.cpp b/tests/cpp/core/camera_wire/test_codec.cpp new file mode 100644 index 0000000000..e3b33b516f --- /dev/null +++ b/tests/cpp/core/camera_wire/test_codec.cpp @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +namespace wire = core::camera::wire; + +namespace +{ + +GroupDescription stereo_group() +{ + GroupDescription group; + group.id = "head"; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = std::string("head/") + eye; + stream.pixel_format = PixelFormat::NV12; + stream.width = 1920; + stream.height = 1080; + stream.plane_count = 2; + stream.planes[0] = { .offset_bytes = 0, .stride_bytes = 2048 }; + stream.planes[1] = { .offset_bytes = 2048 * 1080, .stride_bytes = 2048 }; + stream.slot_bytes = 2048 * 1620; + stream.slot_count = 8; + stream.frame_rate = 59.94; + group.streams.push_back(stream); + } + return group; +} + +GroupDescription encoded_group() +{ + GroupDescription group; + group.id = "head.h265"; + StreamDescription stream; + stream.id = "head/left.h265"; + stream.codec = Codec::H265; + stream.width = 1920; + stream.height = 1080; + stream.slot_bytes = 1 << 20; + stream.slot_count = 16; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + return group; +} + +void require_same(const GroupDescription& actual, const GroupDescription& expected) +{ + REQUIRE(actual.id == expected.id); + REQUIRE(actual.streams.size() == expected.streams.size()); + for (std::size_t i = 0; i < expected.streams.size(); ++i) + { + const StreamDescription& a = actual.streams[i]; + const StreamDescription& e = expected.streams[i]; + REQUIRE(a.id == e.id); + REQUIRE(a.codec == e.codec); + REQUIRE(a.memory == e.memory); + REQUIRE(a.width == e.width); + REQUIRE(a.height == e.height); + REQUIRE(a.plane_count == e.plane_count); + for (std::uint32_t plane = 0; plane < e.plane_count; ++plane) + { + REQUIRE(a.planes[plane].offset_bytes == e.planes[plane].offset_bytes); + REQUIRE(a.planes[plane].stride_bytes == e.planes[plane].stride_bytes); + } + REQUIRE(a.slot_bytes == e.slot_bytes); + REQUIRE(a.slot_count == e.slot_count); + REQUIRE(a.frame_rate == e.frame_rate); + if (!e.is_encoded()) + { + REQUIRE(a.pixel_format == e.pixel_format); + } + } +} + +// What the other process receives: a copy of the bytes, decoded. +template +Message round_trip(WireEncoder& encoder, const Message& message) +{ + const std::span bytes = encoder.encode(message); + const std::vector received(bytes.begin(), bytes.end()); + WireMessage decoded = decode_wire_message(received); + REQUIRE(std::holds_alternative(decoded)); + return std::get(std::move(decoded)); +} + +std::span bytes_of(const flatbuffers::FlatBufferBuilder& builder) +{ + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; +} + +} // namespace + +TEST_CASE("the handshake messages survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + const HelloMessage hello = round_trip(encoder, HelloMessage{ .protocol_version = 7, .reader_name = "viewer" }); + REQUIRE(hello.protocol_version == 7); + REQUIRE(hello.reader_name == "viewer"); + + WelcomeMessage welcome; + welcome.groups = { stereo_group(), encoded_group() }; + const WelcomeMessage received = round_trip(encoder, welcome); + REQUIRE(received.protocol_version == WIRE_PROTOCOL_VERSION); + REQUIRE(received.error.empty()); + REQUIRE(received.groups.size() == 2); + require_same(received.groups[0], stereo_group()); + require_same(received.groups[1], encoded_group()); + + const WelcomeMessage refused = round_trip(encoder, WelcomeMessage{ .error = "protocol version 7 is not supported" }); + REQUIRE(refused.error == "protocol version 7 is not supported"); + REQUIRE(refused.groups.empty()); +} + +TEST_CASE("a subscription and its answer survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + const SubscribeMessage subscribe = + round_trip(encoder, SubscribeMessage{ "head", { .window = 4, .order = DeliveryOrder::IN_ORDER } }); + REQUIRE(subscribe.group == "head"); + REQUIRE(subscribe.options.window == 4); + REQUIRE(subscribe.options.order == DeliveryOrder::IN_ORDER); + + // What a reader that sets nothing asks for. + const SubscribeMessage defaults = round_trip(encoder, SubscribeMessage{ "head", {} }); + REQUIRE(defaults.options.window == 1); + REQUIRE(defaults.options.order == DeliveryOrder::DEFAULT); + + const SubscribedMessage accepted = round_trip(encoder, SubscribedMessage{ "", stereo_group() }); + REQUIRE(accepted.error.empty()); + require_same(accepted.group, stereo_group()); + + const SubscribedMessage refused = round_trip(encoder, SubscribedMessage{ "does not fit the pool", {} }); + REQUIRE(refused.error == "does not fit the pool"); + REQUIRE(refused.group.streams.empty()); +} + +TEST_CASE("a frame set and its release survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + FrameEvent event; + event.lease_id = 0x1122334455667788; + event.sequence = 123456789012; + event.generation = 3; + event.skipped = 2; + event.image_count = 2; + event.timeline = { .capture_ns = 1000, .sdk_ready_ns = 2000, .published_ns = 3000, .sent_ns = 4000 }; + event.images[0] = { + .slot = 5, .flags = IMAGE_FLAG_SDK_WARNING, .bytes_used = 0, .device_frame_id = 77, .device_time_ns = 99 + }; + event.images[1] = { .slot = 6, .flags = IMAGE_FLAG_KEYFRAME, .bytes_used = 4096, .device_frame_id = 78 }; + + const FrameEvent received = round_trip(encoder, event); + REQUIRE(received.lease_id == event.lease_id); + REQUIRE(received.sequence == event.sequence); + REQUIRE(received.generation == 3); + REQUIRE(received.skipped == 2); + REQUIRE(received.image_count == 2); + REQUIRE(received.timeline.capture_ns == 1000); + REQUIRE(received.timeline.sdk_ready_ns == 2000); + REQUIRE(received.timeline.published_ns == 3000); + REQUIRE(received.timeline.sent_ns == 4000); + REQUIRE(received.images[0].slot == 5); + REQUIRE(received.images[0].flags == IMAGE_FLAG_SDK_WARNING); + REQUIRE(received.images[0].device_frame_id == 77); + REQUIRE(received.images[0].device_time_ns == 99); + REQUIRE(received.images[1].slot == 6); + REQUIRE(received.images[1].flags == IMAGE_FLAG_KEYFRAME); + REQUIRE(received.images[1].bytes_used == 4096); + REQUIRE(received.images[1].device_time_ns == TIME_UNKNOWN); + + // Times that are not known stay unknown. + const FrameEvent bare = round_trip(encoder, FrameEvent{}); + REQUIRE(bare.image_count == 0); + REQUIRE(bare.timeline.capture_ns == TIME_UNKNOWN); + REQUIRE(bare.timeline.sent_ns == TIME_UNKNOWN); + + REQUIRE(round_trip(encoder, ReleaseMessage{ 42 }).lease_id == 42); +} + +TEST_CASE("an encoder is reused from message to message", "[unit][camera_wire]") +{ + WireEncoder encoder; + FrameEvent event; + event.image_count = 2; + for (std::uint64_t sequence = 1; sequence <= 100; ++sequence) + { + event.sequence = sequence; + REQUIRE(round_trip(encoder, event).sequence == sequence); + } + // A small message after a large one is not padded to the large one's size. + WelcomeMessage welcome; + welcome.groups = { stereo_group(), encoded_group() }; + const std::size_t large = encoder.encode(welcome).size(); + REQUIRE(encoder.encode(ReleaseMessage{ 1 }).size() < large); +} + +TEST_CASE("bytes that are not a message are rejected", "[unit][camera_wire]") +{ + REQUIRE_THROWS_AS(decode_wire_message({}), WireError); + + const std::vector garbage(64, std::byte{ 0xa5 }); + REQUIRE_THROWS_AS(decode_wire_message(garbage), WireError); + + WireEncoder encoder; + WelcomeMessage welcome; + welcome.groups = { stereo_group() }; + const std::span bytes = encoder.encode(welcome); + const std::vector whole(bytes.begin(), bytes.end()); + REQUIRE_THROWS_AS(decode_wire_message({ whole.data(), whole.size() / 2 }), WireError); + + // No cut of a message may be read out of bounds; each is either refused or read in full. + for (std::size_t length = 0; length < whole.size(); ++length) + { + const std::vector cut(whole.begin(), whole.begin() + static_cast(length)); + try + { + (void)decode_wire_message(cut); + } + catch (const WireError&) + { + } + } +} + +TEST_CASE("a message that does not fit this build's limits is rejected", "[unit][camera_wire]") +{ + WireEncoder encoder; + + GroupDescription crowded = stereo_group(); + crowded.streams.resize(MAX_STREAMS_PER_GROUP + 1, crowded.streams.front()); + const std::span bytes = encoder.encode(SubscribedMessage{ "", crowded }); + REQUIRE_THROWS_AS(decode_wire_message(bytes), WireError); + + // A pixel format added by a newer plugin cannot be read, so the group is refused as a whole. + GroupDescription newer = stereo_group(); + newer.streams[1].pixel_format = static_cast(99); + REQUIRE_THROWS_AS(decode_wire_message(encoder.encode(SubscribedMessage{ "", newer })), WireError); + + flatbuffers::FlatBufferBuilder builder; + + // More planes than an image can have. + const std::vector> planes( + MAX_PLANES_PER_IMAGE + 1, wire::CreatePlaneLayout(builder, 0, 64)); + const auto stream = + wire::CreateStream(builder, builder.CreateString("s"), wire::Codec_None, wire::PixelFormat_Gray8, + wire::MemoryKind_Host, 64, 64, builder.CreateVector(planes), 64 * 64, 4, 30.0); + const auto group = wire::CreateGroup(builder, builder.CreateString("g"), builder.CreateVector(&stream, 1)); + builder.Finish(wire::CreateMessage(builder, wire::Body_Subscribed, wire::CreateSubscribed(builder, 0, group).Union())); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // More images than a group has streams. + builder.Clear(); + const std::vector> images(MAX_STREAMS_PER_GROUP + 1, wire::CreateImage(builder)); + builder.Finish(wire::CreateMessage( + builder, wire::Body_Frame, wire::CreateFrame(builder, 1, 1, 0, 0, 0, builder.CreateVector(images)).Union())); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // A body type with no body behind it. + builder.Clear(); + builder.Finish(wire::CreateMessage(builder, wire::Body_Release, 0)); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // The largest message a receive buffer has to hold. + REQUIRE_THROWS_AS( + encoder.encode(HelloMessage{ .reader_name = std::string(MAX_WIRE_MESSAGE_BYTES, 'x') }), std::length_error); +} + +TEST_CASE("a message from a newer peer is ignored, not an error", "[unit][camera_wire]") +{ + flatbuffers::FlatBufferBuilder builder; + + // A union member this build does not have. + const auto body = wire::CreateRelease(builder, 1); + builder.Finish(wire::CreateMessage(builder, static_cast(wire::Body_MAX + 1), body.Union())); + REQUIRE(std::holds_alternative(decode_wire_message(bytes_of(builder)))); + + // No body at all. + builder.Clear(); + builder.Finish(wire::CreateMessage(builder)); + REQUIRE(std::holds_alternative(decode_wire_message(bytes_of(builder)))); +} diff --git a/tests/cpp/plugins/CMakeLists.txt b/tests/cpp/plugins/CMakeLists.txt index fd7e2d2e68..8e22e0f06c 100644 --- a/tests/cpp/plugins/CMakeLists.txt +++ b/tests/cpp/plugins/CMakeLists.txt @@ -11,6 +11,7 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(spacemouse) add_subdirectory(gamepad) + add_subdirectory(camera) endif() if(BUILD_PLUGIN_OGLO) diff --git a/tests/cpp/plugins/camera/CMakeLists.txt b/tests/cpp/plugins/camera/CMakeLists.txt new file mode 100644 index 0000000000..cda01eed40 --- /dev/null +++ b/tests/cpp/plugins/camera/CMakeLists.txt @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_plugin_tests + test_camera_host.cpp +) + +target_link_libraries(camera_plugin_tests PRIVATE + camera::plugin_core + camera::camera_reader + Catch2::Catch2WithMain +) + +message(STATUS "camera_plugin_tests target enabled with Catch2") +catch_discover_tests(camera_plugin_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/plugins/camera/test_camera_host.cpp b/tests/cpp/plugins/camera/test_camera_host.cpp new file mode 100644 index 0000000000..6b5983c828 --- /dev/null +++ b/tests/cpp/plugins/camera/test_camera_host.cpp @@ -0,0 +1,265 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace plugins::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; +using core::camera::CameraReader; +using core::camera::FrameSet; +using core::camera::GroupDescription; +using core::camera::ImageInfo; +using core::camera::PixelFormat; +using core::camera::StreamDescription; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_host_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +// A camera that publishes when the test says so. `control` outlives the source, which the +// host owns. +struct ManualControl +{ + FrameOutput* output = nullptr; // set while the camera runs + std::size_t streams = 0; + + // Publishes one frame set whose every pixel is `value`. False when it had to be dropped. + bool publish(std::uint8_t value) + { + std::vector images(streams); + for (std::size_t stream = 0; stream < streams; ++stream) + { + const std::optional image = output->acquire(stream); + if (!image) + { + for (std::size_t taken = 0; taken < stream; ++taken) + { + output->abandon(taken, images[taken].slot); + } + return false; + } + std::fill(image->data.begin(), image->data.end(), std::byte{ value }); + images[stream].slot = image->slot; + } + output->publish(images, {}); + return true; + } +}; + +class ManualSource final : public CameraSource +{ +public: + ManualSource(std::string id, std::size_t streams, ManualControl& control, std::uint32_t slot_count = 0) + : m_id(std::move(id)), m_slot_count(slot_count), m_control(control) + { + m_control.streams = streams; + } + + GroupDescription describe() const override + { + GroupDescription group; + group.id = m_id; + for (std::size_t i = 0; i < m_control.streams; ++i) + { + StreamDescription stream; + stream.id = m_id + "/" + std::to_string(i); + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = WIDTH; + stream.slot_bytes = WIDTH * HEIGHT; + stream.slot_count = m_slot_count; + stream.frame_rate = 30.0; + group.streams.push_back(stream); + } + return group; + } + + void start(FrameOutput& output) override + { + m_control.output = &output; + } + + void stop() override + { + m_control.output = nullptr; + } + +private: + std::string m_id; + std::uint32_t m_slot_count; + ManualControl& m_control; +}; + +} // namespace + +TEST_CASE("a camera argument names an id, a backend and the backend's options", "[unit][camera_plugin]") +{ + const CameraConfig bare = parse_camera_argument("head:synthetic"); + REQUIRE(bare.id == "head"); + REQUIRE(bare.backend == "synthetic"); + REQUIRE(bare.options.empty()); + + const CameraConfig full = parse_camera_argument("wrist:synthetic:width=640,height=480,format=gray8"); + REQUIRE(full.id == "wrist"); + REQUIRE(full.options.size() == 3); + REQUIRE(full.options.at("width") == "640"); + REQUIRE(full.options.at("format") == "gray8"); + + for (const char* malformed : { "head", ":synthetic", "head:", "head:synthetic:width", "head:synthetic:=3", + "head:synthetic:width=1,width=2" }) + { + REQUIRE_THROWS_WITH(parse_camera_argument(malformed), ContainsSubstring("ID:BACKEND")); + } +} + +TEST_CASE("options a backend does not read are refused", "[unit][camera_plugin]") +{ + const CameraConfig config = parse_camera_argument("head:zed:width=640,serial=1234,widht=7"); + OptionReader options(config); + REQUIRE(options.number("width", 1280, 16, 8192) == 640); + REQUIRE(options.number("height", 720, 16, 8192) == 720); // not set: the fallback + REQUIRE(options.text("serial", "") == "1234"); + REQUIRE(options.text("mode", "hd720") == "hd720"); + REQUIRE_THROWS_WITH(options.finish(), ContainsSubstring("camera 'head' (zed backend): there is no option 'widht'")); + + REQUIRE_THROWS_WITH(options.number("serial", 0, 0, 100), ContainsSubstring("whole number from 0 to 100")); + const CameraConfig words = parse_camera_argument("head:zed:width=wide,height=-3,fps=60.5"); + OptionReader strict(words); + REQUIRE_THROWS_AS(strict.number("width", 1, 0, 100), std::invalid_argument); + REQUIRE_THROWS_AS(strict.number("height", 1, 0, 100), std::invalid_argument); + REQUIRE_THROWS_AS(strict.number("fps", 1, 0, 100), std::invalid_argument); +} + +TEST_CASE("a backend that is not in the build is named with the ones that are", "[unit][camera_plugin]") +{ + const std::vector backends = { { "synthetic", "", {} }, { "zed", "", {} } }; + REQUIRE(find_backend(backends, "zed").name == "zed"); + REQUIRE_THROWS_WITH(find_backend(backends, "oak"), ContainsSubstring("built without a 'oak' backend") && + ContainsSubstring("synthetic, zed") && + ContainsSubstring("chosen when the plugin is built")); +} + +TEST_CASE("a host gives each camera its pools and serves them all on one endpoint", "[unit][camera_plugin]") +{ + TemporaryDirectory directory; + ManualControl head; + ManualControl wrist; + CameraHost host(directory.socket_path()); + host.add_camera(std::make_unique("head", 2, head)); + host.add_camera(std::make_unique("wrist", 1, wrist, 4), 16); // the camera's own count wins + REQUIRE(host.camera_count() == 2); + + ManualControl unused; + REQUIRE_THROWS_AS(host.add_camera(std::make_unique("head", 1, unused)), std::invalid_argument); + REQUIRE_THROWS_AS(host.add_camera(std::make_unique("empty", 0, unused)), std::invalid_argument); + + host.start(); + REQUIRE(head.output != nullptr); + REQUIRE_THROWS_AS(host.add_camera(std::make_unique("late", 1, unused)), std::logic_error); + + const std::vector groups = CameraReader::list(directory.socket_path()); + REQUIRE(groups.size() == 2); + REQUIRE(groups[0].id == "head"); + REQUIRE(groups[0].streams[0].slot_count == DEFAULT_SLOT_COUNT); + REQUIRE(groups[1].id == "wrist"); + REQUIRE(groups[1].streams[0].slot_count == 4); + + CameraReader head_reader = CameraReader::open(directory.socket_path(), "head"); + CameraReader wrist_reader = CameraReader::open(directory.socket_path(), "wrist"); + REQUIRE(host.reader_count() == 2); + + REQUIRE(head.publish(0x21)); + REQUIRE(wrist.publish(0x42)); + const std::optional pair = head_reader.next(2s); + REQUIRE(pair.has_value()); + REQUIRE(pair->image_count() == 2); + REQUIRE(pair->image(0).data.front() == std::byte{ 0x21 }); + REQUIRE(pair->image(1).data.back() == std::byte{ 0x21 }); + const std::optional single = wrist_reader.next(2s); + REQUIRE(single.has_value()); + REQUIRE(single->image(0).data.front() == std::byte{ 0x42 }); + REQUIRE(host.publisher().stats(0).published == 1); + + host.stop(); + REQUIRE(head.output == nullptr); + REQUIRE(host.reader_count() == 0); +} + +TEST_CASE("a camera drops a frame set when its pool is full, and never overwrites a held one", "[unit][camera_plugin]") +{ + TemporaryDirectory directory; + ManualControl camera; + CameraHost host(directory.socket_path()); + host.add_camera(std::make_unique("head", 2, camera), 3); + host.start(); + + // A window of one in a pool of three: the newest, the reader's, and one to write ahead. + CameraReader reader = CameraReader::open(directory.socket_path(), "head"); + REQUIRE(camera.publish(1)); + const std::optional held = reader.next(2s); + REQUIRE(held.has_value()); + for (std::uint8_t value = 2; value < 20; ++value) + { + REQUIRE(camera.publish(value)); // the ring lets go of the frame set before, so there is room + } + REQUIRE(held->image(0).data.front() == std::byte{ 1 }); + REQUIRE(host.publisher().stats(0).no_free_slot == 0); + + // A backend that takes a slot and neither publishes nor abandons it is what fills a pool. + const std::optional forgotten = camera.output->acquire(0); + REQUIRE(forgotten.has_value()); + REQUIRE_FALSE(camera.publish(99)); // stream 0 has no free slot left; the frame set is dropped + REQUIRE(host.publisher().stats(0).no_free_slot == 1); + REQUIRE(held->image(0).data.front() == std::byte{ 1 }); + + camera.output->abandon(0, forgotten->slot); + REQUIRE(camera.publish(100)); +}