diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 8018cea070..0717f5b7b2 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -26,6 +26,10 @@ add_subdirectory(camera_core) # Camera interface: shared memory and the socket between processes if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) + # Camera interface: the messages, the plugin's end and the reader's end of the connection + add_subdirectory(camera_protocol) + add_subdirectory(camera_endpoint) + add_subdirectory(camera_reader) endif() # Build OXR library (depends on oxr_utils for types) diff --git a/src/core/camera_endpoint/AGENTS.md b/src/core/camera_endpoint/AGENTS.md new file mode 100644 index 0000000000..963c3630a7 --- /dev/null +++ b/src/core/camera_endpoint/AGENTS.md @@ -0,0 +1,20 @@ + + +# Agent notes — `camera_endpoint` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) for what runs on which thread before changing it. + +- **Nothing sits between publish and the socket.** A frame event is encoded and sent inside `send_event`, on the publishing thread. Do not add a queue or a thread hop there, and do not log per frame. +- **`send_event` holds its camera group's lock.** It must not take another lock, block or call the publisher. That lock is also what serialises each connection's event encoder. +- **Unsubscribe before a connection is destroyed.** `Publisher::unsubscribe` returning is the guarantee that no `send_event` is running or will start for that sink. +- **A reader is untrusted.** Whatever it sends can at most get it dropped; it must not stop the thread or affect other readers. +- **Reserve, answer, activate.** The handshake response is the first message a reader receives: `subscribe`, send the response with its descriptors, then `activate`. Every way out of that sequence ends in `drop()`, which is the one place a subscription is given up. +- **A test does not assume which thread runs first.** A reader's request and the camera's publish come from two threads and are served in either order: have the reader ask for what comes after a frame set it names, not for "anything", when the test expects a particular one. Run the suite on one CPU (`taskset -c 0`) before pushing; CI's slower machines find these, a workstation does not. +- **A test's helper thread is stopped and joined however the test ends.** A failed `REQUIRE` throws, and a `std::thread` that is still joinable then aborts the process instead of reporting the failure. Use `std::jthread`, or a guard whose destructor joins. +- **A dropped connection returns host-memory leases only because nothing reads them any more.** Do not extend that to a kind of memory whose readers can leave work running on a slot. +- **The constructor takes the path; only `start()` listens.** Readers find a plugin by connecting to every endpoint of their user, so an endpoint that accepts before it can answer costs each of them a wait. Do not listen in the constructor, and call `listen()` before the service thread begins: a listener that does not listen yet polls as hung up. diff --git a/src/core/camera_endpoint/CMakeLists.txt b/src/core/camera_endpoint/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_endpoint/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_endpoint/README.md b/src/core/camera_endpoint/README.md new file mode 100644 index 0000000000..7a1bda28dd --- /dev/null +++ b/src/core/camera_endpoint/README.md @@ -0,0 +1,78 @@ + + +# Camera interface: the endpoint + +A camera plugin's side of the conversation with readers. It puts a +[`camera_core`](../camera_core) `Publisher` on a socket ([`camera_ipc`](../camera_ipc)) and +speaks the [`camera_protocol`](../camera_protocol). + +```cpp +Publisher publisher; +Endpoint endpoint(publisher, socket_path); // takes the path + +// ... the plugin opens its cameras, and so knows what each one publishes ... +const GroupId head = publisher.add_group(description); +HostPool left(slots, bytes, "head.left"), right(slots, bytes, "head.right"); +endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); +endpoint.start(); // readers can connect from here + +// ... the backend acquires slots, writes pixels and publishes; readers come and go ... +endpoint.stop(); // before the pools go away +``` + +## When readers can reach it + +An endpoint is visible to readers from `start()` to `stop()`, and at no other time. + +- **Constructing it takes the path.** The socket file is there and the path is locked, so a + second process that wants the same path fails at once with "another process has taken ...", + before it has opened a camera. +- **Until `start()` a reader that connects is refused at once.** It is not accepted and then + left waiting for an answer nobody sends. A reader that looks through the user's camera + plugins ([`camera_reader`](../camera_reader)) passes over such an endpoint without waiting. + So a plugin constructs its endpoint first, opens its cameras, and calls `start()` when it + can answer. +- **`stop()` gives the path up.** The socket file is removed and a reader is refused again. + A later `start()` takes the path again. + +## What runs where + +- **One thread of its own** accepts readers and handles what they send: the handshake, the + requests for the newest frame set and the releases. It sleeps in `poll` until a reader + sends something. +- **A frame set is sent from the thread that publishes** to a reader that is waiting for it: + straight from `Publisher::publish`, with a send that does not block. There is no queue and + no hand-over to another thread between a frame set being published and its event being on + the socket. A frame set that already exists when a newest-order reader asks, and the one + that follows a release, are sent from the endpoint's thread. +- **The answer to a handshake comes before any frame set.** The subscription is reserved, + which counts it in the pool's budget, then the response is sent, then delivery starts. + Capture does not wait for any of it. A reader that is gone before it could be answered + leaves no reservation behind. + +## When a reader goes away + +A closed connection is the signal, whether the reader exited, crashed or was killed. Its +subscription ends and every frame set it held goes back to the pool. A reader that sends +bytes that are not a message, a message only a plugin sends, or an operation this build +does not know, is dropped the same way; other readers are not affected. + +Taking the frame sets back at once is right for host memory, which is all there is today: a +reader that is gone reads nothing. A closed connection is not evidence that work a reader +handed to a GPU or a hardware encoder has finished, so a kind of memory where that matters +needs its own rule for these slots before it is served from here (see +[`camera_core`](../camera_core), "What publishing, holding and releasing mean"). + +## Logging + +`isaaccapture.camera_endpoint.Endpoint` logs a reader subscribing, being refused and leaving. +A reader that only asks what is published is logged at debug, because a reader that looks for +its camera asks that of every plugin of its user. Nothing is logged per frame. + +## Tests + +`tests/cpp/core/camera_endpoint`, run with `ctest -L camera_endpoint`. They are tagged `unit` +and play the reader's part with the socket and the codec directly. diff --git a/src/core/camera_endpoint/cpp/CMakeLists.txt b/src/core/camera_endpoint/cpp/CMakeLists.txt new file mode 100644 index 0000000000..13a9ea7ee3 --- /dev/null +++ b/src/core/camera_endpoint/cpp/CMakeLists.txt @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +# The plugin's side of the conversation with readers: serves a Publisher's groups over a socket. +add_library(camera_endpoint STATIC + endpoint.cpp +) + +target_include_directories(camera_endpoint INTERFACE + $ +) + +target_link_libraries(camera_endpoint + PUBLIC + camera::camera_core + PRIVATE + camera::camera_ipc + camera::camera_protocol + log_bridge::log_bridge_core + Threads::Threads +) + +add_library(camera::camera_endpoint ALIAS camera_endpoint) diff --git a/src/core/camera_endpoint/cpp/endpoint.cpp b/src/core/camera_endpoint/cpp/endpoint.cpp new file mode 100644 index 0000000000..1cc904b7a0 --- /dev/null +++ b/src/core/camera_endpoint/cpp/endpoint.cpp @@ -0,0 +1,442 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_endpoint/endpoint.hpp" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// One reader. As an EventSink it is called by the publisher with its group's lock held: on the +// capture thread when a frame set is published, on the endpoint's thread when the reader +// released or asked for one. Everything else runs on the endpoint's thread. +class Connection final : public EventSink +{ +public: + explicit Connection(MessageSocket connected) : socket(std::move(connected)) + { + } + + bool send_event(SubscriptionId /*subscription*/, const FrameEvent& event) noexcept override + { + try + { + return socket.send(m_event_encoder.encode(event)) == SendStatus::SENT; + } + catch (const std::exception&) + { + return false; // the reader gets a later frame set, or its closed socket is noticed + } + } + + MessageSocket socket; + std::string name = "unnamed"; + bool greeted = false; + SubscriptionId subscription = 0; + +private: + MessageEncoder m_event_encoder; // used inside send_event only, which the group's lock serialises +}; + +// How many messages one reader is served before the others get their turn. A reader that sends +// without pause then slows nobody but itself, and cannot keep the endpoint from stopping: its +// socket stays readable, so the rest is handled in the next round. +constexpr int MESSAGES_PER_TURN = 32; + +const char* name_of(DeliveryOrder order) +{ + switch (order) + { + case DeliveryOrder::NEWEST: + return "newest"; + case DeliveryOrder::IN_ORDER: + return "in order"; + default: + return "default order"; + } +} + +} // namespace + +class Endpoint::Impl +{ +public: + Impl(Publisher& publisher, const std::string& path) + : m_publisher(publisher), + m_path(path), + m_listener(std::in_place, path), + m_wake(::eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK)), + m_buffer(MAX_MESSAGE_BYTES), + m_logger(isaaccapture::Logger::get("isaaccapture.camera_endpoint.Endpoint")) + { + if (!m_wake.valid()) + { + throw std::system_error(errno, std::generic_category(), "creating the camera endpoint's wake-up descriptor"); + } + } + + ~Impl() + { + stop(); + } + + void set_group_memory(GroupId group, std::vector descriptors) + { + if (m_thread.joinable()) + { + throw std::logic_error("a camera group's memory is named before the endpoint starts"); + } + const GroupDescription description = m_publisher.group_description(group); + if (descriptors.size() != description.streams.size()) + { + throw std::invalid_argument("camera group '" + description.id + "' needs one descriptor per stream"); + } + m_memory[group] = std::move(descriptors); + } + + void start() + { + if (m_thread.joinable()) + { + return; + } + if (!m_listener) + { + m_listener.emplace(m_path); + } + // Before the thread begins: a listener that does not listen yet polls as hung up, and + // the thread would spin on it. + m_listener->listen(); + m_stopping = false; + m_thread = std::thread([this] { run(); }); + m_logger->info("serving cameras on '{}'", m_path); + } + + void stop() + { + if (!m_thread.joinable()) + { + return; + } + m_stopping = true; + const std::uint64_t one = 1; + // Cannot fail in a way that matters: the counter only overflows after 2^64 wake-ups. + [[maybe_unused]] const ssize_t written = ::write(m_wake.get(), &one, sizeof(one)); + m_thread.join(); + // Without the listener a reader that tries to connect is refused at once, instead of + // waiting in a queue nobody serves. + m_listener.reset(); + } + + std::size_t reader_count() const + { + return m_reader_count.load(); + } + +private: + void run() + { + std::vector waits; + while (!m_stopping) + { + waits.assign({ { m_wake.get(), POLLIN, 0 }, { m_listener->descriptor(), POLLIN, 0 } }); + for (const auto& connection : m_connections) + { + waits.push_back({ connection->socket.descriptor(), POLLIN, 0 }); + } + if (::poll(waits.data(), waits.size(), -1) < 0) + { + if (errno == EINTR) + { + continue; + } + m_logger->error("camera endpoint stops serving: poll failed with errno {}", errno); + break; + } + + // Back to front, so dropping a connection leaves the indices still to visit alone. + for (std::size_t i = m_connections.size(); i-- > 0;) + { + if (waits[i + 2].revents != 0 && !serve(*m_connections[i])) + { + drop(i); + } + } + if ((waits[1].revents & POLLIN) != 0) + { + accept_readers(); + } + } + while (!m_connections.empty()) + { + drop(m_connections.size() - 1); + } + // Empty the wake-up counter, so that serving can start again. + std::uint64_t wakes = 0; + [[maybe_unused]] const ssize_t drained = ::read(m_wake.get(), &wakes, sizeof(wakes)); + } + + void accept_readers() + { + try + { + while (auto socket = m_listener->accept(std::chrono::milliseconds(0))) + { + m_connections.push_back(std::make_unique(std::move(*socket))); + m_reader_count = m_connections.size(); + } + } + catch (const std::exception& error) + { + m_logger->warn("could not accept a camera reader: {}", error.what()); + } + } + + // Ends the reader's subscription before its sink goes away. unsubscribe() returns only once + // no send_event is running or can start for it. + void drop(std::size_t index) + { + Connection& connection = *m_connections[index]; + if (connection.subscription != 0) + { + m_publisher.unsubscribe(connection.subscription); + m_logger->info("camera reader '{}' is gone", connection.name); + } + else + { + // At debug: a reader that looks for its camera asks every plugin and leaves at once. + m_logger->debug("camera reader '{}' is gone", connection.name); + } + m_connections.erase(m_connections.begin() + static_cast(index)); + m_reader_count = m_connections.size(); + } + + // Handles what the reader has sent, up to one turn's worth. Returns false when the + // connection is to be dropped. + bool serve(Connection& connection) + { + try + { + for (int handled = 0; handled < MESSAGES_PER_TURN; ++handled) + { + const ReceivedMessage received = connection.socket.receive(m_buffer, std::chrono::milliseconds(0)); + if (received.status == ReceiveStatus::TIMEOUT) + { + return true; + } + if (received.status == ReceiveStatus::CLOSED) + { + return false; + } + const ProtocolMessage message = decode_message({ m_buffer.data(), received.bytes }); + if (!handle(connection, message)) + { + return false; + } + } + return true; + } + catch (const std::exception& error) + { + m_logger->warn("dropping camera reader '{}': {}", connection.name, error.what()); + return false; + } + } + + bool handle(Connection& connection, const ProtocolMessage& message) + { + if (const auto* request = std::get_if(&message)) + { + return shake_hands(connection, *request); + } + if (const auto* release = std::get_if(&message)) + { + if (connection.subscription != 0) + { + m_publisher.release(connection.subscription, release->lease_id); + } + return true; + } + if (const auto* acquire = std::get_if(&message)) + { + if (connection.subscription == 0 || + !m_publisher.request_newest(connection.subscription, acquire->after_sequence)) + { + m_logger->warn( + "dropping camera reader '{}': it asked for the newest frame set without a " + "newest-order subscription", + connection.name); + return false; + } + return true; + } + if (std::holds_alternative(message)) + { + return true; // something a newer reader sends for information; not for this build + } + m_logger->warn("dropping camera reader '{}': it sent a message only a plugin sends", connection.name); + return false; + } + + // Answers a reader's first message: what is published, or the group it asked for with the + // descriptors of its pools. A reader that is refused is told why and then dropped. + // + // A subscription is reserved, then the reader is told, then delivery starts: the answer is + // the first thing the reader receives, and capture never waits for it to be sent. A + // subscription whose answer could not be sent ends in drop(), like any other. + bool shake_hands(Connection& connection, const HandshakeRequest& request) + { + if (connection.greeted) + { + m_logger->warn("dropping camera reader '{}': it sent a second handshake", connection.name); + return false; + } + connection.greeted = true; + connection.name = request.reader_name.empty() ? "unnamed" : request.reader_name; + + HandshakeResponse response; + // The connection uses the newest version both sides speak. A reader newer than this + // plugin is answered in the plugin's version and decides whether it still speaks it. + response.protocol_version = std::min(request.protocol_version, PROTOCOL_VERSION); + std::span descriptors; + const std::optional group = m_publisher.find_group(request.group); + if (request.protocol_version < OLDEST_PROTOCOL_VERSION) + { + response.protocol_version = OLDEST_PROTOCOL_VERSION; + response.error = "this camera plugin speaks protocol versions " + std::to_string(OLDEST_PROTOCOL_VERSION) + + " to " + std::to_string(PROTOCOL_VERSION) + " and the reader version " + + std::to_string(request.protocol_version); + } + else if (!request.unsupported.empty()) + { + response.error = request.unsupported; + } + else if (request.group.empty()) + { + const std::size_t group_count = m_publisher.group_count(); + for (std::size_t i = 0; i < group_count; ++i) + { + response.groups.push_back({ m_publisher.group_description(static_cast(i)), {} }); + } + // At debug: a reader that looks for its camera asks this of every plugin of its user. + m_logger->debug("camera reader '{}' asked what is published", connection.name); + } + else if (!group) + { + response.error = "this camera plugin publishes no group '" + request.group + "'"; + } + else if (!m_memory.contains(*group)) + { + response.error = "camera group '" + request.group + "' has no memory a reader can map"; + } + else + { + const SubscribeResult result = m_publisher.subscribe(connection, *group, request.options); + if (result) + { + connection.subscription = result.id; + response.groups.push_back({ m_publisher.group_description(*group), {} }); + descriptors = m_memory.at(*group); + } + else + { + response.error = result.error; + } + } + + if (!response.error.empty()) + { + m_logger->warn("refusing camera reader '{}': {}", connection.name, response.error); + } + if (!send(connection, m_encoder.encode(response), descriptors) || !response.error.empty()) + { + return false; + } + if (connection.subscription != 0) + { + m_publisher.activate(connection.subscription); + m_logger->info("camera reader '{}' reads '{}' (window {}, {})", connection.name, request.group, + request.options.window, name_of(resolved_order(request.options))); + } + return true; + } + + static bool send(Connection& connection, std::span bytes, std::span descriptors = {}) + { + return connection.socket.send(bytes, descriptors) == SendStatus::SENT; + } + + Publisher& m_publisher; + std::string m_path; + std::optional m_listener; // absent between stop() and the next start() + UniqueFd m_wake; + std::map> m_memory; + + std::thread m_thread; + std::atomic m_stopping{ false }; + std::atomic m_reader_count{ 0 }; + + // The endpoint's thread only. + std::vector> m_connections; + std::vector m_buffer; + MessageEncoder m_encoder; + + std::shared_ptr m_logger; +}; + +Endpoint::Endpoint(Publisher& publisher, const std::string& path) : m_impl(std::make_unique(publisher, path)) +{ +} + +Endpoint::~Endpoint() = default; + +void Endpoint::set_group_memory(GroupId group, std::vector descriptors) +{ + m_impl->set_group_memory(group, std::move(descriptors)); +} + +void Endpoint::start() +{ + m_impl->start(); +} + +void Endpoint::stop() +{ + m_impl->stop(); +} + +std::size_t Endpoint::reader_count() const +{ + return m_impl->reader_count(); +} + +} // namespace core::camera diff --git a/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp b/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp new file mode 100644 index 0000000000..02da61ac04 --- /dev/null +++ b/src/core/camera_endpoint/cpp/inc/camera_endpoint/endpoint.hpp @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include + +namespace core::camera +{ + +/** + * @brief Serves a Publisher's groups to readers in other processes. + * + * Listens on a socket file. A reader's first message says what it came for, and the answer is + * the first thing it receives: what is published, or the group it asked for with the + * descriptors of the group's pools. After that an in-order reader is sent each frame set as + * it is published and a newest-order reader one each time it asks. A reader that goes away, + * cleanly or not, has every frame set it held taken back. + */ +class Endpoint +{ +public: + /** + * @brief Take `path` without serving it yet: until start(), a reader that connects is + * refused at once. + * @param publisher Must outlive the endpoint. + * @throws std::system_error if another process has taken the path or it cannot be bound. + */ + Endpoint(Publisher& publisher, const std::string& path); + + // Stops serving. + ~Endpoint(); + + Endpoint(const Endpoint&) = delete; + Endpoint& operator=(const Endpoint&) = delete; + + /** + * @brief Name what a reader maps for a group: one descriptor per stream, in stream order, + * for the stream's pool. The endpoint does not own them; they stay open while it serves. + * A group without them cannot be subscribed to. + * @throws std::logic_error after start(). + * @throws std::out_of_range for a group the publisher does not have. + * @throws std::invalid_argument if the count does not match the group's streams. + */ + void set_group_memory(GroupId group, std::vector descriptors); + + /** + * @brief Serve readers: from here on a reader that connects is answered. After a stop() + * this takes the path again first. + * @throws std::system_error if another process has taken the path meanwhile, or the + * socket cannot listen. + */ + void start(); + + /** + * @brief Stop serving: every subscription ends, every reader sees its connection close, and + * the path is given up, so a reader that tries to connect is refused at once. Does nothing + * if the endpoint is not serving. + */ + void stop(); + + // Readers connected right now. + std::size_t reader_count() const; + +private: + class Impl; + std::unique_ptr m_impl; +}; + +} // namespace core::camera diff --git a/src/core/camera_protocol/AGENTS.md b/src/core/camera_protocol/AGENTS.md new file mode 100644 index 0000000000..ae137f279a --- /dev/null +++ b/src/core/camera_protocol/AGENTS.md @@ -0,0 +1,22 @@ + + +# Agent notes — `camera_protocol` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +The schema is [`../schema/fbs/camera_protocol.fbs`](../schema/fbs/camera_protocol.fbs). Read [`../schema/AGENTS.md`](../schema/AGENTS.md) and [`README.md`](README.md) before changing it: a reader and a plugin from different releases must keep understanding each other, and the golden check enforces it. + +- **Few messages, and no fewer than correctness needs.** A connection has one purpose, stated in the handshake; do not add a message for what a handshake field can say. A newest-order reader asks for each frame set (`CameraAcquire`): do not fold that back into sending frame sets as they are published. +- **This is local IPC.** Lease ids, slot indices and attached descriptors name resources of one machine. Do not reuse the envelope as a network or recording format, and do not wrap it in the recorder's `Record` tables or its `DeviceDataTimestamp`. +- **A new message says whether it may be ignored.** One that only informs is sent with `skippable`; an operation the peer must carry out is not, and needs a protocol version so that it is only sent to a peer that knows it. +- **The numbers are the protocol.** Union members, flag bits and enum values are written out in the schema and pinned by `static_assert` and a test; a member keeps its number. +- **FlatBuffers stays inside this module.** Its users see `camera_core` types; only the codec and its tests include the generated header. +- **Decoded bytes are untrusted.** Verify, then bound every count against the fixed arrays it fills, and check for a body behind a body type. The decoder establishes structure only; what a description means is checked by `camera_core`'s `problem_with` where it is used. +- **The encoder refuses, it does not truncate.** A count that does not fit is an error to the caller, never a shorter message. +- **A group this build cannot read is an entry, not an error.** A kind added by a newer plugin marks its group `unsupported` and leaves the other groups of the same response readable. +- **A new enum value needs both sides.** The core enum and the schema enum carry the same numbers (pinned by `static_assert`), and the decoder must accept the value before a plugin sends it. +- **The frame path does not allocate.** Keep `encode(const FrameEvent&)` free of containers that do. +- **Evolution is tested with hand-built messages.** A passing golden comparison shows layout, not that two releases understand each other: a rule about newer or older peers gets a test that builds the message that peer would send. diff --git a/src/core/camera_protocol/CMakeLists.txt b/src/core/camera_protocol/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_protocol/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_protocol/README.md b/src/core/camera_protocol/README.md new file mode 100644 index 0000000000..ac1bbfbef7 --- /dev/null +++ b/src/core/camera_protocol/README.md @@ -0,0 +1,106 @@ + + +# Camera interface: the protocol between a camera plugin and a reader + +What a camera plugin and a reader say to each other over their socket +([`camera_ipc`](../camera_ipc)). Each message is one socket message: + +```text +reader plugin + | handshake request | + | (version, group, window, order) | + |--------------------------------------->| + | handshake response (the group) with | + | one pool descriptor per stream | + |<---------------------------------------| + | acquire (after which frame set) | newest order only: once per frame set + |--------------------------------------->| + | frame set (lease, slot per stream, | + | times) | + |<---------------------------------------| + | release (lease) | + |--------------------------------------->| +``` + +- **One connection, one purpose.** A connection carries one subscription, so the reader says + what it came for in its first message and the plugin's answer settles it. A reader that + wants two groups opens two connections. A request that names no group asks only what the + plugin publishes, and the response lists every group. +- **The answer comes first.** The handshake response is the first message a plugin sends. The + subscription is reserved, the response is sent, and only then does delivery start. +- **A refusal ends the connection.** The response then carries the reason: a protocol version + the plugin no longer speaks, a group that does not exist, or a window that does not fit + the pool. +- **A newest-order reader asks for each frame set.** `acquire` means: hold the newest frame + set for me, now if there is one after the one I name, otherwise as soon as there is. The + frame set is chosen when the request is served, so a reader that was busy gets what the + camera saw last. An in-order reader is sent every frame set without asking. +- **Closing the connection is the unsubscribe.** Every frame set the reader still held goes + back to the pool. + +## What this protocol is not + +It is spoken between two processes on one machine. A lease id means something on one +connection; a slot index and the file descriptors that travel with the handshake response +name memory of that machine. None of that is a network packet's identity, and nothing here +is a recording format. A transport that sends frames to another machine carries what +identifies a capture there (the group's `session_id` and a frame set's `sequence`, its +times, its flags) in its own messages, and a recorder maps the times to its own timestamps. + +The descriptors attached to a handshake response are one per stream, in stream order. That +is the rule for host memory, the one kind there is. A kind of memory that needs something +else describes its attachments when it is added. + +## Schema + +The messages are FlatBuffers tables in +[`schema/fbs/camera_protocol.fbs`](../schema/fbs/camera_protocol.fbs), with the other +schemas of the repository. They are never recorded, but they are a contract in the same +way: a reader and a plugin can come from different releases. The golden check of +[`schema`](../schema/README.md) therefore guards them like any other schema, and the same +rules apply: fields are appended with fresh ids and left optional, nothing is renumbered. +The numbers of the messages in their union and the bits of the frame flags are written out +in the schema and pinned by the codec and its tests. The one struct, a plane's offset and +stride, is complete as it is and is never extended. + +A matching golden says the bytes still line up. It does not say two releases still +understand each other, so the rules below are tested with messages built the way a newer +and an older peer would build them. + +## A reader and a plugin from different releases + +- **Version.** Each side has a newest and an oldest protocol version it speaks. The reader + sends its newest; the plugin answers with the older of that and its own newest, and + refuses a reader that is older than its oldest. The reader checks the answer against its + own range. A version is added when one side comes to depend on an operation the other + must carry out; fields and messages that only inform need none. +- **A message type this build does not know** is ignored when its sender marked it + `skippable`, which is for messages that only inform. Not marked, it is an operation the + sender counts on, and the receiver does not carry on as if it had not been sent: a plugin + drops the reader, which the reader notices, and a reader's `open` or `next` fails with + the reason. +- **A group this build cannot read**, because it uses a codec, a pixel format or a kind of + memory added later, is still listed, with the reason. The groups next to it are + unaffected, and a reader that asks for it is told why it cannot have it. +- **A request for something added later**, such as a delivery order, is refused with the + reason. + +## Codec + +`MessageEncoder` and `decode_message` turn [`camera_core`](../camera_core) types into bytes +and back, so nothing outside this module sees FlatBuffers. The encoder reuses its buffer; a +frame set is encoded without allocating. It refuses a message that names more streams, +images or planes than a group can have, and writes nothing in its place. + +The decoder treats its input as coming from another process: it verifies the buffer and bounds +the counts that have a limit: streams per group, planes per stream, images per frame set. That +establishes that the bytes are a message, not that what they describe makes sense. Whether a +group's planes fit its slots is checked by `problem_with` in [`camera_core`](../camera_core), +by the plugin for what a backend declares and by the reader for what it was sent. + +## Tests + +`tests/cpp/core/camera_protocol`, run with `ctest -L camera_protocol`. They are tagged `unit`. diff --git a/src/core/camera_protocol/cpp/CMakeLists.txt b/src/core/camera_protocol/cpp/CMakeLists.txt new file mode 100644 index 0000000000..2018377860 --- /dev/null +++ b/src/core/camera_protocol/cpp/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Turns the messages between a camera plugin and its readers into bytes and back. The +# schema is schema/fbs/camera_protocol.fbs; its FlatBuffers types stay inside this library, +# and its users see camera_core types. +add_library(camera_protocol STATIC + codec.cpp +) + +target_include_directories(camera_protocol INTERFACE + $ +) + +target_link_libraries(camera_protocol + PUBLIC + camera::camera_core + PRIVATE + isaacteleop_schema +) + +add_library(camera::camera_protocol ALIAS camera_protocol) diff --git a/src/core/camera_protocol/cpp/codec.cpp b/src/core/camera_protocol/cpp/codec.cpp new file mode 100644 index 0000000000..27e4f429d9 --- /dev/null +++ b/src/core/camera_protocol/cpp/codec.cpp @@ -0,0 +1,381 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_protocol/messages.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// The schema's enums, flag bits and message numbers are the core's and the protocol's: a value +// converts by a cast, and a change to either side stops the build here. +static_assert(static_cast(PixelFormat::BGRA8) == CameraPixelFormat_Bgra8); +static_assert(static_cast(PixelFormat::RGBA8) == CameraPixelFormat_Rgba8); +static_assert(static_cast(PixelFormat::NV12) == CameraPixelFormat_Nv12); +static_assert(static_cast(PixelFormat::GRAY8) == CameraPixelFormat_Gray8); +static_assert(static_cast(Codec::NONE) == CameraCodec_None); +static_assert(static_cast(Codec::H264) == CameraCodec_H264); +static_assert(static_cast(Codec::H265) == CameraCodec_H265); +static_assert(static_cast(MemoryKind::HOST) == CameraMemoryKind_Host); +static_assert(static_cast(DeliveryOrder::DEFAULT) == CameraDeliveryOrder_Default); +static_assert(static_cast(DeliveryOrder::NEWEST) == CameraDeliveryOrder_Newest); +static_assert(static_cast(DeliveryOrder::IN_ORDER) == CameraDeliveryOrder_InOrder); +static_assert(std::uint32_t{ IMAGE_FLAG_KEYFRAME } == std::uint32_t{ CameraFrameFlag_Keyframe }); +static_assert(std::uint32_t{ IMAGE_FLAG_SDK_WARNING } == std::uint32_t{ CameraFrameFlag_SdkWarning }); +static_assert(std::uint32_t{ CameraFrameFlag_Keyframe } == 1u && std::uint32_t{ CameraFrameFlag_SdkWarning } == 2u); +static_assert(CameraMessageBody_CameraHandshakeRequest == 1); +static_assert(CameraMessageBody_CameraHandshakeResponse == 2); +static_assert(CameraMessageBody_CameraFrameSet == 3); +static_assert(CameraMessageBody_CameraRelease == 4); +static_assert(CameraMessageBody_CameraAcquire == 5); + +using Builder = flatbuffers::FlatBufferBuilder; + +flatbuffers::Offset build_group(Builder& builder, const GroupDescription& group) +{ + if (group.streams.size() > MAX_STREAMS_PER_GROUP) + { + throw std::invalid_argument("camera group '" + group.id + "' has more streams than a group can have"); + } + std::vector> streams; + streams.reserve(group.streams.size()); + for (const StreamDescription& stream : group.streams) + { + if (stream.plane_count > MAX_PLANES_PER_IMAGE) + { + throw std::invalid_argument("camera stream '" + stream.id + "' has more planes than an image can have"); + } + std::array planes; + for (std::size_t i = 0; i < stream.plane_count; ++i) + { + planes[i] = CameraPlaneLayout(stream.planes[i].offset_bytes, stream.planes[i].stride_bytes); + } + const auto id = builder.CreateString(stream.id); + const auto plane_vector = builder.CreateVectorOfStructs(planes.data(), stream.plane_count); + streams.push_back(CreateCameraStream(builder, id, static_cast(stream.codec), + static_cast(stream.pixel_format), + static_cast(stream.memory), stream.width, stream.height, + plane_vector, stream.slot_bytes, stream.slot_count, stream.frame_rate)); + } + const auto id = builder.CreateString(group.id); + const auto stream_vector = builder.CreateVector(streams); + return CreateCameraGroup(builder, id, stream_vector, group.session_id); +} + +std::string text_of(const flatbuffers::String* string) +{ + return string != nullptr ? string->str() : std::string(); +} + +std::optional pixel_format_of(CameraPixelFormat format) +{ + switch (format) + { + case CameraPixelFormat_Bgra8: + case CameraPixelFormat_Rgba8: + case CameraPixelFormat_Nv12: + case CameraPixelFormat_Gray8: + return static_cast(format); + default: + return std::nullopt; + } +} + +std::optional codec_of(CameraCodec codec) +{ + switch (codec) + { + case CameraCodec_None: + case CameraCodec_H264: + case CameraCodec_H265: + return static_cast(codec); + default: + return std::nullopt; + } +} + +std::optional order_of(CameraDeliveryOrder order) +{ + switch (order) + { + case CameraDeliveryOrder_Default: + case CameraDeliveryOrder_Newest: + case CameraDeliveryOrder_InOrder: + return static_cast(order); + default: + return std::nullopt; + } +} + +// Why this build cannot represent a stream; nothing when it can. A newer plugin may publish +// such a stream next to ones this build reads, so this is a property of the group and not a +// broken message. +std::string unsupported_by_this_build(const CameraStream& stream, StreamDescription& description) +{ + const std::string name = "camera stream '" + description.id + "'"; + const auto codec = codec_of(stream.codec()); + if (!codec) + { + return name + " uses a codec this build does not know (" + std::to_string(stream.codec()) + ")"; + } + if (stream.memory() != CameraMemoryKind_Host) + { + return name + " is in a kind of memory this build does not know (" + std::to_string(stream.memory()) + ")"; + } + const auto pixel_format = pixel_format_of(stream.pixel_format()); + if (*codec == Codec::NONE && !pixel_format) // an encoded stream has no pixel format to know + { + return name + " uses a pixel format this build does not know (" + std::to_string(stream.pixel_format()) + ")"; + } + const auto* planes = stream.planes(); + const std::uint32_t plane_count = planes != nullptr ? planes->size() : 0; + if (plane_count > MAX_PLANES_PER_IMAGE) + { + return name + " has more planes than this build can describe"; + } + description.codec = *codec; + description.pixel_format = pixel_format.value_or(PixelFormat::BGRA8); + description.memory = MemoryKind::HOST; + description.plane_count = plane_count; + for (std::uint32_t i = 0; i < plane_count; ++i) + { + description.planes[i] = { planes->Get(i)->offset_bytes(), planes->Get(i)->stride_bytes() }; + } + return {}; +} + +GroupEntry read_group(const CameraGroup& group) +{ + GroupEntry entry; + entry.group.id = text_of(group.id()); + entry.group.session_id = group.session_id(); + const auto* streams = group.streams(); + if (streams == nullptr) + { + return entry; + } + if (streams->size() > MAX_STREAMS_PER_GROUP) + { + entry.unsupported = "camera group '" + entry.group.id + "' has more streams than this build can read"; + return entry; + } + for (const CameraStream* stream : *streams) + { + StreamDescription description; + description.id = text_of(stream->id()); + entry.unsupported = unsupported_by_this_build(*stream, description); + if (!entry.unsupported.empty()) + { + entry.group.streams.clear(); + return entry; + } + description.width = stream->width(); + description.height = stream->height(); + description.slot_bytes = stream->slot_bytes(); + description.slot_count = stream->slot_count(); + description.frame_rate = stream->frame_rate(); + entry.group.streams.push_back(std::move(description)); + } + return entry; +} + +FrameEvent read_frame_set(const CameraFrameSet& frame_set) +{ + FrameEvent event; + event.lease_id = frame_set.lease_id(); + event.sequence = frame_set.sequence(); + event.generation = frame_set.generation(); + event.skipped = frame_set.skipped(); + event.timeline = { frame_set.capture_ns(), frame_set.sdk_ready_ns(), frame_set.published_ns(), frame_set.sent_ns() }; + if (const auto* frames = frame_set.frames()) + { + if (frames->size() > MAX_STREAMS_PER_GROUP) + { + throw ProtocolError("a camera frame set has more images than a group has streams"); + } + event.image_count = frames->size(); + for (std::uint32_t i = 0; i < frames->size(); ++i) + { + const CameraFrameRef* frame = frames->Get(i); + event.images[i] = { frame->slot_index(), static_cast(frame->flags()), frame->bytes_used(), + frame->device_frame_id(), frame->device_time_ns() }; + } + } + return event; +} + +} // namespace + +struct MessageEncoder::Impl +{ + Builder builder{ 1024 }; + + template + std::span finish(CameraMessageBody type, flatbuffers::Offset body) + { + builder.Finish(CreateCameraMessage(builder, type, body.Union())); + if (builder.GetSize() > MAX_MESSAGE_BYTES) + { + throw std::length_error("a camera message of " + std::to_string(builder.GetSize()) + + " bytes is over the limit of " + std::to_string(MAX_MESSAGE_BYTES)); + } + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; + } +}; + +MessageEncoder::MessageEncoder() : m_impl(std::make_unique()) +{ +} + +MessageEncoder::~MessageEncoder() = default; + +std::span MessageEncoder::encode(const HandshakeRequest& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto name = builder.CreateString(message.reader_name); + const auto group = builder.CreateString(message.group); + return m_impl->finish( + CameraMessageBody_CameraHandshakeRequest, + CreateCameraHandshakeRequest(builder, message.protocol_version, name, group, message.options.window, + static_cast(message.options.order))); +} + +std::span MessageEncoder::encode(const HandshakeResponse& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::vector> groups; + groups.reserve(message.groups.size()); + for (const GroupEntry& entry : message.groups) + { + groups.push_back(build_group(builder, entry.group)); + } + const auto group_vector = builder.CreateVector(groups); + const auto error = builder.CreateString(message.error); + return m_impl->finish(CameraMessageBody_CameraHandshakeResponse, + CreateCameraHandshakeResponse(builder, message.protocol_version, error, group_vector)); +} + +// The per-frame path: nothing here allocates once the builder has grown to a frame's size. +std::span MessageEncoder::encode(const FrameEvent& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + if (message.image_count > MAX_STREAMS_PER_GROUP) + { + throw std::invalid_argument("a camera frame set has more images than a group has streams"); + } + std::array, MAX_STREAMS_PER_GROUP> frames; + for (std::size_t i = 0; i < message.image_count; ++i) + { + const ImageInfo& image = message.images[i]; + frames[i] = CreateCameraFrameRef(builder, image.slot, static_cast(image.flags), + image.bytes_used, image.device_frame_id, image.device_time_ns); + } + const auto frame_vector = builder.CreateVector(frames.data(), message.image_count); + return m_impl->finish(CameraMessageBody_CameraFrameSet, + CreateCameraFrameSet(builder, message.lease_id, message.sequence, message.generation, + message.skipped, message.timeline.capture_ns, message.timeline.sdk_ready_ns, + message.timeline.published_ns, message.timeline.sent_ns, frame_vector)); +} + +std::span MessageEncoder::encode(const ReleaseMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + return m_impl->finish(CameraMessageBody_CameraRelease, CreateCameraRelease(builder, message.lease_id)); +} + +std::span MessageEncoder::encode(const AcquireMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + return m_impl->finish(CameraMessageBody_CameraAcquire, CreateCameraAcquire(builder, message.after_sequence)); +} + +ProtocolMessage decode_message(std::span bytes) +{ + const auto* data = reinterpret_cast(bytes.data()); + flatbuffers::Verifier verifier(data, bytes.size()); + if (bytes.empty() || !VerifyCameraMessageBuffer(verifier)) + { + throw ProtocolError("received bytes that are not a camera message"); + } + const CameraMessage* message = GetCameraMessage(data); + const auto type = static_cast(message->body_type()); + if (type == CameraMessageBody_NONE || type > CameraMessageBody_MAX) + { + // From a newer peer. One that only informs can be ignored. Anything else is an + // operation the peer counts on, and carrying on without it would go wrong quietly. + if (message->skippable()) + { + return std::monostate{}; + } + throw ProtocolError("received a camera message of a type this build does not know (" + std::to_string(type) + + ") and cannot ignore"); + } + // The verifier accepts a body type without a body. + if (message->body() == nullptr) + { + throw ProtocolError("received a camera message without a body"); + } + switch (message->body_type()) + { + case CameraMessageBody_CameraHandshakeRequest: + { + const CameraHandshakeRequest* request = message->body_as_CameraHandshakeRequest(); + HandshakeRequest result; + result.protocol_version = request->protocol_version(); + result.reader_name = text_of(request->reader_name()); + result.group = text_of(request->group_id()); + result.options.window = request->window(); + if (const auto order = order_of(request->order())) + { + result.options.order = *order; + } + else + { + result.unsupported = "the reader asked for a delivery order this build does not know (" + + std::to_string(request->order()) + ")"; + } + return result; + } + case CameraMessageBody_CameraHandshakeResponse: + { + const CameraHandshakeResponse* response = message->body_as_CameraHandshakeResponse(); + HandshakeResponse result; + result.protocol_version = response->protocol_version(); + result.error = text_of(response->error()); + if (const auto* groups = response->groups()) + { + for (const CameraGroup* group : *groups) + { + result.groups.push_back(read_group(*group)); + } + } + return result; + } + case CameraMessageBody_CameraFrameSet: + return read_frame_set(*message->body_as_CameraFrameSet()); + case CameraMessageBody_CameraRelease: + return ReleaseMessage{ message->body_as_CameraRelease()->lease_id() }; + case CameraMessageBody_CameraAcquire: + return AcquireMessage{ message->body_as_CameraAcquire()->after_sequence() }; + default: + throw ProtocolError("received a camera message this build has no reader for"); + } +} + +} // namespace core::camera diff --git a/src/core/camera_protocol/cpp/inc/camera_protocol/messages.hpp b/src/core/camera_protocol/cpp/inc/camera_protocol/messages.hpp new file mode 100644 index 0000000000..02873695d1 --- /dev/null +++ b/src/core/camera_protocol/cpp/inc/camera_protocol/messages.hpp @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// The newest and the oldest protocol version this build speaks. A version is added when an +// operation one side depends on is added; a field or a message that only informs needs none. +// The handshake settles which version a connection uses. +inline constexpr std::uint32_t PROTOCOL_VERSION = 1; +inline constexpr std::uint32_t OLDEST_PROTOCOL_VERSION = 1; + +// No message is larger than this, so a receive buffer of this size always fits one. +inline constexpr std::size_t MAX_MESSAGE_BYTES = 64 * 1024; + +// Reader to plugin, the first message on a connection. A connection carries one subscription, +// so the reader says here what it came for. +struct HandshakeRequest +{ + std::uint32_t protocol_version = PROTOCOL_VERSION; // the newest version the reader speaks + std::string reader_name; // for the plugin's log + std::string group; // the group to read; empty to ask only what the plugin publishes + ReaderOptions options; // for the group + + // Set by decode_message when the request asks for something this build does not know, + // such as a delivery order added later. The plugin refuses the reader with it. + std::string unsupported; +}; + +// A group as a handshake response describes it. +struct GroupEntry +{ + GroupDescription group; + + // Set by decode_message when the group uses a codec, a pixel format or a kind of memory + // this build does not know. `group` then holds its id only: the group exists and can be + // listed, and it cannot be read by this build. + std::string unsupported; +}; + +// Plugin to reader, the answer to HandshakeRequest and the first message the plugin sends. +struct HandshakeResponse +{ + // The version the connection uses: the older of the reader's and the plugin's newest. + std::uint32_t protocol_version = PROTOCOL_VERSION; + std::string error; // why the plugin refuses the reader, if it does; the connection then ends + + // The group that was asked for, with one file descriptor per stream travelling with the + // message, in stream order, for the stream's pool. Or every group the plugin publishes, + // when the request named none. + std::vector groups; +}; + +// Reader to plugin: done with a frame set. +struct ReleaseMessage +{ + std::uint64_t lease_id = 0; +}; + +// Reader to plugin, on a newest-order subscription: hold the newest frame set after +// `after_sequence` for me, now or as soon as there is one. +struct AcquireMessage +{ + std::uint64_t after_sequence = 0; // the last frame set the reader took; 0 for none +}; + +// What decode_message returns. A frame set travels as a FrameEvent. std::monostate is a +// message this build does not know and whose sender marked it as one that may be ignored. +using ProtocolMessage = + std::variant; + +/** + * @brief Bytes that are not a message, or a message this build cannot act on. + */ +class ProtocolError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +/** + * @brief Turns messages into bytes. It reuses one buffer, so after the first few messages + * encoding allocates nothing. Not thread-safe. + */ +class MessageEncoder +{ +public: + MessageEncoder(); + ~MessageEncoder(); + + MessageEncoder(const MessageEncoder&) = delete; + MessageEncoder& operator=(const MessageEncoder&) = delete; + + /** + * @return The message's bytes, valid until the next call on this encoder. + * @throws std::invalid_argument if the message names more streams, images or planes than a + * group can have. Nothing is cut short to make it fit. + * @throws std::length_error if the message is larger than MAX_MESSAGE_BYTES. + */ + std::span encode(const HandshakeRequest& message); + std::span encode(const HandshakeResponse& message); + std::span encode(const FrameEvent& message); + std::span encode(const ReleaseMessage& message); + std::span encode(const AcquireMessage& message); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +/** + * @brief Read one message. The bytes come from another process: their structure is verified + * and the counts that have a limit (streams, planes, images) are bounded here. Whether a group's description makes + * sense (sizes, planes, strides) is not checked here; problem_with() does that where the description is used. + * @throws ProtocolError if they are not a message, name more streams, planes or images than a + * group can have, or are an operation this build does not know. + */ +ProtocolMessage decode_message(std::span bytes); + +} // namespace core::camera diff --git a/src/core/camera_reader/AGENTS.md b/src/core/camera_reader/AGENTS.md new file mode 100644 index 0000000000..6b48a89575 --- /dev/null +++ b/src/core/camera_reader/AGENTS.md @@ -0,0 +1,20 @@ + + +# Agent notes — `camera_reader` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +- **The public header shows `camera_core` types only.** The socket, the pools and the protocol's messages stay in the `.cpp`; an application that links the reader does not see them. +- **No copies and no conversions here.** A view is the plugin's buffer. Pixel conversion, if a consumer wants it, is a separate step on top. +- **A frame set outlives its reader safely.** Both share one state object; keep it that way rather than making `FrameSet` point back at the reader. +- **Releasing must never throw or block.** It runs in destructors, possibly on another thread; a plugin that is gone takes its frames back by itself. +- **The plugin is trusted for content, not for bounds.** Check a frame event against the group before handing out views, and check the handshake answer in `open` before mapping anything: the protocol version, that it is the group that was asked for, `problem_with` for the description, one pool per stream. +- **A newest-order reader asks; it does not drain.** `next` sends the request and takes the answer. Do not go back to reading whatever is queued and keeping the last: what is queued was held when it was published. +- **A full window is an error, not a wait.** `next` by a reader that holds its window's worth throws `CameraWindowExhausted`; it does not time out as if no frame had come. +- **An empty location looks; a named one does not.** With no endpoint and no instance the reader asks every endpoint `default_endpoint_paths()` lists. With either, it talks to that one plugin alone: the full handshake timeout, and an error if the plugin is not there. Do not fall back from one to the other. +- **Looking stays plain.** The endpoints are asked one after another, each with a short fixed timeout. A refused connection is passed over silently. An endpoint that fails to say what it publishes (no answer in time, a refusal, a closed connection, any other failure) is passed over too, named with its reason in `open`'s error and left out by `list`; do not narrow that to the timeout. A group on two endpoints is an error, never a choice. No cache, no index file, no retry, no thread, and no option for any of it. +- **A test waits for what the plugin's thread does; it does not assume it.** `next` returning says nothing about whether the plugin has taken the reader's request, so wait for the effect (`Served::wait_until_sent`) before counting what was sent. See "A test does not assume which thread runs first" in [`camera_endpoint`](../camera_endpoint/AGENTS.md). +- **Tests that look through the user's plugins get a directory of their own.** `EndpointDirectory` (`tests/cpp/core/camera_reader/endpoint_directory.hpp`) points `XDG_RUNTIME_DIR` at a temporary directory and puts it back; a test that uses the real one finds whatever camera plugin the developer has running. diff --git a/src/core/camera_reader/CMakeLists.txt b/src/core/camera_reader/CMakeLists.txt new file mode 100644 index 0000000000..8ce91de360 --- /dev/null +++ b/src/core/camera_reader/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(cpp) diff --git a/src/core/camera_reader/README.md b/src/core/camera_reader/README.md new file mode 100644 index 0000000000..b8dd0334e6 --- /dev/null +++ b/src/core/camera_reader/README.md @@ -0,0 +1,121 @@ + + +# Camera interface: the reader + +What an application links to get camera frames. It finds the camera plugin that publishes a +camera group, talks to that plugin's [endpoint](../camera_endpoint) and hands out frame sets +whose pixels are the plugin's own buffers, mapped read-only; nothing is copied. + +```cpp +#include + +CameraReader reader = CameraReader::open("head"); +while (auto frames = reader.next(std::chrono::seconds(1))) +{ + const ImageView left = frames->image(0); + process(left.plane(0), left.stream->width, left.stream->height, left.stream->planes[0].stride_bytes); +} +``` + +- **A `FrameSet` is a hold on the frames.** While it exists the plugin does not reuse those + buffers, so the pixels stay as they are, across a restart of the camera and after the + reader object is gone. It gives them back when it is destroyed, or earlier with + `release()`; give it back only when nothing reads its views any more. +- **Window and order** are chosen in `open` ([terms](../camera_core/README.md)). The window is + how many frame sets the reader may hold at once. The default, a window of one in newest + order, gives the newest frame set and says in `skipped()` how many were passed over. A + recorder asks for a larger window and gets every frame set in order. +- **Newest means newest when you ask.** In newest order `next` asks the plugin for a frame + set, and the plugin holds the newest one it has at that moment, or the next one published. + Nothing is held for a reader between calls, so a reader that was busy for a second gets + what the camera saw last, not what it saw a second ago. See "A frame set that waited". +- **`next` returns nothing on timeout** and throws `CameraDisconnected` when the plugin is + gone. Called by a reader that already holds its window's worth, it throws + `CameraWindowExhausted` at once, because no frame set could arrive. +- **Opening checks the plugin's answer.** It fails with a `CameraReaderError` that says why: + no plugin publishes the group or two do, nothing answers on the endpoint that was named, + the group does not exist, the subscription does not fit the pool, the plugin speaks a + protocol version this reader does not, the answer is not the group that was asked for, the + group uses a kind of memory or a format this build does not know, or its description would + let a view run off the end of a pool. +- **`list` reports every group**, also one this build cannot read, with the reason in + `unsupported`, and with the `endpoint` of the plugin that publishes it. +- **Which plugin.** The reader finds it; see "How a reader finds its camera". A + `PluginLocation` names one plugin instead, by its instance or by its socket path. +- **The reader does not start a plugin.** Like every Isaac Capture plugin, a camera plugin is + started by the application through the plugin manager, or by hand or a service. A reader + that finds none says where it looked and how to start one. +- **One reader reads one group.** For two groups, open two readers; `descriptor()` lets an + application wait on several at once. +- A `FrameSet` may be given back from another thread than the one that calls `next`, and + after its reader is gone. The connection stays open until the last frame set of a reader + that is gone has been given back, because the hold is the connection's. + +## How a reader finds its camera + +There is one camera plugin per vendor or SDK. Each is a process of its own with an endpoint +of its own, a socket file in the user's endpoint directory +([`camera_ipc`](../camera_ipc), "Where a plugin listens"). An application knows the name of +a camera group, not which plugin publishes it. + +- **An empty `PluginLocation` means: look.** `open` asks each endpoint in that directory what + it publishes, one after another, and opens the group on the one that publishes it. `list` + returns the groups of every one that says what it publishes, in the order of their socket + paths. No plugin running is an empty list for `list`, and for `open` an error. +- **A plugin that does not serve yet is passed over at once.** An endpoint accepts readers + only from its `start()`, so a plugin that is still opening its cameras refuses the + connection, and so does a socket file left by a plugin that was killed. Neither is an + error and neither costs a wait. +- **A plugin that takes the connection and does not say what it publishes is passed over.** + One that stays silent is given half a second; one that refuses the reader, closes the + connection or answers something the reader cannot use costs no wait. An endpoint that + cannot be connected to for another reason than a refusal is passed over the same way. That + includes a plugin that stopped accepting long ago and whose queue of connections is full: + the connect is refused at once, so it costs no wait either. + `list` leaves such a plugin out. If `open` does not find the group, the error names the + endpoint and the reason. +- **Nobody publishes the group:** the error names the group and the directory that was + searched, lists what is published there and by which endpoint, and says how to start a + plugin. +- **Two plugins publish the group:** the error names both endpoints. The reader does not + pick one; a `PluginLocation` says which. +- **A `PluginLocation` with an instance or an endpoint is that one plugin.** Nothing else is + asked. The plugin gets the full two seconds to answer, and one that is not there is an + error, for `list` too. +- **Nothing is remembered.** Every `open` and `list` looks again, so a plugin that was + started or stopped a moment ago is seen as it is now. + +## A frame set that waited + +A newest-order reader that asked and stopped waiting (its `next` timed out) is still owed a +frame set, and is sent the next one published, whether or not it is in `next` at that +moment. By the time it comes back, that frame set may have been overtaken. So a frame set +that is already waiting when `next` is called is not handed out: `next` gives it back and +asks again. The plugin answers at once with the newest one, which is the same frame set +when nothing newer exists, so this costs a round trip and never a frame. A reader that +waits in `next` until its frame set comes never pays it. + +Not covered: a process that is stopped while it waits inside `next` is handed, when it +continues, the frame set that arrived meanwhile. A reader that needs to know how old its +frame is has the times of every step in `timeline()`. + +## Tests + +`tests/cpp/core/camera_reader`, run with `ctest -L camera_reader`. + +- **Unit** (`ctest -L unit`): the reader against a real endpoint in the same process, and + against a stand-in plugin for what a real one does not do on demand. The cases about + finding a plugin point `XDG_RUNTIME_DIR` at a temporary directory and put there what the + case is about: plugins with different groups or the same one, a plugin that has not + started, a socket file nobody is behind, a plugin that does not answer, a plugin that + refuses the reader or goes away while it is asked. +- **Integration** (`ctest -L integration`): the reader in the test process and + `camera_test_peer` as the other process, over the real socket. A reader gets whole frame + sets from a plugin in another process; a reader that was away for sixty frame periods is + handed a frame set published while it was away; a reader process that is killed while it + holds frame sets gives them back; a reader is told when the plugin's process is killed and + can still read what it holds; a reader finds two groups on two plugin processes by name, + and still finds the one after the other was killed. diff --git a/src/core/camera_reader/cpp/CMakeLists.txt b/src/core/camera_reader/cpp/CMakeLists.txt new file mode 100644 index 0000000000..6fcc2729ec --- /dev/null +++ b/src/core/camera_reader/cpp/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# What an application links to read cameras from camera plugins. +add_library(camera_reader STATIC + camera_reader.cpp +) + +target_include_directories(camera_reader INTERFACE + $ +) + +target_link_libraries(camera_reader + PUBLIC + camera::camera_core + PRIVATE + camera::camera_ipc + camera::camera_protocol +) + +add_library(camera::camera_reader ALIAS camera_reader) diff --git a/src/core/camera_reader/cpp/camera_reader.cpp b/src/core/camera_reader/cpp/camera_reader.cpp new file mode 100644 index 0000000000..8408d1f011 --- /dev/null +++ b/src/core/camera_reader/cpp/camera_reader.cpp @@ -0,0 +1,605 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_reader/camera_reader.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// How long the plugin gets to answer the handshake. +constexpr std::chrono::milliseconds ANSWER_TIMEOUT(2000); + +// How long each endpoint gets to say what it publishes while a reader looks for its plugin. +// Short, because the endpoints are asked one after another. +constexpr std::chrono::milliseconds PROBE_TIMEOUT(500); + +// How long the plugin gets to answer when a frame set that waited is exchanged for the newest. +// The answer is a frame set the plugin already has, so it comes in well under a millisecond. +constexpr std::chrono::milliseconds EXCHANGE_TIMEOUT(20); + +struct Received +{ + ProtocolMessage message; + std::vector descriptors; +}; + +// Waits for one message. Nothing on timeout. +std::optional receive(MessageSocket& socket, std::span buffer, std::chrono::milliseconds timeout) +{ + ReceivedMessage received = socket.receive(buffer, timeout); + if (received.status == ReceiveStatus::TIMEOUT) + { + return std::nullopt; + } + if (received.status == ReceiveStatus::CLOSED) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + try + { + return Received{ decode_message(buffer.first(received.bytes)), std::move(received.descriptors) }; + } + catch (const ProtocolError& error) + { + throw CameraReaderError(std::string("the camera plugin sent something this reader cannot act on: ") + + error.what()); + } +} + +// Whether a location names a plugin. One that does not leaves the reader to find it. +bool names_a_plugin(const PluginLocation& plugin) +{ + return !plugin.endpoint.empty() || !plugin.instance.empty(); +} + +// The socket path of the plugin a location names. +std::string endpoint_of(const PluginLocation& plugin) +{ + if (!plugin.endpoint.empty() && !plugin.instance.empty()) + { + throw CameraReaderError("a camera plugin is named by its endpoint or by its instance, not both"); + } + try + { + return plugin.endpoint.empty() ? default_endpoint_path(plugin.instance) : plugin.endpoint; + } + catch (const std::invalid_argument& error) + { + throw CameraReaderError(error.what()); + } +} + +// Connects to the plugin a reader was told to read from, or found. +MessageSocket connect_to(const std::string& endpoint) +{ + try + { + return MessageSocket::connect(endpoint); + } + catch (const std::system_error& error) + { + throw CameraReaderError("no camera plugin answers on '" + endpoint + "': " + error.code().message() + + ". Start the camera plugin for your camera through the plugin manager"); + } + catch (const std::invalid_argument& error) + { + throw CameraReaderError(error.what()); + } +} + +// What a handshake brought back: the plugin's answer and the descriptors that came with it. +struct Answer +{ + HandshakeResponse response; + std::vector descriptors; +}; + +// Sends the reader's first message and waits up to `timeout` for the plugin's answer, which is +// the first message a plugin sends. +Answer shake_hands(MessageSocket& socket, + std::span buffer, + const std::string& group, + const ReaderOptions& options, + std::chrono::milliseconds timeout) +{ + MessageEncoder encoder; + HandshakeRequest request; + request.reader_name = program_invocation_short_name; + request.group = group; + request.options = options; + if (socket.send(encoder.encode(request)) != SendStatus::SENT) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + const auto deadline = std::chrono::steady_clock::now() + timeout; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + std::optional received = receive(socket, buffer, left); + if (!received) + { + throw CameraReaderError("the camera plugin did not answer the reader's handshake"); + } + if (std::holds_alternative(received->message)) + { + continue; // something a newer plugin sends for information + } + auto* response = std::get_if(&received->message); + if (response == nullptr) + { + throw CameraReaderError("the camera plugin sent something else before it answered the reader's handshake"); + } + if (!response->error.empty()) + { + throw CameraReaderError(response->error); + } + if (response->protocol_version < OLDEST_PROTOCOL_VERSION || response->protocol_version > PROTOCOL_VERSION) + { + throw CameraReaderError("the camera plugin answered in protocol version " + + std::to_string(response->protocol_version) + " and this reader speaks versions " + + std::to_string(OLDEST_PROTOCOL_VERSION) + " to " + std::to_string(PROTOCOL_VERSION)); + } + return { std::move(*response), std::move(received->descriptors) }; + } +} + +// What the plugin on a connection publishes. +std::vector ask_what_is_published(MessageSocket& socket, + const std::string& endpoint, + std::chrono::milliseconds timeout) +{ + std::vector buffer(MAX_MESSAGE_BYTES); + std::vector listing; + for (GroupEntry& entry : shake_hands(socket, buffer, {}, {}, timeout).response.groups) + { + listing.push_back({ std::move(entry.group), std::move(entry.unsupported), endpoint }); + } + return listing; +} + +// Connects to an endpoint found in the user's endpoint directory. Nothing when the connection +// is refused, which is a plugin that is still starting or a socket file left by one that was +// killed, or when the file is gone since the directory was read. +std::optional connect_if_served(const std::string& endpoint) +{ + try + { + return MessageSocket::connect(endpoint); + } + catch (const std::system_error& error) + { + if (error.code() == std::errc::connection_refused || error.code() == std::errc::no_such_file_or_directory) + { + return std::nullopt; + } + throw; + } +} + +// What a reader finds when it looks through this user's camera plugins. +struct Found +{ + std::vector groups; // of every plugin that answered, in the order of the endpoints + std::vector unanswered; // endpoints that did not say what they publish, each with why +}; + +// Asks each camera endpoint of this user what it publishes, one after another. One that does +// not say, whatever the reason, does not keep a reader from the others. +Found look_through_endpoints() +{ + std::vector endpoints; + try + { + endpoints = default_endpoint_paths(); + } + catch (const std::system_error& error) + { + throw CameraReaderError(std::string("cannot look for camera plugins: ") + error.what()); + } + Found found; + for (const std::string& endpoint : endpoints) + { + try + { + std::optional socket = connect_if_served(endpoint); + if (!socket) + { + continue; + } + std::vector published = ask_what_is_published(*socket, endpoint, PROBE_TIMEOUT); + std::move(published.begin(), published.end(), std::back_inserter(found.groups)); + } + catch (const std::exception& error) + { + found.unanswered.push_back("'" + endpoint + "' (" + error.what() + ")"); + } + } + return found; +} + +std::string joined(const std::vector& items, const std::string& separator) +{ + std::string list; + for (const std::string& item : items) + { + list += (list.empty() ? "" : separator) + item; + } + return list; +} + +// The endpoint of the one camera plugin of this user that publishes a group. +std::string find_endpoint_of(const std::string& group) +{ + const Found found = look_through_endpoints(); + std::vector publishers; // the endpoints that publish the group + std::vector published; // what is published at all, and by which endpoint + for (const GroupListing& listing : found.groups) + { + if (listing.group.id == group) + { + publishers.push_back(listing.endpoint); + } + published.push_back("'" + listing.group.id + "' by '" + listing.endpoint + "'"); + } + if (publishers.size() == 1) + { + return publishers.front(); + } + if (publishers.size() > 1) + { + // Not the first of them: which camera a reader gets must not depend on a file name. + throw CameraReaderError("camera group '" + group + "' is published by more than one camera plugin, on '" + + joined(publishers, "' and '") + + "'. Name the plugin to read from by its instance or its endpoint"); + } + + std::string message = + "no camera plugin with an endpoint in '" + default_endpoint_directory() + "' publishes group '" + group + "'."; + if (published.empty() && found.unanswered.empty()) + { + message += " No camera plugin is running there."; + } + if (!published.empty()) + { + message += " Published there: " + joined(published, ", ") + "."; + } + if (!found.unanswered.empty()) + { + message += " Passed over: " + joined(found.unanswered, ", ") + "."; + } + throw CameraReaderError( + message + " Start the camera plugin for your camera through the plugin manager, for example camera_synthetic"); +} + +} // namespace + +namespace detail +{ + +// Shared by a reader and the frame sets it handed out, so a frame set can be given back after +// its reader is gone. +class ReaderState +{ +public: + explicit ReaderState(MessageSocket connected) : socket(std::move(connected)), buffer(MAX_MESSAGE_BYTES) + { + } + + // Called from a FrameSet, possibly on another thread than the one that reads. The send + // cannot find the plugin's queue full: a reader sends a release for a frame set it holds + // and a request for the next, and the plugin sends no further frame set before it has + // taken those, so no more than about two messages per frame set of the window are ever on + // their way. That is far below what the socket queues. + void release(std::uint64_t lease_id) noexcept + { + try + { + const std::lock_guard lock(m_send_mutex); + (void)socket.send(m_encoder.encode(ReleaseMessage{ lease_id })); + } + catch (const std::exception&) + { + // The plugin is gone or will drop this reader; either way it takes the frame set back. + } + } + + // Asks the plugin to hold the newest frame set after the last one this reader took. A + // request the plugin has no room for is not open, and the next call asks again. + void ask() + { + const std::lock_guard lock(m_send_mutex); + const SendStatus status = socket.send(m_encoder.encode(AcquireMessage{ last_taken })); + if (status == SendStatus::CLOSED) + { + throw CameraDisconnected("the camera plugin closed the connection"); + } + asked = status == SendStatus::SENT; + } + + // The next frame event from the socket; nothing on timeout. + std::optional take(std::chrono::milliseconds timeout) + { + const auto deadline = std::chrono::steady_clock::now() + timeout; + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + const std::optional received = receive(socket, buffer, left); + if (!received) + { + return std::nullopt; + } + if (const auto* event = std::get_if(&received->message)) + { + return *event; + } + if (!std::holds_alternative(received->message)) + { + throw CameraReaderError("the camera plugin sent a message only a reader sends"); + } + // Something a newer plugin sends for information; keep waiting. + } + } + + // The newest frame set, for a newest-order reader. + std::optional take_newest(std::chrono::milliseconds timeout) + { + auto deadline = std::chrono::steady_clock::now() + timeout; + if (asked) + { + // An earlier call asked and stopped waiting. A frame set that arrived for it since + // was held at that time, and whether it is still the newest only the plugin knows: + // give it back and ask again. The answer exists already, so it comes at once. + if (const std::optional waited = take(std::chrono::milliseconds(0))) + { + release(waited->lease_id); + asked = false; + deadline = std::max(deadline, std::chrono::steady_clock::now() + EXCHANGE_TIMEOUT); + } + } + if (!asked) + { + ask(); + } + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + const std::optional event = take(left); + if (event) + { + asked = false; + } + return event; // nothing: the request stays open, and its answer is taken by a later call + } + + // A frame set that names a slot or a size the pools do not have cannot be read safely. + void check(const FrameEvent& event) const + { + bool fits = event.image_count == group.streams.size(); + for (std::size_t i = 0; fits && i < event.image_count; ++i) + { + fits = event.images[i].slot < group.streams[i].slot_count && + event.images[i].bytes_used <= group.streams[i].slot_bytes; + } + if (!fits) + { + throw CameraReaderError("the camera plugin sent a frame set that does not match group '" + group.id + "'"); + } + } + + MessageSocket socket; + GroupDescription group; + ReaderOptions options; + std::vector pools; // one per stream + std::atomic held{ 0 }; // frame sets handed out and not yet given back + + // The reading thread only. + std::vector buffer; + bool asked = false; // the plugin owes this reader a frame set + std::uint64_t last_taken = 0; // the sequence of the last frame set handed out; 0 for none + +private: + std::mutex m_send_mutex; // guards the encoder and sending on the socket + MessageEncoder m_encoder; +}; + +} // namespace detail + +std::span ImageView::plane(std::uint32_t index) const +{ + if (stream == nullptr || index >= stream->plane_count) + { + throw std::out_of_range("the image has no plane " + std::to_string(index)); + } + const PlaneSize size = plane_size_of(stream->pixel_format, stream->width, stream->height, index); + const std::uint64_t offset = stream->planes[index].offset_bytes; + const std::uint64_t bytes = stream->planes[index].stride_bytes * (size.rows - 1) + size.row_bytes; + if (size.rows == 0 || offset + bytes > data.size()) + { + throw std::out_of_range("plane " + std::to_string(index) + " does not lie inside the image's slot"); + } + return data.subspan(offset, bytes); +} + +FrameSet::FrameSet(std::shared_ptr state, const FrameEvent& event) + : m_state(std::move(state)), m_event(event) +{ + ++m_state->held; +} + +FrameSet::FrameSet(FrameSet&& other) noexcept : m_state(std::move(other.m_state)), m_event(other.m_event) +{ +} + +FrameSet& FrameSet::operator=(FrameSet&& other) noexcept +{ + if (this != &other) + { + release(); + m_state = std::move(other.m_state); + m_event = other.m_event; + } + return *this; +} + +FrameSet::~FrameSet() +{ + release(); +} + +ImageView FrameSet::image(std::size_t stream) const +{ + if (!m_state) + { + throw std::logic_error("the frame set was given back"); + } + if (stream >= m_event.image_count) + { + throw std::out_of_range("the frame set has no image " + std::to_string(stream)); + } + const StreamDescription& description = m_state->group.streams[stream]; + const ImageInfo& info = m_event.images[stream]; + const std::span slot = m_state->pools[stream].slot(info.slot); + return { &description, info, description.is_encoded() ? slot.first(info.bytes_used) : slot }; +} + +void FrameSet::release() noexcept +{ + if (m_state) + { + m_state->release(m_event.lease_id); + --m_state->held; + m_state.reset(); + } +} + +std::vector CameraReader::list(const PluginLocation& plugin) +{ + if (!names_a_plugin(plugin)) + { + return look_through_endpoints().groups; + } + const std::string endpoint = endpoint_of(plugin); + MessageSocket socket = connect_to(endpoint); + return ask_what_is_published(socket, endpoint, ANSWER_TIMEOUT); +} + +CameraReader CameraReader::open(const std::string& group, const ReaderOptions& options, const PluginLocation& plugin) +{ + if (group.empty()) + { + throw CameraReaderError("a camera reader is opened for a group; list() says which there are"); + } + const std::string endpoint = names_a_plugin(plugin) ? endpoint_of(plugin) : find_endpoint_of(group); + auto state = std::make_shared(connect_to(endpoint)); + Answer answer = shake_hands(state->socket, state->buffer, group, options, ANSWER_TIMEOUT); + + // The answer comes from another process: it has to be the group that was asked for, in a + // form this build reads, described in a way that keeps every view inside its pool. + if (answer.response.groups.size() != 1) + { + throw CameraReaderError("the camera plugin answered the request for group '" + group + "' with " + + std::to_string(answer.response.groups.size()) + " groups"); + } + GroupEntry& entry = answer.response.groups.front(); + if (entry.group.id != group) + { + throw CameraReaderError("the camera plugin answered the request for group '" + group + "' with group '" + + entry.group.id + "'"); + } + if (!entry.unsupported.empty()) + { + throw CameraReaderError("camera group '" + group + "' cannot be read by this build: " + entry.unsupported); + } + if (const std::string problem = problem_with(entry.group); !problem.empty()) + { + throw CameraReaderError("the camera plugin described group '" + group + + "' in a way that cannot be read: " + problem); + } + if (answer.descriptors.size() != entry.group.streams.size()) + { + throw CameraReaderError("the camera plugin did not send group '" + group + "' with one pool per stream"); + } + state->group = std::move(entry.group); + state->options = { options.window, resolved_order(options) }; + try + { + for (std::size_t i = 0; i < state->group.streams.size(); ++i) + { + const StreamDescription& stream = state->group.streams[i]; + state->pools.emplace_back(std::move(answer.descriptors[i]), stream.slot_count, stream.slot_bytes); + } + } + catch (const std::exception& error) + { + throw CameraReaderError("cannot map the pools of '" + group + "': " + error.what()); + } + return CameraReader(std::move(state)); +} + +CameraReader::CameraReader(std::shared_ptr state) : m_state(std::move(state)) +{ +} + +CameraReader::CameraReader(CameraReader&& other) noexcept = default; +CameraReader& CameraReader::operator=(CameraReader&& other) noexcept = default; +CameraReader::~CameraReader() = default; + +const GroupDescription& CameraReader::group() const +{ + return m_state->group; +} + +std::optional CameraReader::next(std::chrono::milliseconds timeout) +{ + detail::ReaderState& state = *m_state; + if (state.held >= state.options.window) + { + throw CameraWindowExhausted("this camera reader holds " + std::to_string(state.held.load()) + + " frame set(s), which is its whole window: give one back before asking for " + + "the next, or open the reader with a larger window"); + } + const std::optional event = + state.options.order == DeliveryOrder::NEWEST ? state.take_newest(timeout) : state.take(timeout); + if (!event) + { + return std::nullopt; + } + try + { + state.check(*event); + } + catch (const CameraReaderError&) + { + state.release(event->lease_id); + throw; + } + state.last_taken = event->sequence; + return FrameSet(m_state, *event); +} + +int CameraReader::descriptor() const +{ + return m_state->socket.descriptor(); +} + +} // namespace core::camera diff --git a/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp b/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp new file mode 100644 index 0000000000..026a9afc5e --- /dev/null +++ b/src/core/camera_reader/cpp/inc/camera_reader/camera_reader.hpp @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +namespace detail +{ +class ReaderState; +} + +/** + * @brief A reader could not be opened or cannot go on: no plugin publishes the group or + * answers on the endpoint, the plugin refused the reader, or it sent something a reader cannot + * use. + */ +class CameraReaderError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +/** + * @brief The camera plugin closed the connection or is gone. + */ +class CameraDisconnected : public CameraReaderError +{ +public: + using CameraReaderError::CameraReaderError; +}; + +/** + * @brief next() was called by a reader that already holds as many frame sets as its window. + * Give one back first, or open the reader with a larger window. + */ +class CameraWindowExhausted : public std::logic_error +{ +public: + using std::logic_error::logic_error; +}; + +// One group a plugin publishes, as list() reports it. +struct GroupListing +{ + GroupDescription group; // the id only, when `unsupported` is set + + // Why this build cannot read the group, for one that uses a codec, a pixel format or a + // kind of memory added after it; empty for a group it can open. + std::string unsupported; + + std::string endpoint; // the socket path of the plugin that publishes the group +}; + +// Which camera plugin to read from: one of the two, or neither. Left empty, the reader looks +// through this user's camera plugins. +struct PluginLocation +{ + std::string endpoint; // the plugin's socket path + std::string instance; // the plugin's instance name, for its endpoint in the user's endpoint directory +}; + +/** + * @brief One image of a frame set. The memory is the plugin's pool, mapped read-only, and + * stays as it is for as long as the FrameSet the view came from is held. + */ +struct ImageView +{ + const StreamDescription* stream = nullptr; // what the bytes are; owned by the reader + ImageInfo info; + std::span data; // pixels: the whole slot; encoded video: the access unit + + /** + * @brief The bytes of one plane of a pixel image, from its first row to the end of its last. + * Rows are stream->planes[index].stride_bytes apart. + * @throws std::out_of_range for a plane the image does not have. + */ + std::span plane(std::uint32_t index) const; +}; + +/** + * @brief One frame set, held for this reader: the plugin does not reuse its slots until the + * object is destroyed or release() is called, also when the reader it came from is gone + * first. Hold at most as many as the reader's window. + */ +class FrameSet +{ +public: + FrameSet(FrameSet&& other) noexcept; + FrameSet& operator=(FrameSet&& other) noexcept; + FrameSet(const FrameSet&) = delete; + FrameSet& operator=(const FrameSet&) = delete; + ~FrameSet(); + + std::uint64_t sequence() const + { + return m_event.sequence; + } + + // Restarts of the camera since the plugin began publishing the group. + std::uint32_t generation() const + { + return m_event.generation; + } + + // Frame sets published since the previous one this reader was given, and not given to it. + std::uint32_t skipped() const + { + return m_event.skipped; + } + + const FrameTimeline& timeline() const + { + return m_event.timeline; + } + + std::size_t image_count() const + { + return m_event.image_count; + } + + /** + * @brief The image of one stream, in the order of the group's streams. + * @throws std::out_of_range for a stream the group does not have. + * @throws std::logic_error after release(). + */ + ImageView image(std::size_t stream) const; + + // Gives the frame set back before the object is destroyed. Its views are no longer valid: + // give it back only when nothing reads them any more. + void release() noexcept; + +private: + friend class CameraReader; + FrameSet(std::shared_ptr state, const FrameEvent& event); + + std::shared_ptr m_state; + FrameEvent m_event; +}; + +/** + * @brief Reads one group of cameras from the camera plugin that publishes it. + * + * @code + * CameraReader reader = CameraReader::open("head"); + * while (auto frames = reader.next(std::chrono::seconds(1))) + * { + * std::span left = frames->image(0).data; + * } + * @endcode + */ +class CameraReader +{ +public: + /** + * @brief What is published, including groups this build cannot read: by the plugin the + * location names or, with an empty location, by every camera plugin of this user that + * says what it publishes, in the order of their socket paths; one that does not answer or + * refuses this reader is left out. No plugin running gives an empty list. + * @throws CameraReaderError if the plugin that was named does not answer or refuses this + * reader, or the user's endpoint directory cannot be read. + */ + static std::vector list(const PluginLocation& plugin = {}); + + /** + * @brief Subscribe to one group. A reader for a second group is a second CameraReader. + * + * With an empty location the reader finds the plugin: it asks each camera plugin of this + * user what it publishes and reads the group from the one that publishes it. A plugin that + * is still starting is passed over at once, and one that does not say what it publishes + * after a short wait at most. + * + * @throws CameraReaderError if no plugin answers, none or more than one publishes the group, + * the plugin refuses the subscription, or what it answers is not the group that was asked + * for in a form this build can read; the message says why. + */ + static CameraReader open(const std::string& group, + const ReaderOptions& options = {}, + const PluginLocation& plugin = {}); + + CameraReader(CameraReader&& other) noexcept; + CameraReader& operator=(CameraReader&& other) noexcept; + CameraReader(const CameraReader&) = delete; + CameraReader& operator=(const CameraReader&) = delete; + ~CameraReader(); + + const GroupDescription& group() const; + + /** + * @brief Wait up to `timeout` for a frame set. + * + * With newest order the call asks the plugin, which holds the newest frame set there is at + * that moment, or the next one published: a reader that was busy gets what the camera saw + * last, not what it saw when the reader stopped. A frame set that arrived for an earlier + * call that had stopped waiting is given back and exchanged for the newest; that exchange + * usually takes well under a millisecond and may outlast a short timeout by up to 20 ms. + * With in order the plugin sends every frame set and this returns the next. + * + * @return The frame set, or nothing on timeout. + * @throws CameraWindowExhausted if the reader already holds its window's worth. + * @throws CameraDisconnected if the plugin is gone. + * @throws CameraReaderError if the plugin sent something a reader cannot use. + */ + std::optional next(std::chrono::milliseconds timeout); + + /** + * @brief Becomes readable when a frame set may be waiting; for waiting on several readers + * at once. A newest-order reader is sent a frame set only after next() asked for one, so + * call next() first, with a timeout of zero, and again when the descriptor is readable. + */ + int descriptor() const; + +private: + explicit CameraReader(std::shared_ptr state); + + std::shared_ptr m_state; +}; + +} // namespace core::camera diff --git a/src/core/schema/README.md b/src/core/schema/README.md index 7604db8970..d56299260a 100644 --- a/src/core/schema/README.md +++ b/src/core/schema/README.md @@ -25,6 +25,19 @@ call `check_schema_compat()`, so they agree on what "unreadable" means. Note tha comparison reads the `.bfbs` out of the build tree, so `ctest` on its own re-checks whatever was built last: **build before you run it**, or a breaking edit reads as green. +## A schema that is not recorded + +[`fbs/camera_protocol.fbs`](fbs/camera_protocol.fbs) is the conversation between a camera +plugin and the readers of its frames ([`camera_protocol`](../camera_protocol)). No recording +carries it. It is kept here, with a golden, because it is a contract of the same kind: the +two ends can come from different releases, and a change the comparison rejects would make +them misread each other. + +It is not a recorded schema in disguise. It has no `Record` root and does not use +`DeviceDataTimestamp`: its times are the host's monotonic clock, and its lease ids, slot +indices and attached descriptors mean something between two processes on one machine only. +A recorder that stores camera frames maps those times in its own schema. + ## Adding a field Append it to a table with a fresh `id`, and leave it optional. The test passes, and no diff --git a/src/core/schema/fbs/camera_protocol.fbs b/src/core/schema/fbs/camera_protocol.fbs new file mode 100644 index 0000000000..3eed8ca8a2 --- /dev/null +++ b/src/core/schema/fbs/camera_protocol.fbs @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// What a camera plugin and a reader of its frames say to each other, one message per socket +// message. See src/core/camera_protocol/README.md for the conversation. +// +// This is a protocol between two processes on one machine. Lease ids, slot indices and the +// file descriptors that travel with a message name resources of that machine and mean nothing +// anywhere else, so none of it is a network format or a recording format. +// +// Nothing here is recorded. The file lives with the other schemas so that the same golden +// check guards it: a plugin and a reader can come from different releases, and a change the +// check rejects would make them misread each other. + +namespace core; + +// Zero means "a value this build does not know" for the kinds a newer peer may extend. +enum CameraPixelFormat : uint32 { Unknown = 0, Bgra8 = 1, Rgba8 = 2, Nv12 = 3, Gray8 = 4 } +enum CameraCodec : uint32 { None = 0, H264 = 1, H265 = 2 } +enum CameraMemoryKind : uint32 { Unknown = 0, Host = 1 } +enum CameraDeliveryOrder : uint32 { Default = 0, Newest = 1, InOrder = 2 } + +// Bits of CameraFrameRef.flags. A bit this build does not know is passed on untouched. +enum CameraFrameFlag : uint32 (bit_flags) { + Keyframe = 0, // bit 0, value 1: an encoded stream's access unit decodes on its own + SdkWarning = 1 // bit 1, value 2: the camera SDK delivered the frame but flagged it as suspect +} + +// Where one plane of a pixel image sits inside a slot. A struct, and so never extended: an +// offset and the distance between rows are all a plane of rows has. +struct CameraPlaneLayout { + offset_bytes: uint32; + stride_bytes: uint32; +} + +// One sensor's images or one encoded video stream, and the pool its frames are in. +table CameraStream { + id: string (id: 0); + codec: CameraCodec (id: 1); + pixel_format: CameraPixelFormat (id: 2); + memory: CameraMemoryKind (id: 3); + width: uint32 (id: 4); + height: uint32 (id: 5); + + // One entry per plane of a pixel image, the same for every slot; none for an encoded stream. + planes: [CameraPlaneLayout] (id: 6); + + slot_bytes: uint64 (id: 7); + slot_count: uint32 (id: 8); + frame_rate: float64 (id: 9); +} + +// Streams that are captured together and published as one frame set. +table CameraGroup { + id: string (id: 0); + streams: [CameraStream] (id: 1); + + // Names one run of the group, from the plugin starting to publish it until that plugin + // ends. With CameraFrameSet.sequence it identifies a capture for good. + session_id: uint64 (id: 2); +} + +// Reader to plugin, the first message on a connection. With a group it asks for that group's +// frame sets; without one it only asks what the plugin publishes. +table CameraHandshakeRequest { + // The newest protocol version the reader speaks. + protocol_version: uint32 (id: 0); + reader_name: string (id: 1); + group_id: string (id: 2); + window: uint32 = 1 (id: 3); + order: CameraDeliveryOrder (id: 4); +} + +// Plugin to reader, the answer to the request and the first message the plugin sends. With +// `error` set the reader is refused and the connection ends. Otherwise `groups` is the group +// that was asked for, or every group the plugin publishes when the request named none. +// +// Attachments, for a group that was asked for: one file descriptor per stream, in stream +// order, for the stream's pool. That is the rule for CameraMemoryKind.Host; a kind of memory +// that needs something else says so when it is added. +table CameraHandshakeResponse { + // The version both sides speak on this connection: the reader's or the plugin's newest, + // whichever is older. + protocol_version: uint32 (id: 0); + error: string (id: 1); + groups: [CameraGroup] (id: 2); +} + +// One stream's part of a frame set: the slot its image is in and what the camera SDK said +// about it. +table CameraFrameRef { + slot_index: uint32 (id: 0); + flags: CameraFrameFlag (id: 1); + bytes_used: uint64 (id: 2); + device_frame_id: uint64 (id: 3); + device_time_ns: int64 = -1 (id: 4); +} + +// Plugin to reader: one frame set, held for the reader until it sends CameraRelease with the +// same lease. The times are nanoseconds on the host's monotonic clock, -1 where one is not +// known. +table CameraFrameSet { + // This reader's hold on the frame set. It means something on this connection only. + lease_id: uint64 (id: 0); + + // The frame set's place among the group's captures in its session: the same for every + // reader, so a reader that skips sees a gap. A camera restart does not reset it. + sequence: uint64 (id: 1); + + // Restarts of the camera within the session. + generation: uint32 (id: 2); + + // Frame sets published since the reader's previous one and not given to it. + skipped: uint32 (id: 3); + + capture_ns: int64 = -1 (id: 4); + sdk_ready_ns: int64 = -1 (id: 5); + published_ns: int64 = -1 (id: 6); + sent_ns: int64 = -1 (id: 7); + + // One per stream of the group, in stream order. + frames: [CameraFrameRef] (id: 8); +} + +// Reader to plugin: done with a frame set. +table CameraRelease { + lease_id: uint64 (id: 0); +} + +// Reader to plugin, on a newest-order subscription: hold the newest frame set for me, now if +// there is one after `after_sequence` and otherwise as soon as there is. One CameraFrameSet +// answers it. An in-order subscription is sent its frame sets without asking. +table CameraAcquire { + // The sequence of the last frame set the reader took; 0 for none. + after_sequence: uint64 (id: 0); +} + +// The numbers are part of the protocol: a member keeps its number, and a new one takes the next. +union CameraMessageBody { + CameraHandshakeRequest = 1, + CameraHandshakeResponse = 2, + CameraFrameSet = 3, + CameraRelease = 4, + CameraAcquire = 5 +} + +// The union's type field takes id 0, so the body is id 1. +table CameraMessage { + body: CameraMessageBody (id: 1); + + // Set by the sender of a message that only informs. A receiver that does not know the body's + // type ignores such a message; one it does not know and that is not marked this way is an + // operation it cannot carry out: a plugin drops the reader, and a reader's call fails. + skippable: bool (id: 2); +} + +root_type CameraMessage; diff --git a/src/core/schema/golden/camera_protocol.bfbs b/src/core/schema/golden/camera_protocol.bfbs new file mode 100644 index 0000000000..a41e787b47 Binary files /dev/null and b/src/core/schema/golden/camera_protocol.bfbs differ diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 541201d114..8b13a6585c 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -18,6 +18,10 @@ add_subdirectory(camera_core) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) + add_subdirectory(camera_protocol) + add_subdirectory(camera_endpoint) + add_subdirectory(camera_peer) + add_subdirectory(camera_reader) endif() if(BUILD_OVXR_CAMERA_QUADS) diff --git a/tests/cpp/core/camera_endpoint/CMakeLists.txt b/tests/cpp/core/camera_endpoint/CMakeLists.txt new file mode 100644 index 0000000000..80ce917a68 --- /dev/null +++ b/tests/cpp/core/camera_endpoint/CMakeLists.txt @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(camera_endpoint_tests + test_endpoint.cpp +) + +# The tests play the reader's part with the socket and the codec themselves. The schema is +# linked to send what the codec never would: a message from a newer reader. +target_link_libraries(camera_endpoint_tests PRIVATE + camera::camera_endpoint + camera::camera_ipc + camera::camera_protocol + isaacteleop_schema + Catch2::Catch2WithMain + Threads::Threads +) + +message(STATUS "camera_endpoint_tests target enabled with Catch2") +catch_discover_tests(camera_endpoint_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_endpoint/test_endpoint.cpp b/tests/cpp/core/camera_endpoint/test_endpoint.cpp new file mode 100644 index 0000000000..6e99e07998 --- /dev/null +++ b/tests/cpp/core/camera_endpoint/test_endpoint.cpp @@ -0,0 +1,756 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; +constexpr std::uint32_t SLOTS = 8; + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_endpoint_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +GroupDescription gray_group(const std::string& id, std::size_t streams) +{ + GroupDescription group; + group.id = id; + for (std::size_t i = 0; i < streams; ++i) + { + StreamDescription stream; + stream.id = id + "/" + std::to_string(i); + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = WIDTH; + stream.slot_bytes = WIDTH * HEIGHT; + stream.slot_count = SLOTS; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// A plugin in miniature: a stereo group and a single camera, in host memory, behind an endpoint. +struct Served +{ + TemporaryDirectory directory; + Publisher publisher; + GroupId head = publisher.add_group(gray_group("head", 2)); + GroupId wrist = publisher.add_group(gray_group("wrist", 1)); + HostPool left{ SLOTS, WIDTH* HEIGHT, "test_left" }; + HostPool right{ SLOTS, WIDTH* HEIGHT, "test_right" }; + HostPool hand{ SLOTS, WIDTH* HEIGHT, "test_wrist" }; + Endpoint endpoint{ publisher, directory.socket_path() }; + + Served() + { + endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); + endpoint.set_group_memory(wrist, { hand.descriptor() }); + endpoint.start(); + } + + // Publishes one stereo frame set whose every pixel is `value`. + std::uint64_t publish(std::uint8_t value) + { + std::vector images(2); + HostPool* pools[] = { &left, &right }; + for (std::size_t i = 0; i < 2; ++i) + { + const auto slot = publisher.acquire_slot(head, i); + REQUIRE(slot.has_value()); + std::fill(pools[i]->slot(*slot).begin(), pools[i]->slot(*slot).end(), std::byte{ value }); + images[i].slot = *slot; + } + return publisher.publish(head, images, {}); + } +}; + +// The reader's part, played with the socket and the codec directly. +struct Reader +{ + struct Received + { + ReceiveStatus status = ReceiveStatus::TIMEOUT; + ProtocolMessage message; + std::vector descriptors; + }; + + explicit Reader(const std::string& endpoint_path) : socket(MessageSocket::connect(endpoint_path)) + { + } + + explicit Reader(const Served& served) : Reader(served.directory.socket_path()) + { + } + + template + void send(const Message& message) + { + REQUIRE(socket.send(encoder.encode(message)) == SendStatus::SENT); + } + + Received receive(std::chrono::milliseconds timeout = 2s) + { + Received result; + ReceivedMessage received = socket.receive(buffer, timeout); + result.status = received.status; + if (received.status == ReceiveStatus::MESSAGE) + { + result.message = decode_message({ buffer.data(), received.bytes }); + result.descriptors = std::move(received.descriptors); + } + return result; + } + + template + Message expect(std::chrono::milliseconds timeout = 2s) + { + Received received = receive(timeout); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(std::holds_alternative(received.message)); + return std::get(std::move(received.message)); + } + + // The reader's first message and the plugin's answer, with the descriptors that came with it. + // Without a group it asks what is published. + std::pair> handshake(const std::string& group = {}, + ReaderOptions options = {}) + { + send(HandshakeRequest{ .reader_name = "test reader", .group = group, .options = options }); + Received received = receive(); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + REQUIRE(std::holds_alternative(received.message)); + return { std::get(std::move(received.message)), std::move(received.descriptors) }; + } + + // Asks for the newest frame set after `after_sequence`, as a newest-order reader does. + void ask(std::uint64_t after_sequence = 0) + { + send(AcquireMessage{ after_sequence }); + } + + // A message of a type no build has yet, as a newer reader might send. + void send_unknown(bool skippable) + { + flatbuffers::FlatBufferBuilder builder; + const auto body = core::CreateCameraRelease(builder, 1); + builder.Finish(core::CreateCameraMessage( + builder, static_cast(core::CameraMessageBody_MAX + 1), body.Union(), skippable)); + REQUIRE(socket.send({ reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }) == + SendStatus::SENT); + } + + MessageSocket socket; + MessageEncoder encoder; + std::vector buffer = std::vector(MAX_MESSAGE_BYTES); +}; + +// What a connect to the path fails with; no error if it is accepted. +std::error_code connect_error(const std::string& endpoint_path) +{ + try + { + (void)MessageSocket::connect(endpoint_path); + return {}; + } + catch (const std::system_error& error) + { + return error.code(); + } +} + +bool wait_until(const std::function& condition, std::chrono::milliseconds timeout = 2s) +{ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (!condition()) + { + if (std::chrono::steady_clock::now() > deadline) + { + return false; + } + std::this_thread::sleep_for(1ms); + } + return true; +} + +} // namespace + +TEST_CASE("a reader is told what is published", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + + auto [response, descriptors] = reader.handshake(); + REQUIRE(response.error.empty()); + REQUIRE(response.protocol_version == PROTOCOL_VERSION); + REQUIRE(descriptors.empty()); + REQUIRE(response.groups.size() == 2); + REQUIRE(response.groups[0].group.id == "head"); + REQUIRE(response.groups[0].unsupported.empty()); + REQUIRE(response.groups[0].group.session_id != 0); + REQUIRE(response.groups[0].group.streams.size() == 2); + REQUIRE(response.groups[0].group.streams[1].id == "head/1"); + REQUIRE(response.groups[0].group.streams[1].width == WIDTH); + REQUIRE(problem_with(response.groups[0].group).empty()); + REQUIRE(response.groups[1].group.id == "wrist"); + REQUIRE(response.groups[1].group.session_id != response.groups[0].group.session_id); + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 1; })); +} + +TEST_CASE("a connection uses the newest protocol version both sides speak", "[unit][camera_endpoint]") +{ + Served served; + + // A reader from a later release: the plugin answers in its own version, and it is for the + // reader to say whether it still speaks that. + Reader newer(served); + newer.send(HandshakeRequest{ .protocol_version = PROTOCOL_VERSION + 3, .reader_name = "newer", .group = "head" }); + Reader::Received answer = newer.receive(); + REQUIRE(answer.status == ReceiveStatus::MESSAGE); + const HandshakeResponse& accepted = std::get(answer.message); + REQUIRE(accepted.error.empty()); + REQUIRE(accepted.protocol_version == PROTOCOL_VERSION); + REQUIRE(accepted.groups.size() == 1); + REQUIRE(answer.descriptors.size() == 2); + + // A reader from before the oldest version this plugin still speaks is refused. + Reader older(served); + older.send( + HandshakeRequest{ .protocol_version = OLDEST_PROTOCOL_VERSION - 1, .reader_name = "older", .group = "head" }); + const HandshakeResponse refused = older.expect(); + REQUIRE_THAT(refused.error, ContainsSubstring("protocol versions")); + REQUIRE(refused.groups.empty()); + REQUIRE(older.receive().status == ReceiveStatus::CLOSED); +} + +TEST_CASE("a reader that asks for a group maps its pools and is sent the frame sets it asks for", + "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + + auto [response, descriptors] = reader.handshake("head"); + REQUIRE(response.error.empty()); + REQUIRE(response.groups.size() == 1); + REQUIRE(response.groups[0].group.id == "head"); + REQUIRE(descriptors.size() == 2); + const HostPoolView left(std::move(descriptors[0]), SLOTS, WIDTH * HEIGHT); + const HostPoolView right(std::move(descriptors[1]), SLOTS, WIDTH * HEIGHT); + + // Nothing is sent to a newest-order reader that has not asked. + served.publish(0x11); + REQUIRE(reader.receive(20ms).status == ReceiveStatus::TIMEOUT); + + reader.ask(); + const FrameEvent first = reader.expect(); + REQUIRE(first.sequence == 1); + REQUIRE(first.image_count == 2); + REQUIRE(first.skipped == 0); + REQUIRE(first.timeline.published_ns != TIME_UNKNOWN); + REQUIRE(first.timeline.sent_ns >= first.timeline.published_ns); + REQUIRE(left.slot(first.images[0].slot).front() == std::byte{ 0x11 }); + REQUIRE(right.slot(first.images[1].slot).back() == std::byte{ 0x11 }); + + // A window of one holds one frame set. A request made while it is held is answered when + // it is given back, with the newest frame set there is then. + reader.ask(first.sequence); + served.publish(0x22); + served.publish(0x33); + REQUIRE(reader.receive(20ms).status == ReceiveStatus::TIMEOUT); + REQUIRE(left.slot(first.images[0].slot).front() == std::byte{ 0x11 }); // still held, so not rewritten + + reader.send(ReleaseMessage{ first.lease_id }); + const FrameEvent newest = reader.expect(); + REQUIRE(newest.sequence == 3); + REQUIRE(newest.skipped == 1); + REQUIRE(left.slot(newest.images[0].slot).front() == std::byte{ 0x33 }); +} + +TEST_CASE("a reader that paused is handed what the camera saw last", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + auto [response, descriptors] = reader.handshake("head"); + REQUIRE(response.error.empty()); + const HostPoolView left(std::move(descriptors[0]), SLOTS, WIDTH * HEIGHT); + + reader.ask(); + served.publish(1); + const FrameEvent before = reader.expect(); + reader.send(ReleaseMessage{ before.lease_id }); + + // The reader does something else, holding nothing and asking for nothing, while the camera + // runs through its whole pool several times. No frame set is put aside for the reader, so + // none is waiting for it and the camera never runs short of slots. + for (std::uint8_t value = 2; value <= 3 * SLOTS; ++value) + { + served.publish(value); + std::this_thread::sleep_for(1ms); + } + REQUIRE(reader.receive(20ms).status == ReceiveStatus::TIMEOUT); + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + + // Back again, it gets the last frame set published, held from this moment on. + const auto asked = std::chrono::steady_clock::now().time_since_epoch(); + reader.ask(before.sequence); + const FrameEvent after = reader.expect(); + REQUIRE(after.sequence == 3 * SLOTS); + REQUIRE(after.skipped == 3 * SLOTS - 2); + REQUIRE(left.slot(after.images[0].slot).front() == std::byte{ 3 * SLOTS }); + REQUIRE(after.timeline.sent_ns >= std::chrono::duration_cast(asked).count()); +} + +TEST_CASE("the answer to a handshake comes before any frame set", "[unit][camera_endpoint]") +{ + Served served; + + // A camera that publishes without pause while readers come and go. A jthread, so that it is + // stopped and joined however the test ends. + std::jthread capture( + [&](std::stop_token stop) + { + std::vector images(2); + while (!stop.stop_requested()) + { + const auto left = served.publisher.acquire_slot(served.head, 0); + const auto right = served.publisher.acquire_slot(served.head, 1); + if (left && right) + { + images[0].slot = *left; + images[1].slot = *right; + served.publisher.publish(served.head, images, {}); + } + else + { + if (left) + { + served.publisher.abandon_slot(served.head, 0, *left); + } + if (right) + { + served.publisher.abandon_slot(served.head, 1, *right); + } + std::this_thread::yield(); + } + } + }); + + // An in-order reader is sent frame sets without asking, so one published while its + // handshake is being answered is the one that could arrive first. handshake() requires the + // first message to be the answer. + for (int i = 0; i < 200; ++i) + { + Reader reader(served); + auto [response, descriptors] = reader.handshake("head", { .window = 2, .order = DeliveryOrder::IN_ORDER }); + REQUIRE(response.error.empty()); + REQUIRE(descriptors.size() == 2); + const FrameEvent first = reader.expect(); + REQUIRE(first.image_count == 2); + } +} + +TEST_CASE("a reader that cannot be served is told why and its connection ends", "[unit][camera_endpoint]") +{ + Served served; + { + Reader reader(served); + auto [unknown, descriptors] = reader.handshake("chest"); + REQUIRE_THAT(unknown.error, ContainsSubstring("no group 'chest'")); + REQUIRE(unknown.groups.empty()); + REQUIRE(descriptors.empty()); + REQUIRE(reader.receive().status == ReceiveStatus::CLOSED); + } + { + Reader reader(served); + auto [too_large, descriptors] = reader.handshake("head", { .window = SLOTS }); + REQUIRE_THAT(too_large.error, ContainsSubstring("does not fit the pool")); + REQUIRE(descriptors.empty()); + REQUIRE(reader.receive().status == ReceiveStatus::CLOSED); + } + { + // A delivery order added after this build: refused with the reason, not left unanswered. + Reader reader(served); + auto [newer_order, descriptors] = reader.handshake("head", { .order = static_cast(7) }); + REQUIRE_THAT(newer_order.error, ContainsSubstring("delivery order")); + REQUIRE(descriptors.empty()); + REQUIRE(reader.receive().status == ReceiveStatus::CLOSED); + } + // A refusal leaves nothing behind: the pool has all its room for a reader that fits. + Reader reader(served); + auto [accepted, descriptors] = reader.handshake("head", { .window = 3, .order = DeliveryOrder::IN_ORDER }); + REQUIRE(accepted.error.empty()); + REQUIRE(descriptors.size() == 2); +} + +TEST_CASE("a subscription whose reader left before the answer leaves nothing behind", "[unit][camera_endpoint]") +{ + Served served; + for (int i = 0; i < 20; ++i) + { + // Asks for a place in the pool and is gone before it could be told it has one. + Reader reader(served); + reader.send(HandshakeRequest{ .group = "head", .options = { .window = 2, .order = DeliveryOrder::IN_ORDER } }); + } + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 0; })); + + // Every one of those reservations is gone: a reader that needs all the room there is fits. + Reader reader(served); + auto [accepted, descriptors] = reader.handshake("head", { .window = 3, .order = DeliveryOrder::IN_ORDER }); + REQUIRE(accepted.error.empty()); + served.publish(1); + REQUIRE(reader.expect().sequence == 1); +} + +TEST_CASE("a reader that disappears returns what it held", "[unit][camera_endpoint]") +{ + Served served; + { + Reader reader(served); + REQUIRE(reader.handshake("head", { .window = 2, .order = DeliveryOrder::IN_ORDER }).first.error.empty()); + served.publish(1); + served.publish(2); + REQUIRE(reader.expect().sequence == 1); + REQUIRE(reader.expect().sequence == 2); + REQUIRE(served.endpoint.reader_count() == 1); + // Gone without releasing either frame set. + } + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 0; })); + + // With the held slots back, capture runs through the whole pool again and again. + for (std::uint8_t i = 0; i < 3 * SLOTS; ++i) + { + served.publish(i); + } + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + + // And the room in the pool is there for the next reader. + Reader next(served); + REQUIRE(next.handshake("head", { .window = 2, .order = DeliveryOrder::IN_ORDER }).first.error.empty()); +} + +TEST_CASE("a frame set held across a camera restart stays as it was", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + auto [response, descriptors] = reader.handshake("head"); + REQUIRE(response.error.empty()); + const HostPoolView left(std::move(descriptors[0]), SLOTS, WIDTH * HEIGHT); + + reader.ask(); + served.publish(0x11); + const FrameEvent held = reader.expect(); + REQUIRE(held.generation == 0); + + // The camera restarts and runs through its pool several times. The slot the reader holds + // is not given to the camera again, so what the reader sees does not change. + served.publisher.restart_group(served.head); + for (std::uint8_t i = 0; i < 3 * SLOTS; ++i) + { + served.publish(0x80 + i); + } + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + const std::span pixels = left.slot(held.images[0].slot); + REQUIRE(std::all_of(pixels.begin(), pixels.end(), [](std::byte pixel) { return pixel == std::byte{ 0x11 }; })); + + // What it is sent next is from after the restart, and says so. + reader.send(ReleaseMessage{ held.lease_id }); + reader.ask(held.sequence); + const FrameEvent next = reader.expect(); + REQUIRE(next.generation == 1); + REQUIRE(next.sequence == 1 + 3 * SLOTS); +} + +TEST_CASE("a reader that breaks the protocol is dropped and the others are not", "[unit][camera_endpoint]") +{ + Served served; + Reader good(served); + REQUIRE(good.handshake("head").first.error.empty()); + + Reader garbage(served); + const std::vector noise(32, std::byte{ 0x5a }); + REQUIRE(garbage.socket.send(noise) == SendStatus::SENT); + REQUIRE(garbage.receive().status == ReceiveStatus::CLOSED); + + // A message only a plugin sends. + Reader impostor(served); + impostor.send(HandshakeResponse{}); + REQUIRE(impostor.receive().status == ReceiveStatus::CLOSED); + + // A connection is for one thing, said once. + Reader twice(served); + REQUIRE(twice.handshake().first.error.empty()); + twice.send(HandshakeRequest{ .group = "head" }); + REQUIRE(twice.receive().status == ReceiveStatus::CLOSED); + + // Asking for the newest frame set is for newest-order readers that have a subscription. + Reader ordered(served); + REQUIRE(ordered.handshake("head", { .window = 2, .order = DeliveryOrder::IN_ORDER }).first.error.empty()); + ordered.ask(); + REQUIRE(ordered.receive().status == ReceiveStatus::CLOSED); + Reader lister(served); + REQUIRE(lister.handshake().first.error.empty()); + lister.ask(); + REQUIRE(lister.receive().status == ReceiveStatus::CLOSED); + + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 1; })); + good.ask(); + served.publish(7); + REQUIRE(good.expect().sequence == 1); +} + +TEST_CASE("a newer reader's message is ignored only when it may be", "[unit][camera_endpoint]") +{ + Served served; + + // Something that only informs: the plugin carries on as if it had not been sent. + Reader informing(served); + REQUIRE(informing.handshake("head").first.error.empty()); + informing.send_unknown(true); + informing.ask(); + served.publish(1); + REQUIRE(informing.expect().sequence == 1); + + // An operation the reader counts on and this plugin cannot carry out: the connection ends, + // which the reader notices, instead of the reader waiting for something that never comes. + Reader depending(served); + REQUIRE(depending.handshake("head").first.error.empty()); + depending.send_unknown(false); + REQUIRE(depending.receive().status == ReceiveStatus::CLOSED); + REQUIRE(wait_until([&] { return served.endpoint.reader_count() == 1; })); +} + +TEST_CASE("readers do not slow each other", "[unit][camera_endpoint]") +{ + Served served; + Reader stalled(served); + REQUIRE(stalled.handshake("head").first.error.empty()); + Reader live(served); + REQUIRE(live.handshake("head").first.error.empty()); + stalled.ask(); + live.ask(); + + // One reader never releases; the other still gets every frame set. + for (std::uint64_t sequence = 1; sequence <= 5; ++sequence) + { + REQUIRE(served.publish(static_cast(sequence)) == sequence); + const FrameEvent event = live.expect(); + REQUIRE(event.sequence == sequence); + REQUIRE(event.skipped == 0); + // Whether the release and the request are seen before or after the next publish, that + // frame set is next. + live.send(ReleaseMessage{ event.lease_id }); + live.ask(event.sequence); + } + // The stalled reader asked once and was sent one frame set: the newest there was when its + // request was served, which is the first unless the endpoint got to it later. + const FrameEvent held = stalled.expect(); + REQUIRE(held.sequence >= 1); + REQUIRE(held.sequence <= 5); + REQUIRE(stalled.receive(20ms).status == ReceiveStatus::TIMEOUT); +} + +TEST_CASE("a reader that sends without pause keeps nobody waiting", "[unit][camera_endpoint]") +{ + Served served; + + // One connection, and three threads sending valid requests on it as fast as they can: + // more than the endpoint handles in the same time, so there is always another one waiting. + Reader flooder(served); + REQUIRE(flooder.handshake("head").first.error.empty()); + // Stops and joins the senders however the test ends. + struct Flood + { + ~Flood() + { + flooding = false; + for (std::thread& sender : senders) + { + sender.join(); + } + } + + std::atomic flooding{ true }; + std::vector senders; + } flood; + for (int i = 0; i < 3; ++i) + { + flood.senders.emplace_back( + [&] + { + MessageEncoder encoder; + // A request that is never answered, so nothing piles up for the flooder to read. + const AcquireMessage never{ std::numeric_limits::max() }; + while (flood.flooding) + { + (void)flooder.socket.send(encoder.encode(never)); + } + }); + } + + // Other readers are served all the same. + for (std::uint64_t sequence = 1; sequence <= 5; ++sequence) + { + Reader other(served); + REQUIRE(other.handshake("head").first.error.empty()); + // It asks for what comes after the frame set before, so that it is sent this round's + // whether its request is served before the publish or after it. + other.ask(sequence - 1); + served.publish(static_cast(sequence)); + REQUIRE(other.expect().sequence == sequence); + } + + // And the endpoint stops while the flood goes on. + const auto before = std::chrono::steady_clock::now(); + served.endpoint.stop(); + REQUIRE(std::chrono::steady_clock::now() - before < 2s); +} + +TEST_CASE("stopping the endpoint closes every reader, and it can serve again", "[unit][camera_endpoint]") +{ + Served served; + Reader reader(served); + REQUIRE(reader.handshake("head").first.error.empty()); + + served.endpoint.stop(); + REQUIRE(reader.receive().status == ReceiveStatus::CLOSED); + REQUIRE(served.endpoint.reader_count() == 0); + served.endpoint.stop(); // nothing to stop + + // A stopped endpoint refuses a reader at once; it does not leave it waiting for an answer. + REQUIRE_THROWS_AS(MessageSocket::connect(served.directory.socket_path()), std::system_error); + + served.endpoint.start(); + Reader next(served); + REQUIRE(next.handshake().first.groups.size() == 2); + Reader again(served); + REQUIRE(again.handshake("head").first.error.empty()); + again.ask(); + served.publish(9); + REQUIRE(again.expect().sequence == 1); +} + +TEST_CASE("an endpoint has its path from the start and is reached only while it serves", "[unit][camera_endpoint]") +{ + TemporaryDirectory directory; + const std::string path = directory.socket_path(); + Publisher publisher; + Endpoint endpoint(publisher, path); + + // Not started, as while a plugin opens its cameras. The path is taken: a second endpoint + // is turned away already. + REQUIRE(std::filesystem::is_socket(path)); + try + { + const Endpoint second(publisher, path); + FAIL("a second endpoint took a path that was taken"); + } + catch (const std::system_error& error) + { + REQUIRE(error.code() == std::errc::address_in_use); + REQUIRE_THAT(error.what(), ContainsSubstring("another process has taken")); + } + + // And a reader is refused at once: it is not left waiting for an answer nobody sends. + REQUIRE(connect_error(path) == std::errc::connection_refused); + REQUIRE(endpoint.reader_count() == 0); + + // Its camera open, the plugin adds what the camera publishes. Started, it answers. + publisher.add_group(gray_group("head", 2)); + endpoint.start(); + { + Reader reader(path); + REQUIRE(reader.handshake().first.groups.size() == 1); + } + + // Stopped, it refuses again, and no file is left for a reader to try. + endpoint.stop(); + REQUIRE(connect_error(path)); + REQUIRE_FALSE(std::filesystem::exists(path)); + + endpoint.start(); + Reader reader(path); + REQUIRE(reader.handshake().first.groups.size() == 1); +} + +TEST_CASE("a group needs its memory named before readers can ask for it", "[unit][camera_endpoint]") +{ + TemporaryDirectory directory; + Publisher publisher; + const GroupId head = publisher.add_group(gray_group("head", 2)); + HostPool pool(SLOTS, WIDTH * HEIGHT, "test"); + Endpoint endpoint(publisher, directory.socket_path()); + + REQUIRE_THROWS_AS(endpoint.set_group_memory(head, { pool.descriptor() }), std::invalid_argument); + REQUIRE_THROWS_AS(endpoint.set_group_memory(head + 1, {}), std::out_of_range); + endpoint.start(); + REQUIRE_THROWS_AS(endpoint.set_group_memory(head, { pool.descriptor(), pool.descriptor() }), std::logic_error); + + MessageSocket socket = MessageSocket::connect(directory.socket_path()); + MessageEncoder encoder; + std::vector buffer(MAX_MESSAGE_BYTES); + REQUIRE(socket.send(encoder.encode(HandshakeRequest{ .group = "head" })) == SendStatus::SENT); + const ReceivedMessage received = socket.receive(buffer, 2s); + REQUIRE(received.status == ReceiveStatus::MESSAGE); + const ProtocolMessage answer = decode_message({ buffer.data(), received.bytes }); + REQUIRE_THAT(std::get(answer).error, ContainsSubstring("no memory a reader can map")); + + // Another endpoint cannot take the same path while this one lives. + REQUIRE_THROWS_AS(Endpoint(publisher, directory.socket_path()), std::system_error); +} diff --git a/tests/cpp/core/camera_peer/CMakeLists.txt b/tests/cpp/core/camera_peer/CMakeLists.txt new file mode 100644 index 0000000000..30f7d923e7 --- /dev/null +++ b/tests/cpp/core/camera_peer/CMakeLists.txt @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# The other process of the camera integration tests: it serves a camera group, or reads one +# and holds on to what it read, until it is killed. +add_executable(camera_test_peer + main.cpp +) + +target_link_libraries(camera_test_peer PRIVATE + camera::camera_endpoint + camera::camera_ipc + camera::camera_reader +) diff --git a/tests/cpp/core/camera_peer/main.cpp b/tests/cpp/core/camera_peer/main.cpp new file mode 100644 index 0000000000..005c0f485e --- /dev/null +++ b/tests/cpp/core/camera_peer/main.cpp @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The other process of the camera integration tests. +// +// camera_test_peer serve ENDPOINT [GROUP] serves one group, "head" unless it is named: two gray +// 64x48 streams, 8 slots. Every byte of frame set n is +// n modulo 256. +// camera_test_peer hold ENDPOINT MARKER reads "head" in order with a window of three, keeps +// the three frame sets, then creates the file MARKER. +// +// Either runs until it is killed. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; +constexpr std::uint32_t SLOTS = 8; + +[[noreturn]] void serve(const std::string& endpoint_path, const std::string& group_id) +{ + GroupDescription group; + group.id = group_id; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = group_id + "/" + eye; + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = WIDTH; + stream.slot_bytes = WIDTH * HEIGHT; + stream.slot_count = SLOTS; + stream.frame_rate = 200.0; + group.streams.push_back(stream); + } + + Publisher publisher; + const GroupId head = publisher.add_group(group); + HostPool left(SLOTS, WIDTH * HEIGHT, "peer_left"); + HostPool right(SLOTS, WIDTH * HEIGHT, "peer_right"); + Endpoint endpoint(publisher, endpoint_path); + endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); + endpoint.start(); + + HostPool* pools[] = { &left, &right }; + for (std::uint64_t sequence = 1;; ++sequence) + { + std::vector images(2); + bool complete = true; + for (std::size_t stream = 0; stream < 2 && complete; ++stream) + { + const std::optional slot = publisher.acquire_slot(head, stream); + if (!slot) + { + for (std::size_t taken = 0; taken < stream; ++taken) + { + publisher.abandon_slot(head, taken, images[taken].slot); + } + complete = false; + break; + } + const std::span pixels = pools[stream]->slot(*slot); + std::fill(pixels.begin(), pixels.end(), static_cast(sequence & 0xff)); + images[stream].slot = *slot; + } + if (complete) + { + // A dropped frame set takes no sequence number, so this stays the publisher's count. + if (publisher.publish(head, images, {}) != sequence) + { + std::fprintf(stderr, "camera_test_peer: the publisher's sequence is not the peer's\n"); + ::_exit(4); + } + } + else + { + --sequence; + } + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } +} + +[[noreturn]] void hold(const std::string& endpoint_path, const std::string& marker) +{ + CameraReader reader = + CameraReader::open("head", { .window = 3, .order = DeliveryOrder::IN_ORDER }, { .endpoint = endpoint_path }); + std::vector held; + while (held.size() < 3) + { + if (std::optional frames = reader.next(std::chrono::seconds(5))) + { + held.push_back(std::move(*frames)); + } + } + std::ofstream(marker) << "holding\n"; + for (;;) + { + ::pause(); + } +} + +} // namespace + +int main(int argc, char** argv) +{ + try + { + const std::string mode = argc > 1 ? argv[1] : ""; + if (mode == "serve" && (argc == 3 || argc == 4)) + { + serve(argv[2], argc == 4 ? argv[3] : "head"); + } + if (mode == "hold" && argc == 4) + { + hold(argv[2], argv[3]); + } + std::fprintf(stderr, "usage: camera_test_peer serve ENDPOINT [GROUP] | hold ENDPOINT MARKER\n"); + return 2; + } + catch (const std::exception& error) + { + std::fprintf(stderr, "camera_test_peer: %s\n", error.what()); + return 1; + } +} diff --git a/tests/cpp/core/camera_protocol/CMakeLists.txt b/tests/cpp/core/camera_protocol/CMakeLists.txt new file mode 100644 index 0000000000..d1b2083a7f --- /dev/null +++ b/tests/cpp/core/camera_protocol/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_protocol_tests + test_codec.cpp +) + +# The schema is linked to build messages the encoder never would: ones from a newer peer and +# malformed ones. +target_link_libraries(camera_protocol_tests PRIVATE + camera::camera_protocol + isaacteleop_schema + Catch2::Catch2WithMain +) + +message(STATUS "camera_protocol_tests target enabled with Catch2") +catch_discover_tests(camera_protocol_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_protocol/test_codec.cpp b/tests/cpp/core/camera_protocol/test_codec.cpp new file mode 100644 index 0000000000..4325819970 --- /dev/null +++ b/tests/cpp/core/camera_protocol/test_codec.cpp @@ -0,0 +1,555 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; + +namespace +{ + +GroupDescription stereo_group() +{ + GroupDescription group; + group.id = "head"; + group.session_id = 0x0123456789abcdef; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = std::string("head/") + eye; + stream.pixel_format = PixelFormat::NV12; + stream.width = 1920; + stream.height = 1080; + stream.plane_count = 2; + stream.planes[0] = { .offset_bytes = 0, .stride_bytes = 2048 }; + stream.planes[1] = { .offset_bytes = 2048 * 1080, .stride_bytes = 2048 }; + stream.slot_bytes = 2048 * 1620; + stream.slot_count = 8; + stream.frame_rate = 59.94; + group.streams.push_back(stream); + } + return group; +} + +GroupDescription encoded_group() +{ + GroupDescription group; + group.id = "head.h265"; + StreamDescription stream; + stream.id = "head/left.h265"; + stream.codec = Codec::H265; + stream.width = 1920; + stream.height = 1080; + stream.slot_bytes = 1 << 20; + stream.slot_count = 16; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + return group; +} + +void require_same(const GroupDescription& actual, const GroupDescription& expected) +{ + REQUIRE(actual.id == expected.id); + REQUIRE(actual.session_id == expected.session_id); + REQUIRE(actual.streams.size() == expected.streams.size()); + for (std::size_t i = 0; i < expected.streams.size(); ++i) + { + const StreamDescription& a = actual.streams[i]; + const StreamDescription& e = expected.streams[i]; + REQUIRE(a.id == e.id); + REQUIRE(a.codec == e.codec); + REQUIRE(a.memory == e.memory); + REQUIRE(a.width == e.width); + REQUIRE(a.height == e.height); + REQUIRE(a.plane_count == e.plane_count); + for (std::uint32_t plane = 0; plane < e.plane_count; ++plane) + { + REQUIRE(a.planes[plane].offset_bytes == e.planes[plane].offset_bytes); + REQUIRE(a.planes[plane].stride_bytes == e.planes[plane].stride_bytes); + } + REQUIRE(a.slot_bytes == e.slot_bytes); + REQUIRE(a.slot_count == e.slot_count); + REQUIRE(a.frame_rate == e.frame_rate); + if (!e.is_encoded()) + { + REQUIRE(a.pixel_format == e.pixel_format); + } + } +} + +// What the other process receives: a copy of the bytes, decoded. +template +Message round_trip(MessageEncoder& encoder, const Message& message) +{ + const std::span bytes = encoder.encode(message); + const std::vector received(bytes.begin(), bytes.end()); + ProtocolMessage decoded = decode_message(received); + REQUIRE(std::holds_alternative(decoded)); + return std::get(std::move(decoded)); +} + +std::span bytes_of(const flatbuffers::FlatBufferBuilder& builder) +{ + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; +} + +// Finishes a message whose body the schema's own helpers would not build: a body type this +// build does not have, a missing body, or the mark that lets a receiver ignore it. +void finish_message(flatbuffers::FlatBufferBuilder& builder, + std::uint32_t body_type, + flatbuffers::Offset body, + bool skippable = false) +{ + builder.Finish(core::CreateCameraMessage(builder, static_cast(body_type), body, skippable)); +} + +// A stream as a plugin describes it, with the kinds given as numbers so that ones this build +// does not have can be written. +flatbuffers::Offset build_stream(flatbuffers::FlatBufferBuilder& builder, + const std::string& id, + std::uint32_t codec = core::CameraCodec_None, + std::uint32_t pixel_format = core::CameraPixelFormat_Gray8, + std::uint32_t memory = core::CameraMemoryKind_Host, + std::size_t plane_count = 1) +{ + const std::vector planes(plane_count, core::CameraPlaneLayout(0, 64)); + return core::CreateCameraStream(builder, builder.CreateString(id), static_cast(codec), + static_cast(pixel_format), + static_cast(memory), 64, 64, + builder.CreateVectorOfStructs(planes), 64 * 64, 4, 30.0); +} + +flatbuffers::Offset build_group(flatbuffers::FlatBufferBuilder& builder, + const std::string& id, + const std::vector>& streams) +{ + return core::CreateCameraGroup(builder, builder.CreateString(id), builder.CreateVector(streams), 9); +} + +std::span finish_response(flatbuffers::FlatBufferBuilder& builder, + const std::vector>& groups) +{ + const auto response = core::CreateCameraHandshakeResponse(builder, PROTOCOL_VERSION, 0, builder.CreateVector(groups)); + finish_message(builder, core::CameraMessageBody_CameraHandshakeResponse, response.Union()); + return bytes_of(builder); +} + +HandshakeResponse response_with(std::initializer_list groups) +{ + HandshakeResponse response; + for (const GroupDescription& group : groups) + { + response.groups.push_back({ group, {} }); + } + return response; +} + +} // namespace + +TEST_CASE("a handshake survives the wire in both directions", "[unit][camera_protocol]") +{ + MessageEncoder encoder; + + // A reader that wants a group says so in its first message. + HandshakeRequest request; + request.protocol_version = 7; + request.reader_name = "viewer"; + request.group = "head"; + request.options = { .window = 4, .order = DeliveryOrder::IN_ORDER }; + const HandshakeRequest asked = round_trip(encoder, request); + REQUIRE(asked.protocol_version == 7); + REQUIRE(asked.reader_name == "viewer"); + REQUIRE(asked.group == "head"); + REQUIRE(asked.options.window == 4); + REQUIRE(asked.options.order == DeliveryOrder::IN_ORDER); + REQUIRE(asked.unsupported.empty()); + + // One that sets nothing asks what is published, and would read with the default window. + const HandshakeRequest plain = round_trip(encoder, HandshakeRequest{}); + REQUIRE(plain.protocol_version == PROTOCOL_VERSION); + REQUIRE(plain.group.empty()); + REQUIRE(plain.options.window == 1); + REQUIRE(plain.options.order == DeliveryOrder::DEFAULT); + + const HandshakeResponse answered = round_trip(encoder, response_with({ stereo_group(), encoded_group() })); + REQUIRE(answered.protocol_version == PROTOCOL_VERSION); + REQUIRE(answered.error.empty()); + REQUIRE(answered.groups.size() == 2); + REQUIRE(answered.groups[0].unsupported.empty()); + REQUIRE(answered.groups[1].unsupported.empty()); + require_same(answered.groups[0].group, stereo_group()); + require_same(answered.groups[1].group, encoded_group()); + + const HandshakeResponse refused = round_trip(encoder, HandshakeResponse{ .error = "does not fit the pool" }); + REQUIRE(refused.error == "does not fit the pool"); + REQUIRE(refused.groups.empty()); +} + +TEST_CASE("a frame set, its release and a request for one survive the wire", "[unit][camera_protocol]") +{ + MessageEncoder encoder; + + FrameEvent event; + event.lease_id = 0x1122334455667788; + event.sequence = 123456789012; + event.generation = 3; + event.skipped = 2; + event.image_count = 2; + event.timeline = { .capture_ns = 1000, .sdk_ready_ns = 2000, .published_ns = 3000, .sent_ns = 4000 }; + event.images[0] = { + .slot = 5, .flags = IMAGE_FLAG_SDK_WARNING, .bytes_used = 0, .device_frame_id = 77, .device_time_ns = 99 + }; + // A flag bit this build has no name for is passed on as it is. + event.images[1] = { .slot = 6, .flags = IMAGE_FLAG_KEYFRAME | (1u << 9), .bytes_used = 4096, .device_frame_id = 78 }; + + const FrameEvent received = round_trip(encoder, event); + REQUIRE(received.lease_id == event.lease_id); + REQUIRE(received.sequence == event.sequence); + REQUIRE(received.generation == 3); + REQUIRE(received.skipped == 2); + REQUIRE(received.image_count == 2); + REQUIRE(received.timeline.capture_ns == 1000); + REQUIRE(received.timeline.sdk_ready_ns == 2000); + REQUIRE(received.timeline.published_ns == 3000); + REQUIRE(received.timeline.sent_ns == 4000); + REQUIRE(received.images[0].slot == 5); + REQUIRE(received.images[0].flags == IMAGE_FLAG_SDK_WARNING); + REQUIRE(received.images[0].device_frame_id == 77); + REQUIRE(received.images[0].device_time_ns == 99); + REQUIRE(received.images[1].slot == 6); + REQUIRE(received.images[1].flags == (IMAGE_FLAG_KEYFRAME | (1u << 9))); + REQUIRE(received.images[1].bytes_used == 4096); + REQUIRE(received.images[1].device_time_ns == TIME_UNKNOWN); + + // Times that are not known stay unknown. + const FrameEvent bare = round_trip(encoder, FrameEvent{}); + REQUIRE(bare.image_count == 0); + REQUIRE(bare.timeline.capture_ns == TIME_UNKNOWN); + REQUIRE(bare.timeline.sent_ns == TIME_UNKNOWN); + + REQUIRE(round_trip(encoder, ReleaseMessage{ 42 }).lease_id == 42); + REQUIRE(round_trip(encoder, AcquireMessage{ 123456789012 }).after_sequence == 123456789012); + REQUIRE(round_trip(encoder, AcquireMessage{}).after_sequence == 0); +} + +TEST_CASE("the numbers of the protocol are the ones it was published with", "[unit][camera_protocol]") +{ + // A message's number, a flag's bit and a kind's value are read by builds that will never see + // this source. Changing one here is changing the protocol. + REQUIRE(core::CameraMessageBody_CameraHandshakeRequest == 1); + REQUIRE(core::CameraMessageBody_CameraHandshakeResponse == 2); + REQUIRE(core::CameraMessageBody_CameraFrameSet == 3); + REQUIRE(core::CameraMessageBody_CameraRelease == 4); + REQUIRE(core::CameraMessageBody_CameraAcquire == 5); + REQUIRE(static_cast(core::CameraFrameFlag_Keyframe) == IMAGE_FLAG_KEYFRAME); + REQUIRE(static_cast(core::CameraFrameFlag_SdkWarning) == IMAGE_FLAG_SDK_WARNING); + REQUIRE(IMAGE_FLAG_KEYFRAME == 1u); + REQUIRE(IMAGE_FLAG_SDK_WARNING == 2u); + REQUIRE(core::CameraPixelFormat_Bgra8 == 1); + REQUIRE(core::CameraPixelFormat_Gray8 == 4); + REQUIRE(core::CameraCodec_H265 == 2); + REQUIRE(core::CameraMemoryKind_Host == 1); + REQUIRE(core::CameraDeliveryOrder_InOrder == 2); + REQUIRE(sizeof(core::CameraPlaneLayout) == 8); + REQUIRE(PROTOCOL_VERSION >= OLDEST_PROTOCOL_VERSION); +} + +TEST_CASE("an encoder is reused from message to message", "[unit][camera_protocol]") +{ + MessageEncoder encoder; + FrameEvent event; + event.image_count = 2; + for (std::uint64_t sequence = 1; sequence <= 100; ++sequence) + { + event.sequence = sequence; + REQUIRE(round_trip(encoder, event).sequence == sequence); + } + // A small message after a large one is not padded to the large one's size. + const std::size_t large = encoder.encode(response_with({ stereo_group(), encoded_group() })).size(); + REQUIRE(encoder.encode(ReleaseMessage{ 1 }).size() < large); +} + +TEST_CASE("bytes that are not a message are rejected", "[unit][camera_protocol]") +{ + REQUIRE_THROWS_AS(decode_message({}), ProtocolError); + + const std::vector garbage(64, std::byte{ 0xa5 }); + REQUIRE_THROWS_AS(decode_message(garbage), ProtocolError); + + MessageEncoder encoder; + const std::span bytes = encoder.encode(response_with({ stereo_group() })); + const std::vector whole(bytes.begin(), bytes.end()); + REQUIRE_THROWS_AS(decode_message({ whole.data(), whole.size() / 2 }), ProtocolError); + + // No cut of a message may be read out of bounds; each is either refused or read in full. + for (std::size_t length = 0; length < whole.size(); ++length) + { + const std::vector cut(whole.begin(), whole.begin() + static_cast(length)); + try + { + (void)decode_message(cut); + } + catch (const ProtocolError&) + { + } + } +} + +TEST_CASE("the encoder refuses counts it cannot write, and cuts nothing short", "[unit][camera_protocol]") +{ + MessageEncoder encoder; + + GroupDescription crowded = stereo_group(); + crowded.streams.resize(MAX_STREAMS_PER_GROUP + 1, crowded.streams.front()); + REQUIRE_THROWS_AS(encoder.encode(response_with({ crowded })), std::invalid_argument); + + GroupDescription planes = stereo_group(); + planes.streams[0].plane_count = MAX_PLANES_PER_IMAGE + 1; + REQUIRE_THROWS_AS(encoder.encode(response_with({ planes })), std::invalid_argument); + + FrameEvent event; + event.image_count = MAX_STREAMS_PER_GROUP + 1; + REQUIRE_THROWS_AS(encoder.encode(event), std::invalid_argument); + + // The largest message a receive buffer has to hold. + REQUIRE_THROWS_AS( + encoder.encode(HandshakeRequest{ .reader_name = std::string(MAX_MESSAGE_BYTES, 'x') }), std::length_error); + + // The encoder is still usable after refusing a message. + REQUIRE(round_trip(encoder, ReleaseMessage{ 7 }).lease_id == 7); +} + +TEST_CASE("a message that is broken is rejected", "[unit][camera_protocol]") +{ + flatbuffers::FlatBufferBuilder builder; + + // More images than a group has streams. + const std::vector> frames( + MAX_STREAMS_PER_GROUP + 1, core::CreateCameraFrameRef(builder)); + const auto frame_set = core::CreateCameraFrameSet(builder, 1, 1, 0, 0, -1, -1, -1, -1, builder.CreateVector(frames)); + finish_message(builder, core::CameraMessageBody_CameraFrameSet, frame_set.Union()); + REQUIRE_THROWS_AS(decode_message(bytes_of(builder)), ProtocolError); + + // A body type with no body behind it. + builder.Clear(); + finish_message(builder, core::CameraMessageBody_CameraRelease, 0); + REQUIRE_THROWS_AS(decode_message(bytes_of(builder)), ProtocolError); +} + +TEST_CASE("a description is decoded as it was sent; whether it makes sense is checked where it is used", + "[unit][camera_protocol]") +{ + // The structure of these messages is fine. What they describe is not: rows longer than the + // slot, a plane count that does not match the format, a slot size no pool can have. + MessageEncoder encoder; + GroupDescription rows = stereo_group(); + rows.streams[0].planes[0].stride_bytes = 16; + GroupDescription planes = stereo_group(); + planes.streams[1].plane_count = 1; + GroupDescription slots = stereo_group(); + slots.streams[0].slot_bytes = MAX_SLOT_BYTES + 1; + GroupDescription wraps = stereo_group(); + wraps.streams[0].width = 0xFFFFFFFF; + wraps.streams[0].height = 0xFFFFFFFF; + wraps.streams[0].planes[0] = { .offset_bytes = 0xFFFFFFFF, .stride_bytes = 0xFFFFFFFF }; + + for (const GroupDescription& broken : { rows, planes, slots, wraps }) + { + const HandshakeResponse response = round_trip(encoder, response_with({ broken })); + REQUIRE(response.groups.size() == 1); + REQUIRE(response.groups[0].unsupported.empty()); + REQUIRE_FALSE(problem_with(response.groups[0].group).empty()); + } + REQUIRE(problem_with(round_trip(encoder, response_with({ stereo_group() })).groups[0].group).empty()); +} + +TEST_CASE("a group this build cannot read is listed as such, next to the ones it can", "[unit][camera_protocol]") +{ + flatbuffers::FlatBufferBuilder builder; + const auto decode_groups = [&builder](const std::vector>& groups) + { + ProtocolMessage decoded = decode_message(finish_response(builder, groups)); + REQUIRE(std::holds_alternative(decoded)); + return std::get(std::move(decoded)).groups; + }; + + // A newer plugin publishes one camera in a kind of memory added after this build, one with + // a newer codec and one with a newer pixel format, next to a camera this build reads. + builder.Clear(); + const std::vector groups = decode_groups({ + build_group(builder, "head", { build_stream(builder, "head/left"), build_stream(builder, "head/right") }), + build_group(builder, "wrist", + { build_stream(builder, "wrist/0", core::CameraCodec_None, core::CameraPixelFormat_Gray8, + core::CameraMemoryKind_MAX + 1) }), + build_group(builder, "chest", { build_stream(builder, "chest/0", core::CameraCodec_MAX + 1) }), + build_group(builder, "hip", + { build_stream(builder, "hip/0", core::CameraCodec_None, core::CameraPixelFormat_MAX + 1) }), + }); + REQUIRE(groups.size() == 4); + REQUIRE(groups[0].unsupported.empty()); + REQUIRE(groups[0].group.streams.size() == 2); + REQUIRE(groups[0].group.session_id == 9); + REQUIRE(problem_with(groups[0].group).empty()); + REQUIRE(groups[1].group.id == "wrist"); + REQUIRE(groups[1].group.streams.empty()); + REQUIRE(groups[1].unsupported.find("kind of memory") != std::string::npos); + REQUIRE(groups[2].unsupported.find("codec") != std::string::npos); + REQUIRE(groups[3].unsupported.find("pixel format") != std::string::npos); + + // An encoded stream has no pixel format to know, so one this build has no name for is fine. + builder.Clear(); + REQUIRE(decode_groups( + { build_group(builder, "enc", + { build_stream(builder, "enc/0", core::CameraCodec_H264, core::CameraPixelFormat_MAX + 1, + core::CameraMemoryKind_Host, 0) }) })[0] + .unsupported.empty()); + + // More planes or more streams than this build can describe. + builder.Clear(); + REQUIRE_FALSE(decode_groups({ build_group( + builder, "planar", + { build_stream(builder, "planar/0", core::CameraCodec_None, core::CameraPixelFormat_Gray8, + core::CameraMemoryKind_Host, MAX_PLANES_PER_IMAGE + 1) }) })[0] + .unsupported.empty()); + builder.Clear(); + std::vector> many; + for (std::size_t i = 0; i <= MAX_STREAMS_PER_GROUP; ++i) + { + many.push_back(build_stream(builder, "rig/" + std::to_string(i))); + } + const std::vector crowded = decode_groups({ build_group(builder, "rig", many) }); + REQUIRE(crowded[0].group.id == "rig"); + REQUIRE(crowded[0].unsupported.find("more streams") != std::string::npos); + + // A reader that asks for a delivery order added later is told so, not dropped unanswered. + MessageEncoder encoder; + const HandshakeRequest newer = + round_trip(encoder, HandshakeRequest{ .group = "head", .options = { .order = static_cast(9) } }); + REQUIRE(newer.group == "head"); + REQUIRE(newer.unsupported.find("delivery order") != std::string::npos); +} + +TEST_CASE("a message type this build does not know is ignored only when its sender says it may be", + "[unit][camera_protocol]") +{ + flatbuffers::FlatBufferBuilder builder; + const std::uint32_t unknown = core::CameraMessageBody_MAX + 1; + + // Something that only informs, such as a status a newer plugin sends: ignored. + finish_message(builder, unknown, core::CreateCameraRelease(builder, 1).Union(), true); + REQUIRE(std::holds_alternative(decode_message(bytes_of(builder)))); + + // An operation the peer counts on: carrying on as if it had not been sent would go wrong + // quietly, so it is an error. + builder.Clear(); + finish_message(builder, unknown, core::CreateCameraRelease(builder, 1).Union(), false); + REQUIRE_THROWS_AS(decode_message(bytes_of(builder)), ProtocolError); + + // No body at all, marked or not. + builder.Clear(); + builder.Finish(core::CreateCameraMessage(builder)); + REQUIRE_THROWS_AS(decode_message(bytes_of(builder)), ProtocolError); + builder.Clear(); + finish_message(builder, core::CameraMessageBody_NONE, 0, true); + REQUIRE(std::holds_alternative(decode_message(bytes_of(builder)))); + + // The mark changes nothing for a message this build does know. + builder.Clear(); + finish_message(builder, core::CameraMessageBody_CameraRelease, core::CreateCameraRelease(builder, 5).Union(), true); + const ProtocolMessage known = decode_message(bytes_of(builder)); + REQUIRE(std::get(known).lease_id == 5); +} + +TEST_CASE("fields added by a newer peer are passed over, and fields an older peer lacks take their defaults", + "[unit][camera_protocol]") +{ + flatbuffers::FlatBufferBuilder builder; + // A table's fields sit two bytes apart in its vtable, so the slot after a table's last field + // is where a newer schema's first appended field goes. + constexpr flatbuffers::voffset_t NEXT = 2; + + // Newer reader: a handshake request with a field appended after `order`. + { + const auto name = builder.CreateString("newer"); + const auto group = builder.CreateString("head"); + const auto start = builder.StartTable(); + builder.AddElement(core::CameraHandshakeRequest::VT_PROTOCOL_VERSION, PROTOCOL_VERSION + 1, 0); + builder.AddOffset(core::CameraHandshakeRequest::VT_READER_NAME, name); + builder.AddOffset(core::CameraHandshakeRequest::VT_GROUP_ID, group); + builder.AddElement(core::CameraHandshakeRequest::VT_WINDOW, 2, 1); + builder.AddElement(core::CameraHandshakeRequest::VT_ORDER, core::CameraDeliveryOrder_Newest, 0); + builder.AddElement(core::CameraHandshakeRequest::VT_ORDER + NEXT, 0xfeedULL, 0); + const flatbuffers::Offset request(builder.EndTable(start)); + finish_message(builder, core::CameraMessageBody_CameraHandshakeRequest, request); + const ProtocolMessage decoded = decode_message(bytes_of(builder)); + const HandshakeRequest& asked = std::get(decoded); + REQUIRE(asked.protocol_version == PROTOCOL_VERSION + 1); + REQUIRE(asked.reader_name == "newer"); + REQUIRE(asked.group == "head"); + REQUIRE(asked.options.window == 2); + REQUIRE(asked.options.order == DeliveryOrder::NEWEST); + REQUIRE(asked.unsupported.empty()); + } + + // Newer plugin: a frame set with a field appended after `frames`, and a frame reference + // with one appended after `device_time_ns`. + { + builder.Clear(); + auto start = builder.StartTable(); + builder.AddElement(core::CameraFrameRef::VT_SLOT_INDEX, 3, 0); + builder.AddElement(core::CameraFrameRef::VT_DEVICE_TIME_NS + NEXT, 555, 0); + const flatbuffers::Offset frame(builder.EndTable(start)); + const auto frames = builder.CreateVector(&frame, 1); + start = builder.StartTable(); + builder.AddElement(core::CameraFrameSet::VT_LEASE_ID, 11, 0); + builder.AddElement(core::CameraFrameSet::VT_SEQUENCE, 22, 0); + builder.AddOffset(core::CameraFrameSet::VT_FRAMES, frames); + builder.AddElement(core::CameraFrameSet::VT_FRAMES + NEXT, 7, 0); + const flatbuffers::Offset frame_set(builder.EndTable(start)); + finish_message(builder, core::CameraMessageBody_CameraFrameSet, frame_set); + const ProtocolMessage decoded = decode_message(bytes_of(builder)); + const FrameEvent& event = std::get(decoded); + REQUIRE(event.lease_id == 11); + REQUIRE(event.sequence == 22); + REQUIRE(event.image_count == 1); + REQUIRE(event.images[0].slot == 3); + REQUIRE(event.images[0].device_time_ns == TIME_UNKNOWN); + REQUIRE(event.timeline.capture_ns == TIME_UNKNOWN); + } + + // Older peer: a group written before `session_id` existed, in a message written before + // `skippable` did. What is missing reads as its default. + { + builder.Clear(); + const auto stream = build_stream(builder, "head/left"); + const auto id = builder.CreateString("head"); + const auto streams = builder.CreateVector(&stream, 1); + auto start = builder.StartTable(); + builder.AddOffset(core::CameraGroup::VT_ID, id); + builder.AddOffset(core::CameraGroup::VT_STREAMS, streams); + const flatbuffers::Offset group(builder.EndTable(start)); + const auto response = + core::CreateCameraHandshakeResponse(builder, PROTOCOL_VERSION, 0, builder.CreateVector(&group, 1)); + start = builder.StartTable(); + builder.AddElement( + core::CameraMessage::VT_BODY_TYPE, core::CameraMessageBody_CameraHandshakeResponse, 0); + builder.AddOffset(core::CameraMessage::VT_BODY, response.Union()); + builder.Finish(flatbuffers::Offset(builder.EndTable(start))); + const ProtocolMessage decoded = decode_message(bytes_of(builder)); + const HandshakeResponse& answered = std::get(decoded); + REQUIRE(answered.groups.size() == 1); + REQUIRE(answered.groups[0].group.id == "head"); + REQUIRE(answered.groups[0].group.session_id == 0); + REQUIRE(answered.groups[0].unsupported.empty()); + } +} diff --git a/tests/cpp/core/camera_reader/CMakeLists.txt b/tests/cpp/core/camera_reader/CMakeLists.txt new file mode 100644 index 0000000000..771228c81a --- /dev/null +++ b/tests/cpp/core/camera_reader/CMakeLists.txt @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +find_package(Threads REQUIRED) + +add_executable(camera_reader_tests + test_camera_reader.cpp + test_cross_process.cpp +) + +# The reader is tested against a real endpoint, and against a stand-in plugin built from the +# socket and the codec for what a real one does not do on demand. The schema is linked for +# what the codec never writes: a group from a newer plugin. +target_link_libraries(camera_reader_tests PRIVATE + camera::camera_reader + camera::camera_endpoint + camera::camera_ipc + camera::camera_protocol + isaacteleop_schema + Catch2::Catch2WithMain + Threads::Threads +) + +# The integration cases run camera_test_peer as the other process. +target_compile_definitions(camera_reader_tests PRIVATE + CAMERA_TEST_PEER_EXECUTABLE="$" +) +add_dependencies(camera_reader_tests camera_test_peer) +# That path is into the build tree, so this binary only runs where it was built: keep it off +# the venv PATH of a pip build. +set_target_properties(camera_reader_tests PROPERTIES ISAACTELEOP_NEEDS_SOURCE_TREE TRUE) + +message(STATUS "camera_reader_tests target enabled with Catch2") +catch_discover_tests(camera_reader_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_reader/endpoint_directory.hpp b/tests/cpp/core/camera_reader/endpoint_directory.hpp new file mode 100644 index 0000000000..7747d07e78 --- /dev/null +++ b/tests/cpp/core/camera_reader/endpoint_directory.hpp @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +// This user's endpoint directory for one test. XDG_RUNTIME_DIR points at a short temporary +// directory, so a reader that looks through the user's camera plugins finds what the test put +// there and nothing else. What the variable held is put back afterwards. +class EndpointDirectory +{ +public: + EndpointDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_find_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_runtime = pattern; + if (const char* previous = std::getenv("XDG_RUNTIME_DIR")) + { + m_previous = previous; + } + ::setenv("XDG_RUNTIME_DIR", m_runtime.c_str(), 1); + // As a plugin does before it takes its endpoint. + core::camera::prepare_endpoint_directory(core::camera::default_endpoint_path("synthetic")); + } + + ~EndpointDirectory() + { + if (m_previous) + { + ::setenv("XDG_RUNTIME_DIR", m_previous->c_str(), 1); + } + else + { + ::unsetenv("XDG_RUNTIME_DIR"); + } + std::error_code ignored; + std::filesystem::remove_all(m_runtime, ignored); + } + + EndpointDirectory(const EndpointDirectory&) = delete; + EndpointDirectory& operator=(const EndpointDirectory&) = delete; + + // The directory a reader looks through. + std::string path() const + { + return (m_runtime / "isaaccapture").string(); + } + +private: + std::filesystem::path m_runtime; + std::optional m_previous; +}; diff --git a/tests/cpp/core/camera_reader/test_camera_reader.cpp b/tests/cpp/core/camera_reader/test_camera_reader.cpp new file mode 100644 index 0000000000..a3b73ec544 --- /dev/null +++ b/tests/cpp/core/camera_reader/test_camera_reader.cpp @@ -0,0 +1,1033 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "endpoint_directory.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; + +namespace +{ + +constexpr std::uint32_t WIDTH = 64; +constexpr std::uint32_t HEIGHT = 48; +constexpr std::uint32_t STRIDE = 80; // rows are padded +constexpr std::uint32_t SLOTS = 8; +constexpr std::uint64_t SLOT_BYTES = STRIDE * HEIGHT; + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_reader_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string socket_path() const + { + return (m_path / "cameras.sock").string(); + } + +private: + std::filesystem::path m_path; +}; + +GroupDescription stereo_group(const std::string& id = "head") +{ + GroupDescription group; + group.id = id; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = id + "/" + eye; + stream.pixel_format = PixelFormat::GRAY8; + stream.width = WIDTH; + stream.height = HEIGHT; + stream.plane_count = 1; + stream.planes[0].stride_bytes = STRIDE; + stream.slot_bytes = SLOT_BYTES; + stream.slot_count = SLOTS; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + } + return group; +} + +// A plugin in miniature: a stereo group in host memory behind an endpoint. The endpoint is a +// socket in a directory of its own, or the path it is given: one in the user's endpoint +// directory, where a reader finds it without being told. +struct Served +{ + std::optional directory; + std::string endpoint_path; + Publisher publisher; + GroupId head; + HostPool left{ SLOTS, SLOT_BYTES, "test_left" }; + HostPool right{ SLOTS, SLOT_BYTES, "test_right" }; + Endpoint endpoint; + + explicit Served(const std::string& group = "head", const std::string& path = {}) + : endpoint_path(path.empty() ? directory.emplace().socket_path() : path), + head(publisher.add_group(stereo_group(group))), + endpoint(publisher, endpoint_path) + { + endpoint.set_group_memory(head, { left.descriptor(), right.descriptor() }); + endpoint.start(); + } + + // Publishes one frame set: every left pixel is `value`, every right pixel `value + 1`. + std::uint64_t publish(std::uint8_t value, const FrameTimeline& timeline = {}) + { + std::vector images(2); + HostPool* pools[] = { &left, &right }; + for (std::size_t i = 0; i < 2; ++i) + { + const auto slot = publisher.acquire_slot(head, i); + REQUIRE(slot.has_value()); + const std::byte fill{ static_cast(value + i) }; + std::fill(pools[i]->slot(*slot).begin(), pools[i]->slot(*slot).end(), fill); + images[i].slot = *slot; + images[i].device_frame_id = 1000 + value; + } + return publisher.publish(head, images, timeline); + } + + // Gives the endpoint's thread time to serve a request the reader has made: it is served + // before the publish that answers it or after, and the frame set is sent either way. + void wait_until_sent(std::uint64_t count) + { + for (int i = 0; i < 2000 && publisher.stats(head).events_sent < count; ++i) + { + std::this_thread::sleep_for(1ms); + } + } +}; + +std::byte first_pixel(const FrameSet& frames, std::size_t stream) +{ + return frames.image(stream).data.front(); +} + +// What a plugin that was killed leaves behind: a socket file with nothing behind it. +void leave_socket_file(const std::string& path) +{ + sockaddr_un address = {}; + address.sun_family = AF_UNIX; + REQUIRE(path.size() < sizeof(address.sun_path)); + std::memcpy(address.sun_path, path.data(), path.size()); + const UniqueFd dead(::socket(AF_UNIX, SOCK_SEQPACKET, 0)); + REQUIRE(::bind(dead.get(), reinterpret_cast(&address), sizeof(address)) == 0); +} + +// A stand-in plugin, to answer a reader with what a real plugin does not send on demand. It +// takes one reader and its handshake, then does what the test says. Like Served, it is on a +// socket in a directory of its own or on the path it is given. +class StandIn +{ +public: + using Conversation = std::function; + + explicit StandIn(Conversation conversation, const std::string& path = {}) + : m_path(path.empty() ? m_directory.emplace().socket_path() : path), m_listener(m_path) + { + // Before the constructor returns, so that the reader the test opens next is not refused. + m_listener.listen(); + m_thread = std::thread( + [this, conversation = std::move(conversation)] + { + try + { + std::optional reader = m_listener.accept(5s); + if (!reader) + { + return; + } + std::vector buffer(MAX_MESSAGE_BYTES); + const ReceivedMessage received = reader->receive(buffer, 5s); + conversation(*reader, std::get(decode_message({ buffer.data(), received.bytes }))); + } + catch (const std::exception&) + { + // The reader's side of the test fails. + } + }); + } + + ~StandIn() + { + m_thread.join(); + } + + PluginLocation location() const + { + return { .endpoint = m_path }; + } + + // The answer a real plugin gives to a request for the stereo group. + static void answer(MessageSocket& reader, const HostPool& left, const HostPool& right) + { + MessageEncoder encoder; + HandshakeResponse response; + response.groups.push_back({ stereo_group(), {} }); + const std::array descriptors = { left.descriptor(), right.descriptor() }; + reader.send(encoder.encode(response), descriptors); + } + +private: + std::optional m_directory; + std::string m_path; + MessageListener m_listener; + std::thread m_thread; +}; + +// What the reader does with an answer to a request for "head", built by `respond`. +void open_against(const std::function& respond) +{ + StandIn plugin([&](MessageSocket& reader, const HandshakeRequest&) { respond(reader); }); + CameraReader::open("head", {}, plugin.location()); +} + +std::span bytes_of(const flatbuffers::FlatBufferBuilder& builder) +{ + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; +} + +} // namespace + +TEST_CASE("a reader lists what a plugin publishes", "[unit][camera_reader]") +{ + Served served; + const std::vector groups = CameraReader::list({ .endpoint = served.endpoint_path }); + REQUIRE(groups.size() == 1); + REQUIRE(groups[0].group.id == "head"); + REQUIRE(groups[0].unsupported.empty()); + REQUIRE(groups[0].group.session_id != 0); + REQUIRE(groups[0].group.streams.size() == 2); + REQUIRE(groups[0].group.streams[0].id == "head/left"); + REQUIRE(groups[0].endpoint == served.endpoint_path); +} + +TEST_CASE("a reader gets each frame set with its pixels and what is known about it", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open("head", {}, { .endpoint = served.endpoint_path }); + REQUIRE(reader.group().id == "head"); + REQUIRE(reader.group().streams.size() == 2); + REQUIRE(reader.group().session_id == served.publisher.group_description(served.head).session_id); + REQUIRE(reader.descriptor() >= 0); + + REQUIRE_FALSE(reader.next(10ms).has_value()); // nothing published yet + + served.publish(0x10, { .capture_ns = 111, .sdk_ready_ns = 222 }); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 1); + REQUIRE(frames->skipped() == 0); + REQUIRE(frames->image_count() == 2); + REQUIRE(frames->timeline().capture_ns == 111); + REQUIRE(frames->timeline().sdk_ready_ns == 222); + REQUIRE(frames->timeline().sent_ns >= frames->timeline().published_ns); + + const ImageView left = frames->image(0); + const ImageView right = frames->image(1); + REQUIRE(left.stream->id == "head/left"); + REQUIRE(left.info.device_frame_id == 1000 + 0x10); + REQUIRE(left.data.size() == SLOT_BYTES); + REQUIRE(left.data.front() == std::byte{ 0x10 }); + REQUIRE(right.data.back() == std::byte{ 0x11 }); + + // A plane runs from its first row to the end of its last, without the last row's padding. + REQUIRE(left.plane(0).size() == STRIDE * (HEIGHT - 1) + WIDTH); + REQUIRE(left.plane(0).data() == left.data.data()); + REQUIRE_THROWS_AS(left.plane(1), std::out_of_range); + REQUIRE_THROWS_AS(frames->image(2), std::out_of_range); +} + +TEST_CASE("a held frame set is not rewritten and the next one is the newest", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open("head", {}, { .endpoint = served.endpoint_path }); + + served.publish(1); + std::optional held = reader.next(2s); + REQUIRE(held.has_value()); + + // Capture runs through the pool several times while the reader holds on. + for (std::uint8_t value = 2; value <= 3 * SLOTS; ++value) + { + served.publish(value); + } + REQUIRE(first_pixel(*held, 0) == std::byte{ 1 }); + REQUIRE(first_pixel(*held, 1) == std::byte{ 2 }); + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + + held.reset(); // gives it back + std::optional newest = reader.next(2s); + REQUIRE(newest.has_value()); + REQUIRE(newest->sequence() == 3 * SLOTS); + REQUIRE(newest->skipped() == 3 * SLOTS - 2); + REQUIRE(first_pixel(*newest, 0) == std::byte{ 3 * SLOTS }); + + // release() gives a frame set back before it goes out of scope. + newest->release(); + REQUIRE_THROWS_AS(newest->image(0), std::logic_error); + served.publish(99); + REQUIRE(reader.next(2s)->sequence() == 3 * SLOTS + 1); +} + +TEST_CASE("a reader that was busy gets what the camera saw last", "[unit][camera_reader][newest]") +{ + Served served; + CameraReader reader = CameraReader::open("head", {}, { .endpoint = served.endpoint_path }); + served.publish(1); + REQUIRE(reader.next(2s)->sequence() == 1); // taken and given back at once + + // The reader does something else for a while, holding nothing and not asking. The camera + // goes on; nothing is put aside for the reader. + std::int64_t last_published_ns = 0; + for (std::uint8_t value = 2; value <= 12; ++value) + { + std::this_thread::sleep_for(5ms); + last_published_ns = + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); + served.publish(value); + } + REQUIRE(served.publisher.stats(served.head).events_sent == 1); + + // It is handed the last frame set published, not the one that was newest when it stopped. + const std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 12); + REQUIRE(frames->skipped() == 10); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 12 }); + REQUIRE(frames->timeline().published_ns >= last_published_ns); + REQUIRE(served.publisher.stats(served.head).events_sent == 2); +} + +TEST_CASE("a frame set that arrived after its reader stopped waiting is exchanged for the newest", + "[unit][camera_reader][newest]") +{ + Served served; + CameraReader reader = CameraReader::open("head", {}, { .endpoint = served.endpoint_path }); + + served.publish(1); + REQUIRE(reader.next(2s)->sequence() == 1); + + // The reader asks for the next one and stops waiting before it is published. + REQUIRE_FALSE(reader.next(10ms).has_value()); + + // The camera then publishes: the first frame set answers the reader's request and waits in + // its connection. Two more follow while the reader is away. + served.publish(2); + served.wait_until_sent(2); + served.publish(3); + served.publish(4); + REQUIRE(served.publisher.stats(served.head).events_sent == 2); + + // Back again, the reader does not hand out the one that waited: it gives it back and is + // sent the newest. + const std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 4); + REQUIRE(frames->skipped() == 2); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 4 }); + REQUIRE(served.publisher.stats(served.head).events_sent == 3); +} + +TEST_CASE("a frame set that waited is handed out when nothing newer exists", "[unit][camera_reader][newest]") +{ + Served served; + CameraReader reader = CameraReader::open("head", {}, { .endpoint = served.endpoint_path }); + REQUIRE_FALSE(reader.next(10ms).has_value()); + served.publish(7); + served.wait_until_sent(1); + REQUIRE(served.publisher.stats(served.head).events_sent == 1); + + // The exchange finds the same frame set still the newest, and the reader gets it: asking + // again costs a round trip, never a frame. + const std::optional frames = reader.next(0ms); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 1); + REQUIRE(frames->skipped() == 0); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 7 }); + REQUIRE(served.publisher.stats(served.head).events_sent == 2); +} + +TEST_CASE("an in-order reader gets every frame set", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open("head", { .window = 3 }, { .endpoint = served.endpoint_path }); + + served.publish(1); + served.publish(2); + served.publish(3); + std::vector held; + for (std::uint64_t sequence = 1; sequence <= 3; ++sequence) + { + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == sequence); + REQUIRE(frames->skipped() == 0); + held.push_back(std::move(*frames)); + } + // The window is full: three are held together, all three are still readable, and asking + // for a fourth says what is wrong instead of waiting for something that cannot come. + served.publish(4); + REQUIRE_THROWS_WITH(reader.next(2s), ContainsSubstring("holds 3 frame set(s), which is its whole window")); + REQUIRE_THROWS_AS(reader.next(2s), CameraWindowExhausted); + REQUIRE(first_pixel(held[0], 0) == std::byte{ 1 }); + REQUIRE(first_pixel(held[2], 0) == std::byte{ 3 }); + + held.erase(held.begin()); + REQUIRE(reader.next(2s)->sequence() == 4); +} + +TEST_CASE("a newest-order reader with a larger window holds several frame sets at once", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open( + "head", { .window = 3, .order = DeliveryOrder::NEWEST }, { .endpoint = served.endpoint_path }); + + served.publish(1); + served.publish(2); + served.publish(3); + // It gets the newest of the three, and nothing of the two before it. + std::optional first = reader.next(2s); + REQUIRE(first.has_value()); + REQUIRE(first->sequence() == 3); + REQUIRE(first_pixel(*first, 0) == std::byte{ 3 }); + + // While it holds that one it can take two more. + REQUIRE_FALSE(reader.next(10ms).has_value()); // nothing newer yet + served.publish(4); + std::optional second = reader.next(2s); + REQUIRE(second.has_value()); + REQUIRE(second->sequence() == 4); + served.publish(5); + served.publish(6); + std::optional third = reader.next(2s); + REQUIRE(third.has_value()); + REQUIRE(third->sequence() == 6); + REQUIRE(third->skipped() == 1); + REQUIRE(first_pixel(*first, 0) == std::byte{ 3 }); + REQUIRE(first_pixel(*second, 0) == std::byte{ 4 }); + + // Three held is the window. + REQUIRE_THROWS_AS(reader.next(10ms), CameraWindowExhausted); + second.reset(); + served.publish(7); + REQUIRE(reader.next(2s)->sequence() == 7); +} + +TEST_CASE("a frame set can be given back after its reader is gone", "[unit][camera_reader]") +{ + Served served; + std::optional frames; + { + CameraReader reader = CameraReader::open("head", {}, { .endpoint = served.endpoint_path }); + served.publish(5); + frames = reader.next(2s); + REQUIRE(frames.has_value()); + CameraReader moved = std::move(reader); + REQUIRE(moved.group().id == "head"); + } + // The frame set is still this reader's: its slots are not reused while the camera runs + // through the pool, and the pixels stay as they were. + for (std::uint8_t value = 0; value < 3 * SLOTS; ++value) + { + served.publish(0x80 + value); + } + REQUIRE(served.publisher.stats(served.head).no_free_slot == 0); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 5 }); + REQUIRE(served.endpoint.reader_count() == 1); + + // Giving it back is what ends the connection. + frames.reset(); + for (int i = 0; i < 2000 && served.endpoint.reader_count() != 0; ++i) + { + std::this_thread::sleep_for(1ms); + } + REQUIRE(served.endpoint.reader_count() == 0); +} + +TEST_CASE("a reader says why it could not be opened", "[unit][camera_reader]") +{ + TemporaryDirectory directory; + REQUIRE_THROWS_WITH(CameraReader::open("head", {}, { .endpoint = directory.socket_path() }), + ContainsSubstring("no camera plugin answers on '" + directory.socket_path() + "'") && + ContainsSubstring("Start the camera plugin for your camera through the plugin manager")); + REQUIRE_THROWS_AS(CameraReader::list({ .endpoint = directory.socket_path() }), CameraReaderError); + + Served served; + REQUIRE_THROWS_WITH( + CameraReader::open("chest", {}, { .endpoint = served.endpoint_path }), ContainsSubstring("no group 'chest'")); + REQUIRE_THROWS_WITH(CameraReader::open("head", { .window = SLOTS }, { .endpoint = served.endpoint_path }), + ContainsSubstring("does not fit the pool")); + + // A refusal leaves nothing behind: the pool still has room for a reader that fits. + REQUIRE_NOTHROW(CameraReader::open("head", { .window = 2 }, { .endpoint = served.endpoint_path })); +} + +TEST_CASE("a reader finds each group on the plugin that publishes it", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + Served head_plugin("head", default_endpoint_path("synthetic")); + Served wrist_plugin("wrist", default_endpoint_path("second")); + + // No location: the reader asks the plugins of this user and goes to the one with the group. + CameraReader head = CameraReader::open("head"); + CameraReader wrist = CameraReader::open("wrist"); + REQUIRE(head.group().id == "head"); + REQUIRE(wrist.group().id == "wrist"); + + // Each reads what its own plugin publishes. + head_plugin.publish(0x10); + wrist_plugin.publish(0x20); + const std::optional from_head = head.next(2s); + const std::optional from_wrist = wrist.next(2s); + REQUIRE(from_head.has_value()); + REQUIRE(from_wrist.has_value()); + REQUIRE(first_pixel(*from_head, 0) == std::byte{ 0x10 }); + REQUIRE(first_pixel(*from_wrist, 0) == std::byte{ 0x20 }); + + // Listing gives the groups of every plugin, in the order of the endpoints, each with the + // endpoint that publishes it. + const std::vector groups = CameraReader::list(); + REQUIRE(groups.size() == 2); + REQUIRE(groups[0].group.id == "wrist"); + REQUIRE(groups[0].endpoint == default_endpoint_path("second")); + REQUIRE(groups[0].group.streams.size() == 2); + REQUIRE(groups[1].group.id == "head"); + REQUIRE(groups[1].endpoint == default_endpoint_path("synthetic")); +} + +TEST_CASE("a group that two plugins publish is not opened by guessing", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + Served first("head", default_endpoint_path("first")); + Served second("head", default_endpoint_path("second")); + + REQUIRE_THROWS_WITH(CameraReader::open("head"), + ContainsSubstring("camera group 'head' is published by more than one camera plugin") && + ContainsSubstring("'" + first.endpoint_path + "'") && + ContainsSubstring("'" + second.endpoint_path + "'")); + REQUIRE_THROWS_AS(CameraReader::open("head"), CameraReaderError); + REQUIRE(CameraReader::list().size() == 2); + + // Told which plugin, the reader opens the group there. + CameraReader reader = CameraReader::open("head", {}, { .instance = "second" }); + first.publish(0x10); + second.publish(0x20); + const std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 0x20 }); +} + +TEST_CASE("a reader that finds no plugin for its group says where it looked and what is there", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + const std::string how_to_start = + "Start the camera plugin for your camera through the plugin manager, for example camera_synthetic"; + + // No plugin runs. That is nothing to list, and not an error until a group is asked for. + REQUIRE(CameraReader::list().empty()); + REQUIRE_THROWS_WITH( + CameraReader::open("head"), + ContainsSubstring("no camera plugin with an endpoint in '" + endpoints.path() + "' publishes group 'head'") && + ContainsSubstring("No camera plugin is running there") && ContainsSubstring(how_to_start)); + REQUIRE_THROWS_AS(CameraReader::open("head"), CameraReaderError); + + // Plugins run, with other groups: the message says which, and where each is. + Served wrist_plugin("wrist", default_endpoint_path("synthetic")); + Served chest_plugin("chest", default_endpoint_path("second")); + REQUIRE_THROWS_WITH( + CameraReader::open("head"), ContainsSubstring("in '" + endpoints.path() + "' publishes group 'head'") && + ContainsSubstring("'wrist' by '" + wrist_plugin.endpoint_path + "'") && + ContainsSubstring("'chest' by '" + chest_plugin.endpoint_path + "'") && + !ContainsSubstring("No camera plugin is running") && + !ContainsSubstring("Passed over") && ContainsSubstring(how_to_start)); +} + +TEST_CASE("a reader that cannot look through the user's endpoints says so", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + // Where the endpoints would be, there is something that is not a directory. + std::filesystem::remove(endpoints.path()); + std::ofstream(endpoints.path()) << "not a directory"; + + REQUIRE_THROWS_WITH(CameraReader::list(), ContainsSubstring("cannot look for camera plugins")); + REQUIRE_THROWS_AS(CameraReader::list(), CameraReaderError); + REQUIRE_THROWS_AS(CameraReader::open("head"), CameraReaderError); +} + +TEST_CASE("a plugin that is still starting is passed over at once", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + Served served("head", default_endpoint_path("synthetic")); + + // A plugin that has taken its endpoint and does not serve it yet, as while it opens its + // cameras: a connection to it is refused. + Publisher publisher; + publisher.add_group(stereo_group("wrist")); + Endpoint starting(publisher, default_endpoint_path("starting")); + REQUIRE(default_endpoint_paths().size() == 2); + + const auto before = std::chrono::steady_clock::now(); + CameraReader reader = CameraReader::open("head"); + // Well under the two seconds a plugin gets that a reader was pointed at. That this one was + // refused, and not asked and waited for, is what the checks below show. + REQUIRE(std::chrono::steady_clock::now() - before < 1s); + REQUIRE(reader.group().id == "head"); + + // For a reader it is not there yet: not listed, and not named as a plugin that failed to answer. + REQUIRE(CameraReader::list().size() == 1); + REQUIRE_THROWS_WITH(CameraReader::open("wrist"), ContainsSubstring("'head' by '" + served.endpoint_path + "'") && + !ContainsSubstring(default_endpoint_path("starting"))); + + // A reader that is pointed at it is told that nothing answers, instead of waiting for it. + REQUIRE_THROWS_WITH(CameraReader::open("wrist", {}, { .instance = "starting" }), + ContainsSubstring("no camera plugin answers on '" + default_endpoint_path("starting") + "'")); + + // Once it serves, readers find what it publishes. + starting.start(); + const std::vector groups = CameraReader::list(); + REQUIRE(groups.size() == 2); + REQUIRE(groups[0].group.id == "wrist"); + REQUIRE(groups[0].endpoint == default_endpoint_path("starting")); +} + +TEST_CASE("a socket file left by a plugin that was killed is passed over", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + leave_socket_file(default_endpoint_path("gone")); + REQUIRE(default_endpoint_paths() == std::vector{ default_endpoint_path("gone") }); + + // Alone, it is no plugin at all. + REQUIRE(CameraReader::list().empty()); + REQUIRE_THROWS_WITH(CameraReader::open("head"), ContainsSubstring("No camera plugin is running there")); + + // Next to a plugin that runs, it does not keep the reader from that one. + Served served("head", default_endpoint_path("synthetic")); + CameraReader reader = CameraReader::open("head"); + served.publish(0x10); + REQUIRE(reader.next(2s).has_value()); + REQUIRE(CameraReader::list().size() == 1); + REQUIRE_THROWS_WITH(CameraReader::open("wrist"), ContainsSubstring("'head' by '" + served.endpoint_path + "'") && + !ContainsSubstring(default_endpoint_path("gone"))); +} + +TEST_CASE("a plugin that does not answer keeps a reader from the others only briefly", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + // Connections to it are taken into its queue, and nobody answers them. + MessageListener silent(default_endpoint_path("silent")); + silent.listen(); + Served served("head", default_endpoint_path("synthetic")); + + // The reader gives it a short time to say what it publishes, then goes on to the next. + auto before = std::chrono::steady_clock::now(); + CameraReader reader = CameraReader::open("head"); + auto took = std::chrono::steady_clock::now() - before; + REQUIRE(took >= 400ms); + REQUIRE(took < 2s); // less than a plugin gets that a reader was pointed at + served.publish(0x10); + REQUIRE(reader.next(2s).has_value()); + + // Listing passes it over too, and gives what the others publish. + const std::vector groups = CameraReader::list(); + REQUIRE(groups.size() == 1); + REQUIRE(groups[0].endpoint == served.endpoint_path); + + // A reader that does not find its group is told which plugin gave no answer. + REQUIRE_THROWS_WITH(CameraReader::open("wrist"), + ContainsSubstring("'head' by '" + served.endpoint_path + "'") && + ContainsSubstring("Passed over: '" + default_endpoint_path("silent") + + "' (the camera plugin did not answer the reader's handshake)")); + + // Pointed at that plugin, a reader waits for it as long as for any plugin it is told to use. + before = std::chrono::steady_clock::now(); + REQUIRE_THROWS_WITH( + CameraReader::list({ .instance = "silent" }), ContainsSubstring("did not answer the reader's handshake")); + took = std::chrono::steady_clock::now() - before; + REQUIRE(took >= 1500ms); +} + +TEST_CASE("a plugin that has stopped accepting keeps no reader waiting", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + // A plugin that listens and never accepts: every connection stays in its queue, also after + // its reader has gone, and once the queue is full a connect to it is refused. + const std::string stuck = default_endpoint_path("stuck"); + MessageListener listener(stuck); + listener.listen(); + bool full = false; + for (int attempt = 0; attempt < 200000 && !full; ++attempt) + { + try + { + MessageSocket::connect(stuck); + } + catch (const std::system_error&) + { + full = true; + } + } + REQUIRE(full); + Served served("head", default_endpoint_path("synthetic")); + + // Asked first (its file name sorts first), it turns the reader away at once, so the reader + // does not even wait the short time it gives a plugin to say what it publishes. + const auto before = std::chrono::steady_clock::now(); + CameraReader reader = CameraReader::open("head"); + REQUIRE(std::chrono::steady_clock::now() - before < 400ms); + served.publish(0x10); + REQUIRE(reader.next(2s).has_value()); + + const std::vector groups = CameraReader::list(); + REQUIRE(groups.size() == 1); + REQUIRE(groups[0].endpoint == served.endpoint_path); + + // A reader that does not find its group is told that plugin was passed over. + REQUIRE_THROWS_WITH(CameraReader::open("wrist"), ContainsSubstring("Passed over: '" + stuck + "'")); +} + +TEST_CASE("a plugin that answers with anything but what it publishes does not keep a reader from the others", + "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + Served served("head", default_endpoint_path("synthetic")); + // Asked before the plugin that serves: its file name sorts first. + const std::string other = default_endpoint_path("other"); + MessageEncoder encoder; + const StandIn::Conversation refuse = [&](MessageSocket& reader, const HandshakeRequest&) + { reader.send(encoder.encode(HandshakeResponse{ .error = "not for this reader" })); }; + const StandIn::Conversation leave = [](MessageSocket&, const HandshakeRequest&) {}; + + // A stand-in takes one reader, so every time a reader looks there is a new one. + { + // A plugin that refuses the reader, as one of another protocol version does: the camera + // of the other plugin is found and read all the same. + StandIn plugin(refuse, other); + CameraReader reader = CameraReader::open("head"); + served.publish(0x10); + REQUIRE(reader.next(2s).has_value()); + } + { + // A reader that does not find its group is told of that plugin, in the plugin's words. + StandIn plugin(refuse, other); + REQUIRE_THROWS_WITH( + CameraReader::open("wrist"), ContainsSubstring("'head' by '" + served.endpoint_path + "'") && + ContainsSubstring("Passed over: '" + other + "' (not for this reader)")); + } + { + // A plugin that goes away while it is asked. + StandIn plugin(leave, other); + REQUIRE_THROWS_WITH(CameraReader::open("wrist"), + ContainsSubstring("Passed over: '" + other + "' (the camera plugin closed the connection)")); + } + { + // Listing leaves such a plugin out and gives what the others publish. + StandIn plugin(leave, other); + const std::vector groups = CameraReader::list(); + REQUIRE(groups.size() == 1); + REQUIRE(groups[0].endpoint == served.endpoint_path); + } +} + +TEST_CASE("a reader that is pointed at a plugin asks that plugin only", "[unit][camera_reader]") +{ + const EndpointDirectory endpoints; + Served head_plugin("head", default_endpoint_path("synthetic")); + Served wrist_plugin("wrist", default_endpoint_path("second")); + + // By its instance and by its socket path. + for (const PluginLocation& plugin : + { PluginLocation{ .instance = "synthetic" }, PluginLocation{ .endpoint = head_plugin.endpoint_path } }) + { + const std::vector groups = CameraReader::list(plugin); + REQUIRE(groups.size() == 1); + REQUIRE(groups[0].group.id == "head"); + REQUIRE(groups[0].endpoint == head_plugin.endpoint_path); + REQUIRE(CameraReader::open("head", {}, plugin).group().id == "head"); + + // A group of the other plugin is not looked for there: the answer is this plugin's own. + REQUIRE_THROWS_WITH(CameraReader::open("wrist", {}, plugin), + ContainsSubstring("this camera plugin publishes no group 'wrist'")); + } + + // A plugin that is not there is an error, also for listing; the others are not tried. + const PluginLocation absent{ .instance = "absent" }; + const std::string nobody = "no camera plugin answers on '" + default_endpoint_path("absent") + "'"; + REQUIRE_THROWS_WITH(CameraReader::open("head", {}, absent), ContainsSubstring(nobody)); + REQUIRE_THROWS_WITH(CameraReader::list(absent), ContainsSubstring(nobody)); + + REQUIRE_THROWS_WITH(CameraReader::list({ .endpoint = head_plugin.endpoint_path, .instance = "synthetic" }), + ContainsSubstring("by its endpoint or by its instance, not both")); +} + +TEST_CASE("a reader checks that the answer is what it asked for", "[unit][camera_reader][validation]") +{ + HostPool left(SLOTS, SLOT_BYTES, "test_left"); + HostPool right(SLOTS, SLOT_BYTES, "test_right"); + const std::array pools = { left.descriptor(), right.descriptor() }; + MessageEncoder encoder; + const auto answer_with = [&](HandshakeResponse response, std::span descriptors) + { + return [&encoder, response = std::move(response), descriptors](MessageSocket& reader) + { reader.send(encoder.encode(response), descriptors); }; + }; + const auto describing = [](const GroupDescription& group) + { + HandshakeResponse response; + response.groups.push_back({ group, {} }); + return response; + }; + + // The right answer, to show the stand-in is one a reader accepts. + REQUIRE_NOTHROW(open_against(answer_with(describing(stereo_group()), pools))); + + // Another group than the one asked for, or more than one. + REQUIRE_THROWS_WITH( + open_against(answer_with(describing(stereo_group("wrist")), pools)), ContainsSubstring("with group 'wrist'")); + HandshakeResponse two = describing(stereo_group()); + two.groups.push_back({ stereo_group("wrist"), {} }); + REQUIRE_THROWS_WITH(open_against(answer_with(two, pools)), ContainsSubstring("with 2 groups")); + REQUIRE_THROWS_WITH(open_against(answer_with(HandshakeResponse{}, {})), ContainsSubstring("with 0 groups")); + + // A protocol version this reader does not speak. + HandshakeResponse newer = describing(stereo_group()); + newer.protocol_version = PROTOCOL_VERSION + 1; + REQUIRE_THROWS_WITH(open_against(answer_with(newer, pools)), ContainsSubstring("answered in protocol version")); + HandshakeResponse older = describing(stereo_group()); + older.protocol_version = OLDEST_PROTOCOL_VERSION - 1; + REQUIRE_THROWS_WITH(open_against(answer_with(older, pools)), ContainsSubstring("answered in protocol version")); + + // A description a view could run off the end of a pool with: rows longer than the slot + // holds, a plane missing, sizes whose product wraps, a slot no pool can have. + GroupDescription short_rows = stereo_group(); + short_rows.streams[1].planes[0].stride_bytes = WIDTH - 1; + REQUIRE_THROWS_WITH( + open_against(answer_with(describing(short_rows), pools)), ContainsSubstring("in a way that cannot be read")); + GroupDescription tall = stereo_group(); + tall.streams[0].height = HEIGHT + 1; + REQUIRE_THROWS_WITH( + open_against(answer_with(describing(tall), pools)), ContainsSubstring("do not lie inside the slot")); + GroupDescription no_plane = stereo_group(); + no_plane.streams[0].plane_count = 0; + REQUIRE_THROWS_WITH(open_against(answer_with(describing(no_plane), pools)), ContainsSubstring("every plane")); + GroupDescription wraps = stereo_group(); + wraps.streams[0].width = 0xFFFFFFFF; + wraps.streams[0].height = 0xFFFFFFFF; + wraps.streams[0].planes[0] = { .offset_bytes = 0xFFFFFFFF, .stride_bytes = 0xFFFFFFFF }; + REQUIRE_THROWS_WITH( + open_against(answer_with(describing(wraps), pools)), ContainsSubstring("in a way that cannot be read")); + GroupDescription no_slots = stereo_group(); + no_slots.streams[0].slot_count = 0; + REQUIRE_THROWS_WITH(open_against(answer_with(describing(no_slots), pools)), ContainsSubstring("slots")); + + // Fewer pools than streams, and a pool smaller than its description says. + REQUIRE_THROWS_WITH(open_against(answer_with(describing(stereo_group()), std::span(pools).first(1))), + ContainsSubstring("one pool per stream")); + HostPool small(SLOTS / 2, SLOT_BYTES, "test_small"); + const std::array mismatched = { left.descriptor(), small.descriptor() }; + REQUIRE_THROWS_WITH( + open_against(answer_with(describing(stereo_group()), mismatched)), ContainsSubstring("cannot map the pools")); + + // A frame set before the answer. + REQUIRE_THROWS_WITH(open_against([&](MessageSocket& reader) { reader.send(encoder.encode(FrameEvent{})); }), + ContainsSubstring("before it answered")); + + // A refusal is passed on in the plugin's words. + REQUIRE_THROWS_WITH(open_against(answer_with(HandshakeResponse{ .error = "the pool is full" }, {})), + ContainsSubstring("the pool is full")); +} + +TEST_CASE("a group a newer plugin publishes in a form this build cannot read is listed and not opened", + "[unit][camera_reader][validation]") +{ + // What a newer plugin sends: one camera this build reads and one in a kind of memory added + // after it. The codec of this build never writes such a group, so it is built by hand. + const auto answer = [](MessageSocket& reader, const std::vector& ids, std::uint32_t memory_of_last) + { + flatbuffers::FlatBufferBuilder builder; + std::vector> groups; + for (std::size_t i = 0; i < ids.size(); ++i) + { + const core::CameraPlaneLayout plane(0, STRIDE); + const auto stream = core::CreateCameraStream( + builder, builder.CreateString(ids[i] + "/0"), core::CameraCodec_None, core::CameraPixelFormat_Gray8, + static_cast(i + 1 == ids.size() ? memory_of_last : core::CameraMemoryKind_Host), + WIDTH, HEIGHT, builder.CreateVectorOfStructs(&plane, 1), SLOT_BYTES, SLOTS, 60.0); + groups.push_back( + core::CreateCameraGroup(builder, builder.CreateString(ids[i]), builder.CreateVector(&stream, 1), 5)); + } + const auto response = + core::CreateCameraHandshakeResponse(builder, PROTOCOL_VERSION, 0, builder.CreateVector(groups)); + builder.Finish( + core::CreateCameraMessage(builder, core::CameraMessageBody_CameraHandshakeResponse, response.Union())); + HostPool pool(SLOTS, SLOT_BYTES, "test_pool"); + const std::array descriptors = { pool.descriptor() }; + reader.send(bytes_of(builder), ids.size() == 1 ? std::span(descriptors) : std::span()); + }; + const std::uint32_t newer_memory = core::CameraMemoryKind_MAX + 1; + + { + StandIn plugin( + [&](MessageSocket& reader, const HandshakeRequest& request) + { + if (request.group.empty()) + { + answer(reader, { "head", "wrist" }, newer_memory); + } + }); + const std::vector groups = CameraReader::list(plugin.location()); + REQUIRE(groups.size() == 2); + REQUIRE(groups[0].group.id == "head"); + REQUIRE(groups[0].unsupported.empty()); + REQUIRE(groups[0].group.streams.size() == 1); + REQUIRE(groups[1].group.id == "wrist"); + REQUIRE(groups[1].group.streams.empty()); + REQUIRE_THAT(groups[1].unsupported, ContainsSubstring("kind of memory this build does not know")); + } + { + StandIn plugin([&](MessageSocket& reader, const HandshakeRequest&) + { answer(reader, { "wrist" }, newer_memory); }); + REQUIRE_THROWS_WITH( + CameraReader::open("wrist", {}, plugin.location()), ContainsSubstring("cannot be read by this build")); + } + { + // The group it does read opens as usual. + StandIn plugin([&](MessageSocket& reader, const HandshakeRequest&) + { answer(reader, { "head" }, core::CameraMemoryKind_Host); }); + REQUIRE(CameraReader::open("head", {}, plugin.location()).group().streams.size() == 1); + } +} + +TEST_CASE("a reader is told when the plugin is gone", "[unit][camera_reader]") +{ + Served served; + CameraReader reader = CameraReader::open( + "head", { .window = 2, .order = DeliveryOrder::NEWEST }, { .endpoint = served.endpoint_path }); + served.publish(1); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + + served.endpoint.stop(); + REQUIRE_THROWS_AS(reader.next(2s), CameraDisconnected); + // What the reader holds is still mapped, and giving it back to nobody is harmless. + REQUIRE(first_pixel(*frames, 0) == std::byte{ 1 }); + frames.reset(); +} + +TEST_CASE("a reader copes with what a plugin may send", "[unit][camera_reader]") +{ + HostPool left(SLOTS, SLOT_BYTES, "test_left"); + HostPool right(SLOTS, SLOT_BYTES, "test_right"); + left.slot(2)[0] = std::byte{ 0x42 }; + + FrameEvent good; + good.lease_id = 7; + good.sequence = 1; + good.image_count = 2; + good.images[0].slot = 2; + FrameEvent bad_slot = good; + bad_slot.lease_id = 8; + bad_slot.sequence = 2; + bad_slot.images[1].slot = SLOTS; + + std::vector asked_after; + std::vector released; + { + StandIn plugin( + [&](MessageSocket& reader, const HandshakeRequest&) + { + MessageEncoder encoder; + std::vector buffer(MAX_MESSAGE_BYTES); + const auto receive = [&] { + return decode_message({ buffer.data(), reader.receive(buffer, 5s).bytes }); + }; + StandIn::answer(reader, left, right); + + // The reader asks; a newer plugin says something for information first, which is + // passed over, then sends the frame set. + asked_after.push_back(std::get(receive()).after_sequence); + flatbuffers::FlatBufferBuilder builder; + builder.Finish(core::CreateCameraMessage( + builder, static_cast(core::CameraMessageBody_MAX + 1), + core::CreateCameraRelease(builder, 1).Union(), true)); + reader.send(bytes_of(builder)); + reader.send(encoder.encode(good)); + released.push_back(std::get(receive()).lease_id); + + // A frame set that names a slot the pool does not have. + asked_after.push_back(std::get(receive()).after_sequence); + reader.send(encoder.encode(bad_slot)); + released.push_back(std::get(receive()).lease_id); + + // An operation this reader's build does not know. + asked_after.push_back(std::get(receive()).after_sequence); + builder.Clear(); + builder.Finish(core::CreateCameraMessage( + builder, static_cast(core::CameraMessageBody_MAX + 1), + core::CreateCameraRelease(builder, 1).Union(), false)); + reader.send(bytes_of(builder)); + (void)reader.receive(buffer, 5s); // until the reader goes + }); + + CameraReader reader = CameraReader::open("head", {}, plugin.location()); + std::optional frames = reader.next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->sequence() == 1); + REQUIRE(first_pixel(*frames, 0) == std::byte{ 0x42 }); + frames.reset(); + + // Refused and given back; the reader goes on. + REQUIRE_THROWS_WITH(reader.next(2s), ContainsSubstring("does not match group 'head'")); + // Not something to pass over: the reader says so instead of waiting on. + REQUIRE_THROWS_WITH(reader.next(2s), ContainsSubstring("cannot act on")); + } + REQUIRE(asked_after == std::vector{ 0, 1, 1 }); + REQUIRE(released == std::vector{ 7, 8 }); +} diff --git a/tests/cpp/core/camera_reader/test_cross_process.cpp b/tests/cpp/core/camera_reader/test_cross_process.cpp new file mode 100644 index 0000000000..e11800031b --- /dev/null +++ b/tests/cpp/core/camera_reader/test_cross_process.cpp @@ -0,0 +1,334 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The endpoint, the wire messages and the reader together, with the two ends in different +// processes: what a camera plugin and its readers are. + +#include "endpoint_directory.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +using namespace std::chrono_literals; +using Catch::Matchers::ContainsSubstring; + +namespace +{ + +class TemporaryDirectory +{ +public: + TemporaryDirectory() + { + std::string pattern = (std::filesystem::temp_directory_path() / "camera_peer_XXXXXX").string(); + REQUIRE(::mkdtemp(pattern.data()) != nullptr); + m_path = pattern; + } + + ~TemporaryDirectory() + { + std::error_code ignored; + std::filesystem::remove_all(m_path, ignored); + } + + std::string file(const std::string& name) const + { + return (m_path / name).string(); + } + +private: + std::filesystem::path m_path; +}; + +// camera_test_peer as a process of its own; killed when the test is done with it. +class Peer +{ +public: + explicit Peer(const std::vector& arguments) + { + std::vector owned = { CAMERA_TEST_PEER_EXECUTABLE }; + owned.insert(owned.end(), arguments.begin(), arguments.end()); + std::vector argv; + for (std::string& argument : owned) + { + argv.push_back(argument.data()); + } + argv.push_back(nullptr); + m_pid = ::fork(); + REQUIRE(m_pid >= 0); + if (m_pid == 0) + { + // A test process that dies without running ~Peer takes its peers with it. + ::prctl(PR_SET_PDEATHSIG, SIGKILL); + ::execv(argv[0], argv.data()); + ::_exit(127); + } + } + + ~Peer() + { + kill(); + } + + // Ends the process the way a crash does: no chance to clean up. + void kill() + { + if (m_pid > 0) + { + ::kill(m_pid, SIGKILL); + ::waitpid(m_pid, nullptr, 0); + m_pid = -1; + } + } + +private: + pid_t m_pid = -1; +}; + +bool wait_until(const std::function& condition, std::chrono::seconds timeout = 10s) +{ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (!condition()) + { + if (std::chrono::steady_clock::now() > deadline) + { + return false; + } + std::this_thread::sleep_for(5ms); + } + return true; +} + +// A peer that was just started needs a moment before it serves. +std::optional open_when_ready(const std::string& group, + const ReaderOptions& options = {}, + const PluginLocation& plugin = {}) +{ + std::optional reader; + wait_until( + [&] + { + try + { + reader = CameraReader::open(group, options, plugin); + } + catch (const CameraReaderError&) + { + } + return reader.has_value(); + }); + return reader; +} + +// Every byte of both images of frame set n is n modulo 256. +void require_whole(const FrameSet& frames) +{ + const auto expected = static_cast(frames.sequence() & 0xff); + for (std::size_t stream = 0; stream < frames.image_count(); ++stream) + { + const auto pixels = frames.image(stream).data; + REQUIRE(std::all_of(pixels.begin(), pixels.end(), [expected](std::byte value) { return value == expected; })); + } +} + +} // namespace + +TEST_CASE("a reader gets whole frame sets from a plugin in another process", "[integration][camera_reader]") +{ + TemporaryDirectory directory; + const std::string endpoint = directory.file("cameras.sock"); + Peer plugin({ "serve", endpoint }); + + std::optional reader = open_when_ready("head", {}, { .endpoint = endpoint }); + REQUIRE(reader.has_value()); + REQUIRE(reader->group().streams.size() == 2); + + std::uint64_t previous = 0; + for (int i = 0; i < 50; ++i) + { + const std::optional frames = reader->next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->image_count() == 2); + // The image is the frame set the event named, and all of it: the slot was not rewritten + // between the plugin sending the event and this process reading the pixels. + require_whole(*frames); + if (previous != 0) + { + REQUIRE(frames->sequence() == previous + 1 + frames->skipped()); + } + previous = frames->sequence(); + } + + // A frame set that is held stays whole while the plugin goes through its pool many times. + const std::optional held = reader->next(2s); + REQUIRE(held.has_value()); + std::this_thread::sleep_for(200ms); + require_whole(*held); +} + +TEST_CASE("a reader that was away is handed a fresh frame set by a plugin in another process", + "[integration][camera_reader][newest]") +{ + TemporaryDirectory directory; + const std::string endpoint = directory.file("cameras.sock"); + Peer plugin({ "serve", endpoint }); + std::optional reader = open_when_ready("head", {}, { .endpoint = endpoint }); + REQUIRE(reader.has_value()); + REQUIRE(reader->next(2s).has_value()); // taken and given back at once + + const auto now_ns = [] + { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count(); + }; + for (int round = 0; round < 5; ++round) + { + // Away for sixty frame periods of the peer, holding nothing. + const std::int64_t left_ns = now_ns(); + std::this_thread::sleep_for(300ms); + const std::optional frames = reader->next(2s); + REQUIRE(frames.has_value()); + + // A frame set held for the reader when it left would have been published before it left. + // This one was published well into the time it was away. Neither bound depends on how + // soon this process runs again after next() returns. + const std::int64_t published_ns = frames->timeline().published_ns; + REQUIRE(published_ns > left_ns + std::chrono::nanoseconds(150ms).count()); + REQUIRE(published_ns <= now_ns()); + REQUIRE(frames->skipped() > 0); + require_whole(*frames); + } +} + +TEST_CASE("a reader process that is killed gives back what it held", "[integration][camera_reader]") +{ + TemporaryDirectory directory; + const std::string endpoint = directory.file("cameras.sock"); + const std::string marker = directory.file("holding"); + Peer plugin({ "serve", endpoint }); + REQUIRE(open_when_ready("head", {}, { .endpoint = endpoint }).has_value()); + + // Another process reads in order with a window of three and keeps all three frame sets. + Peer holder({ "hold", endpoint, marker }); + REQUIRE(wait_until([&] { return std::filesystem::exists(marker); })); + + // While it lives, its share of the pool is taken: a reader that needs that room is refused. + const ReaderOptions wide{ .window = 2, .order = DeliveryOrder::NEWEST }; + REQUIRE_THROWS_WITH( + CameraReader::open("head", wide, { .endpoint = endpoint }), ContainsSubstring("does not fit the pool")); + + // Killed, it returns nothing itself. The plugin notices and takes everything back. + holder.kill(); + std::optional reader = open_when_ready("head", wide, { .endpoint = endpoint }); + REQUIRE(reader.has_value()); + for (int i = 0; i < 40; ++i) + { + const std::optional frames = reader->next(2s); + REQUIRE(frames.has_value()); + require_whole(*frames); + } +} + +TEST_CASE("a reader is told when the plugin's process is killed", "[integration][camera_reader]") +{ + TemporaryDirectory directory; + const std::string endpoint = directory.file("cameras.sock"); + Peer plugin({ "serve", endpoint }); + // A window of two, so that the reader can hold one frame set and still ask for another. + std::optional reader = + open_when_ready("head", { .window = 2, .order = DeliveryOrder::NEWEST }, { .endpoint = endpoint }); + REQUIRE(reader.has_value()); + std::optional held = reader->next(2s); + REQUIRE(held.has_value()); + const std::uint64_t sequence = held->sequence(); + + plugin.kill(); + REQUIRE_THROWS_AS( + [&] + { + for (int i = 0; i < 1000; ++i) + { + (void)reader->next(2s); + } + }(), + CameraDisconnected); + + // What the reader holds is still its to read: the memory outlives the process that made it. + REQUIRE(held->sequence() == sequence); + require_whole(*held); + held.reset(); // giving it back to nobody is harmless + + // The killed plugin left its socket file behind; nothing answers there. + REQUIRE_THROWS_WITH( + CameraReader::open("head", {}, { .endpoint = endpoint }), ContainsSubstring("no camera plugin answers on")); +} + +TEST_CASE("a reader finds each camera on the plugin process that publishes it", "[integration][camera_reader]") +{ + const EndpointDirectory endpoints; + Peer head_plugin({ "serve", default_endpoint_path("first"), "head" }); + Peer wrist_plugin({ "serve", default_endpoint_path("second"), "wrist" }); + + // The reader is told the name of each camera group and nothing about where it is served. + std::optional head = open_when_ready("head"); + std::optional wrist = open_when_ready("wrist"); + REQUIRE(head.has_value()); + REQUIRE(wrist.has_value()); + REQUIRE(head->group().streams[0].id == "head/left"); + REQUIRE(wrist->group().streams[0].id == "wrist/left"); + for (CameraReader* reader : { &*head, &*wrist }) + { + const std::optional frames = reader->next(2s); + REQUIRE(frames.has_value()); + REQUIRE(frames->image_count() == 2); + require_whole(*frames); + } + + const std::vector groups = CameraReader::list(); + REQUIRE(groups.size() == 2); + REQUIRE(groups[0].group.id == "head"); + REQUIRE(groups[0].endpoint == default_endpoint_path("first")); + REQUIRE(groups[1].group.id == "wrist"); + REQUIRE(groups[1].endpoint == default_endpoint_path("second")); + + // One plugin is killed and leaves its socket file behind. Its reader is told, and the other + // camera is found and read as before. + head_plugin.kill(); + REQUIRE(std::filesystem::exists(default_endpoint_path("first"))); + REQUIRE_THROWS_AS( + [&] + { + for (int i = 0; i < 1000; ++i) + { + (void)head->next(2s); + } + }(), + CameraDisconnected); + REQUIRE_THROWS_WITH( + CameraReader::open("head"), ContainsSubstring("publishes group 'head'") && + ContainsSubstring("'wrist' by '" + default_endpoint_path("second") + "'")); + CameraReader again = CameraReader::open("wrist"); + const std::optional frames = again.next(2s); + REQUIRE(frames.has_value()); + require_whole(*frames); +}