diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 602cc1969a..7851685616 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -25,6 +25,7 @@ add_subdirectory(camera_core) # Camera frames and messages between processes (shared host memory, descriptor passing) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) + add_subdirectory(camera_wire) endif() # Build OXR library (depends on oxr_utils for types) diff --git a/src/core/camera_wire/AGENTS.md b/src/core/camera_wire/AGENTS.md new file mode 100644 index 0000000000..a7596019c1 --- /dev/null +++ b/src/core/camera_wire/AGENTS.md @@ -0,0 +1,16 @@ + + +# Agent notes — `camera_wire` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) before changing `fbs/camera_wire.fbs`: a reader and a plugin from different builds must keep understanding each other. + +- **The schema only grows.** Append fields with fresh ids, optional; append union members; tables, never structs. Do not reuse or renumber. +- **FlatBuffers stays inside this module.** Its users see `camera_core` types; only the codec and its tests include the generated header. +- **Decoded bytes are untrusted.** Verify, then bound every count against the fixed arrays it fills, and check for a body behind a body type. +- **A new enum value needs both sides.** The core enum and the schema enum carry the same numbers (pinned by `static_assert`), and the decoder must accept the value before a plugin sends it. +- **The frame path does not allocate.** Keep `encode(const FrameEvent&)` free of containers that do. diff --git a/src/core/camera_wire/CMakeLists.txt b/src/core/camera_wire/CMakeLists.txt new file mode 100644 index 0000000000..e162dea055 --- /dev/null +++ b/src/core/camera_wire/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(fbs) +add_subdirectory(cpp) diff --git a/src/core/camera_wire/README.md b/src/core/camera_wire/README.md new file mode 100644 index 0000000000..20effdcb4b --- /dev/null +++ b/src/core/camera_wire/README.md @@ -0,0 +1,58 @@ + + +# Camera interface: messages between the plugin and a reader + +What the camera plugin and a reader say to each other over their socket +([`camera_ipc`](../camera_ipc)). Each socket message is one of these: + +```text +reader plugin + | Hello (protocol version) | + |------------------------------------>| + | Welcome (what is published) | + |<------------------------------------| + | Subscribe (group, window, order) | + |------------------------------------>| + | Subscribed + one pool descriptor | + | per stream | + |<------------------------------------| + | Frame (lease, slot per image) | once per frame set + |<------------------------------------| + | Release (lease) | + |------------------------------------>| +``` + +A connection carries one subscription; a reader that wants two groups opens two. Closing the +connection ends the subscription and returns every frame set the reader still held. A `Frame` +can overtake `Subscribed`, because capture does not wait for the answer to be sent; it belongs +to the subscription all the same. + +## Schema + +[`fbs/camera_wire.fbs`](fbs/camera_wire.fbs) defines the messages as FlatBuffers tables. It is +kept here and not in [`schema`](../schema): that directory holds what goes into recordings, +and these messages are never recorded. + +A reader and a plugin may come from different builds, so the schema only grows: + +- Append fields with fresh ids and leave them optional; append union members. Never reuse or + renumber an id, and use tables, not structs. +- A message type this build does not know decodes to nothing and is ignored. +- A stream whose format or kind of memory this build does not know makes its group + unreadable, and the reader is told so; other groups are unaffected. +- `WIRE_PROTOCOL_VERSION` changes only when the two sides would misread each other. The + plugin refuses a reader with a different version in `Welcome`. + +## Codec + +`WireEncoder` and `decode_wire_message` turn [`camera_core`](../camera_core) types into bytes +and back, so nothing outside this module sees FlatBuffers. The encoder reuses its buffer; a +frame event is encoded without allocating. The decoder treats its input as coming from another +process: it verifies the buffer and refuses counts that would not fit a group. + +## Tests + +`tests/cpp/core/camera_wire`, run with `ctest -L camera_wire`. They are tagged `unit`. diff --git a/src/core/camera_wire/cpp/CMakeLists.txt b/src/core/camera_wire/cpp/CMakeLists.txt new file mode 100644 index 0000000000..a4d4122dbb --- /dev/null +++ b/src/core/camera_wire/cpp/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Turns the messages between the camera plugin and its readers into bytes and back. The +# FlatBuffers types stay inside this library; its users see camera_core types. +add_library(camera_wire STATIC + codec.cpp +) + +target_include_directories(camera_wire INTERFACE + $ +) + +target_link_libraries(camera_wire + PUBLIC + camera::camera_core + PRIVATE + camera::camera_wire_schema +) + +add_library(camera::camera_wire ALIAS camera_wire) diff --git a/src/core/camera_wire/cpp/codec.cpp b/src/core/camera_wire/cpp/codec.cpp new file mode 100644 index 0000000000..d0856fb521 --- /dev/null +++ b/src/core/camera_wire/cpp/codec.cpp @@ -0,0 +1,341 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_wire/messages.hpp" + +#include + +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// The schema's enums carry the same numbers as the core's, so a value converts by a cast. +static_assert(static_cast(PixelFormat::BGRA8) == wire::PixelFormat_Bgra8); +static_assert(static_cast(PixelFormat::RGBA8) == wire::PixelFormat_Rgba8); +static_assert(static_cast(PixelFormat::NV12) == wire::PixelFormat_Nv12); +static_assert(static_cast(PixelFormat::GRAY8) == wire::PixelFormat_Gray8); +static_assert(static_cast(Codec::NONE) == wire::Codec_None); +static_assert(static_cast(Codec::H264) == wire::Codec_H264); +static_assert(static_cast(Codec::H265) == wire::Codec_H265); +static_assert(static_cast(MemoryKind::HOST) == wire::MemoryKind_Host); +static_assert(static_cast(DeliveryOrder::DEFAULT) == wire::DeliveryOrder_Default); +static_assert(static_cast(DeliveryOrder::NEWEST) == wire::DeliveryOrder_Newest); +static_assert(static_cast(DeliveryOrder::IN_ORDER) == wire::DeliveryOrder_InOrder); + +using Builder = flatbuffers::FlatBufferBuilder; + +flatbuffers::Offset build_group(Builder& builder, const GroupDescription& group) +{ + std::vector> streams; + streams.reserve(group.streams.size()); + for (const StreamDescription& stream : group.streams) + { + std::array, MAX_PLANES_PER_IMAGE> planes; + const std::size_t plane_count = std::min(stream.plane_count, MAX_PLANES_PER_IMAGE); + for (std::size_t i = 0; i < plane_count; ++i) + { + planes[i] = wire::CreatePlaneLayout(builder, stream.planes[i].offset_bytes, stream.planes[i].stride_bytes); + } + const auto id = builder.CreateString(stream.id); + const auto plane_vector = builder.CreateVector(planes.data(), plane_count); + streams.push_back(wire::CreateStream(builder, id, static_cast(stream.codec), + static_cast(stream.pixel_format), + static_cast(stream.memory), stream.width, stream.height, + plane_vector, stream.slot_bytes, stream.slot_count, stream.frame_rate)); + } + const auto id = builder.CreateString(group.id); + const auto stream_vector = builder.CreateVector(streams); + return wire::CreateGroup(builder, id, stream_vector); +} + +std::string text_of(const flatbuffers::String* string) +{ + return string != nullptr ? string->str() : std::string(); +} + +std::optional pixel_format_of(wire::PixelFormat format) +{ + switch (format) + { + case wire::PixelFormat_Bgra8: + case wire::PixelFormat_Rgba8: + case wire::PixelFormat_Nv12: + case wire::PixelFormat_Gray8: + return static_cast(format); + default: + return std::nullopt; + } +} + +std::optional codec_of(wire::Codec codec) +{ + switch (codec) + { + case wire::Codec_None: + case wire::Codec_H264: + case wire::Codec_H265: + return static_cast(codec); + default: + return std::nullopt; + } +} + +std::optional order_of(wire::DeliveryOrder order) +{ + switch (order) + { + case wire::DeliveryOrder_Default: + case wire::DeliveryOrder_Newest: + case wire::DeliveryOrder_InOrder: + return static_cast(order); + default: + return std::nullopt; + } +} + +GroupDescription read_group(const wire::Group* group) +{ + GroupDescription result; + if (group == nullptr) + { + return result; + } + result.id = text_of(group->id()); + const auto* streams = group->streams(); + if (streams == nullptr) + { + return result; + } + if (streams->size() > MAX_STREAMS_PER_GROUP) + { + throw WireError("camera group '" + result.id + "' has more streams than a group can have"); + } + for (const wire::Stream* stream : *streams) + { + StreamDescription description; + description.id = text_of(stream->id()); + const auto codec = codec_of(stream->codec()); + const auto pixel_format = pixel_format_of(stream->pixel_format()); + // An encoded stream has no pixel format to know. + if (!codec || stream->memory() != wire::MemoryKind_Host || (*codec == Codec::NONE && !pixel_format)) + { + throw WireError("camera stream '" + description.id + + "' uses a format or a kind of memory this build does not know"); + } + description.codec = *codec; + description.pixel_format = pixel_format.value_or(PixelFormat::BGRA8); + description.memory = MemoryKind::HOST; + description.width = stream->width(); + description.height = stream->height(); + if (const auto* planes = stream->planes()) + { + if (planes->size() > MAX_PLANES_PER_IMAGE) + { + throw WireError("camera stream '" + description.id + "' has more planes than an image can have"); + } + description.plane_count = planes->size(); + for (std::uint32_t i = 0; i < planes->size(); ++i) + { + description.planes[i] = { planes->Get(i)->offset_bytes(), planes->Get(i)->stride_bytes() }; + } + } + description.slot_bytes = stream->slot_bytes(); + description.slot_count = stream->slot_count(); + description.frame_rate = stream->frame_rate(); + result.streams.push_back(std::move(description)); + } + return result; +} + +FrameEvent read_frame(const wire::Frame& frame) +{ + FrameEvent event; + event.lease_id = frame.lease_id(); + event.sequence = frame.sequence(); + event.generation = frame.generation(); + event.skipped = frame.skipped(); + if (const wire::Timeline* timeline = frame.timeline()) + { + event.timeline = { timeline->capture_ns(), timeline->sdk_ready_ns(), timeline->published_ns(), + timeline->sent_ns() }; + } + if (const auto* images = frame.images()) + { + if (images->size() > MAX_STREAMS_PER_GROUP) + { + throw WireError("a camera frame set has more images than a group has streams"); + } + event.image_count = images->size(); + for (std::uint32_t i = 0; i < images->size(); ++i) + { + const wire::Image* image = images->Get(i); + event.images[i] = { image->slot(), image->flags(), image->bytes_used(), image->device_frame_id(), + image->device_time_ns() }; + } + } + return event; +} + +} // namespace + +struct WireEncoder::Impl +{ + Builder builder{ 1024 }; + + template + std::span finish(wire::Body type, flatbuffers::Offset body) + { + builder.Finish(wire::CreateMessage(builder, type, body.Union())); + if (builder.GetSize() > MAX_WIRE_MESSAGE_BYTES) + { + throw std::length_error("a camera wire message of " + std::to_string(builder.GetSize()) + + " bytes is over the limit of " + std::to_string(MAX_WIRE_MESSAGE_BYTES)); + } + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; + } +}; + +WireEncoder::WireEncoder() : m_impl(std::make_unique()) +{ +} + +WireEncoder::~WireEncoder() = default; + +std::span WireEncoder::encode(const HelloMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto name = builder.CreateString(message.reader_name); + return m_impl->finish(wire::Body_Hello, wire::CreateHello(builder, message.protocol_version, name)); +} + +std::span WireEncoder::encode(const WelcomeMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::vector> groups; + groups.reserve(message.groups.size()); + for (const GroupDescription& group : message.groups) + { + groups.push_back(build_group(builder, group)); + } + const auto group_vector = builder.CreateVector(groups); + const auto error = builder.CreateString(message.error); + return m_impl->finish( + wire::Body_Welcome, wire::CreateWelcome(builder, message.protocol_version, group_vector, error)); +} + +std::span WireEncoder::encode(const SubscribeMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto group = builder.CreateString(message.group); + return m_impl->finish( + wire::Body_Subscribe, wire::CreateSubscribe(builder, group, message.options.window, + static_cast(message.options.order))); +} + +std::span WireEncoder::encode(const SubscribedMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto error = builder.CreateString(message.error); + const auto group = build_group(builder, message.group); + return m_impl->finish(wire::Body_Subscribed, wire::CreateSubscribed(builder, error, group)); +} + +// The per-frame path: nothing here allocates once the builder has grown to a frame's size. +std::span WireEncoder::encode(const FrameEvent& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::array, MAX_STREAMS_PER_GROUP> images; + const std::size_t image_count = std::min(message.image_count, MAX_STREAMS_PER_GROUP); + for (std::size_t i = 0; i < image_count; ++i) + { + const ImageInfo& image = message.images[i]; + images[i] = wire::CreateImage( + builder, image.slot, image.flags, image.bytes_used, image.device_frame_id, image.device_time_ns); + } + const auto image_vector = builder.CreateVector(images.data(), image_count); + const auto timeline = wire::CreateTimeline(builder, message.timeline.capture_ns, message.timeline.sdk_ready_ns, + message.timeline.published_ns, message.timeline.sent_ns); + return m_impl->finish( + wire::Body_Frame, wire::CreateFrame(builder, message.lease_id, message.sequence, message.generation, + message.skipped, timeline, image_vector)); +} + +std::span WireEncoder::encode(const ReleaseMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + return m_impl->finish(wire::Body_Release, wire::CreateRelease(builder, message.lease_id)); +} + +WireMessage decode_wire_message(std::span bytes) +{ + const auto* data = reinterpret_cast(bytes.data()); + flatbuffers::Verifier verifier(data, bytes.size()); + if (bytes.empty() || !wire::VerifyMessageBuffer(verifier)) + { + throw WireError("received bytes that are not a camera wire message"); + } + const wire::Message* message = wire::GetMessage(data); + // The verifier accepts a body type without a body. + if (message->body_type() != wire::Body_NONE && message->body() == nullptr) + { + throw WireError("received a camera wire message without a body"); + } + switch (message->body_type()) + { + case wire::Body_Hello: + { + const wire::Hello* hello = message->body_as_Hello(); + return HelloMessage{ hello->protocol_version(), text_of(hello->reader_name()) }; + } + case wire::Body_Welcome: + { + const wire::Welcome* welcome = message->body_as_Welcome(); + WelcomeMessage result; + result.protocol_version = welcome->protocol_version(); + result.error = text_of(welcome->error()); + if (const auto* groups = welcome->groups()) + { + for (const wire::Group* group : *groups) + { + result.groups.push_back(read_group(group)); + } + } + return result; + } + case wire::Body_Subscribe: + { + const wire::Subscribe* subscribe = message->body_as_Subscribe(); + const auto order = order_of(subscribe->order()); + if (!order) + { + throw WireError("a camera reader asked for a delivery order this build does not know"); + } + return SubscribeMessage{ text_of(subscribe->group()), { subscribe->window(), *order } }; + } + case wire::Body_Subscribed: + { + const wire::Subscribed* subscribed = message->body_as_Subscribed(); + return SubscribedMessage{ text_of(subscribed->error()), read_group(subscribed->group()) }; + } + case wire::Body_Frame: + return read_frame(*message->body_as_Frame()); + case wire::Body_Release: + return ReleaseMessage{ message->body_as_Release()->lease_id() }; + default: + return std::monostate{}; + } +} + +} // namespace core::camera diff --git a/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp b/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp new file mode 100644 index 0000000000..e2626c10ee --- /dev/null +++ b/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// Changes only when a reader and a plugin of different versions could misread each other. +// A field or a message added to the schema does not change it. +inline constexpr std::uint32_t WIRE_PROTOCOL_VERSION = 1; + +// No message is larger than this, so a receive buffer of this size always fits one. +inline constexpr std::size_t MAX_WIRE_MESSAGE_BYTES = 64 * 1024; + +// Reader to plugin, first message on a connection. +struct HelloMessage +{ + std::uint32_t protocol_version = WIRE_PROTOCOL_VERSION; + std::string reader_name; // for the plugin's log +}; + +// Plugin to reader, in answer to HelloMessage. +struct WelcomeMessage +{ + std::uint32_t protocol_version = WIRE_PROTOCOL_VERSION; + std::vector groups; // what the plugin publishes + std::string error; // why the plugin refuses the reader, if it does +}; + +// Reader to plugin: start sending a group's frame sets. One subscription per connection. +struct SubscribeMessage +{ + std::string group; + ReaderOptions options; +}; + +// Plugin to reader, in answer to SubscribeMessage. On success one file descriptor per stream +// travels with it, in stream order, for the stream's pool. +struct SubscribedMessage +{ + std::string error; // why the subscription was refused, if it was + GroupDescription group; +}; + +// Reader to plugin: done with a frame set. +struct ReleaseMessage +{ + std::uint64_t lease_id = 0; +}; + +// What decode_wire_message returns. A frame set travels as a FrameEvent. std::monostate is a +// message from a newer peer that this build does not know; ignore it. +using WireMessage = + std::variant; + +/** + * @brief Bytes that are not a message, or a message this build cannot represent. + */ +class WireError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +/** + * @brief Turns messages into bytes. It reuses one buffer, so after the first few messages + * encoding allocates nothing. Not thread-safe. + */ +class WireEncoder +{ +public: + WireEncoder(); + ~WireEncoder(); + + WireEncoder(const WireEncoder&) = delete; + WireEncoder& operator=(const WireEncoder&) = delete; + + /** + * @return The message's bytes, valid until the next call on this encoder. + * @throws std::length_error if the message is larger than MAX_WIRE_MESSAGE_BYTES. + */ + std::span encode(const HelloMessage& message); + std::span encode(const WelcomeMessage& message); + std::span encode(const SubscribeMessage& message); + std::span encode(const SubscribedMessage& message); + std::span encode(const FrameEvent& message); + std::span encode(const ReleaseMessage& message); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +/** + * @brief Read one message. The bytes come from another process and are checked in full. + * @throws WireError if they are not a message, or name more streams, planes or images than a + * group can have, or a format this build does not know. + */ +WireMessage decode_wire_message(std::span bytes); + +} // namespace core::camera diff --git a/src/core/camera_wire/fbs/CMakeLists.txt b/src/core/camera_wire/fbs/CMakeLists.txt new file mode 100644 index 0000000000..02239af735 --- /dev/null +++ b/src/core/camera_wire/fbs/CMakeLists.txt @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# The messages between the camera plugin and its readers, as a FlatBuffer schema. The generated +# header is reached as ; only the codec and its +# tests include it. + +include(${CMAKE_SOURCE_DIR}/cmake/GenerateFlatBuffers.cmake) + +generate_flatbuffer_headers( + CAMERA_WIRE_GENERATED_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/generated/camera_wire_schema + CAMERA_WIRE_GENERATED_BFBS +) + +add_custom_target(camera_wire_schema_generation + DEPENDS ${CAMERA_WIRE_GENERATED_HEADERS} ${CAMERA_WIRE_GENERATED_BFBS} +) + +add_library(camera_wire_schema INTERFACE) +add_dependencies(camera_wire_schema camera_wire_schema_generation) + +target_include_directories(camera_wire_schema INTERFACE + $ +) + +target_link_libraries(camera_wire_schema INTERFACE + flatbuffers +) + +add_library(camera::camera_wire_schema ALIAS camera_wire_schema) diff --git a/src/core/camera_wire/fbs/camera_wire.fbs b/src/core/camera_wire/fbs/camera_wire.fbs new file mode 100644 index 0000000000..5f6bc1dd5c --- /dev/null +++ b/src/core/camera_wire/fbs/camera_wire.fbs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The messages between the camera plugin and a reader, one per socket message. A reader and a +// plugin may be built at different times, so this file only grows: append fields with fresh +// ids and leave them optional, append union members, never reuse or renumber. See README.md. + +namespace core.camera.wire; + +// Zero means "a value this build does not know" for the kinds a newer peer may extend. +enum PixelFormat : uint32 { Unknown = 0, Bgra8 = 1, Rgba8 = 2, Nv12 = 3, Gray8 = 4 } +enum Codec : uint32 { None = 0, H264 = 1, H265 = 2 } +enum MemoryKind : uint32 { Unknown = 0, Host = 1 } +enum DeliveryOrder : uint32 { Default = 0, Newest = 1, InOrder = 2 } + +// Where one plane of an image sits inside a slot. +table PlaneLayout { + offset_bytes: uint32 (id: 0); + stride_bytes: uint32 (id: 1); +} + +// One sensor's images or one encoded video stream, and the pool its frames are in. +table Stream { + id: string (id: 0); + codec: Codec (id: 1); + pixel_format: PixelFormat (id: 2); + memory: MemoryKind (id: 3); + width: uint32 (id: 4); + height: uint32 (id: 5); + planes: [PlaneLayout] (id: 6); + slot_bytes: uint64 (id: 7); + slot_count: uint32 (id: 8); + frame_rate: float64 (id: 9); +} + +// Streams that are captured together and published as one frame set. +table Group { + id: string (id: 0); + streams: [Stream] (id: 1); +} + +// Reader to plugin, first message on a connection. +table Hello { + protocol_version: uint32 (id: 0); + reader_name: string (id: 1); +} + +// Plugin to reader, in answer to Hello. `error` is set when the plugin refuses the reader. +table Welcome { + protocol_version: uint32 (id: 0); + groups: [Group] (id: 1); + error: string (id: 2); +} + +// Reader to plugin: start sending this group's frame sets. One subscription per connection. +table Subscribe { + group: string (id: 0); + window: uint32 = 1 (id: 1); + order: DeliveryOrder (id: 2); +} + +// Plugin to reader, in answer to Subscribe. On success the message carries one file descriptor +// per stream of the group, in stream order, for the stream's pool. `error` is set otherwise. +table Subscribed { + error: string (id: 0); + group: Group (id: 1); +} + +// One image of a frame set: the slot it is in and what the camera SDK said about it. +table Image { + slot: uint32 (id: 0); + flags: uint32 (id: 1); + bytes_used: uint64 (id: 2); + device_frame_id: uint64 (id: 3); + device_time_ns: int64 = -1 (id: 4); +} + +// Host monotonic times in nanoseconds; -1 where a time is not known. +table Timeline { + capture_ns: int64 = -1 (id: 0); + sdk_ready_ns: int64 = -1 (id: 1); + published_ns: int64 = -1 (id: 2); + sent_ns: int64 = -1 (id: 3); +} + +// Plugin to reader: one frame set, held for the reader until it sends Release with the same +// lease. Frames may arrive before Subscribed; they belong to the subscription all the same. +table Frame { + lease_id: uint64 (id: 0); + sequence: uint64 (id: 1); + generation: uint32 (id: 2); + skipped: uint32 (id: 3); + timeline: Timeline (id: 4); + images: [Image] (id: 5); +} + +// Reader to plugin: done with a frame set. +table Release { + lease_id: uint64 (id: 0); +} + +union Body { Hello, Welcome, Subscribe, Subscribed, Frame, Release } + +table Message { + body: Body (id: 1); +} + +root_type Message; diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 705bead328..35bbcad055 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -17,6 +17,7 @@ add_subdirectory(camera_core) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) + add_subdirectory(camera_wire) endif() if(BUILD_OVXR_CAMERA_QUADS) diff --git a/tests/cpp/core/camera_wire/CMakeLists.txt b/tests/cpp/core/camera_wire/CMakeLists.txt new file mode 100644 index 0000000000..69b9c4a7b1 --- /dev/null +++ b/tests/cpp/core/camera_wire/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_wire_tests + test_codec.cpp +) + +# The schema is linked to build messages the encoder never would: ones from a newer peer and +# malformed ones. +target_link_libraries(camera_wire_tests PRIVATE + camera::camera_wire + camera::camera_wire_schema + Catch2::Catch2WithMain +) + +message(STATUS "camera_wire_tests target enabled with Catch2") +catch_discover_tests(camera_wire_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_wire/test_codec.cpp b/tests/cpp/core/camera_wire/test_codec.cpp new file mode 100644 index 0000000000..e3b33b516f --- /dev/null +++ b/tests/cpp/core/camera_wire/test_codec.cpp @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +namespace wire = core::camera::wire; + +namespace +{ + +GroupDescription stereo_group() +{ + GroupDescription group; + group.id = "head"; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = std::string("head/") + eye; + stream.pixel_format = PixelFormat::NV12; + stream.width = 1920; + stream.height = 1080; + stream.plane_count = 2; + stream.planes[0] = { .offset_bytes = 0, .stride_bytes = 2048 }; + stream.planes[1] = { .offset_bytes = 2048 * 1080, .stride_bytes = 2048 }; + stream.slot_bytes = 2048 * 1620; + stream.slot_count = 8; + stream.frame_rate = 59.94; + group.streams.push_back(stream); + } + return group; +} + +GroupDescription encoded_group() +{ + GroupDescription group; + group.id = "head.h265"; + StreamDescription stream; + stream.id = "head/left.h265"; + stream.codec = Codec::H265; + stream.width = 1920; + stream.height = 1080; + stream.slot_bytes = 1 << 20; + stream.slot_count = 16; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + return group; +} + +void require_same(const GroupDescription& actual, const GroupDescription& expected) +{ + REQUIRE(actual.id == expected.id); + REQUIRE(actual.streams.size() == expected.streams.size()); + for (std::size_t i = 0; i < expected.streams.size(); ++i) + { + const StreamDescription& a = actual.streams[i]; + const StreamDescription& e = expected.streams[i]; + REQUIRE(a.id == e.id); + REQUIRE(a.codec == e.codec); + REQUIRE(a.memory == e.memory); + REQUIRE(a.width == e.width); + REQUIRE(a.height == e.height); + REQUIRE(a.plane_count == e.plane_count); + for (std::uint32_t plane = 0; plane < e.plane_count; ++plane) + { + REQUIRE(a.planes[plane].offset_bytes == e.planes[plane].offset_bytes); + REQUIRE(a.planes[plane].stride_bytes == e.planes[plane].stride_bytes); + } + REQUIRE(a.slot_bytes == e.slot_bytes); + REQUIRE(a.slot_count == e.slot_count); + REQUIRE(a.frame_rate == e.frame_rate); + if (!e.is_encoded()) + { + REQUIRE(a.pixel_format == e.pixel_format); + } + } +} + +// What the other process receives: a copy of the bytes, decoded. +template +Message round_trip(WireEncoder& encoder, const Message& message) +{ + const std::span bytes = encoder.encode(message); + const std::vector received(bytes.begin(), bytes.end()); + WireMessage decoded = decode_wire_message(received); + REQUIRE(std::holds_alternative(decoded)); + return std::get(std::move(decoded)); +} + +std::span bytes_of(const flatbuffers::FlatBufferBuilder& builder) +{ + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; +} + +} // namespace + +TEST_CASE("the handshake messages survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + const HelloMessage hello = round_trip(encoder, HelloMessage{ .protocol_version = 7, .reader_name = "viewer" }); + REQUIRE(hello.protocol_version == 7); + REQUIRE(hello.reader_name == "viewer"); + + WelcomeMessage welcome; + welcome.groups = { stereo_group(), encoded_group() }; + const WelcomeMessage received = round_trip(encoder, welcome); + REQUIRE(received.protocol_version == WIRE_PROTOCOL_VERSION); + REQUIRE(received.error.empty()); + REQUIRE(received.groups.size() == 2); + require_same(received.groups[0], stereo_group()); + require_same(received.groups[1], encoded_group()); + + const WelcomeMessage refused = round_trip(encoder, WelcomeMessage{ .error = "protocol version 7 is not supported" }); + REQUIRE(refused.error == "protocol version 7 is not supported"); + REQUIRE(refused.groups.empty()); +} + +TEST_CASE("a subscription and its answer survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + const SubscribeMessage subscribe = + round_trip(encoder, SubscribeMessage{ "head", { .window = 4, .order = DeliveryOrder::IN_ORDER } }); + REQUIRE(subscribe.group == "head"); + REQUIRE(subscribe.options.window == 4); + REQUIRE(subscribe.options.order == DeliveryOrder::IN_ORDER); + + // What a reader that sets nothing asks for. + const SubscribeMessage defaults = round_trip(encoder, SubscribeMessage{ "head", {} }); + REQUIRE(defaults.options.window == 1); + REQUIRE(defaults.options.order == DeliveryOrder::DEFAULT); + + const SubscribedMessage accepted = round_trip(encoder, SubscribedMessage{ "", stereo_group() }); + REQUIRE(accepted.error.empty()); + require_same(accepted.group, stereo_group()); + + const SubscribedMessage refused = round_trip(encoder, SubscribedMessage{ "does not fit the pool", {} }); + REQUIRE(refused.error == "does not fit the pool"); + REQUIRE(refused.group.streams.empty()); +} + +TEST_CASE("a frame set and its release survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + FrameEvent event; + event.lease_id = 0x1122334455667788; + event.sequence = 123456789012; + event.generation = 3; + event.skipped = 2; + event.image_count = 2; + event.timeline = { .capture_ns = 1000, .sdk_ready_ns = 2000, .published_ns = 3000, .sent_ns = 4000 }; + event.images[0] = { + .slot = 5, .flags = IMAGE_FLAG_SDK_WARNING, .bytes_used = 0, .device_frame_id = 77, .device_time_ns = 99 + }; + event.images[1] = { .slot = 6, .flags = IMAGE_FLAG_KEYFRAME, .bytes_used = 4096, .device_frame_id = 78 }; + + const FrameEvent received = round_trip(encoder, event); + REQUIRE(received.lease_id == event.lease_id); + REQUIRE(received.sequence == event.sequence); + REQUIRE(received.generation == 3); + REQUIRE(received.skipped == 2); + REQUIRE(received.image_count == 2); + REQUIRE(received.timeline.capture_ns == 1000); + REQUIRE(received.timeline.sdk_ready_ns == 2000); + REQUIRE(received.timeline.published_ns == 3000); + REQUIRE(received.timeline.sent_ns == 4000); + REQUIRE(received.images[0].slot == 5); + REQUIRE(received.images[0].flags == IMAGE_FLAG_SDK_WARNING); + REQUIRE(received.images[0].device_frame_id == 77); + REQUIRE(received.images[0].device_time_ns == 99); + REQUIRE(received.images[1].slot == 6); + REQUIRE(received.images[1].flags == IMAGE_FLAG_KEYFRAME); + REQUIRE(received.images[1].bytes_used == 4096); + REQUIRE(received.images[1].device_time_ns == TIME_UNKNOWN); + + // Times that are not known stay unknown. + const FrameEvent bare = round_trip(encoder, FrameEvent{}); + REQUIRE(bare.image_count == 0); + REQUIRE(bare.timeline.capture_ns == TIME_UNKNOWN); + REQUIRE(bare.timeline.sent_ns == TIME_UNKNOWN); + + REQUIRE(round_trip(encoder, ReleaseMessage{ 42 }).lease_id == 42); +} + +TEST_CASE("an encoder is reused from message to message", "[unit][camera_wire]") +{ + WireEncoder encoder; + FrameEvent event; + event.image_count = 2; + for (std::uint64_t sequence = 1; sequence <= 100; ++sequence) + { + event.sequence = sequence; + REQUIRE(round_trip(encoder, event).sequence == sequence); + } + // A small message after a large one is not padded to the large one's size. + WelcomeMessage welcome; + welcome.groups = { stereo_group(), encoded_group() }; + const std::size_t large = encoder.encode(welcome).size(); + REQUIRE(encoder.encode(ReleaseMessage{ 1 }).size() < large); +} + +TEST_CASE("bytes that are not a message are rejected", "[unit][camera_wire]") +{ + REQUIRE_THROWS_AS(decode_wire_message({}), WireError); + + const std::vector garbage(64, std::byte{ 0xa5 }); + REQUIRE_THROWS_AS(decode_wire_message(garbage), WireError); + + WireEncoder encoder; + WelcomeMessage welcome; + welcome.groups = { stereo_group() }; + const std::span bytes = encoder.encode(welcome); + const std::vector whole(bytes.begin(), bytes.end()); + REQUIRE_THROWS_AS(decode_wire_message({ whole.data(), whole.size() / 2 }), WireError); + + // No cut of a message may be read out of bounds; each is either refused or read in full. + for (std::size_t length = 0; length < whole.size(); ++length) + { + const std::vector cut(whole.begin(), whole.begin() + static_cast(length)); + try + { + (void)decode_wire_message(cut); + } + catch (const WireError&) + { + } + } +} + +TEST_CASE("a message that does not fit this build's limits is rejected", "[unit][camera_wire]") +{ + WireEncoder encoder; + + GroupDescription crowded = stereo_group(); + crowded.streams.resize(MAX_STREAMS_PER_GROUP + 1, crowded.streams.front()); + const std::span bytes = encoder.encode(SubscribedMessage{ "", crowded }); + REQUIRE_THROWS_AS(decode_wire_message(bytes), WireError); + + // A pixel format added by a newer plugin cannot be read, so the group is refused as a whole. + GroupDescription newer = stereo_group(); + newer.streams[1].pixel_format = static_cast(99); + REQUIRE_THROWS_AS(decode_wire_message(encoder.encode(SubscribedMessage{ "", newer })), WireError); + + flatbuffers::FlatBufferBuilder builder; + + // More planes than an image can have. + const std::vector> planes( + MAX_PLANES_PER_IMAGE + 1, wire::CreatePlaneLayout(builder, 0, 64)); + const auto stream = + wire::CreateStream(builder, builder.CreateString("s"), wire::Codec_None, wire::PixelFormat_Gray8, + wire::MemoryKind_Host, 64, 64, builder.CreateVector(planes), 64 * 64, 4, 30.0); + const auto group = wire::CreateGroup(builder, builder.CreateString("g"), builder.CreateVector(&stream, 1)); + builder.Finish(wire::CreateMessage(builder, wire::Body_Subscribed, wire::CreateSubscribed(builder, 0, group).Union())); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // More images than a group has streams. + builder.Clear(); + const std::vector> images(MAX_STREAMS_PER_GROUP + 1, wire::CreateImage(builder)); + builder.Finish(wire::CreateMessage( + builder, wire::Body_Frame, wire::CreateFrame(builder, 1, 1, 0, 0, 0, builder.CreateVector(images)).Union())); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // A body type with no body behind it. + builder.Clear(); + builder.Finish(wire::CreateMessage(builder, wire::Body_Release, 0)); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // The largest message a receive buffer has to hold. + REQUIRE_THROWS_AS( + encoder.encode(HelloMessage{ .reader_name = std::string(MAX_WIRE_MESSAGE_BYTES, 'x') }), std::length_error); +} + +TEST_CASE("a message from a newer peer is ignored, not an error", "[unit][camera_wire]") +{ + flatbuffers::FlatBufferBuilder builder; + + // A union member this build does not have. + const auto body = wire::CreateRelease(builder, 1); + builder.Finish(wire::CreateMessage(builder, static_cast(wire::Body_MAX + 1), body.Union())); + REQUIRE(std::holds_alternative(decode_wire_message(bytes_of(builder)))); + + // No body at all. + builder.Clear(); + builder.Finish(wire::CreateMessage(builder)); + REQUIRE(std::holds_alternative(decode_wire_message(bytes_of(builder)))); +}