From 1572f956b642f3758f724f1d9a07ed29421d6b93 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Fri, 13 Nov 2026 16:45:19 -0800 Subject: [PATCH] feat(camera): add camera_wire, the messages between the plugin and a reader The plugin and a reader exchange six messages over their socket: Hello and Welcome (the catalog of what is published), Subscribe and Subscribed (with one pool descriptor per stream), then Frame per frame set and Release when the reader is done with it. A connection carries one subscription. The messages are FlatBuffers tables in a schema kept inside the module, so a reader and a plugin from different builds keep understanding each other: the schema only grows, and a message type a build does not know is ignored. It is not under src/core/schema because these messages are never recorded. WireEncoder and decode_wire_message convert to and from camera_core types, so nothing outside the module sees FlatBuffers. Encoding a frame event allocates nothing once the encoder's buffer has grown. Decoding treats its input as coming from another process and checks it in full. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- src/core/CMakeLists.txt | 1 + src/core/camera_wire/AGENTS.md | 16 + src/core/camera_wire/CMakeLists.txt | 7 + src/core/camera_wire/README.md | 58 +++ src/core/camera_wire/cpp/CMakeLists.txt | 23 ++ src/core/camera_wire/cpp/codec.cpp | 341 ++++++++++++++++++ .../cpp/inc/camera_wire/messages.hpp | 113 ++++++ src/core/camera_wire/fbs/CMakeLists.txt | 34 ++ src/core/camera_wire/fbs/camera_wire.fbs | 108 ++++++ tests/cpp/core/CMakeLists.txt | 1 + tests/cpp/core/camera_wire/CMakeLists.txt | 19 + tests/cpp/core/camera_wire/test_codec.cpp | 298 +++++++++++++++ 12 files changed, 1019 insertions(+) create mode 100644 src/core/camera_wire/AGENTS.md create mode 100644 src/core/camera_wire/CMakeLists.txt create mode 100644 src/core/camera_wire/README.md create mode 100644 src/core/camera_wire/cpp/CMakeLists.txt create mode 100644 src/core/camera_wire/cpp/codec.cpp create mode 100644 src/core/camera_wire/cpp/inc/camera_wire/messages.hpp create mode 100644 src/core/camera_wire/fbs/CMakeLists.txt create mode 100644 src/core/camera_wire/fbs/camera_wire.fbs create mode 100644 tests/cpp/core/camera_wire/CMakeLists.txt create mode 100644 tests/cpp/core/camera_wire/test_codec.cpp diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 602cc1969a..7851685616 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -25,6 +25,7 @@ add_subdirectory(camera_core) # Camera frames and messages between processes (shared host memory, descriptor passing) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) + add_subdirectory(camera_wire) endif() # Build OXR library (depends on oxr_utils for types) diff --git a/src/core/camera_wire/AGENTS.md b/src/core/camera_wire/AGENTS.md new file mode 100644 index 0000000000..a7596019c1 --- /dev/null +++ b/src/core/camera_wire/AGENTS.md @@ -0,0 +1,16 @@ + + +# Agent notes — `camera_wire` + +**CRITICAL (non-optional):** Before editing this package, complete the mandatory **`AGENTS.md` preflight** in [`../../../AGENTS.md`](../../../AGENTS.md) (read every applicable `AGENTS.md` on your paths, not just this file). + +Read [`README.md`](README.md) before changing `fbs/camera_wire.fbs`: a reader and a plugin from different builds must keep understanding each other. + +- **The schema only grows.** Append fields with fresh ids, optional; append union members; tables, never structs. Do not reuse or renumber. +- **FlatBuffers stays inside this module.** Its users see `camera_core` types; only the codec and its tests include the generated header. +- **Decoded bytes are untrusted.** Verify, then bound every count against the fixed arrays it fills, and check for a body behind a body type. +- **A new enum value needs both sides.** The core enum and the schema enum carry the same numbers (pinned by `static_assert`), and the decoder must accept the value before a plugin sends it. +- **The frame path does not allocate.** Keep `encode(const FrameEvent&)` free of containers that do. diff --git a/src/core/camera_wire/CMakeLists.txt b/src/core/camera_wire/CMakeLists.txt new file mode 100644 index 0000000000..e162dea055 --- /dev/null +++ b/src/core/camera_wire/CMakeLists.txt @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_subdirectory(fbs) +add_subdirectory(cpp) diff --git a/src/core/camera_wire/README.md b/src/core/camera_wire/README.md new file mode 100644 index 0000000000..20effdcb4b --- /dev/null +++ b/src/core/camera_wire/README.md @@ -0,0 +1,58 @@ + + +# Camera interface: messages between the plugin and a reader + +What the camera plugin and a reader say to each other over their socket +([`camera_ipc`](../camera_ipc)). Each socket message is one of these: + +```text +reader plugin + | Hello (protocol version) | + |------------------------------------>| + | Welcome (what is published) | + |<------------------------------------| + | Subscribe (group, window, order) | + |------------------------------------>| + | Subscribed + one pool descriptor | + | per stream | + |<------------------------------------| + | Frame (lease, slot per image) | once per frame set + |<------------------------------------| + | Release (lease) | + |------------------------------------>| +``` + +A connection carries one subscription; a reader that wants two groups opens two. Closing the +connection ends the subscription and returns every frame set the reader still held. A `Frame` +can overtake `Subscribed`, because capture does not wait for the answer to be sent; it belongs +to the subscription all the same. + +## Schema + +[`fbs/camera_wire.fbs`](fbs/camera_wire.fbs) defines the messages as FlatBuffers tables. It is +kept here and not in [`schema`](../schema): that directory holds what goes into recordings, +and these messages are never recorded. + +A reader and a plugin may come from different builds, so the schema only grows: + +- Append fields with fresh ids and leave them optional; append union members. Never reuse or + renumber an id, and use tables, not structs. +- A message type this build does not know decodes to nothing and is ignored. +- A stream whose format or kind of memory this build does not know makes its group + unreadable, and the reader is told so; other groups are unaffected. +- `WIRE_PROTOCOL_VERSION` changes only when the two sides would misread each other. The + plugin refuses a reader with a different version in `Welcome`. + +## Codec + +`WireEncoder` and `decode_wire_message` turn [`camera_core`](../camera_core) types into bytes +and back, so nothing outside this module sees FlatBuffers. The encoder reuses its buffer; a +frame event is encoded without allocating. The decoder treats its input as coming from another +process: it verifies the buffer and refuses counts that would not fit a group. + +## Tests + +`tests/cpp/core/camera_wire`, run with `ctest -L camera_wire`. They are tagged `unit`. diff --git a/src/core/camera_wire/cpp/CMakeLists.txt b/src/core/camera_wire/cpp/CMakeLists.txt new file mode 100644 index 0000000000..a4d4122dbb --- /dev/null +++ b/src/core/camera_wire/cpp/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# Turns the messages between the camera plugin and its readers into bytes and back. The +# FlatBuffers types stay inside this library; its users see camera_core types. +add_library(camera_wire STATIC + codec.cpp +) + +target_include_directories(camera_wire INTERFACE + $ +) + +target_link_libraries(camera_wire + PUBLIC + camera::camera_core + PRIVATE + camera::camera_wire_schema +) + +add_library(camera::camera_wire ALIAS camera_wire) diff --git a/src/core/camera_wire/cpp/codec.cpp b/src/core/camera_wire/cpp/codec.cpp new file mode 100644 index 0000000000..d0856fb521 --- /dev/null +++ b/src/core/camera_wire/cpp/codec.cpp @@ -0,0 +1,341 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include "inc/camera_wire/messages.hpp" + +#include + +#include +#include +#include + +namespace core::camera +{ + +namespace +{ + +// The schema's enums carry the same numbers as the core's, so a value converts by a cast. +static_assert(static_cast(PixelFormat::BGRA8) == wire::PixelFormat_Bgra8); +static_assert(static_cast(PixelFormat::RGBA8) == wire::PixelFormat_Rgba8); +static_assert(static_cast(PixelFormat::NV12) == wire::PixelFormat_Nv12); +static_assert(static_cast(PixelFormat::GRAY8) == wire::PixelFormat_Gray8); +static_assert(static_cast(Codec::NONE) == wire::Codec_None); +static_assert(static_cast(Codec::H264) == wire::Codec_H264); +static_assert(static_cast(Codec::H265) == wire::Codec_H265); +static_assert(static_cast(MemoryKind::HOST) == wire::MemoryKind_Host); +static_assert(static_cast(DeliveryOrder::DEFAULT) == wire::DeliveryOrder_Default); +static_assert(static_cast(DeliveryOrder::NEWEST) == wire::DeliveryOrder_Newest); +static_assert(static_cast(DeliveryOrder::IN_ORDER) == wire::DeliveryOrder_InOrder); + +using Builder = flatbuffers::FlatBufferBuilder; + +flatbuffers::Offset build_group(Builder& builder, const GroupDescription& group) +{ + std::vector> streams; + streams.reserve(group.streams.size()); + for (const StreamDescription& stream : group.streams) + { + std::array, MAX_PLANES_PER_IMAGE> planes; + const std::size_t plane_count = std::min(stream.plane_count, MAX_PLANES_PER_IMAGE); + for (std::size_t i = 0; i < plane_count; ++i) + { + planes[i] = wire::CreatePlaneLayout(builder, stream.planes[i].offset_bytes, stream.planes[i].stride_bytes); + } + const auto id = builder.CreateString(stream.id); + const auto plane_vector = builder.CreateVector(planes.data(), plane_count); + streams.push_back(wire::CreateStream(builder, id, static_cast(stream.codec), + static_cast(stream.pixel_format), + static_cast(stream.memory), stream.width, stream.height, + plane_vector, stream.slot_bytes, stream.slot_count, stream.frame_rate)); + } + const auto id = builder.CreateString(group.id); + const auto stream_vector = builder.CreateVector(streams); + return wire::CreateGroup(builder, id, stream_vector); +} + +std::string text_of(const flatbuffers::String* string) +{ + return string != nullptr ? string->str() : std::string(); +} + +std::optional pixel_format_of(wire::PixelFormat format) +{ + switch (format) + { + case wire::PixelFormat_Bgra8: + case wire::PixelFormat_Rgba8: + case wire::PixelFormat_Nv12: + case wire::PixelFormat_Gray8: + return static_cast(format); + default: + return std::nullopt; + } +} + +std::optional codec_of(wire::Codec codec) +{ + switch (codec) + { + case wire::Codec_None: + case wire::Codec_H264: + case wire::Codec_H265: + return static_cast(codec); + default: + return std::nullopt; + } +} + +std::optional order_of(wire::DeliveryOrder order) +{ + switch (order) + { + case wire::DeliveryOrder_Default: + case wire::DeliveryOrder_Newest: + case wire::DeliveryOrder_InOrder: + return static_cast(order); + default: + return std::nullopt; + } +} + +GroupDescription read_group(const wire::Group* group) +{ + GroupDescription result; + if (group == nullptr) + { + return result; + } + result.id = text_of(group->id()); + const auto* streams = group->streams(); + if (streams == nullptr) + { + return result; + } + if (streams->size() > MAX_STREAMS_PER_GROUP) + { + throw WireError("camera group '" + result.id + "' has more streams than a group can have"); + } + for (const wire::Stream* stream : *streams) + { + StreamDescription description; + description.id = text_of(stream->id()); + const auto codec = codec_of(stream->codec()); + const auto pixel_format = pixel_format_of(stream->pixel_format()); + // An encoded stream has no pixel format to know. + if (!codec || stream->memory() != wire::MemoryKind_Host || (*codec == Codec::NONE && !pixel_format)) + { + throw WireError("camera stream '" + description.id + + "' uses a format or a kind of memory this build does not know"); + } + description.codec = *codec; + description.pixel_format = pixel_format.value_or(PixelFormat::BGRA8); + description.memory = MemoryKind::HOST; + description.width = stream->width(); + description.height = stream->height(); + if (const auto* planes = stream->planes()) + { + if (planes->size() > MAX_PLANES_PER_IMAGE) + { + throw WireError("camera stream '" + description.id + "' has more planes than an image can have"); + } + description.plane_count = planes->size(); + for (std::uint32_t i = 0; i < planes->size(); ++i) + { + description.planes[i] = { planes->Get(i)->offset_bytes(), planes->Get(i)->stride_bytes() }; + } + } + description.slot_bytes = stream->slot_bytes(); + description.slot_count = stream->slot_count(); + description.frame_rate = stream->frame_rate(); + result.streams.push_back(std::move(description)); + } + return result; +} + +FrameEvent read_frame(const wire::Frame& frame) +{ + FrameEvent event; + event.lease_id = frame.lease_id(); + event.sequence = frame.sequence(); + event.generation = frame.generation(); + event.skipped = frame.skipped(); + if (const wire::Timeline* timeline = frame.timeline()) + { + event.timeline = { timeline->capture_ns(), timeline->sdk_ready_ns(), timeline->published_ns(), + timeline->sent_ns() }; + } + if (const auto* images = frame.images()) + { + if (images->size() > MAX_STREAMS_PER_GROUP) + { + throw WireError("a camera frame set has more images than a group has streams"); + } + event.image_count = images->size(); + for (std::uint32_t i = 0; i < images->size(); ++i) + { + const wire::Image* image = images->Get(i); + event.images[i] = { image->slot(), image->flags(), image->bytes_used(), image->device_frame_id(), + image->device_time_ns() }; + } + } + return event; +} + +} // namespace + +struct WireEncoder::Impl +{ + Builder builder{ 1024 }; + + template + std::span finish(wire::Body type, flatbuffers::Offset body) + { + builder.Finish(wire::CreateMessage(builder, type, body.Union())); + if (builder.GetSize() > MAX_WIRE_MESSAGE_BYTES) + { + throw std::length_error("a camera wire message of " + std::to_string(builder.GetSize()) + + " bytes is over the limit of " + std::to_string(MAX_WIRE_MESSAGE_BYTES)); + } + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; + } +}; + +WireEncoder::WireEncoder() : m_impl(std::make_unique()) +{ +} + +WireEncoder::~WireEncoder() = default; + +std::span WireEncoder::encode(const HelloMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto name = builder.CreateString(message.reader_name); + return m_impl->finish(wire::Body_Hello, wire::CreateHello(builder, message.protocol_version, name)); +} + +std::span WireEncoder::encode(const WelcomeMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::vector> groups; + groups.reserve(message.groups.size()); + for (const GroupDescription& group : message.groups) + { + groups.push_back(build_group(builder, group)); + } + const auto group_vector = builder.CreateVector(groups); + const auto error = builder.CreateString(message.error); + return m_impl->finish( + wire::Body_Welcome, wire::CreateWelcome(builder, message.protocol_version, group_vector, error)); +} + +std::span WireEncoder::encode(const SubscribeMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto group = builder.CreateString(message.group); + return m_impl->finish( + wire::Body_Subscribe, wire::CreateSubscribe(builder, group, message.options.window, + static_cast(message.options.order))); +} + +std::span WireEncoder::encode(const SubscribedMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + const auto error = builder.CreateString(message.error); + const auto group = build_group(builder, message.group); + return m_impl->finish(wire::Body_Subscribed, wire::CreateSubscribed(builder, error, group)); +} + +// The per-frame path: nothing here allocates once the builder has grown to a frame's size. +std::span WireEncoder::encode(const FrameEvent& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + std::array, MAX_STREAMS_PER_GROUP> images; + const std::size_t image_count = std::min(message.image_count, MAX_STREAMS_PER_GROUP); + for (std::size_t i = 0; i < image_count; ++i) + { + const ImageInfo& image = message.images[i]; + images[i] = wire::CreateImage( + builder, image.slot, image.flags, image.bytes_used, image.device_frame_id, image.device_time_ns); + } + const auto image_vector = builder.CreateVector(images.data(), image_count); + const auto timeline = wire::CreateTimeline(builder, message.timeline.capture_ns, message.timeline.sdk_ready_ns, + message.timeline.published_ns, message.timeline.sent_ns); + return m_impl->finish( + wire::Body_Frame, wire::CreateFrame(builder, message.lease_id, message.sequence, message.generation, + message.skipped, timeline, image_vector)); +} + +std::span WireEncoder::encode(const ReleaseMessage& message) +{ + Builder& builder = m_impl->builder; + builder.Clear(); + return m_impl->finish(wire::Body_Release, wire::CreateRelease(builder, message.lease_id)); +} + +WireMessage decode_wire_message(std::span bytes) +{ + const auto* data = reinterpret_cast(bytes.data()); + flatbuffers::Verifier verifier(data, bytes.size()); + if (bytes.empty() || !wire::VerifyMessageBuffer(verifier)) + { + throw WireError("received bytes that are not a camera wire message"); + } + const wire::Message* message = wire::GetMessage(data); + // The verifier accepts a body type without a body. + if (message->body_type() != wire::Body_NONE && message->body() == nullptr) + { + throw WireError("received a camera wire message without a body"); + } + switch (message->body_type()) + { + case wire::Body_Hello: + { + const wire::Hello* hello = message->body_as_Hello(); + return HelloMessage{ hello->protocol_version(), text_of(hello->reader_name()) }; + } + case wire::Body_Welcome: + { + const wire::Welcome* welcome = message->body_as_Welcome(); + WelcomeMessage result; + result.protocol_version = welcome->protocol_version(); + result.error = text_of(welcome->error()); + if (const auto* groups = welcome->groups()) + { + for (const wire::Group* group : *groups) + { + result.groups.push_back(read_group(group)); + } + } + return result; + } + case wire::Body_Subscribe: + { + const wire::Subscribe* subscribe = message->body_as_Subscribe(); + const auto order = order_of(subscribe->order()); + if (!order) + { + throw WireError("a camera reader asked for a delivery order this build does not know"); + } + return SubscribeMessage{ text_of(subscribe->group()), { subscribe->window(), *order } }; + } + case wire::Body_Subscribed: + { + const wire::Subscribed* subscribed = message->body_as_Subscribed(); + return SubscribedMessage{ text_of(subscribed->error()), read_group(subscribed->group()) }; + } + case wire::Body_Frame: + return read_frame(*message->body_as_Frame()); + case wire::Body_Release: + return ReleaseMessage{ message->body_as_Release()->lease_id() }; + default: + return std::monostate{}; + } +} + +} // namespace core::camera diff --git a/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp b/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp new file mode 100644 index 0000000000..e2626c10ee --- /dev/null +++ b/src/core/camera_wire/cpp/inc/camera_wire/messages.hpp @@ -0,0 +1,113 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace core::camera +{ + +// Changes only when a reader and a plugin of different versions could misread each other. +// A field or a message added to the schema does not change it. +inline constexpr std::uint32_t WIRE_PROTOCOL_VERSION = 1; + +// No message is larger than this, so a receive buffer of this size always fits one. +inline constexpr std::size_t MAX_WIRE_MESSAGE_BYTES = 64 * 1024; + +// Reader to plugin, first message on a connection. +struct HelloMessage +{ + std::uint32_t protocol_version = WIRE_PROTOCOL_VERSION; + std::string reader_name; // for the plugin's log +}; + +// Plugin to reader, in answer to HelloMessage. +struct WelcomeMessage +{ + std::uint32_t protocol_version = WIRE_PROTOCOL_VERSION; + std::vector groups; // what the plugin publishes + std::string error; // why the plugin refuses the reader, if it does +}; + +// Reader to plugin: start sending a group's frame sets. One subscription per connection. +struct SubscribeMessage +{ + std::string group; + ReaderOptions options; +}; + +// Plugin to reader, in answer to SubscribeMessage. On success one file descriptor per stream +// travels with it, in stream order, for the stream's pool. +struct SubscribedMessage +{ + std::string error; // why the subscription was refused, if it was + GroupDescription group; +}; + +// Reader to plugin: done with a frame set. +struct ReleaseMessage +{ + std::uint64_t lease_id = 0; +}; + +// What decode_wire_message returns. A frame set travels as a FrameEvent. std::monostate is a +// message from a newer peer that this build does not know; ignore it. +using WireMessage = + std::variant; + +/** + * @brief Bytes that are not a message, or a message this build cannot represent. + */ +class WireError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; + +/** + * @brief Turns messages into bytes. It reuses one buffer, so after the first few messages + * encoding allocates nothing. Not thread-safe. + */ +class WireEncoder +{ +public: + WireEncoder(); + ~WireEncoder(); + + WireEncoder(const WireEncoder&) = delete; + WireEncoder& operator=(const WireEncoder&) = delete; + + /** + * @return The message's bytes, valid until the next call on this encoder. + * @throws std::length_error if the message is larger than MAX_WIRE_MESSAGE_BYTES. + */ + std::span encode(const HelloMessage& message); + std::span encode(const WelcomeMessage& message); + std::span encode(const SubscribeMessage& message); + std::span encode(const SubscribedMessage& message); + std::span encode(const FrameEvent& message); + std::span encode(const ReleaseMessage& message); + +private: + struct Impl; + std::unique_ptr m_impl; +}; + +/** + * @brief Read one message. The bytes come from another process and are checked in full. + * @throws WireError if they are not a message, or name more streams, planes or images than a + * group can have, or a format this build does not know. + */ +WireMessage decode_wire_message(std::span bytes); + +} // namespace core::camera diff --git a/src/core/camera_wire/fbs/CMakeLists.txt b/src/core/camera_wire/fbs/CMakeLists.txt new file mode 100644 index 0000000000..02239af735 --- /dev/null +++ b/src/core/camera_wire/fbs/CMakeLists.txt @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +# The messages between the camera plugin and its readers, as a FlatBuffer schema. The generated +# header is reached as ; only the codec and its +# tests include it. + +include(${CMAKE_SOURCE_DIR}/cmake/GenerateFlatBuffers.cmake) + +generate_flatbuffer_headers( + CAMERA_WIRE_GENERATED_HEADERS + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/generated/camera_wire_schema + CAMERA_WIRE_GENERATED_BFBS +) + +add_custom_target(camera_wire_schema_generation + DEPENDS ${CAMERA_WIRE_GENERATED_HEADERS} ${CAMERA_WIRE_GENERATED_BFBS} +) + +add_library(camera_wire_schema INTERFACE) +add_dependencies(camera_wire_schema camera_wire_schema_generation) + +target_include_directories(camera_wire_schema INTERFACE + $ +) + +target_link_libraries(camera_wire_schema INTERFACE + flatbuffers +) + +add_library(camera::camera_wire_schema ALIAS camera_wire_schema) diff --git a/src/core/camera_wire/fbs/camera_wire.fbs b/src/core/camera_wire/fbs/camera_wire.fbs new file mode 100644 index 0000000000..5f6bc1dd5c --- /dev/null +++ b/src/core/camera_wire/fbs/camera_wire.fbs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// The messages between the camera plugin and a reader, one per socket message. A reader and a +// plugin may be built at different times, so this file only grows: append fields with fresh +// ids and leave them optional, append union members, never reuse or renumber. See README.md. + +namespace core.camera.wire; + +// Zero means "a value this build does not know" for the kinds a newer peer may extend. +enum PixelFormat : uint32 { Unknown = 0, Bgra8 = 1, Rgba8 = 2, Nv12 = 3, Gray8 = 4 } +enum Codec : uint32 { None = 0, H264 = 1, H265 = 2 } +enum MemoryKind : uint32 { Unknown = 0, Host = 1 } +enum DeliveryOrder : uint32 { Default = 0, Newest = 1, InOrder = 2 } + +// Where one plane of an image sits inside a slot. +table PlaneLayout { + offset_bytes: uint32 (id: 0); + stride_bytes: uint32 (id: 1); +} + +// One sensor's images or one encoded video stream, and the pool its frames are in. +table Stream { + id: string (id: 0); + codec: Codec (id: 1); + pixel_format: PixelFormat (id: 2); + memory: MemoryKind (id: 3); + width: uint32 (id: 4); + height: uint32 (id: 5); + planes: [PlaneLayout] (id: 6); + slot_bytes: uint64 (id: 7); + slot_count: uint32 (id: 8); + frame_rate: float64 (id: 9); +} + +// Streams that are captured together and published as one frame set. +table Group { + id: string (id: 0); + streams: [Stream] (id: 1); +} + +// Reader to plugin, first message on a connection. +table Hello { + protocol_version: uint32 (id: 0); + reader_name: string (id: 1); +} + +// Plugin to reader, in answer to Hello. `error` is set when the plugin refuses the reader. +table Welcome { + protocol_version: uint32 (id: 0); + groups: [Group] (id: 1); + error: string (id: 2); +} + +// Reader to plugin: start sending this group's frame sets. One subscription per connection. +table Subscribe { + group: string (id: 0); + window: uint32 = 1 (id: 1); + order: DeliveryOrder (id: 2); +} + +// Plugin to reader, in answer to Subscribe. On success the message carries one file descriptor +// per stream of the group, in stream order, for the stream's pool. `error` is set otherwise. +table Subscribed { + error: string (id: 0); + group: Group (id: 1); +} + +// One image of a frame set: the slot it is in and what the camera SDK said about it. +table Image { + slot: uint32 (id: 0); + flags: uint32 (id: 1); + bytes_used: uint64 (id: 2); + device_frame_id: uint64 (id: 3); + device_time_ns: int64 = -1 (id: 4); +} + +// Host monotonic times in nanoseconds; -1 where a time is not known. +table Timeline { + capture_ns: int64 = -1 (id: 0); + sdk_ready_ns: int64 = -1 (id: 1); + published_ns: int64 = -1 (id: 2); + sent_ns: int64 = -1 (id: 3); +} + +// Plugin to reader: one frame set, held for the reader until it sends Release with the same +// lease. Frames may arrive before Subscribed; they belong to the subscription all the same. +table Frame { + lease_id: uint64 (id: 0); + sequence: uint64 (id: 1); + generation: uint32 (id: 2); + skipped: uint32 (id: 3); + timeline: Timeline (id: 4); + images: [Image] (id: 5); +} + +// Reader to plugin: done with a frame set. +table Release { + lease_id: uint64 (id: 0); +} + +union Body { Hello, Welcome, Subscribe, Subscribed, Frame, Release } + +table Message { + body: Body (id: 1); +} + +root_type Message; diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 705bead328..35bbcad055 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -17,6 +17,7 @@ add_subdirectory(camera_core) if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_subdirectory(camera_ipc) + add_subdirectory(camera_wire) endif() if(BUILD_OVXR_CAMERA_QUADS) diff --git a/tests/cpp/core/camera_wire/CMakeLists.txt b/tests/cpp/core/camera_wire/CMakeLists.txt new file mode 100644 index 0000000000..69b9c4a7b1 --- /dev/null +++ b/tests/cpp/core/camera_wire/CMakeLists.txt @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +cmake_minimum_required(VERSION 3.20) + +add_executable(camera_wire_tests + test_codec.cpp +) + +# The schema is linked to build messages the encoder never would: ones from a newer peer and +# malformed ones. +target_link_libraries(camera_wire_tests PRIVATE + camera::camera_wire + camera::camera_wire_schema + Catch2::Catch2WithMain +) + +message(STATUS "camera_wire_tests target enabled with Catch2") +catch_discover_tests(camera_wire_tests ADD_TAGS_AS_LABELS) diff --git a/tests/cpp/core/camera_wire/test_codec.cpp b/tests/cpp/core/camera_wire/test_codec.cpp new file mode 100644 index 0000000000..e3b33b516f --- /dev/null +++ b/tests/cpp/core/camera_wire/test_codec.cpp @@ -0,0 +1,298 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace core::camera; +namespace wire = core::camera::wire; + +namespace +{ + +GroupDescription stereo_group() +{ + GroupDescription group; + group.id = "head"; + for (const char* eye : { "left", "right" }) + { + StreamDescription stream; + stream.id = std::string("head/") + eye; + stream.pixel_format = PixelFormat::NV12; + stream.width = 1920; + stream.height = 1080; + stream.plane_count = 2; + stream.planes[0] = { .offset_bytes = 0, .stride_bytes = 2048 }; + stream.planes[1] = { .offset_bytes = 2048 * 1080, .stride_bytes = 2048 }; + stream.slot_bytes = 2048 * 1620; + stream.slot_count = 8; + stream.frame_rate = 59.94; + group.streams.push_back(stream); + } + return group; +} + +GroupDescription encoded_group() +{ + GroupDescription group; + group.id = "head.h265"; + StreamDescription stream; + stream.id = "head/left.h265"; + stream.codec = Codec::H265; + stream.width = 1920; + stream.height = 1080; + stream.slot_bytes = 1 << 20; + stream.slot_count = 16; + stream.frame_rate = 60.0; + group.streams.push_back(stream); + return group; +} + +void require_same(const GroupDescription& actual, const GroupDescription& expected) +{ + REQUIRE(actual.id == expected.id); + REQUIRE(actual.streams.size() == expected.streams.size()); + for (std::size_t i = 0; i < expected.streams.size(); ++i) + { + const StreamDescription& a = actual.streams[i]; + const StreamDescription& e = expected.streams[i]; + REQUIRE(a.id == e.id); + REQUIRE(a.codec == e.codec); + REQUIRE(a.memory == e.memory); + REQUIRE(a.width == e.width); + REQUIRE(a.height == e.height); + REQUIRE(a.plane_count == e.plane_count); + for (std::uint32_t plane = 0; plane < e.plane_count; ++plane) + { + REQUIRE(a.planes[plane].offset_bytes == e.planes[plane].offset_bytes); + REQUIRE(a.planes[plane].stride_bytes == e.planes[plane].stride_bytes); + } + REQUIRE(a.slot_bytes == e.slot_bytes); + REQUIRE(a.slot_count == e.slot_count); + REQUIRE(a.frame_rate == e.frame_rate); + if (!e.is_encoded()) + { + REQUIRE(a.pixel_format == e.pixel_format); + } + } +} + +// What the other process receives: a copy of the bytes, decoded. +template +Message round_trip(WireEncoder& encoder, const Message& message) +{ + const std::span bytes = encoder.encode(message); + const std::vector received(bytes.begin(), bytes.end()); + WireMessage decoded = decode_wire_message(received); + REQUIRE(std::holds_alternative(decoded)); + return std::get(std::move(decoded)); +} + +std::span bytes_of(const flatbuffers::FlatBufferBuilder& builder) +{ + return { reinterpret_cast(builder.GetBufferPointer()), builder.GetSize() }; +} + +} // namespace + +TEST_CASE("the handshake messages survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + const HelloMessage hello = round_trip(encoder, HelloMessage{ .protocol_version = 7, .reader_name = "viewer" }); + REQUIRE(hello.protocol_version == 7); + REQUIRE(hello.reader_name == "viewer"); + + WelcomeMessage welcome; + welcome.groups = { stereo_group(), encoded_group() }; + const WelcomeMessage received = round_trip(encoder, welcome); + REQUIRE(received.protocol_version == WIRE_PROTOCOL_VERSION); + REQUIRE(received.error.empty()); + REQUIRE(received.groups.size() == 2); + require_same(received.groups[0], stereo_group()); + require_same(received.groups[1], encoded_group()); + + const WelcomeMessage refused = round_trip(encoder, WelcomeMessage{ .error = "protocol version 7 is not supported" }); + REQUIRE(refused.error == "protocol version 7 is not supported"); + REQUIRE(refused.groups.empty()); +} + +TEST_CASE("a subscription and its answer survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + const SubscribeMessage subscribe = + round_trip(encoder, SubscribeMessage{ "head", { .window = 4, .order = DeliveryOrder::IN_ORDER } }); + REQUIRE(subscribe.group == "head"); + REQUIRE(subscribe.options.window == 4); + REQUIRE(subscribe.options.order == DeliveryOrder::IN_ORDER); + + // What a reader that sets nothing asks for. + const SubscribeMessage defaults = round_trip(encoder, SubscribeMessage{ "head", {} }); + REQUIRE(defaults.options.window == 1); + REQUIRE(defaults.options.order == DeliveryOrder::DEFAULT); + + const SubscribedMessage accepted = round_trip(encoder, SubscribedMessage{ "", stereo_group() }); + REQUIRE(accepted.error.empty()); + require_same(accepted.group, stereo_group()); + + const SubscribedMessage refused = round_trip(encoder, SubscribedMessage{ "does not fit the pool", {} }); + REQUIRE(refused.error == "does not fit the pool"); + REQUIRE(refused.group.streams.empty()); +} + +TEST_CASE("a frame set and its release survive the wire", "[unit][camera_wire]") +{ + WireEncoder encoder; + + FrameEvent event; + event.lease_id = 0x1122334455667788; + event.sequence = 123456789012; + event.generation = 3; + event.skipped = 2; + event.image_count = 2; + event.timeline = { .capture_ns = 1000, .sdk_ready_ns = 2000, .published_ns = 3000, .sent_ns = 4000 }; + event.images[0] = { + .slot = 5, .flags = IMAGE_FLAG_SDK_WARNING, .bytes_used = 0, .device_frame_id = 77, .device_time_ns = 99 + }; + event.images[1] = { .slot = 6, .flags = IMAGE_FLAG_KEYFRAME, .bytes_used = 4096, .device_frame_id = 78 }; + + const FrameEvent received = round_trip(encoder, event); + REQUIRE(received.lease_id == event.lease_id); + REQUIRE(received.sequence == event.sequence); + REQUIRE(received.generation == 3); + REQUIRE(received.skipped == 2); + REQUIRE(received.image_count == 2); + REQUIRE(received.timeline.capture_ns == 1000); + REQUIRE(received.timeline.sdk_ready_ns == 2000); + REQUIRE(received.timeline.published_ns == 3000); + REQUIRE(received.timeline.sent_ns == 4000); + REQUIRE(received.images[0].slot == 5); + REQUIRE(received.images[0].flags == IMAGE_FLAG_SDK_WARNING); + REQUIRE(received.images[0].device_frame_id == 77); + REQUIRE(received.images[0].device_time_ns == 99); + REQUIRE(received.images[1].slot == 6); + REQUIRE(received.images[1].flags == IMAGE_FLAG_KEYFRAME); + REQUIRE(received.images[1].bytes_used == 4096); + REQUIRE(received.images[1].device_time_ns == TIME_UNKNOWN); + + // Times that are not known stay unknown. + const FrameEvent bare = round_trip(encoder, FrameEvent{}); + REQUIRE(bare.image_count == 0); + REQUIRE(bare.timeline.capture_ns == TIME_UNKNOWN); + REQUIRE(bare.timeline.sent_ns == TIME_UNKNOWN); + + REQUIRE(round_trip(encoder, ReleaseMessage{ 42 }).lease_id == 42); +} + +TEST_CASE("an encoder is reused from message to message", "[unit][camera_wire]") +{ + WireEncoder encoder; + FrameEvent event; + event.image_count = 2; + for (std::uint64_t sequence = 1; sequence <= 100; ++sequence) + { + event.sequence = sequence; + REQUIRE(round_trip(encoder, event).sequence == sequence); + } + // A small message after a large one is not padded to the large one's size. + WelcomeMessage welcome; + welcome.groups = { stereo_group(), encoded_group() }; + const std::size_t large = encoder.encode(welcome).size(); + REQUIRE(encoder.encode(ReleaseMessage{ 1 }).size() < large); +} + +TEST_CASE("bytes that are not a message are rejected", "[unit][camera_wire]") +{ + REQUIRE_THROWS_AS(decode_wire_message({}), WireError); + + const std::vector garbage(64, std::byte{ 0xa5 }); + REQUIRE_THROWS_AS(decode_wire_message(garbage), WireError); + + WireEncoder encoder; + WelcomeMessage welcome; + welcome.groups = { stereo_group() }; + const std::span bytes = encoder.encode(welcome); + const std::vector whole(bytes.begin(), bytes.end()); + REQUIRE_THROWS_AS(decode_wire_message({ whole.data(), whole.size() / 2 }), WireError); + + // No cut of a message may be read out of bounds; each is either refused or read in full. + for (std::size_t length = 0; length < whole.size(); ++length) + { + const std::vector cut(whole.begin(), whole.begin() + static_cast(length)); + try + { + (void)decode_wire_message(cut); + } + catch (const WireError&) + { + } + } +} + +TEST_CASE("a message that does not fit this build's limits is rejected", "[unit][camera_wire]") +{ + WireEncoder encoder; + + GroupDescription crowded = stereo_group(); + crowded.streams.resize(MAX_STREAMS_PER_GROUP + 1, crowded.streams.front()); + const std::span bytes = encoder.encode(SubscribedMessage{ "", crowded }); + REQUIRE_THROWS_AS(decode_wire_message(bytes), WireError); + + // A pixel format added by a newer plugin cannot be read, so the group is refused as a whole. + GroupDescription newer = stereo_group(); + newer.streams[1].pixel_format = static_cast(99); + REQUIRE_THROWS_AS(decode_wire_message(encoder.encode(SubscribedMessage{ "", newer })), WireError); + + flatbuffers::FlatBufferBuilder builder; + + // More planes than an image can have. + const std::vector> planes( + MAX_PLANES_PER_IMAGE + 1, wire::CreatePlaneLayout(builder, 0, 64)); + const auto stream = + wire::CreateStream(builder, builder.CreateString("s"), wire::Codec_None, wire::PixelFormat_Gray8, + wire::MemoryKind_Host, 64, 64, builder.CreateVector(planes), 64 * 64, 4, 30.0); + const auto group = wire::CreateGroup(builder, builder.CreateString("g"), builder.CreateVector(&stream, 1)); + builder.Finish(wire::CreateMessage(builder, wire::Body_Subscribed, wire::CreateSubscribed(builder, 0, group).Union())); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // More images than a group has streams. + builder.Clear(); + const std::vector> images(MAX_STREAMS_PER_GROUP + 1, wire::CreateImage(builder)); + builder.Finish(wire::CreateMessage( + builder, wire::Body_Frame, wire::CreateFrame(builder, 1, 1, 0, 0, 0, builder.CreateVector(images)).Union())); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // A body type with no body behind it. + builder.Clear(); + builder.Finish(wire::CreateMessage(builder, wire::Body_Release, 0)); + REQUIRE_THROWS_AS(decode_wire_message(bytes_of(builder)), WireError); + + // The largest message a receive buffer has to hold. + REQUIRE_THROWS_AS( + encoder.encode(HelloMessage{ .reader_name = std::string(MAX_WIRE_MESSAGE_BYTES, 'x') }), std::length_error); +} + +TEST_CASE("a message from a newer peer is ignored, not an error", "[unit][camera_wire]") +{ + flatbuffers::FlatBufferBuilder builder; + + // A union member this build does not have. + const auto body = wire::CreateRelease(builder, 1); + builder.Finish(wire::CreateMessage(builder, static_cast(wire::Body_MAX + 1), body.Union())); + REQUIRE(std::holds_alternative(decode_wire_message(bytes_of(builder)))); + + // No body at all. + builder.Clear(); + builder.Finish(wire::CreateMessage(builder)); + REQUIRE(std::holds_alternative(decode_wire_message(bytes_of(builder)))); +}