From d9a6551e258839001ba3a1c69f92fdd052d8dc90 Mon Sep 17 00:00:00 2001 From: Farbod Motlagh Date: Wed, 18 Nov 2026 12:02:04 -0800 Subject: [PATCH] feat(camera): add the camera reader for Python isaaccapture.camera is the camera reader for Python. from isaaccapture.camera import CameraReader with CameraReader.open("head") as reader: for frames in reader: left = frames.image(0).array() - An array is a read-only view of the plugin's buffer, not a copy. Its base holds the frame set, so it shows its frame for as long as it exists, and counts as a frame set the reader holds. - The window is two by default, so a loop can keep the array of one frame while it asks for the next. A reader that holds its window's worth raises CameraWindowExhausted. frames.borrow() is for code that must not keep anything. - Iterating lets go of the frame set before at each step. next() returns None when no frame set came. Waiting releases the GIL and lets Ctrl-C through; close() from another thread ends a wait. - CameraReader.open() and list() look through this user's camera plugins, as the C++ reader does; endpoint= or instance= names one plugin instead. A listing names the endpoint of its plugin. - The camera_synthetic plugin and its manifest are bundled into the package under isaaccapture/plugins/camera_synthetic, where the plugin manager finds them, as the gamepad and spacemouse plugins are. The module is added to the wheel's dependencies, to stub generation and to the pip build's import check. It is Linux only, and is left out of the package where its extension is not built. Co-Authored-By: Claude Signed-off-by: Farbod Motlagh --- .github/workflows/build-ubuntu.yml | 1 + src/core/camera/AGENTS.md | 4 +- src/core/camera/CMakeLists.txt | 4 + src/core/camera/README.md | 53 +- src/core/camera/python/CMakeLists.txt | 18 + src/core/camera/python/camera_bindings.cpp | 538 +++++++++++++++++ src/core/python/CMakeLists.txt | 9 + src/core/python/pyproject.toml.in | 1 + src/plugins/camera_synthetic/AGENTS.md | 2 +- src/plugins/camera_synthetic/README.md | 5 + .../camera_synthetic/app/CMakeLists.txt | 15 + src/python/CMakeLists.txt | 6 + src/python/isaaccapture/camera/__init__.py | 44 ++ tests/python/core/CMakeLists.txt | 4 + tests/python/core/camera/CMakeLists.txt | 23 + tests/python/core/camera/pyproject.toml | 22 + .../python/core/camera/test_camera_reader.py | 541 ++++++++++++++++++ tests/python/packaging/test_wheel_contents.py | 15 + 18 files changed, 1301 insertions(+), 4 deletions(-) create mode 100644 src/core/camera/python/CMakeLists.txt create mode 100644 src/core/camera/python/camera_bindings.cpp create mode 100644 src/python/isaaccapture/camera/__init__.py create mode 100644 tests/python/core/camera/CMakeLists.txt create mode 100644 tests/python/core/camera/pyproject.toml create mode 100644 tests/python/core/camera/test_camera_reader.py diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index d9565ecea7..9e532d6904 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -534,6 +534,7 @@ jobs: "deviceio_session", "schema", "plugin_manager", + "camera", ): importlib.import_module(f"isaaccapture.{mod}") diff --git a/src/core/camera/AGENTS.md b/src/core/camera/AGENTS.md index cb0f2efef9..6d96fd7f6e 100644 --- a/src/core/camera/AGENTS.md +++ b/src/core/camera/AGENTS.md @@ -21,6 +21,7 @@ Read [`README.md`](README.md) first: it has the terms (stream, group, window, or - **Releasing must never throw or block.** It runs in destructors, possibly on another thread; a plugin that is gone takes its frames back by itself. - **The plugin is trusted for content, not for bounds.** Check a frame event against the group before handing out views, and check the handshake answer in `open` before mapping anything: the protocol version, that it is the group that was asked for, `problem_with` for the description, one pool per stream. - **A newest-order reader asks; it does not drain.** `next` sends the request and takes the answer. Do not go back to reading whatever is queued and keeping the last: what is queued was held when it was published. +- **In a binding, whatever exposes image memory holds the frame set.** An array's base is the lease, not just the mapping, so an array never shows another frame; a scope that must not keep anything checks that count when it ends. Do not hand out a view that outlives its lease, and do not build a scope on something that can be released under a view (a `memoryview` released while `numpy.frombuffer` still points at it dangles). - **A full window is an error, not a wait.** `next` by a reader that holds its window's worth throws `CameraWindowExhausted`; it does not time out as if no frame had come. - **An empty location looks; a named one does not.** With no endpoint and no instance the reader asks every endpoint `default_endpoint_paths()` lists. With either, it talks to that one plugin alone: the full handshake timeout, and an error if the plugin is not there. Do not fall back from one to the other. - **Looking stays plain.** The endpoints are asked one after another, each with a short fixed timeout. A refused connection is passed over silently. An endpoint that fails to say what it publishes (no answer in time, a refusal, a closed connection, any other failure) is passed over too, named with its reason in `open`'s error and left out by `list`; do not narrow that to the timeout. A group on two endpoints is an error, never a choice. No cache, no index file, no retry, no thread, and no option for any of it. @@ -95,5 +96,6 @@ Read "What runs where" in the [`README.md`](README.md) before changing it. - **A test does not assume which thread runs first.** A reader's request and the camera's publish come from two threads and are served in either order: have the reader ask for what comes after a frame set it names, not for "anything", when the test expects a particular one. Run the suite on one CPU (`taskset -c 0`) before pushing; CI's slower machines find these, a workstation does not. - **A test waits for what the plugin's thread does; it does not assume it.** `next` returning says nothing about whether the plugin has taken the reader's request, so wait for the effect (`Served::wait_until_sent`) before counting what was sent. - **A test's helper thread is stopped and joined however the test ends.** A failed `REQUIRE` throws, and a `std::thread` that is still joinable then aborts the process instead of reporting the failure. Use `std::jthread`, or a guard whose destructor joins. -- **Tests that look through the user's plugins, or let a plugin choose its endpoint or its configuration file, get a user's directories of their own.** Point `XDG_RUNTIME_DIR`, and for a plugin's configuration file `XDG_CONFIG_HOME`, at a short temporary directory and put them back, as `EndpointDirectory` (`tests/cpp/core/camera/endpoint_directory.hpp`) and `UserDirectories` (`tests/cpp/core/camera/test_camera_plugin.cpp`) do; a test that uses the real ones finds whatever camera plugin and `cameras.yaml` the developer has. +- **Tests that look through the user's plugins, or let a plugin choose its endpoint or its configuration file, get a user's directories of their own.** Point `XDG_RUNTIME_DIR`, and for a plugin's configuration file `XDG_CONFIG_HOME`, at a short temporary directory and put them back, as `EndpointDirectory` (`tests/cpp/core/camera/endpoint_directory.hpp`) and `UserDirectories` (`tests/cpp/core/camera/test_camera_plugin.cpp`) do, and the `endpoint_directory` fixture of `tests/python/core/camera` for a Python test and the plugins it starts; a test that uses the real ones finds whatever camera plugin and `cameras.yaml` the developer has. - **A test that runs `run_camera_plugin` in the test process aims the stop signal at the plugin's thread**, and starts that thread with the signal blocked (`PluginRun` in `tests/cpp/core/camera/test_camera_plugin.cpp`). A signal sent to the process would end the test instead. +- **A Python test starts plugins through `start_plugin`.** The fixture stops every process it started and waits for it, also when the test fails, before the directory with the socket files is removed. A plugin started through the plugin manager is stopped in a `finally`. diff --git a/src/core/camera/CMakeLists.txt b/src/core/camera/CMakeLists.txt index 8ce91de360..bb23b0836d 100644 --- a/src/core/camera/CMakeLists.txt +++ b/src/core/camera/CMakeLists.txt @@ -4,3 +4,7 @@ cmake_minimum_required(VERSION 3.20) add_subdirectory(cpp) + +if(BUILD_PYTHON_BINDINGS) + add_subdirectory(python) +endif() diff --git a/src/core/camera/README.md b/src/core/camera/README.md index dec377ac0c..6860ea706a 100644 --- a/src/core/camera/README.md +++ b/src/core/camera/README.md @@ -158,6 +158,45 @@ Not covered: a process that is stopped while it waits inside `next` is handed, w continues, the frame set that arrived meanwhile. A reader that needs to know how old its frame is has the times of every step in `timeline()`. +### Python + +The same reader as `isaaccapture.camera`: + +```python +from isaaccapture.camera import CameraReader + +with CameraReader.open("head") as reader: + for frames in reader: + left = frames.image(0).array() # (height, width, 4) uint8, read-only +``` + +- An array is a view of the plugin's buffer, not a copy, and cannot be written to. **It + keeps its frame set held**: it shows that frame for as long as the array exists, after + `release()`, after the loop has moved on and after the reader is closed. The price is the + window: an array that is kept counts as a frame set the reader holds. +- `open` takes the window and the order (`"newest"`, `"in_order"`) as keyword arguments. The + window is two by default, not one as in C++: in the loop above `left` still names the array + of the frame before while the next frame set is asked for. The order is `"newest"` unless + `"in_order"` is given; a larger window alone does not change it, as it does in C++. +- **Which plugin** is as in C++ ("How a reader finds its camera"). `open` and `list` look + through this user's camera plugins; `instance=` or `endpoint=` names one plugin instead. + Each `GroupListing` of `list` has the `endpoint` of the plugin that publishes the group. +- A reader that holds its window's worth raises `CameraWindowExhausted` from `next()`, with + what to do about it: drop an array, or copy what is needed from it. +- `for frames in reader` lets go of the frame set before at each step. With `reader.next()` + the caller does: `frames.release()`, or a `with` block. +- `with frames.borrow() as frames:` is for code that must not keep anything: the frame set + goes back to the plugin when the block ends, and an array of it that still exists then + raises `BufferError` instead of holding the frame set without a word. +- Arrays are NumPy arrays over host memory. A DLPack export is added with the first pool + kind that is not host memory, where it is the way to hand a frame to a tensor library. +- `bgra8` and `rgba8` images are `(height, width, 4)`, `gray8` is `(height, width)`; `nv12` + has two planes, `plane(0)` and `plane(1)`. `data` is the flat bytes, also for encoded video. +- The synthetic camera plugin ships in the package under `plugins/camera_synthetic`. The + plugin manager, given `BUNDLED_PLUGINS_DIR` (the package's `plugins` directory, from + `isaaccapture.teleop_session_manager`) as its search path, starts it by its name, + `camera_synthetic`. + ## Publishing cameras A camera plugin is one vendor's or one SDK's capture code and this library. The capture code @@ -297,7 +336,9 @@ its camera restarted. vendor or SDK called ``: 1. **Directory.** Copy `src/plugins/camera_synthetic` to `src/plugins/camera_`, without - `tools/`: `camera_bench` reads any camera plugin and stays where it is. + `tools/`: `camera_bench` reads any camera plugin and stays where it is. Leave out the block + in `app/CMakeLists.txt` that bundles the plugin into the Python package as well: the wheel + does not ship a vendor's SDK. 2. **`core/`: the backend.** A static library `camera__core` that links `camera::camera` and the SDK. It implements `CameraSource` for one camera and has one function that returns the `Backend`, named ``. No type of the SDK leaves this @@ -697,7 +738,8 @@ its camera asks that of every plugin of its user. Nothing is logged per frame. `tests/cpp/core/camera`, built as one executable, `camera_tests`. `ctest -L camera` runs them. A label is a regular expression, so that also runs the tests of the camera plugins, whose labels -start with `camera_`; `ctest -L "^camera$"` is the form that selects this library's tests alone. +start with `camera_`, and in a build with the Python bindings the Python tests below +(`camera_python`); `ctest -L "^camera$"` is the form that selects this library's tests alone. - **Unit** (`ctest -L unit`): everything that runs in one process. The endpoint's cases play the reader's part with the socket and the codec directly. The reader's cases run it against a @@ -725,3 +767,10 @@ start with `camera_`; `ctest -L "^camera$"` is the form that selects this librar while it holds frame sets gives them back; a reader is told when the plugin's process is killed and can still read what it holds; a reader finds two groups on two plugin processes by name, and still finds the one after the other was killed. + +`tests/python/core/camera` (`ctest -L camera_python`) reads from the synthetic camera plugin +that is bundled into the package, started as a process and, in one case, through the plugin +manager. Besides the reading, the cases cover the finding, each in a user's directories of +its own: a reader that is told nothing finds the plugin; a listing names the endpoint of its +plugin; two plugins side by side are both found by group; a group on two plugins, and a +group nobody publishes, are errors that name the endpoints and the directory. diff --git a/src/core/camera/python/CMakeLists.txt b/src/core/camera/python/CMakeLists.txt new file mode 100644 index 0000000000..411a096fdc --- /dev/null +++ b/src/core/camera/python/CMakeLists.txt @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The camera reader for Python: isaaccapture.camera._camera. +pybind11_add_module(camera_py + camera_bindings.cpp +) + +target_link_libraries(camera_py + PRIVATE + camera::camera +) + +set_target_properties(camera_py PROPERTIES + OUTPUT_NAME "_camera" + # Use genex in output directory - CMake won't add another config subdirectory + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python_package/$/isaaccapture/camera" +) diff --git a/src/core/camera/python/camera_bindings.cpp b/src/core/camera/python/camera_bindings.cpp new file mode 100644 index 0000000000..e2c7ec6a52 --- /dev/null +++ b/src/core/camera/python/camera_bindings.cpp @@ -0,0 +1,538 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace py = pybind11; +using namespace core::camera; + +namespace +{ + +const char* name_of(PixelFormat format) +{ + switch (format) + { + case PixelFormat::BGRA8: + return "bgra8"; + case PixelFormat::RGBA8: + return "rgba8"; + case PixelFormat::NV12: + return "nv12"; + case PixelFormat::GRAY8: + return "gray8"; + } + return "unknown"; +} + +const char* name_of(Codec codec) +{ + switch (codec) + { + case Codec::NONE: + return "none"; + case Codec::H264: + return "h264"; + case Codec::H265: + return "h265"; + } + return "unknown"; +} + +DeliveryOrder order_of(const std::string& name) +{ + if (name == "newest") + { + return DeliveryOrder::NEWEST; + } + if (name == "in_order") + { + return DeliveryOrder::IN_ORDER; + } + throw std::invalid_argument("order is 'newest' or 'in_order', not '" + name + "'"); +} + +PluginLocation location_of(const std::string& endpoint, const std::string& instance) +{ + PluginLocation plugin; + plugin.endpoint = endpoint; + plugin.instance = instance; + return plugin; +} + +// A frame set as Python holds it: by the reader's FrameSet object and by every array that views +// one of its images. It goes back to the plugin when the last of them is gone. +using Lease = std::shared_ptr; + +// The bytes of one plane or of a whole image, with the shape an array of them has. +struct Layout +{ + std::span bytes; + std::vector shape; + std::vector strides; +}; + +// A read-only array over image memory. Its base holds the frame set, so what the array shows +// stays that frame for as long as the array exists. +py::array array_of(const Layout& layout, const Lease& lease) +{ + auto* owner = new Lease(lease); + const py::capsule base(owner, [](void* held) { delete static_cast(held); }); + py::array array(py::dtype::of(), layout.shape, layout.strides, layout.bytes.data(), base); + py::detail::array_proxy(array.ptr())->flags &= ~py::detail::npy_api::NPY_ARRAY_WRITEABLE_; + return array; +} + +// One image of a frame set. It does not hold the frame set itself: its arrays do. +struct Image +{ + ImageView view; + std::shared_ptr group; // owns the stream description + std::size_t stream = 0; + std::weak_ptr frame_set; + + Lease lease() const + { + Lease held = frame_set.lock(); + if (!held) + { + throw std::runtime_error("the frame set was given back"); + } + return held; + } + + const StreamDescription& description() const + { + return group->streams[stream]; + } + + Layout data_layout() const + { + return { view.data, { static_cast(view.data.size()) }, { 1 } }; + } + + Layout plane_layout(std::uint32_t index) const + { + const StreamDescription& stream_description = description(); + ImageView local = view; + local.stream = &stream_description; + const std::span bytes = local.plane(index); + const PlaneSize size = + plane_size_of(stream_description.pixel_format, stream_description.width, stream_description.height, index); + const auto rows = static_cast(size.rows); + const auto stride = static_cast(stream_description.planes[index].stride_bytes); + const auto row_bytes = static_cast(size.row_bytes); + switch (stream_description.pixel_format) + { + case PixelFormat::BGRA8: + case PixelFormat::RGBA8: + return { bytes, { rows, row_bytes / 4, 4 }, { stride, 4, 1 } }; + case PixelFormat::NV12: + if (index == 1) + { + return { bytes, { rows, row_bytes / 2, 2 }, { stride, 2, 1 } }; + } + [[fallthrough]]; + case PixelFormat::GRAY8: + return { bytes, { rows, row_bytes }, { stride, 1 } }; + } + throw std::invalid_argument("the image has a pixel format this binding does not know"); + } + + py::array data() const + { + return array_of(data_layout(), lease()); + } + + py::array plane(std::uint32_t index) const + { + const Lease held = lease(); // before the layout, which reads the image's memory + return array_of(plane_layout(index), held); + } + + py::array array() const + { + if (description().is_encoded() || description().plane_count != 1) + { + throw std::invalid_argument("array() is for images with one plane; use plane(i) for " + + std::string(name_of(description().pixel_format)) + ", data for encoded video"); + } + return plane(0); + } +}; + +// A frame set as the reader handed it out. What is known about it stays readable after it was +// given back; its images do not. +struct Frames +{ + Frames(FrameSet&& set, std::shared_ptr description) + : sequence(set.sequence()), + generation(set.generation()), + skipped(set.skipped()), + timeline(set.timeline()), + image_count(set.image_count()), + group(std::move(description)), + lease(std::make_shared(std::move(set))) + { + } + + Image image(std::size_t stream) const + { + if (!lease) + { + throw std::runtime_error("the frame set was given back"); + } + return { lease->image(stream), group, stream, lease }; + } + + // Lets go of the frame set. It goes back to the plugin now, or when the last array that + // views one of its images is gone. + void release() + { + lease.reset(); + } + + // The end of a `with frames.borrow()` block: gives the frame set back, and says so when an + // array of it still exists instead of leaving the frame set held without a word. When the + // block ends in an exception the arrays are left to keep it; that error is the one to see. + void end_borrow(bool failed) + { + const bool kept = lease && lease.use_count() > 1; + lease.reset(); + if (kept && !failed) + { + throw py::buffer_error( + "an array of this frame set still exists at the end of the block it was " + "borrowed for: drop it before the block ends, or copy what is needed from it"); + } + } + + std::uint64_t sequence; + std::uint32_t generation; + std::uint32_t skipped; + FrameTimeline timeline; + std::size_t image_count; + std::shared_ptr group; + Lease lease; +}; + +// `with frames.borrow() as frames`: the frame set for the length of a block. +struct Borrowed +{ + std::shared_ptr frames; +}; + +// How long one wait for a frame set lasts before the interpreter gets to look at signals. +constexpr std::chrono::milliseconds SIGNAL_CHECK_INTERVAL(100); + +class Reader +{ +public: + explicit Reader(CameraReader reader) + : m_group(std::make_shared(reader.group())), + m_reader(std::make_shared(std::move(reader))) + { + } + + const GroupDescription& group() const + { + return *m_group; + } + + // Waits up to `timeout_seconds`; None when no frame set came. + std::shared_ptr next(double timeout_seconds) + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::duration(timeout_seconds); + for (;;) + { + const auto left = + std::chrono::duration_cast(deadline - std::chrono::steady_clock::now()); + // The wait holds the reader itself: close() on another thread, while this one waits + // without the interpreter lock, ends the reading and does not take the reader away + // from under the wait. + const std::shared_ptr reader = open_reader(); + std::optional frames; + { + const py::gil_scoped_release release; + frames = reader->next(std::clamp(left, std::chrono::milliseconds(0), SIGNAL_CHECK_INTERVAL)); + } + if (m_reader != reader) + { + frames.reset(); // closed meanwhile: what arrived goes back + open_reader(); + } + if (frames) + { + return std::make_shared(std::move(*frames), m_group); + } + if (left <= std::chrono::milliseconds(0)) + { + return nullptr; + } + if (PyErr_CheckSignals() != 0) + { + throw py::error_already_set(); + } + } + } + + // For `for frames in reader`: lets go of the frame set handed out before, then waits for + // the next for as long as it takes. + std::shared_ptr iterate() + { + if (m_iterated) + { + m_iterated->release(); + m_iterated.reset(); + } + while (!m_iterated) + { + m_iterated = next(std::chrono::duration(SIGNAL_CHECK_INTERVAL).count()); + } + return m_iterated; + } + + void close() + { + if (m_iterated) + { + m_iterated->release(); + m_iterated.reset(); + } + m_reader.reset(); + } + + int fileno() const + { + return open_reader()->descriptor(); + } + +private: + // Call with the interpreter lock held. + std::shared_ptr open_reader() const + { + if (!m_reader) + { + throw CameraReaderError("the camera reader is closed"); + } + return m_reader; + } + + std::shared_ptr m_group; + std::shared_ptr m_reader; // empty once closed + std::shared_ptr m_iterated; +}; + +} // namespace + +PYBIND11_MODULE(_camera, m) +{ + m.doc() = "Isaac Capture camera reader: frames from a camera plugin, without a copy"; + + static py::exception reader_error(m, "CameraReaderError", PyExc_RuntimeError); + static py::exception disconnected(m, "CameraDisconnected", reader_error.ptr()); + static py::exception exhausted(m, "CameraWindowExhausted", PyExc_RuntimeError); + py::register_exception_translator( + [](std::exception_ptr pointer) + { + try + { + if (pointer) + { + std::rethrow_exception(pointer); + } + } + catch (const CameraWindowExhausted& error) + { + // In Python it is usually an array, not a frame set, that still holds one. + const std::string message = std::string(error.what()) + + ". An array taken from a frame set keeps it held: drop the array, or " + "copy what is needed from it"; + PyErr_SetString(exhausted.ptr(), message.c_str()); + } + catch (const CameraDisconnected& error) + { + PyErr_SetString(disconnected.ptr(), error.what()); + } + catch (const CameraReaderError& error) + { + PyErr_SetString(reader_error.ptr(), error.what()); + } + }); + + py::class_(m, "StreamDescription", "One sensor's images or one encoded video stream.") + .def_readonly("id", &StreamDescription::id) + .def_property_readonly( + "codec", [](const StreamDescription& s) { return name_of(s.codec); }, + "'none' for pixels, else 'h264' or 'h265'") + .def_property_readonly("pixel_format", [](const StreamDescription& s) { return name_of(s.pixel_format); }) + .def_readonly("width", &StreamDescription::width) + .def_readonly("height", &StreamDescription::height) + .def_readonly("plane_count", &StreamDescription::plane_count) + .def_property_readonly( + "strides", + [](const StreamDescription& s) + { + std::vector strides; + for (std::uint32_t i = 0; i < s.plane_count; ++i) + { + strides.push_back(s.planes[i].stride_bytes); + } + return strides; + }, + "Bytes from one row to the next, per plane") + .def_readonly("slot_bytes", &StreamDescription::slot_bytes) + .def_readonly("slot_count", &StreamDescription::slot_count) + .def_readonly("frame_rate", &StreamDescription::frame_rate) + .def("__repr__", + [](const StreamDescription& s) + { + return ""; + }); + + py::class_( + m, "GroupDescription", "Streams that are captured together and published as one frame set.") + .def_readonly("id", &GroupDescription::id) + .def_readonly("session_id", &GroupDescription::session_id, + "Names one run of the group; with a frame set's sequence it identifies a capture") + .def_readonly("streams", &GroupDescription::streams) + .def("__repr__", [](const GroupDescription& g) + { return ""; }); + + py::class_(m, "GroupListing", "One group a plugin publishes, as CameraReader.list() reports it.") + .def_property_readonly("id", [](const GroupListing& g) { return g.group.id; }) + .def_property_readonly("streams", [](const GroupListing& g) { return g.group.streams; }) + .def_readonly("unsupported", &GroupListing::unsupported, + "Why this build cannot read the group; empty for one it can open") + .def_readonly("endpoint", &GroupListing::endpoint, "The socket path of the plugin that publishes the group") + .def("__repr__", + [](const GroupListing& g) + { + return ""; + }); + + py::class_(m, "FrameTimeline", + "Where a frame set was at each step: nanoseconds on the host's monotonic clock " + "(time.monotonic_ns()), -1 where a time is not known.") + .def_readonly("capture_ns", &FrameTimeline::capture_ns) + .def_readonly("sdk_ready_ns", &FrameTimeline::sdk_ready_ns) + .def_readonly("published_ns", &FrameTimeline::published_ns) + .def_readonly("sent_ns", &FrameTimeline::sent_ns); + + py::class_(m, "ImageView", + "One image of a frame set. Its arrays are read-only views of the plugin's buffer: no copy " + "is made. An array keeps its frame set held, so it shows that frame for as long as it " + "exists and counts against the reader's window until it is gone.") + .def_property_readonly("stream", &Image::description) + .def_property_readonly("slot", [](const Image& i) { return i.view.info.slot; }) + .def_property_readonly("flags", [](const Image& i) { return i.view.info.flags; }) + .def_property_readonly("device_frame_id", [](const Image& i) { return i.view.info.device_frame_id; }) + .def_property_readonly("device_time_ns", [](const Image& i) { return i.view.info.device_time_ns; }) + .def_property_readonly( + "data", &Image::data, "The image's bytes as a flat array: the whole slot, or the access unit") + .def("array", &Image::array, + "The pixels as (height, width, 4) for bgra8 and rgba8, (height, width) for gray8. Rows are stride bytes apart.") + .def("plane", &Image::plane, py::arg("index"), + "One plane: nv12 has (height, width) luma and (height/2, width/2, 2) chroma."); + + py::class_(m, "_Borrowed") + .def("__enter__", [](const Borrowed& borrowed) { return borrowed.frames; }) + .def("__exit__", [](Borrowed& borrowed, const py::object& error, const py::object&, const py::object&) + { borrowed.frames->end_borrow(!error.is_none()); }); + + py::class_>( + m, "FrameSet", + "One frame set, held for this reader: the plugin does not reuse its buffers until it is given back. " + "release(), leaving a `with` block or the object going away let go of it; it goes back then, or when " + "the last array that views one of its images is gone.") + .def_readonly("sequence", &Frames::sequence, + "The frame set's place among the group's captures; the same for every reader") + .def_readonly("generation", &Frames::generation, "Restarts of the camera since the plugin began publishing") + .def_readonly("skipped", &Frames::skipped, + "Frame sets published since the previous one this reader was given, and not given to it") + .def_readonly("timeline", &Frames::timeline) + .def("__len__", [](const Frames& f) { return f.image_count; }) + .def("image", &Frames::image, py::arg("stream"), "The image of one stream, in the order of the group's streams") + .def("release", &Frames::release, "Let go of the frame set; arrays of its images keep it until they are gone") + .def( + "borrow", [](const std::shared_ptr& f) { return Borrowed{ f }; }, + "For `with frames.borrow() as frames`: the frame set goes back to the plugin when the block ends, and " + "every array taken from it has to be gone by then. One that still exists raises BufferError.") + .def("__enter__", [](const std::shared_ptr& f) { return f; }) + .def("__exit__", [](Frames& f, const py::object&, const py::object&, const py::object&) { f.release(); }); + + py::class_(m, "CameraReader", "Reads one group of cameras from the camera plugin that publishes it.") + .def_static( + "open", + [](const std::string& group, std::uint32_t window, const std::string& order, const std::string& endpoint, + const std::string& instance) + { + const ReaderOptions options{ window, order_of(order) }; + const PluginLocation plugin = location_of(endpoint, instance); + const py::gil_scoped_release release; + return std::make_unique(CameraReader::open(group, options, plugin)); + }, + py::arg("group"), py::kw_only(), py::arg("window") = 2, py::arg("order") = "newest", + py::arg("endpoint") = "", py::arg("instance") = "", + "Subscribe to one group. `window` is how many frame sets this reader may hold at a time; the default " + "of two lets a loop keep the array of one frame while it asks for the next. `order` is 'newest' (each " + "call gets the latest) or 'in_order' (every frame set). The reader looks through this user's camera " + "plugins for the one that publishes the group; `instance` or `endpoint` names one plugin instead, by " + "its instance name or its socket path. Raises CameraReaderError, which says why, when no plugin publishes " + "the group or two do, the plugin that was named does not answer, or the plugin refuses the reader.") + .def_static( + "list", + [](const std::string& endpoint, const std::string& instance) + { + const PluginLocation plugin = location_of(endpoint, instance); + const py::gil_scoped_release release; + return CameraReader::list(plugin); + }, + py::kw_only(), py::arg("endpoint") = "", py::arg("instance") = "", + "What is published, each group with the `endpoint` of its plugin: by every camera plugin of this user " + "that says what it publishes, or by the one plugin `instance` or `endpoint` names. No plugin running " + "gives an empty list; a plugin that was named and does not answer raises CameraReaderError.") + .def_property_readonly("group", &Reader::group) + .def( + "next", + // An optional, so that the signature and the generated stub say that None can come back. + [](Reader& reader, double timeout) -> std::optional> + { + std::shared_ptr frames = reader.next(timeout); + if (!frames) + { + return std::nullopt; + } + return frames; + }, + py::arg("timeout") = 1.0, + "Wait up to `timeout` seconds for a frame set; None when none came. Raises CameraWindowExhausted " + "when the reader already holds as many as its window.") + .def("__iter__", [](py::object self) { return self; }) + .def("__next__", &Reader::iterate, + "The next frame set, waiting as long as it takes. The one handed out before is let go of first.") + .def("close", &Reader::close, + "Stop reading. Frame sets and arrays that still exist stay as they are, and the connection to the " + "plugin ends when the last of them is gone. May be called from another thread than the one that " + "reads: a next() that is waiting then raises CameraReaderError within a tenth of a second.") + .def("fileno", &Reader::fileno, "Readable when a frame set may be waiting; for select and asyncio") + .def("__enter__", [](py::object self) { return self; }) + .def("__exit__", [](Reader& reader, const py::object&, const py::object&, const py::object&) { reader.close(); }); +} diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index 6a826f498c..aebc347446 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -99,6 +99,11 @@ if(TARGET ovxr_pip_layer) add_dependencies(python_package ovxr_pip_layer) endif() +# The camera reader is Linux only, so its module is not in the DEPENDS list above. +if(TARGET camera_py) + add_dependencies(python_package camera_py) +endif() + # viz_py only exists when BUILD_VIZ=ON. Use add_dependencies (not a # DEPENDS genex) so the build-order constraint is unconditional under # all generators — Windows ninja runs the wheel build in parallel @@ -136,6 +141,9 @@ set(STUBGEN_MODULES if(BUILD_VIZ) list(APPEND STUBGEN_MODULES isaaccapture.viz._viz) endif() +if(TARGET camera_py) + list(APPEND STUBGEN_MODULES isaaccapture.camera._camera) +endif() # --unset=VIRTUAL_ENV: an activated venv in the developer's shell is inherited # here, and any uv run that resolves no project falls back to it -- where @@ -177,6 +185,7 @@ add_custom_command( # viz_py's .pyd file as a file-level dependency when BUILD_VIZ=ON # — keeps the wheel build from racing the link. Empty otherwise. $<$:$> + $<$:$> WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" COMMENT "Generating .pyi stub files for isaaccapture modules" ) diff --git a/src/core/python/pyproject.toml.in b/src/core/python/pyproject.toml.in index 31ca29b176..73146e96f2 100644 --- a/src/core/python/pyproject.toml.in +++ b/src/core/python/pyproject.toml.in @@ -73,6 +73,7 @@ isaaccapture = ["*.so", "*.pyd", "*.pyi", "py.typed", "plugins/*/*"] # CHROMIUM_LICENSE: the notice for the key table compiled into _deviceio_trackers. "isaaccapture.deviceio_trackers" = ["*.so", "*.pyd", "*.pyi", "CHROMIUM_LICENSE"] "isaaccapture.deviceio_session" = ["*.so", "*.pyd", "*.pyi"] +"isaaccapture.camera" = ["*.so", "*.pyd", "*.pyi"] "isaaccapture.mcap" = ["*.so", "*.pyd", "*.pyi"] "isaaccapture.oxr" = ["*.so", "*.pyd", "*.pyi"] "isaaccapture.plugin_manager" = ["*.so", "*.pyd", "*.pyi"] diff --git a/src/plugins/camera_synthetic/AGENTS.md b/src/plugins/camera_synthetic/AGENTS.md index 7e1ec4b81c..060a3abfdf 100644 --- a/src/plugins/camera_synthetic/AGENTS.md +++ b/src/plugins/camera_synthetic/AGENTS.md @@ -10,6 +10,6 @@ SPDX-License-Identifier: Apache-2.0 The rules every camera plugin follows are under "Publishing cameras" in [`../../core/camera/AGENTS.md`](../../core/camera/AGENTS.md). Read them first; what follows is this plugin's own. - **This plugin is the template.** A vendor's camera plugin is made by copying it ([Writing a camera plugin](../../core/camera/README.md#writing-a-camera-plugin)), so keep it the smallest complete camera plugin: a backend in `core/`, a `main` of a few lines in `app/`, a `plugin.yaml`. What every camera plugin needs goes into the camera library, not here. -- **No SDK and no library beyond the repository's own, ever.** It is the one camera plugin that is built whenever plugins are (`BUILD_PLUGINS`), the tests of the camera interface run it, and it is the camera plugin for the Python wheel to bundle, where a plugin's binary must not need a shared library the wheel does not ship ([`src/core/AGENTS.md`](../../core/AGENTS.md)). It has no build option of its own for the same reason. +- **No SDK and no library beyond the repository's own, ever.** It is the one camera plugin that is built whenever plugins are (`BUILD_PLUGINS`), the tests of the camera interface run it, and the Python wheel bundles it under `isaaccapture/plugins/camera_synthetic`, where a plugin's binary must not need a shared library the wheel does not ship ([`src/core/AGENTS.md`](../../core/AGENTS.md)). It has no build option of its own for the same reason. - **The pattern is an interface.** `synthetic_fill` and `synthetic_frame_number` are what `camera_bench --check-synthetic` and the tests check frames against; change the pattern and all of them in one pass. - **`camera_bench` reads any camera plugin.** It lives here because it checks the synthetic pattern; it links the camera library and this plugin's `core/`, never a camera SDK. diff --git a/src/plugins/camera_synthetic/README.md b/src/plugins/camera_synthetic/README.md index 2b395394f4..779dc52ed8 100644 --- a/src/plugins/camera_synthetic/README.md +++ b/src/plugins/camera_synthetic/README.md @@ -55,6 +55,11 @@ core::PluginManager manager({ "/plugins" }); std::unique_ptr plugin = manager.start("camera_synthetic", "camera", { "--camera=head" }); ``` +The `isaaccapture` Python package bundles the executable and its `plugin.yaml` under +`plugins/camera_synthetic`, so an application that installed the wheel starts the plugin the +same way, with `BUNDLED_PLUGINS_DIR` (from `isaaccapture.teleop_session_manager`) as the +search path. + The arguments are those of every camera plugin; [Running a plugin](../../core/camera/README.md#running-a-plugin) has the whole of it. diff --git a/src/plugins/camera_synthetic/app/CMakeLists.txt b/src/plugins/camera_synthetic/app/CMakeLists.txt index 851bfaccfc..320e4cc8f7 100644 --- a/src/plugins/camera_synthetic/app/CMakeLists.txt +++ b/src/plugins/camera_synthetic/app/CMakeLists.txt @@ -15,3 +15,18 @@ add_custom_command(TARGET camera_synthetic_plugin POST_BUILD COMMENT "Putting the camera_synthetic plugin's manifest next to its executable") install(TARGETS camera_synthetic_plugin RUNTIME DESTINATION plugins/camera_synthetic) + +# Ship the plugin inside the isaaccapture package, so an application in a wheel install can +# start it through the plugin manager. The binary links nothing beyond libc/libstdc++, which +# is why it may be bundled: a vendor's plugin copied from this one leaves this block out, +# because the wheel does not ship its SDK. +if(TARGET python_package) + set(_bundle_dir "${CMAKE_BINARY_DIR}/python_package/$/isaaccapture/plugins/camera_synthetic") + add_custom_command(TARGET camera_synthetic_plugin POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "${_bundle_dir}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" "${CAMERA_SYNTHETIC_PLUGIN_MANIFEST}" "${_bundle_dir}/" + COMMENT "Bundling camera_synthetic_plugin into the isaaccapture package") + # Files staged into the package tree must exist before the wheel is built from it. + add_dependencies(python_package camera_synthetic_plugin) +endif() diff --git a/src/python/CMakeLists.txt b/src/python/CMakeLists.txt index 1134b5fd44..8c3f5092e2 100644 --- a/src/python/CMakeLists.txt +++ b/src/python/CMakeLists.txt @@ -35,6 +35,12 @@ if(NOT BUILD_VIZ) list(FILTER ISAAC_TELEOP_PYTHON_SOURCES EXCLUDE REGEX "^isaaccapture/viz/") endif() +# camera/__init__.py does a top-level `from ._camera import ...`, and _camera is +# built on Linux only, so without it the package is unimportable and is left out. +if(NOT TARGET camera_py) + list(FILTER ISAAC_TELEOP_PYTHON_SOURCES EXCLUDE REGEX "^isaaccapture/camera/") +endif() + set(ISAAC_TELEOP_PYTHON_STAGE_DIR "${CMAKE_BINARY_DIR}/python_package/$") # Staging copies, it never removes: a .py deleted or renamed in the source tree diff --git a/src/python/isaaccapture/camera/__init__.py b/src/python/isaaccapture/camera/__init__.py new file mode 100644 index 0000000000..8d6fc27c98 --- /dev/null +++ b/src/python/isaaccapture/camera/__init__.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Camera frames from the Isaac Capture camera plugins, without a copy. + + from isaaccapture.camera import CameraReader + + with CameraReader.open("head") as reader: + for frames in reader: + left = frames.image(0).array() # (height, width, 4), read-only, the plugin's own buffer + +Each camera vendor has its own plugin, a process that owns its cameras; any number of readers +read its buffers in place. A reader finds the plugin that serves its camera. An array keeps +its frame set held, so it shows that frame for as long as it exists; a reader may hold as many +frame sets at a time as its window. The synthetic camera plugin ships in this package under +``plugins/camera_synthetic`` and is started like every other plugin, through the plugin +manager. Linux only. +""" + +from ._camera import ( + CameraDisconnected, + CameraReader, + CameraReaderError, + CameraWindowExhausted, + FrameSet, + FrameTimeline, + GroupDescription, + GroupListing, + ImageView, + StreamDescription, +) + +__all__ = [ + "CameraDisconnected", + "CameraReader", + "CameraReaderError", + "CameraWindowExhausted", + "FrameSet", + "FrameTimeline", + "GroupDescription", + "GroupListing", + "ImageView", + "StreamDescription", +] diff --git a/tests/python/core/CMakeLists.txt b/tests/python/core/CMakeLists.txt index e9ff6b6993..fcf2438a35 100644 --- a/tests/python/core/CMakeLists.txt +++ b/tests/python/core/CMakeLists.txt @@ -12,4 +12,8 @@ if(BUILD_PYTHON_BINDINGS) # Needs `import isaaccapture`, and drives the C++ half through the emitter # built under tests/cpp/core/log_bridge. add_subdirectory(logging_config) + # Reads from the camera plugin bundled into the package: Linux, with plugins built. + if(TARGET camera_py AND TARGET camera_synthetic_plugin) + add_subdirectory(camera) + endif() endif() diff --git a/tests/python/core/camera/CMakeLists.txt b/tests/python/core/camera/CMakeLists.txt new file mode 100644 index 0000000000..0af727d2fc --- /dev/null +++ b/tests/python/core/camera/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The tests read from the camera plugin that is bundled into the staged package. +file(GLOB TEST_FILES + RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/test_*.py" +) + +foreach(test_file ${TEST_FILES}) + get_filename_component(test_name "${test_file}" NAME_WE) + + add_test( + NAME "camera_python_${test_name}" + COMMAND uv run --python ${ISAAC_TELEOP_PYTHON_VERSION} --extra dev pytest -v --tb=short "${test_file}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) + + set_tests_properties("camera_python_${test_name}" PROPERTIES + ENVIRONMENT "PYTHONPATH=${CMAKE_BINARY_DIR}/python_package/$" + LABELS "camera_python;integration" + ) +endforeach() diff --git a/tests/python/core/camera/pyproject.toml b/tests/python/core/camera/pyproject.toml new file mode 100644 index 0000000000..a4be93404f --- /dev/null +++ b/tests/python/core/camera/pyproject.toml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[project] +name = "isaaccapture-camera-tests" +version = "0.0.0" # Internal tests - not versioned +description = "Tests for the Isaac Capture camera reader" +requires-python = ">=3.11,<3.14" + +[project.optional-dependencies] +dev = [ + "pytest", + "numpy", +] + +[tool.pytest.ini_options] +pythonpath = ["."] +python_files = ["test_*.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +# Prevent pytest from recursing into parent directories +norecursedirs = [".git", ".venv", "build", "dist", "*.egg", "__pycache__"] diff --git a/tests/python/core/camera/test_camera_reader.py b/tests/python/core/camera/test_camera_reader.py new file mode 100644 index 0000000000..079902c11f --- /dev/null +++ b/tests/python/core/camera/test_camera_reader.py @@ -0,0 +1,541 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The camera reader, against the synthetic camera plugin shipped in the package.""" + +import shutil +import signal +import subprocess +import tempfile +import threading +import time +from pathlib import Path + +import numpy as np +import pytest + +from isaaccapture.camera import ( + CameraDisconnected, + CameraReader, + CameraReaderError, + CameraWindowExhausted, +) +from isaaccapture.teleop_session_manager import BUNDLED_PLUGINS_DIR + +# The synthetic camera plugin among the plugins bundled in the package. +PLUGIN = BUNDLED_PLUGINS_DIR / "camera_synthetic" / "camera_synthetic_plugin" + + +def wait_until(condition, timeout=15.0): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if condition(): + return True + time.sleep(0.02) + return False + + +def published(**plugin): + """The ids of the groups this user's camera plugins publish, or the plugin that is named.""" + try: + return [group.id for group in CameraReader.list(**plugin)] + except CameraReaderError: + return [] # the plugin that was named does not serve yet + + +@pytest.fixture +def short_directory(): + """A temporary directory with a short path: a socket path is limited to 107 bytes.""" + path = Path(tempfile.mkdtemp(prefix="cam", dir="/tmp")) + yield path + shutil.rmtree(path, ignore_errors=True) + + +@pytest.fixture +def endpoint_directory(short_directory, monkeypatch): + """The endpoint directory of a user whose directories are this test's own. + + A plugin that is not told where to listen and a reader that looks for a plugin both look + there, and the test's process and the plugins it starts share it. With the real ones a + test would meet the developer's own plugins and cameras.yaml. + """ + (short_directory / "run").mkdir() + (short_directory / "config" / "isaaccapture").mkdir(parents=True) + monkeypatch.setenv("XDG_RUNTIME_DIR", str(short_directory / "run")) + monkeypatch.setenv("XDG_CONFIG_HOME", str(short_directory / "config")) + return short_directory / "run" / "isaaccapture" + + +@pytest.fixture +def start_plugin(short_directory): + """Starts plugin processes for a test; each is stopped and waited for when the test ends.""" + processes = [] + + def start(*arguments): + processes.append(subprocess.Popen([str(PLUGIN), *arguments])) + return processes[-1] + + yield start + for process in processes: + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +@pytest.fixture +def plugin(short_directory, start_plugin): + """A running plugin with synthetic cameras; gives its endpoint and its process.""" + endpoint = str(short_directory / "cameras.sock") + process = start_plugin( + f"--endpoint={endpoint}", + "--camera=head:width=160,height=120,fps=120", + "--camera=wrist:width=64,height=48,streams=1,format=gray8,fps=120", + "--camera=sky:width=64,height=48,streams=1,format=nv12,fps=120", + "--camera=slow:width=64,height=48,streams=1,format=gray8,fps=1", + ) + assert wait_until(lambda: published(endpoint=endpoint)), ( + "the camera plugin did not start serving" + ) + return endpoint, process + + +def assert_synthetic(image, stream): + """Every byte of a synthetic image is known from its frame number and its stream.""" + data = image.data + frame = int.from_bytes(data[:8].tobytes(), "little") + assert image.device_frame_id == frame + assert np.all(data[8:] == (frame + 17 * stream) & 0xFF) + + +def test_a_reader_lists_what_the_plugin_publishes(plugin): + endpoint, _ = plugin + groups = CameraReader.list(endpoint=endpoint) + assert [group.id for group in groups] == ["head", "wrist", "sky", "slow"] + assert all(group.unsupported == "" for group in groups) + # Each listing says which plugin publishes the group, also when the plugin was named. + assert all(group.endpoint == endpoint for group in groups) + left, right = groups[0].streams + assert (left.id, right.id) == ("head/left", "head/right") + assert (left.width, left.height, left.pixel_format, left.codec) == ( + 160, + 120, + "bgra8", + "none", + ) + assert left.strides == [640] + assert left.frame_rate == 120.0 + assert groups[2].streams[0].pixel_format == "nv12" + assert groups[2].streams[0].plane_count == 2 + + +def test_images_are_read_only_views_of_the_plugins_buffers(plugin): + endpoint, _ = plugin + with CameraReader.open("head", endpoint=endpoint) as reader: + assert reader.group.id == "head" + assert reader.group.session_id != 0 + with reader.next(2.0) as frames: + assert len(frames) == 2 + assert frames.sequence >= 1 + assert ( + frames.timeline.capture_ns + <= frames.timeline.sdk_ready_ns + <= frames.timeline.published_ns + ) + for stream in range(2): + image = frames.image(stream) + assert image.stream.id == ("head/left", "head/right")[stream] + assert_synthetic(image, stream) + pixels = image.array() + assert pixels.shape == (120, 160, 4) + assert pixels.dtype == np.uint8 + assert pixels.base is not None # a view, not a copy + assert not pixels.flags.writeable + with pytest.raises(ValueError): + pixels[0, 0, 0] = 1 + assert ( + frames.image(0).array()[5, 7, 2] + == frames.image(0).data[5 * 640 + 7 * 4 + 2] + ) + with pytest.raises(IndexError): + frames.image(2) + + +def test_each_pixel_format_has_its_shape(plugin): + endpoint, _ = plugin + with ( + CameraReader.open("wrist", endpoint=endpoint) as reader, + reader.next(2.0) as frames, + ): + gray = frames.image(0) + assert gray.array().shape == (48, 64) + assert_synthetic(gray, 0) + + with ( + CameraReader.open("sky", endpoint=endpoint) as reader, + reader.next(2.0) as frames, + ): + nv12 = frames.image(0) + with pytest.raises(ValueError): + nv12.array() + assert nv12.plane(0).shape == (48, 64) + assert nv12.plane(1).shape == (24, 32, 2) + assert nv12.data.shape == (64 * 48 * 3 // 2,) + with pytest.raises(IndexError): + nv12.plane(2) + + +def test_a_loop_keeps_the_array_of_one_frame_while_it_asks_for_the_next(plugin): + endpoint, _ = plugin + with CameraReader.open("head", endpoint=endpoint) as reader: + previous = None + for count, frames in enumerate(reader): + # `pixels` still names the array of the frame before while the next frame set is + # asked for. The default window of two is what lets this loop run. + pixels = frames.image(0).array() + assert_synthetic(frames.image(0), 0) + assert pixels.shape == (120, 160, 4) + if previous is not None: + assert frames.sequence == previous + 1 + frames.skipped + previous = frames.sequence + if count == 30: + break + + +def test_an_array_shows_its_frame_for_as_long_as_it_exists(plugin): + endpoint, _ = plugin + reader = CameraReader.open("head", endpoint=endpoint) + frames = reader.next(2.0) + pixels = frames.image(0).array() + kept = pixels.copy() + + # The frame set is let go of, the reader is closed and both are forgotten. The array still + # holds the frame: the camera goes through its pool many times and the pixels stay. + frames.release() + reader.close() + del frames, reader + time.sleep(0.3) + assert np.array_equal(pixels, kept) + assert int.from_bytes(pixels.tobytes()[:8], "little") == int.from_bytes( + kept.tobytes()[:8], "little" + ) + + +def test_a_reader_that_was_busy_gets_what_the_camera_saw_last(plugin): + endpoint, _ = plugin + with CameraReader.open("head", endpoint=endpoint) as reader: + reader.next(2.0).release() + for _ in range(3): + left = time.monotonic_ns() + time.sleep( + 0.3 + ) # busy elsewhere, holding nothing; the camera runs at 120 fps + frames = reader.next(2.0) + # A frame set put aside when the reader left would have been published then, with + # nothing skipped. This one was published while the reader was away. + assert ( + left + 150_000_000 < frames.timeline.published_ns <= time.monotonic_ns() + ) + assert frames.skipped > 0 + frames.release() + + +def test_a_full_window_is_an_error_that_says_what_holds_it(plugin): + endpoint, _ = plugin + with CameraReader.open("head", endpoint=endpoint, window=1) as reader: + held = reader.next(2.0) + assert held is not None + with pytest.raises(CameraWindowExhausted, match="holds 1 frame set"): + reader.next(0.1) + first = held.image(0).data[:8].tobytes() + time.sleep(0.1) # the camera has moved on a dozen frames + # The held frame was not rewritten. + assert held.image(0).data[:8].tobytes() == first + + # Letting go of the frame set is not enough while an array of it is kept. + pixels = held.image(0).array() + held.release() + with pytest.raises(RuntimeError, match="given back"): + held.image(0) + with pytest.raises( + CameraWindowExhausted, match="An array taken from a frame set" + ): + reader.next(0.1) + del pixels + newer = reader.next(2.0) + assert newer is not None + assert newer.skipped > 0 + assert ( + newer.sequence > held.sequence + ) # what is known of a frame set stays readable + newer.release() + + +def test_a_borrowed_frame_set_goes_back_when_its_block_ends(plugin): + endpoint, _ = plugin + with CameraReader.open("head", endpoint=endpoint, window=1) as reader: + with reader.next(2.0).borrow() as frames: + image = frames.image(0) + pixels = image.array() + assert pixels.shape == (120, 160, 4) + assert_synthetic(image, 0) + del pixels + # Given back at the end of the block: its images are no longer to be had, and the + # reader's window of one is free again. + with pytest.raises(RuntimeError, match="given back"): + image.array() + with pytest.raises(RuntimeError, match="given back"): + frames.image(0) + + # Leaving the block while an array still exists is an error, not a frame set that + # stays held without a word. The array itself is still good: it keeps its frame. + with pytest.raises(BufferError, match="still exists at the end of the block"): + with reader.next(2.0).borrow() as frames: + kept = frames.image(0).array() + before = kept.copy() + time.sleep(0.1) + assert np.array_equal(kept, before) + del kept + assert reader.next(2.0) is not None + + # An error inside the block is the one that is seen. + with CameraReader.open("head", endpoint=endpoint) as reader: + with pytest.raises(KeyError): + with reader.next(2.0).borrow() as frames: + kept = frames.image(0).array() + assert kept.shape == (120, 160, 4) + raise KeyError("from the block") + + +def test_an_in_order_reader_gets_every_frame_set(plugin): + endpoint, _ = plugin + with CameraReader.open( + "wrist", endpoint=endpoint, window=3, order="in_order" + ) as reader: + held = [reader.next(2.0) for _ in range(3)] + assert all(frames is not None for frames in held) + with pytest.raises(CameraWindowExhausted): + reader.next(0.05) # three are held together + # Each follows the one before; whatever was passed over is counted, never lost. + for before, after in zip(held, held[1:]): + assert after.sequence == before.sequence + 1 + after.skipped + for frames in held: + assert_synthetic(frames.image(0), 0) + frames.release() + + +def test_a_reader_says_why_it_could_not_be_opened(plugin, short_directory): + endpoint, _ = plugin + with pytest.raises(CameraReaderError, match="no group 'chest'"): + CameraReader.open("chest", endpoint=endpoint) + with pytest.raises(CameraReaderError, match="does not fit the pool"): + CameraReader.open("head", endpoint=endpoint, window=8) + with pytest.raises(ValueError, match="in_order"): + CameraReader.open("head", endpoint=endpoint, order="sideways") + with pytest.raises(CameraReaderError, match="no camera plugin answers on"): + CameraReader.open("head", endpoint=str(short_directory / "nobody.sock")) + closed = CameraReader.open("head", endpoint=endpoint) + closed.close() + with pytest.raises(CameraReaderError, match="closed"): + closed.next(0.1) + + +def test_closing_from_another_thread_ends_a_wait(plugin): + endpoint, _ = plugin + reader = CameraReader.open("slow", endpoint=endpoint) + # Taken right as it is published, so the next frame set is a second away. + reader.next(3.0).release() + reader.next(3.0).release() + + outcome = [] + + def wait(): + try: + outcome.append(reader.next(5.0)) + except CameraReaderError as error: + outcome.append(error) + + thread = threading.Thread(target=wait) + thread.start() + time.sleep(0.2) # the thread now waits, with the interpreter lock released + reader.close() + thread.join(timeout=2.0) + assert not thread.is_alive() + assert isinstance(outcome[0], CameraReaderError) + assert "closed" in str(outcome[0]) + + +def test_a_reader_is_told_when_the_plugin_is_gone(plugin): + endpoint, process = plugin + reader = CameraReader.open("head", endpoint=endpoint) + process.send_signal(signal.SIGTERM) + process.wait(timeout=10) + with pytest.raises(CameraDisconnected): + for _ in range(100): # what was already on its way is still delivered + frames = reader.next(2.0) + if frames is not None: + frames.release() + + +def test_a_reader_finds_the_plugin_that_serves_its_camera( + endpoint_directory, start_plugin +): + # Neither side is told where the other is: the plugin has no --endpoint and the reader no + # endpoint and no instance. + start_plugin("--camera=head:width=160,height=120,fps=120") + assert wait_until(lambda: published() == ["head"]) + # The plugin's instance name is its backend's. + assert (endpoint_directory / "camera-synthetic.sock").is_socket() + with CameraReader.open("head") as reader: + assert reader.group.id == "head" + with reader.next(2.0) as frames: + assert frames.image(0).array().shape == (120, 160, 4) + assert_synthetic(frames.image(0), 0) + + +def test_a_listing_names_the_endpoint_of_its_plugin(endpoint_directory, start_plugin): + assert CameraReader.list() == [] # no plugin running is not an error + + start_plugin( + "--camera=head:width=160,height=120", + "--camera=wrist:width=64,height=48,streams=1", + ) + assert wait_until(lambda: published() == ["head", "wrist"]) + socket_file = str(endpoint_directory / "camera-synthetic.sock") + groups = CameraReader.list() + assert [(group.id, group.endpoint) for group in groups] == [ + ("head", socket_file), + ("wrist", socket_file), + ] + assert ( + repr(groups[0]) == f"" + ) + with pytest.raises(AttributeError): + groups[0].endpoint = "elsewhere" + + # The same when the plugin is named, by its socket path or by its instance. + for named in ({"endpoint": socket_file}, {"instance": "synthetic"}): + assert [(group.id, group.endpoint) for group in CameraReader.list(**named)] == [ + ("head", socket_file), + ("wrist", socket_file), + ] + + +def test_two_plugins_side_by_side_are_both_found_by_group( + endpoint_directory, start_plugin +): + # A second synthetic plugin runs beside the first under another instance name, with a + # camera of its own. + start_plugin("--camera=head:width=160,height=120,fps=120") + start_plugin( + "--instance=bench", + "--camera=wrist:width=64,height=48,streams=1,format=gray8,fps=120", + ) + assert wait_until(lambda: published() == ["wrist", "head"]) + + # In the order of the socket paths, each group with the socket of the plugin that has it. + assert [(group.id, group.endpoint) for group in CameraReader.list()] == [ + ("wrist", str(endpoint_directory / "camera-bench.sock")), + ("head", str(endpoint_directory / "camera-synthetic.sock")), + ] + with CameraReader.open("head") as head, CameraReader.open("wrist") as wrist: + with head.next(2.0) as frames: + assert frames.image(0).array().shape == (120, 160, 4) + assert_synthetic(frames.image(0), 0) + with wrist.next(2.0) as frames: + assert frames.image(0).array().shape == (48, 64) + assert_synthetic(frames.image(0), 0) + + # A reader that names an instance reads from that plugin and no other. + with CameraReader.open("wrist", instance="bench") as wrist: + assert wrist.group.id == "wrist" + with pytest.raises(CameraReaderError, match="no group 'wrist'"): + CameraReader.open("wrist", instance="synthetic") + + # A listing that names an instance is that plugin's alone, and an error if it is not there. + bench = CameraReader.list(instance="bench") + assert [(group.id, group.endpoint) for group in bench] == [ + ("wrist", str(endpoint_directory / "camera-bench.sock")) + ] + with pytest.raises(CameraReaderError, match="no camera plugin answers on"): + CameraReader.list(instance="absent") + + +def test_a_group_on_two_plugins_is_an_error_that_names_both( + endpoint_directory, start_plugin +): + start_plugin("--camera=head:width=64,height=48") + start_plugin("--instance=second", "--camera=head:width=64,height=48") + assert wait_until(lambda: published() == ["head", "head"]) + + # The reader does not pick one of them. + with pytest.raises(CameraReaderError) as error: + CameraReader.open("head") + message = str(error.value) + assert "published by more than one camera plugin" in message + assert f"'{endpoint_directory / 'camera-second.sock'}'" in message + assert f"'{endpoint_directory / 'camera-synthetic.sock'}'" in message + + # Naming the plugin says which. + with CameraReader.open("head", instance="second") as reader: + assert reader.next(2.0) is not None + + +def test_a_group_nobody_publishes_is_an_error_that_says_where_the_reader_looked( + endpoint_directory, start_plugin +): + # No plugin runs. + with pytest.raises(CameraReaderError) as error: + CameraReader.open("head") + message = str(error.value) + assert f"in '{endpoint_directory}' publishes group 'head'" in message + assert "No camera plugin is running there" in message + + # A plugin runs, with another camera: the error also says what is published, and where. + start_plugin("--camera=head:width=64,height=48") + assert wait_until(lambda: published() == ["head"]) + with pytest.raises(CameraReaderError) as error: + CameraReader.open("chest") + message = str(error.value) + assert f"in '{endpoint_directory}' publishes group 'chest'" in message + assert f"'head' by '{endpoint_directory / 'camera-synthetic.sock'}'" in message + + +def test_the_plugin_manager_starts_the_plugin_of_the_package( + endpoint_directory, short_directory +): + from isaaccapture.plugin_manager import PluginManager + + # The user's configuration file names the camera; the plugin is given no argument. + (short_directory / "config" / "isaaccapture" / "cameras.yaml").write_text( + "cameras:\n head:\n backend: synthetic\n width: 160\n height: 120\n fps: 120\n" + ) + socket_file = endpoint_directory / "camera-synthetic.sock" + + # Nothing runs yet, and a reader does not start anything: it says that none is running. + with pytest.raises(CameraReaderError, match="No camera plugin is running there"): + CameraReader.open("head") + assert not socket_file.exists() + + # The plugin in the package is started like any other plugin. Neither it nor the reader is + # told where the other is. + manager = PluginManager([str(BUNDLED_PLUGINS_DIR)]) + assert "camera_synthetic" in manager.get_plugin_names() + plugin = manager.start("camera_synthetic", "camera_test") + try: + assert wait_until(lambda: published() == ["head"]) + assert socket_file.is_socket() + with CameraReader.open("head") as reader: + for count, frames in enumerate(reader): + assert_synthetic(frames.image(0), 0) + if count == 5: + break + plugin.check_health() + finally: + plugin.stop() + + # Stopped through its handle, the plugin leaves nothing behind. + assert wait_until(lambda: not socket_file.exists()) diff --git a/tests/python/packaging/test_wheel_contents.py b/tests/python/packaging/test_wheel_contents.py index a76c736a61..c30dd47180 100644 --- a/tests/python/packaging/test_wheel_contents.py +++ b/tests/python/packaging/test_wheel_contents.py @@ -184,3 +184,18 @@ def test_gamepad_plugin_ships_runnable_for_teleop_session() -> None: assert (binary.external_attr >> 16) & 0o111, ( "gamepad_plugin lost its executable bit" ) + + +@pytest.mark.skipif( + not sys.platform.startswith("linux"), reason="the plugin is Linux only" +) +def test_camera_synthetic_plugin_ships_runnable_for_the_plugin_manager() -> None: + """Apps start the bundled camera plugin from BUNDLED_PLUGINS_DIR, so it needs its manifest and exec bit.""" + bundle = f"{DIST}/plugins/camera_synthetic" + with zipfile.ZipFile(_wheel()) as archive: + names = set(archive.namelist()) + binary = archive.getinfo(f"{bundle}/camera_synthetic_plugin") + assert f"{bundle}/plugin.yaml" in names + assert (binary.external_attr >> 16) & 0o111, ( + "camera_synthetic_plugin lost its executable bit" + )